sqreen 1.19.0 → 1.20.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d9ee940962990208133a70a5b307638cd43dda2b269cccce3f145ed96816e64a
-  data.tar.gz: '012987d55219ddd310337f6c0a9837d1e52aae1b7e165ab24e855c4f5fbeb767'
+  metadata.gz: 7be645cedf514d1af5754fc7244738e17f817aad09761d2da7d7fa032ac55801
+  data.tar.gz: f81295c9ac98714284d6368bc5e01c1a857317c8dad98d53366b24e53cc6cc47
 SHA512:
-  metadata.gz: 602e605c2d82de6d011dba78ce03411973ca25c2fb8a00322af9cb80937eff2997f8eda20e00d08bb2106073a3ffe8bbf4bc7f01409d123d510ebb16a7915405
-  data.tar.gz: 2363c998cb4af54018638c7d6e46f54e6d98afcac94c6be5747a3c1f555bdd59321441b3196c85f1eec08555dc579af80cdf6728ade7dfb4b9bf0f506669d55d
+  metadata.gz: 986748fc2c11ee0a6ea7a997d661246d2840e170fdcbe5d7c1221fb189e3b6a9281e4feb1025f58f1ac6712ff10f598caa7d81867a7a62898138e743eadc9262
+  data.tar.gz: 39b246ce351e780f539a0d2fe3df9791996d8203056545ab5472f0deec2e1526351bfc9cbb4f8c4cd5c145969db0b177b1b559a7491e4a8a15de4e9dd3721665

data/CHANGELOG.md

@@ -1,3 +1,25 @@
+## 1.20.1
+
+* Add fallback mechanisms when connecting to new Sqreen backend API domains
+
+## 1.20.0
+
+* Enable new instrumentation engine by default
+* Add signal-based backend communication
+
+## 1.19.3
+
+* Improve WAF PII protection
+
+## 1.19.2
+
+* Handle unexpected rule callback return values more gracefully
+* Fix incorrect return value for 404 native callback
+
+## 1.19.1
+
+* Fix LocalJumpError when reaching a Rack app nested in a Rails app
+
 ## 1.19.0
 
 * Upgrade WAF features via libsqreen 0.6.1

data/lib/sqreen/agent_message.rb

@@ -0,0 +1,20 @@
+require 'digest'
+
+module Sqreen
+  class AgentMessage
+    def initialize(kind, message, id = nil)
+      id ||= message + "\x00" + kind
+      @hash_hex = Digest::SHA1.hexdigest(id)
+      @kind = kind
+      @message = message
+    end
+
+    def to_h
+      {
+        id: @hash_hex,
+        kind: @kind,
+        message: @message,
+      }
+    end
+  end
+end

data/lib/sqreen/aggregated_metric.rb

@@ -0,0 +1,25 @@
+require 'sqreen/rules/rule_cb'
+require 'sqreen/metrics/base'
+
+module Sqreen
+  class AggregatedMetric
+    def initialize(values = {})
+      values.each do |k, v|
+        public_send "#{k}=", v
+      end
+    end
+
+    # @return [Sqreen::Rules::RuleCB]
+    attr_accessor :rule # optional
+
+    # @return [Sqreen::Metric::Base]
+    attr_accessor :metric
+
+    attr_accessor :start, :finish
+    attr_accessor :data
+
+    def name
+      metric.name
+    end
+  end
+end

data/lib/sqreen/ca.crt

@@ -70,3 +70,27 @@ WE9gyn6CagsCqiUXObXbf+eEZSqVir2G3l6BFoMtEMze/aiCKm0oHw0LxOXnGiYZ
 4fQRbxC1lfznQgUy286dUV4otp6F01vvpX1FQHKOtw5rDgb7MzVIcbidJ4vEZV8N
 hnacRHr2lVz2XTIIM6RUthg/aFzyQkqFOFSDX9HoLPKsEdao7WNq
 -----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIID7zCCAtegAwIBAgIBADANBgkqhkiG9w0BAQsFADCBmDELMAkGA1UEBhMCVVMx
+EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxJTAjBgNVBAoT
+HFN0YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4xOzA5BgNVBAMTMlN0YXJmaWVs
+ZCBTZXJ2aWNlcyBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5
+MDkwMTAwMDAwMFoXDTM3MTIzMTIzNTk1OVowgZgxCzAJBgNVBAYTAlVTMRAwDgYD
+VQQIEwdBcml6b25hMRMwEQYDVQQHEwpTY290dHNkYWxlMSUwIwYDVQQKExxTdGFy
+ZmllbGQgVGVjaG5vbG9naWVzLCBJbmMuMTswOQYDVQQDEzJTdGFyZmllbGQgU2Vy
+dmljZXMgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgLSBHMjCCASIwDQYJKoZI
+hvcNAQEBBQADggEPADCCAQoCggEBANUMOsQq+U7i9b4Zl1+OiFOxHz/Lz58gE20p
+OsgPfTz3a3Y4Y9k2YKibXlwAgLIvWX/2h/klQ4bnaRtSmpDhcePYLQ1Ob/bISdm2
+8xpWriu2dBTrz/sm4xq6HZYuajtYlIlHVv8loJNwU4PahHQUw2eeBGg6345AWh1K
+Ts9DkTvnVtYAcMtS7nt9rjrnvDH5RfbCYM8TWQIrgMw0R9+53pBlbQLPLJGmpufe
+hRhJfGZOozptqbXuNC66DQO4M99H67FrjSXZm86B0UVGMpZwh94CDklDhbZsc7tk
+6mFBrMnUVN+HL8cisibMn1lUaJ/8viovxFUcdUBgF4UCVTmLfwUCAwEAAaNCMEAw
+DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFJxfAN+q
+AdcwKziIorhtSpzyEZGDMA0GCSqGSIb3DQEBCwUAA4IBAQBLNqaEd2ndOxmfZyMI
+bw5hyf2E3F/YNoHN2BtBLZ9g3ccaaNnRbobhiCPPE95Dz+I0swSdHynVv/heyNXB
+ve6SbzJ08pGCL72CQnqtKrcgfU28elUSwhXqvfdqlS5sdJ/PHLTyxQGjhdByPq1z
+qwubdQxtRbeOlKyWN7Wg0I8VRw7j6IPdj/3vQQF3zCepYoUz8jcI73HPdwbeyBkd
+iEDPfUYd/x7H4c7/I9vG+o1VTqkC50cRRj70/b17KSa7qWFiNyi2LSr2EIZkyXCn
+0q23KXB56jzaYyWf/Wi3MOxw+3WKt21gZ7IeyLnp2KhvAotnDU0mV3HaIPzBSlCN
+sSi6
+-----END CERTIFICATE-----

data/lib/sqreen/configuration.rb

@@ -39,11 +39,15 @@ module Sqreen
     { :env => :SQREEN_LIBSQREEN, :name => :libsqreen,
       :default => true, :convert => :to_bool },
     { :env => :SQREEN_WEAVE, :name => :weave,
-      :default => false, :convert => :to_bool },
+      :default => true, :convert => :to_bool },
     { :env => :SQREEN_WEAVE_STRATEGY, :name => :weave_strategy,
-      :default => :chain, :convert => :to_sym },
-    { :env => :SQREEN_URL,       :name => :url,
-      :default => 'https://back.sqreen.io' },
+      :default => :prepend, :convert => :to_sym },
+    { :env => :SQREEN_URL, :name => :url,
+      :default => nil },
+    { :env => :SQREEN_INGESTION_URL, :name => :ingestion_url,
+      :default => nil },
+    { :env => :SQREEN_PROXY_URL, :name => :proxy_url,
+      :default => nil },
     { :env => :SQREEN_TOKEN,     :name => :token,
       :default => nil },
     { :env => :SQREEN_APP_NAME,  :name => :app_name,
@@ -74,6 +78,8 @@ module Sqreen
       :default => nil },
     { :env => :SQREEN_STRIP_SENSITIVE_REGEX, :name => :strip_sensitive_regex,
       :default => nil },
+    { :env => :SQREEN_NO_SNIFF_DOMAINS, :name => :no_sniff_domains,
+      :default => false },
 
   ].freeze
 

data/lib/sqreen/deliveries/batch.rb

@@ -8,6 +8,7 @@
 # TODO: Sqreen::RequestRecord => sqreen/events
 # TODO: Sqreen.time
 
+require 'sqreen/aggregated_metric'
 require 'sqreen/events/attack'
 require 'sqreen/events/remote_exception'
 require 'sqreen/mono_time'
@@ -91,9 +92,11 @@ module Sqreen
       def event_key(event)
         case event
         when Sqreen::Attack
-          "att-#{event.type}"
+          "att-#{event.rule_name}"
         when Sqreen::RemoteException
           "rex-#{event.klass}"
+        when Sqreen::AggregatedMetric
+          "agg-metric"
         end
       end
     end

data/lib/sqreen/deliveries/simple.rb

@@ -7,6 +7,7 @@
 # TODO: Sqreen::RemoteException  => sqreen/events
 # TODO: Sqreen::RequestRecord => sqreen/events
 
+require 'sqreen/log/loggable'
 require 'sqreen/events/attack'
 require 'sqreen/events/remote_exception'
 require 'sqreen/events/request_record'
@@ -15,6 +16,7 @@ module Sqreen
   module Deliveries
     # Simple delivery method that directly call session on event
     class Simple
+      include Log::Loggable
       attr_accessor :session
 
       def initialize(session)
@@ -29,6 +31,8 @@ module Sqreen
           session.post_sqreen_exception(event)
         when Sqreen::RequestRecord
           session.post_request_record(event)
+        when Sqreen::AggregatedMetric
+          logger.warn 'Delivery of metrics using signals is not supported with simple delivery'
         else
           session.post_event(event)
         end

data/lib/sqreen/endpoint_testing.rb

@@ -0,0 +1,184 @@
+require 'net/https'
+require 'sqreen/agent_message'
+require 'sqreen/log/loggable'
+
+module Sqreen
+  class EndpointTesting
+    Endpoint = Struct.new(:url, :ca_store)
+    class ChosenEndpoints
+      def initialize
+        @messages = []
+      end
+
+      # @return [Sqreen::EndpointTesting::Endpoint]
+      attr_accessor :control
+
+      # @return [Sqreen::EndpointTesting::Endpoint]
+      attr_accessor :ingestion
+
+      # @return [Array<Sqreen::AgentMessage>]
+      attr_reader :messages
+
+      # @param [Sqreen::AgentMessage] message
+      def add_message(message)
+        @messages << message
+      end
+    end
+
+    MAIN_CONTROL_HOST = 'back.sqreen.com'.freeze
+    MAIN_INJECTION_HOST = 'ingestion.sqreen.com'.freeze
+    FALLBACK_ENDPOINT_URL = 'https://back.sqreen.io/'.freeze
+    GLOBAL_TIMEOUT = 30
+
+    CONTROL_ERROR_KIND = 'back_sqreen_com_unavailable'.freeze
+    INGESTION_ERROR_KIND = 'ingestion_sqreen_com_unavailable'.freeze
+
+    class << self
+      include Log::Loggable
+
+      # reproduces behaviour before endpoint testing was introduced
+      def no_test_endpoints(config_url, config_ingestion_url)
+        endpoints = ChosenEndpoints.new
+
+        endpoints.control = Endpoint.new(
+          config_url || "https://#{MAIN_CONTROL_HOST}/", cert_store
+        )
+        endpoints.ingestion = Endpoint.new(
+          config_ingestion_url || "https://#{MAIN_INJECTION_HOST}/", nil
+        )
+
+        endpoints
+      end
+
+      def test_endpoints(proxy_url, config_url, config_ingestion_url)
+        proxy_params = create_proxy_params(proxy_url)
+
+        # execute the tests in separate threads and wait for them
+        thread_control = Thread.new do
+          thread_main(config_url, proxy_params, MAIN_CONTROL_HOST)
+        end
+        thread_injection = Thread.new do
+          thread_main(config_ingestion_url, proxy_params, MAIN_INJECTION_HOST)
+        end
+
+        wait_for_threads(thread_control, thread_injection)
+
+        # build and return result
+        fallback = Endpoint.new(FALLBACK_ENDPOINT_URL, cert_store)
+        endpoints = ChosenEndpoints.new
+        endpoints.control = thread_control[:endpoint] || fallback
+        endpoints.ingestion = thread_injection[:endpoint] || fallback
+
+        if thread_control[:endpoint_error]
+          msg = AgentMessage.new(CONTROL_ERROR_KIND, thread_control[:endpoint_error])
+          endpoints.add_message msg
+        end
+        if thread_injection[:endpoint_error]
+          msg = AgentMessage.new(INGESTION_ERROR_KIND, thread_injection[:endpoint_error])
+          endpoints.add_message msg
+        end
+
+        endpoints
+      end
+
+      private
+
+      def thread_main(configured_url, proxy_params, host)
+        res = if configured_url
+                Endpoint.new(configured_url, nil)
+              else
+                EndpointTesting.send(:test_with_store_variants, proxy_params, host)
+              end
+
+        Thread.current[:endpoint] = res
+      rescue StandardError => e
+        Thread.current[:endpoint_error] = e.message
+      end
+
+      def create_proxy_params(proxy_url)
+        return [] unless proxy_url
+
+        proxy = URI.parse(proxy_url)
+
+        return [] unless proxy.scheme == 'http'
+
+        [proxy.host, proxy.port, proxy.user, proxy.password]
+      end
+
+      def test_with_store_variants(proxy_params, server_name)
+        # first without custom store
+        do_test(proxy_params, server_name, false)
+      rescue StandardError => _e
+        do_test(proxy_params, server_name, true)
+      end
+
+      # @param [Array] proxy_params
+      # @param [String] server_name
+      # @param [Boolean] custom_store
+      def do_test(proxy_params, server_name, custom_store)
+        http = Net::HTTP.new(server_name, 443, *proxy_params)
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE if ENV['SQREEN_SSL_NO_VERIFY']
+        http.verify_callback = lambda do |preverify_ok, ctx|
+          unless preverify_ok
+            logger.warn do
+              "Certificate validation failure for certificate issued to " \
+              "#{ctx.chain[0].subject}: #{ctx.error_string}"
+            end
+          end
+
+          preverify_ok
+        end
+
+        http.open_timeout = 13
+        http.ssl_timeout = 7
+        http.read_timeout = 7
+        http.close_on_empty_response = true
+
+        http.cert_store = cert_store if custom_store
+
+        resp = http.get('/ping')
+
+        logger.info do
+          "Got response from #{server_name}'s ping endpoint. " \
+          "Status code is #{resp.code} (custom CA store: #{custom_store})"
+        end
+
+        unless resp.code == '200'
+          raise "Response code for /ping is #{resp.code}, not 200"
+        end
+
+        Endpoint.new("https://#{server_name}/", http.cert_store)
+      rescue StandardError => e
+        logger.info do
+          "Error in request to #{server_name} " \
+          "(custom store: #{custom_store}): #{e.message}"
+        end
+
+        raise "Error in request to #{server_name}: #{e.message}"
+      end
+
+      def wait_for_threads(thread_control, thread_injection)
+        deadline = Time.now + GLOBAL_TIMEOUT
+        [thread_control, thread_injection].each do |thread|
+          rem = deadline - Time.now
+          rem = 0.1 if rem < 0.1
+          next if thread.join(rem)
+          logger.debug { "Timeout for thread #{thread}" }
+          thread.kill
+          thread[:endpoint_error] = "Timeout doing endpoint testing"
+        end
+      end
+
+      def cert_store
+        @cert_store ||= begin
+          cert_file = File.join(File.dirname(__FILE__), 'ca.crt')
+          cert_store = OpenSSL::X509::Store.new
+          cert_store.add_file cert_file
+
+          cert_store
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/event.rb

@@ -8,17 +8,19 @@
 module Sqreen
   # Master interface for point in time events (e.g. Attack, RemoteException)
   class Event
+    # @return [Hash]
     attr_reader :payload
+
+    # @return [Time]
+    attr_accessor :time # writer used only in tests
+
     def initialize(payload)
       @payload = payload
-    end
-
-    def to_hash
-      payload.to_hash
+      @time = Time.now.utc
     end
 
     def to_s
-      "<#{self.class.name}: #{to_hash}>"
+      "<#{self.class.name}: #{payload.to_hash}>"
     end
   end
 end

data/lib/sqreen/events/attack.rb

@@ -11,6 +11,8 @@ module Sqreen
   # Attack
   # When creating a new attack, it gets automatically pushed to the event's
   # queue.
+  # XXX: TURNS OUT THIS CLASS IS ACTUALLY NOT USED ANYMORE
+  # Framework.observe is used instead with unstructured attack details
   class Attack < Event
     def self.record(payload)
       attack = Attack.new(payload)
@@ -26,11 +28,31 @@ module Sqreen
       payload['rule']['rulespack_id']
     end
 
-    def type
+    def rule_name
       return nil unless payload['rule']
       payload['rule']['name']
     end
 
+    def test?
+      return nil unless payload['rule']
+      payload['rule']['test'] ? true : false
+    end
+
+    def beta?
+      return nil unless payload['rule']
+      payload['rule']['beta'] ? true : false
+    end
+
+    def block?
+      return nil unless payload['rule']
+      payload['rule']['block'] ? true : false
+    end
+
+    def attack_type
+      return nil unless payload['rule']
+      payload['rule']['attack_type']
+    end
+
     def time
       return nil unless payload['local']
       payload['local']['time']
@@ -44,22 +66,5 @@ module Sqreen
     def enqueue
       Sqreen.queue.push(self)
     end
-
-    def to_hash
-      res = {}
-      rule_p = payload['rule']
-      request_p = payload['request']
-      res[:rule_name]    = rule_p['name']         if rule_p && rule_p['name']
-      res[:rulespack_id] = rule_p['rulespack_id'] if rule_p && rule_p['rulespack_id']
-      res[:test]         = rule_p['test']         if rule_p && rule_p['test']
-      res[:infos]        = payload['infos']       if payload['infos']
-      res[:time]         = time                   if time
-      res[:client_ip]    = request_p[:addr]       if request_p && request_p[:addr]
-      res[:request]      = request_p              if request_p
-      res[:params]       = payload['params']      if payload['params']
-      res[:context]      = payload['context']     if payload['context']
-      res[:headers]      = payload['headers']     if payload['headers']
-      res
-    end
   end
 end