sqreen 1.19.1 → 1.21.0.beta1

data/lib/sqreen/ecosystem_integration/instrumentation_service.rb

@@ -0,0 +1,38 @@
+require 'sqreen/graft/hook'
+require 'sqreen/ecosystem_integration/around_callbacks'
+
+module Sqreen
+  class EcosystemIntegration
+    module InstrumentationService
+      class << self
+        # @param [String] module_name
+        # @param [String] method in form A::B#c or A::B.c
+        # @param [Hash{Symbol=>Proc}] spec
+        def instrument(module_name, method, spec)
+          hook = Sqreen::Graft::Hook[method].add do
+            if spec[:before]
+              cb = AroundCallbacks.wrap_instrumentation_hook(module_name, 'pre', spec[:before])
+              before(nil, flow: true, &cb)
+            end
+
+            if spec[:after]
+              cb = AroundCallbacks.wrap_instrumentation_hook(module_name, 'post', spec[:after])
+              after(nil, flow: true, &cb)
+            end
+
+            if spec[:raised]
+              cb = AroundCallbacks.wrap_instrumentation_hook(module_name, 'failing', spec[:raised])
+              raised(nil, flow: true, &cb)
+            end
+
+            if spec[:ensured]
+              cb = AroundCallbacks.wrap_instrumentation_hook(module_name, 'finally', spec[:ensured])
+              ensured(nil, flow: true, &cb)
+            end
+          end
+          hook.install
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem_integration/request_lifecycle_tracking.rb

@@ -0,0 +1,56 @@
+require 'sqreen/log/loggable'
+
+module Sqreen
+  class EcosystemIntegration
+    # This class gets notified of request start/end and
+    # 1) distributes such events to listeners (typically ecosystem modules;
+    #    the method add_start_observer is exposed to ecosystem modules through
+    #    +Sqreen::Ecosystem::ModuleApi::EventListener+ and the dispatch table).
+    # 2) keeps track of whether a request is active on this thread. This is
+    #    so that users of this class can have this information without needing
+    #    to subscribe to request start/events and keeping thread local state
+    #    themselves.
+    # XXX: Since the Ecosystem is also notified of request start/end, it could
+    # notify its modules of request start without going through the dispatch
+    # table and call add_start_observer. We need to think if we want to keep
+    # the transaction / request distinction or if they should just be
+    # assumed to be the same, though.
+    class RequestLifecycleTracking
+      include Sqreen::Log::Loggable
+
+      def initialize
+        @start_observers = []
+        @tl_key = "#{object_id}_req_in_flight"
+      end
+
+      # API for classes needing to know the request state
+
+      # @param cb A callback taking a Rack::Request
+      def add_start_observer(cb)
+        @start_observers << cb
+      end
+
+      def in_request?
+        Thread.current[@tl_key] ? true : false
+      end
+
+      # API for classes notifying this one of request events
+
+      def notify_request_start(rack_req)
+        Thread.current[@tl_key] = true
+        return if @start_observers.empty?
+        @start_observers.each do |cb|
+          begin
+            cb.call(rack_req)
+          rescue ::Exception => e # rubocop:disable Lint/RescueException
+            logger.warn { "Error calling #{cb} on request start: #{e.message}" }
+          end
+        end
+      end
+
+      def notify_request_end
+        Thread.current[@tl_key] = false
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem_integration/signal_consumption.rb

@@ -0,0 +1,35 @@
+require 'sqreen/log/loggable'
+
+module Sqreen
+  class EcosystemIntegration
+    class SignalConsumption
+      include Sqreen::Log::Loggable
+
+      PAYLOAD_CREATOR_SECTIONS = %w[request response params headers].freeze
+
+      # @param [Sqreen::Frameworks::GenericFramework] framework
+      # @param [Sqreen::EcosystemIntegration::RequestLifecycleTracking]
+      # @param [Sqreen::CappedQueue]
+      def initialize(framework, req_lifecycle, queue)
+        @framework = framework
+        @req_lifecycle = req_lifecycle
+        @queue = queue
+      end
+
+      def consume_signal(signal)
+        # transitional
+        unless Sqreen.features.fetch('use_signals', DEFAULT_USE_SIGNALS)
+          logger.debug { "Discarding signal #{signal} (signals disabled)" }
+          return
+        end
+
+        if @req_lifecycle.in_request?
+          # add it to the request record
+          @framework.observe(:signals, signal, PAYLOAD_CREATOR_SECTIONS, true)
+        else
+          @queue.push signal
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/endpoint_testing.rb

@@ -0,0 +1,184 @@
+require 'net/https'
+require 'sqreen/agent_message'
+require 'sqreen/log/loggable'
+
+module Sqreen
+  class EndpointTesting
+    Endpoint = Struct.new(:url, :ca_store)
+    class ChosenEndpoints
+      def initialize
+        @messages = []
+      end
+
+      # @return [Sqreen::EndpointTesting::Endpoint]
+      attr_accessor :control
+
+      # @return [Sqreen::EndpointTesting::Endpoint]
+      attr_accessor :ingestion
+
+      # @return [Array<Sqreen::AgentMessage>]
+      attr_reader :messages
+
+      # @param [Sqreen::AgentMessage] message
+      def add_message(message)
+        @messages << message
+      end
+    end
+
+    MAIN_CONTROL_HOST = 'back.sqreen.com'.freeze
+    MAIN_INJECTION_HOST = 'ingestion.sqreen.com'.freeze
+    FALLBACK_ENDPOINT_URL = 'https://back.sqreen.io/'.freeze
+    GLOBAL_TIMEOUT = 30
+
+    CONTROL_ERROR_KIND = 'back_sqreen_com_unavailable'.freeze
+    INGESTION_ERROR_KIND = 'ingestion_sqreen_com_unavailable'.freeze
+
+    class << self
+      include Log::Loggable
+
+      # reproduces behaviour before endpoint testing was introduced
+      def no_test_endpoints(config_url, config_ingestion_url)
+        endpoints = ChosenEndpoints.new
+
+        endpoints.control = Endpoint.new(
+          config_url || "https://#{MAIN_CONTROL_HOST}/", cert_store
+        )
+        endpoints.ingestion = Endpoint.new(
+          config_ingestion_url || "https://#{MAIN_INJECTION_HOST}/", nil
+        )
+
+        endpoints
+      end
+
+      def test_endpoints(proxy_url, config_url, config_ingestion_url)
+        proxy_params = create_proxy_params(proxy_url)
+
+        # execute the tests in separate threads and wait for them
+        thread_control = Thread.new do
+          thread_main(config_url, proxy_params, MAIN_CONTROL_HOST)
+        end
+        thread_injection = Thread.new do
+          thread_main(config_ingestion_url, proxy_params, MAIN_INJECTION_HOST)
+        end
+
+        wait_for_threads(thread_control, thread_injection)
+
+        # build and return result
+        fallback = Endpoint.new(FALLBACK_ENDPOINT_URL, cert_store)
+        endpoints = ChosenEndpoints.new
+        endpoints.control = thread_control[:endpoint] || fallback
+        endpoints.ingestion = thread_injection[:endpoint] || fallback
+
+        if thread_control[:endpoint_error]
+          msg = AgentMessage.new(CONTROL_ERROR_KIND, thread_control[:endpoint_error])
+          endpoints.add_message msg
+        end
+        if thread_injection[:endpoint_error]
+          msg = AgentMessage.new(INGESTION_ERROR_KIND, thread_injection[:endpoint_error])
+          endpoints.add_message msg
+        end
+
+        endpoints
+      end
+
+      private
+
+      def thread_main(configured_url, proxy_params, host)
+        res = if configured_url
+                Endpoint.new(configured_url, nil)
+              else
+                EndpointTesting.send(:test_with_store_variants, proxy_params, host)
+              end
+
+        Thread.current[:endpoint] = res
+      rescue StandardError => e
+        Thread.current[:endpoint_error] = e.message
+      end
+
+      def create_proxy_params(proxy_url)
+        return [] unless proxy_url
+
+        proxy = URI.parse(proxy_url)
+
+        return [] unless proxy.scheme == 'http'
+
+        [proxy.host, proxy.port, proxy.user, proxy.password]
+      end
+
+      def test_with_store_variants(proxy_params, server_name)
+        # first without custom store
+        do_test(proxy_params, server_name, false)
+      rescue StandardError => _e
+        do_test(proxy_params, server_name, true)
+      end
+
+      # @param [Array] proxy_params
+      # @param [String] server_name
+      # @param [Boolean] custom_store
+      def do_test(proxy_params, server_name, custom_store)
+        http = Net::HTTP.new(server_name, 443, *proxy_params)
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE if ENV['SQREEN_SSL_NO_VERIFY']
+        http.verify_callback = lambda do |preverify_ok, ctx|
+          unless preverify_ok
+            logger.warn do
+              "Certificate validation failure for certificate issued to " \
+              "#{ctx.chain[0].subject}: #{ctx.error_string}"
+            end
+          end
+
+          preverify_ok
+        end
+
+        http.open_timeout = 13
+        http.ssl_timeout = 7
+        http.read_timeout = 7
+        http.close_on_empty_response = true
+
+        http.cert_store = cert_store if custom_store
+
+        resp = http.get('/ping')
+
+        logger.info do
+          "Got response from #{server_name}'s ping endpoint. " \
+          "Status code is #{resp.code} (custom CA store: #{custom_store})"
+        end
+
+        unless resp.code == '200'
+          raise "Response code for /ping is #{resp.code}, not 200"
+        end
+
+        Endpoint.new("https://#{server_name}/", http.cert_store)
+      rescue StandardError => e
+        logger.info do
+          "Error in request to #{server_name} " \
+          "(custom store: #{custom_store}): #{e.message}"
+        end
+
+        raise "Error in request to #{server_name}: #{e.message}"
+      end
+
+      def wait_for_threads(thread_control, thread_injection)
+        deadline = Time.now + GLOBAL_TIMEOUT
+        [thread_control, thread_injection].each do |thread|
+          rem = deadline - Time.now
+          rem = 0.1 if rem < 0.1
+          next if thread.join(rem)
+          logger.debug { "Timeout for thread #{thread}" }
+          thread.kill
+          thread[:endpoint_error] = "Timeout doing endpoint testing"
+        end
+      end
+
+      def cert_store
+        @cert_store ||= begin
+          cert_file = File.join(File.dirname(__FILE__), 'ca.crt')
+          cert_store = OpenSSL::X509::Store.new
+          cert_store.add_file cert_file
+
+          cert_store
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/event.rb

@@ -8,17 +8,19 @@
 module Sqreen
   # Master interface for point in time events (e.g. Attack, RemoteException)
   class Event
+    # @return [Hash]
     attr_reader :payload
+
+    # @return [Time]
+    attr_accessor :time # writer used only in tests
+
     def initialize(payload)
       @payload = payload
-    end
-
-    def to_hash
-      payload.to_hash
+      @time = Time.now.utc
     end
 
     def to_s
-      "<#{self.class.name}: #{to_hash}>"
+      "<#{self.class.name}: #{payload.to_hash}>"
     end
   end
 end

data/lib/sqreen/events/attack.rb

@@ -11,6 +11,8 @@ module Sqreen
   # Attack
   # When creating a new attack, it gets automatically pushed to the event's
   # queue.
+  # XXX: TURNS OUT THIS CLASS IS ACTUALLY NOT USED ANYMORE
+  # Framework.observe is used instead with unstructured attack details
   class Attack < Event
     def self.record(payload)
       attack = Attack.new(payload)
@@ -26,11 +28,31 @@ module Sqreen
       payload['rule']['rulespack_id']
     end
 
-    def type
+    def rule_name
       return nil unless payload['rule']
       payload['rule']['name']
     end
 
+    def test?
+      return nil unless payload['rule']
+      payload['rule']['test'] ? true : false
+    end
+
+    def beta?
+      return nil unless payload['rule']
+      payload['rule']['beta'] ? true : false
+    end
+
+    def block?
+      return nil unless payload['rule']
+      payload['rule']['block'] ? true : false
+    end
+
+    def attack_type
+      return nil unless payload['rule']
+      payload['rule']['attack_type']
+    end
+
     def time
       return nil unless payload['local']
       payload['local']['time']
@@ -44,22 +66,5 @@ module Sqreen
     def enqueue
       Sqreen.queue.push(self)
     end
-
-    def to_hash
-      res = {}
-      rule_p = payload['rule']
-      request_p = payload['request']
-      res[:rule_name]    = rule_p['name']         if rule_p && rule_p['name']
-      res[:rulespack_id] = rule_p['rulespack_id'] if rule_p && rule_p['rulespack_id']
-      res[:test]         = rule_p['test']         if rule_p && rule_p['test']
-      res[:infos]        = payload['infos']       if payload['infos']
-      res[:time]         = time                   if time
-      res[:client_ip]    = request_p[:addr]       if request_p && request_p[:addr]
-      res[:request]      = request_p              if request_p
-      res[:params]       = payload['params']      if payload['params']
-      res[:context]      = payload['context']     if payload['context']
-      res[:headers]      = payload['headers']     if payload['headers']
-      res
-    end
   end
 end

data/lib/sqreen/events/remote_exception.rb

@@ -30,27 +30,5 @@ module Sqreen
     def klass
       payload['exception'].class.name
     end
-
-    def to_hash
-      exception = payload['exception']
-      ev = {
-        :klass => exception.class.name,
-        :message => exception.message,
-        :params => payload['request_params'],
-        :time => payload['time'],
-        :infos => {
-          :client_ip => payload['client_ip'],
-        },
-        :request => payload['request_infos'],
-        :headers => payload['headers'],
-        :rule_name => payload['rule_name'],
-        :rulespack_id => payload['rulespack_id'],
-      }
-
-      ev[:infos].merge!(payload['infos']) if payload['infos']
-      return ev unless exception.backtrace
-      ev[:context] = { :backtrace => exception.backtrace.map(&:to_s) }
-      ev
-    end
   end
 end

data/lib/sqreen/events/request_record.rb

@@ -14,6 +14,10 @@ require 'sqreen/sensitive_data_redactor'
 module Sqreen
   # When a request is deeemed worthy of being sent to the backend
   class RequestRecord < Sqreen::Event
+    attr_reader :redactor
+
+    # @param [Hash] payload
+    # @param [Sqreen::SensitiveDataRedactor] redactor
     def initialize(payload, redactor = nil)
       @redactor = redactor
       super(payload)
@@ -23,74 +27,18 @@ module Sqreen
       (payload && payload[:observed]) || {}
     end
 
-    def to_hash
-      res = { :version => '20171208' }
-      if payload[:observed]
-        res[:observed] = payload[:observed].dup
-        rulespack = nil
-        if observed[:attacks]
-          res[:observed][:attacks] = observed[:attacks].map do |att|
-            natt = att.dup
-            rulespack = natt.delete(:rulespack_id) || rulespack
-            natt
-          end
-        end
-        if observed[:sqreen_exceptions]
-          res[:observed][:sqreen_exceptions] = observed[:sqreen_exceptions].map do |exc|
-            nex = exc.dup
-            excp = nex.delete(:exception)
-            if excp
-              nex[:message] = excp.message
-              nex[:klass] = excp.class.name
-            end
-            rulespack = nex.delete(:rulespack_id) || rulespack
-            nex
-          end
-        end
-        res[:rulespack_id] = rulespack unless rulespack.nil?
-        if observed[:observations]
-          res[:observed][:observations] = observed[:observations].map do |cat, key, value, time|
-            { :category => cat, :key => key, :value => value, :time => time }
-          end
-        end
-        if observed[:sdk]
-          res[:observed][:sdk] = processed_sdk_calls
-        end
-      end
-      res[:local] = payload['local'] if payload['local']
-      if payload['request']
-        res[:request] = payload['request'].dup
-        res[:client_ip] = res[:request].delete(:client_ip) if res[:request][:client_ip]
-      else
-        res[:request] = {}
-      end
-      if payload['response']
-        res[:response] = payload['response'].dup
-      else
-        res[:response] = {}
-      end
-
-      res[:request][:parameters] = payload['params'] if payload['params']
-      res[:request][:headers] = payload['headers'] if payload['headers']
-
-      res = Sqreen::EncodingSanitizer.sanitize(res)
+    def last_identify_args
+      return nil unless observed[:sdk]
 
-      if @redactor
-        res[:request], redacted = @redactor.redact(res[:request])
-        if redacted.any? && res[:observed] && res[:observed][:attacks]
-          res[:observed][:attacks] = @redactor.redact_attacks!(res[:observed][:attacks], redacted)
-        end
-        if redacted.any? && res[:observed] && res[:observed][:sqreen_exceptions]
-          res[:observed][:sqreen_exceptions] = @redactor.redact_exceptions!(res[:observed][:sqreen_exceptions], redacted)
-        end
+      observed[:sdk].reverse_each do |meth, _time, *args|
+        next unless meth == :identify
+        return args
       end
-
-      res
+      nil
     end
 
-    private
-
     def processed_sdk_calls
+      return [] unless observed[:sdk]
       auth_keys = last_identify_id
 
       observed[:sdk].map do |meth, time, *args|
@@ -102,6 +50,8 @@ module Sqreen
       end
     end
 
+    private
+
     def inject_identifiers(args, meth, auth_keys)
       return args unless meth == :track && auth_keys
 
@@ -118,13 +68,8 @@ module Sqreen
     end
 
     def last_identify_id
-      return nil unless observed[:sdk]
-
-      observed[:sdk].reverse_each do |meth, _time, *args|
-        next unless meth == :identify
-        return args.first if args.respond_to? :first
-      end
-      nil
+      args = last_identify_args
+      args.first if args.respond_to? :first
     end
   end
 end