sqreen 1.18.6-java → 1.20.0-java

data/lib/sqreen/{instrumentation.rb → legacy/instrumentation.rb}

@@ -1,6 +1,9 @@
+# typed: ignore
+
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
+require 'sqreen/legacy'
 require 'sqreen/cb_tree'
 require 'sqreen/log'
 require 'sqreen/exception'
@@ -36,6 +39,8 @@ require 'set'
 # end
 
 module Sqreen
+
+module Legacy
   class Instrumentation
     OVERTIME_METRIC = 'request_overtime'.freeze
 
@@ -84,6 +89,7 @@ module Sqreen
             start = Sqreen.time
             res = cb.pre(instance, args, budget, &block)
             stop = Sqreen.time
+            Sqreen.log.debug { "ran pre cb #{cb} => #{res.inspect}" }
             # The first few pre callbacks could not have a request & hence a budget just yet so we try harder to find it
             budget = framework.remaining_perf_budget if framework && !budget && Sqreen.performance_budget
             if budget
@@ -139,6 +145,7 @@ module Sqreen
             start = Sqreen.time
             res = cb.post(return_val, instance, args, budget, &block)
             stop = Sqreen.time
+            Sqreen.log.debug { "ran post cb #{cb} => #{res.inspect}" }
             if budget
               budget -= (stop - start)
               cb.overtime! if budget <= 0.0
@@ -193,6 +200,7 @@ module Sqreen
             start = Sqreen.time
             res = cb.failing(exception, instance, args, budget, &block)
             stop = Sqreen.time
+            Sqreen.log.debug { "ran failing cb #{cb} => #{res.inspect}" }
             if budget
               budget -= (stop - start)
               cb.overtime! if budget <= 0.0
@@ -508,6 +516,7 @@ module Sqreen
       saved_meth_name
     end
 
+    ### bad idea anyway
     # WARNING We do not actually remove `meth`
     def unoverride_class_method(klass, meth)
       saved_meth_name = get_saved_method_name(meth)
@@ -526,6 +535,7 @@ module Sqreen
       end
     end
 
+    ### useless now
     if RUBY_VERSION < '1.9'
       def adjust_method_name(method)
         method.to_s
@@ -554,6 +564,8 @@ module Sqreen
       is_instance_method?(obj, method)
     end
 
+    ### is that actually used?
+    ### if so, do not attempt to alter frozen instances
     # Override a singleton method on an instance
     def override_singleton_method(instance, klass_name, meth)
       @@overriden_singleton_methods = true
@@ -647,8 +659,7 @@ module Sqreen
 
       already_overriden = @@overriden_methods.include? key
       unless already_overriden
-        Sqreen.log.debug "#{key} not overriden, returning"
-        return
+        Sqreen.log.debug "#{key} apparently not overridden"
       end
 
       defined_cbs = @@registered_callbacks.get(klass, method).flatten
@@ -674,6 +685,7 @@ module Sqreen
       elsif is_instance_method?(klass, method)
         unoverride_instance_method(klass, method)
       else
+        ### Module#prepend will take care of that
         # FIXME: Override define_method and other dynamic ways to
         # The following should be monitored to make sure we
         # don't forget dynamically added methods:
@@ -701,8 +713,13 @@ module Sqreen
     # @return [Array<Sqreen::CB>]
     def hardcoded_callbacks(framework)
       [
+        ### callback for performing sec responses based on ip
+        ### init redefined to implement smartass way to hook it upon the
+        ### framework's middleware #call
         Sqreen::Rules::RunReqStartActions.new(framework),
+        ### callback for performing sec responses based on user
         Sqreen::Rules::RunUserActions.new(Sqreen, :identify, 0),
+        ### callback for performing sec responses based on user
         Sqreen::Rules::RunUserActions.new(Sqreen, :auth_track, 1),
       ]
     end
@@ -711,6 +728,7 @@ module Sqreen
     # @param rules [Array<Hash>] Rules to instrument
     # @param framework [Sqreen::Frameworks::GenericFramework]
     def instrument!(rules, framework)
+      ### set up rule signature verifier
       verifier = nil
       if Sqreen.features['rules_signature'] &&
          Sqreen.config_get(:rules_verify_signature) == true &&
@@ -720,33 +738,44 @@ module Sqreen
         Sqreen.log.debug('Rules signature is not enabled')
       end
 
+      ### force clean instrumentation callback list
       remove_all_callbacks # Force cb tree to be empty before instrumenting
 
+      ### for each rule description, transform into format for adding callback
       rules.each do |rule|
         rcb = Sqreen::Rules.cb_from_rule(rule, self, metrics_engine, verifier)
         next unless rcb
+        ### attach framework to callback
         rcb.framework = framework
+        ### add callback
         add_callback(rcb)
       end
 
       # add hardcoded callbacks, observing priority
       hardcoded_callbacks(framework).each { |cb| add_callback(cb) }
 
+      ### globally declare instrumentation ready
+      ### from within instance method? not even thread local?
       Sqreen.instrumentation_ready = true
     end
 
     def initialize(metrics_engine = nil)
       self.metrics_engine = metrics_engine
       return if metrics_engine.nil?
+      ### init metric to count calls to sqreen
       metrics_engine.create_metric('name' => CallCountable::COUNT_CALLS,
                                    'period' => 60,
                                    'kind' => 'Sum')
+      ### init metric to count request whitelist matches (ip or path whitelist)
       metrics_engine.create_metric('name' => Sqreen::Rules::RecordRequestContext::WHITELISTED_METRIC,
                                    'period' => 60,
                                    'kind' => 'Sum')
+      ### init metric to count over budget hits
       metrics_engine.create_metric('name' => OVERTIME_METRIC,
                                    'period' => 60,
                                    'kind' => 'Sum')
     end
   end
 end
+
+end

data/lib/sqreen/legacy/old_event_submission_strategy.rb

@@ -0,0 +1,221 @@
+# typed: ignore
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/aggregated_metric'
+require 'sqreen/log/loggable'
+require 'sqreen/legacy/waf_redactions'
+
+module Sqreen
+  module Legacy
+    # see also Sqreen::Signals::SignalsSubmissionStrategy
+    # usage in Sqreen:Session
+    class OldEventSubmissionStrategy
+      include Sqreen::Log::Loggable
+
+      RETRY_MANY = 301
+
+      def initialize(post_proc)
+        @post_proc = post_proc
+      end
+
+      def post_metrics(metrics)
+        return if metrics.nil? || metrics.empty?
+        payload = { metrics: metrics.map { |m| EventToHash.convert_agg_metric(m) } }
+        post('metrics', payload, {}, RETRY_MANY)
+      end
+
+      # @param attack [Sqreen::Attack]
+      def post_attack(attack)
+        post('attack', EventToHash.convert_attack(attack), {}, RETRY_MANY)
+      end
+
+      # @param [Sqreen::RequestRecord] request_record
+      def post_request_record(request_record)
+        rr_hash = EventToHash.convert_request_record(request_record)
+        post('request_record', rr_hash, {}, RETRY_MANY)
+      end
+
+      # Post an exception to Sqreen for analysis
+      # @param exception [RemoteException] Exception and context to be sent over
+      def post_sqreen_exception(exception)
+        data = EventToHash.convert_exception(exception)
+        post('sqreen_exception', data, {}, 5)
+      rescue StandardError => e
+        logger.warn(format('Could not post exception (network down? %s) %s',
+                           e.inspect,
+                           exception.inspect))
+        nil
+      end
+
+      def post_batch(events)
+        batch = events.map do |event|
+          h = case event
+              when AggregatedMetric
+                logger.warn "Aggregated metric event in non-signal mode. Signals disabled at runtime?"
+                next
+              when Attack # in practice only found inside req rec
+                EventToHash.convert_attack event
+              when RemoteException
+                EventToHash.convert_exception event
+              when RequestRecord
+                EventToHash.convert_request_record event
+              else
+                logger.warn "Unexpected event type: #{event}"
+                next
+              end
+          h['event_type'] = event_kind(event)
+          h
+        end
+        Sqreen.log.debug do
+          tally = Hash[events.group_by(&:class).map { |k, v| [k, v.count] }]
+          "Doing batch with the following tally of event types: #{tally}"
+        end
+        post('batch', { batch: batch }, {}, RETRY_MANY)
+      end
+
+      private
+
+      # see +Sqreen::Session.post+
+      def post(*args)
+        @post_proc[*args]
+      end
+
+      def event_kind(event)
+        case event
+        when Sqreen::RemoteException then 'sqreen_exception'
+        when Sqreen::Attack then 'attack'
+        when Sqreen::RequestRecord then 'request_record'
+        end
+      end
+    end
+
+    module EventToHash
+      class << self
+        # @param attack [Sqreen::Attack]
+        def convert_attack(attack)
+          payload = attack.payload
+          res = {}
+          rule_p = payload['rule']
+          request_p = payload['request']
+          res[:rule_name]    = rule_p['name']         if rule_p && rule_p['name']
+          res[:rulespack_id] = rule_p['rulespack_id'] if rule_p && rule_p['rulespack_id']
+          res[:test]         = rule_p['test']         if rule_p && rule_p['test']
+          res[:infos]        = payload['infos']       if payload['infos']
+          res[:time]         = attack.time
+          res[:client_ip]    = request_p[:addr]       if request_p && request_p[:addr]
+          res[:request]      = request_p              if request_p
+          res[:params]       = payload['params']      if payload['params']
+          res[:context]      = payload['context']     if payload['context']
+          res[:headers]      = payload['headers']     if payload['headers']
+          res
+        end
+
+        # @param [Sqreen::RequestRecord] rr
+        def convert_request_record(rr)
+          res = { :version => '20171208' }
+          payload = rr.payload
+
+          if payload[:observed]
+            res[:observed] = payload[:observed].dup
+            rulespack = nil
+            if rr.observed[:attacks]
+              res[:observed][:attacks] = rr.observed[:attacks].map do |att|
+                natt = att.dup
+                [:attack_type, :block].each { |k| natt.delete(k) } # signals stuff
+                rulespack = natt.delete(:rulespack_id) || rulespack
+                natt
+              end
+            end
+            if rr.observed[:sqreen_exceptions]
+              res[:observed][:sqreen_exceptions] = rr.observed[:sqreen_exceptions].map do |exc|
+                nex = exc.dup
+                excp = nex.delete(:exception)
+                if excp
+                  nex[:message] = excp.message
+                  nex[:klass] = excp.class.name
+                end
+                rulespack = nex.delete(:rulespack_id) || rulespack
+                nex
+              end
+            end
+            res[:rulespack_id] = rulespack unless rulespack.nil?
+            if rr.observed[:observations]
+              res[:observed][:observations] = rr.observed[:observations].map do |cat, key, value, time|
+                { :category => cat, :key => key, :value => value, :time => time }
+              end
+            end
+            if rr.observed[:sdk] # rubocop:disable Style/IfUnlessModifier
+              res[:observed][:sdk] = rr.processed_sdk_calls
+            end
+          end
+          res[:local] = payload['local'] if payload['local']
+          if payload['request']
+            res[:request] = payload['request'].dup
+            res[:client_ip] = res[:request].delete(:client_ip) if res[:request][:client_ip]
+          else
+            res[:request] = {}
+          end
+          if payload['response']
+            res[:response] = payload['response'].dup
+          else
+            res[:response] = {}
+          end
+
+          res[:request][:parameters] = payload['params'] if payload['params']
+          res[:request][:headers] = payload['headers'] if payload['headers']
+
+          res = Sqreen::EncodingSanitizer.sanitize(res)
+
+          if rr.redactor
+            res[:request], redacted = rr.redactor.redact(res[:request])
+            redacted = redacted.uniq
+            if redacted.any? && res[:observed] && res[:observed][:attacks]
+              res[:observed][:attacks] = WafRedactions.redact_attacks!(res[:observed][:attacks], redacted)
+            end
+            if redacted.any? && res[:observed] && res[:observed][:sqreen_exceptions]
+              res[:observed][:sqreen_exceptions] = WafRedactions.redact_exceptions!(res[:observed][:sqreen_exceptions], redacted)
+            end
+          end
+
+          res
+        end
+
+        # @param exception_evt [Sqreen::RemoteException]
+        def convert_exception(exception_evt)
+          payload = exception_evt.payload
+          exception = payload['exception']
+          ev = {
+            :klass => exception.class.name,
+            :message => exception.message,
+            :params => payload['request_params'],
+            :time => payload['time'],
+            :infos => {
+              :client_ip => payload['client_ip'],
+            },
+            :request => payload['request_infos'],
+            :headers => payload['headers'],
+            :rule_name => payload['rule_name'],
+            :rulespack_id => payload['rulespack_id'],
+          }
+
+          ev[:infos].merge!(payload['infos']) if payload['infos']
+          return ev unless exception.backtrace
+          ev[:context] = { :backtrace => exception.backtrace.map(&:to_s) }
+          ev
+        end
+
+        # @param [Sqreen::AggregatedMetric] agg_metric
+        def convert_agg_metric(agg_metric)
+          {
+            name: agg_metric.name,
+            observation: agg_metric.data,
+            start: agg_metric.start,
+            finish: agg_metric.finish,
+          }
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/legacy/waf_redactions.rb

@@ -0,0 +1,49 @@
+# typed: ignore
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+module Sqreen
+  module Legacy
+    module WafRedactions
+      class << self
+        def redact_attacks!(attacks, values)
+          return attacks if values.empty?
+
+          values = values.map { |v| v.downcase if v.is_a?(String) }
+
+          attacks.each do |e|
+            next(e) unless e[:infos]
+            next(e) unless e[:infos][:waf_data]
+
+            parsed = JSON.parse(e[:infos][:waf_data])
+            redacted = parsed.each do |w|
+              next unless (filters = w['filter'])
+
+              filters.each do |f|
+                next unless (v = f['resolved_value'])
+                next unless values.include?(v.downcase)
+
+                f['match_status'] = SensitiveDataRedactor::MASK
+                f['resolved_value'] = SensitiveDataRedactor::MASK
+              end
+            end
+            e[:infos][:waf_data] = JSON.dump(redacted)
+          end
+        end
+
+        # see https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000022-waf-data-sanitization.md#changes-to-the-agents
+        def redact_exceptions!(exceptions, values)
+          return exceptions if values.empty?
+
+          exceptions.each do |e|
+            next(e) unless e[:infos]
+            next(e) unless e[:infos][:waf]
+
+            e[:infos][:waf].delete(:args)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/log.rb

@@ -1,3 +1,5 @@
+# typed: ignore
+
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 

data/lib/sqreen/log/loggable.rb

@@ -0,0 +1,28 @@
+# typed: false
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'logger'
+
+module Sqreen; end
+module Sqreen::Log; end
+
+module Sqreen::Log::Loggable
+  def self.included(klass)
+    klass.extend(ClassMethods)
+  end
+
+  module ClassMethods
+    def logger
+      # TODO: use progname for qualified log messages
+      # TODO: qualified logger proxies for selectable levels per component
+      # @logger ||= ::Logger.new(STDOUT, progname: name, level: Logger::DEBUG)
+      Sqreen.log
+    end
+  end
+
+  def logger
+    @logger || self.class.logger
+  end
+end