sqreen 1.19.1-java → 1.21.0.beta3-java

data/lib/sqreen/legacy/old_event_submission_strategy.rb

@@ -0,0 +1,228 @@
+# typed: ignore
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/aggregated_metric'
+require 'sqreen/log/loggable'
+require 'sqreen/legacy/waf_redactions'
+require 'sqreen/kit/string_sanitizer'
+
+module Sqreen
+  module Legacy
+    # see also Sqreen::Signals::SignalsSubmissionStrategy
+    # usage in Sqreen:Session
+    class OldEventSubmissionStrategy
+      include Sqreen::Log::Loggable
+
+      RETRY_MANY = 301
+
+      def initialize(post_proc)
+        @post_proc = post_proc
+      end
+
+      def post_metrics(metrics)
+        return if metrics.nil? || metrics.empty?
+        payload = { metrics: metrics.map { |m| EventToHash.convert_agg_metric(m) } }
+        post('metrics', payload, {}, RETRY_MANY)
+      end
+
+      # @param attack [Sqreen::Attack]
+      def post_attack(attack)
+        post('attack', EventToHash.convert_attack(attack), {}, RETRY_MANY)
+      end
+
+      # @param [Sqreen::RequestRecord] request_record
+      def post_request_record(request_record)
+        rr_hash = EventToHash.convert_request_record(request_record)
+        post('request_record', rr_hash, {}, RETRY_MANY)
+      end
+
+      # Post an exception to Sqreen for analysis
+      # @param exception [RemoteException] Exception and context to be sent over
+      def post_sqreen_exception(exception)
+        data = EventToHash.convert_exception(exception)
+        post('sqreen_exception', data, {}, 5)
+      rescue StandardError => e
+        logger.warn(format('Could not post exception (network down? %s) %s',
+                           e.inspect,
+                           exception.inspect))
+        nil
+      end
+
+      def post_batch(events)
+        batch = events.map do |event|
+          h = case event
+              when AggregatedMetric
+                logger.warn "Aggregated metric event in non-signal mode. Signals disabled at runtime?"
+                next
+              when Sqreen::Kit::Signals::Signal
+                logger.warn "Signal event in non-signal mode"
+                next
+              when Sqreen::Kit::Signals::Trace
+                logger.warn "Trace event in non-signal mode"
+                next
+              when Attack # in practice only found inside req rec
+                EventToHash.convert_attack event
+              when RemoteException
+                EventToHash.convert_exception event
+              when RequestRecord
+                EventToHash.convert_request_record event
+              else
+                logger.warn "Unexpected event type: #{event}"
+                next
+              end
+          h['event_type'] = event_kind(event)
+          h
+        end
+        Sqreen.log.debug do
+          tally = Hash[events.group_by(&:class).map { |k, v| [k, v.count] }]
+          "Doing batch with the following tally of event types: #{tally}"
+        end
+        post('batch', { batch: batch.compact }, {}, RETRY_MANY)
+      end
+
+      private
+
+      # see +Sqreen::Session.post+
+      def post(*args)
+        @post_proc[*args]
+      end
+
+      def event_kind(event)
+        case event
+        when Sqreen::RemoteException then 'sqreen_exception'
+        when Sqreen::Attack then 'attack'
+        when Sqreen::RequestRecord then 'request_record'
+        end
+      end
+    end
+
+    module EventToHash
+      class << self
+        # @param attack [Sqreen::Attack]
+        def convert_attack(attack)
+          payload = attack.payload
+          res = {}
+          rule_p = payload['rule']
+          request_p = payload['request']
+          res[:rule_name]    = rule_p['name']         if rule_p && rule_p['name']
+          res[:rulespack_id] = rule_p['rulespack_id'] if rule_p && rule_p['rulespack_id']
+          res[:test]         = rule_p['test']         if rule_p && rule_p['test']
+          res[:infos]        = payload['infos']       if payload['infos']
+          res[:time]         = attack.time
+          res[:client_ip]    = request_p[:addr]       if request_p && request_p[:addr]
+          res[:request]      = request_p              if request_p
+          res[:params]       = payload['params']      if payload['params']
+          res[:context]      = payload['context']     if payload['context']
+          res[:headers]      = payload['headers']     if payload['headers']
+          res
+        end
+
+        # @param [Sqreen::RequestRecord] rr
+        def convert_request_record(rr)
+          res = { :version => '20171208' }
+          payload = rr.payload
+
+          if payload[:observed]
+            res[:observed] = payload[:observed].dup
+            rulespack = nil
+            if rr.observed[:attacks]
+              res[:observed][:attacks] = rr.observed[:attacks].map do |att|
+                natt = att.dup
+                [:attack_type, :block].each { |k| natt.delete(k) } # signals stuff
+                rulespack = natt.delete(:rulespack_id) || rulespack
+                natt
+              end
+            end
+            if rr.observed[:sqreen_exceptions]
+              res[:observed][:sqreen_exceptions] = rr.observed[:sqreen_exceptions].map do |exc|
+                nex = exc.dup
+                excp = nex.delete(:exception)
+                if excp
+                  nex[:message] = excp.message
+                  nex[:klass] = excp.class.name
+                end
+                rulespack = nex.delete(:rulespack_id) || rulespack
+                nex
+              end
+            end
+            res[:rulespack_id] = rulespack unless rulespack.nil?
+            if rr.observed[:observations]
+              res[:observed][:observations] = rr.observed[:observations].map do |cat, key, value, time|
+                { :category => cat, :key => key, :value => value, :time => time }
+              end
+            end
+            if rr.observed[:sdk] # rubocop:disable Style/IfUnlessModifier
+              res[:observed][:sdk] = rr.processed_sdk_calls
+            end
+          end
+          res[:local] = payload['local'] if payload['local']
+          if payload['request']
+            res[:request] = payload['request'].dup
+            res[:client_ip] = res[:request].delete(:client_ip) if res[:request][:client_ip]
+          else
+            res[:request] = {}
+          end
+          if payload['response']
+            res[:response] = payload['response'].dup
+          else
+            res[:response] = {}
+          end
+
+          res[:request][:parameters] = payload['params'] if payload['params']
+          res[:request][:headers] = payload['headers'] if payload['headers']
+
+          res = Sqreen::Kit::StringSanitizer.sanitize(res)
+
+          if rr.redactor
+            res[:request], redacted = rr.redactor.redact(res[:request])
+            redacted = redacted.uniq
+            if redacted.any? && res[:observed] && res[:observed][:attacks]
+              res[:observed][:attacks] = WafRedactions.redact_attacks!(res[:observed][:attacks], redacted)
+            end
+            if redacted.any? && res[:observed] && res[:observed][:sqreen_exceptions]
+              res[:observed][:sqreen_exceptions] = WafRedactions.redact_exceptions!(res[:observed][:sqreen_exceptions], redacted)
+            end
+          end
+
+          res
+        end
+
+        # @param exception_evt [Sqreen::RemoteException]
+        def convert_exception(exception_evt)
+          payload = exception_evt.payload
+          exception = payload['exception']
+          ev = {
+            :klass => exception.class.name,
+            :message => exception.message,
+            :params => payload['request_params'],
+            :time => payload['time'],
+            :infos => {
+              :client_ip => payload['client_ip'],
+            },
+            :request => payload['request_infos'],
+            :headers => payload['headers'],
+            :rule_name => payload['rule_name'],
+            :rulespack_id => payload['rulespack_id'],
+          }
+
+          ev[:infos].merge!(payload['infos']) if payload['infos']
+          return ev unless exception.backtrace
+          ev[:context] = { :backtrace => exception.backtrace.map(&:to_s) }
+          ev
+        end
+
+        # @param [Sqreen::AggregatedMetric] agg_metric
+        def convert_agg_metric(agg_metric)
+          {
+            name: agg_metric.name,
+            observation: agg_metric.data,
+            start: agg_metric.start,
+            finish: agg_metric.finish,
+          }
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/legacy/waf_redactions.rb

@@ -0,0 +1,49 @@
+# typed: ignore
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+module Sqreen
+  module Legacy
+    module WafRedactions
+      class << self
+        def redact_attacks!(attacks, values)
+          return attacks if values.empty?
+
+          values = values.map { |v| v.downcase if v.is_a?(String) }
+
+          attacks.each do |e|
+            next(e) unless e[:infos]
+            next(e) unless e[:infos][:waf_data]
+
+            parsed = JSON.parse(e[:infos][:waf_data])
+            redacted = parsed.each do |w|
+              next unless (filters = w['filter'])
+
+              filters.each do |f|
+                next unless (v = f['resolved_value'])
+                next unless values.include?(v.downcase)
+
+                f['match_status'] = SensitiveDataRedactor::MASK
+                f['resolved_value'] = SensitiveDataRedactor::MASK
+              end
+            end
+            e[:infos][:waf_data] = JSON.dump(redacted)
+          end
+        end
+
+        # see https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000022-waf-data-sanitization.md#changes-to-the-agents
+        def redact_exceptions!(exceptions, values)
+          return exceptions if values.empty?
+
+          exceptions.each do |e|
+            next(e) unless e[:infos]
+            next(e) unless e[:infos][:waf]
+
+            e[:infos][:waf].delete(:args)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/log.rb

@@ -14,16 +14,17 @@ require 'sqreen/deferred_logger'
 
 module Sqreen
   def self.log_init
+    deferred_logger = @logger
     @logger = Sqreen::Logger.new(
       Sqreen.config_get(:log_level).to_s.upcase,
       Sqreen.config_get(:log_location)
     )
-    Sqreen::DeferredLogger.instance.flush_to(@logger.instance_eval { @logger })
+    deferred_logger.flush_to(@logger.instance_eval { @logger })
   rescue => e
     warn "Sqreen logger exception: #{e}"
   end
 
   def self::log
-    @logger || Sqreen::DeferredLogger.instance
+    @logger ||= Sqreen::DeferredLogger.new
   end
 end

data/lib/sqreen/log/loggable.rb

@@ -4,6 +4,7 @@
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
 require 'logger'
+require 'sqreen/log'
 
 module Sqreen; end
 module Sqreen::Log; end
@@ -23,6 +24,6 @@ module Sqreen::Log::Loggable
   end
 
   def logger
-    @logger || self.class.logger
+    @logger || singleton_class.logger
   end
 end

data/lib/sqreen/logger.rb

@@ -28,6 +28,26 @@ module Sqreen
       create_error_logger
     end
 
+    def debug?
+      @logger.debug?
+    end
+
+    def info?
+      @logger.info?
+    end
+
+    def warn?
+      @logger.warn?
+    end
+
+    def error?
+      @logger.error?
+    end
+
+    def fatal?
+      @logger.fatal?
+    end
+
     def debug(msg = nil, &block)
       @logger.debug(msg, &block)
     end
@@ -45,6 +65,10 @@ module Sqreen
       @logger.error(msg, &block)
     end
 
+    def unknown(msg = nil, &block)
+      @logger.unknown(msg, &block)
+    end
+
     def add(severity, msg = nil, &block)
       send(SEVERITY_TO_METHOD[severity], msg, &block)
     end

data/lib/sqreen/metrics.rb

@@ -7,3 +7,4 @@ require 'sqreen/metrics/collect'
 require 'sqreen/metrics/average'
 require 'sqreen/metrics/sum'
 require 'sqreen/metrics/binning'
+require 'sqreen/metrics/req_detailed'

data/lib/sqreen/metrics/base.rb

@@ -12,6 +12,9 @@ module Sqreen
     FINISH_KEY = 'finish'.freeze
     # Base interface for a metric
     class Base
+      attr_accessor :name, :period # for signals serialization
+      attr_accessor :rule # optional
+
       def initialize(_opts={})
         @sample = nil
       end

data/lib/sqreen/metrics/req_detailed.rb

@@ -0,0 +1,41 @@
+require 'base64'
+require 'sqreen/mono_time'
+require 'sqreen/metrics/base'
+begin
+  require 'sq_detailed_metrics'
+rescue LoadError => _e # rubocop:disable Lint/HandleExceptions
+end
+
+module Sqreen
+  module Metric
+    class ReqDetailed < Base
+      attr_reader :num_requests
+
+      def initialize(opts = {})
+        raise 'SqDetailedMetrics unavailable' unless defined?(SqDetailedMetrics)
+        super(opts)
+        @coll = SqDetailedMetrics::RequestCollection.new
+        @start_time = Sqreen.time
+        @num_requests = 0
+      end
+
+      # @param [SqDetailedMetrics::Request] value
+      def update(_key, value)
+        @coll << value
+        @num_requests += 1
+      end
+
+      def next_sample(time)
+        data = @coll.serialize
+        @num_requests = 0
+        return nil unless data
+
+        {
+          OBSERVATION_KEY => { 'v1' => Base64.strict_encode64(data) },
+          START_KEY => @start_time,
+          FINISH_KEY => (@start_time = time),
+        }
+      end
+    end
+  end
+end

data/lib/sqreen/metrics_store.rb

@@ -3,6 +3,7 @@
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
+require 'sqreen/aggregated_metric'
 require 'sqreen/metrics'
 require 'sqreen/mono_time'
 require 'sqreen/metrics_store/unknown_metric'
@@ -26,12 +27,16 @@ module Sqreen
     def initialize
       @store = []
       @metrics = {} # name => (metric, period, start)
+      @mutex = Mutex.new
     end
 
     # Definition contains a name,period and aggregate at least
     # @param definition [Hash] a metric definition
+    # @param rule [RuleCB] the rule associated with this metric, if any
     # @param mklass [Object] Override metric object (used in testing)
-    def create_metric(definition, mklass = nil)
+    def create_metric(definition, rule = nil, mklass = nil)
+      @mutex.lock
+
       name = definition[NAME_KEY]
       kind = definition[KIND_KEY]
       klass = valid_metric(kind, name)
@@ -43,30 +48,41 @@ module Sqreen
         definition[PERIOD_KEY],
         nil # Start
       ]
+      metric.name = name
+      metric.rule = rule
+      metric.period = definition[PERIOD_KEY]
       metric
+    ensure
+      @mutex.unlock
     end
 
     def metric?(name)
       @metrics.key?(name)
     end
 
-    # @params at [Time] when is the store emptied
+    # @param at [Time] when is the store emptied
     def update(name, at, key, value)
+      @mutex.lock
       metric, period, start = @metrics[name]
       raise UnregisteredMetric, "Unknown metric #{name}" unless metric
       next_sample(name, at) if start.nil? || (start + period) < at
       metric.update(key, value)
+    ensure
+      @mutex.unlock
     end
 
     # Drains every metrics and returns the store content
-    # @params at [Time] when is the store emptied
+    # @param at [Time] when is the store emptied
     def publish(flush = true, at = Sqreen.time)
+      @mutex.lock
       @metrics.each do |name, (_, period, start)|
         next_sample(name, at) if flush || !start.nil? && (start + period) < at
       end
       out = @store
       @store = []
       out
+    ensure
+      @mutex.unlock
     end
 
     protected
@@ -75,15 +91,20 @@ module Sqreen
       metric = @metrics[name][0]
       r = metric.next_sample(at)
       @metrics[name][2] = at # new start
-      if r
-        r[NAME_KEY] = name
-        obs = r[Metric::OBSERVATION_KEY]
-        start_of_mono = Time.now.utc - Sqreen.time
-        r[Metric::START_KEY] = start_of_mono + r[Metric::START_KEY]
-        r[Metric::FINISH_KEY] = start_of_mono + r[Metric::FINISH_KEY]
-        @store << r if obs && (!obs.respond_to?(:empty?) || !obs.empty?)
-      end
-      r
+      return unless r
+
+      r[NAME_KEY] = name
+      obs = r[Metric::OBSERVATION_KEY]
+      return unless obs && (!obs.respond_to?(:empty?) || !obs.empty?)
+      start_of_mono = Time.now.utc - Sqreen.time
+
+      agg = AggregatedMetric.new
+      agg.metric = metric
+      agg.rule = agg.metric.rule
+      agg.start = start_of_mono + r[Metric::START_KEY]
+      agg.finish = start_of_mono + r[Metric::FINISH_KEY]
+      agg.data = obs
+      @store << agg
     end
 
     def valid_metric(kind, name)