sqreen 1.19.4 → 1.20.4.beta1

data/lib/sqreen/kit/signals/specialized/attack.rb

@@ -0,0 +1,57 @@
+# typed: ignore
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/kit/signals/point'
+require 'sqreen/kit/signals/dto_helper'
+
+# reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/payload/attack/2020-01-01T00_00_00_000Z/schema.cue
+
+module Sqreen
+  module Kit
+    module Signals
+      module Specialized
+      end
+    end
+  end
+end
+
+class Sqreen::Kit::Signals::Specialized::Attack < Sqreen::Kit::Signals::Point
+  add_mandatory_attrs :source, :time, :payload
+
+  validate_str_attr :signal_name, /\Asq\.agent\.attack\..+\z/
+  validate_str_attr :source, /\Asqreen:rule:[a-f0-9]{40}:.+\z/
+
+  def initialize(values = {})
+    self.payload_schema = Payload::SCHEMA_VERSION
+    self.time = values[:time] || Time.now
+    super
+  end
+
+  def payload=(payload)
+    unless payload.is_a?(Payload)
+      raise ArgumentError, "Payload should be a #{Payload}"
+    end
+    super
+  end
+
+  class Payload
+    include Sqreen::Kit::Signals::DtoHelper
+
+    SCHEMA_VERSION = 'attack/2020-01-01T00:00:00.000Z'.freeze
+
+    add_mandatory_attrs :test, :block, :infos
+
+    # all are mandatory
+
+    # @return [Boolean]
+    attr_accessor :test
+
+    # @return [Boolean]
+    attr_accessor :block
+
+    # @return [Hash{String|Symbol=>Object}]
+    attr_accessor :infos
+  end
+end

data/lib/sqreen/kit/signals/specialized/binning_metric.rb

@@ -0,0 +1,76 @@
+# typed: ignore
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/kit/signals/metric'
+require 'sqreen/kit/signals/dto_helper'
+
+# reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/payload/binning_metric/2020-01-01T00_00_00_000Z/schema.cue
+
+module Sqreen
+  module Kit
+    module Signals
+      module Specialized
+      end
+    end
+  end
+end
+
+class Sqreen::Kit::Signals::Specialized::BinningMetric < Sqreen::Kit::Signals::Metric
+  add_mandatory_attrs :source, :time, :payload
+
+  validate_str_attr :signal_name, /\Asq\.agent\.metric\..+\z/
+
+  def initialize(args)
+    self.payload_schema = Payload::SCHEMA_VERSION
+    super
+  end
+
+  class Payload
+    include Sqreen::Kit::Signals::DtoHelper
+
+    SCHEMA_VERSION = 'metric_binning/2020-01-01T00:00:00.000Z'.freeze
+
+    add_mandatory_attrs :capture_interval_s,
+                        :date_started,
+                        :date_ended,
+                        :max, :base, :unit, :bins
+
+    # mandatory
+    # @return [Integer]
+    attr_accessor :capture_interval_s
+
+    # mandatory
+    # @param [Time]
+    # @return [String]
+    attr_accessor_time :date_started
+
+    # mandatory
+    # @param [Time]
+    # @return [String]
+    attr_accessor_time :date_ended
+
+    # mandatory
+    # @return [Float]
+    attr_accessor :max
+
+    # mandatory
+    # @return [Float]
+    attr_accessor :base
+
+    # mandatory
+    # @return [Float]
+    attr_accessor :unit
+
+    # mandatory
+    # @return [Hash{Integer=>Integer}]
+    attr_accessor :bins
+
+    def to_h
+      {
+        kind: 'binning',
+      }.merge(super)
+    end
+  end
+end

data/lib/sqreen/kit/signals/specialized/http_trace.rb

@@ -0,0 +1,26 @@
+# typed: ignore
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/kit/signals/trace'
+require 'sqreen/kit/signals/context/http_context'
+require 'sqreen/kit/signals/dto_helper'
+
+module Sqreen
+  module Kit
+    module Signals
+      module Specialized
+      end
+    end
+  end
+end
+
+class Sqreen::Kit::Signals::Specialized::HttpTrace < Sqreen::Kit::Signals::Trace
+  add_mandatory_attrs :context
+
+  def initialize(values = {})
+    self.context_schema = ::Sqreen::Kit::Signals::Context::HttpContext::SCHEMA_VERSION
+    super
+  end
+end

data/lib/sqreen/kit/signals/specialized/sdk_track_call.rb

@@ -0,0 +1,50 @@
+# typed: ignore
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/kit/signals/point'
+require 'sqreen/kit/signals/dto_helper'
+
+# reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/payload/sdk_call/2020-01-01T00_00_00_000Z/schema.cue
+
+module Sqreen
+  module Kit
+    module Signals
+      module Specialized
+      end
+    end
+  end
+end
+
+class Sqreen::Kit::Signals::Specialized::SdkTrackCall < Sqreen::Kit::Signals::Point
+  add_mandatory_attrs :source, :time, :payload
+
+  validate_str_attr :signal_name, /\Asq\.sdk\..+\z/
+
+  def initialize(values = {})
+    self.payload_schema = Payload::SCHEMA_VERSION
+    self.source = "sqreen:sdk:track"
+    self.time = values[:time] || Time.now
+    super
+  end
+
+  def payload=(payload)
+    unless payload.is_a?(Payload)
+      raise ArgumentError, "Payload should be a #{Payload}"
+    end
+    super
+  end
+
+  class Payload
+    include Sqreen::Kit::Signals::DtoHelper
+
+    SCHEMA_VERSION = 'track_event/2020-01-01T00:00:00.000Z'.freeze
+
+    # @return [Hash{String|Symbol=>Object}]
+    attr_accessor :properties
+
+    # @return [Hash{String|Symbol=>String}]
+    attr_accessor :user_identifiers
+  end
+end

data/lib/sqreen/kit/signals/specialized/sqreen_exception.rb

@@ -0,0 +1,57 @@
+# typed: ignore
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/kit/signals/point'
+require 'sqreen/kit/signals/dto_helper'
+
+# reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/payload/sqreen_exception/2020-01-01T00_00_00_000Z/schema.cue
+
+module Sqreen
+  module Kit
+    module Signals
+      module Specialized
+      end
+    end
+  end
+end
+
+class Sqreen::Kit::Signals::Specialized::SqreenException < Sqreen::Kit::Signals::Point
+  PAYLOAD_SCHEMA_VERSION = 'sqreen_exception/2020-01-01T00:00:00.000Z'.freeze
+
+  # @return [Hash]
+  attr_accessor :infos
+
+  # @return [Exception]
+  attr_accessor :ruby_exception
+
+  add_mandatory_attrs :source, :time, :ruby_exception
+
+  validate_str_attr :source, /\A(?:sqreen:rule:[a-f0-9]{40}:.+)|(?:sqreen:agent:.+)\z/
+
+  def self.attributes_for_to_h_self
+    [] # don't include ruby_exception in list of attributes for to_h
+  end
+
+  def initialize(values = {})
+    self.payload_schema = PAYLOAD_SCHEMA_VERSION
+    self.signal_name = 'sq.agent.exception'
+    self.time = values[:time] || Time.now
+    super
+  end
+
+  def payload
+    return nil unless @ruby_exception
+    compact_hash({
+                   klass: @ruby_exception.class.to_s,
+                   message: @ruby_exception.message,
+                   infos: @infos,
+                 })
+  end
+
+  def location
+    return nil unless @ruby_exception
+    Sqreen::Kit::Signals::Location.new(exception: @ruby_exception)
+  end
+end

data/lib/sqreen/legacy/instrumentation.rb

@@ -109,7 +109,7 @@ module Legacy
             break if res.is_a?(Hash) && res[:skip_rem_cbs]
           rescue StandardError => e
             Sqreen.log.warn { "we catch an exception: #{e.inspect}" }
-            Sqreen.log.debug e.backtrace
+            Sqreen.log.debug { e.backtrace }
             if cb.respond_to?(:record_exception)
               cb.record_exception(e)
             else
@@ -162,7 +162,7 @@ module Legacy
             returns << res
           rescue StandardError => e
             Sqreen.log.warn { "we catch an exception: #{e.inspect}" }
-            Sqreen.log.debug e.backtrace
+            Sqreen.log.debug { e.backtrace }
             if cb.respond_to?(:record_exception)
               cb.record_exception(e)
             else
@@ -597,16 +597,25 @@ module Legacy
         method = cb.method
         key = [klass, method]
 
+        if (call_count = ENV['SQREEN_DEBUG_CALL_COUNT'])
+          call_count = JSON.parse(call_count)
+          if cb.respond_to?(:rule_name) && call_count.key?(cb.rule_name)
+            count = call_count[cb.rule_name]
+            Sqreen.log.debug { "override rule:#{cb.rule_name} call_count:#{count.inspect}" }
+            cb.instance_eval { @call_count_interval = call_count[cb.rule_name] }
+          end
+        end
+
         @@record_request_hookpoints << key if cb.is_a?(Sqreen::Rules::RecordRequestContext)
 
         already_overriden = @@overriden_methods.include? key
 
         if !already_overriden
           if is_class_method?(klass, method)
-            Sqreen.log.debug "overriding class method for #{cb}"
+            Sqreen.log.debug { "overriding class method for #{cb}" }
             success = override_class_method(klass, method)
           elsif is_instance_method?(klass, method)
-            Sqreen.log.debug "overriding instance method for #{cb}"
+            Sqreen.log.debug { "overriding instance method for #{cb}" }
             success = override_instance_method(klass, method)
           else
             # FIXME: Override define_method and other dynamic ways to
@@ -623,7 +632,7 @@ module Legacy
 
           @@overriden_methods += [key] if success
         else
-          Sqreen.log.debug "#{key} was already overriden"
+          Sqreen.log.debug { "#{key} was already overriden" }
         end
 
         if klass != Object && klass != Kernel && !Sqreen.features['instrument_all_instances'] && !defined?(::JRUBY_VERSION)
@@ -638,7 +647,7 @@ module Legacy
           end
         end
 
-        Sqreen.log.debug "Adding callback #{cb} for #{klass} #{method}"
+        Sqreen.log.debug { "Adding callback #{cb} for #{klass} #{method}" }
         @@registered_callbacks.add(cb)
         @@unovertimable_hookpoints << key unless cb.overtimeable
         @@instrumented_pid = Process.pid
@@ -659,7 +668,7 @@ module Legacy
 
       already_overriden = @@overriden_methods.include? key
       unless already_overriden
-        Sqreen.log.debug "#{key} apparently not overridden"
+        Sqreen.log.debug { "#{key} apparently not overridden" }
       end
 
       defined_cbs = @@registered_callbacks.get(klass, method).flatten
@@ -667,17 +676,17 @@ module Legacy
       nb_removed = 0
       defined_cbs.each do |found_cb|
         if found_cb == cb
-          Sqreen.log.debug "Removing callback #{found_cb}"
+          Sqreen.log.debug { "Removing callback #{found_cb}" }
           @@registered_callbacks.remove(found_cb)
           nb_removed += 1
         else
-          Sqreen.log.debug "Not removing callback #{found_cb} (remains #{defined_cbs.size} cbs)"
+          Sqreen.log.debug { "Not removing callback #{found_cb} (remains #{defined_cbs.size} cbs)" }
         end
       end
 
       return unless nb_removed == defined_cbs.size
 
-      Sqreen.log.debug "Removing overriden method #{key}"
+      Sqreen.log.debug { "Removing overriden method #{key}" }
       @@overriden_methods.delete(key)
 
       if is_class_method?(klass, method)
@@ -705,6 +714,7 @@ module Legacy
           remove_callback_no_lock(cb)
         end
         Sqreen.instrumentation_ready = false
+        Sqreen.log.info('Instrumentation deactivated')
       end
     end
 
@@ -757,6 +767,8 @@ module Legacy
       ### globally declare instrumentation ready
       ### from within instance method? not even thread local?
       Sqreen.instrumentation_ready = true
+
+      Sqreen.log.info('Instrumentation activated')
     end
 
     def initialize(metrics_engine = nil)

data/lib/sqreen/legacy/old_event_submission_strategy.rb

@@ -0,0 +1,221 @@
+# typed: ignore
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/aggregated_metric'
+require 'sqreen/log/loggable'
+require 'sqreen/legacy/waf_redactions'
+
+module Sqreen
+  module Legacy
+    # see also Sqreen::Signals::SignalsSubmissionStrategy
+    # usage in Sqreen:Session
+    class OldEventSubmissionStrategy
+      include Sqreen::Log::Loggable
+
+      RETRY_MANY = 301
+
+      def initialize(post_proc)
+        @post_proc = post_proc
+      end
+
+      def post_metrics(metrics)
+        return if metrics.nil? || metrics.empty?
+        payload = { metrics: metrics.map { |m| EventToHash.convert_agg_metric(m) } }
+        post('metrics', payload, {}, RETRY_MANY)
+      end
+
+      # @param attack [Sqreen::Attack]
+      def post_attack(attack)
+        post('attack', EventToHash.convert_attack(attack), {}, RETRY_MANY)
+      end
+
+      # @param [Sqreen::RequestRecord] request_record
+      def post_request_record(request_record)
+        rr_hash = EventToHash.convert_request_record(request_record)
+        post('request_record', rr_hash, {}, RETRY_MANY)
+      end
+
+      # Post an exception to Sqreen for analysis
+      # @param exception [RemoteException] Exception and context to be sent over
+      def post_sqreen_exception(exception)
+        data = EventToHash.convert_exception(exception)
+        post('sqreen_exception', data, {}, 5)
+      rescue StandardError => e
+        logger.warn(format('Could not post exception (network down? %s) %s',
+                           e.inspect,
+                           exception.inspect))
+        nil
+      end
+
+      def post_batch(events)
+        batch = events.map do |event|
+          h = case event
+              when AggregatedMetric
+                logger.warn "Aggregated metric event in non-signal mode. Signals disabled at runtime?"
+                next
+              when Attack # in practice only found inside req rec
+                EventToHash.convert_attack event
+              when RemoteException
+                EventToHash.convert_exception event
+              when RequestRecord
+                EventToHash.convert_request_record event
+              else
+                logger.warn "Unexpected event type: #{event}"
+                next
+              end
+          h['event_type'] = event_kind(event)
+          h
+        end
+        Sqreen.log.debug do
+          tally = Hash[events.group_by(&:class).map { |k, v| [k, v.count] }]
+          "Doing batch with the following tally of event types: #{tally}"
+        end
+        post('batch', { batch: batch }, {}, RETRY_MANY)
+      end
+
+      private
+
+      # see +Sqreen::Session.post+
+      def post(*args)
+        @post_proc[*args]
+      end
+
+      def event_kind(event)
+        case event
+        when Sqreen::RemoteException then 'sqreen_exception'
+        when Sqreen::Attack then 'attack'
+        when Sqreen::RequestRecord then 'request_record'
+        end
+      end
+    end
+
+    module EventToHash
+      class << self
+        # @param attack [Sqreen::Attack]
+        def convert_attack(attack)
+          payload = attack.payload
+          res = {}
+          rule_p = payload['rule']
+          request_p = payload['request']
+          res[:rule_name]    = rule_p['name']         if rule_p && rule_p['name']
+          res[:rulespack_id] = rule_p['rulespack_id'] if rule_p && rule_p['rulespack_id']
+          res[:test]         = rule_p['test']         if rule_p && rule_p['test']
+          res[:infos]        = payload['infos']       if payload['infos']
+          res[:time]         = attack.time
+          res[:client_ip]    = request_p[:addr]       if request_p && request_p[:addr]
+          res[:request]      = request_p              if request_p
+          res[:params]       = payload['params']      if payload['params']
+          res[:context]      = payload['context']     if payload['context']
+          res[:headers]      = payload['headers']     if payload['headers']
+          res
+        end
+
+        # @param [Sqreen::RequestRecord] rr
+        def convert_request_record(rr)
+          res = { :version => '20171208' }
+          payload = rr.payload
+
+          if payload[:observed]
+            res[:observed] = payload[:observed].dup
+            rulespack = nil
+            if rr.observed[:attacks]
+              res[:observed][:attacks] = rr.observed[:attacks].map do |att|
+                natt = att.dup
+                [:attack_type, :block].each { |k| natt.delete(k) } # signals stuff
+                rulespack = natt.delete(:rulespack_id) || rulespack
+                natt
+              end
+            end
+            if rr.observed[:sqreen_exceptions]
+              res[:observed][:sqreen_exceptions] = rr.observed[:sqreen_exceptions].map do |exc|
+                nex = exc.dup
+                excp = nex.delete(:exception)
+                if excp
+                  nex[:message] = excp.message
+                  nex[:klass] = excp.class.name
+                end
+                rulespack = nex.delete(:rulespack_id) || rulespack
+                nex
+              end
+            end
+            res[:rulespack_id] = rulespack unless rulespack.nil?
+            if rr.observed[:observations]
+              res[:observed][:observations] = rr.observed[:observations].map do |cat, key, value, time|
+                { :category => cat, :key => key, :value => value, :time => time }
+              end
+            end
+            if rr.observed[:sdk] # rubocop:disable Style/IfUnlessModifier
+              res[:observed][:sdk] = rr.processed_sdk_calls
+            end
+          end
+          res[:local] = payload['local'] if payload['local']
+          if payload['request']
+            res[:request] = payload['request'].dup
+            res[:client_ip] = res[:request].delete(:client_ip) if res[:request][:client_ip]
+          else
+            res[:request] = {}
+          end
+          if payload['response']
+            res[:response] = payload['response'].dup
+          else
+            res[:response] = {}
+          end
+
+          res[:request][:parameters] = payload['params'] if payload['params']
+          res[:request][:headers] = payload['headers'] if payload['headers']
+
+          res = Sqreen::EncodingSanitizer.sanitize(res)
+
+          if rr.redactor
+            res[:request], redacted = rr.redactor.redact(res[:request])
+            redacted = redacted.uniq
+            if redacted.any? && res[:observed] && res[:observed][:attacks]
+              res[:observed][:attacks] = WafRedactions.redact_attacks!(res[:observed][:attacks], redacted)
+            end
+            if redacted.any? && res[:observed] && res[:observed][:sqreen_exceptions]
+              res[:observed][:sqreen_exceptions] = WafRedactions.redact_exceptions!(res[:observed][:sqreen_exceptions], redacted)
+            end
+          end
+
+          res
+        end
+
+        # @param exception_evt [Sqreen::RemoteException]
+        def convert_exception(exception_evt)
+          payload = exception_evt.payload
+          exception = payload['exception']
+          ev = {
+            :klass => exception.class.name,
+            :message => exception.message,
+            :params => payload['request_params'],
+            :time => payload['time'],
+            :infos => {
+              :client_ip => payload['client_ip'],
+            },
+            :request => payload['request_infos'],
+            :headers => payload['headers'],
+            :rule_name => payload['rule_name'],
+            :rulespack_id => payload['rulespack_id'],
+          }
+
+          ev[:infos].merge!(payload['infos']) if payload['infos']
+          return ev unless exception.backtrace
+          ev[:context] = { :backtrace => exception.backtrace.map(&:to_s) }
+          ev
+        end
+
+        # @param [Sqreen::AggregatedMetric] agg_metric
+        def convert_agg_metric(agg_metric)
+          {
+            name: agg_metric.name,
+            observation: agg_metric.data,
+            start: agg_metric.start,
+            finish: agg_metric.finish,
+          }
+        end
+      end
+    end
+  end
+end