sqreen 1.19.1 → 1.21.0.beta1

data/lib/sqreen/frameworks/generic.rb

@@ -22,8 +22,17 @@ module Sqreen
       include RequestRecorder
       attr_accessor :sqreen_configuration
 
+      attr_writer :req_start_cb, :req_end_cb
+
       def initialize
         clean_request_record
+
+        # for notifying the ecosystem of request boundaries
+        # XXX: this should be refactored. It shouldn't be
+        # the framework doing these notifications to the ecosystem
+        # Probably the rule callback should do it itself
+        @req_start_cb = Proc.new {}
+        @req_end_cb = Proc.new {}
       end
 
       # What kind of database is this
@@ -251,8 +260,12 @@ module Sqreen
       # Nota: cleanup should be performed at end of request (see clean_request)
       def store_request(object)
         return unless ensure_rack_loaded
+
+        rack_req = Rack::Request.new(object)
+        @req_start_cb.call(rack_req)
+
         self.remaining_perf_budget = Sqreen.performance_budget
-        SharedStorage.set(:request, Rack::Request.new(object))
+        SharedStorage.set(:request, rack_req)
         SharedStorage.set(:xss_params, nil)
         SharedStorage.set(:whitelisted, nil)
         SharedStorage.set(:request_overtime, nil)
@@ -281,6 +294,7 @@ module Sqreen
         SharedStorage.set(:xss_params, nil)
         SharedStorage.set(:whitelisted, nil)
         SharedStorage.set(:request_overtime, nil)
+        @req_end_cb.call
       end
 
       def remaining_perf_budget

data/lib/sqreen/frameworks/request_recorder.rb

@@ -58,12 +58,20 @@ module Sqreen
       Sqreen.log.debug { "close_request_record called. observed_items: #{observed_items}" }
 
       clean_request_record if observed_items.nil?
-      if only_metric_observation
+      if Sqreen.features['use_signals'] || only_metric_observation
         push_metrics(observations_queue, queue)
-        return clean_request_record
       end
+
+      if only_metric_observation
+        clean_request_record
+        return
+      end
+
+      # signals require request section to be present
+      payload_requests << 'request'
       payload = payload_creator.payload(payload_requests)
       payload[:observed] = observed_items
+
       queue.push create_request_record(payload)
       clean_request_record
     end
@@ -79,10 +87,13 @@ module Sqreen
       @redactor ||= SensitiveDataRedactor.from_config
     end
 
+    # pushes metric observations to the observations queue
+    # and clears the list for the request record
     def push_metrics(observations_queue, event_queue)
       observed_items[:observations].each do |obs|
         observations_queue.push obs
       end
+      observed_items[:observations] = []
       return unless observations_queue.size > MAX_OBS_QUEUE_LENGTH / 2
       event_queue.push Sqreen::METRICS_EVENT
     end

data/lib/sqreen/graft/call.rb

@@ -138,6 +138,15 @@ module Sqreen
         @blips << Timer.read
       end
 
+      def include_measurements(another_timer)
+        @blips += another_timer.instance_variable_get(:@blips)
+      end
+
+      def start_and_end
+        raise 'Not exactly two measurements recorded' unless size == 2
+        @blips
+      end
+
       def size
         @blips.size
       end

data/lib/sqreen/kit/signals/specialized/aggregated_metric.rb

@@ -0,0 +1,72 @@
+# typed: ignore
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/kit/signals/metric'
+require 'sqreen/kit/signals/dto_helper'
+
+# reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/payload/binning_metric/2020-01-01T00_00_00_000Z/schema.cue
+
+module Sqreen
+  module Kit
+    module Signals
+      module Specialized
+      end
+    end
+  end
+end
+
+class Sqreen::Kit::Signals::Specialized::AggregatedMetric < Sqreen::Kit::Signals::Metric
+  add_mandatory_attrs :source, :payload
+
+  validate_str_attr :signal_name, /\Asq\.agent\.metric\..+\z/
+
+  def initialize(args)
+    self.payload_schema = Payload::SCHEMA_VERSION
+    super
+  end
+
+  class Payload
+    include Sqreen::Kit::Signals::DtoHelper
+
+    SCHEMA_VERSION = 'metric/2020-01-01T00:00:00.000Z'.freeze
+
+    add_mandatory_attrs :capture_interval_s,
+                        :date_started,
+                        :date_ended,
+                        :values,
+                        :kind
+
+    # mandatory
+    # @return [Integer]
+    attr_accessor :capture_interval_s
+
+    # mandatory
+    # @param [Time]
+    # @return [String]
+    attr_accessor_time :date_started
+
+    # mandatory
+    # @param [Time]
+    # @return [String]
+    attr_accessor_time :date_ended
+
+    # mandatory
+    # @return [Hash{String=>Object}]
+    attr_writer :values
+    def values
+      return nil if @values.nil?
+      @values.map do |k, v|
+        {
+          key: k.is_a?(Hash) || k.is_a?(Array) ? k : k.to_s,
+          value: v,
+        }
+      end
+    end
+
+    # mandatory
+    # @return [String]
+    attr_accessor :kind
+  end
+end

data/lib/sqreen/kit/signals/specialized/attack.rb

@@ -0,0 +1,57 @@
+# typed: ignore
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/kit/signals/point'
+require 'sqreen/kit/signals/dto_helper'
+
+# reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/payload/attack/2020-01-01T00_00_00_000Z/schema.cue
+
+module Sqreen
+  module Kit
+    module Signals
+      module Specialized
+      end
+    end
+  end
+end
+
+class Sqreen::Kit::Signals::Specialized::Attack < Sqreen::Kit::Signals::Point
+  add_mandatory_attrs :source, :time, :payload
+
+  validate_str_attr :signal_name, /\Asq\.agent\.attack\..+\z/
+  validate_str_attr :source, /\Asqreen:rule:[a-f0-9]{40}:.+\z/
+
+  def initialize(values = {})
+    self.payload_schema = Payload::SCHEMA_VERSION
+    self.time = values[:time] || Time.now
+    super
+  end
+
+  def payload=(payload)
+    unless payload.is_a?(Payload)
+      raise ArgumentError, "Payload should be a #{Payload}"
+    end
+    super
+  end
+
+  class Payload
+    include Sqreen::Kit::Signals::DtoHelper
+
+    SCHEMA_VERSION = 'attack/2020-01-01T00:00:00.000Z'.freeze
+
+    add_mandatory_attrs :test, :block, :infos
+
+    # all are mandatory
+
+    # @return [Boolean]
+    attr_accessor :test
+
+    # @return [Boolean]
+    attr_accessor :block
+
+    # @return [Hash{String|Symbol=>Object}]
+    attr_accessor :infos
+  end
+end

data/lib/sqreen/kit/signals/specialized/binning_metric.rb

@@ -0,0 +1,76 @@
+# typed: ignore
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/kit/signals/metric'
+require 'sqreen/kit/signals/dto_helper'
+
+# reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/payload/binning_metric/2020-01-01T00_00_00_000Z/schema.cue
+
+module Sqreen
+  module Kit
+    module Signals
+      module Specialized
+      end
+    end
+  end
+end
+
+class Sqreen::Kit::Signals::Specialized::BinningMetric < Sqreen::Kit::Signals::Metric
+  add_mandatory_attrs :source, :time, :payload
+
+  validate_str_attr :signal_name, /\Asq\.agent\.metric\..+\z/
+
+  def initialize(args)
+    self.payload_schema = Payload::SCHEMA_VERSION
+    super
+  end
+
+  class Payload
+    include Sqreen::Kit::Signals::DtoHelper
+
+    SCHEMA_VERSION = 'metric_binning/2020-01-01T00:00:00.000Z'.freeze
+
+    add_mandatory_attrs :capture_interval_s,
+                        :date_started,
+                        :date_ended,
+                        :max, :base, :unit, :bins
+
+    # mandatory
+    # @return [Integer]
+    attr_accessor :capture_interval_s
+
+    # mandatory
+    # @param [Time]
+    # @return [String]
+    attr_accessor_time :date_started
+
+    # mandatory
+    # @param [Time]
+    # @return [String]
+    attr_accessor_time :date_ended
+
+    # mandatory
+    # @return [Float]
+    attr_accessor :max
+
+    # mandatory
+    # @return [Float]
+    attr_accessor :base
+
+    # mandatory
+    # @return [Float]
+    attr_accessor :unit
+
+    # mandatory
+    # @return [Hash{Integer=>Integer}]
+    attr_accessor :bins
+
+    def to_h
+      {
+        kind: 'binning',
+      }.merge(super)
+    end
+  end
+end

data/lib/sqreen/kit/signals/specialized/http_trace.rb

@@ -0,0 +1,26 @@
+# typed: ignore
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/kit/signals/trace'
+require 'sqreen/kit/signals/context/http_context'
+require 'sqreen/kit/signals/dto_helper'
+
+module Sqreen
+  module Kit
+    module Signals
+      module Specialized
+      end
+    end
+  end
+end
+
+class Sqreen::Kit::Signals::Specialized::HttpTrace < Sqreen::Kit::Signals::Trace
+  add_mandatory_attrs :context
+
+  def initialize(values = {})
+    self.context_schema = ::Sqreen::Kit::Signals::Context::HttpContext::SCHEMA_VERSION
+    super
+  end
+end

data/lib/sqreen/kit/signals/specialized/sdk_track_call.rb

@@ -0,0 +1,50 @@
+# typed: ignore
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/kit/signals/point'
+require 'sqreen/kit/signals/dto_helper'
+
+# reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/payload/sdk_call/2020-01-01T00_00_00_000Z/schema.cue
+
+module Sqreen
+  module Kit
+    module Signals
+      module Specialized
+      end
+    end
+  end
+end
+
+class Sqreen::Kit::Signals::Specialized::SdkTrackCall < Sqreen::Kit::Signals::Point
+  add_mandatory_attrs :source, :time, :payload
+
+  validate_str_attr :signal_name, /\Asq\.sdk\..+\z/
+
+  def initialize(values = {})
+    self.payload_schema = Payload::SCHEMA_VERSION
+    self.source = "sqreen:sdk:track"
+    self.time = values[:time] || Time.now
+    super
+  end
+
+  def payload=(payload)
+    unless payload.is_a?(Payload)
+      raise ArgumentError, "Payload should be a #{Payload}"
+    end
+    super
+  end
+
+  class Payload
+    include Sqreen::Kit::Signals::DtoHelper
+
+    SCHEMA_VERSION = 'track_event/2020-01-01T00:00:00.000Z'.freeze
+
+    # @return [Hash{String|Symbol=>Object}]
+    attr_accessor :properties
+
+    # @return [Hash{String|Symbol=>String}]
+    attr_accessor :user_identifiers
+  end
+end

data/lib/sqreen/kit/signals/specialized/sqreen_exception.rb

@@ -0,0 +1,57 @@
+# typed: ignore
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/kit/signals/point'
+require 'sqreen/kit/signals/dto_helper'
+
+# reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/payload/sqreen_exception/2020-01-01T00_00_00_000Z/schema.cue
+
+module Sqreen
+  module Kit
+    module Signals
+      module Specialized
+      end
+    end
+  end
+end
+
+class Sqreen::Kit::Signals::Specialized::SqreenException < Sqreen::Kit::Signals::Point
+  PAYLOAD_SCHEMA_VERSION = 'sqreen_exception/2020-01-01T00:00:00.000Z'.freeze
+
+  # @return [Hash]
+  attr_accessor :infos
+
+  # @return [Exception]
+  attr_accessor :ruby_exception
+
+  add_mandatory_attrs :source, :time, :ruby_exception
+
+  validate_str_attr :source, /\A(?:sqreen:rule:[a-f0-9]{40}:.+)|(?:sqreen:agent:.+)\z/
+
+  def self.attributes_for_to_h_self
+    [] # don't include ruby_exception in list of attributes for to_h
+  end
+
+  def initialize(values = {})
+    self.payload_schema = PAYLOAD_SCHEMA_VERSION
+    self.signal_name = 'sq.agent.exception'
+    self.time = values[:time] || Time.now
+    super
+  end
+
+  def payload
+    return nil unless @ruby_exception
+    compact_hash({
+                   klass: @ruby_exception.class.to_s,
+                   message: @ruby_exception.message,
+                   infos: @infos,
+                 })
+  end
+
+  def location
+    return nil unless @ruby_exception
+    Sqreen::Kit::Signals::Location.new(exception: @ruby_exception)
+  end
+end

data/lib/sqreen/legacy/old_event_submission_strategy.rb

@@ -0,0 +1,227 @@
+# typed: ignore
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/aggregated_metric'
+require 'sqreen/log/loggable'
+require 'sqreen/legacy/waf_redactions'
+
+module Sqreen
+  module Legacy
+    # see also Sqreen::Signals::SignalsSubmissionStrategy
+    # usage in Sqreen:Session
+    class OldEventSubmissionStrategy
+      include Sqreen::Log::Loggable
+
+      RETRY_MANY = 301
+
+      def initialize(post_proc)
+        @post_proc = post_proc
+      end
+
+      def post_metrics(metrics)
+        return if metrics.nil? || metrics.empty?
+        payload = { metrics: metrics.map { |m| EventToHash.convert_agg_metric(m) } }
+        post('metrics', payload, {}, RETRY_MANY)
+      end
+
+      # @param attack [Sqreen::Attack]
+      def post_attack(attack)
+        post('attack', EventToHash.convert_attack(attack), {}, RETRY_MANY)
+      end
+
+      # @param [Sqreen::RequestRecord] request_record
+      def post_request_record(request_record)
+        rr_hash = EventToHash.convert_request_record(request_record)
+        post('request_record', rr_hash, {}, RETRY_MANY)
+      end
+
+      # Post an exception to Sqreen for analysis
+      # @param exception [RemoteException] Exception and context to be sent over
+      def post_sqreen_exception(exception)
+        data = EventToHash.convert_exception(exception)
+        post('sqreen_exception', data, {}, 5)
+      rescue StandardError => e
+        logger.warn(format('Could not post exception (network down? %s) %s',
+                           e.inspect,
+                           exception.inspect))
+        nil
+      end
+
+      def post_batch(events)
+        batch = events.map do |event|
+          h = case event
+              when AggregatedMetric
+                logger.warn "Aggregated metric event in non-signal mode. Signals disabled at runtime?"
+                next
+              when Sqreen::Kit::Signals::Signal
+                logger.warn "Signal event in non-signal mode"
+                next
+              when Sqreen::Kit::Signals::Trace
+                logger.warn "Trace event in non-signal mode"
+                next
+              when Attack # in practice only found inside req rec
+                EventToHash.convert_attack event
+              when RemoteException
+                EventToHash.convert_exception event
+              when RequestRecord
+                EventToHash.convert_request_record event
+              else
+                logger.warn "Unexpected event type: #{event}"
+                next
+              end
+          h['event_type'] = event_kind(event)
+          h
+        end
+        Sqreen.log.debug do
+          tally = Hash[events.group_by(&:class).map { |k, v| [k, v.count] }]
+          "Doing batch with the following tally of event types: #{tally}"
+        end
+        post('batch', { batch: batch.compact }, {}, RETRY_MANY)
+      end
+
+      private
+
+      # see +Sqreen::Session.post+
+      def post(*args)
+        @post_proc[*args]
+      end
+
+      def event_kind(event)
+        case event
+        when Sqreen::RemoteException then 'sqreen_exception'
+        when Sqreen::Attack then 'attack'
+        when Sqreen::RequestRecord then 'request_record'
+        end
+      end
+    end
+
+    module EventToHash
+      class << self
+        # @param attack [Sqreen::Attack]
+        def convert_attack(attack)
+          payload = attack.payload
+          res = {}
+          rule_p = payload['rule']
+          request_p = payload['request']
+          res[:rule_name]    = rule_p['name']         if rule_p && rule_p['name']
+          res[:rulespack_id] = rule_p['rulespack_id'] if rule_p && rule_p['rulespack_id']
+          res[:test]         = rule_p['test']         if rule_p && rule_p['test']
+          res[:infos]        = payload['infos']       if payload['infos']
+          res[:time]         = attack.time
+          res[:client_ip]    = request_p[:addr]       if request_p && request_p[:addr]
+          res[:request]      = request_p              if request_p
+          res[:params]       = payload['params']      if payload['params']
+          res[:context]      = payload['context']     if payload['context']
+          res[:headers]      = payload['headers']     if payload['headers']
+          res
+        end
+
+        # @param [Sqreen::RequestRecord] rr
+        def convert_request_record(rr)
+          res = { :version => '20171208' }
+          payload = rr.payload
+
+          if payload[:observed]
+            res[:observed] = payload[:observed].dup
+            rulespack = nil
+            if rr.observed[:attacks]
+              res[:observed][:attacks] = rr.observed[:attacks].map do |att|
+                natt = att.dup
+                [:attack_type, :block].each { |k| natt.delete(k) } # signals stuff
+                rulespack = natt.delete(:rulespack_id) || rulespack
+                natt
+              end
+            end
+            if rr.observed[:sqreen_exceptions]
+              res[:observed][:sqreen_exceptions] = rr.observed[:sqreen_exceptions].map do |exc|
+                nex = exc.dup
+                excp = nex.delete(:exception)
+                if excp
+                  nex[:message] = excp.message
+                  nex[:klass] = excp.class.name
+                end
+                rulespack = nex.delete(:rulespack_id) || rulespack
+                nex
+              end
+            end
+            res[:rulespack_id] = rulespack unless rulespack.nil?
+            if rr.observed[:observations]
+              res[:observed][:observations] = rr.observed[:observations].map do |cat, key, value, time|
+                { :category => cat, :key => key, :value => value, :time => time }
+              end
+            end
+            if rr.observed[:sdk] # rubocop:disable Style/IfUnlessModifier
+              res[:observed][:sdk] = rr.processed_sdk_calls
+            end
+          end
+          res[:local] = payload['local'] if payload['local']
+          if payload['request']
+            res[:request] = payload['request'].dup
+            res[:client_ip] = res[:request].delete(:client_ip) if res[:request][:client_ip]
+          else
+            res[:request] = {}
+          end
+          if payload['response']
+            res[:response] = payload['response'].dup
+          else
+            res[:response] = {}
+          end
+
+          res[:request][:parameters] = payload['params'] if payload['params']
+          res[:request][:headers] = payload['headers'] if payload['headers']
+
+          res = Sqreen::EncodingSanitizer.sanitize(res)
+
+          if rr.redactor
+            res[:request], redacted = rr.redactor.redact(res[:request])
+            redacted = redacted.uniq
+            if redacted.any? && res[:observed] && res[:observed][:attacks]
+              res[:observed][:attacks] = WafRedactions.redact_attacks!(res[:observed][:attacks], redacted)
+            end
+            if redacted.any? && res[:observed] && res[:observed][:sqreen_exceptions]
+              res[:observed][:sqreen_exceptions] = WafRedactions.redact_exceptions!(res[:observed][:sqreen_exceptions], redacted)
+            end
+          end
+
+          res
+        end
+
+        # @param exception_evt [Sqreen::RemoteException]
+        def convert_exception(exception_evt)
+          payload = exception_evt.payload
+          exception = payload['exception']
+          ev = {
+            :klass => exception.class.name,
+            :message => exception.message,
+            :params => payload['request_params'],
+            :time => payload['time'],
+            :infos => {
+              :client_ip => payload['client_ip'],
+            },
+            :request => payload['request_infos'],
+            :headers => payload['headers'],
+            :rule_name => payload['rule_name'],
+            :rulespack_id => payload['rulespack_id'],
+          }
+
+          ev[:infos].merge!(payload['infos']) if payload['infos']
+          return ev unless exception.backtrace
+          ev[:context] = { :backtrace => exception.backtrace.map(&:to_s) }
+          ev
+        end
+
+        # @param [Sqreen::AggregatedMetric] agg_metric
+        def convert_agg_metric(agg_metric)
+          {
+            name: agg_metric.name,
+            observation: agg_metric.data,
+            start: agg_metric.start,
+            finish: agg_metric.finish,
+          }
+        end
+      end
+    end
+  end
+end