sqreen 1.19.0-java → 1.20.1-java

data/lib/sqreen/kit/signals/specialized/sqreen_exception.rb

@@ -0,0 +1,57 @@
+# typed: ignore
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/kit/signals/point'
+require 'sqreen/kit/signals/dto_helper'
+
+# reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/payload/sqreen_exception/2020-01-01T00_00_00_000Z/schema.cue
+
+module Sqreen
+  module Kit
+    module Signals
+      module Specialized
+      end
+    end
+  end
+end
+
+class Sqreen::Kit::Signals::Specialized::SqreenException < Sqreen::Kit::Signals::Point
+  PAYLOAD_SCHEMA_VERSION = 'sqreen_exception/2020-01-01T00:00:00.000Z'.freeze
+
+  # @return [Hash]
+  attr_accessor :infos
+
+  # @return [Exception]
+  attr_accessor :ruby_exception
+
+  add_mandatory_attrs :source, :time, :ruby_exception
+
+  validate_str_attr :source, /\A(?:sqreen:rule:[a-f0-9]{40}:.+)|(?:sqreen:agent:.+)\z/
+
+  def self.attributes_for_to_h_self
+    [] # don't include ruby_exception in list of attributes for to_h
+  end
+
+  def initialize(values = {})
+    self.payload_schema = PAYLOAD_SCHEMA_VERSION
+    self.signal_name = 'sq.agent.exception'
+    self.time = values[:time] || Time.now
+    super
+  end
+
+  def payload
+    return nil unless @ruby_exception
+    compact_hash({
+                   klass: @ruby_exception.class.to_s,
+                   message: @ruby_exception.message,
+                   infos: @infos,
+                 })
+  end
+
+  def location
+    return nil unless @ruby_exception
+    Sqreen::Kit::Signals::Location.new(exception: @ruby_exception)
+  end
+end

data/lib/sqreen/legacy/old_event_submission_strategy.rb

@@ -0,0 +1,221 @@
+# typed: ignore
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/aggregated_metric'
+require 'sqreen/log/loggable'
+require 'sqreen/legacy/waf_redactions'
+
+module Sqreen
+  module Legacy
+    # see also Sqreen::Signals::SignalsSubmissionStrategy
+    # usage in Sqreen:Session
+    class OldEventSubmissionStrategy
+      include Sqreen::Log::Loggable
+
+      RETRY_MANY = 301
+
+      def initialize(post_proc)
+        @post_proc = post_proc
+      end
+
+      def post_metrics(metrics)
+        return if metrics.nil? || metrics.empty?
+        payload = { metrics: metrics.map { |m| EventToHash.convert_agg_metric(m) } }
+        post('metrics', payload, {}, RETRY_MANY)
+      end
+
+      # @param attack [Sqreen::Attack]
+      def post_attack(attack)
+        post('attack', EventToHash.convert_attack(attack), {}, RETRY_MANY)
+      end
+
+      # @param [Sqreen::RequestRecord] request_record
+      def post_request_record(request_record)
+        rr_hash = EventToHash.convert_request_record(request_record)
+        post('request_record', rr_hash, {}, RETRY_MANY)
+      end
+
+      # Post an exception to Sqreen for analysis
+      # @param exception [RemoteException] Exception and context to be sent over
+      def post_sqreen_exception(exception)
+        data = EventToHash.convert_exception(exception)
+        post('sqreen_exception', data, {}, 5)
+      rescue StandardError => e
+        logger.warn(format('Could not post exception (network down? %s) %s',
+                           e.inspect,
+                           exception.inspect))
+        nil
+      end
+
+      def post_batch(events)
+        batch = events.map do |event|
+          h = case event
+              when AggregatedMetric
+                logger.warn "Aggregated metric event in non-signal mode. Signals disabled at runtime?"
+                next
+              when Attack # in practice only found inside req rec
+                EventToHash.convert_attack event
+              when RemoteException
+                EventToHash.convert_exception event
+              when RequestRecord
+                EventToHash.convert_request_record event
+              else
+                logger.warn "Unexpected event type: #{event}"
+                next
+              end
+          h['event_type'] = event_kind(event)
+          h
+        end
+        Sqreen.log.debug do
+          tally = Hash[events.group_by(&:class).map { |k, v| [k, v.count] }]
+          "Doing batch with the following tally of event types: #{tally}"
+        end
+        post('batch', { batch: batch }, {}, RETRY_MANY)
+      end
+
+      private
+
+      # see +Sqreen::Session.post+
+      def post(*args)
+        @post_proc[*args]
+      end
+
+      def event_kind(event)
+        case event
+        when Sqreen::RemoteException then 'sqreen_exception'
+        when Sqreen::Attack then 'attack'
+        when Sqreen::RequestRecord then 'request_record'
+        end
+      end
+    end
+
+    module EventToHash
+      class << self
+        # @param attack [Sqreen::Attack]
+        def convert_attack(attack)
+          payload = attack.payload
+          res = {}
+          rule_p = payload['rule']
+          request_p = payload['request']
+          res[:rule_name]    = rule_p['name']         if rule_p && rule_p['name']
+          res[:rulespack_id] = rule_p['rulespack_id'] if rule_p && rule_p['rulespack_id']
+          res[:test]         = rule_p['test']         if rule_p && rule_p['test']
+          res[:infos]        = payload['infos']       if payload['infos']
+          res[:time]         = attack.time
+          res[:client_ip]    = request_p[:addr]       if request_p && request_p[:addr]
+          res[:request]      = request_p              if request_p
+          res[:params]       = payload['params']      if payload['params']
+          res[:context]      = payload['context']     if payload['context']
+          res[:headers]      = payload['headers']     if payload['headers']
+          res
+        end
+
+        # @param [Sqreen::RequestRecord] rr
+        def convert_request_record(rr)
+          res = { :version => '20171208' }
+          payload = rr.payload
+
+          if payload[:observed]
+            res[:observed] = payload[:observed].dup
+            rulespack = nil
+            if rr.observed[:attacks]
+              res[:observed][:attacks] = rr.observed[:attacks].map do |att|
+                natt = att.dup
+                [:attack_type, :block].each { |k| natt.delete(k) } # signals stuff
+                rulespack = natt.delete(:rulespack_id) || rulespack
+                natt
+              end
+            end
+            if rr.observed[:sqreen_exceptions]
+              res[:observed][:sqreen_exceptions] = rr.observed[:sqreen_exceptions].map do |exc|
+                nex = exc.dup
+                excp = nex.delete(:exception)
+                if excp
+                  nex[:message] = excp.message
+                  nex[:klass] = excp.class.name
+                end
+                rulespack = nex.delete(:rulespack_id) || rulespack
+                nex
+              end
+            end
+            res[:rulespack_id] = rulespack unless rulespack.nil?
+            if rr.observed[:observations]
+              res[:observed][:observations] = rr.observed[:observations].map do |cat, key, value, time|
+                { :category => cat, :key => key, :value => value, :time => time }
+              end
+            end
+            if rr.observed[:sdk] # rubocop:disable Style/IfUnlessModifier
+              res[:observed][:sdk] = rr.processed_sdk_calls
+            end
+          end
+          res[:local] = payload['local'] if payload['local']
+          if payload['request']
+            res[:request] = payload['request'].dup
+            res[:client_ip] = res[:request].delete(:client_ip) if res[:request][:client_ip]
+          else
+            res[:request] = {}
+          end
+          if payload['response']
+            res[:response] = payload['response'].dup
+          else
+            res[:response] = {}
+          end
+
+          res[:request][:parameters] = payload['params'] if payload['params']
+          res[:request][:headers] = payload['headers'] if payload['headers']
+
+          res = Sqreen::EncodingSanitizer.sanitize(res)
+
+          if rr.redactor
+            res[:request], redacted = rr.redactor.redact(res[:request])
+            redacted = redacted.uniq
+            if redacted.any? && res[:observed] && res[:observed][:attacks]
+              res[:observed][:attacks] = WafRedactions.redact_attacks!(res[:observed][:attacks], redacted)
+            end
+            if redacted.any? && res[:observed] && res[:observed][:sqreen_exceptions]
+              res[:observed][:sqreen_exceptions] = WafRedactions.redact_exceptions!(res[:observed][:sqreen_exceptions], redacted)
+            end
+          end
+
+          res
+        end
+
+        # @param exception_evt [Sqreen::RemoteException]
+        def convert_exception(exception_evt)
+          payload = exception_evt.payload
+          exception = payload['exception']
+          ev = {
+            :klass => exception.class.name,
+            :message => exception.message,
+            :params => payload['request_params'],
+            :time => payload['time'],
+            :infos => {
+              :client_ip => payload['client_ip'],
+            },
+            :request => payload['request_infos'],
+            :headers => payload['headers'],
+            :rule_name => payload['rule_name'],
+            :rulespack_id => payload['rulespack_id'],
+          }
+
+          ev[:infos].merge!(payload['infos']) if payload['infos']
+          return ev unless exception.backtrace
+          ev[:context] = { :backtrace => exception.backtrace.map(&:to_s) }
+          ev
+        end
+
+        # @param [Sqreen::AggregatedMetric] agg_metric
+        def convert_agg_metric(agg_metric)
+          {
+            name: agg_metric.name,
+            observation: agg_metric.data,
+            start: agg_metric.start,
+            finish: agg_metric.finish,
+          }
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/legacy/waf_redactions.rb

@@ -0,0 +1,49 @@
+# typed: ignore
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+module Sqreen
+  module Legacy
+    module WafRedactions
+      class << self
+        def redact_attacks!(attacks, values)
+          return attacks if values.empty?
+
+          values = values.map { |v| v.downcase if v.is_a?(String) }
+
+          attacks.each do |e|
+            next(e) unless e[:infos]
+            next(e) unless e[:infos][:waf_data]
+
+            parsed = JSON.parse(e[:infos][:waf_data])
+            redacted = parsed.each do |w|
+              next unless (filters = w['filter'])
+
+              filters.each do |f|
+                next unless (v = f['resolved_value'])
+                next unless values.include?(v.downcase)
+
+                f['match_status'] = SensitiveDataRedactor::MASK
+                f['resolved_value'] = SensitiveDataRedactor::MASK
+              end
+            end
+            e[:infos][:waf_data] = JSON.dump(redacted)
+          end
+        end
+
+        # see https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000022-waf-data-sanitization.md#changes-to-the-agents
+        def redact_exceptions!(exceptions, values)
+          return exceptions if values.empty?
+
+          exceptions.each do |e|
+            next(e) unless e[:infos]
+            next(e) unless e[:infos][:waf]
+
+            e[:infos][:waf].delete(:args)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/log/loggable.rb

@@ -23,6 +23,6 @@ module Sqreen::Log::Loggable
   end
 
   def logger
-    @logger || self.class.logger
+    @logger || singleton_class.logger
   end
 end

data/lib/sqreen/metrics/base.rb

@@ -12,6 +12,9 @@ module Sqreen
     FINISH_KEY = 'finish'.freeze
     # Base interface for a metric
     class Base
+      attr_accessor :name, :period # for signals serialization
+      attr_accessor :rule # optional
+
       def initialize(_opts={})
         @sample = nil
       end

data/lib/sqreen/metrics_store.rb

@@ -3,6 +3,7 @@
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
+require 'sqreen/aggregated_metric'
 require 'sqreen/metrics'
 require 'sqreen/mono_time'
 require 'sqreen/metrics_store/unknown_metric'
@@ -30,8 +31,9 @@ module Sqreen
 
     # Definition contains a name,period and aggregate at least
     # @param definition [Hash] a metric definition
+    # @param rule [RuleCB] the rule associated with this metric, if any
     # @param mklass [Object] Override metric object (used in testing)
-    def create_metric(definition, mklass = nil)
+    def create_metric(definition, rule = nil, mklass = nil)
       name = definition[NAME_KEY]
       kind = definition[KIND_KEY]
       klass = valid_metric(kind, name)
@@ -43,6 +45,9 @@ module Sqreen
         definition[PERIOD_KEY],
         nil # Start
       ]
+      metric.name = name
+      metric.rule = rule
+      metric.period = definition[PERIOD_KEY]
       metric
     end
 
@@ -50,7 +55,7 @@ module Sqreen
       @metrics.key?(name)
     end
 
-    # @params at [Time] when is the store emptied
+    # @param at [Time] when is the store emptied
     def update(name, at, key, value)
       metric, period, start = @metrics[name]
       raise UnregisteredMetric, "Unknown metric #{name}" unless metric
@@ -59,7 +64,7 @@ module Sqreen
     end
 
     # Drains every metrics and returns the store content
-    # @params at [Time] when is the store emptied
+    # @param at [Time] when is the store emptied
     def publish(flush = true, at = Sqreen.time)
       @metrics.each do |name, (_, period, start)|
         next_sample(name, at) if flush || !start.nil? && (start + period) < at
@@ -75,15 +80,20 @@ module Sqreen
       metric = @metrics[name][0]
       r = metric.next_sample(at)
       @metrics[name][2] = at # new start
-      if r
-        r[NAME_KEY] = name
-        obs = r[Metric::OBSERVATION_KEY]
-        start_of_mono = Time.now.utc - Sqreen.time
-        r[Metric::START_KEY] = start_of_mono + r[Metric::START_KEY]
-        r[Metric::FINISH_KEY] = start_of_mono + r[Metric::FINISH_KEY]
-        @store << r if obs && (!obs.respond_to?(:empty?) || !obs.empty?)
-      end
-      r
+      return unless r
+
+      r[NAME_KEY] = name
+      obs = r[Metric::OBSERVATION_KEY]
+      return unless obs && (!obs.respond_to?(:empty?) || !obs.empty?)
+      start_of_mono = Time.now.utc - Sqreen.time
+
+      agg = AggregatedMetric.new
+      agg.metric = metric
+      agg.rule = agg.metric.rule
+      agg.start = start_of_mono + r[Metric::START_KEY]
+      agg.finish = start_of_mono + r[Metric::FINISH_KEY]
+      agg.data = obs
+      @store << agg
     end
 
     def valid_metric(kind, name)

data/lib/sqreen/performance_notifications/binned_metrics.rb

@@ -122,10 +122,16 @@ module Sqreen
       attr_reader :metrics_store
       attr_reader :period
 
-      def ensure_metric(metric_name)
+      def ensure_metric(metric_name, rule = nil)
         return if metrics_store.metric?(metric_name)
         metrics_store.create_metric(
-          'name' => metric_name, 'period' => period, 'kind' => 'Binning', 'options' => @perf_metric_opts
+          {
+            'name' => metric_name,
+            'period' => period,
+            'kind' => 'Binning',
+            'options' => @perf_metric_opts,
+          },
+          rule
         )
       end
 

data/lib/sqreen/rules.rb

@@ -135,13 +135,15 @@ module Sqreen
         return nil
       end
 
+      rule_cb = cb_class.new(instr_class, instr_method, hash_rule)
+
       if metrics_store
         (hash_rule[Attrs::METRICS] || []).each do |metric|
-          metrics_store.create_metric(metric)
+          metrics_store.create_metric(metric, rule_cb)
         end
       end
 
-      cb_class.new(instr_class, instr_method, hash_rule)
+      rule_cb
     rescue => e
       rule_name = nil
       rulespack_id = nil