sqreen 1.19.3-java → 1.20.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7e815895cd811cc70702eb3e3694c9bbf2bf896fc547d671f306981385b6d47c
-  data.tar.gz: fac7b6430073800ba99b34303c33eb2ab0605734b42665f5d64df07d08795393
+  metadata.gz: f45370a0f5c03f939b0098527398d058cf0df4ad026f7002905bdb31b79e857d
+  data.tar.gz: 516269e4979639d390e557f5f382522ebd0fdbc68e1731ea75315cb881dded9d
 SHA512:
-  metadata.gz: 8af3be238e7f0305be2de69682aaa0b739c316b1b4cc0d3e0fbcc404a0211c09561f98504083275b634c40d3bc2a0f122c9f33c1d3524446b9c2ca6b9287fe8e
-  data.tar.gz: 840b44697bf35f5c6ed9608199629c00dd2fe6d5f817368b41efe37e25b939d8bb585ea9e865f05f6f62115aca1ff7730c731cef67d37f2b9ef364606f16c241
+  metadata.gz: dbaec7b5608f60ef43922c970895b0c3aa3b547dcfc27192aff587993ade656ea6e2bdf969f01ce180d389f272b55f8df6c716be73262c251110afb53909bea5
+  data.tar.gz: a901202c4f7949711b2bb9c6fd940ea830ebf6c41e0f5e20ed17a0fa67957d49a10eb68573c41b0e5e72ccb0d8cae2e2dc5a9fd29c0644da4845a56164898505

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 1.20.0
+
+* Enable new instrumentation engine by default
+* Add signal-based backend communication
+
 ## 1.19.3
 
 * Improve WAF PII protection

data/lib/sqreen/aggregated_metric.rb

@@ -0,0 +1,25 @@
+require 'sqreen/rules/rule_cb'
+require 'sqreen/metrics/base'
+
+module Sqreen
+  class AggregatedMetric
+    def initialize(values = {})
+      values.each do |k, v|
+        public_send "#{k}=", v
+      end
+    end
+
+    # @return [Sqreen::Rules::RuleCB]
+    attr_accessor :rule # optional
+
+    # @return [Sqreen::Metric::Base]
+    attr_accessor :metric
+
+    attr_accessor :start, :finish
+    attr_accessor :data
+
+    def name
+      metric.name
+    end
+  end
+end

data/lib/sqreen/configuration.rb

@@ -39,11 +39,15 @@ module Sqreen
     { :env => :SQREEN_LIBSQREEN, :name => :libsqreen,
       :default => true, :convert => :to_bool },
     { :env => :SQREEN_WEAVE, :name => :weave,
-      :default => false, :convert => :to_bool },
+      :default => true, :convert => :to_bool },
     { :env => :SQREEN_WEAVE_STRATEGY, :name => :weave_strategy,
-      :default => :chain, :convert => :to_sym },
-    { :env => :SQREEN_URL,       :name => :url,
+      :default => :prepend, :convert => :to_sym },
+    { :env => :SQREEN_URL, :name => :url,
       :default => 'https://back.sqreen.io' },
+    { :env => :SQREEN_INGESTION_URL, :name => :ingestion_url,
+      :default => 'https://ingestion.sqreen.com/' },
+    { :env => :SQREEN_PROXY_URL, :name => :proxy_url,
+      :default => nil },
     { :env => :SQREEN_TOKEN,     :name => :token,
       :default => nil },
     { :env => :SQREEN_APP_NAME,  :name => :app_name,

data/lib/sqreen/deliveries/batch.rb

@@ -8,6 +8,7 @@
 # TODO: Sqreen::RequestRecord => sqreen/events
 # TODO: Sqreen.time
 
+require 'sqreen/aggregated_metric'
 require 'sqreen/events/attack'
 require 'sqreen/events/remote_exception'
 require 'sqreen/mono_time'
@@ -91,9 +92,11 @@ module Sqreen
       def event_key(event)
         case event
         when Sqreen::Attack
-          "att-#{event.type}"
+          "att-#{event.rule_name}"
         when Sqreen::RemoteException
           "rex-#{event.klass}"
+        when Sqreen::AggregatedMetric
+          "agg-metric"
         end
       end
     end

data/lib/sqreen/deliveries/simple.rb

@@ -7,6 +7,7 @@
 # TODO: Sqreen::RemoteException  => sqreen/events
 # TODO: Sqreen::RequestRecord => sqreen/events
 
+require 'sqreen/log/loggable'
 require 'sqreen/events/attack'
 require 'sqreen/events/remote_exception'
 require 'sqreen/events/request_record'
@@ -15,6 +16,7 @@ module Sqreen
   module Deliveries
     # Simple delivery method that directly call session on event
     class Simple
+      include Log::Loggable
       attr_accessor :session
 
       def initialize(session)
@@ -29,6 +31,8 @@ module Sqreen
           session.post_sqreen_exception(event)
         when Sqreen::RequestRecord
           session.post_request_record(event)
+        when Sqreen::AggregatedMetric
+          logger.warn 'Delivery of metrics using signals is not supported with simple delivery'
         else
           session.post_event(event)
         end

data/lib/sqreen/event.rb

@@ -8,17 +8,19 @@
 module Sqreen
   # Master interface for point in time events (e.g. Attack, RemoteException)
   class Event
+    # @return [Hash]
     attr_reader :payload
+
+    # @return [Time]
+    attr_accessor :time # writer used only in tests
+
     def initialize(payload)
       @payload = payload
-    end
-
-    def to_hash
-      payload.to_hash
+      @time = Time.now.utc
     end
 
     def to_s
-      "<#{self.class.name}: #{to_hash}>"
+      "<#{self.class.name}: #{payload.to_hash}>"
     end
   end
 end

data/lib/sqreen/events/attack.rb

@@ -11,6 +11,8 @@ module Sqreen
   # Attack
   # When creating a new attack, it gets automatically pushed to the event's
   # queue.
+  # XXX: TURNS OUT THIS CLASS IS ACTUALLY NOT USED ANYMORE
+  # Framework.observe is used instead with unstructured attack details
   class Attack < Event
     def self.record(payload)
       attack = Attack.new(payload)
@@ -26,11 +28,31 @@ module Sqreen
       payload['rule']['rulespack_id']
     end
 
-    def type
+    def rule_name
       return nil unless payload['rule']
       payload['rule']['name']
     end
 
+    def test?
+      return nil unless payload['rule']
+      payload['rule']['test'] ? true : false
+    end
+
+    def beta?
+      return nil unless payload['rule']
+      payload['rule']['beta'] ? true : false
+    end
+
+    def block?
+      return nil unless payload['rule']
+      payload['rule']['block'] ? true : false
+    end
+
+    def attack_type
+      return nil unless payload['rule']
+      payload['rule']['attack_type']
+    end
+
     def time
       return nil unless payload['local']
       payload['local']['time']
@@ -44,22 +66,5 @@ module Sqreen
     def enqueue
       Sqreen.queue.push(self)
     end
-
-    def to_hash
-      res = {}
-      rule_p = payload['rule']
-      request_p = payload['request']
-      res[:rule_name]    = rule_p['name']         if rule_p && rule_p['name']
-      res[:rulespack_id] = rule_p['rulespack_id'] if rule_p && rule_p['rulespack_id']
-      res[:test]         = rule_p['test']         if rule_p && rule_p['test']
-      res[:infos]        = payload['infos']       if payload['infos']
-      res[:time]         = time                   if time
-      res[:client_ip]    = request_p[:addr]       if request_p && request_p[:addr]
-      res[:request]      = request_p              if request_p
-      res[:params]       = payload['params']      if payload['params']
-      res[:context]      = payload['context']     if payload['context']
-      res[:headers]      = payload['headers']     if payload['headers']
-      res
-    end
   end
 end

data/lib/sqreen/events/remote_exception.rb

@@ -30,27 +30,5 @@ module Sqreen
     def klass
       payload['exception'].class.name
     end
-
-    def to_hash
-      exception = payload['exception']
-      ev = {
-        :klass => exception.class.name,
-        :message => exception.message,
-        :params => payload['request_params'],
-        :time => payload['time'],
-        :infos => {
-          :client_ip => payload['client_ip'],
-        },
-        :request => payload['request_infos'],
-        :headers => payload['headers'],
-        :rule_name => payload['rule_name'],
-        :rulespack_id => payload['rulespack_id'],
-      }
-
-      ev[:infos].merge!(payload['infos']) if payload['infos']
-      return ev unless exception.backtrace
-      ev[:context] = { :backtrace => exception.backtrace.map(&:to_s) }
-      ev
-    end
   end
 end

data/lib/sqreen/events/request_record.rb

@@ -14,6 +14,10 @@ require 'sqreen/sensitive_data_redactor'
 module Sqreen
   # When a request is deeemed worthy of being sent to the backend
   class RequestRecord < Sqreen::Event
+    attr_reader :redactor
+
+    # @param [Hash] payload
+    # @param [Sqreen::SensitiveDataRedactor] redactor
     def initialize(payload, redactor = nil)
       @redactor = redactor
       super(payload)
@@ -23,74 +27,18 @@ module Sqreen
       (payload && payload[:observed]) || {}
     end
 
-    def to_hash
-      res = { :version => '20171208' }
-      if payload[:observed]
-        res[:observed] = payload[:observed].dup
-        rulespack = nil
-        if observed[:attacks]
-          res[:observed][:attacks] = observed[:attacks].map do |att|
-            natt = att.dup
-            rulespack = natt.delete(:rulespack_id) || rulespack
-            natt
-          end
-        end
-        if observed[:sqreen_exceptions]
-          res[:observed][:sqreen_exceptions] = observed[:sqreen_exceptions].map do |exc|
-            nex = exc.dup
-            excp = nex.delete(:exception)
-            if excp
-              nex[:message] = excp.message
-              nex[:klass] = excp.class.name
-            end
-            rulespack = nex.delete(:rulespack_id) || rulespack
-            nex
-          end
-        end
-        res[:rulespack_id] = rulespack unless rulespack.nil?
-        if observed[:observations]
-          res[:observed][:observations] = observed[:observations].map do |cat, key, value, time|
-            { :category => cat, :key => key, :value => value, :time => time }
-          end
-        end
-        if observed[:sdk]
-          res[:observed][:sdk] = processed_sdk_calls
-        end
-      end
-      res[:local] = payload['local'] if payload['local']
-      if payload['request']
-        res[:request] = payload['request'].dup
-        res[:client_ip] = res[:request].delete(:client_ip) if res[:request][:client_ip]
-      else
-        res[:request] = {}
-      end
-      if payload['response']
-        res[:response] = payload['response'].dup
-      else
-        res[:response] = {}
-      end
-
-      res[:request][:parameters] = payload['params'] if payload['params']
-      res[:request][:headers] = payload['headers'] if payload['headers']
-
-      res = Sqreen::EncodingSanitizer.sanitize(res)
+    def last_identify_args
+      return nil unless observed[:sdk]
 
-      if @redactor
-        res[:request], redacted = @redactor.redact(res[:request])
-        if redacted.any? && res[:observed] && res[:observed][:attacks]
-          res[:observed][:attacks] = @redactor.redact_attacks!(res[:observed][:attacks], redacted)
-        end
-        if redacted.any? && res[:observed] && res[:observed][:sqreen_exceptions]
-          res[:observed][:sqreen_exceptions] = @redactor.redact_exceptions!(res[:observed][:sqreen_exceptions], redacted)
-        end
+      observed[:sdk].reverse_each do |meth, _time, *args|
+        next unless meth == :identify
+        return args
       end
-
-      res
+      nil
     end
 
-    private
-
     def processed_sdk_calls
+      return [] unless observed[:sdk]
       auth_keys = last_identify_id
 
       observed[:sdk].map do |meth, time, *args|
@@ -102,6 +50,8 @@ module Sqreen
       end
     end
 
+    private
+
     def inject_identifiers(args, meth, auth_keys)
       return args unless meth == :track && auth_keys
 
@@ -118,13 +68,8 @@ module Sqreen
     end
 
     def last_identify_id
-      return nil unless observed[:sdk]
-
-      observed[:sdk].reverse_each do |meth, _time, *args|
-        next unless meth == :identify
-        return args.first if args.respond_to? :first
-      end
-      nil
+      args = last_identify_args
+      args.first if args.respond_to? :first
     end
   end
 end

data/lib/sqreen/frameworks/request_recorder.rb

@@ -58,12 +58,20 @@ module Sqreen
       Sqreen.log.debug { "close_request_record called. observed_items: #{observed_items}" }
 
       clean_request_record if observed_items.nil?
-      if only_metric_observation
+      if Sqreen.features['use_signals'] || only_metric_observation
         push_metrics(observations_queue, queue)
-        return clean_request_record
       end
+
+      if only_metric_observation
+        clean_request_record
+        return
+      end
+
+      # signals require request section to be present
+      payload_requests << 'request'
       payload = payload_creator.payload(payload_requests)
       payload[:observed] = observed_items
+
       queue.push create_request_record(payload)
       clean_request_record
     end
@@ -79,10 +87,13 @@ module Sqreen
       @redactor ||= SensitiveDataRedactor.from_config
     end
 
+    # pushes metric observations to the observations queue
+    # and clears the list for the request record
     def push_metrics(observations_queue, event_queue)
       observed_items[:observations].each do |obs|
         observations_queue.push obs
       end
+      observed_items[:observations] = []
       return unless observations_queue.size > MAX_OBS_QUEUE_LENGTH / 2
       event_queue.push Sqreen::METRICS_EVENT
     end

data/lib/sqreen/kit/signals/specialized/aggregated_metric.rb

@@ -0,0 +1,72 @@
+# typed: ignore
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/kit/signals/metric'
+require 'sqreen/kit/signals/dto_helper'
+
+# reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/payload/binning_metric/2020-01-01T00_00_00_000Z/schema.cue
+
+module Sqreen
+  module Kit
+    module Signals
+      module Specialized
+      end
+    end
+  end
+end
+
+class Sqreen::Kit::Signals::Specialized::AggregatedMetric < Sqreen::Kit::Signals::Metric
+  add_mandatory_attrs :source, :payload
+
+  validate_str_attr :signal_name, /\Asq\.agent\.metric\..+\z/
+
+  def initialize(args)
+    self.payload_schema = Payload::SCHEMA_VERSION
+    super
+  end
+
+  class Payload
+    include Sqreen::Kit::Signals::DtoHelper
+
+    SCHEMA_VERSION = 'metric/2020-01-01T00:00:00.000Z'.freeze
+
+    add_mandatory_attrs :capture_interval_s,
+                        :date_started,
+                        :date_ended,
+                        :values,
+                        :kind
+
+    # mandatory
+    # @return [Integer]
+    attr_accessor :capture_interval_s
+
+    # mandatory
+    # @param [Time]
+    # @return [String]
+    attr_accessor_time :date_started
+
+    # mandatory
+    # @param [Time]
+    # @return [String]
+    attr_accessor_time :date_ended
+
+    # mandatory
+    # @return [Hash{String=>Object}]
+    attr_writer :values
+    def values
+      return nil if @values.nil?
+      @values.map do |k, v|
+        {
+          key: k.is_a?(Hash) || k.is_a?(Array) ? k : k.to_s,
+          value: v,
+        }
+      end
+    end
+
+    # mandatory
+    # @return [String]
+    attr_accessor :kind
+  end
+end