sqreen 1.19.4 → 1.20.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a1a0d2f8489dac7835a9d2135b0e35c15848a4fa094f7912f9979a83a9d2670e
-  data.tar.gz: 9bfc5db45531824d4f968f45fe495c9fad5da7c258c368fec5821aba8f66b124
+  metadata.gz: 6f18004c730291041540b696217f1c70b449e179328f959705a07a148364bfe9
+  data.tar.gz: 417f8bb15cbee7faf4f1aa127552e38f0c929255493341d0f683daae37ab9894
 SHA512:
-  metadata.gz: f2c2e5a91c6e415c6003dae1c3ca775b9d40eaacb34e4d28d07c991e46941bcb0bb0f1e9d01bdb1644562fb35852c9fb222bd8f1b83a008be3871ee6a5880cb9
-  data.tar.gz: 9164671762553af1f0cd658e3d654432a3eebfe81050f32d9cf8fc7c36fedb8c40a530629e57a16bec62536a080a931a2f2b34ba1fcb33521ab2964db38bc224
+  metadata.gz: f891f7785362829c23028206d6ba656ae3860e8c13cb265a839608c2cf02ba5be6576337c8d2630d1b0c8eb5c69e9dc732d7c68581f9bb472157e900f7a49a15
+  data.tar.gz: b1e2e6708cfc177b66e5fe25dc85a9839e784dd4afffd49585f06050311d821c3eee9fe06ddff63a4eddabfc94b9307ca54e4a7301ebbcd5fc4d82096ec8efa7

data/CHANGELOG.md

@@ -1,6 +1,7 @@
-## 1.19.4
+## 1.20.0
 
-* Fix signature check
+* Enable new instrumentation engine by default
+* Add signal-based backend communication
 
 ## 1.19.3
 

data/lib/sqreen/aggregated_metric.rb

@@ -0,0 +1,25 @@
+require 'sqreen/rules/rule_cb'
+require 'sqreen/metrics/base'
+
+module Sqreen
+  class AggregatedMetric
+    def initialize(values = {})
+      values.each do |k, v|
+        public_send "#{k}=", v
+      end
+    end
+
+    # @return [Sqreen::Rules::RuleCB]
+    attr_accessor :rule # optional
+
+    # @return [Sqreen::Metric::Base]
+    attr_accessor :metric
+
+    attr_accessor :start, :finish
+    attr_accessor :data
+
+    def name
+      metric.name
+    end
+  end
+end

data/lib/sqreen/configuration.rb

@@ -39,11 +39,15 @@ module Sqreen
     { :env => :SQREEN_LIBSQREEN, :name => :libsqreen,
       :default => true, :convert => :to_bool },
     { :env => :SQREEN_WEAVE, :name => :weave,
-      :default => false, :convert => :to_bool },
+      :default => true, :convert => :to_bool },
     { :env => :SQREEN_WEAVE_STRATEGY, :name => :weave_strategy,
-      :default => :chain, :convert => :to_sym },
-    { :env => :SQREEN_URL,       :name => :url,
+      :default => :prepend, :convert => :to_sym },
+    { :env => :SQREEN_URL, :name => :url,
       :default => 'https://back.sqreen.io' },
+    { :env => :SQREEN_INGESTION_URL, :name => :ingestion_url,
+      :default => 'https://ingestion.sqreen.com/' },
+    { :env => :SQREEN_PROXY_URL, :name => :proxy_url,
+      :default => nil },
     { :env => :SQREEN_TOKEN,     :name => :token,
       :default => nil },
     { :env => :SQREEN_APP_NAME,  :name => :app_name,

data/lib/sqreen/deliveries/batch.rb

@@ -8,6 +8,7 @@
 # TODO: Sqreen::RequestRecord => sqreen/events
 # TODO: Sqreen.time
 
+require 'sqreen/aggregated_metric'
 require 'sqreen/events/attack'
 require 'sqreen/events/remote_exception'
 require 'sqreen/mono_time'
@@ -91,9 +92,11 @@ module Sqreen
       def event_key(event)
         case event
         when Sqreen::Attack
-          "att-#{event.type}"
+          "att-#{event.rule_name}"
         when Sqreen::RemoteException
           "rex-#{event.klass}"
+        when Sqreen::AggregatedMetric
+          "agg-metric"
         end
       end
     end

data/lib/sqreen/deliveries/simple.rb

@@ -7,6 +7,7 @@
 # TODO: Sqreen::RemoteException  => sqreen/events
 # TODO: Sqreen::RequestRecord => sqreen/events
 
+require 'sqreen/log/loggable'
 require 'sqreen/events/attack'
 require 'sqreen/events/remote_exception'
 require 'sqreen/events/request_record'
@@ -15,6 +16,7 @@ module Sqreen
   module Deliveries
     # Simple delivery method that directly call session on event
     class Simple
+      include Log::Loggable
       attr_accessor :session
 
       def initialize(session)
@@ -29,6 +31,8 @@ module Sqreen
           session.post_sqreen_exception(event)
         when Sqreen::RequestRecord
           session.post_request_record(event)
+        when Sqreen::AggregatedMetric
+          logger.warn 'Delivery of metrics using signals is not supported with simple delivery'
         else
           session.post_event(event)
         end

data/lib/sqreen/event.rb

@@ -8,17 +8,19 @@
 module Sqreen
   # Master interface for point in time events (e.g. Attack, RemoteException)
   class Event
+    # @return [Hash]
     attr_reader :payload
+
+    # @return [Time]
+    attr_accessor :time # writer used only in tests
+
     def initialize(payload)
       @payload = payload
-    end
-
-    def to_hash
-      payload.to_hash
+      @time = Time.now.utc
     end
 
     def to_s
-      "<#{self.class.name}: #{to_hash}>"
+      "<#{self.class.name}: #{payload.to_hash}>"
     end
   end
 end

data/lib/sqreen/events/attack.rb

@@ -11,6 +11,8 @@ module Sqreen
   # Attack
   # When creating a new attack, it gets automatically pushed to the event's
   # queue.
+  # XXX: TURNS OUT THIS CLASS IS ACTUALLY NOT USED ANYMORE
+  # Framework.observe is used instead with unstructured attack details
   class Attack < Event
     def self.record(payload)
       attack = Attack.new(payload)
@@ -26,11 +28,31 @@ module Sqreen
       payload['rule']['rulespack_id']
     end
 
-    def type
+    def rule_name
       return nil unless payload['rule']
       payload['rule']['name']
     end
 
+    def test?
+      return nil unless payload['rule']
+      payload['rule']['test'] ? true : false
+    end
+
+    def beta?
+      return nil unless payload['rule']
+      payload['rule']['beta'] ? true : false
+    end
+
+    def block?
+      return nil unless payload['rule']
+      payload['rule']['block'] ? true : false
+    end
+
+    def attack_type
+      return nil unless payload['rule']
+      payload['rule']['attack_type']
+    end
+
     def time
       return nil unless payload['local']
       payload['local']['time']
@@ -44,22 +66,5 @@ module Sqreen
     def enqueue
       Sqreen.queue.push(self)
     end
-
-    def to_hash
-      res = {}
-      rule_p = payload['rule']
-      request_p = payload['request']
-      res[:rule_name]    = rule_p['name']         if rule_p && rule_p['name']
-      res[:rulespack_id] = rule_p['rulespack_id'] if rule_p && rule_p['rulespack_id']
-      res[:test]         = rule_p['test']         if rule_p && rule_p['test']
-      res[:infos]        = payload['infos']       if payload['infos']
-      res[:time]         = time                   if time
-      res[:client_ip]    = request_p[:addr]       if request_p && request_p[:addr]
-      res[:request]      = request_p              if request_p
-      res[:params]       = payload['params']      if payload['params']
-      res[:context]      = payload['context']     if payload['context']
-      res[:headers]      = payload['headers']     if payload['headers']
-      res
-    end
   end
 end

data/lib/sqreen/events/remote_exception.rb

@@ -30,27 +30,5 @@ module Sqreen
     def klass
       payload['exception'].class.name
     end
-
-    def to_hash
-      exception = payload['exception']
-      ev = {
-        :klass => exception.class.name,
-        :message => exception.message,
-        :params => payload['request_params'],
-        :time => payload['time'],
-        :infos => {
-          :client_ip => payload['client_ip'],
-        },
-        :request => payload['request_infos'],
-        :headers => payload['headers'],
-        :rule_name => payload['rule_name'],
-        :rulespack_id => payload['rulespack_id'],
-      }
-
-      ev[:infos].merge!(payload['infos']) if payload['infos']
-      return ev unless exception.backtrace
-      ev[:context] = { :backtrace => exception.backtrace.map(&:to_s) }
-      ev
-    end
   end
 end

data/lib/sqreen/events/request_record.rb

@@ -14,6 +14,10 @@ require 'sqreen/sensitive_data_redactor'
 module Sqreen
   # When a request is deeemed worthy of being sent to the backend
   class RequestRecord < Sqreen::Event
+    attr_reader :redactor
+
+    # @param [Hash] payload
+    # @param [Sqreen::SensitiveDataRedactor] redactor
     def initialize(payload, redactor = nil)
       @redactor = redactor
       super(payload)
@@ -23,74 +27,18 @@ module Sqreen
       (payload && payload[:observed]) || {}
     end
 
-    def to_hash
-      res = { :version => '20171208' }
-      if payload[:observed]
-        res[:observed] = payload[:observed].dup
-        rulespack = nil
-        if observed[:attacks]
-          res[:observed][:attacks] = observed[:attacks].map do |att|
-            natt = att.dup
-            rulespack = natt.delete(:rulespack_id) || rulespack
-            natt
-          end
-        end
-        if observed[:sqreen_exceptions]
-          res[:observed][:sqreen_exceptions] = observed[:sqreen_exceptions].map do |exc|
-            nex = exc.dup
-            excp = nex.delete(:exception)
-            if excp
-              nex[:message] = excp.message
-              nex[:klass] = excp.class.name
-            end
-            rulespack = nex.delete(:rulespack_id) || rulespack
-            nex
-          end
-        end
-        res[:rulespack_id] = rulespack unless rulespack.nil?
-        if observed[:observations]
-          res[:observed][:observations] = observed[:observations].map do |cat, key, value, time|
-            { :category => cat, :key => key, :value => value, :time => time }
-          end
-        end
-        if observed[:sdk]
-          res[:observed][:sdk] = processed_sdk_calls
-        end
-      end
-      res[:local] = payload['local'] if payload['local']
-      if payload['request']
-        res[:request] = payload['request'].dup
-        res[:client_ip] = res[:request].delete(:client_ip) if res[:request][:client_ip]
-      else
-        res[:request] = {}
-      end
-      if payload['response']
-        res[:response] = payload['response'].dup
-      else
-        res[:response] = {}
-      end
-
-      res[:request][:parameters] = payload['params'] if payload['params']
-      res[:request][:headers] = payload['headers'] if payload['headers']
-
-      res = Sqreen::EncodingSanitizer.sanitize(res)
+    def last_identify_args
+      return nil unless observed[:sdk]
 
-      if @redactor
-        res[:request], redacted = @redactor.redact(res[:request])
-        if redacted.any? && res[:observed] && res[:observed][:attacks]
-          res[:observed][:attacks] = @redactor.redact_attacks!(res[:observed][:attacks], redacted)
-        end
-        if redacted.any? && res[:observed] && res[:observed][:sqreen_exceptions]
-          res[:observed][:sqreen_exceptions] = @redactor.redact_exceptions!(res[:observed][:sqreen_exceptions], redacted)
-        end
+      observed[:sdk].reverse_each do |meth, _time, *args|
+        next unless meth == :identify
+        return args
       end
-
-      res
+      nil
     end
 
-    private
-
     def processed_sdk_calls
+      return [] unless observed[:sdk]
       auth_keys = last_identify_id
 
       observed[:sdk].map do |meth, time, *args|
@@ -102,6 +50,8 @@ module Sqreen
       end
     end
 
+    private
+
     def inject_identifiers(args, meth, auth_keys)
       return args unless meth == :track && auth_keys
 
@@ -118,13 +68,8 @@ module Sqreen
     end
 
     def last_identify_id
-      return nil unless observed[:sdk]
-
-      observed[:sdk].reverse_each do |meth, _time, *args|
-        next unless meth == :identify
-        return args.first if args.respond_to? :first
-      end
-      nil
+      args = last_identify_args
+      args.first if args.respond_to? :first
     end
   end
 end

data/lib/sqreen/frameworks/request_recorder.rb

@@ -58,12 +58,20 @@ module Sqreen
       Sqreen.log.debug { "close_request_record called. observed_items: #{observed_items}" }
 
       clean_request_record if observed_items.nil?
-      if only_metric_observation
+      if Sqreen.features['use_signals'] || only_metric_observation
         push_metrics(observations_queue, queue)
-        return clean_request_record
       end
+
+      if only_metric_observation
+        clean_request_record
+        return
+      end
+
+      # signals require request section to be present
+      payload_requests << 'request'
       payload = payload_creator.payload(payload_requests)
       payload[:observed] = observed_items
+
       queue.push create_request_record(payload)
       clean_request_record
     end
@@ -79,10 +87,13 @@ module Sqreen
       @redactor ||= SensitiveDataRedactor.from_config
     end
 
+    # pushes metric observations to the observations queue
+    # and clears the list for the request record
     def push_metrics(observations_queue, event_queue)
       observed_items[:observations].each do |obs|
         observations_queue.push obs
       end
+      observed_items[:observations] = []
       return unless observations_queue.size > MAX_OBS_QUEUE_LENGTH / 2
       event_queue.push Sqreen::METRICS_EVENT
     end

data/lib/sqreen/kit/signals/specialized/aggregated_metric.rb

@@ -0,0 +1,72 @@
+# typed: ignore
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/kit/signals/metric'
+require 'sqreen/kit/signals/dto_helper'
+
+# reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/payload/binning_metric/2020-01-01T00_00_00_000Z/schema.cue
+
+module Sqreen
+  module Kit
+    module Signals
+      module Specialized
+      end
+    end
+  end
+end
+
+class Sqreen::Kit::Signals::Specialized::AggregatedMetric < Sqreen::Kit::Signals::Metric
+  add_mandatory_attrs :source, :payload
+
+  validate_str_attr :signal_name, /\Asq\.agent\.metric\..+\z/
+
+  def initialize(args)
+    self.payload_schema = Payload::SCHEMA_VERSION
+    super
+  end
+
+  class Payload
+    include Sqreen::Kit::Signals::DtoHelper
+
+    SCHEMA_VERSION = 'metric/2020-01-01T00:00:00.000Z'.freeze
+
+    add_mandatory_attrs :capture_interval_s,
+                        :date_started,
+                        :date_ended,
+                        :values,
+                        :kind
+
+    # mandatory
+    # @return [Integer]
+    attr_accessor :capture_interval_s
+
+    # mandatory
+    # @param [Time]
+    # @return [String]
+    attr_accessor_time :date_started
+
+    # mandatory
+    # @param [Time]
+    # @return [String]
+    attr_accessor_time :date_ended
+
+    # mandatory
+    # @return [Hash{String=>Object}]
+    attr_writer :values
+    def values
+      return nil if @values.nil?
+      @values.map do |k, v|
+        {
+          key: k.is_a?(Hash) || k.is_a?(Array) ? k : k.to_s,
+          value: v,
+        }
+      end
+    end
+
+    # mandatory
+    # @return [String]
+    attr_accessor :kind
+  end
+end