sqreen 1.19.1-java → 1.21.0.beta3-java

data/lib/sqreen/event.rb

@@ -8,17 +8,19 @@
 module Sqreen
   # Master interface for point in time events (e.g. Attack, RemoteException)
   class Event
+    # @return [Hash]
     attr_reader :payload
+
+    # @return [Time]
+    attr_accessor :time # writer used only in tests
+
     def initialize(payload)
       @payload = payload
-    end
-
-    def to_hash
-      payload.to_hash
+      @time = Time.now.utc
     end
 
     def to_s
-      "<#{self.class.name}: #{to_hash}>"
+      "<#{self.class.name}: #{payload.to_hash}>"
     end
   end
 end

data/lib/sqreen/events/attack.rb

@@ -11,6 +11,8 @@ module Sqreen
   # Attack
   # When creating a new attack, it gets automatically pushed to the event's
   # queue.
+  # XXX: TURNS OUT THIS CLASS IS ACTUALLY NOT USED ANYMORE
+  # Framework.observe is used instead with unstructured attack details
   class Attack < Event
     def self.record(payload)
       attack = Attack.new(payload)
@@ -26,11 +28,31 @@ module Sqreen
       payload['rule']['rulespack_id']
     end
 
-    def type
+    def rule_name
       return nil unless payload['rule']
       payload['rule']['name']
     end
 
+    def test?
+      return nil unless payload['rule']
+      payload['rule']['test'] ? true : false
+    end
+
+    def beta?
+      return nil unless payload['rule']
+      payload['rule']['beta'] ? true : false
+    end
+
+    def block?
+      return nil unless payload['rule']
+      payload['rule']['block'] ? true : false
+    end
+
+    def attack_type
+      return nil unless payload['rule']
+      payload['rule']['attack_type']
+    end
+
     def time
       return nil unless payload['local']
       payload['local']['time']
@@ -44,22 +66,5 @@ module Sqreen
     def enqueue
       Sqreen.queue.push(self)
     end
-
-    def to_hash
-      res = {}
-      rule_p = payload['rule']
-      request_p = payload['request']
-      res[:rule_name]    = rule_p['name']         if rule_p && rule_p['name']
-      res[:rulespack_id] = rule_p['rulespack_id'] if rule_p && rule_p['rulespack_id']
-      res[:test]         = rule_p['test']         if rule_p && rule_p['test']
-      res[:infos]        = payload['infos']       if payload['infos']
-      res[:time]         = time                   if time
-      res[:client_ip]    = request_p[:addr]       if request_p && request_p[:addr]
-      res[:request]      = request_p              if request_p
-      res[:params]       = payload['params']      if payload['params']
-      res[:context]      = payload['context']     if payload['context']
-      res[:headers]      = payload['headers']     if payload['headers']
-      res
-    end
   end
 end

data/lib/sqreen/events/remote_exception.rb

@@ -30,27 +30,5 @@ module Sqreen
     def klass
       payload['exception'].class.name
     end
-
-    def to_hash
-      exception = payload['exception']
-      ev = {
-        :klass => exception.class.name,
-        :message => exception.message,
-        :params => payload['request_params'],
-        :time => payload['time'],
-        :infos => {
-          :client_ip => payload['client_ip'],
-        },
-        :request => payload['request_infos'],
-        :headers => payload['headers'],
-        :rule_name => payload['rule_name'],
-        :rulespack_id => payload['rulespack_id'],
-      }
-
-      ev[:infos].merge!(payload['infos']) if payload['infos']
-      return ev unless exception.backtrace
-      ev[:context] = { :backtrace => exception.backtrace.map(&:to_s) }
-      ev
-    end
   end
 end

data/lib/sqreen/events/request_record.rb

@@ -8,12 +8,15 @@
 require 'json'
 require 'sqreen/log'
 require 'sqreen/event'
-require 'sqreen/encoding_sanitizer'
 require 'sqreen/sensitive_data_redactor'
 
 module Sqreen
   # When a request is deeemed worthy of being sent to the backend
   class RequestRecord < Sqreen::Event
+    attr_reader :redactor
+
+    # @param [Hash] payload
+    # @param [Sqreen::SensitiveDataRedactor] redactor
     def initialize(payload, redactor = nil)
       @redactor = redactor
       super(payload)
@@ -23,74 +26,18 @@ module Sqreen
       (payload && payload[:observed]) || {}
     end
 
-    def to_hash
-      res = { :version => '20171208' }
-      if payload[:observed]
-        res[:observed] = payload[:observed].dup
-        rulespack = nil
-        if observed[:attacks]
-          res[:observed][:attacks] = observed[:attacks].map do |att|
-            natt = att.dup
-            rulespack = natt.delete(:rulespack_id) || rulespack
-            natt
-          end
-        end
-        if observed[:sqreen_exceptions]
-          res[:observed][:sqreen_exceptions] = observed[:sqreen_exceptions].map do |exc|
-            nex = exc.dup
-            excp = nex.delete(:exception)
-            if excp
-              nex[:message] = excp.message
-              nex[:klass] = excp.class.name
-            end
-            rulespack = nex.delete(:rulespack_id) || rulespack
-            nex
-          end
-        end
-        res[:rulespack_id] = rulespack unless rulespack.nil?
-        if observed[:observations]
-          res[:observed][:observations] = observed[:observations].map do |cat, key, value, time|
-            { :category => cat, :key => key, :value => value, :time => time }
-          end
-        end
-        if observed[:sdk]
-          res[:observed][:sdk] = processed_sdk_calls
-        end
-      end
-      res[:local] = payload['local'] if payload['local']
-      if payload['request']
-        res[:request] = payload['request'].dup
-        res[:client_ip] = res[:request].delete(:client_ip) if res[:request][:client_ip]
-      else
-        res[:request] = {}
-      end
-      if payload['response']
-        res[:response] = payload['response'].dup
-      else
-        res[:response] = {}
-      end
-
-      res[:request][:parameters] = payload['params'] if payload['params']
-      res[:request][:headers] = payload['headers'] if payload['headers']
-
-      res = Sqreen::EncodingSanitizer.sanitize(res)
+    def last_identify_args
+      return nil unless observed[:sdk]
 
-      if @redactor
-        res[:request], redacted = @redactor.redact(res[:request])
-        if redacted.any? && res[:observed] && res[:observed][:attacks]
-          res[:observed][:attacks] = @redactor.redact_attacks!(res[:observed][:attacks], redacted)
-        end
-        if redacted.any? && res[:observed] && res[:observed][:sqreen_exceptions]
-          res[:observed][:sqreen_exceptions] = @redactor.redact_exceptions!(res[:observed][:sqreen_exceptions], redacted)
-        end
+      observed[:sdk].reverse_each do |meth, _time, *args|
+        next unless meth == :identify
+        return args
       end
-
-      res
+      nil
     end
 
-    private
-
     def processed_sdk_calls
+      return [] unless observed[:sdk]
       auth_keys = last_identify_id
 
       observed[:sdk].map do |meth, time, *args|
@@ -102,6 +49,8 @@ module Sqreen
       end
     end
 
+    private
+
     def inject_identifiers(args, meth, auth_keys)
       return args unless meth == :track && auth_keys
 
@@ -118,13 +67,8 @@ module Sqreen
     end
 
     def last_identify_id
-      return nil unless observed[:sdk]
-
-      observed[:sdk].reverse_each do |meth, _time, *args|
-        next unless meth == :identify
-        return args.first if args.respond_to? :first
-      end
-      nil
+      args = last_identify_args
+      args.first if args.respond_to? :first
     end
   end
 end

data/lib/sqreen/frameworks/generic.rb

@@ -22,8 +22,17 @@ module Sqreen
       include RequestRecorder
       attr_accessor :sqreen_configuration
 
+      attr_writer :req_start_cb, :req_end_cb
+
       def initialize
         clean_request_record
+
+        # for notifying the ecosystem of request boundaries
+        # XXX: this should be refactored. It shouldn't be
+        # the framework doing these notifications to the ecosystem
+        # Probably the rule callback should do it itself
+        @req_start_cb = Proc.new {}
+        @req_end_cb = Proc.new {}
       end
 
       # What kind of database is this
@@ -209,7 +218,16 @@ module Sqreen
 
       # Should the agent not be starting up?
       def prevent_startup
+        # SQREEN-880 - prevent Sqreen startup on Sidekiq workers
+        return :sidekiq_cli if defined?(Sidekiq::CLI)
+        return :delayed_job if defined?(Delayed::Command)
+
+        # Prevent Sqreen startup on rake tasks - unless this is a Sqreen test
+        run_in_test = sqreen_configuration.get(:run_in_test)
+        return :rake if !run_in_test && $0.end_with?('rake')
+
         return :irb if $0 == 'irb'
+
         return if sqreen_configuration.nil?
         disable = sqreen_configuration.get(:disable)
         return :config_disable if disable == true || disable.to_s.to_i == 1
@@ -251,8 +269,12 @@ module Sqreen
       # Nota: cleanup should be performed at end of request (see clean_request)
       def store_request(object)
         return unless ensure_rack_loaded
+
+        rack_req = Rack::Request.new(object)
+        @req_start_cb.call(rack_req)
+
         self.remaining_perf_budget = Sqreen.performance_budget
-        SharedStorage.set(:request, Rack::Request.new(object))
+        SharedStorage.set(:request, rack_req)
         SharedStorage.set(:xss_params, nil)
         SharedStorage.set(:whitelisted, nil)
         SharedStorage.set(:request_overtime, nil)
@@ -281,6 +303,7 @@ module Sqreen
         SharedStorage.set(:xss_params, nil)
         SharedStorage.set(:whitelisted, nil)
         SharedStorage.set(:request_overtime, nil)
+        @req_end_cb.call
       end
 
       def remaining_perf_budget

data/lib/sqreen/frameworks/rails.rb

@@ -103,13 +103,6 @@ module Sqreen
         run_in_test = sqreen_configuration.get(:run_in_test)
         return :rails_test if !run_in_test && (Rails.env.test? || Rails.env.cucumber?)
 
-        # SQREEN-880 - prevent Sqreen startup on Sidekiq workers
-        return :sidekiq_cli if defined?(Sidekiq::CLI)
-        return :delayed_job if defined?(Delayed::Command)
-
-        # Prevent Sqreen startup on rake tasks - unless this is a Sqreen test
-        return :rake if !run_in_test && $0.end_with?('rake')
-
         return nil unless defined?(Rails::CommandsTasks)
         return nil if defined?(Rails::Server)
         return :rails_console    if defined?(Rails::Console)

data/lib/sqreen/frameworks/request_recorder.rb

@@ -58,12 +58,22 @@ module Sqreen
       Sqreen.log.debug { "close_request_record called. observed_items: #{observed_items}" }
 
       clean_request_record if observed_items.nil?
-      if only_metric_observation
+      if Sqreen.features['use_signals'] || only_metric_observation
         push_metrics(observations_queue, queue)
-        return clean_request_record
       end
+
+      if only_metric_observation
+        clean_request_record
+        return
+      end
+
+      # signals require request section to be present
+      payload_requests << 'request'
+      # for signals, response is optional, but the backend team wants them
+      payload_requests << 'response'
       payload = payload_creator.payload(payload_requests)
       payload[:observed] = observed_items
+
       queue.push create_request_record(payload)
       clean_request_record
     end
@@ -79,10 +89,13 @@ module Sqreen
       @redactor ||= SensitiveDataRedactor.from_config
     end
 
+    # pushes metric observations to the observations queue
+    # and clears the list for the request record
     def push_metrics(observations_queue, event_queue)
       observed_items[:observations].each do |obs|
         observations_queue.push obs
       end
+      observed_items[:observations] = []
       return unless observations_queue.size > MAX_OBS_QUEUE_LENGTH / 2
       event_queue.push Sqreen::METRICS_EVENT
     end

data/lib/sqreen/graft/call.rb

@@ -27,6 +27,10 @@ module Sqreen
       def raise(value)
         Flow.raise(value)
       end
+
+      def noop
+        Flow.noop
+      end
     end
 
     class Flow
@@ -46,12 +50,17 @@ module Sqreen
         def raise(value)
           new(:raise, value)
         end
+
+        def noop
+          new(:noop, nil)
+        end
       end
 
       def initialize(action, value, brk = false)
         @action = action
         @value = value
         @break = brk
+        @passed_conditions = false
       end
 
       def return?
@@ -91,60 +100,138 @@ module Sqreen
       def break?
         @break ? true : false
       end
+
+      def passed_conditions!
+        @passed_conditions = true
+
+        self
+      end
+
+      def passed_conditions?
+        @passed_conditions
+      end
     end
 
+    class TimerError < StandardError; end
+
     class Timer
       def self.read
         Process.clock_gettime(Process::CLOCK_MONOTONIC)
       end
 
-      attr_reader :tag
+      attr_reader :tag, :size
+      attr_accessor :conditions_passed
 
       def initialize(tag, &block)
         @tag = tag
-        @blips = []
         @block = block
+        @tally = 0
+        @size = 0
       end
 
-      def duration
-        @blips.each_with_index.reduce(0) { |a, (e, i)| i.even? ? a - e : a + e }
+      def elapsed
+        raise(TimerError, 'Timer#elapsed when paused') if @size.even?
+
+        @tally + Timer.read
       end
 
-      def elapsed
-        @blips.each_with_index.reduce(0) { |a, (e, i)| i.even? ? a - e : a + e } + Timer.read
+      def duration
+        raise(TimerError, 'Timer#duration when running') if @size.odd?
+
+        @tally
       end
 
       def ignore
-        @blips << Timer.read
+        raise(TimerError, 'Timer#ignore when paused') if @size.even?
+
+        @size += 1
+        @tally += Timer.read
         yield(self)
       ensure
-        @blips << Timer.read
+        @size += 1
+        @tally -= Timer.read
       end
 
-      def measure
-        @blips << Timer.read
+      def measure(opts = nil)
+        raise(TimerError, 'Timer#measure when running') if @size.odd?
+
+        now = Timer.read
+
+        ignore = opts[:ignore] if opts
+        if ignore
+          ignore.size += 1
+          ignore.tally += now
+        end
+
+        @size += 1
+        @tally -= now
+
         yield(self)
       ensure
-        @blips << Timer.read
+        now = Timer.read
+
+        if ignore
+          ignore.size += 1
+          ignore.tally -= now
+        end
+
+        @size += 1
+        @tally += now
+
         @block.call(self) if @block
-        Sqreen::Graft.logger.debug { "#{@tag}: time=%.03fus" % (duration * 1_000_000) }
       end
 
-      def start
-        @blips << Timer.read
+      def start(at = Timer.read)
+        raise(TimerError, 'Timer#start when started') unless @size.even?
+
+        @size += 1
+        @tally -= at
+
+        at
+      end
+
+      def stop(at = Timer.read)
+        raise(TimerError, 'Timer#stop when unstarted') unless @size.odd?
+
+        @size += 1
+        @tally += at
+
+        at
       end
 
-      def stop
-        @blips << Timer.read
+      def started?
+        @size != 0 && @size.odd?
       end
 
-      def size
-        @blips.size
+      def stopped?
+        @size != 0 && @size.even?
+      end
+
+      def running?
+        @size.odd?
+      end
+
+      def paused?
+        @size.even?
+      end
+
+      def include_measurements(another_timer)
+        @blips += another_timer.instance_variable_get(:@blips)
+      end
+
+      def start_and_end
+        raise 'Not exactly two measurements recorded' unless size == 2
+        @blips
       end
 
       def to_s
-        "#{@tag}: time=%.03fus" % (duration * 1_000_000)
+        "#{@tag}: time=%.03fus" % (@tally * 1_000_000)
       end
+
+      protected
+
+      attr_reader :tally
+      attr_writer :size, :tally
     end
   end
 end