sqreen 1.15.0-java → 1.15.5-java

data/lib/sqreen/frameworks/generic.rb

@@ -246,7 +246,6 @@ module Sqreen
         return unless ensure_rack_loaded
         self.remaining_perf_budget = Sqreen.performance_budget
         SharedStorage.set(:request, Rack::Request.new(object))
-        SharedStorage.inc(:stored_requests)
         SharedStorage.set(:xss_params, nil)
         SharedStorage.set(:whitelisted, nil)
         SharedStorage.set(:request_overtime, nil)
@@ -259,7 +258,6 @@ module Sqreen
 
       # Cleanup request context
       def clean_request
-        return unless SharedStorage.dec(:stored_requests) <= 0
         payload_creator = Sqreen::PayloadCreator.new(self)
         close_request_record(Sqreen.queue, Sqreen.observations_queue, payload_creator)
         self.remaining_perf_budget = nil

data/lib/sqreen/frameworks/request_recorder.rb

@@ -5,6 +5,8 @@ require 'sqreen/shared_storage'
 require 'sqreen/events/request_record'
 require 'sqreen/performance_notifications/log_performance'
 require 'sqreen/performance_notifications/newrelic'
+require 'sqreen/log'
+require 'sqreen/runner'
 
 module Sqreen
   # Store event/observations that happened in this request
@@ -47,6 +49,8 @@ module Sqreen
     end
 
     def close_request_record(queue, observations_queue, payload_creator)
+      Sqreen.log.debug { "close_request_record called. observed_items: #{observed_items}" }
+
       clean_request_record if observed_items.nil?
       if only_metric_observation
         push_metrics(observations_queue, queue)
@@ -54,12 +58,21 @@ module Sqreen
       end
       payload = payload_creator.payload(payload_requests)
       payload[:observed] = observed_items
-      queue.push RequestRecord.new(payload)
+      queue.push create_request_record(payload)
       clean_request_record
     end
 
     protected
 
+    def create_request_record(payload)
+      RequestRecord.new(payload, redactor)
+    end
+
+    def redactor
+      return nil unless Sqreen.config_get(:strip_sensitive_data)
+      @redactor ||= SensitiveDataRedactor.from_config
+    end
+
     def push_metrics(observations_queue, event_queue)
       observed_items[:observations].each do |obs|
         observations_queue.push obs

data/lib/sqreen/instrumentation.rb

@@ -12,6 +12,7 @@ require 'sqreen/shared_storage'
 require 'sqreen/rules_callbacks/record_request_context'
 require 'sqreen/rules_callbacks/run_req_start_actions'
 require 'sqreen/rules_callbacks/run_block_user_actions'
+require 'sqreen/mono_time'
 require 'set'
 
 # How to override a class method:
@@ -37,6 +38,11 @@ require 'set'
 module Sqreen
   class Instrumentation
     OVERTIME_METRIC = 'request_overtime'.freeze
+
+    PRE_CB = 'pre'.freeze
+    POST_CB = 'post'.freeze
+    FAILING_CB = 'failing'.freeze
+
     MGMT_COST = 0.000025
     @@override_semaphore = Mutex.new
     @@overriden_singleton_methods = false
@@ -64,7 +70,7 @@ module Sqreen
       @@overriden_methods
     end
     def self.callback_wrapper_pre(callbacks, framework, budget, _klass, method, instance, args, &block)
-      all_start = Sqreen::PerformanceNotifications.time
+      all_start = Sqreen.time
       #Instrumentation.guard_call(method, []) do
         returns = []
         callbacks.each do |cb|
@@ -76,9 +82,9 @@ module Sqreen
           end
           Sqreen.log.debug { "running pre cb #{cb}" }
           begin
-            start = Sqreen::PerformanceNotifications.time
+            start = Sqreen.time
             res = cb.pre(instance, args, budget, &block)
-            stop = Sqreen::PerformanceNotifications.time
+            stop = Sqreen.time
             # The first few pre callbacks could not have a request & hence a budget just yet so we try harder to find it
             budget = framework.remaining_perf_budget if framework && !budget && Sqreen.performance_budget
             if budget
@@ -95,6 +101,7 @@ module Sqreen
               res[:rule_name] = rule
             end
             returns << res
+            break if res.is_a?(Hash) && res[:skip_rem_cbs]
           rescue StandardError => e
             Sqreen.log.warn { "we catch an exception: #{e.inspect}" }
             Sqreen.log.debug e.backtrace
@@ -105,21 +112,21 @@ module Sqreen
             end
             next
           end
-          Sqreen::PerformanceNotifications.notify("Callbacks/#{rule || cb.class.name}/pre", start, stop)
+          Sqreen::PerformanceNotifications.notify(rule || cb.class.name, PRE_CB, start, stop)
         end
-        all_stop = Sqreen::PerformanceNotifications.time
+        all_stop = Sqreen.time
         framework.remaining_perf_budget = budget - (all_stop - all_start) - MGMT_COST * callbacks.size if framework && budget
-        Sqreen::PerformanceNotifications.notify('Callbacks/hooks_pre/pre', all_start, all_stop)
+        Sqreen::PerformanceNotifications.notify('hooks_pre', PRE_CB, all_start, all_stop)
         returns
       #end
     rescue StandardError => e
-      Sqreen.log.warn { "we catched an exception between cbs: #{e.inspect}" }
+      Sqreen.log.warn { "we caught an exception between cbs: #{e.inspect}" }
       Sqreen::RemoteException.record(e)
       []
     end
 
     def self.callback_wrapper_post(callbacks, framework, budget, _klass, method, return_val, instance, args, &block)
-      all_start = Sqreen::PerformanceNotifications.time
+      all_start = Sqreen.time
       #Instrumentation.guard_call(method, []) do
         returns = []
         callbacks.reverse_each do |cb|
@@ -130,9 +137,9 @@ module Sqreen
           end
           Sqreen.log.debug { "running post cb #{cb}" }
           begin
-            start = Sqreen::PerformanceNotifications.time
+            start = Sqreen.time
             res = cb.post(return_val, instance, args, budget, &block)
-            stop = Sqreen::PerformanceNotifications.time
+            stop = Sqreen.time
             if budget
               budget -= (stop - start)
               cb.overtime! if budget <= 0.0
@@ -157,23 +164,23 @@ module Sqreen
             end
             next
           end
-          Sqreen::PerformanceNotifications.notify("Callbacks/#{rule || cb.class.name}/post", start, stop)
+          Sqreen::PerformanceNotifications.notify(rule || cb.class.name, POST_CB, start, stop)
         end
-        all_stop = Sqreen::PerformanceNotifications.time
+        all_stop = Sqreen.time
         if framework && budget && framework.remaining_perf_budget
           framework.remaining_perf_budget = budget - (all_stop - all_start) - MGMT_COST * callbacks.size 
         end
-        Sqreen::PerformanceNotifications.notify('Callbacks/hooks_post/post', all_start, all_stop)
+        Sqreen::PerformanceNotifications.notify('hooks_post', POST_CB, all_start, all_stop)
         returns
       #end
     rescue StandardError => e
-      Sqreen.log.warn { "we catched an exception between cbs: #{e.inspect}" }
+      Sqreen.log.warn { "we caught an exception between cbs: #{e.inspect}" }
       Sqreen::RemoteException.record(e)
       []
     end
 
     def self.callback_wrapper_failing(callbacks, framework, budget, exception, _klass, method, instance, args, &block)
-      all_start = Sqreen::PerformanceNotifications.time
+      all_start = Sqreen.time
       #Instrumentation.guard_call(method, []) do
         returns = []
         callbacks.each do |cb|
@@ -184,9 +191,9 @@ module Sqreen
           end
           Sqreen.log.debug { "running failing cb #{cb}" }
           begin
-            start = Sqreen::PerformanceNotifications.time
+            start = Sqreen.time
             res = cb.failing(exception, instance, args, budget, &block)
-            stop = Sqreen::PerformanceNotifications.time
+            stop = Sqreen.time
             if budget
               budget -= (stop - start)
               cb.overtime! if budget <= 0.0
@@ -211,17 +218,17 @@ module Sqreen
             end
             next
           end
-          Sqreen::PerformanceNotifications.notify("Callbacks/#{rule || cb.class.name}/failing", start, stop)
+          Sqreen::PerformanceNotifications.notify(rule || cb.class.name, FAILING_CB, start, stop)
         end
-        all_stop = Sqreen::PerformanceNotifications.time
+        all_stop = Sqreen.time
         if framework && budget && framework.remaining_perf_budget
           framework.remaining_perf_budget = budget - (all_stop - all_start) - MGMT_COST * callbacks.size 
         end
-        Sqreen::PerformanceNotifications.notify('Callbacks/hooks_failing/failing', all_start, all_stop)
+        Sqreen::PerformanceNotifications.notify('hooks_failing', FAILING_CB, all_start, all_stop)
         returns
      # end
     rescue StandardError => e
-      Sqreen.log.warn "we catched an exception between cbs: #{e.inspect}"
+      Sqreen.log.warn "we caught an exception between cbs: #{e.inspect}"
       Sqreen::RemoteException.record(e)
       []
     end
@@ -245,11 +252,8 @@ module Sqreen
 
     def self.define_callback_method(meth, original_meth, klass_name)
       @sqreen_multi_instr ||= nil
-      proc do |*args, &block|
-        record_req_hp = @@record_request_hookpoints.include?([klass_name, meth]) &&
-          Sqreen::PerformanceNotifications.listen_for?
-        Sqreen::PerformanceNotifications::BinnedMetrics.start_request if record_req_hp
 
+      proc do |*args, &block|
         budget = nil
         skip_call = Thread.current[:sqreen_in_use]
         begin
@@ -385,16 +389,28 @@ module Sqreen
             end
             result
           ensure
-            if record_req_hp
-              Sqreen::PerformanceNotifications.instrument('Callbacks/hooks_reporting/pre') do
-                Sqreen::PerformanceNotifications::LogPerformance.next_request
-                Sqreen::PerformanceNotifications::NewRelic.next_request
-                Sqreen::PerformanceNotifications::BinnedMetrics.finish_request
-              end
-            end
             Thread.current[:sqreen_in_use] = false
           end
         end
+      end # end of proc
+    end
+
+    def self.request_hookpoint_method(original_meth)
+      proc do |*args, &block|
+        has_notifications = Sqreen::PerformanceNotifications.listen_for?
+        Sqreen::PerformanceNotifications::BinnedMetrics.start_request if has_notifications
+
+        begin
+          send(original_meth, *args, &block)
+        ensure
+          if has_notifications
+            Sqreen::PerformanceNotifications.instrument('next_req_notifs', PRE_CB) do
+              Sqreen::PerformanceNotifications::LogPerformance.next_request
+              Sqreen::PerformanceNotifications::NewRelic.next_request
+              Sqreen::PerformanceNotifications::BinnedMetrics.finish_request
+            end
+          end
+        end
       end
     end
 
@@ -476,6 +492,12 @@ module Sqreen
 
         define_method(new_method, p)
 
+        if @@record_request_hookpoints.include?([klass_name, meth])
+          p = Instrumentation.request_hookpoint_method(new_method)
+          new_method = "#{new_method}_req_hp_wrapper"
+          define_method(new_method, p)
+        end
+
         if public_method_defined?(meth)
           method_kind = :public
         elsif protected_method_defined?(meth)
@@ -568,6 +590,8 @@ module Sqreen
         method = cb.method
         key = [klass, method]
 
+        @@record_request_hookpoints << key if cb.is_a?(Sqreen::Rules::RecordRequestContext)
+
         already_overriden = @@overriden_methods.include? key
 
         if !already_overriden
@@ -609,7 +633,6 @@ module Sqreen
 
         @@registered_callbacks.add(cb)
         @@unovertimable_hookpoints << key unless cb.overtimeable
-        @@record_request_hookpoints << key if cb.is_a?(Sqreen::Rules::RecordRequestContext)
         @@instrumented_pid = Process.pid
       end
     end
@@ -710,6 +733,7 @@ module Sqreen
         add_callback(rcb)
       end
 
+      # add hardcoded callbacks, observing priority
       hardcoded_callbacks(framework).each { |cb| add_callback(cb) }
 
       Sqreen.instrumentation_ready = true

data/lib/sqreen/js/mini_racer_adapter.rb

@@ -82,13 +82,22 @@ module Sqreen
 
       def run_js_cb(cb_name, budget, arguments)
         @pool.with_context do |ctx|
+          if ctx.code_failed?(@code_id)
+            Sqreen.log.debug do
+              "Skipping execution of callback #{cb_name} (code md5 #{@code_id})" \
+              " due to prev failure of definition evaluation"
+            end
+            return nil
+          end
+
           ctx.add_code(@code_id, @code) unless ctx.has_code?(@code_id)
 
           begin
-            json_args = "[#{arguments.map(&method(:fixup_bad_encoding)).map(&:to_json).join(',')}]"
+            json_args = convert_arguments(arguments)
             ctx.eval_unsafe(
-                 "sqreen_data['#{@code_id}']['#{cb_name}'].apply(this, #{json_args})", nil, budget)
+              "sqreen_data['#{@code_id}']['#{cb_name}'].apply(this, #{json_args})", nil, budget)
           rescue @module::ScriptTerminatedError
+            Sqreen.log.debug "ScriptTerminatedError/#{cb_name}"
             nil
           end
         end
@@ -100,9 +109,31 @@ module Sqreen
 
       private
 
-      def fixup_bad_encoding(arg)
-        # NOTE: we don't fix encoding problems in deeper structures
+      def convert_arguments(args)
+        JSON.generate(args)
+      rescue JSON::GeneratorError, Encoding::UndefinedConversionError
+        fixed_args = fixup_bad_encoding(args)
+        JSON.generate(fixed_args)
+      end
+
+      def fixup_bad_encoding(arg, max_depth = 100)
+        return nil if max_depth <= 0
+
+        if arg.is_a?(Array)
+          return arg.map { |it| fixup_bad_encoding(it, max_depth - 1) }
+        end
+
+        if arg.is_a?(Hash)
+          return Hash[
+            arg.map do |k, v|
+              [fixup_bad_encoding(k, max_depth - 1),
+               fixup_bad_encoding(v, max_depth - 1)]
+            end
+          ]
+        end
+
         return arg unless arg.is_a?(String)
+
         unless arg.valid_encoding?
           return arg.dup.force_encoding(Encoding::ISO_8859_1)
         end
@@ -127,13 +158,20 @@ module Sqreen
               @code_ids.include?(code_id)
             end
 
+            def code_failed?(code_id)
+              return false unless @failed_code_ids
+              @failed_code_ids.include?(code_id)
+            end
+
             def add_code(code_id, code)
+              eval_unsafe "(function() { #{code} })()"
+              transf_global_funcs code_id
               @code_ids ||= Set.new
-              # if it fails, we don't try again
               @code_ids << code_id
-
-              eval_unsafe code
-              transf_global_funcs code_id
+            rescue
+              @failed_code_ids ||= Set.new
+              @failed_code_ids << code_id
+              raise
             end
 
             def eval_unsafe(str, filename = nil, timeoutv = nil)

data/lib/sqreen/metrics/average.rb

@@ -9,7 +9,7 @@ module Sqreen
     class Average < Base
       # from class attr_accessor :aggregate
 
-      def update(_at, key, value)
+      def update(key, value)
         super
         @sums[key] ||= 0
         @sums[key] += value

data/lib/sqreen/metrics/base.rb

@@ -18,12 +18,12 @@ module Sqreen
       # @param _at [Time] when was the observation made
       # @param _key [String] which aggregation key was it made for
       # @param _value [Object] The observation
-      def update(_at, _key, _value)
+      def update(_key, _value)
         raise Sqreen::Exception, 'No current sample' unless @sample
       end
 
       # create a new empty sample and publish the last one
-      # @param time [Time] Time of start/finish
+      # @param time [Float] Time of start of new sample/end of the last one
       def next_sample(time)
         finalize_sample(time) unless @sample.nil?
         current_sample = @sample
@@ -33,10 +33,12 @@ module Sqreen
 
       protected
 
+      # @param time [Float]
       def new_sample(time)
         @sample = { OBSERVATION_KEY => {}, START_KEY => time }
       end
 
+      # @param time [Float]
       def finalize_sample(time)
         @sample[FINISH_KEY] = time
       end

data/lib/sqreen/metrics/binning.rb

@@ -1,6 +1,7 @@
 # Copyright (c) 2018 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.io/terms.html
 
+require 'sqreen/mono_time'
 require 'sqreen/metrics/base'
 
 module Sqreen
@@ -20,10 +21,10 @@ module Sqreen
         log_factor = Math.log(@factor)
         @inv_log_base = 1 / log_base
         @add_parcel = - log_factor / log_base
-        new_sample(Time.now.utc)
+        new_sample(Sqreen.time)
       end
 
-      def update(_at, _key, x)
+      def update(_key, x)
         h = @sample[OBSERVATION_KEY]
         bin = bin_no(x)
         h[bin] += 1

data/lib/sqreen/metrics/collect.rb

@@ -11,7 +11,7 @@ module Sqreen
     class Collect < Base
       # from class attr_accessor :aggregate
 
-      def update(_at, key, value)
+      def update(key, value)
         super
         s = @sample[OBSERVATION_KEY]
         s[key] ||= []

data/lib/sqreen/metrics/sum.rb

@@ -9,7 +9,7 @@ module Sqreen
     class Sum < Base
       # from class attr_accessor :aggregate
 
-      def update(_at, key, value)
+      def update(key, value)
         super
         s = @sample[OBSERVATION_KEY]
         s[key] ||= 0