sqreen 1.21.0.beta2 → 1.21.0.beta3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e5b3d6fc37fd5d2431e622d302fd3f58d35dd64a3fc7abca741a84b213688025
-  data.tar.gz: a3443a3f1c95841af9deb7eb5f4a5ab8c47aabf6c6f42eeb902d828edd3ad91d
+  metadata.gz: 88934827e2aa5f84ae21f96fb33afdada009fb118d94e8940fe61973812da4d4
+  data.tar.gz: b30da2199b4dc96d0193c671872ac192396c988ef67845c26fc1ed36a7704bfd
 SHA512:
-  metadata.gz: 487d63ea8f4bc8c5d3da55dced0c8d458c6a6c7f218689027925d8f6ca2808fc4eda478e57ca67764edb8a049fb8fae60b552974ffa39dfaa0dcd15d31f34071
-  data.tar.gz: 23cf04763b76e95d421a36593c2b96d91aad0f40bc206fe8608606fad803a464d03518ac92b40e4c1bb667dbd163ba79c6380f605b1133270f65d66fb8314173
+  metadata.gz: 19942da7597a18aee5b46aa4ef664c5083eadf3a878c5f431f9dde83196fc15f3263509c6cda6537a5c823de310f117dbb7c600130478a0828d2dbc73f0bf53c
+  data.tar.gz: c8b1293e8480043d3b494d6335725f649107c56e7bf282580592de7d680eaeb9475bd9091ea280ad2d2e473b100789a9b80c3dac93b9ddebbddb2f8ec4f3ced6

data/CHANGELOG.md

@@ -1,10 +1,18 @@
-## 1.21.0.beta2
-
-* Improve transport and tracing internals
-
-## 1.21.0.beta1
-
-* Add transport and tracing internals
+## 1.21.0.beta3
+
+* Avoid fd leak in `custom_error_cb` (ARB-109)
+* Support `skip_rem_cbs` (ARB-107)
+* Fix instrumentation in Ruby 2.0
+* Fix encoding exception on `hash_key_include` (ARB-53)
+* Fix erroneous start in non-Rails context (SQREEN-880)
+* Make metrics thread-safe
+* Add restart command
+* Fix `overtime_cb`
+* Add `perf_level` 2
+* Several performance optimizations
+* WAF: rename `budget_in_ms` to `max_budget_ms`
+* Transport/Tracing with http module
+* Update the blocking page
 
 ## 1.20.1
 

data/lib/sqreen/actions/block_user.rb

@@ -22,7 +22,7 @@ module Sqreen
       end
 
       def do_run(identity_params)
-        Sqreen.log.info(
+        Sqreen.log.debug(
           "Will raise due to user being blocked by action #{id}. " \
           "Blocked user identity: #{identity_params}"
         )

data/lib/sqreen/actions/redirect_ip.rb

@@ -25,7 +25,7 @@ module Sqreen
       end
 
       def do_run(client_ip)
-        Sqreen.log.info "Will request redirect for client with IP #{client_ip} " \
+        Sqreen.log.debug "Will request redirect for client with IP #{client_ip} " \
           "(action: #{id})."
         {
           :status => :skip,

data/lib/sqreen/actions/redirect_user.rb

@@ -24,7 +24,7 @@ module Sqreen
       end
 
       def do_run(identity_params)
-        Sqreen.log.info 'Will request redirect for user with identity ' \
+        Sqreen.log.debug 'Will request redirect for user with identity ' \
           "#{identity_params} (action: #{id})."
 
         e = Sqreen::AttackBlocked.new(

data/lib/sqreen/condition_evaluator.rb

@@ -67,7 +67,7 @@ module Sqreen
       return true if rem <= 0
       if hash.is_a?(Array)
         return hash.any? do |v|
-          ConditionEvaluator.hash_key_include?(values, v, min_value_size, rem - 1)
+          hash_key_include?(values, v, min_value_size, rem - 1)
         end
       end
 
@@ -81,7 +81,14 @@ module Sqreen
           if hkey.respond_to?(:empty?) && hkey.empty?
             false
           else
-            values.include?(hkey.to_s) || ConditionEvaluator.hash_key_include?(values, hval, min_value_size, rem - 1)
+            key_incl =
+              if values.is_a?(String)
+                str_include? values, hkey.to_s
+              else
+                values.include?(hkey.to_s)
+              end
+
+            key_incl || hash_key_include?(values, hval, min_value_size, rem - 1)
           end
         end
       end

data/lib/sqreen/conditionable.rb

@@ -28,21 +28,39 @@ module Sqreen
     end
 
     def pre_with_conditions(inst, args, budget = nil, &block)
+      return pre_without_conditions(inst, args, budget, &block) if pre_conditions.nil?
+
       eargs = [nil, framework, inst, args, @data, nil]
-      return nil if !pre_conditions.nil? && !pre_conditions.evaluate(*eargs)
-      pre_without_conditions(inst, args, budget, &block)
+      return nil unless pre_conditions.evaluate(*eargs)
+
+      res = pre_without_conditions(inst, args, budget, &block)
+      return { passed_conditions: true } unless res.is_a?(Hash)
+      res[:passed_conditions] = true
+      res
     end
 
     def post_with_conditions(rv, inst, args, budget = nil, &block)
+      return post_without_conditions(rv, inst, args, budget, &block) if post_conditions.nil?
+
       eargs = [nil, framework, inst, args, @data, rv]
-      return nil if !post_conditions.nil? && !post_conditions.evaluate(*eargs)
-      post_without_conditions(rv, inst, args, budget, &block)
+      return nil unless post_conditions.evaluate(*eargs)
+
+      res = post_without_conditions(rv, inst, args, budget, &block)
+      return { passed_conditions: true } if res.nil?
+      res[:passed_conditions] = true
+      res
     end
 
     def failing_with_conditions(rv, inst, args, budget = nil, &block)
+      return failing_without_conditions(rv, inst, args, budget, &block) if failing_conditions.nil?
+
       eargs = [nil, framework, inst, args, @data, rv]
-      return nil if !failing_conditions.nil? && !failing_conditions.evaluate(*eargs)
-      failing_without_conditions(rv, inst, args, budget, &block)
+      return nil unless failing_conditions.evaluate(*eargs)
+
+      res = failing_without_conditions(rv, inst, args, budget, &block)
+      return { passed_conditions: true } if res.nil?
+      res[:passed_conditions] = true
+      res
     end
 
     protected

data/lib/sqreen/configuration.rb

@@ -57,7 +57,7 @@ module Sqreen
     { :env => :SQREEN_RULES_SIGNATURE, :name => :rules_verify_signature,
       :default => true },
     { :env => :SQREEN_LOG_LEVEL, :name => :log_level,
-      :default => 'WARN', :choice => %w[UNKNOWN FATAL ERROR WARN INFO DEBUG] },
+      :default => 'INFO', :choice => %w[UNKNOWN FATAL ERROR WARN INFO DEBUG] },
     { :env => :SQREEN_LOG_LOCATION, :name => :log_location,
       :default => 'log/sqreen.log' },
     { :env => :SQREEN_RUN_IN_TEST, :name => :run_in_test,

data/lib/sqreen/deferred_logger.rb

@@ -9,35 +9,70 @@ require 'sqreen/logger'
 
 module Sqreen
   class DeferredLogger
-    include Singleton
+    MAX_ENTRIES = 1000
+
+    Entry = Struct.new(:severity, :message)
 
     def initialize
       @buffer = StringIO.new
       @logger = ::Logger.new(@buffer)
+      @entries = []
+      @mutex = Mutex.new
+    end
+
+    def debug?
+      true
+    end
+
+    def info?
+      true
+    end
+
+    def warn?
+      true
+    end
+
+    def error?
+      true
+    end
+
+    def fatal?
+      true
     end
 
     def debug(msg = nil, &block)
-      @logger.debug(msg, &block)
+      add(::Logger::DEBUG, msg, &block)
     end
 
     def info(msg = nil, &block)
-      @logger.info(msg, &block)
+      add(::Logger::INFO, msg, &block)
     end
 
     def warn(msg = nil, &block)
-      @logger.warn(msg, &block)
+      add(::Logger::WARN, msg, &block)
     end
 
     def error(msg = nil, &block)
-      @logger.error(msg, &block)
+      add(::Logger::ERROR, msg, &block)
     end
 
     def fatal(msg = nil, &block)
-      @logger.error(msg, &block)
+      add(::Logger::FATAL, msg, &block)
+    end
+
+    def unknown(msg = nil, &block)
+      add(::Logger::UNKNOWN, msg, &block)
     end
 
     def add(severity, msg = nil, &block)
-      send(Sqreen::Logger::SEVERITY_TO_METHOD[severity], msg, &block)
+      @mutex.synchronize do
+        @entries.shift if @entries.count >= MAX_ENTRIES
+        mark = @buffer.pos
+        @logger.add(severity, msg, &block)
+        @buffer.seek(mark)
+        @entries << Entry.new(severity, @buffer.read)
+        @buffer.truncate(0)
+      end
     end
 
     def formatter=(value)
@@ -45,21 +80,22 @@ module Sqreen
     end
 
     def flush_to(logger)
-      logger.instance_eval { @logdev }.write(read).tap { reset }
+      @mutex.synchronize do
+        @entries.each do |entry|
+          next if entry.severity < logger.level
+          logger.instance_eval { @logdev }.write(entry.message)
+        end
+        reset
+      end
     end
 
     private
 
-    def read
-      @buffer.rewind
-      @buffer.read
-    end
-
     def reset
       buffer = StringIO.new
       logger = ::Logger.new(buffer)
       logger.formatter = @logger.formatter
-      @buffer, @logger = buffer, logger
+      @buffer, @logger, @entries = buffer, logger, []
     end
   end
 end

data/lib/sqreen/deprecation.rb

@@ -0,0 +1,38 @@
+# typed: strong
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/log/loggable'
+
+module Sqreen
+  module Deprecation
+    include Sqreen::Log::Loggable
+
+    module_function
+
+    def deprecate(method)
+      return unless ENV['SQREEN_DEBUG_DEPRECATION']
+
+      owner = method.owner
+      deprecated = :"_deprecated_#{method.name}"
+      klass = owner.is_a?(Module)
+      target = klass ? owner.to_s : owner.class.to_s
+
+      method.owner.instance_eval do
+        alias_method deprecated, method.name
+
+        define_method(method.name) do |*args, &block|
+          msg = [
+            "deprecation",
+            "target:#{target}",
+            "method:#{method.name}",
+            "caller:#{Kernel.caller_locations[0]}",
+          ].join(' ')
+          Sqreen::Deprecation.logger.info(msg)
+          send(deprecated, *args, &block)
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem_integration.rb

@@ -1,6 +1,8 @@
 require 'sqreen/log/loggable'
+require 'sqreen/metrics_store'
 require 'sqreen/ecosystem'
 require 'sqreen/ecosystem/dispatch_table'
+require 'sqreen/ecosystem_integration/around_callbacks'
 require 'sqreen/ecosystem_integration/instrumentation_service'
 require 'sqreen/ecosystem_integration/request_lifecycle_tracking'
 require 'sqreen/ecosystem_integration/signal_consumption'
@@ -17,9 +19,12 @@ module Sqreen
     include Sqreen::Log::Loggable
 
     # @param [Sqreen::Framework] framework
-    def initialize(framework, queue)
+    # @param [Proc] create_binning_metric
+    def initialize(framework, queue, create_binning_metric)
       @framework = framework
       @queue = queue
+      # XXX: created metrics are insulated from feature upgrades
+      @create_binning_metric = create_binning_metric
       @request_lifecycle = RequestLifecycleTracking.new
       @online = false
     end
@@ -28,6 +33,7 @@ module Sqreen
       raise 'already initialized' if @online
 
       setup_dispatch_table
+      AroundCallbacks.create_metric = @create_binning_metric
       Ecosystem.init
       logger.info 'Ecosystem successfully initialized'
       @online = true

data/lib/sqreen/ecosystem_integration/around_callbacks.rb

@@ -1,6 +1,8 @@
+require 'sqreen/metrics_store'
 require 'sqreen/log/loggable'
 require 'sqreen/events/remote_exception'
 require 'sqreen/mono_time'
+require 'sqreen/graft/call'
 
 module Sqreen
   class EcosystemIntegration
@@ -8,27 +10,32 @@ module Sqreen
       class << self
         include Log::Loggable::ClassMethods
 
+        # @return [Proc]
+        attr_accessor :create_metric
+
         # for instrumentation hooks
-        # instrumentation hooks already handle budgets, so nothing
-        # to do in that respect
+        # instrumentation hooks already handle budgets/metrics
         def wrap_instrumentation_hook(module_name, action, callable)
-          perf_notif_name = "ecosystem_#{module_name}"
+          # make tag similar to that of rules
+          # TODO: move to structured tags to avoid silly string manips
+          action_for_metric = if %w[pre post failing finally].include?(action)
+                                action
+                              else
+                                'pre'
+                              end
+          metric_name = "sq.ecosystem_#{module_name}.#{action_for_metric}"
+          create_metric.call(metric_name)
 
           Proc.new do |*args|
             begin
-              start = Sqreen.time
               callable.call(*args)
             rescue ::Exception => e # rubocop:disable Lint/RescueException
               # 2) rescue exceptions
               logger.warn { "Error in #{module_name}:#{action}: #{e.message}" }
               logger.debug { e.backtrace.map { |x| "  #{x}" }.join("\n") }
               Sqreen::RemoteException.record(e)
-            ensure
-              # 3) contribute to performance metrics
-              stop = Sqreen.time
-              Sqreen::PerformanceNotifications.notify(perf_notif_name, action, start, stop)
-            end # end proc
-          end # end begin
+            end # end begin
+          end # end proc
         end
 
         # XXX: not used yet
@@ -36,6 +43,8 @@ module Sqreen
           timer_name = "ecosystem:#{module_name}@#{action}"
           perf_notif_name = "ecosystem_#{module_name}"
 
+          # XXX: register metric
+
           Proc.new do |*args|
             begin
               req_storage = Thread.current[:sqreen_http_request]
@@ -76,6 +85,7 @@ module Sqreen
               if timer
                 req_timer.include_measurements(timer) if req_timer
 
+                # XXX: PerformanceNotifications is used no more
                 Sqreen::PerformanceNotifications.notify(
                   perf_notif_name, action, *timer.start_and_end
                 )

data/lib/sqreen/ecosystem_integration/instrumentation_service.rb

@@ -12,22 +12,26 @@ module Sqreen
           hook = Sqreen::Graft::Hook[method].add do
             if spec[:before]
               cb = AroundCallbacks.wrap_instrumentation_hook(module_name, 'pre', spec[:before])
-              before(nil, flow: true, &cb)
+              tag = "weave,rule=ecosystem_#{module_name}"
+              before(tag, flow: true, &cb)
             end
 
             if spec[:after]
               cb = AroundCallbacks.wrap_instrumentation_hook(module_name, 'post', spec[:after])
-              after(nil, flow: true, &cb)
+              tag = "weave,rule=ecosystem_#{module_name}"
+              after(tag, flow: true, &cb)
             end
 
             if spec[:raised]
               cb = AroundCallbacks.wrap_instrumentation_hook(module_name, 'failing', spec[:raised])
-              raised(nil, flow: true, &cb)
+              tag = "weave,rule=ecosystem_#{module_name}"
+              raised(tag, flow: true, &cb)
             end
 
             if spec[:ensured]
               cb = AroundCallbacks.wrap_instrumentation_hook(module_name, 'finally', spec[:ensured])
-              ensured(nil, flow: true, &cb)
+              tag = "weave,rule=ecosystem_#{module_name}"
+              ensured(tag, flow: true, &cb)
             end
           end
           hook.install

data/lib/sqreen/events/request_record.rb

@@ -8,7 +8,6 @@
 require 'json'
 require 'sqreen/log'
 require 'sqreen/event'
-require 'sqreen/encoding_sanitizer'
 require 'sqreen/sensitive_data_redactor'
 
 module Sqreen

data/lib/sqreen/frameworks/generic.rb

@@ -218,7 +218,16 @@ module Sqreen
 
       # Should the agent not be starting up?
       def prevent_startup
+        # SQREEN-880 - prevent Sqreen startup on Sidekiq workers
+        return :sidekiq_cli if defined?(Sidekiq::CLI)
+        return :delayed_job if defined?(Delayed::Command)
+
+        # Prevent Sqreen startup on rake tasks - unless this is a Sqreen test
+        run_in_test = sqreen_configuration.get(:run_in_test)
+        return :rake if !run_in_test && $0.end_with?('rake')
+
         return :irb if $0 == 'irb'
+
         return if sqreen_configuration.nil?
         disable = sqreen_configuration.get(:disable)
         return :config_disable if disable == true || disable.to_s.to_i == 1