sqreen 1.20.4-java → 1.21.0.beta3-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 67f60f1ad0bd4392e5a60272ef50f406aa511a5e
-  data.tar.gz: bbd2e7baacc6c6cdaa7bb95d06300fa325a5ef09
+  metadata.gz: 00e4d67d3fbef516336bc81d189df526cfd4e67c
+  data.tar.gz: d66dc33e04ec02c6f7e3118ba7d2727ee20dbd4d
 SHA512:
-  metadata.gz: 86bdc374c4a1e7a01cfc371ef975fd17d6e58709a06dde323a8b7006766472cd974f4fb22ce663c693b840d3cbb19556d7cb0310df931217872283da5f1c2292
-  data.tar.gz: 6ee3811d3d1292d8b4d818323624b98837cb24c4f62f994f83a6c1b6d0eb3616961067073f20527ef0f246ad5f2665fe0c193d2b8ed79d95642ccb02d2125d7c
+  metadata.gz: b9ea001f33c1daf64e8e6cecfdb3463f70c67bbec7db64b1b138216178a769451b0190d3101a67d0b0d5326f7cb3d717fb0b277720fbedcdeefb9bfcaf05018a
+  data.tar.gz: f56453335a9e3340e5434b376322a8e0993df25ddfb16df34c26e88aa30eccb3bab4bb31c04a0c6174f5119f65a2ede9d3732103809ee12ffa8da1d22213c361

data/CHANGELOG.md

@@ -1,27 +1,18 @@
-## 1.20.4
-
-* Fix missing budget check
-* Improve performance
-* Align internal setting name for WAF
-* Include response information in all payloads
-* Improve robustness against invalid Unicode
-* Prevent rule execution to pursue in early block cases
-
-## 1.20.4.beta1
-
-* Add optional dynamic time budget
-* Add advanced per request metrics
-* Improve robustness against exception in instrumentation
-* Improve metric engine thread safety
-* Restrict deferred logger to final logger severity on agent boot
-
-## 1.20.3
-
-* Fix signature check
-
-## 1.20.2
-
-* Fix performance regression in instrumentation engine
+## 1.21.0.beta3
+
+* Avoid fd leak in `custom_error_cb` (ARB-109)
+* Support `skip_rem_cbs` (ARB-107)
+* Fix instrumentation in Ruby 2.0
+* Fix encoding exception on `hash_key_include` (ARB-53)
+* Fix erroneous start in non-Rails context (SQREEN-880)
+* Make metrics thread-safe
+* Add restart command
+* Fix `overtime_cb`
+* Add `perf_level` 2
+* Several performance optimizations
+* WAF: rename `budget_in_ms` to `max_budget_ms`
+* Transport/Tracing with http module
+* Update the blocking page
 
 ## 1.20.1
 

data/lib/sqreen/condition_evaluator.rb

@@ -81,11 +81,12 @@ module Sqreen
           if hkey.respond_to?(:empty?) && hkey.empty?
             false
           else
-            key_incl = if values.is_a?(String)
-                         str_include?(values, hkey.to_s)
-                       else
-                         values.include?(hkey.to_s)
-                       end
+            key_incl =
+              if values.is_a?(String)
+                str_include? values, hkey.to_s
+              else
+                values.include?(hkey.to_s)
+              end
 
             key_incl || hash_key_include?(values, hval, min_value_size, rem - 1)
           end

data/lib/sqreen/conditionable.rb

@@ -28,21 +28,39 @@ module Sqreen
     end
 
     def pre_with_conditions(inst, args, budget = nil, &block)
+      return pre_without_conditions(inst, args, budget, &block) if pre_conditions.nil?
+
       eargs = [nil, framework, inst, args, @data, nil]
-      return nil if !pre_conditions.nil? && !pre_conditions.evaluate(*eargs)
-      pre_without_conditions(inst, args, budget, &block)
+      return nil unless pre_conditions.evaluate(*eargs)
+
+      res = pre_without_conditions(inst, args, budget, &block)
+      return { passed_conditions: true } unless res.is_a?(Hash)
+      res[:passed_conditions] = true
+      res
     end
 
     def post_with_conditions(rv, inst, args, budget = nil, &block)
+      return post_without_conditions(rv, inst, args, budget, &block) if post_conditions.nil?
+
       eargs = [nil, framework, inst, args, @data, rv]
-      return nil if !post_conditions.nil? && !post_conditions.evaluate(*eargs)
-      post_without_conditions(rv, inst, args, budget, &block)
+      return nil unless post_conditions.evaluate(*eargs)
+
+      res = post_without_conditions(rv, inst, args, budget, &block)
+      return { passed_conditions: true } if res.nil?
+      res[:passed_conditions] = true
+      res
     end
 
     def failing_with_conditions(rv, inst, args, budget = nil, &block)
+      return failing_without_conditions(rv, inst, args, budget, &block) if failing_conditions.nil?
+
       eargs = [nil, framework, inst, args, @data, rv]
-      return nil if !failing_conditions.nil? && !failing_conditions.evaluate(*eargs)
-      failing_without_conditions(rv, inst, args, budget, &block)
+      return nil unless failing_conditions.evaluate(*eargs)
+
+      res = failing_without_conditions(rv, inst, args, budget, &block)
+      return { passed_conditions: true } if res.nil?
+      res[:passed_conditions] = true
+      res
     end
 
     protected

data/lib/sqreen/deliveries/batch.rb

@@ -13,6 +13,8 @@ require 'sqreen/events/attack'
 require 'sqreen/events/remote_exception'
 require 'sqreen/mono_time'
 require 'sqreen/deliveries/simple'
+require 'sqreen/kit/signals/signal'
+require 'sqreen/kit/signals/trace'
 
 module Sqreen
   module Deliveries
@@ -58,7 +60,7 @@ module Sqreen
       def post_batch_needed?(event)
         now = Sqreen.time
         # do not use any? {} due to side effects inside block
-        event_keys(event).map do |key|
+        event_keys(event).uniq.map do |key|
           was = @first_seen[key]
           @first_seen[key] ||= now
           was.nil? || current_batch.size > max_batch || now > (was + max_staleness)
@@ -86,6 +88,7 @@ module Sqreen
         res += event.observed.fetch(:sdk, []).select { |e|
             e[0] == :track
         }.map { |e| "sdk-track".freeze }
+        res += event.observed.fetch(:signals, []).map { "signal".freeze }
         return res
       end
 
@@ -97,6 +100,10 @@ module Sqreen
           "rex-#{event.klass}"
         when Sqreen::AggregatedMetric
           "agg-metric"
+        when Sqreen::Kit::Signals::Signal
+          "signal"
+        when Sqreen::Kit::Signals::Trace
+          "signal"
         end
       end
     end

data/lib/sqreen/ecosystem/dispatch_table.rb

@@ -0,0 +1,43 @@
+require 'logger'
+
+module Sqreen
+  module Ecosystem
+    # Configured by the ecosystem client
+    module DispatchTable
+      class << self
+        # data consumption
+        # argument: +Sqreen::Kit::Signals::Signal+
+        # see +Sqreen::EcosystemIntegration::SignalConsumption#consume_signal+
+        attr_accessor :consume_signal
+
+        # argument: block taking a Rack::Request
+        # see +Sqreen::EcosystemIntegration::RequestLifecycleTracking#add_start_observer+
+        attr_accessor :add_request_start_listener
+
+        attr_accessor :fetch_logger
+
+        # argument: callback taking:
+        # * the method to instrument
+        # * A Hash{Symbol=>Proc} with the advice. The proc takes the
+        # arguments and the ball, so these details of the instrumentation
+        # implementation leak through the abstraction
+        # see +Sqreen::EcosystemIntegration::InstrumentationService+
+        attr_accessor :instrument
+
+        def reset
+          instance_variables.each do |ia|
+            instance_variable_set(ia, nil)
+          end
+
+          # set default logger
+          logger = ::Logger.new(STDERR)
+          logger.level = ::Logger::WARN
+          logger.progname = 'sqreen-ecosystem'
+          self.fetch_logger = proc { logger }
+        end
+      end
+
+      reset
+    end
+  end
+end

data/lib/sqreen/ecosystem/exception_reporting.rb

@@ -0,0 +1,26 @@
+require 'sqreen/ecosystem/dispatch_table'
+require 'sqreen/ecosystem/loggable'
+require 'sqreen/kit/signals/specialized/sqreen_exception'
+
+module Sqreen
+  module Ecosystem
+    module ExceptionReporting
+      include Loggable
+
+      private
+
+      # @param [String] message
+      # @param [Exception] e
+      def report_exception(message, e)
+        logger.warn { "#{message}: #{e.message}" }
+        logger.debug { e.backtrace.map { |x| "  #{x}" }.join("\n") }
+
+        signal = Sqreen::Kit::Signals::Specialized::SqreenException.new(
+          ruby_exception: e
+        )
+
+        DispatchTable.consume_signal signal
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/http/net_http.rb

@@ -0,0 +1,50 @@
+require 'sqreen/ecosystem/module_api'
+require 'sqreen/ecosystem/module_api/instrumentation'
+require 'sqreen/ecosystem/module_api/message_producer'
+require 'sqreen/ecosystem/module_api/tracing_id_generation'
+require 'sqreen/ecosystem/module_api/tracing/client_data'
+
+module Sqreen
+  module Ecosystem
+    module Http
+      class NetHttp
+        class HttpConnectionData
+          include ModuleApi::Tracing::ClientData
+        end
+
+        include ModuleApi::Instrumentation
+        include ModuleApi::MessageProducer
+        include ModuleApi::TracingIdGeneration
+
+        def setup
+          advice = wrap_for_interest(HttpConnectionData, &method(:before_advice))
+          instrument 'Net::HTTP#request', before: advice
+        end
+
+        private
+
+        # instr. def request(req, body = nil, &block)  # :yield: +response+
+        # req is of type +Net::HTTPGenericRequest+
+        def before_advice(call, _ball)
+          tracing_id = create_tracing_id
+
+          # build & submit signal
+          host = call.instance.address
+          port = call.instance.port
+
+          # add tracing header
+          req = call.args[0]
+          req[ModuleApi::TRACE_ID_HEADER] = tracing_id
+
+          host += ':' + port.to_s if port != 80 && port != 443
+
+          HttpConnectionData.new(
+            transport: 'http',
+            host: host,
+            tracing_identifier: tracing_id
+          )
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/http/rack_request.rb

@@ -0,0 +1,39 @@
+require 'sqreen/ecosystem/module_api'
+require 'sqreen/ecosystem/module_api/event_listener'
+require 'sqreen/ecosystem/module_api/message_producer'
+require 'sqreen/ecosystem/module_api/tracing/server_data'
+
+module Sqreen
+  module Ecosystem
+    module Http
+      class RackRequest
+        class HttpServerData
+          include ModuleApi::Tracing::ServerData
+        end
+
+        include ModuleApi::EventListener
+        include ModuleApi::MessageProducer
+
+        def setup
+          advice = wrap_for_interest(
+            ModuleApi::Tracing::ServerData,
+            &method(:handle_request)
+          )
+          on_request_start(&advice)
+        end
+
+        private
+
+        def handle_request(rack_request)
+          trace_id = rack_request.env[ModuleApi::TRACE_ID_ENV_KEY]
+
+          HttpServerData.new(
+            transport: 'http',
+            client_ip: rack_request.ip,
+            tracing_identifier: trace_id
+          )
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/loggable.rb

@@ -0,0 +1,13 @@
+require 'sqreen/ecosystem/dispatch_table'
+
+module Sqreen
+  module Ecosystem
+    module Loggable
+      private
+
+      def logger
+        DispatchTable.fetch_logger.call
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api/event_listener.rb

@@ -0,0 +1,18 @@
+require 'sqreen/ecosystem/dispatch_table'
+
+module Sqreen
+  module Ecosystem
+    module ModuleApi
+      module EventListener
+        private
+
+        # XXX: callbacks need to be wrapped in order ot handle
+        # perfcap, exceptions, and maybe other concerns applying
+        # across the board
+        def on_request_start(&cb)
+          DispatchTable.add_request_start_listener.call(cb)
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api/instrumentation.rb

@@ -0,0 +1,23 @@
+require 'sqreen/ecosystem/module_api'
+
+module Sqreen
+  module Ecosystem
+    module ModuleApi
+      module Instrumentation
+        def self.included(mod)
+          mod.send :include, ModuleApi unless mod.ancestors.include?(ModuleApi)
+        end
+
+        private
+
+        # Just forwards the call to the instrumentation service
+        # @param [String] method
+        # @param [Hash{Symbol=>Proc}] advice keys are one of: :before, :after,
+        #                             :raised,
+        def instrument(method, advice)
+          DispatchTable.instrument.call(self.class.module_name, method, advice)
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api/message_producer.rb

@@ -0,0 +1,51 @@
+require 'sqreen/ecosystem/loggable'
+
+module Sqreen
+  module Ecosystem
+    module ModuleApi
+      module MessageProducer
+        include Loggable
+
+        # method for ecosystem to inject the config
+        # @param [Sqreen::Ecosystem::TracingBroker]
+        attr_writer :tracing_broker
+
+        private
+
+        def determine_interest(type, hints = {})
+          @tracing_broker.interested_consumers(type, hints)
+        end
+
+        def publish(data, interested)
+          @tracing_broker.publish(data, interested)
+        end
+
+        # Convenience wrapper.
+        # Wraps a callback, skipping it if there is no interest in the type
+        # produced and submitting the return value as a message to the
+        # tracing broker
+        def wrap_for_interest(type, gen_hints = nil, &block)
+          raise ArgumentError, 'no block passed' if block.nil?
+
+          proc do |*args|
+            hints = gen_hints[*args] if gen_hints
+            interested = determine_interest(type, hints || {})
+
+            unless interested
+              logger.debug { "No interested consumers in #{type}" }
+              next
+            end
+
+            res = block[*args]
+
+            next if res.nil?
+
+            raise 'unexpected return type' unless res.is_a?(type)
+
+            @tracing_broker.publish(res, interested)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api/signal_producer.rb

@@ -0,0 +1,24 @@
+require 'sqreen/ecosystem/dispatch_table'
+
+module Sqreen
+  module Ecosystem
+    module ModuleApi
+      module SignalProducer
+        # for injection
+        # callable taking no arguments and generating a tracing id
+        attr_writer :tracing_id_producer
+
+        private
+
+        def create_tracing_id
+          @tracing_id_producer.call
+        end
+
+        # @param [Sqreen::Kit::Signals::Signal] signal
+        def submit_signal(signal)
+          DispatchTable.consume_signal.call(signal)
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api/tracing/client_data.rb

@@ -0,0 +1,31 @@
+require 'sqreen/ecosystem/util/call_writers_from_init'
+
+module Sqreen
+  module Ecosystem
+    module ModuleApi
+      module Tracing
+        # The data the tracing module needs in order to populate
+        # +Sqreen::Ecosystem::Tracing::Signals::TracingClient::Payload+
+        #
+        # Signals are not produced by the data producers (transport)
+        # because of superior orders, as the only current use of this
+        # data is to generate signals.
+        module ClientData
+          include Util::CallWritersFromInit
+
+          # @return [Symbol]
+          attr_accessor :transport
+
+          # @return [String]
+          attr_accessor :host
+
+          # @return [String]
+          attr_accessor :ip
+
+          # @return [String]
+          attr_accessor :tracing_identifier
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api/tracing/server_data.rb

@@ -0,0 +1,27 @@
+require 'sqreen/ecosystem/util/call_writers_from_init'
+
+module Sqreen
+  module Ecosystem
+    module ModuleApi
+      module Tracing
+        # The data the tracing module needs in order to populate
+        # +Sqreen::Ecosystem::Tracing::Signals::TracingServer::Payload+
+        module ServerData
+          include Util::CallWritersFromInit
+
+          # @return [Symbol]
+          attr_accessor :transport
+
+          # @return [String]
+          attr_accessor :client_ip
+
+          # @return [Array<String>]
+          attr_accessor :previous_hops
+
+          # @return [String]
+          attr_accessor :tracing_identifier
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api/tracing.rb

@@ -0,0 +1,45 @@
+require 'sqreen/ecosystem/module_api/signal_producer'
+
+module Sqreen
+  module Ecosystem
+    module ModuleApi
+      module Tracing
+        include SignalProducer
+
+        def self.included(mod)
+          mod.extend(ClassMethods)
+        end
+
+        module ClassMethods
+          private
+
+          # @param [Module] type The type the including module is interested in
+          def consumes(type)
+            @consumes = type
+          end
+
+          # A fixed (non-virtual) scope for this tracing module
+          # @param [String] scope
+          def fixed_scope(scope)
+            @fixed_scope = scope
+          end
+        end
+
+        def consumed_type
+          self.class.instance_variable_get(:@consumes) \
+            || raise('@consumes not specified')
+        end
+
+        def scope(_hints = {})
+          self.class.instance_variable_get(:@fixed_scope) \
+            || raise('@fixed_scope not set')
+        end
+
+        # including class must implement it
+        def receive(_data)
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api/tracing_id_generation.rb

@@ -0,0 +1,16 @@
+module Sqreen
+  module Ecosystem
+    module ModuleApi
+      module TracingIdGeneration
+        # for injection
+        attr_writer :tracing_id_producer
+
+        private
+
+        def create_tracing_id
+          @tracing_id_producer.call
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api/transaction_storage.rb

@@ -0,0 +1,71 @@
+require 'sqreen/ecosystem/transaction_storage'
+require 'sqreen/ecosystem/loggable'
+
+module Sqreen
+  module Ecosystem
+    module ModuleApi
+      module TransactionStorage
+        class TxLocalVariables
+          class << self
+            class << self
+              include Sqreen::Ecosystem::Loggable
+
+              private
+
+              def attr_reader(attr, _opts = {})
+                define_method attr do
+                  tx_storage = Ecosystem::TransactionStorage.fetch_thread_local
+                  return unless tx_storage
+                  tx_storage[attr]
+                end
+              end
+
+              def attr_accessor(attr, opts = {})
+                # reader
+                attr_reader attr, opts
+
+                # writer (2 variants)
+                do_assign = proc do |value|
+                  tx_storage = Ecosystem::TransactionStorage.fetch_thread_local
+                  unless tx_storage
+                    logger.debug do
+                      "Assignment of tx local attribute #{attr} to #{value} has no effect"
+                    end
+                    return
+                  end
+
+                  tx_storage[attr] = value
+                end
+
+                if opts.fetch(:allow_overwrite, false)
+                  define "#{attr}=", &do_assign
+                else
+                  define_method "#{attr}=" do |value|
+                    cur = public_send(attr)
+                    unless cur.nil?
+                      raise "Cannot override value of #{attr} from #{cur} with #{value}"
+                    end
+
+                    do_assign.call(value)
+                  end
+                end
+              end
+            end # TxLocalVariables.singleton_class.singleton_class
+
+            # usage:
+            # attr_reader :xxx
+
+            # in the future, we'll possibly need to expose the full
+            # TransactionStorage to the modules, at least if we don't
+            # opt for a more structured fashion of data exchange between
+            # the modules.
+          end # TxLocalVariables.singleton_class
+        end # TxLocalVariables
+
+        def tx_local_vars
+          TxLocalVariables
+        end
+      end # TransactionStorage module
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api.rb

@@ -0,0 +1,30 @@
+require 'sqreen/ecosystem/loggable'
+
+module Sqreen
+  module Ecosystem
+    # The API that the transport/tracing modules are written against
+    module ModuleApi
+      TRACE_ID_HEADER = 'X-Sqreen-Trace-Identifier'.freeze
+      TRACE_ID_ENV_KEY = 'HTTP_X_SQREEN_TRACE_IDENTIFIER'.freeze
+
+      Loggable = Sqreen::Ecosystem::Loggable
+
+      module ClassMethods
+        attr_writer :module_name
+
+        def module_name
+          if instance_variable_defined?(:@module_name)
+            @module_name
+          else
+            # to snake case
+            @module_name = to_s.sub(/.*::/, '').gsub(/([a-z])([A-Z])/, '\1_\2').downcase
+          end
+        end
+      end
+
+      def self.included(mod)
+        mod.extend(ClassMethods)
+      end
+    end
+  end
+end