sqreen 1.20.4.beta1 → 1.21.0

data/lib/sqreen/remote_command.rb

@@ -23,6 +23,7 @@ module Sqreen
       :ips_whitelist => :change_whitelisted_ips,
       :get_bundle => :upload_bundle,
       :performance_budget => :change_performance_budget,
+      :tracing_enable => :tracing_enable,
     }.freeze
 
     attr_reader :uuid
@@ -40,6 +41,8 @@ module Sqreen
       begin
         output = runner.send(KNOWN_COMMANDS[@name], *@params, context_infos)
       rescue => e
+        Sqreen.log.warn { "Command failed with #{e}" }
+        Sqreen.log.debug { e.backtrace.map { |x| "  #{x}" }.join("\n") }
         Sqreen::RemoteException.record(e)
         return { :status => false, :reason => "error: #{e.inspect}" }
       end

data/lib/sqreen/rules.rb

@@ -120,9 +120,13 @@ module Sqreen
 
       cb_class = ExecJSCB if js
 
-      if cbname && Rules.const_defined?(cbname)
-        # Only load callbacks from sqreen
-        cb_class = Rules.const_get(cbname)
+      if cbname
+        cb_class = if cbname.include?('::')
+                     # Only load callbacks from sqreen
+                     Rules.walk_const_get(cbname) if cbname.start_with?('::Sqreen::', 'Sqreen::')
+                   else
+                     Rules.const_get(cbname) if Rules.const_defined?(cbname) # rubocop:disable Style/IfInsideElse
+                   end
       end
 
       if cb_class.nil?

data/lib/sqreen/rules/custom_error_cb.rb

@@ -55,12 +55,12 @@ module Sqreen
       end
 
       def respond_page
-        page = open(File.join(File.dirname(__FILE__), '../attack_detected.html'))
+        @page ||= File.open(File.join(File.dirname(__FILE__), '../attack_detected.html'), 'rb', &:read)
         headers = {
           'Content-Type' => 'text/html',
-          'Content-Length' => page.size.to_s,
+          'Content-Length' => @page.size.to_s,
         }
-        [@status_code, headers, page]
+        [@status_code, headers, [@page]]
       end
     end
   end

data/lib/sqreen/rules/waf_cb.rb

@@ -11,7 +11,7 @@ require 'sqreen/safe_json'
 require 'sqreen/exception'
 require 'sqreen/util/capper'
 require 'sqreen/dependency/libsqreen'
-require 'sqreen/encoding_sanitizer'
+require 'sqreen/kit/string_sanitizer'
 
 module Sqreen
   module Rules
@@ -60,7 +60,7 @@ module Sqreen
         end
 
         # 0 for using defaults (PW_RUN_TIMEOUT)
-        @max_run_budget_us = (@data['values'].fetch('budget_in_ms', 0) * 1000).to_i
+        @max_run_budget_us = (@data['values'].fetch('max_budget_ms', 0) * 1000).to_i
         @max_run_budget_us = INFINITE_BUDGET_US if @max_run_budget_us >= INFINITE_BUDGET_US
 
         Sqreen.log.debug { "Max WAF run budget for #{@waf_rule_name} set to #{@max_run_budget_us} us" }
@@ -82,7 +82,7 @@ module Sqreen
         waf_args = binding_accessors.each_with_object({}) do |(e, b), h|
           h[e] = capper.call(b.resolve(*env))
         end
-        waf_args = Sqreen::EncodingSanitizer.sanitize(waf_args)
+        waf_args = Sqreen::Kit::StringSanitizer.sanitize(waf_args)
 
         if budget
           rem_budget_s = budget - (Sqreen.time - start)

data/lib/sqreen/runner.rb

@@ -14,6 +14,7 @@ require 'sqreen/log'
 require 'sqreen/agent_message'
 require 'sqreen/rules'
 require 'sqreen/session'
+require 'sqreen/version'
 require 'sqreen/remote_command'
 require 'sqreen/capped_queue'
 require 'sqreen/metrics_store'
@@ -26,6 +27,7 @@ require 'sqreen/legacy/instrumentation'
 require 'sqreen/call_countable'
 require 'sqreen/weave/legacy/instrumentation'
 require 'sqreen/kit/configuration'
+require 'sqreen/ecosystem_integration'
 
 module Sqreen
   @features = {}
@@ -52,10 +54,6 @@ module Sqreen
       @queue ||= CappedQueue.new(MAX_QUEUE_LENGTH)
     end
 
-    def update_queue(queue)
-      @queue = queue
-    end
-
     def observations_queue
       @observations_queue ||= CappedQueue.new(MAX_OBS_QUEUE_LENGTH)
     end
@@ -104,8 +102,8 @@ module Sqreen
     # we may want to do that in a thread in order to prevent delaying app
     # startup
     # set_at_exit do not place a global at_exit (used for testing)
+    # @param [Sqreen::Frameworks::GenericFramework] framework
     def initialize(configuration, framework, set_at_exit = true, session_class = Sqreen::Session)
-      Sqreen.update_queue(CappedQueue.new(MAX_QUEUE_LENGTH))
       @logged_out_tried = false
       @configuration = configuration
       @framework = framework
@@ -132,6 +130,7 @@ module Sqreen
       Sqreen::Kit::Configuration.ingestion_url = chosen_endpoints.ingestion.url
       Sqreen::Kit::Configuration.certificate_store = chosen_endpoints.ingestion.ca_store
       Sqreen::Kit::Configuration.proxy_url = @proxy_url
+      Sqreen::Kit::Configuration.default_source = "sqreen:agent:ruby:#{Sqreen::VERSION}"
 
       register_exit_cb if set_at_exit
 
@@ -168,6 +167,10 @@ module Sqreen
       end
       self.features = wanted_features
 
+      @ecosystem_integration = EcosystemIntegration.new(framework, Sqreen.queue)
+      framework.req_start_cb = @ecosystem_integration.method(:request_start)
+      framework.req_end_cb = @ecosystem_integration.method(:request_end)
+
       # Ensure a deliverer is there unless features have set it first
       self.deliverer ||= Deliveries::Simple.new(session)
       context_infos = {}
@@ -268,6 +271,10 @@ module Sqreen
       rulespack_id, rules = load_rules(context_infos)
       @framework.instrument_when_ready!(instrumenter, rules)
       Sqreen.log.info 'Instrumentation set up'
+
+      # XXX: ecosystem instrumentation should likely be deferred
+      #      the same way the rest might be
+      @ecosystem_integration.init
       rulespack_id.to_s
     end
 
@@ -409,6 +416,13 @@ module Sqreen
       { :was => prev }
     end
 
+    # @param [String] tracing_id_prefix
+    # @param [Array<Hash{String=>Object}>] sampling_config
+    def tracing_enable(tracing_id_prefix, sampling_config, _context_infos = {})
+      @ecosystem_integration.handle_tracing_command(tracing_id_prefix, sampling_config)
+      { status: true }
+    end
+
     def upload_bundle(_context_infos = {})
       t = Time.now
       session.post_bundle(RuntimeInfos.dependencies_signature, RuntimeInfos.dependencies)

data/lib/sqreen/session.rb

@@ -249,8 +249,10 @@ module Sqreen
       end
       Sqreen.log.info 'Login success.'
       @session_id = res['session_id']
+
       Kit::Configuration.session_key = @session_id
       Kit.reset
+
       Sqreen.log.debug { "received session_id #{@session_id}" }
       Sqreen.logged_in = true
       res

data/lib/sqreen/signals/conversions.rb

@@ -118,6 +118,7 @@ module Sqreen
           signals += req_rec.processed_sdk_calls
                             .select { |h| h[:name] == :track }
                             .map { |h| convert_track(h) }
+          signals += (observed[:signals] || [])
 
           trace = Kit::Signals::Specialized::HttpTrace.new(
             actor: Kit::Signals::Actor.new(
@@ -137,7 +138,7 @@ module Sqreen
           trace
         end
 
-        # @param [Array<Sqreen::Kit::Signals::Signal|Sqreen::Kit::Signals::Trace>] batch
+        # @return [Array<Sqreen::Kit::Signals::Signal|Sqreen::Kit::Signals::Trace>]
         def convert_batch(batch)
           batch.map do |evt|
             case evt
@@ -147,6 +148,10 @@ module Sqreen
               convert_metric_sample(evt)
             when RequestRecord
               convert_req_record(evt)
+            when Sqreen::Kit::Signals::Signal
+              evt
+            when Sqreen::Kit::Signals::Trace
+              evt
             else
               raise NotImplementedError, "Unknown type of event in batch: #{evt}"
             end

data/lib/sqreen/version.rb

@@ -4,5 +4,5 @@
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
 module Sqreen
-  VERSION = '1.20.4.beta1'.freeze
+  VERSION = '1.21.0'.freeze
 end

data/lib/sqreen/weave/legacy/instrumentation.rb

@@ -5,6 +5,7 @@
 
 require 'sqreen/weave/legacy'
 require 'sqreen/weave/budget'
+require 'sqreen/graft/hook'
 require 'sqreen/graft/hook_point'
 require 'sqreen/call_countable'
 require 'sqreen/rules'
@@ -172,8 +173,8 @@ class Sqreen::Weave::Legacy::Instrumentation
         request_timer.start
         sqreen_timer = Sqreen::Graft::Timer.new("sqreen")
         budget = Sqreen::Weave::Budget.current
-        request_budget_threshold = budget.threshold
-        request_budget_ratio = budget.ratio
+        request_budget_threshold = budget.threshold if budget
+        request_budget_ratio = budget.ratio if budget
         request_budget_is_dynamic = !request_budget_ratio.nil?
         request_budget = !request_budget_threshold.nil?
         timed_level = (Sqreen.features['perf_level'] || 1).to_i
@@ -212,12 +213,16 @@ class Sqreen::Weave::Legacy::Instrumentation
         if request[:timed_level] >= 1
           request[:timed_callbacks].each do |timer|
             duration = timer.duration
+
             timer.tag =~ /weave,rule=(.*)$/ && rule = $1
-            timer.tag =~ /@before/ && whence = 'pre'
-            timer.tag =~ /@after/  && whence = 'post'
-            timer.tag =~ /@raised/ && whence = 'failing'
+            next unless rule
 
-            next unless rule && whence
+            whence = case timer.tag
+                     when /@before/ then 'pre'
+                     when /@after/  then 'post'
+                     when /@raised/ then 'failing'
+                     end
+            next unless whence
 
             metric_name = "sq.#{rule}.#{whence}"
             metrics_engine.update(metric_name, now, nil, duration * 1000)
@@ -270,12 +275,16 @@ class Sqreen::Weave::Legacy::Instrumentation
           tallies = Hash.new(0.0)
           request[:timed_callbacks].each do |timer|
             duration = timer.duration
+
             timer.tag =~ /weave,rule=(.*)$/ && rule = $1
-            timer.tag =~ /@before/ && whence = 'pre'
-            timer.tag =~ /@after/  && whence = 'post'
-            timer.tag =~ /@raised/ && whence = 'failing'
+            next unless rule
 
-            next unless rule && whence
+            whence = case timer.tag
+                     when /@before/ then 'pre'
+                     when /@after/  then 'post'
+                     when /@raised/ then 'failing'
+                     end
+            next unless whence
 
             metric_name = "req.sq.#{rule}.#{whence}"
             tallies[metric_name] += duration
@@ -364,15 +373,25 @@ class Sqreen::Weave::Legacy::Instrumentation
             end
           end
 
-          case ret[:status]
-          when :skip, 'skip'
-            throw(b, b.return(ret[:new_return_value]).break!) if ret.key?(:new_return_value)
-          when :modify_args, 'modify_args'
-            throw(b, b.args(ret[:args]))
-          when :raise, 'raise'
-            throw(b, b.raise(ret[:exception])) if ret.key?(:exception)
-            throw(b, b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required.")))
-          end unless ret.nil? || !ret.is_a?(Hash)
+          next if ret.nil? || !ret.is_a?(Hash)
+
+          throw_val =
+            case ret[:status]
+            when :skip, 'skip'
+              b.return(ret[:new_return_value]).break! if ret.key?(:new_return_value)
+            when :modify_args, 'modify_args'
+              b.args(ret[:args])
+            when :raise, 'raise'
+              if ret.key?(:exception)
+                b.raise(ret[:exception])
+              else
+                b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required."))
+              end
+            end
+
+          next unless throw_val
+          throw_val.break! if ret[:skip_rem_cbs]
+          throw(b, throw_val)
         end
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sqreen
 version: !ruby/object:Gem::Version
-  version: 1.20.4.beta1
+  version: 1.21.0
 platform: ruby
 authors:
 - Sqreen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-08-14 00:00:00.000000000 Z
+date: 2020-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sqreen-backport
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.1
+        version: 0.2.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.1
+        version: 0.2.2
 - !ruby/object:Gem::Dependency
   name: sq_mini_racer
   requirement: !ruby/object:Gem::Requirement
@@ -123,7 +123,55 @@ files:
 - lib/sqreen/dependency/sentry.rb
 - lib/sqreen/dependency/sinatra.rb
 - lib/sqreen/deprecation.rb
-- lib/sqreen/encoding_sanitizer.rb
+- lib/sqreen/ecosystem.rb
+- lib/sqreen/ecosystem/databases/database_connection_data.rb
+- lib/sqreen/ecosystem/databases/mongo.rb
+- lib/sqreen/ecosystem/databases/mysql.rb
+- lib/sqreen/ecosystem/databases/postgres.rb
+- lib/sqreen/ecosystem/databases/redis.rb
+- lib/sqreen/ecosystem/dispatch_table.rb
+- lib/sqreen/ecosystem/exception_reporting.rb
+- lib/sqreen/ecosystem/http/net_http.rb
+- lib/sqreen/ecosystem/http/rack_request.rb
+- lib/sqreen/ecosystem/loggable.rb
+- lib/sqreen/ecosystem/messaging/bunny.rb
+- lib/sqreen/ecosystem/messaging/kafka.rb
+- lib/sqreen/ecosystem/messaging/kinesis.rb
+- lib/sqreen/ecosystem/messaging/sqs.rb
+- lib/sqreen/ecosystem/module_api.rb
+- lib/sqreen/ecosystem/module_api/event_listener.rb
+- lib/sqreen/ecosystem/module_api/instrumentation.rb
+- lib/sqreen/ecosystem/module_api/message_producer.rb
+- lib/sqreen/ecosystem/module_api/signal_producer.rb
+- lib/sqreen/ecosystem/module_api/tracing.rb
+- lib/sqreen/ecosystem/module_api/tracing/client_data.rb
+- lib/sqreen/ecosystem/module_api/tracing/consumer_data.rb
+- lib/sqreen/ecosystem/module_api/tracing/messaging_data.rb
+- lib/sqreen/ecosystem/module_api/tracing/producer_data.rb
+- lib/sqreen/ecosystem/module_api/tracing/server_data.rb
+- lib/sqreen/ecosystem/module_api/tracing_id_generation.rb
+- lib/sqreen/ecosystem/module_api/transaction_storage.rb
+- lib/sqreen/ecosystem/module_registry.rb
+- lib/sqreen/ecosystem/tracing/modules/client.rb
+- lib/sqreen/ecosystem/tracing/modules/consumer.rb
+- lib/sqreen/ecosystem/tracing/modules/determine_ip.rb
+- lib/sqreen/ecosystem/tracing/modules/producer.rb
+- lib/sqreen/ecosystem/tracing/modules/server.rb
+- lib/sqreen/ecosystem/tracing/sampler.rb
+- lib/sqreen/ecosystem/tracing/sampling_configuration.rb
+- lib/sqreen/ecosystem/tracing/signals/tracing_client.rb
+- lib/sqreen/ecosystem/tracing/signals/tracing_consumer.rb
+- lib/sqreen/ecosystem/tracing/signals/tracing_producer.rb
+- lib/sqreen/ecosystem/tracing/signals/tracing_server.rb
+- lib/sqreen/ecosystem/tracing_broker.rb
+- lib/sqreen/ecosystem/tracing_id_setup.rb
+- lib/sqreen/ecosystem/transaction_storage.rb
+- lib/sqreen/ecosystem/util/call_writers_from_init.rb
+- lib/sqreen/ecosystem_integration.rb
+- lib/sqreen/ecosystem_integration/around_callbacks.rb
+- lib/sqreen/ecosystem_integration/instrumentation_service.rb
+- lib/sqreen/ecosystem_integration/request_lifecycle_tracking.rb
+- lib/sqreen/ecosystem_integration/signal_consumption.rb
 - lib/sqreen/endpoint_testing.rb
 - lib/sqreen/error_handling_middleware.rb
 - lib/sqreen/event.rb
@@ -279,9 +327,7 @@ metadata:
   changelog_uri: https://docs.sqreen.com/ruby/release-notes/
   source_code_uri: https://github.com/sqreen/ruby-agent
   bug_tracker_uri: https://github.com/sqreen/ruby-agent/issues
-post_install_message: |2
-      This is a Sqreen beta release and may not work in all situations.
-      Make sure to review CHANGELOG.md for important details.
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -292,11 +338,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Sqreen Ruby agent

data/lib/sqreen/encoding_sanitizer.rb

@@ -1,27 +0,0 @@
-# typed: true
-
-# Copyright (c) 2015 Sqreen. All Rights Reserved.
-# Please refer to our terms for more information: https://www.sqreen.com/terms.html
-
-module Sqreen
-  class EncodingSanitizer
-    def self.sanitize(obj)
-      case obj
-      when String
-        sanitize_string(obj)
-      when Array
-        obj.map { |e| sanitize(e) }
-      when Hash
-        obj.each_with_object({}) { |(k, v), h| h[k] = sanitize(v) }
-      else
-        obj
-      end
-    end
-
-    def self.sanitize_string(s)
-      return s if s.encoding.name == 'UTF-8' && s.valid_encoding?
-
-      s.encode('UTF-16', :invalid => :replace, :undef => :replace).encode('UTF-8')
-    end
-  end
-end