sqreen 1.16.2 → 1.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 424c794c9315a7e1a0544fd6be427194ec7d7155e665ba86c1c3413306078941
-  data.tar.gz: 815fa9b893dd2577360a0408cc44194879c9fb1bbe5074cdce6c8c8629e82704
+  metadata.gz: edfd8cc358ff6408be042a8d11d473b85b00c2a1cf80c97cf5b154a6fa7da037
+  data.tar.gz: f558c9aef036f11b3e20fe0f8d41d7020b7d302233a4385d7cb760a983e91116
 SHA512:
-  metadata.gz: 9bc336364648476b1e6a0b6a3955215646c008a418b35f39abacc1cee9fccd5bab9374e8e90b444bc2a1482eebc25ddf463526154479acca14acc7605e438bdf
-  data.tar.gz: 53433d834097073f8ec98dd4396243029fb55e60c675bb6da379d12090000cbb31be0a23e736a9cc70615dd5fe7991559cc4a3c3c884d02cbbe6e5c787e5b415
+  metadata.gz: aa39b3c2a3af1e2b2135dd735b736dce973eab3810cbbe7dc7d017d25ca69d7f309f9029e9f0281d51cd523f996f86c75963d83b77a2294f6a359f847305f100
+  data.tar.gz: daa34349243dc5a56abadff975c776098db0e1d520f15720bae17443b157cc586ba77bc954ab4c7687866dab4bd4b8607cb6fee4c2e9e3bd68cc9f7b46b78faf

data/README.md

@@ -75,3 +75,8 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/sqreen/RubyAgent. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](contributor-covenant.org) code of conduct.
+
+## LICENSE
+
+Sqreen for Ruby is free-to-use, proprietary software.
+

data/lib/sqreen.rb

@@ -18,14 +18,21 @@ require 'thread'
 
 Sqreen.framework.on_start do |framework|
   if Sqreen.framework.on_pre_fork_preload?
+    Sqreen.log.debug "Sqreen detected a forking server with preloading"
     next
+  else
+    Sqreen.log.debug "Sqreen detected a single-process server"
   end
   Thread.new do
     begin
       runner = nil
+      Sqreen.log.debug("Reading configuration")
       configuration = Sqreen.config_init(framework)
-      Sqreen.log.debug("Starting Sqreen #{Sqreen::VERSION}")
       framework.sqreen_configuration = configuration
+      Sqreen.log.debug("Initializing logs")
+      Sqreen.log_init
+      Sqreen.log.debug("Starting Sqreen #{Sqreen::VERSION}")
+      warn "[#{Process.pid}] Sqreen logging at level #{Sqreen.log.instance_eval { @logger }.level} to #{Sqreen.log.instance_eval { @logger }.instance_eval { @logdev.filename }}"
       prevent_startup = Sqreen.framework.prevent_startup
       if !prevent_startup
         runner = Sqreen::Runner.new(configuration, framework)
@@ -38,33 +45,33 @@ Sqreen.framework.on_start do |framework|
     rescue Sqreen::TokenInvalidException
       Sqreen.log.error "Sorry but your Sqreen token appears to be invalid.\nYour application is NOT currently protected by Sqreen.\n\nHave you correctly filled your config/sqreen.yml?\n\n"
     rescue Exception => e
-      Sqreen.log.error e.inspect
-      Sqreen.log.debug e.backtrace.join("\n")
+      Sqreen.log.debug("General exception caught: #{e.inspect}")
+      Sqreen.log.debug e.backtrace
       if runner
-        # immediately post exception
+        Sqreen.log.debug("Immediately posting exception for runner #{runner.inspect}")
         runner.session.post_sqreen_exception(Sqreen::RemoteException.new(e))
-        Sqreen.log.debug("runner = #{runner.inspect}")
         begin
           runner.remove_instrumentation
-        rescue => remove_exception
-          Sqreen.log.debug(remove_exception.inspect)
-          # We did not manage to remove instrumentation, state is unclear:
-          # terminate thread
+        rescue => e
+          Sqreen.log.debug("Unexpected exception when removing instrumentation: #{e.inspect}")
+          Sqreen.log.debug e.backtrace
+          Sqreen.log.error("Terminating Sqreen thread")
           return nil
         end
         begin
           runner.logout(false)
-        rescue => logout_exception
-          Sqreen.log.debug(logout_exception.inspect)
+        rescue StandardError => e
+          Sqreen.log.debug("Unexpected exception when logging out: #{remove_exception.inspect}")
+          Sqreen.log.debug(e.backtrace)
           nil
         end
       end
       # Wait a few seconds before retrying
       delay = rand(120)
-      Sqreen.log.debug("Sleeping #{delay} seconds before retry")
+      Sqreen.log.debug("Sleeping #{delay} seconds before restarting Sqreen thread")
       sleep(delay)
       retry
     end
-    Sqreen.log.debug("shutting down Sqreen #{Sqreen::VERSION}")
+    Sqreen.log.debug("Shutting down Sqreen #{Sqreen::VERSION}")
   end
 end unless Sqreen::to_bool(ENV['SQREEN_DISABLE'])

data/lib/sqreen/configuration.rb

@@ -96,6 +96,8 @@ module Sqreen
       if path
         file_config = parse_configuration_file(path)
         @config.merge!(file_config)
+      else
+        Sqreen.log.warn("could not find sqreen configuration file")
       end
 
       env_config = from_environment
@@ -162,8 +164,17 @@ module Sqreen
     protected
 
     def config_file_from_env
-      path = ENV[Sqreen::CONFIG_FILE_BY_ENV]
-      return path if path && File.exist?(path)
+      return unless (path = ENV[Sqreen::CONFIG_FILE_BY_ENV])
+
+      path = File.expand_path(path)
+
+      unless File.exist?(path)
+        Sqreen.log.warn("could not find sqreen configuration file provided by #{Sqreen::CONFIG_FILE_BY_ENV}: '#{path}'")
+      end
+
+      Sqreen.log.info("using config file provided by #{Sqreen::CONFIG_FILE_BY_ENV}: '#{path}'")
+
+      path
     end
 
     def local_config_file

data/lib/sqreen/events/request_record.rb

@@ -57,10 +57,17 @@ module Sqreen
       else
         res[:request] = {}
       end
+      if payload['response']
+        res[:response] = payload['response'].dup
+      else
+        res[:response] = {}
+      end
 
       res[:request][:parameters] = payload['params'] if payload['params']
       res[:request][:headers] = payload['headers'] if payload['headers']
 
+      res = Sqreen::EncodingSanitizer.sanitize(res)
+
       if @redactor
         res[:request] = @redactor.redact(res[:request])
       end
@@ -108,6 +115,27 @@ module Sqreen
     end
   end
 
+  class EncodingSanitizer
+    def self.sanitize(obj)
+      case obj
+      when String
+        sanitize_string(obj)
+      when Array
+        obj.map { |e| sanitize(e) }
+      when Hash
+        obj.each_with_object({}) { |(k, v), h| h[k] = sanitize(v) }
+      else
+        obj
+      end
+    end
+
+    def self.sanitize_string(s)
+      return s if s.encoding.name == 'UTF-8' && s.valid_encoding?
+
+      s.encode('UTF-16', :invalid => :replace, :undef => :replace).encode('UTF-8')
+    end
+  end
+
   # For redacting sensitive data and avoid having it sent to our servers
   class SensitiveDataRedactor
     DEFAULT_SENSITIVE_KEYS = Set.new(%w[password secret passwd authorization api_key apikey access_token]).freeze

data/lib/sqreen/frameworks.rb

@@ -20,7 +20,9 @@ module Sqreen
                 require 'sqreen/frameworks/rails3'
                 Sqreen::Frameworks::Rails3Framework
               else
-                raise "Rails version #{Rails.version} not supported"
+                Sqreen.log.warn "Detected unsupported Rails version #{Rails.version}, but continuing"
+                require 'sqreen/frameworks/rails'
+                Sqreen::Frameworks::RailsFramework
               end
             when defined?(::Sinatra)
               require 'sqreen/frameworks/sinatra'
@@ -29,8 +31,7 @@ module Sqreen
               require 'sqreen/frameworks/sqreen_test'
               Sqreen::Frameworks::SqreenTestFramework
             else
-              # FIXME: use sqreen logger before configuration?
-              STDERR.puts "Error: cannot find any framework\n"
+              Sqreen.log.error "Error: cannot find any framework\n"
               require 'sqreen/frameworks/generic'
               Sqreen::Frameworks::GenericFramework
             end

data/lib/sqreen/frameworks/generic.rb

@@ -44,6 +44,10 @@ module Sqreen
         ENV['RACK_ENV'] == 'development'
       end
 
+      def test?
+        ENV['RACK_ENV'] == 'test'
+      end
+
       PREFERRED_IP_HEADERS = %w[HTTP_X_FORWARDED_FOR HTTP_X_REAL_IP
                                 HTTP_CLIENT_IP HTTP_X_FORWARDED
                                 HTTP_X_CLUSTER_CLIENT_IP HTTP_FORWARDED_FOR
@@ -161,6 +165,23 @@ module Sqreen
         }
       end
 
+      def response_infos
+        return {} if response.nil?
+
+        content_type = response.header['Content-Type']
+        content_length = begin
+                           Integer(response.header['Content-Length'])
+                         rescue ArgumentError, TypeError
+                           nil
+                         end
+
+        {
+          :status => response.status,
+          :content_type => content_type,
+          :content_length => content_length,
+        }
+      end
+
       # Request URL path
       def request_path
         req = request
@@ -257,17 +278,26 @@ module Sqreen
         SharedStorage.set(:request_overtime, nil)
       end
 
+      def store_response(rv, _env)
+        SharedStorage.set(:response, Rack::Response.new([], rv[0], rv[1]))
+      end
+
       # Get the currently stored request
       def request
         SharedStorage.get(:request)
       end
 
+      def response
+        SharedStorage.get(:response)
+      end
+
       # Cleanup request context
       def clean_request
         payload_creator = Sqreen::PayloadCreator.new(self)
         close_request_record(Sqreen.queue, Sqreen.observations_queue, payload_creator)
         self.remaining_perf_budget = nil
         SharedStorage.set(:request, nil)
+        SharedStorage.set(:response, nil)
         SharedStorage.set(:xss_params, nil)
         SharedStorage.set(:whitelisted, nil)
         SharedStorage.set(:request_overtime, nil)
@@ -397,7 +427,7 @@ module Sqreen
           each_key_value_for_hash(p) do |value|
             next unless value.is_a?(String)
             next if value.size < 5
-            value = value.force_encoding(Encoding::ISO_8859_1).encode(Encoding::UTF_8) unless value.valid_encoding?
+            value = value.dup.force_encoding(Encoding::ISO_8859_1).encode(Encoding::UTF_8) unless value.valid_encoding?
             next if regexp && !regexp.match?(value)
             parm << value
           end
@@ -415,7 +445,7 @@ module Sqreen
           each_key_value_for_hash(p) do |value|
             next unless value.is_a?(String)
             next if value.size < 5
-            value = value.force_encoding(Encoding::ISO_8859_1).encode(Encoding::UTF_8) unless value.valid_encoding?
+            value = value.dup.force_encoding(Encoding::ISO_8859_1).encode(Encoding::UTF_8) unless value.valid_encoding?
             next if regexp && !regexp.match(value)
             parm << value
           end
@@ -564,7 +594,13 @@ module Sqreen
       end
 
       def split_ip_addresses(ip_addresses)
-        ip_addresses ? ip_addresses.strip.split(/[,\s]+/) : []
+        return [] unless ip_addresses
+
+        r = ip_addresses.strip.split(/[,\s]+/)
+        r.map! do |ip|
+          m = /^(?:(?<ip>.*\..*)\:|\[(?<ip>.*)\]\:)/.match(ip)
+          m ? m[:ip] : ip
+        end
       end
 
       def valid_ip?(ip)

data/lib/sqreen/instrumentation.rb

@@ -415,36 +415,31 @@ module Sqreen
     end
 
     def override_class_method(klass, meth)
-      # FIXME: This is somehow ugly. We should reduce the amount of
-      # `evaled` code.
-      str = " class << #{klass}
+      original = meth.to_sym
+      saved_meth_name = get_saved_method_name(meth).to_sym
+      new_method      = "#{meth}_modified".to_sym
 
-      original = '#{meth}'.to_sym
-      saved_meth_name = '#{get_saved_method_name(meth)}'.to_sym
-      new_method      = '#{meth}_modified'.to_sym
+      klass.singleton_class.instance_eval do
+        alias_method saved_meth_name, original
 
-      alias_method saved_meth_name, original
+        p = Instrumentation.define_callback_method(original, saved_meth_name, klass)
+        define_method(new_method, p)
 
-      p = Instrumentation.define_callback_method(original, saved_meth_name,
-                                                 #{klass})
-      define_method(new_method, p)
+        private new_method
 
-      private new_method
+        method_kind = nil
+        if public_method_defined?(original)
+          method_kind = :public
+        elsif protected_method_defined?(original)
+          method_kind = :protected
+        elsif private_method_defined?(original)
+          method_kind = :private
+        end
 
-      method_kind = nil
-      case
-      when public_method_defined?(original)
-        method_kind = :public
-      when protected_method_defined?(original)
-        method_kind = :protected
-      when private_method_defined?(original)
-        method_kind = :private
+        alias_method original, new_method
+        send(method_kind, original)
+        private saved_meth_name
       end
-      alias_method original, new_method
-      send(method_kind, original)
-      private saved_meth_name
-      end "
-      eval str
     end
 
     def unoverride_instance_method(obj, meth)
@@ -516,19 +511,19 @@ module Sqreen
     # WARNING We do not actually remove `meth`
     def unoverride_class_method(klass, meth)
       saved_meth_name = get_saved_method_name(meth)
+      method_kind = nil
 
-      eval "method_kind = nil; class << #{klass}
-          case
-          when public_method_defined?(#{meth.to_sym.inspect})
-            method_kind = :public
-          when protected_method_defined?(original)
-            method_kind = :protected
-          when private_method_defined?(#{meth.to_sym.inspect})
-            method_kind = :private
-          end
-          alias_method #{meth.to_sym.inspect}, #{saved_meth_name.to_sym.inspect}
-          send(method_kind, #{meth.to_sym.inspect})
-      end "
+      klass.singleton_class.instance_eval do
+        if public_method_defined?(meth.to_sym)
+          method_kind = :public
+        elsif protected_method_defined?(original)
+          method_kind = :protected
+        elsif private_method_defined?(meth.to_sym)
+          method_kind = :private
+        end
+        alias_method meth.to_sym, saved_meth_name.to_sym
+        send(method_kind, meth.to_sym)
+      end
     end
 
     if RUBY_VERSION < '1.9'

data/lib/sqreen/log.rb

@@ -9,18 +9,20 @@ require 'sqreen/performance_notifications/log_performance'
 require 'sqreen/configuration'
 
 module Sqreen
-  def self::log
-    @logger ||= nil
-    return @logger unless @logger.nil?
-    return NoopLogger.instance unless Sqreen.configured?
+  def self.log_init
     @logger = Logger.new(
       Sqreen.config_get(:log_level).to_s.upcase,
       Sqreen.config_get(:log_location)
     )
+    DeferredLogger.instance.flush_to(@logger.instance_eval { @logger })
   rescue => e
     warn "Sqreen logger exception: #{e}"
   end
 
+  def self::log
+    @logger || DeferredLogger.instance
+  end
+
   # Ruby default formatter modified to display current thread_id
   class FormatterWithTid
     Format = "%s, [%s#%d.%s] %5s -- %s: %s\n".freeze
@@ -113,7 +115,7 @@ module Sqreen
     end
 
     def create_error_logger
-      @error_logger = ::Logger.new(STDERR)
+      @error_logger = Kernel.const_defined?('MiniTest') ? NullLogger.instance : ::Logger.new(STDERR)
       enforce_log_format(@error_logger)
     end
 
@@ -122,7 +124,7 @@ module Sqreen
     end
   end
 
-  class NoopLogger
+  class NullLogger
     include Singleton
 
     def debug(_msg = nil); end
@@ -132,5 +134,54 @@ module Sqreen
     def warn(_msg = nil); end
 
     def error(_msg = nil); end
+
+    def formatter=(_); end
+  end
+
+  class DeferredLogger
+    include Singleton
+
+    def initialize
+      @buffer = StringIO.new
+      @logger = ::Logger.new(@buffer)
+    end
+
+    def debug(msg = nil, &block)
+      @logger.debug(msg, &block)
+    end
+
+    def info(msg = nil, &block)
+      @logger.info(msg, &block)
+    end
+
+    def warn(msg = nil, &block)
+      @logger.warn(msg, &block)
+    end
+
+    def error(msg = nil, &block)
+      @logger.error(msg, &block)
+    end
+
+    def formatter=(value)
+      @logger.formatter = value
+    end
+
+    def flush_to(logger)
+      logger.instance_eval { @logdev }.write(read).tap { reset }
+    end
+
+    private
+
+    def read
+      @buffer.rewind
+      @buffer.read
+    end
+
+    def reset
+      buffer = StringIO.new
+      logger = ::Logger.new(buffer)
+      logger.formatter = @logger.formatter
+      @buffer, @logger = buffer, logger
+    end
   end
 end

data/lib/sqreen/payload_creator.rb

@@ -54,6 +54,7 @@ module Sqreen
 
     FULL_SECTIONS = {
       'request' => 'request_infos',
+      'response' => 'response_infos',
       'params' => 'filtered_request_params',
       'headers' => 'ip_headers',
       'local' => 'local_infos',
@@ -64,6 +65,9 @@ module Sqreen
         'addr' => 'client_ip',
         'rid' => 'request_id',
       },
+      'response' => {
+        'status' => 'status',
+      },
       'local' => {
         'name' => 'hostname',
       },

data/lib/sqreen/rules_callbacks/record_request_context.rb

@@ -30,7 +30,8 @@ module Sqreen
         advise_action(nil)
       end
 
-      def post(_rv, _inst, _args, _budget = nil, &_block)
+      def post(rv, _inst, args, _budget = nil, &_block)
+        framework.store_response(rv, args[0])
         framework.clean_request
         advise_action(nil)
       end

data/lib/sqreen/runner.rb

@@ -228,16 +228,18 @@ module Sqreen
 
 
     def setup_instrumentation(context_infos = {})
-      Sqreen.log.info 'setup instrumentation'
+      Sqreen.log.info 'Setting up instrumentation'
       rulespack_id, rules = load_rules(context_infos)
       @framework.instrument_when_ready!(instrumenter, rules)
+      Sqreen.log.info 'Instrumentation set up'
       rulespack_id.to_s
     end
 
     def remove_instrumentation(_context_infos = {})
-      Sqreen.log.debug 'removing instrumentation'
+      Sqreen.log.debug 'Removing instrumentation'
       instrumenter.remove_all_callbacks
       Sqreen::Actions::Repository.instance.clear
+      Sqreen.log.debug 'Instrumentation removed'
       true
     end
 
@@ -419,6 +421,7 @@ module Sqreen
 
     def logout(retrying = true)
       return unless session
+      Sqreen.log.debug("Logging out")
       if @framework.development?
         @running = false
         return
@@ -440,7 +443,13 @@ module Sqreen
         if exit_from_sinatra_startup? && try_again
           register_exit_cb(false)
         else
-          logout
+          begin
+            logout
+          rescue StandardError => e
+            Sqreen.log.debug(e.inspect)
+            Sqreen.log.debug(e.backtrace)
+            nil
+          end
         end
       end
     end

data/lib/sqreen/session.rb

@@ -89,16 +89,6 @@ module Sqreen
       @http.finish if connected?
     end
 
-    NET_ERRORS = [Timeout::Error,
-                  Errno::EINVAL,
-                  Errno::ECONNRESET,
-                  Errno::ECONNREFUSED,
-                  EOFError,
-                  Net::HTTPBadResponse,
-                  Net::HTTPHeaderSyntaxError,
-                  SocketError,
-                  Net::ProtocolError].freeze
-
     def connect
       return if connected?
       Sqreen.log.warn "connection to #{@server_url}..."
@@ -106,7 +96,9 @@ module Sqreen
       @conn_retry = 0
       begin
         @con = @http.start
-      rescue *NET_ERRORS
+      rescue StandardError => e
+        Sqreen.log.debug { "Caught exception during request: #{e.inspect}" }
+        Sqreen.log.debug { e.backtrace }
         Sqreen.log.debug "Cannot connect, retry in #{RETRY_CONNECT_SECONDS} seconds"
         sleep RETRY_CONNECT_SECONDS
         @conn_retry += 1
@@ -133,16 +125,18 @@ module Sqreen
     end
 
     def resiliently(retry_request_seconds, max_retry, current_retry = 0)
+      connect unless connected?
       return yield
     rescue => e
-      Sqreen.log.debug { e.inspect }
+      Sqreen.log.debug { "Caught exception during request: #{e.inspect}" }
+      Sqreen.log.debug { e.backtrace }
 
       current_retry += 1
 
       raise e if current_retry >= max_retry || e.is_a?(Sqreen::NotImplementedYet)
 
       sleep_delay = [MAX_DELAY, retry_request_seconds * current_retry].min
-      Sqreen.log.debug format('Sleeping %ds', sleep_delay)
+      Sqreen.log.debug format("Sleeping %ds before retry #{current_retry}/#{max_retry}", sleep_delay)
       sleep(sleep_delay)
 
       retry
@@ -159,7 +153,6 @@ module Sqreen
     end
 
     def do_http_request(method, path, data, headers = {}, max_retry = 2)
-      connect unless connected?
       now = Time.now.utc
       headers['X-Session-Key'] = @session_id if @session_id
       headers['X-Sqreen-Time'] = now.to_f.to_s
@@ -170,6 +163,7 @@ module Sqreen
                                         @req_nb,
                                         Time.now.utc.to_f)
       headers['Content-Type'] = 'application/json'
+      headers['Accept'] = 'application/json'
       if request_compression && !method.casecmp(:GET).zero?
         headers['Content-Encoding'] = 'gzip'
       end
@@ -198,11 +192,15 @@ module Sqreen
                    raise Sqreen::NotImplementedYet
                  end
         end
-
         if json && json.body
-          res = JSON.parse(json.body)
-          unless res['status']
-            Sqreen.log.debug(format('Cannot %s %s.', method, path))
+          if json['Content-Type'] && json['Content-Type'].start_with?('application/json')
+            res = JSON.parse(json.body)
+            unless res['status']
+              Sqreen.log.debug(format('Cannot %s %s. Parsed response body was: %s', method, path, res.inspect))
+            end
+          else
+            Sqreen.log.debug "Unexpected response Content-Type: #{json['Content-Type']}"
+            Sqreen.log.debug "Unexpected response body: #{json.body.inspect}"
           end
         else
           Sqreen.log.debug 'warning: empty return value'
@@ -283,7 +281,7 @@ module Sqreen
     # @param exception [RemoteException] Exception and context to be sent over
     def post_sqreen_exception(exception)
       post('sqreen_exception', exception.to_hash, {}, 5)
-    rescue *NET_ERRORS => e
+    rescue StandardError => e
       Sqreen.log.warn(format('Could not post exception (network down? %s) %s',
                              e.inspect,
                              exception.to_hash.inspect))

data/lib/sqreen/version.rb

@@ -1,5 +1,5 @@
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.io/terms.html
 module Sqreen
-  VERSION = '1.16.2'.freeze
+  VERSION = '1.17.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sqreen
 version: !ruby/object:Gem::Version
-  version: 1.16.2
+  version: 1.17.0
 platform: ruby
 authors:
 - Sqreen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-02-14 00:00:00.000000000 Z
+date: 2019-03-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sq_mini_racer