sqreen 1.17.0 → 1.17.2.beta1

data/lib/sqreen/dependency/new_relic.rb

@@ -0,0 +1,17 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+module Sqreen
+  module Dependency
+    module NewRelic
+      module_function
+
+      def ignore_sqreen_exceptions
+        return unless defined?(NewRelic::Agent::Agent)
+        NewRelic::Agent::Agent.instance.error_collector.ignore(['Sqreen::AttackBlocked'])
+      rescue ::Exception => e # rubocop:disable Lint/RescueException
+        Sqreen.log.warn "Failed ignoring AttackBlocked on NewRelic: #{e.inspect}"
+      end
+    end
+  end
+end

data/lib/sqreen/dependency/rack.rb

@@ -0,0 +1,36 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+module Sqreen
+  module Dependency
+    module Rack
+      module_function
+
+      def find_handler(&block)
+        Sqreen::Dependency::Hook.add('Rack::Server#server') do
+          after do |callback, _, server, _|
+            block.call(server)
+            callback.disable # do this once, :server is a lazy init accessor
+          end
+        end
+        Sqreen::Dependency::Hook['Rack::Server#server'].install
+      end
+
+      def on_run(handler, &block)
+        Sqreen.log.debug "[#{Process.pid}] #{handler.inspect}"
+        hookpoint_name = "#{handler.name}.run"
+
+        Sqreen::Dependency::Hook.add(hookpoint_name) do
+          before { block.call(handler) }
+        end
+        Sqreen::Dependency::Hook[hookpoint_name].install
+      end
+
+      def rackup?
+        return false if Sqreen::Dependency::Rails.server?
+
+        Sqreen::Dependency.const_exist?('Rack::Server') && ObjectSpace.each_object(::Rack::Server).count > 0
+      end
+    end
+  end
+end

data/lib/sqreen/dependency/rails.rb

@@ -0,0 +1,30 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+module Sqreen
+  module Dependency
+    module Rails
+      module_function
+
+      def required?
+        Sqreen::Dependency.const_exist?('Rails::Application')
+      end
+
+      def server?
+        Sqreen::Dependency.const_exist?('Rails::Server') && ObjectSpace.each_object(::Rails::Server).count > 0
+      end
+
+      def inspect_middlewares
+        Sqreen.log.debug { "Middlewares: " << ::Rails.application.middleware.map(&:inspect).inspect }
+      end
+
+      def insert_sqreen_middlewares
+        Sqreen.log.debug { 'Inserting Sqreen middlewares for Rails' }
+        app = ::Rails.application
+        app.middleware.insert_after(::Rack::Runtime, Sqreen::Middleware)
+        app.middleware.insert_after(::ActionDispatch::DebugExceptions, Sqreen::RailsMiddleware)
+        app.middleware.insert_after(::ActionDispatch::DebugExceptions, Sqreen::ErrorHandlingMiddleware)
+      end
+    end
+  end
+end

data/lib/sqreen/dependency/sentry.rb

@@ -0,0 +1,17 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+module Sqreen
+  module Dependency
+    module Sentry
+      module_function
+
+      def ignore_sqreen_exceptions
+        return unless defined?(Raven) && Raven.respond_to?(:configuration)
+        Raven.configuration.excluded_exceptions += ['Sqreen::AttackBlocked']
+      rescue ::Exception => e # rubocop:disable Lint/RescueException
+        Sqreen.log.warn "Failed setting Sentry's excluded_exceptions: #{e.inspect}"
+      end
+    end
+  end
+end

data/lib/sqreen/exception.rb

@@ -40,4 +40,7 @@ module Sqreen
 
   class InvalidSignatureException < Exception
   end
+
+  class Unauthorized < Exception
+  end
 end

data/lib/sqreen/frameworks/generic.rb

@@ -1,5 +1,6 @@
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
 require 'ipaddr'
 require 'set'
 
@@ -17,11 +18,6 @@ module Sqreen
       attr_accessor :sqreen_configuration
 
       def initialize
-        if defined?(Rack::Builder)
-          hook_rack_builder
-        else
-          to_app_done(Process.pid)
-        end
         clean_request_record
       end
 
@@ -205,12 +201,6 @@ module Sqreen
         nil
       end
 
-      # Main entry point for sqreen.
-      # launch whenever we are ready
-      def on_start
-        yield self
-      end
-
       # Should the agent not be starting up?
       def prevent_startup
         return :irb if $0 == 'irb'
@@ -221,23 +211,7 @@ module Sqreen
 
       # Instrument with our rules when the framework as finished loading
       def instrument_when_ready!(instrumentor, rules)
-        wait_for_to_app do
-          instrumentor.instrument!(rules, self)
-        end
-      end
-
-      def to_app_done(val)
-        return if @to_app_done
-        @to_app_done = val
-        return unless @wait
-        @wait.each(&:call)
-        @wait.clear
-      end
-
-      def wait_for_to_app(&block)
-        yield && return if @to_app_done
-        @wait ||= []
-        @wait << block
+        instrumentor.instrument!(rules, self)
       end
 
       # Does the parameters value include this value
@@ -456,15 +430,6 @@ module Sqreen
         end
       end
 
-      def on_pre_fork_preload?
-        # unlike with worker_fork_detection, we can't be instrument Puma::Cluster
-        # in time for intercepting run
-        stack = Kernel.caller_locations
-
-        puma_preload?(stack) || unicorn_preload?(stack)
-      end
-
-
       protected
 
       # Is this a whitelisted path?
@@ -485,22 +450,6 @@ module Sqreen
         ret.first
       end
 
-      def hook_rack_request(klass)
-        @calling_pid = Process.pid
-        klass.class_eval do
-          define_method(:call_with_sqreen) do |*args, &block|
-            rv = call_without_sqreen(*args, &block)
-            if Sqreen.framework.instance_variable_get('@calling_pid') != Process.pid
-              Sqreen.framework.instance_variable_set('@calling_pid', Process.pid)
-              yield Sqreen.framework
-            end
-            rv
-          end
-          alias_method :call_without_sqreen, :call
-          alias_method :call, :call_with_sqreen
-        end
-      end
-
       def hook_rack_builder
         Rack::Builder.class_eval do
           define_method(:to_app_with_sqreen) do |*args, &block|
@@ -547,52 +496,8 @@ module Sqreen
         false
       end
 
-      def sentry_ignore_exceptions
-        return unless defined?(Raven) && Raven.respond_to?(:configuration)
-        Raven.configuration.excluded_exceptions += ['Sqreen::AttackBlocked']
-      rescue
-        Sqreen.log.warn "Failed setting Sentry's excluded_exceptions: #{e.inspect}"
-      end
-
-      def newrelic_ignore_errors
-        return unless defined?(NewRelic::Agent::Agent)
-        NewRelic::Agent::Agent.instance.
-          error_collector.ignore(['Sqreen::AttackBlocked'])
-      rescue
-        Sqreen.log.warn "Failed ignoring AttackBlocked on NewRelic: #{e.inspect}"
-      end
-
-      # if it doesn't detect the fork it's not a problem
-      def worker_fork_detection
-        # only Puma currently supported
-        return unless defined?(Puma::Cluster) && Puma::Cluster.instance_methods.include?(:worker)
-        cur_worker_meth = Puma::Cluster.instance_method(:worker)
-        Puma::Cluster.class_eval do
-          define_method(:worker) do |*args|
-            Sqreen.on_forked_worker = true
-            cur_worker_meth.bind(self)[*args]
-          end
-        end
-      end
-
       private
 
-      def puma_preload?(stack)
-        cluster_run = stack.each_with_index.find { |b, _i| b.path =~ /puma\/cluster\.rb\z/ && b.label == 'run' }
-        return false unless cluster_run
-        frame_atop = stack[cluster_run[1] - 1]
-        frame_atop.label == 'load_and_bind'
-      end
-
-      def unicorn_preload?(stack)
-        build_app = stack.each_with_index.find do |b, _i|
-          b.path =~ /unicorn\/http_server\.rb\z/ && b.label == 'build_app!'
-        end
-        return false unless build_app
-        frame_below = stack[build_app[1] + 1]
-        frame_below.label == 'start'
-      end
-
       def split_ip_addresses(ip_addresses)
         return [] unless ip_addresses
 

data/lib/sqreen/frameworks/rails.rb

@@ -15,6 +15,10 @@ module Sqreen
         'Mysql2' => :mysql,
       }.freeze
 
+      def initialize
+        super
+      end
+
       def framework_infos
         {
           :framework_type => 'Rails',
@@ -89,19 +93,6 @@ module Sqreen
         end
       end
 
-      def on_start(&block)
-        @calling_pid = Process.pid
-        Init.startup do |app|
-          worker_fork_detection
-          sentry_ignore_exceptions
-          newrelic_ignore_errors
-          hook_rack_request(app.class, &block)
-          app.config.after_initialize do
-            yield self
-          end
-        end
-      end
-
       def prevent_startup
         res = super
         return res if res

data/lib/sqreen/frameworks/sinatra.rb

@@ -15,15 +15,6 @@ module Sqreen
         h
       end
 
-      def on_start(&block)
-        worker_fork_detection
-        sentry_ignore_exceptions
-        newrelic_ignore_errors
-        hook_app_build(Sinatra::Base)
-        hook_rack_request(Sinatra::Application, &block)
-        yield self
-      end
-
       def db_settings(options = {})
         adapter = options[:connection_adapter]
         return nil unless adapter
@@ -39,22 +30,6 @@ module Sqreen
         db_infos = { :name => adapter_name }
         [db_type, db_infos]
       end
-
-      def hook_app_build(klass)
-        klass.singleton_class.class_eval do
-          define_method(:setup_default_middleware_with_sqreen) do |builder|
-            ret = setup_default_middleware_without_sqreen(builder)
-            builder.instance_variable_get('@use').insert(2, proc do |app|
-              # Inject error middle just before sinatra one
-              Sqreen::ErrorHandlingMiddleware.new(app)
-            end)
-            ret
-          end
-
-          alias_method :setup_default_middleware_without_sqreen, :setup_default_middleware
-          alias_method :setup_default_middleware, :setup_default_middleware_with_sqreen
-        end
-      end
     end
   end
 end

data/lib/sqreen/instrumentation.rb

@@ -43,7 +43,6 @@ module Sqreen
     POST_CB = 'post'.freeze
     FAILING_CB = 'failing'.freeze
 
-    MGMT_COST = 0.000025
     @@override_semaphore = Mutex.new
     @@overriden_singleton_methods = false
 
@@ -115,7 +114,7 @@ module Sqreen
           Sqreen::PerformanceNotifications.notify(rule || cb.class.name, PRE_CB, start, stop)
         end
         all_stop = Sqreen.time
-        framework.remaining_perf_budget = budget - (all_stop - all_start) - MGMT_COST * callbacks.size if framework && budget
+        framework.remaining_perf_budget = budget - (all_stop - all_start) if framework && budget
         Sqreen::PerformanceNotifications.notify('hooks_pre', PRE_CB, all_start, all_stop)
         returns
       #end
@@ -168,7 +167,7 @@ module Sqreen
         end
         all_stop = Sqreen.time
         if framework && budget && framework.remaining_perf_budget
-          framework.remaining_perf_budget = budget - (all_stop - all_start) - MGMT_COST * callbacks.size 
+          framework.remaining_perf_budget = budget - (all_stop - all_start)
         end
         Sqreen::PerformanceNotifications.notify('hooks_post', POST_CB, all_start, all_stop)
         returns
@@ -222,7 +221,7 @@ module Sqreen
         end
         all_stop = Sqreen.time
         if framework && budget && framework.remaining_perf_budget
-          framework.remaining_perf_budget = budget - (all_stop - all_start) - MGMT_COST * callbacks.size 
+          framework.remaining_perf_budget = budget - (all_stop - all_start)
         end
         Sqreen::PerformanceNotifications.notify('hooks_failing', FAILING_CB, all_start, all_stop)
         returns
@@ -254,6 +253,7 @@ module Sqreen
       @sqreen_multi_instr ||= nil
 
       proc do |*args, &block|
+        Sqreen.log.debug { "Calling instrumented #{klass_name} #{original_meth} => #{meth}" }
         budget = nil
         skip_call = Thread.current[:sqreen_in_use]
         begin
@@ -626,6 +626,7 @@ module Sqreen
           end
         end
 
+        Sqreen.log.debug "Adding callback #{cb} for #{klass} #{method}"
         @@registered_callbacks.add(cb)
         @@unovertimable_hookpoints << key unless cb.overtimeable
         @@instrumented_pid = Process.pid

data/lib/sqreen/rules_callbacks/execjs.rb

@@ -50,18 +50,21 @@ module Sqreen
       end
 
       def pre(inst, args, budget = nil, &_block)
+        Sqreen.log.debug { "#{self.class} pre args: #{args.inspect}" }
         return unless pre?
 
         call_callback('pre', budget, inst, @cb_bas['pre'], args)
       end
 
       def post(rv, inst, args, budget = nil, &_block)
+        Sqreen.log.debug { "#{self.class} post args: #{args.inspect}" }
         return unless post?
 
         call_callback('post', budget, inst, @cb_bas['post'], args, rv)
       end
 
       def failing(rv, inst, args, budget = nil, &_block)
+        Sqreen.log.debug { "#{self.class} failing args: #{args.inspect}" }
         return unless failing?
 
         call_callback('failing', budget, inst, @cb_bas['failing'], args, rv)

data/lib/sqreen/rules_callbacks/record_request_context.rb

@@ -18,6 +18,7 @@ module Sqreen
       end
 
       def pre(_inst, args, _budget = nil, &_block)
+        Sqreen.log.debug { "RecordRequestContext pre args: #{args.inspect}" }
         framework.store_request(args[0])
         wh = framework.whitelisted_match
         if wh
@@ -31,12 +32,14 @@ module Sqreen
       end
 
       def post(rv, _inst, args, _budget = nil, &_block)
+        Sqreen.log.debug { "RecordRequestContext post args: #{args.inspect}" }
         framework.store_response(rv, args[0])
         framework.clean_request
         advise_action(nil)
       end
 
-      def failing(_exception, _inst, _args, _budget = nil, &_block)
+      def failing(_exception, _inst, args, _budget = nil, &_block)
+        Sqreen.log.debug { "RecordRequestContext failing args: #{args.inspect}" }
         framework.clean_request
         advise_action(nil)
       end

data/lib/sqreen/runner.rb

@@ -58,9 +58,6 @@ module Sqreen
     attr_accessor :logged_in
     alias logged_in? logged_in
 
-    attr_accessor :on_forked_worker
-    alias on_forked_worker? on_forked_worker
-
     attr_reader :whitelisted_paths
     def update_whitelisted_paths(paths)
       @whitelisted_paths = paths.freeze

data/lib/sqreen/session.rb

@@ -33,9 +33,10 @@ module Sqreen
     RETRY_CONNECT_SECONDS = 10
     RETRY_REQUEST_SECONDS = 10
 
-    MAX_DELAY = 60 * 30
+    MAX_DELAY = 300
 
-    RETRY_LONG = 128
+    RETRY_FOREVER = :forever
+    RETRY_MANY = 301
 
     MUTEX = Mutex.new
     METRICS_KEY = 'metrics'.freeze
@@ -108,14 +109,6 @@ module Sqreen
       end
     end
 
-    def resilient_post(path, data, headers = {})
-      post(path, data, headers, RETRY_LONG)
-    end
-
-    def resilient_get(path, headers = {})
-      get(path, headers, RETRY_LONG)
-    end
-
     def post(path, data, headers = {}, max_retry = 2)
       do_http_request(:POST, path, data, headers, max_retry)
     end
@@ -133,7 +126,7 @@ module Sqreen
 
       current_retry += 1
 
-      raise e if current_retry >= max_retry || e.is_a?(Sqreen::NotImplementedYet)
+      raise e if max_retry != RETRY_FOREVER && current_retry >= max_retry || e.is_a?(Sqreen::NotImplementedYet) || e.is_a?(Sqreen::Unauthorized)
 
       sleep_delay = [MAX_DELAY, retry_request_seconds * current_retry].min
       Sqreen.log.debug format("Sleeping %ds before retry #{current_retry}/#{max_retry}", sleep_delay)
@@ -173,41 +166,44 @@ module Sqreen
       path = prefix_path(path)
       Sqreen.log.debug format('%s %s (%s)', method, path, @token)
 
-      res = {}
+      payload = {}
       resiliently(RETRY_REQUEST_SECONDS, max_retry) do
-        json = nil
+        res = nil
         MUTEX.synchronize do
-          json = case method.upcase
-                 when :GET
-                   @con.get(path, headers)
-                 when :POST
-                   json_data = nil
-                   unless data.nil?
-                     serialized = Serializer.serialize(data)
-                     json_data = compress(SafeJSON.dump(serialized))
-                   end
-                   @con.post(path, json_data, headers)
-                 else
-                   Sqreen.log.debug format('unknown method %s', method)
-                   raise Sqreen::NotImplementedYet
-                 end
+          res = case method.upcase
+                when :GET
+                  @con.get(path, headers)
+                when :POST
+                  json_data = nil
+                  unless data.nil?
+                    serialized = Serializer.serialize(data)
+                    json_data = compress(SafeJSON.dump(serialized))
+                  end
+                  @con.post(path, json_data, headers)
+                else
+                  Sqreen.log.debug format('unknown method %s', method)
+                  raise Sqreen::NotImplementedYet
+                end
+        end
+        if res && res.code == '401'
+          raise Sqreen::Unauthorized, 'HTTP 401: shall relogin'
         end
-        if json && json.body
-          if json['Content-Type'] && json['Content-Type'].start_with?('application/json')
-            res = JSON.parse(json.body)
-            unless res['status']
-              Sqreen.log.debug(format('Cannot %s %s. Parsed response body was: %s', method, path, res.inspect))
+        if res && res.body
+          if res['Content-Type'] && res['Content-Type'].start_with?('application/json')
+            payload = JSON.parse(res.body)
+            unless payload['status']
+              Sqreen.log.debug(format('Cannot %s %s. Parsed response body was: %s', method, path, payload.inspect))
             end
           else
-            Sqreen.log.debug "Unexpected response Content-Type: #{json['Content-Type']}"
-            Sqreen.log.debug "Unexpected response body: #{json.body.inspect}"
+            Sqreen.log.debug "Unexpected response Content-Type: #{res['Content-Type']}"
+            Sqreen.log.debug "Unexpected response body: #{res.body.inspect}"
           end
         else
           Sqreen.log.debug 'warning: empty return value'
         end
       end
       Sqreen.log.debug format('%s %s (DONE in %f ms)', method, path, (Time.now.utc - now) * 1000)
-      res
+      payload
     end
 
     def compress(data)
@@ -227,7 +223,7 @@ module Sqreen
 
       Sqreen.log.warn "Using app name: #{headers['x-app-name']}"
 
-      res = resilient_post('app-login', RuntimeInfos.all(framework), headers)
+      res = post('app-login', RuntimeInfos.all(framework), headers, RETRY_FOREVER)
 
       if !res || !res['status']
         public_error = format('Cannot login. Token may be invalid: %s', @token)
@@ -243,7 +239,7 @@ module Sqreen
     end
 
     def rules
-      resilient_get('rulespack')
+      get('rulespack', {}, RETRY_MANY)
     end
 
     def heartbeat(cmd_res = {}, metrics = [])
@@ -251,30 +247,29 @@ module Sqreen
       payload['metrics'] = metrics unless metrics.nil? || metrics.empty?
       payload['command_results'] = cmd_res unless cmd_res.nil? || cmd_res.empty?
 
-      post('app-beat', payload.empty? ? nil : payload, {}, 5)
+      post('app-beat', payload.empty? ? nil : payload, {}, RETRY_MANY)
     end
 
     def post_metrics(metrics)
       return if metrics.nil? || metrics.empty?
       payload = { METRICS_KEY => metrics }
-      resilient_post(METRICS_KEY, payload)
+      post(METRICS_KEY, payload, {}, RETRY_MANY)
     end
 
     def post_attack(attack)
-      resilient_post('attack', attack.to_hash)
+      post('attack', attack.to_hash, {}, RETRY_MANY)
     end
 
     def post_bundle(bundle_sig, dependencies)
-      resilient_post('bundle', 'bundle_signature' => bundle_sig,
-                               'dependencies' => dependencies)
+      post('bundle', { 'bundle_signature' => bundle_sig, 'dependencies' => dependencies }, {}, RETRY_MANY)
     end
 
     def get_actionspack
-      resilient_get('actionspack')
+      get('actionspack', {}, RETRY_MANY)
     end
 
     def post_request_record(request_record)
-      resilient_post('request_record', request_record.to_hash)
+      post('request_record', request_record.to_hash, {}, RETRY_MANY)
     end
 
     # Post an exception to Sqreen for analysis
@@ -300,7 +295,7 @@ module Sqreen
         tally = Hash[events.group_by(&:class).map{ |k,v| [k, v.count] }]
         "Doing batch with the following tally of event types: #{tally}"
       end
-      resilient_post(BATCH_KEY, BATCH_KEY => batch)
+      post(BATCH_KEY, { BATCH_KEY => batch }, {}, RETRY_MANY)
     end
 
     # Perform agent logout