sqreen 1.18.3.beta1 → 1.18.3.beta2

data/lib/sqreen/error_handling_middleware.rb

@@ -0,0 +1,30 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/attack_blocked'
+
+module Sqreen
+  class ErrorHandlingMiddleware
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      @app.call(env)
+    rescue StandardError => e
+      sqreen_attack = nil
+      if e.is_a?(Sqreen::AttackBlocked)
+        sqreen_attack = e
+      elsif e.respond_to?(:original_exception) &&
+            e.original_exception.is_a?(Sqreen::AttackBlocked)
+        sqreen_attack = e.original_exception
+      end
+
+      if sqreen_attack && sqreen_attack.redirect_url
+        return [303, { 'Location' => sqreen_attack.redirect_url }, ['']]
+      end
+
+      raise
+    end
+  end
+end

data/lib/sqreen/event.rb

@@ -1,6 +1,8 @@
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
+# TODO: see sqreen/events
+
 module Sqreen
   # Master interface for point in time events (e.g. Attack, RemoteException)
   class Event

data/lib/sqreen/events/attack.rb

@@ -1,6 +1,8 @@
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
+# TODO: sqreen/events
+
 require 'sqreen/event'
 
 module Sqreen

data/lib/sqreen/events/request_record.rb

@@ -1,9 +1,13 @@
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
+# TODO: sqreen/events
+
 require 'json'
+require 'sqreen/log'
 require 'sqreen/event'
 require 'sqreen/encoding_sanitizer'
+require 'sqreen/sensitive_data_redactor'
 
 module Sqreen
   # When a request is deeemed worthy of being sent to the backend
@@ -70,7 +74,13 @@ module Sqreen
       res = Sqreen::EncodingSanitizer.sanitize(res)
 
       if @redactor
-        res[:request] = @redactor.redact(res[:request])
+        res[:request], redacted = @redactor.redact(res[:request])
+        if redacted.any? && res[:observed] && res[:observed][:attacks]
+          res[:observed][:attacks] = @redactor.redact_attacks!(res[:observed][:attacks], redacted)
+        end
+        if redacted.any? && res[:observed] && res[:observed][:sqreen_exceptions]
+          res[:observed][:sqreen_exceptions] = @redactor.redact_exceptions!(res[:observed][:sqreen_exceptions], redacted)
+        end
       end
 
       res
@@ -115,59 +125,4 @@ module Sqreen
       nil
     end
   end
-
-  # For redacting sensitive data and avoid having it sent to our servers
-  class SensitiveDataRedactor
-    DEFAULT_SENSITIVE_KEYS = Set.new(%w[password secret passwd authorization api_key apikey access_token]).freeze
-    DEFAULT_REGEX = /\A(?:\d[ -]*?){13,16}\z/
-    MASK = '<Redacted by Sqreen>'.freeze
-
-    def self.from_config
-      keys = Sqreen.config_get(:strip_sensitive_keys)
-      if keys && keys.is_a?(String)
-        keys = keys.split(',')
-      else
-        keys = nil
-      end
-
-      regex = Sqreen.config_get(:strip_sensitive_regex)
-      if regex && regex.is_a?(String)
-        begin
-          regex = Regexp.compile(regex)
-        rescue RegexpError
-          Sqreen.log.warn("Invalid regular expression given in strip_sensitive_regex: #{regex}")
-          regex = nil
-        end
-      else
-        regex = nil
-      end
-
-      new(keys: keys, regex: regex)
-    end
-
-    def initialize(params = {})
-      @regex = params[:regex] || DEFAULT_REGEX
-      @keys = (params[:keys] || DEFAULT_SENSITIVE_KEYS).map(&:downcase)
-    end
-
-    def redact(obj)
-      case obj
-      when String
-        return MASK if obj =~ @regex
-
-      when Array
-        return obj.map(&method(:redact))
-
-      when Hash
-        return Hash[
-          obj.map do |k, v|
-            ck = k.is_a?(String) ? k.downcase : k
-            [k, @keys.include?(ck) ? MASK : redact(v)]
-          end
-        ]
-      end
-
-      obj
-    end
-  end
 end

data/lib/sqreen/exception.rb

@@ -4,6 +4,7 @@
 require 'sqreen/log'
 
 module Sqreen
+  # TODO: do we really want this to be StandardError?
   # Base exeception class for sqreen
   class Exception < ::StandardError
     def initialize(msg = nil, *args)
@@ -15,44 +16,12 @@ module Sqreen
       Sqreen.log.error(msg)
     end
   end
-
-  # When the token is not found
-  class TokenNotFoundException < Exception
-  end
-
-  # When the token is invalid
-  class TokenInvalidException < Exception
-  end
-
-  # This exception name is particularly important since it is often seen by
-  # Sqreen users when watching their logs. It should not raise any concern to
-  # them.
-  class AttackBlocked < Exception
-    attr_accessor :redirect_url
-
-    def log_message(msg)
-      Sqreen.log.warn(msg)
-    end
-  end
-
-  class NotImplementedYet < Exception
-  end
-
-  class InvalidSignatureException < Exception
-  end
-
-  class Unauthorized < Exception
-  end
-
-  class WAFError < Exception
-    attr_reader :rule_name, :error, :data, :args
-
-    def initialize(rule_name, error, data = nil, args = nil)
-      super(error.to_s)
-      @rule_name = rule_name
-      @error = error
-      @data = data
-      @args = args
-    end
-  end
 end
+
+require 'sqreen/token_not_found_exception'
+require 'sqreen/token_invalid_exception'
+require 'sqreen/attack_blocked'
+require 'sqreen/not_implemented_yet'
+require 'sqreen/invalid_signature_exception'
+require 'sqreen/unauthorized'
+require 'sqreen/waf_error'

data/lib/sqreen/formatter_with_tid.rb

@@ -0,0 +1,45 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/log'
+
+module Sqreen
+  # Ruby default formatter modified to display current thread_id
+  class FormatterWithTid
+    # TODO: constant name
+    Format = "%s, [%s#%d.%s] %5s -- %s: %s\n".freeze
+    DatetimeFormat = '%Y-%m-%dT%H:%M:%S.%6N '.freeze
+
+    attr_accessor :datetime_format
+
+    def initialize
+      @datetime_format = nil
+    end
+
+    def call(severity, time, progname, msg)
+      format(
+        Format,
+        severity[0..0], format_datetime(time), $$,
+        Thread.current.object_id.to_s(36),
+        severity, progname, msg2str(msg),
+      )
+    end
+
+    private
+
+    def format_datetime(time)
+      time.strftime(DatetimeFormat)
+    end
+
+    def msg2str(msg)
+      case msg
+      when ::String
+        msg
+      when ::Exception
+        "#{msg.message} (#{msg.class})\n" << (msg.backtrace || []).join("\n")
+      else
+        msg.inspect
+      end
+    end
+  end
+end

data/lib/sqreen/framework_cb.rb

@@ -0,0 +1,28 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/cb'
+require 'sqreen/shared_storage'
+
+module Sqreen
+  # Framework-aware callback
+  class FrameworkCB < CB
+    attr_accessor :framework
+
+    def whitelisted?
+      whitelisted = SharedStorage.get(:whitelisted)
+      return whitelisted unless whitelisted.nil?
+      framework && !framework.whitelisted_match.nil?
+    end
+
+    # Record a metric observation
+    # @param category [String] Name of the metric observed
+    # @param key [String] aggregation key
+    # @param observation [Object] data observed
+    # @param at [Time] time when observation was made
+    def record_observation(category, key, observation, at = Time.now.utc)
+      return unless framework
+      framework.observe(:observations, [category, key, observation, at], [], false)
+    end
+  end
+end

data/lib/sqreen/frameworks.rb

@@ -1,7 +1,14 @@
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
+# TODO: @@framework global of hell, misscoped (move to Sqreen::Framework?)
+# TODO: Sqreen::Frameworks => Sqreen::Framework
+
+require 'sqreen/log'
+
 module Sqreen
+  module Frameworks; end
+
   @@framework = nil
 
   def self::set_framework(fwk)

data/lib/sqreen/frameworks/generic.rb

@@ -1,13 +1,16 @@
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
+# TODO: Sqreen::NotImplementedYet => sqreen/exceptions
+
 require 'ipaddr'
 require 'set'
 
 require 'sqreen/events/remote_exception'
-require 'sqreen/callbacks'
+require 'sqreen/shared_storage'
 require 'sqreen/exception'
 require 'sqreen/log'
+
 require 'sqreen/frameworks/request_recorder'
 
 module Sqreen
@@ -49,6 +52,7 @@ module Sqreen
                                 HTTP_X_CLUSTER_CLIENT_IP HTTP_FORWARDED_FOR
                                 HTTP_FORWARDED HTTP_VIA].freeze
 
+      # TODO: remove global config_get
       def preferred_ip_headers
         @preferred_ip_headers ||=
           begin

data/lib/sqreen/frameworks/rails.rb

@@ -5,6 +5,8 @@
 
 require 'sqreen/frameworks/generic'
 require 'sqreen/middleware'
+require 'sqreen/error_handling_middleware'
+require 'sqreen/rails_middleware'
 
 module Sqreen
   module Frameworks

data/lib/sqreen/frameworks/request_recorder.rb

@@ -1,6 +1,9 @@
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
+# TODO: Sqreen::Framework::RequestRecorder misnamed/misplaced?
+# TODO: deps?
+
 require 'set'
 require 'sqreen/shared_storage'
 require 'sqreen/events/request_record'

data/lib/sqreen/frameworks/sinatra.rb

@@ -3,6 +3,8 @@
 
 require 'sqreen/frameworks/generic'
 require 'sqreen/middleware'
+require 'sqreen/error_handling_middleware'
+require 'sqreen/sinatra_middleware'
 
 module Sqreen
   module Frameworks

data/lib/sqreen/frameworks/sqreen_test.rb

@@ -1,6 +1,8 @@
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
+# TODO: drop
+
 require 'sqreen/frameworks/generic'
 
 module Sqreen

data/lib/sqreen/instrumentation.rb

@@ -1,17 +1,17 @@
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
-require 'sqreen/callback_tree'
+require 'sqreen/cb_tree'
 require 'sqreen/log'
 require 'sqreen/exception'
 require 'sqreen/performance_notifications'
 require 'sqreen/call_countable'
 require 'sqreen/events/remote_exception'
-require 'sqreen/rules_signature'
+require 'sqreen/sqreen_signed_verifier'
 require 'sqreen/shared_storage'
-require 'sqreen/rules_callbacks/record_request_context'
-require 'sqreen/rules_callbacks/run_req_start_actions'
-require 'sqreen/rules_callbacks/run_user_actions'
+require 'sqreen/rules/record_request_context'
+require 'sqreen/rules/run_req_start_actions'
+require 'sqreen/rules/run_user_actions'
 require 'sqreen/mono_time'
 require 'set'
 

data/lib/sqreen/invalid_signature_exception.rb

@@ -0,0 +1,8 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/exception'
+
+module Sqreen
+  class InvalidSignatureException < Sqreen::Exception; end
+end

data/lib/{sqreen-alt.rb → sqreen/js.rb}

@@ -1,4 +1,9 @@
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
-require "sqreen"
+# TODO: case JS/Js?
+
+module Sqreen
+  module Js
+  end
+end

data/lib/sqreen/js/call_context.rb

@@ -0,0 +1,10 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+# TODO: useless?
+
+module Sqreen
+  module Js
+    CallContext = Struct.new(:inst, :args, :rv)
+  end
+end

data/lib/sqreen/js/context_pool.rb

@@ -0,0 +1,60 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+# TODO: => Sqreen::JS:MiniRacer ?
+
+module Sqreen
+  module Js
+    class ContextPool
+      def initialize
+        @mutex = Mutex.new
+        @total_ctxs = 0
+        @contexts = []
+      end
+
+      def with_context(&block)
+        isolate = context
+        begin
+          block[isolate]
+        ensure
+          give_back_context isolate
+        end
+      end
+
+      private
+
+      def context
+        @mutex.synchronize do
+          if @contexts.empty?
+            @total_ctxs += 1
+            Sqreen.log.debug "Creating new V8 context (#{@total_ctxs})"
+            SqreenContext.new
+          else
+            @contexts.pop
+          end
+        end
+      end
+
+      def give_back_context(context)
+        context.possibly_gc
+
+        if context.gc_load > 30
+          if context.gc_threshold_in_bytes == DEFAULT_GC_THRESHOLD
+            context.gc_threshold_in_bytes *= 2
+            Sqreen.log.warn("Context #{context} had too many close garbage " \
+                            'collections; doubling the threshold to ' \
+                            "#{context.gc_threshold_in_bytes} bytes")
+            context.gc_load = 0
+          else
+            Sqreen.log.warn("Context #{context} had too many close garbage " \
+                            'collections; discarding it')
+            context.dispose
+            return
+          end
+        end
+
+        @mutex.synchronize { @contexts.push(context); }
+      end
+    end
+  end
+end