sqreen 1.18.2-java → 1.19.0-java

data/lib/sqreen/dependency/rails.rb

@@ -1,6 +1,13 @@
+# typed: ignore
+
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
+require 'sqreen/shrink_wrap'
+require 'sqreen/middleware'
+require 'sqreen/error_handling_middleware'
+require 'sqreen/rails_middleware'
+
 module Sqreen
   module Dependency
     module Rails
@@ -23,6 +30,7 @@ module Sqreen
       def insert_sqreen_middlewares
         Sqreen.log.debug { 'Inserting Sqreen middlewares for Rails' }
         app = ::Rails.application
+        app.middleware.insert(0, Sqreen::ShrinkWrap)
         app.middleware.insert_after(::Rack::Runtime, Sqreen::Middleware)
         app.middleware.insert_after(::ActionDispatch::DebugExceptions, Sqreen::RailsMiddleware)
         app.middleware.insert_after(::ActionDispatch::DebugExceptions, Sqreen::ErrorHandlingMiddleware)

data/lib/sqreen/dependency/sentry.rb

@@ -1,3 +1,5 @@
+# typed: ignore
+
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 

data/lib/sqreen/dependency/sinatra.rb

@@ -1,6 +1,13 @@
+# typed: ignore
+
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
+require 'sqreen/shrink_wrap'
+require 'sqreen/middleware'
+require 'sqreen/error_handling_middleware'
+require 'sqreen/sinatra_middleware'
+
 module Sqreen
   module Dependency
     module Sinatra
@@ -12,25 +19,62 @@ module Sqreen
 
       def insert_sqreen_middlewares(builder, *args, &block)
         Sqreen.log.debug { 'Inserting Sqreen middlewares for Sinatra' }
-        middleware = Sqreen::ErrorHandlingMiddleware
-        use = builder.instance_variable_get('@use')
 
-        p = proc { |app| middleware.new(app, *args, &block) }
+        insert_middleware(builder, Sqreen::ErrorHandlingMiddleware, args, block) do |p, u|
+          if middlewares(builder).include?(::Sinatra::ShowExceptions)
+            Sqreen.log.warn('Sinatra :show_exceptions detected: Sinatra exception handling may prevent the Sqreen error page to display on attacks.')
+          end
+
+          if (i = middlewares(builder).index(::Rack::Head))
+            u.insert(i, p)
+          elsif (i = middlewares(builder).index(::Rack::MethodOverride))
+            u.insert(i + 1, p)
+          elsif (i = middlewares(builder).index(::Sinatra::ExtendedRack))
+            u.insert(i + 1, p)
+          else
+            u.insert(0, p)
+          end
+        end
 
-        return if middlewares(builder).include?(middleware)
+        insert_middleware(builder, Sqreen::ShrinkWrap, args, block) do |p, u|
+          if (i = middlewares(builder).index(::Sinatra::ExtendedRack))
+            u.insert(i, p)
+          else
+            u.insert(0, p)
+          end
+        end
 
-        if middlewares(builder).include?(::Sinatra::ShowExceptions)
-          Sqreen.log.warn('Sinatra :show_exceptions detected: Sinatra exception handling may prevent the Sqreen error page to display on attacks.')
+        insert_middleware(builder, Sqreen::Middleware, args, block) do |p, u|
+          if (i = middlewares(builder).index(::Sinatra::ExtendedRack))
+            u.insert(i, p)
+          else
+            u.insert(1, p)
+          end
         end
 
-        if (i = middlewares(builder).index(::Rack::Head))
-          use.insert(i, p)
-        elsif (i = middlewares(builder).index(::Rack::MethodOverride))
-          use.insert(i + 1, p)
-        elsif (i = middlewares(builder).index(::Sinatra::ExtendedRack))
-          use.insert(i + 1, p)
-        else
-          use.insert(0, p)
+        insert_middleware(builder, Sqreen::SinatraMiddleware, args, block) do |p, u|
+          if ::Sqreen::Dependency.const_exist?('Rack::PostBodyContentTypeParser') && (i = middlewares(builder).index(::Rack::PostBodyContentTypeParser))
+            u.insert(i + 1, p)
+          elsif (i = middlewares(builder).index(::Rack::Protection))
+            u.insert(i + 1, p)
+          else
+            u.append(p)
+          end
+        end
+      end
+
+      def wrap_middleware(middleware, *args, &block)
+        proc { |app| middleware.new(app, *args, &block) }
+      end
+
+      def insert_middleware(builder, middleware, args, block)
+        use = builder.instance_variable_get('@use')
+        wrapped = wrap_middleware(middleware, *args, &block)
+
+        catch(:skip) do
+          throw(:skip) if middlewares(builder).include?(middleware)
+
+          yield(wrapped, use)
         end
       end
 

data/lib/sqreen/encoding_sanitizer.rb

@@ -1,3 +1,5 @@
+# typed: true
+
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 

data/lib/sqreen/error_handling_middleware.rb

@@ -0,0 +1,32 @@
+# typed: false
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/attack_blocked'
+
+module Sqreen
+  class ErrorHandlingMiddleware
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      @app.call(env)
+    rescue StandardError => e
+      sqreen_attack = nil
+      if e.is_a?(Sqreen::AttackBlocked)
+        sqreen_attack = e
+      elsif e.respond_to?(:original_exception) &&
+            e.original_exception.is_a?(Sqreen::AttackBlocked)
+        sqreen_attack = e.original_exception
+      end
+
+      if sqreen_attack && sqreen_attack.redirect_url
+        return [303, { 'Location' => sqreen_attack.redirect_url }, ['']]
+      end
+
+      raise
+    end
+  end
+end

data/lib/sqreen/event.rb

@@ -1,6 +1,10 @@
+# typed: true
+
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
+# TODO: see sqreen/events
+
 module Sqreen
   # Master interface for point in time events (e.g. Attack, RemoteException)
   class Event

data/lib/sqreen/events/attack.rb

@@ -1,6 +1,10 @@
+# typed: true
+
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
+# TODO: sqreen/events
+
 require 'sqreen/event'
 
 module Sqreen

data/lib/sqreen/events/remote_exception.rb

@@ -1,3 +1,5 @@
+# typed: true
+
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 

data/lib/sqreen/events/request_record.rb

@@ -1,9 +1,15 @@
+# typed: false
+
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
+# TODO: sqreen/events
+
 require 'json'
+require 'sqreen/log'
 require 'sqreen/event'
 require 'sqreen/encoding_sanitizer'
+require 'sqreen/sensitive_data_redactor'
 
 module Sqreen
   # When a request is deeemed worthy of being sent to the backend
@@ -70,7 +76,13 @@ module Sqreen
       res = Sqreen::EncodingSanitizer.sanitize(res)
 
       if @redactor
-        res[:request] = @redactor.redact(res[:request])
+        res[:request], redacted = @redactor.redact(res[:request])
+        if redacted.any? && res[:observed] && res[:observed][:attacks]
+          res[:observed][:attacks] = @redactor.redact_attacks!(res[:observed][:attacks], redacted)
+        end
+        if redacted.any? && res[:observed] && res[:observed][:sqreen_exceptions]
+          res[:observed][:sqreen_exceptions] = @redactor.redact_exceptions!(res[:observed][:sqreen_exceptions], redacted)
+        end
       end
 
       res
@@ -115,59 +127,4 @@ module Sqreen
       nil
     end
   end
-
-  # For redacting sensitive data and avoid having it sent to our servers
-  class SensitiveDataRedactor
-    DEFAULT_SENSITIVE_KEYS = Set.new(%w[password secret passwd authorization api_key apikey access_token]).freeze
-    DEFAULT_REGEX = /\A(?:\d[ -]*?){13,16}\z/
-    MASK = '<Redacted by Sqreen>'.freeze
-
-    def self.from_config
-      keys = Sqreen.config_get(:strip_sensitive_keys)
-      if keys && keys.is_a?(String)
-        keys = keys.split(',')
-      else
-        keys = nil
-      end
-
-      regex = Sqreen.config_get(:strip_sensitive_regex)
-      if regex && regex.is_a?(String)
-        begin
-          regex = Regexp.compile(regex)
-        rescue RegexpError
-          Sqreen.log.warn("Invalid regular expression given in strip_sensitive_regex: #{regex}")
-          regex = nil
-        end
-      else
-        regex = nil
-      end
-
-      new(keys: keys, regex: regex)
-    end
-
-    def initialize(params = {})
-      @regex = params[:regex] || DEFAULT_REGEX
-      @keys = (params[:keys] || DEFAULT_SENSITIVE_KEYS).map(&:downcase)
-    end
-
-    def redact(obj)
-      case obj
-      when String
-        return MASK if obj =~ @regex
-
-      when Array
-        return obj.map(&method(:redact))
-
-      when Hash
-        return Hash[
-          obj.map do |k, v|
-            ck = k.is_a?(String) ? k.downcase : k
-            [k, @keys.include?(ck) ? MASK : redact(v)]
-          end
-        ]
-      end
-
-      obj
-    end
-  end
 end

data/lib/sqreen/exception.rb

@@ -1,9 +1,12 @@
+# typed: false
+
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
 require 'sqreen/log'
 
 module Sqreen
+  # TODO: do we really want this to be StandardError?
   # Base exeception class for sqreen
   class Exception < ::StandardError
     def initialize(msg = nil, *args)
@@ -15,44 +18,12 @@ module Sqreen
       Sqreen.log.error(msg)
     end
   end
-
-  # When the token is not found
-  class TokenNotFoundException < Exception
-  end
-
-  # When the token is invalid
-  class TokenInvalidException < Exception
-  end
-
-  # This exception name is particularly important since it is often seen by
-  # Sqreen users when watching their logs. It should not raise any concern to
-  # them.
-  class AttackBlocked < Exception
-    attr_accessor :redirect_url
-
-    def log_message(msg)
-      Sqreen.log.warn(msg)
-    end
-  end
-
-  class NotImplementedYet < Exception
-  end
-
-  class InvalidSignatureException < Exception
-  end
-
-  class Unauthorized < Exception
-  end
-
-  class WAFError < Exception
-    attr_reader :rule_name, :error, :data, :args
-
-    def initialize(rule_name, error, data = nil, args = nil)
-      super(error.to_s)
-      @rule_name = rule_name
-      @error = error
-      @data = data
-      @args = args
-    end
-  end
 end
+
+require 'sqreen/token_not_found_exception'
+require 'sqreen/token_invalid_exception'
+require 'sqreen/attack_blocked'
+require 'sqreen/not_implemented_yet'
+require 'sqreen/invalid_signature_exception'
+require 'sqreen/unauthorized'
+require 'sqreen/waf_error'

data/lib/sqreen/formatter_with_tid.rb

@@ -0,0 +1,47 @@
+# typed: true
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/log'
+
+module Sqreen
+  # Ruby default formatter modified to display current thread_id
+  class FormatterWithTid
+    # TODO: constant name
+    Format = "%s, [%s#%d.%s] %5s -- %s: %s\n".freeze
+    DatetimeFormat = '%Y-%m-%dT%H:%M:%S.%6N '.freeze
+
+    attr_accessor :datetime_format
+
+    def initialize
+      @datetime_format = nil
+    end
+
+    def call(severity, time, progname, msg)
+      format(
+        Format,
+        severity[0..0], format_datetime(time), $$,
+        Thread.current.object_id.to_s(36),
+        severity, progname, msg2str(msg),
+      )
+    end
+
+    private
+
+    def format_datetime(time)
+      time.strftime(DatetimeFormat)
+    end
+
+    def msg2str(msg)
+      case msg
+      when ::String
+        msg
+      when ::Exception
+        "#{msg.message} (#{msg.class})\n" << (msg.backtrace || []).join("\n")
+      else
+        msg.inspect
+      end
+    end
+  end
+end

data/lib/sqreen/framework_cb.rb

@@ -0,0 +1,30 @@
+# typed: true
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/cb'
+require 'sqreen/shared_storage'
+
+module Sqreen
+  # Framework-aware callback
+  class FrameworkCB < CB
+    attr_accessor :framework
+
+    def whitelisted?
+      whitelisted = SharedStorage.get(:whitelisted)
+      return whitelisted unless whitelisted.nil?
+      framework && !framework.whitelisted_match.nil?
+    end
+
+    # Record a metric observation
+    # @param category [String] Name of the metric observed
+    # @param key [String] aggregation key
+    # @param observation [Object] data observed
+    # @param at [Time] time when observation was made
+    def record_observation(category, key, observation, at = Time.now.utc)
+      return unless framework
+      framework.observe(:observations, [category, key, observation, at], [], false)
+    end
+  end
+end

data/lib/sqreen/frameworks.rb

@@ -1,7 +1,16 @@
+# typed: ignore
+
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
+# TODO: @@framework global of hell, misscoped (move to Sqreen::Framework?)
+# TODO: Sqreen::Frameworks => Sqreen::Framework
+
+require 'sqreen/log'
+
 module Sqreen
+  module Frameworks; end
+
   @@framework = nil
 
   def self::set_framework(fwk)

data/lib/sqreen/frameworks/generic.rb

@@ -1,13 +1,18 @@
+# typed: ignore
+
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
+# TODO: Sqreen::NotImplementedYet => sqreen/exceptions
+
 require 'ipaddr'
 require 'set'
 
 require 'sqreen/events/remote_exception'
-require 'sqreen/callbacks'
+require 'sqreen/shared_storage'
 require 'sqreen/exception'
 require 'sqreen/log'
+
 require 'sqreen/frameworks/request_recorder'
 
 module Sqreen
@@ -49,6 +54,7 @@ module Sqreen
                                 HTTP_X_CLUSTER_CLIENT_IP HTTP_FORWARDED_FOR
                                 HTTP_FORWARDED HTTP_VIA].freeze
 
+      # TODO: remove global config_get
       def preferred_ip_headers
         @preferred_ip_headers ||=
           begin
@@ -295,13 +301,14 @@ module Sqreen
         params
       end
 
-      %w(form query cookies).each do |section|
+      %w(form query cookies rack).each do |section|
         define_method("#{section}_params") do
           self.class.send("#{section}_params", request)
         end
       end
 
       P_FORM = 'form'.freeze
+      P_RACK = 'rack'.freeze
       P_QUERY = 'query'.freeze
       P_COOKIE = 'cookies'.freeze
       P_GRAPE = 'grape_params'.freeze
@@ -317,6 +324,16 @@ module Sqreen
         end
       end
 
+      def self.rack_params(request)
+        return nil unless request
+        begin
+          request.params
+        rescue => e
+          Sqreen.log.debug("Rack Parameters are invalid #{e.inspect}")
+          nil
+        end
+      end
+
       def self.cookies_params(request)
         return nil unless request
         begin
@@ -345,6 +362,9 @@ module Sqreen
           P_QUERY  => query_params(request),
           P_COOKIE => cookies_params(request),
         }
+        if (p = rack_params(request))
+          r[P_RACK] = p
+        end
         # Add grape parameters if seen
         p = request.env['grape.request.params']
         r[P_GRAPE] = p if p