sqreen 1.15.0-java → 1.15.5-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ae1a29b376a222af3ccfc344c0c1010e5043f5f249113e917038af356d8ac24e
-  data.tar.gz: 3c5bad999caff0579ee484efe534e7b2b2b36fa0f68fac9f8900422c816b4ed3
+  metadata.gz: bb1cc783655676697b23d94b34643b1990d7ef8011d4304d1f8602ee39830e0e
+  data.tar.gz: d569d191c90467462c4b9431750e31df1eac0f2a7fd5f8b9f85707937b460da6
 SHA512:
-  metadata.gz: 571a374c56db67eaed6775693c261467299a41e0f183e1d1847281d9998ded1611ca4cd6805372bab3bdfa42e1b0fb98d00dcc4433ee2aea89284a3d3cde9779
-  data.tar.gz: dabdb93c32dced05d2d7b56e89fa6e829b0e83cf4e07b98c3b12c15ba82ace39e64210c08e16a4f7f059c2d6cdeea42a54f4f29a0646db36cea098d611d661e6
+  metadata.gz: 0e415a37c0cccbb9776f1c46d39e7d6a2da002950cf1a97d680c777c636ebfd225aeaebc552d50b1bf92e934946ceaf5a04214efa74ccb2b51e9d4f8787851be
+  data.tar.gz: 624b68dbc3150ec7f9637c9926c2ee9f02134310ee69327d161ea45d1959846b97c0458fdadf69842e6042a2e59dbbbc6e27733e5e12eb85d8bca6982bc44e2f

data/lib/sqreen/actions.rb

@@ -2,6 +2,7 @@
 # Please refer to our terms for more information: https://www.sqreen.io/terms.html
 
 require 'ipaddr'
+require 'sqreen/trie'
 require 'sqreen/log'
 require 'sqreen/exception'
 require 'sqreen/sdk'
@@ -24,21 +25,17 @@ module Sqreen
     class Repository
       include Singleton
 
-      def initialize
-        @actions = {} # indexed by subclass
-        @actions.default_proc = proc { |h, k| h[k] = [] }
+      def add(params, action)
+        action.class.index(params || {}, action)
       end
 
-      def <<(action)
-        @actions[action.class] << action
-      end
-
-      def [](action_class)
-        @actions[action_class]
+      def get(action_class, key)
+        action_class = Base.get_type_class(action_class) unless action_class.class == Class
+        action_class.actions_matching key
       end
 
       def clear
-        @actions.clear
+        Base.known_subclasses.each(&:clear)
       end
     end
 
@@ -68,6 +65,8 @@ module Sqreen
     end
 
     # Base class for actions
+    # subclasses must also implement some methods in their singleton classes
+    # (actions_matching, index and clear)
     class Base
       attr_reader :id, :expiry, :send_response
 
@@ -124,10 +123,27 @@ module Sqreen
           @@subclasses[name]
         end
 
+        def known_subclasses
+          @@subclasses.values
+        end
+
         def known_types
           @@subclasses.keys
         end
 
+        # all actions matching, possibly already expired
+        def actions_matching(_key)
+          raise 'implement in singletons of subclasses'
+        end
+
+        def index(_params, _action)
+          raise 'implement in singletons of subclasses'
+        end
+
+        def clear
+          raise 'implement in singletons of subclasses'
+        end
+
         def inherited(subclass)
           class << subclass
             public :new
@@ -143,43 +159,75 @@ module Sqreen
       end
     end
 
-    module IpRanges
-      attr_reader :ranges
+    module IpRangesIndex
+      def add_prefix(prefix_str, data)
+        @trie_v4 ||= Sqreen::Trie.new
+        @trie_v6 ||= Sqreen::Trie.new(nil, nil, Socket::AF_INET6)
+        prefix = Sqreen::Prefix.from_str(prefix_str, data)
+        trie = prefix.family == Socket::AF_INET6 ? @trie_v6 : @trie_v4
+        trie.insert prefix
+      end
+
+      def matching_actions(client_ip)
+        parsed_ip = IPAddr.new(client_ip)
+        trie = parsed_ip.family == Socket::AF_INET6 ? @trie_v6 : @trie_v4
+        return [] unless trie
+        found = trie.search_matching(parsed_ip.to_i, parsed_ip.family)
+        return [] unless found.size > 0
+
+        Sqreen.log.debug("Client ip #{client_ip} matches #{found.inspect}")
+        found.map(&:data)
+      end
+
+      def clear
+        @trie_v4 = Sqreen::Trie.new
+        @trie_v6 = Sqreen::Trie.new(nil, nil, Socket::AF_INET6)
+      end
+    end
+
+    module IpRangeIndexedActionClass
+      include IpRangesIndex
+
+      def actions_matching(client_ip)
+         matching_actions client_ip
+      end
+
+      def index(params, action)
+        ranges = parse_ip_ranges params
+
+        ranges.each do |r|
+          add_prefix r, action
+        end
+      end
+
+      private
 
+      # returns array of prefixes in string form
       def parse_ip_ranges(params)
         ranges = params['ip_cidr']
         unless ranges && ranges.is_a?(Array) && !ranges.empty?
           raise 'no non-empty ip_cidr array present'
         end
 
-        @ranges = ranges.map &IPAddr.method(:new)
-      end
-
-      def matches_ip?(client_ip)
-        parsed_ip = IPAddr.new client_ip
-        found = ranges.find { |r| r.include? parsed_ip }
-        return false unless found
-
-        Sqreen.log.debug("Client ip #{client_ip} matches #{found.inspect}")
-        true
+        ranges
       end
     end
 
     # Block a list of IP address ranges. Standard "raise" behavior.
     class BlockIp < Base
-      include IpRanges
+      extend IpRangeIndexedActionClass
+
       self.type_name = 'block_ip'
 
       def initialize(id, opts, params = {})
+        # no need to store the ranges for this action, the index filter the class
         super(id, opts)
-        parse_ip_ranges params
       end
 
       def do_run(client_ip)
-        return nil unless matches_ip? client_ip
         e = Sqreen::AttackBlocked.new("Blocked client's IP #{client_ip} " \
-          "(action: #{id} covering range(s) #{ranges}). No action is required")
-        { :status => :raise, :exception => e }
+          "(action: #{id}). No action is required")
+        { :status => :raise, :exception => e, :skip_rem_cbs => true }
       end
 
       def event_properties(client_ip)
@@ -190,7 +238,8 @@ module Sqreen
     # Block a list of IP address ranges by forcefully redirecting the user
     # to a specific URL.
     class RedirectIp < Base
-      include IpRanges
+      extend IpRangeIndexedActionClass
+
       self.type_name = 'redirect_ip'
 
       attr_reader :redirect_url
@@ -199,16 +248,15 @@ module Sqreen
         super(id, opts)
         @redirect_url = params['url']
         raise "no url provided for action #{id}" unless @redirect_url
-        parse_ip_ranges params
       end
 
       def do_run(client_ip)
-        return nil unless matches_ip? client_ip
         Sqreen.log.info "Will request redirect for client with IP #{client_ip} " \
-          "(action: #{id} covering range(s) #{ranges})."
+          "(action: #{id})."
         {
           :status => :skip,
           :new_return_value => [303, { 'Location' => @redirect_url }, ['']],
+          :skip_rem_cbs => true,
         }
       end
 
@@ -222,15 +270,46 @@ module Sqreen
     class BlockUser < Base
       self.type_name = 'block_user'
 
+      class << self
+        def actions_matching(identity_params)
+          return [] unless @idx
+          key = stringify_keys(identity_params)
+          actions = @idx[key]
+          actions || []
+        end
+
+        def index(params, action)
+          @idx ||= {}
+          users = params['users']
+          raise ::Sqreen::Exception, 'nil "users" param for block_user action' if users.nil?
+          raise ::Sqreen::Exception, '"users" param must be an array' unless users.is_a? Array
+
+          users.each do |u|
+            @idx[u] ||= []
+            @idx[u] << action
+          end
+        end
+
+        def clear
+          @idx = {}
+        end
+
+        private
+
+        def stringify_keys(hash)
+          Hash[
+            hash.map { |k, v| [k.to_s, v] }
+          ]
+        end
+      end
+
+      # BlockUser proper definition continues
+
       def initialize(id, opts, params = {})
         super(id, opts)
-        @users = params['users']
-        raise ::Sqreen::Exception, 'nil "users" param for block_user action' if @users.nil?
-        raise ::Sqreen::Exception, '"users" param must be an array' unless @users.is_a? Array
       end
 
       def do_run(identity_params)
-        return unless @users.include? stringify_keys(identity_params)
         Sqreen.log.info(
           "Will raise due to user being blocked by action #{id}. " \
           "Blocked user identity: #{identity_params}"
@@ -250,14 +329,6 @@ module Sqreen
       def event_properties(identity_params)
         { 'user' => identity_params }
       end
-
-      private
-
-      def stringify_keys(hash)
-        Hash[
-          hash.map { |k, v| [k.to_s, v] }
-        ]
-      end
     end
   end
 end

data/lib/sqreen/callback_tree.rb

@@ -25,15 +25,15 @@ module Sqreen
 
       if cb.pre?
         methods[0] ||= []
-        methods[0] << cb
+        add_to_array(methods[0], cb)
       end
       if cb.post?
         methods[1] ||= []
-        methods[1] << cb
+        add_to_array(methods[1], cb)
       end
       if cb.failing?
         methods[2] ||= []
-        methods[2] << cb
+        add_to_array(methods[2], cb)
       end
     end
 
@@ -88,5 +88,18 @@ module Sqreen
         end
       end
     end
+
+    private
+
+    def add_to_array(arr, cb)
+      # find last element with prio equal or smaller; insert after
+      pos = arr.size
+      arr.reverse_each do |arr_cb|
+        break if arr_cb.priority <= cb.priority
+        pos -= 1
+      end
+
+      arr.insert(pos, cb)
+    end
   end
 end

data/lib/sqreen/callbacks.rb

@@ -36,6 +36,8 @@ module Sqreen
     #
     #  CB can also declare that they are whitelisted and should not be run at the moment.
 
+    DEFAULT_PRIORITY = 100
+
     attr_reader :klass, :method
     attr_reader :overtimeable
 
@@ -55,6 +57,11 @@ module Sqreen
       false
     end
 
+    # the lower, the closer to the beginning of the list
+    def priority
+      DEFAULT_PRIORITY
+    end
+
     def pre?
       @has_pre
     end
@@ -120,23 +127,6 @@ module Sqreen
       framework && !framework.whitelisted_match.nil?
     end
 
-    # Record an attack event into Sqreen system
-    # @param infos [Hash] Additional information about request
-    def record_event(infos, at = Time.now.utc)
-      return unless framework
-      payload = {
-        :infos => infos,
-        :rulespack_id => rulespack_id,
-        :rule_name => rule_name,
-        :test => test,
-        :time => at,
-      }
-      if payload_tpl.include?('context')
-        payload[:backtrace] = Sqreen::Context.new.bt
-      end
-      framework.observe(:attacks, payload, payload_tpl)
-    end
-
     # Record a metric observation
     # @param category [String] Name of the metric observed
     # @param key [String] aggregation key
@@ -146,22 +136,5 @@ module Sqreen
       return unless framework
       framework.observe(:observations, [category, key, observation, at], [], false)
     end
-
-    # Record an exception that just occurred
-    # @param exception [Exception] Exception to send over
-    # @param infos [Hash] Additional contextual information
-    def record_exception(exception, infos = {}, at = Time.now.utc)
-      return unless framework
-      payload = {
-        :exception => exception,
-        :infos => infos,
-        :rulespack_id => rulespack_id,
-        :rule_name => rule_name,
-        :test => test,
-        :time => at,
-        :backtrace => exception.backtrace || Sqreen::Context.bt,
-      }
-      framework.observe(:sqreen_exceptions, payload)
-    end
   end
 end

data/lib/sqreen/capped_queue.rb

@@ -15,7 +15,11 @@ module Sqreen
     alias original_push push
 
     def push(value)
-      pop until size < @capacity
+      until size < @capacity
+        discarded = pop
+        Sqreen.log.debug { "Discarded from queue: #{discarded}" }
+      end
+      Sqreen.log.debug { "Pushed to the queue: #{value}" }
       original_push(value)
     end
   end

data/lib/sqreen/configuration.rb

@@ -53,6 +53,10 @@ module Sqreen
       :default => nil },
     { :env => :SQREEN_STRIP_SENSITIVE_DATA, :name => :strip_sensitive_data,
       :default => true, :convert => :to_bool },
+    { :env => :SQREEN_STRIP_SENSITIVE_KEYS, :name => :strip_sensitive_keys,
+      :default => nil },
+    { :env => :SQREEN_STRIP_SENSITIVE_REGEX, :name => :strip_sensitive_regex,
+      :default => nil },
 
   ].freeze
 

data/lib/sqreen/deliveries/batch.rb

@@ -3,6 +3,8 @@
 
 require 'sqreen/deliveries/simple'
 require 'sqreen/events/remote_exception'
+require 'sqreen/mono_time'
+
 module Sqreen
   module Deliveries
     # Simple delivery method that batch event already seen in a batch
@@ -41,22 +43,23 @@ module Sqreen
       def stale?
         min = @first_seen.values.min
         return false if min.nil?
-        (min + max_staleness) < Time.now
+        (min + max_staleness) < Sqreen.time
       end
 
       def post_batch_needed?(event)
-        now = Time.now
+        now = Sqreen.time
+        # do not use any? {} due to side effects inside block
         event_keys(event).map do |key|
           was = @first_seen[key]
           @first_seen[key] ||= now
-          was.nil? || current_batch.size > max_batch || (was + max_staleness) < now
+          was.nil? || current_batch.size > max_batch || now > (was + max_staleness)
         end.any?
       end
 
       def post_batch
         session.post_batch(current_batch)
         current_batch.clear
-        now = Time.now
+        now = Sqreen.time
         @first_seen.each_key do |key|
           @first_seen[key] = now
         end

data/lib/sqreen/event.rb

@@ -12,5 +12,9 @@ module Sqreen
     def to_hash
       payload.to_hash
     end
+
+    def to_s
+      "<#{self.class.name}: #{to_hash}>"
+    end
   end
 end

data/lib/sqreen/events/request_record.rb

@@ -7,6 +7,11 @@ require 'sqreen/event'
 module Sqreen
   # When a request is deeemed worthy of being sent to the backend
   class RequestRecord < Sqreen::Event
+    def initialize(payload, redactor = nil)
+      @redactor = redactor
+      super(payload)
+    end
+
     def observed
       (payload && payload[:observed]) || {}
     end
@@ -56,8 +61,8 @@ module Sqreen
       res[:request][:parameters] = payload['params'] if payload['params']
       res[:request][:headers] = payload['headers'] if payload['headers']
 
-      if Sqreen.config_get(:strip_sensitive_data)
-        res[:request] = SensitiveDataRedactor.redact(res[:request])
+      if @redactor
+        res[:request] = @redactor.redact(res[:request])
       end
 
       res
@@ -105,14 +110,42 @@ module Sqreen
 
   # For redacting sensitive data and avoid having it sent to our servers
   class SensitiveDataRedactor
-    SENSITIVE_KEYS = Set.new(%w[password secret passwd authorization api_key apikey access_token]).freeze
+    DEFAULT_SENSITIVE_KEYS = Set.new(%w[password secret passwd authorization api_key apikey access_token]).freeze
+    DEFAULT_REGEX = /\A(?:\d[ -]*?){13,16}\z/
     MASK = '<Redacted by Sqreen>'.freeze
-    REGEX = /\A(?:\d[ -]*?){13,16}\z/
 
-    def self.redact(obj)
+    def self.from_config
+      keys = Sqreen.config_get(:strip_sensitive_keys)
+      if keys && keys.is_a?(String)
+        keys = keys.split(',')
+      else
+        keys = nil
+      end
+
+      regex = Sqreen.config_get(:strip_sensitive_regex)
+      if regex && regex.is_a?(String)
+        begin
+          regex = Regexp.compile(regex)
+        rescue RegexpError
+          Sqreen.log.warn("Invalid regular expression given in strip_sensitive_keys: #{regex}")
+          regex = nil
+        end
+      else
+        regex = nil
+      end
+
+      new(keys: keys, regex: regex)
+    end
+
+    def initialize(params = {})
+      @regex = params[:regex] || DEFAULT_REGEX
+      @keys = params[:keys] || DEFAULT_SENSITIVE_KEYS
+    end
+
+    def redact(obj)
       case obj
       when String
-        return MASK if obj =~ REGEX
+        return MASK if obj =~ @regex
 
       when Array
         return obj.map(&method(:redact))
@@ -121,7 +154,7 @@ module Sqreen
         return Hash[
           obj.map do |k, v|
             ck = k.is_a?(String) ? k.downcase : k
-            [k, SENSITIVE_KEYS.include?(ck) ? MASK : redact(v)]
+            [k, @keys.include?(ck) ? MASK : redact(v)]
           end
         ]
       end