sqreen 1.6.5 → 1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cc4c4bcf86488934c2048d33804dd07398484e09
-  data.tar.gz: 4d4bb6a78d00ef04d6d824f01f1194a156cb019b
+  metadata.gz: dc11a3a48f65cd5aaa1b4e0444b122cf12ae2c21
+  data.tar.gz: 85c6e6f370a1bb862f554679063057774f96e7ab
 SHA512:
-  metadata.gz: 1ab651db3fb58f44551e4c04885da9bf0c7543a51c479565e8baf23cb7739d83b0b14196ac6958515462b34103542916875da9516cac8517b0f54a74cfa437ee
-  data.tar.gz: b6886860a4d6a9d118bc675c4ea6b1e70fe6093c9cbda35691f89a9058e845b1270f75b67c9b2bbacd4f4e3a40b013fc5ba733c9cc8d269e5b2806a510c4b699
+  metadata.gz: b0d671840b6c605359512b50e23efef360f29d58944031ce0589683c64254a9d7205e91e7ecb6c17797a05e42c09b0d649a36c50de78468280fd4720eefe50b6
+  data.tar.gz: 7664d0468f7fd8e0ee71e03b325f02f5b0c76f53379046d2013232592dda05db0f3c9970db1ac046944ada579a42e87175077955485d649fdb207fdbe0e91589

data/lib/sqreen/callbacks.rb

@@ -58,6 +58,8 @@ module Sqreen
     #  - { :status => :reraise }: reraise
     #  - { :status => :override }: eat exception
     #  - { :retry => :retry }: try the block again
+    #
+    #  CB can also declare that they are whitelisted and should not be run at the moment.
 
     attr_reader :klass, :method
 
@@ -72,6 +74,10 @@ module Sqreen
       raise(Sqreen::Exception, 'No callback provided') unless @has_pre || @has_post || @has_failing
     end
 
+    def whitelisted?
+      false
+    end
+
     def pre?
       @has_pre
     end

data/lib/sqreen/events/attack.rb

@@ -45,7 +45,6 @@ module Sqreen
       res = {}
       rule_p = payload['rule']
       request_p = payload['request']
-      whitelisted = request_p.delete('whitelisted') if request_p
       res[:rule_name]    = rule_p['name']         if rule_p && rule_p['name']
       res[:rulespack_id] = rule_p['rulespack_id'] if rule_p && rule_p['rulespack_id']
       res[:test]         = rule_p['test']         if rule_p && rule_p['test']
@@ -56,7 +55,6 @@ module Sqreen
       res[:params]       = payload['params']      if payload['params']
       res[:context]      = payload['context']     if payload['context']
       res[:headers]      = payload['headers']     if payload['headers']
-      res[:whitelist_match] = whitelisted         if whitelisted
       res
     end
   end

data/lib/sqreen/frameworks/generic.rb

@@ -291,8 +291,52 @@ module Sqreen
         Dir.getwd
       end
 
+      WHITELIST_KEY = 'sqreen.whitelisted_request'.freeze
+
+      # Return the current item that whitelist this request
+      # returns nil if request is not whitelisted
+      def whitelisted_match
+        return nil unless request
+        return request.env[WHITELIST_KEY] if request.env.key?(WHITELIST_KEY)
+        request.env[WHITELIST_KEY] = whitelisted_ip || whitelisted_path
+      end
+
+      # Returns the current path that whitelist the request
+      def whitelisted_path
+        path = request_path
+        return nil unless path
+        find_whitelisted_path(path)
+      end
+
+      # Returns the current path that whitelist the request
+      def whitelisted_ip
+        ip = client_ip
+        return nil unless ip
+        find_whitelisted_ip(ip)
+      rescue
+        nil
+      end
+
       protected
 
+      # Is this a whitelisted path?
+      # return the path witelisted prefix that match path
+      def find_whitelisted_path(rpath)
+        (Sqreen.whitelisted_paths || []).find do |path|
+          rpath.start_with?(path)
+        end
+      end
+
+      # Is this a whitelisted ip?
+      # return the ip witelisted range that match ip
+      def find_whitelisted_ip(rip)
+        ret = (Sqreen.whitelisted_ips || {}).find do |_, ip|
+          ip.include?(rip)
+        end
+        return nil unless ret
+        ret.first
+      end
+
       def hook_rack_request(klass)
         @calling_pid = Process.pid
         klass.class_eval do

data/lib/sqreen/instrumentation.rb

@@ -33,6 +33,7 @@ require 'set'
 
 module Sqreen
   class Instrumentation
+    WHITELISTED_METRIC='whitelisted'.freeze
     @@override_semaphore = Mutex.new
 
     ## Overriden methods and callbacks globals
@@ -54,6 +55,7 @@ module Sqreen
     def self.callback_wrapper_pre(klass, method, instance, *args, &block)
       Instrumentation.guard_call(method, []) do
         callbacks = @@registered_callbacks.get(klass, method, :pre)
+        callbacks.reject!(&:whitelisted?)
 
         returns = []
         callbacks.each do |cb|
@@ -90,6 +92,7 @@ module Sqreen
     def self.callback_wrapper_post(klass, method, return_val, instance, *args, &block)
       Instrumentation.guard_call(method, []) do
         callbacks = @@registered_callbacks.get(klass, method, :post)
+        callbacks.reject!(&:whitelisted?)
 
         returns = []
         callbacks.reverse_each do |cb|
@@ -126,6 +129,7 @@ module Sqreen
     def self.callback_wrapper_failing(exception, klass, method, instance, *args, &block)
       Instrumentation.guard_call(method, []) do
         callbacks = @@registered_callbacks.get(klass, method, :failing)
+        callbacks.reject!(&:whitelisted?)
 
         returns = []
         callbacks.each do |cb|
@@ -506,11 +510,13 @@ module Sqreen
 
     def initialize(metrics_engine = nil)
       self.metrics_engine = metrics_engine
-      unless metrics_engine.nil?
-        metrics_engine.create_metric('name' => CallCountable::COUNT_CALLS,
-                                     'period' => 60,
-                                     'kind' => 'Sum')
-      end
+      return if metrics_engine.nil?
+      metrics_engine.create_metric('name' => CallCountable::COUNT_CALLS,
+                                   'period' => 60,
+                                   'kind' => 'Sum')
+      metrics_engine.create_metric('name' => WHITELISTED_METRIC,
+                                   'period' => 60,
+                                   'kind' => 'Sum')
     end
   end
 end

data/lib/sqreen/remote_command.rb

@@ -1,6 +1,7 @@
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.io/terms.html
-require "sqreen/log"
+require 'sqreen/log'
+require 'sqreen/events/remote_exception'
 
 module Sqreen
   # Execute and sanitize remote commands
@@ -13,6 +14,7 @@ module Sqreen
       :features_change => :change_features,
       :force_logout => :shutdown,
       :paths_whitelist => :change_whitelisted_paths,
+      :ips_whitelist => :change_whitelisted_ips,
     }.freeze
 
     attr_reader :uuid
@@ -27,7 +29,12 @@ module Sqreen
       failing = validate_command(runner)
       return failing if failing
       Sqreen.log.debug format('processing command %s', @name)
-      output = runner.send(KNOWN_COMMANDS[@name], *@params, context_infos)
+      begin
+        output = runner.send(KNOWN_COMMANDS[@name], *@params, context_infos)
+      rescue => e
+        Sqreen::RemoteException.record(e)
+        return { :status => false, :reason => "error: #{e.inspect}" }
+      end
       format_output(output)
     end
 

data/lib/sqreen/rule_callback.rb

@@ -46,39 +46,24 @@ module Sqreen
         @rule[Attrs::RULESPACK_ID]
       end
 
+      def whitelisted?
+        framework && !framework.whitelisted_match.nil?
+      end
+
       # Recommend taking an action (optionnally adding more data/context)
       #
       # This will format the requested action and optionnally
       # override it if it should not be taken (should not block for example)
       def advise_action(action, additional_data = {})
         return if action.nil? && additional_data.empty?
-        if %w(override raise).include?(action.to_s) && framework
-          prefix = find_whitelisted_path(framework.request_path.to_s)
-          if prefix
-            Sqreen.log.debug { "Trying to block on whitelisted path #{prefix}" }
-            action = nil
-          end
-        end
         additional_data.merge(:status => action)
       end
 
-      # Is this a whitelisted path?
-      # return the path witelisted prefix that match path
-      def find_whitelisted_path(rpath)
-        (Sqreen.whitelisted_paths || []).find do |path|
-          rpath.start_with?(path)
-        end
-      end
-
       # Record an attack event into Sqreen system
       # @param infos [Hash] Additional information about request
       def record_event(infos)
         payload = @payload_generator.payload(framework, @rule)
         payload['infos'] = infos
-        if framework && payload['request']
-          prefix = find_whitelisted_path(framework.request_path.to_s)
-          payload['request']['whitelisted'] = prefix
-        end
         Attack.record(payload)
       end
 

data/lib/sqreen/rules_callbacks.rb

@@ -8,6 +8,7 @@ require 'sqreen/rules_callbacks/record_request_context'
 require 'sqreen/rules_callbacks/rails_parameters'
 
 require 'sqreen/rules_callbacks/headers_insert'
+require 'sqreen/rules_callbacks/blacklist_ips'
 
 require 'sqreen/rules_callbacks/inspect_rule'
 

data/lib/sqreen/rules_callbacks/blacklist_ips.rb

@@ -0,0 +1,44 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+require 'ipaddr'
+
+require 'sqreen/rule_callback'
+
+module Sqreen
+  module Rules
+    # Looks for a blacklisted ip and block
+    class BlacklistIPsCB < RuleCB
+      def initialize(klass, method, rule_hash)
+        super(klass, method, rule_hash)
+        @ips = Hash[@data['values'].map { |v| [v, IPAddr.new(v)] }]
+        raise ArgumentError.new("no ips given") if @ips.empty?
+      end
+
+      def pre(_inst, *_args, &_block)
+        return unless framework
+        ip = framework.client_ip
+        return unless ip
+        found = find_blacklisted_ip(ip)
+        return unless found
+        Sqreen.log.debug { "Found blacklisted IP #{ip} - found: #{found}" }
+        record_observation('blacklisted', found, 1)
+        advise_action(:raise)
+      end
+
+      protected
+
+      # Is this a blacklisted ip?
+      # return the ip blacklisted range that match ip
+      def find_blacklisted_ip(rip)
+        ret = (@ips || {}).find do |_, ip|
+          ip.include?(rip)
+        end
+        return nil unless ret
+        ret.first
+      rescue
+        nil
+      end
+    end
+  end
+end

data/lib/sqreen/rules_callbacks/count_http_codes.rb

@@ -3,6 +3,7 @@
 
 require 'sqreen/rule_attributes'
 require 'sqreen/rule_callback'
+require 'sqreen/safe_json'
 
 module Sqreen
   module Rules
@@ -31,7 +32,7 @@ module Sqreen
         key = @accessors.map do |accessor|
           accessor.resolve(binding, framework, inst, args, @data, rv)
         end
-        record_observation(@metric_category, JSON.dump(key), 1)
+        record_observation(@metric_category, SafeJSON.dump(key), 1)
         advise_action(nil)
       end
     end

data/lib/sqreen/rules_callbacks/matcher_rule.rb

@@ -73,6 +73,7 @@ module Sqreen
 
       def match(str)
         return if str.nil? || str.empty?
+        str = enforce_encoding(str)
         istr = str.downcase
 
         @string.each do |type, cases|
@@ -92,11 +93,35 @@ module Sqreen
           end
         end
 
-        @regexp_patterns.each do |p|
-          return p if p.match(str)
+        if defined?(Encoding)
+          @regexp_patterns.each do |p|
+            next unless Encoding.compatible?(p, str)
+            return p if p.match(str)
+          end
+        else
+          @regexp_patterns.each do |p|
+            warn(Encoding.compatible?(p, str).inspect)
+            return p if p.match(str)
+          end
         end
         nil
       end
+
+      private
+
+      def enforce_encoding(str)
+        return str unless str.is_a?(String)
+        return str if str.ascii_only?
+        encoded8bit = str.encoding.name == 'ASCII-8BIT'
+        return str if !encoded8bit && str.valid_encoding?
+        str.chars.map do |v|
+          if !v.valid_encoding? || (encoded8bit && !v.ascii_only?)
+            ''
+          else
+            v
+          end
+        end.join
+      end
     end
 
     # A configurable matcher rule

data/lib/sqreen/rules_callbacks/record_request_context.rb

@@ -2,13 +2,26 @@
 # Please refer to our terms for more information: https://www.sqreen.io/terms.html
 
 require 'sqreen/rule_callback'
+require 'sqreen/instrumentation'
 
 module Sqreen
   module Rules
     # Save request context for handling further down
     class RecordRequestContext < RuleCB
+      def whitelisted?
+        false
+      end
+
       def pre(_inst, *args, &_block)
         framework.store_request(args[0])
+        wh = framework.whitelisted_match
+        if wh
+          unless Sqreen.features.key?('whitelisted_metric') &&
+                 !Sqreen.features['whitelisted_metric']
+            record_observation(Instrumentation::WHITELISTED_METRIC, wh, 1)
+          end
+          Sqreen.log.debug { "Request was whitelisted because of #{wh}" }
+        end
         advise_action(nil)
       end
 

data/lib/sqreen/rules_callbacks/reflected_xss.rb

@@ -47,8 +47,7 @@ module Sqreen
         return unless report_dangerous_xss?(saved_value)
 
         # potential XSS! let's escape
-        if block &&
-           (!framework || !find_whitelisted_path(framework.request_path.to_s))
+        if block
           args[0].replace(CGI.escape_html(value))
         end
 
@@ -150,9 +149,6 @@ module Sqreen
 
         return if !found_xss || !block
         # potential XSS! let's escape
-        if framework && find_whitelisted_path(framework.request_path.to_s)
-          return nil
-        end
         args[-1] = new_attrs
         r = inst.send(method, *args)
         { :status => :skip, :new_return_value => r }
@@ -213,8 +209,7 @@ module Sqreen
 
         return unless block
         # potential XSS! let's escape
-        if block &&
-           (!framework || !find_whitelisted_path(framework.request_path.to_s))
+        if block
           args[0] = CGI.escape_html(value)
           r = inst.send(method, *args)
           return { :status => :skip, :new_return_value => r }

data/lib/sqreen/rules_signature.rb

@@ -1,7 +1,6 @@
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.io/terms.html
 
-require 'sqreen/rules_callbacks'
 require 'sqreen/exception'
 
 require 'set'

data/lib/sqreen/runner.rb

@@ -1,6 +1,7 @@
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.io/terms.html
 
+require 'ipaddr'
 require 'timeout'
 require 'json'
 
@@ -53,6 +54,11 @@ module Sqreen
     def update_whitelisted_paths(paths)
       @whitelisted_paths = paths.freeze
     end
+
+    attr_reader :whitelisted_ips
+    def update_whitelisted_ips(paths)
+      @whitelisted_ips = Hash[paths.map { |v| [v, IPAddr.new(v)] }].freeze
+    end
   end
 
   # Main running job class for the agent
@@ -87,6 +93,7 @@ module Sqreen
       @token = @configuration.get(:token)
       @url = @configuration.get(:url)
       Sqreen.update_whitelisted_paths([])
+      Sqreen.update_whitelisted_ips({})
       raise(Sqreen::Exception, 'no url found') unless @url
       raise(Sqreen::TokenNotFoundException, 'no token found') unless @token
 
@@ -244,6 +251,12 @@ module Sqreen
       true
     end
 
+    def change_whitelisted_ips(ips, _context_infos = {})
+      return false unless ips.respond_to?(:each)
+      Sqreen.update_whitelisted_ips(ips)
+      true
+    end
+
     def change_features(new_features, _context_infos = {})
       old = features
       self.features = new_features

data/lib/sqreen/safe_json.rb

@@ -0,0 +1,60 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+require 'json'
+
+require 'sqreen/log'
+
+module Sqreen
+  # Safely dump datastructure in json (more resilient to encoding errors)
+  class SafeJSON
+    def self.dump(data)
+      JSON.generate(data)
+    rescue JSON::GeneratorError, Encoding::UndefinedConversionError
+      Sqreen.log.debug('Payload could not be encoded enforcing recode')
+      JSON.generate(rencode_payload(data))
+    end
+
+    def self.rencode_payload(obj, max_depth = 20)
+      max_depth -= 1
+      return obj if max_depth < 0
+      return rencode_array(obj, max_depth) if obj.is_a?(Array)
+      return enforce_encoding(obj) unless obj.is_a?(Hash)
+      nobj = {}
+      obj.each do |k, v|
+        safe_k = rencode_payload(k, max_depth)
+        nobj[safe_k] = case v
+                       when Array
+                         rencode_array(v, max_depth)
+                       when Hash
+                         rencode_payload(v, max_depth)
+                       when String
+                         enforce_encoding(v)
+                       else # for example integers
+                         v
+                       end
+      end
+      nobj
+    end
+
+    def self.rencode_array(array, max_depth)
+      array.map! { |e| rencode_payload(e, max_depth - 1) }
+      array
+    end
+
+    def self.enforce_encoding(str)
+      return str unless str.is_a?(String)
+      return str if str.ascii_only?
+      encoded8bit = str.encoding.name == 'ASCII-8BIT'
+      return str if !encoded8bit && str.valid_encoding?
+      r = str.chars.map do |v|
+        if !v.valid_encoding? || (encoded8bit && !v.ascii_only?)
+          v.bytes.map { |c| "\\x#{c.to_s(16).upcase}" }.join
+        else
+          v
+        end
+      end.join
+      "SqBytes[#{r}]"
+    end
+  end
+end

data/lib/sqreen/session.rb

@@ -6,9 +6,9 @@ require 'sqreen/serializer'
 require 'sqreen/runtime_infos'
 require 'sqreen/events/remote_exception'
 require 'sqreen/exception'
+require 'sqreen/safe_json'
 
 require 'net/https'
-require 'json'
 require 'uri'
 require 'openssl'
 require 'zlib'
@@ -187,7 +187,7 @@ module Sqreen
                    json_data = nil
                    unless data.nil?
                      serialized = Serializer.serialize(data)
-                     json_data = compress(self.class.encode_payload(serialized))
+                     json_data = compress(SafeJSON.dump(serialized))
                    end
                    @con.post(path, json_data, headers)
                  else
@@ -209,13 +209,6 @@ module Sqreen
       res
     end
 
-    def self.encode_payload(data)
-      JSON.generate(data)
-    rescue JSON::GeneratorError, Encoding::UndefinedConversionError
-      Sqreen.log.debug('Payload could not be encoded enforcing recode')
-      JSON.generate(rencode_payload(data))
-    end
-
     def compress(data)
       return data unless request_compression
       out = StringIO.new
@@ -225,47 +218,6 @@ module Sqreen
       out.string
     end
 
-    def self.rencode_payload(obj, max_depth = 20)
-      max_depth -= 1
-      return obj if max_depth < 0
-      return rencode_array(obj, max_depth) if obj.is_a?(Array)
-      return enforce_encoding(obj) unless obj.is_a?(Hash)
-      nobj = {}
-      obj.each do |k, v|
-        safe_k = rencode_payload(k, max_depth)
-        nobj[safe_k] = case v
-                       when Array
-                         rencode_array(v, max_depth)
-                       when Hash
-                         rencode_payload(v, max_depth)
-                       when String
-                         enforce_encoding(v)
-                       else # for example integers
-                         v
-                       end
-      end
-      nobj
-    end
-
-    def self.rencode_array(array, max_depth)
-      array.map! { |e| rencode_payload(e, max_depth - 1) }
-      array
-    end
-
-    def self.enforce_encoding(str)
-      return str unless str.is_a?(String)
-      return str if str.ascii_only?
-      encoded8bit = str.encoding.name == 'ASCII-8BIT'
-      return str if !encoded8bit && str.valid_encoding?
-      str.chars.map do |v|
-        if !v.valid_encoding? || (encoded8bit && !v.ascii_only?)
-          v.bytes.map { |c| "\\x#{c.to_s(16).upcase}" }.join
-        else
-          v
-        end
-      end.join
-    end
-
     def login(framework)
       headers = { 'x-api-key' => @token }
 

data/lib/sqreen/version.rb

@@ -1,5 +1,5 @@
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.io/terms.html
 module Sqreen
-  VERSION = '1.6.5'.freeze
+  VERSION = '1.7.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sqreen
 version: !ruby/object:Gem::Version
-  version: 1.6.5
+  version: 1.7.0
 platform: ruby
 authors:
 - Sqreen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-09 00:00:00.000000000 Z
+date: 2017-06-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: execjs
@@ -93,6 +93,7 @@ files:
 - lib/sqreen/rules_callbacks.rb
 - lib/sqreen/rules_callbacks/binding_accessor_matcher.rb
 - lib/sqreen/rules_callbacks/binding_accessor_metrics.rb
+- lib/sqreen/rules_callbacks/blacklist_ips.rb
 - lib/sqreen/rules_callbacks/count_http_codes.rb
 - lib/sqreen/rules_callbacks/crawler_user_agent_matches.rb
 - lib/sqreen/rules_callbacks/crawler_user_agent_matches_metrics.rb
@@ -111,6 +112,7 @@ files:
 - lib/sqreen/rules_signature.rb
 - lib/sqreen/runner.rb
 - lib/sqreen/runtime_infos.rb
+- lib/sqreen/safe_json.rb
 - lib/sqreen/sdk.rb
 - lib/sqreen/serializer.rb
 - lib/sqreen/session.rb
@@ -135,7 +137,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.6.12
 signing_key: 
 specification_version: 4
 summary: Sqreen Ruby agent