sqreen-alt 1.10.0

data/lib/sqreen/rules_callbacks.rb

@@ -0,0 +1,29 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+require 'sqreen/rules_callbacks/regexp_rule'
+require 'sqreen/rules_callbacks/matcher_rule'
+
+require 'sqreen/rules_callbacks/record_request_context'
+require 'sqreen/rules_callbacks/rails_parameters'
+
+require 'sqreen/rules_callbacks/headers_insert'
+require 'sqreen/rules_callbacks/blacklist_ips'
+
+require 'sqreen/rules_callbacks/inspect_rule'
+
+require 'sqreen/rules_callbacks/shell_env'
+
+require 'sqreen/rules_callbacks/url_matches'
+require 'sqreen/rules_callbacks/user_agent_matches'
+require 'sqreen/rules_callbacks/crawler_user_agent_matches'
+
+require 'sqreen/rules_callbacks/reflected_xss'
+require 'sqreen/rules_callbacks/execjs'
+
+require 'sqreen/rules_callbacks/binding_accessor_metrics'
+require 'sqreen/rules_callbacks/binding_accessor_matcher'
+require 'sqreen/rules_callbacks/count_http_codes'
+require 'sqreen/rules_callbacks/crawler_user_agent_matches_metrics'
+
+require 'sqreen/rules_callbacks/custom_error'

data/lib/sqreen/rules_callbacks/binding_accessor_matcher.rb

@@ -0,0 +1,77 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+require 'sqreen/rule_callback'
+require 'sqreen/binding_accessor'
+require 'sqreen/rules_callbacks/matcher_rule'
+
+module Sqreen
+  module Rules
+    # Callback that match on a list or matcher+binding accessor
+    class BindingAccessorMatcherCB < RuleCB
+      attr_reader :rules
+
+      # matcher on one elem
+      class MatcherElem
+        def initialize(expr)
+          prepare([expr])
+        end
+        include Matcher
+      end
+
+      def initialize(klass, method, rule_hash)
+        super(klass, method, rule_hash)
+        @rules = []
+        if @data.empty? || @data['values'].nil? || @data['values'].empty?
+          msg = "no rules in data (had #{@data.keys})"
+          raise Sqreen::Exception, msg
+        end
+        prepare_rules(@data['values'])
+      end
+
+      def prepare_rules(rules)
+        accessors = Hash.new do |hash, key|
+          hash[key] = BindingAccessor.new(key, true)
+        end
+        @rules = rules.map do |r|
+          if r['binding_accessor'].empty?
+            raise Sqreen::Exception, "no accessors #{r['id']}"
+          end
+          [
+            r['id'],
+            r['binding_accessor'].map { |expression| accessors[expression] },
+            MatcherElem.new(r['matcher']),
+            r['matcher']['value'],
+          ]
+        end
+      end
+
+      def pre(inst, *args, &_block)
+        resol_cache = Hash.new do |hash, accessor|
+          hash[accessor] = accessor.resolve(binding, framework, inst, args)
+        end
+        @rules.each do |id, accessors, matcher, matcher_ref|
+          accessors.each do |accessor|
+            val = resol_cache[accessor]
+            val = [val] if val.is_a?(String)
+            next unless val.respond_to?(:each)
+            next if val.respond_to?(:seek)
+            val.each do |v|
+              next if !v.is_a?(String) || (!matcher.min_size.nil? && v.size < matcher.min_size)
+              next if matcher.match(v).nil?
+              infos = {
+                'id' => id,
+                'binding_accessor' => accessor.expression,
+                'matcher' => matcher_ref,
+                'found' => v,
+              }
+              record_event(infos)
+              return advise_action(:raise, :infos => infos)
+            end
+          end
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/sqreen/rules_callbacks/binding_accessor_metrics.rb

@@ -0,0 +1,79 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+require 'sqreen/rule_callback'
+require 'sqreen/binding_accessor'
+require 'sqreen/events/remote_exception'
+
+module Sqreen
+  module Rules
+    # Publish metrics about data taken from the binding accessor
+    class BindingAccessorMetrics < RuleCB
+      # Exception thrown when no expression are present
+      class NoExpressions < Sqreen::Exception
+        def initialize(expr)
+          super("No valid expressions defined in #{expr.inspect}")
+        end
+      end
+
+      def initialize(klass, method, rule_hash)
+        super(klass, method, rule_hash)
+        @expr = {}
+        build_expressions(rule_hash[Attrs::CALLBACKS])
+      end
+
+      PRE_CB = 'pre'.freeze
+      POST_CB = 'post'.freeze
+      FAILING_CB = 'failing'.freeze
+
+      def pre?
+        @expr[PRE_CB]
+      end
+
+      def post?
+        @expr[POST_CB]
+      end
+
+      def failing?
+        @expr[FAILING_CB]
+      end
+
+      def pre(inst, *args, &_block)
+        return unless pre?
+
+        add_metrics(PRE_CB, inst, args)
+      end
+
+      def post(rv, inst, *args, &_block)
+        return unless post?
+
+        add_metrics(POST_CB, inst, args, rv)
+      end
+
+      def failing(exception, inst, *args, &_block)
+        return unless failing?
+
+        add_metrics(FAILING_CB, inst, args, exception)
+      end
+
+      protected
+
+      def add_metrics(name, inst, args, rv = nil)
+        category, key, value, = @expr[name].map do |accessor|
+          accessor.resolve(binding, framework, inst, args, @data, rv)
+        end
+        record_observation(category, key, value)
+        advise_action(nil)
+      end
+
+      def build_expressions(callbacks)
+        raise NoExpressions, callbacks if callbacks.nil? || callbacks.empty?
+        [PRE_CB, POST_CB, FAILING_CB].each do |c|
+          next if callbacks[c].nil? || callbacks[c].size < 3
+          @expr[c] = callbacks[c].map { |req| BindingAccessor.new(req, true) }
+        end
+        raise NoExpressions, callbacks if @expr.empty?
+      end
+    end
+  end
+end

data/lib/sqreen/rules_callbacks/blacklist_ips.rb

@@ -0,0 +1,44 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+require 'ipaddr'
+
+require 'sqreen/rule_callback'
+
+module Sqreen
+  module Rules
+    # Looks for a blacklisted ip and block
+    class BlacklistIPsCB < RuleCB
+      def initialize(klass, method, rule_hash)
+        super(klass, method, rule_hash)
+        @ips = Hash[@data['values'].map { |v| [v, IPAddr.new(v)] }]
+        raise ArgumentError.new("no ips given") if @ips.empty?
+      end
+
+      def pre(_inst, *_args, &_block)
+        return unless framework
+        ip = framework.client_ip
+        return unless ip
+        found = find_blacklisted_ip(ip)
+        return unless found
+        Sqreen.log.debug { "Found blacklisted IP #{ip} - found: #{found}" }
+        record_observation('blacklisted', found, 1)
+        advise_action(:raise)
+      end
+
+      protected
+
+      # Is this a blacklisted ip?
+      # return the ip blacklisted range that match ip
+      def find_blacklisted_ip(rip)
+        ret = (@ips || {}).find do |_, ip|
+          ip.include?(rip)
+        end
+        return nil unless ret
+        ret.first
+      rescue
+        nil
+      end
+    end
+  end
+end

data/lib/sqreen/rules_callbacks/count_http_codes.rb

@@ -0,0 +1,40 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+require 'sqreen/rule_attributes'
+require 'sqreen/rule_callback'
+require 'sqreen/safe_json'
+
+module Sqreen
+  module Rules
+    # Save request context for handling further down
+    class CountHTTPCodes < RuleCB
+      METRIC_CATEGORY = 'http_code'.freeze
+      def post(rv, _inst, *_args, &_block)
+        return unless rv.is_a?(Array) && !rv.empty?
+        record_observation(METRIC_CATEGORY, rv[0], 1)
+        advise_action(nil)
+      end
+    end
+
+    # Count 1 for each things located by the binding accessor
+    class BindingAccessorCounter < RuleCB
+      def initialize(klass, method, rule_hash)
+        super(klass, method, rule_hash)
+        @accessors = @data['values'].map do |expr|
+          BindingAccessor.new(expr, true)
+        end
+        @metric_category = rule_hash[Attrs::METRICS].first['name']
+      end
+
+      def post(rv, inst, *args, &_block)
+        return unless rv.is_a?(Array) && !rv.empty?
+        key = @accessors.map do |accessor|
+          accessor.resolve(binding, framework, inst, args, @data, rv)
+        end
+        record_observation(@metric_category, SafeJSON.dump(key), 1)
+        advise_action(nil)
+      end
+    end
+  end
+end

data/lib/sqreen/rules_callbacks/crawler_user_agent_matches.rb

@@ -0,0 +1,24 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+require 'sqreen/rules_callbacks/matcher_rule'
+require 'sqreen/frameworks'
+
+module Sqreen
+  module Rules
+    # FIXME: Factor with UserAgentMatchesCB
+    # Look for crawlers
+    class CrawlerUserAgentMatchesCB < MatcherRuleCB
+      def pre(_inst, *_args, &_block)
+        ua = framework.client_user_agent
+        return unless ua
+        found = match(ua)
+        return unless found
+        Sqreen.log.debug { "Found UA #{ua} - found: #{found}" }
+        infos = { :found => found }
+        record_event(infos)
+        advise_action(nil)
+      end
+    end
+  end
+end

data/lib/sqreen/rules_callbacks/crawler_user_agent_matches_metrics.rb

@@ -0,0 +1,24 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+require 'sqreen/rules_callbacks/matcher_rule'
+require 'sqreen/frameworks'
+
+module Sqreen
+  module Rules
+    # Look for crawlers and post them in metrics
+    class CrawlerUserAgentMatchesMetricsCB < MatcherRuleCB
+      CRAWLER_CATEGORY = 'crawler'.freeze
+
+      def pre(_inst, *_args, &_block)
+        ua = framework.client_user_agent
+        return unless ua
+        found = match(ua)
+        return unless found
+        Sqreen.log.debug { "Found UA #{ua} - found: #{found}" }
+        record_observation(CRAWLER_CATEGORY, ua, 1)
+        advise_action(nil)
+      end
+    end
+  end
+end

data/lib/sqreen/rules_callbacks/custom_error.rb

@@ -0,0 +1,64 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+require 'sqreen/rule_callback'
+require 'sqreen/exception'
+
+module Sqreen
+  module Rules
+    # Display sqreen presence
+    class CustomErrorCB < RuleCB
+      attr_reader :status_code, :redirect_url
+      def initialize(klass, method, rule_hash)
+        @redirect_url = nil
+        @status_code = nil
+        super(klass, method, rule_hash)
+        if @data.nil? || @data['values'].empty?
+          raise Sqreen::Exception, 'No data'
+        end
+        configure_custom_error(@data['values'][0])
+      end
+
+      def configure_custom_error(custom_error)
+        case custom_error['type']
+        when 'custom_error_page' then
+          @status_code = custom_error['status_code'].to_i
+        when 'redirection' then
+          @redirect_url = custom_error['redirection_url']
+          @status_code = custom_error.fetch('status_code', 303).to_i
+        else
+          raise Sqreen::Exception, "No custom error #{custom_error['type']}"
+        end
+      end
+
+      def failing(except, _inst, *_args, &_block)
+        oexcept = nil
+        if except.respond_to?(:original_exception)
+          oexcept = except.original_exception
+        end
+        if !except.is_a?(Sqreen::AttackBlocked) &&
+           !oexcept.is_a?(Sqreen::AttackBlocked)
+          return advise_action(nil)
+        end
+        if @redirect_url
+          advise_action(:override, :new_return_value => respond_redirect)
+        else
+          advise_action(:override, :new_return_value => respond_page)
+        end
+      end
+
+      def respond_redirect
+        [@status_code, { 'Location' => @redirect_url }, ['']]
+      end
+
+      def respond_page
+        page = open(File.join(File.dirname(__FILE__), '../attack_detected.html'))
+        headers = {
+          'Content-Type' => 'text/html',
+          'Content-Length' => page.size.to_s,
+        }
+        [@status_code, headers, page]
+      end
+    end
+  end
+end

data/lib/sqreen/rules_callbacks/execjs.rb

@@ -0,0 +1,241 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+if defined?(::JRUBY_VERSION)
+  require 'rhino'
+  SQREEN_V8_THREAD = false
+else
+  begin
+    require 'mini_racer'
+    SQREEN_V8_THREAD = true
+  rescue LoadError
+    require 'therubyracer'
+    SQREEN_V8_THREAD = false
+  end
+end
+
+require 'execjs'
+
+require 'sqreen/rule_attributes'
+require 'sqreen/rule_callback'
+require 'sqreen/condition_evaluator'
+require 'sqreen/binding_accessor'
+require 'sqreen/events/remote_exception'
+
+module Sqreen
+  module Rules
+    # Exec js callbacks
+    class ExecJSCB < RuleCB
+      attr_accessor :restrict_max_depth
+      def initialize(klass, method, rule_hash)
+        super(klass, method, rule_hash)
+        callbacks = @rule[Attrs::CALLBACKS]
+        @conditions = @rule.fetch(Attrs::CONDITIONS, {})
+
+        if callbacks['pre'].nil? &&
+           callbacks['post'].nil? &&
+           callbacks['failing'].nil?
+          raise(Sqreen::Exception, 'no JS CB provided')
+        end
+
+        build_runnable(callbacks)
+        if !SQREEN_V8_THREAD
+          @compiled = ExecJS.compile(@source)
+        end
+        @restrict_max_depth = 20
+      end
+
+      def pre?
+        @js_pre
+      end
+
+      def post?
+        @js_post
+      end
+
+      def failing?
+        @js_failing
+      end
+
+      def pre(inst, *args, &_block)
+        return unless pre?
+
+        call_callback('pre', inst, args)
+      end
+
+      def post(rv, inst, *args, &_block)
+        return unless post?
+
+        call_callback('post', inst, args, rv)
+      end
+
+      def failing(rv, inst, *args, &_block)
+        return unless failing?
+
+        call_callback('failing', inst, args, rv)
+      end
+
+      def self.hash_val_included(needed, haystack, min_length = 8, max_depth = 20)
+        new_obj = {}
+        insert = []
+        to_do = haystack.map { |k, v| [new_obj, k, v, 0] }
+        until to_do.empty?
+          where, key, value, deepness = to_do.pop
+          safe_key = key.kind_of?(Integer) ? key : key.to_s
+          if value.is_a?(Hash) && deepness < max_depth
+            val = {}
+            insert << [where, safe_key, val]
+            to_do += value.map { |k, v| [val, k, v, deepness + 1] }
+          elsif value.is_a?(Array) && deepness < max_depth
+            val = []
+            insert << [where, safe_key, val]
+            i = -1
+            to_do += value.map { |v| [val, i += 1, v, deepness + 1] }
+          elsif deepness >= max_depth # if we are after max_depth don't try to filter
+            insert << [where, safe_key, value]
+          else
+            v = value.to_s
+            if v.size >= min_length && ConditionEvaluator.str_include?(needed.to_s, v)
+              case where
+              when Array
+                where << value
+              else
+                where[safe_key] = value
+              end
+            end
+          end
+        end
+        insert.reverse.each do |wh, ikey, ival|
+          case wh
+          when Array
+            wh << ival unless ival.respond_to?(:empty?) && ival.empty?
+          else
+            wh[ikey] = ival unless ival.respond_to?(:empty?) && ival.empty?
+          end
+        end
+        new_obj
+      end
+
+      protected
+
+      def record_and_continue?(ret)
+        case ret
+        when NilClass
+          return false
+        when Hash
+          ret.keys.each do |k|
+            ret[(begin
+                                     k.to_sym
+                                   rescue
+                                     k
+                                   end)] = ret[k] end
+          record_event(ret[:record]) unless ret[:record].nil?
+          unless ret['observations'].nil?
+            ret['observations'].each do |obs|
+              obs[3] = Time.parse(obs[3]) if obs.size >= 3 && obs[3].is_a?(String)
+              record_observation(*obs)
+            end
+          end
+          return !ret[:call].nil?
+        else
+          raise Sqreen::Exception, "Invalid return type #{ret.inspect}"
+        end
+      end
+
+      def call_callback(name, inst, args, rv = nil)
+        if SQREEN_V8_THREAD
+          Thread.current["SQREEN_EXECJS_SOURCE#{self.object_id}"] ||= ExecJS.compile(@source)
+        end
+        ret = nil
+        args_override = nil
+        arguments = nil
+        loop do
+          arguments = (args_override || @argument_requirements[name]).map do |accessor|
+            accessor.resolve(binding, framework, inst, args, @data, rv)
+          end
+          arguments = restrict(name, arguments) if @conditions.key?(name)
+          Sqreen.log.debug { [name, arguments].inspect }
+          if SQREEN_V8_THREAD
+            ret = Thread.current["SQREEN_EXECJS_SOURCE#{self.object_id}"].call(name, *arguments)
+          else
+            ret = @compiled.call(name, *arguments)
+          end
+          unless record_and_continue?(ret)
+            return nil if ret.nil?
+            return advise_action(ret[:status], ret)
+          end
+          name = ret[:call]
+          rv = ret[:data]
+          args_override = ret[:args]
+          args_override = build_accessor(args_override) if args_override
+        end
+      rescue => e
+        Sqreen.log.warn "we catch a JScb exception: #{e.inspect}"
+        Sqreen.log.debug e.backtrace
+        record_exception(e, :cb => name, :args => arguments)
+        nil
+      end
+
+      def each_hash_val_include(condition, depth = 10)
+        return if depth <= 0
+        condition.each do |key, values|
+          if key == ConditionEvaluator::HASH_INC_OPERATOR
+            yield values
+          else
+            values.map do |v|
+              each_hash_val_include(v, depth - 1) { |vals| yield vals } if v.is_a?(Hash)
+            end
+          end
+        end
+      end
+
+      def restrict(cbname, arguments)
+        condition = @conditions[cbname]
+        return arguments if condition.nil? or @argument_requirements[cbname].nil?
+
+        each_hash_val_include(condition) do |needle, haystack, min_length|
+          # We could actually run the binding accessor expression here.
+          needed_idx = @argument_requirements[cbname].map(&:expression).index(needle)
+          next unless needed_idx
+
+          haystack_idx = @argument_requirements[cbname].map(&:expression).index(haystack)
+          next unless haystack_idx
+
+          arguments[haystack_idx] = ExecJSCB.hash_val_included(
+            arguments[needed_idx],
+            arguments[haystack_idx],
+            min_length.to_i,
+            @restrict_max_depth
+          )
+        end
+
+        arguments
+      end
+
+      def build_accessor(reqs)
+        reqs.map do |req|
+          BindingAccessor.new(req, true)
+        end
+      end
+
+      def build_runnable(callbacks)
+        @argument_requirements = {}
+        @source = ''
+        @js_pre = !callbacks['pre'].nil?
+        @js_post = !callbacks['post'].nil?
+        @js_failing = !callbacks['failing'].nil?
+        callbacks.each do |name, args_or_func|
+          @source << "var #{name} = "
+          if args_or_func.is_a?(Array)
+            @source << args_or_func.pop
+            @argument_requirements[name] = build_accessor(args_or_func)
+          else
+            @source << args_or_func
+            @argument_requirements[name] = []
+          end
+          @source << ";\n"
+        end
+      end
+    end
+  end
+end