sqreen 0.8.11465220943 → 1.0.0.pre1480953244

data/lib/sqreen/instrumentation.rb

@@ -159,15 +159,13 @@ module Sqreen
     end
 
     def self.guard_call(method, retval)
-      if @sqreen_in_instr && @sqreen_in_instr.member?(method)
-        return retval
-      else
-        @sqreen_in_instr ||= Set.new
-        @sqreen_in_instr.add(method)
-        r = yield
-        @sqreen_in_instr.delete(method)
-        return r
-      end
+      @sqreen_in_instr ||= nil
+      return retval if @sqreen_in_instr && @sqreen_in_instr.member?(method)
+      @sqreen_in_instr ||= Set.new
+      @sqreen_in_instr.add(method)
+      r = yield
+      @sqreen_in_instr.delete(method)
+      return r
     rescue Exception => e
       @sqreen_in_instr.delete(method)
       raise e
@@ -338,7 +336,6 @@ module Sqreen
       saved_meth_name = get_saved_method_name(meth)
 
       eval "method_kind = nil; class << #{klass}
-          new_method      = '#{meth}_modified'.to_sym
           case
           when public_method_defined?(#{meth.to_sym.inspect})
             method_kind = :public

data/lib/sqreen/log.rb

@@ -8,6 +8,7 @@ require 'sqreen/configuration'
 
 module Sqreen
   def self::log
+    @logger ||= nil
     return @logger unless @logger.nil?
     @logger = Logger.new(
       Sqreen.config_get(:log_level).to_s.upcase,

data/lib/sqreen/metrics/base.rb

@@ -10,6 +10,10 @@ module Sqreen
     FINISH_KEY = 'finish'.freeze
     # Base interface for a metric
     class Base
+      def initialize
+        @sample = nil
+      end
+
       # Update the current metric with a new observation
       # @param _at [Time] when was the observation made
       # @param _key [String] which aggregation key was it made for

data/lib/sqreen/metrics_store.rb

@@ -49,15 +49,15 @@ module Sqreen
       at = at.utc
       metric, period, start = @metrics[name]
       raise UnregisteredMetric, "Unknown metric #{name}" unless metric
-      next_sample(name, at) if start.nil? || start + period < at
+      next_sample(name, at) if start.nil? || (start + period) < at
       metric.update(at, key, value)
     end
 
     # Drains every metrics and returns the store content
     # @params at [Time] when is the store emptied
-    def publish(at = Time.now.utc)
-      @metrics.each_key do |name|
-        next_sample(name, at)
+    def publish(flush = true, at = Time.now.utc)
+      @metrics.each do |name, (_, period, start)|
+        next_sample(name, at) if flush || !start.nil? && (start + period) < at
       end
       out = @store
       @store = []

data/lib/sqreen/remote_command.rb

@@ -1,5 +1,6 @@
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.io/terms.html
+require "sqreen/log"
 
 module Sqreen
   # Execute and sanitize remote commands
@@ -20,15 +21,15 @@ module Sqreen
       @uuid = json_desc['uuid']
     end
 
-    def process(runner)
+    def process(runner, context_infos = {})
       failing = validate_command(runner)
       return failing if failing
       Sqreen.log.debug format('processing command %s', @name)
-      output = runner.send(KNOWN_COMMANDS[@name], *@params)
+      output = runner.send(KNOWN_COMMANDS[@name], *@params, context_infos)
       format_output(output)
     end
 
-    def self.process_list(runner, commands)
+    def self.process_list(runner, commands, context_infos = {})
       res_list = {}
 
       return res_list unless commands
@@ -43,7 +44,7 @@ module Sqreen
         cmd = RemoteCommand.new(cmd_json)
         Sqreen.log.debug cmd.inspect
         uuid = cmd.uuid
-        res_list[uuid] = cmd.process(runner)
+        res_list[uuid] = cmd.process(runner, context_infos)
       end
       res_list
     end

data/lib/sqreen/rules_callbacks.rb

@@ -11,12 +11,8 @@ require 'sqreen/rules_callbacks/headers_insert'
 
 require 'sqreen/rules_callbacks/inspect_rule'
 
-require 'sqreen/rules_callbacks/shell'
-require 'sqreen/rules_callbacks/system_shell'
 require 'sqreen/rules_callbacks/shell_env'
 
-require 'sqreen/rules_callbacks/sql'
-
 require 'sqreen/rules_callbacks/url_matches'
 require 'sqreen/rules_callbacks/user_agent_matches'
 require 'sqreen/rules_callbacks/crawler_user_agent_matches'

data/lib/sqreen/rules_callbacks/matcher_rule.rb

@@ -19,6 +19,7 @@ module Sqreen
         Regexp.compile(value, res)
       end
 
+      ANYWHERE_OPT = 'anywhere'.freeze
       def prepare
         @string = {}
         @regex_patterns = []
@@ -30,10 +31,10 @@ module Sqreen
         end
 
         @funs = {
-          'anywhere'    => lambda { |value, str| str.include?(value)    },
-          'starts_with' => lambda { |value, str| str.start_with?(value) },
-          'ends_with'   => lambda { |value, str| str.end_with?(value)   },
-          'equals'      => lambda { |value, str| str == value           },
+          ANYWHERE_OPT       => lambda { |value, str| str.include?(value)   },
+          'starts_with'.freeze => lambda { |value, str| str.start_with?(value) },
+          'ends_with'.freeze => lambda { |value, str| str.end_with?(value)   },
+          'equals'.freeze    => lambda { |value, str| str == value           },
         }
 
         patterns.each do |entry|
@@ -41,7 +42,8 @@ module Sqreen
           type = entry['type']
           val = entry['value']
           opts = entry['options']
-          opt = (opts && opts.first && opts.first != '') ? opts.first : 'anywhere'
+          opt = ANYWHERE_OPT
+          opt = opts.first.freeze if opts && opts.first && opts.first != ''
           case_sensitive = entry['case_sensitive'] || false
           case type
           when 'string'
@@ -49,7 +51,7 @@ module Sqreen
               case_type = :cs
             else
               case_type = :ci
-              val = val.downcase
+              val.downcase!
             end
 
             unless @funs.keys.include?(opt)

data/lib/sqreen/rules_callbacks/reflected_xss.rb

@@ -7,35 +7,79 @@ require 'sqreen/rules_callbacks/regexp_rule'
 
 module Sqreen
   module Rules
-    # look for reflected XSS
+    # look for reflected XSS with erb template engine
     class ReflectedXSSCB < RegexpRuleCB
       def pre(_inst, *args, &_block)
         value = args[0]
-        return if value.nil?
+
+        return unless value.is_a?(String)
+
         # If the value is not marked as html_safe, it will be escaped later
         return unless value.html_safe?
 
         # Sqreen::log.debug value
-        # Sqreen::log.debug params
 
         return unless framework.params_include?(value)
 
         Sqreen.log.debug { format('Found unescaped user param: %s', value) }
 
-        return unless value.is_a?(String)
-
         saved_value = value.dup
         # potential XSS! let's escape
         args[0].replace(CGI.escape_html(value)) if block
-        # The remaining code is only to find out if user entry was an attack,
-        # and record it. Since we don't rely on it to respond to user, it would
-        # be better to do it in background.
-        found = match_regexp(saved_value)
+
+        report_dangerous_xss(saved_value)
+
+        nil
+      end
+
+      # The remaining code is only to find out if user entry was an attack,
+      # and record it. Since we don't rely on it to respond to user, it would
+      # be better to do it in background.
+      def report_dangerous_xss(value)
+        found = match_regexp(value)
 
         return unless found
-        infos = { :found => found }
+        infos = {
+          :found => found,
+          :payload => value
+        }
         record_event(infos)
-        nil
+      end
+    end
+    # look for reflected XSS with haml template engine
+    # hook function arguments of
+    # Haml::Buffer.format_script(result, preserve_script, in_tag, preserve_tag,
+    #                            escape_html, nuke_inner_whitespace,
+    #                            interpolated, ugly)
+    class ReflectedXSSHamlCB < ReflectedXSSCB
+      def pre(inst, *args, &_block)
+        value = args[0]
+
+        return unless value.is_a?(String)
+
+        # if escape_html is already true, it will be escaped later
+        return if args[4]
+
+        return unless framework.params_include?(value)
+
+        Sqreen.log.debug { format('Found unescaped user param: %s', value) }
+
+        # potential XSS ! Call the same method with the argument
+        # escape_html true
+        if block
+          nargs = args.dup
+          nargs[4] = true
+          return_value = {
+            :status => :skip,
+            # 'method' is an attribut reader in class CB, it's the name of the
+            # hooked method
+            :new_return_value => inst.send(method, *nargs),
+          }
+        end
+
+        report_dangerous_xss(value)
+
+        return_value
       end
     end
   end

data/lib/sqreen/runner.rb

@@ -2,18 +2,21 @@
 # Please refer to our terms for more information: https://www.sqreen.io/terms.html
 
 require 'timeout'
+require 'json'
 
 require 'sqreen/events/attack'
 
 require 'sqreen/log'
 
 require 'sqreen/rules'
+require 'sqreen/session'
 require 'sqreen/remote_command'
 require 'sqreen/capped_queue'
 require 'sqreen/metrics_store'
 require 'sqreen/deliveries/simple'
 require 'sqreen/deliveries/batch'
 require 'sqreen/performance_notifications/metrics'
+require 'sqreen/instrumentation'
 
 module Sqreen
   @features = {}
@@ -48,19 +51,18 @@ module Sqreen
 
   # Main running job class for the agent
   class Runner
-    # At start, heartbeat is every 15 seconds
-    HEARTBEAT_INITIAL_DELAY = 15
     # During one hour
     HEARTBEAT_WARMUP = 60 * 60
-    # Then delay raises to 5 minutes
+    # Initail delay is 5 minutes
     HEARTBEAT_MAX_DELAY = 5 * 60
 
-    attr_reader :heartbeat_delay
+    attr_accessor :heartbeat_delay
     attr_accessor :metrics_engine
-    attr_reader :publish_metrics_delay
     attr_reader :deliverer
     attr_reader :session
     attr_reader :instrumenter
+    attr_accessor :next_command_results
+    attr_accessor :next_metrics
 
     # we may want to do that in a thread in order to prevent delaying app
     # startup
@@ -69,12 +71,10 @@ module Sqreen
       @logged_out_tried = false
       @configuration = configuration
       @framework = framework
-      @heartbeat_delay = HEARTBEAT_INITIAL_DELAY
-      @publish_metrics_delay = HEARTBEAT_MAX_DELAY
-      @sleep_delay = HEARTBEAT_INITIAL_DELAY
-      @last_heartbeat_request = Time.at(0)
-      @last_post_metrics_request = Time.now
-      @started = Time.now
+      @heartbeat_delay = HEARTBEAT_MAX_DELAY
+      @last_heartbeat_request = Time.now
+      @next_command_results = {}
+      @next_metrics = []
 
       @token = @configuration.get(:token)
       @url = @configuration.get(:url)
@@ -83,13 +83,37 @@ module Sqreen
 
       register_exit_cb if set_at_exit
 
+      self.metrics_engine = MetricsStore.new
+      @instrumenter = Instrumentation.new(metrics_engine)
+
       Sqreen.log.warn "using token #{@token}"
-      self.features = create_session(session_class)
+      response = create_session(session_class)
+      wanted_features = response.fetch('features', {})
+      conf_initial_features = configuration.get(:initial_features)
+      unless conf_initial_features.nil?
+        begin
+          conf_features = JSON.parse(conf_initial_features)
+          raise 'Invalid Type' unless conf_features.is_a?(Hash)
+          Sqreen.log.debug do
+            "Override initial features with #{conf_features.inspect}"
+          end
+          wanted_features = conf_features
+        rescue
+          Sqreen.log.error do
+            "NOT using Invalid inital features #{conf_initial_features}"
+          end
+        end
+      end
+      self.features = wanted_features
+
       # Ensure a deliverer is there unless features have set it first
       self.deliverer ||= Deliveries::Simple.new(session)
 
-      self.metrics_engine = MetricsStore.new
-      @instrumenter = Instrumentation.new(metrics_engine)
+      context_infos = {}
+      %w(rules pack_id).each do |p|
+        context_infos[p] = response[p] unless response[p].nil?
+      end
+      process_commands(response.fetch('commands', []), context_infos)
     end
 
     def create_session(session_class)
@@ -112,20 +136,27 @@ module Sqreen
                        end
     end
 
-    def load_rules
-      rules_pack = session.rules
-      rulespack_id = rules_pack['pack_id']
-      rules = rules_pack['rules'].each { |r| r['rulespack_id'] = rulespack_id }
-      Sqreen.log.info format('retrieved rulespack id: %s', rulespack_id)
-      Sqreen.log.debug format('retrieved %d rules', rules.size)
+    def load_rules(context_infos = {})
+      rules_pack = context_infos['rules']
+      rulespack_id = context_infos['pack_id']
+      if rules_pack.nil? || rulespack_id.nil?
+        session_rules = session.rules
+        rules_pack = session_rules['rules']
+        rulespack_id = session_rules['pack_id']
+      end
+      rules = rules_pack.each { |r| r['rulespack_id'] = rulespack_id }
+      Sqreen.log.info { format('retrieved rulespack id: %s', rulespack_id) }
+      Sqreen.log.debug { format('retrieved %d rules', rules.size) }
       local_rules = Sqreen::Rules.local(@configuration) || []
       rules += local_rules.
                select { |rule| rule['enabled'] }.
                each { |r| r['rulespack_id'] = 'local' }
-      Sqreen.log.debug format('rules: %s', rules.
-                              sort_by { |r| r['name'] }.
-                              map { |r| format('(%s, %s)', r['name'], r.to_json.size) }.
-                              join(', '))
+      Sqreen.log.debug do
+        format('rules: %s', rules.
+               sort_by { |r| r['name'] }.
+               map { |r| format('(%s, %s)', r['name'], r.to_json.size) }.
+               join(', '))
+      end
       [rulespack_id, rules]
     end
 
@@ -138,20 +169,20 @@ module Sqreen
       end
     end
 
-    def setup_instrumentation
+    def setup_instrumentation(context_infos = {})
       Sqreen.log.info 'setup instrumentation'
-      rulespack_id, rules = load_rules
+      rulespack_id, rules = load_rules(context_infos)
       @framework.instrument_when_ready!(instrumenter, rules)
       rulespack_id.to_s
     end
 
-    def remove_instrumentation
+    def remove_instrumentation(_context_infos = {})
       Sqreen.log.debug 'removing instrumentation'
       instrumenter.remove_all_callbacks
       true
     end
 
-    def reload_rules
+    def reload_rules(_context_infos = {})
       Sqreen.log.debug 'Reloading rules'
       rulespack_id, rules = load_rules
       instrumenter.remove_all_callbacks
@@ -161,22 +192,21 @@ module Sqreen
       rulespack_id.to_s
     end
 
-    def process_commands(commands)
-      while commands && !commands.empty?
-        res = RemoteCommand.process_list(self, commands)
-        res = session.post_commands_result(res)
-        commands = res['commands']
-      end
+    def process_commands(commands, context_infos = {})
+      return if commands.nil? || commands.empty?
+      res = RemoteCommand.process_list(self, commands, context_infos)
+      @next_command_results = res
     end
 
     def do_heartbeat
       @last_heartbeat_request = Time.now
-      res = session.heartbeat
-      update_heartbeat_delay
+      res = session.heartbeat(next_command_results, next_metrics)
+      next_command_results.clear
+      next_metrics.clear
       process_commands(res['commands'])
     end
 
-    def features
+    def features(_context_infos = {})
       Sqreen.features
     end
 
@@ -184,13 +214,13 @@ module Sqreen
       Sqreen.update_features(features)
       session.request_compression = features['request_compression'] if session
       self.performance_metrics_period = features['performance_metrics_period']
-      md = features['publish_metrics_delay'].to_i
-      self.publish_metrics_delay = md if md > 0
+      hd = features['heartbeat_delay'].to_i
+      self.heartbeat_delay = hd if hd > 0
       return if features['batch_size'].nil?
       batch_events(features['batch_size'], features['max_staleness'])
     end
 
-    def change_features(new_features)
+    def change_features(new_features, _context_infos = {})
       old = features
       self.features = new_features
       {
@@ -208,7 +238,7 @@ module Sqreen
     end
 
     def run_watcher_once
-      event = Timeout.timeout(@sleep_delay) do
+      event = Timeout.timeout(heartbeat_delay) do
         Sqreen.queue.pop
       end
     rescue Timeout::Error
@@ -231,8 +261,10 @@ module Sqreen
       @deliverer.tick
       aggregate_observations
       t = Time.now
-      do_heartbeat if (@last_heartbeat_request + heartbeat_delay) < t
-      post_metrics if (@last_post_metrics_request + publish_metrics_delay) < t
+      if (@last_heartbeat_request + heartbeat_delay) < t
+        @next_metrics.concat(metrics_engine.publish(false)) if metrics_engine
+        do_heartbeat
+      end
     end
 
     def handle_event(event)
@@ -249,35 +281,6 @@ module Sqreen
       end
     end
 
-    def update_heartbeat_delay
-      return unless Time.now - @started > HEARTBEAT_WARMUP
-      return if heartbeat_delay == HEARTBEAT_MAX_DELAY
-      self.heartbeat_delay = HEARTBEAT_MAX_DELAY
-    end
-
-    def update_sleep_delay
-      @sleep_delay = [heartbeat_delay, publish_metrics_delay].min
-      Sqreen.log.debug { format('sleep delay %f', @sleep_delay) }
-    end
-
-    def heartbeat_delay=(x)
-      @heartbeat_delay = x
-      update_sleep_delay
-      x
-    end
-
-    def publish_metrics_delay=(x)
-      @publish_metrics_delay = x
-      update_sleep_delay
-      x
-    end
-
-    def post_metrics
-      return unless metrics_engine
-      @last_post_metrics_request = Time.now
-      session.post_metrics(metrics_engine.publish)
-    end
-
     # Sinatra is using at_exit to run the application, see:
     # https://github.com/sinatra/sinatra/blob/cd503e6c590cd48c2c9bb7869522494bfc62cb14/lib/sinatra/main.rb#L25
     def exit_from_sinatra_startup?
@@ -295,7 +298,7 @@ module Sqreen
       @logged_out_tried = true
       @deliverer.drain if @deliverer
       aggregate_observations
-      post_metrics
+      session.post_metrics(metrics_engine.publish) if metrics_engine
       session.logout(retrying)
     end