sqreen 0.1.0.pre → 0.7.01461158029

data/lib/sqreen/performance_notifications.rb

@@ -0,0 +1,86 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+module Sqreen
+  # This module enable us to keep track of sqreen resource usage
+  #
+  # It is inspired by ActiveSupport::Notifications
+  #
+  module PerformanceNotifications
+    @subscriptions_all = {}
+    @subscriptions_regexp = {}
+    @subscriptions_val = Hash.new { |h, k| h[k] = [] }
+    @subscription_id = 0
+    class << self
+      # Subsribe to receive notificiations about an event
+      # returns a subscription indentitifcation
+      def subscribe(pattern = nil, &block)
+        id = (@subscription_id += 1)
+        case pattern
+        when NilClass
+          @subscriptions_all[id] = block
+        when Regexp
+          @subscriptions_regexp[id] = [pattern, block]
+        else
+          @subscriptions_val[pattern].push([id, block])
+        end
+        id
+      end
+
+      # Is there a subscriber for this key
+      def listen_for?(key)
+        return true unless @subscriptions_all.empty?
+        return true if @subscriptions_val.key?(key)
+        @subscriptions_regexp.values.any? { |r| r.first.match(key) }
+      end
+
+      # Instrument a call identified by key
+      def instrument(key, meta = {}, &block)
+        return yield unless listen_for?(key)
+        _instrument(key, meta, &block)
+      end
+
+      # Unsubscrube for a given subscription
+      def unsubscribe(subscription)
+        return unless @subscriptions_all.delete(subscription).nil?
+        return unless @subscriptions_regexp.delete(subscription).nil?
+        @subscriptions_val.delete_if do |_, v|
+          v.delete_if { |r| r.first == subscription }
+          v.empty?
+        end
+      end
+
+      # Unsubscribe from everything
+      # not threadsafe
+      def unsubscribe_all!
+        @subscriptions_all.clear
+        @subscriptions_regexp.clear
+        @subscriptions_val.clear
+      end
+
+      private
+
+      def notifiers_for(key)
+        reg = @subscriptions_regexp.values.map do |r|
+          r.first.match(key) && r.last
+        end
+        reg.compact!
+        str = []
+        if @subscriptions_val.key?(key)
+          str = @subscriptions_val[key].map(&:last)
+        end
+        @subscriptions_all.values + str + reg
+      end
+
+      def _instrument(key, meta)
+        start = Time.now
+        yield
+      ensure
+        stop = Time.now
+        notifiers_for(key).each do |callable|
+          callable.call(key, start, stop, meta)
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/performance_notifications/log.rb

@@ -0,0 +1,36 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+require 'sqreen/performance_notifications'
+
+module Sqreen
+  module PerformanceNotifications
+    # Log performances on the console
+    class Log
+      @subid = nil
+      @facility = nil
+      class << self
+        def log(event, start, finish, meta)
+          (@facility || Sqreen.log).debug do
+            meta_str = nil
+            meta_str = ": #{meta.inspect}" unless meta.empty?
+            format('%s took %.2fms%s', event, (finish - start) * 1000, meta_str)
+          end
+        end
+
+        def enable(facility = nil)
+          return unless @subid.nil?
+          @facility = facility
+          @subid = Sqreen::PerformanceNotifications.subscribe(nil,
+                                                              &method(:log))
+        end
+
+        def disable
+          return if @subid.nil?
+          Sqreen::PerformanceNotifications.unsubscribe(@subid)
+          @subid = nil
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/performance_notifications/metrics.rb

@@ -0,0 +1,36 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+require 'sqreen/performance_notifications'
+
+module Sqreen
+  module PerformanceNotifications
+    # Log performances in sqreen metrics_store
+    class Metrics
+      @subid = nil
+      @facility = nil
+      class << self
+        EVENT_CAT = 'sqreen_time'.freeze
+        def log(event, start, finish, _meta)
+          evt = [EVENT_CAT, event, (finish - start) * 1000, finish]
+          Sqreen.observations_queue.push(evt)
+        end
+
+        def enable(metrics_engine, period = 60)
+          return unless @subid.nil?
+          metrics_engine.create_metric('name' => EVENT_CAT,
+                                       'period' => period,
+                                       'kind' => 'Average')
+          @subid = Sqreen::PerformanceNotifications.subscribe(nil,
+                                                              &method(:log))
+        end
+
+        def disable
+          return if @subid.nil?
+          Sqreen::PerformanceNotifications.unsubscribe(@subid)
+          @subid = nil
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/performance_notifications/newrelic.rb

@@ -0,0 +1,36 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+require 'sqreen/performance_notifications'
+
+module Sqreen
+  module PerformanceNotifications
+    # Log performances on the console
+    class NewRelic
+      @subid = nil
+
+      @nr_name_regexp = %r{/([^/]+)$}
+
+      class << self
+        def log(event, start, finish, _meta)
+          event_name = "Custom/Sqreen#{event.sub(@nr_name_regexp, '_\1')}"
+          ::NewRelic::Agent.record_metric(event_name, finish - start)
+        end
+
+        def enable
+          return unless @subid.nil?
+          return unless defined?(::NewRelic::Agent)
+          Sqreen.log.debug('Enabling New Relic reporting')
+          @subid = Sqreen::PerformanceNotifications.subscribe(nil,
+                                                              &method(:log))
+        end
+
+        def disable
+          return if @subid.nil?
+          Sqreen::PerformanceNotifications.unsubscribe(@subid)
+          @subid = nil
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/remote_command.rb

@@ -0,0 +1,82 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+module Sqreen
+  # Execute and sanitize remote commands
+  class RemoteCommand
+    KNOWN_COMMANDS = {
+      :instrumentation_enable => :setup_instrumentation,
+      :instrumentation_remove => :remove_instrumentation,
+      :rules_reload => :reload_rules,
+      :features_get => :features,
+      :features_change => :change_features,
+    }.freeze
+
+    attr_reader :uuid
+
+    def initialize(json_desc)
+      @name = json_desc['name'].to_sym
+      @params = json_desc.fetch('params', [])
+      @uuid = json_desc['uuid']
+    end
+
+    def process(runner)
+      failing = validate_command(runner)
+      return failing if failing
+      Sqreen.log.debug format('processing command %s', @name)
+      output = runner.send(KNOWN_COMMANDS[@name], *@params)
+      format_output(output)
+    end
+
+    def self.process_list(runner, commands)
+      res_list = {}
+
+      return res_list unless commands
+
+      unless commands.is_a? Array
+        Sqreen.log.debug format('Wrong commands type %s', commands.class)
+        Sqreen.log.debug commands.inspect
+        return res_list
+      end
+      commands.each do |cmd_json|
+        Sqreen.log.debug cmd_json
+        cmd = RemoteCommand.new(cmd_json)
+        Sqreen.log.debug cmd.inspect
+        uuid = cmd.uuid
+        res_list[uuid] = cmd.process(runner)
+      end
+      res_list
+    end
+
+    def to_h
+      {
+        :name => @name,
+      }
+    end
+
+    protected
+
+    def validate_command(runner)
+      unless KNOWN_COMMANDS.include?(@name)
+        msg = format("unknown command name '%s'", @name)
+        Sqreen.log.debug msg
+        return { :status => false, :reason => msg }
+      end
+      return nil if runner.respond_to?(KNOWN_COMMANDS[@name])
+      msg = format("not implemented '%s'", @name)
+      Sqreen.log.debug msg
+      { :status => false, :reason => msg }
+    end
+
+    def format_output(output)
+      case output
+      when NilClass
+        return { :status => false, :reason => 'nil returned' }
+      when TrueClass
+        return { :status => true }
+      else
+        return { :status => true, :output => output }
+      end
+    end
+  end
+end

data/lib/sqreen/rule_attributes.rb

@@ -0,0 +1,25 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+module Sqreen
+  module Rules
+    # Common field names in a rule
+    module Attrs
+      CALLBACKS = 'callbacks'.freeze
+      BLOCK = 'block'.freeze
+      TEST = 'test'.freeze
+      DATA = 'data'.freeze
+      PAYLOAD = 'payload'.freeze
+      NAME = 'name'.freeze
+      RULESPACK_ID = 'rulespack_id'.freeze
+      HOOKPOINT = 'hookpoint'.freeze
+      KLASS = 'klass'.freeze
+      METHOD = 'method'.freeze
+      CALLBACK_CLASS = 'callback_class'.freeze
+      METRICS = 'metrics'.freeze
+      CONDITIONS = 'conditions'.freeze
+
+      freeze
+    end
+  end
+end

data/lib/sqreen/rule_callback.rb

@@ -0,0 +1,97 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+require 'sqreen/callbacks'
+require 'sqreen/conditionable'
+require 'sqreen/rule_attributes'
+require 'sqreen/events/attack'
+require 'sqreen/events/remote_exception'
+require 'sqreen/payload_creator'
+
+# Rules defined here can be instanciated from JSON.
+module Sqreen
+  module Rules
+    # Base class for callback that are initialized by rules from Sqreen
+    class RuleCB < CB
+      include Conditionable
+      # If nothing was asked by the rule we will ask for all sections available
+      # These information will be pruned later when exporting in #to_hash
+      DEFAULT_PAYLOAD = PayloadCreator::METHODS.keys.freeze
+      attr_reader :test
+      attr_reader :block
+      attr_accessor :framework
+
+      # @params klass [String] class instrumented
+      # @params method [String] method that was instrumented
+      # @params rule_hash [Hash] Rule data that govern the current behavior
+      def initialize(klass, method, rule_hash)
+        super(klass, method)
+        @block = rule_hash[Attrs::BLOCK] == true
+        @test = rule_hash[Attrs::TEST] == true
+        @data = rule_hash[Attrs::DATA]
+        @rule = rule_hash
+        payload_tpl = @rule[Attrs::PAYLOAD] || DEFAULT_PAYLOAD
+        @payload_generator = PayloadCreator.new(payload_tpl)
+        conditions = @rule[Attrs::CONDITIONS]
+        condition_callbacks(conditions) unless conditions.nil?
+      end
+
+      def rule_name
+        @rule[Attrs::NAME]
+      end
+
+      def rulespack_id
+        @rule[Attrs::RULESPACK_ID]
+      end
+
+      # Record an attack event into Sqreen system
+      # @param infos [Hash] Additional information about request
+      def record_event(infos)
+        payload = @payload_generator.payload(framework, @rule)
+        payload['infos'] = infos
+        Attack.record(payload)
+      end
+
+      # Record a metric observation
+      # @param category [String] Name of the metric observed
+      # @param key [String] aggregation key
+      # @param observation [Object] data observed
+      # @param at [Time] time when observation was made
+      def record_observation(category, key, observation, at = Time.now.utc)
+        Sqreen.observations_queue.push [category, key, observation, at]
+        if Sqreen.observations_queue.size > MAX_OBS_QUEUE_LENGTH / 2
+          Sqreen.queue.push Sqreen::METRICS_EVENT
+        end
+      end
+
+      # Record an exception that just occurred
+      # @param exception [Exception] Exception to send over
+      # @param infos [Hash] Additional contextual information
+      def record_exception(exception, infos = {})
+        payload = {}
+        payload['exception']      = exception
+        payload['rulespack_id']   = rulespack_id
+        payload['rule_name']      = rule_name
+        begin
+          payload['request_infos'] = framework.request_infos
+        rescue => e
+          Sqreen.log.debug("No framework request infos #{e}")
+        end
+        begin
+          payload['request_params'] = framework.request_params
+        rescue => e
+          Sqreen.log.debug("No framework request params #{e}")
+        end
+        payload['time']           = Time.now.utc
+        payload['infos']          = infos
+        payload['backtrace']      = Sqreen::Context.new.bt
+        begin
+          payload['client_ip'] = framework.client_ip.to_s
+        rescue => e
+          Sqreen.log.debug("No framework client_ip #{e}")
+        end
+        RemoteException.record(payload)
+      end
+    end
+  end
+end

data/lib/sqreen/rules.rb

@@ -0,0 +1,116 @@
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.io/terms.html
+
+require 'sqreen/rule_attributes'
+require 'sqreen/rules_callbacks'
+
+
+## Rules
+#
+# Rule example:
+#  {
+#      :class => 'ActionController::Metal',
+#      :method => 'dispatch',
+#      :arguments => {:type => 'position', :options => {:position => 1}}
+#      :callback_class => 'RackCB',
+#  }
+# We instrument ActionController::Metal#dispatch. We are interested in the first
+# argument. When this method is called, we will provide it's argument to the
+# callback RackCB.
+#
+# Another option for execution is to delegate the callback to a JS helper,
+# rather than to a class. The JS callback will be executed with the requested
+# arguments.
+
+module Sqreen
+  # Rules related method/functions
+  module Rules
+    def self::local(configuration)
+      # Parse and return local rules (path defined in environment)
+
+      path = configuration.get(:local_rules)
+      return [] unless path
+      begin
+        File.open(path) { |f| JSON.load(f) }
+      rescue Errno::ENOENT
+        Sqreen.log.error "File '#{path}' not found"
+        []
+      end
+    end
+
+    # Given a rule, will instantiate the related callback.
+    # @param hash_rule     [Hash]                 Rules metadata
+    # @param metrics_store [MetricStore]          Metrics storage facility
+    # @param verifier      [SqreenSignedVerifier] Signed verifier
+    def self::cb_from_rule(hash_rule, metrics_store = nil, verifier = nil)
+      # Check rules signature
+      verifier.verify(hash_rule) if verifier
+
+      hook = hash_rule[Attrs::HOOKPOINT]
+      klass = hook[Attrs::KLASS]
+
+      # The instrumented class can be from anywhere
+      instr_class = Rules.walk_const_get klass
+
+      if instr_class.nil?
+        rule_name = hash_rule[Attrs::NAME]
+        Sqreen.log.debug "#{klass} does not exists. Skipping #{rule_name}"
+        return nil
+      end
+
+      instr_method = hook[Attrs::METHOD]
+      instr_method = instr_method.to_sym
+
+      cbname = hook[Attrs::CALLBACK_CLASS]
+
+      cb_class = nil
+      js = hash_rule[Attrs::CALLBACKS]
+      cb_class = ExecJSCB if js
+
+      if cbname && Rules.const_defined?(cbname)
+        # Only load callbacks from sqreen
+        cb_class = Rules.const_get(cbname)
+      end
+
+      if cb_class.nil?
+        Sqreen.log.debug "Cannot setup #{cbname.inspect} [#{rule_name}]"
+        return nil
+      end
+
+      unless cb_class.ancestors.include?(RuleCB)
+        Sqreen.log.debug "#{cb_class} does not inherit from RuleCB"
+        return nil
+      end
+
+      if metrics_store
+        (hash_rule[Attrs::METRICS] || []).each do |metric|
+          metrics_store.create_metric(metric)
+        end
+      end
+
+      cb_class.new(instr_class, instr_method, hash_rule)
+    rescue => e
+      rule_name = nil
+      rulespack_id = nil
+      if hash_rule.respond_to?(:[])
+        rule_name = hash_rule[Attrs::NAME]
+        rulespack_id = hash_rule[Attrs::RULESPACK_ID]
+      end
+      Sqreen::RemoteException.record(
+        'exception' => e,
+        'rulespack_id' => rulespack_id,
+        'rule_name' => rule_name)
+      Sqreen.log.debug("Creating cb from rule #{rule_name} failed (#{e.inspect})")
+      nil
+    end
+
+    def self::walk_const_get(str)
+      obj = Object
+      str.split('::').compact.each do |part|
+        return nil unless obj.const_defined?(part)
+        obj = obj.const_get(part)
+      end
+      obj
+    end
+  end
+end