sqreen 0.7.01461158029-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. checksums.yaml +7 -0
  2. data/CODE_OF_CONDUCT.md +22 -0
  3. data/README.md +77 -0
  4. data/Rakefile +40 -0
  5. data/lib/sqreen.rb +67 -0
  6. data/lib/sqreen/binding_accessor.rb +184 -0
  7. data/lib/sqreen/ca.crt +72 -0
  8. data/lib/sqreen/callback_tree.rb +78 -0
  9. data/lib/sqreen/callbacks.rb +120 -0
  10. data/lib/sqreen/capped_queue.rb +23 -0
  11. data/lib/sqreen/condition_evaluator.rb +169 -0
  12. data/lib/sqreen/conditionable.rb +50 -0
  13. data/lib/sqreen/configuration.rb +151 -0
  14. data/lib/sqreen/context.rb +22 -0
  15. data/lib/sqreen/deliveries/batch.rb +80 -0
  16. data/lib/sqreen/deliveries/simple.rb +36 -0
  17. data/lib/sqreen/detect.rb +14 -0
  18. data/lib/sqreen/detect/shell_injection.rb +61 -0
  19. data/lib/sqreen/detect/sql_injection.rb +115 -0
  20. data/lib/sqreen/event.rb +16 -0
  21. data/lib/sqreen/events/attack.rb +60 -0
  22. data/lib/sqreen/events/remote_exception.rb +53 -0
  23. data/lib/sqreen/exception.rb +31 -0
  24. data/lib/sqreen/frameworks.rb +40 -0
  25. data/lib/sqreen/frameworks/generic.rb +243 -0
  26. data/lib/sqreen/frameworks/rails.rb +155 -0
  27. data/lib/sqreen/frameworks/rails3.rb +36 -0
  28. data/lib/sqreen/frameworks/sinatra.rb +34 -0
  29. data/lib/sqreen/frameworks/sqreen_test.rb +26 -0
  30. data/lib/sqreen/instrumentation.rb +504 -0
  31. data/lib/sqreen/log.rb +116 -0
  32. data/lib/sqreen/metrics.rb +6 -0
  33. data/lib/sqreen/metrics/average.rb +39 -0
  34. data/lib/sqreen/metrics/base.rb +41 -0
  35. data/lib/sqreen/metrics/collect.rb +22 -0
  36. data/lib/sqreen/metrics/sum.rb +20 -0
  37. data/lib/sqreen/metrics_store.rb +94 -0
  38. data/lib/sqreen/parsers/sql.rb +98 -0
  39. data/lib/sqreen/parsers/sql_tokenizer.rb +266 -0
  40. data/lib/sqreen/parsers/unix.rb +110 -0
  41. data/lib/sqreen/payload_creator.rb +132 -0
  42. data/lib/sqreen/performance_notifications.rb +86 -0
  43. data/lib/sqreen/performance_notifications/log.rb +36 -0
  44. data/lib/sqreen/performance_notifications/metrics.rb +36 -0
  45. data/lib/sqreen/performance_notifications/newrelic.rb +36 -0
  46. data/lib/sqreen/remote_command.rb +82 -0
  47. data/lib/sqreen/rule_attributes.rb +25 -0
  48. data/lib/sqreen/rule_callback.rb +97 -0
  49. data/lib/sqreen/rules.rb +116 -0
  50. data/lib/sqreen/rules_callbacks.rb +29 -0
  51. data/lib/sqreen/rules_callbacks/binding_accessor_metrics.rb +79 -0
  52. data/lib/sqreen/rules_callbacks/count_http_codes.rb +18 -0
  53. data/lib/sqreen/rules_callbacks/crawler_user_agent_matches.rb +24 -0
  54. data/lib/sqreen/rules_callbacks/crawler_user_agent_matches_metrics.rb +25 -0
  55. data/lib/sqreen/rules_callbacks/execjs.rb +136 -0
  56. data/lib/sqreen/rules_callbacks/headers_insert.rb +20 -0
  57. data/lib/sqreen/rules_callbacks/inspect_rule.rb +20 -0
  58. data/lib/sqreen/rules_callbacks/matcher_rule.rb +103 -0
  59. data/lib/sqreen/rules_callbacks/rails_parameters.rb +14 -0
  60. data/lib/sqreen/rules_callbacks/record_request_context.rb +23 -0
  61. data/lib/sqreen/rules_callbacks/reflected_xss.rb +40 -0
  62. data/lib/sqreen/rules_callbacks/regexp_rule.rb +36 -0
  63. data/lib/sqreen/rules_callbacks/shell.rb +33 -0
  64. data/lib/sqreen/rules_callbacks/shell_env.rb +32 -0
  65. data/lib/sqreen/rules_callbacks/sql.rb +41 -0
  66. data/lib/sqreen/rules_callbacks/system_shell.rb +25 -0
  67. data/lib/sqreen/rules_callbacks/url_matches.rb +25 -0
  68. data/lib/sqreen/rules_callbacks/user_agent_matches.rb +22 -0
  69. data/lib/sqreen/rules_signature.rb +142 -0
  70. data/lib/sqreen/runner.rb +312 -0
  71. data/lib/sqreen/runtime_infos.rb +127 -0
  72. data/lib/sqreen/session.rb +340 -0
  73. data/lib/sqreen/stats.rb +18 -0
  74. data/lib/sqreen/version.rb +6 -0
  75. metadata +143 -0
@@ -0,0 +1,86 @@
1
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
+ # Please refer to our terms for more information: https://www.sqreen.io/terms.html
3
+
4
+ module Sqreen
5
+ # This module enable us to keep track of sqreen resource usage
6
+ #
7
+ # It is inspired by ActiveSupport::Notifications
8
+ #
9
+ module PerformanceNotifications
10
+ @subscriptions_all = {}
11
+ @subscriptions_regexp = {}
12
+ @subscriptions_val = Hash.new { |h, k| h[k] = [] }
13
+ @subscription_id = 0
14
+ class << self
15
+ # Subsribe to receive notificiations about an event
16
+ # returns a subscription indentitifcation
17
+ def subscribe(pattern = nil, &block)
18
+ id = (@subscription_id += 1)
19
+ case pattern
20
+ when NilClass
21
+ @subscriptions_all[id] = block
22
+ when Regexp
23
+ @subscriptions_regexp[id] = [pattern, block]
24
+ else
25
+ @subscriptions_val[pattern].push([id, block])
26
+ end
27
+ id
28
+ end
29
+
30
+ # Is there a subscriber for this key
31
+ def listen_for?(key)
32
+ return true unless @subscriptions_all.empty?
33
+ return true if @subscriptions_val.key?(key)
34
+ @subscriptions_regexp.values.any? { |r| r.first.match(key) }
35
+ end
36
+
37
+ # Instrument a call identified by key
38
+ def instrument(key, meta = {}, &block)
39
+ return yield unless listen_for?(key)
40
+ _instrument(key, meta, &block)
41
+ end
42
+
43
+ # Unsubscrube for a given subscription
44
+ def unsubscribe(subscription)
45
+ return unless @subscriptions_all.delete(subscription).nil?
46
+ return unless @subscriptions_regexp.delete(subscription).nil?
47
+ @subscriptions_val.delete_if do |_, v|
48
+ v.delete_if { |r| r.first == subscription }
49
+ v.empty?
50
+ end
51
+ end
52
+
53
+ # Unsubscribe from everything
54
+ # not threadsafe
55
+ def unsubscribe_all!
56
+ @subscriptions_all.clear
57
+ @subscriptions_regexp.clear
58
+ @subscriptions_val.clear
59
+ end
60
+
61
+ private
62
+
63
+ def notifiers_for(key)
64
+ reg = @subscriptions_regexp.values.map do |r|
65
+ r.first.match(key) && r.last
66
+ end
67
+ reg.compact!
68
+ str = []
69
+ if @subscriptions_val.key?(key)
70
+ str = @subscriptions_val[key].map(&:last)
71
+ end
72
+ @subscriptions_all.values + str + reg
73
+ end
74
+
75
+ def _instrument(key, meta)
76
+ start = Time.now
77
+ yield
78
+ ensure
79
+ stop = Time.now
80
+ notifiers_for(key).each do |callable|
81
+ callable.call(key, start, stop, meta)
82
+ end
83
+ end
84
+ end
85
+ end
86
+ end
@@ -0,0 +1,36 @@
1
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
+ # Please refer to our terms for more information: https://www.sqreen.io/terms.html
3
+
4
+ require 'sqreen/performance_notifications'
5
+
6
+ module Sqreen
7
+ module PerformanceNotifications
8
+ # Log performances on the console
9
+ class Log
10
+ @subid = nil
11
+ @facility = nil
12
+ class << self
13
+ def log(event, start, finish, meta)
14
+ (@facility || Sqreen.log).debug do
15
+ meta_str = nil
16
+ meta_str = ": #{meta.inspect}" unless meta.empty?
17
+ format('%s took %.2fms%s', event, (finish - start) * 1000, meta_str)
18
+ end
19
+ end
20
+
21
+ def enable(facility = nil)
22
+ return unless @subid.nil?
23
+ @facility = facility
24
+ @subid = Sqreen::PerformanceNotifications.subscribe(nil,
25
+ &method(:log))
26
+ end
27
+
28
+ def disable
29
+ return if @subid.nil?
30
+ Sqreen::PerformanceNotifications.unsubscribe(@subid)
31
+ @subid = nil
32
+ end
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,36 @@
1
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
+ # Please refer to our terms for more information: https://www.sqreen.io/terms.html
3
+
4
+ require 'sqreen/performance_notifications'
5
+
6
+ module Sqreen
7
+ module PerformanceNotifications
8
+ # Log performances in sqreen metrics_store
9
+ class Metrics
10
+ @subid = nil
11
+ @facility = nil
12
+ class << self
13
+ EVENT_CAT = 'sqreen_time'.freeze
14
+ def log(event, start, finish, _meta)
15
+ evt = [EVENT_CAT, event, (finish - start) * 1000, finish]
16
+ Sqreen.observations_queue.push(evt)
17
+ end
18
+
19
+ def enable(metrics_engine, period = 60)
20
+ return unless @subid.nil?
21
+ metrics_engine.create_metric('name' => EVENT_CAT,
22
+ 'period' => period,
23
+ 'kind' => 'Average')
24
+ @subid = Sqreen::PerformanceNotifications.subscribe(nil,
25
+ &method(:log))
26
+ end
27
+
28
+ def disable
29
+ return if @subid.nil?
30
+ Sqreen::PerformanceNotifications.unsubscribe(@subid)
31
+ @subid = nil
32
+ end
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,36 @@
1
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
+ # Please refer to our terms for more information: https://www.sqreen.io/terms.html
3
+
4
+ require 'sqreen/performance_notifications'
5
+
6
+ module Sqreen
7
+ module PerformanceNotifications
8
+ # Log performances on the console
9
+ class NewRelic
10
+ @subid = nil
11
+
12
+ @nr_name_regexp = %r{/([^/]+)$}
13
+
14
+ class << self
15
+ def log(event, start, finish, _meta)
16
+ event_name = "Custom/Sqreen#{event.sub(@nr_name_regexp, '_\1')}"
17
+ ::NewRelic::Agent.record_metric(event_name, finish - start)
18
+ end
19
+
20
+ def enable
21
+ return unless @subid.nil?
22
+ return unless defined?(::NewRelic::Agent)
23
+ Sqreen.log.debug('Enabling New Relic reporting')
24
+ @subid = Sqreen::PerformanceNotifications.subscribe(nil,
25
+ &method(:log))
26
+ end
27
+
28
+ def disable
29
+ return if @subid.nil?
30
+ Sqreen::PerformanceNotifications.unsubscribe(@subid)
31
+ @subid = nil
32
+ end
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,82 @@
1
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
+ # Please refer to our terms for more information: https://www.sqreen.io/terms.html
3
+
4
+ module Sqreen
5
+ # Execute and sanitize remote commands
6
+ class RemoteCommand
7
+ KNOWN_COMMANDS = {
8
+ :instrumentation_enable => :setup_instrumentation,
9
+ :instrumentation_remove => :remove_instrumentation,
10
+ :rules_reload => :reload_rules,
11
+ :features_get => :features,
12
+ :features_change => :change_features,
13
+ }.freeze
14
+
15
+ attr_reader :uuid
16
+
17
+ def initialize(json_desc)
18
+ @name = json_desc['name'].to_sym
19
+ @params = json_desc.fetch('params', [])
20
+ @uuid = json_desc['uuid']
21
+ end
22
+
23
+ def process(runner)
24
+ failing = validate_command(runner)
25
+ return failing if failing
26
+ Sqreen.log.debug format('processing command %s', @name)
27
+ output = runner.send(KNOWN_COMMANDS[@name], *@params)
28
+ format_output(output)
29
+ end
30
+
31
+ def self.process_list(runner, commands)
32
+ res_list = {}
33
+
34
+ return res_list unless commands
35
+
36
+ unless commands.is_a? Array
37
+ Sqreen.log.debug format('Wrong commands type %s', commands.class)
38
+ Sqreen.log.debug commands.inspect
39
+ return res_list
40
+ end
41
+ commands.each do |cmd_json|
42
+ Sqreen.log.debug cmd_json
43
+ cmd = RemoteCommand.new(cmd_json)
44
+ Sqreen.log.debug cmd.inspect
45
+ uuid = cmd.uuid
46
+ res_list[uuid] = cmd.process(runner)
47
+ end
48
+ res_list
49
+ end
50
+
51
+ def to_h
52
+ {
53
+ :name => @name,
54
+ }
55
+ end
56
+
57
+ protected
58
+
59
+ def validate_command(runner)
60
+ unless KNOWN_COMMANDS.include?(@name)
61
+ msg = format("unknown command name '%s'", @name)
62
+ Sqreen.log.debug msg
63
+ return { :status => false, :reason => msg }
64
+ end
65
+ return nil if runner.respond_to?(KNOWN_COMMANDS[@name])
66
+ msg = format("not implemented '%s'", @name)
67
+ Sqreen.log.debug msg
68
+ { :status => false, :reason => msg }
69
+ end
70
+
71
+ def format_output(output)
72
+ case output
73
+ when NilClass
74
+ return { :status => false, :reason => 'nil returned' }
75
+ when TrueClass
76
+ return { :status => true }
77
+ else
78
+ return { :status => true, :output => output }
79
+ end
80
+ end
81
+ end
82
+ end
@@ -0,0 +1,25 @@
1
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
+ # Please refer to our terms for more information: https://www.sqreen.io/terms.html
3
+
4
+ module Sqreen
5
+ module Rules
6
+ # Common field names in a rule
7
+ module Attrs
8
+ CALLBACKS = 'callbacks'.freeze
9
+ BLOCK = 'block'.freeze
10
+ TEST = 'test'.freeze
11
+ DATA = 'data'.freeze
12
+ PAYLOAD = 'payload'.freeze
13
+ NAME = 'name'.freeze
14
+ RULESPACK_ID = 'rulespack_id'.freeze
15
+ HOOKPOINT = 'hookpoint'.freeze
16
+ KLASS = 'klass'.freeze
17
+ METHOD = 'method'.freeze
18
+ CALLBACK_CLASS = 'callback_class'.freeze
19
+ METRICS = 'metrics'.freeze
20
+ CONDITIONS = 'conditions'.freeze
21
+
22
+ freeze
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,97 @@
1
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
+ # Please refer to our terms for more information: https://www.sqreen.io/terms.html
3
+
4
+ require 'sqreen/callbacks'
5
+ require 'sqreen/conditionable'
6
+ require 'sqreen/rule_attributes'
7
+ require 'sqreen/events/attack'
8
+ require 'sqreen/events/remote_exception'
9
+ require 'sqreen/payload_creator'
10
+
11
+ # Rules defined here can be instanciated from JSON.
12
+ module Sqreen
13
+ module Rules
14
+ # Base class for callback that are initialized by rules from Sqreen
15
+ class RuleCB < CB
16
+ include Conditionable
17
+ # If nothing was asked by the rule we will ask for all sections available
18
+ # These information will be pruned later when exporting in #to_hash
19
+ DEFAULT_PAYLOAD = PayloadCreator::METHODS.keys.freeze
20
+ attr_reader :test
21
+ attr_reader :block
22
+ attr_accessor :framework
23
+
24
+ # @params klass [String] class instrumented
25
+ # @params method [String] method that was instrumented
26
+ # @params rule_hash [Hash] Rule data that govern the current behavior
27
+ def initialize(klass, method, rule_hash)
28
+ super(klass, method)
29
+ @block = rule_hash[Attrs::BLOCK] == true
30
+ @test = rule_hash[Attrs::TEST] == true
31
+ @data = rule_hash[Attrs::DATA]
32
+ @rule = rule_hash
33
+ payload_tpl = @rule[Attrs::PAYLOAD] || DEFAULT_PAYLOAD
34
+ @payload_generator = PayloadCreator.new(payload_tpl)
35
+ conditions = @rule[Attrs::CONDITIONS]
36
+ condition_callbacks(conditions) unless conditions.nil?
37
+ end
38
+
39
+ def rule_name
40
+ @rule[Attrs::NAME]
41
+ end
42
+
43
+ def rulespack_id
44
+ @rule[Attrs::RULESPACK_ID]
45
+ end
46
+
47
+ # Record an attack event into Sqreen system
48
+ # @param infos [Hash] Additional information about request
49
+ def record_event(infos)
50
+ payload = @payload_generator.payload(framework, @rule)
51
+ payload['infos'] = infos
52
+ Attack.record(payload)
53
+ end
54
+
55
+ # Record a metric observation
56
+ # @param category [String] Name of the metric observed
57
+ # @param key [String] aggregation key
58
+ # @param observation [Object] data observed
59
+ # @param at [Time] time when observation was made
60
+ def record_observation(category, key, observation, at = Time.now.utc)
61
+ Sqreen.observations_queue.push [category, key, observation, at]
62
+ if Sqreen.observations_queue.size > MAX_OBS_QUEUE_LENGTH / 2
63
+ Sqreen.queue.push Sqreen::METRICS_EVENT
64
+ end
65
+ end
66
+
67
+ # Record an exception that just occurred
68
+ # @param exception [Exception] Exception to send over
69
+ # @param infos [Hash] Additional contextual information
70
+ def record_exception(exception, infos = {})
71
+ payload = {}
72
+ payload['exception'] = exception
73
+ payload['rulespack_id'] = rulespack_id
74
+ payload['rule_name'] = rule_name
75
+ begin
76
+ payload['request_infos'] = framework.request_infos
77
+ rescue => e
78
+ Sqreen.log.debug("No framework request infos #{e}")
79
+ end
80
+ begin
81
+ payload['request_params'] = framework.request_params
82
+ rescue => e
83
+ Sqreen.log.debug("No framework request params #{e}")
84
+ end
85
+ payload['time'] = Time.now.utc
86
+ payload['infos'] = infos
87
+ payload['backtrace'] = Sqreen::Context.new.bt
88
+ begin
89
+ payload['client_ip'] = framework.client_ip.to_s
90
+ rescue => e
91
+ Sqreen.log.debug("No framework client_ip #{e}")
92
+ end
93
+ RemoteException.record(payload)
94
+ end
95
+ end
96
+ end
97
+ end
@@ -0,0 +1,116 @@
1
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
+ # Please refer to our terms for more information: https://www.sqreen.io/terms.html
3
+
4
+ require 'sqreen/rule_attributes'
5
+ require 'sqreen/rules_callbacks'
6
+
7
+
8
+ ## Rules
9
+ #
10
+ # Rule example:
11
+ # {
12
+ # :class => 'ActionController::Metal',
13
+ # :method => 'dispatch',
14
+ # :arguments => {:type => 'position', :options => {:position => 1}}
15
+ # :callback_class => 'RackCB',
16
+ # }
17
+ # We instrument ActionController::Metal#dispatch. We are interested in the first
18
+ # argument. When this method is called, we will provide it's argument to the
19
+ # callback RackCB.
20
+ #
21
+ # Another option for execution is to delegate the callback to a JS helper,
22
+ # rather than to a class. The JS callback will be executed with the requested
23
+ # arguments.
24
+
25
+ module Sqreen
26
+ # Rules related method/functions
27
+ module Rules
28
+ def self::local(configuration)
29
+ # Parse and return local rules (path defined in environment)
30
+
31
+ path = configuration.get(:local_rules)
32
+ return [] unless path
33
+ begin
34
+ File.open(path) { |f| JSON.load(f) }
35
+ rescue Errno::ENOENT
36
+ Sqreen.log.error "File '#{path}' not found"
37
+ []
38
+ end
39
+ end
40
+
41
+ # Given a rule, will instantiate the related callback.
42
+ # @param hash_rule [Hash] Rules metadata
43
+ # @param metrics_store [MetricStore] Metrics storage facility
44
+ # @param verifier [SqreenSignedVerifier] Signed verifier
45
+ def self::cb_from_rule(hash_rule, metrics_store = nil, verifier = nil)
46
+ # Check rules signature
47
+ verifier.verify(hash_rule) if verifier
48
+
49
+ hook = hash_rule[Attrs::HOOKPOINT]
50
+ klass = hook[Attrs::KLASS]
51
+
52
+ # The instrumented class can be from anywhere
53
+ instr_class = Rules.walk_const_get klass
54
+
55
+ if instr_class.nil?
56
+ rule_name = hash_rule[Attrs::NAME]
57
+ Sqreen.log.debug "#{klass} does not exists. Skipping #{rule_name}"
58
+ return nil
59
+ end
60
+
61
+ instr_method = hook[Attrs::METHOD]
62
+ instr_method = instr_method.to_sym
63
+
64
+ cbname = hook[Attrs::CALLBACK_CLASS]
65
+
66
+ cb_class = nil
67
+ js = hash_rule[Attrs::CALLBACKS]
68
+ cb_class = ExecJSCB if js
69
+
70
+ if cbname && Rules.const_defined?(cbname)
71
+ # Only load callbacks from sqreen
72
+ cb_class = Rules.const_get(cbname)
73
+ end
74
+
75
+ if cb_class.nil?
76
+ Sqreen.log.debug "Cannot setup #{cbname.inspect} [#{rule_name}]"
77
+ return nil
78
+ end
79
+
80
+ unless cb_class.ancestors.include?(RuleCB)
81
+ Sqreen.log.debug "#{cb_class} does not inherit from RuleCB"
82
+ return nil
83
+ end
84
+
85
+ if metrics_store
86
+ (hash_rule[Attrs::METRICS] || []).each do |metric|
87
+ metrics_store.create_metric(metric)
88
+ end
89
+ end
90
+
91
+ cb_class.new(instr_class, instr_method, hash_rule)
92
+ rescue => e
93
+ rule_name = nil
94
+ rulespack_id = nil
95
+ if hash_rule.respond_to?(:[])
96
+ rule_name = hash_rule[Attrs::NAME]
97
+ rulespack_id = hash_rule[Attrs::RULESPACK_ID]
98
+ end
99
+ Sqreen::RemoteException.record(
100
+ 'exception' => e,
101
+ 'rulespack_id' => rulespack_id,
102
+ 'rule_name' => rule_name)
103
+ Sqreen.log.debug("Creating cb from rule #{rule_name} failed (#{e.inspect})")
104
+ nil
105
+ end
106
+
107
+ def self::walk_const_get(str)
108
+ obj = Object
109
+ str.split('::').compact.each do |part|
110
+ return nil unless obj.const_defined?(part)
111
+ obj = obj.const_get(part)
112
+ end
113
+ obj
114
+ end
115
+ end
116
+ end