sqreen-alt 1.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. checksums.yaml +7 -0
  2. data/CODE_OF_CONDUCT.md +22 -0
  3. data/README.md +77 -0
  4. data/Rakefile +20 -0
  5. data/lib/sqreen-alt.rb +1 -0
  6. data/lib/sqreen.rb +68 -0
  7. data/lib/sqreen/attack_detected.html +2 -0
  8. data/lib/sqreen/binding_accessor.rb +288 -0
  9. data/lib/sqreen/ca.crt +72 -0
  10. data/lib/sqreen/call_countable.rb +67 -0
  11. data/lib/sqreen/callback_tree.rb +78 -0
  12. data/lib/sqreen/callbacks.rb +100 -0
  13. data/lib/sqreen/capped_queue.rb +23 -0
  14. data/lib/sqreen/condition_evaluator.rb +235 -0
  15. data/lib/sqreen/conditionable.rb +50 -0
  16. data/lib/sqreen/configuration.rb +168 -0
  17. data/lib/sqreen/context.rb +26 -0
  18. data/lib/sqreen/deliveries/batch.rb +84 -0
  19. data/lib/sqreen/deliveries/simple.rb +39 -0
  20. data/lib/sqreen/event.rb +16 -0
  21. data/lib/sqreen/events/attack.rb +61 -0
  22. data/lib/sqreen/events/remote_exception.rb +54 -0
  23. data/lib/sqreen/events/request_record.rb +62 -0
  24. data/lib/sqreen/exception.rb +34 -0
  25. data/lib/sqreen/frameworks.rb +40 -0
  26. data/lib/sqreen/frameworks/generic.rb +446 -0
  27. data/lib/sqreen/frameworks/rails.rb +148 -0
  28. data/lib/sqreen/frameworks/rails3.rb +36 -0
  29. data/lib/sqreen/frameworks/request_recorder.rb +69 -0
  30. data/lib/sqreen/frameworks/sinatra.rb +57 -0
  31. data/lib/sqreen/frameworks/sqreen_test.rb +26 -0
  32. data/lib/sqreen/instrumentation.rb +542 -0
  33. data/lib/sqreen/log.rb +119 -0
  34. data/lib/sqreen/metrics.rb +6 -0
  35. data/lib/sqreen/metrics/average.rb +39 -0
  36. data/lib/sqreen/metrics/base.rb +45 -0
  37. data/lib/sqreen/metrics/collect.rb +22 -0
  38. data/lib/sqreen/metrics/sum.rb +20 -0
  39. data/lib/sqreen/metrics_store.rb +96 -0
  40. data/lib/sqreen/middleware.rb +34 -0
  41. data/lib/sqreen/payload_creator.rb +137 -0
  42. data/lib/sqreen/performance_notifications.rb +86 -0
  43. data/lib/sqreen/performance_notifications/log.rb +36 -0
  44. data/lib/sqreen/performance_notifications/metrics.rb +36 -0
  45. data/lib/sqreen/performance_notifications/newrelic.rb +36 -0
  46. data/lib/sqreen/remote_command.rb +93 -0
  47. data/lib/sqreen/rule_attributes.rb +26 -0
  48. data/lib/sqreen/rule_callback.rb +108 -0
  49. data/lib/sqreen/rules.rb +126 -0
  50. data/lib/sqreen/rules_callbacks.rb +29 -0
  51. data/lib/sqreen/rules_callbacks/binding_accessor_matcher.rb +77 -0
  52. data/lib/sqreen/rules_callbacks/binding_accessor_metrics.rb +79 -0
  53. data/lib/sqreen/rules_callbacks/blacklist_ips.rb +44 -0
  54. data/lib/sqreen/rules_callbacks/count_http_codes.rb +40 -0
  55. data/lib/sqreen/rules_callbacks/crawler_user_agent_matches.rb +24 -0
  56. data/lib/sqreen/rules_callbacks/crawler_user_agent_matches_metrics.rb +24 -0
  57. data/lib/sqreen/rules_callbacks/custom_error.rb +64 -0
  58. data/lib/sqreen/rules_callbacks/execjs.rb +241 -0
  59. data/lib/sqreen/rules_callbacks/headers_insert.rb +22 -0
  60. data/lib/sqreen/rules_callbacks/inspect_rule.rb +25 -0
  61. data/lib/sqreen/rules_callbacks/matcher_rule.rb +138 -0
  62. data/lib/sqreen/rules_callbacks/rails_parameters.rb +14 -0
  63. data/lib/sqreen/rules_callbacks/record_request_context.rb +39 -0
  64. data/lib/sqreen/rules_callbacks/reflected_xss.rb +254 -0
  65. data/lib/sqreen/rules_callbacks/regexp_rule.rb +36 -0
  66. data/lib/sqreen/rules_callbacks/shell_env.rb +32 -0
  67. data/lib/sqreen/rules_callbacks/url_matches.rb +25 -0
  68. data/lib/sqreen/rules_callbacks/user_agent_matches.rb +22 -0
  69. data/lib/sqreen/rules_signature.rb +151 -0
  70. data/lib/sqreen/runner.rb +365 -0
  71. data/lib/sqreen/runtime_infos.rb +138 -0
  72. data/lib/sqreen/safe_json.rb +60 -0
  73. data/lib/sqreen/sdk.rb +22 -0
  74. data/lib/sqreen/serializer.rb +46 -0
  75. data/lib/sqreen/session.rb +317 -0
  76. data/lib/sqreen/shared_storage.rb +31 -0
  77. data/lib/sqreen/stats.rb +18 -0
  78. data/lib/sqreen/version.rb +5 -0
  79. metadata +148 -0
@@ -0,0 +1,119 @@
1
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
+ # Please refer to our terms for more information: https://www.sqreen.io/terms.html
3
+
4
+ require 'logger'
5
+
6
+ require 'sqreen/performance_notifications/log'
7
+ require 'sqreen/configuration'
8
+
9
+ module Sqreen
10
+ def self::log
11
+ @logger ||= nil
12
+ return @logger unless @logger.nil?
13
+ @logger = Logger.new(
14
+ Sqreen.config_get(:log_level).to_s.upcase,
15
+ Sqreen.config_get(:log_location)
16
+ )
17
+ rescue => e
18
+ warn "Sqreen logger exception: #{e}"
19
+ end
20
+
21
+ # Ruby default formatter modified to display current thread_id
22
+ class FormatterWithTid
23
+ Format = "%s, [%s#%d.%s] %5s -- %s: %s\n".freeze
24
+ DatetimeFormat = '%Y-%m-%dT%H:%M:%S.%6N '.freeze
25
+
26
+ attr_accessor :datetime_format
27
+
28
+ def initialize
29
+ @datetime_format = nil
30
+ end
31
+
32
+ def call(severity, time, progname, msg)
33
+ format(Format,
34
+ severity[0..0], format_datetime(time), $$,
35
+ Thread.current.object_id.to_s(36),
36
+ severity, progname, msg2str(msg)
37
+ )
38
+ end
39
+
40
+ private
41
+
42
+ def format_datetime(time)
43
+ time.strftime(DatetimeFormat)
44
+ end
45
+
46
+ def msg2str(msg)
47
+ case msg
48
+ when ::String
49
+ msg
50
+ when ::Exception
51
+ "#{msg.message} (#{msg.class})\n" << (msg.backtrace || []).join("\n")
52
+ else
53
+ msg.inspect
54
+ end
55
+ end
56
+ end
57
+
58
+ # Wrapper class for sqreen logging
59
+ class Logger
60
+ def initialize(desired_level, log_location, force_logger = nil)
61
+ if force_logger
62
+ @logger = force_logger
63
+ else
64
+ init_logger_output(log_location)
65
+ end
66
+ init_log_level(desired_level)
67
+ enforce_log_format(@logger)
68
+ create_error_logger
69
+ end
70
+
71
+ def debug(msg = nil, &block)
72
+ @logger.debug(msg, &block)
73
+ end
74
+
75
+ def info(msg = nil, &block)
76
+ @logger.info(msg, &block)
77
+ end
78
+
79
+ def warn(msg = nil, &block)
80
+ @logger.warn(msg, &block)
81
+ end
82
+
83
+ def error(msg = nil, &block)
84
+ @error_logger.error(msg, &block)
85
+ @logger.error(msg, &block)
86
+ end
87
+
88
+ protected
89
+
90
+ def init_logger_output(path)
91
+ path = File.expand_path(path)
92
+ if File.writable?(path) || File.writable?(File.dirname(path))
93
+ @logger = ::Logger.new(path)
94
+ else
95
+ @logger = ::Logger.new(STDOUT)
96
+ @logger.info("Cannot access #{path} for writing. Defaulting to stdout")
97
+ end
98
+ rescue => e
99
+ @logger = ::Logger.new(STDOUT)
100
+ @logger.error('Got error while trying to setting logger up, '\
101
+ "falling back to stdout #{e.inspect}")
102
+ end
103
+
104
+ def init_log_level(level)
105
+ log_level = ::Logger.const_get(level)
106
+ @logger.level = log_level
107
+ Sqreen::PerformanceNotifications::Log.enable if level == 'DEBUG'
108
+ end
109
+
110
+ def create_error_logger
111
+ @error_logger = ::Logger.new(STDERR)
112
+ enforce_log_format(@error_logger)
113
+ end
114
+
115
+ def enforce_log_format(logger)
116
+ logger.formatter = FormatterWithTid.new
117
+ end
118
+ end
119
+ end
@@ -0,0 +1,6 @@
1
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
+ # Please refer to our terms for more information: https://www.sqreen.io/terms.html
3
+
4
+ require 'sqreen/metrics/collect'
5
+ require 'sqreen/metrics/average'
6
+ require 'sqreen/metrics/sum'
@@ -0,0 +1,39 @@
1
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
+ # Please refer to our terms for more information: https://www.sqreen.io/terms.html
3
+
4
+ require 'sqreen/metrics/base'
5
+
6
+ module Sqreen
7
+ module Metric
8
+ # This perform an average aggregation
9
+ class Average < Base
10
+ # from class attr_accessor :aggregate
11
+
12
+ def update(_at, key, value)
13
+ super
14
+ @sums[key] ||= 0
15
+ @sums[key] += value
16
+ @counts[key] ||= 0
17
+ @counts[key] += 1
18
+ end
19
+
20
+ protected
21
+
22
+ def new_sample(time)
23
+ super(time)
24
+ @sums = {}
25
+ @counts = {}
26
+ end
27
+
28
+ def finalize_sample(time)
29
+ super(time)
30
+ @sample[FINISH_KEY] = time
31
+ h = {}
32
+ @sums.each do |k, v|
33
+ h[k] = v.to_f / @counts[k]
34
+ end
35
+ @sample[OBSERVATION_KEY] = h
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,45 @@
1
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
+ # Please refer to our terms for more information: https://www.sqreen.io/terms.html
3
+
4
+ require 'sqreen/exception'
5
+
6
+ module Sqreen
7
+ module Metric
8
+ OBSERVATION_KEY = 'observation'.freeze
9
+ START_KEY = 'start'.freeze
10
+ FINISH_KEY = 'finish'.freeze
11
+ # Base interface for a metric
12
+ class Base
13
+ def initialize
14
+ @sample = nil
15
+ end
16
+
17
+ # Update the current metric with a new observation
18
+ # @param _at [Time] when was the observation made
19
+ # @param _key [String] which aggregation key was it made for
20
+ # @param _value [Object] The observation
21
+ def update(_at, _key, _value)
22
+ raise Sqreen::Exception, 'No current sample' unless @sample
23
+ end
24
+
25
+ # create a new empty sample and publish the last one
26
+ # @param time [Time] Time of start/finish
27
+ def next_sample(time)
28
+ finalize_sample(time) unless @sample.nil?
29
+ current_sample = @sample
30
+ new_sample(time)
31
+ current_sample
32
+ end
33
+
34
+ protected
35
+
36
+ def new_sample(time)
37
+ @sample = { OBSERVATION_KEY => {}, START_KEY => time }
38
+ end
39
+
40
+ def finalize_sample(time)
41
+ @sample[FINISH_KEY] = time
42
+ end
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,22 @@
1
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
+ # Please refer to our terms for more information: https://www.sqreen.io/terms.html
3
+
4
+ require 'sqreen/metrics/base'
5
+
6
+ module Sqreen
7
+ module Metric
8
+ # This is an aggregated statistic definition
9
+ # This is a base class to collect metrics in a hash based structure
10
+ # that does not aggregate anything
11
+ class Collect < Base
12
+ # from class attr_accessor :aggregate
13
+
14
+ def update(_at, key, value)
15
+ super
16
+ s = @sample[OBSERVATION_KEY]
17
+ s[key] ||= []
18
+ s[key] << value
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,20 @@
1
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
+ # Please refer to our terms for more information: https://www.sqreen.io/terms.html
3
+
4
+ require 'sqreen/metrics/base'
5
+
6
+ module Sqreen
7
+ module Metric
8
+ # This perform a sum aggregation
9
+ class Sum < Base
10
+ # from class attr_accessor :aggregate
11
+
12
+ def update(_at, key, value)
13
+ super
14
+ s = @sample[OBSERVATION_KEY]
15
+ s[key] ||= 0
16
+ s[key] += value
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,96 @@
1
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
+ # Please refer to our terms for more information: https://www.sqreen.io/terms.html
3
+
4
+ require 'sqreen/exception'
5
+ require 'sqreen/metrics'
6
+
7
+ module Sqreen
8
+ # This store and register metrics
9
+ class MetricsStore
10
+ # When a metric is not yet created
11
+ class UnregisteredMetric < Sqreen::Exception
12
+ end
13
+ # When the metric is unknown
14
+ class UnknownMetric < Sqreen::Exception
15
+ end
16
+ # When this name as already been declared with another kind
17
+ class AlreadyRegisteredMetric < Sqreen::Exception
18
+ end
19
+
20
+ NAME_KEY = 'name'.freeze
21
+ KIND_KEY = 'kind'.freeze
22
+ PERIOD_KEY = 'period'.freeze
23
+
24
+ # Currently ready samples
25
+ attr_reader :store
26
+ # All known metrics
27
+ attr_reader :metrics
28
+
29
+ def initialize
30
+ @store = []
31
+ @metrics = {}
32
+ end
33
+
34
+ # Definition contains a name,period and aggregate at least
35
+ # @param definition [Hash] a metric definition
36
+ # @param klass [Object] Override metric class (used in testing)
37
+ def create_metric(definition, mklass = nil)
38
+ name = definition[NAME_KEY]
39
+ kind = definition[KIND_KEY]
40
+ klass = valid_metric(kind, name)
41
+ metric = mklass || klass.new
42
+ @metrics[name] = [
43
+ metric,
44
+ definition[PERIOD_KEY],
45
+ nil # Start
46
+ ]
47
+ metric
48
+ end
49
+
50
+ def update(name, at, key, value)
51
+ at = at.utc
52
+ metric, period, start = @metrics[name]
53
+ raise UnregisteredMetric, "Unknown metric #{name}" unless metric
54
+ next_sample(name, at) if start.nil? || (start + period) < at
55
+ metric.update(at, key, value)
56
+ end
57
+
58
+ # Drains every metrics and returns the store content
59
+ # @params at [Time] when is the store emptied
60
+ def publish(flush = true, at = Time.now.utc)
61
+ @metrics.each do |name, (_, period, start)|
62
+ next_sample(name, at) if flush || !start.nil? && (start + period) < at
63
+ end
64
+ out = @store
65
+ @store = []
66
+ out
67
+ end
68
+
69
+ protected
70
+
71
+ def next_sample(name, at)
72
+ metric = @metrics[name][0]
73
+ r = metric.next_sample(at)
74
+ @metrics[name][2] = at
75
+ if r
76
+ r[NAME_KEY] = name
77
+ obs = r[Metric::OBSERVATION_KEY]
78
+ @store << r if obs && (!obs.respond_to?(:empty?) || !obs.empty?)
79
+ end
80
+ r
81
+ end
82
+
83
+ def valid_metric(kind, name)
84
+ unless Sqreen::Metric.const_defined?(kind)
85
+ raise UnknownMetric, "No such #{kind} metric"
86
+ end
87
+ klass = Sqreen::Metric.const_get(kind)
88
+ metric = @metrics[name] && @metrics[name][0]
89
+ if metric && metric.class != klass
90
+ msg = "Was a #{metric.class.name} not a #{klass.name} "
91
+ raise AlreadyRegisteredMetric, msg
92
+ end
93
+ klass
94
+ end
95
+ end
96
+ end
@@ -0,0 +1,34 @@
1
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
+ # Please refer to our terms for more information: https://www.sqreen.io/terms.html
3
+
4
+ module Sqreen
5
+ class Middleware
6
+ def initialize(app)
7
+ @app = app
8
+ end
9
+
10
+ def call(env)
11
+ @app.call(env)
12
+ end
13
+ end
14
+
15
+ class ErrorHandlingMiddleware
16
+ def initialize(app)
17
+ @app = app
18
+ end
19
+
20
+ def call(env)
21
+ @app.call(env)
22
+ end
23
+ end
24
+
25
+ class RailsMiddleware
26
+ def initialize(app)
27
+ @app = app
28
+ end
29
+
30
+ def call(env)
31
+ @app.call(env)
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,137 @@
1
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
+ # Please refer to our terms for more information: https://www.sqreen.io/terms.html
3
+
4
+ require 'sqreen/runtime_infos'
5
+ require 'sqreen/events/remote_exception'
6
+
7
+ module Sqreen
8
+ # Create a payload from a given query
9
+ #
10
+ # Template elements are made of sections and subsections.
11
+ # This class is able to send the full content of section or
12
+ # only the required subsections as needed.
13
+ #
14
+ # The payload will always be outputed as a
15
+ # Hash of section => subsection.
16
+ class PayloadCreator
17
+ attr_reader :framework
18
+ def initialize(framework)
19
+ @framework = framework
20
+ end
21
+
22
+ def query=(keys)
23
+ @sections = {}
24
+ keys.each do |key|
25
+ section, subsection = key.split('.', 2)
26
+ @sections[section] = true if subsection.nil?
27
+ next if @sections[section] == true
28
+ @sections[section] ||= []
29
+ @sections[section].push(subsection)
30
+ end
31
+ end
32
+
33
+ def payload(query)
34
+ self.query = query
35
+ ret = {}
36
+ METHODS.each_key do |section|
37
+ ret = fill(section, ret, @framework)
38
+ end
39
+ ret
40
+ end
41
+
42
+ protected
43
+
44
+ def fill(key, base, framework)
45
+ subsection = @sections[key]
46
+ return base if subsection.nil?
47
+ if subsection == true
48
+ return base.merge!(key => full_section(key, framework))
49
+ end
50
+ return base if subsection.empty?
51
+ base[key] = fields(key, framework)
52
+ base
53
+ end
54
+
55
+ FULL_SECTIONS = {
56
+ 'request' => 'request_infos',
57
+ 'params' => 'filtered_request_params',
58
+ 'headers' => 'ip_headers',
59
+ 'local' => 'local_infos',
60
+ }.freeze
61
+
62
+ METHODS = {
63
+ 'request' => {
64
+ 'addr' => 'client_ip',
65
+ 'rid' => 'request_id',
66
+ },
67
+ 'local' => {
68
+ 'name' => 'hostname',
69
+ },
70
+ 'params' => {
71
+ 'form' => 'form_params',
72
+ 'query' => 'query_params',
73
+ 'cookies' => 'cookies_params',
74
+ 'rails' => 'rails_params',
75
+ },
76
+ 'headers' => {},
77
+ }.freeze
78
+
79
+ def section_object(section, framework)
80
+ return RuntimeInfos if section == 'local'
81
+ return HeaderSection.new(framework) if section == 'headers'
82
+ framework
83
+ end
84
+
85
+ def full_section(section, framework)
86
+ # fast path prevent initializing a HeaderSection
87
+ return framework.ip_headers if section == 'headers'
88
+ so = section_object(section, framework)
89
+ so.send(FULL_SECTIONS[section])
90
+ end
91
+
92
+ def fields(section, framework)
93
+ out = {}
94
+ object = section_object(section, framework)
95
+ remove = []
96
+ @sections[section].each do |key|
97
+ meth = METHODS[section][key]
98
+ invoke(out, key, object, meth || key, remove)
99
+ end
100
+ remove.each { |k| @sections[section].delete(k) }
101
+ Hash[out]
102
+ end
103
+
104
+ def invoke(out, key, object, method, remove)
105
+ out[key] = if object.respond_to?(:[])
106
+ object[method]
107
+ else
108
+ object.send(method)
109
+ end
110
+ rescue NoMethodError => e
111
+ remove.push(key)
112
+ Sqreen::RemoteException.record(e)
113
+ end
114
+
115
+ # object that default to call on framework header
116
+ class HeaderSection
117
+ def initialize(framework)
118
+ @framework = framework
119
+ end
120
+
121
+ def [](value)
122
+ if %w[rack_client_ip rails_client_ip ip_headers].include?(value)
123
+ return @framework.send(value)
124
+ end
125
+ @framework.header(value)
126
+ end
127
+
128
+ def ip_headers
129
+ @framework.ip_headers
130
+ end
131
+ end
132
+
133
+ def section_headers(framework)
134
+ HeaderSection.new(framework)
135
+ end
136
+ end
137
+ end