sqreen 1.19.1-java → 1.21.0.beta3-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +34 -0
  3. data/lib/sqreen/actions/block_user.rb +1 -1
  4. data/lib/sqreen/actions/redirect_ip.rb +1 -1
  5. data/lib/sqreen/actions/redirect_user.rb +1 -1
  6. data/lib/sqreen/agent_message.rb +20 -0
  7. data/lib/sqreen/aggregated_metric.rb +25 -0
  8. data/lib/sqreen/attack_detected.html +1 -2
  9. data/lib/sqreen/ca.crt +24 -0
  10. data/lib/sqreen/condition_evaluator.rb +9 -2
  11. data/lib/sqreen/conditionable.rb +24 -6
  12. data/lib/sqreen/configuration.rb +11 -5
  13. data/lib/sqreen/deferred_logger.rb +50 -14
  14. data/lib/sqreen/deliveries/batch.rb +12 -2
  15. data/lib/sqreen/deliveries/simple.rb +4 -0
  16. data/lib/sqreen/deprecation.rb +38 -0
  17. data/lib/sqreen/ecosystem.rb +96 -0
  18. data/lib/sqreen/ecosystem/dispatch_table.rb +43 -0
  19. data/lib/sqreen/ecosystem/exception_reporting.rb +26 -0
  20. data/lib/sqreen/ecosystem/http/net_http.rb +50 -0
  21. data/lib/sqreen/ecosystem/http/rack_request.rb +39 -0
  22. data/lib/sqreen/ecosystem/loggable.rb +13 -0
  23. data/lib/sqreen/ecosystem/module_api.rb +30 -0
  24. data/lib/sqreen/ecosystem/module_api/event_listener.rb +18 -0
  25. data/lib/sqreen/ecosystem/module_api/instrumentation.rb +23 -0
  26. data/lib/sqreen/ecosystem/module_api/message_producer.rb +51 -0
  27. data/lib/sqreen/ecosystem/module_api/signal_producer.rb +24 -0
  28. data/lib/sqreen/ecosystem/module_api/tracing.rb +45 -0
  29. data/lib/sqreen/ecosystem/module_api/tracing/client_data.rb +31 -0
  30. data/lib/sqreen/ecosystem/module_api/tracing/server_data.rb +27 -0
  31. data/lib/sqreen/ecosystem/module_api/tracing_id_generation.rb +16 -0
  32. data/lib/sqreen/ecosystem/module_api/transaction_storage.rb +71 -0
  33. data/lib/sqreen/ecosystem/module_registry.rb +44 -0
  34. data/lib/sqreen/ecosystem/redis/redis_connection.rb +43 -0
  35. data/lib/sqreen/ecosystem/tracing/modules/client.rb +31 -0
  36. data/lib/sqreen/ecosystem/tracing/modules/server.rb +30 -0
  37. data/lib/sqreen/ecosystem/tracing/sampler.rb +160 -0
  38. data/lib/sqreen/ecosystem/tracing/sampling_configuration.rb +150 -0
  39. data/lib/sqreen/ecosystem/tracing/signals/tracing_client.rb +53 -0
  40. data/lib/sqreen/ecosystem/tracing/signals/tracing_server.rb +53 -0
  41. data/lib/sqreen/ecosystem/tracing_broker.rb +101 -0
  42. data/lib/sqreen/ecosystem/tracing_id_setup.rb +34 -0
  43. data/lib/sqreen/ecosystem/transaction_storage.rb +64 -0
  44. data/lib/sqreen/ecosystem/util/call_writers_from_init.rb +13 -0
  45. data/lib/sqreen/ecosystem_integration.rb +87 -0
  46. data/lib/sqreen/ecosystem_integration/around_callbacks.rb +99 -0
  47. data/lib/sqreen/ecosystem_integration/instrumentation_service.rb +42 -0
  48. data/lib/sqreen/ecosystem_integration/request_lifecycle_tracking.rb +58 -0
  49. data/lib/sqreen/ecosystem_integration/signal_consumption.rb +35 -0
  50. data/lib/sqreen/endpoint_testing.rb +184 -0
  51. data/lib/sqreen/event.rb +7 -5
  52. data/lib/sqreen/events/attack.rb +23 -18
  53. data/lib/sqreen/events/remote_exception.rb +0 -22
  54. data/lib/sqreen/events/request_record.rb +15 -71
  55. data/lib/sqreen/frameworks/generic.rb +24 -1
  56. data/lib/sqreen/frameworks/rails.rb +0 -7
  57. data/lib/sqreen/frameworks/request_recorder.rb +15 -2
  58. data/lib/sqreen/graft/call.rb +106 -19
  59. data/lib/sqreen/graft/callback.rb +1 -1
  60. data/lib/sqreen/graft/hook.rb +212 -100
  61. data/lib/sqreen/graft/hook_point.rb +18 -11
  62. data/lib/sqreen/kit/signals/specialized/aggregated_metric.rb +72 -0
  63. data/lib/sqreen/kit/signals/specialized/attack.rb +57 -0
  64. data/lib/sqreen/kit/signals/specialized/binning_metric.rb +76 -0
  65. data/lib/sqreen/kit/signals/specialized/http_trace.rb +26 -0
  66. data/lib/sqreen/kit/signals/specialized/sdk_track_call.rb +50 -0
  67. data/lib/sqreen/kit/signals/specialized/sqreen_exception.rb +57 -0
  68. data/lib/sqreen/legacy/instrumentation.rb +22 -10
  69. data/lib/sqreen/legacy/old_event_submission_strategy.rb +228 -0
  70. data/lib/sqreen/legacy/waf_redactions.rb +49 -0
  71. data/lib/sqreen/log.rb +3 -2
  72. data/lib/sqreen/log/loggable.rb +2 -1
  73. data/lib/sqreen/logger.rb +24 -0
  74. data/lib/sqreen/metrics.rb +1 -0
  75. data/lib/sqreen/metrics/base.rb +3 -0
  76. data/lib/sqreen/metrics/req_detailed.rb +41 -0
  77. data/lib/sqreen/metrics_store.rb +33 -12
  78. data/lib/sqreen/null_logger.rb +22 -0
  79. data/lib/sqreen/performance_notifications/binned_metrics.rb +8 -2
  80. data/lib/sqreen/remote_command.rb +4 -0
  81. data/lib/sqreen/rules.rb +12 -6
  82. data/lib/sqreen/rules/blacklist_ips_cb.rb +2 -2
  83. data/lib/sqreen/rules/custom_error_cb.rb +3 -3
  84. data/lib/sqreen/rules/not_found_cb.rb +2 -0
  85. data/lib/sqreen/rules/rule_cb.rb +6 -2
  86. data/lib/sqreen/rules/waf_cb.rb +16 -13
  87. data/lib/sqreen/runner.rb +138 -16
  88. data/lib/sqreen/sensitive_data_redactor.rb +19 -31
  89. data/lib/sqreen/session.rb +53 -43
  90. data/lib/sqreen/signals/conversions.rb +288 -0
  91. data/lib/sqreen/signals/http_trace_redaction.rb +111 -0
  92. data/lib/sqreen/signals/signals_submission_strategy.rb +78 -0
  93. data/lib/sqreen/version.rb +1 -1
  94. data/lib/sqreen/weave/budget.rb +35 -0
  95. data/lib/sqreen/weave/legacy/instrumentation.rb +277 -135
  96. data/lib/sqreen/worker.rb +6 -2
  97. metadata +86 -10
  98. data/lib/sqreen/backport.rb +0 -9
  99. data/lib/sqreen/backport/clock_gettime.rb +0 -74
  100. data/lib/sqreen/backport/original_name.rb +0 -88
  101. data/lib/sqreen/encoding_sanitizer.rb +0 -27
@@ -0,0 +1,111 @@
1
+ require 'json'
2
+ require 'sqreen/kit/loggable'
3
+ require 'sqreen/kit/signals/specialized/http_trace'
4
+
5
+ module Sqreen
6
+ module Signals
7
+ module HttpTraceRedaction
8
+ class << self
9
+ include Sqreen::Kit::Loggable
10
+
11
+ # @param [Sqreen::Kit::Signals::Specialized::HttpTrace] trace
12
+ # @param [Sqreen::SensitiveDataRedactor] redactor
13
+ def redact_trace!(trace, redactor)
14
+ return unless redactor
15
+ # redact headers (keys unsafe)
16
+ # @type [Sqreen::Kit::Signals::Context::HttpContext]
17
+ http_context = trace.context
18
+
19
+ all_redacted = []
20
+
21
+ # Redact headers; save redacted values
22
+ # headers are encoded as [key, value], not a hash, so
23
+ # they require some transformation
24
+ orig_headers = http_context.headers
25
+ if orig_headers
26
+ headers = orig_headers.map { |(k, v)| { k => v } }
27
+ headers, redacted = redactor.redact(headers)
28
+ http_context.headers = headers.map(&:first)
29
+ all_redacted += redacted
30
+ end
31
+
32
+ # Redact params; save redacted values
33
+ Kit::Signals::Context::HttpContext::PARAMS_ATTRS.each do |attr|
34
+ value = http_context.public_send(attr)
35
+ next unless value
36
+ value, redacted = redactor.redact(value)
37
+ all_redacted += redacted
38
+ http_context.public_send(:"#{attr}=", value)
39
+ end
40
+
41
+ all_redacted = all_redacted.uniq.map(&:downcase)
42
+
43
+ # Redact attacks and exceptions
44
+ # XXX: no redaction for infos in attacks/exceptions except for WAF data
45
+ # Is this the correct behavior?
46
+ redact_attacks!(trace, redactor, all_redacted)
47
+ redact_exceptions!(trace, redactor, all_redacted)
48
+ end
49
+
50
+ private
51
+
52
+ # @param [Sqreen::Kit::Signals::Specialized::HttpTrace] trace
53
+ # @param [Sqreen::SensitiveDataRedactor] redactor
54
+ # Redacts WAF data according to specific rules therefor
55
+ # Redacts infos according to general rules
56
+ def redact_attacks!(trace, redactor, redacted_data)
57
+ trace.data.each do |signal|
58
+ next unless signal.is_a?(Kit::Signals::Specialized::Attack)
59
+ # @type [Sqreen::Kit::Signals::Specialized::Attack::Payload] payload
60
+ payload = signal.payload
61
+ next unless payload.infos
62
+
63
+ if payload.infos[:waf_data]
64
+ redact_waf_attack_data!(payload.infos, redacted_data)
65
+ end
66
+ payload.infos, = redactor.redact(payload.infos)
67
+ end
68
+ end
69
+
70
+ def redact_exceptions!(trace, redactor, redacted_data)
71
+ trace.data.each do |signal|
72
+ next unless signal.is_a?(Kit::Signals::Specialized::SqreenException)
73
+ infos = signal.infos
74
+ next unless infos
75
+
76
+ redact_waf_exception_data!(signal.infos, redacted_data) if signal.infos[:waf]
77
+ signal.infos, = redactor.redact(infos)
78
+ end
79
+ end
80
+
81
+ # @param [Hash] infos from WAF attack
82
+ def redact_waf_attack_data!(infos, redacted_data)
83
+ begin
84
+ parsed = JSON.parse(infos[:waf_data])
85
+ rescue JSON::JSONError => e
86
+ logger.warn("waf_data is not valid json: #{e.message}")
87
+ return
88
+ end
89
+ redacted = parsed.each do |w|
90
+ next unless (filters = w['filter'])
91
+
92
+ filters.each do |f|
93
+ next unless (v = f['resolved_value'])
94
+ next unless redacted_data.include?(v.downcase)
95
+
96
+ f['match_status'] = SensitiveDataRedactor::MASK
97
+ f['resolved_value'] = SensitiveDataRedactor::MASK
98
+ end
99
+ end
100
+ infos[:waf_data] = JSON.dump(redacted)
101
+ end
102
+
103
+ # see https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000022-waf-data-sanitization.md#changes-to-the-agents
104
+ def redact_waf_exception_data!(infos, redacted_data)
105
+ return if redacted_data.empty?
106
+ infos[:waf].delete(:args)
107
+ end
108
+ end
109
+ end
110
+ end
111
+ end
@@ -0,0 +1,78 @@
1
+ require 'sqreen/aggregated_metric'
2
+ require 'sqreen/kit'
3
+ require 'sqreen/kit/string_sanitizer'
4
+ require 'sqreen/signals/conversions'
5
+ require 'sqreen/log/loggable'
6
+
7
+ module Sqreen
8
+ module Signals
9
+ # see also Sqreen::Legacy::OldEventSubmissionStrategy
10
+ # usage in Sqreen:Session
11
+ class SignalsSubmissionStrategy
12
+ include Sqreen::Log::Loggable
13
+
14
+ # @param [Array<Sqreen::AggregatedMetric>] metrics
15
+ def post_metrics(metrics)
16
+ return if metrics.nil? || metrics.empty?
17
+
18
+ guarded 'Failed to serialize or submit aggregated metrics' do
19
+ batch = metrics.map do |m|
20
+ Conversions.convert_metric_sample(m)
21
+ end
22
+ client.report_batch(batch)
23
+ end
24
+ end
25
+
26
+ # @param _attack [Sqreen::Attack]
27
+ # XXX: unused
28
+ def post_attack(_attack)
29
+ raise NotImplementedError
30
+ end
31
+
32
+ # @param request_record [Sqreen::RequestRecord]
33
+ def post_request_record(request_record)
34
+ guarded 'Failed to serialize or submit request record' do
35
+ trace = Conversions.convert_req_record(request_record)
36
+ append_sanitizing_filter(trace)
37
+ client.report_trace(trace)
38
+ end
39
+ end
40
+
41
+ # Post an exception to Sqreen for analysis
42
+ # @param exception [RemoteException] Exception and context to be sent over
43
+ def post_sqreen_exception(exception)
44
+ guarded 'Failed to serialize or submit exception', false do
45
+ data = Conversions.convert_exception(exception)
46
+ append_sanitizing_filter(data)
47
+ client.report_signal(data)
48
+ end
49
+ end
50
+
51
+ def post_batch(events)
52
+ guarded 'Failed to serialize or submit batch of events' do
53
+ batch = Conversions.convert_batch(events)
54
+ batch.each { |sig_or_trace| append_sanitizing_filter(sig_or_trace) }
55
+ client.report_batch(batch)
56
+ end
57
+ end
58
+
59
+ private
60
+
61
+ def append_sanitizing_filter(sig_or_trace)
62
+ sig_or_trace.append_to_h_filter Kit::StringSanitizer.method(:sanitize)
63
+ end
64
+
65
+ # we don't want exceptions to propagate and kill the worker thread
66
+ def guarded(msg, report = true)
67
+ yield
68
+ rescue StandardError => e
69
+ logger.warn "#{msg}: #{e.message}\n#{e.backtrace.map { |x| " #{x}" }.join("\n")}"
70
+ post_sqreen_exception(RemoteException.new(e)) if report
71
+ end
72
+
73
+ def client
74
+ Sqreen::Kit.auth_signals_client
75
+ end
76
+ end
77
+ end
78
+ end
@@ -4,5 +4,5 @@
4
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
5
5
 
6
6
  module Sqreen
7
- VERSION = '1.19.1'.freeze
7
+ VERSION = '1.21.0.beta3'.freeze
8
8
  end
@@ -0,0 +1,35 @@
1
+ # typed: false
2
+
3
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
4
+ # Please refer to our terms for more information: https://www.sqreen.com/terms.html
5
+
6
+ require 'sqreen/log/loggable'
7
+ require 'sqreen/weave'
8
+
9
+ class Sqreen::Weave::Budget
10
+ include Sqreen::Log::Loggable
11
+
12
+ def initialize(threshold)
13
+ @threshold = threshold
14
+ end
15
+
16
+ attr_reader :threshold
17
+
18
+ def to_h
19
+ { threshold: threshold }
20
+ end
21
+
22
+ class << self
23
+ attr_reader :current
24
+
25
+ def update(opts = nil)
26
+ Sqreen::Weave.logger.info("budget update:#{opts.inspect}")
27
+
28
+ return @current = nil if opts.nil? || opts.empty?
29
+
30
+ threshold = opts[:threshold]
31
+
32
+ @current = threshold
33
+ end
34
+ end
35
+ end
@@ -4,23 +4,41 @@
4
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
5
5
 
6
6
  require 'sqreen/weave/legacy'
7
+ require 'sqreen/weave/budget'
8
+ require 'sqreen/graft/hook'
7
9
  require 'sqreen/graft/hook_point'
8
10
  require 'sqreen/call_countable'
9
11
  require 'sqreen/rules'
10
12
  require 'sqreen/rules/record_request_context'
13
+ require 'sqreen/sqreen_signed_verifier'
14
+ require 'rack/request'
15
+ begin
16
+ require 'sq_detailed_metrics'
17
+ rescue LoadError => _e # rubocop:disable Lint/HandleExceptions
18
+ end
11
19
 
12
20
  class Sqreen::Weave::Legacy::Instrumentation
13
21
  attr_accessor :metrics_engine
14
22
 
23
+ HAS_SQ_DETAILED_METRICS = defined?(::SqDetailedMetrics)
24
+ REQ_LVL_2_METRIC = 'request_level_perf'.freeze
25
+
15
26
  def initialize(metrics_engine, opts = {})
16
27
  Sqreen::Weave.logger.debug { "#{self.class.name}#initialize #{metrics_engine}" }
17
28
  @hooks = []
18
29
 
30
+ unless HAS_SQ_DETAILED_METRICS
31
+ Sqreen::Weave.logger.warn { "Detailed metrics are unavailable" }
32
+ end
33
+
19
34
  self.metrics_engine = metrics_engine
20
35
 
21
36
  ### bail out if no metric engine
22
37
  return if metrics_engine.nil?
23
38
 
39
+ # XXX: these metric definitions do not support change of opts
40
+ # due to features updates!
41
+
24
42
  ### init metric to count calls to sqreen
25
43
  metrics_engine.create_metric(
26
44
  'name' => 'sqreen_call_counts',
@@ -60,12 +78,42 @@ class Sqreen::Weave::Legacy::Instrumentation
60
78
  'options' => opts[:perf_metric_percent] || { 'base' => 1.3, 'factor' => 1.0 },
61
79
  )
62
80
 
81
+ metrics_engine.create_metric(
82
+ 'name' => 'req.sq.hook.overhead',
83
+ 'period' => 60,
84
+ 'kind' => 'Binning',
85
+ 'options' => { 'base' => 2.0, 'factor' => 0.1 },
86
+ )
87
+
88
+ metrics_engine.create_metric(
89
+ 'name' => 'sq.hook.overhead',
90
+ 'period' => 60,
91
+ 'kind' => 'Binning',
92
+ 'options' => { 'base' => 2.0, 'factor' => 0.1 },
93
+ )
94
+
95
+ metrics_engine.create_metric(
96
+ 'name' => 'sq.shrinkwrap',
97
+ 'period' => 60,
98
+ 'kind' => 'Binning',
99
+ 'options' => { 'base' => 2.0, 'factor' => 0.1 },
100
+ )
101
+
63
102
  Sqreen.thread_cpu_time? && metrics_engine.create_metric(
64
103
  'name' => 'sq_thread_cpu_pct',
65
104
  'period' => opts[:period] || 60,
66
105
  'kind' => 'Binning',
67
106
  'options' => opts[:perf_metric_percent] || { 'base' => 1.3, 'factor' => 1.0 },
68
107
  )
108
+
109
+ if HAS_SQ_DETAILED_METRICS # rubocop:disable Style/GuardClause
110
+ @lvl_2_metric = metrics_engine.create_metric(
111
+ 'name' => REQ_LVL_2_METRIC,
112
+ 'period' => opts[:perf_req_metrics_period] || 60,
113
+ 'kind' => 'ReqDetailed',
114
+ )
115
+ @lvl_2_max_reqs = opts[:perf_req_metrics_max_reqs] || 100
116
+ end
69
117
  end
70
118
 
71
119
  # needed by Sqreen::Runner#initialize
@@ -84,6 +132,15 @@ class Sqreen::Weave::Legacy::Instrumentation
84
132
 
85
133
  ### set up rule signature verifier
86
134
  verifier = nil
135
+ if Sqreen.features['rules_signature'] &&
136
+ Sqreen.config_get(:rules_verify_signature) == true &&
137
+ !defined?(::JRUBY_VERSION)
138
+ verifier = Sqreen::SqreenSignedVerifier.new
139
+ Sqreen::Weave.logger.debug('Rules signature enabled')
140
+ else
141
+ Sqreen::Weave.logger.debug('Rules signature disabled')
142
+ end
143
+
87
144
  ### force clean instrumentation callback list
88
145
  @hooks = []
89
146
  ### for each rule description
@@ -94,6 +151,25 @@ class Sqreen::Weave::Legacy::Instrumentation
94
151
  next unless rule_callback
95
152
  ### attach framework to callback
96
153
  rule_callback.framework = framework
154
+ ## create metric
155
+ Sqreen::Weave.logger.debug { "Adding rule metric: #{rule_callback}" }
156
+ [:pre, :post, :failing].each do |whence|
157
+ next unless rule_callback.send(:"#{whence}?")
158
+ metric_name = "sq.#{rule['name']}.#{whence}"
159
+ metrics_engine.create_metric(
160
+ 'name' => metric_name,
161
+ 'period' => 60,
162
+ 'kind' => 'Binning',
163
+ 'options' => { 'base' => 2.0, 'factor' => 0.1 },
164
+ )
165
+ metric_name = "req.sq.#{rule['name']}.#{whence}"
166
+ metrics_engine.create_metric(
167
+ 'name' => metric_name,
168
+ 'period' => 60,
169
+ 'kind' => 'Binning',
170
+ 'options' => { 'base' => 2.0, 'factor' => 0.1 },
171
+ )
172
+ end
97
173
  ### install callback, observing priority
98
174
  Sqreen::Weave.logger.debug { "Adding rule callback: #{rule_callback}" }
99
175
  @hooks << add_callback("weave,rule=#{rule['name']}", rule_callback, strategy)
@@ -107,30 +183,62 @@ class Sqreen::Weave::Legacy::Instrumentation
107
183
  end
108
184
 
109
185
  metrics_engine = self.metrics_engine
186
+ lvl_2_metric = @lvl_2_metric
187
+ lvl_2_max_reqs = @lvl_2_max_reqs
188
+
110
189
  request_hook = Sqreen::Graft::Hook['Sqreen::ShrinkWrap#call', strategy]
111
190
  @hooks << request_hook
112
191
  request_hook.add do
113
- before('wave,meta,request', rank: -100000, mandatory: true) do |_call|
192
+ before('wave,meta,request', rank: -100000, mandatory: true) do |call|
114
193
  next unless Sqreen.instrumentation_ready
115
194
 
116
- uuid = SecureRandom.uuid
117
- now = Sqreen::Graft::Timer.read
195
+ # shrinkwrap_timer = Sqreen::Graft::Timer.new('weave,shrinkwrap')
196
+ # shrinkwrap_timer.start
197
+
198
+ request_timer = Sqreen::Graft::Timer.new("request")
199
+ request_timer.start
200
+ sqreen_timer = Sqreen::Graft::Timer.new("sqreen")
201
+ budget = Sqreen::Weave::Budget.current
202
+
203
+ timed_level = (Sqreen.features['perf_level'] || 1).to_i
204
+ timed_level = 1 if !HAS_SQ_DETAILED_METRICS && timed_level == 2
205
+ if timed_level == 2 && lvl_2_metric.num_requests >= lvl_2_max_reqs
206
+ timed_level = 1
207
+ Sqreen::Weave.logger.debug { "Reducing timed level to 1 (#{lvl_2_metric.num_requests} reqs accumulated)" }
208
+ end
209
+
210
+ Sqreen::Weave.logger.debug { "request budget: #{budget} timed.level: #{timed_level}" } if Sqreen::Weave.logger.debug?
211
+
212
+ route_found = nil
213
+ if timed_level >= 2
214
+ rack_env, = call.args
215
+ rack_request = Rack::Request.new(rack_env) if rack_env
216
+
217
+ # TODO: Rails engines
218
+ # TODO: Struct
219
+ # TODO: Sinatra
220
+ # TODO: Rack?
221
+ Rails.application.routes.router.recognize(rack_request) do |route, params|
222
+ route = ActionDispatch::Routing::RouteWrapper.new(route)
223
+ route_found = { name: route.name, verb: route.verb, path: route.path, reqs: route.reqs, params: params }
224
+ end if defined?(Rails) && Rails.application && defined?(ActionDispatch::Routing::RouteWrapper)
225
+ end
226
+
227
+ # TODO: Struct
118
228
  Thread.current[:sqreen_http_request] = {
119
- uuid: uuid,
120
- start_time: now,
121
- time_budget: Sqreen.performance_budget,
229
+ request_timer: request_timer,
230
+ sqreen_timer: sqreen_timer,
122
231
  time_budget_expended: false,
123
- timer: Sqreen::Graft::Timer.new("request_#{uuid}"),
232
+ time_budget: budget,
124
233
  timed_callbacks: [],
125
234
  timed_hooks: [],
126
- timed_hooks_before: [],
127
- timed_hooks_after: [],
128
- timed_hooks_raised: [],
129
- timed_hooks_ensured: [],
235
+ timed_level: timed_level,
130
236
  skipped_callbacks: [],
237
+ route: ("#{route_found[:verb]} #{route_found[:path]}" if route_found),
238
+ # timed_shrinkwrap: shrinkwrap_timer,
131
239
  }
132
240
 
133
- Sqreen::Weave.logger.debug { "request.uuid: #{uuid}" }
241
+ # shrinkwrap_timer.stop
134
242
  end
135
243
 
136
244
  ensured('weave,meta,request', rank: 100000, mandatory: true) do |_call|
@@ -138,105 +246,89 @@ class Sqreen::Weave::Legacy::Instrumentation
138
246
 
139
247
  next if request.nil?
140
248
 
249
+ timed_level = request[:timed_level]
250
+ req_detailed = SqDetailedMetrics::Request.new if timed_level >= 2
251
+
252
+ # shrinkwrap_timer = request[:timed_shrinkwrap]
253
+ # shrinkwrap_timer.start
254
+
141
255
  Thread.current[:sqreen_http_request] = nil
142
- now = Sqreen::Graft::Timer.read
143
- utc_now = Time.now.utc
144
-
145
- request[:timed_callbacks].each do |timer|
146
- duration = timer.duration
147
- # stop = now
148
- # start = now - duration
149
- timer.tag =~ /weave,rule=(.*)$/ && rule = $1
150
- timer.tag =~ /@before/ && whence = 'pre'
151
- timer.tag =~ /@after/ && whence = 'post'
152
- timer.tag =~ /@raised/ && whence = 'failing'
153
-
154
- next unless rule && whence
155
-
156
- # Sqreen::PerformanceNotifications.notify(rule, whence, start, stop)
157
- # => BinnedMetrics
158
- metric_name = "sq.#{rule}.#{whence}"
159
- unless metrics_engine.metric?(metric_name)
160
- metrics_engine.create_metric(
161
- 'name' => metric_name,
162
- 'period' => 60,
163
- 'kind' => 'Binning',
164
- 'options' => { 'base' => 2.0, 'factor' => 0.1 },
165
- )
256
+ request_timer = request[:request_timer]
257
+ now = request_timer.stop
258
+
259
+ if timed_level >= 1
260
+ request[:timed_callbacks].each do |timer|
261
+ duration_ms = timer.duration * 1000.0
262
+ # XXX: the timer tag should have this structured data;
263
+ # it would be better than recomputing this for every measurement
264
+ metric_name = ::Sqreen::Weave::Legacy::Instrumentation.tag_to_metric_name(timer.tag)
265
+
266
+ next unless metric_name
267
+
268
+ metrics_engine.update(metric_name, now, nil, duration_ms)
269
+ duration_ms *= -1.0 if timer.conditions_passed
270
+ req_detailed.add_measurement metric_name, duration_ms if req_detailed
166
271
  end
167
- metrics_engine.update(metric_name, now, nil, duration * 1000)
168
272
  end
169
273
 
170
- metric_name = 'sq.hooks_pre.pre'
171
- duration = request[:timed_hooks_before].sum(&:duration)
172
- unless metrics_engine.metric?(metric_name)
173
- metrics_engine.create_metric(
174
- 'name' => metric_name,
175
- 'period' => 60,
176
- 'kind' => 'Binning',
177
- 'options' => { 'base' => 2.0, 'factor' => 0.1 },
178
- )
179
- end
180
- metrics_engine.update(metric_name, now, nil, duration * 1000)
181
-
182
- metric_name = 'sq.hooks_post.post'
183
- duration = request[:timed_hooks_after].sum(&:duration)
184
- unless metrics_engine.metric?(metric_name)
185
- metrics_engine.create_metric(
186
- 'name' => metric_name,
187
- 'period' => 60,
188
- 'kind' => 'Binning',
189
- 'options' => { 'base' => 2.0, 'factor' => 0.1 },
190
- )
191
- end
192
- metrics_engine.update(metric_name, now, nil, duration * 1000)
193
-
194
- metric_name = 'sq.hooks_failing.failing'
195
- duration = request[:timed_hooks_raised].sum(&:duration)
196
- unless metrics_engine.metric?(metric_name)
197
- metrics_engine.create_metric(
198
- 'name' => metric_name,
199
- 'period' => 60,
200
- 'kind' => 'Binning',
201
- 'options' => { 'base' => 2.0, 'factor' => 0.1 },
202
- )
274
+ sqreen_timer = request[:sqreen_timer]
275
+ Sqreen::Weave.logger.debug do
276
+ "request sqreen_timer.total: #{'%.03fus' % (sqreen_timer.duration * 1_000_000)}"
277
+ end if Sqreen::Weave.logger.debug?
278
+ Sqreen::Weave.logger.debug do
279
+ "request request_timer.total: #{'%.03fus' % (request_timer.duration * 1_000_000)}"
280
+ end if Sqreen::Weave.logger.debug?
281
+
282
+ if timed_level >= 1 && Sqreen::Weave.logger.debug?
283
+ skipped = request[:skipped_callbacks].map(&:name)
284
+ Sqreen::Weave.logger.debug { "request callback.skipped.count: #{skipped.count}" } if Sqreen::Weave.logger.debug?
285
+ timings = request[:timed_callbacks].map(&:to_s)
286
+ total = request[:timed_callbacks].sum(&:duration)
287
+ Sqreen::Weave.logger.debug { "request callback.total: #{'%.03fus' % (total * 1_000_000)} callback.count: #{timings.count}" } if Sqreen::Weave.logger.debug?
288
+ timings = request[:timed_hooks].map(&:to_s)
289
+ total = request[:timed_hooks].sum(&:duration)
290
+ Sqreen::Weave.logger.debug { "request hook.total: #{'%.03fus' % (total * 1_000_000)} hook.count: #{timings.count}" } if Sqreen::Weave.logger.debug?
203
291
  end
204
- metrics_engine.update(metric_name, now, nil, duration * 1000)
205
-
206
- skipped = request[:skipped_callbacks].map(&:name)
207
- Sqreen::Weave.logger.debug { "request:#{request[:uuid]} callback.skipped.size: #{skipped.count} callback.skipped: [#{skipped.join(', ')}]" }
208
- timer = request[:timer]
209
- total = timer.duration
210
- Sqreen::Weave.logger.debug { "request:#{request[:uuid]} timer.total: #{'%.03fus' % (total * 1_000_000)} timer.size: #{timer.size}" }
211
- timings = request[:timed_callbacks].map(&:to_s)
212
- total = request[:timed_callbacks].sum(&:duration)
213
- Sqreen::Weave.logger.debug { "request:#{request[:uuid]} callback.total: #{'%.03fus' % (total * 1_000_000)} callback.timings: [#{timings.join(', ')}]" }
214
- timings = request[:timed_hooks].map(&:to_s)
215
- total = request[:timed_hooks].sum(&:duration)
216
- Sqreen::Weave.logger.debug { "request:#{request[:uuid]} hook.total: #{'%.03fus' % (total * 1_000_000)} hook.timings: [#{timings.join(', ')}]" }
217
-
218
- skipped = request[:skipped_callbacks].map(&:name)
219
- skipped_rule_name = skipped.first && skipped.first =~ /weave,rule=(.*)$/ && $1
220
- Sqreen.observations_queue.push(['request_overtime', skipped_rule_name, 1, utc_now]) if skipped_rule_name
221
-
222
- sqreen_request_duration = total
223
- Sqreen.observations_queue.push(['sq', nil, sqreen_request_duration * 1000, utc_now])
224
-
225
- request_duration = now - request[:start_time]
226
- Sqreen.observations_queue.push(['req', nil, request_duration * 1000, utc_now])
292
+
293
+ overtime_cb = ::Sqreen::Weave::Legacy::Instrumentation.tag_to_metric_name(request[:overtime_cb]) \
294
+ if request[:overtime_cb]
295
+ metrics_engine.update('request_overtime', now, overtime_cb, 1) if overtime_cb
296
+
297
+ sqreen_request_duration = sqreen_timer.duration * 1000.0
298
+ metrics_engine.update('sq', now, nil, sqreen_request_duration)
299
+
300
+ request_duration = request_timer.duration * 1000.0
301
+ metrics_engine.update('req', now, nil, request_duration)
227
302
 
228
303
  sqreen_request_ratio = (sqreen_request_duration * 100.0) / (request_duration - sqreen_request_duration)
229
- Sqreen.observations_queue.push(['pct', nil, sqreen_request_ratio, utc_now])
304
+ metrics_engine.update('pct', now, nil, sqreen_request_ratio)
305
+ Sqreen::Weave.logger.debug { "request sqreen_timer.ratio: #{'%.03f' % (sqreen_request_ratio / 100.0)}" } if Sqreen::Weave.logger.debug?
306
+
307
+ if req_detailed
308
+ req_detailed.route = request[:route]
309
+ req_detailed.overtime_cb = overtime_cb if overtime_cb
310
+ req_detailed.add_measurement 'sq', sqreen_request_duration
311
+ req_detailed.add_measurement 'req', request_duration
312
+
313
+ metrics_engine.update(REQ_LVL_2_METRIC, now, nil, req_detailed)
314
+ end
315
+
316
+ # shrinkwrap_timer.stop
317
+
318
+ # duration = shrinkwrap_timer.duration
319
+ # metrics_engine.update('sq.shrinkwrap', now, nil, duration * 1000)
230
320
  end
231
321
  end.install
232
322
 
233
323
  ### globally declare instrumentation ready
234
324
  Sqreen.instrumentation_ready = true
325
+ Sqreen::Weave.logger.info { "Instrumentation activated" }
235
326
  end
236
327
 
237
328
  # needed by Sqreen::Runner
238
329
  def remove_all_callbacks
239
330
  Sqreen.instrumentation_ready = false
331
+ Sqreen::Weave.logger.info { "Instrumentation deactivated" }
240
332
 
241
333
  loop do
242
334
  hook = @hooks.pop
@@ -253,6 +345,15 @@ class Sqreen::Weave::Legacy::Instrumentation
253
345
  klass = callback.klass
254
346
  method = callback.method
255
347
 
348
+ if (call_count = ENV['SQREEN_DEBUG_CALL_COUNT'])
349
+ call_count = JSON.parse(call_count)
350
+ if callback.respond_to?(:rule_name) && call_count.key?(callback.rule_name)
351
+ count = call_count[callback.rule_name]
352
+ Sqreen::Weave.logger.debug { "override rule: #{callback.rule_name} call_count: #{count.inspect}" }
353
+ callback.instance_eval { @call_count_interval = call_count[callback.rule_name] }
354
+ end
355
+ end
356
+
256
357
  if Sqreen::Graft::HookPoint.new("#{klass}.#{method}").exist?
257
358
  hook_point = "#{klass}.#{method}"
258
359
  elsif Sqreen::Graft::HookPoint.new("#{klass}##{method}").exist?
@@ -268,14 +369,14 @@ class Sqreen::Weave::Legacy::Instrumentation
268
369
  hook = Sqreen::Graft::Hook[hook_point, strategy]
269
370
  hook.add do
270
371
  if callback.pre?
271
- before(rule, rank: priority, mandatory: !callback.overtimeable, flow: block, ignore: ignore) do |call, b|
372
+ use_flow = block || callback.is_a?(::Sqreen::Conditionable)
373
+ before(rule, rank: priority, mandatory: !callback.overtimeable, flow: use_flow, ignore: ignore) do |call, b|
272
374
  next unless Thread.current[:sqreen_http_request]
273
375
 
274
376
  i = call.instance
275
377
  a = call.args
276
378
  r = call.remaining
277
379
 
278
- Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#pre instance=#{i}" }
279
380
  begin
280
381
  ret = callback.pre(i, a, r)
281
382
  rescue StandardError => e
@@ -286,17 +387,30 @@ class Sqreen::Weave::Legacy::Instrumentation
286
387
  Sqreen::RemoteException.record(e)
287
388
  end
288
389
  end
289
- Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#pre instance=#{i} => return=#{ret.inspect}" }
290
-
291
- case ret[:status]
292
- when :skip, 'skip'
293
- throw(b, b.return(ret[:new_return_value]).break!) if ret.key?(:new_return_value)
294
- when :modify_args, 'modify_args'
295
- throw(b, b.args(ret[:args]))
296
- when :raise, 'raise'
297
- throw(b, b.raise(ret[:exception])) if ret.key?(:exception)
298
- throw(b, b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required.")))
299
- end unless ret.nil?
390
+
391
+ next if ret.nil? || !ret.is_a?(Hash)
392
+
393
+ throw_val =
394
+ case ret[:status]
395
+ when :skip, 'skip'
396
+ b.return(ret[:new_return_value]).break! if ret.key?(:new_return_value)
397
+ when :modify_args, 'modify_args'
398
+ b.args(ret[:args])
399
+ when :raise, 'raise'
400
+ if ret.key?(:exception)
401
+ b.raise(ret[:exception])
402
+ else
403
+ b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required."))
404
+ end
405
+ end if block
406
+
407
+ if ret && ret[:passed_conditions]
408
+ throw_val ||= b.noop
409
+ throw_val.passed_conditions!
410
+ end
411
+ next unless throw_val
412
+ throw_val.break! if ret[:skip_rem_cbs]
413
+ throw(b, throw_val)
300
414
  end
301
415
  end
302
416
 
@@ -309,7 +423,6 @@ class Sqreen::Weave::Legacy::Instrumentation
309
423
  a = call.args
310
424
  r = call.remaining
311
425
 
312
- Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#post instance=#{i}" }
313
426
  begin
314
427
  ret = callback.post(v, i, a, r)
315
428
  rescue StandardError => e
@@ -320,15 +433,22 @@ class Sqreen::Weave::Legacy::Instrumentation
320
433
  Sqreen::RemoteException.record(e)
321
434
  end
322
435
  end
323
- Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#post instance=#{i} => return=#{ret.inspect}" }
324
-
325
- case ret[:status]
326
- when :override, 'override'
327
- throw(b, b.return(ret[:new_return_value])) if ret.key?(:new_return_value)
328
- when :raise, 'raise'
329
- throw(b, b.raise(ret[:exception])) if ret.key?(:exception)
330
- throw(b, b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required.")))
331
- end unless ret.nil?
436
+
437
+ throw_val =
438
+ case ret[:status]
439
+ when :override, 'override'
440
+ b.return(ret[:new_return_value]) if ret.key?(:new_return_value)
441
+ when :raise, 'raise'
442
+ b.raise(ret[:exception]) if ret.key?(:exception)
443
+ b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required."))
444
+ end unless ret.nil? || !ret.is_a?(Hash) || !block
445
+
446
+ if ret && ret[:passed_conditions]
447
+ throw_val ||= b.noop
448
+ throw_val.passed_conditions!
449
+ end
450
+ next unless throw_val
451
+ throw(b, throw_val)
332
452
  end
333
453
  end
334
454
 
@@ -341,7 +461,6 @@ class Sqreen::Weave::Legacy::Instrumentation
341
461
  a = call.args
342
462
  r = call.remaining
343
463
 
344
- Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#failing instance=#{i}" }
345
464
  begin
346
465
  ret = callback.failing(e, i, a, r)
347
466
  rescue StandardError => e
@@ -352,23 +471,30 @@ class Sqreen::Weave::Legacy::Instrumentation
352
471
  Sqreen::RemoteException.record(e)
353
472
  end
354
473
  end
355
- Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#failing instance=#{i} => return=#{ret.inspect}" }
356
-
357
- raise e if ret.nil?
358
-
359
- case ret[:status]
360
- when :override, 'override'
361
- throw(b, b.return(ret[:new_return_value])) if ret.key?(:new_return_value)
362
- when :retry, 'retry'
363
- throw(b, b.retry)
364
- when :raise, 'raise'
365
- throw(b, b.raise(ret[:exception])) if ret.key?(:exception)
366
- throw(b, b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required.")))
367
- when :reraise, 'reraise'
368
- throw(b, b.raise(e))
369
- else
370
- throw(b, b.raise(e))
371
- end unless ret.nil?
474
+
475
+ throw(b, b.raise(e)) if ret.nil? || !ret.is_a?(Hash)
476
+
477
+ throw_val =
478
+ case ret[:status]
479
+ when :override, 'override'
480
+ b.return(ret[:new_return_value]) if ret.key?(:new_return_value)
481
+ when :retry, 'retry'
482
+ b.retry
483
+ when :raise, 'raise'
484
+ b.raise(ret[:exception]) if ret.key?(:exception)
485
+ b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required."))
486
+ when :reraise, 'reraise'
487
+ b.raise(e)
488
+ else
489
+ b.raise(e)
490
+ end unless ret.nil? || !ret.is_a?(Hash) || !block
491
+
492
+ if ret && ret[:passed_conditions]
493
+ throw_val ||= b.noop
494
+ throw_val.passed_conditions!
495
+ end
496
+ next unless throw_val
497
+ throw(b, throw_val)
372
498
  end
373
499
  end
374
500
  end.install
@@ -403,4 +529,20 @@ class Sqreen::Weave::Legacy::Instrumentation
403
529
  Sqreen::Rules::RunUserActions.new(Sqreen, :auth_track, 1),
404
530
  ]
405
531
  end
532
+
533
+ def self.tag_to_metric_name(tag)
534
+ cached = @cache_tag_to_metric[tag]
535
+ return cached unless cached.nil?
536
+
537
+ tag =~ /weave,rule=(.*)$/ && rule = $1 and # rubocop:disable Style/AndOr
538
+ (tag =~ /@before/ && whence = 'pre' or # rubocop:disable Style/AndOr
539
+ tag =~ /@after/ && whence = 'post' or # rubocop:disable Style/AndOr
540
+ tag =~ /@raised/ && whence = 'failing' or # rubocop:disable Style/AndOr
541
+ tag =~ /@ensured/ && whence = 'finally')
542
+
543
+ @cache_tag_to_metric[tag] =
544
+ rule && whence ? "sq.#{rule}.#{whence}" : false
545
+ end
546
+
547
+ @cache_tag_to_metric = {}
406
548
  end