sqreen 1.19.1-java → 1.21.0.beta3-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (101) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +34 -0
  3. data/lib/sqreen/actions/block_user.rb +1 -1
  4. data/lib/sqreen/actions/redirect_ip.rb +1 -1
  5. data/lib/sqreen/actions/redirect_user.rb +1 -1
  6. data/lib/sqreen/agent_message.rb +20 -0
  7. data/lib/sqreen/aggregated_metric.rb +25 -0
  8. data/lib/sqreen/attack_detected.html +1 -2
  9. data/lib/sqreen/ca.crt +24 -0
  10. data/lib/sqreen/condition_evaluator.rb +9 -2
  11. data/lib/sqreen/conditionable.rb +24 -6
  12. data/lib/sqreen/configuration.rb +11 -5
  13. data/lib/sqreen/deferred_logger.rb +50 -14
  14. data/lib/sqreen/deliveries/batch.rb +12 -2
  15. data/lib/sqreen/deliveries/simple.rb +4 -0
  16. data/lib/sqreen/deprecation.rb +38 -0
  17. data/lib/sqreen/ecosystem.rb +96 -0
  18. data/lib/sqreen/ecosystem/dispatch_table.rb +43 -0
  19. data/lib/sqreen/ecosystem/exception_reporting.rb +26 -0
  20. data/lib/sqreen/ecosystem/http/net_http.rb +50 -0
  21. data/lib/sqreen/ecosystem/http/rack_request.rb +39 -0
  22. data/lib/sqreen/ecosystem/loggable.rb +13 -0
  23. data/lib/sqreen/ecosystem/module_api.rb +30 -0
  24. data/lib/sqreen/ecosystem/module_api/event_listener.rb +18 -0
  25. data/lib/sqreen/ecosystem/module_api/instrumentation.rb +23 -0
  26. data/lib/sqreen/ecosystem/module_api/message_producer.rb +51 -0
  27. data/lib/sqreen/ecosystem/module_api/signal_producer.rb +24 -0
  28. data/lib/sqreen/ecosystem/module_api/tracing.rb +45 -0
  29. data/lib/sqreen/ecosystem/module_api/tracing/client_data.rb +31 -0
  30. data/lib/sqreen/ecosystem/module_api/tracing/server_data.rb +27 -0
  31. data/lib/sqreen/ecosystem/module_api/tracing_id_generation.rb +16 -0
  32. data/lib/sqreen/ecosystem/module_api/transaction_storage.rb +71 -0
  33. data/lib/sqreen/ecosystem/module_registry.rb +44 -0
  34. data/lib/sqreen/ecosystem/redis/redis_connection.rb +43 -0
  35. data/lib/sqreen/ecosystem/tracing/modules/client.rb +31 -0
  36. data/lib/sqreen/ecosystem/tracing/modules/server.rb +30 -0
  37. data/lib/sqreen/ecosystem/tracing/sampler.rb +160 -0
  38. data/lib/sqreen/ecosystem/tracing/sampling_configuration.rb +150 -0
  39. data/lib/sqreen/ecosystem/tracing/signals/tracing_client.rb +53 -0
  40. data/lib/sqreen/ecosystem/tracing/signals/tracing_server.rb +53 -0
  41. data/lib/sqreen/ecosystem/tracing_broker.rb +101 -0
  42. data/lib/sqreen/ecosystem/tracing_id_setup.rb +34 -0
  43. data/lib/sqreen/ecosystem/transaction_storage.rb +64 -0
  44. data/lib/sqreen/ecosystem/util/call_writers_from_init.rb +13 -0
  45. data/lib/sqreen/ecosystem_integration.rb +87 -0
  46. data/lib/sqreen/ecosystem_integration/around_callbacks.rb +99 -0
  47. data/lib/sqreen/ecosystem_integration/instrumentation_service.rb +42 -0
  48. data/lib/sqreen/ecosystem_integration/request_lifecycle_tracking.rb +58 -0
  49. data/lib/sqreen/ecosystem_integration/signal_consumption.rb +35 -0
  50. data/lib/sqreen/endpoint_testing.rb +184 -0
  51. data/lib/sqreen/event.rb +7 -5
  52. data/lib/sqreen/events/attack.rb +23 -18
  53. data/lib/sqreen/events/remote_exception.rb +0 -22
  54. data/lib/sqreen/events/request_record.rb +15 -71
  55. data/lib/sqreen/frameworks/generic.rb +24 -1
  56. data/lib/sqreen/frameworks/rails.rb +0 -7
  57. data/lib/sqreen/frameworks/request_recorder.rb +15 -2
  58. data/lib/sqreen/graft/call.rb +106 -19
  59. data/lib/sqreen/graft/callback.rb +1 -1
  60. data/lib/sqreen/graft/hook.rb +212 -100
  61. data/lib/sqreen/graft/hook_point.rb +18 -11
  62. data/lib/sqreen/kit/signals/specialized/aggregated_metric.rb +72 -0
  63. data/lib/sqreen/kit/signals/specialized/attack.rb +57 -0
  64. data/lib/sqreen/kit/signals/specialized/binning_metric.rb +76 -0
  65. data/lib/sqreen/kit/signals/specialized/http_trace.rb +26 -0
  66. data/lib/sqreen/kit/signals/specialized/sdk_track_call.rb +50 -0
  67. data/lib/sqreen/kit/signals/specialized/sqreen_exception.rb +57 -0
  68. data/lib/sqreen/legacy/instrumentation.rb +22 -10
  69. data/lib/sqreen/legacy/old_event_submission_strategy.rb +228 -0
  70. data/lib/sqreen/legacy/waf_redactions.rb +49 -0
  71. data/lib/sqreen/log.rb +3 -2
  72. data/lib/sqreen/log/loggable.rb +2 -1
  73. data/lib/sqreen/logger.rb +24 -0
  74. data/lib/sqreen/metrics.rb +1 -0
  75. data/lib/sqreen/metrics/base.rb +3 -0
  76. data/lib/sqreen/metrics/req_detailed.rb +41 -0
  77. data/lib/sqreen/metrics_store.rb +33 -12
  78. data/lib/sqreen/null_logger.rb +22 -0
  79. data/lib/sqreen/performance_notifications/binned_metrics.rb +8 -2
  80. data/lib/sqreen/remote_command.rb +4 -0
  81. data/lib/sqreen/rules.rb +12 -6
  82. data/lib/sqreen/rules/blacklist_ips_cb.rb +2 -2
  83. data/lib/sqreen/rules/custom_error_cb.rb +3 -3
  84. data/lib/sqreen/rules/not_found_cb.rb +2 -0
  85. data/lib/sqreen/rules/rule_cb.rb +6 -2
  86. data/lib/sqreen/rules/waf_cb.rb +16 -13
  87. data/lib/sqreen/runner.rb +138 -16
  88. data/lib/sqreen/sensitive_data_redactor.rb +19 -31
  89. data/lib/sqreen/session.rb +53 -43
  90. data/lib/sqreen/signals/conversions.rb +288 -0
  91. data/lib/sqreen/signals/http_trace_redaction.rb +111 -0
  92. data/lib/sqreen/signals/signals_submission_strategy.rb +78 -0
  93. data/lib/sqreen/version.rb +1 -1
  94. data/lib/sqreen/weave/budget.rb +35 -0
  95. data/lib/sqreen/weave/legacy/instrumentation.rb +277 -135
  96. data/lib/sqreen/worker.rb +6 -2
  97. metadata +86 -10
  98. data/lib/sqreen/backport.rb +0 -9
  99. data/lib/sqreen/backport/clock_gettime.rb +0 -74
  100. data/lib/sqreen/backport/original_name.rb +0 -88
  101. data/lib/sqreen/encoding_sanitizer.rb +0 -27
@@ -0,0 +1,111 @@
1
+ require 'json'
2
+ require 'sqreen/kit/loggable'
3
+ require 'sqreen/kit/signals/specialized/http_trace'
4
+
5
+ module Sqreen
6
+ module Signals
7
+ module HttpTraceRedaction
8
+ class << self
9
+ include Sqreen::Kit::Loggable
10
+
11
+ # @param [Sqreen::Kit::Signals::Specialized::HttpTrace] trace
12
+ # @param [Sqreen::SensitiveDataRedactor] redactor
13
+ def redact_trace!(trace, redactor)
14
+ return unless redactor
15
+ # redact headers (keys unsafe)
16
+ # @type [Sqreen::Kit::Signals::Context::HttpContext]
17
+ http_context = trace.context
18
+
19
+ all_redacted = []
20
+
21
+ # Redact headers; save redacted values
22
+ # headers are encoded as [key, value], not a hash, so
23
+ # they require some transformation
24
+ orig_headers = http_context.headers
25
+ if orig_headers
26
+ headers = orig_headers.map { |(k, v)| { k => v } }
27
+ headers, redacted = redactor.redact(headers)
28
+ http_context.headers = headers.map(&:first)
29
+ all_redacted += redacted
30
+ end
31
+
32
+ # Redact params; save redacted values
33
+ Kit::Signals::Context::HttpContext::PARAMS_ATTRS.each do |attr|
34
+ value = http_context.public_send(attr)
35
+ next unless value
36
+ value, redacted = redactor.redact(value)
37
+ all_redacted += redacted
38
+ http_context.public_send(:"#{attr}=", value)
39
+ end
40
+
41
+ all_redacted = all_redacted.uniq.map(&:downcase)
42
+
43
+ # Redact attacks and exceptions
44
+ # XXX: no redaction for infos in attacks/exceptions except for WAF data
45
+ # Is this the correct behavior?
46
+ redact_attacks!(trace, redactor, all_redacted)
47
+ redact_exceptions!(trace, redactor, all_redacted)
48
+ end
49
+
50
+ private
51
+
52
+ # @param [Sqreen::Kit::Signals::Specialized::HttpTrace] trace
53
+ # @param [Sqreen::SensitiveDataRedactor] redactor
54
+ # Redacts WAF data according to specific rules therefor
55
+ # Redacts infos according to general rules
56
+ def redact_attacks!(trace, redactor, redacted_data)
57
+ trace.data.each do |signal|
58
+ next unless signal.is_a?(Kit::Signals::Specialized::Attack)
59
+ # @type [Sqreen::Kit::Signals::Specialized::Attack::Payload] payload
60
+ payload = signal.payload
61
+ next unless payload.infos
62
+
63
+ if payload.infos[:waf_data]
64
+ redact_waf_attack_data!(payload.infos, redacted_data)
65
+ end
66
+ payload.infos, = redactor.redact(payload.infos)
67
+ end
68
+ end
69
+
70
+ def redact_exceptions!(trace, redactor, redacted_data)
71
+ trace.data.each do |signal|
72
+ next unless signal.is_a?(Kit::Signals::Specialized::SqreenException)
73
+ infos = signal.infos
74
+ next unless infos
75
+
76
+ redact_waf_exception_data!(signal.infos, redacted_data) if signal.infos[:waf]
77
+ signal.infos, = redactor.redact(infos)
78
+ end
79
+ end
80
+
81
+ # @param [Hash] infos from WAF attack
82
+ def redact_waf_attack_data!(infos, redacted_data)
83
+ begin
84
+ parsed = JSON.parse(infos[:waf_data])
85
+ rescue JSON::JSONError => e
86
+ logger.warn("waf_data is not valid json: #{e.message}")
87
+ return
88
+ end
89
+ redacted = parsed.each do |w|
90
+ next unless (filters = w['filter'])
91
+
92
+ filters.each do |f|
93
+ next unless (v = f['resolved_value'])
94
+ next unless redacted_data.include?(v.downcase)
95
+
96
+ f['match_status'] = SensitiveDataRedactor::MASK
97
+ f['resolved_value'] = SensitiveDataRedactor::MASK
98
+ end
99
+ end
100
+ infos[:waf_data] = JSON.dump(redacted)
101
+ end
102
+
103
+ # see https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000022-waf-data-sanitization.md#changes-to-the-agents
104
+ def redact_waf_exception_data!(infos, redacted_data)
105
+ return if redacted_data.empty?
106
+ infos[:waf].delete(:args)
107
+ end
108
+ end
109
+ end
110
+ end
111
+ end
@@ -0,0 +1,78 @@
1
+ require 'sqreen/aggregated_metric'
2
+ require 'sqreen/kit'
3
+ require 'sqreen/kit/string_sanitizer'
4
+ require 'sqreen/signals/conversions'
5
+ require 'sqreen/log/loggable'
6
+
7
+ module Sqreen
8
+ module Signals
9
+ # see also Sqreen::Legacy::OldEventSubmissionStrategy
10
+ # usage in Sqreen:Session
11
+ class SignalsSubmissionStrategy
12
+ include Sqreen::Log::Loggable
13
+
14
+ # @param [Array<Sqreen::AggregatedMetric>] metrics
15
+ def post_metrics(metrics)
16
+ return if metrics.nil? || metrics.empty?
17
+
18
+ guarded 'Failed to serialize or submit aggregated metrics' do
19
+ batch = metrics.map do |m|
20
+ Conversions.convert_metric_sample(m)
21
+ end
22
+ client.report_batch(batch)
23
+ end
24
+ end
25
+
26
+ # @param _attack [Sqreen::Attack]
27
+ # XXX: unused
28
+ def post_attack(_attack)
29
+ raise NotImplementedError
30
+ end
31
+
32
+ # @param request_record [Sqreen::RequestRecord]
33
+ def post_request_record(request_record)
34
+ guarded 'Failed to serialize or submit request record' do
35
+ trace = Conversions.convert_req_record(request_record)
36
+ append_sanitizing_filter(trace)
37
+ client.report_trace(trace)
38
+ end
39
+ end
40
+
41
+ # Post an exception to Sqreen for analysis
42
+ # @param exception [RemoteException] Exception and context to be sent over
43
+ def post_sqreen_exception(exception)
44
+ guarded 'Failed to serialize or submit exception', false do
45
+ data = Conversions.convert_exception(exception)
46
+ append_sanitizing_filter(data)
47
+ client.report_signal(data)
48
+ end
49
+ end
50
+
51
+ def post_batch(events)
52
+ guarded 'Failed to serialize or submit batch of events' do
53
+ batch = Conversions.convert_batch(events)
54
+ batch.each { |sig_or_trace| append_sanitizing_filter(sig_or_trace) }
55
+ client.report_batch(batch)
56
+ end
57
+ end
58
+
59
+ private
60
+
61
+ def append_sanitizing_filter(sig_or_trace)
62
+ sig_or_trace.append_to_h_filter Kit::StringSanitizer.method(:sanitize)
63
+ end
64
+
65
+ # we don't want exceptions to propagate and kill the worker thread
66
+ def guarded(msg, report = true)
67
+ yield
68
+ rescue StandardError => e
69
+ logger.warn "#{msg}: #{e.message}\n#{e.backtrace.map { |x| " #{x}" }.join("\n")}"
70
+ post_sqreen_exception(RemoteException.new(e)) if report
71
+ end
72
+
73
+ def client
74
+ Sqreen::Kit.auth_signals_client
75
+ end
76
+ end
77
+ end
78
+ end
@@ -4,5 +4,5 @@
4
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
5
5
 
6
6
  module Sqreen
7
- VERSION = '1.19.1'.freeze
7
+ VERSION = '1.21.0.beta3'.freeze
8
8
  end
@@ -0,0 +1,35 @@
1
+ # typed: false
2
+
3
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
4
+ # Please refer to our terms for more information: https://www.sqreen.com/terms.html
5
+
6
+ require 'sqreen/log/loggable'
7
+ require 'sqreen/weave'
8
+
9
+ class Sqreen::Weave::Budget
10
+ include Sqreen::Log::Loggable
11
+
12
+ def initialize(threshold)
13
+ @threshold = threshold
14
+ end
15
+
16
+ attr_reader :threshold
17
+
18
+ def to_h
19
+ { threshold: threshold }
20
+ end
21
+
22
+ class << self
23
+ attr_reader :current
24
+
25
+ def update(opts = nil)
26
+ Sqreen::Weave.logger.info("budget update:#{opts.inspect}")
27
+
28
+ return @current = nil if opts.nil? || opts.empty?
29
+
30
+ threshold = opts[:threshold]
31
+
32
+ @current = threshold
33
+ end
34
+ end
35
+ end
@@ -4,23 +4,41 @@
4
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
5
5
 
6
6
  require 'sqreen/weave/legacy'
7
+ require 'sqreen/weave/budget'
8
+ require 'sqreen/graft/hook'
7
9
  require 'sqreen/graft/hook_point'
8
10
  require 'sqreen/call_countable'
9
11
  require 'sqreen/rules'
10
12
  require 'sqreen/rules/record_request_context'
13
+ require 'sqreen/sqreen_signed_verifier'
14
+ require 'rack/request'
15
+ begin
16
+ require 'sq_detailed_metrics'
17
+ rescue LoadError => _e # rubocop:disable Lint/HandleExceptions
18
+ end
11
19
 
12
20
  class Sqreen::Weave::Legacy::Instrumentation
13
21
  attr_accessor :metrics_engine
14
22
 
23
+ HAS_SQ_DETAILED_METRICS = defined?(::SqDetailedMetrics)
24
+ REQ_LVL_2_METRIC = 'request_level_perf'.freeze
25
+
15
26
  def initialize(metrics_engine, opts = {})
16
27
  Sqreen::Weave.logger.debug { "#{self.class.name}#initialize #{metrics_engine}" }
17
28
  @hooks = []
18
29
 
30
+ unless HAS_SQ_DETAILED_METRICS
31
+ Sqreen::Weave.logger.warn { "Detailed metrics are unavailable" }
32
+ end
33
+
19
34
  self.metrics_engine = metrics_engine
20
35
 
21
36
  ### bail out if no metric engine
22
37
  return if metrics_engine.nil?
23
38
 
39
+ # XXX: these metric definitions do not support change of opts
40
+ # due to features updates!
41
+
24
42
  ### init metric to count calls to sqreen
25
43
  metrics_engine.create_metric(
26
44
  'name' => 'sqreen_call_counts',
@@ -60,12 +78,42 @@ class Sqreen::Weave::Legacy::Instrumentation
60
78
  'options' => opts[:perf_metric_percent] || { 'base' => 1.3, 'factor' => 1.0 },
61
79
  )
62
80
 
81
+ metrics_engine.create_metric(
82
+ 'name' => 'req.sq.hook.overhead',
83
+ 'period' => 60,
84
+ 'kind' => 'Binning',
85
+ 'options' => { 'base' => 2.0, 'factor' => 0.1 },
86
+ )
87
+
88
+ metrics_engine.create_metric(
89
+ 'name' => 'sq.hook.overhead',
90
+ 'period' => 60,
91
+ 'kind' => 'Binning',
92
+ 'options' => { 'base' => 2.0, 'factor' => 0.1 },
93
+ )
94
+
95
+ metrics_engine.create_metric(
96
+ 'name' => 'sq.shrinkwrap',
97
+ 'period' => 60,
98
+ 'kind' => 'Binning',
99
+ 'options' => { 'base' => 2.0, 'factor' => 0.1 },
100
+ )
101
+
63
102
  Sqreen.thread_cpu_time? && metrics_engine.create_metric(
64
103
  'name' => 'sq_thread_cpu_pct',
65
104
  'period' => opts[:period] || 60,
66
105
  'kind' => 'Binning',
67
106
  'options' => opts[:perf_metric_percent] || { 'base' => 1.3, 'factor' => 1.0 },
68
107
  )
108
+
109
+ if HAS_SQ_DETAILED_METRICS # rubocop:disable Style/GuardClause
110
+ @lvl_2_metric = metrics_engine.create_metric(
111
+ 'name' => REQ_LVL_2_METRIC,
112
+ 'period' => opts[:perf_req_metrics_period] || 60,
113
+ 'kind' => 'ReqDetailed',
114
+ )
115
+ @lvl_2_max_reqs = opts[:perf_req_metrics_max_reqs] || 100
116
+ end
69
117
  end
70
118
 
71
119
  # needed by Sqreen::Runner#initialize
@@ -84,6 +132,15 @@ class Sqreen::Weave::Legacy::Instrumentation
84
132
 
85
133
  ### set up rule signature verifier
86
134
  verifier = nil
135
+ if Sqreen.features['rules_signature'] &&
136
+ Sqreen.config_get(:rules_verify_signature) == true &&
137
+ !defined?(::JRUBY_VERSION)
138
+ verifier = Sqreen::SqreenSignedVerifier.new
139
+ Sqreen::Weave.logger.debug('Rules signature enabled')
140
+ else
141
+ Sqreen::Weave.logger.debug('Rules signature disabled')
142
+ end
143
+
87
144
  ### force clean instrumentation callback list
88
145
  @hooks = []
89
146
  ### for each rule description
@@ -94,6 +151,25 @@ class Sqreen::Weave::Legacy::Instrumentation
94
151
  next unless rule_callback
95
152
  ### attach framework to callback
96
153
  rule_callback.framework = framework
154
+ ## create metric
155
+ Sqreen::Weave.logger.debug { "Adding rule metric: #{rule_callback}" }
156
+ [:pre, :post, :failing].each do |whence|
157
+ next unless rule_callback.send(:"#{whence}?")
158
+ metric_name = "sq.#{rule['name']}.#{whence}"
159
+ metrics_engine.create_metric(
160
+ 'name' => metric_name,
161
+ 'period' => 60,
162
+ 'kind' => 'Binning',
163
+ 'options' => { 'base' => 2.0, 'factor' => 0.1 },
164
+ )
165
+ metric_name = "req.sq.#{rule['name']}.#{whence}"
166
+ metrics_engine.create_metric(
167
+ 'name' => metric_name,
168
+ 'period' => 60,
169
+ 'kind' => 'Binning',
170
+ 'options' => { 'base' => 2.0, 'factor' => 0.1 },
171
+ )
172
+ end
97
173
  ### install callback, observing priority
98
174
  Sqreen::Weave.logger.debug { "Adding rule callback: #{rule_callback}" }
99
175
  @hooks << add_callback("weave,rule=#{rule['name']}", rule_callback, strategy)
@@ -107,30 +183,62 @@ class Sqreen::Weave::Legacy::Instrumentation
107
183
  end
108
184
 
109
185
  metrics_engine = self.metrics_engine
186
+ lvl_2_metric = @lvl_2_metric
187
+ lvl_2_max_reqs = @lvl_2_max_reqs
188
+
110
189
  request_hook = Sqreen::Graft::Hook['Sqreen::ShrinkWrap#call', strategy]
111
190
  @hooks << request_hook
112
191
  request_hook.add do
113
- before('wave,meta,request', rank: -100000, mandatory: true) do |_call|
192
+ before('wave,meta,request', rank: -100000, mandatory: true) do |call|
114
193
  next unless Sqreen.instrumentation_ready
115
194
 
116
- uuid = SecureRandom.uuid
117
- now = Sqreen::Graft::Timer.read
195
+ # shrinkwrap_timer = Sqreen::Graft::Timer.new('weave,shrinkwrap')
196
+ # shrinkwrap_timer.start
197
+
198
+ request_timer = Sqreen::Graft::Timer.new("request")
199
+ request_timer.start
200
+ sqreen_timer = Sqreen::Graft::Timer.new("sqreen")
201
+ budget = Sqreen::Weave::Budget.current
202
+
203
+ timed_level = (Sqreen.features['perf_level'] || 1).to_i
204
+ timed_level = 1 if !HAS_SQ_DETAILED_METRICS && timed_level == 2
205
+ if timed_level == 2 && lvl_2_metric.num_requests >= lvl_2_max_reqs
206
+ timed_level = 1
207
+ Sqreen::Weave.logger.debug { "Reducing timed level to 1 (#{lvl_2_metric.num_requests} reqs accumulated)" }
208
+ end
209
+
210
+ Sqreen::Weave.logger.debug { "request budget: #{budget} timed.level: #{timed_level}" } if Sqreen::Weave.logger.debug?
211
+
212
+ route_found = nil
213
+ if timed_level >= 2
214
+ rack_env, = call.args
215
+ rack_request = Rack::Request.new(rack_env) if rack_env
216
+
217
+ # TODO: Rails engines
218
+ # TODO: Struct
219
+ # TODO: Sinatra
220
+ # TODO: Rack?
221
+ Rails.application.routes.router.recognize(rack_request) do |route, params|
222
+ route = ActionDispatch::Routing::RouteWrapper.new(route)
223
+ route_found = { name: route.name, verb: route.verb, path: route.path, reqs: route.reqs, params: params }
224
+ end if defined?(Rails) && Rails.application && defined?(ActionDispatch::Routing::RouteWrapper)
225
+ end
226
+
227
+ # TODO: Struct
118
228
  Thread.current[:sqreen_http_request] = {
119
- uuid: uuid,
120
- start_time: now,
121
- time_budget: Sqreen.performance_budget,
229
+ request_timer: request_timer,
230
+ sqreen_timer: sqreen_timer,
122
231
  time_budget_expended: false,
123
- timer: Sqreen::Graft::Timer.new("request_#{uuid}"),
232
+ time_budget: budget,
124
233
  timed_callbacks: [],
125
234
  timed_hooks: [],
126
- timed_hooks_before: [],
127
- timed_hooks_after: [],
128
- timed_hooks_raised: [],
129
- timed_hooks_ensured: [],
235
+ timed_level: timed_level,
130
236
  skipped_callbacks: [],
237
+ route: ("#{route_found[:verb]} #{route_found[:path]}" if route_found),
238
+ # timed_shrinkwrap: shrinkwrap_timer,
131
239
  }
132
240
 
133
- Sqreen::Weave.logger.debug { "request.uuid: #{uuid}" }
241
+ # shrinkwrap_timer.stop
134
242
  end
135
243
 
136
244
  ensured('weave,meta,request', rank: 100000, mandatory: true) do |_call|
@@ -138,105 +246,89 @@ class Sqreen::Weave::Legacy::Instrumentation
138
246
 
139
247
  next if request.nil?
140
248
 
249
+ timed_level = request[:timed_level]
250
+ req_detailed = SqDetailedMetrics::Request.new if timed_level >= 2
251
+
252
+ # shrinkwrap_timer = request[:timed_shrinkwrap]
253
+ # shrinkwrap_timer.start
254
+
141
255
  Thread.current[:sqreen_http_request] = nil
142
- now = Sqreen::Graft::Timer.read
143
- utc_now = Time.now.utc
144
-
145
- request[:timed_callbacks].each do |timer|
146
- duration = timer.duration
147
- # stop = now
148
- # start = now - duration
149
- timer.tag =~ /weave,rule=(.*)$/ && rule = $1
150
- timer.tag =~ /@before/ && whence = 'pre'
151
- timer.tag =~ /@after/ && whence = 'post'
152
- timer.tag =~ /@raised/ && whence = 'failing'
153
-
154
- next unless rule && whence
155
-
156
- # Sqreen::PerformanceNotifications.notify(rule, whence, start, stop)
157
- # => BinnedMetrics
158
- metric_name = "sq.#{rule}.#{whence}"
159
- unless metrics_engine.metric?(metric_name)
160
- metrics_engine.create_metric(
161
- 'name' => metric_name,
162
- 'period' => 60,
163
- 'kind' => 'Binning',
164
- 'options' => { 'base' => 2.0, 'factor' => 0.1 },
165
- )
256
+ request_timer = request[:request_timer]
257
+ now = request_timer.stop
258
+
259
+ if timed_level >= 1
260
+ request[:timed_callbacks].each do |timer|
261
+ duration_ms = timer.duration * 1000.0
262
+ # XXX: the timer tag should have this structured data;
263
+ # it would be better than recomputing this for every measurement
264
+ metric_name = ::Sqreen::Weave::Legacy::Instrumentation.tag_to_metric_name(timer.tag)
265
+
266
+ next unless metric_name
267
+
268
+ metrics_engine.update(metric_name, now, nil, duration_ms)
269
+ duration_ms *= -1.0 if timer.conditions_passed
270
+ req_detailed.add_measurement metric_name, duration_ms if req_detailed
166
271
  end
167
- metrics_engine.update(metric_name, now, nil, duration * 1000)
168
272
  end
169
273
 
170
- metric_name = 'sq.hooks_pre.pre'
171
- duration = request[:timed_hooks_before].sum(&:duration)
172
- unless metrics_engine.metric?(metric_name)
173
- metrics_engine.create_metric(
174
- 'name' => metric_name,
175
- 'period' => 60,
176
- 'kind' => 'Binning',
177
- 'options' => { 'base' => 2.0, 'factor' => 0.1 },
178
- )
179
- end
180
- metrics_engine.update(metric_name, now, nil, duration * 1000)
181
-
182
- metric_name = 'sq.hooks_post.post'
183
- duration = request[:timed_hooks_after].sum(&:duration)
184
- unless metrics_engine.metric?(metric_name)
185
- metrics_engine.create_metric(
186
- 'name' => metric_name,
187
- 'period' => 60,
188
- 'kind' => 'Binning',
189
- 'options' => { 'base' => 2.0, 'factor' => 0.1 },
190
- )
191
- end
192
- metrics_engine.update(metric_name, now, nil, duration * 1000)
193
-
194
- metric_name = 'sq.hooks_failing.failing'
195
- duration = request[:timed_hooks_raised].sum(&:duration)
196
- unless metrics_engine.metric?(metric_name)
197
- metrics_engine.create_metric(
198
- 'name' => metric_name,
199
- 'period' => 60,
200
- 'kind' => 'Binning',
201
- 'options' => { 'base' => 2.0, 'factor' => 0.1 },
202
- )
274
+ sqreen_timer = request[:sqreen_timer]
275
+ Sqreen::Weave.logger.debug do
276
+ "request sqreen_timer.total: #{'%.03fus' % (sqreen_timer.duration * 1_000_000)}"
277
+ end if Sqreen::Weave.logger.debug?
278
+ Sqreen::Weave.logger.debug do
279
+ "request request_timer.total: #{'%.03fus' % (request_timer.duration * 1_000_000)}"
280
+ end if Sqreen::Weave.logger.debug?
281
+
282
+ if timed_level >= 1 && Sqreen::Weave.logger.debug?
283
+ skipped = request[:skipped_callbacks].map(&:name)
284
+ Sqreen::Weave.logger.debug { "request callback.skipped.count: #{skipped.count}" } if Sqreen::Weave.logger.debug?
285
+ timings = request[:timed_callbacks].map(&:to_s)
286
+ total = request[:timed_callbacks].sum(&:duration)
287
+ Sqreen::Weave.logger.debug { "request callback.total: #{'%.03fus' % (total * 1_000_000)} callback.count: #{timings.count}" } if Sqreen::Weave.logger.debug?
288
+ timings = request[:timed_hooks].map(&:to_s)
289
+ total = request[:timed_hooks].sum(&:duration)
290
+ Sqreen::Weave.logger.debug { "request hook.total: #{'%.03fus' % (total * 1_000_000)} hook.count: #{timings.count}" } if Sqreen::Weave.logger.debug?
203
291
  end
204
- metrics_engine.update(metric_name, now, nil, duration * 1000)
205
-
206
- skipped = request[:skipped_callbacks].map(&:name)
207
- Sqreen::Weave.logger.debug { "request:#{request[:uuid]} callback.skipped.size: #{skipped.count} callback.skipped: [#{skipped.join(', ')}]" }
208
- timer = request[:timer]
209
- total = timer.duration
210
- Sqreen::Weave.logger.debug { "request:#{request[:uuid]} timer.total: #{'%.03fus' % (total * 1_000_000)} timer.size: #{timer.size}" }
211
- timings = request[:timed_callbacks].map(&:to_s)
212
- total = request[:timed_callbacks].sum(&:duration)
213
- Sqreen::Weave.logger.debug { "request:#{request[:uuid]} callback.total: #{'%.03fus' % (total * 1_000_000)} callback.timings: [#{timings.join(', ')}]" }
214
- timings = request[:timed_hooks].map(&:to_s)
215
- total = request[:timed_hooks].sum(&:duration)
216
- Sqreen::Weave.logger.debug { "request:#{request[:uuid]} hook.total: #{'%.03fus' % (total * 1_000_000)} hook.timings: [#{timings.join(', ')}]" }
217
-
218
- skipped = request[:skipped_callbacks].map(&:name)
219
- skipped_rule_name = skipped.first && skipped.first =~ /weave,rule=(.*)$/ && $1
220
- Sqreen.observations_queue.push(['request_overtime', skipped_rule_name, 1, utc_now]) if skipped_rule_name
221
-
222
- sqreen_request_duration = total
223
- Sqreen.observations_queue.push(['sq', nil, sqreen_request_duration * 1000, utc_now])
224
-
225
- request_duration = now - request[:start_time]
226
- Sqreen.observations_queue.push(['req', nil, request_duration * 1000, utc_now])
292
+
293
+ overtime_cb = ::Sqreen::Weave::Legacy::Instrumentation.tag_to_metric_name(request[:overtime_cb]) \
294
+ if request[:overtime_cb]
295
+ metrics_engine.update('request_overtime', now, overtime_cb, 1) if overtime_cb
296
+
297
+ sqreen_request_duration = sqreen_timer.duration * 1000.0
298
+ metrics_engine.update('sq', now, nil, sqreen_request_duration)
299
+
300
+ request_duration = request_timer.duration * 1000.0
301
+ metrics_engine.update('req', now, nil, request_duration)
227
302
 
228
303
  sqreen_request_ratio = (sqreen_request_duration * 100.0) / (request_duration - sqreen_request_duration)
229
- Sqreen.observations_queue.push(['pct', nil, sqreen_request_ratio, utc_now])
304
+ metrics_engine.update('pct', now, nil, sqreen_request_ratio)
305
+ Sqreen::Weave.logger.debug { "request sqreen_timer.ratio: #{'%.03f' % (sqreen_request_ratio / 100.0)}" } if Sqreen::Weave.logger.debug?
306
+
307
+ if req_detailed
308
+ req_detailed.route = request[:route]
309
+ req_detailed.overtime_cb = overtime_cb if overtime_cb
310
+ req_detailed.add_measurement 'sq', sqreen_request_duration
311
+ req_detailed.add_measurement 'req', request_duration
312
+
313
+ metrics_engine.update(REQ_LVL_2_METRIC, now, nil, req_detailed)
314
+ end
315
+
316
+ # shrinkwrap_timer.stop
317
+
318
+ # duration = shrinkwrap_timer.duration
319
+ # metrics_engine.update('sq.shrinkwrap', now, nil, duration * 1000)
230
320
  end
231
321
  end.install
232
322
 
233
323
  ### globally declare instrumentation ready
234
324
  Sqreen.instrumentation_ready = true
325
+ Sqreen::Weave.logger.info { "Instrumentation activated" }
235
326
  end
236
327
 
237
328
  # needed by Sqreen::Runner
238
329
  def remove_all_callbacks
239
330
  Sqreen.instrumentation_ready = false
331
+ Sqreen::Weave.logger.info { "Instrumentation deactivated" }
240
332
 
241
333
  loop do
242
334
  hook = @hooks.pop
@@ -253,6 +345,15 @@ class Sqreen::Weave::Legacy::Instrumentation
253
345
  klass = callback.klass
254
346
  method = callback.method
255
347
 
348
+ if (call_count = ENV['SQREEN_DEBUG_CALL_COUNT'])
349
+ call_count = JSON.parse(call_count)
350
+ if callback.respond_to?(:rule_name) && call_count.key?(callback.rule_name)
351
+ count = call_count[callback.rule_name]
352
+ Sqreen::Weave.logger.debug { "override rule: #{callback.rule_name} call_count: #{count.inspect}" }
353
+ callback.instance_eval { @call_count_interval = call_count[callback.rule_name] }
354
+ end
355
+ end
356
+
256
357
  if Sqreen::Graft::HookPoint.new("#{klass}.#{method}").exist?
257
358
  hook_point = "#{klass}.#{method}"
258
359
  elsif Sqreen::Graft::HookPoint.new("#{klass}##{method}").exist?
@@ -268,14 +369,14 @@ class Sqreen::Weave::Legacy::Instrumentation
268
369
  hook = Sqreen::Graft::Hook[hook_point, strategy]
269
370
  hook.add do
270
371
  if callback.pre?
271
- before(rule, rank: priority, mandatory: !callback.overtimeable, flow: block, ignore: ignore) do |call, b|
372
+ use_flow = block || callback.is_a?(::Sqreen::Conditionable)
373
+ before(rule, rank: priority, mandatory: !callback.overtimeable, flow: use_flow, ignore: ignore) do |call, b|
272
374
  next unless Thread.current[:sqreen_http_request]
273
375
 
274
376
  i = call.instance
275
377
  a = call.args
276
378
  r = call.remaining
277
379
 
278
- Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#pre instance=#{i}" }
279
380
  begin
280
381
  ret = callback.pre(i, a, r)
281
382
  rescue StandardError => e
@@ -286,17 +387,30 @@ class Sqreen::Weave::Legacy::Instrumentation
286
387
  Sqreen::RemoteException.record(e)
287
388
  end
288
389
  end
289
- Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#pre instance=#{i} => return=#{ret.inspect}" }
290
-
291
- case ret[:status]
292
- when :skip, 'skip'
293
- throw(b, b.return(ret[:new_return_value]).break!) if ret.key?(:new_return_value)
294
- when :modify_args, 'modify_args'
295
- throw(b, b.args(ret[:args]))
296
- when :raise, 'raise'
297
- throw(b, b.raise(ret[:exception])) if ret.key?(:exception)
298
- throw(b, b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required.")))
299
- end unless ret.nil?
390
+
391
+ next if ret.nil? || !ret.is_a?(Hash)
392
+
393
+ throw_val =
394
+ case ret[:status]
395
+ when :skip, 'skip'
396
+ b.return(ret[:new_return_value]).break! if ret.key?(:new_return_value)
397
+ when :modify_args, 'modify_args'
398
+ b.args(ret[:args])
399
+ when :raise, 'raise'
400
+ if ret.key?(:exception)
401
+ b.raise(ret[:exception])
402
+ else
403
+ b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required."))
404
+ end
405
+ end if block
406
+
407
+ if ret && ret[:passed_conditions]
408
+ throw_val ||= b.noop
409
+ throw_val.passed_conditions!
410
+ end
411
+ next unless throw_val
412
+ throw_val.break! if ret[:skip_rem_cbs]
413
+ throw(b, throw_val)
300
414
  end
301
415
  end
302
416
 
@@ -309,7 +423,6 @@ class Sqreen::Weave::Legacy::Instrumentation
309
423
  a = call.args
310
424
  r = call.remaining
311
425
 
312
- Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#post instance=#{i}" }
313
426
  begin
314
427
  ret = callback.post(v, i, a, r)
315
428
  rescue StandardError => e
@@ -320,15 +433,22 @@ class Sqreen::Weave::Legacy::Instrumentation
320
433
  Sqreen::RemoteException.record(e)
321
434
  end
322
435
  end
323
- Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#post instance=#{i} => return=#{ret.inspect}" }
324
-
325
- case ret[:status]
326
- when :override, 'override'
327
- throw(b, b.return(ret[:new_return_value])) if ret.key?(:new_return_value)
328
- when :raise, 'raise'
329
- throw(b, b.raise(ret[:exception])) if ret.key?(:exception)
330
- throw(b, b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required.")))
331
- end unless ret.nil?
436
+
437
+ throw_val =
438
+ case ret[:status]
439
+ when :override, 'override'
440
+ b.return(ret[:new_return_value]) if ret.key?(:new_return_value)
441
+ when :raise, 'raise'
442
+ b.raise(ret[:exception]) if ret.key?(:exception)
443
+ b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required."))
444
+ end unless ret.nil? || !ret.is_a?(Hash) || !block
445
+
446
+ if ret && ret[:passed_conditions]
447
+ throw_val ||= b.noop
448
+ throw_val.passed_conditions!
449
+ end
450
+ next unless throw_val
451
+ throw(b, throw_val)
332
452
  end
333
453
  end
334
454
 
@@ -341,7 +461,6 @@ class Sqreen::Weave::Legacy::Instrumentation
341
461
  a = call.args
342
462
  r = call.remaining
343
463
 
344
- Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#failing instance=#{i}" }
345
464
  begin
346
465
  ret = callback.failing(e, i, a, r)
347
466
  rescue StandardError => e
@@ -352,23 +471,30 @@ class Sqreen::Weave::Legacy::Instrumentation
352
471
  Sqreen::RemoteException.record(e)
353
472
  end
354
473
  end
355
- Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#failing instance=#{i} => return=#{ret.inspect}" }
356
-
357
- raise e if ret.nil?
358
-
359
- case ret[:status]
360
- when :override, 'override'
361
- throw(b, b.return(ret[:new_return_value])) if ret.key?(:new_return_value)
362
- when :retry, 'retry'
363
- throw(b, b.retry)
364
- when :raise, 'raise'
365
- throw(b, b.raise(ret[:exception])) if ret.key?(:exception)
366
- throw(b, b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required.")))
367
- when :reraise, 'reraise'
368
- throw(b, b.raise(e))
369
- else
370
- throw(b, b.raise(e))
371
- end unless ret.nil?
474
+
475
+ throw(b, b.raise(e)) if ret.nil? || !ret.is_a?(Hash)
476
+
477
+ throw_val =
478
+ case ret[:status]
479
+ when :override, 'override'
480
+ b.return(ret[:new_return_value]) if ret.key?(:new_return_value)
481
+ when :retry, 'retry'
482
+ b.retry
483
+ when :raise, 'raise'
484
+ b.raise(ret[:exception]) if ret.key?(:exception)
485
+ b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required."))
486
+ when :reraise, 'reraise'
487
+ b.raise(e)
488
+ else
489
+ b.raise(e)
490
+ end unless ret.nil? || !ret.is_a?(Hash) || !block
491
+
492
+ if ret && ret[:passed_conditions]
493
+ throw_val ||= b.noop
494
+ throw_val.passed_conditions!
495
+ end
496
+ next unless throw_val
497
+ throw(b, throw_val)
372
498
  end
373
499
  end
374
500
  end.install
@@ -403,4 +529,20 @@ class Sqreen::Weave::Legacy::Instrumentation
403
529
  Sqreen::Rules::RunUserActions.new(Sqreen, :auth_track, 1),
404
530
  ]
405
531
  end
532
+
533
+ def self.tag_to_metric_name(tag)
534
+ cached = @cache_tag_to_metric[tag]
535
+ return cached unless cached.nil?
536
+
537
+ tag =~ /weave,rule=(.*)$/ && rule = $1 and # rubocop:disable Style/AndOr
538
+ (tag =~ /@before/ && whence = 'pre' or # rubocop:disable Style/AndOr
539
+ tag =~ /@after/ && whence = 'post' or # rubocop:disable Style/AndOr
540
+ tag =~ /@raised/ && whence = 'failing' or # rubocop:disable Style/AndOr
541
+ tag =~ /@ensured/ && whence = 'finally')
542
+
543
+ @cache_tag_to_metric[tag] =
544
+ rule && whence ? "sq.#{rule}.#{whence}" : false
545
+ end
546
+
547
+ @cache_tag_to_metric = {}
406
548
  end