sqreen 1.19.3-java → 1.21.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +38 -0
  3. data/lib/sqreen/actions/block_user.rb +1 -1
  4. data/lib/sqreen/actions/redirect_ip.rb +1 -1
  5. data/lib/sqreen/actions/redirect_user.rb +1 -1
  6. data/lib/sqreen/agent_message.rb +20 -0
  7. data/lib/sqreen/aggregated_metric.rb +25 -0
  8. data/lib/sqreen/attack_detected.html +1 -2
  9. data/lib/sqreen/ca.crt +24 -0
  10. data/lib/sqreen/condition_evaluator.rb +8 -2
  11. data/lib/sqreen/configuration.rb +11 -5
  12. data/lib/sqreen/deferred_logger.rb +50 -14
  13. data/lib/sqreen/deliveries/batch.rb +12 -2
  14. data/lib/sqreen/deliveries/simple.rb +4 -0
  15. data/lib/sqreen/deprecation.rb +38 -0
  16. data/lib/sqreen/ecosystem.rb +123 -0
  17. data/lib/sqreen/ecosystem/databases/database_connection_data.rb +23 -0
  18. data/lib/sqreen/ecosystem/databases/mongo.rb +39 -0
  19. data/lib/sqreen/ecosystem/databases/mysql.rb +54 -0
  20. data/lib/sqreen/ecosystem/databases/postgres.rb +51 -0
  21. data/lib/sqreen/ecosystem/databases/redis.rb +36 -0
  22. data/lib/sqreen/ecosystem/dispatch_table.rb +43 -0
  23. data/lib/sqreen/ecosystem/exception_reporting.rb +28 -0
  24. data/lib/sqreen/ecosystem/http/net_http.rb +50 -0
  25. data/lib/sqreen/ecosystem/http/rack_request.rb +39 -0
  26. data/lib/sqreen/ecosystem/loggable.rb +13 -0
  27. data/lib/sqreen/ecosystem/messaging/bunny.rb +61 -0
  28. data/lib/sqreen/ecosystem/messaging/kafka.rb +70 -0
  29. data/lib/sqreen/ecosystem/messaging/kinesis.rb +66 -0
  30. data/lib/sqreen/ecosystem/messaging/sqs.rb +68 -0
  31. data/lib/sqreen/ecosystem/module_api.rb +30 -0
  32. data/lib/sqreen/ecosystem/module_api/event_listener.rb +18 -0
  33. data/lib/sqreen/ecosystem/module_api/instrumentation.rb +23 -0
  34. data/lib/sqreen/ecosystem/module_api/message_producer.rb +57 -0
  35. data/lib/sqreen/ecosystem/module_api/signal_producer.rb +24 -0
  36. data/lib/sqreen/ecosystem/module_api/tracing.rb +45 -0
  37. data/lib/sqreen/ecosystem/module_api/tracing/client_data.rb +31 -0
  38. data/lib/sqreen/ecosystem/module_api/tracing/consumer_data.rb +13 -0
  39. data/lib/sqreen/ecosystem/module_api/tracing/messaging_data.rb +35 -0
  40. data/lib/sqreen/ecosystem/module_api/tracing/producer_data.rb +13 -0
  41. data/lib/sqreen/ecosystem/module_api/tracing/server_data.rb +27 -0
  42. data/lib/sqreen/ecosystem/module_api/tracing_id_generation.rb +16 -0
  43. data/lib/sqreen/ecosystem/module_api/transaction_storage.rb +71 -0
  44. data/lib/sqreen/ecosystem/module_registry.rb +48 -0
  45. data/lib/sqreen/ecosystem/tracing/modules/client.rb +35 -0
  46. data/lib/sqreen/ecosystem/tracing/modules/consumer.rb +35 -0
  47. data/lib/sqreen/ecosystem/tracing/modules/determine_ip.rb +28 -0
  48. data/lib/sqreen/ecosystem/tracing/modules/producer.rb +35 -0
  49. data/lib/sqreen/ecosystem/tracing/modules/server.rb +30 -0
  50. data/lib/sqreen/ecosystem/tracing/sampler.rb +160 -0
  51. data/lib/sqreen/ecosystem/tracing/sampling_configuration.rb +150 -0
  52. data/lib/sqreen/ecosystem/tracing/signals/tracing_client.rb +53 -0
  53. data/lib/sqreen/ecosystem/tracing/signals/tracing_consumer.rb +56 -0
  54. data/lib/sqreen/ecosystem/tracing/signals/tracing_producer.rb +56 -0
  55. data/lib/sqreen/ecosystem/tracing/signals/tracing_server.rb +53 -0
  56. data/lib/sqreen/ecosystem/tracing_broker.rb +101 -0
  57. data/lib/sqreen/ecosystem/tracing_id_setup.rb +34 -0
  58. data/lib/sqreen/ecosystem/transaction_storage.rb +64 -0
  59. data/lib/sqreen/ecosystem/util/call_writers_from_init.rb +13 -0
  60. data/lib/sqreen/ecosystem_integration.rb +81 -0
  61. data/lib/sqreen/ecosystem_integration/around_callbacks.rb +89 -0
  62. data/lib/sqreen/ecosystem_integration/instrumentation_service.rb +38 -0
  63. data/lib/sqreen/ecosystem_integration/request_lifecycle_tracking.rb +58 -0
  64. data/lib/sqreen/ecosystem_integration/signal_consumption.rb +35 -0
  65. data/lib/sqreen/endpoint_testing.rb +184 -0
  66. data/lib/sqreen/event.rb +7 -5
  67. data/lib/sqreen/events/attack.rb +23 -18
  68. data/lib/sqreen/events/remote_exception.rb +0 -22
  69. data/lib/sqreen/events/request_record.rb +15 -71
  70. data/lib/sqreen/frameworks/generic.rb +24 -1
  71. data/lib/sqreen/frameworks/rails.rb +0 -7
  72. data/lib/sqreen/frameworks/request_recorder.rb +15 -2
  73. data/lib/sqreen/graft/call.rb +85 -18
  74. data/lib/sqreen/graft/callback.rb +1 -1
  75. data/lib/sqreen/graft/hook.rb +192 -88
  76. data/lib/sqreen/graft/hook_point.rb +18 -11
  77. data/lib/sqreen/kit/signals/specialized/aggregated_metric.rb +72 -0
  78. data/lib/sqreen/kit/signals/specialized/attack.rb +57 -0
  79. data/lib/sqreen/kit/signals/specialized/binning_metric.rb +76 -0
  80. data/lib/sqreen/kit/signals/specialized/http_trace.rb +26 -0
  81. data/lib/sqreen/kit/signals/specialized/sdk_track_call.rb +50 -0
  82. data/lib/sqreen/kit/signals/specialized/sqreen_exception.rb +59 -0
  83. data/lib/sqreen/legacy/instrumentation.rb +22 -10
  84. data/lib/sqreen/legacy/old_event_submission_strategy.rb +228 -0
  85. data/lib/sqreen/legacy/waf_redactions.rb +49 -0
  86. data/lib/sqreen/log.rb +3 -2
  87. data/lib/sqreen/log/loggable.rb +2 -1
  88. data/lib/sqreen/logger.rb +24 -0
  89. data/lib/sqreen/metrics/base.rb +3 -0
  90. data/lib/sqreen/metrics_store.rb +33 -12
  91. data/lib/sqreen/null_logger.rb +22 -0
  92. data/lib/sqreen/performance_notifications/binned_metrics.rb +8 -2
  93. data/lib/sqreen/remote_command.rb +4 -0
  94. data/lib/sqreen/rules.rb +12 -6
  95. data/lib/sqreen/rules/blacklist_ips_cb.rb +2 -2
  96. data/lib/sqreen/rules/custom_error_cb.rb +3 -3
  97. data/lib/sqreen/rules/rule_cb.rb +4 -0
  98. data/lib/sqreen/rules/waf_cb.rb +14 -11
  99. data/lib/sqreen/runner.rb +122 -15
  100. data/lib/sqreen/sensitive_data_redactor.rb +19 -31
  101. data/lib/sqreen/session.rb +53 -43
  102. data/lib/sqreen/signals/conversions.rb +288 -0
  103. data/lib/sqreen/signals/http_trace_redaction.rb +111 -0
  104. data/lib/sqreen/signals/signals_submission_strategy.rb +78 -0
  105. data/lib/sqreen/version.rb +1 -1
  106. data/lib/sqreen/weave/budget.rb +46 -0
  107. data/lib/sqreen/weave/legacy/instrumentation.rb +194 -103
  108. data/lib/sqreen/worker.rb +6 -2
  109. metadata +96 -7
  110. data/lib/sqreen/backport.rb +0 -9
  111. data/lib/sqreen/backport/clock_gettime.rb +0 -74
  112. data/lib/sqreen/backport/original_name.rb +0 -88
  113. data/lib/sqreen/encoding_sanitizer.rb +0 -27
@@ -0,0 +1,111 @@
1
+ require 'json'
2
+ require 'sqreen/kit/loggable'
3
+ require 'sqreen/kit/signals/specialized/http_trace'
4
+
5
+ module Sqreen
6
+ module Signals
7
+ module HttpTraceRedaction
8
+ class << self
9
+ include Sqreen::Kit::Loggable
10
+
11
+ # @param [Sqreen::Kit::Signals::Specialized::HttpTrace] trace
12
+ # @param [Sqreen::SensitiveDataRedactor] redactor
13
+ def redact_trace!(trace, redactor)
14
+ return unless redactor
15
+ # redact headers (keys unsafe)
16
+ # @type [Sqreen::Kit::Signals::Context::HttpContext]
17
+ http_context = trace.context
18
+
19
+ all_redacted = []
20
+
21
+ # Redact headers; save redacted values
22
+ # headers are encoded as [key, value], not a hash, so
23
+ # they require some transformation
24
+ orig_headers = http_context.headers
25
+ if orig_headers
26
+ headers = orig_headers.map { |(k, v)| { k => v } }
27
+ headers, redacted = redactor.redact(headers)
28
+ http_context.headers = headers.map(&:first)
29
+ all_redacted += redacted
30
+ end
31
+
32
+ # Redact params; save redacted values
33
+ Kit::Signals::Context::HttpContext::PARAMS_ATTRS.each do |attr|
34
+ value = http_context.public_send(attr)
35
+ next unless value
36
+ value, redacted = redactor.redact(value)
37
+ all_redacted += redacted
38
+ http_context.public_send(:"#{attr}=", value)
39
+ end
40
+
41
+ all_redacted = all_redacted.uniq.map(&:downcase)
42
+
43
+ # Redact attacks and exceptions
44
+ # XXX: no redaction for infos in attacks/exceptions except for WAF data
45
+ # Is this the correct behavior?
46
+ redact_attacks!(trace, redactor, all_redacted)
47
+ redact_exceptions!(trace, redactor, all_redacted)
48
+ end
49
+
50
+ private
51
+
52
+ # @param [Sqreen::Kit::Signals::Specialized::HttpTrace] trace
53
+ # @param [Sqreen::SensitiveDataRedactor] redactor
54
+ # Redacts WAF data according to specific rules therefor
55
+ # Redacts infos according to general rules
56
+ def redact_attacks!(trace, redactor, redacted_data)
57
+ trace.data.each do |signal|
58
+ next unless signal.is_a?(Kit::Signals::Specialized::Attack)
59
+ # @type [Sqreen::Kit::Signals::Specialized::Attack::Payload] payload
60
+ payload = signal.payload
61
+ next unless payload.infos
62
+
63
+ if payload.infos[:waf_data]
64
+ redact_waf_attack_data!(payload.infos, redacted_data)
65
+ end
66
+ payload.infos, = redactor.redact(payload.infos)
67
+ end
68
+ end
69
+
70
+ def redact_exceptions!(trace, redactor, redacted_data)
71
+ trace.data.each do |signal|
72
+ next unless signal.is_a?(Kit::Signals::Specialized::SqreenException)
73
+ infos = signal.infos
74
+ next unless infos
75
+
76
+ redact_waf_exception_data!(signal.infos, redacted_data) if signal.infos[:waf]
77
+ signal.infos, = redactor.redact(infos)
78
+ end
79
+ end
80
+
81
+ # @param [Hash] infos from WAF attack
82
+ def redact_waf_attack_data!(infos, redacted_data)
83
+ begin
84
+ parsed = JSON.parse(infos[:waf_data])
85
+ rescue JSON::JSONError => e
86
+ logger.warn("waf_data is not valid json: #{e.message}")
87
+ return
88
+ end
89
+ redacted = parsed.each do |w|
90
+ next unless (filters = w['filter'])
91
+
92
+ filters.each do |f|
93
+ next unless (v = f['resolved_value'])
94
+ next unless redacted_data.include?(v.downcase)
95
+
96
+ f['match_status'] = SensitiveDataRedactor::MASK
97
+ f['resolved_value'] = SensitiveDataRedactor::MASK
98
+ end
99
+ end
100
+ infos[:waf_data] = JSON.dump(redacted)
101
+ end
102
+
103
+ # see https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000022-waf-data-sanitization.md#changes-to-the-agents
104
+ def redact_waf_exception_data!(infos, redacted_data)
105
+ return if redacted_data.empty?
106
+ infos[:waf].delete(:args)
107
+ end
108
+ end
109
+ end
110
+ end
111
+ end
@@ -0,0 +1,78 @@
1
+ require 'sqreen/aggregated_metric'
2
+ require 'sqreen/kit'
3
+ require 'sqreen/kit/string_sanitizer'
4
+ require 'sqreen/signals/conversions'
5
+ require 'sqreen/log/loggable'
6
+
7
+ module Sqreen
8
+ module Signals
9
+ # see also Sqreen::Legacy::OldEventSubmissionStrategy
10
+ # usage in Sqreen:Session
11
+ class SignalsSubmissionStrategy
12
+ include Sqreen::Log::Loggable
13
+
14
+ # @param [Array<Sqreen::AggregatedMetric>] metrics
15
+ def post_metrics(metrics)
16
+ return if metrics.nil? || metrics.empty?
17
+
18
+ guarded 'Failed to serialize or submit aggregated metrics' do
19
+ batch = metrics.map do |m|
20
+ Conversions.convert_metric_sample(m)
21
+ end
22
+ client.report_batch(batch)
23
+ end
24
+ end
25
+
26
+ # @param _attack [Sqreen::Attack]
27
+ # XXX: unused
28
+ def post_attack(_attack)
29
+ raise NotImplementedError
30
+ end
31
+
32
+ # @param request_record [Sqreen::RequestRecord]
33
+ def post_request_record(request_record)
34
+ guarded 'Failed to serialize or submit request record' do
35
+ trace = Conversions.convert_req_record(request_record)
36
+ append_sanitizing_filter(trace)
37
+ client.report_trace(trace)
38
+ end
39
+ end
40
+
41
+ # Post an exception to Sqreen for analysis
42
+ # @param exception [RemoteException] Exception and context to be sent over
43
+ def post_sqreen_exception(exception)
44
+ guarded 'Failed to serialize or submit exception', false do
45
+ data = Conversions.convert_exception(exception)
46
+ append_sanitizing_filter(data)
47
+ client.report_signal(data)
48
+ end
49
+ end
50
+
51
+ def post_batch(events)
52
+ guarded 'Failed to serialize or submit batch of events' do
53
+ batch = Conversions.convert_batch(events)
54
+ batch.each { |sig_or_trace| append_sanitizing_filter(sig_or_trace) }
55
+ client.report_batch(batch)
56
+ end
57
+ end
58
+
59
+ private
60
+
61
+ def append_sanitizing_filter(sig_or_trace)
62
+ sig_or_trace.append_to_h_filter Kit::StringSanitizer.method(:sanitize)
63
+ end
64
+
65
+ # we don't want exceptions to propagate and kill the worker thread
66
+ def guarded(msg, report = true)
67
+ yield
68
+ rescue StandardError => e
69
+ logger.warn "#{msg}: #{e.message}\n#{e.backtrace.map { |x| " #{x}" }.join("\n")}"
70
+ post_sqreen_exception(RemoteException.new(e)) if report
71
+ end
72
+
73
+ def client
74
+ Sqreen::Kit.auth_signals_client
75
+ end
76
+ end
77
+ end
78
+ end
@@ -4,5 +4,5 @@
4
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
5
5
 
6
6
  module Sqreen
7
- VERSION = '1.19.3'.freeze
7
+ VERSION = '1.21.0'.freeze
8
8
  end
@@ -0,0 +1,46 @@
1
+ # typed: false
2
+
3
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
4
+ # Please refer to our terms for more information: https://www.sqreen.com/terms.html
5
+
6
+ require 'sqreen/log/loggable'
7
+ require 'sqreen/weave'
8
+
9
+ class Sqreen::Weave::Budget
10
+ include Sqreen::Log::Loggable
11
+
12
+ def initialize(threshold, ratio = nil)
13
+ @threshold = threshold
14
+ @ratio = ratio
15
+ end
16
+
17
+ def static?
18
+ threshold && !ratio
19
+ end
20
+
21
+ def dynamic?
22
+ threshold && ratio
23
+ end
24
+
25
+ attr_reader :threshold
26
+ attr_reader :ratio
27
+
28
+ def to_h
29
+ { threshold: threshold, ratio: ratio }
30
+ end
31
+
32
+ class << self
33
+ attr_reader :current
34
+
35
+ def update(opts = nil)
36
+ Sqreen::Weave.logger.info("budget update:#{opts.inspect}")
37
+
38
+ return @current = nil if opts.nil? || opts.empty?
39
+
40
+ threshold = opts[:threshold]
41
+ ratio = opts[:ratio]
42
+
43
+ @current = new(threshold, ratio)
44
+ end
45
+ end
46
+ end
@@ -4,10 +4,13 @@
4
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
5
5
 
6
6
  require 'sqreen/weave/legacy'
7
+ require 'sqreen/weave/budget'
8
+ require 'sqreen/graft/hook'
7
9
  require 'sqreen/graft/hook_point'
8
10
  require 'sqreen/call_countable'
9
11
  require 'sqreen/rules'
10
12
  require 'sqreen/rules/record_request_context'
13
+ require 'sqreen/sqreen_signed_verifier'
11
14
 
12
15
  class Sqreen::Weave::Legacy::Instrumentation
13
16
  attr_accessor :metrics_engine
@@ -60,6 +63,27 @@ class Sqreen::Weave::Legacy::Instrumentation
60
63
  'options' => opts[:perf_metric_percent] || { 'base' => 1.3, 'factor' => 1.0 },
61
64
  )
62
65
 
66
+ metrics_engine.create_metric(
67
+ 'name' => 'req.sq.hook.overhead',
68
+ 'period' => 60,
69
+ 'kind' => 'Binning',
70
+ 'options' => { 'base' => 2.0, 'factor' => 0.1 },
71
+ )
72
+
73
+ metrics_engine.create_metric(
74
+ 'name' => 'sq.hook.overhead',
75
+ 'period' => 60,
76
+ 'kind' => 'Binning',
77
+ 'options' => { 'base' => 2.0, 'factor' => 0.1 },
78
+ )
79
+
80
+ metrics_engine.create_metric(
81
+ 'name' => 'sq.shrinkwrap',
82
+ 'period' => 60,
83
+ 'kind' => 'Binning',
84
+ 'options' => { 'base' => 2.0, 'factor' => 0.1 },
85
+ )
86
+
63
87
  Sqreen.thread_cpu_time? && metrics_engine.create_metric(
64
88
  'name' => 'sq_thread_cpu_pct',
65
89
  'period' => opts[:period] || 60,
@@ -84,6 +108,15 @@ class Sqreen::Weave::Legacy::Instrumentation
84
108
 
85
109
  ### set up rule signature verifier
86
110
  verifier = nil
111
+ if Sqreen.features['rules_signature'] &&
112
+ Sqreen.config_get(:rules_verify_signature) == true &&
113
+ !defined?(::JRUBY_VERSION)
114
+ verifier = Sqreen::SqreenSignedVerifier.new
115
+ Sqreen::Weave.logger.debug('Rules signature enabled')
116
+ else
117
+ Sqreen::Weave.logger.debug('Rules signature disabled')
118
+ end
119
+
87
120
  ### force clean instrumentation callback list
88
121
  @hooks = []
89
122
  ### for each rule description
@@ -94,6 +127,25 @@ class Sqreen::Weave::Legacy::Instrumentation
94
127
  next unless rule_callback
95
128
  ### attach framework to callback
96
129
  rule_callback.framework = framework
130
+ ## create metric
131
+ Sqreen::Weave.logger.debug { "Adding rule metric: #{rule_callback}" }
132
+ [:pre, :post, :failing].each do |whence|
133
+ next unless rule_callback.send(:"#{whence}?")
134
+ metric_name = "sq.#{rule['name']}.#{whence}"
135
+ metrics_engine.create_metric(
136
+ 'name' => metric_name,
137
+ 'period' => 60,
138
+ 'kind' => 'Binning',
139
+ 'options' => { 'base' => 2.0, 'factor' => 0.1 },
140
+ )
141
+ metric_name = "req.sq.#{rule['name']}.#{whence}"
142
+ metrics_engine.create_metric(
143
+ 'name' => metric_name,
144
+ 'period' => 60,
145
+ 'kind' => 'Binning',
146
+ 'options' => { 'base' => 2.0, 'factor' => 0.1 },
147
+ )
148
+ end
97
149
  ### install callback, observing priority
98
150
  Sqreen::Weave.logger.debug { "Adding rule callback: #{rule_callback}" }
99
151
  @hooks << add_callback("weave,rule=#{rule['name']}", rule_callback, strategy)
@@ -107,30 +159,43 @@ class Sqreen::Weave::Legacy::Instrumentation
107
159
  end
108
160
 
109
161
  metrics_engine = self.metrics_engine
162
+
110
163
  request_hook = Sqreen::Graft::Hook['Sqreen::ShrinkWrap#call', strategy]
111
164
  @hooks << request_hook
112
165
  request_hook.add do
113
166
  before('wave,meta,request', rank: -100000, mandatory: true) do |_call|
114
167
  next unless Sqreen.instrumentation_ready
115
168
 
116
- uuid = SecureRandom.uuid
117
- now = Sqreen::Graft::Timer.read
169
+ # shrinkwrap_timer = Sqreen::Graft::Timer.new('weave,shrinkwrap')
170
+ # shrinkwrap_timer.start
171
+
172
+ request_timer = Sqreen::Graft::Timer.new("request")
173
+ request_timer.start
174
+ sqreen_timer = Sqreen::Graft::Timer.new("sqreen")
175
+ budget = Sqreen::Weave::Budget.current
176
+ request_budget_threshold = budget.threshold if budget
177
+ request_budget_ratio = budget.ratio if budget
178
+ request_budget_is_dynamic = !request_budget_ratio.nil?
179
+ request_budget = !request_budget_threshold.nil?
180
+ timed_level = (Sqreen.features['perf_level'] || 1).to_i
181
+ Sqreen::Weave.logger.debug { "request budget: #{budget.to_h} timed.level: #{timed_level}" } if Sqreen::Weave.logger.debug?
182
+
118
183
  Thread.current[:sqreen_http_request] = {
119
- uuid: uuid,
120
- start_time: now,
121
- time_budget: Sqreen.performance_budget,
184
+ request_timer: request_timer,
185
+ sqreen_timer: sqreen_timer,
122
186
  time_budget_expended: false,
123
- timer: Sqreen::Graft::Timer.new("request_#{uuid}"),
187
+ time_budget_threshold: request_budget_threshold,
188
+ time_budget_dynamic: request_budget_is_dynamic,
189
+ time_budget_ratio: request_budget_ratio,
190
+ time_budget: request_budget,
124
191
  timed_callbacks: [],
125
192
  timed_hooks: [],
126
- timed_hooks_before: [],
127
- timed_hooks_after: [],
128
- timed_hooks_raised: [],
129
- timed_hooks_ensured: [],
193
+ timed_level: timed_level,
130
194
  skipped_callbacks: [],
195
+ # timed_shrinkwrap: shrinkwrap_timer,
131
196
  }
132
197
 
133
- Sqreen::Weave.logger.debug { "request.uuid: #{uuid}" }
198
+ # shrinkwrap_timer.stop
134
199
  end
135
200
 
136
201
  ensured('weave,meta,request', rank: 100000, mandatory: true) do |_call|
@@ -138,105 +203,118 @@ class Sqreen::Weave::Legacy::Instrumentation
138
203
 
139
204
  next if request.nil?
140
205
 
206
+ # shrinkwrap_timer = request[:timed_shrinkwrap]
207
+ # shrinkwrap_timer.start
208
+
141
209
  Thread.current[:sqreen_http_request] = nil
142
- now = Sqreen::Graft::Timer.read
143
- utc_now = Time.now.utc
144
-
145
- request[:timed_callbacks].each do |timer|
146
- duration = timer.duration
147
- # stop = now
148
- # start = now - duration
149
- timer.tag =~ /weave,rule=(.*)$/ && rule = $1
150
- timer.tag =~ /@before/ && whence = 'pre'
151
- timer.tag =~ /@after/ && whence = 'post'
152
- timer.tag =~ /@raised/ && whence = 'failing'
153
-
154
- next unless rule && whence
155
-
156
- # Sqreen::PerformanceNotifications.notify(rule, whence, start, stop)
157
- # => BinnedMetrics
158
- metric_name = "sq.#{rule}.#{whence}"
159
- unless metrics_engine.metric?(metric_name)
160
- metrics_engine.create_metric(
161
- 'name' => metric_name,
162
- 'period' => 60,
163
- 'kind' => 'Binning',
164
- 'options' => { 'base' => 2.0, 'factor' => 0.1 },
165
- )
210
+ request_timer = request[:request_timer]
211
+ now = request_timer.stop
212
+
213
+ if request[:timed_level] >= 1
214
+ request[:timed_callbacks].each do |timer|
215
+ duration = timer.duration
216
+
217
+ timer.tag =~ /weave,rule=(.*)$/ && rule = $1
218
+ next unless rule
219
+
220
+ whence = case timer.tag
221
+ when /@before/ then 'pre'
222
+ when /@after/ then 'post'
223
+ when /@raised/ then 'failing'
224
+ end
225
+ next unless whence
226
+
227
+ metric_name = "sq.#{rule}.#{whence}"
228
+ metrics_engine.update(metric_name, now, nil, duration * 1000)
229
+ # Sqreen.observations_queue.push([metric_name, nil, duration * 1000, utc_now])
166
230
  end
167
- metrics_engine.update(metric_name, now, nil, duration * 1000)
168
- end
169
231
 
170
- metric_name = 'sq.hooks_pre.pre'
171
- duration = request[:timed_hooks_before].sum(&:duration)
172
- unless metrics_engine.metric?(metric_name)
173
- metrics_engine.create_metric(
174
- 'name' => metric_name,
175
- 'period' => 60,
176
- 'kind' => 'Binning',
177
- 'options' => { 'base' => 2.0, 'factor' => 0.1 },
178
- )
179
- end
180
- metrics_engine.update(metric_name, now, nil, duration * 1000)
181
-
182
- metric_name = 'sq.hooks_post.post'
183
- duration = request[:timed_hooks_after].sum(&:duration)
184
- unless metrics_engine.metric?(metric_name)
185
- metrics_engine.create_metric(
186
- 'name' => metric_name,
187
- 'period' => 60,
188
- 'kind' => 'Binning',
189
- 'options' => { 'base' => 2.0, 'factor' => 0.1 },
190
- )
191
- end
192
- metrics_engine.update(metric_name, now, nil, duration * 1000)
193
-
194
- metric_name = 'sq.hooks_failing.failing'
195
- duration = request[:timed_hooks_raised].sum(&:duration)
196
- unless metrics_engine.metric?(metric_name)
197
- metrics_engine.create_metric(
198
- 'name' => metric_name,
199
- 'period' => 60,
200
- 'kind' => 'Binning',
201
- 'options' => { 'base' => 2.0, 'factor' => 0.1 },
202
- )
232
+ request[:timed_hooks].each do |timer|
233
+ duration = timer.duration
234
+ metrics_engine.update('sq.hook.overhead', now, nil, duration * 1000)
235
+ # Sqreen.observations_queue.push(['sq.hook.overhead', nil, duration * 1000, utc_now])
236
+ end
203
237
  end
204
- metrics_engine.update(metric_name, now, nil, duration * 1000)
205
238
 
206
- skipped = request[:skipped_callbacks].map(&:name)
207
- Sqreen::Weave.logger.debug { "request:#{request[:uuid]} callback.skipped.size: #{skipped.count} callback.skipped: [#{skipped.join(', ')}]" }
208
- timer = request[:timer]
209
- total = timer.duration
210
- Sqreen::Weave.logger.debug { "request:#{request[:uuid]} timer.total: #{'%.03fus' % (total * 1_000_000)} timer.size: #{timer.size}" }
211
- timings = request[:timed_callbacks].map(&:to_s)
212
- total = request[:timed_callbacks].sum(&:duration)
213
- Sqreen::Weave.logger.debug { "request:#{request[:uuid]} callback.total: #{'%.03fus' % (total * 1_000_000)} callback.timings: [#{timings.join(', ')}]" }
214
- timings = request[:timed_hooks].map(&:to_s)
215
- total = request[:timed_hooks].sum(&:duration)
216
- Sqreen::Weave.logger.debug { "request:#{request[:uuid]} hook.total: #{'%.03fus' % (total * 1_000_000)} hook.timings: [#{timings.join(', ')}]" }
239
+ sqreen_timer = request[:sqreen_timer]
240
+ total = sqreen_timer.duration
241
+ Sqreen::Weave.logger.debug { "request sqreen_timer.total: #{'%.03fus' % (total * 1_000_000)}" } if Sqreen::Weave.logger.debug?
242
+ total = request_timer.duration
243
+ Sqreen::Weave.logger.debug { "request request_timer.total: #{'%.03fus' % (total * 1_000_000)}" } if Sqreen::Weave.logger.debug?
244
+
245
+ if request[:timed_level] >= 2
246
+ skipped = request[:skipped_callbacks].map(&:name)
247
+ Sqreen::Weave.logger.debug { "request callback.skipped.count: #{skipped.count}" } if Sqreen::Weave.logger.debug?
248
+ timings = request[:timed_callbacks].map(&:to_s)
249
+ total = request[:timed_callbacks].sum(&:duration)
250
+ Sqreen::Weave.logger.debug { "request callback.total: #{'%.03fus' % (total * 1_000_000)} callback.count: #{timings.count}" } if Sqreen::Weave.logger.debug?
251
+ timings = request[:timed_hooks].map(&:to_s)
252
+ total = request[:timed_hooks].sum(&:duration)
253
+ Sqreen::Weave.logger.debug { "request hook.total: #{'%.03fus' % (total * 1_000_000)} hook.count: #{timings.count}" } if Sqreen::Weave.logger.debug?
254
+ end
217
255
 
218
256
  skipped = request[:skipped_callbacks].map(&:name)
219
257
  skipped_rule_name = skipped.first && skipped.first =~ /weave,rule=(.*)$/ && $1
220
- Sqreen.observations_queue.push(['request_overtime', skipped_rule_name, 1, utc_now]) if skipped_rule_name
258
+ metrics_engine.update('request_overtime', now, skipped_rule_name, 1) if skipped_rule_name
259
+ # Sqreen.observations_queue.push(['request_overtime', skipped_rule_name, 1, utc_now]) if skipped_rule_name
221
260
 
222
- sqreen_request_duration = total
223
- Sqreen.observations_queue.push(['sq', nil, sqreen_request_duration * 1000, utc_now])
261
+ sqreen_request_duration = sqreen_timer.duration
262
+ metrics_engine.update('sq', now, nil, sqreen_request_duration * 1000)
263
+ # Sqreen.observations_queue.push(['sq', nil, sqreen_request_duration * 1000, utc_now])
224
264
 
225
- request_duration = now - request[:start_time]
226
- Sqreen.observations_queue.push(['req', nil, request_duration * 1000, utc_now])
265
+ request_duration = request_timer.duration
266
+ metrics_engine.update('req', now, nil, request_duration * 1000)
267
+ # Sqreen.observations_queue.push(['req', nil, request_duration * 1000, utc_now])
227
268
 
228
269
  sqreen_request_ratio = (sqreen_request_duration * 100.0) / (request_duration - sqreen_request_duration)
229
- Sqreen.observations_queue.push(['pct', nil, sqreen_request_ratio, utc_now])
270
+ metrics_engine.update('pct', now, nil, sqreen_request_ratio)
271
+ # Sqreen.observations_queue.push(['pct', nil, sqreen_request_ratio, utc_now])
272
+ Sqreen::Weave.logger.debug { "request sqreen_timer.ratio: #{'%.03f' % (sqreen_request_ratio / 100.0)}" } if Sqreen::Weave.logger.debug?
273
+
274
+ if request[:timed_level] >= 2
275
+ tallies = Hash.new(0.0)
276
+ request[:timed_callbacks].each do |timer|
277
+ duration = timer.duration
278
+
279
+ timer.tag =~ /weave,rule=(.*)$/ && rule = $1
280
+ next unless rule
281
+
282
+ whence = case timer.tag
283
+ when /@before/ then 'pre'
284
+ when /@after/ then 'post'
285
+ when /@raised/ then 'failing'
286
+ end
287
+ next unless whence
288
+
289
+ metric_name = "req.sq.#{rule}.#{whence}"
290
+ tallies[metric_name] += duration
291
+ end
292
+ tallies.each do |metric_name, duration|
293
+ metrics_engine.update(metric_name, now, nil, duration * 1000)
294
+ # Sqreen.observations_queue.push([metric_name, nil, duration * 1000, utc_now])
295
+ end
296
+
297
+ duration = request[:timed_hooks].sum(&:duration)
298
+ metrics_engine.update('req.sq.hook.overhead', now, nil, duration * 1000)
299
+ # Sqreen.observations_queue.push(['req.sq.hook.overhead', nil, duration * 1000, utc_now])
300
+ end
301
+
302
+ # shrinkwrap_timer.stop
303
+
304
+ # duration = shrinkwrap_timer.duration
305
+ # metrics_engine.update('sq.shrinkwrap', now, nil, duration * 1000)
230
306
  end
231
307
  end.install
232
308
 
233
309
  ### globally declare instrumentation ready
234
310
  Sqreen.instrumentation_ready = true
311
+ Sqreen::Weave.logger.info { "Instrumentation activated" }
235
312
  end
236
313
 
237
314
  # needed by Sqreen::Runner
238
315
  def remove_all_callbacks
239
316
  Sqreen.instrumentation_ready = false
317
+ Sqreen::Weave.logger.info { "Instrumentation deactivated" }
240
318
 
241
319
  loop do
242
320
  hook = @hooks.pop
@@ -253,6 +331,15 @@ class Sqreen::Weave::Legacy::Instrumentation
253
331
  klass = callback.klass
254
332
  method = callback.method
255
333
 
334
+ if (call_count = ENV['SQREEN_DEBUG_CALL_COUNT'])
335
+ call_count = JSON.parse(call_count)
336
+ if callback.respond_to?(:rule_name) && call_count.key?(callback.rule_name)
337
+ count = call_count[callback.rule_name]
338
+ Sqreen::Weave.logger.debug { "override rule: #{callback.rule_name} call_count: #{count.inspect}" }
339
+ callback.instance_eval { @call_count_interval = call_count[callback.rule_name] }
340
+ end
341
+ end
342
+
256
343
  if Sqreen::Graft::HookPoint.new("#{klass}.#{method}").exist?
257
344
  hook_point = "#{klass}.#{method}"
258
345
  elsif Sqreen::Graft::HookPoint.new("#{klass}##{method}").exist?
@@ -275,7 +362,6 @@ class Sqreen::Weave::Legacy::Instrumentation
275
362
  a = call.args
276
363
  r = call.remaining
277
364
 
278
- Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#pre instance=#{i}" }
279
365
  begin
280
366
  ret = callback.pre(i, a, r)
281
367
  rescue StandardError => e
@@ -286,17 +372,26 @@ class Sqreen::Weave::Legacy::Instrumentation
286
372
  Sqreen::RemoteException.record(e)
287
373
  end
288
374
  end
289
- Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#pre instance=#{i} => return=#{ret.inspect}" }
290
375
 
291
- case ret[:status]
292
- when :skip, 'skip'
293
- throw(b, b.return(ret[:new_return_value]).break!) if ret.key?(:new_return_value)
294
- when :modify_args, 'modify_args'
295
- throw(b, b.args(ret[:args]))
296
- when :raise, 'raise'
297
- throw(b, b.raise(ret[:exception])) if ret.key?(:exception)
298
- throw(b, b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required.")))
299
- end unless ret.nil? || !ret.is_a?(Hash)
376
+ next if ret.nil? || !ret.is_a?(Hash)
377
+
378
+ throw_val =
379
+ case ret[:status]
380
+ when :skip, 'skip'
381
+ b.return(ret[:new_return_value]).break! if ret.key?(:new_return_value)
382
+ when :modify_args, 'modify_args'
383
+ b.args(ret[:args])
384
+ when :raise, 'raise'
385
+ if ret.key?(:exception)
386
+ b.raise(ret[:exception])
387
+ else
388
+ b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required."))
389
+ end
390
+ end
391
+
392
+ next unless throw_val
393
+ throw_val.break! if ret[:skip_rem_cbs]
394
+ throw(b, throw_val)
300
395
  end
301
396
  end
302
397
 
@@ -309,7 +404,6 @@ class Sqreen::Weave::Legacy::Instrumentation
309
404
  a = call.args
310
405
  r = call.remaining
311
406
 
312
- Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#post instance=#{i}" }
313
407
  begin
314
408
  ret = callback.post(v, i, a, r)
315
409
  rescue StandardError => e
@@ -320,7 +414,6 @@ class Sqreen::Weave::Legacy::Instrumentation
320
414
  Sqreen::RemoteException.record(e)
321
415
  end
322
416
  end
323
- Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#post instance=#{i} => return=#{ret.inspect}" }
324
417
 
325
418
  case ret[:status]
326
419
  when :override, 'override'
@@ -341,7 +434,6 @@ class Sqreen::Weave::Legacy::Instrumentation
341
434
  a = call.args
342
435
  r = call.remaining
343
436
 
344
- Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#failing instance=#{i}" }
345
437
  begin
346
438
  ret = callback.failing(e, i, a, r)
347
439
  rescue StandardError => e
@@ -352,7 +444,6 @@ class Sqreen::Weave::Legacy::Instrumentation
352
444
  Sqreen::RemoteException.record(e)
353
445
  end
354
446
  end
355
- Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#failing instance=#{i} => return=#{ret.inspect}" }
356
447
 
357
448
  throw(b, b.raise(e)) if ret.nil? || !ret.is_a?(Hash)
358
449