sqreen 1.19.1 → 1.20.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (50) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +22 -0
  3. data/lib/sqreen/agent_message.rb +20 -0
  4. data/lib/sqreen/aggregated_metric.rb +25 -0
  5. data/lib/sqreen/attack_detected.html +1 -2
  6. data/lib/sqreen/ca.crt +24 -0
  7. data/lib/sqreen/configuration.rb +10 -4
  8. data/lib/sqreen/deferred_logger.rb +4 -0
  9. data/lib/sqreen/deliveries/batch.rb +4 -1
  10. data/lib/sqreen/deliveries/simple.rb +4 -0
  11. data/lib/sqreen/endpoint_testing.rb +184 -0
  12. data/lib/sqreen/event.rb +7 -5
  13. data/lib/sqreen/events/attack.rb +23 -18
  14. data/lib/sqreen/events/remote_exception.rb +0 -22
  15. data/lib/sqreen/events/request_record.rb +15 -70
  16. data/lib/sqreen/frameworks/request_recorder.rb +13 -2
  17. data/lib/sqreen/graft/call.rb +32 -19
  18. data/lib/sqreen/graft/callback.rb +1 -1
  19. data/lib/sqreen/graft/hook.rb +97 -116
  20. data/lib/sqreen/graft/hook_point.rb +1 -1
  21. data/lib/sqreen/kit/signals/specialized/aggregated_metric.rb +72 -0
  22. data/lib/sqreen/kit/signals/specialized/attack.rb +57 -0
  23. data/lib/sqreen/kit/signals/specialized/binning_metric.rb +76 -0
  24. data/lib/sqreen/kit/signals/specialized/http_trace.rb +26 -0
  25. data/lib/sqreen/kit/signals/specialized/sdk_track_call.rb +50 -0
  26. data/lib/sqreen/kit/signals/specialized/sqreen_exception.rb +57 -0
  27. data/lib/sqreen/legacy/instrumentation.rb +10 -10
  28. data/lib/sqreen/legacy/old_event_submission_strategy.rb +221 -0
  29. data/lib/sqreen/legacy/waf_redactions.rb +49 -0
  30. data/lib/sqreen/log/loggable.rb +2 -1
  31. data/lib/sqreen/logger.rb +4 -0
  32. data/lib/sqreen/metrics/base.rb +3 -0
  33. data/lib/sqreen/metrics_store.rb +22 -12
  34. data/lib/sqreen/performance_notifications/binned_metrics.rb +8 -2
  35. data/lib/sqreen/rules.rb +4 -2
  36. data/lib/sqreen/rules/not_found_cb.rb +2 -0
  37. data/lib/sqreen/rules/rule_cb.rb +2 -0
  38. data/lib/sqreen/rules/waf_cb.rb +13 -10
  39. data/lib/sqreen/runner.rb +75 -8
  40. data/lib/sqreen/sensitive_data_redactor.rb +19 -31
  41. data/lib/sqreen/session.rb +51 -43
  42. data/lib/sqreen/signals/conversions.rb +283 -0
  43. data/lib/sqreen/signals/http_trace_redaction.rb +111 -0
  44. data/lib/sqreen/signals/signals_submission_strategy.rb +78 -0
  45. data/lib/sqreen/version.rb +1 -1
  46. data/lib/sqreen/weave/legacy/instrumentation.rb +56 -53
  47. metadata +45 -7
  48. data/lib/sqreen/backport.rb +0 -9
  49. data/lib/sqreen/backport/clock_gettime.rb +0 -74
  50. data/lib/sqreen/backport/original_name.rb +0 -88
@@ -61,7 +61,7 @@ module Sqreen
61
61
  obj.each do |k, v|
62
62
  ck = k.is_a?(String) ? k.downcase : k
63
63
  if @keys.include?(ck)
64
- redacted << v
64
+ redacted += SensitiveDataRedactor.all_strings(v)
65
65
  v = MASK
66
66
  else
67
67
  v, r = redact(v)
@@ -74,39 +74,27 @@ module Sqreen
74
74
  [result, redacted]
75
75
  end
76
76
 
77
- def redact_attacks!(attacks, values)
78
- return attacks if values.empty?
79
-
80
- values = values.map { |v| v.downcase if v.is_a?(String) }
81
-
82
- attacks.each do |e|
83
- next(e) unless e[:infos]
84
- next(e) unless e[:infos][:waf_data]
85
-
86
- parsed = JSON.parse(e[:infos][:waf_data])
87
- redacted = parsed.each do |w|
88
- next unless (filters = w['filter'])
89
-
90
- filters.each do |f|
91
- next unless (v = f['resolved_value'])
92
- next unless values.include?(v.downcase)
77
+ class << self
78
+ def all_strings(v)
79
+ accum = []
80
+ all_strings_impl(v, accum)
81
+ accum
82
+ end
93
83
 
94
- f['match_status'] = MASK
95
- f['resolved_value'] = MASK
84
+ private
85
+
86
+ def all_strings_impl(obj, accum)
87
+ case obj
88
+ when String
89
+ accum << obj
90
+ when Array
91
+ obj.each { |el| all_strings_impl(el, accum) }
92
+ when Hash
93
+ obj.each do |k, v|
94
+ all_strings_impl(k, accum)
95
+ all_strings_impl(v, accum)
96
96
  end
97
97
  end
98
- e[:infos][:waf_data] = JSON.dump(redacted)
99
- end
100
- end
101
-
102
- def redact_exceptions!(exceptions, values)
103
- return exceptions if values.empty?
104
-
105
- exceptions.each do |e|
106
- next(e) unless e[:infos]
107
- next(e) unless e[:infos][:waf]
108
-
109
- e[:infos][:waf].delete(:args)
110
98
  end
111
99
  end
112
100
  end
@@ -11,6 +11,10 @@ require 'sqreen/events/attack'
11
11
  require 'sqreen/events/request_record'
12
12
  require 'sqreen/exception'
13
13
  require 'sqreen/safe_json'
14
+ require 'sqreen/kit'
15
+ require 'sqreen/kit/configuration'
16
+ require 'sqreen/signals/signals_submission_strategy'
17
+ require 'sqreen/legacy/old_event_submission_strategy'
14
18
 
15
19
  require 'net/https'
16
20
  require 'uri'
@@ -41,13 +45,12 @@ module Sqreen
41
45
  RETRY_MANY = 301
42
46
 
43
47
  MUTEX = Mutex.new
44
- METRICS_KEY = 'metrics'.freeze
45
48
 
46
49
  @@path_prefix = '/sqreen/v0/'
47
50
 
48
51
  attr_accessor :request_compression
49
52
 
50
- def initialize(server_url, token, app_name = nil)
53
+ def initialize(server_url, cert_store, token, app_name = nil, proxy_url = nil)
51
54
  @token = token
52
55
  @app_name = app_name
53
56
  @session_id = nil
@@ -59,15 +62,29 @@ module Sqreen
59
62
  uri = parse_uri(server_url)
60
63
  use_ssl = (uri.scheme == 'https')
61
64
 
65
+ proxy_params = []
66
+ if proxy_url
67
+ proxy_uri = parse_uri(proxy_url)
68
+ proxy_params = [proxy_uri.host, proxy_uri.port, proxy_uri.user, proxy_uri.password]
69
+ end
70
+
62
71
  @req_nb = 0
63
72
 
64
- @http = Net::HTTP.new(uri.host, uri.port)
73
+ @http = Net::HTTP.new(uri.host, uri.port, *proxy_params)
65
74
  @http.use_ssl = use_ssl
66
- if use_ssl
67
- cert_file = File.join(File.dirname(__FILE__), 'ca.crt')
68
- cert_store = OpenSSL::X509::Store.new
69
- cert_store.add_file cert_file
70
- @http.cert_store = cert_store
75
+ @http.verify_mode = OpenSSL::SSL::VERIFY_NONE if ENV['SQREEN_SSL_NO_VERIFY'] # for testing
76
+ @http.cert_store = cert_store if use_ssl
77
+ self.use_signals = false
78
+ end
79
+
80
+ def use_signals=(do_use)
81
+ return if do_use == @use_signals
82
+
83
+ @use_signals = do_use
84
+ if do_use
85
+ @evt_sub_strategy = Sqreen::Signals::SignalsSubmissionStrategy.new
86
+ else
87
+ @evt_sub_strategy = Sqreen::Legacy::OldEventSubmissionStrategy.new(method(:post))
71
88
  end
72
89
  end
73
90
 
@@ -218,10 +235,7 @@ module Sqreen
218
235
  end
219
236
 
220
237
  def login(framework)
221
- headers = {
222
- 'x-api-key' => @token,
223
- 'x-app-name' => @app_name || framework.application_name,
224
- }.reject { |k, v| v == nil }
238
+ headers = prelogin_auth_headers(framework)
225
239
 
226
240
  Sqreen.log.warn "Using app name: #{headers['x-app-name']}"
227
241
 
@@ -235,6 +249,8 @@ module Sqreen
235
249
  end
236
250
  Sqreen.log.info 'Login success.'
237
251
  @session_id = res['session_id']
252
+ Kit::Configuration.session_key = @session_id
253
+ Kit.reset
238
254
  Sqreen.log.debug { "received session_id #{@session_id}" }
239
255
  Sqreen.logged_in = true
240
256
  res
@@ -246,20 +262,24 @@ module Sqreen
246
262
 
247
263
  def heartbeat(cmd_res = {}, metrics = [])
248
264
  payload = {}
249
- payload['metrics'] = metrics unless metrics.nil? || metrics.empty?
265
+ unless metrics.nil? || metrics.empty?
266
+ # never reached with signals
267
+ payload['metrics'] = metrics.map do |m|
268
+ Sqreen::Legacy::EventToHash.convert_agg_metric(m)
269
+ end
270
+ end
250
271
  payload['command_results'] = cmd_res unless cmd_res.nil? || cmd_res.empty?
251
272
 
252
273
  post('app-beat', payload.empty? ? nil : payload, {}, RETRY_MANY)
253
274
  end
254
275
 
255
276
  def post_metrics(metrics)
256
- return if metrics.nil? || metrics.empty?
257
- payload = { METRICS_KEY => metrics }
258
- post(METRICS_KEY, payload, {}, RETRY_MANY)
277
+ @evt_sub_strategy.post_metrics(metrics)
259
278
  end
260
279
 
280
+ # XXX never called
261
281
  def post_attack(attack)
262
- post('attack', attack.to_hash, {}, RETRY_MANY)
282
+ @evt_sub_strategy.post_attack(attack)
263
283
  end
264
284
 
265
285
  def post_bundle(bundle_sig, dependencies)
@@ -271,33 +291,22 @@ module Sqreen
271
291
  end
272
292
 
273
293
  def post_request_record(request_record)
274
- post('request_record', request_record.to_hash, {}, RETRY_MANY)
294
+ @evt_sub_strategy.post_request_record(request_record)
275
295
  end
276
296
 
277
297
  # Post an exception to Sqreen for analysis
278
298
  # @param exception [RemoteException] Exception and context to be sent over
279
299
  def post_sqreen_exception(exception)
280
- post('sqreen_exception', exception.to_hash, {}, 5)
281
- rescue StandardError => e
282
- Sqreen.log.warn(format('Could not post exception (network down? %s) %s',
283
- e.inspect,
284
- exception.to_hash.inspect))
285
- nil
300
+ @evt_sub_strategy.post_sqreen_exception(exception)
286
301
  end
287
302
 
288
- BATCH_KEY = 'batch'.freeze
289
- EVENT_TYPE_KEY = 'event_type'.freeze
290
303
  def post_batch(events)
291
- batch = events.map do |event|
292
- h = event.to_hash
293
- h[EVENT_TYPE_KEY] = event_kind(event)
294
- h
295
- end
296
- Sqreen.log.debug do
297
- tally = Hash[events.group_by(&:class).map{ |k,v| [k, v.count] }]
298
- "Doing batch with the following tally of event types: #{tally}"
299
- end
300
- post(BATCH_KEY, { BATCH_KEY => batch }, {}, RETRY_MANY)
304
+ @evt_sub_strategy.post_batch(events)
305
+ end
306
+
307
+ def post_agent_message(framework, agent_message)
308
+ headers = prelogin_auth_headers(framework)
309
+ post('app_agent_message', agent_message.to_h, headers, 0)
301
310
  end
302
311
 
303
312
  # Perform agent logout
@@ -314,14 +323,13 @@ module Sqreen
314
323
  disconnect
315
324
  end
316
325
 
317
- protected
326
+ private
318
327
 
319
- def event_kind(event)
320
- case event
321
- when Sqreen::RemoteException then 'sqreen_exception'
322
- when Sqreen::Attack then 'attack'
323
- when Sqreen::RequestRecord then 'request_record'
324
- end
328
+ def prelogin_auth_headers(framework)
329
+ {
330
+ 'x-api-key' => @token,
331
+ 'x-app-name' => @app_name || framework.application_name,
332
+ }.reject { |_k, v| v == nil }
325
333
  end
326
334
  end
327
335
  end
@@ -0,0 +1,283 @@
1
+ require 'sqreen/version'
2
+ require 'sqreen/rules/rule_cb'
3
+ require 'sqreen/metrics/base'
4
+ require 'sqreen/metrics/binning'
5
+ require 'sqreen/signals/http_trace_redaction'
6
+ require 'sqreen/kit/signals/signal_attributes'
7
+ require 'sqreen/kit/signals/specialized/aggregated_metric'
8
+ require 'sqreen/kit/signals/specialized/attack'
9
+ require 'sqreen/kit/signals/specialized/binning_metric'
10
+ require 'sqreen/kit/signals/specialized/sqreen_exception'
11
+ require 'sqreen/kit/signals/specialized/http_trace'
12
+ require 'sqreen/kit/signals/specialized/sdk_track_call'
13
+
14
+ module Sqreen
15
+ module Signals
16
+ module Conversions # rubocop:disable Metrics/ModuleLength
17
+ class << self
18
+ # @param [Sqreen::AggregatedMetric] agg
19
+ # @return [Sqreen::Kit::Signals::Metric]
20
+ def convert_metric_sample(agg)
21
+ attrs = {
22
+ signal_name: "sq.agent.metric.#{agg.name}",
23
+ source: if agg.rule
24
+ "sqreen:rules:#{agg.rule.rulespack_id}:#{agg.rule.rule_name}"
25
+ else
26
+ agent_gen_source
27
+ end,
28
+ time: agg.finish,
29
+ }
30
+
31
+ if agg.metric.is_a?(Sqreen::Metric::Binning)
32
+ conv_binning_metric(agg, attrs)
33
+ else
34
+ conv_generic_metric(agg, attrs)
35
+ end
36
+ end
37
+
38
+ # @param [Sqreen::Attack] attack
39
+ # XXX: not used because we don't use Sqreen::Attack
40
+ def convert_attack(attack)
41
+ # no need to set actor/context as we only include them in request records/traces
42
+ Kit::Signals::Specialized::Attack.new(
43
+ signal_name: "sq.agent.attack.#{attack.attack_type}",
44
+ source: "sqreen:rule:#{attack.rulespack_id}:#{attack.rule_name}",
45
+ time: attack.time,
46
+ location: Kit::Signals::Location.new(stack_trace: attack.backtrace),
47
+ payload: Kit::Signals::Specialized::Attack::Payload.new(
48
+ test: attack.test?,
49
+ block: attack.block?,
50
+ infos: attack.infos
51
+ )
52
+ )
53
+ end
54
+
55
+ # see Sqreen::Rules::RuleCB.record_event
56
+ def convert_unstructured_attack(payload)
57
+ Kit::Signals::Specialized::Attack.new(
58
+ signal_name: "sq.agent.attack.#{payload[:attack_type]}",
59
+ source: "sqreen:rule:#{payload[:rulespack_id]}:#{payload[:rule_name]}",
60
+ time: payload[:time],
61
+ location: (Kit::Signals::Location.new(stack_trace: payload[:backtrace]) if payload[:backtrace]),
62
+ payload: Kit::Signals::Specialized::Attack::Payload.new(
63
+ test: payload[:test],
64
+ block: payload[:block],
65
+ infos: payload[:infos]
66
+ )
67
+ )
68
+ end
69
+
70
+ # @param [Sqreen::RemoteException] exception
71
+ # @return [Sqreen::Kit::Signals::Specialized::SqreenException]
72
+ def convert_exception(exception)
73
+ payload = exception.payload
74
+
75
+ infos = payload['client_ip'] ? { client_ip: payload['client_ip'] } : {}
76
+ infos.merge!(payload['infos'] || {})
77
+
78
+ Kit::Signals::Specialized::SqreenException.new(
79
+ source: if payload['rule_name']
80
+ "sqreen:rule:#{payload['rulespack_id']}:#{payload['rule_name']}"
81
+ else
82
+ agent_gen_source
83
+ end,
84
+ time: exception.time,
85
+ ruby_exception: payload['exception'],
86
+ infos: infos
87
+ )
88
+ end
89
+
90
+ # see Sqreen::Rules::RuleCB.record_exception
91
+ # @param [Hash] payload
92
+ # @return [Sqreen::Kit::Signals::Specialized::SqreenException]
93
+ def convert_unstructured_exception(payload)
94
+ Kit::Signals::Specialized::SqreenException.new(
95
+ source: "sqreen:rule:#{payload[:rulespack_id]}:#{payload[:rule_name]}",
96
+ time: payload[:time],
97
+ ruby_exception: payload[:exception],
98
+ infos: payload[:infos]
99
+ )
100
+ end
101
+
102
+ # @param [Sqreen::RequestRecord] req_rec
103
+ # @return [Sqreen::Kit::Signals::Specialized::HttpTrace]
104
+ def convert_req_record(req_rec)
105
+ payload = req_rec.payload
106
+
107
+ request_p = payload['request']
108
+ id_args = req_rec.last_identify_args
109
+ identifiers = id_args[0] if id_args
110
+ traits = id_args[1] if id_args
111
+
112
+ observed = payload[:observed] || {}
113
+ signals = []
114
+ signals += (observed[:attacks] || [])
115
+ .map { |att| convert_unstructured_attack(att) }
116
+ signals += (observed[:sqreen_exceptions] || [])
117
+ .map { |sq_exc| convert_unstructured_exception(sq_exc) }
118
+ signals += req_rec.processed_sdk_calls
119
+ .select { |h| h[:name] == :track }
120
+ .map { |h| convert_track(h) }
121
+
122
+ trace = Kit::Signals::Specialized::HttpTrace.new(
123
+ actor: Kit::Signals::Actor.new(
124
+ ip_addresses: [request_p[:client_ip]].compact,
125
+ user_agent: request_p[:user_agent],
126
+ identifiers: identifiers,
127
+ traits: traits,
128
+ ),
129
+ location_infra: location_infra,
130
+ context: convert_request(request_p,
131
+ payload['response'],
132
+ payload['headers'],
133
+ payload['params']),
134
+ data: signals
135
+ )
136
+ HttpTraceRedaction.redact_trace!(trace, req_rec.redactor)
137
+ trace
138
+ end
139
+
140
+ # @param [Array<Sqreen::Kit::Signals::Signal|Sqreen::Kit::Signals::Trace>] batch
141
+ def convert_batch(batch)
142
+ batch.map do |evt|
143
+ case evt
144
+ when RemoteException
145
+ convert_exception(evt)
146
+ when AggregatedMetric
147
+ convert_metric_sample(evt)
148
+ when RequestRecord
149
+ convert_req_record(evt)
150
+ else
151
+ raise NotImplementedError, "Unknown type of event in batch: #{evt}"
152
+ end
153
+ end
154
+ end
155
+
156
+ private
157
+
158
+ def agent_gen_source
159
+ "sqreen:agent:ruby:#{Sqreen::VERSION}"
160
+ end
161
+
162
+ def location_infra
163
+ @location_infra ||= begin
164
+ Kit::Signals::LocationInfra.new(
165
+ agent_version: Sqreen::VERSION,
166
+ os_type: RuntimeInfos.os[:os_type],
167
+ hostname: RuntimeInfos.hostname,
168
+ runtime_type: RuntimeInfos.runtime[:runtime_type],
169
+ runtime_version: RuntimeInfos.runtime[:runtime_version],
170
+ libsqreen_version: RuntimeInfos.libsqreen_version,
171
+ )
172
+ end
173
+ end
174
+
175
+ # see Sqreen::RequestRecord.processed_sdk_calls
176
+ def convert_track(call_info)
177
+ options = call_info[:args][1] || {}
178
+ Kit::Signals::Specialized::SdkTrackCall.new(
179
+ signal_name: "sq.sdk.#{call_info[:args][0]}",
180
+ time: call_info[:time],
181
+ payload: Kit::Signals::Specialized::SdkTrackCall::Payload.new(
182
+ properties: options[:properties],
183
+ user_identifiers: options[:user_identifiers]
184
+ )
185
+ )
186
+ end
187
+
188
+ # @param [Hash] req_payload
189
+ # @param [Hash] headers_payload
190
+ # @param [Hash] params_payload
191
+ # see the PayloadCreator abomination for reference
192
+ # TODO: do not convert from the old payload to the new payload
193
+ # Have an intermediate object that gets the data from the framework.
194
+ # (Or convert directly from the framework, but this needs to be
195
+ # done during the request, not just before event is transmitted)
196
+ def convert_request(req_payload, resp_payload, headers_payload, params_payload)
197
+ req_payload ||= {}
198
+ headers_payload ||= {}
199
+ resp_payload ||= {}
200
+ params_payload ||= {}
201
+
202
+ other = params_payload['other']
203
+ other = merge_hash_append(other, params_payload['rack'])
204
+ other = merge_hash_append(other, params_payload['grape_params'])
205
+ other = merge_hash_append(other, params_payload['rack_routing'])
206
+
207
+ Sqreen::Kit::Signals::Context::HttpContext.new(
208
+ {
209
+ rid: req_payload[:rid],
210
+ headers: headers_payload,
211
+ user_agent: req_payload[:user_agent],
212
+ scheme: req_payload[:scheme],
213
+ verb: req_payload[:verb],
214
+ host: req_payload[:host],
215
+ port: req_payload[:port],
216
+ remote_ip: req_payload[:remote_ip],
217
+ remote_port: req_payload[:remote_port] || 0,
218
+ path: req_payload[:path],
219
+ referer: req_payload[:referer],
220
+ params_query: params_payload['query'],
221
+ params_form: params_payload['form'],
222
+ params_other: other,
223
+ # endpoint, is_reveal_replayed not set
224
+ status: resp_payload[:status],
225
+ content_length: resp_payload[:content_length],
226
+ content_type: resp_payload[:content_type],
227
+ }
228
+ )
229
+ end
230
+
231
+ def merge_hash_append(hash1, hash2)
232
+ return nil if hash1.nil? && hash2.nil?
233
+ return hash1 if hash2.nil? || hash2.empty?
234
+ return hash2 if hash1.nil? || hash1.empty?
235
+
236
+ pairs = (hash1.keys + hash2.keys).map do |key|
237
+ values1 = hash1[key]
238
+ values2 = hash2[key]
239
+ values = [values1, values2].compact
240
+ values = values.first if values.size == 1
241
+ [key, values]
242
+ end
243
+ Hash[pairs]
244
+ end
245
+
246
+ # @param [Sqreen::AggregatedMetric] agg
247
+ # @param [Hash] attrs
248
+ def conv_generic_metric(agg, attrs)
249
+ attrs[:payload] = Kit::Signals::Specialized::AggregatedMetric::Payload.new(
250
+ kind: metric_kind(agg.metric),
251
+ capture_interval_s: agg.metric.period,
252
+ date_started: agg.start,
253
+ date_ended: agg.finish,
254
+ values: agg.data
255
+ )
256
+
257
+ Kit::Signals::Specialized::AggregatedMetric.new(attrs)
258
+ end
259
+
260
+ # @param [Sqreen::AggregatedMetric] agg
261
+ # @param [Hash] attrs
262
+ def conv_binning_metric(agg, attrs)
263
+ attrs[:payload] = Kit::Signals::Specialized::BinningMetric::Payload.new(
264
+ capture_interval_s: agg.metric.period,
265
+ date_started: agg.start,
266
+ date_ended: agg.finish,
267
+ base: agg.data['b'],
268
+ unit: agg.data['u'],
269
+ max: agg.data['v']['max'],
270
+ bins: agg.data['v'].reject { |k, _v| k == 'max' }
271
+ )
272
+
273
+ Kit::Signals::Specialized::BinningMetric.new(attrs)
274
+ end
275
+
276
+ # @param [Sqreen::Metric::Base] metric
277
+ def metric_kind(metric)
278
+ metric.class.name.sub(/.*::/, '').sub(/Metric$/, '')
279
+ end
280
+ end
281
+ end
282
+ end
283
+ end