sqreen 1.19.0 → 1.20.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (42) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +22 -0
  3. data/lib/sqreen/agent_message.rb +20 -0
  4. data/lib/sqreen/aggregated_metric.rb +25 -0
  5. data/lib/sqreen/ca.crt +24 -0
  6. data/lib/sqreen/configuration.rb +10 -4
  7. data/lib/sqreen/deliveries/batch.rb +4 -1
  8. data/lib/sqreen/deliveries/simple.rb +4 -0
  9. data/lib/sqreen/endpoint_testing.rb +184 -0
  10. data/lib/sqreen/event.rb +7 -5
  11. data/lib/sqreen/events/attack.rb +23 -18
  12. data/lib/sqreen/events/remote_exception.rb +0 -22
  13. data/lib/sqreen/events/request_record.rb +15 -70
  14. data/lib/sqreen/frameworks/request_recorder.rb +13 -2
  15. data/lib/sqreen/kit/signals/specialized/aggregated_metric.rb +72 -0
  16. data/lib/sqreen/kit/signals/specialized/attack.rb +57 -0
  17. data/lib/sqreen/kit/signals/specialized/binning_metric.rb +76 -0
  18. data/lib/sqreen/kit/signals/specialized/http_trace.rb +26 -0
  19. data/lib/sqreen/kit/signals/specialized/sdk_track_call.rb +50 -0
  20. data/lib/sqreen/kit/signals/specialized/sqreen_exception.rb +57 -0
  21. data/lib/sqreen/legacy/old_event_submission_strategy.rb +221 -0
  22. data/lib/sqreen/legacy/waf_redactions.rb +49 -0
  23. data/lib/sqreen/log/loggable.rb +1 -1
  24. data/lib/sqreen/metrics/base.rb +3 -0
  25. data/lib/sqreen/metrics_store.rb +22 -12
  26. data/lib/sqreen/performance_notifications/binned_metrics.rb +8 -2
  27. data/lib/sqreen/rules.rb +4 -2
  28. data/lib/sqreen/rules/not_found_cb.rb +2 -0
  29. data/lib/sqreen/rules/rule_cb.rb +2 -0
  30. data/lib/sqreen/rules/waf_cb.rb +13 -10
  31. data/lib/sqreen/runner.rb +75 -8
  32. data/lib/sqreen/sensitive_data_redactor.rb +19 -31
  33. data/lib/sqreen/session.rb +51 -43
  34. data/lib/sqreen/signals/conversions.rb +283 -0
  35. data/lib/sqreen/signals/http_trace_redaction.rb +111 -0
  36. data/lib/sqreen/signals/signals_submission_strategy.rb +78 -0
  37. data/lib/sqreen/version.rb +1 -1
  38. data/lib/sqreen/weave/legacy/instrumentation.rb +7 -7
  39. metadata +50 -6
  40. data/lib/sqreen/backport.rb +0 -9
  41. data/lib/sqreen/backport/clock_gettime.rb +0 -74
  42. data/lib/sqreen/backport/original_name.rb +0 -88
@@ -0,0 +1,283 @@
1
+ require 'sqreen/version'
2
+ require 'sqreen/rules/rule_cb'
3
+ require 'sqreen/metrics/base'
4
+ require 'sqreen/metrics/binning'
5
+ require 'sqreen/signals/http_trace_redaction'
6
+ require 'sqreen/kit/signals/signal_attributes'
7
+ require 'sqreen/kit/signals/specialized/aggregated_metric'
8
+ require 'sqreen/kit/signals/specialized/attack'
9
+ require 'sqreen/kit/signals/specialized/binning_metric'
10
+ require 'sqreen/kit/signals/specialized/sqreen_exception'
11
+ require 'sqreen/kit/signals/specialized/http_trace'
12
+ require 'sqreen/kit/signals/specialized/sdk_track_call'
13
+
14
+ module Sqreen
15
+ module Signals
16
+ module Conversions # rubocop:disable Metrics/ModuleLength
17
+ class << self
18
+ # @param [Sqreen::AggregatedMetric] agg
19
+ # @return [Sqreen::Kit::Signals::Metric]
20
+ def convert_metric_sample(agg)
21
+ attrs = {
22
+ signal_name: "sq.agent.metric.#{agg.name}",
23
+ source: if agg.rule
24
+ "sqreen:rules:#{agg.rule.rulespack_id}:#{agg.rule.rule_name}"
25
+ else
26
+ agent_gen_source
27
+ end,
28
+ time: agg.finish,
29
+ }
30
+
31
+ if agg.metric.is_a?(Sqreen::Metric::Binning)
32
+ conv_binning_metric(agg, attrs)
33
+ else
34
+ conv_generic_metric(agg, attrs)
35
+ end
36
+ end
37
+
38
+ # @param [Sqreen::Attack] attack
39
+ # XXX: not used because we don't use Sqreen::Attack
40
+ def convert_attack(attack)
41
+ # no need to set actor/context as we only include them in request records/traces
42
+ Kit::Signals::Specialized::Attack.new(
43
+ signal_name: "sq.agent.attack.#{attack.attack_type}",
44
+ source: "sqreen:rule:#{attack.rulespack_id}:#{attack.rule_name}",
45
+ time: attack.time,
46
+ location: Kit::Signals::Location.new(stack_trace: attack.backtrace),
47
+ payload: Kit::Signals::Specialized::Attack::Payload.new(
48
+ test: attack.test?,
49
+ block: attack.block?,
50
+ infos: attack.infos
51
+ )
52
+ )
53
+ end
54
+
55
+ # see Sqreen::Rules::RuleCB.record_event
56
+ def convert_unstructured_attack(payload)
57
+ Kit::Signals::Specialized::Attack.new(
58
+ signal_name: "sq.agent.attack.#{payload[:attack_type]}",
59
+ source: "sqreen:rule:#{payload[:rulespack_id]}:#{payload[:rule_name]}",
60
+ time: payload[:time],
61
+ location: (Kit::Signals::Location.new(stack_trace: payload[:backtrace]) if payload[:backtrace]),
62
+ payload: Kit::Signals::Specialized::Attack::Payload.new(
63
+ test: payload[:test],
64
+ block: payload[:block],
65
+ infos: payload[:infos]
66
+ )
67
+ )
68
+ end
69
+
70
+ # @param [Sqreen::RemoteException] exception
71
+ # @return [Sqreen::Kit::Signals::Specialized::SqreenException]
72
+ def convert_exception(exception)
73
+ payload = exception.payload
74
+
75
+ infos = payload['client_ip'] ? { client_ip: payload['client_ip'] } : {}
76
+ infos.merge!(payload['infos'] || {})
77
+
78
+ Kit::Signals::Specialized::SqreenException.new(
79
+ source: if payload['rule_name']
80
+ "sqreen:rule:#{payload['rulespack_id']}:#{payload['rule_name']}"
81
+ else
82
+ agent_gen_source
83
+ end,
84
+ time: exception.time,
85
+ ruby_exception: payload['exception'],
86
+ infos: infos
87
+ )
88
+ end
89
+
90
+ # see Sqreen::Rules::RuleCB.record_exception
91
+ # @param [Hash] payload
92
+ # @return [Sqreen::Kit::Signals::Specialized::SqreenException]
93
+ def convert_unstructured_exception(payload)
94
+ Kit::Signals::Specialized::SqreenException.new(
95
+ source: "sqreen:rule:#{payload[:rulespack_id]}:#{payload[:rule_name]}",
96
+ time: payload[:time],
97
+ ruby_exception: payload[:exception],
98
+ infos: payload[:infos]
99
+ )
100
+ end
101
+
102
+ # @param [Sqreen::RequestRecord] req_rec
103
+ # @return [Sqreen::Kit::Signals::Specialized::HttpTrace]
104
+ def convert_req_record(req_rec)
105
+ payload = req_rec.payload
106
+
107
+ request_p = payload['request']
108
+ id_args = req_rec.last_identify_args
109
+ identifiers = id_args[0] if id_args
110
+ traits = id_args[1] if id_args
111
+
112
+ observed = payload[:observed] || {}
113
+ signals = []
114
+ signals += (observed[:attacks] || [])
115
+ .map { |att| convert_unstructured_attack(att) }
116
+ signals += (observed[:sqreen_exceptions] || [])
117
+ .map { |sq_exc| convert_unstructured_exception(sq_exc) }
118
+ signals += req_rec.processed_sdk_calls
119
+ .select { |h| h[:name] == :track }
120
+ .map { |h| convert_track(h) }
121
+
122
+ trace = Kit::Signals::Specialized::HttpTrace.new(
123
+ actor: Kit::Signals::Actor.new(
124
+ ip_addresses: [request_p[:client_ip]].compact,
125
+ user_agent: request_p[:user_agent],
126
+ identifiers: identifiers,
127
+ traits: traits,
128
+ ),
129
+ location_infra: location_infra,
130
+ context: convert_request(request_p,
131
+ payload['response'],
132
+ payload['headers'],
133
+ payload['params']),
134
+ data: signals
135
+ )
136
+ HttpTraceRedaction.redact_trace!(trace, req_rec.redactor)
137
+ trace
138
+ end
139
+
140
+ # @param [Array<Sqreen::Kit::Signals::Signal|Sqreen::Kit::Signals::Trace>] batch
141
+ def convert_batch(batch)
142
+ batch.map do |evt|
143
+ case evt
144
+ when RemoteException
145
+ convert_exception(evt)
146
+ when AggregatedMetric
147
+ convert_metric_sample(evt)
148
+ when RequestRecord
149
+ convert_req_record(evt)
150
+ else
151
+ raise NotImplementedError, "Unknown type of event in batch: #{evt}"
152
+ end
153
+ end
154
+ end
155
+
156
+ private
157
+
158
+ def agent_gen_source
159
+ "sqreen:agent:ruby:#{Sqreen::VERSION}"
160
+ end
161
+
162
+ def location_infra
163
+ @location_infra ||= begin
164
+ Kit::Signals::LocationInfra.new(
165
+ agent_version: Sqreen::VERSION,
166
+ os_type: RuntimeInfos.os[:os_type],
167
+ hostname: RuntimeInfos.hostname,
168
+ runtime_type: RuntimeInfos.runtime[:runtime_type],
169
+ runtime_version: RuntimeInfos.runtime[:runtime_version],
170
+ libsqreen_version: RuntimeInfos.libsqreen_version,
171
+ )
172
+ end
173
+ end
174
+
175
+ # see Sqreen::RequestRecord.processed_sdk_calls
176
+ def convert_track(call_info)
177
+ options = call_info[:args][1] || {}
178
+ Kit::Signals::Specialized::SdkTrackCall.new(
179
+ signal_name: "sq.sdk.#{call_info[:args][0]}",
180
+ time: call_info[:time],
181
+ payload: Kit::Signals::Specialized::SdkTrackCall::Payload.new(
182
+ properties: options[:properties],
183
+ user_identifiers: options[:user_identifiers]
184
+ )
185
+ )
186
+ end
187
+
188
+ # @param [Hash] req_payload
189
+ # @param [Hash] headers_payload
190
+ # @param [Hash] params_payload
191
+ # see the PayloadCreator abomination for reference
192
+ # TODO: do not convert from the old payload to the new payload
193
+ # Have an intermediate object that gets the data from the framework.
194
+ # (Or convert directly from the framework, but this needs to be
195
+ # done during the request, not just before event is transmitted)
196
+ def convert_request(req_payload, resp_payload, headers_payload, params_payload)
197
+ req_payload ||= {}
198
+ headers_payload ||= {}
199
+ resp_payload ||= {}
200
+ params_payload ||= {}
201
+
202
+ other = params_payload['other']
203
+ other = merge_hash_append(other, params_payload['rack'])
204
+ other = merge_hash_append(other, params_payload['grape_params'])
205
+ other = merge_hash_append(other, params_payload['rack_routing'])
206
+
207
+ Sqreen::Kit::Signals::Context::HttpContext.new(
208
+ {
209
+ rid: req_payload[:rid],
210
+ headers: headers_payload,
211
+ user_agent: req_payload[:user_agent],
212
+ scheme: req_payload[:scheme],
213
+ verb: req_payload[:verb],
214
+ host: req_payload[:host],
215
+ port: req_payload[:port],
216
+ remote_ip: req_payload[:remote_ip],
217
+ remote_port: req_payload[:remote_port] || 0,
218
+ path: req_payload[:path],
219
+ referer: req_payload[:referer],
220
+ params_query: params_payload['query'],
221
+ params_form: params_payload['form'],
222
+ params_other: other,
223
+ # endpoint, is_reveal_replayed not set
224
+ status: resp_payload[:status],
225
+ content_length: resp_payload[:content_length],
226
+ content_type: resp_payload[:content_type],
227
+ }
228
+ )
229
+ end
230
+
231
+ def merge_hash_append(hash1, hash2)
232
+ return nil if hash1.nil? && hash2.nil?
233
+ return hash1 if hash2.nil? || hash2.empty?
234
+ return hash2 if hash1.nil? || hash1.empty?
235
+
236
+ pairs = (hash1.keys + hash2.keys).map do |key|
237
+ values1 = hash1[key]
238
+ values2 = hash2[key]
239
+ values = [values1, values2].compact
240
+ values = values.first if values.size == 1
241
+ [key, values]
242
+ end
243
+ Hash[pairs]
244
+ end
245
+
246
+ # @param [Sqreen::AggregatedMetric] agg
247
+ # @param [Hash] attrs
248
+ def conv_generic_metric(agg, attrs)
249
+ attrs[:payload] = Kit::Signals::Specialized::AggregatedMetric::Payload.new(
250
+ kind: metric_kind(agg.metric),
251
+ capture_interval_s: agg.metric.period,
252
+ date_started: agg.start,
253
+ date_ended: agg.finish,
254
+ values: agg.data
255
+ )
256
+
257
+ Kit::Signals::Specialized::AggregatedMetric.new(attrs)
258
+ end
259
+
260
+ # @param [Sqreen::AggregatedMetric] agg
261
+ # @param [Hash] attrs
262
+ def conv_binning_metric(agg, attrs)
263
+ attrs[:payload] = Kit::Signals::Specialized::BinningMetric::Payload.new(
264
+ capture_interval_s: agg.metric.period,
265
+ date_started: agg.start,
266
+ date_ended: agg.finish,
267
+ base: agg.data['b'],
268
+ unit: agg.data['u'],
269
+ max: agg.data['v']['max'],
270
+ bins: agg.data['v'].reject { |k, _v| k == 'max' }
271
+ )
272
+
273
+ Kit::Signals::Specialized::BinningMetric.new(attrs)
274
+ end
275
+
276
+ # @param [Sqreen::Metric::Base] metric
277
+ def metric_kind(metric)
278
+ metric.class.name.sub(/.*::/, '').sub(/Metric$/, '')
279
+ end
280
+ end
281
+ end
282
+ end
283
+ end
@@ -0,0 +1,111 @@
1
+ require 'json'
2
+ require 'sqreen/kit/loggable'
3
+ require 'sqreen/kit/signals/specialized/http_trace'
4
+
5
+ module Sqreen
6
+ module Signals
7
+ module HttpTraceRedaction
8
+ class << self
9
+ include Sqreen::Kit::Loggable
10
+
11
+ # @param [Sqreen::Kit::Signals::Specialized::HttpTrace] trace
12
+ # @param [Sqreen::SensitiveDataRedactor] redactor
13
+ def redact_trace!(trace, redactor)
14
+ return unless redactor
15
+ # redact headers (keys unsafe)
16
+ # @type [Sqreen::Kit::Signals::Context::HttpContext]
17
+ http_context = trace.context
18
+
19
+ all_redacted = []
20
+
21
+ # Redact headers; save redacted values
22
+ # headers are encoded as [key, value], not a hash, so
23
+ # they require some transformation
24
+ orig_headers = http_context.headers
25
+ if orig_headers
26
+ headers = orig_headers.map { |(k, v)| { k => v } }
27
+ headers, redacted = redactor.redact(headers)
28
+ http_context.headers = headers.map(&:first)
29
+ all_redacted += redacted
30
+ end
31
+
32
+ # Redact params; save redacted values
33
+ Kit::Signals::Context::HttpContext::PARAMS_ATTRS.each do |attr|
34
+ value = http_context.public_send(attr)
35
+ next unless value
36
+ value, redacted = redactor.redact(value)
37
+ all_redacted += redacted
38
+ http_context.public_send(:"#{attr}=", value)
39
+ end
40
+
41
+ all_redacted = all_redacted.uniq.map(&:downcase)
42
+
43
+ # Redact attacks and exceptions
44
+ # XXX: no redaction for infos in attacks/exceptions except for WAF data
45
+ # Is this the correct behavior?
46
+ redact_attacks!(trace, redactor, all_redacted)
47
+ redact_exceptions!(trace, redactor, all_redacted)
48
+ end
49
+
50
+ private
51
+
52
+ # @param [Sqreen::Kit::Signals::Specialized::HttpTrace] trace
53
+ # @param [Sqreen::SensitiveDataRedactor] redactor
54
+ # Redacts WAF data according to specific rules therefor
55
+ # Redacts infos according to general rules
56
+ def redact_attacks!(trace, redactor, redacted_data)
57
+ trace.data.each do |signal|
58
+ next unless signal.is_a?(Kit::Signals::Specialized::Attack)
59
+ # @type [Sqreen::Kit::Signals::Specialized::Attack::Payload] payload
60
+ payload = signal.payload
61
+ next unless payload.infos
62
+
63
+ if payload.infos[:waf_data]
64
+ redact_waf_attack_data!(payload.infos, redacted_data)
65
+ end
66
+ payload.infos, = redactor.redact(payload.infos)
67
+ end
68
+ end
69
+
70
+ def redact_exceptions!(trace, redactor, redacted_data)
71
+ trace.data.each do |signal|
72
+ next unless signal.is_a?(Kit::Signals::Specialized::SqreenException)
73
+ infos = signal.infos
74
+ next unless infos
75
+
76
+ redact_waf_exception_data!(signal.infos, redacted_data) if signal.infos[:waf]
77
+ signal.infos, = redactor.redact(infos)
78
+ end
79
+ end
80
+
81
+ # @param [Hash] infos from WAF attack
82
+ def redact_waf_attack_data!(infos, redacted_data)
83
+ begin
84
+ parsed = JSON.parse(infos[:waf_data])
85
+ rescue JSON::JSONError => e
86
+ logger.warn("waf_data is not valid json: #{e.message}")
87
+ return
88
+ end
89
+ redacted = parsed.each do |w|
90
+ next unless (filters = w['filter'])
91
+
92
+ filters.each do |f|
93
+ next unless (v = f['resolved_value'])
94
+ next unless redacted_data.include?(v.downcase)
95
+
96
+ f['match_status'] = SensitiveDataRedactor::MASK
97
+ f['resolved_value'] = SensitiveDataRedactor::MASK
98
+ end
99
+ end
100
+ infos[:waf_data] = JSON.dump(redacted)
101
+ end
102
+
103
+ # see https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000022-waf-data-sanitization.md#changes-to-the-agents
104
+ def redact_waf_exception_data!(infos, redacted_data)
105
+ return if redacted_data.empty?
106
+ infos[:waf].delete(:args)
107
+ end
108
+ end
109
+ end
110
+ end
111
+ end
@@ -0,0 +1,78 @@
1
+ require 'sqreen/aggregated_metric'
2
+ require 'sqreen/kit'
3
+ require 'sqreen/kit/string_sanitizer'
4
+ require 'sqreen/signals/conversions'
5
+ require 'sqreen/log/loggable'
6
+
7
+ module Sqreen
8
+ module Signals
9
+ # see also Sqreen::Legacy::OldEventSubmissionStrategy
10
+ # usage in Sqreen:Session
11
+ class SignalsSubmissionStrategy
12
+ include Sqreen::Log::Loggable
13
+
14
+ # @param [Array<Sqreen::AggregatedMetric>] metrics
15
+ def post_metrics(metrics)
16
+ return if metrics.nil? || metrics.empty?
17
+
18
+ guarded 'Failed to serialize or submit aggregated metrics' do
19
+ batch = metrics.map do |m|
20
+ Conversions.convert_metric_sample(m)
21
+ end
22
+ client.report_batch(batch)
23
+ end
24
+ end
25
+
26
+ # @param _attack [Sqreen::Attack]
27
+ # XXX: unused
28
+ def post_attack(_attack)
29
+ raise NotImplementedError
30
+ end
31
+
32
+ # @param request_record [Sqreen::RequestRecord]
33
+ def post_request_record(request_record)
34
+ guarded 'Failed to serialize or submit request record' do
35
+ trace = Conversions.convert_req_record(request_record)
36
+ append_sanitizing_filter(trace)
37
+ client.report_trace(trace)
38
+ end
39
+ end
40
+
41
+ # Post an exception to Sqreen for analysis
42
+ # @param exception [RemoteException] Exception and context to be sent over
43
+ def post_sqreen_exception(exception)
44
+ guarded 'Failed to serialize or submit exception', false do
45
+ data = Conversions.convert_exception(exception)
46
+ append_sanitizing_filter(data)
47
+ client.report_signal(data)
48
+ end
49
+ end
50
+
51
+ def post_batch(events)
52
+ guarded 'Failed to serialize or submit batch of events' do
53
+ batch = Conversions.convert_batch(events)
54
+ batch.each { |sig_or_trace| append_sanitizing_filter(sig_or_trace) }
55
+ client.report_batch(batch)
56
+ end
57
+ end
58
+
59
+ private
60
+
61
+ def append_sanitizing_filter(sig_or_trace)
62
+ sig_or_trace.append_to_h_filter Kit::StringSanitizer.method(:sanitize)
63
+ end
64
+
65
+ # we don't want exceptions to propagate and kill the worker thread
66
+ def guarded(msg, report = true)
67
+ yield
68
+ rescue StandardError => e
69
+ logger.warn "#{msg}: #{e.message}\n#{e.backtrace.map { |x| " #{x}" }.join("\n")}"
70
+ post_sqreen_exception(RemoteException.new(e)) if report
71
+ end
72
+
73
+ def client
74
+ Sqreen::Kit.auth_signals_client
75
+ end
76
+ end
77
+ end
78
+ end