sqreen 1.19.0.beta1 → 1.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +22 -2
- data/lib/sqreen/aggregated_metric.rb +25 -0
- data/lib/sqreen/configuration.rb +7 -3
- data/lib/sqreen/deliveries/batch.rb +4 -1
- data/lib/sqreen/deliveries/simple.rb +4 -0
- data/lib/sqreen/event.rb +7 -5
- data/lib/sqreen/events/attack.rb +23 -18
- data/lib/sqreen/events/remote_exception.rb +0 -22
- data/lib/sqreen/events/request_record.rb +15 -70
- data/lib/sqreen/frameworks/request_recorder.rb +13 -2
- data/lib/sqreen/kit/signals/specialized/aggregated_metric.rb +72 -0
- data/lib/sqreen/kit/signals/specialized/attack.rb +57 -0
- data/lib/sqreen/kit/signals/specialized/binning_metric.rb +76 -0
- data/lib/sqreen/kit/signals/specialized/http_trace.rb +26 -0
- data/lib/sqreen/kit/signals/specialized/sdk_track_call.rb +50 -0
- data/lib/sqreen/kit/signals/specialized/sqreen_exception.rb +57 -0
- data/lib/sqreen/legacy/old_event_submission_strategy.rb +221 -0
- data/lib/sqreen/legacy/waf_redactions.rb +49 -0
- data/lib/sqreen/metrics/base.rb +3 -0
- data/lib/sqreen/metrics_store.rb +22 -12
- data/lib/sqreen/performance_notifications/binned_metrics.rb +8 -2
- data/lib/sqreen/rules.rb +4 -2
- data/lib/sqreen/rules/not_found_cb.rb +2 -0
- data/lib/sqreen/rules/rule_cb.rb +2 -0
- data/lib/sqreen/rules/waf_cb.rb +39 -16
- data/lib/sqreen/runner.rb +48 -6
- data/lib/sqreen/sensitive_data_redactor.rb +19 -31
- data/lib/sqreen/session.rb +39 -37
- data/lib/sqreen/signals/conversions.rb +283 -0
- data/lib/sqreen/signals/http_trace_redaction.rb +111 -0
- data/lib/sqreen/signals/signals_submission_strategy.rb +78 -0
- data/lib/sqreen/version.rb +1 -1
- data/lib/sqreen/weave/legacy/instrumentation.rb +15 -7
- metadata +55 -14
- data/lib/sqreen/backport.rb +0 -9
- data/lib/sqreen/backport/clock_gettime.rb +0 -74
- data/lib/sqreen/backport/original_name.rb +0 -88
data/lib/sqreen/session.rb
CHANGED
@@ -11,6 +11,10 @@ require 'sqreen/events/attack'
|
|
11
11
|
require 'sqreen/events/request_record'
|
12
12
|
require 'sqreen/exception'
|
13
13
|
require 'sqreen/safe_json'
|
14
|
+
require 'sqreen/kit'
|
15
|
+
require 'sqreen/kit/configuration'
|
16
|
+
require 'sqreen/signals/signals_submission_strategy'
|
17
|
+
require 'sqreen/legacy/old_event_submission_strategy'
|
14
18
|
|
15
19
|
require 'net/https'
|
16
20
|
require 'uri'
|
@@ -41,13 +45,12 @@ module Sqreen
|
|
41
45
|
RETRY_MANY = 301
|
42
46
|
|
43
47
|
MUTEX = Mutex.new
|
44
|
-
METRICS_KEY = 'metrics'.freeze
|
45
48
|
|
46
49
|
@@path_prefix = '/sqreen/v0/'
|
47
50
|
|
48
51
|
attr_accessor :request_compression
|
49
52
|
|
50
|
-
def initialize(server_url, token, app_name = nil)
|
53
|
+
def initialize(server_url, token, app_name = nil, proxy_url = nil)
|
51
54
|
@token = token
|
52
55
|
@app_name = app_name
|
53
56
|
@session_id = nil
|
@@ -59,16 +62,35 @@ module Sqreen
|
|
59
62
|
uri = parse_uri(server_url)
|
60
63
|
use_ssl = (uri.scheme == 'https')
|
61
64
|
|
65
|
+
proxy_params = []
|
66
|
+
if proxy_url
|
67
|
+
proxy_uri = parse_uri(proxy_url)
|
68
|
+
proxy_params = [proxy_uri.host, proxy_uri.port, proxy_uri.user, proxy_uri.password]
|
69
|
+
end
|
70
|
+
|
62
71
|
@req_nb = 0
|
63
72
|
|
64
|
-
@http = Net::HTTP.new(uri.host, uri.port)
|
73
|
+
@http = Net::HTTP.new(uri.host, uri.port, *proxy_params)
|
65
74
|
@http.use_ssl = use_ssl
|
75
|
+
@http.verify_mode = OpenSSL::SSL::VERIFY_NONE if ENV['SQREEN_SSL_NO_VERIFY'] # for testing
|
66
76
|
if use_ssl
|
67
77
|
cert_file = File.join(File.dirname(__FILE__), 'ca.crt')
|
68
78
|
cert_store = OpenSSL::X509::Store.new
|
69
79
|
cert_store.add_file cert_file
|
70
80
|
@http.cert_store = cert_store
|
71
81
|
end
|
82
|
+
self.use_signals = false
|
83
|
+
end
|
84
|
+
|
85
|
+
def use_signals=(do_use)
|
86
|
+
return if do_use == @use_signals
|
87
|
+
|
88
|
+
@use_signals = do_use
|
89
|
+
if do_use
|
90
|
+
@evt_sub_strategy = Sqreen::Signals::SignalsSubmissionStrategy.new
|
91
|
+
else
|
92
|
+
@evt_sub_strategy = Sqreen::Legacy::OldEventSubmissionStrategy.new(method(:post))
|
93
|
+
end
|
72
94
|
end
|
73
95
|
|
74
96
|
def parse_uri(uri)
|
@@ -235,6 +257,8 @@ module Sqreen
|
|
235
257
|
end
|
236
258
|
Sqreen.log.info 'Login success.'
|
237
259
|
@session_id = res['session_id']
|
260
|
+
Kit::Configuration.session_key = @session_id
|
261
|
+
Kit.reset
|
238
262
|
Sqreen.log.debug { "received session_id #{@session_id}" }
|
239
263
|
Sqreen.logged_in = true
|
240
264
|
res
|
@@ -246,20 +270,24 @@ module Sqreen
|
|
246
270
|
|
247
271
|
def heartbeat(cmd_res = {}, metrics = [])
|
248
272
|
payload = {}
|
249
|
-
|
273
|
+
unless metrics.nil? || metrics.empty?
|
274
|
+
# never reached with signals
|
275
|
+
payload['metrics'] = metrics.map do |m|
|
276
|
+
Sqreen::Legacy::EventToHash.convert_agg_metric(m)
|
277
|
+
end
|
278
|
+
end
|
250
279
|
payload['command_results'] = cmd_res unless cmd_res.nil? || cmd_res.empty?
|
251
280
|
|
252
281
|
post('app-beat', payload.empty? ? nil : payload, {}, RETRY_MANY)
|
253
282
|
end
|
254
283
|
|
255
284
|
def post_metrics(metrics)
|
256
|
-
|
257
|
-
payload = { METRICS_KEY => metrics }
|
258
|
-
post(METRICS_KEY, payload, {}, RETRY_MANY)
|
285
|
+
@evt_sub_strategy.post_metrics(metrics)
|
259
286
|
end
|
260
287
|
|
288
|
+
# XXX never called
|
261
289
|
def post_attack(attack)
|
262
|
-
|
290
|
+
@evt_sub_strategy.post_attack(attack)
|
263
291
|
end
|
264
292
|
|
265
293
|
def post_bundle(bundle_sig, dependencies)
|
@@ -271,33 +299,17 @@ module Sqreen
|
|
271
299
|
end
|
272
300
|
|
273
301
|
def post_request_record(request_record)
|
274
|
-
|
302
|
+
@evt_sub_strategy.post_request_record(request_record)
|
275
303
|
end
|
276
304
|
|
277
305
|
# Post an exception to Sqreen for analysis
|
278
306
|
# @param exception [RemoteException] Exception and context to be sent over
|
279
307
|
def post_sqreen_exception(exception)
|
280
|
-
|
281
|
-
rescue StandardError => e
|
282
|
-
Sqreen.log.warn(format('Could not post exception (network down? %s) %s',
|
283
|
-
e.inspect,
|
284
|
-
exception.to_hash.inspect))
|
285
|
-
nil
|
308
|
+
@evt_sub_strategy.post_sqreen_exception(exception)
|
286
309
|
end
|
287
310
|
|
288
|
-
BATCH_KEY = 'batch'.freeze
|
289
|
-
EVENT_TYPE_KEY = 'event_type'.freeze
|
290
311
|
def post_batch(events)
|
291
|
-
|
292
|
-
h = event.to_hash
|
293
|
-
h[EVENT_TYPE_KEY] = event_kind(event)
|
294
|
-
h
|
295
|
-
end
|
296
|
-
Sqreen.log.debug do
|
297
|
-
tally = Hash[events.group_by(&:class).map{ |k,v| [k, v.count] }]
|
298
|
-
"Doing batch with the following tally of event types: #{tally}"
|
299
|
-
end
|
300
|
-
post(BATCH_KEY, { BATCH_KEY => batch }, {}, RETRY_MANY)
|
312
|
+
@evt_sub_strategy.post_batch(events)
|
301
313
|
end
|
302
314
|
|
303
315
|
# Perform agent logout
|
@@ -313,15 +325,5 @@ module Sqreen
|
|
313
325
|
Sqreen.logged_in = false
|
314
326
|
disconnect
|
315
327
|
end
|
316
|
-
|
317
|
-
protected
|
318
|
-
|
319
|
-
def event_kind(event)
|
320
|
-
case event
|
321
|
-
when Sqreen::RemoteException then 'sqreen_exception'
|
322
|
-
when Sqreen::Attack then 'attack'
|
323
|
-
when Sqreen::RequestRecord then 'request_record'
|
324
|
-
end
|
325
|
-
end
|
326
328
|
end
|
327
329
|
end
|
@@ -0,0 +1,283 @@
|
|
1
|
+
require 'sqreen/version'
|
2
|
+
require 'sqreen/rules/rule_cb'
|
3
|
+
require 'sqreen/metrics/base'
|
4
|
+
require 'sqreen/metrics/binning'
|
5
|
+
require 'sqreen/signals/http_trace_redaction'
|
6
|
+
require 'sqreen/kit/signals/signal_attributes'
|
7
|
+
require 'sqreen/kit/signals/specialized/aggregated_metric'
|
8
|
+
require 'sqreen/kit/signals/specialized/attack'
|
9
|
+
require 'sqreen/kit/signals/specialized/binning_metric'
|
10
|
+
require 'sqreen/kit/signals/specialized/sqreen_exception'
|
11
|
+
require 'sqreen/kit/signals/specialized/http_trace'
|
12
|
+
require 'sqreen/kit/signals/specialized/sdk_track_call'
|
13
|
+
|
14
|
+
module Sqreen
|
15
|
+
module Signals
|
16
|
+
module Conversions # rubocop:disable Metrics/ModuleLength
|
17
|
+
class << self
|
18
|
+
# @param [Sqreen::AggregatedMetric] agg
|
19
|
+
# @return [Sqreen::Kit::Signals::Metric]
|
20
|
+
def convert_metric_sample(agg)
|
21
|
+
attrs = {
|
22
|
+
signal_name: "sq.agent.metric.#{agg.name}",
|
23
|
+
source: if agg.rule
|
24
|
+
"sqreen:rules:#{agg.rule.rulespack_id}:#{agg.rule.rule_name}"
|
25
|
+
else
|
26
|
+
agent_gen_source
|
27
|
+
end,
|
28
|
+
time: agg.finish,
|
29
|
+
}
|
30
|
+
|
31
|
+
if agg.metric.is_a?(Sqreen::Metric::Binning)
|
32
|
+
conv_binning_metric(agg, attrs)
|
33
|
+
else
|
34
|
+
conv_generic_metric(agg, attrs)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
# @param [Sqreen::Attack] attack
|
39
|
+
# XXX: not used because we don't use Sqreen::Attack
|
40
|
+
def convert_attack(attack)
|
41
|
+
# no need to set actor/context as we only include them in request records/traces
|
42
|
+
Kit::Signals::Specialized::Attack.new(
|
43
|
+
signal_name: "sq.agent.attack.#{attack.attack_type}",
|
44
|
+
source: "sqreen:rule:#{attack.rulespack_id}:#{attack.rule_name}",
|
45
|
+
time: attack.time,
|
46
|
+
location: Kit::Signals::Location.new(stack_trace: attack.backtrace),
|
47
|
+
payload: Kit::Signals::Specialized::Attack::Payload.new(
|
48
|
+
test: attack.test?,
|
49
|
+
block: attack.block?,
|
50
|
+
infos: attack.infos
|
51
|
+
)
|
52
|
+
)
|
53
|
+
end
|
54
|
+
|
55
|
+
# see Sqreen::Rules::RuleCB.record_event
|
56
|
+
def convert_unstructured_attack(payload)
|
57
|
+
Kit::Signals::Specialized::Attack.new(
|
58
|
+
signal_name: "sq.agent.attack.#{payload[:attack_type]}",
|
59
|
+
source: "sqreen:rule:#{payload[:rulespack_id]}:#{payload[:rule_name]}",
|
60
|
+
time: payload[:time],
|
61
|
+
location: (Kit::Signals::Location.new(stack_trace: payload[:backtrace]) if payload[:backtrace]),
|
62
|
+
payload: Kit::Signals::Specialized::Attack::Payload.new(
|
63
|
+
test: payload[:test],
|
64
|
+
block: payload[:block],
|
65
|
+
infos: payload[:infos]
|
66
|
+
)
|
67
|
+
)
|
68
|
+
end
|
69
|
+
|
70
|
+
# @param [Sqreen::RemoteException] exception
|
71
|
+
# @return [Sqreen::Kit::Signals::Specialized::SqreenException]
|
72
|
+
def convert_exception(exception)
|
73
|
+
payload = exception.payload
|
74
|
+
|
75
|
+
infos = payload['client_ip'] ? { client_ip: payload['client_ip'] } : {}
|
76
|
+
infos.merge!(payload['infos'] || {})
|
77
|
+
|
78
|
+
Kit::Signals::Specialized::SqreenException.new(
|
79
|
+
source: if payload['rule_name']
|
80
|
+
"sqreen:rule:#{payload['rulespack_id']}:#{payload['rule_name']}"
|
81
|
+
else
|
82
|
+
agent_gen_source
|
83
|
+
end,
|
84
|
+
time: exception.time,
|
85
|
+
ruby_exception: payload['exception'],
|
86
|
+
infos: infos
|
87
|
+
)
|
88
|
+
end
|
89
|
+
|
90
|
+
# see Sqreen::Rules::RuleCB.record_exception
|
91
|
+
# @param [Hash] payload
|
92
|
+
# @return [Sqreen::Kit::Signals::Specialized::SqreenException]
|
93
|
+
def convert_unstructured_exception(payload)
|
94
|
+
Kit::Signals::Specialized::SqreenException.new(
|
95
|
+
source: "sqreen:rule:#{payload[:rulespack_id]}:#{payload[:rule_name]}",
|
96
|
+
time: payload[:time],
|
97
|
+
ruby_exception: payload[:exception],
|
98
|
+
infos: payload[:infos]
|
99
|
+
)
|
100
|
+
end
|
101
|
+
|
102
|
+
# @param [Sqreen::RequestRecord] req_rec
|
103
|
+
# @return [Sqreen::Kit::Signals::Specialized::HttpTrace]
|
104
|
+
def convert_req_record(req_rec)
|
105
|
+
payload = req_rec.payload
|
106
|
+
|
107
|
+
request_p = payload['request']
|
108
|
+
id_args = req_rec.last_identify_args
|
109
|
+
identifiers = id_args[0] if id_args
|
110
|
+
traits = id_args[1] if id_args
|
111
|
+
|
112
|
+
observed = payload[:observed] || {}
|
113
|
+
signals = []
|
114
|
+
signals += (observed[:attacks] || [])
|
115
|
+
.map { |att| convert_unstructured_attack(att) }
|
116
|
+
signals += (observed[:sqreen_exceptions] || [])
|
117
|
+
.map { |sq_exc| convert_unstructured_exception(sq_exc) }
|
118
|
+
signals += req_rec.processed_sdk_calls
|
119
|
+
.select { |h| h[:name] == :track }
|
120
|
+
.map { |h| convert_track(h) }
|
121
|
+
|
122
|
+
trace = Kit::Signals::Specialized::HttpTrace.new(
|
123
|
+
actor: Kit::Signals::Actor.new(
|
124
|
+
ip_addresses: [request_p[:client_ip]].compact,
|
125
|
+
user_agent: request_p[:user_agent],
|
126
|
+
identifiers: identifiers,
|
127
|
+
traits: traits,
|
128
|
+
),
|
129
|
+
location_infra: location_infra,
|
130
|
+
context: convert_request(request_p,
|
131
|
+
payload['response'],
|
132
|
+
payload['headers'],
|
133
|
+
payload['params']),
|
134
|
+
data: signals
|
135
|
+
)
|
136
|
+
HttpTraceRedaction.redact_trace!(trace, req_rec.redactor)
|
137
|
+
trace
|
138
|
+
end
|
139
|
+
|
140
|
+
# @param [Array<Sqreen::Kit::Signals::Signal|Sqreen::Kit::Signals::Trace>] batch
|
141
|
+
def convert_batch(batch)
|
142
|
+
batch.map do |evt|
|
143
|
+
case evt
|
144
|
+
when RemoteException
|
145
|
+
convert_exception(evt)
|
146
|
+
when AggregatedMetric
|
147
|
+
convert_metric_sample(evt)
|
148
|
+
when RequestRecord
|
149
|
+
convert_req_record(evt)
|
150
|
+
else
|
151
|
+
raise NotImplementedError, "Unknown type of event in batch: #{evt}"
|
152
|
+
end
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
private
|
157
|
+
|
158
|
+
def agent_gen_source
|
159
|
+
"sqreen:agent:ruby:#{Sqreen::VERSION}"
|
160
|
+
end
|
161
|
+
|
162
|
+
def location_infra
|
163
|
+
@location_infra ||= begin
|
164
|
+
Kit::Signals::LocationInfra.new(
|
165
|
+
agent_version: Sqreen::VERSION,
|
166
|
+
os_type: RuntimeInfos.os[:os_type],
|
167
|
+
hostname: RuntimeInfos.hostname,
|
168
|
+
runtime_type: RuntimeInfos.runtime[:runtime_type],
|
169
|
+
runtime_version: RuntimeInfos.runtime[:runtime_version],
|
170
|
+
libsqreen_version: RuntimeInfos.libsqreen_version,
|
171
|
+
)
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
# see Sqreen::RequestRecord.processed_sdk_calls
|
176
|
+
def convert_track(call_info)
|
177
|
+
options = call_info[:args][1] || {}
|
178
|
+
Kit::Signals::Specialized::SdkTrackCall.new(
|
179
|
+
signal_name: "sq.sdk.#{call_info[:args][0]}",
|
180
|
+
time: call_info[:time],
|
181
|
+
payload: Kit::Signals::Specialized::SdkTrackCall::Payload.new(
|
182
|
+
properties: options[:properties],
|
183
|
+
user_identifiers: options[:user_identifiers]
|
184
|
+
)
|
185
|
+
)
|
186
|
+
end
|
187
|
+
|
188
|
+
# @param [Hash] req_payload
|
189
|
+
# @param [Hash] headers_payload
|
190
|
+
# @param [Hash] params_payload
|
191
|
+
# see the PayloadCreator abomination for reference
|
192
|
+
# TODO: do not convert from the old payload to the new payload
|
193
|
+
# Have an intermediate object that gets the data from the framework.
|
194
|
+
# (Or convert directly from the framework, but this needs to be
|
195
|
+
# done during the request, not just before event is transmitted)
|
196
|
+
def convert_request(req_payload, resp_payload, headers_payload, params_payload)
|
197
|
+
req_payload ||= {}
|
198
|
+
headers_payload ||= {}
|
199
|
+
resp_payload ||= {}
|
200
|
+
params_payload ||= {}
|
201
|
+
|
202
|
+
other = params_payload['other']
|
203
|
+
other = merge_hash_append(other, params_payload['rack'])
|
204
|
+
other = merge_hash_append(other, params_payload['grape_params'])
|
205
|
+
other = merge_hash_append(other, params_payload['rack_routing'])
|
206
|
+
|
207
|
+
Sqreen::Kit::Signals::Context::HttpContext.new(
|
208
|
+
{
|
209
|
+
rid: req_payload[:rid],
|
210
|
+
headers: headers_payload,
|
211
|
+
user_agent: req_payload[:user_agent],
|
212
|
+
scheme: req_payload[:scheme],
|
213
|
+
verb: req_payload[:verb],
|
214
|
+
host: req_payload[:host],
|
215
|
+
port: req_payload[:port],
|
216
|
+
remote_ip: req_payload[:remote_ip],
|
217
|
+
remote_port: req_payload[:remote_port] || 0,
|
218
|
+
path: req_payload[:path],
|
219
|
+
referer: req_payload[:referer],
|
220
|
+
params_query: params_payload['query'],
|
221
|
+
params_form: params_payload['form'],
|
222
|
+
params_other: other,
|
223
|
+
# endpoint, is_reveal_replayed not set
|
224
|
+
status: resp_payload[:status],
|
225
|
+
content_length: resp_payload[:content_length],
|
226
|
+
content_type: resp_payload[:content_type],
|
227
|
+
}
|
228
|
+
)
|
229
|
+
end
|
230
|
+
|
231
|
+
def merge_hash_append(hash1, hash2)
|
232
|
+
return nil if hash1.nil? && hash2.nil?
|
233
|
+
return hash1 if hash2.nil? || hash2.empty?
|
234
|
+
return hash2 if hash1.nil? || hash1.empty?
|
235
|
+
|
236
|
+
pairs = (hash1.keys + hash2.keys).map do |key|
|
237
|
+
values1 = hash1[key]
|
238
|
+
values2 = hash2[key]
|
239
|
+
values = [values1, values2].compact
|
240
|
+
values = values.first if values.size == 1
|
241
|
+
[key, values]
|
242
|
+
end
|
243
|
+
Hash[pairs]
|
244
|
+
end
|
245
|
+
|
246
|
+
# @param [Sqreen::AggregatedMetric] agg
|
247
|
+
# @param [Hash] attrs
|
248
|
+
def conv_generic_metric(agg, attrs)
|
249
|
+
attrs[:payload] = Kit::Signals::Specialized::AggregatedMetric::Payload.new(
|
250
|
+
kind: metric_kind(agg.metric),
|
251
|
+
capture_interval_s: agg.metric.period,
|
252
|
+
date_started: agg.start,
|
253
|
+
date_ended: agg.finish,
|
254
|
+
values: agg.data
|
255
|
+
)
|
256
|
+
|
257
|
+
Kit::Signals::Specialized::AggregatedMetric.new(attrs)
|
258
|
+
end
|
259
|
+
|
260
|
+
# @param [Sqreen::AggregatedMetric] agg
|
261
|
+
# @param [Hash] attrs
|
262
|
+
def conv_binning_metric(agg, attrs)
|
263
|
+
attrs[:payload] = Kit::Signals::Specialized::BinningMetric::Payload.new(
|
264
|
+
capture_interval_s: agg.metric.period,
|
265
|
+
date_started: agg.start,
|
266
|
+
date_ended: agg.finish,
|
267
|
+
base: agg.data['b'],
|
268
|
+
unit: agg.data['u'],
|
269
|
+
max: agg.data['v']['max'],
|
270
|
+
bins: agg.data['v'].reject { |k, _v| k == 'max' }
|
271
|
+
)
|
272
|
+
|
273
|
+
Kit::Signals::Specialized::BinningMetric.new(attrs)
|
274
|
+
end
|
275
|
+
|
276
|
+
# @param [Sqreen::Metric::Base] metric
|
277
|
+
def metric_kind(metric)
|
278
|
+
metric.class.name.sub(/.*::/, '').sub(/Metric$/, '')
|
279
|
+
end
|
280
|
+
end
|
281
|
+
end
|
282
|
+
end
|
283
|
+
end
|
@@ -0,0 +1,111 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'sqreen/kit/loggable'
|
3
|
+
require 'sqreen/kit/signals/specialized/http_trace'
|
4
|
+
|
5
|
+
module Sqreen
|
6
|
+
module Signals
|
7
|
+
module HttpTraceRedaction
|
8
|
+
class << self
|
9
|
+
include Sqreen::Kit::Loggable
|
10
|
+
|
11
|
+
# @param [Sqreen::Kit::Signals::Specialized::HttpTrace] trace
|
12
|
+
# @param [Sqreen::SensitiveDataRedactor] redactor
|
13
|
+
def redact_trace!(trace, redactor)
|
14
|
+
return unless redactor
|
15
|
+
# redact headers (keys unsafe)
|
16
|
+
# @type [Sqreen::Kit::Signals::Context::HttpContext]
|
17
|
+
http_context = trace.context
|
18
|
+
|
19
|
+
all_redacted = []
|
20
|
+
|
21
|
+
# Redact headers; save redacted values
|
22
|
+
# headers are encoded as [key, value], not a hash, so
|
23
|
+
# they require some transformation
|
24
|
+
orig_headers = http_context.headers
|
25
|
+
if orig_headers
|
26
|
+
headers = orig_headers.map { |(k, v)| { k => v } }
|
27
|
+
headers, redacted = redactor.redact(headers)
|
28
|
+
http_context.headers = headers.map(&:first)
|
29
|
+
all_redacted += redacted
|
30
|
+
end
|
31
|
+
|
32
|
+
# Redact params; save redacted values
|
33
|
+
Kit::Signals::Context::HttpContext::PARAMS_ATTRS.each do |attr|
|
34
|
+
value = http_context.public_send(attr)
|
35
|
+
next unless value
|
36
|
+
value, redacted = redactor.redact(value)
|
37
|
+
all_redacted += redacted
|
38
|
+
http_context.public_send(:"#{attr}=", value)
|
39
|
+
end
|
40
|
+
|
41
|
+
all_redacted = all_redacted.uniq.map(&:downcase)
|
42
|
+
|
43
|
+
# Redact attacks and exceptions
|
44
|
+
# XXX: no redaction for infos in attacks/exceptions except for WAF data
|
45
|
+
# Is this the correct behavior?
|
46
|
+
redact_attacks!(trace, redactor, all_redacted)
|
47
|
+
redact_exceptions!(trace, redactor, all_redacted)
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
|
52
|
+
# @param [Sqreen::Kit::Signals::Specialized::HttpTrace] trace
|
53
|
+
# @param [Sqreen::SensitiveDataRedactor] redactor
|
54
|
+
# Redacts WAF data according to specific rules therefor
|
55
|
+
# Redacts infos according to general rules
|
56
|
+
def redact_attacks!(trace, redactor, redacted_data)
|
57
|
+
trace.data.each do |signal|
|
58
|
+
next unless signal.is_a?(Kit::Signals::Specialized::Attack)
|
59
|
+
# @type [Sqreen::Kit::Signals::Specialized::Attack::Payload] payload
|
60
|
+
payload = signal.payload
|
61
|
+
next unless payload.infos
|
62
|
+
|
63
|
+
if payload.infos[:waf_data]
|
64
|
+
redact_waf_attack_data!(payload.infos, redacted_data)
|
65
|
+
end
|
66
|
+
payload.infos, = redactor.redact(payload.infos)
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def redact_exceptions!(trace, redactor, redacted_data)
|
71
|
+
trace.data.each do |signal|
|
72
|
+
next unless signal.is_a?(Kit::Signals::Specialized::SqreenException)
|
73
|
+
infos = signal.infos
|
74
|
+
next unless infos
|
75
|
+
|
76
|
+
redact_waf_exception_data!(signal.infos, redacted_data) if signal.infos[:waf]
|
77
|
+
signal.infos, = redactor.redact(infos)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
# @param [Hash] infos from WAF attack
|
82
|
+
def redact_waf_attack_data!(infos, redacted_data)
|
83
|
+
begin
|
84
|
+
parsed = JSON.parse(infos[:waf_data])
|
85
|
+
rescue JSON::JSONError => e
|
86
|
+
logger.warn("waf_data is not valid json: #{e.message}")
|
87
|
+
return
|
88
|
+
end
|
89
|
+
redacted = parsed.each do |w|
|
90
|
+
next unless (filters = w['filter'])
|
91
|
+
|
92
|
+
filters.each do |f|
|
93
|
+
next unless (v = f['resolved_value'])
|
94
|
+
next unless redacted_data.include?(v.downcase)
|
95
|
+
|
96
|
+
f['match_status'] = SensitiveDataRedactor::MASK
|
97
|
+
f['resolved_value'] = SensitiveDataRedactor::MASK
|
98
|
+
end
|
99
|
+
end
|
100
|
+
infos[:waf_data] = JSON.dump(redacted)
|
101
|
+
end
|
102
|
+
|
103
|
+
# see https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000022-waf-data-sanitization.md#changes-to-the-agents
|
104
|
+
def redact_waf_exception_data!(infos, redacted_data)
|
105
|
+
return if redacted_data.empty?
|
106
|
+
infos[:waf].delete(:args)
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|