sqreen 1.19.1 → 1.20.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +22 -0
- data/lib/sqreen/agent_message.rb +20 -0
- data/lib/sqreen/aggregated_metric.rb +25 -0
- data/lib/sqreen/attack_detected.html +1 -2
- data/lib/sqreen/ca.crt +24 -0
- data/lib/sqreen/configuration.rb +10 -4
- data/lib/sqreen/deferred_logger.rb +4 -0
- data/lib/sqreen/deliveries/batch.rb +4 -1
- data/lib/sqreen/deliveries/simple.rb +4 -0
- data/lib/sqreen/endpoint_testing.rb +184 -0
- data/lib/sqreen/event.rb +7 -5
- data/lib/sqreen/events/attack.rb +23 -18
- data/lib/sqreen/events/remote_exception.rb +0 -22
- data/lib/sqreen/events/request_record.rb +15 -70
- data/lib/sqreen/frameworks/request_recorder.rb +13 -2
- data/lib/sqreen/graft/call.rb +32 -19
- data/lib/sqreen/graft/callback.rb +1 -1
- data/lib/sqreen/graft/hook.rb +97 -116
- data/lib/sqreen/graft/hook_point.rb +1 -1
- data/lib/sqreen/kit/signals/specialized/aggregated_metric.rb +72 -0
- data/lib/sqreen/kit/signals/specialized/attack.rb +57 -0
- data/lib/sqreen/kit/signals/specialized/binning_metric.rb +76 -0
- data/lib/sqreen/kit/signals/specialized/http_trace.rb +26 -0
- data/lib/sqreen/kit/signals/specialized/sdk_track_call.rb +50 -0
- data/lib/sqreen/kit/signals/specialized/sqreen_exception.rb +57 -0
- data/lib/sqreen/legacy/instrumentation.rb +10 -10
- data/lib/sqreen/legacy/old_event_submission_strategy.rb +221 -0
- data/lib/sqreen/legacy/waf_redactions.rb +49 -0
- data/lib/sqreen/log/loggable.rb +2 -1
- data/lib/sqreen/logger.rb +4 -0
- data/lib/sqreen/metrics/base.rb +3 -0
- data/lib/sqreen/metrics_store.rb +22 -12
- data/lib/sqreen/performance_notifications/binned_metrics.rb +8 -2
- data/lib/sqreen/rules.rb +4 -2
- data/lib/sqreen/rules/not_found_cb.rb +2 -0
- data/lib/sqreen/rules/rule_cb.rb +2 -0
- data/lib/sqreen/rules/waf_cb.rb +13 -10
- data/lib/sqreen/runner.rb +75 -8
- data/lib/sqreen/sensitive_data_redactor.rb +19 -31
- data/lib/sqreen/session.rb +51 -43
- data/lib/sqreen/signals/conversions.rb +283 -0
- data/lib/sqreen/signals/http_trace_redaction.rb +111 -0
- data/lib/sqreen/signals/signals_submission_strategy.rb +78 -0
- data/lib/sqreen/version.rb +1 -1
- data/lib/sqreen/weave/legacy/instrumentation.rb +56 -53
- metadata +45 -7
- data/lib/sqreen/backport.rb +0 -9
- data/lib/sqreen/backport/clock_gettime.rb +0 -74
- data/lib/sqreen/backport/original_name.rb +0 -88
@@ -0,0 +1,49 @@
|
|
1
|
+
# typed: ignore
|
2
|
+
|
3
|
+
# Copyright (c) 2015 Sqreen. All Rights Reserved.
|
4
|
+
# Please refer to our terms for more information: https://www.sqreen.com/terms.html
|
5
|
+
|
6
|
+
module Sqreen
|
7
|
+
module Legacy
|
8
|
+
module WafRedactions
|
9
|
+
class << self
|
10
|
+
def redact_attacks!(attacks, values)
|
11
|
+
return attacks if values.empty?
|
12
|
+
|
13
|
+
values = values.map { |v| v.downcase if v.is_a?(String) }
|
14
|
+
|
15
|
+
attacks.each do |e|
|
16
|
+
next(e) unless e[:infos]
|
17
|
+
next(e) unless e[:infos][:waf_data]
|
18
|
+
|
19
|
+
parsed = JSON.parse(e[:infos][:waf_data])
|
20
|
+
redacted = parsed.each do |w|
|
21
|
+
next unless (filters = w['filter'])
|
22
|
+
|
23
|
+
filters.each do |f|
|
24
|
+
next unless (v = f['resolved_value'])
|
25
|
+
next unless values.include?(v.downcase)
|
26
|
+
|
27
|
+
f['match_status'] = SensitiveDataRedactor::MASK
|
28
|
+
f['resolved_value'] = SensitiveDataRedactor::MASK
|
29
|
+
end
|
30
|
+
end
|
31
|
+
e[:infos][:waf_data] = JSON.dump(redacted)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
# see https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000022-waf-data-sanitization.md#changes-to-the-agents
|
36
|
+
def redact_exceptions!(exceptions, values)
|
37
|
+
return exceptions if values.empty?
|
38
|
+
|
39
|
+
exceptions.each do |e|
|
40
|
+
next(e) unless e[:infos]
|
41
|
+
next(e) unless e[:infos][:waf]
|
42
|
+
|
43
|
+
e[:infos][:waf].delete(:args)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
data/lib/sqreen/log/loggable.rb
CHANGED
@@ -4,6 +4,7 @@
|
|
4
4
|
# Please refer to our terms for more information: https://www.sqreen.com/terms.html
|
5
5
|
|
6
6
|
require 'logger'
|
7
|
+
require 'sqreen/log'
|
7
8
|
|
8
9
|
module Sqreen; end
|
9
10
|
module Sqreen::Log; end
|
@@ -23,6 +24,6 @@ module Sqreen::Log::Loggable
|
|
23
24
|
end
|
24
25
|
|
25
26
|
def logger
|
26
|
-
@logger ||
|
27
|
+
@logger || singleton_class.logger
|
27
28
|
end
|
28
29
|
end
|
data/lib/sqreen/logger.rb
CHANGED
data/lib/sqreen/metrics/base.rb
CHANGED
data/lib/sqreen/metrics_store.rb
CHANGED
@@ -3,6 +3,7 @@
|
|
3
3
|
# Copyright (c) 2015 Sqreen. All Rights Reserved.
|
4
4
|
# Please refer to our terms for more information: https://www.sqreen.com/terms.html
|
5
5
|
|
6
|
+
require 'sqreen/aggregated_metric'
|
6
7
|
require 'sqreen/metrics'
|
7
8
|
require 'sqreen/mono_time'
|
8
9
|
require 'sqreen/metrics_store/unknown_metric'
|
@@ -30,8 +31,9 @@ module Sqreen
|
|
30
31
|
|
31
32
|
# Definition contains a name,period and aggregate at least
|
32
33
|
# @param definition [Hash] a metric definition
|
34
|
+
# @param rule [RuleCB] the rule associated with this metric, if any
|
33
35
|
# @param mklass [Object] Override metric object (used in testing)
|
34
|
-
def create_metric(definition, mklass = nil)
|
36
|
+
def create_metric(definition, rule = nil, mklass = nil)
|
35
37
|
name = definition[NAME_KEY]
|
36
38
|
kind = definition[KIND_KEY]
|
37
39
|
klass = valid_metric(kind, name)
|
@@ -43,6 +45,9 @@ module Sqreen
|
|
43
45
|
definition[PERIOD_KEY],
|
44
46
|
nil # Start
|
45
47
|
]
|
48
|
+
metric.name = name
|
49
|
+
metric.rule = rule
|
50
|
+
metric.period = definition[PERIOD_KEY]
|
46
51
|
metric
|
47
52
|
end
|
48
53
|
|
@@ -50,7 +55,7 @@ module Sqreen
|
|
50
55
|
@metrics.key?(name)
|
51
56
|
end
|
52
57
|
|
53
|
-
# @
|
58
|
+
# @param at [Time] when is the store emptied
|
54
59
|
def update(name, at, key, value)
|
55
60
|
metric, period, start = @metrics[name]
|
56
61
|
raise UnregisteredMetric, "Unknown metric #{name}" unless metric
|
@@ -59,7 +64,7 @@ module Sqreen
|
|
59
64
|
end
|
60
65
|
|
61
66
|
# Drains every metrics and returns the store content
|
62
|
-
# @
|
67
|
+
# @param at [Time] when is the store emptied
|
63
68
|
def publish(flush = true, at = Sqreen.time)
|
64
69
|
@metrics.each do |name, (_, period, start)|
|
65
70
|
next_sample(name, at) if flush || !start.nil? && (start + period) < at
|
@@ -75,15 +80,20 @@ module Sqreen
|
|
75
80
|
metric = @metrics[name][0]
|
76
81
|
r = metric.next_sample(at)
|
77
82
|
@metrics[name][2] = at # new start
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
83
|
+
return unless r
|
84
|
+
|
85
|
+
r[NAME_KEY] = name
|
86
|
+
obs = r[Metric::OBSERVATION_KEY]
|
87
|
+
return unless obs && (!obs.respond_to?(:empty?) || !obs.empty?)
|
88
|
+
start_of_mono = Time.now.utc - Sqreen.time
|
89
|
+
|
90
|
+
agg = AggregatedMetric.new
|
91
|
+
agg.metric = metric
|
92
|
+
agg.rule = agg.metric.rule
|
93
|
+
agg.start = start_of_mono + r[Metric::START_KEY]
|
94
|
+
agg.finish = start_of_mono + r[Metric::FINISH_KEY]
|
95
|
+
agg.data = obs
|
96
|
+
@store << agg
|
87
97
|
end
|
88
98
|
|
89
99
|
def valid_metric(kind, name)
|
@@ -122,10 +122,16 @@ module Sqreen
|
|
122
122
|
attr_reader :metrics_store
|
123
123
|
attr_reader :period
|
124
124
|
|
125
|
-
def ensure_metric(metric_name)
|
125
|
+
def ensure_metric(metric_name, rule = nil)
|
126
126
|
return if metrics_store.metric?(metric_name)
|
127
127
|
metrics_store.create_metric(
|
128
|
-
|
128
|
+
{
|
129
|
+
'name' => metric_name,
|
130
|
+
'period' => period,
|
131
|
+
'kind' => 'Binning',
|
132
|
+
'options' => @perf_metric_opts,
|
133
|
+
},
|
134
|
+
rule
|
129
135
|
)
|
130
136
|
end
|
131
137
|
|
data/lib/sqreen/rules.rb
CHANGED
@@ -135,13 +135,15 @@ module Sqreen
|
|
135
135
|
return nil
|
136
136
|
end
|
137
137
|
|
138
|
+
rule_cb = cb_class.new(instr_class, instr_method, hash_rule)
|
139
|
+
|
138
140
|
if metrics_store
|
139
141
|
(hash_rule[Attrs::METRICS] || []).each do |metric|
|
140
|
-
metrics_store.create_metric(metric)
|
142
|
+
metrics_store.create_metric(metric, rule_cb)
|
141
143
|
end
|
142
144
|
end
|
143
145
|
|
144
|
-
|
146
|
+
rule_cb
|
145
147
|
rescue => e
|
146
148
|
rule_name = nil
|
147
149
|
rulespack_id = nil
|
data/lib/sqreen/rules/rule_cb.rb
CHANGED
@@ -61,7 +61,9 @@ module Sqreen
|
|
61
61
|
:infos => infos,
|
62
62
|
:rulespack_id => rulespack_id,
|
63
63
|
:rule_name => rule_name,
|
64
|
+
:attack_type => @rule['attack_type'], # for signal
|
64
65
|
:test => test,
|
66
|
+
:block => @rule['block'], # for signal
|
65
67
|
:time => at,
|
66
68
|
}
|
67
69
|
if payload_tpl.include?('context')
|
data/lib/sqreen/rules/waf_cb.rb
CHANGED
@@ -98,10 +98,10 @@ module Sqreen
|
|
98
98
|
|
99
99
|
case action
|
100
100
|
when :monitor
|
101
|
-
record_event({
|
101
|
+
record_event({ waf_data: data })
|
102
102
|
advise_action(nil)
|
103
103
|
when :block
|
104
|
-
record_event({
|
104
|
+
record_event({ waf_data: data })
|
105
105
|
advise_action(:raise)
|
106
106
|
when :good
|
107
107
|
advise_action(nil)
|
@@ -132,20 +132,23 @@ module Sqreen
|
|
132
132
|
end
|
133
133
|
|
134
134
|
def record_exception(exception, infos = {}, at = Time.now.utc)
|
135
|
-
infos.merge!(
|
135
|
+
infos.merge!(waf_infos(exception)) if exception.is_a?(Sqreen::WAFError)
|
136
136
|
super(exception, infos, at)
|
137
137
|
end
|
138
138
|
|
139
139
|
private
|
140
140
|
|
141
|
-
|
141
|
+
# see https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000016-waf-integration.md#error-management
|
142
|
+
def waf_infos(e)
|
142
143
|
{
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
144
|
+
waf: {
|
145
|
+
waf_rule: e.rule_name,
|
146
|
+
error_code: ERROR_CODES[e.error],
|
147
|
+
}.tap do |r|
|
148
|
+
r[:error_data] = e.data if e.data
|
149
|
+
r[:args] = e.args if e.arg
|
150
|
+
end,
|
151
|
+
}
|
149
152
|
end
|
150
153
|
|
151
154
|
ERROR_CODES = {
|
data/lib/sqreen/runner.rb
CHANGED
@@ -11,6 +11,7 @@ require 'sqreen/events/attack'
|
|
11
11
|
|
12
12
|
require 'sqreen/log'
|
13
13
|
|
14
|
+
require 'sqreen/agent_message'
|
14
15
|
require 'sqreen/rules'
|
15
16
|
require 'sqreen/session'
|
16
17
|
require 'sqreen/remote_command'
|
@@ -18,11 +19,13 @@ require 'sqreen/capped_queue'
|
|
18
19
|
require 'sqreen/metrics_store'
|
19
20
|
require 'sqreen/deliveries/simple'
|
20
21
|
require 'sqreen/deliveries/batch'
|
22
|
+
require 'sqreen/endpoint_testing'
|
21
23
|
require 'sqreen/performance_notifications/metrics'
|
22
24
|
require 'sqreen/performance_notifications/binned_metrics'
|
23
25
|
require 'sqreen/legacy/instrumentation'
|
24
26
|
require 'sqreen/call_countable'
|
25
27
|
require 'sqreen/weave/legacy/instrumentation'
|
28
|
+
require 'sqreen/kit/configuration'
|
26
29
|
|
27
30
|
module Sqreen
|
28
31
|
@features = {}
|
@@ -37,6 +40,8 @@ module Sqreen
|
|
37
40
|
PERF_METRICS_PERIOD = 60 # 1 min
|
38
41
|
DEFAULT_PERF_LEVEL = 0 # disabled
|
39
42
|
|
43
|
+
DEFAULT_USE_SIGNALS = false
|
44
|
+
|
40
45
|
class << self
|
41
46
|
attr_reader :features
|
42
47
|
def update_features(features)
|
@@ -87,7 +92,9 @@ module Sqreen
|
|
87
92
|
|
88
93
|
attr_accessor :heartbeat_delay
|
89
94
|
attr_accessor :metrics_engine
|
95
|
+
# @return [Sqreen::Deliveries::Simple]
|
90
96
|
attr_reader :deliverer
|
97
|
+
# @return [Sqreen::Session]
|
91
98
|
attr_reader :session
|
92
99
|
attr_reader :instrumenter
|
93
100
|
attr_accessor :running
|
@@ -108,15 +115,24 @@ module Sqreen
|
|
108
115
|
@next_metrics = []
|
109
116
|
@running = true
|
110
117
|
|
118
|
+
@proxy_url = @configuration.get(:proxy_url)
|
119
|
+
chosen_endpoints = determine_endpoints
|
120
|
+
|
111
121
|
@token = @configuration.get(:token)
|
112
122
|
@app_name = @configuration.get(:app_name)
|
113
|
-
@url =
|
123
|
+
@url = chosen_endpoints.control.url
|
124
|
+
@cert_store = chosen_endpoints.control.ca_store
|
125
|
+
|
114
126
|
Sqreen.update_whitelisted_paths([])
|
115
127
|
Sqreen.update_whitelisted_ips({})
|
116
128
|
Sqreen.update_performance_budget(nil)
|
117
|
-
raise(Sqreen::Exception, 'no url found') unless @url
|
118
129
|
raise(Sqreen::TokenNotFoundException, 'no token found') unless @token
|
119
130
|
|
131
|
+
Sqreen::Kit::Configuration.logger = Sqreen.log
|
132
|
+
Sqreen::Kit::Configuration.ingestion_url = chosen_endpoints.ingestion.url
|
133
|
+
Sqreen::Kit::Configuration.certificate_store = chosen_endpoints.ingestion.ca_store
|
134
|
+
Sqreen::Kit::Configuration.proxy_url = @proxy_url
|
135
|
+
|
120
136
|
register_exit_cb if set_at_exit
|
121
137
|
|
122
138
|
self.metrics_engine = MetricsStore.new
|
@@ -133,6 +149,7 @@ module Sqreen
|
|
133
149
|
|
134
150
|
Sqreen.log.debug "Using token #{@token}"
|
135
151
|
response = create_session(session_class)
|
152
|
+
post_endpoint_testing_msgs(chosen_endpoints)
|
136
153
|
wanted_features = response.fetch('features', {})
|
137
154
|
conf_initial_features = configuration.get(:initial_features)
|
138
155
|
unless conf_initial_features.nil?
|
@@ -142,10 +159,10 @@ module Sqreen
|
|
142
159
|
Sqreen.log.debug do
|
143
160
|
"Override initial features with #{conf_features.inspect}"
|
144
161
|
end
|
145
|
-
wanted_features = conf_features
|
162
|
+
wanted_features = wanted_features.merge(conf_features)
|
146
163
|
rescue
|
147
164
|
Sqreen.log.warn do
|
148
|
-
"NOT using invalid
|
165
|
+
"NOT using invalid initial features #{conf_initial_features}"
|
149
166
|
end
|
150
167
|
end
|
151
168
|
end
|
@@ -161,7 +178,7 @@ module Sqreen
|
|
161
178
|
end
|
162
179
|
|
163
180
|
def create_session(session_class)
|
164
|
-
@session = session_class.new(@url, @token, @app_name)
|
181
|
+
@session = session_class.new(@url, @cert_store, @token, @app_name, @proxy_url)
|
165
182
|
session.login(@framework)
|
166
183
|
end
|
167
184
|
|
@@ -170,8 +187,18 @@ module Sqreen
|
|
170
187
|
@deliverer = new_deliverer
|
171
188
|
end
|
172
189
|
|
173
|
-
def batch_events(batch_size, max_staleness = nil)
|
190
|
+
def batch_events(batch_size, max_staleness = nil, use_signals = false)
|
174
191
|
size = batch_size.to_i
|
192
|
+
|
193
|
+
if size <= 1 && use_signals
|
194
|
+
Sqreen.log.warn do
|
195
|
+
"Using signals with no delivery batching is unsupported. " \
|
196
|
+
"Using instead batching with batch size = 30, max_staleness = 60"
|
197
|
+
end
|
198
|
+
size = 30
|
199
|
+
max_staleness = 60
|
200
|
+
end
|
201
|
+
|
175
202
|
self.deliverer = if size < 1
|
176
203
|
Deliveries::Simple.new(session)
|
177
204
|
else
|
@@ -301,19 +328,37 @@ module Sqreen
|
|
301
328
|
def do_heartbeat
|
302
329
|
@last_heartbeat_request = Time.now
|
303
330
|
@next_metrics.concat(metrics_engine.publish(false)) if metrics_engine
|
304
|
-
|
331
|
+
metrics_in_hb = use_signals? ? nil : next_metrics
|
332
|
+
|
333
|
+
res = session.heartbeat(next_command_results, metrics_in_hb)
|
305
334
|
next_command_results.clear
|
335
|
+
|
336
|
+
deliver_metrics_as_event if use_signals?
|
306
337
|
next_metrics.clear
|
338
|
+
|
307
339
|
process_commands(res['commands'])
|
308
340
|
end
|
309
341
|
|
342
|
+
def deliver_metrics_as_event
|
343
|
+
# this is disastrous withe simple delivery strategy,
|
344
|
+
# as each aggregated metric would trigger an http request
|
345
|
+
# Sending of metrics is therefore not supported with simple delivery strategy
|
346
|
+
# TODO: Confirm that only batch is used in production
|
347
|
+
next_metrics.each { |x| deliverer.post_event(x) }
|
348
|
+
end
|
349
|
+
|
310
350
|
def features(_context_infos = {})
|
311
351
|
Sqreen.features
|
312
352
|
end
|
313
353
|
|
354
|
+
def use_signals?
|
355
|
+
features.fetch('use_signals', DEFAULT_USE_SIGNALS)
|
356
|
+
end
|
357
|
+
|
314
358
|
def features=(features)
|
315
359
|
Sqreen.update_features(features)
|
316
360
|
session.request_compression = features['request_compression'] if session
|
361
|
+
session.use_signals = use_signals?
|
317
362
|
self.performance_metrics_period = features['performance_metrics_period']
|
318
363
|
|
319
364
|
unless @configuration.get(:weave)
|
@@ -331,7 +376,7 @@ module Sqreen
|
|
331
376
|
hd = features['heartbeat_delay'].to_i
|
332
377
|
self.heartbeat_delay = hd if hd > 0
|
333
378
|
return if features['batch_size'].nil?
|
334
|
-
batch_events(features['batch_size'], features['max_staleness'])
|
379
|
+
batch_events(features['batch_size'], features['max_staleness'], use_signals?)
|
335
380
|
end
|
336
381
|
|
337
382
|
def change_whitelisted_paths(paths, _context_infos = {})
|
@@ -470,6 +515,28 @@ module Sqreen
|
|
470
515
|
|
471
516
|
private
|
472
517
|
|
518
|
+
def post_endpoint_testing_msgs(chosen_endpoints)
|
519
|
+
chosen_endpoints.messages.each do |msg|
|
520
|
+
session.post_agent_message(@framework, msg)
|
521
|
+
end
|
522
|
+
rescue => e
|
523
|
+
Sqreen.log.warn "Error submitting agent message: #{e}"
|
524
|
+
RemoteException.record(e)
|
525
|
+
end
|
526
|
+
|
527
|
+
def determine_endpoints
|
528
|
+
# there's no sniffing going on; just a misnamed config setting
|
529
|
+
if @configuration.get(:no_sniff_domains)
|
530
|
+
# reproduces behaviour before endpoint testing was introduced
|
531
|
+
EndpointTesting.no_test_endpoints(@configuration.get(:url),
|
532
|
+
@configuration.get(:ingestion_url))
|
533
|
+
else
|
534
|
+
EndpointTesting.test_endpoints(@proxy_url,
|
535
|
+
@configuration.get(:url),
|
536
|
+
@configuration.get(:ingestion_url))
|
537
|
+
end
|
538
|
+
end
|
539
|
+
|
473
540
|
def load_actions(hashes)
|
474
541
|
unsupported = Set.new
|
475
542
|
|