sqreen 1.19.1-java → 1.21.0.beta3-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/CHANGELOG.md +34 -0
- data/lib/sqreen/actions/block_user.rb +1 -1
- data/lib/sqreen/actions/redirect_ip.rb +1 -1
- data/lib/sqreen/actions/redirect_user.rb +1 -1
- data/lib/sqreen/agent_message.rb +20 -0
- data/lib/sqreen/aggregated_metric.rb +25 -0
- data/lib/sqreen/attack_detected.html +1 -2
- data/lib/sqreen/ca.crt +24 -0
- data/lib/sqreen/condition_evaluator.rb +9 -2
- data/lib/sqreen/conditionable.rb +24 -6
- data/lib/sqreen/configuration.rb +11 -5
- data/lib/sqreen/deferred_logger.rb +50 -14
- data/lib/sqreen/deliveries/batch.rb +12 -2
- data/lib/sqreen/deliveries/simple.rb +4 -0
- data/lib/sqreen/deprecation.rb +38 -0
- data/lib/sqreen/ecosystem.rb +96 -0
- data/lib/sqreen/ecosystem/dispatch_table.rb +43 -0
- data/lib/sqreen/ecosystem/exception_reporting.rb +26 -0
- data/lib/sqreen/ecosystem/http/net_http.rb +50 -0
- data/lib/sqreen/ecosystem/http/rack_request.rb +39 -0
- data/lib/sqreen/ecosystem/loggable.rb +13 -0
- data/lib/sqreen/ecosystem/module_api.rb +30 -0
- data/lib/sqreen/ecosystem/module_api/event_listener.rb +18 -0
- data/lib/sqreen/ecosystem/module_api/instrumentation.rb +23 -0
- data/lib/sqreen/ecosystem/module_api/message_producer.rb +51 -0
- data/lib/sqreen/ecosystem/module_api/signal_producer.rb +24 -0
- data/lib/sqreen/ecosystem/module_api/tracing.rb +45 -0
- data/lib/sqreen/ecosystem/module_api/tracing/client_data.rb +31 -0
- data/lib/sqreen/ecosystem/module_api/tracing/server_data.rb +27 -0
- data/lib/sqreen/ecosystem/module_api/tracing_id_generation.rb +16 -0
- data/lib/sqreen/ecosystem/module_api/transaction_storage.rb +71 -0
- data/lib/sqreen/ecosystem/module_registry.rb +44 -0
- data/lib/sqreen/ecosystem/redis/redis_connection.rb +43 -0
- data/lib/sqreen/ecosystem/tracing/modules/client.rb +31 -0
- data/lib/sqreen/ecosystem/tracing/modules/server.rb +30 -0
- data/lib/sqreen/ecosystem/tracing/sampler.rb +160 -0
- data/lib/sqreen/ecosystem/tracing/sampling_configuration.rb +150 -0
- data/lib/sqreen/ecosystem/tracing/signals/tracing_client.rb +53 -0
- data/lib/sqreen/ecosystem/tracing/signals/tracing_server.rb +53 -0
- data/lib/sqreen/ecosystem/tracing_broker.rb +101 -0
- data/lib/sqreen/ecosystem/tracing_id_setup.rb +34 -0
- data/lib/sqreen/ecosystem/transaction_storage.rb +64 -0
- data/lib/sqreen/ecosystem/util/call_writers_from_init.rb +13 -0
- data/lib/sqreen/ecosystem_integration.rb +87 -0
- data/lib/sqreen/ecosystem_integration/around_callbacks.rb +99 -0
- data/lib/sqreen/ecosystem_integration/instrumentation_service.rb +42 -0
- data/lib/sqreen/ecosystem_integration/request_lifecycle_tracking.rb +58 -0
- data/lib/sqreen/ecosystem_integration/signal_consumption.rb +35 -0
- data/lib/sqreen/endpoint_testing.rb +184 -0
- data/lib/sqreen/event.rb +7 -5
- data/lib/sqreen/events/attack.rb +23 -18
- data/lib/sqreen/events/remote_exception.rb +0 -22
- data/lib/sqreen/events/request_record.rb +15 -71
- data/lib/sqreen/frameworks/generic.rb +24 -1
- data/lib/sqreen/frameworks/rails.rb +0 -7
- data/lib/sqreen/frameworks/request_recorder.rb +15 -2
- data/lib/sqreen/graft/call.rb +106 -19
- data/lib/sqreen/graft/callback.rb +1 -1
- data/lib/sqreen/graft/hook.rb +212 -100
- data/lib/sqreen/graft/hook_point.rb +18 -11
- data/lib/sqreen/kit/signals/specialized/aggregated_metric.rb +72 -0
- data/lib/sqreen/kit/signals/specialized/attack.rb +57 -0
- data/lib/sqreen/kit/signals/specialized/binning_metric.rb +76 -0
- data/lib/sqreen/kit/signals/specialized/http_trace.rb +26 -0
- data/lib/sqreen/kit/signals/specialized/sdk_track_call.rb +50 -0
- data/lib/sqreen/kit/signals/specialized/sqreen_exception.rb +57 -0
- data/lib/sqreen/legacy/instrumentation.rb +22 -10
- data/lib/sqreen/legacy/old_event_submission_strategy.rb +228 -0
- data/lib/sqreen/legacy/waf_redactions.rb +49 -0
- data/lib/sqreen/log.rb +3 -2
- data/lib/sqreen/log/loggable.rb +2 -1
- data/lib/sqreen/logger.rb +24 -0
- data/lib/sqreen/metrics.rb +1 -0
- data/lib/sqreen/metrics/base.rb +3 -0
- data/lib/sqreen/metrics/req_detailed.rb +41 -0
- data/lib/sqreen/metrics_store.rb +33 -12
- data/lib/sqreen/null_logger.rb +22 -0
- data/lib/sqreen/performance_notifications/binned_metrics.rb +8 -2
- data/lib/sqreen/remote_command.rb +4 -0
- data/lib/sqreen/rules.rb +12 -6
- data/lib/sqreen/rules/blacklist_ips_cb.rb +2 -2
- data/lib/sqreen/rules/custom_error_cb.rb +3 -3
- data/lib/sqreen/rules/not_found_cb.rb +2 -0
- data/lib/sqreen/rules/rule_cb.rb +6 -2
- data/lib/sqreen/rules/waf_cb.rb +16 -13
- data/lib/sqreen/runner.rb +138 -16
- data/lib/sqreen/sensitive_data_redactor.rb +19 -31
- data/lib/sqreen/session.rb +53 -43
- data/lib/sqreen/signals/conversions.rb +288 -0
- data/lib/sqreen/signals/http_trace_redaction.rb +111 -0
- data/lib/sqreen/signals/signals_submission_strategy.rb +78 -0
- data/lib/sqreen/version.rb +1 -1
- data/lib/sqreen/weave/budget.rb +35 -0
- data/lib/sqreen/weave/legacy/instrumentation.rb +277 -135
- data/lib/sqreen/worker.rb +6 -2
- metadata +86 -10
- data/lib/sqreen/backport.rb +0 -9
- data/lib/sqreen/backport/clock_gettime.rb +0 -74
- data/lib/sqreen/backport/original_name.rb +0 -88
- data/lib/sqreen/encoding_sanitizer.rb +0 -27
@@ -0,0 +1,111 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'sqreen/kit/loggable'
|
3
|
+
require 'sqreen/kit/signals/specialized/http_trace'
|
4
|
+
|
5
|
+
module Sqreen
|
6
|
+
module Signals
|
7
|
+
module HttpTraceRedaction
|
8
|
+
class << self
|
9
|
+
include Sqreen::Kit::Loggable
|
10
|
+
|
11
|
+
# @param [Sqreen::Kit::Signals::Specialized::HttpTrace] trace
|
12
|
+
# @param [Sqreen::SensitiveDataRedactor] redactor
|
13
|
+
def redact_trace!(trace, redactor)
|
14
|
+
return unless redactor
|
15
|
+
# redact headers (keys unsafe)
|
16
|
+
# @type [Sqreen::Kit::Signals::Context::HttpContext]
|
17
|
+
http_context = trace.context
|
18
|
+
|
19
|
+
all_redacted = []
|
20
|
+
|
21
|
+
# Redact headers; save redacted values
|
22
|
+
# headers are encoded as [key, value], not a hash, so
|
23
|
+
# they require some transformation
|
24
|
+
orig_headers = http_context.headers
|
25
|
+
if orig_headers
|
26
|
+
headers = orig_headers.map { |(k, v)| { k => v } }
|
27
|
+
headers, redacted = redactor.redact(headers)
|
28
|
+
http_context.headers = headers.map(&:first)
|
29
|
+
all_redacted += redacted
|
30
|
+
end
|
31
|
+
|
32
|
+
# Redact params; save redacted values
|
33
|
+
Kit::Signals::Context::HttpContext::PARAMS_ATTRS.each do |attr|
|
34
|
+
value = http_context.public_send(attr)
|
35
|
+
next unless value
|
36
|
+
value, redacted = redactor.redact(value)
|
37
|
+
all_redacted += redacted
|
38
|
+
http_context.public_send(:"#{attr}=", value)
|
39
|
+
end
|
40
|
+
|
41
|
+
all_redacted = all_redacted.uniq.map(&:downcase)
|
42
|
+
|
43
|
+
# Redact attacks and exceptions
|
44
|
+
# XXX: no redaction for infos in attacks/exceptions except for WAF data
|
45
|
+
# Is this the correct behavior?
|
46
|
+
redact_attacks!(trace, redactor, all_redacted)
|
47
|
+
redact_exceptions!(trace, redactor, all_redacted)
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
|
52
|
+
# @param [Sqreen::Kit::Signals::Specialized::HttpTrace] trace
|
53
|
+
# @param [Sqreen::SensitiveDataRedactor] redactor
|
54
|
+
# Redacts WAF data according to specific rules therefor
|
55
|
+
# Redacts infos according to general rules
|
56
|
+
def redact_attacks!(trace, redactor, redacted_data)
|
57
|
+
trace.data.each do |signal|
|
58
|
+
next unless signal.is_a?(Kit::Signals::Specialized::Attack)
|
59
|
+
# @type [Sqreen::Kit::Signals::Specialized::Attack::Payload] payload
|
60
|
+
payload = signal.payload
|
61
|
+
next unless payload.infos
|
62
|
+
|
63
|
+
if payload.infos[:waf_data]
|
64
|
+
redact_waf_attack_data!(payload.infos, redacted_data)
|
65
|
+
end
|
66
|
+
payload.infos, = redactor.redact(payload.infos)
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def redact_exceptions!(trace, redactor, redacted_data)
|
71
|
+
trace.data.each do |signal|
|
72
|
+
next unless signal.is_a?(Kit::Signals::Specialized::SqreenException)
|
73
|
+
infos = signal.infos
|
74
|
+
next unless infos
|
75
|
+
|
76
|
+
redact_waf_exception_data!(signal.infos, redacted_data) if signal.infos[:waf]
|
77
|
+
signal.infos, = redactor.redact(infos)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
# @param [Hash] infos from WAF attack
|
82
|
+
def redact_waf_attack_data!(infos, redacted_data)
|
83
|
+
begin
|
84
|
+
parsed = JSON.parse(infos[:waf_data])
|
85
|
+
rescue JSON::JSONError => e
|
86
|
+
logger.warn("waf_data is not valid json: #{e.message}")
|
87
|
+
return
|
88
|
+
end
|
89
|
+
redacted = parsed.each do |w|
|
90
|
+
next unless (filters = w['filter'])
|
91
|
+
|
92
|
+
filters.each do |f|
|
93
|
+
next unless (v = f['resolved_value'])
|
94
|
+
next unless redacted_data.include?(v.downcase)
|
95
|
+
|
96
|
+
f['match_status'] = SensitiveDataRedactor::MASK
|
97
|
+
f['resolved_value'] = SensitiveDataRedactor::MASK
|
98
|
+
end
|
99
|
+
end
|
100
|
+
infos[:waf_data] = JSON.dump(redacted)
|
101
|
+
end
|
102
|
+
|
103
|
+
# see https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000022-waf-data-sanitization.md#changes-to-the-agents
|
104
|
+
def redact_waf_exception_data!(infos, redacted_data)
|
105
|
+
return if redacted_data.empty?
|
106
|
+
infos[:waf].delete(:args)
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
@@ -0,0 +1,78 @@
|
|
1
|
+
require 'sqreen/aggregated_metric'
|
2
|
+
require 'sqreen/kit'
|
3
|
+
require 'sqreen/kit/string_sanitizer'
|
4
|
+
require 'sqreen/signals/conversions'
|
5
|
+
require 'sqreen/log/loggable'
|
6
|
+
|
7
|
+
module Sqreen
|
8
|
+
module Signals
|
9
|
+
# see also Sqreen::Legacy::OldEventSubmissionStrategy
|
10
|
+
# usage in Sqreen:Session
|
11
|
+
class SignalsSubmissionStrategy
|
12
|
+
include Sqreen::Log::Loggable
|
13
|
+
|
14
|
+
# @param [Array<Sqreen::AggregatedMetric>] metrics
|
15
|
+
def post_metrics(metrics)
|
16
|
+
return if metrics.nil? || metrics.empty?
|
17
|
+
|
18
|
+
guarded 'Failed to serialize or submit aggregated metrics' do
|
19
|
+
batch = metrics.map do |m|
|
20
|
+
Conversions.convert_metric_sample(m)
|
21
|
+
end
|
22
|
+
client.report_batch(batch)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
# @param _attack [Sqreen::Attack]
|
27
|
+
# XXX: unused
|
28
|
+
def post_attack(_attack)
|
29
|
+
raise NotImplementedError
|
30
|
+
end
|
31
|
+
|
32
|
+
# @param request_record [Sqreen::RequestRecord]
|
33
|
+
def post_request_record(request_record)
|
34
|
+
guarded 'Failed to serialize or submit request record' do
|
35
|
+
trace = Conversions.convert_req_record(request_record)
|
36
|
+
append_sanitizing_filter(trace)
|
37
|
+
client.report_trace(trace)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
# Post an exception to Sqreen for analysis
|
42
|
+
# @param exception [RemoteException] Exception and context to be sent over
|
43
|
+
def post_sqreen_exception(exception)
|
44
|
+
guarded 'Failed to serialize or submit exception', false do
|
45
|
+
data = Conversions.convert_exception(exception)
|
46
|
+
append_sanitizing_filter(data)
|
47
|
+
client.report_signal(data)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def post_batch(events)
|
52
|
+
guarded 'Failed to serialize or submit batch of events' do
|
53
|
+
batch = Conversions.convert_batch(events)
|
54
|
+
batch.each { |sig_or_trace| append_sanitizing_filter(sig_or_trace) }
|
55
|
+
client.report_batch(batch)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
private
|
60
|
+
|
61
|
+
def append_sanitizing_filter(sig_or_trace)
|
62
|
+
sig_or_trace.append_to_h_filter Kit::StringSanitizer.method(:sanitize)
|
63
|
+
end
|
64
|
+
|
65
|
+
# we don't want exceptions to propagate and kill the worker thread
|
66
|
+
def guarded(msg, report = true)
|
67
|
+
yield
|
68
|
+
rescue StandardError => e
|
69
|
+
logger.warn "#{msg}: #{e.message}\n#{e.backtrace.map { |x| " #{x}" }.join("\n")}"
|
70
|
+
post_sqreen_exception(RemoteException.new(e)) if report
|
71
|
+
end
|
72
|
+
|
73
|
+
def client
|
74
|
+
Sqreen::Kit.auth_signals_client
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
data/lib/sqreen/version.rb
CHANGED
@@ -0,0 +1,35 @@
|
|
1
|
+
# typed: false
|
2
|
+
|
3
|
+
# Copyright (c) 2015 Sqreen. All Rights Reserved.
|
4
|
+
# Please refer to our terms for more information: https://www.sqreen.com/terms.html
|
5
|
+
|
6
|
+
require 'sqreen/log/loggable'
|
7
|
+
require 'sqreen/weave'
|
8
|
+
|
9
|
+
class Sqreen::Weave::Budget
|
10
|
+
include Sqreen::Log::Loggable
|
11
|
+
|
12
|
+
def initialize(threshold)
|
13
|
+
@threshold = threshold
|
14
|
+
end
|
15
|
+
|
16
|
+
attr_reader :threshold
|
17
|
+
|
18
|
+
def to_h
|
19
|
+
{ threshold: threshold }
|
20
|
+
end
|
21
|
+
|
22
|
+
class << self
|
23
|
+
attr_reader :current
|
24
|
+
|
25
|
+
def update(opts = nil)
|
26
|
+
Sqreen::Weave.logger.info("budget update:#{opts.inspect}")
|
27
|
+
|
28
|
+
return @current = nil if opts.nil? || opts.empty?
|
29
|
+
|
30
|
+
threshold = opts[:threshold]
|
31
|
+
|
32
|
+
@current = threshold
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -4,23 +4,41 @@
|
|
4
4
|
# Please refer to our terms for more information: https://www.sqreen.com/terms.html
|
5
5
|
|
6
6
|
require 'sqreen/weave/legacy'
|
7
|
+
require 'sqreen/weave/budget'
|
8
|
+
require 'sqreen/graft/hook'
|
7
9
|
require 'sqreen/graft/hook_point'
|
8
10
|
require 'sqreen/call_countable'
|
9
11
|
require 'sqreen/rules'
|
10
12
|
require 'sqreen/rules/record_request_context'
|
13
|
+
require 'sqreen/sqreen_signed_verifier'
|
14
|
+
require 'rack/request'
|
15
|
+
begin
|
16
|
+
require 'sq_detailed_metrics'
|
17
|
+
rescue LoadError => _e # rubocop:disable Lint/HandleExceptions
|
18
|
+
end
|
11
19
|
|
12
20
|
class Sqreen::Weave::Legacy::Instrumentation
|
13
21
|
attr_accessor :metrics_engine
|
14
22
|
|
23
|
+
HAS_SQ_DETAILED_METRICS = defined?(::SqDetailedMetrics)
|
24
|
+
REQ_LVL_2_METRIC = 'request_level_perf'.freeze
|
25
|
+
|
15
26
|
def initialize(metrics_engine, opts = {})
|
16
27
|
Sqreen::Weave.logger.debug { "#{self.class.name}#initialize #{metrics_engine}" }
|
17
28
|
@hooks = []
|
18
29
|
|
30
|
+
unless HAS_SQ_DETAILED_METRICS
|
31
|
+
Sqreen::Weave.logger.warn { "Detailed metrics are unavailable" }
|
32
|
+
end
|
33
|
+
|
19
34
|
self.metrics_engine = metrics_engine
|
20
35
|
|
21
36
|
### bail out if no metric engine
|
22
37
|
return if metrics_engine.nil?
|
23
38
|
|
39
|
+
# XXX: these metric definitions do not support change of opts
|
40
|
+
# due to features updates!
|
41
|
+
|
24
42
|
### init metric to count calls to sqreen
|
25
43
|
metrics_engine.create_metric(
|
26
44
|
'name' => 'sqreen_call_counts',
|
@@ -60,12 +78,42 @@ class Sqreen::Weave::Legacy::Instrumentation
|
|
60
78
|
'options' => opts[:perf_metric_percent] || { 'base' => 1.3, 'factor' => 1.0 },
|
61
79
|
)
|
62
80
|
|
81
|
+
metrics_engine.create_metric(
|
82
|
+
'name' => 'req.sq.hook.overhead',
|
83
|
+
'period' => 60,
|
84
|
+
'kind' => 'Binning',
|
85
|
+
'options' => { 'base' => 2.0, 'factor' => 0.1 },
|
86
|
+
)
|
87
|
+
|
88
|
+
metrics_engine.create_metric(
|
89
|
+
'name' => 'sq.hook.overhead',
|
90
|
+
'period' => 60,
|
91
|
+
'kind' => 'Binning',
|
92
|
+
'options' => { 'base' => 2.0, 'factor' => 0.1 },
|
93
|
+
)
|
94
|
+
|
95
|
+
metrics_engine.create_metric(
|
96
|
+
'name' => 'sq.shrinkwrap',
|
97
|
+
'period' => 60,
|
98
|
+
'kind' => 'Binning',
|
99
|
+
'options' => { 'base' => 2.0, 'factor' => 0.1 },
|
100
|
+
)
|
101
|
+
|
63
102
|
Sqreen.thread_cpu_time? && metrics_engine.create_metric(
|
64
103
|
'name' => 'sq_thread_cpu_pct',
|
65
104
|
'period' => opts[:period] || 60,
|
66
105
|
'kind' => 'Binning',
|
67
106
|
'options' => opts[:perf_metric_percent] || { 'base' => 1.3, 'factor' => 1.0 },
|
68
107
|
)
|
108
|
+
|
109
|
+
if HAS_SQ_DETAILED_METRICS # rubocop:disable Style/GuardClause
|
110
|
+
@lvl_2_metric = metrics_engine.create_metric(
|
111
|
+
'name' => REQ_LVL_2_METRIC,
|
112
|
+
'period' => opts[:perf_req_metrics_period] || 60,
|
113
|
+
'kind' => 'ReqDetailed',
|
114
|
+
)
|
115
|
+
@lvl_2_max_reqs = opts[:perf_req_metrics_max_reqs] || 100
|
116
|
+
end
|
69
117
|
end
|
70
118
|
|
71
119
|
# needed by Sqreen::Runner#initialize
|
@@ -84,6 +132,15 @@ class Sqreen::Weave::Legacy::Instrumentation
|
|
84
132
|
|
85
133
|
### set up rule signature verifier
|
86
134
|
verifier = nil
|
135
|
+
if Sqreen.features['rules_signature'] &&
|
136
|
+
Sqreen.config_get(:rules_verify_signature) == true &&
|
137
|
+
!defined?(::JRUBY_VERSION)
|
138
|
+
verifier = Sqreen::SqreenSignedVerifier.new
|
139
|
+
Sqreen::Weave.logger.debug('Rules signature enabled')
|
140
|
+
else
|
141
|
+
Sqreen::Weave.logger.debug('Rules signature disabled')
|
142
|
+
end
|
143
|
+
|
87
144
|
### force clean instrumentation callback list
|
88
145
|
@hooks = []
|
89
146
|
### for each rule description
|
@@ -94,6 +151,25 @@ class Sqreen::Weave::Legacy::Instrumentation
|
|
94
151
|
next unless rule_callback
|
95
152
|
### attach framework to callback
|
96
153
|
rule_callback.framework = framework
|
154
|
+
## create metric
|
155
|
+
Sqreen::Weave.logger.debug { "Adding rule metric: #{rule_callback}" }
|
156
|
+
[:pre, :post, :failing].each do |whence|
|
157
|
+
next unless rule_callback.send(:"#{whence}?")
|
158
|
+
metric_name = "sq.#{rule['name']}.#{whence}"
|
159
|
+
metrics_engine.create_metric(
|
160
|
+
'name' => metric_name,
|
161
|
+
'period' => 60,
|
162
|
+
'kind' => 'Binning',
|
163
|
+
'options' => { 'base' => 2.0, 'factor' => 0.1 },
|
164
|
+
)
|
165
|
+
metric_name = "req.sq.#{rule['name']}.#{whence}"
|
166
|
+
metrics_engine.create_metric(
|
167
|
+
'name' => metric_name,
|
168
|
+
'period' => 60,
|
169
|
+
'kind' => 'Binning',
|
170
|
+
'options' => { 'base' => 2.0, 'factor' => 0.1 },
|
171
|
+
)
|
172
|
+
end
|
97
173
|
### install callback, observing priority
|
98
174
|
Sqreen::Weave.logger.debug { "Adding rule callback: #{rule_callback}" }
|
99
175
|
@hooks << add_callback("weave,rule=#{rule['name']}", rule_callback, strategy)
|
@@ -107,30 +183,62 @@ class Sqreen::Weave::Legacy::Instrumentation
|
|
107
183
|
end
|
108
184
|
|
109
185
|
metrics_engine = self.metrics_engine
|
186
|
+
lvl_2_metric = @lvl_2_metric
|
187
|
+
lvl_2_max_reqs = @lvl_2_max_reqs
|
188
|
+
|
110
189
|
request_hook = Sqreen::Graft::Hook['Sqreen::ShrinkWrap#call', strategy]
|
111
190
|
@hooks << request_hook
|
112
191
|
request_hook.add do
|
113
|
-
before('wave,meta,request', rank: -100000, mandatory: true) do |
|
192
|
+
before('wave,meta,request', rank: -100000, mandatory: true) do |call|
|
114
193
|
next unless Sqreen.instrumentation_ready
|
115
194
|
|
116
|
-
|
117
|
-
|
195
|
+
# shrinkwrap_timer = Sqreen::Graft::Timer.new('weave,shrinkwrap')
|
196
|
+
# shrinkwrap_timer.start
|
197
|
+
|
198
|
+
request_timer = Sqreen::Graft::Timer.new("request")
|
199
|
+
request_timer.start
|
200
|
+
sqreen_timer = Sqreen::Graft::Timer.new("sqreen")
|
201
|
+
budget = Sqreen::Weave::Budget.current
|
202
|
+
|
203
|
+
timed_level = (Sqreen.features['perf_level'] || 1).to_i
|
204
|
+
timed_level = 1 if !HAS_SQ_DETAILED_METRICS && timed_level == 2
|
205
|
+
if timed_level == 2 && lvl_2_metric.num_requests >= lvl_2_max_reqs
|
206
|
+
timed_level = 1
|
207
|
+
Sqreen::Weave.logger.debug { "Reducing timed level to 1 (#{lvl_2_metric.num_requests} reqs accumulated)" }
|
208
|
+
end
|
209
|
+
|
210
|
+
Sqreen::Weave.logger.debug { "request budget: #{budget} timed.level: #{timed_level}" } if Sqreen::Weave.logger.debug?
|
211
|
+
|
212
|
+
route_found = nil
|
213
|
+
if timed_level >= 2
|
214
|
+
rack_env, = call.args
|
215
|
+
rack_request = Rack::Request.new(rack_env) if rack_env
|
216
|
+
|
217
|
+
# TODO: Rails engines
|
218
|
+
# TODO: Struct
|
219
|
+
# TODO: Sinatra
|
220
|
+
# TODO: Rack?
|
221
|
+
Rails.application.routes.router.recognize(rack_request) do |route, params|
|
222
|
+
route = ActionDispatch::Routing::RouteWrapper.new(route)
|
223
|
+
route_found = { name: route.name, verb: route.verb, path: route.path, reqs: route.reqs, params: params }
|
224
|
+
end if defined?(Rails) && Rails.application && defined?(ActionDispatch::Routing::RouteWrapper)
|
225
|
+
end
|
226
|
+
|
227
|
+
# TODO: Struct
|
118
228
|
Thread.current[:sqreen_http_request] = {
|
119
|
-
|
120
|
-
|
121
|
-
time_budget: Sqreen.performance_budget,
|
229
|
+
request_timer: request_timer,
|
230
|
+
sqreen_timer: sqreen_timer,
|
122
231
|
time_budget_expended: false,
|
123
|
-
|
232
|
+
time_budget: budget,
|
124
233
|
timed_callbacks: [],
|
125
234
|
timed_hooks: [],
|
126
|
-
|
127
|
-
timed_hooks_after: [],
|
128
|
-
timed_hooks_raised: [],
|
129
|
-
timed_hooks_ensured: [],
|
235
|
+
timed_level: timed_level,
|
130
236
|
skipped_callbacks: [],
|
237
|
+
route: ("#{route_found[:verb]} #{route_found[:path]}" if route_found),
|
238
|
+
# timed_shrinkwrap: shrinkwrap_timer,
|
131
239
|
}
|
132
240
|
|
133
|
-
|
241
|
+
# shrinkwrap_timer.stop
|
134
242
|
end
|
135
243
|
|
136
244
|
ensured('weave,meta,request', rank: 100000, mandatory: true) do |_call|
|
@@ -138,105 +246,89 @@ class Sqreen::Weave::Legacy::Instrumentation
|
|
138
246
|
|
139
247
|
next if request.nil?
|
140
248
|
|
249
|
+
timed_level = request[:timed_level]
|
250
|
+
req_detailed = SqDetailedMetrics::Request.new if timed_level >= 2
|
251
|
+
|
252
|
+
# shrinkwrap_timer = request[:timed_shrinkwrap]
|
253
|
+
# shrinkwrap_timer.start
|
254
|
+
|
141
255
|
Thread.current[:sqreen_http_request] = nil
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
# => BinnedMetrics
|
158
|
-
metric_name = "sq.#{rule}.#{whence}"
|
159
|
-
unless metrics_engine.metric?(metric_name)
|
160
|
-
metrics_engine.create_metric(
|
161
|
-
'name' => metric_name,
|
162
|
-
'period' => 60,
|
163
|
-
'kind' => 'Binning',
|
164
|
-
'options' => { 'base' => 2.0, 'factor' => 0.1 },
|
165
|
-
)
|
256
|
+
request_timer = request[:request_timer]
|
257
|
+
now = request_timer.stop
|
258
|
+
|
259
|
+
if timed_level >= 1
|
260
|
+
request[:timed_callbacks].each do |timer|
|
261
|
+
duration_ms = timer.duration * 1000.0
|
262
|
+
# XXX: the timer tag should have this structured data;
|
263
|
+
# it would be better than recomputing this for every measurement
|
264
|
+
metric_name = ::Sqreen::Weave::Legacy::Instrumentation.tag_to_metric_name(timer.tag)
|
265
|
+
|
266
|
+
next unless metric_name
|
267
|
+
|
268
|
+
metrics_engine.update(metric_name, now, nil, duration_ms)
|
269
|
+
duration_ms *= -1.0 if timer.conditions_passed
|
270
|
+
req_detailed.add_measurement metric_name, duration_ms if req_detailed
|
166
271
|
end
|
167
|
-
metrics_engine.update(metric_name, now, nil, duration * 1000)
|
168
272
|
end
|
169
273
|
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
'period' => 60,
|
188
|
-
'kind' => 'Binning',
|
189
|
-
'options' => { 'base' => 2.0, 'factor' => 0.1 },
|
190
|
-
)
|
191
|
-
end
|
192
|
-
metrics_engine.update(metric_name, now, nil, duration * 1000)
|
193
|
-
|
194
|
-
metric_name = 'sq.hooks_failing.failing'
|
195
|
-
duration = request[:timed_hooks_raised].sum(&:duration)
|
196
|
-
unless metrics_engine.metric?(metric_name)
|
197
|
-
metrics_engine.create_metric(
|
198
|
-
'name' => metric_name,
|
199
|
-
'period' => 60,
|
200
|
-
'kind' => 'Binning',
|
201
|
-
'options' => { 'base' => 2.0, 'factor' => 0.1 },
|
202
|
-
)
|
274
|
+
sqreen_timer = request[:sqreen_timer]
|
275
|
+
Sqreen::Weave.logger.debug do
|
276
|
+
"request sqreen_timer.total: #{'%.03fus' % (sqreen_timer.duration * 1_000_000)}"
|
277
|
+
end if Sqreen::Weave.logger.debug?
|
278
|
+
Sqreen::Weave.logger.debug do
|
279
|
+
"request request_timer.total: #{'%.03fus' % (request_timer.duration * 1_000_000)}"
|
280
|
+
end if Sqreen::Weave.logger.debug?
|
281
|
+
|
282
|
+
if timed_level >= 1 && Sqreen::Weave.logger.debug?
|
283
|
+
skipped = request[:skipped_callbacks].map(&:name)
|
284
|
+
Sqreen::Weave.logger.debug { "request callback.skipped.count: #{skipped.count}" } if Sqreen::Weave.logger.debug?
|
285
|
+
timings = request[:timed_callbacks].map(&:to_s)
|
286
|
+
total = request[:timed_callbacks].sum(&:duration)
|
287
|
+
Sqreen::Weave.logger.debug { "request callback.total: #{'%.03fus' % (total * 1_000_000)} callback.count: #{timings.count}" } if Sqreen::Weave.logger.debug?
|
288
|
+
timings = request[:timed_hooks].map(&:to_s)
|
289
|
+
total = request[:timed_hooks].sum(&:duration)
|
290
|
+
Sqreen::Weave.logger.debug { "request hook.total: #{'%.03fus' % (total * 1_000_000)} hook.count: #{timings.count}" } if Sqreen::Weave.logger.debug?
|
203
291
|
end
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
timings = request[:timed_hooks].map(&:to_s)
|
215
|
-
total = request[:timed_hooks].sum(&:duration)
|
216
|
-
Sqreen::Weave.logger.debug { "request:#{request[:uuid]} hook.total: #{'%.03fus' % (total * 1_000_000)} hook.timings: [#{timings.join(', ')}]" }
|
217
|
-
|
218
|
-
skipped = request[:skipped_callbacks].map(&:name)
|
219
|
-
skipped_rule_name = skipped.first && skipped.first =~ /weave,rule=(.*)$/ && $1
|
220
|
-
Sqreen.observations_queue.push(['request_overtime', skipped_rule_name, 1, utc_now]) if skipped_rule_name
|
221
|
-
|
222
|
-
sqreen_request_duration = total
|
223
|
-
Sqreen.observations_queue.push(['sq', nil, sqreen_request_duration * 1000, utc_now])
|
224
|
-
|
225
|
-
request_duration = now - request[:start_time]
|
226
|
-
Sqreen.observations_queue.push(['req', nil, request_duration * 1000, utc_now])
|
292
|
+
|
293
|
+
overtime_cb = ::Sqreen::Weave::Legacy::Instrumentation.tag_to_metric_name(request[:overtime_cb]) \
|
294
|
+
if request[:overtime_cb]
|
295
|
+
metrics_engine.update('request_overtime', now, overtime_cb, 1) if overtime_cb
|
296
|
+
|
297
|
+
sqreen_request_duration = sqreen_timer.duration * 1000.0
|
298
|
+
metrics_engine.update('sq', now, nil, sqreen_request_duration)
|
299
|
+
|
300
|
+
request_duration = request_timer.duration * 1000.0
|
301
|
+
metrics_engine.update('req', now, nil, request_duration)
|
227
302
|
|
228
303
|
sqreen_request_ratio = (sqreen_request_duration * 100.0) / (request_duration - sqreen_request_duration)
|
229
|
-
|
304
|
+
metrics_engine.update('pct', now, nil, sqreen_request_ratio)
|
305
|
+
Sqreen::Weave.logger.debug { "request sqreen_timer.ratio: #{'%.03f' % (sqreen_request_ratio / 100.0)}" } if Sqreen::Weave.logger.debug?
|
306
|
+
|
307
|
+
if req_detailed
|
308
|
+
req_detailed.route = request[:route]
|
309
|
+
req_detailed.overtime_cb = overtime_cb if overtime_cb
|
310
|
+
req_detailed.add_measurement 'sq', sqreen_request_duration
|
311
|
+
req_detailed.add_measurement 'req', request_duration
|
312
|
+
|
313
|
+
metrics_engine.update(REQ_LVL_2_METRIC, now, nil, req_detailed)
|
314
|
+
end
|
315
|
+
|
316
|
+
# shrinkwrap_timer.stop
|
317
|
+
|
318
|
+
# duration = shrinkwrap_timer.duration
|
319
|
+
# metrics_engine.update('sq.shrinkwrap', now, nil, duration * 1000)
|
230
320
|
end
|
231
321
|
end.install
|
232
322
|
|
233
323
|
### globally declare instrumentation ready
|
234
324
|
Sqreen.instrumentation_ready = true
|
325
|
+
Sqreen::Weave.logger.info { "Instrumentation activated" }
|
235
326
|
end
|
236
327
|
|
237
328
|
# needed by Sqreen::Runner
|
238
329
|
def remove_all_callbacks
|
239
330
|
Sqreen.instrumentation_ready = false
|
331
|
+
Sqreen::Weave.logger.info { "Instrumentation deactivated" }
|
240
332
|
|
241
333
|
loop do
|
242
334
|
hook = @hooks.pop
|
@@ -253,6 +345,15 @@ class Sqreen::Weave::Legacy::Instrumentation
|
|
253
345
|
klass = callback.klass
|
254
346
|
method = callback.method
|
255
347
|
|
348
|
+
if (call_count = ENV['SQREEN_DEBUG_CALL_COUNT'])
|
349
|
+
call_count = JSON.parse(call_count)
|
350
|
+
if callback.respond_to?(:rule_name) && call_count.key?(callback.rule_name)
|
351
|
+
count = call_count[callback.rule_name]
|
352
|
+
Sqreen::Weave.logger.debug { "override rule: #{callback.rule_name} call_count: #{count.inspect}" }
|
353
|
+
callback.instance_eval { @call_count_interval = call_count[callback.rule_name] }
|
354
|
+
end
|
355
|
+
end
|
356
|
+
|
256
357
|
if Sqreen::Graft::HookPoint.new("#{klass}.#{method}").exist?
|
257
358
|
hook_point = "#{klass}.#{method}"
|
258
359
|
elsif Sqreen::Graft::HookPoint.new("#{klass}##{method}").exist?
|
@@ -268,14 +369,14 @@ class Sqreen::Weave::Legacy::Instrumentation
|
|
268
369
|
hook = Sqreen::Graft::Hook[hook_point, strategy]
|
269
370
|
hook.add do
|
270
371
|
if callback.pre?
|
271
|
-
|
372
|
+
use_flow = block || callback.is_a?(::Sqreen::Conditionable)
|
373
|
+
before(rule, rank: priority, mandatory: !callback.overtimeable, flow: use_flow, ignore: ignore) do |call, b|
|
272
374
|
next unless Thread.current[:sqreen_http_request]
|
273
375
|
|
274
376
|
i = call.instance
|
275
377
|
a = call.args
|
276
378
|
r = call.remaining
|
277
379
|
|
278
|
-
Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#pre instance=#{i}" }
|
279
380
|
begin
|
280
381
|
ret = callback.pre(i, a, r)
|
281
382
|
rescue StandardError => e
|
@@ -286,17 +387,30 @@ class Sqreen::Weave::Legacy::Instrumentation
|
|
286
387
|
Sqreen::RemoteException.record(e)
|
287
388
|
end
|
288
389
|
end
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
|
390
|
+
|
391
|
+
next if ret.nil? || !ret.is_a?(Hash)
|
392
|
+
|
393
|
+
throw_val =
|
394
|
+
case ret[:status]
|
395
|
+
when :skip, 'skip'
|
396
|
+
b.return(ret[:new_return_value]).break! if ret.key?(:new_return_value)
|
397
|
+
when :modify_args, 'modify_args'
|
398
|
+
b.args(ret[:args])
|
399
|
+
when :raise, 'raise'
|
400
|
+
if ret.key?(:exception)
|
401
|
+
b.raise(ret[:exception])
|
402
|
+
else
|
403
|
+
b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required."))
|
404
|
+
end
|
405
|
+
end if block
|
406
|
+
|
407
|
+
if ret && ret[:passed_conditions]
|
408
|
+
throw_val ||= b.noop
|
409
|
+
throw_val.passed_conditions!
|
410
|
+
end
|
411
|
+
next unless throw_val
|
412
|
+
throw_val.break! if ret[:skip_rem_cbs]
|
413
|
+
throw(b, throw_val)
|
300
414
|
end
|
301
415
|
end
|
302
416
|
|
@@ -309,7 +423,6 @@ class Sqreen::Weave::Legacy::Instrumentation
|
|
309
423
|
a = call.args
|
310
424
|
r = call.remaining
|
311
425
|
|
312
|
-
Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#post instance=#{i}" }
|
313
426
|
begin
|
314
427
|
ret = callback.post(v, i, a, r)
|
315
428
|
rescue StandardError => e
|
@@ -320,15 +433,22 @@ class Sqreen::Weave::Legacy::Instrumentation
|
|
320
433
|
Sqreen::RemoteException.record(e)
|
321
434
|
end
|
322
435
|
end
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
436
|
+
|
437
|
+
throw_val =
|
438
|
+
case ret[:status]
|
439
|
+
when :override, 'override'
|
440
|
+
b.return(ret[:new_return_value]) if ret.key?(:new_return_value)
|
441
|
+
when :raise, 'raise'
|
442
|
+
b.raise(ret[:exception]) if ret.key?(:exception)
|
443
|
+
b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required."))
|
444
|
+
end unless ret.nil? || !ret.is_a?(Hash) || !block
|
445
|
+
|
446
|
+
if ret && ret[:passed_conditions]
|
447
|
+
throw_val ||= b.noop
|
448
|
+
throw_val.passed_conditions!
|
449
|
+
end
|
450
|
+
next unless throw_val
|
451
|
+
throw(b, throw_val)
|
332
452
|
end
|
333
453
|
end
|
334
454
|
|
@@ -341,7 +461,6 @@ class Sqreen::Weave::Legacy::Instrumentation
|
|
341
461
|
a = call.args
|
342
462
|
r = call.remaining
|
343
463
|
|
344
|
-
Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#failing instance=#{i}" }
|
345
464
|
begin
|
346
465
|
ret = callback.failing(e, i, a, r)
|
347
466
|
rescue StandardError => e
|
@@ -352,23 +471,30 @@ class Sqreen::Weave::Legacy::Instrumentation
|
|
352
471
|
Sqreen::RemoteException.record(e)
|
353
472
|
end
|
354
473
|
end
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
474
|
+
|
475
|
+
throw(b, b.raise(e)) if ret.nil? || !ret.is_a?(Hash)
|
476
|
+
|
477
|
+
throw_val =
|
478
|
+
case ret[:status]
|
479
|
+
when :override, 'override'
|
480
|
+
b.return(ret[:new_return_value]) if ret.key?(:new_return_value)
|
481
|
+
when :retry, 'retry'
|
482
|
+
b.retry
|
483
|
+
when :raise, 'raise'
|
484
|
+
b.raise(ret[:exception]) if ret.key?(:exception)
|
485
|
+
b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required."))
|
486
|
+
when :reraise, 'reraise'
|
487
|
+
b.raise(e)
|
488
|
+
else
|
489
|
+
b.raise(e)
|
490
|
+
end unless ret.nil? || !ret.is_a?(Hash) || !block
|
491
|
+
|
492
|
+
if ret && ret[:passed_conditions]
|
493
|
+
throw_val ||= b.noop
|
494
|
+
throw_val.passed_conditions!
|
495
|
+
end
|
496
|
+
next unless throw_val
|
497
|
+
throw(b, throw_val)
|
372
498
|
end
|
373
499
|
end
|
374
500
|
end.install
|
@@ -403,4 +529,20 @@ class Sqreen::Weave::Legacy::Instrumentation
|
|
403
529
|
Sqreen::Rules::RunUserActions.new(Sqreen, :auth_track, 1),
|
404
530
|
]
|
405
531
|
end
|
532
|
+
|
533
|
+
def self.tag_to_metric_name(tag)
|
534
|
+
cached = @cache_tag_to_metric[tag]
|
535
|
+
return cached unless cached.nil?
|
536
|
+
|
537
|
+
tag =~ /weave,rule=(.*)$/ && rule = $1 and # rubocop:disable Style/AndOr
|
538
|
+
(tag =~ /@before/ && whence = 'pre' or # rubocop:disable Style/AndOr
|
539
|
+
tag =~ /@after/ && whence = 'post' or # rubocop:disable Style/AndOr
|
540
|
+
tag =~ /@raised/ && whence = 'failing' or # rubocop:disable Style/AndOr
|
541
|
+
tag =~ /@ensured/ && whence = 'finally')
|
542
|
+
|
543
|
+
@cache_tag_to_metric[tag] =
|
544
|
+
rule && whence ? "sq.#{rule}.#{whence}" : false
|
545
|
+
end
|
546
|
+
|
547
|
+
@cache_tag_to_metric = {}
|
406
548
|
end
|