sqreen 1.19.3-java → 1.21.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/CHANGELOG.md +38 -0
- data/lib/sqreen/actions/block_user.rb +1 -1
- data/lib/sqreen/actions/redirect_ip.rb +1 -1
- data/lib/sqreen/actions/redirect_user.rb +1 -1
- data/lib/sqreen/agent_message.rb +20 -0
- data/lib/sqreen/aggregated_metric.rb +25 -0
- data/lib/sqreen/attack_detected.html +1 -2
- data/lib/sqreen/ca.crt +24 -0
- data/lib/sqreen/condition_evaluator.rb +8 -2
- data/lib/sqreen/configuration.rb +11 -5
- data/lib/sqreen/deferred_logger.rb +50 -14
- data/lib/sqreen/deliveries/batch.rb +12 -2
- data/lib/sqreen/deliveries/simple.rb +4 -0
- data/lib/sqreen/deprecation.rb +38 -0
- data/lib/sqreen/ecosystem.rb +123 -0
- data/lib/sqreen/ecosystem/databases/database_connection_data.rb +23 -0
- data/lib/sqreen/ecosystem/databases/mongo.rb +39 -0
- data/lib/sqreen/ecosystem/databases/mysql.rb +54 -0
- data/lib/sqreen/ecosystem/databases/postgres.rb +51 -0
- data/lib/sqreen/ecosystem/databases/redis.rb +36 -0
- data/lib/sqreen/ecosystem/dispatch_table.rb +43 -0
- data/lib/sqreen/ecosystem/exception_reporting.rb +28 -0
- data/lib/sqreen/ecosystem/http/net_http.rb +50 -0
- data/lib/sqreen/ecosystem/http/rack_request.rb +39 -0
- data/lib/sqreen/ecosystem/loggable.rb +13 -0
- data/lib/sqreen/ecosystem/messaging/bunny.rb +61 -0
- data/lib/sqreen/ecosystem/messaging/kafka.rb +70 -0
- data/lib/sqreen/ecosystem/messaging/kinesis.rb +66 -0
- data/lib/sqreen/ecosystem/messaging/sqs.rb +68 -0
- data/lib/sqreen/ecosystem/module_api.rb +30 -0
- data/lib/sqreen/ecosystem/module_api/event_listener.rb +18 -0
- data/lib/sqreen/ecosystem/module_api/instrumentation.rb +23 -0
- data/lib/sqreen/ecosystem/module_api/message_producer.rb +57 -0
- data/lib/sqreen/ecosystem/module_api/signal_producer.rb +24 -0
- data/lib/sqreen/ecosystem/module_api/tracing.rb +45 -0
- data/lib/sqreen/ecosystem/module_api/tracing/client_data.rb +31 -0
- data/lib/sqreen/ecosystem/module_api/tracing/consumer_data.rb +13 -0
- data/lib/sqreen/ecosystem/module_api/tracing/messaging_data.rb +35 -0
- data/lib/sqreen/ecosystem/module_api/tracing/producer_data.rb +13 -0
- data/lib/sqreen/ecosystem/module_api/tracing/server_data.rb +27 -0
- data/lib/sqreen/ecosystem/module_api/tracing_id_generation.rb +16 -0
- data/lib/sqreen/ecosystem/module_api/transaction_storage.rb +71 -0
- data/lib/sqreen/ecosystem/module_registry.rb +48 -0
- data/lib/sqreen/ecosystem/tracing/modules/client.rb +35 -0
- data/lib/sqreen/ecosystem/tracing/modules/consumer.rb +35 -0
- data/lib/sqreen/ecosystem/tracing/modules/determine_ip.rb +28 -0
- data/lib/sqreen/ecosystem/tracing/modules/producer.rb +35 -0
- data/lib/sqreen/ecosystem/tracing/modules/server.rb +30 -0
- data/lib/sqreen/ecosystem/tracing/sampler.rb +160 -0
- data/lib/sqreen/ecosystem/tracing/sampling_configuration.rb +150 -0
- data/lib/sqreen/ecosystem/tracing/signals/tracing_client.rb +53 -0
- data/lib/sqreen/ecosystem/tracing/signals/tracing_consumer.rb +56 -0
- data/lib/sqreen/ecosystem/tracing/signals/tracing_producer.rb +56 -0
- data/lib/sqreen/ecosystem/tracing/signals/tracing_server.rb +53 -0
- data/lib/sqreen/ecosystem/tracing_broker.rb +101 -0
- data/lib/sqreen/ecosystem/tracing_id_setup.rb +34 -0
- data/lib/sqreen/ecosystem/transaction_storage.rb +64 -0
- data/lib/sqreen/ecosystem/util/call_writers_from_init.rb +13 -0
- data/lib/sqreen/ecosystem_integration.rb +81 -0
- data/lib/sqreen/ecosystem_integration/around_callbacks.rb +89 -0
- data/lib/sqreen/ecosystem_integration/instrumentation_service.rb +38 -0
- data/lib/sqreen/ecosystem_integration/request_lifecycle_tracking.rb +58 -0
- data/lib/sqreen/ecosystem_integration/signal_consumption.rb +35 -0
- data/lib/sqreen/endpoint_testing.rb +184 -0
- data/lib/sqreen/event.rb +7 -5
- data/lib/sqreen/events/attack.rb +23 -18
- data/lib/sqreen/events/remote_exception.rb +0 -22
- data/lib/sqreen/events/request_record.rb +15 -71
- data/lib/sqreen/frameworks/generic.rb +24 -1
- data/lib/sqreen/frameworks/rails.rb +0 -7
- data/lib/sqreen/frameworks/request_recorder.rb +15 -2
- data/lib/sqreen/graft/call.rb +85 -18
- data/lib/sqreen/graft/callback.rb +1 -1
- data/lib/sqreen/graft/hook.rb +192 -88
- data/lib/sqreen/graft/hook_point.rb +18 -11
- data/lib/sqreen/kit/signals/specialized/aggregated_metric.rb +72 -0
- data/lib/sqreen/kit/signals/specialized/attack.rb +57 -0
- data/lib/sqreen/kit/signals/specialized/binning_metric.rb +76 -0
- data/lib/sqreen/kit/signals/specialized/http_trace.rb +26 -0
- data/lib/sqreen/kit/signals/specialized/sdk_track_call.rb +50 -0
- data/lib/sqreen/kit/signals/specialized/sqreen_exception.rb +59 -0
- data/lib/sqreen/legacy/instrumentation.rb +22 -10
- data/lib/sqreen/legacy/old_event_submission_strategy.rb +228 -0
- data/lib/sqreen/legacy/waf_redactions.rb +49 -0
- data/lib/sqreen/log.rb +3 -2
- data/lib/sqreen/log/loggable.rb +2 -1
- data/lib/sqreen/logger.rb +24 -0
- data/lib/sqreen/metrics/base.rb +3 -0
- data/lib/sqreen/metrics_store.rb +33 -12
- data/lib/sqreen/null_logger.rb +22 -0
- data/lib/sqreen/performance_notifications/binned_metrics.rb +8 -2
- data/lib/sqreen/remote_command.rb +4 -0
- data/lib/sqreen/rules.rb +12 -6
- data/lib/sqreen/rules/blacklist_ips_cb.rb +2 -2
- data/lib/sqreen/rules/custom_error_cb.rb +3 -3
- data/lib/sqreen/rules/rule_cb.rb +4 -0
- data/lib/sqreen/rules/waf_cb.rb +14 -11
- data/lib/sqreen/runner.rb +122 -15
- data/lib/sqreen/sensitive_data_redactor.rb +19 -31
- data/lib/sqreen/session.rb +53 -43
- data/lib/sqreen/signals/conversions.rb +288 -0
- data/lib/sqreen/signals/http_trace_redaction.rb +111 -0
- data/lib/sqreen/signals/signals_submission_strategy.rb +78 -0
- data/lib/sqreen/version.rb +1 -1
- data/lib/sqreen/weave/budget.rb +46 -0
- data/lib/sqreen/weave/legacy/instrumentation.rb +194 -103
- data/lib/sqreen/worker.rb +6 -2
- metadata +96 -7
- data/lib/sqreen/backport.rb +0 -9
- data/lib/sqreen/backport/clock_gettime.rb +0 -74
- data/lib/sqreen/backport/original_name.rb +0 -88
- data/lib/sqreen/encoding_sanitizer.rb +0 -27
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
require 'json'
|
|
2
|
+
require 'sqreen/kit/loggable'
|
|
3
|
+
require 'sqreen/kit/signals/specialized/http_trace'
|
|
4
|
+
|
|
5
|
+
module Sqreen
|
|
6
|
+
module Signals
|
|
7
|
+
module HttpTraceRedaction
|
|
8
|
+
class << self
|
|
9
|
+
include Sqreen::Kit::Loggable
|
|
10
|
+
|
|
11
|
+
# @param [Sqreen::Kit::Signals::Specialized::HttpTrace] trace
|
|
12
|
+
# @param [Sqreen::SensitiveDataRedactor] redactor
|
|
13
|
+
def redact_trace!(trace, redactor)
|
|
14
|
+
return unless redactor
|
|
15
|
+
# redact headers (keys unsafe)
|
|
16
|
+
# @type [Sqreen::Kit::Signals::Context::HttpContext]
|
|
17
|
+
http_context = trace.context
|
|
18
|
+
|
|
19
|
+
all_redacted = []
|
|
20
|
+
|
|
21
|
+
# Redact headers; save redacted values
|
|
22
|
+
# headers are encoded as [key, value], not a hash, so
|
|
23
|
+
# they require some transformation
|
|
24
|
+
orig_headers = http_context.headers
|
|
25
|
+
if orig_headers
|
|
26
|
+
headers = orig_headers.map { |(k, v)| { k => v } }
|
|
27
|
+
headers, redacted = redactor.redact(headers)
|
|
28
|
+
http_context.headers = headers.map(&:first)
|
|
29
|
+
all_redacted += redacted
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Redact params; save redacted values
|
|
33
|
+
Kit::Signals::Context::HttpContext::PARAMS_ATTRS.each do |attr|
|
|
34
|
+
value = http_context.public_send(attr)
|
|
35
|
+
next unless value
|
|
36
|
+
value, redacted = redactor.redact(value)
|
|
37
|
+
all_redacted += redacted
|
|
38
|
+
http_context.public_send(:"#{attr}=", value)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
all_redacted = all_redacted.uniq.map(&:downcase)
|
|
42
|
+
|
|
43
|
+
# Redact attacks and exceptions
|
|
44
|
+
# XXX: no redaction for infos in attacks/exceptions except for WAF data
|
|
45
|
+
# Is this the correct behavior?
|
|
46
|
+
redact_attacks!(trace, redactor, all_redacted)
|
|
47
|
+
redact_exceptions!(trace, redactor, all_redacted)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
private
|
|
51
|
+
|
|
52
|
+
# @param [Sqreen::Kit::Signals::Specialized::HttpTrace] trace
|
|
53
|
+
# @param [Sqreen::SensitiveDataRedactor] redactor
|
|
54
|
+
# Redacts WAF data according to specific rules therefor
|
|
55
|
+
# Redacts infos according to general rules
|
|
56
|
+
def redact_attacks!(trace, redactor, redacted_data)
|
|
57
|
+
trace.data.each do |signal|
|
|
58
|
+
next unless signal.is_a?(Kit::Signals::Specialized::Attack)
|
|
59
|
+
# @type [Sqreen::Kit::Signals::Specialized::Attack::Payload] payload
|
|
60
|
+
payload = signal.payload
|
|
61
|
+
next unless payload.infos
|
|
62
|
+
|
|
63
|
+
if payload.infos[:waf_data]
|
|
64
|
+
redact_waf_attack_data!(payload.infos, redacted_data)
|
|
65
|
+
end
|
|
66
|
+
payload.infos, = redactor.redact(payload.infos)
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def redact_exceptions!(trace, redactor, redacted_data)
|
|
71
|
+
trace.data.each do |signal|
|
|
72
|
+
next unless signal.is_a?(Kit::Signals::Specialized::SqreenException)
|
|
73
|
+
infos = signal.infos
|
|
74
|
+
next unless infos
|
|
75
|
+
|
|
76
|
+
redact_waf_exception_data!(signal.infos, redacted_data) if signal.infos[:waf]
|
|
77
|
+
signal.infos, = redactor.redact(infos)
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# @param [Hash] infos from WAF attack
|
|
82
|
+
def redact_waf_attack_data!(infos, redacted_data)
|
|
83
|
+
begin
|
|
84
|
+
parsed = JSON.parse(infos[:waf_data])
|
|
85
|
+
rescue JSON::JSONError => e
|
|
86
|
+
logger.warn("waf_data is not valid json: #{e.message}")
|
|
87
|
+
return
|
|
88
|
+
end
|
|
89
|
+
redacted = parsed.each do |w|
|
|
90
|
+
next unless (filters = w['filter'])
|
|
91
|
+
|
|
92
|
+
filters.each do |f|
|
|
93
|
+
next unless (v = f['resolved_value'])
|
|
94
|
+
next unless redacted_data.include?(v.downcase)
|
|
95
|
+
|
|
96
|
+
f['match_status'] = SensitiveDataRedactor::MASK
|
|
97
|
+
f['resolved_value'] = SensitiveDataRedactor::MASK
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
infos[:waf_data] = JSON.dump(redacted)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# see https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000022-waf-data-sanitization.md#changes-to-the-agents
|
|
104
|
+
def redact_waf_exception_data!(infos, redacted_data)
|
|
105
|
+
return if redacted_data.empty?
|
|
106
|
+
infos[:waf].delete(:args)
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
require 'sqreen/aggregated_metric'
|
|
2
|
+
require 'sqreen/kit'
|
|
3
|
+
require 'sqreen/kit/string_sanitizer'
|
|
4
|
+
require 'sqreen/signals/conversions'
|
|
5
|
+
require 'sqreen/log/loggable'
|
|
6
|
+
|
|
7
|
+
module Sqreen
|
|
8
|
+
module Signals
|
|
9
|
+
# see also Sqreen::Legacy::OldEventSubmissionStrategy
|
|
10
|
+
# usage in Sqreen:Session
|
|
11
|
+
class SignalsSubmissionStrategy
|
|
12
|
+
include Sqreen::Log::Loggable
|
|
13
|
+
|
|
14
|
+
# @param [Array<Sqreen::AggregatedMetric>] metrics
|
|
15
|
+
def post_metrics(metrics)
|
|
16
|
+
return if metrics.nil? || metrics.empty?
|
|
17
|
+
|
|
18
|
+
guarded 'Failed to serialize or submit aggregated metrics' do
|
|
19
|
+
batch = metrics.map do |m|
|
|
20
|
+
Conversions.convert_metric_sample(m)
|
|
21
|
+
end
|
|
22
|
+
client.report_batch(batch)
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# @param _attack [Sqreen::Attack]
|
|
27
|
+
# XXX: unused
|
|
28
|
+
def post_attack(_attack)
|
|
29
|
+
raise NotImplementedError
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# @param request_record [Sqreen::RequestRecord]
|
|
33
|
+
def post_request_record(request_record)
|
|
34
|
+
guarded 'Failed to serialize or submit request record' do
|
|
35
|
+
trace = Conversions.convert_req_record(request_record)
|
|
36
|
+
append_sanitizing_filter(trace)
|
|
37
|
+
client.report_trace(trace)
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Post an exception to Sqreen for analysis
|
|
42
|
+
# @param exception [RemoteException] Exception and context to be sent over
|
|
43
|
+
def post_sqreen_exception(exception)
|
|
44
|
+
guarded 'Failed to serialize or submit exception', false do
|
|
45
|
+
data = Conversions.convert_exception(exception)
|
|
46
|
+
append_sanitizing_filter(data)
|
|
47
|
+
client.report_signal(data)
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def post_batch(events)
|
|
52
|
+
guarded 'Failed to serialize or submit batch of events' do
|
|
53
|
+
batch = Conversions.convert_batch(events)
|
|
54
|
+
batch.each { |sig_or_trace| append_sanitizing_filter(sig_or_trace) }
|
|
55
|
+
client.report_batch(batch)
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
private
|
|
60
|
+
|
|
61
|
+
def append_sanitizing_filter(sig_or_trace)
|
|
62
|
+
sig_or_trace.append_to_h_filter Kit::StringSanitizer.method(:sanitize)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# we don't want exceptions to propagate and kill the worker thread
|
|
66
|
+
def guarded(msg, report = true)
|
|
67
|
+
yield
|
|
68
|
+
rescue StandardError => e
|
|
69
|
+
logger.warn "#{msg}: #{e.message}\n#{e.backtrace.map { |x| " #{x}" }.join("\n")}"
|
|
70
|
+
post_sqreen_exception(RemoteException.new(e)) if report
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def client
|
|
74
|
+
Sqreen::Kit.auth_signals_client
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
data/lib/sqreen/version.rb
CHANGED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# typed: false
|
|
2
|
+
|
|
3
|
+
# Copyright (c) 2015 Sqreen. All Rights Reserved.
|
|
4
|
+
# Please refer to our terms for more information: https://www.sqreen.com/terms.html
|
|
5
|
+
|
|
6
|
+
require 'sqreen/log/loggable'
|
|
7
|
+
require 'sqreen/weave'
|
|
8
|
+
|
|
9
|
+
class Sqreen::Weave::Budget
|
|
10
|
+
include Sqreen::Log::Loggable
|
|
11
|
+
|
|
12
|
+
def initialize(threshold, ratio = nil)
|
|
13
|
+
@threshold = threshold
|
|
14
|
+
@ratio = ratio
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def static?
|
|
18
|
+
threshold && !ratio
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def dynamic?
|
|
22
|
+
threshold && ratio
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
attr_reader :threshold
|
|
26
|
+
attr_reader :ratio
|
|
27
|
+
|
|
28
|
+
def to_h
|
|
29
|
+
{ threshold: threshold, ratio: ratio }
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
class << self
|
|
33
|
+
attr_reader :current
|
|
34
|
+
|
|
35
|
+
def update(opts = nil)
|
|
36
|
+
Sqreen::Weave.logger.info("budget update:#{opts.inspect}")
|
|
37
|
+
|
|
38
|
+
return @current = nil if opts.nil? || opts.empty?
|
|
39
|
+
|
|
40
|
+
threshold = opts[:threshold]
|
|
41
|
+
ratio = opts[:ratio]
|
|
42
|
+
|
|
43
|
+
@current = new(threshold, ratio)
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
@@ -4,10 +4,13 @@
|
|
|
4
4
|
# Please refer to our terms for more information: https://www.sqreen.com/terms.html
|
|
5
5
|
|
|
6
6
|
require 'sqreen/weave/legacy'
|
|
7
|
+
require 'sqreen/weave/budget'
|
|
8
|
+
require 'sqreen/graft/hook'
|
|
7
9
|
require 'sqreen/graft/hook_point'
|
|
8
10
|
require 'sqreen/call_countable'
|
|
9
11
|
require 'sqreen/rules'
|
|
10
12
|
require 'sqreen/rules/record_request_context'
|
|
13
|
+
require 'sqreen/sqreen_signed_verifier'
|
|
11
14
|
|
|
12
15
|
class Sqreen::Weave::Legacy::Instrumentation
|
|
13
16
|
attr_accessor :metrics_engine
|
|
@@ -60,6 +63,27 @@ class Sqreen::Weave::Legacy::Instrumentation
|
|
|
60
63
|
'options' => opts[:perf_metric_percent] || { 'base' => 1.3, 'factor' => 1.0 },
|
|
61
64
|
)
|
|
62
65
|
|
|
66
|
+
metrics_engine.create_metric(
|
|
67
|
+
'name' => 'req.sq.hook.overhead',
|
|
68
|
+
'period' => 60,
|
|
69
|
+
'kind' => 'Binning',
|
|
70
|
+
'options' => { 'base' => 2.0, 'factor' => 0.1 },
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
metrics_engine.create_metric(
|
|
74
|
+
'name' => 'sq.hook.overhead',
|
|
75
|
+
'period' => 60,
|
|
76
|
+
'kind' => 'Binning',
|
|
77
|
+
'options' => { 'base' => 2.0, 'factor' => 0.1 },
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
metrics_engine.create_metric(
|
|
81
|
+
'name' => 'sq.shrinkwrap',
|
|
82
|
+
'period' => 60,
|
|
83
|
+
'kind' => 'Binning',
|
|
84
|
+
'options' => { 'base' => 2.0, 'factor' => 0.1 },
|
|
85
|
+
)
|
|
86
|
+
|
|
63
87
|
Sqreen.thread_cpu_time? && metrics_engine.create_metric(
|
|
64
88
|
'name' => 'sq_thread_cpu_pct',
|
|
65
89
|
'period' => opts[:period] || 60,
|
|
@@ -84,6 +108,15 @@ class Sqreen::Weave::Legacy::Instrumentation
|
|
|
84
108
|
|
|
85
109
|
### set up rule signature verifier
|
|
86
110
|
verifier = nil
|
|
111
|
+
if Sqreen.features['rules_signature'] &&
|
|
112
|
+
Sqreen.config_get(:rules_verify_signature) == true &&
|
|
113
|
+
!defined?(::JRUBY_VERSION)
|
|
114
|
+
verifier = Sqreen::SqreenSignedVerifier.new
|
|
115
|
+
Sqreen::Weave.logger.debug('Rules signature enabled')
|
|
116
|
+
else
|
|
117
|
+
Sqreen::Weave.logger.debug('Rules signature disabled')
|
|
118
|
+
end
|
|
119
|
+
|
|
87
120
|
### force clean instrumentation callback list
|
|
88
121
|
@hooks = []
|
|
89
122
|
### for each rule description
|
|
@@ -94,6 +127,25 @@ class Sqreen::Weave::Legacy::Instrumentation
|
|
|
94
127
|
next unless rule_callback
|
|
95
128
|
### attach framework to callback
|
|
96
129
|
rule_callback.framework = framework
|
|
130
|
+
## create metric
|
|
131
|
+
Sqreen::Weave.logger.debug { "Adding rule metric: #{rule_callback}" }
|
|
132
|
+
[:pre, :post, :failing].each do |whence|
|
|
133
|
+
next unless rule_callback.send(:"#{whence}?")
|
|
134
|
+
metric_name = "sq.#{rule['name']}.#{whence}"
|
|
135
|
+
metrics_engine.create_metric(
|
|
136
|
+
'name' => metric_name,
|
|
137
|
+
'period' => 60,
|
|
138
|
+
'kind' => 'Binning',
|
|
139
|
+
'options' => { 'base' => 2.0, 'factor' => 0.1 },
|
|
140
|
+
)
|
|
141
|
+
metric_name = "req.sq.#{rule['name']}.#{whence}"
|
|
142
|
+
metrics_engine.create_metric(
|
|
143
|
+
'name' => metric_name,
|
|
144
|
+
'period' => 60,
|
|
145
|
+
'kind' => 'Binning',
|
|
146
|
+
'options' => { 'base' => 2.0, 'factor' => 0.1 },
|
|
147
|
+
)
|
|
148
|
+
end
|
|
97
149
|
### install callback, observing priority
|
|
98
150
|
Sqreen::Weave.logger.debug { "Adding rule callback: #{rule_callback}" }
|
|
99
151
|
@hooks << add_callback("weave,rule=#{rule['name']}", rule_callback, strategy)
|
|
@@ -107,30 +159,43 @@ class Sqreen::Weave::Legacy::Instrumentation
|
|
|
107
159
|
end
|
|
108
160
|
|
|
109
161
|
metrics_engine = self.metrics_engine
|
|
162
|
+
|
|
110
163
|
request_hook = Sqreen::Graft::Hook['Sqreen::ShrinkWrap#call', strategy]
|
|
111
164
|
@hooks << request_hook
|
|
112
165
|
request_hook.add do
|
|
113
166
|
before('wave,meta,request', rank: -100000, mandatory: true) do |_call|
|
|
114
167
|
next unless Sqreen.instrumentation_ready
|
|
115
168
|
|
|
116
|
-
|
|
117
|
-
|
|
169
|
+
# shrinkwrap_timer = Sqreen::Graft::Timer.new('weave,shrinkwrap')
|
|
170
|
+
# shrinkwrap_timer.start
|
|
171
|
+
|
|
172
|
+
request_timer = Sqreen::Graft::Timer.new("request")
|
|
173
|
+
request_timer.start
|
|
174
|
+
sqreen_timer = Sqreen::Graft::Timer.new("sqreen")
|
|
175
|
+
budget = Sqreen::Weave::Budget.current
|
|
176
|
+
request_budget_threshold = budget.threshold if budget
|
|
177
|
+
request_budget_ratio = budget.ratio if budget
|
|
178
|
+
request_budget_is_dynamic = !request_budget_ratio.nil?
|
|
179
|
+
request_budget = !request_budget_threshold.nil?
|
|
180
|
+
timed_level = (Sqreen.features['perf_level'] || 1).to_i
|
|
181
|
+
Sqreen::Weave.logger.debug { "request budget: #{budget.to_h} timed.level: #{timed_level}" } if Sqreen::Weave.logger.debug?
|
|
182
|
+
|
|
118
183
|
Thread.current[:sqreen_http_request] = {
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
time_budget: Sqreen.performance_budget,
|
|
184
|
+
request_timer: request_timer,
|
|
185
|
+
sqreen_timer: sqreen_timer,
|
|
122
186
|
time_budget_expended: false,
|
|
123
|
-
|
|
187
|
+
time_budget_threshold: request_budget_threshold,
|
|
188
|
+
time_budget_dynamic: request_budget_is_dynamic,
|
|
189
|
+
time_budget_ratio: request_budget_ratio,
|
|
190
|
+
time_budget: request_budget,
|
|
124
191
|
timed_callbacks: [],
|
|
125
192
|
timed_hooks: [],
|
|
126
|
-
|
|
127
|
-
timed_hooks_after: [],
|
|
128
|
-
timed_hooks_raised: [],
|
|
129
|
-
timed_hooks_ensured: [],
|
|
193
|
+
timed_level: timed_level,
|
|
130
194
|
skipped_callbacks: [],
|
|
195
|
+
# timed_shrinkwrap: shrinkwrap_timer,
|
|
131
196
|
}
|
|
132
197
|
|
|
133
|
-
|
|
198
|
+
# shrinkwrap_timer.stop
|
|
134
199
|
end
|
|
135
200
|
|
|
136
201
|
ensured('weave,meta,request', rank: 100000, mandatory: true) do |_call|
|
|
@@ -138,105 +203,118 @@ class Sqreen::Weave::Legacy::Instrumentation
|
|
|
138
203
|
|
|
139
204
|
next if request.nil?
|
|
140
205
|
|
|
206
|
+
# shrinkwrap_timer = request[:timed_shrinkwrap]
|
|
207
|
+
# shrinkwrap_timer.start
|
|
208
|
+
|
|
141
209
|
Thread.current[:sqreen_http_request] = nil
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
request[:
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
metrics_engine.
|
|
161
|
-
|
|
162
|
-
'period' => 60,
|
|
163
|
-
'kind' => 'Binning',
|
|
164
|
-
'options' => { 'base' => 2.0, 'factor' => 0.1 },
|
|
165
|
-
)
|
|
210
|
+
request_timer = request[:request_timer]
|
|
211
|
+
now = request_timer.stop
|
|
212
|
+
|
|
213
|
+
if request[:timed_level] >= 1
|
|
214
|
+
request[:timed_callbacks].each do |timer|
|
|
215
|
+
duration = timer.duration
|
|
216
|
+
|
|
217
|
+
timer.tag =~ /weave,rule=(.*)$/ && rule = $1
|
|
218
|
+
next unless rule
|
|
219
|
+
|
|
220
|
+
whence = case timer.tag
|
|
221
|
+
when /@before/ then 'pre'
|
|
222
|
+
when /@after/ then 'post'
|
|
223
|
+
when /@raised/ then 'failing'
|
|
224
|
+
end
|
|
225
|
+
next unless whence
|
|
226
|
+
|
|
227
|
+
metric_name = "sq.#{rule}.#{whence}"
|
|
228
|
+
metrics_engine.update(metric_name, now, nil, duration * 1000)
|
|
229
|
+
# Sqreen.observations_queue.push([metric_name, nil, duration * 1000, utc_now])
|
|
166
230
|
end
|
|
167
|
-
metrics_engine.update(metric_name, now, nil, duration * 1000)
|
|
168
|
-
end
|
|
169
231
|
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
'period' => 60,
|
|
176
|
-
'kind' => 'Binning',
|
|
177
|
-
'options' => { 'base' => 2.0, 'factor' => 0.1 },
|
|
178
|
-
)
|
|
179
|
-
end
|
|
180
|
-
metrics_engine.update(metric_name, now, nil, duration * 1000)
|
|
181
|
-
|
|
182
|
-
metric_name = 'sq.hooks_post.post'
|
|
183
|
-
duration = request[:timed_hooks_after].sum(&:duration)
|
|
184
|
-
unless metrics_engine.metric?(metric_name)
|
|
185
|
-
metrics_engine.create_metric(
|
|
186
|
-
'name' => metric_name,
|
|
187
|
-
'period' => 60,
|
|
188
|
-
'kind' => 'Binning',
|
|
189
|
-
'options' => { 'base' => 2.0, 'factor' => 0.1 },
|
|
190
|
-
)
|
|
191
|
-
end
|
|
192
|
-
metrics_engine.update(metric_name, now, nil, duration * 1000)
|
|
193
|
-
|
|
194
|
-
metric_name = 'sq.hooks_failing.failing'
|
|
195
|
-
duration = request[:timed_hooks_raised].sum(&:duration)
|
|
196
|
-
unless metrics_engine.metric?(metric_name)
|
|
197
|
-
metrics_engine.create_metric(
|
|
198
|
-
'name' => metric_name,
|
|
199
|
-
'period' => 60,
|
|
200
|
-
'kind' => 'Binning',
|
|
201
|
-
'options' => { 'base' => 2.0, 'factor' => 0.1 },
|
|
202
|
-
)
|
|
232
|
+
request[:timed_hooks].each do |timer|
|
|
233
|
+
duration = timer.duration
|
|
234
|
+
metrics_engine.update('sq.hook.overhead', now, nil, duration * 1000)
|
|
235
|
+
# Sqreen.observations_queue.push(['sq.hook.overhead', nil, duration * 1000, utc_now])
|
|
236
|
+
end
|
|
203
237
|
end
|
|
204
|
-
metrics_engine.update(metric_name, now, nil, duration * 1000)
|
|
205
238
|
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
total =
|
|
210
|
-
Sqreen::Weave.logger.debug { "request
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
239
|
+
sqreen_timer = request[:sqreen_timer]
|
|
240
|
+
total = sqreen_timer.duration
|
|
241
|
+
Sqreen::Weave.logger.debug { "request sqreen_timer.total: #{'%.03fus' % (total * 1_000_000)}" } if Sqreen::Weave.logger.debug?
|
|
242
|
+
total = request_timer.duration
|
|
243
|
+
Sqreen::Weave.logger.debug { "request request_timer.total: #{'%.03fus' % (total * 1_000_000)}" } if Sqreen::Weave.logger.debug?
|
|
244
|
+
|
|
245
|
+
if request[:timed_level] >= 2
|
|
246
|
+
skipped = request[:skipped_callbacks].map(&:name)
|
|
247
|
+
Sqreen::Weave.logger.debug { "request callback.skipped.count: #{skipped.count}" } if Sqreen::Weave.logger.debug?
|
|
248
|
+
timings = request[:timed_callbacks].map(&:to_s)
|
|
249
|
+
total = request[:timed_callbacks].sum(&:duration)
|
|
250
|
+
Sqreen::Weave.logger.debug { "request callback.total: #{'%.03fus' % (total * 1_000_000)} callback.count: #{timings.count}" } if Sqreen::Weave.logger.debug?
|
|
251
|
+
timings = request[:timed_hooks].map(&:to_s)
|
|
252
|
+
total = request[:timed_hooks].sum(&:duration)
|
|
253
|
+
Sqreen::Weave.logger.debug { "request hook.total: #{'%.03fus' % (total * 1_000_000)} hook.count: #{timings.count}" } if Sqreen::Weave.logger.debug?
|
|
254
|
+
end
|
|
217
255
|
|
|
218
256
|
skipped = request[:skipped_callbacks].map(&:name)
|
|
219
257
|
skipped_rule_name = skipped.first && skipped.first =~ /weave,rule=(.*)$/ && $1
|
|
220
|
-
|
|
258
|
+
metrics_engine.update('request_overtime', now, skipped_rule_name, 1) if skipped_rule_name
|
|
259
|
+
# Sqreen.observations_queue.push(['request_overtime', skipped_rule_name, 1, utc_now]) if skipped_rule_name
|
|
221
260
|
|
|
222
|
-
sqreen_request_duration =
|
|
223
|
-
|
|
261
|
+
sqreen_request_duration = sqreen_timer.duration
|
|
262
|
+
metrics_engine.update('sq', now, nil, sqreen_request_duration * 1000)
|
|
263
|
+
# Sqreen.observations_queue.push(['sq', nil, sqreen_request_duration * 1000, utc_now])
|
|
224
264
|
|
|
225
|
-
request_duration =
|
|
226
|
-
|
|
265
|
+
request_duration = request_timer.duration
|
|
266
|
+
metrics_engine.update('req', now, nil, request_duration * 1000)
|
|
267
|
+
# Sqreen.observations_queue.push(['req', nil, request_duration * 1000, utc_now])
|
|
227
268
|
|
|
228
269
|
sqreen_request_ratio = (sqreen_request_duration * 100.0) / (request_duration - sqreen_request_duration)
|
|
229
|
-
|
|
270
|
+
metrics_engine.update('pct', now, nil, sqreen_request_ratio)
|
|
271
|
+
# Sqreen.observations_queue.push(['pct', nil, sqreen_request_ratio, utc_now])
|
|
272
|
+
Sqreen::Weave.logger.debug { "request sqreen_timer.ratio: #{'%.03f' % (sqreen_request_ratio / 100.0)}" } if Sqreen::Weave.logger.debug?
|
|
273
|
+
|
|
274
|
+
if request[:timed_level] >= 2
|
|
275
|
+
tallies = Hash.new(0.0)
|
|
276
|
+
request[:timed_callbacks].each do |timer|
|
|
277
|
+
duration = timer.duration
|
|
278
|
+
|
|
279
|
+
timer.tag =~ /weave,rule=(.*)$/ && rule = $1
|
|
280
|
+
next unless rule
|
|
281
|
+
|
|
282
|
+
whence = case timer.tag
|
|
283
|
+
when /@before/ then 'pre'
|
|
284
|
+
when /@after/ then 'post'
|
|
285
|
+
when /@raised/ then 'failing'
|
|
286
|
+
end
|
|
287
|
+
next unless whence
|
|
288
|
+
|
|
289
|
+
metric_name = "req.sq.#{rule}.#{whence}"
|
|
290
|
+
tallies[metric_name] += duration
|
|
291
|
+
end
|
|
292
|
+
tallies.each do |metric_name, duration|
|
|
293
|
+
metrics_engine.update(metric_name, now, nil, duration * 1000)
|
|
294
|
+
# Sqreen.observations_queue.push([metric_name, nil, duration * 1000, utc_now])
|
|
295
|
+
end
|
|
296
|
+
|
|
297
|
+
duration = request[:timed_hooks].sum(&:duration)
|
|
298
|
+
metrics_engine.update('req.sq.hook.overhead', now, nil, duration * 1000)
|
|
299
|
+
# Sqreen.observations_queue.push(['req.sq.hook.overhead', nil, duration * 1000, utc_now])
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
# shrinkwrap_timer.stop
|
|
303
|
+
|
|
304
|
+
# duration = shrinkwrap_timer.duration
|
|
305
|
+
# metrics_engine.update('sq.shrinkwrap', now, nil, duration * 1000)
|
|
230
306
|
end
|
|
231
307
|
end.install
|
|
232
308
|
|
|
233
309
|
### globally declare instrumentation ready
|
|
234
310
|
Sqreen.instrumentation_ready = true
|
|
311
|
+
Sqreen::Weave.logger.info { "Instrumentation activated" }
|
|
235
312
|
end
|
|
236
313
|
|
|
237
314
|
# needed by Sqreen::Runner
|
|
238
315
|
def remove_all_callbacks
|
|
239
316
|
Sqreen.instrumentation_ready = false
|
|
317
|
+
Sqreen::Weave.logger.info { "Instrumentation deactivated" }
|
|
240
318
|
|
|
241
319
|
loop do
|
|
242
320
|
hook = @hooks.pop
|
|
@@ -253,6 +331,15 @@ class Sqreen::Weave::Legacy::Instrumentation
|
|
|
253
331
|
klass = callback.klass
|
|
254
332
|
method = callback.method
|
|
255
333
|
|
|
334
|
+
if (call_count = ENV['SQREEN_DEBUG_CALL_COUNT'])
|
|
335
|
+
call_count = JSON.parse(call_count)
|
|
336
|
+
if callback.respond_to?(:rule_name) && call_count.key?(callback.rule_name)
|
|
337
|
+
count = call_count[callback.rule_name]
|
|
338
|
+
Sqreen::Weave.logger.debug { "override rule: #{callback.rule_name} call_count: #{count.inspect}" }
|
|
339
|
+
callback.instance_eval { @call_count_interval = call_count[callback.rule_name] }
|
|
340
|
+
end
|
|
341
|
+
end
|
|
342
|
+
|
|
256
343
|
if Sqreen::Graft::HookPoint.new("#{klass}.#{method}").exist?
|
|
257
344
|
hook_point = "#{klass}.#{method}"
|
|
258
345
|
elsif Sqreen::Graft::HookPoint.new("#{klass}##{method}").exist?
|
|
@@ -275,7 +362,6 @@ class Sqreen::Weave::Legacy::Instrumentation
|
|
|
275
362
|
a = call.args
|
|
276
363
|
r = call.remaining
|
|
277
364
|
|
|
278
|
-
Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#pre instance=#{i}" }
|
|
279
365
|
begin
|
|
280
366
|
ret = callback.pre(i, a, r)
|
|
281
367
|
rescue StandardError => e
|
|
@@ -286,17 +372,26 @@ class Sqreen::Weave::Legacy::Instrumentation
|
|
|
286
372
|
Sqreen::RemoteException.record(e)
|
|
287
373
|
end
|
|
288
374
|
end
|
|
289
|
-
Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#pre instance=#{i} => return=#{ret.inspect}" }
|
|
290
375
|
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
376
|
+
next if ret.nil? || !ret.is_a?(Hash)
|
|
377
|
+
|
|
378
|
+
throw_val =
|
|
379
|
+
case ret[:status]
|
|
380
|
+
when :skip, 'skip'
|
|
381
|
+
b.return(ret[:new_return_value]).break! if ret.key?(:new_return_value)
|
|
382
|
+
when :modify_args, 'modify_args'
|
|
383
|
+
b.args(ret[:args])
|
|
384
|
+
when :raise, 'raise'
|
|
385
|
+
if ret.key?(:exception)
|
|
386
|
+
b.raise(ret[:exception])
|
|
387
|
+
else
|
|
388
|
+
b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required."))
|
|
389
|
+
end
|
|
390
|
+
end
|
|
391
|
+
|
|
392
|
+
next unless throw_val
|
|
393
|
+
throw_val.break! if ret[:skip_rem_cbs]
|
|
394
|
+
throw(b, throw_val)
|
|
300
395
|
end
|
|
301
396
|
end
|
|
302
397
|
|
|
@@ -309,7 +404,6 @@ class Sqreen::Weave::Legacy::Instrumentation
|
|
|
309
404
|
a = call.args
|
|
310
405
|
r = call.remaining
|
|
311
406
|
|
|
312
|
-
Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#post instance=#{i}" }
|
|
313
407
|
begin
|
|
314
408
|
ret = callback.post(v, i, a, r)
|
|
315
409
|
rescue StandardError => e
|
|
@@ -320,7 +414,6 @@ class Sqreen::Weave::Legacy::Instrumentation
|
|
|
320
414
|
Sqreen::RemoteException.record(e)
|
|
321
415
|
end
|
|
322
416
|
end
|
|
323
|
-
Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#post instance=#{i} => return=#{ret.inspect}" }
|
|
324
417
|
|
|
325
418
|
case ret[:status]
|
|
326
419
|
when :override, 'override'
|
|
@@ -341,7 +434,6 @@ class Sqreen::Weave::Legacy::Instrumentation
|
|
|
341
434
|
a = call.args
|
|
342
435
|
r = call.remaining
|
|
343
436
|
|
|
344
|
-
Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#failing instance=#{i}" }
|
|
345
437
|
begin
|
|
346
438
|
ret = callback.failing(e, i, a, r)
|
|
347
439
|
rescue StandardError => e
|
|
@@ -352,7 +444,6 @@ class Sqreen::Weave::Legacy::Instrumentation
|
|
|
352
444
|
Sqreen::RemoteException.record(e)
|
|
353
445
|
end
|
|
354
446
|
end
|
|
355
|
-
Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#failing instance=#{i} => return=#{ret.inspect}" }
|
|
356
447
|
|
|
357
448
|
throw(b, b.raise(e)) if ret.nil? || !ret.is_a?(Hash)
|
|
358
449
|
|