sqreen 1.18.6-java → 1.20.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +27 -0
- data/lib/sqreen/actions.rb +2 -0
- data/lib/sqreen/actions/actions_index.rb +16 -0
- data/lib/sqreen/actions/base.rb +4 -10
- data/lib/sqreen/actions/block_ip.rb +2 -0
- data/lib/sqreen/actions/block_user.rb +2 -0
- data/lib/sqreen/actions/ip_range_indexed_action_class.rb +4 -24
- data/lib/sqreen/actions/ip_ranges_index.rb +32 -11
- data/lib/sqreen/actions/redirect_ip.rb +2 -0
- data/lib/sqreen/actions/redirect_user.rb +2 -0
- data/lib/sqreen/actions/repository.rb +27 -8
- data/lib/sqreen/actions/unknown_action_type.rb +4 -0
- data/lib/sqreen/actions/user_action_class.rb +5 -30
- data/lib/sqreen/actions/users_index.rb +35 -0
- data/lib/sqreen/agent.rb +2 -1
- data/lib/sqreen/aggregated_metric.rb +25 -0
- data/lib/sqreen/attack_blocked.rb +2 -0
- data/lib/sqreen/binding_accessor.rb +2 -0
- data/lib/sqreen/binding_accessor/path_elem.rb +2 -0
- data/lib/sqreen/binding_accessor/transforms.rb +8 -1
- data/lib/sqreen/call_countable.rb +2 -0
- data/lib/sqreen/capped_queue.rb +2 -0
- data/lib/sqreen/cb.rb +2 -0
- data/lib/sqreen/cb_tree.rb +2 -0
- data/lib/sqreen/condition_evaluator.rb +2 -0
- data/lib/sqreen/conditionable.rb +2 -0
- data/lib/sqreen/configuration.rb +19 -1
- data/lib/sqreen/context.rb +2 -0
- data/lib/sqreen/default_cb.rb +2 -0
- data/lib/sqreen/deferred_logger.rb +2 -0
- data/lib/sqreen/deliveries.rb +2 -0
- data/lib/sqreen/deliveries/batch.rb +6 -1
- data/lib/sqreen/deliveries/simple.rb +6 -0
- data/lib/sqreen/dependency.rb +3 -1
- data/lib/sqreen/dependency/detector.rb +22 -14
- data/lib/sqreen/dependency/libsqreen.rb +4 -0
- data/lib/sqreen/dependency/new_relic.rb +2 -0
- data/lib/sqreen/dependency/rack.rb +10 -5
- data/lib/sqreen/dependency/rails.rb +4 -0
- data/lib/sqreen/dependency/sentry.rb +2 -0
- data/lib/sqreen/dependency/sinatra.rb +12 -1
- data/lib/sqreen/encoding_sanitizer.rb +2 -0
- data/lib/sqreen/error_handling_middleware.rb +2 -0
- data/lib/sqreen/event.rb +9 -5
- data/lib/sqreen/events/attack.rb +25 -18
- data/lib/sqreen/events/remote_exception.rb +2 -22
- data/lib/sqreen/events/request_record.rb +17 -70
- data/lib/sqreen/exception.rb +2 -0
- data/lib/sqreen/formatter_with_tid.rb +2 -0
- data/lib/sqreen/framework_cb.rb +2 -0
- data/lib/sqreen/frameworks.rb +2 -0
- data/lib/sqreen/frameworks/generic.rb +2 -0
- data/lib/sqreen/frameworks/rails.rb +1 -0
- data/lib/sqreen/frameworks/rails3.rb +2 -0
- data/lib/sqreen/frameworks/request_recorder.rb +15 -2
- data/lib/sqreen/frameworks/sinatra.rb +2 -0
- data/lib/sqreen/frameworks/sqreen_test.rb +2 -0
- data/lib/sqreen/graft.rb +12 -0
- data/lib/sqreen/graft/call.rb +150 -0
- data/lib/sqreen/{dependency → graft}/callback.rb +12 -4
- data/lib/sqreen/graft/hook.rb +316 -0
- data/lib/sqreen/{dependency → graft}/hook_point.rb +152 -33
- data/lib/sqreen/graft/hook_point_error.rb +10 -0
- data/lib/sqreen/invalid_signature_exception.rb +2 -0
- data/lib/sqreen/js.rb +2 -0
- data/lib/sqreen/js/call_context.rb +2 -0
- data/lib/sqreen/js/context_pool.rb +2 -0
- data/lib/sqreen/js/exec_js_runnable.rb +2 -0
- data/lib/sqreen/js/execjs_adapter.rb +2 -0
- data/lib/sqreen/js/executable_js.rb +2 -0
- data/lib/sqreen/js/js_service.rb +2 -0
- data/lib/sqreen/js/js_service_adapter.rb +2 -0
- data/lib/sqreen/js/mini_racer_adapter.rb +2 -0
- data/lib/sqreen/js/mini_racer_executable_js.rb +2 -0
- data/lib/sqreen/js/thread_local_exec_js_runnable.rb +2 -0
- data/lib/sqreen/kit/signals/specialized/aggregated_metric.rb +72 -0
- data/lib/sqreen/kit/signals/specialized/attack.rb +57 -0
- data/lib/sqreen/kit/signals/specialized/binning_metric.rb +76 -0
- data/lib/sqreen/kit/signals/specialized/http_trace.rb +26 -0
- data/lib/sqreen/kit/signals/specialized/sdk_track_call.rb +50 -0
- data/lib/sqreen/kit/signals/specialized/sqreen_exception.rb +57 -0
- data/lib/sqreen/{backport.rb → legacy.rb} +3 -2
- data/lib/sqreen/{instrumentation.rb → legacy/instrumentation.rb} +31 -2
- data/lib/sqreen/legacy/old_event_submission_strategy.rb +221 -0
- data/lib/sqreen/legacy/waf_redactions.rb +49 -0
- data/lib/sqreen/log.rb +2 -0
- data/lib/sqreen/log/loggable.rb +28 -0
- data/lib/sqreen/logger.rb +2 -0
- data/lib/sqreen/metrics.rb +2 -0
- data/lib/sqreen/metrics/average.rb +2 -0
- data/lib/sqreen/metrics/base.rb +5 -0
- data/lib/sqreen/metrics/binning.rb +2 -0
- data/lib/sqreen/metrics/collect.rb +2 -0
- data/lib/sqreen/metrics/sum.rb +2 -0
- data/lib/sqreen/metrics_store.rb +24 -12
- data/lib/sqreen/metrics_store/already_registered_metric.rb +2 -0
- data/lib/sqreen/metrics_store/unknown_metric.rb +2 -0
- data/lib/sqreen/metrics_store/unregistered_metric.rb +2 -0
- data/lib/sqreen/middleware.rb +2 -0
- data/lib/sqreen/mono_time.rb +2 -0
- data/lib/sqreen/node.rb +2 -0
- data/lib/sqreen/not_implemented_yet.rb +2 -0
- data/lib/sqreen/null_logger.rb +2 -0
- data/lib/sqreen/payload_creator.rb +2 -0
- data/lib/sqreen/payload_creator/header_section.rb +2 -0
- data/lib/sqreen/performance_notifications.rb +2 -0
- data/lib/sqreen/performance_notifications/binned_metrics.rb +10 -2
- data/lib/sqreen/performance_notifications/log.rb +2 -0
- data/lib/sqreen/performance_notifications/log_performance.rb +2 -0
- data/lib/sqreen/performance_notifications/metrics.rb +2 -0
- data/lib/sqreen/performance_notifications/newrelic.rb +2 -0
- data/lib/sqreen/prefix.rb +2 -0
- data/lib/sqreen/rails_middleware.rb +2 -0
- data/lib/sqreen/remote_command.rb +2 -0
- data/lib/sqreen/remote_command/failure_output.rb +5 -0
- data/lib/sqreen/rules.rb +6 -2
- data/lib/sqreen/rules/attrs.rb +2 -0
- data/lib/sqreen/rules/auth_track_cb.rb +2 -0
- data/lib/sqreen/rules/binding_accessor_matcher_cb.rb +2 -0
- data/lib/sqreen/rules/binding_accessor_metrics.rb +2 -0
- data/lib/sqreen/rules/blacklist_ips_cb.rb +2 -0
- data/lib/sqreen/rules/count_http_codes.rb +2 -0
- data/lib/sqreen/rules/crawler_user_agent_matches_cb.rb +2 -0
- data/lib/sqreen/rules/crawler_user_agent_matches_metrics_cb.rb +2 -0
- data/lib/sqreen/rules/custom_error_cb.rb +2 -0
- data/lib/sqreen/rules/devise_auth_track_cb.rb +2 -0
- data/lib/sqreen/rules/devise_signup_track_cb.rb +2 -0
- data/lib/sqreen/rules/execjs_cb.rb +2 -0
- data/lib/sqreen/rules/headers_insert_cb.rb +7 -0
- data/lib/sqreen/rules/matcher_rule.rb +2 -0
- data/lib/sqreen/rules/not_found_cb.rb +7 -0
- data/lib/sqreen/rules/rails_parameters_cb.rb +2 -0
- data/lib/sqreen/rules/record_request_context.rb +2 -0
- data/lib/sqreen/rules/regexp_rule_cb.rb +2 -0
- data/lib/sqreen/rules/rule_cb.rb +4 -0
- data/lib/sqreen/rules/run_req_start_actions.rb +3 -1
- data/lib/sqreen/rules/run_user_actions.rb +3 -1
- data/lib/sqreen/rules/shell_env_cb.rb +2 -0
- data/lib/sqreen/rules/signup_track_cb.rb +2 -0
- data/lib/sqreen/rules/update_request_context.rb +2 -0
- data/lib/sqreen/rules/url_matches_cb.rb +2 -0
- data/lib/sqreen/rules/user_agent_matches_cb.rb +2 -0
- data/lib/sqreen/rules/waf_cb.rb +41 -16
- data/lib/sqreen/rules/xss_cb.rb +2 -0
- data/lib/sqreen/run_when_called_cb.rb +2 -0
- data/lib/sqreen/runner.rb +68 -12
- data/lib/sqreen/runtime_infos.rb +2 -0
- data/lib/sqreen/safe_json.rb +2 -0
- data/lib/sqreen/sdk.rb +4 -0
- data/lib/sqreen/sensitive_data_redactor.rb +21 -31
- data/lib/sqreen/serializer.rb +2 -0
- data/lib/sqreen/session.rb +41 -37
- data/lib/sqreen/shared_storage.rb +2 -0
- data/lib/sqreen/shared_storage23.rb +2 -0
- data/lib/sqreen/shrink_wrap.rb +16 -0
- data/lib/sqreen/signals/conversions.rb +283 -0
- data/lib/sqreen/signals/http_trace_redaction.rb +111 -0
- data/lib/sqreen/signals/signals_submission_strategy.rb +78 -0
- data/lib/sqreen/signature_verifier.rb +2 -0
- data/lib/sqreen/sinatra_middleware.rb +2 -0
- data/lib/sqreen/sqreen_signed_verifier.rb +2 -0
- data/lib/sqreen/token_invalid_exception.rb +2 -0
- data/lib/sqreen/token_not_found_exception.rb +2 -0
- data/lib/sqreen/trie.rb +2 -0
- data/lib/sqreen/unauthorized.rb +2 -0
- data/lib/sqreen/util.rb +5 -0
- data/lib/sqreen/util/capped_array.rb +2 -0
- data/lib/sqreen/util/capped_hash.rb +2 -0
- data/lib/sqreen/util/capped_string.rb +2 -0
- data/lib/sqreen/util/capper.rb +2 -0
- data/lib/sqreen/version.rb +3 -1
- data/lib/sqreen/waf_error.rb +2 -0
- data/lib/sqreen/weave.rb +12 -0
- data/lib/sqreen/weave/hardcoded.rb +19 -0
- data/lib/sqreen/weave/instrumentor.rb +48 -0
- data/lib/sqreen/weave/legacy.rb +12 -0
- data/lib/sqreen/weave/legacy/instrumentation.rb +406 -0
- data/lib/sqreen/web_server.rb +2 -0
- data/lib/sqreen/web_server/generic.rb +2 -0
- data/lib/sqreen/web_server/passenger.rb +2 -0
- data/lib/sqreen/web_server/puma.rb +2 -0
- data/lib/sqreen/web_server/rainbows.rb +2 -0
- data/lib/sqreen/web_server/thin.rb +2 -0
- data/lib/sqreen/web_server/unicorn.rb +2 -0
- data/lib/sqreen/web_server/webrick.rb +2 -0
- data/lib/sqreen/worker.rb +2 -0
- metadata +65 -9
- data/lib/sqreen/backport/original_name.rb +0 -86
- data/lib/sqreen/dependency/hook.rb +0 -102
@@ -0,0 +1,111 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'sqreen/kit/loggable'
|
3
|
+
require 'sqreen/kit/signals/specialized/http_trace'
|
4
|
+
|
5
|
+
module Sqreen
|
6
|
+
module Signals
|
7
|
+
module HttpTraceRedaction
|
8
|
+
class << self
|
9
|
+
include Sqreen::Kit::Loggable
|
10
|
+
|
11
|
+
# @param [Sqreen::Kit::Signals::Specialized::HttpTrace] trace
|
12
|
+
# @param [Sqreen::SensitiveDataRedactor] redactor
|
13
|
+
def redact_trace!(trace, redactor)
|
14
|
+
return unless redactor
|
15
|
+
# redact headers (keys unsafe)
|
16
|
+
# @type [Sqreen::Kit::Signals::Context::HttpContext]
|
17
|
+
http_context = trace.context
|
18
|
+
|
19
|
+
all_redacted = []
|
20
|
+
|
21
|
+
# Redact headers; save redacted values
|
22
|
+
# headers are encoded as [key, value], not a hash, so
|
23
|
+
# they require some transformation
|
24
|
+
orig_headers = http_context.headers
|
25
|
+
if orig_headers
|
26
|
+
headers = orig_headers.map { |(k, v)| { k => v } }
|
27
|
+
headers, redacted = redactor.redact(headers)
|
28
|
+
http_context.headers = headers.map(&:first)
|
29
|
+
all_redacted += redacted
|
30
|
+
end
|
31
|
+
|
32
|
+
# Redact params; save redacted values
|
33
|
+
Kit::Signals::Context::HttpContext::PARAMS_ATTRS.each do |attr|
|
34
|
+
value = http_context.public_send(attr)
|
35
|
+
next unless value
|
36
|
+
value, redacted = redactor.redact(value)
|
37
|
+
all_redacted += redacted
|
38
|
+
http_context.public_send(:"#{attr}=", value)
|
39
|
+
end
|
40
|
+
|
41
|
+
all_redacted = all_redacted.uniq.map(&:downcase)
|
42
|
+
|
43
|
+
# Redact attacks and exceptions
|
44
|
+
# XXX: no redaction for infos in attacks/exceptions except for WAF data
|
45
|
+
# Is this the correct behavior?
|
46
|
+
redact_attacks!(trace, redactor, all_redacted)
|
47
|
+
redact_exceptions!(trace, redactor, all_redacted)
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
|
52
|
+
# @param [Sqreen::Kit::Signals::Specialized::HttpTrace] trace
|
53
|
+
# @param [Sqreen::SensitiveDataRedactor] redactor
|
54
|
+
# Redacts WAF data according to specific rules therefor
|
55
|
+
# Redacts infos according to general rules
|
56
|
+
def redact_attacks!(trace, redactor, redacted_data)
|
57
|
+
trace.data.each do |signal|
|
58
|
+
next unless signal.is_a?(Kit::Signals::Specialized::Attack)
|
59
|
+
# @type [Sqreen::Kit::Signals::Specialized::Attack::Payload] payload
|
60
|
+
payload = signal.payload
|
61
|
+
next unless payload.infos
|
62
|
+
|
63
|
+
if payload.infos[:waf_data]
|
64
|
+
redact_waf_attack_data!(payload.infos, redacted_data)
|
65
|
+
end
|
66
|
+
payload.infos, = redactor.redact(payload.infos)
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def redact_exceptions!(trace, redactor, redacted_data)
|
71
|
+
trace.data.each do |signal|
|
72
|
+
next unless signal.is_a?(Kit::Signals::Specialized::SqreenException)
|
73
|
+
infos = signal.infos
|
74
|
+
next unless infos
|
75
|
+
|
76
|
+
redact_waf_exception_data!(signal.infos, redacted_data) if signal.infos[:waf]
|
77
|
+
signal.infos, = redactor.redact(infos)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
# @param [Hash] infos from WAF attack
|
82
|
+
def redact_waf_attack_data!(infos, redacted_data)
|
83
|
+
begin
|
84
|
+
parsed = JSON.parse(infos[:waf_data])
|
85
|
+
rescue JSON::JSONError => e
|
86
|
+
logger.warn("waf_data is not valid json: #{e.message}")
|
87
|
+
return
|
88
|
+
end
|
89
|
+
redacted = parsed.each do |w|
|
90
|
+
next unless (filters = w['filter'])
|
91
|
+
|
92
|
+
filters.each do |f|
|
93
|
+
next unless (v = f['resolved_value'])
|
94
|
+
next unless redacted_data.include?(v.downcase)
|
95
|
+
|
96
|
+
f['match_status'] = SensitiveDataRedactor::MASK
|
97
|
+
f['resolved_value'] = SensitiveDataRedactor::MASK
|
98
|
+
end
|
99
|
+
end
|
100
|
+
infos[:waf_data] = JSON.dump(redacted)
|
101
|
+
end
|
102
|
+
|
103
|
+
# see https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000022-waf-data-sanitization.md#changes-to-the-agents
|
104
|
+
def redact_waf_exception_data!(infos, redacted_data)
|
105
|
+
return if redacted_data.empty?
|
106
|
+
infos[:waf].delete(:args)
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
@@ -0,0 +1,78 @@
|
|
1
|
+
require 'sqreen/aggregated_metric'
|
2
|
+
require 'sqreen/kit'
|
3
|
+
require 'sqreen/kit/string_sanitizer'
|
4
|
+
require 'sqreen/signals/conversions'
|
5
|
+
require 'sqreen/log/loggable'
|
6
|
+
|
7
|
+
module Sqreen
|
8
|
+
module Signals
|
9
|
+
# see also Sqreen::Legacy::OldEventSubmissionStrategy
|
10
|
+
# usage in Sqreen:Session
|
11
|
+
class SignalsSubmissionStrategy
|
12
|
+
include Sqreen::Log::Loggable
|
13
|
+
|
14
|
+
# @param [Array<Sqreen::AggregatedMetric>] metrics
|
15
|
+
def post_metrics(metrics)
|
16
|
+
return if metrics.nil? || metrics.empty?
|
17
|
+
|
18
|
+
guarded 'Failed to serialize or submit aggregated metrics' do
|
19
|
+
batch = metrics.map do |m|
|
20
|
+
Conversions.convert_metric_sample(m)
|
21
|
+
end
|
22
|
+
client.report_batch(batch)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
# @param _attack [Sqreen::Attack]
|
27
|
+
# XXX: unused
|
28
|
+
def post_attack(_attack)
|
29
|
+
raise NotImplementedError
|
30
|
+
end
|
31
|
+
|
32
|
+
# @param request_record [Sqreen::RequestRecord]
|
33
|
+
def post_request_record(request_record)
|
34
|
+
guarded 'Failed to serialize or submit request record' do
|
35
|
+
trace = Conversions.convert_req_record(request_record)
|
36
|
+
append_sanitizing_filter(trace)
|
37
|
+
client.report_trace(trace)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
# Post an exception to Sqreen for analysis
|
42
|
+
# @param exception [RemoteException] Exception and context to be sent over
|
43
|
+
def post_sqreen_exception(exception)
|
44
|
+
guarded 'Failed to serialize or submit exception', false do
|
45
|
+
data = Conversions.convert_exception(exception)
|
46
|
+
append_sanitizing_filter(data)
|
47
|
+
client.report_signal(data)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def post_batch(events)
|
52
|
+
guarded 'Failed to serialize or submit batch of events' do
|
53
|
+
batch = Conversions.convert_batch(events)
|
54
|
+
batch.each { |sig_or_trace| append_sanitizing_filter(sig_or_trace) }
|
55
|
+
client.report_batch(batch)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
private
|
60
|
+
|
61
|
+
def append_sanitizing_filter(sig_or_trace)
|
62
|
+
sig_or_trace.append_to_h_filter Kit::StringSanitizer.method(:sanitize)
|
63
|
+
end
|
64
|
+
|
65
|
+
# we don't want exceptions to propagate and kill the worker thread
|
66
|
+
def guarded(msg, report = true)
|
67
|
+
yield
|
68
|
+
rescue StandardError => e
|
69
|
+
logger.warn "#{msg}: #{e.message}\n#{e.backtrace.map { |x| " #{x}" }.join("\n")}"
|
70
|
+
post_sqreen_exception(RemoteException.new(e)) if report
|
71
|
+
end
|
72
|
+
|
73
|
+
def client
|
74
|
+
Sqreen::Kit.auth_signals_client
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
data/lib/sqreen/trie.rb
CHANGED
data/lib/sqreen/unauthorized.rb
CHANGED
data/lib/sqreen/util.rb
CHANGED
data/lib/sqreen/util/capper.rb
CHANGED
data/lib/sqreen/version.rb
CHANGED
data/lib/sqreen/waf_error.rb
CHANGED
data/lib/sqreen/weave.rb
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
# typed: strong
|
2
|
+
|
3
|
+
# Copyright (c) 2015 Sqreen. All Rights Reserved.
|
4
|
+
# Please refer to our terms for more information: https://www.sqreen.com/terms.html
|
5
|
+
|
6
|
+
require 'sqreen/log/loggable'
|
7
|
+
|
8
|
+
module Sqreen
|
9
|
+
module Weave
|
10
|
+
include Sqreen::Log::Loggable
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# typed: strong
|
2
|
+
|
3
|
+
# Copyright (c) 2015 Sqreen. All Rights Reserved.
|
4
|
+
# Please refer to our terms for more information: https://www.sqreen.com/terms.html
|
5
|
+
|
6
|
+
require 'sqreen/weave'
|
7
|
+
|
8
|
+
class Sqreen::Weave::Hardcoded
|
9
|
+
# [
|
10
|
+
# ### callback for performing sec responses based on ip
|
11
|
+
# ### init redefined to implement smartass way to hook it upon the
|
12
|
+
# ### framework's middleware #call
|
13
|
+
# Sqreen::Rules::RunReqStartActions.new(framework),
|
14
|
+
# ### callback for performing sec responses based on user
|
15
|
+
# Sqreen::Rules::RunUserActions.new(Sqreen, :identify, 0),
|
16
|
+
# ### callback for performing sec responses based on user
|
17
|
+
# Sqreen::Rules::RunUserActions.new(Sqreen, :auth_track, 1),
|
18
|
+
# ]
|
19
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
# typed: true
|
2
|
+
|
3
|
+
# Copyright (c) 2015 Sqreen. All Rights Reserved.
|
4
|
+
# Please refer to our terms for more information: https://www.sqreen.com/terms.html
|
5
|
+
|
6
|
+
require 'sqreen/weave'
|
7
|
+
|
8
|
+
# rule loader: decouple from runner
|
9
|
+
# remote rules from back
|
10
|
+
# local rules from local files
|
11
|
+
# => rule list (what is a rule?)
|
12
|
+
# => to callback (what is a callback?)
|
13
|
+
# => to instrumentation (== attach callbacks to their targets using graft)
|
14
|
+
|
15
|
+
# make shit like instrument framework independent (block passing?)
|
16
|
+
# => too much things assume only one framework
|
17
|
+
# possible to do run req actions without hardcoded cbs?
|
18
|
+
# (data comes from actions command, native cb merely binds to middleware)
|
19
|
+
# can cb be a form of abstraction?
|
20
|
+
|
21
|
+
# rule sig: decouple/split
|
22
|
+
# - data signer/checker
|
23
|
+
# apply this to rule data
|
24
|
+
|
25
|
+
# whitelist is mixed in
|
26
|
+
|
27
|
+
# metrics
|
28
|
+
# three dedicated metrics: abstract and isolate
|
29
|
+
|
30
|
+
class Sqreen::Weave::Instrumentor
|
31
|
+
def initialize(metrics_engine)
|
32
|
+
### bail out if no metric engine
|
33
|
+
### init metric to count calls to sqreen
|
34
|
+
### init metric to count request whitelist matches (ip or path whitelist)
|
35
|
+
### init metric to count over budget hits
|
36
|
+
end
|
37
|
+
|
38
|
+
def instrument!(rules, framework)
|
39
|
+
### set up rule signature verifier
|
40
|
+
### force clean instrumentation callback list
|
41
|
+
### for each rule description, transform into format for adding callback
|
42
|
+
### attach framework to callback
|
43
|
+
### install callback, observing priority
|
44
|
+
### for each hardcoded callback
|
45
|
+
### install hardcoded callbacks, observing priority
|
46
|
+
### globally declare instrumentation ready
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,406 @@
|
|
1
|
+
# typed: false
|
2
|
+
|
3
|
+
# Copyright (c) 2015 Sqreen. All Rights Reserved.
|
4
|
+
# Please refer to our terms for more information: https://www.sqreen.com/terms.html
|
5
|
+
|
6
|
+
require 'sqreen/weave/legacy'
|
7
|
+
require 'sqreen/graft/hook_point'
|
8
|
+
require 'sqreen/call_countable'
|
9
|
+
require 'sqreen/rules'
|
10
|
+
require 'sqreen/rules/record_request_context'
|
11
|
+
|
12
|
+
class Sqreen::Weave::Legacy::Instrumentation
|
13
|
+
attr_accessor :metrics_engine
|
14
|
+
|
15
|
+
def initialize(metrics_engine, opts = {})
|
16
|
+
Sqreen::Weave.logger.debug { "#{self.class.name}#initialize #{metrics_engine}" }
|
17
|
+
@hooks = []
|
18
|
+
|
19
|
+
self.metrics_engine = metrics_engine
|
20
|
+
|
21
|
+
### bail out if no metric engine
|
22
|
+
return if metrics_engine.nil?
|
23
|
+
|
24
|
+
### init metric to count calls to sqreen
|
25
|
+
metrics_engine.create_metric(
|
26
|
+
'name' => 'sqreen_call_counts',
|
27
|
+
'period' => 60,
|
28
|
+
'kind' => 'Sum',
|
29
|
+
)
|
30
|
+
### init metric to count request whitelist matches (ip or path whitelist)
|
31
|
+
metrics_engine.create_metric(
|
32
|
+
'name' => 'whitelisted',
|
33
|
+
'period' => 60,
|
34
|
+
'kind' => 'Sum',
|
35
|
+
)
|
36
|
+
### init metric to count over budget hits
|
37
|
+
metrics_engine.create_metric(
|
38
|
+
'name' => 'request_overtime',
|
39
|
+
'period' => 60,
|
40
|
+
'kind' => 'Sum',
|
41
|
+
)
|
42
|
+
|
43
|
+
# PerformanceNotifications::Binning
|
44
|
+
metrics_engine.create_metric(
|
45
|
+
'name' => 'req',
|
46
|
+
'period' => opts[:period] || 60,
|
47
|
+
'kind' => 'Binning',
|
48
|
+
'options' => opts[:perf_metric] || { 'base' => 2.0, 'factor' => 0.1 },
|
49
|
+
)
|
50
|
+
metrics_engine.create_metric(
|
51
|
+
'name' => 'sq',
|
52
|
+
'period' => opts[:period] || 60,
|
53
|
+
'kind' => 'Binning',
|
54
|
+
'options' => opts[:perf_metric] || { 'base' => 2.0, 'factor' => 0.1 },
|
55
|
+
)
|
56
|
+
metrics_engine.create_metric(
|
57
|
+
'name' => 'pct',
|
58
|
+
'period' => opts[:period] || 60,
|
59
|
+
'kind' => 'Binning',
|
60
|
+
'options' => opts[:perf_metric_percent] || { 'base' => 1.3, 'factor' => 1.0 },
|
61
|
+
)
|
62
|
+
|
63
|
+
Sqreen.thread_cpu_time? && metrics_engine.create_metric(
|
64
|
+
'name' => 'sq_thread_cpu_pct',
|
65
|
+
'period' => opts[:period] || 60,
|
66
|
+
'kind' => 'Binning',
|
67
|
+
'options' => opts[:perf_metric_percent] || { 'base' => 1.3, 'factor' => 1.0 },
|
68
|
+
)
|
69
|
+
end
|
70
|
+
|
71
|
+
# needed by Sqreen::Runner#initialize
|
72
|
+
def instrument!(rules, framework)
|
73
|
+
Sqreen::Weave.logger.debug { "#{rules.count} rules, #{framework}" }
|
74
|
+
|
75
|
+
strategy = Sqreen.config_get(:weave_strategy)
|
76
|
+
if strategy == :prepend && !Module.respond_to?(:prepend)
|
77
|
+
Sqreen::Weave.logger.warn { "strategy: #{strategy.inspect} unavailable, falling back to :chain" }
|
78
|
+
strategy = :chain
|
79
|
+
elsif strategy == :chain && Gem::Specification.select { |s| s.name == 'scout_apm' && Gem::Requirement.new('>= 2.5.2').satisfied_by?(Gem::Version.new(s.version)) }.any?
|
80
|
+
Sqreen::Weave.logger.warn { "strategy: #{strategy.inspect} unavailable with scout_apm >= 2.5.2, switching to :prepend" }
|
81
|
+
strategy = :prepend
|
82
|
+
end
|
83
|
+
Sqreen::Weave.logger.debug { "strategy: #{strategy.inspect}" }
|
84
|
+
|
85
|
+
### set up rule signature verifier
|
86
|
+
verifier = nil
|
87
|
+
### force clean instrumentation callback list
|
88
|
+
@hooks = []
|
89
|
+
### for each rule description
|
90
|
+
rules.each do |rule|
|
91
|
+
Sqreen::Weave.logger.debug { "Processing rule: #{rule['name']}" }
|
92
|
+
### transform into format for adding callback
|
93
|
+
rule_callback = Sqreen::Rules.cb_from_rule(rule, self, metrics_engine, verifier)
|
94
|
+
next unless rule_callback
|
95
|
+
### attach framework to callback
|
96
|
+
rule_callback.framework = framework
|
97
|
+
### install callback, observing priority
|
98
|
+
Sqreen::Weave.logger.debug { "Adding rule callback: #{rule_callback}" }
|
99
|
+
@hooks << add_callback("weave,rule=#{rule['name']}", rule_callback, strategy)
|
100
|
+
end
|
101
|
+
|
102
|
+
### for each hardcoded callback
|
103
|
+
hardcoded_callbacks(framework).each do |hard_callback|
|
104
|
+
Sqreen::Weave.logger.debug { "Adding hardcoded callback: #{hard_callback}" }
|
105
|
+
### install hardcoded callbacks, observing priority
|
106
|
+
@hooks << add_callback('weave,hardcoded', hard_callback, strategy)
|
107
|
+
end
|
108
|
+
|
109
|
+
metrics_engine = self.metrics_engine
|
110
|
+
request_hook = Sqreen::Graft::Hook['Sqreen::ShrinkWrap#call', strategy]
|
111
|
+
@hooks << request_hook
|
112
|
+
request_hook.add do
|
113
|
+
before('wave,meta,request', rank: -100000, mandatory: true) do |_call|
|
114
|
+
next unless Sqreen.instrumentation_ready
|
115
|
+
|
116
|
+
uuid = SecureRandom.uuid
|
117
|
+
now = Sqreen::Graft::Timer.read
|
118
|
+
Thread.current[:sqreen_http_request] = {
|
119
|
+
uuid: uuid,
|
120
|
+
start_time: now,
|
121
|
+
time_budget: Sqreen.performance_budget,
|
122
|
+
time_budget_expended: false,
|
123
|
+
timer: Sqreen::Graft::Timer.new("request_#{uuid}"),
|
124
|
+
timed_callbacks: [],
|
125
|
+
timed_hooks: [],
|
126
|
+
timed_hooks_before: [],
|
127
|
+
timed_hooks_after: [],
|
128
|
+
timed_hooks_raised: [],
|
129
|
+
timed_hooks_ensured: [],
|
130
|
+
skipped_callbacks: [],
|
131
|
+
}
|
132
|
+
|
133
|
+
Sqreen::Weave.logger.debug { "request.uuid: #{uuid}" }
|
134
|
+
end
|
135
|
+
|
136
|
+
ensured('weave,meta,request', rank: 100000, mandatory: true) do |_call|
|
137
|
+
request = Thread.current[:sqreen_http_request]
|
138
|
+
|
139
|
+
next if request.nil?
|
140
|
+
|
141
|
+
Thread.current[:sqreen_http_request] = nil
|
142
|
+
now = Sqreen::Graft::Timer.read
|
143
|
+
utc_now = Time.now.utc
|
144
|
+
|
145
|
+
request[:timed_callbacks].each do |timer|
|
146
|
+
duration = timer.duration
|
147
|
+
# stop = now
|
148
|
+
# start = now - duration
|
149
|
+
timer.tag =~ /weave,rule=(.*)$/ && rule = $1
|
150
|
+
timer.tag =~ /@before/ && whence = 'pre'
|
151
|
+
timer.tag =~ /@after/ && whence = 'post'
|
152
|
+
timer.tag =~ /@raised/ && whence = 'failing'
|
153
|
+
|
154
|
+
next unless rule && whence
|
155
|
+
|
156
|
+
# Sqreen::PerformanceNotifications.notify(rule, whence, start, stop)
|
157
|
+
# => BinnedMetrics
|
158
|
+
metric_name = "sq.#{rule}.#{whence}"
|
159
|
+
unless metrics_engine.metric?(metric_name)
|
160
|
+
metrics_engine.create_metric(
|
161
|
+
'name' => metric_name,
|
162
|
+
'period' => 60,
|
163
|
+
'kind' => 'Binning',
|
164
|
+
'options' => { 'base' => 2.0, 'factor' => 0.1 },
|
165
|
+
)
|
166
|
+
end
|
167
|
+
metrics_engine.update(metric_name, now, nil, duration * 1000)
|
168
|
+
end
|
169
|
+
|
170
|
+
metric_name = 'sq.hooks_pre.pre'
|
171
|
+
duration = request[:timed_hooks_before].sum(&:duration)
|
172
|
+
unless metrics_engine.metric?(metric_name)
|
173
|
+
metrics_engine.create_metric(
|
174
|
+
'name' => metric_name,
|
175
|
+
'period' => 60,
|
176
|
+
'kind' => 'Binning',
|
177
|
+
'options' => { 'base' => 2.0, 'factor' => 0.1 },
|
178
|
+
)
|
179
|
+
end
|
180
|
+
metrics_engine.update(metric_name, now, nil, duration * 1000)
|
181
|
+
|
182
|
+
metric_name = 'sq.hooks_post.post'
|
183
|
+
duration = request[:timed_hooks_after].sum(&:duration)
|
184
|
+
unless metrics_engine.metric?(metric_name)
|
185
|
+
metrics_engine.create_metric(
|
186
|
+
'name' => metric_name,
|
187
|
+
'period' => 60,
|
188
|
+
'kind' => 'Binning',
|
189
|
+
'options' => { 'base' => 2.0, 'factor' => 0.1 },
|
190
|
+
)
|
191
|
+
end
|
192
|
+
metrics_engine.update(metric_name, now, nil, duration * 1000)
|
193
|
+
|
194
|
+
metric_name = 'sq.hooks_failing.failing'
|
195
|
+
duration = request[:timed_hooks_raised].sum(&:duration)
|
196
|
+
unless metrics_engine.metric?(metric_name)
|
197
|
+
metrics_engine.create_metric(
|
198
|
+
'name' => metric_name,
|
199
|
+
'period' => 60,
|
200
|
+
'kind' => 'Binning',
|
201
|
+
'options' => { 'base' => 2.0, 'factor' => 0.1 },
|
202
|
+
)
|
203
|
+
end
|
204
|
+
metrics_engine.update(metric_name, now, nil, duration * 1000)
|
205
|
+
|
206
|
+
skipped = request[:skipped_callbacks].map(&:name)
|
207
|
+
Sqreen::Weave.logger.debug { "request:#{request[:uuid]} callback.skipped.size: #{skipped.count} callback.skipped: [#{skipped.join(', ')}]" }
|
208
|
+
timer = request[:timer]
|
209
|
+
total = timer.duration
|
210
|
+
Sqreen::Weave.logger.debug { "request:#{request[:uuid]} timer.total: #{'%.03fus' % (total * 1_000_000)} timer.size: #{timer.size}" }
|
211
|
+
timings = request[:timed_callbacks].map(&:to_s)
|
212
|
+
total = request[:timed_callbacks].sum(&:duration)
|
213
|
+
Sqreen::Weave.logger.debug { "request:#{request[:uuid]} callback.total: #{'%.03fus' % (total * 1_000_000)} callback.timings: [#{timings.join(', ')}]" }
|
214
|
+
timings = request[:timed_hooks].map(&:to_s)
|
215
|
+
total = request[:timed_hooks].sum(&:duration)
|
216
|
+
Sqreen::Weave.logger.debug { "request:#{request[:uuid]} hook.total: #{'%.03fus' % (total * 1_000_000)} hook.timings: [#{timings.join(', ')}]" }
|
217
|
+
|
218
|
+
skipped = request[:skipped_callbacks].map(&:name)
|
219
|
+
skipped_rule_name = skipped.first && skipped.first =~ /weave,rule=(.*)$/ && $1
|
220
|
+
Sqreen.observations_queue.push(['request_overtime', skipped_rule_name, 1, utc_now]) if skipped_rule_name
|
221
|
+
|
222
|
+
sqreen_request_duration = total
|
223
|
+
Sqreen.observations_queue.push(['sq', nil, sqreen_request_duration * 1000, utc_now])
|
224
|
+
|
225
|
+
request_duration = now - request[:start_time]
|
226
|
+
Sqreen.observations_queue.push(['req', nil, request_duration * 1000, utc_now])
|
227
|
+
|
228
|
+
sqreen_request_ratio = (sqreen_request_duration * 100.0) / (request_duration - sqreen_request_duration)
|
229
|
+
Sqreen.observations_queue.push(['pct', nil, sqreen_request_ratio, utc_now])
|
230
|
+
end
|
231
|
+
end.install
|
232
|
+
|
233
|
+
### globally declare instrumentation ready
|
234
|
+
Sqreen.instrumentation_ready = true
|
235
|
+
end
|
236
|
+
|
237
|
+
# needed by Sqreen::Runner
|
238
|
+
def remove_all_callbacks
|
239
|
+
Sqreen.instrumentation_ready = false
|
240
|
+
|
241
|
+
loop do
|
242
|
+
hook = @hooks.pop
|
243
|
+
break unless hook
|
244
|
+
Sqreen::Weave.logger.debug { "hook.deinstrument: #{hook}" }
|
245
|
+
hook.uninstall
|
246
|
+
hook.clear
|
247
|
+
end
|
248
|
+
end
|
249
|
+
|
250
|
+
# needed by #instrument!
|
251
|
+
def add_callback(rule, callback, strategy)
|
252
|
+
Sqreen::Weave.logger.debug { "Adding rule: #{rule} callback: #{callback}" }
|
253
|
+
klass = callback.klass
|
254
|
+
method = callback.method
|
255
|
+
|
256
|
+
if Sqreen::Graft::HookPoint.new("#{klass}.#{method}").exist?
|
257
|
+
hook_point = "#{klass}.#{method}"
|
258
|
+
elsif Sqreen::Graft::HookPoint.new("#{klass}##{method}").exist?
|
259
|
+
hook_point = "#{klass}##{method}"
|
260
|
+
end
|
261
|
+
|
262
|
+
return if hook_point.nil?
|
263
|
+
|
264
|
+
priority = callback.priority || 100
|
265
|
+
block = callback.respond_to?(:block) ? callback.block : true
|
266
|
+
ignore = -> { callback.whitelisted? } if callback.respond_to?(:whitelisted?)
|
267
|
+
|
268
|
+
hook = Sqreen::Graft::Hook[hook_point, strategy]
|
269
|
+
hook.add do
|
270
|
+
if callback.pre?
|
271
|
+
before(rule, rank: priority, mandatory: !callback.overtimeable, flow: block, ignore: ignore) do |call, b|
|
272
|
+
next unless Thread.current[:sqreen_http_request]
|
273
|
+
|
274
|
+
i = call.instance
|
275
|
+
a = call.args
|
276
|
+
r = call.remaining
|
277
|
+
|
278
|
+
Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#pre instance=#{i}" }
|
279
|
+
begin
|
280
|
+
ret = callback.pre(i, a, r)
|
281
|
+
rescue StandardError => e
|
282
|
+
Sqreen::Weave.logger.warn { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#pre instance=#{i} => exception=#{e}" }
|
283
|
+
if callback.respond_to?(:record_exception)
|
284
|
+
callback.record_exception(e)
|
285
|
+
else
|
286
|
+
Sqreen::RemoteException.record(e)
|
287
|
+
end
|
288
|
+
end
|
289
|
+
Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#pre instance=#{i} => return=#{ret.inspect}" }
|
290
|
+
|
291
|
+
case ret[:status]
|
292
|
+
when :skip, 'skip'
|
293
|
+
throw(b, b.return(ret[:new_return_value]).break!) if ret.key?(:new_return_value)
|
294
|
+
when :modify_args, 'modify_args'
|
295
|
+
throw(b, b.args(ret[:args]))
|
296
|
+
when :raise, 'raise'
|
297
|
+
throw(b, b.raise(ret[:exception])) if ret.key?(:exception)
|
298
|
+
throw(b, b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required.")))
|
299
|
+
end unless ret.nil? || !ret.is_a?(Hash)
|
300
|
+
end
|
301
|
+
end
|
302
|
+
|
303
|
+
if callback.post?
|
304
|
+
after(rule, rank: -priority, mandatory: !callback.overtimeable, flow: block, ignore: ignore) do |call, b|
|
305
|
+
next unless Thread.current[:sqreen_http_request]
|
306
|
+
|
307
|
+
i = call.instance
|
308
|
+
v = call.returned
|
309
|
+
a = call.args
|
310
|
+
r = call.remaining
|
311
|
+
|
312
|
+
Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#post instance=#{i}" }
|
313
|
+
begin
|
314
|
+
ret = callback.post(v, i, a, r)
|
315
|
+
rescue StandardError => e
|
316
|
+
Sqreen::Weave.logger.warn { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#post instance=#{i} => exception=#{e}" }
|
317
|
+
if callback.respond_to?(:record_exception)
|
318
|
+
callback.record_exception(e)
|
319
|
+
else
|
320
|
+
Sqreen::RemoteException.record(e)
|
321
|
+
end
|
322
|
+
end
|
323
|
+
Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#post instance=#{i} => return=#{ret.inspect}" }
|
324
|
+
|
325
|
+
case ret[:status]
|
326
|
+
when :override, 'override'
|
327
|
+
throw(b, b.return(ret[:new_return_value])) if ret.key?(:new_return_value)
|
328
|
+
when :raise, 'raise'
|
329
|
+
throw(b, b.raise(ret[:exception])) if ret.key?(:exception)
|
330
|
+
throw(b, b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required.")))
|
331
|
+
end unless ret.nil? || !ret.is_a?(Hash)
|
332
|
+
end
|
333
|
+
end
|
334
|
+
|
335
|
+
if callback.failing?
|
336
|
+
raised(rule, rank: priority, mandatory: !callback.overtimeable, flow: block, ignore: ignore) do |call, b|
|
337
|
+
next unless Thread.current[:sqreen_http_request]
|
338
|
+
|
339
|
+
i = call.instance
|
340
|
+
e = call.raised
|
341
|
+
a = call.args
|
342
|
+
r = call.remaining
|
343
|
+
|
344
|
+
Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#failing instance=#{i}" }
|
345
|
+
begin
|
346
|
+
ret = callback.failing(e, i, a, r)
|
347
|
+
rescue StandardError => e
|
348
|
+
Sqreen::Weave.logger.warn { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#failing instance=#{i} => exception=#{e}" }
|
349
|
+
if callback.respond_to?(:record_exception)
|
350
|
+
callback.record_exception(e)
|
351
|
+
else
|
352
|
+
Sqreen::RemoteException.record(e)
|
353
|
+
end
|
354
|
+
end
|
355
|
+
Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#failing instance=#{i} => return=#{ret.inspect}" }
|
356
|
+
|
357
|
+
throw(b, b.raise(e)) if ret.nil? || !ret.is_a?(Hash)
|
358
|
+
|
359
|
+
case ret[:status]
|
360
|
+
when :override, 'override'
|
361
|
+
throw(b, b.return(ret[:new_return_value])) if ret.key?(:new_return_value)
|
362
|
+
when :retry, 'retry'
|
363
|
+
throw(b, b.retry)
|
364
|
+
when :raise, 'raise'
|
365
|
+
throw(b, b.raise(ret[:exception])) if ret.key?(:exception)
|
366
|
+
throw(b, b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required.")))
|
367
|
+
when :reraise, 'reraise'
|
368
|
+
throw(b, b.raise(e))
|
369
|
+
else
|
370
|
+
throw(b, b.raise(e))
|
371
|
+
end unless ret.nil? || !ret.is_a?(Hash)
|
372
|
+
end
|
373
|
+
end
|
374
|
+
end.install
|
375
|
+
|
376
|
+
hook
|
377
|
+
end
|
378
|
+
|
379
|
+
# needed by Sqreen::Rules.cb_from_rule
|
380
|
+
def valid_method?(klass, method)
|
381
|
+
if Sqreen::Graft::HookPoint.new("#{klass}.#{method}").exist?
|
382
|
+
Sqreen::Weave.logger.debug { "HookPoint found: #{klass}.#{method}" }
|
383
|
+
true
|
384
|
+
elsif Sqreen::Graft::HookPoint.new("#{klass}##{method}").exist?
|
385
|
+
Sqreen::Weave.logger.debug { "HookPoint found: #{klass}##{method}" }
|
386
|
+
true
|
387
|
+
else
|
388
|
+
Sqreen::Weave.logger.debug { "HookPoint not found: #{klass} #{method}" }
|
389
|
+
false
|
390
|
+
end
|
391
|
+
end
|
392
|
+
|
393
|
+
# needed by #instrument!
|
394
|
+
def hardcoded_callbacks(framework)
|
395
|
+
[
|
396
|
+
### callback for performing sec responses based on ip
|
397
|
+
### init redefined to implement smartass way to hook it upon the
|
398
|
+
### framework's middleware #call
|
399
|
+
Sqreen::Rules::RunReqStartActions.new(framework),
|
400
|
+
### callback for performing sec responses based on user
|
401
|
+
Sqreen::Rules::RunUserActions.new(Sqreen, :identify, 0),
|
402
|
+
### callback for performing sec responses based on user
|
403
|
+
Sqreen::Rules::RunUserActions.new(Sqreen, :auth_track, 1),
|
404
|
+
]
|
405
|
+
end
|
406
|
+
end
|