sqreen 1.20.1 → 1.21.0.beta1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/sqreen/attack_detected.html +1 -2
- data/lib/sqreen/deliveries/batch.rb +8 -1
- data/lib/sqreen/ecosystem.rb +80 -0
- data/lib/sqreen/ecosystem/dispatch_table.rb +43 -0
- data/lib/sqreen/ecosystem/http/net_http.rb +51 -0
- data/lib/sqreen/ecosystem/http/rack_request.rb +38 -0
- data/lib/sqreen/ecosystem/loggable.rb +13 -0
- data/lib/sqreen/ecosystem/module_api.rb +30 -0
- data/lib/sqreen/ecosystem/module_api/event_listener.rb +18 -0
- data/lib/sqreen/ecosystem/module_api/instrumentation.rb +23 -0
- data/lib/sqreen/ecosystem/module_api/signal_producer.rb +26 -0
- data/lib/sqreen/ecosystem/module_api/tracing_push_down.rb +34 -0
- data/lib/sqreen/ecosystem/module_api/transaction_storage.rb +71 -0
- data/lib/sqreen/ecosystem/module_registry.rb +39 -0
- data/lib/sqreen/ecosystem/redis/redis_connection.rb +35 -0
- data/lib/sqreen/ecosystem/tracing/sampler.rb +160 -0
- data/lib/sqreen/ecosystem/tracing/sampling_configuration.rb +150 -0
- data/lib/sqreen/ecosystem/tracing/signals/tracing_client.rb +53 -0
- data/lib/sqreen/ecosystem/tracing/signals/tracing_server.rb +53 -0
- data/lib/sqreen/ecosystem/tracing_id_setup.rb +34 -0
- data/lib/sqreen/ecosystem/transaction_storage.rb +64 -0
- data/lib/sqreen/ecosystem_integration.rb +70 -0
- data/lib/sqreen/ecosystem_integration/around_callbacks.rb +89 -0
- data/lib/sqreen/ecosystem_integration/instrumentation_service.rb +38 -0
- data/lib/sqreen/ecosystem_integration/request_lifecycle_tracking.rb +56 -0
- data/lib/sqreen/ecosystem_integration/signal_consumption.rb +35 -0
- data/lib/sqreen/frameworks/generic.rb +15 -1
- data/lib/sqreen/graft/call.rb +9 -0
- data/lib/sqreen/legacy/old_event_submission_strategy.rb +7 -1
- data/lib/sqreen/remote_command.rb +3 -0
- data/lib/sqreen/runner.rb +19 -5
- data/lib/sqreen/session.rb +2 -0
- data/lib/sqreen/signals/conversions.rb +6 -1
- data/lib/sqreen/version.rb +1 -1
- metadata +32 -7
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'sqreen/ecosystem/module_registry'
|
2
|
+
require 'sqreen/ecosystem/transaction_storage'
|
3
|
+
require 'sqreen/ecosystem/module_api/signal_producer'
|
4
|
+
|
5
|
+
module Sqreen
|
6
|
+
module Ecosystem
|
7
|
+
class TracingIdSetup
|
8
|
+
# @param [Sqreen::Ecosystem::ModuleRegistry] registry
|
9
|
+
def initialize(registry)
|
10
|
+
@registry = registry
|
11
|
+
@tracing_id_prefix = nil
|
12
|
+
end
|
13
|
+
|
14
|
+
def setup_modules
|
15
|
+
inject_out_of_tx_tracing_id_gen
|
16
|
+
end
|
17
|
+
|
18
|
+
attr_writer :tracing_id_prefix
|
19
|
+
|
20
|
+
private
|
21
|
+
|
22
|
+
def inject_out_of_tx_tracing_id_gen
|
23
|
+
@registry.each_module(Sqreen::Ecosystem::ModuleApi::SignalProducer) do |mod|
|
24
|
+
mod.tracing_id_producer = method(:generate_tracing_id)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def generate_tracing_id
|
29
|
+
return nil unless @tracing_id_prefix
|
30
|
+
"#{@tracing_id_prefix}.#{SecureRandom.uuid}"
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
require 'sqreen/ecosystem/loggable'
|
2
|
+
|
3
|
+
module Sqreen
|
4
|
+
module Ecosystem
|
5
|
+
# The transaction storage is a mechanism for the modules to share
|
6
|
+
# request-scoped data with each other or to keep request-scoped objects
|
7
|
+
# themselves, although, for this last use case to be effective,
|
8
|
+
# propagation of request start/end events to the modules will likely be
|
9
|
+
# needed. A more generic notification of data availability to modules
|
10
|
+
# also may be needed in the future.
|
11
|
+
#
|
12
|
+
# This is not be used to share data with the Ecosystem client
|
13
|
+
#
|
14
|
+
# This class is not thread safe because it can call the lazy getter
|
15
|
+
# twice if [] is called concurrently.
|
16
|
+
class TransactionStorage
|
17
|
+
include Loggable
|
18
|
+
|
19
|
+
class << self
|
20
|
+
# @return [Sqreen::Ecosystem::TransactionStorage]
|
21
|
+
def create_thread_local
|
22
|
+
Thread.current[:tx_storage] = new
|
23
|
+
end
|
24
|
+
|
25
|
+
# @return [Sqreen::Ecosystem::TransactionStorage]
|
26
|
+
def fetch_thread_local
|
27
|
+
Thread.current[:tx_storage]
|
28
|
+
end
|
29
|
+
|
30
|
+
def destroy_thread_local
|
31
|
+
Thread.current[:tx_storage] = nil
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def initialize
|
36
|
+
@values = {}
|
37
|
+
@values_lazy = {}
|
38
|
+
end
|
39
|
+
|
40
|
+
def []=(key, value)
|
41
|
+
@values[key] = value
|
42
|
+
end
|
43
|
+
|
44
|
+
def set_lazy(key, &block)
|
45
|
+
@values_lazy[key] = block
|
46
|
+
end
|
47
|
+
|
48
|
+
def [](key)
|
49
|
+
v = @values[key]
|
50
|
+
return v unless v.nil?
|
51
|
+
|
52
|
+
v = @values_lazy[key]
|
53
|
+
return nil if v.nil?
|
54
|
+
|
55
|
+
begin
|
56
|
+
@values[key] = v.call
|
57
|
+
rescue StandardError => e
|
58
|
+
logger.warn { "Error resolving key #{e} with lazy value: #{e.message}" }
|
59
|
+
raise
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
require 'sqreen/log/loggable'
|
2
|
+
require 'sqreen/ecosystem'
|
3
|
+
require 'sqreen/ecosystem/dispatch_table'
|
4
|
+
require 'sqreen/ecosystem_integration/instrumentation_service'
|
5
|
+
require 'sqreen/ecosystem_integration/request_lifecycle_tracking'
|
6
|
+
require 'sqreen/ecosystem_integration/signal_consumption'
|
7
|
+
|
8
|
+
module Sqreen
|
9
|
+
# This class is the interface through which the agent interacts
|
10
|
+
# with the ecosystem.
|
11
|
+
#
|
12
|
+
# Other classes in the EcosystemIntegration module implement the
|
13
|
+
# functionality that the ecosystem requires in order to deliver
|
14
|
+
# data to the agent and to be informed by the agent of certain
|
15
|
+
# key events (see Sqreen::Ecosystem::DispatchTable).
|
16
|
+
class EcosystemIntegration
|
17
|
+
include Sqreen::Log::Loggable
|
18
|
+
|
19
|
+
# @param [Sqreen::Framework] framework
|
20
|
+
def initialize(framework, queue)
|
21
|
+
@framework = framework
|
22
|
+
@queue = queue
|
23
|
+
@request_lifecycle = RequestLifecycleTracking.new
|
24
|
+
end
|
25
|
+
|
26
|
+
def init
|
27
|
+
setup_dispatch_table
|
28
|
+
Ecosystem.init
|
29
|
+
logger.info 'Ecosystem successfully initialized'
|
30
|
+
rescue ::Exception => e # rubocop:disable Lint/RescueException
|
31
|
+
logger.warn { "Error initializing Ecosystem: #{e.message}" }
|
32
|
+
logger.debug { e.backtrace.map { |x| " #{x}" }.join("\n") }
|
33
|
+
end
|
34
|
+
|
35
|
+
def disable
|
36
|
+
raise NotImplementedYet
|
37
|
+
end
|
38
|
+
|
39
|
+
def request_start(rack_request)
|
40
|
+
Ecosystem.start_transaction
|
41
|
+
@request_lifecycle.notify_request_start(rack_request)
|
42
|
+
end
|
43
|
+
|
44
|
+
def request_end
|
45
|
+
Ecosystem.end_transaction
|
46
|
+
end
|
47
|
+
|
48
|
+
def handle_tracing_command(trace_id_prefix, scopes_config)
|
49
|
+
Ecosystem.configure_sampling(trace_id_prefix, scopes_config)
|
50
|
+
end
|
51
|
+
|
52
|
+
private
|
53
|
+
|
54
|
+
def setup_dispatch_table
|
55
|
+
Ecosystem::DispatchTable.consume_signal =
|
56
|
+
create_signal_consumption.method(:consume_signal)
|
57
|
+
|
58
|
+
Ecosystem::DispatchTable.add_request_start_listener =
|
59
|
+
@request_lifecycle.method(:add_start_observer)
|
60
|
+
|
61
|
+
Ecosystem::DispatchTable.fetch_logger = lambda { logger }
|
62
|
+
|
63
|
+
Ecosystem::DispatchTable.instrument = InstrumentationService.method(:instrument)
|
64
|
+
end
|
65
|
+
|
66
|
+
def create_signal_consumption
|
67
|
+
SignalConsumption.new(@framework, @request_lifecycle, @queue)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
@@ -0,0 +1,89 @@
|
|
1
|
+
require 'sqreen/log/loggable'
|
2
|
+
require 'sqreen/events/remote_exception'
|
3
|
+
require 'sqreen/mono_time'
|
4
|
+
|
5
|
+
module Sqreen
|
6
|
+
class EcosystemIntegration
|
7
|
+
module AroundCallbacks
|
8
|
+
class << self
|
9
|
+
include Log::Loggable::ClassMethods
|
10
|
+
|
11
|
+
# for instrumentation hooks
|
12
|
+
# instrumentation hooks already handle budgets, so nothing
|
13
|
+
# to do in that respect
|
14
|
+
def wrap_instrumentation_hook(module_name, action, callable)
|
15
|
+
perf_notif_name = "ecosystem_#{module_name}"
|
16
|
+
|
17
|
+
Proc.new do |*args|
|
18
|
+
begin
|
19
|
+
start = Sqreen.time
|
20
|
+
callable.call(*args)
|
21
|
+
rescue ::Exception => e # rubocop:disable Lint/RescueException
|
22
|
+
# 2) rescue exceptions
|
23
|
+
logger.warn { "Error in #{module_name}:#{action}: #{e.message}" }
|
24
|
+
logger.debug { e.backtrace.map { |x| " #{x}" }.join("\n") }
|
25
|
+
Sqreen::RemoteException.record(e)
|
26
|
+
ensure
|
27
|
+
# 3) contribute to performance metrics
|
28
|
+
stop = Sqreen.time
|
29
|
+
Sqreen::PerformanceNotifications.notify(perf_notif_name, action, start, stop)
|
30
|
+
end # end proc
|
31
|
+
end # end begin
|
32
|
+
end
|
33
|
+
|
34
|
+
# XXX: not used yet
|
35
|
+
def wrap_generic_callback(module_name, action, callable)
|
36
|
+
timer_name = "ecosystem:#{module_name}@#{action}"
|
37
|
+
perf_notif_name = "ecosystem_#{module_name}"
|
38
|
+
|
39
|
+
Proc.new do |*args|
|
40
|
+
begin
|
41
|
+
req_storage = Thread.current[:sqreen_http_request]
|
42
|
+
|
43
|
+
timer = Graft::Timer.new(timer_name) do |t|
|
44
|
+
# this is an epilogue to measure()
|
45
|
+
req_storage && req_storage[:timed_hooks] << t
|
46
|
+
end
|
47
|
+
|
48
|
+
req_timer = nil
|
49
|
+
timer.measure do
|
50
|
+
# not in a request, no budget; call cb
|
51
|
+
next callable.call(*args) unless req_storage
|
52
|
+
|
53
|
+
# 1) budget enforcement
|
54
|
+
# skip callback if budget already expended
|
55
|
+
next if req_storage[:time_budget_expended]
|
56
|
+
|
57
|
+
budget = req_storage[:time_budget]
|
58
|
+
if budget
|
59
|
+
req_timer = req_storage[:timer]
|
60
|
+
remaining = budget - req_timer.elapsed
|
61
|
+
unless remaining > 0
|
62
|
+
req_storage[:time_budget_expended] = true
|
63
|
+
next # skip callback
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
callable.call(*args)
|
68
|
+
end
|
69
|
+
rescue ::Exception => e # rubocop:disable Lint/RescueException
|
70
|
+
# 2) rescue exceptions
|
71
|
+
logger.warn { "Error in #{module_name}:#{action}: #{e.message}" }
|
72
|
+
logger.debug { e.backtrace.map { |x| " #{x}" }.join("\n") }
|
73
|
+
Sqreen::RemoteException.record(e)
|
74
|
+
ensure
|
75
|
+
# 3) contribute to performance metrics
|
76
|
+
if timer
|
77
|
+
req_timer.include_measurements(timer) if req_timer
|
78
|
+
|
79
|
+
Sqreen::PerformanceNotifications.notify(
|
80
|
+
perf_notif_name, action, *timer.start_and_end
|
81
|
+
)
|
82
|
+
end
|
83
|
+
end # end begin
|
84
|
+
end # end proc
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'sqreen/graft/hook'
|
2
|
+
require 'sqreen/ecosystem_integration/around_callbacks'
|
3
|
+
|
4
|
+
module Sqreen
|
5
|
+
class EcosystemIntegration
|
6
|
+
module InstrumentationService
|
7
|
+
class << self
|
8
|
+
# @param [String] module_name
|
9
|
+
# @param [String] method in form A::B#c or A::B.c
|
10
|
+
# @param [Hash{Symbol=>Proc}] spec
|
11
|
+
def instrument(module_name, method, spec)
|
12
|
+
hook = Sqreen::Graft::Hook[method].add do
|
13
|
+
if spec[:before]
|
14
|
+
cb = AroundCallbacks.wrap_instrumentation_hook(module_name, 'pre', spec[:before])
|
15
|
+
before(nil, flow: true, &cb)
|
16
|
+
end
|
17
|
+
|
18
|
+
if spec[:after]
|
19
|
+
cb = AroundCallbacks.wrap_instrumentation_hook(module_name, 'post', spec[:after])
|
20
|
+
after(nil, flow: true, &cb)
|
21
|
+
end
|
22
|
+
|
23
|
+
if spec[:raised]
|
24
|
+
cb = AroundCallbacks.wrap_instrumentation_hook(module_name, 'failing', spec[:raised])
|
25
|
+
raised(nil, flow: true, &cb)
|
26
|
+
end
|
27
|
+
|
28
|
+
if spec[:ensured]
|
29
|
+
cb = AroundCallbacks.wrap_instrumentation_hook(module_name, 'finally', spec[:ensured])
|
30
|
+
ensured(nil, flow: true, &cb)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
hook.install
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
require 'sqreen/log/loggable'
|
2
|
+
|
3
|
+
module Sqreen
|
4
|
+
class EcosystemIntegration
|
5
|
+
# This class gets notified of request start/end and
|
6
|
+
# 1) distributes such events to listeners (typically ecosystem modules;
|
7
|
+
# the method add_start_observer is exposed to ecosystem modules through
|
8
|
+
# +Sqreen::Ecosystem::ModuleApi::EventListener+ and the dispatch table).
|
9
|
+
# 2) keeps track of whether a request is active on this thread. This is
|
10
|
+
# so that users of this class can have this information without needing
|
11
|
+
# to subscribe to request start/events and keeping thread local state
|
12
|
+
# themselves.
|
13
|
+
# XXX: Since the Ecosystem is also notified of request start/end, it could
|
14
|
+
# notify its modules of request start without going through the dispatch
|
15
|
+
# table and call add_start_observer. We need to think if we want to keep
|
16
|
+
# the transaction / request distinction or if they should just be
|
17
|
+
# assumed to be the same, though.
|
18
|
+
class RequestLifecycleTracking
|
19
|
+
include Sqreen::Log::Loggable
|
20
|
+
|
21
|
+
def initialize
|
22
|
+
@start_observers = []
|
23
|
+
@tl_key = "#{object_id}_req_in_flight"
|
24
|
+
end
|
25
|
+
|
26
|
+
# API for classes needing to know the request state
|
27
|
+
|
28
|
+
# @param cb A callback taking a Rack::Request
|
29
|
+
def add_start_observer(cb)
|
30
|
+
@start_observers << cb
|
31
|
+
end
|
32
|
+
|
33
|
+
def in_request?
|
34
|
+
Thread.current[@tl_key] ? true : false
|
35
|
+
end
|
36
|
+
|
37
|
+
# API for classes notifying this one of request events
|
38
|
+
|
39
|
+
def notify_request_start(rack_req)
|
40
|
+
Thread.current[@tl_key] = true
|
41
|
+
return if @start_observers.empty?
|
42
|
+
@start_observers.each do |cb|
|
43
|
+
begin
|
44
|
+
cb.call(rack_req)
|
45
|
+
rescue ::Exception => e # rubocop:disable Lint/RescueException
|
46
|
+
logger.warn { "Error calling #{cb} on request start: #{e.message}" }
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def notify_request_end
|
52
|
+
Thread.current[@tl_key] = false
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
require 'sqreen/log/loggable'
|
2
|
+
|
3
|
+
module Sqreen
|
4
|
+
class EcosystemIntegration
|
5
|
+
class SignalConsumption
|
6
|
+
include Sqreen::Log::Loggable
|
7
|
+
|
8
|
+
PAYLOAD_CREATOR_SECTIONS = %w[request response params headers].freeze
|
9
|
+
|
10
|
+
# @param [Sqreen::Frameworks::GenericFramework] framework
|
11
|
+
# @param [Sqreen::EcosystemIntegration::RequestLifecycleTracking]
|
12
|
+
# @param [Sqreen::CappedQueue]
|
13
|
+
def initialize(framework, req_lifecycle, queue)
|
14
|
+
@framework = framework
|
15
|
+
@req_lifecycle = req_lifecycle
|
16
|
+
@queue = queue
|
17
|
+
end
|
18
|
+
|
19
|
+
def consume_signal(signal)
|
20
|
+
# transitional
|
21
|
+
unless Sqreen.features.fetch('use_signals', DEFAULT_USE_SIGNALS)
|
22
|
+
logger.debug { "Discarding signal #{signal} (signals disabled)" }
|
23
|
+
return
|
24
|
+
end
|
25
|
+
|
26
|
+
if @req_lifecycle.in_request?
|
27
|
+
# add it to the request record
|
28
|
+
@framework.observe(:signals, signal, PAYLOAD_CREATOR_SECTIONS, true)
|
29
|
+
else
|
30
|
+
@queue.push signal
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -22,8 +22,17 @@ module Sqreen
|
|
22
22
|
include RequestRecorder
|
23
23
|
attr_accessor :sqreen_configuration
|
24
24
|
|
25
|
+
attr_writer :req_start_cb, :req_end_cb
|
26
|
+
|
25
27
|
def initialize
|
26
28
|
clean_request_record
|
29
|
+
|
30
|
+
# for notifying the ecosystem of request boundaries
|
31
|
+
# XXX: this should be refactored. It shouldn't be
|
32
|
+
# the framework doing these notifications to the ecosystem
|
33
|
+
# Probably the rule callback should do it itself
|
34
|
+
@req_start_cb = Proc.new {}
|
35
|
+
@req_end_cb = Proc.new {}
|
27
36
|
end
|
28
37
|
|
29
38
|
# What kind of database is this
|
@@ -251,8 +260,12 @@ module Sqreen
|
|
251
260
|
# Nota: cleanup should be performed at end of request (see clean_request)
|
252
261
|
def store_request(object)
|
253
262
|
return unless ensure_rack_loaded
|
263
|
+
|
264
|
+
rack_req = Rack::Request.new(object)
|
265
|
+
@req_start_cb.call(rack_req)
|
266
|
+
|
254
267
|
self.remaining_perf_budget = Sqreen.performance_budget
|
255
|
-
SharedStorage.set(:request,
|
268
|
+
SharedStorage.set(:request, rack_req)
|
256
269
|
SharedStorage.set(:xss_params, nil)
|
257
270
|
SharedStorage.set(:whitelisted, nil)
|
258
271
|
SharedStorage.set(:request_overtime, nil)
|
@@ -281,6 +294,7 @@ module Sqreen
|
|
281
294
|
SharedStorage.set(:xss_params, nil)
|
282
295
|
SharedStorage.set(:whitelisted, nil)
|
283
296
|
SharedStorage.set(:request_overtime, nil)
|
297
|
+
@req_end_cb.call
|
284
298
|
end
|
285
299
|
|
286
300
|
def remaining_perf_budget
|
data/lib/sqreen/graft/call.rb
CHANGED
@@ -138,6 +138,15 @@ module Sqreen
|
|
138
138
|
@blips << Timer.read
|
139
139
|
end
|
140
140
|
|
141
|
+
def include_measurements(another_timer)
|
142
|
+
@blips += another_timer.instance_variable_get(:@blips)
|
143
|
+
end
|
144
|
+
|
145
|
+
def start_and_end
|
146
|
+
raise 'Not exactly two measurements recorded' unless size == 2
|
147
|
+
@blips
|
148
|
+
end
|
149
|
+
|
141
150
|
def size
|
142
151
|
@blips.size
|
143
152
|
end
|