sqreen 1.20.4 → 1.21.0.beta1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +0 -25
  3. data/lib/sqreen/actions/block_user.rb +1 -1
  4. data/lib/sqreen/actions/redirect_ip.rb +1 -1
  5. data/lib/sqreen/actions/redirect_user.rb +1 -1
  6. data/lib/sqreen/condition_evaluator.rb +2 -8
  7. data/lib/sqreen/configuration.rb +1 -1
  8. data/lib/sqreen/deferred_logger.rb +14 -50
  9. data/lib/sqreen/deliveries/batch.rb +8 -1
  10. data/lib/sqreen/ecosystem.rb +80 -0
  11. data/lib/sqreen/ecosystem/dispatch_table.rb +43 -0
  12. data/lib/sqreen/ecosystem/http/net_http.rb +51 -0
  13. data/lib/sqreen/ecosystem/http/rack_request.rb +38 -0
  14. data/lib/sqreen/ecosystem/loggable.rb +13 -0
  15. data/lib/sqreen/ecosystem/module_api.rb +30 -0
  16. data/lib/sqreen/ecosystem/module_api/event_listener.rb +18 -0
  17. data/lib/sqreen/ecosystem/module_api/instrumentation.rb +23 -0
  18. data/lib/sqreen/ecosystem/module_api/signal_producer.rb +26 -0
  19. data/lib/sqreen/ecosystem/module_api/tracing_push_down.rb +34 -0
  20. data/lib/sqreen/ecosystem/module_api/transaction_storage.rb +71 -0
  21. data/lib/sqreen/ecosystem/module_registry.rb +39 -0
  22. data/lib/sqreen/ecosystem/redis/redis_connection.rb +35 -0
  23. data/lib/sqreen/ecosystem/tracing/sampler.rb +160 -0
  24. data/lib/sqreen/ecosystem/tracing/sampling_configuration.rb +150 -0
  25. data/lib/sqreen/ecosystem/tracing/signals/tracing_client.rb +53 -0
  26. data/lib/sqreen/ecosystem/tracing/signals/tracing_server.rb +53 -0
  27. data/lib/sqreen/ecosystem/tracing_id_setup.rb +34 -0
  28. data/lib/sqreen/ecosystem/transaction_storage.rb +64 -0
  29. data/lib/sqreen/ecosystem_integration.rb +70 -0
  30. data/lib/sqreen/ecosystem_integration/around_callbacks.rb +89 -0
  31. data/lib/sqreen/ecosystem_integration/instrumentation_service.rb +38 -0
  32. data/lib/sqreen/ecosystem_integration/request_lifecycle_tracking.rb +56 -0
  33. data/lib/sqreen/ecosystem_integration/signal_consumption.rb +35 -0
  34. data/lib/sqreen/encoding_sanitizer.rb +27 -0
  35. data/lib/sqreen/events/request_record.rb +1 -0
  36. data/lib/sqreen/frameworks/generic.rb +15 -10
  37. data/lib/sqreen/frameworks/rails.rb +7 -0
  38. data/lib/sqreen/frameworks/request_recorder.rb +0 -2
  39. data/lib/sqreen/graft/call.rb +23 -72
  40. data/lib/sqreen/graft/callback.rb +1 -1
  41. data/lib/sqreen/graft/hook.rb +85 -187
  42. data/lib/sqreen/graft/hook_point.rb +1 -1
  43. data/lib/sqreen/legacy/instrumentation.rb +10 -22
  44. data/lib/sqreen/legacy/old_event_submission_strategy.rb +8 -3
  45. data/lib/sqreen/log.rb +2 -3
  46. data/lib/sqreen/log/loggable.rb +0 -1
  47. data/lib/sqreen/logger.rb +0 -24
  48. data/lib/sqreen/metrics_store.rb +0 -11
  49. data/lib/sqreen/null_logger.rb +0 -22
  50. data/lib/sqreen/remote_command.rb +3 -1
  51. data/lib/sqreen/rules.rb +4 -8
  52. data/lib/sqreen/rules/blacklist_ips_cb.rb +2 -2
  53. data/lib/sqreen/rules/custom_error_cb.rb +3 -3
  54. data/lib/sqreen/rules/rule_cb.rb +0 -2
  55. data/lib/sqreen/rules/waf_cb.rb +3 -3
  56. data/lib/sqreen/runner.rb +21 -33
  57. data/lib/sqreen/session.rb +2 -0
  58. data/lib/sqreen/signals/conversions.rb +6 -1
  59. data/lib/sqreen/version.rb +1 -1
  60. data/lib/sqreen/weave/legacy/instrumentation.rb +103 -194
  61. data/lib/sqreen/worker.rb +2 -6
  62. metadata +35 -10
  63. data/lib/sqreen/deprecation.rb +0 -38
  64. data/lib/sqreen/weave/budget.rb +0 -46
@@ -0,0 +1,150 @@
1
+ require 'thread'
2
+ require 'sqreen/ecosystem/loggable'
3
+ require 'sqreen/ecosystem/tracing/sampler'
4
+
5
+ module Sqreen
6
+ module Ecosystem
7
+ module Tracing
8
+ # tracing sampling configuration, as specified by the 2nd argument of the
9
+ # tracing_enable command.
10
+ # See https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000025-enabling-tracing.md
11
+ class SamplingConfiguration
12
+ include Loggable
13
+
14
+ DEFAULT_SCOPE = '*'.freeze
15
+
16
+ def initialize(sampling_config)
17
+ @sampling_config = sampling_config
18
+ @samplers = {}
19
+ @samplers_virtual = build_virtual_holders(sampling_config)
20
+ end
21
+
22
+ # @param [String] scope either the scope or the part behind @ in a virtual scope
23
+ # @param [String] qualifier the part after @ in a virtual scope or nil
24
+ def should_sample?(scope, qualifier = nil)
25
+ if qualifier
26
+ fetch_sampler_virtual(scope, qualifier).should_sample?
27
+ else
28
+ fetch_sampler(scope).should_sample?
29
+ end
30
+ end
31
+
32
+ def forget_virtual_scope(scope, qualifier)
33
+ holder = @samplers_virtual[scope]
34
+ return unless holder.delete!(qualifier)
35
+ end
36
+
37
+ private
38
+
39
+ def fetch_sampler_virtual(scope, qualifier)
40
+ holder = @samplers_virtual[scope]
41
+
42
+ # no virtual scope configured, fallback to plain scope
43
+ return fetch_sampler(scope) unless holder
44
+
45
+ holder[qualifier]
46
+ end
47
+
48
+ def fetch_sampler(scope)
49
+ sampler = @samplers[scope]
50
+ return sampler if sampler
51
+
52
+ cfg = @sampling_config[scope] || @sampling_config[DEFAULT_SCOPE]
53
+ if cfg
54
+ @samplers[scope] = SamplingConfiguration.build_sampler(scope, cfg)
55
+ else
56
+ logger.info { "Disabling scope #{scope} due to its not being configured" }
57
+ @samplers[scope] = DisabledScopeSampler
58
+ end
59
+ end
60
+
61
+ def build_virtual_holders(sampling_config)
62
+ Hash[
63
+ sampling_config
64
+ .select { |scope, _cfg| scope.end_with?('@*') }
65
+ .map do |scope, cfg|
66
+ parent = scope[0...-2] # remove trailing @*
67
+ [parent, VirtualScopesHolder.new(parent, cfg)]
68
+ end
69
+ ]
70
+ end
71
+
72
+ class << self
73
+ include Loggable
74
+
75
+ def build_sampler(scope, cfg)
76
+ do_build_sampler(cfg['enabled'], cfg['sampling'] || [{}])
77
+ rescue StandardError => e
78
+ logger.warn "Invalid sampling configuration for #{scope}: #{e.inspect}"
79
+ logger.debug { e.backtrace }
80
+ DisabledScopeSampler
81
+ end
82
+
83
+ private
84
+
85
+ def do_build_sampler(enabled, sampling)
86
+ if enabled
87
+ Sampler.new(sampling)
88
+ else
89
+ logger.debug { "Disabling scope #{scope}" }
90
+ DisabledScopeSampler
91
+ end
92
+ end
93
+ end
94
+ end
95
+
96
+ class VirtualScopesHolder
97
+ include Loggable
98
+
99
+ MAX_VIRTUAL_SCOPES = 120
100
+ DISCARD_SIZE = 20
101
+
102
+ def initialize(parent, cfg)
103
+ @parent = parent
104
+ @cfg = cfg
105
+ @virtual_scopes = {}
106
+ @mutex = Mutex.new
107
+ end
108
+
109
+ def [](qualifier)
110
+ return false unless @cfg['enabled']
111
+
112
+ @mutex.synchronize do
113
+ sampler = @virtual_scopes[qualifier]
114
+ return sampler if sampler
115
+
116
+ @virtual_scopes[qualifier] = create_virtual(qualifier)
117
+ end
118
+ end
119
+
120
+ def delete!(qualifier)
121
+ @virtual_scopes.delete(qualifier)
122
+ end
123
+
124
+ private
125
+
126
+ def create_virtual(qualifier)
127
+ discard if @virtual_scopes.size >= MAX_VIRTUAL_SCOPES
128
+
129
+ @virtual_scopes[qualifier] =
130
+ SamplingConfiguration.build_sampler("#{@parent}@#{qualifier}", @cfg)
131
+ end
132
+
133
+ def discard
134
+ logger.info { "Discarding excess virtual scopes for scope '#{@parent}'" }
135
+ discard_keys = @virtual_scopes.keys.sample(DISCARD_SIZE)
136
+ @virtual_scopes.delete_if { |k, _v| discard_keys.include? k }
137
+ end
138
+ end
139
+
140
+ # fake sampler that always returns false
141
+ class DisabledScopeSampler
142
+ class << self
143
+ def should_sample?
144
+ false
145
+ end
146
+ end
147
+ end
148
+ end
149
+ end
150
+ end
@@ -0,0 +1,53 @@
1
+ require 'sqreen/kit/configuration'
2
+ require 'sqreen/kit/signals/point'
3
+ require 'sqreen/kit/signals/dto_helper'
4
+
5
+ # reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/payload/tracing/client-2020-04-21/schema.cue
6
+
7
+ module Sqreen
8
+ module Ecosystem
9
+ module Tracing
10
+ module Signals
11
+ end
12
+ end
13
+ end
14
+ end
15
+
16
+ class Sqreen::Ecosystem::Tracing::Signals::TracingClient < Sqreen::Kit::Signals::Point
17
+ readonly_attrs :payload_schema, :source, :signal_name
18
+
19
+ def initialize(values = {})
20
+ self.payload_schema = Payload::SCHEMA_VERSION
21
+ self.source = Sqreen::Kit::Configuration.default_source
22
+ self.signal_name = 'tracing.client'
23
+ self.time = values[:time] || Time.now
24
+ super
25
+ end
26
+
27
+ def payload=(payload)
28
+ unless payload.is_a?(Payload)
29
+ raise ArgumentError, "Payload should be a #{Payload}"
30
+ end
31
+ super
32
+ end
33
+
34
+ class Payload
35
+ include Sqreen::Kit::Signals::DtoHelper
36
+
37
+ add_mandatory_attrs :transport, :host
38
+
39
+ SCHEMA_VERSION = 'tracing/client-2020-04-21'.freeze
40
+
41
+ # @return [Symbol]
42
+ attr_accessor :transport
43
+
44
+ # @return [String]
45
+ attr_accessor :host
46
+
47
+ # @return [String]
48
+ attr_accessor :ip
49
+
50
+ # @return [String]
51
+ attr_accessor :tracing_identifier
52
+ end
53
+ end
@@ -0,0 +1,53 @@
1
+ require 'sqreen/kit/configuration'
2
+ require 'sqreen/kit/signals/point'
3
+ require 'sqreen/kit/signals/dto_helper'
4
+
5
+ # reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/payload/tracing/server-2020-04-21/schema.cue
6
+
7
+ module Sqreen
8
+ module Ecosystem
9
+ module Tracing
10
+ module Signals
11
+ end
12
+ end
13
+ end
14
+ end
15
+
16
+ class Sqreen::Ecosystem::Tracing::Signals::TracingServer < Sqreen::Kit::Signals::Point
17
+ readonly_attrs :payload_schema, :source, :signal_name
18
+
19
+ def initialize(values = {})
20
+ self.payload_schema = Payload::SCHEMA_VERSION
21
+ self.source = Sqreen::Kit::Configuration.default_source
22
+ self.signal_name = 'tracing.server'
23
+ self.time = values[:time] || Time.now
24
+ super
25
+ end
26
+
27
+ def payload=(payload)
28
+ unless payload.is_a?(Payload)
29
+ raise ArgumentError, "Payload should be a #{Payload}"
30
+ end
31
+ super
32
+ end
33
+
34
+ class Payload
35
+ include Sqreen::Kit::Signals::DtoHelper
36
+
37
+ add_mandatory_attrs :transport, :client_ip
38
+
39
+ SCHEMA_VERSION = 'tracing/server-2020-04-21'.freeze
40
+
41
+ # @return [Symbol]
42
+ attr_accessor :transport
43
+
44
+ # @return [String]
45
+ attr_accessor :client_ip
46
+
47
+ # @return [Array<String>]
48
+ attr_accessor :previous_hops
49
+
50
+ # @return [String]
51
+ attr_accessor :tracing_identifier
52
+ end
53
+ end
@@ -0,0 +1,34 @@
1
+ require 'sqreen/ecosystem/module_registry'
2
+ require 'sqreen/ecosystem/transaction_storage'
3
+ require 'sqreen/ecosystem/module_api/signal_producer'
4
+
5
+ module Sqreen
6
+ module Ecosystem
7
+ class TracingIdSetup
8
+ # @param [Sqreen::Ecosystem::ModuleRegistry] registry
9
+ def initialize(registry)
10
+ @registry = registry
11
+ @tracing_id_prefix = nil
12
+ end
13
+
14
+ def setup_modules
15
+ inject_out_of_tx_tracing_id_gen
16
+ end
17
+
18
+ attr_writer :tracing_id_prefix
19
+
20
+ private
21
+
22
+ def inject_out_of_tx_tracing_id_gen
23
+ @registry.each_module(Sqreen::Ecosystem::ModuleApi::SignalProducer) do |mod|
24
+ mod.tracing_id_producer = method(:generate_tracing_id)
25
+ end
26
+ end
27
+
28
+ def generate_tracing_id
29
+ return nil unless @tracing_id_prefix
30
+ "#{@tracing_id_prefix}.#{SecureRandom.uuid}"
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,64 @@
1
+ require 'sqreen/ecosystem/loggable'
2
+
3
+ module Sqreen
4
+ module Ecosystem
5
+ # The transaction storage is a mechanism for the modules to share
6
+ # request-scoped data with each other or to keep request-scoped objects
7
+ # themselves, although, for this last use case to be effective,
8
+ # propagation of request start/end events to the modules will likely be
9
+ # needed. A more generic notification of data availability to modules
10
+ # also may be needed in the future.
11
+ #
12
+ # This is not be used to share data with the Ecosystem client
13
+ #
14
+ # This class is not thread safe because it can call the lazy getter
15
+ # twice if [] is called concurrently.
16
+ class TransactionStorage
17
+ include Loggable
18
+
19
+ class << self
20
+ # @return [Sqreen::Ecosystem::TransactionStorage]
21
+ def create_thread_local
22
+ Thread.current[:tx_storage] = new
23
+ end
24
+
25
+ # @return [Sqreen::Ecosystem::TransactionStorage]
26
+ def fetch_thread_local
27
+ Thread.current[:tx_storage]
28
+ end
29
+
30
+ def destroy_thread_local
31
+ Thread.current[:tx_storage] = nil
32
+ end
33
+ end
34
+
35
+ def initialize
36
+ @values = {}
37
+ @values_lazy = {}
38
+ end
39
+
40
+ def []=(key, value)
41
+ @values[key] = value
42
+ end
43
+
44
+ def set_lazy(key, &block)
45
+ @values_lazy[key] = block
46
+ end
47
+
48
+ def [](key)
49
+ v = @values[key]
50
+ return v unless v.nil?
51
+
52
+ v = @values_lazy[key]
53
+ return nil if v.nil?
54
+
55
+ begin
56
+ @values[key] = v.call
57
+ rescue StandardError => e
58
+ logger.warn { "Error resolving key #{e} with lazy value: #{e.message}" }
59
+ raise
60
+ end
61
+ end
62
+ end
63
+ end
64
+ end
@@ -0,0 +1,70 @@
1
+ require 'sqreen/log/loggable'
2
+ require 'sqreen/ecosystem'
3
+ require 'sqreen/ecosystem/dispatch_table'
4
+ require 'sqreen/ecosystem_integration/instrumentation_service'
5
+ require 'sqreen/ecosystem_integration/request_lifecycle_tracking'
6
+ require 'sqreen/ecosystem_integration/signal_consumption'
7
+
8
+ module Sqreen
9
+ # This class is the interface through which the agent interacts
10
+ # with the ecosystem.
11
+ #
12
+ # Other classes in the EcosystemIntegration module implement the
13
+ # functionality that the ecosystem requires in order to deliver
14
+ # data to the agent and to be informed by the agent of certain
15
+ # key events (see Sqreen::Ecosystem::DispatchTable).
16
+ class EcosystemIntegration
17
+ include Sqreen::Log::Loggable
18
+
19
+ # @param [Sqreen::Framework] framework
20
+ def initialize(framework, queue)
21
+ @framework = framework
22
+ @queue = queue
23
+ @request_lifecycle = RequestLifecycleTracking.new
24
+ end
25
+
26
+ def init
27
+ setup_dispatch_table
28
+ Ecosystem.init
29
+ logger.info 'Ecosystem successfully initialized'
30
+ rescue ::Exception => e # rubocop:disable Lint/RescueException
31
+ logger.warn { "Error initializing Ecosystem: #{e.message}" }
32
+ logger.debug { e.backtrace.map { |x| " #{x}" }.join("\n") }
33
+ end
34
+
35
+ def disable
36
+ raise NotImplementedYet
37
+ end
38
+
39
+ def request_start(rack_request)
40
+ Ecosystem.start_transaction
41
+ @request_lifecycle.notify_request_start(rack_request)
42
+ end
43
+
44
+ def request_end
45
+ Ecosystem.end_transaction
46
+ end
47
+
48
+ def handle_tracing_command(trace_id_prefix, scopes_config)
49
+ Ecosystem.configure_sampling(trace_id_prefix, scopes_config)
50
+ end
51
+
52
+ private
53
+
54
+ def setup_dispatch_table
55
+ Ecosystem::DispatchTable.consume_signal =
56
+ create_signal_consumption.method(:consume_signal)
57
+
58
+ Ecosystem::DispatchTable.add_request_start_listener =
59
+ @request_lifecycle.method(:add_start_observer)
60
+
61
+ Ecosystem::DispatchTable.fetch_logger = lambda { logger }
62
+
63
+ Ecosystem::DispatchTable.instrument = InstrumentationService.method(:instrument)
64
+ end
65
+
66
+ def create_signal_consumption
67
+ SignalConsumption.new(@framework, @request_lifecycle, @queue)
68
+ end
69
+ end
70
+ end
@@ -0,0 +1,89 @@
1
+ require 'sqreen/log/loggable'
2
+ require 'sqreen/events/remote_exception'
3
+ require 'sqreen/mono_time'
4
+
5
+ module Sqreen
6
+ class EcosystemIntegration
7
+ module AroundCallbacks
8
+ class << self
9
+ include Log::Loggable::ClassMethods
10
+
11
+ # for instrumentation hooks
12
+ # instrumentation hooks already handle budgets, so nothing
13
+ # to do in that respect
14
+ def wrap_instrumentation_hook(module_name, action, callable)
15
+ perf_notif_name = "ecosystem_#{module_name}"
16
+
17
+ Proc.new do |*args|
18
+ begin
19
+ start = Sqreen.time
20
+ callable.call(*args)
21
+ rescue ::Exception => e # rubocop:disable Lint/RescueException
22
+ # 2) rescue exceptions
23
+ logger.warn { "Error in #{module_name}:#{action}: #{e.message}" }
24
+ logger.debug { e.backtrace.map { |x| " #{x}" }.join("\n") }
25
+ Sqreen::RemoteException.record(e)
26
+ ensure
27
+ # 3) contribute to performance metrics
28
+ stop = Sqreen.time
29
+ Sqreen::PerformanceNotifications.notify(perf_notif_name, action, start, stop)
30
+ end # end proc
31
+ end # end begin
32
+ end
33
+
34
+ # XXX: not used yet
35
+ def wrap_generic_callback(module_name, action, callable)
36
+ timer_name = "ecosystem:#{module_name}@#{action}"
37
+ perf_notif_name = "ecosystem_#{module_name}"
38
+
39
+ Proc.new do |*args|
40
+ begin
41
+ req_storage = Thread.current[:sqreen_http_request]
42
+
43
+ timer = Graft::Timer.new(timer_name) do |t|
44
+ # this is an epilogue to measure()
45
+ req_storage && req_storage[:timed_hooks] << t
46
+ end
47
+
48
+ req_timer = nil
49
+ timer.measure do
50
+ # not in a request, no budget; call cb
51
+ next callable.call(*args) unless req_storage
52
+
53
+ # 1) budget enforcement
54
+ # skip callback if budget already expended
55
+ next if req_storage[:time_budget_expended]
56
+
57
+ budget = req_storage[:time_budget]
58
+ if budget
59
+ req_timer = req_storage[:timer]
60
+ remaining = budget - req_timer.elapsed
61
+ unless remaining > 0
62
+ req_storage[:time_budget_expended] = true
63
+ next # skip callback
64
+ end
65
+ end
66
+
67
+ callable.call(*args)
68
+ end
69
+ rescue ::Exception => e # rubocop:disable Lint/RescueException
70
+ # 2) rescue exceptions
71
+ logger.warn { "Error in #{module_name}:#{action}: #{e.message}" }
72
+ logger.debug { e.backtrace.map { |x| " #{x}" }.join("\n") }
73
+ Sqreen::RemoteException.record(e)
74
+ ensure
75
+ # 3) contribute to performance metrics
76
+ if timer
77
+ req_timer.include_measurements(timer) if req_timer
78
+
79
+ Sqreen::PerformanceNotifications.notify(
80
+ perf_notif_name, action, *timer.start_and_end
81
+ )
82
+ end
83
+ end # end begin
84
+ end # end proc
85
+ end
86
+ end
87
+ end
88
+ end
89
+ end