sqreen 1.19.1-java → 1.21.0.beta3-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +34 -0
  3. data/lib/sqreen/actions/block_user.rb +1 -1
  4. data/lib/sqreen/actions/redirect_ip.rb +1 -1
  5. data/lib/sqreen/actions/redirect_user.rb +1 -1
  6. data/lib/sqreen/agent_message.rb +20 -0
  7. data/lib/sqreen/aggregated_metric.rb +25 -0
  8. data/lib/sqreen/attack_detected.html +1 -2
  9. data/lib/sqreen/ca.crt +24 -0
  10. data/lib/sqreen/condition_evaluator.rb +9 -2
  11. data/lib/sqreen/conditionable.rb +24 -6
  12. data/lib/sqreen/configuration.rb +11 -5
  13. data/lib/sqreen/deferred_logger.rb +50 -14
  14. data/lib/sqreen/deliveries/batch.rb +12 -2
  15. data/lib/sqreen/deliveries/simple.rb +4 -0
  16. data/lib/sqreen/deprecation.rb +38 -0
  17. data/lib/sqreen/ecosystem.rb +96 -0
  18. data/lib/sqreen/ecosystem/dispatch_table.rb +43 -0
  19. data/lib/sqreen/ecosystem/exception_reporting.rb +26 -0
  20. data/lib/sqreen/ecosystem/http/net_http.rb +50 -0
  21. data/lib/sqreen/ecosystem/http/rack_request.rb +39 -0
  22. data/lib/sqreen/ecosystem/loggable.rb +13 -0
  23. data/lib/sqreen/ecosystem/module_api.rb +30 -0
  24. data/lib/sqreen/ecosystem/module_api/event_listener.rb +18 -0
  25. data/lib/sqreen/ecosystem/module_api/instrumentation.rb +23 -0
  26. data/lib/sqreen/ecosystem/module_api/message_producer.rb +51 -0
  27. data/lib/sqreen/ecosystem/module_api/signal_producer.rb +24 -0
  28. data/lib/sqreen/ecosystem/module_api/tracing.rb +45 -0
  29. data/lib/sqreen/ecosystem/module_api/tracing/client_data.rb +31 -0
  30. data/lib/sqreen/ecosystem/module_api/tracing/server_data.rb +27 -0
  31. data/lib/sqreen/ecosystem/module_api/tracing_id_generation.rb +16 -0
  32. data/lib/sqreen/ecosystem/module_api/transaction_storage.rb +71 -0
  33. data/lib/sqreen/ecosystem/module_registry.rb +44 -0
  34. data/lib/sqreen/ecosystem/redis/redis_connection.rb +43 -0
  35. data/lib/sqreen/ecosystem/tracing/modules/client.rb +31 -0
  36. data/lib/sqreen/ecosystem/tracing/modules/server.rb +30 -0
  37. data/lib/sqreen/ecosystem/tracing/sampler.rb +160 -0
  38. data/lib/sqreen/ecosystem/tracing/sampling_configuration.rb +150 -0
  39. data/lib/sqreen/ecosystem/tracing/signals/tracing_client.rb +53 -0
  40. data/lib/sqreen/ecosystem/tracing/signals/tracing_server.rb +53 -0
  41. data/lib/sqreen/ecosystem/tracing_broker.rb +101 -0
  42. data/lib/sqreen/ecosystem/tracing_id_setup.rb +34 -0
  43. data/lib/sqreen/ecosystem/transaction_storage.rb +64 -0
  44. data/lib/sqreen/ecosystem/util/call_writers_from_init.rb +13 -0
  45. data/lib/sqreen/ecosystem_integration.rb +87 -0
  46. data/lib/sqreen/ecosystem_integration/around_callbacks.rb +99 -0
  47. data/lib/sqreen/ecosystem_integration/instrumentation_service.rb +42 -0
  48. data/lib/sqreen/ecosystem_integration/request_lifecycle_tracking.rb +58 -0
  49. data/lib/sqreen/ecosystem_integration/signal_consumption.rb +35 -0
  50. data/lib/sqreen/endpoint_testing.rb +184 -0
  51. data/lib/sqreen/event.rb +7 -5
  52. data/lib/sqreen/events/attack.rb +23 -18
  53. data/lib/sqreen/events/remote_exception.rb +0 -22
  54. data/lib/sqreen/events/request_record.rb +15 -71
  55. data/lib/sqreen/frameworks/generic.rb +24 -1
  56. data/lib/sqreen/frameworks/rails.rb +0 -7
  57. data/lib/sqreen/frameworks/request_recorder.rb +15 -2
  58. data/lib/sqreen/graft/call.rb +106 -19
  59. data/lib/sqreen/graft/callback.rb +1 -1
  60. data/lib/sqreen/graft/hook.rb +212 -100
  61. data/lib/sqreen/graft/hook_point.rb +18 -11
  62. data/lib/sqreen/kit/signals/specialized/aggregated_metric.rb +72 -0
  63. data/lib/sqreen/kit/signals/specialized/attack.rb +57 -0
  64. data/lib/sqreen/kit/signals/specialized/binning_metric.rb +76 -0
  65. data/lib/sqreen/kit/signals/specialized/http_trace.rb +26 -0
  66. data/lib/sqreen/kit/signals/specialized/sdk_track_call.rb +50 -0
  67. data/lib/sqreen/kit/signals/specialized/sqreen_exception.rb +57 -0
  68. data/lib/sqreen/legacy/instrumentation.rb +22 -10
  69. data/lib/sqreen/legacy/old_event_submission_strategy.rb +228 -0
  70. data/lib/sqreen/legacy/waf_redactions.rb +49 -0
  71. data/lib/sqreen/log.rb +3 -2
  72. data/lib/sqreen/log/loggable.rb +2 -1
  73. data/lib/sqreen/logger.rb +24 -0
  74. data/lib/sqreen/metrics.rb +1 -0
  75. data/lib/sqreen/metrics/base.rb +3 -0
  76. data/lib/sqreen/metrics/req_detailed.rb +41 -0
  77. data/lib/sqreen/metrics_store.rb +33 -12
  78. data/lib/sqreen/null_logger.rb +22 -0
  79. data/lib/sqreen/performance_notifications/binned_metrics.rb +8 -2
  80. data/lib/sqreen/remote_command.rb +4 -0
  81. data/lib/sqreen/rules.rb +12 -6
  82. data/lib/sqreen/rules/blacklist_ips_cb.rb +2 -2
  83. data/lib/sqreen/rules/custom_error_cb.rb +3 -3
  84. data/lib/sqreen/rules/not_found_cb.rb +2 -0
  85. data/lib/sqreen/rules/rule_cb.rb +6 -2
  86. data/lib/sqreen/rules/waf_cb.rb +16 -13
  87. data/lib/sqreen/runner.rb +138 -16
  88. data/lib/sqreen/sensitive_data_redactor.rb +19 -31
  89. data/lib/sqreen/session.rb +53 -43
  90. data/lib/sqreen/signals/conversions.rb +288 -0
  91. data/lib/sqreen/signals/http_trace_redaction.rb +111 -0
  92. data/lib/sqreen/signals/signals_submission_strategy.rb +78 -0
  93. data/lib/sqreen/version.rb +1 -1
  94. data/lib/sqreen/weave/budget.rb +35 -0
  95. data/lib/sqreen/weave/legacy/instrumentation.rb +277 -135
  96. data/lib/sqreen/worker.rb +6 -2
  97. metadata +86 -10
  98. data/lib/sqreen/backport.rb +0 -9
  99. data/lib/sqreen/backport/clock_gettime.rb +0 -74
  100. data/lib/sqreen/backport/original_name.rb +0 -88
  101. data/lib/sqreen/encoding_sanitizer.rb +0 -27
@@ -0,0 +1,45 @@
1
+ require 'sqreen/ecosystem/module_api/signal_producer'
2
+
3
+ module Sqreen
4
+ module Ecosystem
5
+ module ModuleApi
6
+ module Tracing
7
+ include SignalProducer
8
+
9
+ def self.included(mod)
10
+ mod.extend(ClassMethods)
11
+ end
12
+
13
+ module ClassMethods
14
+ private
15
+
16
+ # @param [Module] type The type the including module is interested in
17
+ def consumes(type)
18
+ @consumes = type
19
+ end
20
+
21
+ # A fixed (non-virtual) scope for this tracing module
22
+ # @param [String] scope
23
+ def fixed_scope(scope)
24
+ @fixed_scope = scope
25
+ end
26
+ end
27
+
28
+ def consumed_type
29
+ self.class.instance_variable_get(:@consumes) \
30
+ || raise('@consumes not specified')
31
+ end
32
+
33
+ def scope(_hints = {})
34
+ self.class.instance_variable_get(:@fixed_scope) \
35
+ || raise('@fixed_scope not set')
36
+ end
37
+
38
+ # including class must implement it
39
+ def receive(_data)
40
+ raise NotImplementedError
41
+ end
42
+ end
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,31 @@
1
+ require 'sqreen/ecosystem/util/call_writers_from_init'
2
+
3
+ module Sqreen
4
+ module Ecosystem
5
+ module ModuleApi
6
+ module Tracing
7
+ # The data the tracing module needs in order to populate
8
+ # +Sqreen::Ecosystem::Tracing::Signals::TracingClient::Payload+
9
+ #
10
+ # Signals are not produced by the data producers (transport)
11
+ # because of superior orders, as the only current use of this
12
+ # data is to generate signals.
13
+ module ClientData
14
+ include Util::CallWritersFromInit
15
+
16
+ # @return [Symbol]
17
+ attr_accessor :transport
18
+
19
+ # @return [String]
20
+ attr_accessor :host
21
+
22
+ # @return [String]
23
+ attr_accessor :ip
24
+
25
+ # @return [String]
26
+ attr_accessor :tracing_identifier
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,27 @@
1
+ require 'sqreen/ecosystem/util/call_writers_from_init'
2
+
3
+ module Sqreen
4
+ module Ecosystem
5
+ module ModuleApi
6
+ module Tracing
7
+ # The data the tracing module needs in order to populate
8
+ # +Sqreen::Ecosystem::Tracing::Signals::TracingServer::Payload+
9
+ module ServerData
10
+ include Util::CallWritersFromInit
11
+
12
+ # @return [Symbol]
13
+ attr_accessor :transport
14
+
15
+ # @return [String]
16
+ attr_accessor :client_ip
17
+
18
+ # @return [Array<String>]
19
+ attr_accessor :previous_hops
20
+
21
+ # @return [String]
22
+ attr_accessor :tracing_identifier
23
+ end
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,16 @@
1
+ module Sqreen
2
+ module Ecosystem
3
+ module ModuleApi
4
+ module TracingIdGeneration
5
+ # for injection
6
+ attr_writer :tracing_id_producer
7
+
8
+ private
9
+
10
+ def create_tracing_id
11
+ @tracing_id_producer.call
12
+ end
13
+ end
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,71 @@
1
+ require 'sqreen/ecosystem/transaction_storage'
2
+ require 'sqreen/ecosystem/loggable'
3
+
4
+ module Sqreen
5
+ module Ecosystem
6
+ module ModuleApi
7
+ module TransactionStorage
8
+ class TxLocalVariables
9
+ class << self
10
+ class << self
11
+ include Sqreen::Ecosystem::Loggable
12
+
13
+ private
14
+
15
+ def attr_reader(attr, _opts = {})
16
+ define_method attr do
17
+ tx_storage = Ecosystem::TransactionStorage.fetch_thread_local
18
+ return unless tx_storage
19
+ tx_storage[attr]
20
+ end
21
+ end
22
+
23
+ def attr_accessor(attr, opts = {})
24
+ # reader
25
+ attr_reader attr, opts
26
+
27
+ # writer (2 variants)
28
+ do_assign = proc do |value|
29
+ tx_storage = Ecosystem::TransactionStorage.fetch_thread_local
30
+ unless tx_storage
31
+ logger.debug do
32
+ "Assignment of tx local attribute #{attr} to #{value} has no effect"
33
+ end
34
+ return
35
+ end
36
+
37
+ tx_storage[attr] = value
38
+ end
39
+
40
+ if opts.fetch(:allow_overwrite, false)
41
+ define "#{attr}=", &do_assign
42
+ else
43
+ define_method "#{attr}=" do |value|
44
+ cur = public_send(attr)
45
+ unless cur.nil?
46
+ raise "Cannot override value of #{attr} from #{cur} with #{value}"
47
+ end
48
+
49
+ do_assign.call(value)
50
+ end
51
+ end
52
+ end
53
+ end # TxLocalVariables.singleton_class.singleton_class
54
+
55
+ # usage:
56
+ # attr_reader :xxx
57
+
58
+ # in the future, we'll possibly need to expose the full
59
+ # TransactionStorage to the modules, at least if we don't
60
+ # opt for a more structured fashion of data exchange between
61
+ # the modules.
62
+ end # TxLocalVariables.singleton_class
63
+ end # TxLocalVariables
64
+
65
+ def tx_local_vars
66
+ TxLocalVariables
67
+ end
68
+ end # TransactionStorage module
69
+ end
70
+ end
71
+ end
@@ -0,0 +1,44 @@
1
+ require 'sqreen/ecosystem/loggable'
2
+
3
+ module Sqreen
4
+ module Ecosystem
5
+ class ModuleRegistry
6
+ include Sqreen::Ecosystem::Loggable
7
+
8
+ def initialize
9
+ @mods = []
10
+ end
11
+
12
+ def register(mod)
13
+ @mods << mod
14
+ end
15
+
16
+ def init_all
17
+ logger.info { "Initializing #{@mods.size} ecosystem modules" }
18
+ each_module do |mod|
19
+ next unless mod.respond_to? :setup
20
+ logger.debug { "Initializing module with type #{mod.class}" }
21
+ mod.setup
22
+ end
23
+ end
24
+
25
+ def destroy_all
26
+ # not implemented
27
+ end
28
+
29
+ # @param [Class] type
30
+ def each_module(type = nil, &block)
31
+ selected_mods = type ? (@mods.select { |mod| mod.is_a?(type) }) : @mods
32
+ if block_given?
33
+ selected_mods.each(&block)
34
+ else
35
+ selected_mods.each
36
+ end
37
+ end
38
+
39
+ def module_subset(type)
40
+ each_module(type).to_a
41
+ end
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,43 @@
1
+ require 'sqreen/ecosystem/module_api'
2
+ require 'sqreen/ecosystem/module_api/instrumentation'
3
+ require 'sqreen/ecosystem/module_api/message_producer'
4
+ require 'sqreen/ecosystem/module_api/tracing_id_generation'
5
+ require 'sqreen/ecosystem/module_api/tracing/client_data'
6
+
7
+ module Sqreen
8
+ module Ecosystem
9
+ module Redis
10
+ class RedisConnection
11
+ class RedisConnectionData
12
+ include ModuleApi::Tracing::ClientData
13
+
14
+ attr_accessor :port
15
+ end
16
+
17
+ include ModuleApi::Instrumentation
18
+ include ModuleApi::MessageProducer
19
+ include ModuleApi::TracingIdGeneration
20
+
21
+ def setup
22
+ advice = wrap_for_interest(ModuleApi::Tracing::ClientData, &method(:before_advice))
23
+ instrument 'Redis::Connection::TCPSocket#connect',
24
+ before: advice
25
+ end
26
+
27
+ private
28
+
29
+ def before_advice
30
+ host = call.args[0]
31
+ port = call.args[1]
32
+
33
+ RedisConnectionData.new(
34
+ transport: 'redis',
35
+ host: host,
36
+ port: port,
37
+ tracing_identifier: create_tracing_id
38
+ )
39
+ end
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,31 @@
1
+ require 'sqreen/ecosystem/tracing/signals/tracing_client'
2
+ require 'sqreen/ecosystem/module_api/tracing'
3
+ require 'sqreen/ecosystem/module_api/tracing/client_data'
4
+
5
+ module Sqreen
6
+ module Ecosystem
7
+ module Tracing
8
+ module Modules
9
+ class Client
10
+ include ModuleApi::Tracing
11
+
12
+ consumes ModuleApi::Tracing::ClientData
13
+ fixed_scope 'client'
14
+
15
+ # @param [Sqreen::Ecosystem::ModuleApi::Tracing::ClientData] data
16
+ def receive(data)
17
+ signal = Tracing::Signals::TracingClient.new
18
+ signal.payload = Tracing::Signals::TracingClient::Payload.new(
19
+ transport: data.transport,
20
+ host: data.host,
21
+ ip: data.ip,
22
+ tracing_identifier: data.tracing_identifier
23
+ )
24
+
25
+ submit_signal signal
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,30 @@
1
+ require 'sqreen/ecosystem/tracing/signals/tracing_server'
2
+ require 'sqreen/ecosystem/module_api/tracing'
3
+ require 'sqreen/ecosystem/module_api/tracing/server_data'
4
+
5
+ module Sqreen
6
+ module Ecosystem
7
+ module Tracing
8
+ module Modules
9
+ class Server
10
+ include ModuleApi::Tracing
11
+
12
+ consumes ModuleApi::Tracing::ServerData
13
+ fixed_scope 'server'
14
+
15
+ # @param [Sqreen::Ecosystem::ModuleApi::Tracing::ServerData] data
16
+ def receive(data)
17
+ signal = Tracing::Signals::TracingServer.new
18
+ signal.payload = Tracing::Signals::TracingServer::Payload.new(
19
+ transport: data.transport,
20
+ client_ip: data.client_ip,
21
+ tracing_identifier: data.tracing_identifier
22
+ )
23
+
24
+ submit_signal signal
25
+ end
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,160 @@
1
+ require 'thread' # for Mutex
2
+ require 'singleton'
3
+ require 'sqreen/ecosystem/loggable'
4
+
5
+ # see https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000024-sampling.md
6
+
7
+ module Sqreen
8
+ module Ecosystem
9
+ module Tracing
10
+ class Sampler
11
+ # @param [Array<Hash{String=>Object}>] definition
12
+ def initialize(definition)
13
+ @lines = definition.map { |h| Line.new(h) }
14
+ end
15
+
16
+ def should_sample?
17
+ line = @lines.find(&:triggers?)
18
+ line ? line.saved_definition : false
19
+ end
20
+
21
+ class Line
22
+ include Loggable
23
+
24
+ attr_reader :saved_definition
25
+
26
+ # @param [Hash{String=>Object}] definition
27
+ def initialize(definition)
28
+ @saved_definition = definition
29
+ @primitives = []
30
+
31
+ unknown = definition.keys - PRIMITIVES_MAP.keys
32
+ unless unknown.empty?
33
+ logger.warn "Unknown primitives: #{unknown}"
34
+ @primitives << AlwaysFalsePrimitive.instance
35
+ return
36
+ end
37
+
38
+ PRIMITIVES_MAP.each do |key, prim_class|
39
+ next unless definition[key]
40
+ @primitives << prim_class.new(definition[key])
41
+ end
42
+ # if @primitives is empty the line will always
43
+ # return true: [].all?(&:triggers?) is true
44
+ end
45
+
46
+ def triggers?
47
+ @primitives.all?(&:triggers?)
48
+ end
49
+ end
50
+
51
+ class AlwaysFalsePrimitive
52
+ include Singleton
53
+
54
+ def triggers?
55
+ false
56
+ end
57
+ end
58
+
59
+ class CallsPrimitive
60
+ def initialize(calls_period)
61
+ @calls_period = calls_period
62
+ @count = 0
63
+ @mutex = Mutex.new
64
+ end
65
+
66
+ def triggers?
67
+ prev_count = nil
68
+ @mutex.synchronize do
69
+ prev_count = @count
70
+ @count += 1
71
+ end
72
+
73
+ (prev_count % @calls_period).zero?
74
+ end
75
+ end
76
+
77
+ class RandomPrimitive
78
+ def initialize(probability)
79
+ @probability = probability
80
+ end
81
+
82
+ def triggers?
83
+ @probability >= rand
84
+ end
85
+ end
86
+
87
+ class MaxDurationMinutesPrimitive
88
+ def initialize(time_in_minutes)
89
+ @deadline = Sqreen.time + time_in_minutes * 60
90
+ @passed = false # no locking needed
91
+ end
92
+
93
+ def triggers?
94
+ return false if @passed
95
+ if Sqreen.time > @deadline
96
+ @passed = true
97
+ return false
98
+ end
99
+ true
100
+ end
101
+ end
102
+
103
+ class TargetPerMinutePrimitive
104
+ def initialize(max_calls)
105
+ @max_calls = max_calls
106
+ @minute_last_call = cur_minute
107
+ @calls_accumulated = 0
108
+ @mutex = Mutex.new
109
+ end
110
+
111
+ def triggers?
112
+ this_minute = cur_minute
113
+ calls_cur_minute = @mutex.synchronize do
114
+ if @minute_last_call == this_minute
115
+ @calls_accumulated += 1
116
+ else
117
+ @minute_last_call = this_minute
118
+ @calls_accumulated = 1
119
+ end
120
+ end
121
+
122
+ calls_cur_minute <= @max_calls
123
+ end
124
+
125
+ private
126
+
127
+ def cur_minute
128
+ (Sqreen.time / 60).floor
129
+ end
130
+ end
131
+
132
+ class MaxCallsPrimitive
133
+ def initialize(max_calls)
134
+ @max_calls = max_calls
135
+ @disabled = false # to avoid lock
136
+ @mutex = Mutex.new
137
+ @num_calls = 0
138
+ end
139
+
140
+ def triggers?
141
+ return false if @disabled
142
+ num_calls = @mutex.synchronize do
143
+ @num_calls += 1
144
+ end
145
+
146
+ num_calls <= @max_calls
147
+ end
148
+ end
149
+
150
+ PRIMITIVES_MAP = {
151
+ "calls" => CallsPrimitive,
152
+ "random" => RandomPrimitive,
153
+ "max_duration_minutes" => MaxDurationMinutesPrimitive,
154
+ "target_per_minute" => TargetPerMinutePrimitive,
155
+ "max_calls" => MaxCallsPrimitive,
156
+ }.freeze
157
+ end
158
+ end
159
+ end
160
+ end