sqreen 1.20.1-java → 1.21.0.beta3-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +16 -0
  3. data/lib/sqreen/actions/block_user.rb +1 -1
  4. data/lib/sqreen/actions/redirect_ip.rb +1 -1
  5. data/lib/sqreen/actions/redirect_user.rb +1 -1
  6. data/lib/sqreen/attack_detected.html +1 -2
  7. data/lib/sqreen/condition_evaluator.rb +9 -2
  8. data/lib/sqreen/conditionable.rb +24 -6
  9. data/lib/sqreen/configuration.rb +1 -1
  10. data/lib/sqreen/deferred_logger.rb +50 -14
  11. data/lib/sqreen/deliveries/batch.rb +8 -1
  12. data/lib/sqreen/deprecation.rb +38 -0
  13. data/lib/sqreen/ecosystem.rb +96 -0
  14. data/lib/sqreen/ecosystem/dispatch_table.rb +43 -0
  15. data/lib/sqreen/ecosystem/exception_reporting.rb +26 -0
  16. data/lib/sqreen/ecosystem/http/net_http.rb +50 -0
  17. data/lib/sqreen/ecosystem/http/rack_request.rb +39 -0
  18. data/lib/sqreen/ecosystem/loggable.rb +13 -0
  19. data/lib/sqreen/ecosystem/module_api.rb +30 -0
  20. data/lib/sqreen/ecosystem/module_api/event_listener.rb +18 -0
  21. data/lib/sqreen/ecosystem/module_api/instrumentation.rb +23 -0
  22. data/lib/sqreen/ecosystem/module_api/message_producer.rb +51 -0
  23. data/lib/sqreen/ecosystem/module_api/signal_producer.rb +24 -0
  24. data/lib/sqreen/ecosystem/module_api/tracing.rb +45 -0
  25. data/lib/sqreen/ecosystem/module_api/tracing/client_data.rb +31 -0
  26. data/lib/sqreen/ecosystem/module_api/tracing/server_data.rb +27 -0
  27. data/lib/sqreen/ecosystem/module_api/tracing_id_generation.rb +16 -0
  28. data/lib/sqreen/ecosystem/module_api/transaction_storage.rb +71 -0
  29. data/lib/sqreen/ecosystem/module_registry.rb +44 -0
  30. data/lib/sqreen/ecosystem/redis/redis_connection.rb +43 -0
  31. data/lib/sqreen/ecosystem/tracing/modules/client.rb +31 -0
  32. data/lib/sqreen/ecosystem/tracing/modules/server.rb +30 -0
  33. data/lib/sqreen/ecosystem/tracing/sampler.rb +160 -0
  34. data/lib/sqreen/ecosystem/tracing/sampling_configuration.rb +150 -0
  35. data/lib/sqreen/ecosystem/tracing/signals/tracing_client.rb +53 -0
  36. data/lib/sqreen/ecosystem/tracing/signals/tracing_server.rb +53 -0
  37. data/lib/sqreen/ecosystem/tracing_broker.rb +101 -0
  38. data/lib/sqreen/ecosystem/tracing_id_setup.rb +34 -0
  39. data/lib/sqreen/ecosystem/transaction_storage.rb +64 -0
  40. data/lib/sqreen/ecosystem/util/call_writers_from_init.rb +13 -0
  41. data/lib/sqreen/ecosystem_integration.rb +87 -0
  42. data/lib/sqreen/ecosystem_integration/around_callbacks.rb +99 -0
  43. data/lib/sqreen/ecosystem_integration/instrumentation_service.rb +42 -0
  44. data/lib/sqreen/ecosystem_integration/request_lifecycle_tracking.rb +58 -0
  45. data/lib/sqreen/ecosystem_integration/signal_consumption.rb +35 -0
  46. data/lib/sqreen/events/request_record.rb +0 -1
  47. data/lib/sqreen/frameworks/generic.rb +24 -1
  48. data/lib/sqreen/frameworks/rails.rb +0 -7
  49. data/lib/sqreen/frameworks/request_recorder.rb +2 -0
  50. data/lib/sqreen/graft/call.rb +106 -19
  51. data/lib/sqreen/graft/callback.rb +1 -1
  52. data/lib/sqreen/graft/hook.rb +212 -100
  53. data/lib/sqreen/graft/hook_point.rb +18 -11
  54. data/lib/sqreen/legacy/instrumentation.rb +22 -10
  55. data/lib/sqreen/legacy/old_event_submission_strategy.rb +9 -2
  56. data/lib/sqreen/log.rb +3 -2
  57. data/lib/sqreen/log/loggable.rb +1 -0
  58. data/lib/sqreen/logger.rb +24 -0
  59. data/lib/sqreen/metrics.rb +1 -0
  60. data/lib/sqreen/metrics/req_detailed.rb +41 -0
  61. data/lib/sqreen/metrics_store.rb +11 -0
  62. data/lib/sqreen/null_logger.rb +22 -0
  63. data/lib/sqreen/remote_command.rb +4 -0
  64. data/lib/sqreen/rules.rb +8 -4
  65. data/lib/sqreen/rules/blacklist_ips_cb.rb +2 -2
  66. data/lib/sqreen/rules/custom_error_cb.rb +3 -3
  67. data/lib/sqreen/rules/rule_cb.rb +4 -2
  68. data/lib/sqreen/rules/waf_cb.rb +3 -3
  69. data/lib/sqreen/runner.rb +63 -8
  70. data/lib/sqreen/session.rb +2 -0
  71. data/lib/sqreen/signals/conversions.rb +6 -1
  72. data/lib/sqreen/version.rb +1 -1
  73. data/lib/sqreen/weave/budget.rb +35 -0
  74. data/lib/sqreen/weave/legacy/instrumentation.rb +274 -132
  75. data/lib/sqreen/worker.rb +6 -2
  76. metadata +46 -9
  77. data/lib/sqreen/encoding_sanitizer.rb +0 -27
@@ -23,6 +23,8 @@ module Sqreen
23
23
  end
24
24
 
25
25
  class HookPoint
26
+ DEFAULT_STRATEGY = Module.respond_to?(:prepend) ? :prepend : :chain
27
+
26
28
  def self.parse(hook_point)
27
29
  klass_name, separator, method_name = hook_point.split(/(\#|\.)/, 2)
28
30
 
@@ -36,13 +38,13 @@ module Sqreen
36
38
 
37
39
  attr_reader :klass_name, :method_kind, :method_name
38
40
 
39
- def initialize(hook_point, strategy = :chain)
41
+ def initialize(hook_point, strategy = DEFAULT_STRATEGY)
40
42
  @klass_name, @method_kind, @method_name = Sqreen::Graft::HookPoint.parse(hook_point)
41
43
  @strategy = strategy
42
44
  end
43
45
 
44
46
  def to_s
45
- "#{@klass_name}#{@method_kind == :instance_method ? '#' : '.'}#{@method_name}"
47
+ @to_s ||= "#{@klass_name}#{@method_kind == :instance_method ? '#' : '.'}#{@method_name}"
46
48
  end
47
49
 
48
50
  def exist?
@@ -177,23 +179,30 @@ module Sqreen
177
179
 
178
180
  private
179
181
 
180
- def prepend(key)
182
+ def hook_spot(key)
181
183
  target = klass_method? ? klass.singleton_class : klass
182
184
  mod = target.ancestors.each { |e| break if e == target; break(e) if e.class == HookSpot && e.key == key }
185
+ raise "Inconsistency detected: #{target} missing from its own ancestors" if mod.is_a?(Array)
186
+
187
+ [target, mod]
188
+ end
189
+
190
+ def prepend(key)
191
+ target, mod = hook_spot(key)
192
+
183
193
  mod ||= HookSpot.new(key)
194
+
184
195
  target.instance_eval { prepend(mod) }
185
196
  end
186
197
 
187
198
  def prepended?(key)
188
- target = klass_method? ? klass.singleton_class : klass
189
- mod = target.ancestors.each { |e| break if e == target; break(e) if e.class == HookSpot && e.key == key }
199
+ _, mod = hook_spot(key)
190
200
 
191
201
  mod != nil
192
202
  end
193
203
 
194
204
  def overridden?(key)
195
- target = klass_method? ? klass.singleton_class : klass
196
- mod = target.ancestors.each { |e| break if e == target; break(e) if e.class == HookSpot && e.key == key }
205
+ _, mod = hook_spot(key)
197
206
 
198
207
  (mod.instance_methods(false) + mod.protected_instance_methods(false) + mod.private_instance_methods(false)).include?(method_name)
199
208
  end
@@ -202,8 +211,7 @@ module Sqreen
202
211
  hook_point = self
203
212
  method_name = @method_name
204
213
 
205
- target = klass_method? ? klass.singleton_class : klass
206
- mod = target.ancestors.each { |e| break if e == target; break(e) if e.class == HookSpot && e.key == key }
214
+ _, mod = hook_spot(key)
207
215
 
208
216
  mod.instance_eval do
209
217
  if hook_point.private_method?
@@ -221,8 +229,7 @@ module Sqreen
221
229
  def unoverride(key)
222
230
  method_name = @method_name
223
231
 
224
- target = klass_method? ? klass.singleton_class : klass
225
- mod = target.ancestors.each { |e| break if e == target; break(e) if e.class == HookSpot && e.key == key }
232
+ _, mod = hook_spot(key)
226
233
 
227
234
  mod.instance_eval { remove_method(method_name) }
228
235
  end
@@ -109,7 +109,7 @@ module Legacy
109
109
  break if res.is_a?(Hash) && res[:skip_rem_cbs]
110
110
  rescue StandardError => e
111
111
  Sqreen.log.warn { "we catch an exception: #{e.inspect}" }
112
- Sqreen.log.debug e.backtrace
112
+ Sqreen.log.debug { e.backtrace }
113
113
  if cb.respond_to?(:record_exception)
114
114
  cb.record_exception(e)
115
115
  else
@@ -162,7 +162,7 @@ module Legacy
162
162
  returns << res
163
163
  rescue StandardError => e
164
164
  Sqreen.log.warn { "we catch an exception: #{e.inspect}" }
165
- Sqreen.log.debug e.backtrace
165
+ Sqreen.log.debug { e.backtrace }
166
166
  if cb.respond_to?(:record_exception)
167
167
  cb.record_exception(e)
168
168
  else
@@ -597,16 +597,25 @@ module Legacy
597
597
  method = cb.method
598
598
  key = [klass, method]
599
599
 
600
+ if (call_count = ENV['SQREEN_DEBUG_CALL_COUNT'])
601
+ call_count = JSON.parse(call_count)
602
+ if cb.respond_to?(:rule_name) && call_count.key?(cb.rule_name)
603
+ count = call_count[cb.rule_name]
604
+ Sqreen.log.debug { "override rule:#{cb.rule_name} call_count:#{count.inspect}" }
605
+ cb.instance_eval { @call_count_interval = call_count[cb.rule_name] }
606
+ end
607
+ end
608
+
600
609
  @@record_request_hookpoints << key if cb.is_a?(Sqreen::Rules::RecordRequestContext)
601
610
 
602
611
  already_overriden = @@overriden_methods.include? key
603
612
 
604
613
  if !already_overriden
605
614
  if is_class_method?(klass, method)
606
- Sqreen.log.debug "overriding class method for #{cb}"
615
+ Sqreen.log.debug { "overriding class method for #{cb}" }
607
616
  success = override_class_method(klass, method)
608
617
  elsif is_instance_method?(klass, method)
609
- Sqreen.log.debug "overriding instance method for #{cb}"
618
+ Sqreen.log.debug { "overriding instance method for #{cb}" }
610
619
  success = override_instance_method(klass, method)
611
620
  else
612
621
  # FIXME: Override define_method and other dynamic ways to
@@ -623,7 +632,7 @@ module Legacy
623
632
 
624
633
  @@overriden_methods += [key] if success
625
634
  else
626
- Sqreen.log.debug "#{key} was already overriden"
635
+ Sqreen.log.debug { "#{key} was already overriden" }
627
636
  end
628
637
 
629
638
  if klass != Object && klass != Kernel && !Sqreen.features['instrument_all_instances'] && !defined?(::JRUBY_VERSION)
@@ -638,7 +647,7 @@ module Legacy
638
647
  end
639
648
  end
640
649
 
641
- Sqreen.log.debug "Adding callback #{cb} for #{klass} #{method}"
650
+ Sqreen.log.debug { "Adding callback #{cb} for #{klass} #{method}" }
642
651
  @@registered_callbacks.add(cb)
643
652
  @@unovertimable_hookpoints << key unless cb.overtimeable
644
653
  @@instrumented_pid = Process.pid
@@ -659,7 +668,7 @@ module Legacy
659
668
 
660
669
  already_overriden = @@overriden_methods.include? key
661
670
  unless already_overriden
662
- Sqreen.log.debug "#{key} apparently not overridden"
671
+ Sqreen.log.debug { "#{key} apparently not overridden" }
663
672
  end
664
673
 
665
674
  defined_cbs = @@registered_callbacks.get(klass, method).flatten
@@ -667,17 +676,17 @@ module Legacy
667
676
  nb_removed = 0
668
677
  defined_cbs.each do |found_cb|
669
678
  if found_cb == cb
670
- Sqreen.log.debug "Removing callback #{found_cb}"
679
+ Sqreen.log.debug { "Removing callback #{found_cb}" }
671
680
  @@registered_callbacks.remove(found_cb)
672
681
  nb_removed += 1
673
682
  else
674
- Sqreen.log.debug "Not removing callback #{found_cb} (remains #{defined_cbs.size} cbs)"
683
+ Sqreen.log.debug { "Not removing callback #{found_cb} (remains #{defined_cbs.size} cbs)" }
675
684
  end
676
685
  end
677
686
 
678
687
  return unless nb_removed == defined_cbs.size
679
688
 
680
- Sqreen.log.debug "Removing overriden method #{key}"
689
+ Sqreen.log.debug { "Removing overriden method #{key}" }
681
690
  @@overriden_methods.delete(key)
682
691
 
683
692
  if is_class_method?(klass, method)
@@ -705,6 +714,7 @@ module Legacy
705
714
  remove_callback_no_lock(cb)
706
715
  end
707
716
  Sqreen.instrumentation_ready = false
717
+ Sqreen.log.info('Instrumentation deactivated')
708
718
  end
709
719
  end
710
720
 
@@ -757,6 +767,8 @@ module Legacy
757
767
  ### globally declare instrumentation ready
758
768
  ### from within instance method? not even thread local?
759
769
  Sqreen.instrumentation_ready = true
770
+
771
+ Sqreen.log.info('Instrumentation activated')
760
772
  end
761
773
 
762
774
  def initialize(metrics_engine = nil)
@@ -6,6 +6,7 @@
6
6
  require 'sqreen/aggregated_metric'
7
7
  require 'sqreen/log/loggable'
8
8
  require 'sqreen/legacy/waf_redactions'
9
+ require 'sqreen/kit/string_sanitizer'
9
10
 
10
11
  module Sqreen
11
12
  module Legacy
@@ -55,6 +56,12 @@ module Sqreen
55
56
  when AggregatedMetric
56
57
  logger.warn "Aggregated metric event in non-signal mode. Signals disabled at runtime?"
57
58
  next
59
+ when Sqreen::Kit::Signals::Signal
60
+ logger.warn "Signal event in non-signal mode"
61
+ next
62
+ when Sqreen::Kit::Signals::Trace
63
+ logger.warn "Trace event in non-signal mode"
64
+ next
58
65
  when Attack # in practice only found inside req rec
59
66
  EventToHash.convert_attack event
60
67
  when RemoteException
@@ -72,7 +79,7 @@ module Sqreen
72
79
  tally = Hash[events.group_by(&:class).map { |k, v| [k, v.count] }]
73
80
  "Doing batch with the following tally of event types: #{tally}"
74
81
  end
75
- post('batch', { batch: batch }, {}, RETRY_MANY)
82
+ post('batch', { batch: batch.compact }, {}, RETRY_MANY)
76
83
  end
77
84
 
78
85
  private
@@ -166,7 +173,7 @@ module Sqreen
166
173
  res[:request][:parameters] = payload['params'] if payload['params']
167
174
  res[:request][:headers] = payload['headers'] if payload['headers']
168
175
 
169
- res = Sqreen::EncodingSanitizer.sanitize(res)
176
+ res = Sqreen::Kit::StringSanitizer.sanitize(res)
170
177
 
171
178
  if rr.redactor
172
179
  res[:request], redacted = rr.redactor.redact(res[:request])
@@ -14,16 +14,17 @@ require 'sqreen/deferred_logger'
14
14
 
15
15
  module Sqreen
16
16
  def self.log_init
17
+ deferred_logger = @logger
17
18
  @logger = Sqreen::Logger.new(
18
19
  Sqreen.config_get(:log_level).to_s.upcase,
19
20
  Sqreen.config_get(:log_location)
20
21
  )
21
- Sqreen::DeferredLogger.instance.flush_to(@logger.instance_eval { @logger })
22
+ deferred_logger.flush_to(@logger.instance_eval { @logger })
22
23
  rescue => e
23
24
  warn "Sqreen logger exception: #{e}"
24
25
  end
25
26
 
26
27
  def self::log
27
- @logger || Sqreen::DeferredLogger.instance
28
+ @logger ||= Sqreen::DeferredLogger.new
28
29
  end
29
30
  end
@@ -4,6 +4,7 @@
4
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
5
5
 
6
6
  require 'logger'
7
+ require 'sqreen/log'
7
8
 
8
9
  module Sqreen; end
9
10
  module Sqreen::Log; end
@@ -28,6 +28,26 @@ module Sqreen
28
28
  create_error_logger
29
29
  end
30
30
 
31
+ def debug?
32
+ @logger.debug?
33
+ end
34
+
35
+ def info?
36
+ @logger.info?
37
+ end
38
+
39
+ def warn?
40
+ @logger.warn?
41
+ end
42
+
43
+ def error?
44
+ @logger.error?
45
+ end
46
+
47
+ def fatal?
48
+ @logger.fatal?
49
+ end
50
+
31
51
  def debug(msg = nil, &block)
32
52
  @logger.debug(msg, &block)
33
53
  end
@@ -45,6 +65,10 @@ module Sqreen
45
65
  @logger.error(msg, &block)
46
66
  end
47
67
 
68
+ def unknown(msg = nil, &block)
69
+ @logger.unknown(msg, &block)
70
+ end
71
+
48
72
  def add(severity, msg = nil, &block)
49
73
  send(SEVERITY_TO_METHOD[severity], msg, &block)
50
74
  end
@@ -7,3 +7,4 @@ require 'sqreen/metrics/collect'
7
7
  require 'sqreen/metrics/average'
8
8
  require 'sqreen/metrics/sum'
9
9
  require 'sqreen/metrics/binning'
10
+ require 'sqreen/metrics/req_detailed'
@@ -0,0 +1,41 @@
1
+ require 'base64'
2
+ require 'sqreen/mono_time'
3
+ require 'sqreen/metrics/base'
4
+ begin
5
+ require 'sq_detailed_metrics'
6
+ rescue LoadError => _e # rubocop:disable Lint/HandleExceptions
7
+ end
8
+
9
+ module Sqreen
10
+ module Metric
11
+ class ReqDetailed < Base
12
+ attr_reader :num_requests
13
+
14
+ def initialize(opts = {})
15
+ raise 'SqDetailedMetrics unavailable' unless defined?(SqDetailedMetrics)
16
+ super(opts)
17
+ @coll = SqDetailedMetrics::RequestCollection.new
18
+ @start_time = Sqreen.time
19
+ @num_requests = 0
20
+ end
21
+
22
+ # @param [SqDetailedMetrics::Request] value
23
+ def update(_key, value)
24
+ @coll << value
25
+ @num_requests += 1
26
+ end
27
+
28
+ def next_sample(time)
29
+ data = @coll.serialize
30
+ @num_requests = 0
31
+ return nil unless data
32
+
33
+ {
34
+ OBSERVATION_KEY => { 'v1' => Base64.strict_encode64(data) },
35
+ START_KEY => @start_time,
36
+ FINISH_KEY => (@start_time = time),
37
+ }
38
+ end
39
+ end
40
+ end
41
+ end
@@ -27,6 +27,7 @@ module Sqreen
27
27
  def initialize
28
28
  @store = []
29
29
  @metrics = {} # name => (metric, period, start)
30
+ @mutex = Mutex.new
30
31
  end
31
32
 
32
33
  # Definition contains a name,period and aggregate at least
@@ -34,6 +35,8 @@ module Sqreen
34
35
  # @param rule [RuleCB] the rule associated with this metric, if any
35
36
  # @param mklass [Object] Override metric object (used in testing)
36
37
  def create_metric(definition, rule = nil, mklass = nil)
38
+ @mutex.lock
39
+
37
40
  name = definition[NAME_KEY]
38
41
  kind = definition[KIND_KEY]
39
42
  klass = valid_metric(kind, name)
@@ -49,6 +52,8 @@ module Sqreen
49
52
  metric.rule = rule
50
53
  metric.period = definition[PERIOD_KEY]
51
54
  metric
55
+ ensure
56
+ @mutex.unlock
52
57
  end
53
58
 
54
59
  def metric?(name)
@@ -57,21 +62,27 @@ module Sqreen
57
62
 
58
63
  # @param at [Time] when is the store emptied
59
64
  def update(name, at, key, value)
65
+ @mutex.lock
60
66
  metric, period, start = @metrics[name]
61
67
  raise UnregisteredMetric, "Unknown metric #{name}" unless metric
62
68
  next_sample(name, at) if start.nil? || (start + period) < at
63
69
  metric.update(key, value)
70
+ ensure
71
+ @mutex.unlock
64
72
  end
65
73
 
66
74
  # Drains every metrics and returns the store content
67
75
  # @param at [Time] when is the store emptied
68
76
  def publish(flush = true, at = Sqreen.time)
77
+ @mutex.lock
69
78
  @metrics.each do |name, (_, period, start)|
70
79
  next_sample(name, at) if flush || !start.nil? && (start + period) < at
71
80
  end
72
81
  out = @store
73
82
  @store = []
74
83
  out
84
+ ensure
85
+ @mutex.unlock
75
86
  end
76
87
 
77
88
  protected
@@ -9,6 +9,26 @@ module Sqreen
9
9
  class NullLogger
10
10
  include Singleton
11
11
 
12
+ def debug?
13
+ false
14
+ end
15
+
16
+ def info?
17
+ false
18
+ end
19
+
20
+ def warn?
21
+ false
22
+ end
23
+
24
+ def error?
25
+ false
26
+ end
27
+
28
+ def fatal?
29
+ false
30
+ end
31
+
12
32
  def debug(_msg = nil); end
13
33
 
14
34
  def info(_msg = nil); end
@@ -19,6 +39,8 @@ module Sqreen
19
39
 
20
40
  def fatal(_msg = nil); end
21
41
 
42
+ def unknown(_msg = nil); end
43
+
22
44
  def add(_severity, _msg = nil); end
23
45
 
24
46
  def formatter=(_); end
@@ -18,10 +18,12 @@ module Sqreen
18
18
  :features_get => :features,
19
19
  :features_change => :change_features,
20
20
  :force_logout => :shutdown,
21
+ :force_restart => :restart,
21
22
  :paths_whitelist => :change_whitelisted_paths,
22
23
  :ips_whitelist => :change_whitelisted_ips,
23
24
  :get_bundle => :upload_bundle,
24
25
  :performance_budget => :change_performance_budget,
26
+ :tracing_enable => :tracing_enable,
25
27
  }.freeze
26
28
 
27
29
  attr_reader :uuid
@@ -39,6 +41,8 @@ module Sqreen
39
41
  begin
40
42
  output = runner.send(KNOWN_COMMANDS[@name], *@params, context_infos)
41
43
  rescue => e
44
+ Sqreen.log.warn { "Command failed with #{e}" }
45
+ Sqreen.log.debug { e.backtrace.map { |x| " #{x}" }.join("\n") }
42
46
  Sqreen::RemoteException.record(e)
43
47
  return { :status => false, :reason => "error: #{e.inspect}" }
44
48
  end