sqreen 1.18.6-java → 1.20.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (190) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +27 -0
  3. data/lib/sqreen/actions.rb +2 -0
  4. data/lib/sqreen/actions/actions_index.rb +16 -0
  5. data/lib/sqreen/actions/base.rb +4 -10
  6. data/lib/sqreen/actions/block_ip.rb +2 -0
  7. data/lib/sqreen/actions/block_user.rb +2 -0
  8. data/lib/sqreen/actions/ip_range_indexed_action_class.rb +4 -24
  9. data/lib/sqreen/actions/ip_ranges_index.rb +32 -11
  10. data/lib/sqreen/actions/redirect_ip.rb +2 -0
  11. data/lib/sqreen/actions/redirect_user.rb +2 -0
  12. data/lib/sqreen/actions/repository.rb +27 -8
  13. data/lib/sqreen/actions/unknown_action_type.rb +4 -0
  14. data/lib/sqreen/actions/user_action_class.rb +5 -30
  15. data/lib/sqreen/actions/users_index.rb +35 -0
  16. data/lib/sqreen/agent.rb +2 -1
  17. data/lib/sqreen/aggregated_metric.rb +25 -0
  18. data/lib/sqreen/attack_blocked.rb +2 -0
  19. data/lib/sqreen/binding_accessor.rb +2 -0
  20. data/lib/sqreen/binding_accessor/path_elem.rb +2 -0
  21. data/lib/sqreen/binding_accessor/transforms.rb +8 -1
  22. data/lib/sqreen/call_countable.rb +2 -0
  23. data/lib/sqreen/capped_queue.rb +2 -0
  24. data/lib/sqreen/cb.rb +2 -0
  25. data/lib/sqreen/cb_tree.rb +2 -0
  26. data/lib/sqreen/condition_evaluator.rb +2 -0
  27. data/lib/sqreen/conditionable.rb +2 -0
  28. data/lib/sqreen/configuration.rb +19 -1
  29. data/lib/sqreen/context.rb +2 -0
  30. data/lib/sqreen/default_cb.rb +2 -0
  31. data/lib/sqreen/deferred_logger.rb +2 -0
  32. data/lib/sqreen/deliveries.rb +2 -0
  33. data/lib/sqreen/deliveries/batch.rb +6 -1
  34. data/lib/sqreen/deliveries/simple.rb +6 -0
  35. data/lib/sqreen/dependency.rb +3 -1
  36. data/lib/sqreen/dependency/detector.rb +22 -14
  37. data/lib/sqreen/dependency/libsqreen.rb +4 -0
  38. data/lib/sqreen/dependency/new_relic.rb +2 -0
  39. data/lib/sqreen/dependency/rack.rb +10 -5
  40. data/lib/sqreen/dependency/rails.rb +4 -0
  41. data/lib/sqreen/dependency/sentry.rb +2 -0
  42. data/lib/sqreen/dependency/sinatra.rb +12 -1
  43. data/lib/sqreen/encoding_sanitizer.rb +2 -0
  44. data/lib/sqreen/error_handling_middleware.rb +2 -0
  45. data/lib/sqreen/event.rb +9 -5
  46. data/lib/sqreen/events/attack.rb +25 -18
  47. data/lib/sqreen/events/remote_exception.rb +2 -22
  48. data/lib/sqreen/events/request_record.rb +17 -70
  49. data/lib/sqreen/exception.rb +2 -0
  50. data/lib/sqreen/formatter_with_tid.rb +2 -0
  51. data/lib/sqreen/framework_cb.rb +2 -0
  52. data/lib/sqreen/frameworks.rb +2 -0
  53. data/lib/sqreen/frameworks/generic.rb +2 -0
  54. data/lib/sqreen/frameworks/rails.rb +1 -0
  55. data/lib/sqreen/frameworks/rails3.rb +2 -0
  56. data/lib/sqreen/frameworks/request_recorder.rb +15 -2
  57. data/lib/sqreen/frameworks/sinatra.rb +2 -0
  58. data/lib/sqreen/frameworks/sqreen_test.rb +2 -0
  59. data/lib/sqreen/graft.rb +12 -0
  60. data/lib/sqreen/graft/call.rb +150 -0
  61. data/lib/sqreen/{dependency → graft}/callback.rb +12 -4
  62. data/lib/sqreen/graft/hook.rb +316 -0
  63. data/lib/sqreen/{dependency → graft}/hook_point.rb +152 -33
  64. data/lib/sqreen/graft/hook_point_error.rb +10 -0
  65. data/lib/sqreen/invalid_signature_exception.rb +2 -0
  66. data/lib/sqreen/js.rb +2 -0
  67. data/lib/sqreen/js/call_context.rb +2 -0
  68. data/lib/sqreen/js/context_pool.rb +2 -0
  69. data/lib/sqreen/js/exec_js_runnable.rb +2 -0
  70. data/lib/sqreen/js/execjs_adapter.rb +2 -0
  71. data/lib/sqreen/js/executable_js.rb +2 -0
  72. data/lib/sqreen/js/js_service.rb +2 -0
  73. data/lib/sqreen/js/js_service_adapter.rb +2 -0
  74. data/lib/sqreen/js/mini_racer_adapter.rb +2 -0
  75. data/lib/sqreen/js/mini_racer_executable_js.rb +2 -0
  76. data/lib/sqreen/js/thread_local_exec_js_runnable.rb +2 -0
  77. data/lib/sqreen/kit/signals/specialized/aggregated_metric.rb +72 -0
  78. data/lib/sqreen/kit/signals/specialized/attack.rb +57 -0
  79. data/lib/sqreen/kit/signals/specialized/binning_metric.rb +76 -0
  80. data/lib/sqreen/kit/signals/specialized/http_trace.rb +26 -0
  81. data/lib/sqreen/kit/signals/specialized/sdk_track_call.rb +50 -0
  82. data/lib/sqreen/kit/signals/specialized/sqreen_exception.rb +57 -0
  83. data/lib/sqreen/{backport.rb → legacy.rb} +3 -2
  84. data/lib/sqreen/{instrumentation.rb → legacy/instrumentation.rb} +31 -2
  85. data/lib/sqreen/legacy/old_event_submission_strategy.rb +221 -0
  86. data/lib/sqreen/legacy/waf_redactions.rb +49 -0
  87. data/lib/sqreen/log.rb +2 -0
  88. data/lib/sqreen/log/loggable.rb +28 -0
  89. data/lib/sqreen/logger.rb +2 -0
  90. data/lib/sqreen/metrics.rb +2 -0
  91. data/lib/sqreen/metrics/average.rb +2 -0
  92. data/lib/sqreen/metrics/base.rb +5 -0
  93. data/lib/sqreen/metrics/binning.rb +2 -0
  94. data/lib/sqreen/metrics/collect.rb +2 -0
  95. data/lib/sqreen/metrics/sum.rb +2 -0
  96. data/lib/sqreen/metrics_store.rb +24 -12
  97. data/lib/sqreen/metrics_store/already_registered_metric.rb +2 -0
  98. data/lib/sqreen/metrics_store/unknown_metric.rb +2 -0
  99. data/lib/sqreen/metrics_store/unregistered_metric.rb +2 -0
  100. data/lib/sqreen/middleware.rb +2 -0
  101. data/lib/sqreen/mono_time.rb +2 -0
  102. data/lib/sqreen/node.rb +2 -0
  103. data/lib/sqreen/not_implemented_yet.rb +2 -0
  104. data/lib/sqreen/null_logger.rb +2 -0
  105. data/lib/sqreen/payload_creator.rb +2 -0
  106. data/lib/sqreen/payload_creator/header_section.rb +2 -0
  107. data/lib/sqreen/performance_notifications.rb +2 -0
  108. data/lib/sqreen/performance_notifications/binned_metrics.rb +10 -2
  109. data/lib/sqreen/performance_notifications/log.rb +2 -0
  110. data/lib/sqreen/performance_notifications/log_performance.rb +2 -0
  111. data/lib/sqreen/performance_notifications/metrics.rb +2 -0
  112. data/lib/sqreen/performance_notifications/newrelic.rb +2 -0
  113. data/lib/sqreen/prefix.rb +2 -0
  114. data/lib/sqreen/rails_middleware.rb +2 -0
  115. data/lib/sqreen/remote_command.rb +2 -0
  116. data/lib/sqreen/remote_command/failure_output.rb +5 -0
  117. data/lib/sqreen/rules.rb +6 -2
  118. data/lib/sqreen/rules/attrs.rb +2 -0
  119. data/lib/sqreen/rules/auth_track_cb.rb +2 -0
  120. data/lib/sqreen/rules/binding_accessor_matcher_cb.rb +2 -0
  121. data/lib/sqreen/rules/binding_accessor_metrics.rb +2 -0
  122. data/lib/sqreen/rules/blacklist_ips_cb.rb +2 -0
  123. data/lib/sqreen/rules/count_http_codes.rb +2 -0
  124. data/lib/sqreen/rules/crawler_user_agent_matches_cb.rb +2 -0
  125. data/lib/sqreen/rules/crawler_user_agent_matches_metrics_cb.rb +2 -0
  126. data/lib/sqreen/rules/custom_error_cb.rb +2 -0
  127. data/lib/sqreen/rules/devise_auth_track_cb.rb +2 -0
  128. data/lib/sqreen/rules/devise_signup_track_cb.rb +2 -0
  129. data/lib/sqreen/rules/execjs_cb.rb +2 -0
  130. data/lib/sqreen/rules/headers_insert_cb.rb +7 -0
  131. data/lib/sqreen/rules/matcher_rule.rb +2 -0
  132. data/lib/sqreen/rules/not_found_cb.rb +7 -0
  133. data/lib/sqreen/rules/rails_parameters_cb.rb +2 -0
  134. data/lib/sqreen/rules/record_request_context.rb +2 -0
  135. data/lib/sqreen/rules/regexp_rule_cb.rb +2 -0
  136. data/lib/sqreen/rules/rule_cb.rb +4 -0
  137. data/lib/sqreen/rules/run_req_start_actions.rb +3 -1
  138. data/lib/sqreen/rules/run_user_actions.rb +3 -1
  139. data/lib/sqreen/rules/shell_env_cb.rb +2 -0
  140. data/lib/sqreen/rules/signup_track_cb.rb +2 -0
  141. data/lib/sqreen/rules/update_request_context.rb +2 -0
  142. data/lib/sqreen/rules/url_matches_cb.rb +2 -0
  143. data/lib/sqreen/rules/user_agent_matches_cb.rb +2 -0
  144. data/lib/sqreen/rules/waf_cb.rb +41 -16
  145. data/lib/sqreen/rules/xss_cb.rb +2 -0
  146. data/lib/sqreen/run_when_called_cb.rb +2 -0
  147. data/lib/sqreen/runner.rb +68 -12
  148. data/lib/sqreen/runtime_infos.rb +2 -0
  149. data/lib/sqreen/safe_json.rb +2 -0
  150. data/lib/sqreen/sdk.rb +4 -0
  151. data/lib/sqreen/sensitive_data_redactor.rb +21 -31
  152. data/lib/sqreen/serializer.rb +2 -0
  153. data/lib/sqreen/session.rb +41 -37
  154. data/lib/sqreen/shared_storage.rb +2 -0
  155. data/lib/sqreen/shared_storage23.rb +2 -0
  156. data/lib/sqreen/shrink_wrap.rb +16 -0
  157. data/lib/sqreen/signals/conversions.rb +283 -0
  158. data/lib/sqreen/signals/http_trace_redaction.rb +111 -0
  159. data/lib/sqreen/signals/signals_submission_strategy.rb +78 -0
  160. data/lib/sqreen/signature_verifier.rb +2 -0
  161. data/lib/sqreen/sinatra_middleware.rb +2 -0
  162. data/lib/sqreen/sqreen_signed_verifier.rb +2 -0
  163. data/lib/sqreen/token_invalid_exception.rb +2 -0
  164. data/lib/sqreen/token_not_found_exception.rb +2 -0
  165. data/lib/sqreen/trie.rb +2 -0
  166. data/lib/sqreen/unauthorized.rb +2 -0
  167. data/lib/sqreen/util.rb +5 -0
  168. data/lib/sqreen/util/capped_array.rb +2 -0
  169. data/lib/sqreen/util/capped_hash.rb +2 -0
  170. data/lib/sqreen/util/capped_string.rb +2 -0
  171. data/lib/sqreen/util/capper.rb +2 -0
  172. data/lib/sqreen/version.rb +3 -1
  173. data/lib/sqreen/waf_error.rb +2 -0
  174. data/lib/sqreen/weave.rb +12 -0
  175. data/lib/sqreen/weave/hardcoded.rb +19 -0
  176. data/lib/sqreen/weave/instrumentor.rb +48 -0
  177. data/lib/sqreen/weave/legacy.rb +12 -0
  178. data/lib/sqreen/weave/legacy/instrumentation.rb +406 -0
  179. data/lib/sqreen/web_server.rb +2 -0
  180. data/lib/sqreen/web_server/generic.rb +2 -0
  181. data/lib/sqreen/web_server/passenger.rb +2 -0
  182. data/lib/sqreen/web_server/puma.rb +2 -0
  183. data/lib/sqreen/web_server/rainbows.rb +2 -0
  184. data/lib/sqreen/web_server/thin.rb +2 -0
  185. data/lib/sqreen/web_server/unicorn.rb +2 -0
  186. data/lib/sqreen/web_server/webrick.rb +2 -0
  187. data/lib/sqreen/worker.rb +2 -0
  188. metadata +65 -9
  189. data/lib/sqreen/backport/original_name.rb +0 -86
  190. data/lib/sqreen/dependency/hook.rb +0 -102
@@ -0,0 +1,111 @@
1
+ require 'json'
2
+ require 'sqreen/kit/loggable'
3
+ require 'sqreen/kit/signals/specialized/http_trace'
4
+
5
+ module Sqreen
6
+ module Signals
7
+ module HttpTraceRedaction
8
+ class << self
9
+ include Sqreen::Kit::Loggable
10
+
11
+ # @param [Sqreen::Kit::Signals::Specialized::HttpTrace] trace
12
+ # @param [Sqreen::SensitiveDataRedactor] redactor
13
+ def redact_trace!(trace, redactor)
14
+ return unless redactor
15
+ # redact headers (keys unsafe)
16
+ # @type [Sqreen::Kit::Signals::Context::HttpContext]
17
+ http_context = trace.context
18
+
19
+ all_redacted = []
20
+
21
+ # Redact headers; save redacted values
22
+ # headers are encoded as [key, value], not a hash, so
23
+ # they require some transformation
24
+ orig_headers = http_context.headers
25
+ if orig_headers
26
+ headers = orig_headers.map { |(k, v)| { k => v } }
27
+ headers, redacted = redactor.redact(headers)
28
+ http_context.headers = headers.map(&:first)
29
+ all_redacted += redacted
30
+ end
31
+
32
+ # Redact params; save redacted values
33
+ Kit::Signals::Context::HttpContext::PARAMS_ATTRS.each do |attr|
34
+ value = http_context.public_send(attr)
35
+ next unless value
36
+ value, redacted = redactor.redact(value)
37
+ all_redacted += redacted
38
+ http_context.public_send(:"#{attr}=", value)
39
+ end
40
+
41
+ all_redacted = all_redacted.uniq.map(&:downcase)
42
+
43
+ # Redact attacks and exceptions
44
+ # XXX: no redaction for infos in attacks/exceptions except for WAF data
45
+ # Is this the correct behavior?
46
+ redact_attacks!(trace, redactor, all_redacted)
47
+ redact_exceptions!(trace, redactor, all_redacted)
48
+ end
49
+
50
+ private
51
+
52
+ # @param [Sqreen::Kit::Signals::Specialized::HttpTrace] trace
53
+ # @param [Sqreen::SensitiveDataRedactor] redactor
54
+ # Redacts WAF data according to specific rules therefor
55
+ # Redacts infos according to general rules
56
+ def redact_attacks!(trace, redactor, redacted_data)
57
+ trace.data.each do |signal|
58
+ next unless signal.is_a?(Kit::Signals::Specialized::Attack)
59
+ # @type [Sqreen::Kit::Signals::Specialized::Attack::Payload] payload
60
+ payload = signal.payload
61
+ next unless payload.infos
62
+
63
+ if payload.infos[:waf_data]
64
+ redact_waf_attack_data!(payload.infos, redacted_data)
65
+ end
66
+ payload.infos, = redactor.redact(payload.infos)
67
+ end
68
+ end
69
+
70
+ def redact_exceptions!(trace, redactor, redacted_data)
71
+ trace.data.each do |signal|
72
+ next unless signal.is_a?(Kit::Signals::Specialized::SqreenException)
73
+ infos = signal.infos
74
+ next unless infos
75
+
76
+ redact_waf_exception_data!(signal.infos, redacted_data) if signal.infos[:waf]
77
+ signal.infos, = redactor.redact(infos)
78
+ end
79
+ end
80
+
81
+ # @param [Hash] infos from WAF attack
82
+ def redact_waf_attack_data!(infos, redacted_data)
83
+ begin
84
+ parsed = JSON.parse(infos[:waf_data])
85
+ rescue JSON::JSONError => e
86
+ logger.warn("waf_data is not valid json: #{e.message}")
87
+ return
88
+ end
89
+ redacted = parsed.each do |w|
90
+ next unless (filters = w['filter'])
91
+
92
+ filters.each do |f|
93
+ next unless (v = f['resolved_value'])
94
+ next unless redacted_data.include?(v.downcase)
95
+
96
+ f['match_status'] = SensitiveDataRedactor::MASK
97
+ f['resolved_value'] = SensitiveDataRedactor::MASK
98
+ end
99
+ end
100
+ infos[:waf_data] = JSON.dump(redacted)
101
+ end
102
+
103
+ # see https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000022-waf-data-sanitization.md#changes-to-the-agents
104
+ def redact_waf_exception_data!(infos, redacted_data)
105
+ return if redacted_data.empty?
106
+ infos[:waf].delete(:args)
107
+ end
108
+ end
109
+ end
110
+ end
111
+ end
@@ -0,0 +1,78 @@
1
+ require 'sqreen/aggregated_metric'
2
+ require 'sqreen/kit'
3
+ require 'sqreen/kit/string_sanitizer'
4
+ require 'sqreen/signals/conversions'
5
+ require 'sqreen/log/loggable'
6
+
7
+ module Sqreen
8
+ module Signals
9
+ # see also Sqreen::Legacy::OldEventSubmissionStrategy
10
+ # usage in Sqreen:Session
11
+ class SignalsSubmissionStrategy
12
+ include Sqreen::Log::Loggable
13
+
14
+ # @param [Array<Sqreen::AggregatedMetric>] metrics
15
+ def post_metrics(metrics)
16
+ return if metrics.nil? || metrics.empty?
17
+
18
+ guarded 'Failed to serialize or submit aggregated metrics' do
19
+ batch = metrics.map do |m|
20
+ Conversions.convert_metric_sample(m)
21
+ end
22
+ client.report_batch(batch)
23
+ end
24
+ end
25
+
26
+ # @param _attack [Sqreen::Attack]
27
+ # XXX: unused
28
+ def post_attack(_attack)
29
+ raise NotImplementedError
30
+ end
31
+
32
+ # @param request_record [Sqreen::RequestRecord]
33
+ def post_request_record(request_record)
34
+ guarded 'Failed to serialize or submit request record' do
35
+ trace = Conversions.convert_req_record(request_record)
36
+ append_sanitizing_filter(trace)
37
+ client.report_trace(trace)
38
+ end
39
+ end
40
+
41
+ # Post an exception to Sqreen for analysis
42
+ # @param exception [RemoteException] Exception and context to be sent over
43
+ def post_sqreen_exception(exception)
44
+ guarded 'Failed to serialize or submit exception', false do
45
+ data = Conversions.convert_exception(exception)
46
+ append_sanitizing_filter(data)
47
+ client.report_signal(data)
48
+ end
49
+ end
50
+
51
+ def post_batch(events)
52
+ guarded 'Failed to serialize or submit batch of events' do
53
+ batch = Conversions.convert_batch(events)
54
+ batch.each { |sig_or_trace| append_sanitizing_filter(sig_or_trace) }
55
+ client.report_batch(batch)
56
+ end
57
+ end
58
+
59
+ private
60
+
61
+ def append_sanitizing_filter(sig_or_trace)
62
+ sig_or_trace.append_to_h_filter Kit::StringSanitizer.method(:sanitize)
63
+ end
64
+
65
+ # we don't want exceptions to propagate and kill the worker thread
66
+ def guarded(msg, report = true)
67
+ yield
68
+ rescue StandardError => e
69
+ logger.warn "#{msg}: #{e.message}\n#{e.backtrace.map { |x| " #{x}" }.join("\n")}"
70
+ post_sqreen_exception(RemoteException.new(e)) if report
71
+ end
72
+
73
+ def client
74
+ Sqreen::Kit.auth_signals_client
75
+ end
76
+ end
77
+ end
78
+ end
@@ -1,3 +1,5 @@
1
+ # typed: true
2
+
1
3
  # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
3
5
 
@@ -1,3 +1,5 @@
1
+ # typed: true
2
+
1
3
  # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
3
5
 
@@ -1,3 +1,5 @@
1
+ # typed: false
2
+
1
3
  # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
3
5
 
@@ -1,3 +1,5 @@
1
+ # typed: strong
2
+
1
3
  # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
3
5
 
@@ -1,3 +1,5 @@
1
+ # typed: strong
2
+
1
3
  # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
3
5
 
@@ -1,3 +1,5 @@
1
+ # typed: false
2
+
1
3
  # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
3
5
 
@@ -1,3 +1,5 @@
1
+ # typed: strong
2
+
1
3
  # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
3
5
 
@@ -1,2 +1,7 @@
1
+ # typed: strong
2
+
3
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
4
+ # Please refer to our terms for more information: https://www.sqreen.com/terms.html
5
+
1
6
  module Sqreen; end
2
7
  module Sqreen::Util; end
@@ -1,3 +1,5 @@
1
+ # typed: false
2
+
1
3
  # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
3
5
 
@@ -1,3 +1,5 @@
1
+ # typed: false
2
+
1
3
  # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
3
5
 
@@ -1,3 +1,5 @@
1
+ # typed: false
2
+
1
3
  # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
3
5
 
@@ -1,3 +1,5 @@
1
+ # typed: true
2
+
1
3
  # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
3
5
 
@@ -1,6 +1,8 @@
1
+ # typed: true
2
+
1
3
  # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
3
5
 
4
6
  module Sqreen
5
- VERSION = '1.18.6'.freeze
7
+ VERSION = '1.20.0'.freeze
6
8
  end
@@ -1,3 +1,5 @@
1
+ # typed: true
2
+
1
3
  # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
3
5
 
@@ -0,0 +1,12 @@
1
+ # typed: strong
2
+
3
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
4
+ # Please refer to our terms for more information: https://www.sqreen.com/terms.html
5
+
6
+ require 'sqreen/log/loggable'
7
+
8
+ module Sqreen
9
+ module Weave
10
+ include Sqreen::Log::Loggable
11
+ end
12
+ end
@@ -0,0 +1,19 @@
1
+ # typed: strong
2
+
3
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
4
+ # Please refer to our terms for more information: https://www.sqreen.com/terms.html
5
+
6
+ require 'sqreen/weave'
7
+
8
+ class Sqreen::Weave::Hardcoded
9
+ # [
10
+ # ### callback for performing sec responses based on ip
11
+ # ### init redefined to implement smartass way to hook it upon the
12
+ # ### framework's middleware #call
13
+ # Sqreen::Rules::RunReqStartActions.new(framework),
14
+ # ### callback for performing sec responses based on user
15
+ # Sqreen::Rules::RunUserActions.new(Sqreen, :identify, 0),
16
+ # ### callback for performing sec responses based on user
17
+ # Sqreen::Rules::RunUserActions.new(Sqreen, :auth_track, 1),
18
+ # ]
19
+ end
@@ -0,0 +1,48 @@
1
+ # typed: true
2
+
3
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
4
+ # Please refer to our terms for more information: https://www.sqreen.com/terms.html
5
+
6
+ require 'sqreen/weave'
7
+
8
+ # rule loader: decouple from runner
9
+ # remote rules from back
10
+ # local rules from local files
11
+ # => rule list (what is a rule?)
12
+ # => to callback (what is a callback?)
13
+ # => to instrumentation (== attach callbacks to their targets using graft)
14
+
15
+ # make shit like instrument framework independent (block passing?)
16
+ # => too much things assume only one framework
17
+ # possible to do run req actions without hardcoded cbs?
18
+ # (data comes from actions command, native cb merely binds to middleware)
19
+ # can cb be a form of abstraction?
20
+
21
+ # rule sig: decouple/split
22
+ # - data signer/checker
23
+ # apply this to rule data
24
+
25
+ # whitelist is mixed in
26
+
27
+ # metrics
28
+ # three dedicated metrics: abstract and isolate
29
+
30
+ class Sqreen::Weave::Instrumentor
31
+ def initialize(metrics_engine)
32
+ ### bail out if no metric engine
33
+ ### init metric to count calls to sqreen
34
+ ### init metric to count request whitelist matches (ip or path whitelist)
35
+ ### init metric to count over budget hits
36
+ end
37
+
38
+ def instrument!(rules, framework)
39
+ ### set up rule signature verifier
40
+ ### force clean instrumentation callback list
41
+ ### for each rule description, transform into format for adding callback
42
+ ### attach framework to callback
43
+ ### install callback, observing priority
44
+ ### for each hardcoded callback
45
+ ### install hardcoded callbacks, observing priority
46
+ ### globally declare instrumentation ready
47
+ end
48
+ end
@@ -0,0 +1,12 @@
1
+ # typed: strong
2
+
3
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
4
+ # Please refer to our terms for more information: https://www.sqreen.com/terms.html
5
+
6
+ require 'sqreen/weave'
7
+
8
+ module Sqreen
9
+ module Weave
10
+ module Legacy; end
11
+ end
12
+ end
@@ -0,0 +1,406 @@
1
+ # typed: false
2
+
3
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
4
+ # Please refer to our terms for more information: https://www.sqreen.com/terms.html
5
+
6
+ require 'sqreen/weave/legacy'
7
+ require 'sqreen/graft/hook_point'
8
+ require 'sqreen/call_countable'
9
+ require 'sqreen/rules'
10
+ require 'sqreen/rules/record_request_context'
11
+
12
+ class Sqreen::Weave::Legacy::Instrumentation
13
+ attr_accessor :metrics_engine
14
+
15
+ def initialize(metrics_engine, opts = {})
16
+ Sqreen::Weave.logger.debug { "#{self.class.name}#initialize #{metrics_engine}" }
17
+ @hooks = []
18
+
19
+ self.metrics_engine = metrics_engine
20
+
21
+ ### bail out if no metric engine
22
+ return if metrics_engine.nil?
23
+
24
+ ### init metric to count calls to sqreen
25
+ metrics_engine.create_metric(
26
+ 'name' => 'sqreen_call_counts',
27
+ 'period' => 60,
28
+ 'kind' => 'Sum',
29
+ )
30
+ ### init metric to count request whitelist matches (ip or path whitelist)
31
+ metrics_engine.create_metric(
32
+ 'name' => 'whitelisted',
33
+ 'period' => 60,
34
+ 'kind' => 'Sum',
35
+ )
36
+ ### init metric to count over budget hits
37
+ metrics_engine.create_metric(
38
+ 'name' => 'request_overtime',
39
+ 'period' => 60,
40
+ 'kind' => 'Sum',
41
+ )
42
+
43
+ # PerformanceNotifications::Binning
44
+ metrics_engine.create_metric(
45
+ 'name' => 'req',
46
+ 'period' => opts[:period] || 60,
47
+ 'kind' => 'Binning',
48
+ 'options' => opts[:perf_metric] || { 'base' => 2.0, 'factor' => 0.1 },
49
+ )
50
+ metrics_engine.create_metric(
51
+ 'name' => 'sq',
52
+ 'period' => opts[:period] || 60,
53
+ 'kind' => 'Binning',
54
+ 'options' => opts[:perf_metric] || { 'base' => 2.0, 'factor' => 0.1 },
55
+ )
56
+ metrics_engine.create_metric(
57
+ 'name' => 'pct',
58
+ 'period' => opts[:period] || 60,
59
+ 'kind' => 'Binning',
60
+ 'options' => opts[:perf_metric_percent] || { 'base' => 1.3, 'factor' => 1.0 },
61
+ )
62
+
63
+ Sqreen.thread_cpu_time? && metrics_engine.create_metric(
64
+ 'name' => 'sq_thread_cpu_pct',
65
+ 'period' => opts[:period] || 60,
66
+ 'kind' => 'Binning',
67
+ 'options' => opts[:perf_metric_percent] || { 'base' => 1.3, 'factor' => 1.0 },
68
+ )
69
+ end
70
+
71
+ # needed by Sqreen::Runner#initialize
72
+ def instrument!(rules, framework)
73
+ Sqreen::Weave.logger.debug { "#{rules.count} rules, #{framework}" }
74
+
75
+ strategy = Sqreen.config_get(:weave_strategy)
76
+ if strategy == :prepend && !Module.respond_to?(:prepend)
77
+ Sqreen::Weave.logger.warn { "strategy: #{strategy.inspect} unavailable, falling back to :chain" }
78
+ strategy = :chain
79
+ elsif strategy == :chain && Gem::Specification.select { |s| s.name == 'scout_apm' && Gem::Requirement.new('>= 2.5.2').satisfied_by?(Gem::Version.new(s.version)) }.any?
80
+ Sqreen::Weave.logger.warn { "strategy: #{strategy.inspect} unavailable with scout_apm >= 2.5.2, switching to :prepend" }
81
+ strategy = :prepend
82
+ end
83
+ Sqreen::Weave.logger.debug { "strategy: #{strategy.inspect}" }
84
+
85
+ ### set up rule signature verifier
86
+ verifier = nil
87
+ ### force clean instrumentation callback list
88
+ @hooks = []
89
+ ### for each rule description
90
+ rules.each do |rule|
91
+ Sqreen::Weave.logger.debug { "Processing rule: #{rule['name']}" }
92
+ ### transform into format for adding callback
93
+ rule_callback = Sqreen::Rules.cb_from_rule(rule, self, metrics_engine, verifier)
94
+ next unless rule_callback
95
+ ### attach framework to callback
96
+ rule_callback.framework = framework
97
+ ### install callback, observing priority
98
+ Sqreen::Weave.logger.debug { "Adding rule callback: #{rule_callback}" }
99
+ @hooks << add_callback("weave,rule=#{rule['name']}", rule_callback, strategy)
100
+ end
101
+
102
+ ### for each hardcoded callback
103
+ hardcoded_callbacks(framework).each do |hard_callback|
104
+ Sqreen::Weave.logger.debug { "Adding hardcoded callback: #{hard_callback}" }
105
+ ### install hardcoded callbacks, observing priority
106
+ @hooks << add_callback('weave,hardcoded', hard_callback, strategy)
107
+ end
108
+
109
+ metrics_engine = self.metrics_engine
110
+ request_hook = Sqreen::Graft::Hook['Sqreen::ShrinkWrap#call', strategy]
111
+ @hooks << request_hook
112
+ request_hook.add do
113
+ before('wave,meta,request', rank: -100000, mandatory: true) do |_call|
114
+ next unless Sqreen.instrumentation_ready
115
+
116
+ uuid = SecureRandom.uuid
117
+ now = Sqreen::Graft::Timer.read
118
+ Thread.current[:sqreen_http_request] = {
119
+ uuid: uuid,
120
+ start_time: now,
121
+ time_budget: Sqreen.performance_budget,
122
+ time_budget_expended: false,
123
+ timer: Sqreen::Graft::Timer.new("request_#{uuid}"),
124
+ timed_callbacks: [],
125
+ timed_hooks: [],
126
+ timed_hooks_before: [],
127
+ timed_hooks_after: [],
128
+ timed_hooks_raised: [],
129
+ timed_hooks_ensured: [],
130
+ skipped_callbacks: [],
131
+ }
132
+
133
+ Sqreen::Weave.logger.debug { "request.uuid: #{uuid}" }
134
+ end
135
+
136
+ ensured('weave,meta,request', rank: 100000, mandatory: true) do |_call|
137
+ request = Thread.current[:sqreen_http_request]
138
+
139
+ next if request.nil?
140
+
141
+ Thread.current[:sqreen_http_request] = nil
142
+ now = Sqreen::Graft::Timer.read
143
+ utc_now = Time.now.utc
144
+
145
+ request[:timed_callbacks].each do |timer|
146
+ duration = timer.duration
147
+ # stop = now
148
+ # start = now - duration
149
+ timer.tag =~ /weave,rule=(.*)$/ && rule = $1
150
+ timer.tag =~ /@before/ && whence = 'pre'
151
+ timer.tag =~ /@after/ && whence = 'post'
152
+ timer.tag =~ /@raised/ && whence = 'failing'
153
+
154
+ next unless rule && whence
155
+
156
+ # Sqreen::PerformanceNotifications.notify(rule, whence, start, stop)
157
+ # => BinnedMetrics
158
+ metric_name = "sq.#{rule}.#{whence}"
159
+ unless metrics_engine.metric?(metric_name)
160
+ metrics_engine.create_metric(
161
+ 'name' => metric_name,
162
+ 'period' => 60,
163
+ 'kind' => 'Binning',
164
+ 'options' => { 'base' => 2.0, 'factor' => 0.1 },
165
+ )
166
+ end
167
+ metrics_engine.update(metric_name, now, nil, duration * 1000)
168
+ end
169
+
170
+ metric_name = 'sq.hooks_pre.pre'
171
+ duration = request[:timed_hooks_before].sum(&:duration)
172
+ unless metrics_engine.metric?(metric_name)
173
+ metrics_engine.create_metric(
174
+ 'name' => metric_name,
175
+ 'period' => 60,
176
+ 'kind' => 'Binning',
177
+ 'options' => { 'base' => 2.0, 'factor' => 0.1 },
178
+ )
179
+ end
180
+ metrics_engine.update(metric_name, now, nil, duration * 1000)
181
+
182
+ metric_name = 'sq.hooks_post.post'
183
+ duration = request[:timed_hooks_after].sum(&:duration)
184
+ unless metrics_engine.metric?(metric_name)
185
+ metrics_engine.create_metric(
186
+ 'name' => metric_name,
187
+ 'period' => 60,
188
+ 'kind' => 'Binning',
189
+ 'options' => { 'base' => 2.0, 'factor' => 0.1 },
190
+ )
191
+ end
192
+ metrics_engine.update(metric_name, now, nil, duration * 1000)
193
+
194
+ metric_name = 'sq.hooks_failing.failing'
195
+ duration = request[:timed_hooks_raised].sum(&:duration)
196
+ unless metrics_engine.metric?(metric_name)
197
+ metrics_engine.create_metric(
198
+ 'name' => metric_name,
199
+ 'period' => 60,
200
+ 'kind' => 'Binning',
201
+ 'options' => { 'base' => 2.0, 'factor' => 0.1 },
202
+ )
203
+ end
204
+ metrics_engine.update(metric_name, now, nil, duration * 1000)
205
+
206
+ skipped = request[:skipped_callbacks].map(&:name)
207
+ Sqreen::Weave.logger.debug { "request:#{request[:uuid]} callback.skipped.size: #{skipped.count} callback.skipped: [#{skipped.join(', ')}]" }
208
+ timer = request[:timer]
209
+ total = timer.duration
210
+ Sqreen::Weave.logger.debug { "request:#{request[:uuid]} timer.total: #{'%.03fus' % (total * 1_000_000)} timer.size: #{timer.size}" }
211
+ timings = request[:timed_callbacks].map(&:to_s)
212
+ total = request[:timed_callbacks].sum(&:duration)
213
+ Sqreen::Weave.logger.debug { "request:#{request[:uuid]} callback.total: #{'%.03fus' % (total * 1_000_000)} callback.timings: [#{timings.join(', ')}]" }
214
+ timings = request[:timed_hooks].map(&:to_s)
215
+ total = request[:timed_hooks].sum(&:duration)
216
+ Sqreen::Weave.logger.debug { "request:#{request[:uuid]} hook.total: #{'%.03fus' % (total * 1_000_000)} hook.timings: [#{timings.join(', ')}]" }
217
+
218
+ skipped = request[:skipped_callbacks].map(&:name)
219
+ skipped_rule_name = skipped.first && skipped.first =~ /weave,rule=(.*)$/ && $1
220
+ Sqreen.observations_queue.push(['request_overtime', skipped_rule_name, 1, utc_now]) if skipped_rule_name
221
+
222
+ sqreen_request_duration = total
223
+ Sqreen.observations_queue.push(['sq', nil, sqreen_request_duration * 1000, utc_now])
224
+
225
+ request_duration = now - request[:start_time]
226
+ Sqreen.observations_queue.push(['req', nil, request_duration * 1000, utc_now])
227
+
228
+ sqreen_request_ratio = (sqreen_request_duration * 100.0) / (request_duration - sqreen_request_duration)
229
+ Sqreen.observations_queue.push(['pct', nil, sqreen_request_ratio, utc_now])
230
+ end
231
+ end.install
232
+
233
+ ### globally declare instrumentation ready
234
+ Sqreen.instrumentation_ready = true
235
+ end
236
+
237
+ # needed by Sqreen::Runner
238
+ def remove_all_callbacks
239
+ Sqreen.instrumentation_ready = false
240
+
241
+ loop do
242
+ hook = @hooks.pop
243
+ break unless hook
244
+ Sqreen::Weave.logger.debug { "hook.deinstrument: #{hook}" }
245
+ hook.uninstall
246
+ hook.clear
247
+ end
248
+ end
249
+
250
+ # needed by #instrument!
251
+ def add_callback(rule, callback, strategy)
252
+ Sqreen::Weave.logger.debug { "Adding rule: #{rule} callback: #{callback}" }
253
+ klass = callback.klass
254
+ method = callback.method
255
+
256
+ if Sqreen::Graft::HookPoint.new("#{klass}.#{method}").exist?
257
+ hook_point = "#{klass}.#{method}"
258
+ elsif Sqreen::Graft::HookPoint.new("#{klass}##{method}").exist?
259
+ hook_point = "#{klass}##{method}"
260
+ end
261
+
262
+ return if hook_point.nil?
263
+
264
+ priority = callback.priority || 100
265
+ block = callback.respond_to?(:block) ? callback.block : true
266
+ ignore = -> { callback.whitelisted? } if callback.respond_to?(:whitelisted?)
267
+
268
+ hook = Sqreen::Graft::Hook[hook_point, strategy]
269
+ hook.add do
270
+ if callback.pre?
271
+ before(rule, rank: priority, mandatory: !callback.overtimeable, flow: block, ignore: ignore) do |call, b|
272
+ next unless Thread.current[:sqreen_http_request]
273
+
274
+ i = call.instance
275
+ a = call.args
276
+ r = call.remaining
277
+
278
+ Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#pre instance=#{i}" }
279
+ begin
280
+ ret = callback.pre(i, a, r)
281
+ rescue StandardError => e
282
+ Sqreen::Weave.logger.warn { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#pre instance=#{i} => exception=#{e}" }
283
+ if callback.respond_to?(:record_exception)
284
+ callback.record_exception(e)
285
+ else
286
+ Sqreen::RemoteException.record(e)
287
+ end
288
+ end
289
+ Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#pre instance=#{i} => return=#{ret.inspect}" }
290
+
291
+ case ret[:status]
292
+ when :skip, 'skip'
293
+ throw(b, b.return(ret[:new_return_value]).break!) if ret.key?(:new_return_value)
294
+ when :modify_args, 'modify_args'
295
+ throw(b, b.args(ret[:args]))
296
+ when :raise, 'raise'
297
+ throw(b, b.raise(ret[:exception])) if ret.key?(:exception)
298
+ throw(b, b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required.")))
299
+ end unless ret.nil? || !ret.is_a?(Hash)
300
+ end
301
+ end
302
+
303
+ if callback.post?
304
+ after(rule, rank: -priority, mandatory: !callback.overtimeable, flow: block, ignore: ignore) do |call, b|
305
+ next unless Thread.current[:sqreen_http_request]
306
+
307
+ i = call.instance
308
+ v = call.returned
309
+ a = call.args
310
+ r = call.remaining
311
+
312
+ Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#post instance=#{i}" }
313
+ begin
314
+ ret = callback.post(v, i, a, r)
315
+ rescue StandardError => e
316
+ Sqreen::Weave.logger.warn { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#post instance=#{i} => exception=#{e}" }
317
+ if callback.respond_to?(:record_exception)
318
+ callback.record_exception(e)
319
+ else
320
+ Sqreen::RemoteException.record(e)
321
+ end
322
+ end
323
+ Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#post instance=#{i} => return=#{ret.inspect}" }
324
+
325
+ case ret[:status]
326
+ when :override, 'override'
327
+ throw(b, b.return(ret[:new_return_value])) if ret.key?(:new_return_value)
328
+ when :raise, 'raise'
329
+ throw(b, b.raise(ret[:exception])) if ret.key?(:exception)
330
+ throw(b, b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required.")))
331
+ end unless ret.nil? || !ret.is_a?(Hash)
332
+ end
333
+ end
334
+
335
+ if callback.failing?
336
+ raised(rule, rank: priority, mandatory: !callback.overtimeable, flow: block, ignore: ignore) do |call, b|
337
+ next unless Thread.current[:sqreen_http_request]
338
+
339
+ i = call.instance
340
+ e = call.raised
341
+ a = call.args
342
+ r = call.remaining
343
+
344
+ Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#failing instance=#{i}" }
345
+ begin
346
+ ret = callback.failing(e, i, a, r)
347
+ rescue StandardError => e
348
+ Sqreen::Weave.logger.warn { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#failing instance=#{i} => exception=#{e}" }
349
+ if callback.respond_to?(:record_exception)
350
+ callback.record_exception(e)
351
+ else
352
+ Sqreen::RemoteException.record(e)
353
+ end
354
+ end
355
+ Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#failing instance=#{i} => return=#{ret.inspect}" }
356
+
357
+ throw(b, b.raise(e)) if ret.nil? || !ret.is_a?(Hash)
358
+
359
+ case ret[:status]
360
+ when :override, 'override'
361
+ throw(b, b.return(ret[:new_return_value])) if ret.key?(:new_return_value)
362
+ when :retry, 'retry'
363
+ throw(b, b.retry)
364
+ when :raise, 'raise'
365
+ throw(b, b.raise(ret[:exception])) if ret.key?(:exception)
366
+ throw(b, b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required.")))
367
+ when :reraise, 'reraise'
368
+ throw(b, b.raise(e))
369
+ else
370
+ throw(b, b.raise(e))
371
+ end unless ret.nil? || !ret.is_a?(Hash)
372
+ end
373
+ end
374
+ end.install
375
+
376
+ hook
377
+ end
378
+
379
+ # needed by Sqreen::Rules.cb_from_rule
380
+ def valid_method?(klass, method)
381
+ if Sqreen::Graft::HookPoint.new("#{klass}.#{method}").exist?
382
+ Sqreen::Weave.logger.debug { "HookPoint found: #{klass}.#{method}" }
383
+ true
384
+ elsif Sqreen::Graft::HookPoint.new("#{klass}##{method}").exist?
385
+ Sqreen::Weave.logger.debug { "HookPoint found: #{klass}##{method}" }
386
+ true
387
+ else
388
+ Sqreen::Weave.logger.debug { "HookPoint not found: #{klass} #{method}" }
389
+ false
390
+ end
391
+ end
392
+
393
+ # needed by #instrument!
394
+ def hardcoded_callbacks(framework)
395
+ [
396
+ ### callback for performing sec responses based on ip
397
+ ### init redefined to implement smartass way to hook it upon the
398
+ ### framework's middleware #call
399
+ Sqreen::Rules::RunReqStartActions.new(framework),
400
+ ### callback for performing sec responses based on user
401
+ Sqreen::Rules::RunUserActions.new(Sqreen, :identify, 0),
402
+ ### callback for performing sec responses based on user
403
+ Sqreen::Rules::RunUserActions.new(Sqreen, :auth_track, 1),
404
+ ]
405
+ end
406
+ end