sqreen 1.18.6-java → 1.20.0-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (190) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +27 -0
  3. data/lib/sqreen/actions.rb +2 -0
  4. data/lib/sqreen/actions/actions_index.rb +16 -0
  5. data/lib/sqreen/actions/base.rb +4 -10
  6. data/lib/sqreen/actions/block_ip.rb +2 -0
  7. data/lib/sqreen/actions/block_user.rb +2 -0
  8. data/lib/sqreen/actions/ip_range_indexed_action_class.rb +4 -24
  9. data/lib/sqreen/actions/ip_ranges_index.rb +32 -11
  10. data/lib/sqreen/actions/redirect_ip.rb +2 -0
  11. data/lib/sqreen/actions/redirect_user.rb +2 -0
  12. data/lib/sqreen/actions/repository.rb +27 -8
  13. data/lib/sqreen/actions/unknown_action_type.rb +4 -0
  14. data/lib/sqreen/actions/user_action_class.rb +5 -30
  15. data/lib/sqreen/actions/users_index.rb +35 -0
  16. data/lib/sqreen/agent.rb +2 -1
  17. data/lib/sqreen/aggregated_metric.rb +25 -0
  18. data/lib/sqreen/attack_blocked.rb +2 -0
  19. data/lib/sqreen/binding_accessor.rb +2 -0
  20. data/lib/sqreen/binding_accessor/path_elem.rb +2 -0
  21. data/lib/sqreen/binding_accessor/transforms.rb +8 -1
  22. data/lib/sqreen/call_countable.rb +2 -0
  23. data/lib/sqreen/capped_queue.rb +2 -0
  24. data/lib/sqreen/cb.rb +2 -0
  25. data/lib/sqreen/cb_tree.rb +2 -0
  26. data/lib/sqreen/condition_evaluator.rb +2 -0
  27. data/lib/sqreen/conditionable.rb +2 -0
  28. data/lib/sqreen/configuration.rb +19 -1
  29. data/lib/sqreen/context.rb +2 -0
  30. data/lib/sqreen/default_cb.rb +2 -0
  31. data/lib/sqreen/deferred_logger.rb +2 -0
  32. data/lib/sqreen/deliveries.rb +2 -0
  33. data/lib/sqreen/deliveries/batch.rb +6 -1
  34. data/lib/sqreen/deliveries/simple.rb +6 -0
  35. data/lib/sqreen/dependency.rb +3 -1
  36. data/lib/sqreen/dependency/detector.rb +22 -14
  37. data/lib/sqreen/dependency/libsqreen.rb +4 -0
  38. data/lib/sqreen/dependency/new_relic.rb +2 -0
  39. data/lib/sqreen/dependency/rack.rb +10 -5
  40. data/lib/sqreen/dependency/rails.rb +4 -0
  41. data/lib/sqreen/dependency/sentry.rb +2 -0
  42. data/lib/sqreen/dependency/sinatra.rb +12 -1
  43. data/lib/sqreen/encoding_sanitizer.rb +2 -0
  44. data/lib/sqreen/error_handling_middleware.rb +2 -0
  45. data/lib/sqreen/event.rb +9 -5
  46. data/lib/sqreen/events/attack.rb +25 -18
  47. data/lib/sqreen/events/remote_exception.rb +2 -22
  48. data/lib/sqreen/events/request_record.rb +17 -70
  49. data/lib/sqreen/exception.rb +2 -0
  50. data/lib/sqreen/formatter_with_tid.rb +2 -0
  51. data/lib/sqreen/framework_cb.rb +2 -0
  52. data/lib/sqreen/frameworks.rb +2 -0
  53. data/lib/sqreen/frameworks/generic.rb +2 -0
  54. data/lib/sqreen/frameworks/rails.rb +1 -0
  55. data/lib/sqreen/frameworks/rails3.rb +2 -0
  56. data/lib/sqreen/frameworks/request_recorder.rb +15 -2
  57. data/lib/sqreen/frameworks/sinatra.rb +2 -0
  58. data/lib/sqreen/frameworks/sqreen_test.rb +2 -0
  59. data/lib/sqreen/graft.rb +12 -0
  60. data/lib/sqreen/graft/call.rb +150 -0
  61. data/lib/sqreen/{dependency → graft}/callback.rb +12 -4
  62. data/lib/sqreen/graft/hook.rb +316 -0
  63. data/lib/sqreen/{dependency → graft}/hook_point.rb +152 -33
  64. data/lib/sqreen/graft/hook_point_error.rb +10 -0
  65. data/lib/sqreen/invalid_signature_exception.rb +2 -0
  66. data/lib/sqreen/js.rb +2 -0
  67. data/lib/sqreen/js/call_context.rb +2 -0
  68. data/lib/sqreen/js/context_pool.rb +2 -0
  69. data/lib/sqreen/js/exec_js_runnable.rb +2 -0
  70. data/lib/sqreen/js/execjs_adapter.rb +2 -0
  71. data/lib/sqreen/js/executable_js.rb +2 -0
  72. data/lib/sqreen/js/js_service.rb +2 -0
  73. data/lib/sqreen/js/js_service_adapter.rb +2 -0
  74. data/lib/sqreen/js/mini_racer_adapter.rb +2 -0
  75. data/lib/sqreen/js/mini_racer_executable_js.rb +2 -0
  76. data/lib/sqreen/js/thread_local_exec_js_runnable.rb +2 -0
  77. data/lib/sqreen/kit/signals/specialized/aggregated_metric.rb +72 -0
  78. data/lib/sqreen/kit/signals/specialized/attack.rb +57 -0
  79. data/lib/sqreen/kit/signals/specialized/binning_metric.rb +76 -0
  80. data/lib/sqreen/kit/signals/specialized/http_trace.rb +26 -0
  81. data/lib/sqreen/kit/signals/specialized/sdk_track_call.rb +50 -0
  82. data/lib/sqreen/kit/signals/specialized/sqreen_exception.rb +57 -0
  83. data/lib/sqreen/{backport.rb → legacy.rb} +3 -2
  84. data/lib/sqreen/{instrumentation.rb → legacy/instrumentation.rb} +31 -2
  85. data/lib/sqreen/legacy/old_event_submission_strategy.rb +221 -0
  86. data/lib/sqreen/legacy/waf_redactions.rb +49 -0
  87. data/lib/sqreen/log.rb +2 -0
  88. data/lib/sqreen/log/loggable.rb +28 -0
  89. data/lib/sqreen/logger.rb +2 -0
  90. data/lib/sqreen/metrics.rb +2 -0
  91. data/lib/sqreen/metrics/average.rb +2 -0
  92. data/lib/sqreen/metrics/base.rb +5 -0
  93. data/lib/sqreen/metrics/binning.rb +2 -0
  94. data/lib/sqreen/metrics/collect.rb +2 -0
  95. data/lib/sqreen/metrics/sum.rb +2 -0
  96. data/lib/sqreen/metrics_store.rb +24 -12
  97. data/lib/sqreen/metrics_store/already_registered_metric.rb +2 -0
  98. data/lib/sqreen/metrics_store/unknown_metric.rb +2 -0
  99. data/lib/sqreen/metrics_store/unregistered_metric.rb +2 -0
  100. data/lib/sqreen/middleware.rb +2 -0
  101. data/lib/sqreen/mono_time.rb +2 -0
  102. data/lib/sqreen/node.rb +2 -0
  103. data/lib/sqreen/not_implemented_yet.rb +2 -0
  104. data/lib/sqreen/null_logger.rb +2 -0
  105. data/lib/sqreen/payload_creator.rb +2 -0
  106. data/lib/sqreen/payload_creator/header_section.rb +2 -0
  107. data/lib/sqreen/performance_notifications.rb +2 -0
  108. data/lib/sqreen/performance_notifications/binned_metrics.rb +10 -2
  109. data/lib/sqreen/performance_notifications/log.rb +2 -0
  110. data/lib/sqreen/performance_notifications/log_performance.rb +2 -0
  111. data/lib/sqreen/performance_notifications/metrics.rb +2 -0
  112. data/lib/sqreen/performance_notifications/newrelic.rb +2 -0
  113. data/lib/sqreen/prefix.rb +2 -0
  114. data/lib/sqreen/rails_middleware.rb +2 -0
  115. data/lib/sqreen/remote_command.rb +2 -0
  116. data/lib/sqreen/remote_command/failure_output.rb +5 -0
  117. data/lib/sqreen/rules.rb +6 -2
  118. data/lib/sqreen/rules/attrs.rb +2 -0
  119. data/lib/sqreen/rules/auth_track_cb.rb +2 -0
  120. data/lib/sqreen/rules/binding_accessor_matcher_cb.rb +2 -0
  121. data/lib/sqreen/rules/binding_accessor_metrics.rb +2 -0
  122. data/lib/sqreen/rules/blacklist_ips_cb.rb +2 -0
  123. data/lib/sqreen/rules/count_http_codes.rb +2 -0
  124. data/lib/sqreen/rules/crawler_user_agent_matches_cb.rb +2 -0
  125. data/lib/sqreen/rules/crawler_user_agent_matches_metrics_cb.rb +2 -0
  126. data/lib/sqreen/rules/custom_error_cb.rb +2 -0
  127. data/lib/sqreen/rules/devise_auth_track_cb.rb +2 -0
  128. data/lib/sqreen/rules/devise_signup_track_cb.rb +2 -0
  129. data/lib/sqreen/rules/execjs_cb.rb +2 -0
  130. data/lib/sqreen/rules/headers_insert_cb.rb +7 -0
  131. data/lib/sqreen/rules/matcher_rule.rb +2 -0
  132. data/lib/sqreen/rules/not_found_cb.rb +7 -0
  133. data/lib/sqreen/rules/rails_parameters_cb.rb +2 -0
  134. data/lib/sqreen/rules/record_request_context.rb +2 -0
  135. data/lib/sqreen/rules/regexp_rule_cb.rb +2 -0
  136. data/lib/sqreen/rules/rule_cb.rb +4 -0
  137. data/lib/sqreen/rules/run_req_start_actions.rb +3 -1
  138. data/lib/sqreen/rules/run_user_actions.rb +3 -1
  139. data/lib/sqreen/rules/shell_env_cb.rb +2 -0
  140. data/lib/sqreen/rules/signup_track_cb.rb +2 -0
  141. data/lib/sqreen/rules/update_request_context.rb +2 -0
  142. data/lib/sqreen/rules/url_matches_cb.rb +2 -0
  143. data/lib/sqreen/rules/user_agent_matches_cb.rb +2 -0
  144. data/lib/sqreen/rules/waf_cb.rb +41 -16
  145. data/lib/sqreen/rules/xss_cb.rb +2 -0
  146. data/lib/sqreen/run_when_called_cb.rb +2 -0
  147. data/lib/sqreen/runner.rb +68 -12
  148. data/lib/sqreen/runtime_infos.rb +2 -0
  149. data/lib/sqreen/safe_json.rb +2 -0
  150. data/lib/sqreen/sdk.rb +4 -0
  151. data/lib/sqreen/sensitive_data_redactor.rb +21 -31
  152. data/lib/sqreen/serializer.rb +2 -0
  153. data/lib/sqreen/session.rb +41 -37
  154. data/lib/sqreen/shared_storage.rb +2 -0
  155. data/lib/sqreen/shared_storage23.rb +2 -0
  156. data/lib/sqreen/shrink_wrap.rb +16 -0
  157. data/lib/sqreen/signals/conversions.rb +283 -0
  158. data/lib/sqreen/signals/http_trace_redaction.rb +111 -0
  159. data/lib/sqreen/signals/signals_submission_strategy.rb +78 -0
  160. data/lib/sqreen/signature_verifier.rb +2 -0
  161. data/lib/sqreen/sinatra_middleware.rb +2 -0
  162. data/lib/sqreen/sqreen_signed_verifier.rb +2 -0
  163. data/lib/sqreen/token_invalid_exception.rb +2 -0
  164. data/lib/sqreen/token_not_found_exception.rb +2 -0
  165. data/lib/sqreen/trie.rb +2 -0
  166. data/lib/sqreen/unauthorized.rb +2 -0
  167. data/lib/sqreen/util.rb +5 -0
  168. data/lib/sqreen/util/capped_array.rb +2 -0
  169. data/lib/sqreen/util/capped_hash.rb +2 -0
  170. data/lib/sqreen/util/capped_string.rb +2 -0
  171. data/lib/sqreen/util/capper.rb +2 -0
  172. data/lib/sqreen/version.rb +3 -1
  173. data/lib/sqreen/waf_error.rb +2 -0
  174. data/lib/sqreen/weave.rb +12 -0
  175. data/lib/sqreen/weave/hardcoded.rb +19 -0
  176. data/lib/sqreen/weave/instrumentor.rb +48 -0
  177. data/lib/sqreen/weave/legacy.rb +12 -0
  178. data/lib/sqreen/weave/legacy/instrumentation.rb +406 -0
  179. data/lib/sqreen/web_server.rb +2 -0
  180. data/lib/sqreen/web_server/generic.rb +2 -0
  181. data/lib/sqreen/web_server/passenger.rb +2 -0
  182. data/lib/sqreen/web_server/puma.rb +2 -0
  183. data/lib/sqreen/web_server/rainbows.rb +2 -0
  184. data/lib/sqreen/web_server/thin.rb +2 -0
  185. data/lib/sqreen/web_server/unicorn.rb +2 -0
  186. data/lib/sqreen/web_server/webrick.rb +2 -0
  187. data/lib/sqreen/worker.rb +2 -0
  188. metadata +65 -9
  189. data/lib/sqreen/backport/original_name.rb +0 -86
  190. data/lib/sqreen/dependency/hook.rb +0 -102
@@ -0,0 +1,111 @@
1
+ require 'json'
2
+ require 'sqreen/kit/loggable'
3
+ require 'sqreen/kit/signals/specialized/http_trace'
4
+
5
+ module Sqreen
6
+ module Signals
7
+ module HttpTraceRedaction
8
+ class << self
9
+ include Sqreen::Kit::Loggable
10
+
11
+ # @param [Sqreen::Kit::Signals::Specialized::HttpTrace] trace
12
+ # @param [Sqreen::SensitiveDataRedactor] redactor
13
+ def redact_trace!(trace, redactor)
14
+ return unless redactor
15
+ # redact headers (keys unsafe)
16
+ # @type [Sqreen::Kit::Signals::Context::HttpContext]
17
+ http_context = trace.context
18
+
19
+ all_redacted = []
20
+
21
+ # Redact headers; save redacted values
22
+ # headers are encoded as [key, value], not a hash, so
23
+ # they require some transformation
24
+ orig_headers = http_context.headers
25
+ if orig_headers
26
+ headers = orig_headers.map { |(k, v)| { k => v } }
27
+ headers, redacted = redactor.redact(headers)
28
+ http_context.headers = headers.map(&:first)
29
+ all_redacted += redacted
30
+ end
31
+
32
+ # Redact params; save redacted values
33
+ Kit::Signals::Context::HttpContext::PARAMS_ATTRS.each do |attr|
34
+ value = http_context.public_send(attr)
35
+ next unless value
36
+ value, redacted = redactor.redact(value)
37
+ all_redacted += redacted
38
+ http_context.public_send(:"#{attr}=", value)
39
+ end
40
+
41
+ all_redacted = all_redacted.uniq.map(&:downcase)
42
+
43
+ # Redact attacks and exceptions
44
+ # XXX: no redaction for infos in attacks/exceptions except for WAF data
45
+ # Is this the correct behavior?
46
+ redact_attacks!(trace, redactor, all_redacted)
47
+ redact_exceptions!(trace, redactor, all_redacted)
48
+ end
49
+
50
+ private
51
+
52
+ # @param [Sqreen::Kit::Signals::Specialized::HttpTrace] trace
53
+ # @param [Sqreen::SensitiveDataRedactor] redactor
54
+ # Redacts WAF data according to specific rules therefor
55
+ # Redacts infos according to general rules
56
+ def redact_attacks!(trace, redactor, redacted_data)
57
+ trace.data.each do |signal|
58
+ next unless signal.is_a?(Kit::Signals::Specialized::Attack)
59
+ # @type [Sqreen::Kit::Signals::Specialized::Attack::Payload] payload
60
+ payload = signal.payload
61
+ next unless payload.infos
62
+
63
+ if payload.infos[:waf_data]
64
+ redact_waf_attack_data!(payload.infos, redacted_data)
65
+ end
66
+ payload.infos, = redactor.redact(payload.infos)
67
+ end
68
+ end
69
+
70
+ def redact_exceptions!(trace, redactor, redacted_data)
71
+ trace.data.each do |signal|
72
+ next unless signal.is_a?(Kit::Signals::Specialized::SqreenException)
73
+ infos = signal.infos
74
+ next unless infos
75
+
76
+ redact_waf_exception_data!(signal.infos, redacted_data) if signal.infos[:waf]
77
+ signal.infos, = redactor.redact(infos)
78
+ end
79
+ end
80
+
81
+ # @param [Hash] infos from WAF attack
82
+ def redact_waf_attack_data!(infos, redacted_data)
83
+ begin
84
+ parsed = JSON.parse(infos[:waf_data])
85
+ rescue JSON::JSONError => e
86
+ logger.warn("waf_data is not valid json: #{e.message}")
87
+ return
88
+ end
89
+ redacted = parsed.each do |w|
90
+ next unless (filters = w['filter'])
91
+
92
+ filters.each do |f|
93
+ next unless (v = f['resolved_value'])
94
+ next unless redacted_data.include?(v.downcase)
95
+
96
+ f['match_status'] = SensitiveDataRedactor::MASK
97
+ f['resolved_value'] = SensitiveDataRedactor::MASK
98
+ end
99
+ end
100
+ infos[:waf_data] = JSON.dump(redacted)
101
+ end
102
+
103
+ # see https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000022-waf-data-sanitization.md#changes-to-the-agents
104
+ def redact_waf_exception_data!(infos, redacted_data)
105
+ return if redacted_data.empty?
106
+ infos[:waf].delete(:args)
107
+ end
108
+ end
109
+ end
110
+ end
111
+ end
@@ -0,0 +1,78 @@
1
+ require 'sqreen/aggregated_metric'
2
+ require 'sqreen/kit'
3
+ require 'sqreen/kit/string_sanitizer'
4
+ require 'sqreen/signals/conversions'
5
+ require 'sqreen/log/loggable'
6
+
7
+ module Sqreen
8
+ module Signals
9
+ # see also Sqreen::Legacy::OldEventSubmissionStrategy
10
+ # usage in Sqreen:Session
11
+ class SignalsSubmissionStrategy
12
+ include Sqreen::Log::Loggable
13
+
14
+ # @param [Array<Sqreen::AggregatedMetric>] metrics
15
+ def post_metrics(metrics)
16
+ return if metrics.nil? || metrics.empty?
17
+
18
+ guarded 'Failed to serialize or submit aggregated metrics' do
19
+ batch = metrics.map do |m|
20
+ Conversions.convert_metric_sample(m)
21
+ end
22
+ client.report_batch(batch)
23
+ end
24
+ end
25
+
26
+ # @param _attack [Sqreen::Attack]
27
+ # XXX: unused
28
+ def post_attack(_attack)
29
+ raise NotImplementedError
30
+ end
31
+
32
+ # @param request_record [Sqreen::RequestRecord]
33
+ def post_request_record(request_record)
34
+ guarded 'Failed to serialize or submit request record' do
35
+ trace = Conversions.convert_req_record(request_record)
36
+ append_sanitizing_filter(trace)
37
+ client.report_trace(trace)
38
+ end
39
+ end
40
+
41
+ # Post an exception to Sqreen for analysis
42
+ # @param exception [RemoteException] Exception and context to be sent over
43
+ def post_sqreen_exception(exception)
44
+ guarded 'Failed to serialize or submit exception', false do
45
+ data = Conversions.convert_exception(exception)
46
+ append_sanitizing_filter(data)
47
+ client.report_signal(data)
48
+ end
49
+ end
50
+
51
+ def post_batch(events)
52
+ guarded 'Failed to serialize or submit batch of events' do
53
+ batch = Conversions.convert_batch(events)
54
+ batch.each { |sig_or_trace| append_sanitizing_filter(sig_or_trace) }
55
+ client.report_batch(batch)
56
+ end
57
+ end
58
+
59
+ private
60
+
61
+ def append_sanitizing_filter(sig_or_trace)
62
+ sig_or_trace.append_to_h_filter Kit::StringSanitizer.method(:sanitize)
63
+ end
64
+
65
+ # we don't want exceptions to propagate and kill the worker thread
66
+ def guarded(msg, report = true)
67
+ yield
68
+ rescue StandardError => e
69
+ logger.warn "#{msg}: #{e.message}\n#{e.backtrace.map { |x| " #{x}" }.join("\n")}"
70
+ post_sqreen_exception(RemoteException.new(e)) if report
71
+ end
72
+
73
+ def client
74
+ Sqreen::Kit.auth_signals_client
75
+ end
76
+ end
77
+ end
78
+ end
@@ -1,3 +1,5 @@
1
+ # typed: true
2
+
1
3
  # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
3
5
 
@@ -1,3 +1,5 @@
1
+ # typed: true
2
+
1
3
  # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
3
5
 
@@ -1,3 +1,5 @@
1
+ # typed: false
2
+
1
3
  # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
3
5
 
@@ -1,3 +1,5 @@
1
+ # typed: strong
2
+
1
3
  # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
3
5
 
@@ -1,3 +1,5 @@
1
+ # typed: strong
2
+
1
3
  # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
3
5
 
@@ -1,3 +1,5 @@
1
+ # typed: false
2
+
1
3
  # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
3
5
 
@@ -1,3 +1,5 @@
1
+ # typed: strong
2
+
1
3
  # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
3
5
 
@@ -1,2 +1,7 @@
1
+ # typed: strong
2
+
3
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
4
+ # Please refer to our terms for more information: https://www.sqreen.com/terms.html
5
+
1
6
  module Sqreen; end
2
7
  module Sqreen::Util; end
@@ -1,3 +1,5 @@
1
+ # typed: false
2
+
1
3
  # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
3
5
 
@@ -1,3 +1,5 @@
1
+ # typed: false
2
+
1
3
  # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
3
5
 
@@ -1,3 +1,5 @@
1
+ # typed: false
2
+
1
3
  # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
3
5
 
@@ -1,3 +1,5 @@
1
+ # typed: true
2
+
1
3
  # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
3
5
 
@@ -1,6 +1,8 @@
1
+ # typed: true
2
+
1
3
  # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
3
5
 
4
6
  module Sqreen
5
- VERSION = '1.18.6'.freeze
7
+ VERSION = '1.20.0'.freeze
6
8
  end
@@ -1,3 +1,5 @@
1
+ # typed: true
2
+
1
3
  # Copyright (c) 2015 Sqreen. All Rights Reserved.
2
4
  # Please refer to our terms for more information: https://www.sqreen.com/terms.html
3
5
 
@@ -0,0 +1,12 @@
1
+ # typed: strong
2
+
3
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
4
+ # Please refer to our terms for more information: https://www.sqreen.com/terms.html
5
+
6
+ require 'sqreen/log/loggable'
7
+
8
+ module Sqreen
9
+ module Weave
10
+ include Sqreen::Log::Loggable
11
+ end
12
+ end
@@ -0,0 +1,19 @@
1
+ # typed: strong
2
+
3
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
4
+ # Please refer to our terms for more information: https://www.sqreen.com/terms.html
5
+
6
+ require 'sqreen/weave'
7
+
8
+ class Sqreen::Weave::Hardcoded
9
+ # [
10
+ # ### callback for performing sec responses based on ip
11
+ # ### init redefined to implement smartass way to hook it upon the
12
+ # ### framework's middleware #call
13
+ # Sqreen::Rules::RunReqStartActions.new(framework),
14
+ # ### callback for performing sec responses based on user
15
+ # Sqreen::Rules::RunUserActions.new(Sqreen, :identify, 0),
16
+ # ### callback for performing sec responses based on user
17
+ # Sqreen::Rules::RunUserActions.new(Sqreen, :auth_track, 1),
18
+ # ]
19
+ end
@@ -0,0 +1,48 @@
1
+ # typed: true
2
+
3
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
4
+ # Please refer to our terms for more information: https://www.sqreen.com/terms.html
5
+
6
+ require 'sqreen/weave'
7
+
8
+ # rule loader: decouple from runner
9
+ # remote rules from back
10
+ # local rules from local files
11
+ # => rule list (what is a rule?)
12
+ # => to callback (what is a callback?)
13
+ # => to instrumentation (== attach callbacks to their targets using graft)
14
+
15
+ # make shit like instrument framework independent (block passing?)
16
+ # => too much things assume only one framework
17
+ # possible to do run req actions without hardcoded cbs?
18
+ # (data comes from actions command, native cb merely binds to middleware)
19
+ # can cb be a form of abstraction?
20
+
21
+ # rule sig: decouple/split
22
+ # - data signer/checker
23
+ # apply this to rule data
24
+
25
+ # whitelist is mixed in
26
+
27
+ # metrics
28
+ # three dedicated metrics: abstract and isolate
29
+
30
+ class Sqreen::Weave::Instrumentor
31
+ def initialize(metrics_engine)
32
+ ### bail out if no metric engine
33
+ ### init metric to count calls to sqreen
34
+ ### init metric to count request whitelist matches (ip or path whitelist)
35
+ ### init metric to count over budget hits
36
+ end
37
+
38
+ def instrument!(rules, framework)
39
+ ### set up rule signature verifier
40
+ ### force clean instrumentation callback list
41
+ ### for each rule description, transform into format for adding callback
42
+ ### attach framework to callback
43
+ ### install callback, observing priority
44
+ ### for each hardcoded callback
45
+ ### install hardcoded callbacks, observing priority
46
+ ### globally declare instrumentation ready
47
+ end
48
+ end
@@ -0,0 +1,12 @@
1
+ # typed: strong
2
+
3
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
4
+ # Please refer to our terms for more information: https://www.sqreen.com/terms.html
5
+
6
+ require 'sqreen/weave'
7
+
8
+ module Sqreen
9
+ module Weave
10
+ module Legacy; end
11
+ end
12
+ end
@@ -0,0 +1,406 @@
1
+ # typed: false
2
+
3
+ # Copyright (c) 2015 Sqreen. All Rights Reserved.
4
+ # Please refer to our terms for more information: https://www.sqreen.com/terms.html
5
+
6
+ require 'sqreen/weave/legacy'
7
+ require 'sqreen/graft/hook_point'
8
+ require 'sqreen/call_countable'
9
+ require 'sqreen/rules'
10
+ require 'sqreen/rules/record_request_context'
11
+
12
+ class Sqreen::Weave::Legacy::Instrumentation
13
+ attr_accessor :metrics_engine
14
+
15
+ def initialize(metrics_engine, opts = {})
16
+ Sqreen::Weave.logger.debug { "#{self.class.name}#initialize #{metrics_engine}" }
17
+ @hooks = []
18
+
19
+ self.metrics_engine = metrics_engine
20
+
21
+ ### bail out if no metric engine
22
+ return if metrics_engine.nil?
23
+
24
+ ### init metric to count calls to sqreen
25
+ metrics_engine.create_metric(
26
+ 'name' => 'sqreen_call_counts',
27
+ 'period' => 60,
28
+ 'kind' => 'Sum',
29
+ )
30
+ ### init metric to count request whitelist matches (ip or path whitelist)
31
+ metrics_engine.create_metric(
32
+ 'name' => 'whitelisted',
33
+ 'period' => 60,
34
+ 'kind' => 'Sum',
35
+ )
36
+ ### init metric to count over budget hits
37
+ metrics_engine.create_metric(
38
+ 'name' => 'request_overtime',
39
+ 'period' => 60,
40
+ 'kind' => 'Sum',
41
+ )
42
+
43
+ # PerformanceNotifications::Binning
44
+ metrics_engine.create_metric(
45
+ 'name' => 'req',
46
+ 'period' => opts[:period] || 60,
47
+ 'kind' => 'Binning',
48
+ 'options' => opts[:perf_metric] || { 'base' => 2.0, 'factor' => 0.1 },
49
+ )
50
+ metrics_engine.create_metric(
51
+ 'name' => 'sq',
52
+ 'period' => opts[:period] || 60,
53
+ 'kind' => 'Binning',
54
+ 'options' => opts[:perf_metric] || { 'base' => 2.0, 'factor' => 0.1 },
55
+ )
56
+ metrics_engine.create_metric(
57
+ 'name' => 'pct',
58
+ 'period' => opts[:period] || 60,
59
+ 'kind' => 'Binning',
60
+ 'options' => opts[:perf_metric_percent] || { 'base' => 1.3, 'factor' => 1.0 },
61
+ )
62
+
63
+ Sqreen.thread_cpu_time? && metrics_engine.create_metric(
64
+ 'name' => 'sq_thread_cpu_pct',
65
+ 'period' => opts[:period] || 60,
66
+ 'kind' => 'Binning',
67
+ 'options' => opts[:perf_metric_percent] || { 'base' => 1.3, 'factor' => 1.0 },
68
+ )
69
+ end
70
+
71
+ # needed by Sqreen::Runner#initialize
72
+ def instrument!(rules, framework)
73
+ Sqreen::Weave.logger.debug { "#{rules.count} rules, #{framework}" }
74
+
75
+ strategy = Sqreen.config_get(:weave_strategy)
76
+ if strategy == :prepend && !Module.respond_to?(:prepend)
77
+ Sqreen::Weave.logger.warn { "strategy: #{strategy.inspect} unavailable, falling back to :chain" }
78
+ strategy = :chain
79
+ elsif strategy == :chain && Gem::Specification.select { |s| s.name == 'scout_apm' && Gem::Requirement.new('>= 2.5.2').satisfied_by?(Gem::Version.new(s.version)) }.any?
80
+ Sqreen::Weave.logger.warn { "strategy: #{strategy.inspect} unavailable with scout_apm >= 2.5.2, switching to :prepend" }
81
+ strategy = :prepend
82
+ end
83
+ Sqreen::Weave.logger.debug { "strategy: #{strategy.inspect}" }
84
+
85
+ ### set up rule signature verifier
86
+ verifier = nil
87
+ ### force clean instrumentation callback list
88
+ @hooks = []
89
+ ### for each rule description
90
+ rules.each do |rule|
91
+ Sqreen::Weave.logger.debug { "Processing rule: #{rule['name']}" }
92
+ ### transform into format for adding callback
93
+ rule_callback = Sqreen::Rules.cb_from_rule(rule, self, metrics_engine, verifier)
94
+ next unless rule_callback
95
+ ### attach framework to callback
96
+ rule_callback.framework = framework
97
+ ### install callback, observing priority
98
+ Sqreen::Weave.logger.debug { "Adding rule callback: #{rule_callback}" }
99
+ @hooks << add_callback("weave,rule=#{rule['name']}", rule_callback, strategy)
100
+ end
101
+
102
+ ### for each hardcoded callback
103
+ hardcoded_callbacks(framework).each do |hard_callback|
104
+ Sqreen::Weave.logger.debug { "Adding hardcoded callback: #{hard_callback}" }
105
+ ### install hardcoded callbacks, observing priority
106
+ @hooks << add_callback('weave,hardcoded', hard_callback, strategy)
107
+ end
108
+
109
+ metrics_engine = self.metrics_engine
110
+ request_hook = Sqreen::Graft::Hook['Sqreen::ShrinkWrap#call', strategy]
111
+ @hooks << request_hook
112
+ request_hook.add do
113
+ before('wave,meta,request', rank: -100000, mandatory: true) do |_call|
114
+ next unless Sqreen.instrumentation_ready
115
+
116
+ uuid = SecureRandom.uuid
117
+ now = Sqreen::Graft::Timer.read
118
+ Thread.current[:sqreen_http_request] = {
119
+ uuid: uuid,
120
+ start_time: now,
121
+ time_budget: Sqreen.performance_budget,
122
+ time_budget_expended: false,
123
+ timer: Sqreen::Graft::Timer.new("request_#{uuid}"),
124
+ timed_callbacks: [],
125
+ timed_hooks: [],
126
+ timed_hooks_before: [],
127
+ timed_hooks_after: [],
128
+ timed_hooks_raised: [],
129
+ timed_hooks_ensured: [],
130
+ skipped_callbacks: [],
131
+ }
132
+
133
+ Sqreen::Weave.logger.debug { "request.uuid: #{uuid}" }
134
+ end
135
+
136
+ ensured('weave,meta,request', rank: 100000, mandatory: true) do |_call|
137
+ request = Thread.current[:sqreen_http_request]
138
+
139
+ next if request.nil?
140
+
141
+ Thread.current[:sqreen_http_request] = nil
142
+ now = Sqreen::Graft::Timer.read
143
+ utc_now = Time.now.utc
144
+
145
+ request[:timed_callbacks].each do |timer|
146
+ duration = timer.duration
147
+ # stop = now
148
+ # start = now - duration
149
+ timer.tag =~ /weave,rule=(.*)$/ && rule = $1
150
+ timer.tag =~ /@before/ && whence = 'pre'
151
+ timer.tag =~ /@after/ && whence = 'post'
152
+ timer.tag =~ /@raised/ && whence = 'failing'
153
+
154
+ next unless rule && whence
155
+
156
+ # Sqreen::PerformanceNotifications.notify(rule, whence, start, stop)
157
+ # => BinnedMetrics
158
+ metric_name = "sq.#{rule}.#{whence}"
159
+ unless metrics_engine.metric?(metric_name)
160
+ metrics_engine.create_metric(
161
+ 'name' => metric_name,
162
+ 'period' => 60,
163
+ 'kind' => 'Binning',
164
+ 'options' => { 'base' => 2.0, 'factor' => 0.1 },
165
+ )
166
+ end
167
+ metrics_engine.update(metric_name, now, nil, duration * 1000)
168
+ end
169
+
170
+ metric_name = 'sq.hooks_pre.pre'
171
+ duration = request[:timed_hooks_before].sum(&:duration)
172
+ unless metrics_engine.metric?(metric_name)
173
+ metrics_engine.create_metric(
174
+ 'name' => metric_name,
175
+ 'period' => 60,
176
+ 'kind' => 'Binning',
177
+ 'options' => { 'base' => 2.0, 'factor' => 0.1 },
178
+ )
179
+ end
180
+ metrics_engine.update(metric_name, now, nil, duration * 1000)
181
+
182
+ metric_name = 'sq.hooks_post.post'
183
+ duration = request[:timed_hooks_after].sum(&:duration)
184
+ unless metrics_engine.metric?(metric_name)
185
+ metrics_engine.create_metric(
186
+ 'name' => metric_name,
187
+ 'period' => 60,
188
+ 'kind' => 'Binning',
189
+ 'options' => { 'base' => 2.0, 'factor' => 0.1 },
190
+ )
191
+ end
192
+ metrics_engine.update(metric_name, now, nil, duration * 1000)
193
+
194
+ metric_name = 'sq.hooks_failing.failing'
195
+ duration = request[:timed_hooks_raised].sum(&:duration)
196
+ unless metrics_engine.metric?(metric_name)
197
+ metrics_engine.create_metric(
198
+ 'name' => metric_name,
199
+ 'period' => 60,
200
+ 'kind' => 'Binning',
201
+ 'options' => { 'base' => 2.0, 'factor' => 0.1 },
202
+ )
203
+ end
204
+ metrics_engine.update(metric_name, now, nil, duration * 1000)
205
+
206
+ skipped = request[:skipped_callbacks].map(&:name)
207
+ Sqreen::Weave.logger.debug { "request:#{request[:uuid]} callback.skipped.size: #{skipped.count} callback.skipped: [#{skipped.join(', ')}]" }
208
+ timer = request[:timer]
209
+ total = timer.duration
210
+ Sqreen::Weave.logger.debug { "request:#{request[:uuid]} timer.total: #{'%.03fus' % (total * 1_000_000)} timer.size: #{timer.size}" }
211
+ timings = request[:timed_callbacks].map(&:to_s)
212
+ total = request[:timed_callbacks].sum(&:duration)
213
+ Sqreen::Weave.logger.debug { "request:#{request[:uuid]} callback.total: #{'%.03fus' % (total * 1_000_000)} callback.timings: [#{timings.join(', ')}]" }
214
+ timings = request[:timed_hooks].map(&:to_s)
215
+ total = request[:timed_hooks].sum(&:duration)
216
+ Sqreen::Weave.logger.debug { "request:#{request[:uuid]} hook.total: #{'%.03fus' % (total * 1_000_000)} hook.timings: [#{timings.join(', ')}]" }
217
+
218
+ skipped = request[:skipped_callbacks].map(&:name)
219
+ skipped_rule_name = skipped.first && skipped.first =~ /weave,rule=(.*)$/ && $1
220
+ Sqreen.observations_queue.push(['request_overtime', skipped_rule_name, 1, utc_now]) if skipped_rule_name
221
+
222
+ sqreen_request_duration = total
223
+ Sqreen.observations_queue.push(['sq', nil, sqreen_request_duration * 1000, utc_now])
224
+
225
+ request_duration = now - request[:start_time]
226
+ Sqreen.observations_queue.push(['req', nil, request_duration * 1000, utc_now])
227
+
228
+ sqreen_request_ratio = (sqreen_request_duration * 100.0) / (request_duration - sqreen_request_duration)
229
+ Sqreen.observations_queue.push(['pct', nil, sqreen_request_ratio, utc_now])
230
+ end
231
+ end.install
232
+
233
+ ### globally declare instrumentation ready
234
+ Sqreen.instrumentation_ready = true
235
+ end
236
+
237
+ # needed by Sqreen::Runner
238
+ def remove_all_callbacks
239
+ Sqreen.instrumentation_ready = false
240
+
241
+ loop do
242
+ hook = @hooks.pop
243
+ break unless hook
244
+ Sqreen::Weave.logger.debug { "hook.deinstrument: #{hook}" }
245
+ hook.uninstall
246
+ hook.clear
247
+ end
248
+ end
249
+
250
+ # needed by #instrument!
251
+ def add_callback(rule, callback, strategy)
252
+ Sqreen::Weave.logger.debug { "Adding rule: #{rule} callback: #{callback}" }
253
+ klass = callback.klass
254
+ method = callback.method
255
+
256
+ if Sqreen::Graft::HookPoint.new("#{klass}.#{method}").exist?
257
+ hook_point = "#{klass}.#{method}"
258
+ elsif Sqreen::Graft::HookPoint.new("#{klass}##{method}").exist?
259
+ hook_point = "#{klass}##{method}"
260
+ end
261
+
262
+ return if hook_point.nil?
263
+
264
+ priority = callback.priority || 100
265
+ block = callback.respond_to?(:block) ? callback.block : true
266
+ ignore = -> { callback.whitelisted? } if callback.respond_to?(:whitelisted?)
267
+
268
+ hook = Sqreen::Graft::Hook[hook_point, strategy]
269
+ hook.add do
270
+ if callback.pre?
271
+ before(rule, rank: priority, mandatory: !callback.overtimeable, flow: block, ignore: ignore) do |call, b|
272
+ next unless Thread.current[:sqreen_http_request]
273
+
274
+ i = call.instance
275
+ a = call.args
276
+ r = call.remaining
277
+
278
+ Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#pre instance=#{i}" }
279
+ begin
280
+ ret = callback.pre(i, a, r)
281
+ rescue StandardError => e
282
+ Sqreen::Weave.logger.warn { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#pre instance=#{i} => exception=#{e}" }
283
+ if callback.respond_to?(:record_exception)
284
+ callback.record_exception(e)
285
+ else
286
+ Sqreen::RemoteException.record(e)
287
+ end
288
+ end
289
+ Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#pre instance=#{i} => return=#{ret.inspect}" }
290
+
291
+ case ret[:status]
292
+ when :skip, 'skip'
293
+ throw(b, b.return(ret[:new_return_value]).break!) if ret.key?(:new_return_value)
294
+ when :modify_args, 'modify_args'
295
+ throw(b, b.args(ret[:args]))
296
+ when :raise, 'raise'
297
+ throw(b, b.raise(ret[:exception])) if ret.key?(:exception)
298
+ throw(b, b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required.")))
299
+ end unless ret.nil? || !ret.is_a?(Hash)
300
+ end
301
+ end
302
+
303
+ if callback.post?
304
+ after(rule, rank: -priority, mandatory: !callback.overtimeable, flow: block, ignore: ignore) do |call, b|
305
+ next unless Thread.current[:sqreen_http_request]
306
+
307
+ i = call.instance
308
+ v = call.returned
309
+ a = call.args
310
+ r = call.remaining
311
+
312
+ Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#post instance=#{i}" }
313
+ begin
314
+ ret = callback.post(v, i, a, r)
315
+ rescue StandardError => e
316
+ Sqreen::Weave.logger.warn { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#post instance=#{i} => exception=#{e}" }
317
+ if callback.respond_to?(:record_exception)
318
+ callback.record_exception(e)
319
+ else
320
+ Sqreen::RemoteException.record(e)
321
+ end
322
+ end
323
+ Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#post instance=#{i} => return=#{ret.inspect}" }
324
+
325
+ case ret[:status]
326
+ when :override, 'override'
327
+ throw(b, b.return(ret[:new_return_value])) if ret.key?(:new_return_value)
328
+ when :raise, 'raise'
329
+ throw(b, b.raise(ret[:exception])) if ret.key?(:exception)
330
+ throw(b, b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required.")))
331
+ end unless ret.nil? || !ret.is_a?(Hash)
332
+ end
333
+ end
334
+
335
+ if callback.failing?
336
+ raised(rule, rank: priority, mandatory: !callback.overtimeable, flow: block, ignore: ignore) do |call, b|
337
+ next unless Thread.current[:sqreen_http_request]
338
+
339
+ i = call.instance
340
+ e = call.raised
341
+ a = call.args
342
+ r = call.remaining
343
+
344
+ Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#failing instance=#{i}" }
345
+ begin
346
+ ret = callback.failing(e, i, a, r)
347
+ rescue StandardError => e
348
+ Sqreen::Weave.logger.warn { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#failing instance=#{i} => exception=#{e}" }
349
+ if callback.respond_to?(:record_exception)
350
+ callback.record_exception(e)
351
+ else
352
+ Sqreen::RemoteException.record(e)
353
+ end
354
+ end
355
+ Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#failing instance=#{i} => return=#{ret.inspect}" }
356
+
357
+ throw(b, b.raise(e)) if ret.nil? || !ret.is_a?(Hash)
358
+
359
+ case ret[:status]
360
+ when :override, 'override'
361
+ throw(b, b.return(ret[:new_return_value])) if ret.key?(:new_return_value)
362
+ when :retry, 'retry'
363
+ throw(b, b.retry)
364
+ when :raise, 'raise'
365
+ throw(b, b.raise(ret[:exception])) if ret.key?(:exception)
366
+ throw(b, b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required.")))
367
+ when :reraise, 'reraise'
368
+ throw(b, b.raise(e))
369
+ else
370
+ throw(b, b.raise(e))
371
+ end unless ret.nil? || !ret.is_a?(Hash)
372
+ end
373
+ end
374
+ end.install
375
+
376
+ hook
377
+ end
378
+
379
+ # needed by Sqreen::Rules.cb_from_rule
380
+ def valid_method?(klass, method)
381
+ if Sqreen::Graft::HookPoint.new("#{klass}.#{method}").exist?
382
+ Sqreen::Weave.logger.debug { "HookPoint found: #{klass}.#{method}" }
383
+ true
384
+ elsif Sqreen::Graft::HookPoint.new("#{klass}##{method}").exist?
385
+ Sqreen::Weave.logger.debug { "HookPoint found: #{klass}##{method}" }
386
+ true
387
+ else
388
+ Sqreen::Weave.logger.debug { "HookPoint not found: #{klass} #{method}" }
389
+ false
390
+ end
391
+ end
392
+
393
+ # needed by #instrument!
394
+ def hardcoded_callbacks(framework)
395
+ [
396
+ ### callback for performing sec responses based on ip
397
+ ### init redefined to implement smartass way to hook it upon the
398
+ ### framework's middleware #call
399
+ Sqreen::Rules::RunReqStartActions.new(framework),
400
+ ### callback for performing sec responses based on user
401
+ Sqreen::Rules::RunUserActions.new(Sqreen, :identify, 0),
402
+ ### callback for performing sec responses based on user
403
+ Sqreen::Rules::RunUserActions.new(Sqreen, :auth_track, 1),
404
+ ]
405
+ end
406
+ end