sqreen 1.19.0 → 1.20.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (42) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +22 -0
  3. data/lib/sqreen/agent_message.rb +20 -0
  4. data/lib/sqreen/aggregated_metric.rb +25 -0
  5. data/lib/sqreen/ca.crt +24 -0
  6. data/lib/sqreen/configuration.rb +10 -4
  7. data/lib/sqreen/deliveries/batch.rb +4 -1
  8. data/lib/sqreen/deliveries/simple.rb +4 -0
  9. data/lib/sqreen/endpoint_testing.rb +184 -0
  10. data/lib/sqreen/event.rb +7 -5
  11. data/lib/sqreen/events/attack.rb +23 -18
  12. data/lib/sqreen/events/remote_exception.rb +0 -22
  13. data/lib/sqreen/events/request_record.rb +15 -70
  14. data/lib/sqreen/frameworks/request_recorder.rb +13 -2
  15. data/lib/sqreen/kit/signals/specialized/aggregated_metric.rb +72 -0
  16. data/lib/sqreen/kit/signals/specialized/attack.rb +57 -0
  17. data/lib/sqreen/kit/signals/specialized/binning_metric.rb +76 -0
  18. data/lib/sqreen/kit/signals/specialized/http_trace.rb +26 -0
  19. data/lib/sqreen/kit/signals/specialized/sdk_track_call.rb +50 -0
  20. data/lib/sqreen/kit/signals/specialized/sqreen_exception.rb +57 -0
  21. data/lib/sqreen/legacy/old_event_submission_strategy.rb +221 -0
  22. data/lib/sqreen/legacy/waf_redactions.rb +49 -0
  23. data/lib/sqreen/log/loggable.rb +1 -1
  24. data/lib/sqreen/metrics/base.rb +3 -0
  25. data/lib/sqreen/metrics_store.rb +22 -12
  26. data/lib/sqreen/performance_notifications/binned_metrics.rb +8 -2
  27. data/lib/sqreen/rules.rb +4 -2
  28. data/lib/sqreen/rules/not_found_cb.rb +2 -0
  29. data/lib/sqreen/rules/rule_cb.rb +2 -0
  30. data/lib/sqreen/rules/waf_cb.rb +13 -10
  31. data/lib/sqreen/runner.rb +75 -8
  32. data/lib/sqreen/sensitive_data_redactor.rb +19 -31
  33. data/lib/sqreen/session.rb +51 -43
  34. data/lib/sqreen/signals/conversions.rb +283 -0
  35. data/lib/sqreen/signals/http_trace_redaction.rb +111 -0
  36. data/lib/sqreen/signals/signals_submission_strategy.rb +78 -0
  37. data/lib/sqreen/version.rb +1 -1
  38. data/lib/sqreen/weave/legacy/instrumentation.rb +7 -7
  39. metadata +50 -6
  40. data/lib/sqreen/backport.rb +0 -9
  41. data/lib/sqreen/backport/clock_gettime.rb +0 -74
  42. data/lib/sqreen/backport/original_name.rb +0 -88
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d9ee940962990208133a70a5b307638cd43dda2b269cccce3f145ed96816e64a
4
- data.tar.gz: '012987d55219ddd310337f6c0a9837d1e52aae1b7e165ab24e855c4f5fbeb767'
3
+ metadata.gz: 7be645cedf514d1af5754fc7244738e17f817aad09761d2da7d7fa032ac55801
4
+ data.tar.gz: f81295c9ac98714284d6368bc5e01c1a857317c8dad98d53366b24e53cc6cc47
5
5
  SHA512:
6
- metadata.gz: 602e605c2d82de6d011dba78ce03411973ca25c2fb8a00322af9cb80937eff2997f8eda20e00d08bb2106073a3ffe8bbf4bc7f01409d123d510ebb16a7915405
7
- data.tar.gz: 2363c998cb4af54018638c7d6e46f54e6d98afcac94c6be5747a3c1f555bdd59321441b3196c85f1eec08555dc579af80cdf6728ade7dfb4b9bf0f506669d55d
6
+ metadata.gz: 986748fc2c11ee0a6ea7a997d661246d2840e170fdcbe5d7c1221fb189e3b6a9281e4feb1025f58f1ac6712ff10f598caa7d81867a7a62898138e743eadc9262
7
+ data.tar.gz: 39b246ce351e780f539a0d2fe3df9791996d8203056545ab5472f0deec2e1526351bfc9cbb4f8c4cd5c145969db0b177b1b559a7491e4a8a15de4e9dd3721665
@@ -1,3 +1,25 @@
1
+ ## 1.20.1
2
+
3
+ * Add fallback mechanisms when connecting to new Sqreen backend API domains
4
+
5
+ ## 1.20.0
6
+
7
+ * Enable new instrumentation engine by default
8
+ * Add signal-based backend communication
9
+
10
+ ## 1.19.3
11
+
12
+ * Improve WAF PII protection
13
+
14
+ ## 1.19.2
15
+
16
+ * Handle unexpected rule callback return values more gracefully
17
+ * Fix incorrect return value for 404 native callback
18
+
19
+ ## 1.19.1
20
+
21
+ * Fix LocalJumpError when reaching a Rack app nested in a Rails app
22
+
1
23
  ## 1.19.0
2
24
 
3
25
  * Upgrade WAF features via libsqreen 0.6.1
@@ -0,0 +1,20 @@
1
+ require 'digest'
2
+
3
+ module Sqreen
4
+ class AgentMessage
5
+ def initialize(kind, message, id = nil)
6
+ id ||= message + "\x00" + kind
7
+ @hash_hex = Digest::SHA1.hexdigest(id)
8
+ @kind = kind
9
+ @message = message
10
+ end
11
+
12
+ def to_h
13
+ {
14
+ id: @hash_hex,
15
+ kind: @kind,
16
+ message: @message,
17
+ }
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,25 @@
1
+ require 'sqreen/rules/rule_cb'
2
+ require 'sqreen/metrics/base'
3
+
4
+ module Sqreen
5
+ class AggregatedMetric
6
+ def initialize(values = {})
7
+ values.each do |k, v|
8
+ public_send "#{k}=", v
9
+ end
10
+ end
11
+
12
+ # @return [Sqreen::Rules::RuleCB]
13
+ attr_accessor :rule # optional
14
+
15
+ # @return [Sqreen::Metric::Base]
16
+ attr_accessor :metric
17
+
18
+ attr_accessor :start, :finish
19
+ attr_accessor :data
20
+
21
+ def name
22
+ metric.name
23
+ end
24
+ end
25
+ end
@@ -70,3 +70,27 @@ WE9gyn6CagsCqiUXObXbf+eEZSqVir2G3l6BFoMtEMze/aiCKm0oHw0LxOXnGiYZ
70
70
  4fQRbxC1lfznQgUy286dUV4otp6F01vvpX1FQHKOtw5rDgb7MzVIcbidJ4vEZV8N
71
71
  hnacRHr2lVz2XTIIM6RUthg/aFzyQkqFOFSDX9HoLPKsEdao7WNq
72
72
  -----END CERTIFICATE-----
73
+ -----BEGIN CERTIFICATE-----
74
+ MIID7zCCAtegAwIBAgIBADANBgkqhkiG9w0BAQsFADCBmDELMAkGA1UEBhMCVVMx
75
+ EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxJTAjBgNVBAoT
76
+ HFN0YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4xOzA5BgNVBAMTMlN0YXJmaWVs
77
+ ZCBTZXJ2aWNlcyBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5
78
+ MDkwMTAwMDAwMFoXDTM3MTIzMTIzNTk1OVowgZgxCzAJBgNVBAYTAlVTMRAwDgYD
79
+ VQQIEwdBcml6b25hMRMwEQYDVQQHEwpTY290dHNkYWxlMSUwIwYDVQQKExxTdGFy
80
+ ZmllbGQgVGVjaG5vbG9naWVzLCBJbmMuMTswOQYDVQQDEzJTdGFyZmllbGQgU2Vy
81
+ dmljZXMgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgLSBHMjCCASIwDQYJKoZI
82
+ hvcNAQEBBQADggEPADCCAQoCggEBANUMOsQq+U7i9b4Zl1+OiFOxHz/Lz58gE20p
83
+ OsgPfTz3a3Y4Y9k2YKibXlwAgLIvWX/2h/klQ4bnaRtSmpDhcePYLQ1Ob/bISdm2
84
+ 8xpWriu2dBTrz/sm4xq6HZYuajtYlIlHVv8loJNwU4PahHQUw2eeBGg6345AWh1K
85
+ Ts9DkTvnVtYAcMtS7nt9rjrnvDH5RfbCYM8TWQIrgMw0R9+53pBlbQLPLJGmpufe
86
+ hRhJfGZOozptqbXuNC66DQO4M99H67FrjSXZm86B0UVGMpZwh94CDklDhbZsc7tk
87
+ 6mFBrMnUVN+HL8cisibMn1lUaJ/8viovxFUcdUBgF4UCVTmLfwUCAwEAAaNCMEAw
88
+ DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFJxfAN+q
89
+ AdcwKziIorhtSpzyEZGDMA0GCSqGSIb3DQEBCwUAA4IBAQBLNqaEd2ndOxmfZyMI
90
+ bw5hyf2E3F/YNoHN2BtBLZ9g3ccaaNnRbobhiCPPE95Dz+I0swSdHynVv/heyNXB
91
+ ve6SbzJ08pGCL72CQnqtKrcgfU28elUSwhXqvfdqlS5sdJ/PHLTyxQGjhdByPq1z
92
+ qwubdQxtRbeOlKyWN7Wg0I8VRw7j6IPdj/3vQQF3zCepYoUz8jcI73HPdwbeyBkd
93
+ iEDPfUYd/x7H4c7/I9vG+o1VTqkC50cRRj70/b17KSa7qWFiNyi2LSr2EIZkyXCn
94
+ 0q23KXB56jzaYyWf/Wi3MOxw+3WKt21gZ7IeyLnp2KhvAotnDU0mV3HaIPzBSlCN
95
+ sSi6
96
+ -----END CERTIFICATE-----
@@ -39,11 +39,15 @@ module Sqreen
39
39
  { :env => :SQREEN_LIBSQREEN, :name => :libsqreen,
40
40
  :default => true, :convert => :to_bool },
41
41
  { :env => :SQREEN_WEAVE, :name => :weave,
42
- :default => false, :convert => :to_bool },
42
+ :default => true, :convert => :to_bool },
43
43
  { :env => :SQREEN_WEAVE_STRATEGY, :name => :weave_strategy,
44
- :default => :chain, :convert => :to_sym },
45
- { :env => :SQREEN_URL, :name => :url,
46
- :default => 'https://back.sqreen.io' },
44
+ :default => :prepend, :convert => :to_sym },
45
+ { :env => :SQREEN_URL, :name => :url,
46
+ :default => nil },
47
+ { :env => :SQREEN_INGESTION_URL, :name => :ingestion_url,
48
+ :default => nil },
49
+ { :env => :SQREEN_PROXY_URL, :name => :proxy_url,
50
+ :default => nil },
47
51
  { :env => :SQREEN_TOKEN, :name => :token,
48
52
  :default => nil },
49
53
  { :env => :SQREEN_APP_NAME, :name => :app_name,
@@ -74,6 +78,8 @@ module Sqreen
74
78
  :default => nil },
75
79
  { :env => :SQREEN_STRIP_SENSITIVE_REGEX, :name => :strip_sensitive_regex,
76
80
  :default => nil },
81
+ { :env => :SQREEN_NO_SNIFF_DOMAINS, :name => :no_sniff_domains,
82
+ :default => false },
77
83
 
78
84
  ].freeze
79
85
 
@@ -8,6 +8,7 @@
8
8
  # TODO: Sqreen::RequestRecord => sqreen/events
9
9
  # TODO: Sqreen.time
10
10
 
11
+ require 'sqreen/aggregated_metric'
11
12
  require 'sqreen/events/attack'
12
13
  require 'sqreen/events/remote_exception'
13
14
  require 'sqreen/mono_time'
@@ -91,9 +92,11 @@ module Sqreen
91
92
  def event_key(event)
92
93
  case event
93
94
  when Sqreen::Attack
94
- "att-#{event.type}"
95
+ "att-#{event.rule_name}"
95
96
  when Sqreen::RemoteException
96
97
  "rex-#{event.klass}"
98
+ when Sqreen::AggregatedMetric
99
+ "agg-metric"
97
100
  end
98
101
  end
99
102
  end
@@ -7,6 +7,7 @@
7
7
  # TODO: Sqreen::RemoteException => sqreen/events
8
8
  # TODO: Sqreen::RequestRecord => sqreen/events
9
9
 
10
+ require 'sqreen/log/loggable'
10
11
  require 'sqreen/events/attack'
11
12
  require 'sqreen/events/remote_exception'
12
13
  require 'sqreen/events/request_record'
@@ -15,6 +16,7 @@ module Sqreen
15
16
  module Deliveries
16
17
  # Simple delivery method that directly call session on event
17
18
  class Simple
19
+ include Log::Loggable
18
20
  attr_accessor :session
19
21
 
20
22
  def initialize(session)
@@ -29,6 +31,8 @@ module Sqreen
29
31
  session.post_sqreen_exception(event)
30
32
  when Sqreen::RequestRecord
31
33
  session.post_request_record(event)
34
+ when Sqreen::AggregatedMetric
35
+ logger.warn 'Delivery of metrics using signals is not supported with simple delivery'
32
36
  else
33
37
  session.post_event(event)
34
38
  end
@@ -0,0 +1,184 @@
1
+ require 'net/https'
2
+ require 'sqreen/agent_message'
3
+ require 'sqreen/log/loggable'
4
+
5
+ module Sqreen
6
+ class EndpointTesting
7
+ Endpoint = Struct.new(:url, :ca_store)
8
+ class ChosenEndpoints
9
+ def initialize
10
+ @messages = []
11
+ end
12
+
13
+ # @return [Sqreen::EndpointTesting::Endpoint]
14
+ attr_accessor :control
15
+
16
+ # @return [Sqreen::EndpointTesting::Endpoint]
17
+ attr_accessor :ingestion
18
+
19
+ # @return [Array<Sqreen::AgentMessage>]
20
+ attr_reader :messages
21
+
22
+ # @param [Sqreen::AgentMessage] message
23
+ def add_message(message)
24
+ @messages << message
25
+ end
26
+ end
27
+
28
+ MAIN_CONTROL_HOST = 'back.sqreen.com'.freeze
29
+ MAIN_INJECTION_HOST = 'ingestion.sqreen.com'.freeze
30
+ FALLBACK_ENDPOINT_URL = 'https://back.sqreen.io/'.freeze
31
+ GLOBAL_TIMEOUT = 30
32
+
33
+ CONTROL_ERROR_KIND = 'back_sqreen_com_unavailable'.freeze
34
+ INGESTION_ERROR_KIND = 'ingestion_sqreen_com_unavailable'.freeze
35
+
36
+ class << self
37
+ include Log::Loggable
38
+
39
+ # reproduces behaviour before endpoint testing was introduced
40
+ def no_test_endpoints(config_url, config_ingestion_url)
41
+ endpoints = ChosenEndpoints.new
42
+
43
+ endpoints.control = Endpoint.new(
44
+ config_url || "https://#{MAIN_CONTROL_HOST}/", cert_store
45
+ )
46
+ endpoints.ingestion = Endpoint.new(
47
+ config_ingestion_url || "https://#{MAIN_INJECTION_HOST}/", nil
48
+ )
49
+
50
+ endpoints
51
+ end
52
+
53
+ def test_endpoints(proxy_url, config_url, config_ingestion_url)
54
+ proxy_params = create_proxy_params(proxy_url)
55
+
56
+ # execute the tests in separate threads and wait for them
57
+ thread_control = Thread.new do
58
+ thread_main(config_url, proxy_params, MAIN_CONTROL_HOST)
59
+ end
60
+ thread_injection = Thread.new do
61
+ thread_main(config_ingestion_url, proxy_params, MAIN_INJECTION_HOST)
62
+ end
63
+
64
+ wait_for_threads(thread_control, thread_injection)
65
+
66
+ # build and return result
67
+ fallback = Endpoint.new(FALLBACK_ENDPOINT_URL, cert_store)
68
+ endpoints = ChosenEndpoints.new
69
+ endpoints.control = thread_control[:endpoint] || fallback
70
+ endpoints.ingestion = thread_injection[:endpoint] || fallback
71
+
72
+ if thread_control[:endpoint_error]
73
+ msg = AgentMessage.new(CONTROL_ERROR_KIND, thread_control[:endpoint_error])
74
+ endpoints.add_message msg
75
+ end
76
+ if thread_injection[:endpoint_error]
77
+ msg = AgentMessage.new(INGESTION_ERROR_KIND, thread_injection[:endpoint_error])
78
+ endpoints.add_message msg
79
+ end
80
+
81
+ endpoints
82
+ end
83
+
84
+ private
85
+
86
+ def thread_main(configured_url, proxy_params, host)
87
+ res = if configured_url
88
+ Endpoint.new(configured_url, nil)
89
+ else
90
+ EndpointTesting.send(:test_with_store_variants, proxy_params, host)
91
+ end
92
+
93
+ Thread.current[:endpoint] = res
94
+ rescue StandardError => e
95
+ Thread.current[:endpoint_error] = e.message
96
+ end
97
+
98
+ def create_proxy_params(proxy_url)
99
+ return [] unless proxy_url
100
+
101
+ proxy = URI.parse(proxy_url)
102
+
103
+ return [] unless proxy.scheme == 'http'
104
+
105
+ [proxy.host, proxy.port, proxy.user, proxy.password]
106
+ end
107
+
108
+ def test_with_store_variants(proxy_params, server_name)
109
+ # first without custom store
110
+ do_test(proxy_params, server_name, false)
111
+ rescue StandardError => _e
112
+ do_test(proxy_params, server_name, true)
113
+ end
114
+
115
+ # @param [Array] proxy_params
116
+ # @param [String] server_name
117
+ # @param [Boolean] custom_store
118
+ def do_test(proxy_params, server_name, custom_store)
119
+ http = Net::HTTP.new(server_name, 443, *proxy_params)
120
+ http.use_ssl = true
121
+ http.verify_mode = OpenSSL::SSL::VERIFY_NONE if ENV['SQREEN_SSL_NO_VERIFY']
122
+ http.verify_callback = lambda do |preverify_ok, ctx|
123
+ unless preverify_ok
124
+ logger.warn do
125
+ "Certificate validation failure for certificate issued to " \
126
+ "#{ctx.chain[0].subject}: #{ctx.error_string}"
127
+ end
128
+ end
129
+
130
+ preverify_ok
131
+ end
132
+
133
+ http.open_timeout = 13
134
+ http.ssl_timeout = 7
135
+ http.read_timeout = 7
136
+ http.close_on_empty_response = true
137
+
138
+ http.cert_store = cert_store if custom_store
139
+
140
+ resp = http.get('/ping')
141
+
142
+ logger.info do
143
+ "Got response from #{server_name}'s ping endpoint. " \
144
+ "Status code is #{resp.code} (custom CA store: #{custom_store})"
145
+ end
146
+
147
+ unless resp.code == '200'
148
+ raise "Response code for /ping is #{resp.code}, not 200"
149
+ end
150
+
151
+ Endpoint.new("https://#{server_name}/", http.cert_store)
152
+ rescue StandardError => e
153
+ logger.info do
154
+ "Error in request to #{server_name} " \
155
+ "(custom store: #{custom_store}): #{e.message}"
156
+ end
157
+
158
+ raise "Error in request to #{server_name}: #{e.message}"
159
+ end
160
+
161
+ def wait_for_threads(thread_control, thread_injection)
162
+ deadline = Time.now + GLOBAL_TIMEOUT
163
+ [thread_control, thread_injection].each do |thread|
164
+ rem = deadline - Time.now
165
+ rem = 0.1 if rem < 0.1
166
+ next if thread.join(rem)
167
+ logger.debug { "Timeout for thread #{thread}" }
168
+ thread.kill
169
+ thread[:endpoint_error] = "Timeout doing endpoint testing"
170
+ end
171
+ end
172
+
173
+ def cert_store
174
+ @cert_store ||= begin
175
+ cert_file = File.join(File.dirname(__FILE__), 'ca.crt')
176
+ cert_store = OpenSSL::X509::Store.new
177
+ cert_store.add_file cert_file
178
+
179
+ cert_store
180
+ end
181
+ end
182
+ end
183
+ end
184
+ end
@@ -8,17 +8,19 @@
8
8
  module Sqreen
9
9
  # Master interface for point in time events (e.g. Attack, RemoteException)
10
10
  class Event
11
+ # @return [Hash]
11
12
  attr_reader :payload
13
+
14
+ # @return [Time]
15
+ attr_accessor :time # writer used only in tests
16
+
12
17
  def initialize(payload)
13
18
  @payload = payload
14
- end
15
-
16
- def to_hash
17
- payload.to_hash
19
+ @time = Time.now.utc
18
20
  end
19
21
 
20
22
  def to_s
21
- "<#{self.class.name}: #{to_hash}>"
23
+ "<#{self.class.name}: #{payload.to_hash}>"
22
24
  end
23
25
  end
24
26
  end
@@ -11,6 +11,8 @@ module Sqreen
11
11
  # Attack
12
12
  # When creating a new attack, it gets automatically pushed to the event's
13
13
  # queue.
14
+ # XXX: TURNS OUT THIS CLASS IS ACTUALLY NOT USED ANYMORE
15
+ # Framework.observe is used instead with unstructured attack details
14
16
  class Attack < Event
15
17
  def self.record(payload)
16
18
  attack = Attack.new(payload)
@@ -26,11 +28,31 @@ module Sqreen
26
28
  payload['rule']['rulespack_id']
27
29
  end
28
30
 
29
- def type
31
+ def rule_name
30
32
  return nil unless payload['rule']
31
33
  payload['rule']['name']
32
34
  end
33
35
 
36
+ def test?
37
+ return nil unless payload['rule']
38
+ payload['rule']['test'] ? true : false
39
+ end
40
+
41
+ def beta?
42
+ return nil unless payload['rule']
43
+ payload['rule']['beta'] ? true : false
44
+ end
45
+
46
+ def block?
47
+ return nil unless payload['rule']
48
+ payload['rule']['block'] ? true : false
49
+ end
50
+
51
+ def attack_type
52
+ return nil unless payload['rule']
53
+ payload['rule']['attack_type']
54
+ end
55
+
34
56
  def time
35
57
  return nil unless payload['local']
36
58
  payload['local']['time']
@@ -44,22 +66,5 @@ module Sqreen
44
66
  def enqueue
45
67
  Sqreen.queue.push(self)
46
68
  end
47
-
48
- def to_hash
49
- res = {}
50
- rule_p = payload['rule']
51
- request_p = payload['request']
52
- res[:rule_name] = rule_p['name'] if rule_p && rule_p['name']
53
- res[:rulespack_id] = rule_p['rulespack_id'] if rule_p && rule_p['rulespack_id']
54
- res[:test] = rule_p['test'] if rule_p && rule_p['test']
55
- res[:infos] = payload['infos'] if payload['infos']
56
- res[:time] = time if time
57
- res[:client_ip] = request_p[:addr] if request_p && request_p[:addr]
58
- res[:request] = request_p if request_p
59
- res[:params] = payload['params'] if payload['params']
60
- res[:context] = payload['context'] if payload['context']
61
- res[:headers] = payload['headers'] if payload['headers']
62
- res
63
- end
64
69
  end
65
70
  end