sqreen 1.19.0 → 1.20.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (42) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +22 -0
  3. data/lib/sqreen/agent_message.rb +20 -0
  4. data/lib/sqreen/aggregated_metric.rb +25 -0
  5. data/lib/sqreen/ca.crt +24 -0
  6. data/lib/sqreen/configuration.rb +10 -4
  7. data/lib/sqreen/deliveries/batch.rb +4 -1
  8. data/lib/sqreen/deliveries/simple.rb +4 -0
  9. data/lib/sqreen/endpoint_testing.rb +184 -0
  10. data/lib/sqreen/event.rb +7 -5
  11. data/lib/sqreen/events/attack.rb +23 -18
  12. data/lib/sqreen/events/remote_exception.rb +0 -22
  13. data/lib/sqreen/events/request_record.rb +15 -70
  14. data/lib/sqreen/frameworks/request_recorder.rb +13 -2
  15. data/lib/sqreen/kit/signals/specialized/aggregated_metric.rb +72 -0
  16. data/lib/sqreen/kit/signals/specialized/attack.rb +57 -0
  17. data/lib/sqreen/kit/signals/specialized/binning_metric.rb +76 -0
  18. data/lib/sqreen/kit/signals/specialized/http_trace.rb +26 -0
  19. data/lib/sqreen/kit/signals/specialized/sdk_track_call.rb +50 -0
  20. data/lib/sqreen/kit/signals/specialized/sqreen_exception.rb +57 -0
  21. data/lib/sqreen/legacy/old_event_submission_strategy.rb +221 -0
  22. data/lib/sqreen/legacy/waf_redactions.rb +49 -0
  23. data/lib/sqreen/log/loggable.rb +1 -1
  24. data/lib/sqreen/metrics/base.rb +3 -0
  25. data/lib/sqreen/metrics_store.rb +22 -12
  26. data/lib/sqreen/performance_notifications/binned_metrics.rb +8 -2
  27. data/lib/sqreen/rules.rb +4 -2
  28. data/lib/sqreen/rules/not_found_cb.rb +2 -0
  29. data/lib/sqreen/rules/rule_cb.rb +2 -0
  30. data/lib/sqreen/rules/waf_cb.rb +13 -10
  31. data/lib/sqreen/runner.rb +75 -8
  32. data/lib/sqreen/sensitive_data_redactor.rb +19 -31
  33. data/lib/sqreen/session.rb +51 -43
  34. data/lib/sqreen/signals/conversions.rb +283 -0
  35. data/lib/sqreen/signals/http_trace_redaction.rb +111 -0
  36. data/lib/sqreen/signals/signals_submission_strategy.rb +78 -0
  37. data/lib/sqreen/version.rb +1 -1
  38. data/lib/sqreen/weave/legacy/instrumentation.rb +7 -7
  39. metadata +50 -6
  40. data/lib/sqreen/backport.rb +0 -9
  41. data/lib/sqreen/backport/clock_gettime.rb +0 -74
  42. data/lib/sqreen/backport/original_name.rb +0 -88
@@ -24,6 +24,8 @@ module Sqreen
24
24
  exception = env['action_dispatch.exception']
25
25
 
26
26
  record_from_env(ua, script_name, path_info, verb, override, host, exception)
27
+
28
+ nil
27
29
  end
28
30
 
29
31
  def record_from_env(ua, script_name, path_info, verb, override, host, exception)
@@ -61,7 +61,9 @@ module Sqreen
61
61
  :infos => infos,
62
62
  :rulespack_id => rulespack_id,
63
63
  :rule_name => rule_name,
64
+ :attack_type => @rule['attack_type'], # for signal
64
65
  :test => test,
66
+ :block => @rule['block'], # for signal
65
67
  :time => at,
66
68
  }
67
69
  if payload_tpl.include?('context')
@@ -98,10 +98,10 @@ module Sqreen
98
98
 
99
99
  case action
100
100
  when :monitor
101
- record_event({ 'waf_data' => data })
101
+ record_event({ waf_data: data })
102
102
  advise_action(nil)
103
103
  when :block
104
- record_event({ 'waf_data' => data })
104
+ record_event({ waf_data: data })
105
105
  advise_action(:raise)
106
106
  when :good
107
107
  advise_action(nil)
@@ -132,20 +132,23 @@ module Sqreen
132
132
  end
133
133
 
134
134
  def record_exception(exception, infos = {}, at = Time.now.utc)
135
- infos.merge!(exception_to_infos(exception)) if exception.is_a?(Sqreen::WAFError)
135
+ infos.merge!(waf_infos(exception)) if exception.is_a?(Sqreen::WAFError)
136
136
  super(exception, infos, at)
137
137
  end
138
138
 
139
139
  private
140
140
 
141
- def exception_to_infos(e)
141
+ # see https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000016-waf-integration.md#error-management
142
+ def waf_infos(e)
142
143
  {
143
- waf_rule: e.rule_name,
144
- error_code: ERROR_CODES[e.error],
145
- }.tap do |r|
146
- r[:error_data] = e.data if e.data
147
- r[:args] = e.args if e.args
148
- end
144
+ waf: {
145
+ waf_rule: e.rule_name,
146
+ error_code: ERROR_CODES[e.error],
147
+ }.tap do |r|
148
+ r[:error_data] = e.data if e.data
149
+ r[:args] = e.args if e.arg
150
+ end,
151
+ }
149
152
  end
150
153
 
151
154
  ERROR_CODES = {
@@ -11,6 +11,7 @@ require 'sqreen/events/attack'
11
11
 
12
12
  require 'sqreen/log'
13
13
 
14
+ require 'sqreen/agent_message'
14
15
  require 'sqreen/rules'
15
16
  require 'sqreen/session'
16
17
  require 'sqreen/remote_command'
@@ -18,11 +19,13 @@ require 'sqreen/capped_queue'
18
19
  require 'sqreen/metrics_store'
19
20
  require 'sqreen/deliveries/simple'
20
21
  require 'sqreen/deliveries/batch'
22
+ require 'sqreen/endpoint_testing'
21
23
  require 'sqreen/performance_notifications/metrics'
22
24
  require 'sqreen/performance_notifications/binned_metrics'
23
25
  require 'sqreen/legacy/instrumentation'
24
26
  require 'sqreen/call_countable'
25
27
  require 'sqreen/weave/legacy/instrumentation'
28
+ require 'sqreen/kit/configuration'
26
29
 
27
30
  module Sqreen
28
31
  @features = {}
@@ -37,6 +40,8 @@ module Sqreen
37
40
  PERF_METRICS_PERIOD = 60 # 1 min
38
41
  DEFAULT_PERF_LEVEL = 0 # disabled
39
42
 
43
+ DEFAULT_USE_SIGNALS = false
44
+
40
45
  class << self
41
46
  attr_reader :features
42
47
  def update_features(features)
@@ -87,7 +92,9 @@ module Sqreen
87
92
 
88
93
  attr_accessor :heartbeat_delay
89
94
  attr_accessor :metrics_engine
95
+ # @return [Sqreen::Deliveries::Simple]
90
96
  attr_reader :deliverer
97
+ # @return [Sqreen::Session]
91
98
  attr_reader :session
92
99
  attr_reader :instrumenter
93
100
  attr_accessor :running
@@ -108,15 +115,24 @@ module Sqreen
108
115
  @next_metrics = []
109
116
  @running = true
110
117
 
118
+ @proxy_url = @configuration.get(:proxy_url)
119
+ chosen_endpoints = determine_endpoints
120
+
111
121
  @token = @configuration.get(:token)
112
122
  @app_name = @configuration.get(:app_name)
113
- @url = @configuration.get(:url)
123
+ @url = chosen_endpoints.control.url
124
+ @cert_store = chosen_endpoints.control.ca_store
125
+
114
126
  Sqreen.update_whitelisted_paths([])
115
127
  Sqreen.update_whitelisted_ips({})
116
128
  Sqreen.update_performance_budget(nil)
117
- raise(Sqreen::Exception, 'no url found') unless @url
118
129
  raise(Sqreen::TokenNotFoundException, 'no token found') unless @token
119
130
 
131
+ Sqreen::Kit::Configuration.logger = Sqreen.log
132
+ Sqreen::Kit::Configuration.ingestion_url = chosen_endpoints.ingestion.url
133
+ Sqreen::Kit::Configuration.certificate_store = chosen_endpoints.ingestion.ca_store
134
+ Sqreen::Kit::Configuration.proxy_url = @proxy_url
135
+
120
136
  register_exit_cb if set_at_exit
121
137
 
122
138
  self.metrics_engine = MetricsStore.new
@@ -133,6 +149,7 @@ module Sqreen
133
149
 
134
150
  Sqreen.log.debug "Using token #{@token}"
135
151
  response = create_session(session_class)
152
+ post_endpoint_testing_msgs(chosen_endpoints)
136
153
  wanted_features = response.fetch('features', {})
137
154
  conf_initial_features = configuration.get(:initial_features)
138
155
  unless conf_initial_features.nil?
@@ -142,10 +159,10 @@ module Sqreen
142
159
  Sqreen.log.debug do
143
160
  "Override initial features with #{conf_features.inspect}"
144
161
  end
145
- wanted_features = conf_features
162
+ wanted_features = wanted_features.merge(conf_features)
146
163
  rescue
147
164
  Sqreen.log.warn do
148
- "NOT using invalid inital features #{conf_initial_features}"
165
+ "NOT using invalid initial features #{conf_initial_features}"
149
166
  end
150
167
  end
151
168
  end
@@ -161,7 +178,7 @@ module Sqreen
161
178
  end
162
179
 
163
180
  def create_session(session_class)
164
- @session = session_class.new(@url, @token, @app_name)
181
+ @session = session_class.new(@url, @cert_store, @token, @app_name, @proxy_url)
165
182
  session.login(@framework)
166
183
  end
167
184
 
@@ -170,8 +187,18 @@ module Sqreen
170
187
  @deliverer = new_deliverer
171
188
  end
172
189
 
173
- def batch_events(batch_size, max_staleness = nil)
190
+ def batch_events(batch_size, max_staleness = nil, use_signals = false)
174
191
  size = batch_size.to_i
192
+
193
+ if size <= 1 && use_signals
194
+ Sqreen.log.warn do
195
+ "Using signals with no delivery batching is unsupported. " \
196
+ "Using instead batching with batch size = 30, max_staleness = 60"
197
+ end
198
+ size = 30
199
+ max_staleness = 60
200
+ end
201
+
175
202
  self.deliverer = if size < 1
176
203
  Deliveries::Simple.new(session)
177
204
  else
@@ -301,19 +328,37 @@ module Sqreen
301
328
  def do_heartbeat
302
329
  @last_heartbeat_request = Time.now
303
330
  @next_metrics.concat(metrics_engine.publish(false)) if metrics_engine
304
- res = session.heartbeat(next_command_results, next_metrics)
331
+ metrics_in_hb = use_signals? ? nil : next_metrics
332
+
333
+ res = session.heartbeat(next_command_results, metrics_in_hb)
305
334
  next_command_results.clear
335
+
336
+ deliver_metrics_as_event if use_signals?
306
337
  next_metrics.clear
338
+
307
339
  process_commands(res['commands'])
308
340
  end
309
341
 
342
+ def deliver_metrics_as_event
343
+ # this is disastrous withe simple delivery strategy,
344
+ # as each aggregated metric would trigger an http request
345
+ # Sending of metrics is therefore not supported with simple delivery strategy
346
+ # TODO: Confirm that only batch is used in production
347
+ next_metrics.each { |x| deliverer.post_event(x) }
348
+ end
349
+
310
350
  def features(_context_infos = {})
311
351
  Sqreen.features
312
352
  end
313
353
 
354
+ def use_signals?
355
+ features.fetch('use_signals', DEFAULT_USE_SIGNALS)
356
+ end
357
+
314
358
  def features=(features)
315
359
  Sqreen.update_features(features)
316
360
  session.request_compression = features['request_compression'] if session
361
+ session.use_signals = use_signals?
317
362
  self.performance_metrics_period = features['performance_metrics_period']
318
363
 
319
364
  unless @configuration.get(:weave)
@@ -331,7 +376,7 @@ module Sqreen
331
376
  hd = features['heartbeat_delay'].to_i
332
377
  self.heartbeat_delay = hd if hd > 0
333
378
  return if features['batch_size'].nil?
334
- batch_events(features['batch_size'], features['max_staleness'])
379
+ batch_events(features['batch_size'], features['max_staleness'], use_signals?)
335
380
  end
336
381
 
337
382
  def change_whitelisted_paths(paths, _context_infos = {})
@@ -470,6 +515,28 @@ module Sqreen
470
515
 
471
516
  private
472
517
 
518
+ def post_endpoint_testing_msgs(chosen_endpoints)
519
+ chosen_endpoints.messages.each do |msg|
520
+ session.post_agent_message(@framework, msg)
521
+ end
522
+ rescue => e
523
+ Sqreen.log.warn "Error submitting agent message: #{e}"
524
+ RemoteException.record(e)
525
+ end
526
+
527
+ def determine_endpoints
528
+ # there's no sniffing going on; just a misnamed config setting
529
+ if @configuration.get(:no_sniff_domains)
530
+ # reproduces behaviour before endpoint testing was introduced
531
+ EndpointTesting.no_test_endpoints(@configuration.get(:url),
532
+ @configuration.get(:ingestion_url))
533
+ else
534
+ EndpointTesting.test_endpoints(@proxy_url,
535
+ @configuration.get(:url),
536
+ @configuration.get(:ingestion_url))
537
+ end
538
+ end
539
+
473
540
  def load_actions(hashes)
474
541
  unsupported = Set.new
475
542
 
@@ -61,7 +61,7 @@ module Sqreen
61
61
  obj.each do |k, v|
62
62
  ck = k.is_a?(String) ? k.downcase : k
63
63
  if @keys.include?(ck)
64
- redacted << v
64
+ redacted += SensitiveDataRedactor.all_strings(v)
65
65
  v = MASK
66
66
  else
67
67
  v, r = redact(v)
@@ -74,39 +74,27 @@ module Sqreen
74
74
  [result, redacted]
75
75
  end
76
76
 
77
- def redact_attacks!(attacks, values)
78
- return attacks if values.empty?
79
-
80
- values = values.map { |v| v.downcase if v.is_a?(String) }
81
-
82
- attacks.each do |e|
83
- next(e) unless e[:infos]
84
- next(e) unless e[:infos][:waf_data]
85
-
86
- parsed = JSON.parse(e[:infos][:waf_data])
87
- redacted = parsed.each do |w|
88
- next unless (filters = w['filter'])
89
-
90
- filters.each do |f|
91
- next unless (v = f['resolved_value'])
92
- next unless values.include?(v.downcase)
77
+ class << self
78
+ def all_strings(v)
79
+ accum = []
80
+ all_strings_impl(v, accum)
81
+ accum
82
+ end
93
83
 
94
- f['match_status'] = MASK
95
- f['resolved_value'] = MASK
84
+ private
85
+
86
+ def all_strings_impl(obj, accum)
87
+ case obj
88
+ when String
89
+ accum << obj
90
+ when Array
91
+ obj.each { |el| all_strings_impl(el, accum) }
92
+ when Hash
93
+ obj.each do |k, v|
94
+ all_strings_impl(k, accum)
95
+ all_strings_impl(v, accum)
96
96
  end
97
97
  end
98
- e[:infos][:waf_data] = JSON.dump(redacted)
99
- end
100
- end
101
-
102
- def redact_exceptions!(exceptions, values)
103
- return exceptions if values.empty?
104
-
105
- exceptions.each do |e|
106
- next(e) unless e[:infos]
107
- next(e) unless e[:infos][:waf]
108
-
109
- e[:infos][:waf].delete(:args)
110
98
  end
111
99
  end
112
100
  end
@@ -11,6 +11,10 @@ require 'sqreen/events/attack'
11
11
  require 'sqreen/events/request_record'
12
12
  require 'sqreen/exception'
13
13
  require 'sqreen/safe_json'
14
+ require 'sqreen/kit'
15
+ require 'sqreen/kit/configuration'
16
+ require 'sqreen/signals/signals_submission_strategy'
17
+ require 'sqreen/legacy/old_event_submission_strategy'
14
18
 
15
19
  require 'net/https'
16
20
  require 'uri'
@@ -41,13 +45,12 @@ module Sqreen
41
45
  RETRY_MANY = 301
42
46
 
43
47
  MUTEX = Mutex.new
44
- METRICS_KEY = 'metrics'.freeze
45
48
 
46
49
  @@path_prefix = '/sqreen/v0/'
47
50
 
48
51
  attr_accessor :request_compression
49
52
 
50
- def initialize(server_url, token, app_name = nil)
53
+ def initialize(server_url, cert_store, token, app_name = nil, proxy_url = nil)
51
54
  @token = token
52
55
  @app_name = app_name
53
56
  @session_id = nil
@@ -59,15 +62,29 @@ module Sqreen
59
62
  uri = parse_uri(server_url)
60
63
  use_ssl = (uri.scheme == 'https')
61
64
 
65
+ proxy_params = []
66
+ if proxy_url
67
+ proxy_uri = parse_uri(proxy_url)
68
+ proxy_params = [proxy_uri.host, proxy_uri.port, proxy_uri.user, proxy_uri.password]
69
+ end
70
+
62
71
  @req_nb = 0
63
72
 
64
- @http = Net::HTTP.new(uri.host, uri.port)
73
+ @http = Net::HTTP.new(uri.host, uri.port, *proxy_params)
65
74
  @http.use_ssl = use_ssl
66
- if use_ssl
67
- cert_file = File.join(File.dirname(__FILE__), 'ca.crt')
68
- cert_store = OpenSSL::X509::Store.new
69
- cert_store.add_file cert_file
70
- @http.cert_store = cert_store
75
+ @http.verify_mode = OpenSSL::SSL::VERIFY_NONE if ENV['SQREEN_SSL_NO_VERIFY'] # for testing
76
+ @http.cert_store = cert_store if use_ssl
77
+ self.use_signals = false
78
+ end
79
+
80
+ def use_signals=(do_use)
81
+ return if do_use == @use_signals
82
+
83
+ @use_signals = do_use
84
+ if do_use
85
+ @evt_sub_strategy = Sqreen::Signals::SignalsSubmissionStrategy.new
86
+ else
87
+ @evt_sub_strategy = Sqreen::Legacy::OldEventSubmissionStrategy.new(method(:post))
71
88
  end
72
89
  end
73
90
 
@@ -218,10 +235,7 @@ module Sqreen
218
235
  end
219
236
 
220
237
  def login(framework)
221
- headers = {
222
- 'x-api-key' => @token,
223
- 'x-app-name' => @app_name || framework.application_name,
224
- }.reject { |k, v| v == nil }
238
+ headers = prelogin_auth_headers(framework)
225
239
 
226
240
  Sqreen.log.warn "Using app name: #{headers['x-app-name']}"
227
241
 
@@ -235,6 +249,8 @@ module Sqreen
235
249
  end
236
250
  Sqreen.log.info 'Login success.'
237
251
  @session_id = res['session_id']
252
+ Kit::Configuration.session_key = @session_id
253
+ Kit.reset
238
254
  Sqreen.log.debug { "received session_id #{@session_id}" }
239
255
  Sqreen.logged_in = true
240
256
  res
@@ -246,20 +262,24 @@ module Sqreen
246
262
 
247
263
  def heartbeat(cmd_res = {}, metrics = [])
248
264
  payload = {}
249
- payload['metrics'] = metrics unless metrics.nil? || metrics.empty?
265
+ unless metrics.nil? || metrics.empty?
266
+ # never reached with signals
267
+ payload['metrics'] = metrics.map do |m|
268
+ Sqreen::Legacy::EventToHash.convert_agg_metric(m)
269
+ end
270
+ end
250
271
  payload['command_results'] = cmd_res unless cmd_res.nil? || cmd_res.empty?
251
272
 
252
273
  post('app-beat', payload.empty? ? nil : payload, {}, RETRY_MANY)
253
274
  end
254
275
 
255
276
  def post_metrics(metrics)
256
- return if metrics.nil? || metrics.empty?
257
- payload = { METRICS_KEY => metrics }
258
- post(METRICS_KEY, payload, {}, RETRY_MANY)
277
+ @evt_sub_strategy.post_metrics(metrics)
259
278
  end
260
279
 
280
+ # XXX never called
261
281
  def post_attack(attack)
262
- post('attack', attack.to_hash, {}, RETRY_MANY)
282
+ @evt_sub_strategy.post_attack(attack)
263
283
  end
264
284
 
265
285
  def post_bundle(bundle_sig, dependencies)
@@ -271,33 +291,22 @@ module Sqreen
271
291
  end
272
292
 
273
293
  def post_request_record(request_record)
274
- post('request_record', request_record.to_hash, {}, RETRY_MANY)
294
+ @evt_sub_strategy.post_request_record(request_record)
275
295
  end
276
296
 
277
297
  # Post an exception to Sqreen for analysis
278
298
  # @param exception [RemoteException] Exception and context to be sent over
279
299
  def post_sqreen_exception(exception)
280
- post('sqreen_exception', exception.to_hash, {}, 5)
281
- rescue StandardError => e
282
- Sqreen.log.warn(format('Could not post exception (network down? %s) %s',
283
- e.inspect,
284
- exception.to_hash.inspect))
285
- nil
300
+ @evt_sub_strategy.post_sqreen_exception(exception)
286
301
  end
287
302
 
288
- BATCH_KEY = 'batch'.freeze
289
- EVENT_TYPE_KEY = 'event_type'.freeze
290
303
  def post_batch(events)
291
- batch = events.map do |event|
292
- h = event.to_hash
293
- h[EVENT_TYPE_KEY] = event_kind(event)
294
- h
295
- end
296
- Sqreen.log.debug do
297
- tally = Hash[events.group_by(&:class).map{ |k,v| [k, v.count] }]
298
- "Doing batch with the following tally of event types: #{tally}"
299
- end
300
- post(BATCH_KEY, { BATCH_KEY => batch }, {}, RETRY_MANY)
304
+ @evt_sub_strategy.post_batch(events)
305
+ end
306
+
307
+ def post_agent_message(framework, agent_message)
308
+ headers = prelogin_auth_headers(framework)
309
+ post('app_agent_message', agent_message.to_h, headers, 0)
301
310
  end
302
311
 
303
312
  # Perform agent logout
@@ -314,14 +323,13 @@ module Sqreen
314
323
  disconnect
315
324
  end
316
325
 
317
- protected
326
+ private
318
327
 
319
- def event_kind(event)
320
- case event
321
- when Sqreen::RemoteException then 'sqreen_exception'
322
- when Sqreen::Attack then 'attack'
323
- when Sqreen::RequestRecord then 'request_record'
324
- end
328
+ def prelogin_auth_headers(framework)
329
+ {
330
+ 'x-api-key' => @token,
331
+ 'x-app-name' => @app_name || framework.application_name,
332
+ }.reject { |_k, v| v == nil }
325
333
  end
326
334
  end
327
335
  end