sqreen 1.19.0 → 1.20.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +22 -0
- data/lib/sqreen/agent_message.rb +20 -0
- data/lib/sqreen/aggregated_metric.rb +25 -0
- data/lib/sqreen/ca.crt +24 -0
- data/lib/sqreen/configuration.rb +10 -4
- data/lib/sqreen/deliveries/batch.rb +4 -1
- data/lib/sqreen/deliveries/simple.rb +4 -0
- data/lib/sqreen/endpoint_testing.rb +184 -0
- data/lib/sqreen/event.rb +7 -5
- data/lib/sqreen/events/attack.rb +23 -18
- data/lib/sqreen/events/remote_exception.rb +0 -22
- data/lib/sqreen/events/request_record.rb +15 -70
- data/lib/sqreen/frameworks/request_recorder.rb +13 -2
- data/lib/sqreen/kit/signals/specialized/aggregated_metric.rb +72 -0
- data/lib/sqreen/kit/signals/specialized/attack.rb +57 -0
- data/lib/sqreen/kit/signals/specialized/binning_metric.rb +76 -0
- data/lib/sqreen/kit/signals/specialized/http_trace.rb +26 -0
- data/lib/sqreen/kit/signals/specialized/sdk_track_call.rb +50 -0
- data/lib/sqreen/kit/signals/specialized/sqreen_exception.rb +57 -0
- data/lib/sqreen/legacy/old_event_submission_strategy.rb +221 -0
- data/lib/sqreen/legacy/waf_redactions.rb +49 -0
- data/lib/sqreen/log/loggable.rb +1 -1
- data/lib/sqreen/metrics/base.rb +3 -0
- data/lib/sqreen/metrics_store.rb +22 -12
- data/lib/sqreen/performance_notifications/binned_metrics.rb +8 -2
- data/lib/sqreen/rules.rb +4 -2
- data/lib/sqreen/rules/not_found_cb.rb +2 -0
- data/lib/sqreen/rules/rule_cb.rb +2 -0
- data/lib/sqreen/rules/waf_cb.rb +13 -10
- data/lib/sqreen/runner.rb +75 -8
- data/lib/sqreen/sensitive_data_redactor.rb +19 -31
- data/lib/sqreen/session.rb +51 -43
- data/lib/sqreen/signals/conversions.rb +283 -0
- data/lib/sqreen/signals/http_trace_redaction.rb +111 -0
- data/lib/sqreen/signals/signals_submission_strategy.rb +78 -0
- data/lib/sqreen/version.rb +1 -1
- data/lib/sqreen/weave/legacy/instrumentation.rb +7 -7
- metadata +50 -6
- data/lib/sqreen/backport.rb +0 -9
- data/lib/sqreen/backport/clock_gettime.rb +0 -74
- data/lib/sqreen/backport/original_name.rb +0 -88
@@ -30,27 +30,5 @@ module Sqreen
|
|
30
30
|
def klass
|
31
31
|
payload['exception'].class.name
|
32
32
|
end
|
33
|
-
|
34
|
-
def to_hash
|
35
|
-
exception = payload['exception']
|
36
|
-
ev = {
|
37
|
-
:klass => exception.class.name,
|
38
|
-
:message => exception.message,
|
39
|
-
:params => payload['request_params'],
|
40
|
-
:time => payload['time'],
|
41
|
-
:infos => {
|
42
|
-
:client_ip => payload['client_ip'],
|
43
|
-
},
|
44
|
-
:request => payload['request_infos'],
|
45
|
-
:headers => payload['headers'],
|
46
|
-
:rule_name => payload['rule_name'],
|
47
|
-
:rulespack_id => payload['rulespack_id'],
|
48
|
-
}
|
49
|
-
|
50
|
-
ev[:infos].merge!(payload['infos']) if payload['infos']
|
51
|
-
return ev unless exception.backtrace
|
52
|
-
ev[:context] = { :backtrace => exception.backtrace.map(&:to_s) }
|
53
|
-
ev
|
54
|
-
end
|
55
33
|
end
|
56
34
|
end
|
@@ -14,6 +14,10 @@ require 'sqreen/sensitive_data_redactor'
|
|
14
14
|
module Sqreen
|
15
15
|
# When a request is deeemed worthy of being sent to the backend
|
16
16
|
class RequestRecord < Sqreen::Event
|
17
|
+
attr_reader :redactor
|
18
|
+
|
19
|
+
# @param [Hash] payload
|
20
|
+
# @param [Sqreen::SensitiveDataRedactor] redactor
|
17
21
|
def initialize(payload, redactor = nil)
|
18
22
|
@redactor = redactor
|
19
23
|
super(payload)
|
@@ -23,74 +27,18 @@ module Sqreen
|
|
23
27
|
(payload && payload[:observed]) || {}
|
24
28
|
end
|
25
29
|
|
26
|
-
def
|
27
|
-
|
28
|
-
if payload[:observed]
|
29
|
-
res[:observed] = payload[:observed].dup
|
30
|
-
rulespack = nil
|
31
|
-
if observed[:attacks]
|
32
|
-
res[:observed][:attacks] = observed[:attacks].map do |att|
|
33
|
-
natt = att.dup
|
34
|
-
rulespack = natt.delete(:rulespack_id) || rulespack
|
35
|
-
natt
|
36
|
-
end
|
37
|
-
end
|
38
|
-
if observed[:sqreen_exceptions]
|
39
|
-
res[:observed][:sqreen_exceptions] = observed[:sqreen_exceptions].map do |exc|
|
40
|
-
nex = exc.dup
|
41
|
-
excp = nex.delete(:exception)
|
42
|
-
if excp
|
43
|
-
nex[:message] = excp.message
|
44
|
-
nex[:klass] = excp.class.name
|
45
|
-
end
|
46
|
-
rulespack = nex.delete(:rulespack_id) || rulespack
|
47
|
-
nex
|
48
|
-
end
|
49
|
-
end
|
50
|
-
res[:rulespack_id] = rulespack unless rulespack.nil?
|
51
|
-
if observed[:observations]
|
52
|
-
res[:observed][:observations] = observed[:observations].map do |cat, key, value, time|
|
53
|
-
{ :category => cat, :key => key, :value => value, :time => time }
|
54
|
-
end
|
55
|
-
end
|
56
|
-
if observed[:sdk]
|
57
|
-
res[:observed][:sdk] = processed_sdk_calls
|
58
|
-
end
|
59
|
-
end
|
60
|
-
res[:local] = payload['local'] if payload['local']
|
61
|
-
if payload['request']
|
62
|
-
res[:request] = payload['request'].dup
|
63
|
-
res[:client_ip] = res[:request].delete(:client_ip) if res[:request][:client_ip]
|
64
|
-
else
|
65
|
-
res[:request] = {}
|
66
|
-
end
|
67
|
-
if payload['response']
|
68
|
-
res[:response] = payload['response'].dup
|
69
|
-
else
|
70
|
-
res[:response] = {}
|
71
|
-
end
|
72
|
-
|
73
|
-
res[:request][:parameters] = payload['params'] if payload['params']
|
74
|
-
res[:request][:headers] = payload['headers'] if payload['headers']
|
75
|
-
|
76
|
-
res = Sqreen::EncodingSanitizer.sanitize(res)
|
30
|
+
def last_identify_args
|
31
|
+
return nil unless observed[:sdk]
|
77
32
|
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
res[:observed][:attacks] = @redactor.redact_attacks!(res[:observed][:attacks], redacted)
|
82
|
-
end
|
83
|
-
if redacted.any? && res[:observed] && res[:observed][:sqreen_exceptions]
|
84
|
-
res[:observed][:sqreen_exceptions] = @redactor.redact_exceptions!(res[:observed][:sqreen_exceptions], redacted)
|
85
|
-
end
|
33
|
+
observed[:sdk].reverse_each do |meth, _time, *args|
|
34
|
+
next unless meth == :identify
|
35
|
+
return args
|
86
36
|
end
|
87
|
-
|
88
|
-
res
|
37
|
+
nil
|
89
38
|
end
|
90
39
|
|
91
|
-
private
|
92
|
-
|
93
40
|
def processed_sdk_calls
|
41
|
+
return [] unless observed[:sdk]
|
94
42
|
auth_keys = last_identify_id
|
95
43
|
|
96
44
|
observed[:sdk].map do |meth, time, *args|
|
@@ -102,6 +50,8 @@ module Sqreen
|
|
102
50
|
end
|
103
51
|
end
|
104
52
|
|
53
|
+
private
|
54
|
+
|
105
55
|
def inject_identifiers(args, meth, auth_keys)
|
106
56
|
return args unless meth == :track && auth_keys
|
107
57
|
|
@@ -118,13 +68,8 @@ module Sqreen
|
|
118
68
|
end
|
119
69
|
|
120
70
|
def last_identify_id
|
121
|
-
|
122
|
-
|
123
|
-
observed[:sdk].reverse_each do |meth, _time, *args|
|
124
|
-
next unless meth == :identify
|
125
|
-
return args.first if args.respond_to? :first
|
126
|
-
end
|
127
|
-
nil
|
71
|
+
args = last_identify_args
|
72
|
+
args.first if args.respond_to? :first
|
128
73
|
end
|
129
74
|
end
|
130
75
|
end
|
@@ -58,12 +58,20 @@ module Sqreen
|
|
58
58
|
Sqreen.log.debug { "close_request_record called. observed_items: #{observed_items}" }
|
59
59
|
|
60
60
|
clean_request_record if observed_items.nil?
|
61
|
-
if only_metric_observation
|
61
|
+
if Sqreen.features['use_signals'] || only_metric_observation
|
62
62
|
push_metrics(observations_queue, queue)
|
63
|
-
return clean_request_record
|
64
63
|
end
|
64
|
+
|
65
|
+
if only_metric_observation
|
66
|
+
clean_request_record
|
67
|
+
return
|
68
|
+
end
|
69
|
+
|
70
|
+
# signals require request section to be present
|
71
|
+
payload_requests << 'request'
|
65
72
|
payload = payload_creator.payload(payload_requests)
|
66
73
|
payload[:observed] = observed_items
|
74
|
+
|
67
75
|
queue.push create_request_record(payload)
|
68
76
|
clean_request_record
|
69
77
|
end
|
@@ -79,10 +87,13 @@ module Sqreen
|
|
79
87
|
@redactor ||= SensitiveDataRedactor.from_config
|
80
88
|
end
|
81
89
|
|
90
|
+
# pushes metric observations to the observations queue
|
91
|
+
# and clears the list for the request record
|
82
92
|
def push_metrics(observations_queue, event_queue)
|
83
93
|
observed_items[:observations].each do |obs|
|
84
94
|
observations_queue.push obs
|
85
95
|
end
|
96
|
+
observed_items[:observations] = []
|
86
97
|
return unless observations_queue.size > MAX_OBS_QUEUE_LENGTH / 2
|
87
98
|
event_queue.push Sqreen::METRICS_EVENT
|
88
99
|
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
# typed: ignore
|
2
|
+
|
3
|
+
# Copyright (c) 2015 Sqreen. All Rights Reserved.
|
4
|
+
# Please refer to our terms for more information: https://www.sqreen.com/terms.html
|
5
|
+
|
6
|
+
require 'sqreen/kit/signals/metric'
|
7
|
+
require 'sqreen/kit/signals/dto_helper'
|
8
|
+
|
9
|
+
# reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/payload/binning_metric/2020-01-01T00_00_00_000Z/schema.cue
|
10
|
+
|
11
|
+
module Sqreen
|
12
|
+
module Kit
|
13
|
+
module Signals
|
14
|
+
module Specialized
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
class Sqreen::Kit::Signals::Specialized::AggregatedMetric < Sqreen::Kit::Signals::Metric
|
21
|
+
add_mandatory_attrs :source, :payload
|
22
|
+
|
23
|
+
validate_str_attr :signal_name, /\Asq\.agent\.metric\..+\z/
|
24
|
+
|
25
|
+
def initialize(args)
|
26
|
+
self.payload_schema = Payload::SCHEMA_VERSION
|
27
|
+
super
|
28
|
+
end
|
29
|
+
|
30
|
+
class Payload
|
31
|
+
include Sqreen::Kit::Signals::DtoHelper
|
32
|
+
|
33
|
+
SCHEMA_VERSION = 'metric/2020-01-01T00:00:00.000Z'.freeze
|
34
|
+
|
35
|
+
add_mandatory_attrs :capture_interval_s,
|
36
|
+
:date_started,
|
37
|
+
:date_ended,
|
38
|
+
:values,
|
39
|
+
:kind
|
40
|
+
|
41
|
+
# mandatory
|
42
|
+
# @return [Integer]
|
43
|
+
attr_accessor :capture_interval_s
|
44
|
+
|
45
|
+
# mandatory
|
46
|
+
# @param [Time]
|
47
|
+
# @return [String]
|
48
|
+
attr_accessor_time :date_started
|
49
|
+
|
50
|
+
# mandatory
|
51
|
+
# @param [Time]
|
52
|
+
# @return [String]
|
53
|
+
attr_accessor_time :date_ended
|
54
|
+
|
55
|
+
# mandatory
|
56
|
+
# @return [Hash{String=>Object}]
|
57
|
+
attr_writer :values
|
58
|
+
def values
|
59
|
+
return nil if @values.nil?
|
60
|
+
@values.map do |k, v|
|
61
|
+
{
|
62
|
+
key: k.is_a?(Hash) || k.is_a?(Array) ? k : k.to_s,
|
63
|
+
value: v,
|
64
|
+
}
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
# mandatory
|
69
|
+
# @return [String]
|
70
|
+
attr_accessor :kind
|
71
|
+
end
|
72
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
# typed: ignore
|
2
|
+
|
3
|
+
# Copyright (c) 2015 Sqreen. All Rights Reserved.
|
4
|
+
# Please refer to our terms for more information: https://www.sqreen.com/terms.html
|
5
|
+
|
6
|
+
require 'sqreen/kit/signals/point'
|
7
|
+
require 'sqreen/kit/signals/dto_helper'
|
8
|
+
|
9
|
+
# reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/payload/attack/2020-01-01T00_00_00_000Z/schema.cue
|
10
|
+
|
11
|
+
module Sqreen
|
12
|
+
module Kit
|
13
|
+
module Signals
|
14
|
+
module Specialized
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
class Sqreen::Kit::Signals::Specialized::Attack < Sqreen::Kit::Signals::Point
|
21
|
+
add_mandatory_attrs :source, :time, :payload
|
22
|
+
|
23
|
+
validate_str_attr :signal_name, /\Asq\.agent\.attack\..+\z/
|
24
|
+
validate_str_attr :source, /\Asqreen:rule:[a-f0-9]{40}:.+\z/
|
25
|
+
|
26
|
+
def initialize(values = {})
|
27
|
+
self.payload_schema = Payload::SCHEMA_VERSION
|
28
|
+
self.time = values[:time] || Time.now
|
29
|
+
super
|
30
|
+
end
|
31
|
+
|
32
|
+
def payload=(payload)
|
33
|
+
unless payload.is_a?(Payload)
|
34
|
+
raise ArgumentError, "Payload should be a #{Payload}"
|
35
|
+
end
|
36
|
+
super
|
37
|
+
end
|
38
|
+
|
39
|
+
class Payload
|
40
|
+
include Sqreen::Kit::Signals::DtoHelper
|
41
|
+
|
42
|
+
SCHEMA_VERSION = 'attack/2020-01-01T00:00:00.000Z'.freeze
|
43
|
+
|
44
|
+
add_mandatory_attrs :test, :block, :infos
|
45
|
+
|
46
|
+
# all are mandatory
|
47
|
+
|
48
|
+
# @return [Boolean]
|
49
|
+
attr_accessor :test
|
50
|
+
|
51
|
+
# @return [Boolean]
|
52
|
+
attr_accessor :block
|
53
|
+
|
54
|
+
# @return [Hash{String|Symbol=>Object}]
|
55
|
+
attr_accessor :infos
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,76 @@
|
|
1
|
+
# typed: ignore
|
2
|
+
|
3
|
+
# Copyright (c) 2015 Sqreen. All Rights Reserved.
|
4
|
+
# Please refer to our terms for more information: https://www.sqreen.com/terms.html
|
5
|
+
|
6
|
+
require 'sqreen/kit/signals/metric'
|
7
|
+
require 'sqreen/kit/signals/dto_helper'
|
8
|
+
|
9
|
+
# reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/payload/binning_metric/2020-01-01T00_00_00_000Z/schema.cue
|
10
|
+
|
11
|
+
module Sqreen
|
12
|
+
module Kit
|
13
|
+
module Signals
|
14
|
+
module Specialized
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
class Sqreen::Kit::Signals::Specialized::BinningMetric < Sqreen::Kit::Signals::Metric
|
21
|
+
add_mandatory_attrs :source, :time, :payload
|
22
|
+
|
23
|
+
validate_str_attr :signal_name, /\Asq\.agent\.metric\..+\z/
|
24
|
+
|
25
|
+
def initialize(args)
|
26
|
+
self.payload_schema = Payload::SCHEMA_VERSION
|
27
|
+
super
|
28
|
+
end
|
29
|
+
|
30
|
+
class Payload
|
31
|
+
include Sqreen::Kit::Signals::DtoHelper
|
32
|
+
|
33
|
+
SCHEMA_VERSION = 'metric_binning/2020-01-01T00:00:00.000Z'.freeze
|
34
|
+
|
35
|
+
add_mandatory_attrs :capture_interval_s,
|
36
|
+
:date_started,
|
37
|
+
:date_ended,
|
38
|
+
:max, :base, :unit, :bins
|
39
|
+
|
40
|
+
# mandatory
|
41
|
+
# @return [Integer]
|
42
|
+
attr_accessor :capture_interval_s
|
43
|
+
|
44
|
+
# mandatory
|
45
|
+
# @param [Time]
|
46
|
+
# @return [String]
|
47
|
+
attr_accessor_time :date_started
|
48
|
+
|
49
|
+
# mandatory
|
50
|
+
# @param [Time]
|
51
|
+
# @return [String]
|
52
|
+
attr_accessor_time :date_ended
|
53
|
+
|
54
|
+
# mandatory
|
55
|
+
# @return [Float]
|
56
|
+
attr_accessor :max
|
57
|
+
|
58
|
+
# mandatory
|
59
|
+
# @return [Float]
|
60
|
+
attr_accessor :base
|
61
|
+
|
62
|
+
# mandatory
|
63
|
+
# @return [Float]
|
64
|
+
attr_accessor :unit
|
65
|
+
|
66
|
+
# mandatory
|
67
|
+
# @return [Hash{Integer=>Integer}]
|
68
|
+
attr_accessor :bins
|
69
|
+
|
70
|
+
def to_h
|
71
|
+
{
|
72
|
+
kind: 'binning',
|
73
|
+
}.merge(super)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# typed: ignore
|
2
|
+
|
3
|
+
# Copyright (c) 2015 Sqreen. All Rights Reserved.
|
4
|
+
# Please refer to our terms for more information: https://www.sqreen.com/terms.html
|
5
|
+
|
6
|
+
require 'sqreen/kit/signals/trace'
|
7
|
+
require 'sqreen/kit/signals/context/http_context'
|
8
|
+
require 'sqreen/kit/signals/dto_helper'
|
9
|
+
|
10
|
+
module Sqreen
|
11
|
+
module Kit
|
12
|
+
module Signals
|
13
|
+
module Specialized
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
class Sqreen::Kit::Signals::Specialized::HttpTrace < Sqreen::Kit::Signals::Trace
|
20
|
+
add_mandatory_attrs :context
|
21
|
+
|
22
|
+
def initialize(values = {})
|
23
|
+
self.context_schema = ::Sqreen::Kit::Signals::Context::HttpContext::SCHEMA_VERSION
|
24
|
+
super
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
# typed: ignore
|
2
|
+
|
3
|
+
# Copyright (c) 2015 Sqreen. All Rights Reserved.
|
4
|
+
# Please refer to our terms for more information: https://www.sqreen.com/terms.html
|
5
|
+
|
6
|
+
require 'sqreen/kit/signals/point'
|
7
|
+
require 'sqreen/kit/signals/dto_helper'
|
8
|
+
|
9
|
+
# reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/payload/sdk_call/2020-01-01T00_00_00_000Z/schema.cue
|
10
|
+
|
11
|
+
module Sqreen
|
12
|
+
module Kit
|
13
|
+
module Signals
|
14
|
+
module Specialized
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
class Sqreen::Kit::Signals::Specialized::SdkTrackCall < Sqreen::Kit::Signals::Point
|
21
|
+
add_mandatory_attrs :source, :time, :payload
|
22
|
+
|
23
|
+
validate_str_attr :signal_name, /\Asq\.sdk\..+\z/
|
24
|
+
|
25
|
+
def initialize(values = {})
|
26
|
+
self.payload_schema = Payload::SCHEMA_VERSION
|
27
|
+
self.source = "sqreen:sdk:track"
|
28
|
+
self.time = values[:time] || Time.now
|
29
|
+
super
|
30
|
+
end
|
31
|
+
|
32
|
+
def payload=(payload)
|
33
|
+
unless payload.is_a?(Payload)
|
34
|
+
raise ArgumentError, "Payload should be a #{Payload}"
|
35
|
+
end
|
36
|
+
super
|
37
|
+
end
|
38
|
+
|
39
|
+
class Payload
|
40
|
+
include Sqreen::Kit::Signals::DtoHelper
|
41
|
+
|
42
|
+
SCHEMA_VERSION = 'track_event/2020-01-01T00:00:00.000Z'.freeze
|
43
|
+
|
44
|
+
# @return [Hash{String|Symbol=>Object}]
|
45
|
+
attr_accessor :properties
|
46
|
+
|
47
|
+
# @return [Hash{String|Symbol=>String}]
|
48
|
+
attr_accessor :user_identifiers
|
49
|
+
end
|
50
|
+
end
|