sqreen-kit 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +5 -0
- data/LICENSE +3 -0
- data/lib/sqreen-kit.rb +3 -0
- data/lib/sqreen.rb +1 -0
- data/lib/sqreen/kit.rb +76 -0
- data/lib/sqreen/kit/configuration.rb +67 -0
- data/lib/sqreen/kit/http_client.rb +160 -0
- data/lib/sqreen/kit/http_client/authentication_error.rb +13 -0
- data/lib/sqreen/kit/http_client/unexpected_status_error.rb +18 -0
- data/lib/sqreen/kit/loggable.rb +14 -0
- data/lib/sqreen/kit/retry_policy.rb +56 -0
- data/lib/sqreen/kit/signals/actor.rb +26 -0
- data/lib/sqreen/kit/signals/auth_signals_client.rb +45 -0
- data/lib/sqreen/kit/signals/batch_collector.rb +177 -0
- data/lib/sqreen/kit/signals/context/http_context.rb +139 -0
- data/lib/sqreen/kit/signals/dto_helper.rb +147 -0
- data/lib/sqreen/kit/signals/metric.rb +16 -0
- data/lib/sqreen/kit/signals/point.rb +16 -0
- data/lib/sqreen/kit/signals/signal.rb +28 -0
- data/lib/sqreen/kit/signals/signal_attributes.rb +104 -0
- data/lib/sqreen/kit/signals/signals_client.rb +33 -0
- data/lib/sqreen/kit/signals/stack_trace.rb +140 -0
- data/lib/sqreen/kit/signals/trace.rb +34 -0
- data/lib/sqreen/kit/string_sanitizer.rb +46 -0
- metadata +82 -0
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'sqreen/kit/signals/signals_client'
|
2
|
+
|
3
|
+
module Sqreen
|
4
|
+
module Kit
|
5
|
+
module Signals
|
6
|
+
class AuthSignalsClient
|
7
|
+
attr_reader :headers # for testing only
|
8
|
+
|
9
|
+
# @param signals_client [SignalsClient]
|
10
|
+
# @param auth_data [Hash] :session_key, :api_key, :app_name
|
11
|
+
def initialize(signals_client, auth_data)
|
12
|
+
@signals_client = signals_client
|
13
|
+
|
14
|
+
@headers = {}
|
15
|
+
session_key = auth_data[:session_key]
|
16
|
+
api_key = auth_data[:api_key]
|
17
|
+
app_name = auth_data[:app_name]
|
18
|
+
if session_key
|
19
|
+
@headers['X-Session-Key'] = session_key
|
20
|
+
elsif api_key
|
21
|
+
@headers['X-Api-Key'] = api_key
|
22
|
+
@headers['X-App-Name'] = app_name if app_name
|
23
|
+
else
|
24
|
+
raise ArgumentError, 'Authentication data not provided'
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
# @param [Array<Sqreen::Kit::Signals::Signal|Sqreen::Kit::Signals::Trace>] signals_and_traces
|
29
|
+
def report_batch(signals_and_traces)
|
30
|
+
@signals_client.report_batch(signals_and_traces, @headers)
|
31
|
+
end
|
32
|
+
|
33
|
+
# @param [Sqreen::Kit::Signals::Signal] signal
|
34
|
+
def report_signal(signal)
|
35
|
+
@signals_client.report_signal(signal, @headers)
|
36
|
+
end
|
37
|
+
|
38
|
+
# @param [Sqreen::Kit::Signals::Trace] trace
|
39
|
+
def report_trace(trace)
|
40
|
+
@signals_client.report_trace(trace, @headers)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,177 @@
|
|
1
|
+
require 'sqreen/kit/loggable'
|
2
|
+
|
3
|
+
module Sqreen
|
4
|
+
module Kit
|
5
|
+
module Signals
|
6
|
+
class BatchCollector
|
7
|
+
include Loggable
|
8
|
+
|
9
|
+
EXIT_SENTINEL = Object.new.freeze
|
10
|
+
DEFAULT_MAX_DELAY_S = 45
|
11
|
+
DEFAULT_FLUSH_SIZE = 30
|
12
|
+
DEFAULT_MAX_BATCH_SIZE = 100
|
13
|
+
|
14
|
+
attr_reader :auth_sig_client,
|
15
|
+
:flush_size,
|
16
|
+
:max_delay_s,
|
17
|
+
:max_batch_size,
|
18
|
+
:queue
|
19
|
+
|
20
|
+
# @param auth_sig_client [AuthSignalsClient]
|
21
|
+
def initialize(auth_sig_client, opts = {})
|
22
|
+
@auth_sig_client = auth_sig_client
|
23
|
+
@flush_size = opts[:flush_size] || DEFAULT_FLUSH_SIZE
|
24
|
+
@max_batch_size = opts[:max_batch_size] || DEFAULT_MAX_BATCH_SIZE
|
25
|
+
@max_delay_s = opts[:max_delay_s] || DEFAULT_MAX_DELAY_S
|
26
|
+
@queue = QueueWithTimeout.new
|
27
|
+
@thread = nil
|
28
|
+
|
29
|
+
if max_batch_size < flush_size # rubocop:disable Style/GuardClause
|
30
|
+
raise ArgumentError, 'max batch size < flush size'
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def <<(signal_or_trace)
|
35
|
+
@queue << signal_or_trace
|
36
|
+
end
|
37
|
+
|
38
|
+
def start
|
39
|
+
@processing_loop = ProcessingLoop.new(self)
|
40
|
+
@thread = Thread.new do
|
41
|
+
@processing_loop.run
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def running?
|
46
|
+
return false if thread.nil?
|
47
|
+
@thread.alive?
|
48
|
+
end
|
49
|
+
|
50
|
+
def close
|
51
|
+
return if @thread.nil?
|
52
|
+
|
53
|
+
@queue << EXIT_SENTINEL
|
54
|
+
@thread.join
|
55
|
+
end
|
56
|
+
|
57
|
+
class ProcessingLoop
|
58
|
+
include Loggable
|
59
|
+
|
60
|
+
# @param [BatchCollector] collector
|
61
|
+
def initialize(collector)
|
62
|
+
@collector = collector
|
63
|
+
@next_batch = []
|
64
|
+
@deadline = nil
|
65
|
+
end
|
66
|
+
|
67
|
+
def queue
|
68
|
+
@collector.queue
|
69
|
+
end
|
70
|
+
|
71
|
+
def max_batch_size
|
72
|
+
@collector.max_batch_size
|
73
|
+
end
|
74
|
+
|
75
|
+
def flush_size
|
76
|
+
@collector.flush_size
|
77
|
+
end
|
78
|
+
|
79
|
+
def run
|
80
|
+
while run_loop_once; end
|
81
|
+
logger.info 'Collector thread exiting'
|
82
|
+
end
|
83
|
+
|
84
|
+
private
|
85
|
+
|
86
|
+
def run_loop_once
|
87
|
+
el = queue.pop(@deadline)
|
88
|
+
if el.nil? # deadline passed
|
89
|
+
submit
|
90
|
+
elsif el.equal?(EXIT_SENTINEL)
|
91
|
+
return false
|
92
|
+
else
|
93
|
+
# a signal or a trace
|
94
|
+
if @next_batch.empty?
|
95
|
+
# first object, set a deadline
|
96
|
+
@deadline = Time.now.to_f + @collector.max_delay_s
|
97
|
+
end
|
98
|
+
@next_batch << el
|
99
|
+
# drain the queue completely
|
100
|
+
until @next_batch.size >= max_batch_size || (el = queue.pop_nb).nil?
|
101
|
+
if el.equal?(EXIT_SENTINEL)
|
102
|
+
queue << EXIT_SENTINEL # push it back
|
103
|
+
break
|
104
|
+
end
|
105
|
+
@next_batch << el
|
106
|
+
end
|
107
|
+
|
108
|
+
submit if @next_batch.size >= flush_size
|
109
|
+
end
|
110
|
+
|
111
|
+
true
|
112
|
+
end
|
113
|
+
|
114
|
+
def submit
|
115
|
+
logger.debug { "Batch submit. Batch size: #{@next_batch.size}" }
|
116
|
+
@deadline = nil
|
117
|
+
return if @next_batch.empty?
|
118
|
+
cur_batch = @next_batch
|
119
|
+
@next_batch = []
|
120
|
+
@collector.auth_sig_client.report_batch(cur_batch)
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
# Adapted from https://spin.atomicobject.com/2014/07/07/ruby-queue-pop-timeout/
|
125
|
+
class QueueWithTimeout
|
126
|
+
include Loggable
|
127
|
+
|
128
|
+
MAX_QUEUE_SIZE = 1000
|
129
|
+
|
130
|
+
def initialize
|
131
|
+
@mutex = Mutex.new
|
132
|
+
@queue = []
|
133
|
+
@received = ConditionVariable.new
|
134
|
+
end
|
135
|
+
|
136
|
+
def <<(x)
|
137
|
+
@mutex.synchronize do
|
138
|
+
if @queue.size >= MAX_QUEUE_SIZE
|
139
|
+
# processing loop is prob spending too much time on http requests
|
140
|
+
logger.warn "Queue is full! Dropping #{x}"
|
141
|
+
next
|
142
|
+
end
|
143
|
+
@queue << x
|
144
|
+
@received.signal
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
# non-blocking pop
|
149
|
+
def pop_nb
|
150
|
+
@mutex.synchronize do
|
151
|
+
return nil if @queue.empty?
|
152
|
+
@queue.shift
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
# @param deadline [Float]
|
157
|
+
def pop(deadline = nil)
|
158
|
+
@mutex.synchronize do
|
159
|
+
if deadline.nil?
|
160
|
+
# wait indefinitely until there is an element in the queue
|
161
|
+
@received.wait(@mutex) while @queue.empty?
|
162
|
+
elsif @queue.empty?
|
163
|
+
# wait for element or timeout
|
164
|
+
while @queue.empty? && (remaining_time = deadline - Time.now.to_f) > 0
|
165
|
+
@received.wait(@mutex, remaining_time)
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
return nil if @queue.empty?
|
170
|
+
@queue.shift
|
171
|
+
end
|
172
|
+
end
|
173
|
+
end # end QueueWithTimeout
|
174
|
+
end
|
175
|
+
end
|
176
|
+
end
|
177
|
+
end
|
@@ -0,0 +1,139 @@
|
|
1
|
+
require 'sqreen/kit/signals/dto_helper'
|
2
|
+
|
3
|
+
# reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/context/http/2020-01-01T00_00_00_000Z/schema.cue
|
4
|
+
|
5
|
+
module Sqreen
|
6
|
+
module Kit
|
7
|
+
module Signals
|
8
|
+
module Context
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
class Sqreen::Kit::Signals::Context::HttpContext
|
15
|
+
include Sqreen::Kit::Signals::DtoHelper
|
16
|
+
|
17
|
+
PARAMS_ATTRS = [:params_query, :params_form, :params_cookies,
|
18
|
+
:params_json, :params_other].freeze
|
19
|
+
|
20
|
+
SCHEMA_VERSION = 'http/2020-01-01T00:00:00.000Z'.freeze
|
21
|
+
add_mandatory_attrs :headers, :user_agent, :scheme, :verb,
|
22
|
+
:host, :port, :remote_ip, :remote_port, :path
|
23
|
+
|
24
|
+
# @return [String]
|
25
|
+
attr_accessor :rid
|
26
|
+
|
27
|
+
# mandatory
|
28
|
+
# @return [Array<Array<String>>] inner arrays have two values: [name, value]
|
29
|
+
attr_accessor :headers
|
30
|
+
|
31
|
+
# mandatory
|
32
|
+
# @return [String]
|
33
|
+
attr_accessor :user_agent
|
34
|
+
|
35
|
+
# mandatory
|
36
|
+
# @return [String]
|
37
|
+
attr_accessor :scheme
|
38
|
+
|
39
|
+
# mandatory
|
40
|
+
# @param [String]
|
41
|
+
attr_writer :verb
|
42
|
+
def verb
|
43
|
+
raise 'verb not set' unless defined?(@verb) && @verb
|
44
|
+
@verb.upcase
|
45
|
+
end
|
46
|
+
|
47
|
+
# mandatory
|
48
|
+
# Host header; may include the port
|
49
|
+
# @return [String]
|
50
|
+
attr_accessor :host
|
51
|
+
|
52
|
+
# mandatory
|
53
|
+
# @return [Integer|String]
|
54
|
+
attr_accessor :port
|
55
|
+
|
56
|
+
# mandatory
|
57
|
+
# @return [String]
|
58
|
+
attr_accessor :remote_ip
|
59
|
+
|
60
|
+
# mandatory
|
61
|
+
# @return [Integer|String]
|
62
|
+
attr_accessor :remote_port
|
63
|
+
|
64
|
+
# mandatory
|
65
|
+
# XXX: Unclear is decoded or not
|
66
|
+
# @return [String]
|
67
|
+
attr_accessor :path
|
68
|
+
|
69
|
+
# @return [String]
|
70
|
+
attr_accessor :referer
|
71
|
+
|
72
|
+
# @return [Hash]
|
73
|
+
attr_accessor :params_query
|
74
|
+
|
75
|
+
# @return [Hash]
|
76
|
+
attr_accessor :params_form
|
77
|
+
|
78
|
+
# @return [Hash]
|
79
|
+
attr_accessor :params_cookies
|
80
|
+
|
81
|
+
# @return [Hash]
|
82
|
+
attr_accessor :params_json
|
83
|
+
|
84
|
+
# @return [Hash]
|
85
|
+
attr_accessor :params_other
|
86
|
+
|
87
|
+
# @return [String]
|
88
|
+
attr_accessor :endpoint
|
89
|
+
|
90
|
+
# @return [Boolean]
|
91
|
+
attr_accessor :reveal_replayed
|
92
|
+
|
93
|
+
# Response status
|
94
|
+
# @return [Integer]
|
95
|
+
attr_accessor :status
|
96
|
+
|
97
|
+
# Response content length
|
98
|
+
# @return [Integer]
|
99
|
+
attr_accessor :content_length
|
100
|
+
|
101
|
+
# Response content type
|
102
|
+
# @return [String]
|
103
|
+
attr_accessor :content_type
|
104
|
+
|
105
|
+
def to_h
|
106
|
+
check_mandatories
|
107
|
+
|
108
|
+
{
|
109
|
+
request: compact_hash({
|
110
|
+
rid: rid,
|
111
|
+
headers: headers,
|
112
|
+
user_agent: user_agent,
|
113
|
+
scheme: scheme,
|
114
|
+
verb: verb,
|
115
|
+
host: host,
|
116
|
+
port: port,
|
117
|
+
remote_ip: remote_ip,
|
118
|
+
remote_port: remote_port,
|
119
|
+
path: path,
|
120
|
+
referer: referer,
|
121
|
+
parameters: compact_hash({
|
122
|
+
query: params_query,
|
123
|
+
form: params_form,
|
124
|
+
cookies: params_cookies,
|
125
|
+
json: params_json,
|
126
|
+
other: params_other,
|
127
|
+
}),
|
128
|
+
endpoint: endpoint,
|
129
|
+
# yes, this one is in CamelCase
|
130
|
+
isRevealReplayed: reveal_replayed,
|
131
|
+
}),
|
132
|
+
response: compact_hash({
|
133
|
+
status: status,
|
134
|
+
content_type: content_type,
|
135
|
+
content_length: content_length,
|
136
|
+
}),
|
137
|
+
}
|
138
|
+
end
|
139
|
+
end
|
@@ -0,0 +1,147 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
module Sqreen
|
4
|
+
module Kit
|
5
|
+
module Signals
|
6
|
+
# Provides a helper constructor, default to_h, and mandatory field checking
|
7
|
+
module DtoHelper
|
8
|
+
DO_NOT_CONVERT_TYPES = [
|
9
|
+
NilClass, Hash, Array, Numeric, TrueClass, FalseClass
|
10
|
+
].freeze
|
11
|
+
|
12
|
+
module ClassMethods
|
13
|
+
# method should have been defined initially in an ancestor
|
14
|
+
def validate_str_attr(attr, regex)
|
15
|
+
define_method(:"#{attr}=") do |val|
|
16
|
+
unless val =~ regex
|
17
|
+
raise "Unexpected format for attribute #{attr}: " \
|
18
|
+
"'#{val}' does not match #{regex}"
|
19
|
+
end
|
20
|
+
super(val)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
# All the classes/modules in the ancestor chain
|
25
|
+
# including DtoHelper
|
26
|
+
def all_implementing_modules
|
27
|
+
ancestors
|
28
|
+
.select { |c| c != DtoHelper && c.ancestors.include?(DtoHelper) }
|
29
|
+
end
|
30
|
+
|
31
|
+
def attributes_for_to_h
|
32
|
+
@all_attributes ||= begin
|
33
|
+
all_implementing_modules
|
34
|
+
.map(&:attributes_for_to_h_self)
|
35
|
+
.reduce(:+)
|
36
|
+
.uniq
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def attributes_for_to_h_self
|
41
|
+
methods = public_instance_methods(false)
|
42
|
+
|
43
|
+
methods.reject { |m| m.to_s.end_with? '=' }
|
44
|
+
.select { |m| methods.include?(:"#{m}=") }
|
45
|
+
end
|
46
|
+
|
47
|
+
def add_mandatory_attrs(*args)
|
48
|
+
@self_mandatory_attrs ||= []
|
49
|
+
@self_mandatory_attrs += args
|
50
|
+
end
|
51
|
+
|
52
|
+
def self_mandatory_attrs
|
53
|
+
@self_mandatory_attrs || []
|
54
|
+
end
|
55
|
+
|
56
|
+
def mandatory_attrs
|
57
|
+
@mandatory_attrs ||= begin
|
58
|
+
all_implementing_modules
|
59
|
+
.map(&:self_mandatory_attrs)
|
60
|
+
.reduce(:+)
|
61
|
+
.uniq
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def attr_accessor_time(attr)
|
66
|
+
define_method :"#{attr}=" do |value|
|
67
|
+
unless value.is_a?(Time)
|
68
|
+
unless value.is_a?(String)
|
69
|
+
raise ArgumentError, 'expected Time or String object'
|
70
|
+
end
|
71
|
+
unless value =~ /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/
|
72
|
+
raise ArgumentError, "Invalid time format for #{value}"
|
73
|
+
end
|
74
|
+
end
|
75
|
+
instance_variable_set("@#{attr}", value)
|
76
|
+
end
|
77
|
+
|
78
|
+
define_method attr do
|
79
|
+
cur_val = instance_variable_get("@#{attr}")
|
80
|
+
|
81
|
+
return nil if cur_val.nil?
|
82
|
+
return cur_val if cur_val.is_a?(String)
|
83
|
+
cur_val.strftime('%Y-%m-%dT%H:%M:%S.%L%z')
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
def included(mod)
|
88
|
+
# make sure that classes/modules indirectly including DtoHelper
|
89
|
+
# also have their singleton class including ClassMethods
|
90
|
+
return if mod.singleton_class.ancestors.include?(ClassMethods)
|
91
|
+
mod.extend(ClassMethods)
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
def self.included(mod)
|
96
|
+
mod.extend(ClassMethods)
|
97
|
+
end
|
98
|
+
|
99
|
+
def initialize(values = {})
|
100
|
+
values.each do |attr, val|
|
101
|
+
public_send("#{attr}=", val)
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
def compact_hash(h)
|
106
|
+
h.delete_if { |_k, v| v.nil? }
|
107
|
+
end
|
108
|
+
|
109
|
+
def to_h
|
110
|
+
check_mandatories
|
111
|
+
|
112
|
+
res = {}
|
113
|
+
self.class.attributes_for_to_h.each do |attr|
|
114
|
+
value = public_send(attr)
|
115
|
+
if (value.class.ancestors & DO_NOT_CONVERT_TYPES).empty? && \
|
116
|
+
value.respond_to?(:to_h)
|
117
|
+
value = value.to_h
|
118
|
+
end
|
119
|
+
|
120
|
+
res[attr] = value unless value.nil?
|
121
|
+
end
|
122
|
+
res
|
123
|
+
end
|
124
|
+
|
125
|
+
def append_to_h_filter(proc)
|
126
|
+
@filters ||= []
|
127
|
+
@filters << proc
|
128
|
+
end
|
129
|
+
|
130
|
+
def to_json
|
131
|
+
res = (@filters || []).reduce(to_h) { |accum, filter| filter[accum] }
|
132
|
+
res.to_json
|
133
|
+
end
|
134
|
+
|
135
|
+
private
|
136
|
+
|
137
|
+
def check_mandatories
|
138
|
+
self.class.mandatory_attrs.each do |attr|
|
139
|
+
if public_send(attr).nil?
|
140
|
+
raise "The attribute #{attr} is not set in #{inspect}"
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
147
|
+
end
|