sqreen-kit 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,45 @@
1
+ require 'sqreen/kit/signals/signals_client'
2
+
3
+ module Sqreen
4
+ module Kit
5
+ module Signals
6
+ class AuthSignalsClient
7
+ attr_reader :headers # for testing only
8
+
9
+ # @param signals_client [SignalsClient]
10
+ # @param auth_data [Hash] :session_key, :api_key, :app_name
11
+ def initialize(signals_client, auth_data)
12
+ @signals_client = signals_client
13
+
14
+ @headers = {}
15
+ session_key = auth_data[:session_key]
16
+ api_key = auth_data[:api_key]
17
+ app_name = auth_data[:app_name]
18
+ if session_key
19
+ @headers['X-Session-Key'] = session_key
20
+ elsif api_key
21
+ @headers['X-Api-Key'] = api_key
22
+ @headers['X-App-Name'] = app_name if app_name
23
+ else
24
+ raise ArgumentError, 'Authentication data not provided'
25
+ end
26
+ end
27
+
28
+ # @param [Array<Sqreen::Kit::Signals::Signal|Sqreen::Kit::Signals::Trace>] signals_and_traces
29
+ def report_batch(signals_and_traces)
30
+ @signals_client.report_batch(signals_and_traces, @headers)
31
+ end
32
+
33
+ # @param [Sqreen::Kit::Signals::Signal] signal
34
+ def report_signal(signal)
35
+ @signals_client.report_signal(signal, @headers)
36
+ end
37
+
38
+ # @param [Sqreen::Kit::Signals::Trace] trace
39
+ def report_trace(trace)
40
+ @signals_client.report_trace(trace, @headers)
41
+ end
42
+ end
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,177 @@
1
+ require 'sqreen/kit/loggable'
2
+
3
+ module Sqreen
4
+ module Kit
5
+ module Signals
6
+ class BatchCollector
7
+ include Loggable
8
+
9
+ EXIT_SENTINEL = Object.new.freeze
10
+ DEFAULT_MAX_DELAY_S = 45
11
+ DEFAULT_FLUSH_SIZE = 30
12
+ DEFAULT_MAX_BATCH_SIZE = 100
13
+
14
+ attr_reader :auth_sig_client,
15
+ :flush_size,
16
+ :max_delay_s,
17
+ :max_batch_size,
18
+ :queue
19
+
20
+ # @param auth_sig_client [AuthSignalsClient]
21
+ def initialize(auth_sig_client, opts = {})
22
+ @auth_sig_client = auth_sig_client
23
+ @flush_size = opts[:flush_size] || DEFAULT_FLUSH_SIZE
24
+ @max_batch_size = opts[:max_batch_size] || DEFAULT_MAX_BATCH_SIZE
25
+ @max_delay_s = opts[:max_delay_s] || DEFAULT_MAX_DELAY_S
26
+ @queue = QueueWithTimeout.new
27
+ @thread = nil
28
+
29
+ if max_batch_size < flush_size # rubocop:disable Style/GuardClause
30
+ raise ArgumentError, 'max batch size < flush size'
31
+ end
32
+ end
33
+
34
+ def <<(signal_or_trace)
35
+ @queue << signal_or_trace
36
+ end
37
+
38
+ def start
39
+ @processing_loop = ProcessingLoop.new(self)
40
+ @thread = Thread.new do
41
+ @processing_loop.run
42
+ end
43
+ end
44
+
45
+ def running?
46
+ return false if thread.nil?
47
+ @thread.alive?
48
+ end
49
+
50
+ def close
51
+ return if @thread.nil?
52
+
53
+ @queue << EXIT_SENTINEL
54
+ @thread.join
55
+ end
56
+
57
+ class ProcessingLoop
58
+ include Loggable
59
+
60
+ # @param [BatchCollector] collector
61
+ def initialize(collector)
62
+ @collector = collector
63
+ @next_batch = []
64
+ @deadline = nil
65
+ end
66
+
67
+ def queue
68
+ @collector.queue
69
+ end
70
+
71
+ def max_batch_size
72
+ @collector.max_batch_size
73
+ end
74
+
75
+ def flush_size
76
+ @collector.flush_size
77
+ end
78
+
79
+ def run
80
+ while run_loop_once; end
81
+ logger.info 'Collector thread exiting'
82
+ end
83
+
84
+ private
85
+
86
+ def run_loop_once
87
+ el = queue.pop(@deadline)
88
+ if el.nil? # deadline passed
89
+ submit
90
+ elsif el.equal?(EXIT_SENTINEL)
91
+ return false
92
+ else
93
+ # a signal or a trace
94
+ if @next_batch.empty?
95
+ # first object, set a deadline
96
+ @deadline = Time.now.to_f + @collector.max_delay_s
97
+ end
98
+ @next_batch << el
99
+ # drain the queue completely
100
+ until @next_batch.size >= max_batch_size || (el = queue.pop_nb).nil?
101
+ if el.equal?(EXIT_SENTINEL)
102
+ queue << EXIT_SENTINEL # push it back
103
+ break
104
+ end
105
+ @next_batch << el
106
+ end
107
+
108
+ submit if @next_batch.size >= flush_size
109
+ end
110
+
111
+ true
112
+ end
113
+
114
+ def submit
115
+ logger.debug { "Batch submit. Batch size: #{@next_batch.size}" }
116
+ @deadline = nil
117
+ return if @next_batch.empty?
118
+ cur_batch = @next_batch
119
+ @next_batch = []
120
+ @collector.auth_sig_client.report_batch(cur_batch)
121
+ end
122
+ end
123
+
124
+ # Adapted from https://spin.atomicobject.com/2014/07/07/ruby-queue-pop-timeout/
125
+ class QueueWithTimeout
126
+ include Loggable
127
+
128
+ MAX_QUEUE_SIZE = 1000
129
+
130
+ def initialize
131
+ @mutex = Mutex.new
132
+ @queue = []
133
+ @received = ConditionVariable.new
134
+ end
135
+
136
+ def <<(x)
137
+ @mutex.synchronize do
138
+ if @queue.size >= MAX_QUEUE_SIZE
139
+ # processing loop is prob spending too much time on http requests
140
+ logger.warn "Queue is full! Dropping #{x}"
141
+ next
142
+ end
143
+ @queue << x
144
+ @received.signal
145
+ end
146
+ end
147
+
148
+ # non-blocking pop
149
+ def pop_nb
150
+ @mutex.synchronize do
151
+ return nil if @queue.empty?
152
+ @queue.shift
153
+ end
154
+ end
155
+
156
+ # @param deadline [Float]
157
+ def pop(deadline = nil)
158
+ @mutex.synchronize do
159
+ if deadline.nil?
160
+ # wait indefinitely until there is an element in the queue
161
+ @received.wait(@mutex) while @queue.empty?
162
+ elsif @queue.empty?
163
+ # wait for element or timeout
164
+ while @queue.empty? && (remaining_time = deadline - Time.now.to_f) > 0
165
+ @received.wait(@mutex, remaining_time)
166
+ end
167
+ end
168
+
169
+ return nil if @queue.empty?
170
+ @queue.shift
171
+ end
172
+ end
173
+ end # end QueueWithTimeout
174
+ end
175
+ end
176
+ end
177
+ end
@@ -0,0 +1,139 @@
1
+ require 'sqreen/kit/signals/dto_helper'
2
+
3
+ # reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/context/http/2020-01-01T00_00_00_000Z/schema.cue
4
+
5
+ module Sqreen
6
+ module Kit
7
+ module Signals
8
+ module Context
9
+ end
10
+ end
11
+ end
12
+ end
13
+
14
+ class Sqreen::Kit::Signals::Context::HttpContext
15
+ include Sqreen::Kit::Signals::DtoHelper
16
+
17
+ PARAMS_ATTRS = [:params_query, :params_form, :params_cookies,
18
+ :params_json, :params_other].freeze
19
+
20
+ SCHEMA_VERSION = 'http/2020-01-01T00:00:00.000Z'.freeze
21
+ add_mandatory_attrs :headers, :user_agent, :scheme, :verb,
22
+ :host, :port, :remote_ip, :remote_port, :path
23
+
24
+ # @return [String]
25
+ attr_accessor :rid
26
+
27
+ # mandatory
28
+ # @return [Array<Array<String>>] inner arrays have two values: [name, value]
29
+ attr_accessor :headers
30
+
31
+ # mandatory
32
+ # @return [String]
33
+ attr_accessor :user_agent
34
+
35
+ # mandatory
36
+ # @return [String]
37
+ attr_accessor :scheme
38
+
39
+ # mandatory
40
+ # @param [String]
41
+ attr_writer :verb
42
+ def verb
43
+ raise 'verb not set' unless defined?(@verb) && @verb
44
+ @verb.upcase
45
+ end
46
+
47
+ # mandatory
48
+ # Host header; may include the port
49
+ # @return [String]
50
+ attr_accessor :host
51
+
52
+ # mandatory
53
+ # @return [Integer|String]
54
+ attr_accessor :port
55
+
56
+ # mandatory
57
+ # @return [String]
58
+ attr_accessor :remote_ip
59
+
60
+ # mandatory
61
+ # @return [Integer|String]
62
+ attr_accessor :remote_port
63
+
64
+ # mandatory
65
+ # XXX: Unclear is decoded or not
66
+ # @return [String]
67
+ attr_accessor :path
68
+
69
+ # @return [String]
70
+ attr_accessor :referer
71
+
72
+ # @return [Hash]
73
+ attr_accessor :params_query
74
+
75
+ # @return [Hash]
76
+ attr_accessor :params_form
77
+
78
+ # @return [Hash]
79
+ attr_accessor :params_cookies
80
+
81
+ # @return [Hash]
82
+ attr_accessor :params_json
83
+
84
+ # @return [Hash]
85
+ attr_accessor :params_other
86
+
87
+ # @return [String]
88
+ attr_accessor :endpoint
89
+
90
+ # @return [Boolean]
91
+ attr_accessor :reveal_replayed
92
+
93
+ # Response status
94
+ # @return [Integer]
95
+ attr_accessor :status
96
+
97
+ # Response content length
98
+ # @return [Integer]
99
+ attr_accessor :content_length
100
+
101
+ # Response content type
102
+ # @return [String]
103
+ attr_accessor :content_type
104
+
105
+ def to_h
106
+ check_mandatories
107
+
108
+ {
109
+ request: compact_hash({
110
+ rid: rid,
111
+ headers: headers,
112
+ user_agent: user_agent,
113
+ scheme: scheme,
114
+ verb: verb,
115
+ host: host,
116
+ port: port,
117
+ remote_ip: remote_ip,
118
+ remote_port: remote_port,
119
+ path: path,
120
+ referer: referer,
121
+ parameters: compact_hash({
122
+ query: params_query,
123
+ form: params_form,
124
+ cookies: params_cookies,
125
+ json: params_json,
126
+ other: params_other,
127
+ }),
128
+ endpoint: endpoint,
129
+ # yes, this one is in CamelCase
130
+ isRevealReplayed: reveal_replayed,
131
+ }),
132
+ response: compact_hash({
133
+ status: status,
134
+ content_type: content_type,
135
+ content_length: content_length,
136
+ }),
137
+ }
138
+ end
139
+ end
@@ -0,0 +1,147 @@
1
+ require 'json'
2
+
3
+ module Sqreen
4
+ module Kit
5
+ module Signals
6
+ # Provides a helper constructor, default to_h, and mandatory field checking
7
+ module DtoHelper
8
+ DO_NOT_CONVERT_TYPES = [
9
+ NilClass, Hash, Array, Numeric, TrueClass, FalseClass
10
+ ].freeze
11
+
12
+ module ClassMethods
13
+ # method should have been defined initially in an ancestor
14
+ def validate_str_attr(attr, regex)
15
+ define_method(:"#{attr}=") do |val|
16
+ unless val =~ regex
17
+ raise "Unexpected format for attribute #{attr}: " \
18
+ "'#{val}' does not match #{regex}"
19
+ end
20
+ super(val)
21
+ end
22
+ end
23
+
24
+ # All the classes/modules in the ancestor chain
25
+ # including DtoHelper
26
+ def all_implementing_modules
27
+ ancestors
28
+ .select { |c| c != DtoHelper && c.ancestors.include?(DtoHelper) }
29
+ end
30
+
31
+ def attributes_for_to_h
32
+ @all_attributes ||= begin
33
+ all_implementing_modules
34
+ .map(&:attributes_for_to_h_self)
35
+ .reduce(:+)
36
+ .uniq
37
+ end
38
+ end
39
+
40
+ def attributes_for_to_h_self
41
+ methods = public_instance_methods(false)
42
+
43
+ methods.reject { |m| m.to_s.end_with? '=' }
44
+ .select { |m| methods.include?(:"#{m}=") }
45
+ end
46
+
47
+ def add_mandatory_attrs(*args)
48
+ @self_mandatory_attrs ||= []
49
+ @self_mandatory_attrs += args
50
+ end
51
+
52
+ def self_mandatory_attrs
53
+ @self_mandatory_attrs || []
54
+ end
55
+
56
+ def mandatory_attrs
57
+ @mandatory_attrs ||= begin
58
+ all_implementing_modules
59
+ .map(&:self_mandatory_attrs)
60
+ .reduce(:+)
61
+ .uniq
62
+ end
63
+ end
64
+
65
+ def attr_accessor_time(attr)
66
+ define_method :"#{attr}=" do |value|
67
+ unless value.is_a?(Time)
68
+ unless value.is_a?(String)
69
+ raise ArgumentError, 'expected Time or String object'
70
+ end
71
+ unless value =~ /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/
72
+ raise ArgumentError, "Invalid time format for #{value}"
73
+ end
74
+ end
75
+ instance_variable_set("@#{attr}", value)
76
+ end
77
+
78
+ define_method attr do
79
+ cur_val = instance_variable_get("@#{attr}")
80
+
81
+ return nil if cur_val.nil?
82
+ return cur_val if cur_val.is_a?(String)
83
+ cur_val.strftime('%Y-%m-%dT%H:%M:%S.%L%z')
84
+ end
85
+ end
86
+
87
+ def included(mod)
88
+ # make sure that classes/modules indirectly including DtoHelper
89
+ # also have their singleton class including ClassMethods
90
+ return if mod.singleton_class.ancestors.include?(ClassMethods)
91
+ mod.extend(ClassMethods)
92
+ end
93
+ end
94
+
95
+ def self.included(mod)
96
+ mod.extend(ClassMethods)
97
+ end
98
+
99
+ def initialize(values = {})
100
+ values.each do |attr, val|
101
+ public_send("#{attr}=", val)
102
+ end
103
+ end
104
+
105
+ def compact_hash(h)
106
+ h.delete_if { |_k, v| v.nil? }
107
+ end
108
+
109
+ def to_h
110
+ check_mandatories
111
+
112
+ res = {}
113
+ self.class.attributes_for_to_h.each do |attr|
114
+ value = public_send(attr)
115
+ if (value.class.ancestors & DO_NOT_CONVERT_TYPES).empty? && \
116
+ value.respond_to?(:to_h)
117
+ value = value.to_h
118
+ end
119
+
120
+ res[attr] = value unless value.nil?
121
+ end
122
+ res
123
+ end
124
+
125
+ def append_to_h_filter(proc)
126
+ @filters ||= []
127
+ @filters << proc
128
+ end
129
+
130
+ def to_json
131
+ res = (@filters || []).reduce(to_h) { |accum, filter| filter[accum] }
132
+ res.to_json
133
+ end
134
+
135
+ private
136
+
137
+ def check_mandatories
138
+ self.class.mandatory_attrs.each do |attr|
139
+ if public_send(attr).nil?
140
+ raise "The attribute #{attr} is not set in #{inspect}"
141
+ end
142
+ end
143
+ end
144
+ end
145
+ end
146
+ end
147
+ end