logbrew-sdk 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +211 -3
- data/examples/Makefile +13 -1
- data/examples/automatic_delivery.rb +32 -0
- data/examples/persistent_worker_delivery.rb +33 -0
- data/examples/sidekiq_tracing.rb +22 -0
- data/lib/logbrew/automatic_delivery.rb +493 -0
- data/lib/logbrew/bounded_event_queue.rb +196 -0
- data/lib/logbrew/event_batcher.rb +67 -0
- data/lib/logbrew/faraday_tracing.rb +55 -0
- data/lib/logbrew/http_client_tracing.rb +282 -0
- data/lib/logbrew/operation_tracing.rb +16 -1
- data/lib/logbrew/persistent_event_store.rb +403 -0
- data/lib/logbrew/sidekiq.rb +466 -0
- data/lib/logbrew/span_events.rb +34 -0
- data/lib/logbrew/worker_lifecycle.rb +211 -0
- data/lib/logbrew.rb +357 -33
- metadata +14 -2
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module LogBrew
|
|
4
|
+
class EventBatcher
|
|
5
|
+
DEFAULT_MAX_SIZE = 100
|
|
6
|
+
DEFAULT_MAX_BYTES = 262_144
|
|
7
|
+
BATCH_SUFFIX = "]}"
|
|
8
|
+
|
|
9
|
+
Batch = Struct.new(:body, :event_count, :event_bytes)
|
|
10
|
+
private_constant :Batch
|
|
11
|
+
|
|
12
|
+
attr_reader :max_event_bytes
|
|
13
|
+
|
|
14
|
+
def initialize(sdk:, max_size:, max_bytes:)
|
|
15
|
+
validate_positive_integer("max_batch_size", max_size)
|
|
16
|
+
validate_positive_integer("max_batch_bytes", max_bytes)
|
|
17
|
+
|
|
18
|
+
@max_size = max_size
|
|
19
|
+
@max_bytes = max_bytes
|
|
20
|
+
@batch_prefix = "{\"sdk\":#{JSON.generate(sdk)},\"events\":["
|
|
21
|
+
@base_bytes = @batch_prefix.bytesize + BATCH_SUFFIX.bytesize
|
|
22
|
+
if @base_bytes >= @max_bytes
|
|
23
|
+
raise SdkError.new("validation_error", "max_batch_bytes must fit the SDK envelope")
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
@max_event_bytes = @max_bytes - @base_bytes
|
|
27
|
+
rescue JSON::GeneratorError, EncodingError
|
|
28
|
+
raise SdkError.new("validation_error", "sdk identity must be JSON serializable")
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def next_batch(serialized_events, limit:)
|
|
32
|
+
event_json = []
|
|
33
|
+
event_bytes = 0
|
|
34
|
+
body_bytes = @base_bytes
|
|
35
|
+
maximum = [serialized_events.length, limit, @max_size].min
|
|
36
|
+
|
|
37
|
+
maximum.times do |index|
|
|
38
|
+
serialized = serialized_events.fetch(index)
|
|
39
|
+
next_body_bytes = body_bytes + (event_json.empty? ? 0 : 1) + serialized.bytesize
|
|
40
|
+
break if next_body_bytes > @max_bytes
|
|
41
|
+
|
|
42
|
+
event_json << serialized
|
|
43
|
+
event_bytes += serialized.bytesize
|
|
44
|
+
body_bytes = next_body_bytes
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
if event_json.empty?
|
|
48
|
+
raise SdkError.new("transport_error", "queued event cannot fit the configured batch byte limit")
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
Batch.new(
|
|
52
|
+
(@batch_prefix + event_json.join(",") + BATCH_SUFFIX).freeze,
|
|
53
|
+
event_json.length,
|
|
54
|
+
event_bytes
|
|
55
|
+
).freeze
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
private
|
|
59
|
+
|
|
60
|
+
def validate_positive_integer(label, value)
|
|
61
|
+
return if value.is_a?(Integer) && value.positive?
|
|
62
|
+
|
|
63
|
+
raise SdkError.new("validation_error", "#{label} must be a positive integer")
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
private_constant :EventBatcher
|
|
67
|
+
end
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "faraday"
|
|
4
|
+
require_relative "../logbrew"
|
|
5
|
+
|
|
6
|
+
module LogBrew
|
|
7
|
+
class FaradayTracingMiddleware < ::Faraday::Middleware
|
|
8
|
+
def initialize(app, client:, on_capture_error: nil)
|
|
9
|
+
super(app)
|
|
10
|
+
@client = client
|
|
11
|
+
@on_capture_error = on_capture_error
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def call(env)
|
|
15
|
+
prepared = HttpClientTracing.prepare(
|
|
16
|
+
client: @client,
|
|
17
|
+
source: "faraday",
|
|
18
|
+
on_capture_error: @on_capture_error
|
|
19
|
+
) do
|
|
20
|
+
url = env.url
|
|
21
|
+
[env.method, url && url.host, HttpClientTracing::FaradayHeaderSnapshot.new(env.request_headers)]
|
|
22
|
+
end
|
|
23
|
+
return @app.call(env) unless prepared
|
|
24
|
+
|
|
25
|
+
operation, header = prepared
|
|
26
|
+
begin
|
|
27
|
+
header.inject(operation.traceparent)
|
|
28
|
+
rescue StandardError => error
|
|
29
|
+
operation.capture_error(error)
|
|
30
|
+
HttpClientTracing.reset_header(header, operation)
|
|
31
|
+
return @app.call(env)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
response = nil
|
|
35
|
+
begin
|
|
36
|
+
response = operation.around { @app.call(env) }
|
|
37
|
+
rescue StandardError => error
|
|
38
|
+
operation.finish(error: error)
|
|
39
|
+
raise
|
|
40
|
+
ensure
|
|
41
|
+
HttpClientTracing.reset_header(header, operation)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
begin
|
|
45
|
+
response.on_complete do |completed|
|
|
46
|
+
operation.finish(status_code: HttpClientTracing.read_status(completed, :status, operation))
|
|
47
|
+
end
|
|
48
|
+
rescue StandardError => error
|
|
49
|
+
operation.capture_error(error)
|
|
50
|
+
operation.finish(status_code: HttpClientTracing.read_status(response, :status, operation))
|
|
51
|
+
end
|
|
52
|
+
response
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "ipaddr"
|
|
4
|
+
|
|
5
|
+
module LogBrew
|
|
6
|
+
class HttpClientTraceOperation
|
|
7
|
+
attr_reader :context, :traceparent
|
|
8
|
+
|
|
9
|
+
def initialize(client:, parent:, source:, method:, host:, on_capture_error:)
|
|
10
|
+
@client = client
|
|
11
|
+
@context = Trace.create(
|
|
12
|
+
trace_id: parent.trace_id,
|
|
13
|
+
span_id: Trace.generate_span_id,
|
|
14
|
+
parent_span_id: parent.span_id,
|
|
15
|
+
trace_flags: parent.trace_flags
|
|
16
|
+
)
|
|
17
|
+
@traceparent = Trace.create_headers(@context).fetch("traceparent")
|
|
18
|
+
@source = source
|
|
19
|
+
@method = HttpClientTracing.normalize_method(method)
|
|
20
|
+
@host = HttpClientTracing.normalize_host(host)
|
|
21
|
+
@on_capture_error = on_capture_error
|
|
22
|
+
@started_at = HttpClientTracing.monotonic_time
|
|
23
|
+
@mutex = Mutex.new
|
|
24
|
+
@finished = false
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def around
|
|
28
|
+
HttpClientTracing.with_operation(self) { yield }
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def finish(status_code: nil, error: nil)
|
|
32
|
+
should_capture = @mutex.synchronize do
|
|
33
|
+
next false if @finished
|
|
34
|
+
|
|
35
|
+
@finished = true
|
|
36
|
+
true
|
|
37
|
+
end
|
|
38
|
+
return unless should_capture
|
|
39
|
+
|
|
40
|
+
capture(status_code, error)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def capture_error(error)
|
|
44
|
+
HttpClientTracing.report_capture_error(@on_capture_error, error)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
private
|
|
48
|
+
|
|
49
|
+
def capture(status_code, error)
|
|
50
|
+
normalized_status = HttpClientTracing.normalize_status_code(status_code)
|
|
51
|
+
metadata = {
|
|
52
|
+
method: @method,
|
|
53
|
+
source: @source,
|
|
54
|
+
sampled: @context.sampled
|
|
55
|
+
}
|
|
56
|
+
metadata[:host] = @host if @host
|
|
57
|
+
metadata[:statusCode] = normalized_status if normalized_status
|
|
58
|
+
metadata[:exceptionType] = error.class.name if error
|
|
59
|
+
status = error || (normalized_status && normalized_status >= 400) ? "error" : "ok"
|
|
60
|
+
|
|
61
|
+
@client.span(
|
|
62
|
+
"ruby_http_span_#{@context.span_id}",
|
|
63
|
+
Time.now.utc.iso8601,
|
|
64
|
+
{
|
|
65
|
+
name: "http.client:#{@method}",
|
|
66
|
+
traceId: @context.trace_id,
|
|
67
|
+
spanId: @context.span_id,
|
|
68
|
+
parentSpanId: @context.parent_span_id,
|
|
69
|
+
status: status,
|
|
70
|
+
durationMs: ((HttpClientTracing.monotonic_time - @started_at) * 1000.0).round(3),
|
|
71
|
+
metadata: metadata
|
|
72
|
+
}
|
|
73
|
+
)
|
|
74
|
+
rescue StandardError => capture_error
|
|
75
|
+
HttpClientTracing.report_capture_error(@on_capture_error, capture_error)
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
class NetHttpTracingClient
|
|
80
|
+
attr_reader :http
|
|
81
|
+
|
|
82
|
+
def initialize(http, client:, on_capture_error: nil)
|
|
83
|
+
@http = http
|
|
84
|
+
@client = client
|
|
85
|
+
@on_capture_error = on_capture_error
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def request(request, body = nil, &block)
|
|
89
|
+
prepared = HttpClientTracing.prepare(
|
|
90
|
+
client: @client,
|
|
91
|
+
source: "net_http",
|
|
92
|
+
on_capture_error: @on_capture_error
|
|
93
|
+
) do
|
|
94
|
+
[request.method, address, HttpClientTracing::NetHttpHeaderSnapshot.new(request)]
|
|
95
|
+
end
|
|
96
|
+
return @http.request(request, body, &block) unless prepared
|
|
97
|
+
|
|
98
|
+
operation, header = prepared
|
|
99
|
+
begin
|
|
100
|
+
header.inject(operation.traceparent)
|
|
101
|
+
rescue StandardError => error
|
|
102
|
+
operation.capture_error(error)
|
|
103
|
+
HttpClientTracing.reset_header(header, operation)
|
|
104
|
+
return @http.request(request, body, &block)
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
response = nil
|
|
108
|
+
begin
|
|
109
|
+
response = operation.around { @http.request(request, body, &block) }
|
|
110
|
+
rescue StandardError => error
|
|
111
|
+
operation.finish(error: error)
|
|
112
|
+
raise
|
|
113
|
+
ensure
|
|
114
|
+
HttpClientTracing.reset_header(header, operation)
|
|
115
|
+
end
|
|
116
|
+
operation.finish(status_code: HttpClientTracing.read_status(response, :code, operation))
|
|
117
|
+
response
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def start(*arguments)
|
|
121
|
+
unless block_given?
|
|
122
|
+
started = @http.start(*arguments)
|
|
123
|
+
return started.equal?(@http) ? self : started
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
@http.start(*arguments) { yield self }
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def method_missing(name, *arguments, &block)
|
|
130
|
+
return super unless @http.respond_to?(name)
|
|
131
|
+
|
|
132
|
+
@http.public_send(name, *arguments, &block)
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def respond_to_missing?(name, include_private = false)
|
|
136
|
+
@http.respond_to?(name, include_private) || super
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
module HttpClientTracing
|
|
141
|
+
ACTIVE_OPERATION_KEY = :logbrew_http_client_operation
|
|
142
|
+
SUPPRESSION_KEY = :logbrew_http_client_suppression
|
|
143
|
+
SOURCES = %w[net_http faraday].freeze
|
|
144
|
+
private_constant :ACTIVE_OPERATION_KEY, :SUPPRESSION_KEY, :SOURCES
|
|
145
|
+
|
|
146
|
+
class NetHttpHeaderSnapshot
|
|
147
|
+
def initialize(request)
|
|
148
|
+
@request = request
|
|
149
|
+
@values = request.get_fields("traceparent")
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def inject(traceparent)
|
|
153
|
+
@request["traceparent"] = traceparent
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def reset
|
|
157
|
+
@request.delete("traceparent")
|
|
158
|
+
Array(@values).each { |value| @request.add_field("traceparent", value) }
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
class FaradayHeaderSnapshot
|
|
163
|
+
def initialize(headers)
|
|
164
|
+
@headers = headers
|
|
165
|
+
@present = headers.key?("traceparent")
|
|
166
|
+
@value = headers["traceparent"]
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def inject(traceparent)
|
|
170
|
+
@headers["traceparent"] = traceparent
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def reset
|
|
174
|
+
if @present
|
|
175
|
+
@headers["traceparent"] = @value
|
|
176
|
+
else
|
|
177
|
+
@headers.delete("traceparent")
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
module_function
|
|
183
|
+
|
|
184
|
+
def wrap_net_http(http, client:, on_capture_error: nil)
|
|
185
|
+
return http if http.is_a?(NetHttpTracingClient)
|
|
186
|
+
|
|
187
|
+
NetHttpTracingClient.new(http, client: client, on_capture_error: on_capture_error)
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
def prepare(client:, source:, on_capture_error: nil)
|
|
191
|
+
parent = Trace.current
|
|
192
|
+
return nil unless parent
|
|
193
|
+
return nil if suppressed? || active_operation?
|
|
194
|
+
return nil unless SOURCES.include?(source)
|
|
195
|
+
|
|
196
|
+
method, host, header = yield
|
|
197
|
+
operation = HttpClientTraceOperation.new(
|
|
198
|
+
client: client,
|
|
199
|
+
parent: parent,
|
|
200
|
+
source: source,
|
|
201
|
+
method: method,
|
|
202
|
+
host: host,
|
|
203
|
+
on_capture_error: on_capture_error
|
|
204
|
+
)
|
|
205
|
+
[operation, header]
|
|
206
|
+
rescue StandardError => error
|
|
207
|
+
report_capture_error(on_capture_error, error)
|
|
208
|
+
nil
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
def with_operation(operation)
|
|
212
|
+
previous = Thread.current[ACTIVE_OPERATION_KEY]
|
|
213
|
+
Thread.current[ACTIVE_OPERATION_KEY] = operation
|
|
214
|
+
Trace.with_context(operation.context) { yield }
|
|
215
|
+
ensure
|
|
216
|
+
Thread.current[ACTIVE_OPERATION_KEY] = previous
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
def suppress
|
|
220
|
+
previous = Thread.current[SUPPRESSION_KEY]
|
|
221
|
+
Thread.current[SUPPRESSION_KEY] = true
|
|
222
|
+
yield
|
|
223
|
+
ensure
|
|
224
|
+
Thread.current[SUPPRESSION_KEY] = previous
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
def active_operation?
|
|
228
|
+
!Thread.current[ACTIVE_OPERATION_KEY].nil?
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
def suppressed?
|
|
232
|
+
Thread.current[SUPPRESSION_KEY] == true
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
def normalize_method(method)
|
|
236
|
+
value = method.to_s.strip.upcase
|
|
237
|
+
value.empty? ? "HTTP" : value[0, 32]
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
def normalize_host(host)
|
|
241
|
+
value = host.to_s.strip.downcase.sub(/\.+\z/, "")
|
|
242
|
+
return nil if value.empty? || value.bytesize > 253
|
|
243
|
+
return nil if value.match?(/\A[0-9.]+\z/)
|
|
244
|
+
return nil unless value.match?(/\A[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?\z/)
|
|
245
|
+
|
|
246
|
+
begin
|
|
247
|
+
IPAddr.new(value)
|
|
248
|
+
nil
|
|
249
|
+
rescue IPAddr::InvalidAddressError
|
|
250
|
+
value
|
|
251
|
+
end
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
def normalize_status_code(status_code)
|
|
255
|
+
value = status_code.to_i
|
|
256
|
+
value.positive? && value <= 999 ? value : nil
|
|
257
|
+
end
|
|
258
|
+
|
|
259
|
+
def read_status(response, method, operation)
|
|
260
|
+
response.public_send(method)
|
|
261
|
+
rescue StandardError => error
|
|
262
|
+
operation.capture_error(error)
|
|
263
|
+
nil
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
def reset_header(header, operation)
|
|
267
|
+
header.reset
|
|
268
|
+
rescue StandardError => error
|
|
269
|
+
operation.capture_error(error)
|
|
270
|
+
end
|
|
271
|
+
|
|
272
|
+
def monotonic_time
|
|
273
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
def report_capture_error(callback, error)
|
|
277
|
+
callback.call(error) if callback.respond_to?(:call)
|
|
278
|
+
rescue StandardError
|
|
279
|
+
nil
|
|
280
|
+
end
|
|
281
|
+
end
|
|
282
|
+
end
|
|
@@ -80,7 +80,8 @@ module LogBrew
|
|
|
80
80
|
parentSpanId: context.parent_span_id,
|
|
81
81
|
status: error ? "error" : "ok",
|
|
82
82
|
durationMs: duration_ms(started_at, options),
|
|
83
|
-
metadata: span_metadata(kind, options, error)
|
|
83
|
+
metadata: span_metadata(kind, options, error),
|
|
84
|
+
events: span_events(error)
|
|
84
85
|
}
|
|
85
86
|
)
|
|
86
87
|
rescue StandardError => capture_error
|
|
@@ -114,6 +115,20 @@ module LogBrew
|
|
|
114
115
|
end
|
|
115
116
|
end
|
|
116
117
|
|
|
118
|
+
def span_events(error)
|
|
119
|
+
return nil unless error
|
|
120
|
+
|
|
121
|
+
[
|
|
122
|
+
{
|
|
123
|
+
name: "exception",
|
|
124
|
+
metadata: {
|
|
125
|
+
exceptionType: error.class.name,
|
|
126
|
+
exceptionEscaped: true
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
]
|
|
130
|
+
end
|
|
131
|
+
|
|
117
132
|
def sanitized_metadata(metadata)
|
|
118
133
|
return {} if metadata.nil?
|
|
119
134
|
raise SdkError.new("validation_error", "operation metadata must be an object") unless metadata.is_a?(Hash)
|