visionary-sdk 1.1.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +17 -0
- data/LICENSE +87 -69
- data/README.md +73 -7
- data/lib/visionary/agent.rb +13 -2
- data/lib/visionary/config.rb +9 -2
- data/lib/visionary/context.rb +9 -4
- data/lib/visionary/logs/logger.rb +2 -3
- data/lib/visionary/logs/pipeline.rb +5 -2
- data/lib/visionary/metrics.rb +392 -0
- data/lib/visionary/middleware.rb +10 -5
- data/lib/visionary/queue.rb +4 -0
- data/lib/visionary/schema/mysql/metrics.sql +20 -0
- data/lib/visionary/schema/postgres/metrics.sql +20 -0
- data/lib/visionary/schema/sqlite/metrics.sql +20 -0
- data/lib/visionary/schema.rb +27 -7
- data/lib/visionary/schemas.rb +1 -0
- data/lib/visionary/store/active_record.rb +1 -0
- data/lib/visionary/store/sql.rb +94 -24
- data/lib/visionary/telemetry/hooks.rb +18 -5
- data/lib/visionary/telemetry.rb +9 -4
- data/lib/visionary/trace_context.rb +111 -0
- data/lib/visionary/version.rb +1 -1
- data/lib/visionary/writer.rb +89 -17
- data/lib/visionary.rb +30 -7
- metadata +8 -3
data/lib/visionary/store/sql.rb
CHANGED
|
@@ -6,18 +6,17 @@ module Visionary
|
|
|
6
6
|
module Store
|
|
7
7
|
class ConfigError < StandardError; end
|
|
8
8
|
|
|
9
|
-
#
|
|
10
|
-
# the pg, mysql2 or sqlite3 gem (whichever the app has installed). One
|
|
11
|
-
# connection, used by one background thread at a time, reopened after an
|
|
12
|
-
# error.
|
|
9
|
+
# Up to four connections on PostgreSQL and MySQL, one on SQLite
|
|
13
10
|
class SQL
|
|
14
|
-
|
|
11
|
+
PARALLEL = 4
|
|
12
|
+
|
|
13
|
+
attr_reader :dialect, :parallel
|
|
15
14
|
|
|
16
15
|
def initialize(dialect, &connect)
|
|
17
16
|
@dialect = dialect
|
|
18
17
|
@connect = connect
|
|
19
|
-
@
|
|
20
|
-
|
|
18
|
+
@parallel = %w[postgres mysql].include?(dialect) ? PARALLEL : 1
|
|
19
|
+
reset_pool
|
|
21
20
|
end
|
|
22
21
|
|
|
23
22
|
def string_params? = false
|
|
@@ -44,12 +43,14 @@ module Visionary
|
|
|
44
43
|
run { |c| c.rows(sql) }
|
|
45
44
|
end
|
|
46
45
|
|
|
47
|
-
#
|
|
46
|
+
# Repeated statements are prepared once, multi-row INSERTs run directly
|
|
48
47
|
def batch(stmts)
|
|
48
|
+
uses = Hash.new(0)
|
|
49
|
+
stmts.each { |sql, _| uses[sql] += 1 }
|
|
49
50
|
run do |c|
|
|
50
51
|
c.execute("BEGIN", [])
|
|
51
52
|
begin
|
|
52
|
-
stmts.each { |sql, args| c.execute(sql, args) }
|
|
53
|
+
stmts.each { |sql, args| uses[sql] > 1 ? c.execute(sql, args) : c.execute_once(sql, args) }
|
|
53
54
|
c.execute("COMMIT", [])
|
|
54
55
|
rescue StandardError
|
|
55
56
|
begin
|
|
@@ -66,37 +67,84 @@ module Visionary
|
|
|
66
67
|
run { |c| c.execute(sql, args) }
|
|
67
68
|
end
|
|
68
69
|
|
|
69
|
-
# The parent's
|
|
70
|
-
# the child: it is kept referenced (so no finalizer closes it) and a new
|
|
71
|
-
# one is opened lazily.
|
|
70
|
+
# The parent's sockets stay referenced so no finalizer closes them from the child
|
|
72
71
|
def after_fork
|
|
73
|
-
@
|
|
74
|
-
|
|
75
|
-
@conn = nil
|
|
72
|
+
(@inherited ||= []).concat(@all)
|
|
73
|
+
reset_pool
|
|
76
74
|
end
|
|
77
75
|
|
|
78
76
|
def close
|
|
79
|
-
@mutex.synchronize
|
|
77
|
+
idle = @mutex.synchronize do
|
|
78
|
+
@all -= @idle
|
|
79
|
+
@idle.dup.tap { @idle.clear }
|
|
80
|
+
end
|
|
81
|
+
idle.each { |c| shut(c) }
|
|
80
82
|
end
|
|
81
83
|
|
|
82
84
|
private
|
|
83
85
|
|
|
86
|
+
def reset_pool
|
|
87
|
+
@mutex = Mutex.new
|
|
88
|
+
@free = ConditionVariable.new
|
|
89
|
+
@idle = []
|
|
90
|
+
@all = []
|
|
91
|
+
@opening = 0
|
|
92
|
+
end
|
|
93
|
+
|
|
84
94
|
def run
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
yield
|
|
95
|
+
conn = checkout
|
|
96
|
+
begin
|
|
97
|
+
yield conn
|
|
88
98
|
rescue StandardError
|
|
89
|
-
|
|
99
|
+
if conn.broken?
|
|
100
|
+
drop(conn)
|
|
101
|
+
conn = nil
|
|
102
|
+
end
|
|
90
103
|
raise
|
|
104
|
+
ensure
|
|
105
|
+
checkin(conn) if conn
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def checkout
|
|
110
|
+
@mutex.synchronize do
|
|
111
|
+
loop do
|
|
112
|
+
return @idle.pop unless @idle.empty?
|
|
113
|
+
break if @all.size + @opening < @parallel
|
|
114
|
+
@free.wait(@mutex)
|
|
115
|
+
end
|
|
116
|
+
@opening += 1
|
|
117
|
+
end
|
|
118
|
+
begin
|
|
119
|
+
conn = @connect.call
|
|
120
|
+
ensure
|
|
121
|
+
@mutex.synchronize do
|
|
122
|
+
@opening -= 1
|
|
123
|
+
conn ? @all << conn : @free.signal
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
conn
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def checkin(conn)
|
|
130
|
+
@mutex.synchronize do
|
|
131
|
+
@idle << conn
|
|
132
|
+
@free.signal
|
|
91
133
|
end
|
|
92
134
|
end
|
|
93
135
|
|
|
94
|
-
def
|
|
95
|
-
@
|
|
136
|
+
def drop(conn)
|
|
137
|
+
@mutex.synchronize do
|
|
138
|
+
@all.delete(conn)
|
|
139
|
+
@free.signal
|
|
140
|
+
end
|
|
141
|
+
shut(conn)
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def shut(conn)
|
|
145
|
+
conn.close
|
|
96
146
|
rescue StandardError
|
|
97
147
|
nil
|
|
98
|
-
ensure
|
|
99
|
-
@conn = nil
|
|
100
148
|
end
|
|
101
149
|
end
|
|
102
150
|
|
|
@@ -110,6 +158,7 @@ module Visionary
|
|
|
110
158
|
def self.rebind(sql)
|
|
111
159
|
return sql unless sql.include?("?")
|
|
112
160
|
n = 0
|
|
161
|
+
return sql.gsub("?") { "$#{n += 1}" } unless sql.include?("'") || sql.include?('"')
|
|
113
162
|
quote = nil
|
|
114
163
|
out = +""
|
|
115
164
|
sql.each_char do |ch|
|
|
@@ -175,6 +224,8 @@ module Visionary
|
|
|
175
224
|
res&.clear
|
|
176
225
|
end
|
|
177
226
|
|
|
227
|
+
alias execute_once execute
|
|
228
|
+
|
|
178
229
|
def column(sql)
|
|
179
230
|
res = @conn.exec(sql)
|
|
180
231
|
res.column_values(0).map(&:to_s)
|
|
@@ -239,6 +290,17 @@ module Visionary
|
|
|
239
290
|
stmt.affected_rows
|
|
240
291
|
end
|
|
241
292
|
|
|
293
|
+
# Not cached since the server caps prepared statements per connection
|
|
294
|
+
def execute_once(sql, args)
|
|
295
|
+
return execute(sql, args) if args.empty?
|
|
296
|
+
|
|
297
|
+
stmt = @client.prepare(sql)
|
|
298
|
+
stmt.execute(*args)
|
|
299
|
+
stmt.affected_rows
|
|
300
|
+
ensure
|
|
301
|
+
stmt&.close
|
|
302
|
+
end
|
|
303
|
+
|
|
242
304
|
def column(sql)
|
|
243
305
|
@client.query(sql, as: :array).map { |r| r[0].to_s }
|
|
244
306
|
end
|
|
@@ -266,7 +328,13 @@ module Visionary
|
|
|
266
328
|
class Sqlite3Conn
|
|
267
329
|
def initialize(path)
|
|
268
330
|
@db = SQLite3::Database.new(path)
|
|
331
|
+
# The Visionary server shares the file, so wait for its lock
|
|
269
332
|
@db.busy_timeout = 5000
|
|
333
|
+
begin
|
|
334
|
+
@db.execute("PRAGMA journal_mode=WAL")
|
|
335
|
+
rescue StandardError
|
|
336
|
+
nil
|
|
337
|
+
end
|
|
270
338
|
end
|
|
271
339
|
|
|
272
340
|
def execute(sql, args)
|
|
@@ -274,6 +342,8 @@ module Visionary
|
|
|
274
342
|
@db.changes
|
|
275
343
|
end
|
|
276
344
|
|
|
345
|
+
alias execute_once execute
|
|
346
|
+
|
|
277
347
|
def column(sql)
|
|
278
348
|
@db.execute(sql).map { |r| r[0].to_s }
|
|
279
349
|
end
|
|
@@ -66,13 +66,25 @@ module Visionary
|
|
|
66
66
|
end
|
|
67
67
|
|
|
68
68
|
# Outbound HTTP spans from Net::HTTP (and the many clients built on it).
|
|
69
|
+
# The traceparent comes off the caller's request object afterwards
|
|
69
70
|
module NetHTTPHook
|
|
70
71
|
def request(req, body = nil, &block)
|
|
71
|
-
rec = Visionary.telemetry
|
|
72
72
|
# Net::HTTP#request calls itself again once the connection is up;
|
|
73
73
|
# that inner call is the one measured.
|
|
74
|
-
return super unless
|
|
74
|
+
return super unless started? && !Telemetry.internal?
|
|
75
|
+
|
|
76
|
+
rec = Visionary.telemetry
|
|
77
|
+
unless rec&.spans_on
|
|
78
|
+
added = TraceContext.inject(req, TraceContext.current)
|
|
79
|
+
begin
|
|
80
|
+
return super
|
|
81
|
+
ensure
|
|
82
|
+
added.each { |h| req.delete(h) }
|
|
83
|
+
end
|
|
84
|
+
end
|
|
75
85
|
|
|
86
|
+
trace = TraceContext.child
|
|
87
|
+
added = TraceContext.inject(req, trace[0])
|
|
76
88
|
t0 = Telemetry.now_ms
|
|
77
89
|
status = "ok"
|
|
78
90
|
error = nil
|
|
@@ -85,11 +97,12 @@ module Visionary
|
|
|
85
97
|
error = "#{e.class}: #{e.message}"
|
|
86
98
|
raise
|
|
87
99
|
ensure
|
|
88
|
-
|
|
100
|
+
added.each { |h| req.delete(h) }
|
|
101
|
+
Telemetry::NetHTTPHook.record(rec, self, req, t0, status, error, trace)
|
|
89
102
|
end
|
|
90
103
|
end
|
|
91
104
|
|
|
92
|
-
def self.record(rec, http, req, t0, status, error)
|
|
105
|
+
def self.record(rec, http, req, t0, status, error, trace = nil)
|
|
93
106
|
host = http.address.to_s
|
|
94
107
|
scheme = http.use_ssl? ? "https" : "http"
|
|
95
108
|
port = http.port == (http.use_ssl? ? 443 : 80) ? "" : ":#{http.port}"
|
|
@@ -98,7 +111,7 @@ module Visionary
|
|
|
98
111
|
rec.span(
|
|
99
112
|
kind: "http", name: "#{req.method} #{host}", target: host,
|
|
100
113
|
detail: "#{req.method} #{scheme}://#{host}#{port}#{path}",
|
|
101
|
-
start_mono: t0, duration_ms: Telemetry.now_ms - t0, status: status, error: error
|
|
114
|
+
start_mono: t0, duration_ms: Telemetry.now_ms - t0, status: status, error: error, trace: trace
|
|
102
115
|
)
|
|
103
116
|
rescue StandardError
|
|
104
117
|
nil
|
data/lib/visionary/telemetry.rb
CHANGED
|
@@ -12,6 +12,7 @@ module Visionary
|
|
|
12
12
|
def self.internal!
|
|
13
13
|
Thread.current[INTERNAL] = true
|
|
14
14
|
RequestContext.current = nil
|
|
15
|
+
TraceContext.current = nil
|
|
15
16
|
end
|
|
16
17
|
|
|
17
18
|
def self.internal? = Thread.current[INTERNAL] == true
|
|
@@ -41,11 +42,13 @@ module Visionary
|
|
|
41
42
|
@limiter = Errors::Limiter.new
|
|
42
43
|
end
|
|
43
44
|
|
|
44
|
-
# start_mono
|
|
45
|
-
def span(kind:, name:, start_mono:, duration_ms:, target: nil, detail: nil, status: "ok", error: nil,
|
|
45
|
+
# start_mono and duration_ms are CLOCK_MONOTONIC milliseconds
|
|
46
|
+
def span(kind:, name:, start_mono:, duration_ms:, target: nil, detail: nil, status: "ok", error: nil,
|
|
47
|
+
row_count: nil, trace: nil)
|
|
46
48
|
return unless @spans_on && @writer
|
|
47
49
|
return if Telemetry.internal?
|
|
48
50
|
|
|
51
|
+
sc, parent_id = trace || TraceContext.child
|
|
49
52
|
ctx = RequestContext.current
|
|
50
53
|
duration_ms = duration_ms.to_f
|
|
51
54
|
started_at = Time.now - ((Telemetry.now_ms - start_mono) / 1000.0)
|
|
@@ -54,7 +57,8 @@ module Visionary
|
|
|
54
57
|
detail: detail && cut(detail.to_s, SQLText::MAX_DETAIL), started_at: started_at,
|
|
55
58
|
offset_ms: ctx ? (start_mono - ctx.started_mono).round(3) : nil,
|
|
56
59
|
duration_ms: duration_ms.round(3), status: status, error: error && cut(error.to_s, 500),
|
|
57
|
-
row_count: row_count, request_id: ctx&.request_id
|
|
60
|
+
row_count: row_count, request_id: ctx&.request_id,
|
|
61
|
+
trace_id: sc.trace_id, span_id: sc.span_id, parent_span_id: parent_id
|
|
58
62
|
)
|
|
59
63
|
if ctx
|
|
60
64
|
# A late span (its request was already logged) is written on its
|
|
@@ -70,7 +74,8 @@ module Visionary
|
|
|
70
74
|
|
|
71
75
|
def span_row(span)
|
|
72
76
|
[Visionary.timestamp(span.started_at), span.request_id, @cfg.service, @instance, span.kind, span.name,
|
|
73
|
-
span.target, span.detail, span.offset_ms, span.duration_ms, span.status, span.error, span.row_count
|
|
77
|
+
span.target, span.detail, span.offset_ms, span.duration_ms, span.status, span.error, span.row_count,
|
|
78
|
+
span.trace_id, span.span_id, span.parent_span_id]
|
|
74
79
|
end
|
|
75
80
|
|
|
76
81
|
# Rows for a finished request's spans, kept for every request.
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Visionary
|
|
4
|
+
# W3C trace context (https://www.w3.org/TR/trace-context/)
|
|
5
|
+
module TraceContext
|
|
6
|
+
KEY = :visionary_span
|
|
7
|
+
TRACEPARENT = "traceparent"
|
|
8
|
+
TRACESTATE = "tracestate"
|
|
9
|
+
|
|
10
|
+
# state is the caller's tracestate, passed on unchanged
|
|
11
|
+
SpanContext = Struct.new(:trace_id, :span_id, :flags, :state) do
|
|
12
|
+
def traceparent = "00-#{trace_id}-#{span_id}-#{flags || '01'}"
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
HEX = /\A[0-9a-f]+\z/
|
|
16
|
+
|
|
17
|
+
module_function
|
|
18
|
+
|
|
19
|
+
# The innermost custom, background or request span, nil outside a trace
|
|
20
|
+
if Fiber.respond_to?(:[])
|
|
21
|
+
def current = Fiber[KEY]
|
|
22
|
+
|
|
23
|
+
def current=(sc)
|
|
24
|
+
Fiber[KEY] = sc
|
|
25
|
+
end
|
|
26
|
+
else
|
|
27
|
+
def current = Thread.current[KEY]
|
|
28
|
+
|
|
29
|
+
def current=(sc)
|
|
30
|
+
Thread.current[KEY] = sc
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def with(sc)
|
|
35
|
+
outer = current
|
|
36
|
+
self.current = sc
|
|
37
|
+
yield
|
|
38
|
+
ensure
|
|
39
|
+
self.current = outer
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# nil when malformed, version ff or an all-zero id
|
|
43
|
+
def parse(header)
|
|
44
|
+
parts = String.new(header.to_s.strip, encoding: Encoding::UTF_8).split("-", -1)
|
|
45
|
+
return nil if parts.size < 4
|
|
46
|
+
|
|
47
|
+
version, trace, span, flags = parts
|
|
48
|
+
return nil unless version.size == 2 && version != "ff" && trace.size == 32 && span.size == 16 && flags.size == 2
|
|
49
|
+
# Version 00 has exactly four fields, later versions may append more
|
|
50
|
+
return nil if version == "00" && parts.size != 4
|
|
51
|
+
return nil unless [version, trace, span, flags].all? { |p| p.match?(HEX) }
|
|
52
|
+
return nil if trace.delete("0").empty? || span.delete("0").empty?
|
|
53
|
+
|
|
54
|
+
SpanContext.new(trace, span, flags, nil)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def new_trace_id = random_hex(16)
|
|
58
|
+
|
|
59
|
+
def new_span_id = random_hex(8)
|
|
60
|
+
|
|
61
|
+
# Not secrets, so Ruby's seeded generator is enough
|
|
62
|
+
def random_hex(bytes)
|
|
63
|
+
loop do
|
|
64
|
+
hex = Random.bytes(bytes).unpack1("H*")
|
|
65
|
+
return hex unless hex.delete("0").empty?
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def request_span(env)
|
|
70
|
+
if (otel = otel_span)
|
|
71
|
+
return [SpanContext.new(otel.trace_id, new_span_id, otel.flags, env["HTTP_TRACESTATE"]), otel.span_id]
|
|
72
|
+
end
|
|
73
|
+
if (sc = parse(env["HTTP_TRACEPARENT"]))
|
|
74
|
+
state = String.new(env["HTTP_TRACESTATE"].to_s, encoding: Encoding::UTF_8)
|
|
75
|
+
return [SpanContext.new(sc.trace_id, new_span_id, sc.flags, state.empty? ? nil : state), sc.span_id]
|
|
76
|
+
end
|
|
77
|
+
[SpanContext.new(new_trace_id, new_span_id, "01", nil), nil]
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def child(parent = current)
|
|
81
|
+
return [SpanContext.new(new_trace_id, new_span_id, "01", nil), nil] unless parent
|
|
82
|
+
|
|
83
|
+
[SpanContext.new(parent.trace_id, new_span_id, parent.flags, parent.state), parent.span_id]
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Leaves a traceparent the caller set alone and returns the header names it added
|
|
87
|
+
def inject(req, sc)
|
|
88
|
+
return [] if sc.nil? || req[TRACEPARENT]
|
|
89
|
+
|
|
90
|
+
req[TRACEPARENT] = sc.traceparent
|
|
91
|
+
added = [TRACEPARENT]
|
|
92
|
+
if sc.state && !sc.state.empty? && req[TRACESTATE].nil?
|
|
93
|
+
req[TRACESTATE] = sc.state
|
|
94
|
+
added << TRACESTATE
|
|
95
|
+
end
|
|
96
|
+
added
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# An OpenTelemetry SDK in the app has already read the header
|
|
100
|
+
def otel_span
|
|
101
|
+
return nil unless defined?(::OpenTelemetry::Trace)
|
|
102
|
+
|
|
103
|
+
sc = ::OpenTelemetry::Trace.current_span.context
|
|
104
|
+
return nil unless sc.valid?
|
|
105
|
+
|
|
106
|
+
SpanContext.new(sc.hex_trace_id, sc.hex_span_id, sc.trace_flags.sampled? ? "01" : "00", nil)
|
|
107
|
+
rescue StandardError
|
|
108
|
+
nil
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|
data/lib/visionary/version.rb
CHANGED
data/lib/visionary/writer.rb
CHANGED
|
@@ -7,16 +7,28 @@ module Visionary
|
|
|
7
7
|
:request_id, :started_at, :method, :path, :route, :query, :status, :duration_ms, :ip, :user_agent,
|
|
8
8
|
:user_id, :user_handle, :user_role, :login_account, :headers, :request_content_type, :request_body, :request_truncated,
|
|
9
9
|
:request_size, :response_content_type, :response_body, :response_truncated, :response_size,
|
|
10
|
-
:totals, :span_rows, :error_rows,
|
|
10
|
+
:totals, :span_rows, :error_rows, :trace_id, :span_id, :parent_span_id,
|
|
11
11
|
keyword_init: true
|
|
12
12
|
)
|
|
13
13
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
14
|
+
WriterStats = Struct.new(:queued, :written, :dropped, keyword_init: true) do
|
|
15
|
+
def +(other)
|
|
16
|
+
WriterStats.new(queued: queued + other.queued, written: written + other.written,
|
|
17
|
+
dropped: dropped + other.dropped)
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# Queue items are [:request, Entry], [:span, row], [:error, row], [:release, row] or [:profile, row]
|
|
17
22
|
class Writer
|
|
23
|
+
# D1 takes 50 statements a call, SQLite allows 32766 parameters
|
|
24
|
+
D1_BATCH = 50
|
|
25
|
+
SQL_BATCH = 500
|
|
26
|
+
MAX_SQL_BATCH = 2000
|
|
27
|
+
MAX_PARAMS = 30_000
|
|
28
|
+
|
|
18
29
|
INSERTS = {
|
|
19
|
-
request: "api_logs", span: "spans", error: "errors", release: "releases", profile: "profiles", log: "app_logs"
|
|
30
|
+
request: "api_logs", span: "spans", error: "errors", release: "releases", profile: "profiles", log: "app_logs",
|
|
31
|
+
metric: "metrics"
|
|
20
32
|
}.freeze
|
|
21
33
|
|
|
22
34
|
def initialize(cfg, store, schemas, logger)
|
|
@@ -25,7 +37,11 @@ module Visionary
|
|
|
25
37
|
@schemas = schemas
|
|
26
38
|
@logger = logger
|
|
27
39
|
@redactor = Redactor.new(cfg.redact_headers, cfg.redact_fields, cfg.redact_request_fields)
|
|
28
|
-
@batch_size =
|
|
40
|
+
@batch_size = if store.string_params?
|
|
41
|
+
cfg.batch_size.between?(1, D1_BATCH) ? cfg.batch_size : D1_BATCH
|
|
42
|
+
else
|
|
43
|
+
cfg.batch_size.between?(1, MAX_SQL_BATCH) ? cfg.batch_size : SQL_BATCH
|
|
44
|
+
end
|
|
29
45
|
@flush_interval = cfg.flush_interval.to_f.positive? ? cfg.flush_interval.to_f : 3.0
|
|
30
46
|
@queue_size = cfg.queue_size.positive? ? cfg.queue_size : 5000
|
|
31
47
|
@sql = INSERTS.transform_values { |t| Schema.insert_sql(t, store.string_params?) }
|
|
@@ -52,6 +68,12 @@ module Visionary
|
|
|
52
68
|
@queue.push(item)
|
|
53
69
|
end
|
|
54
70
|
|
|
71
|
+
def stats
|
|
72
|
+
@stats.synchronize do
|
|
73
|
+
WriterStats.new(queued: @queue.size, written: @written, dropped: @lost + @queue.dropped_total)
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
55
77
|
# Stops the thread after it has drained the queue, waiting at most
|
|
56
78
|
# timeout seconds. False if it was still busy then.
|
|
57
79
|
def close(timeout)
|
|
@@ -72,11 +94,14 @@ module Visionary
|
|
|
72
94
|
def reset
|
|
73
95
|
@queue = BoundedQueue.new(@queue_size)
|
|
74
96
|
@thread = nil
|
|
97
|
+
@stats = Mutex.new
|
|
98
|
+
@written = @lost = 0
|
|
75
99
|
end
|
|
76
100
|
|
|
77
101
|
def run
|
|
78
102
|
Telemetry.internal!
|
|
79
103
|
@groups.each { |g| @schemas.ensure(g) }
|
|
104
|
+
work = start_flushers
|
|
80
105
|
next_flush = mono + @flush_interval
|
|
81
106
|
batch = []
|
|
82
107
|
loop do
|
|
@@ -85,7 +110,11 @@ module Visionary
|
|
|
85
110
|
batch.concat(items)
|
|
86
111
|
closed = @queue.closed? && @queue.size.zero?
|
|
87
112
|
if batch.size >= @batch_size || mono >= next_flush || closed
|
|
88
|
-
|
|
113
|
+
if work
|
|
114
|
+
work.push(batch) unless batch.empty?
|
|
115
|
+
elsif !batch.empty?
|
|
116
|
+
write(batch)
|
|
117
|
+
end
|
|
89
118
|
batch = []
|
|
90
119
|
report_drops
|
|
91
120
|
next_flush = mono + @flush_interval
|
|
@@ -94,6 +123,28 @@ module Visionary
|
|
|
94
123
|
end
|
|
95
124
|
rescue StandardError => e
|
|
96
125
|
@logger.error("visionary: writer stopped: #{e.class}: #{e.message}")
|
|
126
|
+
ensure
|
|
127
|
+
if work
|
|
128
|
+
work.close
|
|
129
|
+
@flushers.each(&:join)
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# SQLite, D1 and an ActiveRecord pool take one batch at a time, on this thread
|
|
134
|
+
def start_flushers
|
|
135
|
+
n = @store.respond_to?(:parallel) ? @store.parallel : 1
|
|
136
|
+
return nil if n <= 1
|
|
137
|
+
|
|
138
|
+
work = SizedQueue.new(1)
|
|
139
|
+
@flushers = Array.new(n) do
|
|
140
|
+
Thread.new do
|
|
141
|
+
Telemetry.internal!
|
|
142
|
+
while (b = work.pop)
|
|
143
|
+
write(b)
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
work
|
|
97
148
|
end
|
|
98
149
|
|
|
99
150
|
def report_drops
|
|
@@ -110,11 +161,13 @@ module Visionary
|
|
|
110
161
|
end
|
|
111
162
|
|
|
112
163
|
def commit(batch)
|
|
113
|
-
|
|
164
|
+
begin
|
|
114
165
|
users = lookup_users(batch)
|
|
115
|
-
batch.flat_map { |kind, item| statements(kind, item, users) }
|
|
166
|
+
rows = batch.flat_map { |kind, item| statements(kind, item, users) }
|
|
167
|
+
stmts = inserts(rows)
|
|
116
168
|
rescue StandardError => e
|
|
117
169
|
@logger.error("visionary: building rows failed; #{batch.size} entries dropped: #{e.class}: #{e.message}")
|
|
170
|
+
count(:@lost, batch.size)
|
|
118
171
|
return
|
|
119
172
|
end
|
|
120
173
|
err = nil
|
|
@@ -122,28 +175,47 @@ module Visionary
|
|
|
122
175
|
@groups.each { |g| @schemas.ensure(g) }
|
|
123
176
|
begin
|
|
124
177
|
@store.batch(stmts)
|
|
178
|
+
count(:@written, rows.size)
|
|
125
179
|
return
|
|
126
180
|
rescue StandardError => e
|
|
127
181
|
err = e
|
|
128
182
|
end
|
|
129
183
|
sleep(attempt + 1) if attempt < 2
|
|
130
184
|
end
|
|
131
|
-
@logger.error("visionary: batch write failed; #{
|
|
185
|
+
@logger.error("visionary: batch write failed; #{rows.size} rows dropped: #{err.message}")
|
|
186
|
+
count(:@lost, rows.size)
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
def count(var, n)
|
|
190
|
+
@stats.synchronize { instance_variable_set(var, instance_variable_get(var) + n) }
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
# One INSERT per row for D1, else multi-row INSERTs split at MAX_PARAMS
|
|
194
|
+
def inserts(rows)
|
|
195
|
+
return rows.map { |kind, values| [@sql.fetch(kind), values] } if @store.string_params?
|
|
196
|
+
|
|
197
|
+
rows.group_by(&:first).flat_map do |kind, group|
|
|
198
|
+
table = INSERTS.fetch(kind)
|
|
199
|
+
per = [MAX_PARAMS / Schema::TABLES.fetch(table).size, 1].max
|
|
200
|
+
group.each_slice(per).map do |slice|
|
|
201
|
+
[Schema.insert_rows_sql(table, slice.size), slice.flat_map(&:last)]
|
|
202
|
+
end
|
|
203
|
+
end
|
|
132
204
|
end
|
|
133
205
|
|
|
134
206
|
def statements(kind, item, users)
|
|
135
207
|
if kind == :request
|
|
136
|
-
out = [
|
|
137
|
-
item.span_rows&.each { |r| out <<
|
|
138
|
-
item.error_rows&.each { |r| out <<
|
|
208
|
+
out = [fit(:request, request_row(item, users))]
|
|
209
|
+
item.span_rows&.each { |r| out << fit(:span, r) }
|
|
210
|
+
item.error_rows&.each { |r| out << fit(:error, r) }
|
|
139
211
|
out
|
|
140
212
|
else
|
|
141
|
-
[
|
|
213
|
+
[fit(kind, item)]
|
|
142
214
|
end
|
|
143
215
|
end
|
|
144
216
|
|
|
145
|
-
def
|
|
146
|
-
[
|
|
217
|
+
def fit(kind, row)
|
|
218
|
+
[kind, Store.fit(@store.dialect, INSERTS.fetch(kind), row)]
|
|
147
219
|
end
|
|
148
220
|
|
|
149
221
|
# Resolves each distinct user missing a handle or role once per batch.
|
|
@@ -189,7 +261,7 @@ module Visionary
|
|
|
189
261
|
nz(@redactor.headers(e.headers)), nz(e.request_content_type), nz(req_body), e.request_truncated ? 1 : 0,
|
|
190
262
|
e.request_size, nz(e.response_content_type), nz(resp_body), e.response_truncated ? 1 : 0, e.response_size,
|
|
191
263
|
nz(code), nz(key), @cfg.service, @cfg.version, db_count, db_ms, http_count, http_ms,
|
|
192
|
-
nz(clip_login(e.login_account))
|
|
264
|
+
nz(clip_login(e.login_account)), e.trace_id, e.span_id, e.parent_span_id
|
|
193
265
|
]
|
|
194
266
|
end
|
|
195
267
|
|