visionary-sdk 1.1.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +17 -0
- data/LICENSE +87 -69
- data/README.md +73 -7
- data/lib/visionary/agent.rb +13 -2
- data/lib/visionary/config.rb +9 -2
- data/lib/visionary/context.rb +9 -4
- data/lib/visionary/logs/logger.rb +2 -3
- data/lib/visionary/logs/pipeline.rb +5 -2
- data/lib/visionary/metrics.rb +392 -0
- data/lib/visionary/middleware.rb +10 -5
- data/lib/visionary/queue.rb +4 -0
- data/lib/visionary/schema/mysql/metrics.sql +20 -0
- data/lib/visionary/schema/postgres/metrics.sql +20 -0
- data/lib/visionary/schema/sqlite/metrics.sql +20 -0
- data/lib/visionary/schema.rb +27 -7
- data/lib/visionary/schemas.rb +1 -0
- data/lib/visionary/store/active_record.rb +1 -0
- data/lib/visionary/store/sql.rb +94 -24
- data/lib/visionary/telemetry/hooks.rb +18 -5
- data/lib/visionary/telemetry.rb +9 -4
- data/lib/visionary/trace_context.rb +111 -0
- data/lib/visionary/version.rb +1 -1
- data/lib/visionary/writer.rb +89 -17
- data/lib/visionary.rb +30 -7
- metadata +8 -3
|
@@ -0,0 +1,392 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
|
|
5
|
+
module Visionary
|
|
6
|
+
# Counters, gauges and histograms, written as one metrics row per series every interval
|
|
7
|
+
module Metrics
|
|
8
|
+
MAX_NAME = 200
|
|
9
|
+
MAX_LABELS = 20
|
|
10
|
+
MAX_KEY = 100
|
|
11
|
+
MAX_VALUE = 200
|
|
12
|
+
DEFAULT_MAX_SERIES = 1000
|
|
13
|
+
|
|
14
|
+
# OpenTelemetry's defaults, in milliseconds
|
|
15
|
+
DEFAULT_BUCKETS = [0, 5, 10, 25, 50, 75, 100, 250, 500, 750, 1000, 2500, 5000, 7500, 10_000].map(&:to_f).freeze
|
|
16
|
+
|
|
17
|
+
class Series
|
|
18
|
+
attr_reader :name, :kind, :unit, :labels, :id, :bounds
|
|
19
|
+
|
|
20
|
+
def initialize(name, kind, unit, labels, id, bounds)
|
|
21
|
+
@name = name
|
|
22
|
+
@kind = kind
|
|
23
|
+
@unit = unit
|
|
24
|
+
@labels = labels
|
|
25
|
+
@id = id
|
|
26
|
+
@bounds = bounds
|
|
27
|
+
@mutex = Mutex.new
|
|
28
|
+
@touched = false
|
|
29
|
+
@total = 0.0
|
|
30
|
+
@value = 0.0
|
|
31
|
+
reset_histogram
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def add(n)
|
|
35
|
+
@mutex.synchronize { @total += n }
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def set(v)
|
|
39
|
+
@mutex.synchronize do
|
|
40
|
+
@value = v
|
|
41
|
+
@touched = true
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def move(delta)
|
|
46
|
+
@mutex.synchronize do
|
|
47
|
+
@value += delta
|
|
48
|
+
@touched = true
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def observe(v)
|
|
53
|
+
i = @bounds.bsearch_index { |b| b >= v } || @bounds.size
|
|
54
|
+
@mutex.synchronize do
|
|
55
|
+
@counts[i] += 1
|
|
56
|
+
@min = v if @count.zero? || v < @min
|
|
57
|
+
@max = v if @count.zero? || v > @max
|
|
58
|
+
@count += 1
|
|
59
|
+
@sum += v
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Counters and histograms start the next interval from zero
|
|
64
|
+
def take
|
|
65
|
+
@mutex.synchronize do
|
|
66
|
+
case @kind
|
|
67
|
+
when "counter"
|
|
68
|
+
next nil unless @total.positive?
|
|
69
|
+
v = @total
|
|
70
|
+
@total = 0.0
|
|
71
|
+
{ value: v }
|
|
72
|
+
when "gauge"
|
|
73
|
+
@touched ? { value: @value } : nil
|
|
74
|
+
else
|
|
75
|
+
next nil if @count.zero?
|
|
76
|
+
row = { value: @sum / @count, count: @count, sum: @sum, min: @min, max: @max,
|
|
77
|
+
buckets: Metrics.encode_buckets(@bounds, @counts) }
|
|
78
|
+
reset_histogram
|
|
79
|
+
row
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
private
|
|
85
|
+
|
|
86
|
+
def reset_histogram
|
|
87
|
+
return unless @bounds
|
|
88
|
+
|
|
89
|
+
@counts = Array.new(@bounds.size + 1, 0)
|
|
90
|
+
@count = 0
|
|
91
|
+
@sum = 0.0
|
|
92
|
+
@min = @max = 0.0
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
class Counter
|
|
97
|
+
def initialize(series) = @s = series
|
|
98
|
+
|
|
99
|
+
# Ignored unless positive and finite
|
|
100
|
+
def add(n = 1)
|
|
101
|
+
n = Metrics.number(n)
|
|
102
|
+
@s.add(n) if @s && n&.positive?
|
|
103
|
+
nil
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def inc = add(1)
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
class Gauge
|
|
110
|
+
def initialize(series) = @s = series
|
|
111
|
+
|
|
112
|
+
def set(v)
|
|
113
|
+
v = Metrics.number(v)
|
|
114
|
+
@s.set(v) if @s && v
|
|
115
|
+
nil
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# Moves the gauge by delta, from 0 when it was never set
|
|
119
|
+
def add(delta)
|
|
120
|
+
delta = Metrics.number(delta)
|
|
121
|
+
@s.move(delta) if @s && delta
|
|
122
|
+
nil
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
class Histogram
|
|
127
|
+
def initialize(series) = @s = series
|
|
128
|
+
|
|
129
|
+
def observe(v)
|
|
130
|
+
v = Metrics.number(v)
|
|
131
|
+
@s.observe(v) if @s && v
|
|
132
|
+
nil
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
class Registry
|
|
137
|
+
attr_accessor :max_series
|
|
138
|
+
|
|
139
|
+
def initialize
|
|
140
|
+
@mutex = Mutex.new
|
|
141
|
+
@series = {}
|
|
142
|
+
@per_name = Hash.new(0)
|
|
143
|
+
@dropped = Hash.new(0)
|
|
144
|
+
@opts = {}
|
|
145
|
+
@max_series = DEFAULT_MAX_SERIES
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def describe(name, unit, buckets)
|
|
149
|
+
name = Metrics.clean_name(name)
|
|
150
|
+
return if name.empty?
|
|
151
|
+
|
|
152
|
+
buckets = buckets&.filter_map { |b| Metrics.number(b) }&.sort&.uniq
|
|
153
|
+
unit = unit.to_s.strip
|
|
154
|
+
@mutex.synchronize { @opts[name] = { unit: unit.empty? ? nil : unit, buckets: buckets && !buckets.empty? ? buckets : nil } }
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# nil when the name is empty or over its cap
|
|
158
|
+
def get(name, kind, labels)
|
|
159
|
+
name = Metrics.clean_name(name)
|
|
160
|
+
return nil if name.empty?
|
|
161
|
+
|
|
162
|
+
labels = Metrics.clean_labels(labels)
|
|
163
|
+
key = [name, kind, labels]
|
|
164
|
+
@mutex.synchronize do
|
|
165
|
+
s = @series[key]
|
|
166
|
+
return s if s
|
|
167
|
+
if @per_name[name] >= @max_series
|
|
168
|
+
@dropped[name] += 1
|
|
169
|
+
return nil
|
|
170
|
+
end
|
|
171
|
+
text, id = Metrics.series(name, labels)
|
|
172
|
+
opts = @opts[name] || {}
|
|
173
|
+
bounds = kind == "histogram" ? (opts[:buckets] || DEFAULT_BUCKETS) : nil
|
|
174
|
+
@per_name[name] += 1
|
|
175
|
+
@series[key] = Series.new(name, kind, opts[:unit], text, id, bounds)
|
|
176
|
+
end
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
def collect
|
|
180
|
+
all, dropped = @mutex.synchronize do
|
|
181
|
+
d = @dropped
|
|
182
|
+
@dropped = Hash.new(0) unless d.empty?
|
|
183
|
+
[@series.values, d]
|
|
184
|
+
end
|
|
185
|
+
rows = all.filter_map do |s|
|
|
186
|
+
v = s.take
|
|
187
|
+
v && [s, v]
|
|
188
|
+
end
|
|
189
|
+
rows.sort_by! { |s, _| s.id }
|
|
190
|
+
[rows.map { |s, v| Metrics.row(s, v) }, dropped]
|
|
191
|
+
end
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
# Columns after created_at, service and instance
|
|
195
|
+
def self.row(s, v)
|
|
196
|
+
[s.name, s.kind, s.unit, s.labels, s.id, v[:value], v[:count], v[:sum], v[:min], v[:max], v[:buckets]]
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
@registry = Registry.new
|
|
200
|
+
|
|
201
|
+
class << self
|
|
202
|
+
attr_reader :registry
|
|
203
|
+
|
|
204
|
+
def counter(name, labels) = Counter.new(get(name, "counter", labels))
|
|
205
|
+
def gauge(name, labels) = Gauge.new(get(name, "gauge", labels))
|
|
206
|
+
def histogram(name, labels) = Histogram.new(get(name, "histogram", labels))
|
|
207
|
+
|
|
208
|
+
def get(name, kind, labels)
|
|
209
|
+
Visionary.config.metrics ? registry.get(name, kind, labels) : nil
|
|
210
|
+
end
|
|
211
|
+
def describe(name, unit:, buckets:) = registry.describe(name, unit, buckets)
|
|
212
|
+
|
|
213
|
+
def labels(hash, kw)
|
|
214
|
+
return kw if hash.nil?
|
|
215
|
+
return hash if kw.empty?
|
|
216
|
+
|
|
217
|
+
hash.to_h.merge(kw)
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
def number(v)
|
|
221
|
+
f = Float(v)
|
|
222
|
+
f.finite? ? f : nil
|
|
223
|
+
rescue ArgumentError, TypeError
|
|
224
|
+
nil
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
def clean_name(name) = cut(name.to_s.strip, MAX_NAME)
|
|
228
|
+
|
|
229
|
+
def clean_labels(labels)
|
|
230
|
+
return {} if labels.nil? || labels.empty?
|
|
231
|
+
|
|
232
|
+
out = {}
|
|
233
|
+
labels.each do |k, v|
|
|
234
|
+
k = cut(k.to_s.strip, MAX_KEY)
|
|
235
|
+
next if k.empty?
|
|
236
|
+
out[k] = cut(v.nil? ? "" : v.to_s, MAX_VALUE)
|
|
237
|
+
end
|
|
238
|
+
out.keys.sort.first(MAX_LABELS).to_h { |k| [k, out[k]] }
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
# Labels as JSON with sorted keys, and the first 16 hex digits of SHA-256 over name, newline, JSON
|
|
242
|
+
def series(name, labels)
|
|
243
|
+
text = +"{"
|
|
244
|
+
labels.keys.sort.each_with_index do |k, i|
|
|
245
|
+
text << "," if i.positive?
|
|
246
|
+
text << json_string(k) << ":" << json_string(labels[k])
|
|
247
|
+
end
|
|
248
|
+
text << "}"
|
|
249
|
+
[text, Digest::SHA256.hexdigest("#{name}\n#{text}")[0, 16]]
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
# Escapes only what JSON requires, so every SDK produces the same bytes
|
|
253
|
+
def json_string(s)
|
|
254
|
+
out = +"\""
|
|
255
|
+
s.each_char do |c|
|
|
256
|
+
out << case c
|
|
257
|
+
when "\"" then "\\\""
|
|
258
|
+
when "\\" then "\\\\"
|
|
259
|
+
else c.ord < 0x20 ? format("\\u%04x", c.ord) : c
|
|
260
|
+
end
|
|
261
|
+
end
|
|
262
|
+
out << "\""
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
def encode_buckets(bounds, counts)
|
|
266
|
+
"{\"bounds\":[#{bounds.map { |b| num(b) }.join(',')}],\"counts\":[#{counts.join(',')}]}"
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
def num(f) = f == f.floor && f.abs < 1e15 ? f.to_i.to_s : f.to_s
|
|
270
|
+
|
|
271
|
+
def cut(s, max)
|
|
272
|
+
s = Visionary.utf8(s)
|
|
273
|
+
return s if s.bytesize <= max
|
|
274
|
+
|
|
275
|
+
s.byteslice(0, max).scrub("")
|
|
276
|
+
end
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
class Recorder
|
|
280
|
+
def initialize(cfg, instance, store, schemas, logger, registry: Metrics.registry)
|
|
281
|
+
@cfg = cfg
|
|
282
|
+
@instance = instance
|
|
283
|
+
@logger = logger
|
|
284
|
+
@registry = registry
|
|
285
|
+
@registry.max_series = cfg.metrics_max_series if cfg.metrics_max_series.to_i.positive?
|
|
286
|
+
@interval = cfg.metrics_interval.to_f.positive? ? cfg.metrics_interval.to_f : 15.0
|
|
287
|
+
@writer = Writer.new(cfg, store, schemas, logger)
|
|
288
|
+
reset
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
attr_reader :writer
|
|
292
|
+
|
|
293
|
+
def start
|
|
294
|
+
@writer.start
|
|
295
|
+
spawn
|
|
296
|
+
self
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
def close(timeout)
|
|
300
|
+
@mutex.synchronize do
|
|
301
|
+
@stopped = true
|
|
302
|
+
@cond.broadcast
|
|
303
|
+
end
|
|
304
|
+
@thread&.join(timeout)
|
|
305
|
+
@writer.close(timeout)
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
# The parent writes what it recorded before the fork
|
|
309
|
+
def after_fork
|
|
310
|
+
@registry.collect
|
|
311
|
+
reset
|
|
312
|
+
@writer.after_fork
|
|
313
|
+
spawn
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
def flush(at = Time.now)
|
|
317
|
+
rows, dropped = @registry.collect
|
|
318
|
+
dropped.each do |name, n|
|
|
319
|
+
@logger.warn("visionary: metric series dropped over the limit (metric=#{name}, dropped=#{n}, " \
|
|
320
|
+
"limit=#{@registry.max_series})")
|
|
321
|
+
end
|
|
322
|
+
return if rows.empty?
|
|
323
|
+
|
|
324
|
+
# Services without custom metrics never get the table
|
|
325
|
+
@writer.use_table!
|
|
326
|
+
stamp = Visionary.timestamp(at)
|
|
327
|
+
rows.each do |r|
|
|
328
|
+
@writer.enqueue([:metric, [stamp, @cfg.service, @instance, *r]])
|
|
329
|
+
end
|
|
330
|
+
end
|
|
331
|
+
|
|
332
|
+
private
|
|
333
|
+
|
|
334
|
+
def reset
|
|
335
|
+
@mutex = Mutex.new
|
|
336
|
+
@cond = ConditionVariable.new
|
|
337
|
+
@stopped = false
|
|
338
|
+
@thread = nil
|
|
339
|
+
end
|
|
340
|
+
|
|
341
|
+
def spawn
|
|
342
|
+
@thread = Thread.new { run }
|
|
343
|
+
@thread.name = "visionary-metrics" if @thread.respond_to?(:name=)
|
|
344
|
+
end
|
|
345
|
+
|
|
346
|
+
def run
|
|
347
|
+
Telemetry.internal!
|
|
348
|
+
loop do
|
|
349
|
+
stop = @mutex.synchronize do
|
|
350
|
+
deadline = mono + @interval
|
|
351
|
+
until @stopped || (left = deadline - mono) <= 0
|
|
352
|
+
@cond.wait(@mutex, left)
|
|
353
|
+
end
|
|
354
|
+
@stopped
|
|
355
|
+
end
|
|
356
|
+
flush
|
|
357
|
+
break if stop
|
|
358
|
+
end
|
|
359
|
+
rescue StandardError => e
|
|
360
|
+
@logger.error("visionary: metrics stopped: #{e.message}")
|
|
361
|
+
end
|
|
362
|
+
|
|
363
|
+
def mono = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
364
|
+
end
|
|
365
|
+
|
|
366
|
+
class Writer < Visionary::Writer
|
|
367
|
+
QUEUE_SIZE = 20_000
|
|
368
|
+
|
|
369
|
+
def initialize(cfg, store, schemas, logger)
|
|
370
|
+
super
|
|
371
|
+
@groups = []
|
|
372
|
+
@queue_size = QUEUE_SIZE
|
|
373
|
+
reset
|
|
374
|
+
end
|
|
375
|
+
|
|
376
|
+
def use_table!
|
|
377
|
+
@groups = [:metrics] if @groups.empty?
|
|
378
|
+
end
|
|
379
|
+
|
|
380
|
+
private
|
|
381
|
+
|
|
382
|
+
def thread_name = "visionary-metrics-writer"
|
|
383
|
+
|
|
384
|
+
def report_drops
|
|
385
|
+
n = @queue.take_dropped
|
|
386
|
+
@logger.warn("visionary: metrics queue full; #{n} rows dropped") if n.positive?
|
|
387
|
+
end
|
|
388
|
+
|
|
389
|
+
def statements(kind, row, _users) = [fit(kind, row)]
|
|
390
|
+
end
|
|
391
|
+
end
|
|
392
|
+
end
|
data/lib/visionary/middleware.rb
CHANGED
|
@@ -26,9 +26,13 @@ module Visionary
|
|
|
26
26
|
|
|
27
27
|
path = "#{env['SCRIPT_NAME']}#{env['PATH_INFO']}"
|
|
28
28
|
path = "/" if path.empty?
|
|
29
|
-
|
|
29
|
+
# Unlogged requests still pass the trace on to their outbound calls
|
|
30
|
+
span, parent = TraceContext.request_span(env)
|
|
31
|
+
TraceContext.with(span) do
|
|
32
|
+
next passthrough(agent, env) unless agent.log_path?(path)
|
|
30
33
|
|
|
31
|
-
|
|
34
|
+
log(agent, env, path, span, parent)
|
|
35
|
+
end
|
|
32
36
|
end
|
|
33
37
|
|
|
34
38
|
def self.capturable?(content_type)
|
|
@@ -91,7 +95,7 @@ module Visionary
|
|
|
91
95
|
end
|
|
92
96
|
end
|
|
93
97
|
|
|
94
|
-
def log(agent, env, path)
|
|
98
|
+
def log(agent, env, path, span, parent)
|
|
95
99
|
cfg = agent.config
|
|
96
100
|
started_at = Time.now
|
|
97
101
|
t0 = Telemetry.now_ms
|
|
@@ -107,7 +111,7 @@ module Visionary
|
|
|
107
111
|
ctx = RequestContext.new(
|
|
108
112
|
request_id: request_id, method: env["REQUEST_METHOD"].to_s, path: path, started_at: started_at,
|
|
109
113
|
started_mono: t0, span_max: cfg.span_max.to_i, db_visible: Telemetry::ActiveRecordHook.installed?,
|
|
110
|
-
|
|
114
|
+
span: span, parent_span_id: parent
|
|
111
115
|
)
|
|
112
116
|
agent.http.started
|
|
113
117
|
Profiler::Tally.started!
|
|
@@ -165,7 +169,8 @@ module Visionary
|
|
|
165
169
|
response_size: [self.class.header(res_headers, "content-length").to_i, resp_capture.size].max,
|
|
166
170
|
totals: tel&.spans_on ? ctx.totals : [nil, nil, nil, nil],
|
|
167
171
|
span_rows: tel ? tel.request_span_rows(ctx) : [],
|
|
168
|
-
error_rows: tel ? ctx.errors.map { |row| tel.error_row(row, ctx, route, user_id) } : []
|
|
172
|
+
error_rows: tel ? ctx.errors.map { |row| tel.error_row(row, ctx, route, user_id) } : [],
|
|
173
|
+
trace_id: ctx.trace_id, span_id: ctx.span_id, parent_span_id: ctx.parent_span_id
|
|
169
174
|
)
|
|
170
175
|
agent.enqueue([:request, entry])
|
|
171
176
|
rescue StandardError => e
|
data/lib/visionary/queue.rb
CHANGED
|
@@ -9,6 +9,7 @@ module Visionary
|
|
|
9
9
|
@max = max
|
|
10
10
|
@items = []
|
|
11
11
|
@dropped = 0
|
|
12
|
+
@dropped_total = 0
|
|
12
13
|
@closed = false
|
|
13
14
|
@mutex = Mutex.new
|
|
14
15
|
@cond = ConditionVariable.new
|
|
@@ -18,6 +19,7 @@ module Visionary
|
|
|
18
19
|
@mutex.synchronize do
|
|
19
20
|
if @items.size >= @max
|
|
20
21
|
@dropped += 1
|
|
22
|
+
@dropped_total += 1
|
|
21
23
|
return false
|
|
22
24
|
end
|
|
23
25
|
@items << item
|
|
@@ -41,6 +43,8 @@ module Visionary
|
|
|
41
43
|
|
|
42
44
|
def size = @mutex.synchronize { @items.size }
|
|
43
45
|
|
|
46
|
+
def dropped_total = @mutex.synchronize { @dropped_total }
|
|
47
|
+
|
|
44
48
|
def take_dropped
|
|
45
49
|
@mutex.synchronize { @dropped.tap { @dropped = 0 } }
|
|
46
50
|
end
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
CREATE TABLE IF NOT EXISTS metrics (
|
|
2
|
+
id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
|
|
3
|
+
created_at VARCHAR(32) NOT NULL,
|
|
4
|
+
service VARCHAR(255),
|
|
5
|
+
instance VARCHAR(255),
|
|
6
|
+
name VARCHAR(255) NOT NULL,
|
|
7
|
+
kind VARCHAR(16) NOT NULL,
|
|
8
|
+
unit VARCHAR(64),
|
|
9
|
+
labels TEXT NOT NULL,
|
|
10
|
+
series VARCHAR(16) NOT NULL,
|
|
11
|
+
value DOUBLE,
|
|
12
|
+
count BIGINT,
|
|
13
|
+
sum DOUBLE,
|
|
14
|
+
min DOUBLE,
|
|
15
|
+
max DOUBLE,
|
|
16
|
+
buckets TEXT
|
|
17
|
+
) CHARACTER SET utf8mb4;
|
|
18
|
+
CREATE INDEX idx_metrics_name_created_at ON metrics (name, created_at);
|
|
19
|
+
CREATE INDEX idx_metrics_series_created_at ON metrics (series, created_at);
|
|
20
|
+
CREATE INDEX idx_metrics_created_at ON metrics (created_at);
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
CREATE TABLE IF NOT EXISTS metrics (
|
|
2
|
+
id BIGSERIAL PRIMARY KEY,
|
|
3
|
+
created_at TEXT NOT NULL,
|
|
4
|
+
service TEXT,
|
|
5
|
+
instance TEXT,
|
|
6
|
+
name TEXT NOT NULL,
|
|
7
|
+
kind TEXT NOT NULL,
|
|
8
|
+
unit TEXT,
|
|
9
|
+
labels TEXT NOT NULL,
|
|
10
|
+
series TEXT NOT NULL,
|
|
11
|
+
value DOUBLE PRECISION,
|
|
12
|
+
count BIGINT,
|
|
13
|
+
sum DOUBLE PRECISION,
|
|
14
|
+
min DOUBLE PRECISION,
|
|
15
|
+
max DOUBLE PRECISION,
|
|
16
|
+
buckets TEXT
|
|
17
|
+
);
|
|
18
|
+
CREATE INDEX IF NOT EXISTS idx_metrics_name_created_at ON metrics (name, created_at);
|
|
19
|
+
CREATE INDEX IF NOT EXISTS idx_metrics_series_created_at ON metrics (series, created_at);
|
|
20
|
+
CREATE INDEX IF NOT EXISTS idx_metrics_created_at ON metrics (created_at);
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
CREATE TABLE IF NOT EXISTS metrics (
|
|
2
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
3
|
+
created_at TEXT NOT NULL,
|
|
4
|
+
service TEXT,
|
|
5
|
+
instance TEXT,
|
|
6
|
+
name TEXT NOT NULL,
|
|
7
|
+
kind TEXT NOT NULL,
|
|
8
|
+
unit TEXT,
|
|
9
|
+
labels TEXT NOT NULL,
|
|
10
|
+
series TEXT NOT NULL,
|
|
11
|
+
value REAL,
|
|
12
|
+
count INTEGER,
|
|
13
|
+
sum REAL,
|
|
14
|
+
min REAL,
|
|
15
|
+
max REAL,
|
|
16
|
+
buckets TEXT
|
|
17
|
+
);
|
|
18
|
+
CREATE INDEX IF NOT EXISTS idx_metrics_name_created_at ON metrics (name, created_at);
|
|
19
|
+
CREATE INDEX IF NOT EXISTS idx_metrics_series_created_at ON metrics (series, created_at);
|
|
20
|
+
CREATE INDEX IF NOT EXISTS idx_metrics_created_at ON metrics (created_at);
|
data/lib/visionary/schema.rb
CHANGED
|
@@ -17,7 +17,7 @@ module Visionary
|
|
|
17
17
|
# api_logs columns added after the first release, in insert order.
|
|
18
18
|
ADDED_REQUEST_COLUMNS = [
|
|
19
19
|
%w[service text], %w[version text], %w[db_count int], %w[db_ms real], %w[http_count int], %w[http_ms real],
|
|
20
|
-
%w[login_account text]
|
|
20
|
+
%w[login_account text], %w[trace_id text], %w[span_id text], %w[parent_span_id text]
|
|
21
21
|
].freeze
|
|
22
22
|
|
|
23
23
|
SYSTEM_BASE_COLUMNS = %w[
|
|
@@ -58,6 +58,9 @@ module Visionary
|
|
|
58
58
|
created_at request_id service instance kind name target detail offset_ms duration_ms status error row_count
|
|
59
59
|
].freeze
|
|
60
60
|
|
|
61
|
+
# Added after the first release, in insert order
|
|
62
|
+
ADDED_SPAN_COLUMNS = [%w[trace_id text], %w[span_id text], %w[parent_span_id text]].freeze
|
|
63
|
+
|
|
61
64
|
ERROR_COLUMNS = %w[
|
|
62
65
|
created_at service instance version request_id method route path user_id
|
|
63
66
|
type message stack fingerprint handled runtime
|
|
@@ -74,22 +77,27 @@ module Visionary
|
|
|
74
77
|
code_func size
|
|
75
78
|
].freeze
|
|
76
79
|
|
|
80
|
+
METRIC_COLUMNS = %w[
|
|
81
|
+
created_at service instance name kind unit labels series value count sum min max buckets
|
|
82
|
+
].freeze
|
|
83
|
+
|
|
77
84
|
TABLES = {
|
|
78
85
|
"api_logs" => REQUEST_COLUMNS + ADDED_REQUEST_COLUMNS.map(&:first),
|
|
79
86
|
"system_metrics" => SYSTEM_BASE_COLUMNS + ADDED_SYSTEM_COLUMNS.map(&:first),
|
|
80
|
-
"spans" => SPAN_COLUMNS,
|
|
87
|
+
"spans" => SPAN_COLUMNS + ADDED_SPAN_COLUMNS.map(&:first),
|
|
81
88
|
"errors" => ERROR_COLUMNS,
|
|
82
89
|
"releases" => RELEASE_COLUMNS,
|
|
83
90
|
"profiles" => PROFILE_COLUMNS,
|
|
84
|
-
"app_logs" => LOG_COLUMNS
|
|
91
|
+
"app_logs" => LOG_COLUMNS,
|
|
92
|
+
"metrics" => METRIC_COLUMNS
|
|
85
93
|
}.freeze
|
|
86
94
|
|
|
87
|
-
# Which script creates a table, and the columns older copies may lack.
|
|
88
95
|
GROUPS = {
|
|
89
|
-
requests: { file: "requests.sql", table: "api_logs", added: ADDED_REQUEST_COLUMNS },
|
|
96
|
+
requests: { file: "requests.sql", table: "api_logs", added: ADDED_REQUEST_COLUMNS, trace_index: true },
|
|
90
97
|
system: { file: "system.sql", table: "system_metrics", added: ADDED_SYSTEM_COLUMNS },
|
|
91
|
-
telemetry: { file: "telemetry.sql", table:
|
|
92
|
-
logs: { file: "logs.sql", table: "app_logs", added: [] }
|
|
98
|
+
telemetry: { file: "telemetry.sql", table: "spans", added: ADDED_SPAN_COLUMNS, trace_index: true },
|
|
99
|
+
logs: { file: "logs.sql", table: "app_logs", added: [] },
|
|
100
|
+
metrics: { file: "metrics.sql", table: "metrics", added: [] }
|
|
93
101
|
}.freeze
|
|
94
102
|
|
|
95
103
|
PURGE_CHUNK = 5000
|
|
@@ -124,6 +132,12 @@ module Visionary
|
|
|
124
132
|
end
|
|
125
133
|
end
|
|
126
134
|
|
|
135
|
+
# Not in the shipped scripts since older tables only get trace_id on upgrade
|
|
136
|
+
def trace_index(dialect, table)
|
|
137
|
+
ine = dialect == "mysql" ? "" : "IF NOT EXISTS "
|
|
138
|
+
"CREATE INDEX #{ine}idx_#{table}_trace_id ON #{table} (trace_id)"
|
|
139
|
+
end
|
|
140
|
+
|
|
127
141
|
# ALTER statements for the added columns missing from existing names.
|
|
128
142
|
def missing_columns(dialect, table, added, existing)
|
|
129
143
|
have = existing.map { |c| c.to_s.downcase }
|
|
@@ -155,6 +169,12 @@ module Visionary
|
|
|
155
169
|
"INSERT INTO #{table} (#{cols.join(', ')}) VALUES (#{([mark] * cols.size).join(', ')})"
|
|
156
170
|
end
|
|
157
171
|
|
|
172
|
+
def insert_rows_sql(table, n)
|
|
173
|
+
cols = TABLES.fetch(table)
|
|
174
|
+
row = "(#{(['?'] * cols.size).join(', ')})"
|
|
175
|
+
"INSERT INTO #{table} (#{cols.join(', ')}) VALUES #{([row] * n).join(', ')}"
|
|
176
|
+
end
|
|
177
|
+
|
|
158
178
|
# MySQL in strict mode rejects over-long values, which would fail the
|
|
159
179
|
# whole batch, so values are cut to the column size first: characters for
|
|
160
180
|
# VARCHAR(n), bytes for TEXT and MEDIUMTEXT. Read from the shipped schema.
|
data/lib/visionary/schemas.rb
CHANGED