visionary-sdk 1.1.1 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,392 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "digest"
4
+
5
+ module Visionary
6
+ # Counters, gauges and histograms, written as one metrics row per series every interval
7
+ module Metrics
8
+ MAX_NAME = 200
9
+ MAX_LABELS = 20
10
+ MAX_KEY = 100
11
+ MAX_VALUE = 200
12
+ DEFAULT_MAX_SERIES = 1000
13
+
14
+ # OpenTelemetry's defaults, in milliseconds
15
+ DEFAULT_BUCKETS = [0, 5, 10, 25, 50, 75, 100, 250, 500, 750, 1000, 2500, 5000, 7500, 10_000].map(&:to_f).freeze
16
+
17
+ class Series
18
+ attr_reader :name, :kind, :unit, :labels, :id, :bounds
19
+
20
+ def initialize(name, kind, unit, labels, id, bounds)
21
+ @name = name
22
+ @kind = kind
23
+ @unit = unit
24
+ @labels = labels
25
+ @id = id
26
+ @bounds = bounds
27
+ @mutex = Mutex.new
28
+ @touched = false
29
+ @total = 0.0
30
+ @value = 0.0
31
+ reset_histogram
32
+ end
33
+
34
+ def add(n)
35
+ @mutex.synchronize { @total += n }
36
+ end
37
+
38
+ def set(v)
39
+ @mutex.synchronize do
40
+ @value = v
41
+ @touched = true
42
+ end
43
+ end
44
+
45
+ def move(delta)
46
+ @mutex.synchronize do
47
+ @value += delta
48
+ @touched = true
49
+ end
50
+ end
51
+
52
+ def observe(v)
53
+ i = @bounds.bsearch_index { |b| b >= v } || @bounds.size
54
+ @mutex.synchronize do
55
+ @counts[i] += 1
56
+ @min = v if @count.zero? || v < @min
57
+ @max = v if @count.zero? || v > @max
58
+ @count += 1
59
+ @sum += v
60
+ end
61
+ end
62
+
63
+ # Counters and histograms start the next interval from zero
64
+ def take
65
+ @mutex.synchronize do
66
+ case @kind
67
+ when "counter"
68
+ next nil unless @total.positive?
69
+ v = @total
70
+ @total = 0.0
71
+ { value: v }
72
+ when "gauge"
73
+ @touched ? { value: @value } : nil
74
+ else
75
+ next nil if @count.zero?
76
+ row = { value: @sum / @count, count: @count, sum: @sum, min: @min, max: @max,
77
+ buckets: Metrics.encode_buckets(@bounds, @counts) }
78
+ reset_histogram
79
+ row
80
+ end
81
+ end
82
+ end
83
+
84
+ private
85
+
86
+ def reset_histogram
87
+ return unless @bounds
88
+
89
+ @counts = Array.new(@bounds.size + 1, 0)
90
+ @count = 0
91
+ @sum = 0.0
92
+ @min = @max = 0.0
93
+ end
94
+ end
95
+
96
+ class Counter
97
+ def initialize(series) = @s = series
98
+
99
+ # Ignored unless positive and finite
100
+ def add(n = 1)
101
+ n = Metrics.number(n)
102
+ @s.add(n) if @s && n&.positive?
103
+ nil
104
+ end
105
+
106
+ def inc = add(1)
107
+ end
108
+
109
+ class Gauge
110
+ def initialize(series) = @s = series
111
+
112
+ def set(v)
113
+ v = Metrics.number(v)
114
+ @s.set(v) if @s && v
115
+ nil
116
+ end
117
+
118
+ # Moves the gauge by delta, from 0 when it was never set
119
+ def add(delta)
120
+ delta = Metrics.number(delta)
121
+ @s.move(delta) if @s && delta
122
+ nil
123
+ end
124
+ end
125
+
126
+ class Histogram
127
+ def initialize(series) = @s = series
128
+
129
+ def observe(v)
130
+ v = Metrics.number(v)
131
+ @s.observe(v) if @s && v
132
+ nil
133
+ end
134
+ end
135
+
136
+ class Registry
137
+ attr_accessor :max_series
138
+
139
+ def initialize
140
+ @mutex = Mutex.new
141
+ @series = {}
142
+ @per_name = Hash.new(0)
143
+ @dropped = Hash.new(0)
144
+ @opts = {}
145
+ @max_series = DEFAULT_MAX_SERIES
146
+ end
147
+
148
+ def describe(name, unit, buckets)
149
+ name = Metrics.clean_name(name)
150
+ return if name.empty?
151
+
152
+ buckets = buckets&.filter_map { |b| Metrics.number(b) }&.sort&.uniq
153
+ unit = unit.to_s.strip
154
+ @mutex.synchronize { @opts[name] = { unit: unit.empty? ? nil : unit, buckets: buckets && !buckets.empty? ? buckets : nil } }
155
+ end
156
+
157
+ # nil when the name is empty or over its cap
158
+ def get(name, kind, labels)
159
+ name = Metrics.clean_name(name)
160
+ return nil if name.empty?
161
+
162
+ labels = Metrics.clean_labels(labels)
163
+ key = [name, kind, labels]
164
+ @mutex.synchronize do
165
+ s = @series[key]
166
+ return s if s
167
+ if @per_name[name] >= @max_series
168
+ @dropped[name] += 1
169
+ return nil
170
+ end
171
+ text, id = Metrics.series(name, labels)
172
+ opts = @opts[name] || {}
173
+ bounds = kind == "histogram" ? (opts[:buckets] || DEFAULT_BUCKETS) : nil
174
+ @per_name[name] += 1
175
+ @series[key] = Series.new(name, kind, opts[:unit], text, id, bounds)
176
+ end
177
+ end
178
+
179
+ def collect
180
+ all, dropped = @mutex.synchronize do
181
+ d = @dropped
182
+ @dropped = Hash.new(0) unless d.empty?
183
+ [@series.values, d]
184
+ end
185
+ rows = all.filter_map do |s|
186
+ v = s.take
187
+ v && [s, v]
188
+ end
189
+ rows.sort_by! { |s, _| s.id }
190
+ [rows.map { |s, v| Metrics.row(s, v) }, dropped]
191
+ end
192
+ end
193
+
194
+ # Columns after created_at, service and instance
195
+ def self.row(s, v)
196
+ [s.name, s.kind, s.unit, s.labels, s.id, v[:value], v[:count], v[:sum], v[:min], v[:max], v[:buckets]]
197
+ end
198
+
199
+ @registry = Registry.new
200
+
201
+ class << self
202
+ attr_reader :registry
203
+
204
+ def counter(name, labels) = Counter.new(get(name, "counter", labels))
205
+ def gauge(name, labels) = Gauge.new(get(name, "gauge", labels))
206
+ def histogram(name, labels) = Histogram.new(get(name, "histogram", labels))
207
+
208
+ def get(name, kind, labels)
209
+ Visionary.config.metrics ? registry.get(name, kind, labels) : nil
210
+ end
211
+ def describe(name, unit:, buckets:) = registry.describe(name, unit, buckets)
212
+
213
+ def labels(hash, kw)
214
+ return kw if hash.nil?
215
+ return hash if kw.empty?
216
+
217
+ hash.to_h.merge(kw)
218
+ end
219
+
220
+ def number(v)
221
+ f = Float(v)
222
+ f.finite? ? f : nil
223
+ rescue ArgumentError, TypeError
224
+ nil
225
+ end
226
+
227
+ def clean_name(name) = cut(name.to_s.strip, MAX_NAME)
228
+
229
+ def clean_labels(labels)
230
+ return {} if labels.nil? || labels.empty?
231
+
232
+ out = {}
233
+ labels.each do |k, v|
234
+ k = cut(k.to_s.strip, MAX_KEY)
235
+ next if k.empty?
236
+ out[k] = cut(v.nil? ? "" : v.to_s, MAX_VALUE)
237
+ end
238
+ out.keys.sort.first(MAX_LABELS).to_h { |k| [k, out[k]] }
239
+ end
240
+
241
+ # Labels as JSON with sorted keys, and the first 16 hex digits of SHA-256 over name, newline, JSON
242
+ def series(name, labels)
243
+ text = +"{"
244
+ labels.keys.sort.each_with_index do |k, i|
245
+ text << "," if i.positive?
246
+ text << json_string(k) << ":" << json_string(labels[k])
247
+ end
248
+ text << "}"
249
+ [text, Digest::SHA256.hexdigest("#{name}\n#{text}")[0, 16]]
250
+ end
251
+
252
+ # Escapes only what JSON requires, so every SDK produces the same bytes
253
+ def json_string(s)
254
+ out = +"\""
255
+ s.each_char do |c|
256
+ out << case c
257
+ when "\"" then "\\\""
258
+ when "\\" then "\\\\"
259
+ else c.ord < 0x20 ? format("\\u%04x", c.ord) : c
260
+ end
261
+ end
262
+ out << "\""
263
+ end
264
+
265
+ def encode_buckets(bounds, counts)
266
+ "{\"bounds\":[#{bounds.map { |b| num(b) }.join(',')}],\"counts\":[#{counts.join(',')}]}"
267
+ end
268
+
269
+ def num(f) = f == f.floor && f.abs < 1e15 ? f.to_i.to_s : f.to_s
270
+
271
+ def cut(s, max)
272
+ s = Visionary.utf8(s)
273
+ return s if s.bytesize <= max
274
+
275
+ s.byteslice(0, max).scrub("")
276
+ end
277
+ end
278
+
279
+ class Recorder
280
+ def initialize(cfg, instance, store, schemas, logger, registry: Metrics.registry)
281
+ @cfg = cfg
282
+ @instance = instance
283
+ @logger = logger
284
+ @registry = registry
285
+ @registry.max_series = cfg.metrics_max_series if cfg.metrics_max_series.to_i.positive?
286
+ @interval = cfg.metrics_interval.to_f.positive? ? cfg.metrics_interval.to_f : 15.0
287
+ @writer = Writer.new(cfg, store, schemas, logger)
288
+ reset
289
+ end
290
+
291
+ attr_reader :writer
292
+
293
+ def start
294
+ @writer.start
295
+ spawn
296
+ self
297
+ end
298
+
299
+ def close(timeout)
300
+ @mutex.synchronize do
301
+ @stopped = true
302
+ @cond.broadcast
303
+ end
304
+ @thread&.join(timeout)
305
+ @writer.close(timeout)
306
+ end
307
+
308
+ # The parent writes what it recorded before the fork
309
+ def after_fork
310
+ @registry.collect
311
+ reset
312
+ @writer.after_fork
313
+ spawn
314
+ end
315
+
316
+ def flush(at = Time.now)
317
+ rows, dropped = @registry.collect
318
+ dropped.each do |name, n|
319
+ @logger.warn("visionary: metric series dropped over the limit (metric=#{name}, dropped=#{n}, " \
320
+ "limit=#{@registry.max_series})")
321
+ end
322
+ return if rows.empty?
323
+
324
+ # Services without custom metrics never get the table
325
+ @writer.use_table!
326
+ stamp = Visionary.timestamp(at)
327
+ rows.each do |r|
328
+ @writer.enqueue([:metric, [stamp, @cfg.service, @instance, *r]])
329
+ end
330
+ end
331
+
332
+ private
333
+
334
+ def reset
335
+ @mutex = Mutex.new
336
+ @cond = ConditionVariable.new
337
+ @stopped = false
338
+ @thread = nil
339
+ end
340
+
341
+ def spawn
342
+ @thread = Thread.new { run }
343
+ @thread.name = "visionary-metrics" if @thread.respond_to?(:name=)
344
+ end
345
+
346
+ def run
347
+ Telemetry.internal!
348
+ loop do
349
+ stop = @mutex.synchronize do
350
+ deadline = mono + @interval
351
+ until @stopped || (left = deadline - mono) <= 0
352
+ @cond.wait(@mutex, left)
353
+ end
354
+ @stopped
355
+ end
356
+ flush
357
+ break if stop
358
+ end
359
+ rescue StandardError => e
360
+ @logger.error("visionary: metrics stopped: #{e.message}")
361
+ end
362
+
363
+ def mono = Process.clock_gettime(Process::CLOCK_MONOTONIC)
364
+ end
365
+
366
+ class Writer < Visionary::Writer
367
+ QUEUE_SIZE = 20_000
368
+
369
+ def initialize(cfg, store, schemas, logger)
370
+ super
371
+ @groups = []
372
+ @queue_size = QUEUE_SIZE
373
+ reset
374
+ end
375
+
376
+ def use_table!
377
+ @groups = [:metrics] if @groups.empty?
378
+ end
379
+
380
+ private
381
+
382
+ def thread_name = "visionary-metrics-writer"
383
+
384
+ def report_drops
385
+ n = @queue.take_dropped
386
+ @logger.warn("visionary: metrics queue full; #{n} rows dropped") if n.positive?
387
+ end
388
+
389
+ def statements(kind, row, _users) = [fit(kind, row)]
390
+ end
391
+ end
392
+ end
@@ -26,9 +26,13 @@ module Visionary
26
26
 
27
27
  path = "#{env['SCRIPT_NAME']}#{env['PATH_INFO']}"
28
28
  path = "/" if path.empty?
29
- return passthrough(agent, env) unless agent.log_path?(path)
29
+ # Unlogged requests still pass the trace on to their outbound calls
30
+ span, parent = TraceContext.request_span(env)
31
+ TraceContext.with(span) do
32
+ next passthrough(agent, env) unless agent.log_path?(path)
30
33
 
31
- log(agent, env, path)
34
+ log(agent, env, path, span, parent)
35
+ end
32
36
  end
33
37
 
34
38
  def self.capturable?(content_type)
@@ -91,7 +95,7 @@ module Visionary
91
95
  end
92
96
  end
93
97
 
94
- def log(agent, env, path)
98
+ def log(agent, env, path, span, parent)
95
99
  cfg = agent.config
96
100
  started_at = Time.now
97
101
  t0 = Telemetry.now_ms
@@ -107,7 +111,7 @@ module Visionary
107
111
  ctx = RequestContext.new(
108
112
  request_id: request_id, method: env["REQUEST_METHOD"].to_s, path: path, started_at: started_at,
109
113
  started_mono: t0, span_max: cfg.span_max.to_i, db_visible: Telemetry::ActiveRecordHook.installed?,
110
- trace_id: Logs.trace_parent_id(env["HTTP_TRACEPARENT"])
114
+ span: span, parent_span_id: parent
111
115
  )
112
116
  agent.http.started
113
117
  Profiler::Tally.started!
@@ -165,7 +169,8 @@ module Visionary
165
169
  response_size: [self.class.header(res_headers, "content-length").to_i, resp_capture.size].max,
166
170
  totals: tel&.spans_on ? ctx.totals : [nil, nil, nil, nil],
167
171
  span_rows: tel ? tel.request_span_rows(ctx) : [],
168
- error_rows: tel ? ctx.errors.map { |row| tel.error_row(row, ctx, route, user_id) } : []
172
+ error_rows: tel ? ctx.errors.map { |row| tel.error_row(row, ctx, route, user_id) } : [],
173
+ trace_id: ctx.trace_id, span_id: ctx.span_id, parent_span_id: ctx.parent_span_id
169
174
  )
170
175
  agent.enqueue([:request, entry])
171
176
  rescue StandardError => e
@@ -9,6 +9,7 @@ module Visionary
9
9
  @max = max
10
10
  @items = []
11
11
  @dropped = 0
12
+ @dropped_total = 0
12
13
  @closed = false
13
14
  @mutex = Mutex.new
14
15
  @cond = ConditionVariable.new
@@ -18,6 +19,7 @@ module Visionary
18
19
  @mutex.synchronize do
19
20
  if @items.size >= @max
20
21
  @dropped += 1
22
+ @dropped_total += 1
21
23
  return false
22
24
  end
23
25
  @items << item
@@ -41,6 +43,8 @@ module Visionary
41
43
 
42
44
  def size = @mutex.synchronize { @items.size }
43
45
 
46
+ def dropped_total = @mutex.synchronize { @dropped_total }
47
+
44
48
  def take_dropped
45
49
  @mutex.synchronize { @dropped.tap { @dropped = 0 } }
46
50
  end
@@ -0,0 +1,20 @@
1
+ CREATE TABLE IF NOT EXISTS metrics (
2
+ id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
3
+ created_at VARCHAR(32) NOT NULL,
4
+ service VARCHAR(255),
5
+ instance VARCHAR(255),
6
+ name VARCHAR(255) NOT NULL,
7
+ kind VARCHAR(16) NOT NULL,
8
+ unit VARCHAR(64),
9
+ labels TEXT NOT NULL,
10
+ series VARCHAR(16) NOT NULL,
11
+ value DOUBLE,
12
+ count BIGINT,
13
+ sum DOUBLE,
14
+ min DOUBLE,
15
+ max DOUBLE,
16
+ buckets TEXT
17
+ ) CHARACTER SET utf8mb4;
18
+ CREATE INDEX idx_metrics_name_created_at ON metrics (name, created_at);
19
+ CREATE INDEX idx_metrics_series_created_at ON metrics (series, created_at);
20
+ CREATE INDEX idx_metrics_created_at ON metrics (created_at);
@@ -0,0 +1,20 @@
1
+ CREATE TABLE IF NOT EXISTS metrics (
2
+ id BIGSERIAL PRIMARY KEY,
3
+ created_at TEXT NOT NULL,
4
+ service TEXT,
5
+ instance TEXT,
6
+ name TEXT NOT NULL,
7
+ kind TEXT NOT NULL,
8
+ unit TEXT,
9
+ labels TEXT NOT NULL,
10
+ series TEXT NOT NULL,
11
+ value DOUBLE PRECISION,
12
+ count BIGINT,
13
+ sum DOUBLE PRECISION,
14
+ min DOUBLE PRECISION,
15
+ max DOUBLE PRECISION,
16
+ buckets TEXT
17
+ );
18
+ CREATE INDEX IF NOT EXISTS idx_metrics_name_created_at ON metrics (name, created_at);
19
+ CREATE INDEX IF NOT EXISTS idx_metrics_series_created_at ON metrics (series, created_at);
20
+ CREATE INDEX IF NOT EXISTS idx_metrics_created_at ON metrics (created_at);
@@ -0,0 +1,20 @@
1
+ CREATE TABLE IF NOT EXISTS metrics (
2
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
3
+ created_at TEXT NOT NULL,
4
+ service TEXT,
5
+ instance TEXT,
6
+ name TEXT NOT NULL,
7
+ kind TEXT NOT NULL,
8
+ unit TEXT,
9
+ labels TEXT NOT NULL,
10
+ series TEXT NOT NULL,
11
+ value REAL,
12
+ count INTEGER,
13
+ sum REAL,
14
+ min REAL,
15
+ max REAL,
16
+ buckets TEXT
17
+ );
18
+ CREATE INDEX IF NOT EXISTS idx_metrics_name_created_at ON metrics (name, created_at);
19
+ CREATE INDEX IF NOT EXISTS idx_metrics_series_created_at ON metrics (series, created_at);
20
+ CREATE INDEX IF NOT EXISTS idx_metrics_created_at ON metrics (created_at);
@@ -17,7 +17,7 @@ module Visionary
17
17
  # api_logs columns added after the first release, in insert order.
18
18
  ADDED_REQUEST_COLUMNS = [
19
19
  %w[service text], %w[version text], %w[db_count int], %w[db_ms real], %w[http_count int], %w[http_ms real],
20
- %w[login_account text]
20
+ %w[login_account text], %w[trace_id text], %w[span_id text], %w[parent_span_id text]
21
21
  ].freeze
22
22
 
23
23
  SYSTEM_BASE_COLUMNS = %w[
@@ -58,6 +58,9 @@ module Visionary
58
58
  created_at request_id service instance kind name target detail offset_ms duration_ms status error row_count
59
59
  ].freeze
60
60
 
61
+ # Added after the first release, in insert order
62
+ ADDED_SPAN_COLUMNS = [%w[trace_id text], %w[span_id text], %w[parent_span_id text]].freeze
63
+
61
64
  ERROR_COLUMNS = %w[
62
65
  created_at service instance version request_id method route path user_id
63
66
  type message stack fingerprint handled runtime
@@ -74,22 +77,27 @@ module Visionary
74
77
  code_func size
75
78
  ].freeze
76
79
 
80
+ METRIC_COLUMNS = %w[
81
+ created_at service instance name kind unit labels series value count sum min max buckets
82
+ ].freeze
83
+
77
84
  TABLES = {
78
85
  "api_logs" => REQUEST_COLUMNS + ADDED_REQUEST_COLUMNS.map(&:first),
79
86
  "system_metrics" => SYSTEM_BASE_COLUMNS + ADDED_SYSTEM_COLUMNS.map(&:first),
80
- "spans" => SPAN_COLUMNS,
87
+ "spans" => SPAN_COLUMNS + ADDED_SPAN_COLUMNS.map(&:first),
81
88
  "errors" => ERROR_COLUMNS,
82
89
  "releases" => RELEASE_COLUMNS,
83
90
  "profiles" => PROFILE_COLUMNS,
84
- "app_logs" => LOG_COLUMNS
91
+ "app_logs" => LOG_COLUMNS,
92
+ "metrics" => METRIC_COLUMNS
85
93
  }.freeze
86
94
 
87
- # Which script creates a table, and the columns older copies may lack.
88
95
  GROUPS = {
89
- requests: { file: "requests.sql", table: "api_logs", added: ADDED_REQUEST_COLUMNS },
96
+ requests: { file: "requests.sql", table: "api_logs", added: ADDED_REQUEST_COLUMNS, trace_index: true },
90
97
  system: { file: "system.sql", table: "system_metrics", added: ADDED_SYSTEM_COLUMNS },
91
- telemetry: { file: "telemetry.sql", table: nil, added: [] },
92
- logs: { file: "logs.sql", table: "app_logs", added: [] }
98
+ telemetry: { file: "telemetry.sql", table: "spans", added: ADDED_SPAN_COLUMNS, trace_index: true },
99
+ logs: { file: "logs.sql", table: "app_logs", added: [] },
100
+ metrics: { file: "metrics.sql", table: "metrics", added: [] }
93
101
  }.freeze
94
102
 
95
103
  PURGE_CHUNK = 5000
@@ -124,6 +132,12 @@ module Visionary
124
132
  end
125
133
  end
126
134
 
135
+ # Not in the shipped scripts since older tables only get trace_id on upgrade
136
+ def trace_index(dialect, table)
137
+ ine = dialect == "mysql" ? "" : "IF NOT EXISTS "
138
+ "CREATE INDEX #{ine}idx_#{table}_trace_id ON #{table} (trace_id)"
139
+ end
140
+
127
141
  # ALTER statements for the added columns missing from existing names.
128
142
  def missing_columns(dialect, table, added, existing)
129
143
  have = existing.map { |c| c.to_s.downcase }
@@ -155,6 +169,12 @@ module Visionary
155
169
  "INSERT INTO #{table} (#{cols.join(', ')}) VALUES (#{([mark] * cols.size).join(', ')})"
156
170
  end
157
171
 
172
+ def insert_rows_sql(table, n)
173
+ cols = TABLES.fetch(table)
174
+ row = "(#{(['?'] * cols.size).join(', ')})"
175
+ "INSERT INTO #{table} (#{cols.join(', ')}) VALUES #{([row] * n).join(', ')}"
176
+ end
177
+
158
178
  # MySQL in strict mode rejects over-long values, which would fail the
159
179
  # whole batch, so values are cut to the column size first: characters for
160
180
  # VARCHAR(n), bytes for TEXT and MEDIUMTEXT. Read from the shipped schema.
@@ -59,6 +59,7 @@ module Visionary
59
59
  rescue StandardError => e
60
60
  raise unless Schema.duplicate_column?(e)
61
61
  end
62
+ @store.exec_script([Schema.trace_index(dialect, g[:table])]) if g[:trace_index]
62
63
  end
63
64
  end
64
65
  end
@@ -88,6 +88,7 @@ module Visionary
88
88
 
89
89
  def bind(conn, sql, args)
90
90
  i = -1
91
+ return sql.gsub("?") { conn.quote(args[i += 1]) } unless sql.include?("'")
91
92
  quote = nil
92
93
  out = +""
93
94
  sql.each_char do |ch|