visionary-sdk 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +19 -0
- data/LICENSE +202 -0
- data/README.md +198 -0
- data/lib/visionary/agent.rb +174 -0
- data/lib/visionary/body.rb +162 -0
- data/lib/visionary/config.rb +151 -0
- data/lib/visionary/context.rb +97 -0
- data/lib/visionary/fork.rb +16 -0
- data/lib/visionary/middleware.rb +169 -0
- data/lib/visionary/queue.rb +57 -0
- data/lib/visionary/railtie.rb +48 -0
- data/lib/visionary/redactor.rb +94 -0
- data/lib/visionary/retention.rb +90 -0
- data/lib/visionary/schema/mysql/requests.sql +37 -0
- data/lib/visionary/schema/mysql/system.sql +76 -0
- data/lib/visionary/schema/mysql/telemetry.sql +52 -0
- data/lib/visionary/schema/postgres/requests.sql +37 -0
- data/lib/visionary/schema/postgres/system.sql +76 -0
- data/lib/visionary/schema/postgres/telemetry.sql +52 -0
- data/lib/visionary/schema/sqlite/requests.sql +37 -0
- data/lib/visionary/schema/sqlite/system.sql +76 -0
- data/lib/visionary/schema/sqlite/telemetry.sql +52 -0
- data/lib/visionary/schema.rb +163 -0
- data/lib/visionary/schemas.rb +64 -0
- data/lib/visionary/store/active_record.rb +103 -0
- data/lib/visionary/store/d1.rb +129 -0
- data/lib/visionary/store/sql.rb +272 -0
- data/lib/visionary/store.rb +53 -0
- data/lib/visionary/system/recorder.rb +353 -0
- data/lib/visionary/system/sources.rb +191 -0
- data/lib/visionary/telemetry/errors.rb +118 -0
- data/lib/visionary/telemetry/hooks.rb +115 -0
- data/lib/visionary/telemetry/sql_text.rb +44 -0
- data/lib/visionary/telemetry.rb +116 -0
- data/lib/visionary/version.rb +5 -0
- data/lib/visionary/writer.rb +185 -0
- data/lib/visionary.rb +157 -0
- metadata +90 -0
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Visionary
|
|
4
|
+
# Copies up to max bytes of a body as it streams past, and counts all of it.
|
|
5
|
+
class Capture
|
|
6
|
+
attr_reader :size
|
|
7
|
+
|
|
8
|
+
def initialize(max, enabled)
|
|
9
|
+
@max = max
|
|
10
|
+
@enabled = enabled
|
|
11
|
+
@buf = enabled ? String.new(capacity: [max, 1024].min, encoding: Encoding::BINARY) : nil
|
|
12
|
+
@size = 0
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def feed(chunk, at = @size)
|
|
16
|
+
n = chunk.bytesize
|
|
17
|
+
if @enabled && at == @buf.bytesize && at < @max
|
|
18
|
+
@buf << chunk.byteslice(0, @max - at).b
|
|
19
|
+
end
|
|
20
|
+
@size = at + n if at + n > @size
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def bytes = @buf || ""
|
|
24
|
+
def truncated? = @enabled && @size > @max
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# Wraps rack.input: the app reads as before, and the first max bytes it
|
|
28
|
+
# reads are kept for the log.
|
|
29
|
+
class InputProxy
|
|
30
|
+
def initialize(input, capture)
|
|
31
|
+
@input = input
|
|
32
|
+
@capture = capture
|
|
33
|
+
@pos = 0
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def read(*args)
|
|
37
|
+
data = @input.read(*args)
|
|
38
|
+
observe(data)
|
|
39
|
+
data
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def gets(*args)
|
|
43
|
+
observe(@input.gets(*args))
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def each
|
|
47
|
+
return enum_for(:each) unless block_given?
|
|
48
|
+
@input.each { |line| yield observe(line) }
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def rewind
|
|
52
|
+
@pos = 0
|
|
53
|
+
@input.rewind
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def close = @input.respond_to?(:close) ? @input.close : nil
|
|
57
|
+
|
|
58
|
+
def respond_to_missing?(name, include_all = false) = @input.respond_to?(name, include_all)
|
|
59
|
+
|
|
60
|
+
def method_missing(name, *args, &block)
|
|
61
|
+
@input.respond_to?(name) ? @input.public_send(name, *args, &block) : super
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
private
|
|
65
|
+
|
|
66
|
+
def observe(data)
|
|
67
|
+
if data.is_a?(String) && !data.empty?
|
|
68
|
+
@capture.feed(data, @pos)
|
|
69
|
+
@pos += data.bytesize
|
|
70
|
+
end
|
|
71
|
+
data
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# Wraps a Rack response body. Chunks pass through untouched (streaming
|
|
76
|
+
# still streams); a capped copy is kept, and done is called once when the
|
|
77
|
+
# server closes the body.
|
|
78
|
+
class BodyProxy
|
|
79
|
+
def initialize(body, capture, &done)
|
|
80
|
+
@body = body
|
|
81
|
+
@capture = capture
|
|
82
|
+
@done = done
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def each(&block)
|
|
86
|
+
return enum_for(:each) unless block
|
|
87
|
+
@body.each do |chunk|
|
|
88
|
+
@capture.feed(chunk.to_s)
|
|
89
|
+
yield chunk
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Rack 3: a body with to_ary must close itself when it is called.
|
|
94
|
+
def to_ary
|
|
95
|
+
arr = @body.to_ary
|
|
96
|
+
arr.each { |chunk| @capture.feed(chunk.to_s) }
|
|
97
|
+
arr
|
|
98
|
+
ensure
|
|
99
|
+
close
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Rack 3 streaming bodies.
|
|
103
|
+
def call(stream)
|
|
104
|
+
@body.call(StreamProxy.new(stream, @capture))
|
|
105
|
+
ensure
|
|
106
|
+
finish
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def to_path = @body.to_path
|
|
110
|
+
|
|
111
|
+
def respond_to?(name, include_all = false)
|
|
112
|
+
case name.to_sym
|
|
113
|
+
when :to_ary, :to_path, :call, :each then @body.respond_to?(name, include_all)
|
|
114
|
+
else super
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def close
|
|
119
|
+
@body.close if @body.respond_to?(:close) && !@closed
|
|
120
|
+
ensure
|
|
121
|
+
@closed = true
|
|
122
|
+
finish
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
private
|
|
126
|
+
|
|
127
|
+
def finish
|
|
128
|
+
done = @done
|
|
129
|
+
@done = nil
|
|
130
|
+
done&.call
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def respond_to_missing?(name, include_all = false) = @body.respond_to?(name, include_all)
|
|
134
|
+
|
|
135
|
+
def method_missing(name, *args, &block)
|
|
136
|
+
@body.respond_to?(name) ? @body.public_send(name, *args, &block) : super
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
class StreamProxy
|
|
141
|
+
def initialize(stream, capture)
|
|
142
|
+
@stream = stream
|
|
143
|
+
@capture = capture
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def write(data)
|
|
147
|
+
@capture.feed(data.to_s)
|
|
148
|
+
@stream.write(data)
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def <<(data)
|
|
152
|
+
write(data)
|
|
153
|
+
self
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def respond_to_missing?(name, include_all = false) = @stream.respond_to?(name, include_all)
|
|
157
|
+
|
|
158
|
+
def method_missing(name, *args, &block)
|
|
159
|
+
@stream.respond_to?(name) ? @stream.public_send(name, *args, &block) : super
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
end
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Visionary
|
|
4
|
+
# Everything the agent needs. Config.from_env fills it from VISIONARY_*
|
|
5
|
+
# (see sdks/SPEC.md "Environment"); set attributes in Visionary.configure to
|
|
6
|
+
# override or to add hooks.
|
|
7
|
+
#
|
|
8
|
+
# Storage is chosen in this order: +connection+ (an ActiveRecord connection
|
|
9
|
+
# class such as ActiveRecord::Base), then +database_url+, then the D1
|
|
10
|
+
# credentials.
|
|
11
|
+
class Config
|
|
12
|
+
COMMIT_VARIABLES = %w[
|
|
13
|
+
RAILWAY_GIT_COMMIT_SHA RENDER_GIT_COMMIT VERCEL_GIT_COMMIT_SHA HEROKU_SLUG_COMMIT
|
|
14
|
+
SOURCE_VERSION GIT_COMMIT GITHUB_SHA K_REVISION
|
|
15
|
+
].freeze
|
|
16
|
+
|
|
17
|
+
attr_accessor :database_url, :d1_account_id, :d1_database_id, :d1_api_token, :d1_api_base,
|
|
18
|
+
:connection, :instance, :logger,
|
|
19
|
+
:log_requests, :path_prefix, :capture_bodies, :max_body_bytes,
|
|
20
|
+
:queue_size, :batch_size, :flush_interval, :log_retention_days,
|
|
21
|
+
:redact_headers, :redact_fields, :redact_request_fields,
|
|
22
|
+
:error_code_field, :error_key_field,
|
|
23
|
+
:system_metrics, :system_interval, :system_retention_days, :disk_path,
|
|
24
|
+
:service, :version, :spans, :span_threshold_ms, :span_max, :errors
|
|
25
|
+
|
|
26
|
+
# Runs on the writer thread, once per distinct user per batch, for users
|
|
27
|
+
# missing a handle or role: ->(id) { [handle, role] } or nil.
|
|
28
|
+
attr_accessor :lookup_user
|
|
29
|
+
# Returns the route pattern for a Rack env when the integration can't
|
|
30
|
+
# see it: ->(env) { "/users/:id" }.
|
|
31
|
+
attr_accessor :route
|
|
32
|
+
# Returns pool stats as a Hash: total, idle, in_use, max, acquires,
|
|
33
|
+
# empty_acquires, acquire_wait_ms (the last three cumulative). Defaults to
|
|
34
|
+
# the ActiveRecord pool when ActiveRecord is loaded.
|
|
35
|
+
attr_accessor :db_stats
|
|
36
|
+
|
|
37
|
+
def initialize
|
|
38
|
+
@database_url = ""
|
|
39
|
+
@d1_account_id = @d1_database_id = @d1_api_token = @d1_api_base = ""
|
|
40
|
+
@connection = nil
|
|
41
|
+
@instance = ""
|
|
42
|
+
@logger = nil
|
|
43
|
+
@log_requests = true
|
|
44
|
+
@path_prefix = ""
|
|
45
|
+
@capture_bodies = true
|
|
46
|
+
@max_body_bytes = 8192
|
|
47
|
+
@queue_size = 5000
|
|
48
|
+
@batch_size = 50
|
|
49
|
+
@flush_interval = 3.0
|
|
50
|
+
@log_retention_days = 30
|
|
51
|
+
@redact_headers = []
|
|
52
|
+
@redact_fields = []
|
|
53
|
+
@redact_request_fields = []
|
|
54
|
+
@error_code_field = "error.code"
|
|
55
|
+
@error_key_field = "error.key"
|
|
56
|
+
@system_metrics = true
|
|
57
|
+
@system_interval = 15.0
|
|
58
|
+
@system_retention_days = 30
|
|
59
|
+
@disk_path = ""
|
|
60
|
+
@service = nil
|
|
61
|
+
@version = nil
|
|
62
|
+
@spans = true
|
|
63
|
+
@span_threshold_ms = 100.0
|
|
64
|
+
@span_max = 100
|
|
65
|
+
@errors = true
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def self.from_env(env = ENV)
|
|
69
|
+
s = ->(key) { env[key].to_s.strip }
|
|
70
|
+
c = new
|
|
71
|
+
c.database_url = s.("VISIONARY_DATABASE_URL")
|
|
72
|
+
c.d1_account_id = s.("D1_ACCOUNT_ID")
|
|
73
|
+
c.d1_database_id = s.("D1_DATABASE_ID")
|
|
74
|
+
c.d1_api_token = s.("D1_API_TOKEN")
|
|
75
|
+
c.d1_api_base = s.("D1_API_BASE")
|
|
76
|
+
c.instance = s.("VISIONARY_INSTANCE")
|
|
77
|
+
c.log_requests = parse_bool(s.("VISIONARY_LOG_REQUESTS"), true)
|
|
78
|
+
c.path_prefix = s.("VISIONARY_LOG_PATH_PREFIX")
|
|
79
|
+
c.capture_bodies = parse_bool(s.("VISIONARY_LOG_BODIES"), true)
|
|
80
|
+
c.max_body_bytes = parse_int(s.("VISIONARY_LOG_MAX_BODY_BYTES"), 8192)
|
|
81
|
+
c.log_retention_days = parse_int(s.("VISIONARY_LOG_RETENTION_DAYS"), 30)
|
|
82
|
+
c.redact_headers = parse_list(s.("VISIONARY_LOG_REDACT_HEADERS"))
|
|
83
|
+
c.redact_fields = parse_list(s.("VISIONARY_LOG_REDACT_FIELDS"))
|
|
84
|
+
c.redact_request_fields = parse_list(s.("VISIONARY_LOG_REDACT_REQUEST_FIELDS"))
|
|
85
|
+
c.error_code_field = s.("VISIONARY_LOG_ERROR_CODE_FIELD").then { |v| v.empty? ? "error.code" : v }
|
|
86
|
+
c.error_key_field = s.("VISIONARY_LOG_ERROR_KEY_FIELD").then { |v| v.empty? ? "error.key" : v }
|
|
87
|
+
c.system_metrics = parse_bool(s.("VISIONARY_SYSTEM_METRICS"), true)
|
|
88
|
+
c.system_interval = parse_duration(s.("VISIONARY_SYSTEM_INTERVAL"), 15.0)
|
|
89
|
+
c.system_retention_days = parse_int(s.("VISIONARY_SYSTEM_RETENTION_DAYS"), 30)
|
|
90
|
+
c.service = s.("VISIONARY_SERVICE").then { |v| v.empty? ? nil : v }
|
|
91
|
+
c.version = release_from(env)
|
|
92
|
+
c.spans = parse_bool(s.("VISIONARY_SPANS"), true)
|
|
93
|
+
c.span_threshold_ms = parse_float(s.("VISIONARY_SPAN_THRESHOLD_MS"), 100.0)
|
|
94
|
+
c.span_max = parse_int(s.("VISIONARY_SPAN_MAX"), 100)
|
|
95
|
+
c.errors = parse_bool(s.("VISIONARY_ERRORS"), true)
|
|
96
|
+
c
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# VISIONARY_RELEASE, else the platform's commit variable; commit hashes
|
|
100
|
+
# are shortened to 12 characters.
|
|
101
|
+
def self.release_from(env)
|
|
102
|
+
["VISIONARY_RELEASE", *COMMIT_VARIABLES].each do |key|
|
|
103
|
+
v = env[key].to_s.strip
|
|
104
|
+
next if v.empty?
|
|
105
|
+
return v.match?(/\A\h{13,64}\z/) ? v[0, 12] : v
|
|
106
|
+
end
|
|
107
|
+
nil
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def self.parse_bool(v, default)
|
|
111
|
+
case v.to_s.strip.downcase
|
|
112
|
+
when "true", "1", "yes" then true
|
|
113
|
+
when "false", "0", "no" then false
|
|
114
|
+
else default
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def self.parse_int(v, default)
|
|
119
|
+
n = Integer(v.to_s.strip, 10)
|
|
120
|
+
n >= 0 ? n : default
|
|
121
|
+
rescue ArgumentError
|
|
122
|
+
default
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def self.parse_float(v, default)
|
|
126
|
+
n = Float(v.to_s.strip)
|
|
127
|
+
n >= 0 ? n : default
|
|
128
|
+
rescue ArgumentError
|
|
129
|
+
default
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def self.parse_list(v)
|
|
133
|
+
v.to_s.split(",").map(&:strip).reject(&:empty?)
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# Seconds from "15", "15s", "500ms", "1m", "1h".
|
|
137
|
+
def self.parse_duration(v, default)
|
|
138
|
+
v = v.to_s.strip.downcase
|
|
139
|
+
return default if v.empty?
|
|
140
|
+
m = v.match(/\A(\d+(?:\.\d+)?)(ms|s|m|h)?\z/)
|
|
141
|
+
return default unless m
|
|
142
|
+
n = m[1].to_f
|
|
143
|
+
case m[2]
|
|
144
|
+
when "ms" then n / 1000.0
|
|
145
|
+
when "m" then n * 60
|
|
146
|
+
when "h" then n * 3600
|
|
147
|
+
else n
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
end
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Visionary
|
|
4
|
+
# What the request being handled has said about itself: the user, the route
|
|
5
|
+
# pattern, and the spans and errors recorded while it ran.
|
|
6
|
+
#
|
|
7
|
+
# The middleware puts one object in fiber storage (Ruby 3.2+, inherited by
|
|
8
|
+
# child fibers and threads, so it works under Falcon and in Async tasks) or a
|
|
9
|
+
# fiber-local variable on Ruby 3.1. Code anywhere in the request then
|
|
10
|
+
# mutates the same object, behind a mutex.
|
|
11
|
+
class RequestContext
|
|
12
|
+
KEY = :visionary_request
|
|
13
|
+
|
|
14
|
+
attr_reader :request_id, :method, :path, :started_at, :started_mono, :errors, :db_visible
|
|
15
|
+
|
|
16
|
+
def initialize(request_id:, method:, path:, started_at:, started_mono:, span_max:, db_visible:)
|
|
17
|
+
@request_id = request_id
|
|
18
|
+
@method = method
|
|
19
|
+
@path = path
|
|
20
|
+
@started_at = started_at
|
|
21
|
+
@started_mono = started_mono
|
|
22
|
+
@span_max = span_max
|
|
23
|
+
@db_visible = db_visible
|
|
24
|
+
@mutex = Mutex.new
|
|
25
|
+
@user = ["", "", ""]
|
|
26
|
+
@route = ""
|
|
27
|
+
@spans = []
|
|
28
|
+
@errors = []
|
|
29
|
+
@db_count = @http_count = 0
|
|
30
|
+
@db_ms = @http_ms = 0.0
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def set_user(id, handle, role)
|
|
34
|
+
@mutex.synchronize { @user = [id.nil? ? "" : id.to_s, handle.to_s, role.to_s] }
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def user
|
|
38
|
+
@mutex.synchronize { @user }
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def route=(pattern)
|
|
42
|
+
@mutex.synchronize { @route = pattern.to_s }
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def route
|
|
46
|
+
@mutex.synchronize { @route }
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Database and HTTP calls always count; rows are kept up to the cap.
|
|
50
|
+
def add_span(span)
|
|
51
|
+
@mutex.synchronize do
|
|
52
|
+
case span.kind
|
|
53
|
+
when "db"
|
|
54
|
+
@db_count += 1
|
|
55
|
+
@db_ms += span.duration_ms
|
|
56
|
+
@db_visible = true
|
|
57
|
+
when "http"
|
|
58
|
+
@http_count += 1
|
|
59
|
+
@http_ms += span.duration_ms
|
|
60
|
+
end
|
|
61
|
+
@spans << span if @spans.size < @span_max
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def add_error(row)
|
|
66
|
+
@mutex.synchronize { @errors << row }
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# [db_count, db_ms, http_count, http_ms]; database totals are NULL when no
|
|
70
|
+
# database hook could see this request's calls.
|
|
71
|
+
def totals
|
|
72
|
+
@mutex.synchronize do
|
|
73
|
+
[@db_visible ? @db_count : nil, @db_visible ? @db_ms.round(3) : nil, @http_count, @http_ms.round(3)]
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def spans
|
|
78
|
+
@mutex.synchronize { @spans.sort_by(&:offset_ms) }
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
if Fiber.respond_to?(:[])
|
|
82
|
+
def self.current = Fiber[KEY]
|
|
83
|
+
def self.current=(ctx)
|
|
84
|
+
Fiber[KEY] = ctx
|
|
85
|
+
end
|
|
86
|
+
else
|
|
87
|
+
def self.current = Thread.current[KEY]
|
|
88
|
+
def self.current=(ctx)
|
|
89
|
+
Thread.current[KEY] = ctx
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# One database, HTTP, cache, queue or custom call.
|
|
95
|
+
Span = Struct.new(:kind, :name, :target, :detail, :started_at, :offset_ms, :duration_ms,
|
|
96
|
+
:status, :error, :row_count, :request_id, keyword_init: true)
|
|
97
|
+
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Visionary
|
|
4
|
+
# Process._fork (Ruby 3.1+) runs for every fork: Kernel#fork, Process.fork
|
|
5
|
+
# and the forks Puma and Unicorn make for their workers. Only the child
|
|
6
|
+
# restarts Visionary's threads.
|
|
7
|
+
module ForkHook
|
|
8
|
+
def _fork
|
|
9
|
+
pid = super
|
|
10
|
+
Visionary.after_fork if pid.zero?
|
|
11
|
+
pid
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
Process.singleton_class.prepend(ForkHook) if Process.respond_to?(:_fork)
|
|
16
|
+
end
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "securerandom"
|
|
4
|
+
|
|
5
|
+
module Visionary
|
|
6
|
+
# Rack middleware: logs each request to api_logs, counts throughput for
|
|
7
|
+
# system_metrics, collects the request's spans and records exceptions.
|
|
8
|
+
# Mount it outermost so it sees the final status. Works with any Rack app
|
|
9
|
+
# (Rails via the Railtie, Sinatra, Hanami, Roda, Grape).
|
|
10
|
+
#
|
|
11
|
+
# use Visionary::Middleware
|
|
12
|
+
class Middleware
|
|
13
|
+
REQUEST_ID = "HTTP_X_REQUEST_ID"
|
|
14
|
+
# Where frameworks leave an exception they turned into an error page.
|
|
15
|
+
EXCEPTION_KEYS = %w[visionary.exception action_dispatch.exception sinatra.error rack.exception].freeze
|
|
16
|
+
UNCAPTURABLE = %w[multipart/ image/ video/ audio/ application/octet-stream application/zip application/pdf].freeze
|
|
17
|
+
|
|
18
|
+
def initialize(app, agent = nil)
|
|
19
|
+
@app = app
|
|
20
|
+
@agent = agent
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def call(env)
|
|
24
|
+
agent = @agent || Visionary.agent
|
|
25
|
+
return @app.call(env) unless agent&.enabled?
|
|
26
|
+
|
|
27
|
+
path = "#{env['SCRIPT_NAME']}#{env['PATH_INFO']}"
|
|
28
|
+
path = "/" if path.empty?
|
|
29
|
+
return passthrough(agent, env) unless agent.log_path?(path)
|
|
30
|
+
|
|
31
|
+
log(agent, env, path)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def self.capturable?(content_type)
|
|
35
|
+
ct = content_type.to_s.downcase
|
|
36
|
+
ct.empty? || UNCAPTURABLE.none? { |p| ct.start_with?(p) }
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Rack env headers as [[name, value], ...], e.g. ["User-Agent", "curl/8"].
|
|
40
|
+
def self.header_pairs(env)
|
|
41
|
+
out = []
|
|
42
|
+
env.each do |k, v|
|
|
43
|
+
next unless k.is_a?(String)
|
|
44
|
+
if k.start_with?("HTTP_") && k != "HTTP_VERSION"
|
|
45
|
+
out << [k[5..].split("_").map(&:capitalize).join("-"), v.to_s]
|
|
46
|
+
elsif k == "CONTENT_TYPE" || k == "CONTENT_LENGTH"
|
|
47
|
+
out << [k.split("_").map(&:capitalize).join("-"), v.to_s] unless v.to_s.empty?
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
out
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# A response header by lowercase name, whatever case the app used.
|
|
54
|
+
def self.header(headers, name)
|
|
55
|
+
return nil unless headers.respond_to?(:each)
|
|
56
|
+
v = headers[name]
|
|
57
|
+
if v.nil?
|
|
58
|
+
headers.each do |k, val|
|
|
59
|
+
next unless k.to_s.casecmp?(name)
|
|
60
|
+
v = val
|
|
61
|
+
break
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
v.is_a?(Array) ? v.join(", ") : v
|
|
65
|
+
rescue StandardError
|
|
66
|
+
nil
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
private
|
|
70
|
+
|
|
71
|
+
# Not logged: count it, and still record what escapes.
|
|
72
|
+
def passthrough(agent, env)
|
|
73
|
+
agent.http.started
|
|
74
|
+
status = 500
|
|
75
|
+
begin
|
|
76
|
+
res = @app.call(env)
|
|
77
|
+
status = res[0].to_i
|
|
78
|
+
res
|
|
79
|
+
rescue Exception => e # re-raised below; the app's handling is unchanged
|
|
80
|
+
agent.telemetry&.exception(e, handled: false, ctx: nil)
|
|
81
|
+
raise
|
|
82
|
+
ensure
|
|
83
|
+
agent.http.finished(status)
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def log(agent, env, path)
|
|
88
|
+
cfg = agent.config
|
|
89
|
+
started_at = Time.now
|
|
90
|
+
t0 = Telemetry.now_ms
|
|
91
|
+
request_id = env[REQUEST_ID].to_s
|
|
92
|
+
request_id = SecureRandom.hex(16) if request_id.empty? || request_id.length > 128
|
|
93
|
+
headers = self.class.header_pairs(env)
|
|
94
|
+
env[REQUEST_ID] = request_id # so the app (Rails' RequestId) uses the same id
|
|
95
|
+
|
|
96
|
+
req_type = env["CONTENT_TYPE"].to_s
|
|
97
|
+
req_capture = Capture.new(cfg.max_body_bytes, cfg.capture_bodies && self.class.capturable?(req_type))
|
|
98
|
+
env["rack.input"] = InputProxy.new(env["rack.input"], req_capture) if env["rack.input"] && cfg.capture_bodies
|
|
99
|
+
|
|
100
|
+
ctx = RequestContext.new(
|
|
101
|
+
request_id: request_id, method: env["REQUEST_METHOD"].to_s, path: path, started_at: started_at,
|
|
102
|
+
started_mono: t0, span_max: cfg.span_max.to_i, db_visible: Telemetry::ActiveRecordHook.installed?
|
|
103
|
+
)
|
|
104
|
+
agent.http.started
|
|
105
|
+
outer = RequestContext.current
|
|
106
|
+
RequestContext.current = ctx
|
|
107
|
+
begin
|
|
108
|
+
status, res_headers, body = @app.call(env)
|
|
109
|
+
rescue Exception => e # re-raised below; the app's handling is unchanged
|
|
110
|
+
agent.telemetry&.exception(e, handled: false, ctx: ctx)
|
|
111
|
+
finish(agent, env, ctx, headers, req_capture, 500, nil, Capture.new(0, false), t0)
|
|
112
|
+
raise
|
|
113
|
+
ensure
|
|
114
|
+
RequestContext.current = outer
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
status = status.to_i
|
|
118
|
+
begin
|
|
119
|
+
res_headers["x-request-id"] = request_id unless self.class.header(res_headers, "x-request-id")
|
|
120
|
+
rescue StandardError
|
|
121
|
+
nil
|
|
122
|
+
end
|
|
123
|
+
if status >= 500 && (exc = EXCEPTION_KEYS.lazy.map { |k| env[k] }.find { |x| x.is_a?(Exception) })
|
|
124
|
+
agent.telemetry&.exception(exc, handled: false, ctx: ctx)
|
|
125
|
+
end
|
|
126
|
+
resp_type = self.class.header(res_headers, "content-type").to_s
|
|
127
|
+
resp_capture = Capture.new(cfg.max_body_bytes, cfg.capture_bodies && self.class.capturable?(resp_type))
|
|
128
|
+
proxy = BodyProxy.new(body, resp_capture) do
|
|
129
|
+
finish(agent, env, ctx, headers, req_capture, status, res_headers, resp_capture, t0)
|
|
130
|
+
end
|
|
131
|
+
[status, res_headers, proxy]
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def finish(agent, env, ctx, headers, req_capture, status, res_headers, resp_capture, t0)
|
|
135
|
+
agent.http.finished(status)
|
|
136
|
+
duration = (Telemetry.now_ms - t0).round(3)
|
|
137
|
+
route = ctx.route
|
|
138
|
+
route = agent.route_for(env) if route.empty?
|
|
139
|
+
user_id, handle, role = ctx.user
|
|
140
|
+
tel = agent.telemetry
|
|
141
|
+
entry = Entry.new(
|
|
142
|
+
request_id: ctx.request_id, started_at: ctx.started_at, method: ctx.method, path: ctx.path,
|
|
143
|
+
route: route, query: env["QUERY_STRING"].to_s, status: status, duration_ms: duration,
|
|
144
|
+
ip: client_ip(env), user_agent: env["HTTP_USER_AGENT"].to_s,
|
|
145
|
+
user_id: user_id, user_handle: handle, user_role: role, headers: headers,
|
|
146
|
+
request_content_type: env["CONTENT_TYPE"].to_s, request_body: req_capture.bytes,
|
|
147
|
+
request_truncated: req_capture.truncated?,
|
|
148
|
+
request_size: [env["CONTENT_LENGTH"].to_i, req_capture.size].max,
|
|
149
|
+
response_content_type: self.class.header(res_headers, "content-type").to_s,
|
|
150
|
+
response_body: resp_capture.bytes, response_truncated: resp_capture.truncated?,
|
|
151
|
+
response_size: [self.class.header(res_headers, "content-length").to_i, resp_capture.size].max,
|
|
152
|
+
totals: tel&.spans_on ? ctx.totals : [nil, nil, nil, nil],
|
|
153
|
+
span_rows: tel ? tel.request_span_rows(ctx, duration, status) : [],
|
|
154
|
+
error_rows: tel ? ctx.errors.map { |row| tel.error_row(row, ctx, route, user_id) } : []
|
|
155
|
+
)
|
|
156
|
+
agent.enqueue([:request, entry])
|
|
157
|
+
rescue StandardError => e
|
|
158
|
+
agent.logger.error("visionary: dropping request log entry: #{e.class}: #{e.message}")
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# The first X-Forwarded-For hop (most deployments sit behind a proxy),
|
|
162
|
+
# else the peer. Informational only.
|
|
163
|
+
def client_ip(env)
|
|
164
|
+
fwd = env["HTTP_X_FORWARDED_FOR"].to_s
|
|
165
|
+
return fwd.split(",", 2).first.strip unless fwd.empty?
|
|
166
|
+
env["REMOTE_ADDR"].to_s
|
|
167
|
+
end
|
|
168
|
+
end
|
|
169
|
+
end
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Visionary
|
|
4
|
+
# A bounded queue that never blocks producers: when full, push drops the
|
|
5
|
+
# item and counts it. Consumers wait with a deadline (Thread::Queue#pop
|
|
6
|
+
# gained a timeout only in Ruby 3.2).
|
|
7
|
+
class BoundedQueue
|
|
8
|
+
def initialize(max)
|
|
9
|
+
@max = max
|
|
10
|
+
@items = []
|
|
11
|
+
@dropped = 0
|
|
12
|
+
@closed = false
|
|
13
|
+
@mutex = Mutex.new
|
|
14
|
+
@cond = ConditionVariable.new
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def push(item)
|
|
18
|
+
@mutex.synchronize do
|
|
19
|
+
if @items.size >= @max
|
|
20
|
+
@dropped += 1
|
|
21
|
+
return false
|
|
22
|
+
end
|
|
23
|
+
@items << item
|
|
24
|
+
@cond.signal
|
|
25
|
+
true
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Up to n items, waiting until at least one is there, the deadline (a
|
|
30
|
+
# monotonic time in seconds) passes, or the queue is closed.
|
|
31
|
+
def take(n, deadline)
|
|
32
|
+
@mutex.synchronize do
|
|
33
|
+
while @items.empty? && !@closed
|
|
34
|
+
left = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
35
|
+
break if left <= 0
|
|
36
|
+
@cond.wait(@mutex, left)
|
|
37
|
+
end
|
|
38
|
+
@items.shift(n)
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def size = @mutex.synchronize { @items.size }
|
|
43
|
+
|
|
44
|
+
def take_dropped
|
|
45
|
+
@mutex.synchronize { @dropped.tap { @dropped = 0 } }
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def close
|
|
49
|
+
@mutex.synchronize do
|
|
50
|
+
@closed = true
|
|
51
|
+
@cond.broadcast
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def closed? = @mutex.synchronize { @closed }
|
|
56
|
+
end
|
|
57
|
+
end
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Visionary
|
|
4
|
+
# Rails integration: inserts the middleware first in the stack, records
|
|
5
|
+
# database spans from ActiveRecord, sees exceptions Rails turns into error
|
|
6
|
+
# pages, and reads route patterns from Rails routing. Configure in an
|
|
7
|
+
# initializer with Visionary.configure.
|
|
8
|
+
class Railtie < ::Rails::Railtie
|
|
9
|
+
initializer "visionary.middleware" do |app|
|
|
10
|
+
app.middleware.insert_before 0, Visionary::Middleware
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
initializer "visionary.active_record" do
|
|
14
|
+
ActiveSupport.on_load(:active_record) do
|
|
15
|
+
Visionary::Telemetry::ActiveRecordHook.install if Visionary.config.spans
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# ShowExceptions and DebugExceptions rescue errors below the middleware;
|
|
20
|
+
# the interceptor hands each one to it (it is recorded when the response
|
|
21
|
+
# is a 5xx, so RecordNotFound turned into a 404 is not an error row).
|
|
22
|
+
initializer "visionary.exceptions" do
|
|
23
|
+
ActiveSupport.on_load(:action_controller) do
|
|
24
|
+
if ActionDispatch::DebugExceptions.respond_to?(:register_interceptor)
|
|
25
|
+
ActionDispatch::DebugExceptions.register_interceptor do |request, exception|
|
|
26
|
+
request.set_header("visionary.exception", exception)
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
config.after_initialize do
|
|
33
|
+
cfg = Visionary.config
|
|
34
|
+
cfg.logger ||= Rails.logger
|
|
35
|
+
# Rails 7.1+ leaves the pattern in the env (request.route_uri_pattern);
|
|
36
|
+
# older versions are asked to recognize the request again.
|
|
37
|
+
cfg.route ||= Visionary::Railtie.method(:recognize) unless ActionDispatch::Request.method_defined?(:route_uri_pattern)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def self.recognize(env)
|
|
41
|
+
req = ActionDispatch::Request.new(env)
|
|
42
|
+
Rails.application.routes.router.recognize(req) { |route, _| return route.path.spec.to_s }
|
|
43
|
+
""
|
|
44
|
+
rescue StandardError
|
|
45
|
+
""
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|