visionary-sdk 1.1.1 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +20 -0
- data/LICENSE +87 -69
- data/README.md +79 -8
- data/lib/visionary/agent.rb +21 -3
- data/lib/visionary/config.rb +9 -2
- data/lib/visionary/context.rb +9 -4
- data/lib/visionary/logs/logger.rb +2 -3
- data/lib/visionary/logs/pipeline.rb +5 -2
- data/lib/visionary/metrics.rb +392 -0
- data/lib/visionary/middleware.rb +10 -5
- data/lib/visionary/queue.rb +4 -0
- data/lib/visionary/schema/mysql/metrics.sql +20 -0
- data/lib/visionary/schema/mysql/system.sql +26 -1
- data/lib/visionary/schema/postgres/metrics.sql +20 -0
- data/lib/visionary/schema/postgres/system.sql +26 -1
- data/lib/visionary/schema/sqlite/metrics.sql +20 -0
- data/lib/visionary/schema/sqlite/system.sql +26 -1
- data/lib/visionary/schema.rb +38 -10
- data/lib/visionary/schemas.rb +1 -0
- data/lib/visionary/skip_list.rb +86 -0
- data/lib/visionary/store/active_record.rb +1 -0
- data/lib/visionary/store/sql.rb +94 -24
- data/lib/visionary/system/cgroup.rb +37 -2
- data/lib/visionary/system/host_stats.rb +141 -0
- data/lib/visionary/system/recorder.rb +18 -1
- data/lib/visionary/telemetry/hooks.rb +18 -5
- data/lib/visionary/telemetry.rb +9 -4
- data/lib/visionary/trace_context.rb +111 -0
- data/lib/visionary/version.rb +1 -1
- data/lib/visionary/writer.rb +89 -17
- data/lib/visionary.rb +32 -7
- metadata +10 -3
data/lib/visionary/schema.rb
CHANGED
|
@@ -17,7 +17,7 @@ module Visionary
|
|
|
17
17
|
# api_logs columns added after the first release, in insert order.
|
|
18
18
|
ADDED_REQUEST_COLUMNS = [
|
|
19
19
|
%w[service text], %w[version text], %w[db_count int], %w[db_ms real], %w[http_count int], %w[http_ms real],
|
|
20
|
-
%w[login_account text]
|
|
20
|
+
%w[login_account text], %w[trace_id text], %w[span_id text], %w[parent_span_id text]
|
|
21
21
|
].freeze
|
|
22
22
|
|
|
23
23
|
SYSTEM_BASE_COLUMNS = %w[
|
|
@@ -51,13 +51,25 @@ module Visionary
|
|
|
51
51
|
%w[sdk text], %w[runtime text],
|
|
52
52
|
%w[container_cpu_cores real], %w[container_cpu_limit real],
|
|
53
53
|
%w[container_cpu_throttled_percent real], %w[container_mem_used_mb real],
|
|
54
|
-
%w[container_mem_limit_mb real], %w[container_oom_kills int]
|
|
54
|
+
%w[container_mem_limit_mb real], %w[container_oom_kills int],
|
|
55
|
+
%w[psi_cpu_some_avg10 real], %w[psi_cpu_some_avg60 real],
|
|
56
|
+
%w[psi_mem_some_avg10 real], %w[psi_mem_some_avg60 real], %w[psi_mem_full_avg10 real],
|
|
57
|
+
%w[psi_io_some_avg10 real], %w[psi_io_some_avg60 real], %w[psi_io_full_avg10 real],
|
|
58
|
+
%w[minor_faults_ps real], %w[major_faults_ps real], %w[swap_in_ps real], %w[swap_out_ps real],
|
|
59
|
+
%w[pages_scanned_ps real], %w[pages_stolen_ps real], %w[host_oom_kills int],
|
|
60
|
+
%w[mem_dirty_mb real], %w[mem_writeback_mb real], %w[mem_slab_mb real],
|
|
61
|
+
%w[mem_shmem_mb real], %w[mem_committed_mb real], %w[mem_commit_limit_mb real],
|
|
62
|
+
%w[container_minor_faults_ps real], %w[container_major_faults_ps real],
|
|
63
|
+
%w[container_mem_high_events int], %w[container_mem_max_events int]
|
|
55
64
|
].freeze
|
|
56
65
|
|
|
57
66
|
SPAN_COLUMNS = %w[
|
|
58
67
|
created_at request_id service instance kind name target detail offset_ms duration_ms status error row_count
|
|
59
68
|
].freeze
|
|
60
69
|
|
|
70
|
+
# Added after the first release, in insert order
|
|
71
|
+
ADDED_SPAN_COLUMNS = [%w[trace_id text], %w[span_id text], %w[parent_span_id text]].freeze
|
|
72
|
+
|
|
61
73
|
ERROR_COLUMNS = %w[
|
|
62
74
|
created_at service instance version request_id method route path user_id
|
|
63
75
|
type message stack fingerprint handled runtime
|
|
@@ -74,22 +86,27 @@ module Visionary
|
|
|
74
86
|
code_func size
|
|
75
87
|
].freeze
|
|
76
88
|
|
|
89
|
+
METRIC_COLUMNS = %w[
|
|
90
|
+
created_at service instance name kind unit labels series value count sum min max buckets
|
|
91
|
+
].freeze
|
|
92
|
+
|
|
77
93
|
TABLES = {
|
|
78
94
|
"api_logs" => REQUEST_COLUMNS + ADDED_REQUEST_COLUMNS.map(&:first),
|
|
79
95
|
"system_metrics" => SYSTEM_BASE_COLUMNS + ADDED_SYSTEM_COLUMNS.map(&:first),
|
|
80
|
-
"spans" => SPAN_COLUMNS,
|
|
96
|
+
"spans" => SPAN_COLUMNS + ADDED_SPAN_COLUMNS.map(&:first),
|
|
81
97
|
"errors" => ERROR_COLUMNS,
|
|
82
98
|
"releases" => RELEASE_COLUMNS,
|
|
83
99
|
"profiles" => PROFILE_COLUMNS,
|
|
84
|
-
"app_logs" => LOG_COLUMNS
|
|
100
|
+
"app_logs" => LOG_COLUMNS,
|
|
101
|
+
"metrics" => METRIC_COLUMNS
|
|
85
102
|
}.freeze
|
|
86
103
|
|
|
87
|
-
# Which script creates a table, and the columns older copies may lack.
|
|
88
104
|
GROUPS = {
|
|
89
|
-
requests: { file: "requests.sql", table: "api_logs", added: ADDED_REQUEST_COLUMNS },
|
|
105
|
+
requests: { file: "requests.sql", table: "api_logs", added: ADDED_REQUEST_COLUMNS, trace_index: true },
|
|
90
106
|
system: { file: "system.sql", table: "system_metrics", added: ADDED_SYSTEM_COLUMNS },
|
|
91
|
-
telemetry: { file: "telemetry.sql", table:
|
|
92
|
-
logs: { file: "logs.sql", table: "app_logs", added: [] }
|
|
107
|
+
telemetry: { file: "telemetry.sql", table: "spans", added: ADDED_SPAN_COLUMNS, trace_index: true },
|
|
108
|
+
logs: { file: "logs.sql", table: "app_logs", added: [] },
|
|
109
|
+
metrics: { file: "metrics.sql", table: "metrics", added: [] }
|
|
93
110
|
}.freeze
|
|
94
111
|
|
|
95
112
|
PURGE_CHUNK = 5000
|
|
@@ -124,6 +141,12 @@ module Visionary
|
|
|
124
141
|
end
|
|
125
142
|
end
|
|
126
143
|
|
|
144
|
+
# Not in the shipped scripts since older tables only get trace_id on upgrade
|
|
145
|
+
def trace_index(dialect, table)
|
|
146
|
+
ine = dialect == "mysql" ? "" : "IF NOT EXISTS "
|
|
147
|
+
"CREATE INDEX #{ine}idx_#{table}_trace_id ON #{table} (trace_id)"
|
|
148
|
+
end
|
|
149
|
+
|
|
127
150
|
# ALTER statements for the added columns missing from existing names.
|
|
128
151
|
def missing_columns(dialect, table, added, existing)
|
|
129
152
|
have = existing.map { |c| c.to_s.downcase }
|
|
@@ -149,12 +172,17 @@ module Visionary
|
|
|
149
172
|
|
|
150
173
|
# D1 takes string params only, so there each placeholder becomes
|
|
151
174
|
# NULLIF(?, '') and nulls travel as empty strings.
|
|
152
|
-
def insert_sql(table, string_params)
|
|
153
|
-
cols = TABLES.fetch(table)
|
|
175
|
+
def insert_sql(table, string_params, cols = TABLES.fetch(table))
|
|
154
176
|
mark = string_params ? "NULLIF(?, '')" : "?"
|
|
155
177
|
"INSERT INTO #{table} (#{cols.join(', ')}) VALUES (#{([mark] * cols.size).join(', ')})"
|
|
156
178
|
end
|
|
157
179
|
|
|
180
|
+
def insert_rows_sql(table, n)
|
|
181
|
+
cols = TABLES.fetch(table)
|
|
182
|
+
row = "(#{(['?'] * cols.size).join(', ')})"
|
|
183
|
+
"INSERT INTO #{table} (#{cols.join(', ')}) VALUES #{([row] * n).join(', ')}"
|
|
184
|
+
end
|
|
185
|
+
|
|
158
186
|
# MySQL in strict mode rejects over-long values, which would fail the
|
|
159
187
|
# whole batch, so values are cut to the column size first: characters for
|
|
160
188
|
# VARCHAR(n), bytes for TEXT and MEDIUMTEXT. Read from the shipped schema.
|
data/lib/visionary/schemas.rb
CHANGED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
module Visionary
|
|
6
|
+
# Settings → Data: requests.skip in visionary_settings, a JSON array of
|
|
7
|
+
# exact paths or prefixes ending in "*". A skipped request is logged like
|
|
8
|
+
# one outside the path prefix. One thread reads it at start and every 30 s;
|
|
9
|
+
# a failed read (no server or table yet) keeps the last list.
|
|
10
|
+
class SkipList
|
|
11
|
+
POLL_S = 30
|
|
12
|
+
SETTING = "requests.skip"
|
|
13
|
+
SETTINGS_SQL = "SELECT name, value FROM visionary_settings WHERE name LIKE 'requests.%'"
|
|
14
|
+
|
|
15
|
+
# Anything but a JSON array of strings skips nothing.
|
|
16
|
+
def self.parse(value)
|
|
17
|
+
list = JSON.parse(value.to_s)
|
|
18
|
+
return [] unless list.is_a?(Array) && list.all?(String)
|
|
19
|
+
list.map(&:strip).reject(&:empty?)
|
|
20
|
+
rescue JSON::ParserError
|
|
21
|
+
[]
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def self.match?(list, path)
|
|
25
|
+
list.any? { |p| p.end_with?("*") ? path.start_with?(p.chomp("*")) : p == path }
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def initialize(store, logger)
|
|
29
|
+
@store = store
|
|
30
|
+
@logger = logger
|
|
31
|
+
@list = [].freeze
|
|
32
|
+
reset
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def skip?(path) = SkipList.match?(@list, path)
|
|
36
|
+
|
|
37
|
+
def poll
|
|
38
|
+
list = []
|
|
39
|
+
@store.rows(SETTINGS_SQL).each { |name, value| list = SkipList.parse(value) if name.to_s == SETTING }
|
|
40
|
+
@list = list.freeze
|
|
41
|
+
rescue StandardError
|
|
42
|
+
nil
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def start
|
|
46
|
+
@thread = Thread.new { run }
|
|
47
|
+
@thread.name = "visionary-skip" if @thread.respond_to?(:name=)
|
|
48
|
+
self
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def after_fork
|
|
52
|
+
reset
|
|
53
|
+
start
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def close
|
|
57
|
+
@mutex.synchronize do
|
|
58
|
+
@stopped = true
|
|
59
|
+
@cond.broadcast
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
private
|
|
64
|
+
|
|
65
|
+
def reset
|
|
66
|
+
@mutex = Mutex.new
|
|
67
|
+
@cond = ConditionVariable.new
|
|
68
|
+
@stopped = false
|
|
69
|
+
@thread = nil
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def run
|
|
73
|
+
Telemetry.internal!
|
|
74
|
+
loop do
|
|
75
|
+
poll
|
|
76
|
+
stopped = @mutex.synchronize do
|
|
77
|
+
@cond.wait(@mutex, POLL_S) unless @stopped
|
|
78
|
+
@stopped
|
|
79
|
+
end
|
|
80
|
+
break if stopped
|
|
81
|
+
end
|
|
82
|
+
rescue StandardError => e
|
|
83
|
+
@logger.error("visionary: skip list stopped: #{e.message}")
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
data/lib/visionary/store/sql.rb
CHANGED
|
@@ -6,18 +6,17 @@ module Visionary
|
|
|
6
6
|
module Store
|
|
7
7
|
class ConfigError < StandardError; end
|
|
8
8
|
|
|
9
|
-
#
|
|
10
|
-
# the pg, mysql2 or sqlite3 gem (whichever the app has installed). One
|
|
11
|
-
# connection, used by one background thread at a time, reopened after an
|
|
12
|
-
# error.
|
|
9
|
+
# Up to four connections on PostgreSQL and MySQL, one on SQLite
|
|
13
10
|
class SQL
|
|
14
|
-
|
|
11
|
+
PARALLEL = 4
|
|
12
|
+
|
|
13
|
+
attr_reader :dialect, :parallel
|
|
15
14
|
|
|
16
15
|
def initialize(dialect, &connect)
|
|
17
16
|
@dialect = dialect
|
|
18
17
|
@connect = connect
|
|
19
|
-
@
|
|
20
|
-
|
|
18
|
+
@parallel = %w[postgres mysql].include?(dialect) ? PARALLEL : 1
|
|
19
|
+
reset_pool
|
|
21
20
|
end
|
|
22
21
|
|
|
23
22
|
def string_params? = false
|
|
@@ -44,12 +43,14 @@ module Visionary
|
|
|
44
43
|
run { |c| c.rows(sql) }
|
|
45
44
|
end
|
|
46
45
|
|
|
47
|
-
#
|
|
46
|
+
# Repeated statements are prepared once, multi-row INSERTs run directly
|
|
48
47
|
def batch(stmts)
|
|
48
|
+
uses = Hash.new(0)
|
|
49
|
+
stmts.each { |sql, _| uses[sql] += 1 }
|
|
49
50
|
run do |c|
|
|
50
51
|
c.execute("BEGIN", [])
|
|
51
52
|
begin
|
|
52
|
-
stmts.each { |sql, args| c.execute(sql, args) }
|
|
53
|
+
stmts.each { |sql, args| uses[sql] > 1 ? c.execute(sql, args) : c.execute_once(sql, args) }
|
|
53
54
|
c.execute("COMMIT", [])
|
|
54
55
|
rescue StandardError
|
|
55
56
|
begin
|
|
@@ -66,37 +67,84 @@ module Visionary
|
|
|
66
67
|
run { |c| c.execute(sql, args) }
|
|
67
68
|
end
|
|
68
69
|
|
|
69
|
-
# The parent's
|
|
70
|
-
# the child: it is kept referenced (so no finalizer closes it) and a new
|
|
71
|
-
# one is opened lazily.
|
|
70
|
+
# The parent's sockets stay referenced so no finalizer closes them from the child
|
|
72
71
|
def after_fork
|
|
73
|
-
@
|
|
74
|
-
|
|
75
|
-
@conn = nil
|
|
72
|
+
(@inherited ||= []).concat(@all)
|
|
73
|
+
reset_pool
|
|
76
74
|
end
|
|
77
75
|
|
|
78
76
|
def close
|
|
79
|
-
@mutex.synchronize
|
|
77
|
+
idle = @mutex.synchronize do
|
|
78
|
+
@all -= @idle
|
|
79
|
+
@idle.dup.tap { @idle.clear }
|
|
80
|
+
end
|
|
81
|
+
idle.each { |c| shut(c) }
|
|
80
82
|
end
|
|
81
83
|
|
|
82
84
|
private
|
|
83
85
|
|
|
86
|
+
def reset_pool
|
|
87
|
+
@mutex = Mutex.new
|
|
88
|
+
@free = ConditionVariable.new
|
|
89
|
+
@idle = []
|
|
90
|
+
@all = []
|
|
91
|
+
@opening = 0
|
|
92
|
+
end
|
|
93
|
+
|
|
84
94
|
def run
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
yield
|
|
95
|
+
conn = checkout
|
|
96
|
+
begin
|
|
97
|
+
yield conn
|
|
88
98
|
rescue StandardError
|
|
89
|
-
|
|
99
|
+
if conn.broken?
|
|
100
|
+
drop(conn)
|
|
101
|
+
conn = nil
|
|
102
|
+
end
|
|
90
103
|
raise
|
|
104
|
+
ensure
|
|
105
|
+
checkin(conn) if conn
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def checkout
|
|
110
|
+
@mutex.synchronize do
|
|
111
|
+
loop do
|
|
112
|
+
return @idle.pop unless @idle.empty?
|
|
113
|
+
break if @all.size + @opening < @parallel
|
|
114
|
+
@free.wait(@mutex)
|
|
115
|
+
end
|
|
116
|
+
@opening += 1
|
|
117
|
+
end
|
|
118
|
+
begin
|
|
119
|
+
conn = @connect.call
|
|
120
|
+
ensure
|
|
121
|
+
@mutex.synchronize do
|
|
122
|
+
@opening -= 1
|
|
123
|
+
conn ? @all << conn : @free.signal
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
conn
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def checkin(conn)
|
|
130
|
+
@mutex.synchronize do
|
|
131
|
+
@idle << conn
|
|
132
|
+
@free.signal
|
|
91
133
|
end
|
|
92
134
|
end
|
|
93
135
|
|
|
94
|
-
def
|
|
95
|
-
@
|
|
136
|
+
def drop(conn)
|
|
137
|
+
@mutex.synchronize do
|
|
138
|
+
@all.delete(conn)
|
|
139
|
+
@free.signal
|
|
140
|
+
end
|
|
141
|
+
shut(conn)
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def shut(conn)
|
|
145
|
+
conn.close
|
|
96
146
|
rescue StandardError
|
|
97
147
|
nil
|
|
98
|
-
ensure
|
|
99
|
-
@conn = nil
|
|
100
148
|
end
|
|
101
149
|
end
|
|
102
150
|
|
|
@@ -110,6 +158,7 @@ module Visionary
|
|
|
110
158
|
def self.rebind(sql)
|
|
111
159
|
return sql unless sql.include?("?")
|
|
112
160
|
n = 0
|
|
161
|
+
return sql.gsub("?") { "$#{n += 1}" } unless sql.include?("'") || sql.include?('"')
|
|
113
162
|
quote = nil
|
|
114
163
|
out = +""
|
|
115
164
|
sql.each_char do |ch|
|
|
@@ -175,6 +224,8 @@ module Visionary
|
|
|
175
224
|
res&.clear
|
|
176
225
|
end
|
|
177
226
|
|
|
227
|
+
alias execute_once execute
|
|
228
|
+
|
|
178
229
|
def column(sql)
|
|
179
230
|
res = @conn.exec(sql)
|
|
180
231
|
res.column_values(0).map(&:to_s)
|
|
@@ -239,6 +290,17 @@ module Visionary
|
|
|
239
290
|
stmt.affected_rows
|
|
240
291
|
end
|
|
241
292
|
|
|
293
|
+
# Not cached since the server caps prepared statements per connection
|
|
294
|
+
def execute_once(sql, args)
|
|
295
|
+
return execute(sql, args) if args.empty?
|
|
296
|
+
|
|
297
|
+
stmt = @client.prepare(sql)
|
|
298
|
+
stmt.execute(*args)
|
|
299
|
+
stmt.affected_rows
|
|
300
|
+
ensure
|
|
301
|
+
stmt&.close
|
|
302
|
+
end
|
|
303
|
+
|
|
242
304
|
def column(sql)
|
|
243
305
|
@client.query(sql, as: :array).map { |r| r[0].to_s }
|
|
244
306
|
end
|
|
@@ -266,7 +328,13 @@ module Visionary
|
|
|
266
328
|
class Sqlite3Conn
|
|
267
329
|
def initialize(path)
|
|
268
330
|
@db = SQLite3::Database.new(path)
|
|
331
|
+
# The Visionary server shares the file, so wait for its lock
|
|
269
332
|
@db.busy_timeout = 5000
|
|
333
|
+
begin
|
|
334
|
+
@db.execute("PRAGMA journal_mode=WAL")
|
|
335
|
+
rescue StandardError
|
|
336
|
+
nil
|
|
337
|
+
end
|
|
270
338
|
end
|
|
271
339
|
|
|
272
340
|
def execute(sql, args)
|
|
@@ -274,6 +342,8 @@ module Visionary
|
|
|
274
342
|
@db.changes
|
|
275
343
|
end
|
|
276
344
|
|
|
345
|
+
alias execute_once execute
|
|
346
|
+
|
|
277
347
|
def column(sql)
|
|
278
348
|
@db.execute(sql).map { |r| r[0].to_s }
|
|
279
349
|
end
|
|
@@ -25,9 +25,13 @@ module Visionary
|
|
|
25
25
|
|
|
26
26
|
attr_reader :cpu_dir, :mem_dir
|
|
27
27
|
|
|
28
|
+
def v2? = @v2
|
|
29
|
+
|
|
30
|
+
def in_container? = @in_ctr
|
|
31
|
+
|
|
28
32
|
def initialize(root, text, env)
|
|
29
33
|
@root = root
|
|
30
|
-
@in_ctr =
|
|
34
|
+
@in_ctr = container?(text, env)
|
|
31
35
|
@v2 = false
|
|
32
36
|
@cpu_dir = @cpu_acct = @mem_dir = nil
|
|
33
37
|
text.strip.each_line do |line|
|
|
@@ -55,6 +59,7 @@ module Visionary
|
|
|
55
59
|
# The baseline the first sample's deltas start from.
|
|
56
60
|
def prime
|
|
57
61
|
@usage, @periods, @throttled, @ooms = counters
|
|
62
|
+
@mem = read_mem
|
|
58
63
|
@last = mono
|
|
59
64
|
@primed = true
|
|
60
65
|
end
|
|
@@ -74,6 +79,7 @@ module Visionary
|
|
|
74
79
|
v["container_cpu_throttled_percent"] = 0.0
|
|
75
80
|
end
|
|
76
81
|
v["container_oom_kills"] = ooms - @ooms if ooms && @ooms && ooms >= @ooms
|
|
82
|
+
sample_mem(v, secs)
|
|
77
83
|
end
|
|
78
84
|
@usage, @periods, @throttled, @ooms = usage, periods, throttled, ooms
|
|
79
85
|
@last = now
|
|
@@ -100,9 +106,38 @@ module Visionary
|
|
|
100
106
|
File.directory?(dir) ? dir : File.join(@root, mount)
|
|
101
107
|
end
|
|
102
108
|
|
|
109
|
+
# Fault counters from memory.stat and limit hits from memory.events (v2)
|
|
110
|
+
def read_mem
|
|
111
|
+
return {} unless @mem_dir
|
|
112
|
+
m = {}
|
|
113
|
+
m[:faults] = HostStats.parse_cgroup_mem_stat(Sources.read(File.join(@mem_dir, "memory.stat")))
|
|
114
|
+
m[:events] = HostStats.parse_cgroup_mem_events(Sources.read(File.join(@mem_dir, "memory.events"))) if @v2
|
|
115
|
+
m
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def sample_mem(v, secs)
|
|
119
|
+
now = read_mem
|
|
120
|
+
prev = @mem || {}
|
|
121
|
+
@mem = now
|
|
122
|
+
if now[:faults] && prev[:faults]
|
|
123
|
+
f, maj = now[:faults]
|
|
124
|
+
pf, pmaj = prev[:faults]
|
|
125
|
+
if f >= maj && pf >= pmaj && maj >= pmaj && f - maj >= pf - pmaj
|
|
126
|
+
v["container_minor_faults_ps"] = (((f - maj) - (pf - pmaj)) / secs).round(1)
|
|
127
|
+
v["container_major_faults_ps"] = ((maj - pmaj) / secs).round(1)
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
return unless now[:events] && prev[:events]
|
|
131
|
+
h, x = now[:events]
|
|
132
|
+
ph, px = prev[:events]
|
|
133
|
+
return unless h >= ph && x >= px
|
|
134
|
+
v["container_mem_high_events"] = h - ph
|
|
135
|
+
v["container_mem_max_events"] = x - px
|
|
136
|
+
end
|
|
137
|
+
|
|
103
138
|
# Tells a container from a process on the host by the runtimes' marker
|
|
104
139
|
# files, the platforms' variables and the cgroup path.
|
|
105
|
-
def
|
|
140
|
+
def container?(text, env)
|
|
106
141
|
File.exist?(path(".dockerenv")) || File.exist?(path("run/.containerenv")) ||
|
|
107
142
|
CONTAINER_ENV.any? { |k| !env[k].to_s.empty? } || MARKERS.any? { |k| text.include?(k) }
|
|
108
143
|
end
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Visionary
|
|
4
|
+
module System
|
|
5
|
+
# Pressure stall information and virtual memory figures from small Linux
|
|
6
|
+
# files, once per sample (sdks/go/hoststats is the reference). In a
|
|
7
|
+
# container pressure comes from the cgroup v2 directory, since
|
|
8
|
+
# /proc/pressure is the whole machine's
|
|
9
|
+
class HostStats
|
|
10
|
+
PRESSURE = [%w[cpu psi_cpu], %w[memory psi_mem], %w[io psi_io]].freeze
|
|
11
|
+
|
|
12
|
+
# root and linux are for tests
|
|
13
|
+
def initialize(container: false, cgroup_dir: nil, root: "/", linux: Sources::LINUX)
|
|
14
|
+
@container = container
|
|
15
|
+
@cgroup_dir = cgroup_dir
|
|
16
|
+
@proc = File.join(root, "proc")
|
|
17
|
+
@linux = linux
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def prime
|
|
21
|
+
@vm = read_vmstat if @linux
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Adds the columns it could read to v. secs is the time since the
|
|
25
|
+
# previous sample, for rates
|
|
26
|
+
def sample(v, secs)
|
|
27
|
+
return unless @linux && secs.positive?
|
|
28
|
+
sample_pressure(v)
|
|
29
|
+
sample_vm(v, secs)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def self.parse_pressure(text)
|
|
33
|
+
out = {}
|
|
34
|
+
text.to_s.each_line do |line|
|
|
35
|
+
kind, rest = line.split(" ", 2)
|
|
36
|
+
a10 = pressure_field(rest, "avg10=")
|
|
37
|
+
a60 = pressure_field(rest, "avg60=")
|
|
38
|
+
out[kind] = [a10, a60] if a10 && a60 && %w[some full].include?(kind)
|
|
39
|
+
end
|
|
40
|
+
out["some"] ? out : nil
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def self.pressure_field(rest, key)
|
|
44
|
+
m = rest.to_s.match(/#{key}(\S+)/) or return nil
|
|
45
|
+
Float(m[1], exception: false)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Sums reclaim over kswapd, direct and khugepaged scans, but not
|
|
49
|
+
# pgscan_anon or pgscan_file, which split the same pages by type
|
|
50
|
+
def self.parse_vmstat(text)
|
|
51
|
+
s = { pgscan: 0, pgsteal: 0 }
|
|
52
|
+
each_kv(text, " ") do |k, n|
|
|
53
|
+
case k
|
|
54
|
+
when "pgfault", "pgmajfault", "pswpin", "pswpout", "oom_kill" then s[k.to_sym] = n
|
|
55
|
+
when "pgscan_direct_throttle" then nil
|
|
56
|
+
when /\Apgscan_(kswapd|direct|khugepaged)/ then s[:pgscan] += n
|
|
57
|
+
when /\Apgsteal_(kswapd|direct|khugepaged)/ then s[:pgsteal] += n
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
s[:pgfault] ? s : nil
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# The /proc/meminfo fields in kB, nil unless Dirty, Committed_AS and
|
|
64
|
+
# CommitLimit are all there
|
|
65
|
+
def self.parse_meminfo(text)
|
|
66
|
+
m = {}
|
|
67
|
+
each_kv(text, ":") { |k, n| m[k] = n }
|
|
68
|
+
%w[Dirty Committed_AS CommitLimit].all? { |k| m.key?(k) } ? m : nil
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# The fault counters of a cgroup's memory.stat, v2 or v1
|
|
72
|
+
def self.parse_cgroup_mem_stat(text)
|
|
73
|
+
faults = major = nil
|
|
74
|
+
each_kv(text, " ") do |k, n|
|
|
75
|
+
case k
|
|
76
|
+
when "pgfault", "total_pgfault" then faults = n
|
|
77
|
+
when "pgmajfault", "total_pgmajfault" then major = n
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
faults ? [faults, major.to_i] : nil
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# high and max of a v2 memory.events
|
|
84
|
+
def self.parse_cgroup_mem_events(text)
|
|
85
|
+
ev = {}
|
|
86
|
+
each_kv(text, " ") { |k, n| ev[k] = n if %w[high max].include?(k) }
|
|
87
|
+
ev.size == 2 ? [ev["high"], ev["max"]] : nil
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# "key<sep> value [unit]" lines with an unsigned value
|
|
91
|
+
def self.each_kv(text, sep)
|
|
92
|
+
text.to_s.each_line do |line|
|
|
93
|
+
i = line.index(sep)
|
|
94
|
+
next if i.nil? || i.zero?
|
|
95
|
+
n = line[(i + 1)..].split.first
|
|
96
|
+
yield line[0, i], n.to_i if n&.match?(/\A\d+\z/)
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
private
|
|
101
|
+
|
|
102
|
+
def sample_pressure(v)
|
|
103
|
+
if @container
|
|
104
|
+
return unless @cgroup_dir
|
|
105
|
+
files = PRESSURE.map { |name, _| File.join(@cgroup_dir, "#{name}.pressure") }
|
|
106
|
+
else
|
|
107
|
+
files = PRESSURE.map { |name, _| File.join(@proc, "pressure", name) }
|
|
108
|
+
end
|
|
109
|
+
PRESSURE.each_with_index do |(name, col), i|
|
|
110
|
+
p = self.class.parse_pressure(Sources.read(files[i])) or next
|
|
111
|
+
v["#{col}_some_avg10"], v["#{col}_some_avg60"] = p["some"]
|
|
112
|
+
v["#{col}_full_avg10"] = p["full"][0] if p["full"] && name != "cpu"
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def read_vmstat = self.class.parse_vmstat(Sources.read(File.join(@proc, "vmstat")))
|
|
117
|
+
|
|
118
|
+
def sample_vm(v, secs)
|
|
119
|
+
now = read_vmstat
|
|
120
|
+
if now && (p = @vm)
|
|
121
|
+
rate = ->(col, a, b) { v[col] = ((a - b) / secs).round(1) if a >= b }
|
|
122
|
+
if now[:pgfault] >= now[:pgmajfault].to_i && p[:pgfault] >= p[:pgmajfault].to_i
|
|
123
|
+
rate.call("minor_faults_ps", now[:pgfault] - now[:pgmajfault].to_i, p[:pgfault] - p[:pgmajfault].to_i)
|
|
124
|
+
end
|
|
125
|
+
rate.call("major_faults_ps", now[:pgmajfault].to_i, p[:pgmajfault].to_i)
|
|
126
|
+
rate.call("swap_in_ps", now[:pswpin].to_i, p[:pswpin].to_i)
|
|
127
|
+
rate.call("swap_out_ps", now[:pswpout].to_i, p[:pswpout].to_i)
|
|
128
|
+
rate.call("pages_scanned_ps", now[:pgscan], p[:pgscan])
|
|
129
|
+
rate.call("pages_stolen_ps", now[:pgsteal], p[:pgsteal])
|
|
130
|
+
v["host_oom_kills"] = now[:oom_kill] - p[:oom_kill] if now[:oom_kill] && p[:oom_kill] && now[:oom_kill] >= p[:oom_kill]
|
|
131
|
+
end
|
|
132
|
+
@vm = now
|
|
133
|
+
m = self.class.parse_meminfo(Sources.read(File.join(@proc, "meminfo"))) or return
|
|
134
|
+
{ "Dirty" => "mem_dirty_mb", "Writeback" => "mem_writeback_mb", "Slab" => "mem_slab_mb", "Shmem" => "mem_shmem_mb",
|
|
135
|
+
"Committed_AS" => "mem_committed_mb", "CommitLimit" => "mem_commit_limit_mb" }.each do |k, col|
|
|
136
|
+
v[col] = (m[k].to_i / 1024.0).round(1)
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
end
|
|
@@ -28,6 +28,8 @@ module Visionary
|
|
|
28
28
|
# background thread and writes them four at a time.
|
|
29
29
|
class Recorder
|
|
30
30
|
SAMPLES_PER_WRITE = 4
|
|
31
|
+
# D1 binds at most 100 params a statement
|
|
32
|
+
D1_MAX_PARAMS = 100
|
|
31
33
|
MB = 1024.0 * 1024
|
|
32
34
|
|
|
33
35
|
def initialize(cfg, instance, store, schemas, http, logger)
|
|
@@ -91,6 +93,7 @@ module Visionary
|
|
|
91
93
|
v["sdk"] = "ruby"
|
|
92
94
|
v["runtime"] = Telemetry.runtime
|
|
93
95
|
@cgroup&.sample(v)
|
|
96
|
+
@stats.sample(v, secs)
|
|
94
97
|
Schema::TABLES["system_metrics"].map { |c| v[c] }
|
|
95
98
|
end
|
|
96
99
|
|
|
@@ -122,6 +125,8 @@ module Visionary
|
|
|
122
125
|
@prev = read_counters(stat, Sources::LINUX ? Sources.tcp : {})
|
|
123
126
|
@cgroup = Cgroup.detect # nil off Linux
|
|
124
127
|
@cgroup&.prime
|
|
128
|
+
@stats = HostStats.new(container: @cgroup&.in_container? || false, cgroup_dir: @cgroup&.v2? ? @cgroup.cpu_dir : nil)
|
|
129
|
+
@stats.prime
|
|
125
130
|
end
|
|
126
131
|
|
|
127
132
|
def run
|
|
@@ -159,7 +164,7 @@ module Visionary
|
|
|
159
164
|
return if @buf.empty?
|
|
160
165
|
@schemas.ensure(:system)
|
|
161
166
|
begin
|
|
162
|
-
@store.batch(@buf.
|
|
167
|
+
@store.batch(@buf.flat_map { |row| row_stmts(Store.fit(@store.dialect, "system_metrics", row)) })
|
|
163
168
|
@buf.clear
|
|
164
169
|
rescue StandardError => e
|
|
165
170
|
@logger.warn("visionary: system metrics write failed (#{@buf.size} rows kept): #{e.message}")
|
|
@@ -168,6 +173,18 @@ module Visionary
|
|
|
168
173
|
end
|
|
169
174
|
end
|
|
170
175
|
|
|
176
|
+
# On D1 the columns past the first 100 follow in an UPDATE of the row
|
|
177
|
+
# just inserted, in the same batch
|
|
178
|
+
def row_stmts(row)
|
|
179
|
+
return [[@sql, row]] unless @store.string_params? && row.size > D1_MAX_PARAMS
|
|
180
|
+
@d1_insert ||= begin
|
|
181
|
+
cols = Schema::TABLES["system_metrics"]
|
|
182
|
+
@d1_update = "UPDATE system_metrics SET #{cols[D1_MAX_PARAMS..].map { |c| "#{c} = NULLIF(?, '')" }.join(', ')} WHERE id = last_insert_rowid()"
|
|
183
|
+
Schema.insert_sql("system_metrics", true, cols[0, D1_MAX_PARAMS])
|
|
184
|
+
end
|
|
185
|
+
[[@d1_insert, row[0, D1_MAX_PARAMS]], [@d1_update, row[D1_MAX_PARAMS..]]]
|
|
186
|
+
end
|
|
187
|
+
|
|
171
188
|
def host(v, stat, secs)
|
|
172
189
|
if (cpu = stat[:cpu]) && @cpu_prev
|
|
173
190
|
total = cpu_total(cpu) - cpu_total(@cpu_prev)
|