visionary-sdk 1.1.1 → 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -17,7 +17,7 @@ module Visionary
17
17
  # api_logs columns added after the first release, in insert order.
18
18
  ADDED_REQUEST_COLUMNS = [
19
19
  %w[service text], %w[version text], %w[db_count int], %w[db_ms real], %w[http_count int], %w[http_ms real],
20
- %w[login_account text]
20
+ %w[login_account text], %w[trace_id text], %w[span_id text], %w[parent_span_id text]
21
21
  ].freeze
22
22
 
23
23
  SYSTEM_BASE_COLUMNS = %w[
@@ -51,13 +51,25 @@ module Visionary
51
51
  %w[sdk text], %w[runtime text],
52
52
  %w[container_cpu_cores real], %w[container_cpu_limit real],
53
53
  %w[container_cpu_throttled_percent real], %w[container_mem_used_mb real],
54
- %w[container_mem_limit_mb real], %w[container_oom_kills int]
54
+ %w[container_mem_limit_mb real], %w[container_oom_kills int],
55
+ %w[psi_cpu_some_avg10 real], %w[psi_cpu_some_avg60 real],
56
+ %w[psi_mem_some_avg10 real], %w[psi_mem_some_avg60 real], %w[psi_mem_full_avg10 real],
57
+ %w[psi_io_some_avg10 real], %w[psi_io_some_avg60 real], %w[psi_io_full_avg10 real],
58
+ %w[minor_faults_ps real], %w[major_faults_ps real], %w[swap_in_ps real], %w[swap_out_ps real],
59
+ %w[pages_scanned_ps real], %w[pages_stolen_ps real], %w[host_oom_kills int],
60
+ %w[mem_dirty_mb real], %w[mem_writeback_mb real], %w[mem_slab_mb real],
61
+ %w[mem_shmem_mb real], %w[mem_committed_mb real], %w[mem_commit_limit_mb real],
62
+ %w[container_minor_faults_ps real], %w[container_major_faults_ps real],
63
+ %w[container_mem_high_events int], %w[container_mem_max_events int]
55
64
  ].freeze
56
65
 
57
66
  SPAN_COLUMNS = %w[
58
67
  created_at request_id service instance kind name target detail offset_ms duration_ms status error row_count
59
68
  ].freeze
60
69
 
70
+ # Added after the first release, in insert order
71
+ ADDED_SPAN_COLUMNS = [%w[trace_id text], %w[span_id text], %w[parent_span_id text]].freeze
72
+
61
73
  ERROR_COLUMNS = %w[
62
74
  created_at service instance version request_id method route path user_id
63
75
  type message stack fingerprint handled runtime
@@ -74,22 +86,27 @@ module Visionary
74
86
  code_func size
75
87
  ].freeze
76
88
 
89
+ METRIC_COLUMNS = %w[
90
+ created_at service instance name kind unit labels series value count sum min max buckets
91
+ ].freeze
92
+
77
93
  TABLES = {
78
94
  "api_logs" => REQUEST_COLUMNS + ADDED_REQUEST_COLUMNS.map(&:first),
79
95
  "system_metrics" => SYSTEM_BASE_COLUMNS + ADDED_SYSTEM_COLUMNS.map(&:first),
80
- "spans" => SPAN_COLUMNS,
96
+ "spans" => SPAN_COLUMNS + ADDED_SPAN_COLUMNS.map(&:first),
81
97
  "errors" => ERROR_COLUMNS,
82
98
  "releases" => RELEASE_COLUMNS,
83
99
  "profiles" => PROFILE_COLUMNS,
84
- "app_logs" => LOG_COLUMNS
100
+ "app_logs" => LOG_COLUMNS,
101
+ "metrics" => METRIC_COLUMNS
85
102
  }.freeze
86
103
 
87
- # Which script creates a table, and the columns older copies may lack.
88
104
  GROUPS = {
89
- requests: { file: "requests.sql", table: "api_logs", added: ADDED_REQUEST_COLUMNS },
105
+ requests: { file: "requests.sql", table: "api_logs", added: ADDED_REQUEST_COLUMNS, trace_index: true },
90
106
  system: { file: "system.sql", table: "system_metrics", added: ADDED_SYSTEM_COLUMNS },
91
- telemetry: { file: "telemetry.sql", table: nil, added: [] },
92
- logs: { file: "logs.sql", table: "app_logs", added: [] }
107
+ telemetry: { file: "telemetry.sql", table: "spans", added: ADDED_SPAN_COLUMNS, trace_index: true },
108
+ logs: { file: "logs.sql", table: "app_logs", added: [] },
109
+ metrics: { file: "metrics.sql", table: "metrics", added: [] }
93
110
  }.freeze
94
111
 
95
112
  PURGE_CHUNK = 5000
@@ -124,6 +141,12 @@ module Visionary
124
141
  end
125
142
  end
126
143
 
144
+ # Not in the shipped scripts since older tables only get trace_id on upgrade
145
+ def trace_index(dialect, table)
146
+ ine = dialect == "mysql" ? "" : "IF NOT EXISTS "
147
+ "CREATE INDEX #{ine}idx_#{table}_trace_id ON #{table} (trace_id)"
148
+ end
149
+
127
150
  # ALTER statements for the added columns missing from existing names.
128
151
  def missing_columns(dialect, table, added, existing)
129
152
  have = existing.map { |c| c.to_s.downcase }
@@ -149,12 +172,17 @@ module Visionary
149
172
 
150
173
  # D1 takes string params only, so there each placeholder becomes
151
174
  # NULLIF(?, '') and nulls travel as empty strings.
152
- def insert_sql(table, string_params)
153
- cols = TABLES.fetch(table)
175
+ def insert_sql(table, string_params, cols = TABLES.fetch(table))
154
176
  mark = string_params ? "NULLIF(?, '')" : "?"
155
177
  "INSERT INTO #{table} (#{cols.join(', ')}) VALUES (#{([mark] * cols.size).join(', ')})"
156
178
  end
157
179
 
180
+ def insert_rows_sql(table, n)
181
+ cols = TABLES.fetch(table)
182
+ row = "(#{(['?'] * cols.size).join(', ')})"
183
+ "INSERT INTO #{table} (#{cols.join(', ')}) VALUES #{([row] * n).join(', ')}"
184
+ end
185
+
158
186
  # MySQL in strict mode rejects over-long values, which would fail the
159
187
  # whole batch, so values are cut to the column size first: characters for
160
188
  # VARCHAR(n), bytes for TEXT and MEDIUMTEXT. Read from the shipped schema.
@@ -59,6 +59,7 @@ module Visionary
59
59
  rescue StandardError => e
60
60
  raise unless Schema.duplicate_column?(e)
61
61
  end
62
+ @store.exec_script([Schema.trace_index(dialect, g[:table])]) if g[:trace_index]
62
63
  end
63
64
  end
64
65
  end
@@ -0,0 +1,86 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+
5
+ module Visionary
6
+ # Settings → Data: requests.skip in visionary_settings, a JSON array of
7
+ # exact paths or prefixes ending in "*". A skipped request is logged like
8
+ # one outside the path prefix. One thread reads it at start and every 30 s;
9
+ # a failed read (no server or table yet) keeps the last list.
10
+ class SkipList
11
+ POLL_S = 30
12
+ SETTING = "requests.skip"
13
+ SETTINGS_SQL = "SELECT name, value FROM visionary_settings WHERE name LIKE 'requests.%'"
14
+
15
+ # Anything but a JSON array of strings skips nothing.
16
+ def self.parse(value)
17
+ list = JSON.parse(value.to_s)
18
+ return [] unless list.is_a?(Array) && list.all?(String)
19
+ list.map(&:strip).reject(&:empty?)
20
+ rescue JSON::ParserError
21
+ []
22
+ end
23
+
24
+ def self.match?(list, path)
25
+ list.any? { |p| p.end_with?("*") ? path.start_with?(p.chomp("*")) : p == path }
26
+ end
27
+
28
+ def initialize(store, logger)
29
+ @store = store
30
+ @logger = logger
31
+ @list = [].freeze
32
+ reset
33
+ end
34
+
35
+ def skip?(path) = SkipList.match?(@list, path)
36
+
37
+ def poll
38
+ list = []
39
+ @store.rows(SETTINGS_SQL).each { |name, value| list = SkipList.parse(value) if name.to_s == SETTING }
40
+ @list = list.freeze
41
+ rescue StandardError
42
+ nil
43
+ end
44
+
45
+ def start
46
+ @thread = Thread.new { run }
47
+ @thread.name = "visionary-skip" if @thread.respond_to?(:name=)
48
+ self
49
+ end
50
+
51
+ def after_fork
52
+ reset
53
+ start
54
+ end
55
+
56
+ def close
57
+ @mutex.synchronize do
58
+ @stopped = true
59
+ @cond.broadcast
60
+ end
61
+ end
62
+
63
+ private
64
+
65
+ def reset
66
+ @mutex = Mutex.new
67
+ @cond = ConditionVariable.new
68
+ @stopped = false
69
+ @thread = nil
70
+ end
71
+
72
+ def run
73
+ Telemetry.internal!
74
+ loop do
75
+ poll
76
+ stopped = @mutex.synchronize do
77
+ @cond.wait(@mutex, POLL_S) unless @stopped
78
+ @stopped
79
+ end
80
+ break if stopped
81
+ end
82
+ rescue StandardError => e
83
+ @logger.error("visionary: skip list stopped: #{e.message}")
84
+ end
85
+ end
86
+ end
@@ -88,6 +88,7 @@ module Visionary
88
88
 
89
89
  def bind(conn, sql, args)
90
90
  i = -1
91
+ return sql.gsub("?") { conn.quote(args[i += 1]) } unless sql.include?("'")
91
92
  quote = nil
92
93
  out = +""
93
94
  sql.each_char do |ch|
@@ -6,18 +6,17 @@ module Visionary
6
6
  module Store
7
7
  class ConfigError < StandardError; end
8
8
 
9
- # A connection the SDK opens itself from VISIONARY_DATABASE_URL, through
10
- # the pg, mysql2 or sqlite3 gem (whichever the app has installed). One
11
- # connection, used by one background thread at a time, reopened after an
12
- # error.
9
+ # Up to four connections on PostgreSQL and MySQL, one on SQLite
13
10
  class SQL
14
- attr_reader :dialect
11
+ PARALLEL = 4
12
+
13
+ attr_reader :dialect, :parallel
15
14
 
16
15
  def initialize(dialect, &connect)
17
16
  @dialect = dialect
18
17
  @connect = connect
19
- @mutex = Mutex.new
20
- @conn = nil
18
+ @parallel = %w[postgres mysql].include?(dialect) ? PARALLEL : 1
19
+ reset_pool
21
20
  end
22
21
 
23
22
  def string_params? = false
@@ -44,12 +43,14 @@ module Visionary
44
43
  run { |c| c.rows(sql) }
45
44
  end
46
45
 
47
- # One transaction for the whole batch.
46
+ # Repeated statements are prepared once, multi-row INSERTs run directly
48
47
  def batch(stmts)
48
+ uses = Hash.new(0)
49
+ stmts.each { |sql, _| uses[sql] += 1 }
49
50
  run do |c|
50
51
  c.execute("BEGIN", [])
51
52
  begin
52
- stmts.each { |sql, args| c.execute(sql, args) }
53
+ stmts.each { |sql, args| uses[sql] > 1 ? c.execute(sql, args) : c.execute_once(sql, args) }
53
54
  c.execute("COMMIT", [])
54
55
  rescue StandardError
55
56
  begin
@@ -66,37 +67,84 @@ module Visionary
66
67
  run { |c| c.execute(sql, args) }
67
68
  end
68
69
 
69
- # The parent's connection (a socket) must not be used or closed from
70
- # the child: it is kept referenced (so no finalizer closes it) and a new
71
- # one is opened lazily.
70
+ # The parent's sockets stay referenced so no finalizer closes them from the child
72
71
  def after_fork
73
- @mutex = Mutex.new
74
- (@inherited ||= []) << @conn if @conn
75
- @conn = nil
72
+ (@inherited ||= []).concat(@all)
73
+ reset_pool
76
74
  end
77
75
 
78
76
  def close
79
- @mutex.synchronize { discard }
77
+ idle = @mutex.synchronize do
78
+ @all -= @idle
79
+ @idle.dup.tap { @idle.clear }
80
+ end
81
+ idle.each { |c| shut(c) }
80
82
  end
81
83
 
82
84
  private
83
85
 
86
+ def reset_pool
87
+ @mutex = Mutex.new
88
+ @free = ConditionVariable.new
89
+ @idle = []
90
+ @all = []
91
+ @opening = 0
92
+ end
93
+
84
94
  def run
85
- @mutex.synchronize do
86
- @conn ||= @connect.call
87
- yield @conn
95
+ conn = checkout
96
+ begin
97
+ yield conn
88
98
  rescue StandardError
89
- discard if @conn&.broken?
99
+ if conn.broken?
100
+ drop(conn)
101
+ conn = nil
102
+ end
90
103
  raise
104
+ ensure
105
+ checkin(conn) if conn
106
+ end
107
+ end
108
+
109
+ def checkout
110
+ @mutex.synchronize do
111
+ loop do
112
+ return @idle.pop unless @idle.empty?
113
+ break if @all.size + @opening < @parallel
114
+ @free.wait(@mutex)
115
+ end
116
+ @opening += 1
117
+ end
118
+ begin
119
+ conn = @connect.call
120
+ ensure
121
+ @mutex.synchronize do
122
+ @opening -= 1
123
+ conn ? @all << conn : @free.signal
124
+ end
125
+ end
126
+ conn
127
+ end
128
+
129
+ def checkin(conn)
130
+ @mutex.synchronize do
131
+ @idle << conn
132
+ @free.signal
91
133
  end
92
134
  end
93
135
 
94
- def discard
95
- @conn&.close
136
+ def drop(conn)
137
+ @mutex.synchronize do
138
+ @all.delete(conn)
139
+ @free.signal
140
+ end
141
+ shut(conn)
142
+ end
143
+
144
+ def shut(conn)
145
+ conn.close
96
146
  rescue StandardError
97
147
  nil
98
- ensure
99
- @conn = nil
100
148
  end
101
149
  end
102
150
 
@@ -110,6 +158,7 @@ module Visionary
110
158
  def self.rebind(sql)
111
159
  return sql unless sql.include?("?")
112
160
  n = 0
161
+ return sql.gsub("?") { "$#{n += 1}" } unless sql.include?("'") || sql.include?('"')
113
162
  quote = nil
114
163
  out = +""
115
164
  sql.each_char do |ch|
@@ -175,6 +224,8 @@ module Visionary
175
224
  res&.clear
176
225
  end
177
226
 
227
+ alias execute_once execute
228
+
178
229
  def column(sql)
179
230
  res = @conn.exec(sql)
180
231
  res.column_values(0).map(&:to_s)
@@ -239,6 +290,17 @@ module Visionary
239
290
  stmt.affected_rows
240
291
  end
241
292
 
293
+ # Not cached since the server caps prepared statements per connection
294
+ def execute_once(sql, args)
295
+ return execute(sql, args) if args.empty?
296
+
297
+ stmt = @client.prepare(sql)
298
+ stmt.execute(*args)
299
+ stmt.affected_rows
300
+ ensure
301
+ stmt&.close
302
+ end
303
+
242
304
  def column(sql)
243
305
  @client.query(sql, as: :array).map { |r| r[0].to_s }
244
306
  end
@@ -266,7 +328,13 @@ module Visionary
266
328
  class Sqlite3Conn
267
329
  def initialize(path)
268
330
  @db = SQLite3::Database.new(path)
331
+ # The Visionary server shares the file, so wait for its lock
269
332
  @db.busy_timeout = 5000
333
+ begin
334
+ @db.execute("PRAGMA journal_mode=WAL")
335
+ rescue StandardError
336
+ nil
337
+ end
270
338
  end
271
339
 
272
340
  def execute(sql, args)
@@ -274,6 +342,8 @@ module Visionary
274
342
  @db.changes
275
343
  end
276
344
 
345
+ alias execute_once execute
346
+
277
347
  def column(sql)
278
348
  @db.execute(sql).map { |r| r[0].to_s }
279
349
  end
@@ -25,9 +25,13 @@ module Visionary
25
25
 
26
26
  attr_reader :cpu_dir, :mem_dir
27
27
 
28
+ def v2? = @v2
29
+
30
+ def in_container? = @in_ctr
31
+
28
32
  def initialize(root, text, env)
29
33
  @root = root
30
- @in_ctr = in_container?(text, env)
34
+ @in_ctr = container?(text, env)
31
35
  @v2 = false
32
36
  @cpu_dir = @cpu_acct = @mem_dir = nil
33
37
  text.strip.each_line do |line|
@@ -55,6 +59,7 @@ module Visionary
55
59
  # The baseline the first sample's deltas start from.
56
60
  def prime
57
61
  @usage, @periods, @throttled, @ooms = counters
62
+ @mem = read_mem
58
63
  @last = mono
59
64
  @primed = true
60
65
  end
@@ -74,6 +79,7 @@ module Visionary
74
79
  v["container_cpu_throttled_percent"] = 0.0
75
80
  end
76
81
  v["container_oom_kills"] = ooms - @ooms if ooms && @ooms && ooms >= @ooms
82
+ sample_mem(v, secs)
77
83
  end
78
84
  @usage, @periods, @throttled, @ooms = usage, periods, throttled, ooms
79
85
  @last = now
@@ -100,9 +106,38 @@ module Visionary
100
106
  File.directory?(dir) ? dir : File.join(@root, mount)
101
107
  end
102
108
 
109
+ # Fault counters from memory.stat and limit hits from memory.events (v2)
110
+ def read_mem
111
+ return {} unless @mem_dir
112
+ m = {}
113
+ m[:faults] = HostStats.parse_cgroup_mem_stat(Sources.read(File.join(@mem_dir, "memory.stat")))
114
+ m[:events] = HostStats.parse_cgroup_mem_events(Sources.read(File.join(@mem_dir, "memory.events"))) if @v2
115
+ m
116
+ end
117
+
118
+ def sample_mem(v, secs)
119
+ now = read_mem
120
+ prev = @mem || {}
121
+ @mem = now
122
+ if now[:faults] && prev[:faults]
123
+ f, maj = now[:faults]
124
+ pf, pmaj = prev[:faults]
125
+ if f >= maj && pf >= pmaj && maj >= pmaj && f - maj >= pf - pmaj
126
+ v["container_minor_faults_ps"] = (((f - maj) - (pf - pmaj)) / secs).round(1)
127
+ v["container_major_faults_ps"] = ((maj - pmaj) / secs).round(1)
128
+ end
129
+ end
130
+ return unless now[:events] && prev[:events]
131
+ h, x = now[:events]
132
+ ph, px = prev[:events]
133
+ return unless h >= ph && x >= px
134
+ v["container_mem_high_events"] = h - ph
135
+ v["container_mem_max_events"] = x - px
136
+ end
137
+
103
138
  # Tells a container from a process on the host by the runtimes' marker
104
139
  # files, the platforms' variables and the cgroup path.
105
- def in_container?(text, env)
140
+ def container?(text, env)
106
141
  File.exist?(path(".dockerenv")) || File.exist?(path("run/.containerenv")) ||
107
142
  CONTAINER_ENV.any? { |k| !env[k].to_s.empty? } || MARKERS.any? { |k| text.include?(k) }
108
143
  end
@@ -0,0 +1,141 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Visionary
4
+ module System
5
+ # Pressure stall information and virtual memory figures from small Linux
6
+ # files, once per sample (sdks/go/hoststats is the reference). In a
7
+ # container pressure comes from the cgroup v2 directory, since
8
+ # /proc/pressure is the whole machine's
9
+ class HostStats
10
+ PRESSURE = [%w[cpu psi_cpu], %w[memory psi_mem], %w[io psi_io]].freeze
11
+
12
+ # root and linux are for tests
13
+ def initialize(container: false, cgroup_dir: nil, root: "/", linux: Sources::LINUX)
14
+ @container = container
15
+ @cgroup_dir = cgroup_dir
16
+ @proc = File.join(root, "proc")
17
+ @linux = linux
18
+ end
19
+
20
+ def prime
21
+ @vm = read_vmstat if @linux
22
+ end
23
+
24
+ # Adds the columns it could read to v. secs is the time since the
25
+ # previous sample, for rates
26
+ def sample(v, secs)
27
+ return unless @linux && secs.positive?
28
+ sample_pressure(v)
29
+ sample_vm(v, secs)
30
+ end
31
+
32
+ def self.parse_pressure(text)
33
+ out = {}
34
+ text.to_s.each_line do |line|
35
+ kind, rest = line.split(" ", 2)
36
+ a10 = pressure_field(rest, "avg10=")
37
+ a60 = pressure_field(rest, "avg60=")
38
+ out[kind] = [a10, a60] if a10 && a60 && %w[some full].include?(kind)
39
+ end
40
+ out["some"] ? out : nil
41
+ end
42
+
43
+ def self.pressure_field(rest, key)
44
+ m = rest.to_s.match(/#{key}(\S+)/) or return nil
45
+ Float(m[1], exception: false)
46
+ end
47
+
48
+ # Sums reclaim over kswapd, direct and khugepaged scans, but not
49
+ # pgscan_anon or pgscan_file, which split the same pages by type
50
+ def self.parse_vmstat(text)
51
+ s = { pgscan: 0, pgsteal: 0 }
52
+ each_kv(text, " ") do |k, n|
53
+ case k
54
+ when "pgfault", "pgmajfault", "pswpin", "pswpout", "oom_kill" then s[k.to_sym] = n
55
+ when "pgscan_direct_throttle" then nil
56
+ when /\Apgscan_(kswapd|direct|khugepaged)/ then s[:pgscan] += n
57
+ when /\Apgsteal_(kswapd|direct|khugepaged)/ then s[:pgsteal] += n
58
+ end
59
+ end
60
+ s[:pgfault] ? s : nil
61
+ end
62
+
63
+ # The /proc/meminfo fields in kB, nil unless Dirty, Committed_AS and
64
+ # CommitLimit are all there
65
+ def self.parse_meminfo(text)
66
+ m = {}
67
+ each_kv(text, ":") { |k, n| m[k] = n }
68
+ %w[Dirty Committed_AS CommitLimit].all? { |k| m.key?(k) } ? m : nil
69
+ end
70
+
71
+ # The fault counters of a cgroup's memory.stat, v2 or v1
72
+ def self.parse_cgroup_mem_stat(text)
73
+ faults = major = nil
74
+ each_kv(text, " ") do |k, n|
75
+ case k
76
+ when "pgfault", "total_pgfault" then faults = n
77
+ when "pgmajfault", "total_pgmajfault" then major = n
78
+ end
79
+ end
80
+ faults ? [faults, major.to_i] : nil
81
+ end
82
+
83
+ # high and max of a v2 memory.events
84
+ def self.parse_cgroup_mem_events(text)
85
+ ev = {}
86
+ each_kv(text, " ") { |k, n| ev[k] = n if %w[high max].include?(k) }
87
+ ev.size == 2 ? [ev["high"], ev["max"]] : nil
88
+ end
89
+
90
+ # "key<sep> value [unit]" lines with an unsigned value
91
+ def self.each_kv(text, sep)
92
+ text.to_s.each_line do |line|
93
+ i = line.index(sep)
94
+ next if i.nil? || i.zero?
95
+ n = line[(i + 1)..].split.first
96
+ yield line[0, i], n.to_i if n&.match?(/\A\d+\z/)
97
+ end
98
+ end
99
+
100
+ private
101
+
102
+ def sample_pressure(v)
103
+ if @container
104
+ return unless @cgroup_dir
105
+ files = PRESSURE.map { |name, _| File.join(@cgroup_dir, "#{name}.pressure") }
106
+ else
107
+ files = PRESSURE.map { |name, _| File.join(@proc, "pressure", name) }
108
+ end
109
+ PRESSURE.each_with_index do |(name, col), i|
110
+ p = self.class.parse_pressure(Sources.read(files[i])) or next
111
+ v["#{col}_some_avg10"], v["#{col}_some_avg60"] = p["some"]
112
+ v["#{col}_full_avg10"] = p["full"][0] if p["full"] && name != "cpu"
113
+ end
114
+ end
115
+
116
+ def read_vmstat = self.class.parse_vmstat(Sources.read(File.join(@proc, "vmstat")))
117
+
118
+ def sample_vm(v, secs)
119
+ now = read_vmstat
120
+ if now && (p = @vm)
121
+ rate = ->(col, a, b) { v[col] = ((a - b) / secs).round(1) if a >= b }
122
+ if now[:pgfault] >= now[:pgmajfault].to_i && p[:pgfault] >= p[:pgmajfault].to_i
123
+ rate.call("minor_faults_ps", now[:pgfault] - now[:pgmajfault].to_i, p[:pgfault] - p[:pgmajfault].to_i)
124
+ end
125
+ rate.call("major_faults_ps", now[:pgmajfault].to_i, p[:pgmajfault].to_i)
126
+ rate.call("swap_in_ps", now[:pswpin].to_i, p[:pswpin].to_i)
127
+ rate.call("swap_out_ps", now[:pswpout].to_i, p[:pswpout].to_i)
128
+ rate.call("pages_scanned_ps", now[:pgscan], p[:pgscan])
129
+ rate.call("pages_stolen_ps", now[:pgsteal], p[:pgsteal])
130
+ v["host_oom_kills"] = now[:oom_kill] - p[:oom_kill] if now[:oom_kill] && p[:oom_kill] && now[:oom_kill] >= p[:oom_kill]
131
+ end
132
+ @vm = now
133
+ m = self.class.parse_meminfo(Sources.read(File.join(@proc, "meminfo"))) or return
134
+ { "Dirty" => "mem_dirty_mb", "Writeback" => "mem_writeback_mb", "Slab" => "mem_slab_mb", "Shmem" => "mem_shmem_mb",
135
+ "Committed_AS" => "mem_committed_mb", "CommitLimit" => "mem_commit_limit_mb" }.each do |k, col|
136
+ v[col] = (m[k].to_i / 1024.0).round(1)
137
+ end
138
+ end
139
+ end
140
+ end
141
+ end
@@ -28,6 +28,8 @@ module Visionary
28
28
  # background thread and writes them four at a time.
29
29
  class Recorder
30
30
  SAMPLES_PER_WRITE = 4
31
+ # D1 binds at most 100 params a statement
32
+ D1_MAX_PARAMS = 100
31
33
  MB = 1024.0 * 1024
32
34
 
33
35
  def initialize(cfg, instance, store, schemas, http, logger)
@@ -91,6 +93,7 @@ module Visionary
91
93
  v["sdk"] = "ruby"
92
94
  v["runtime"] = Telemetry.runtime
93
95
  @cgroup&.sample(v)
96
+ @stats.sample(v, secs)
94
97
  Schema::TABLES["system_metrics"].map { |c| v[c] }
95
98
  end
96
99
 
@@ -122,6 +125,8 @@ module Visionary
122
125
  @prev = read_counters(stat, Sources::LINUX ? Sources.tcp : {})
123
126
  @cgroup = Cgroup.detect # nil off Linux
124
127
  @cgroup&.prime
128
+ @stats = HostStats.new(container: @cgroup&.in_container? || false, cgroup_dir: @cgroup&.v2? ? @cgroup.cpu_dir : nil)
129
+ @stats.prime
125
130
  end
126
131
 
127
132
  def run
@@ -159,7 +164,7 @@ module Visionary
159
164
  return if @buf.empty?
160
165
  @schemas.ensure(:system)
161
166
  begin
162
- @store.batch(@buf.map { |row| [@sql, Store.fit(@store.dialect, "system_metrics", row)] })
167
+ @store.batch(@buf.flat_map { |row| row_stmts(Store.fit(@store.dialect, "system_metrics", row)) })
163
168
  @buf.clear
164
169
  rescue StandardError => e
165
170
  @logger.warn("visionary: system metrics write failed (#{@buf.size} rows kept): #{e.message}")
@@ -168,6 +173,18 @@ module Visionary
168
173
  end
169
174
  end
170
175
 
176
+ # On D1 the columns past the first 100 follow in an UPDATE of the row
177
+ # just inserted, in the same batch
178
+ def row_stmts(row)
179
+ return [[@sql, row]] unless @store.string_params? && row.size > D1_MAX_PARAMS
180
+ @d1_insert ||= begin
181
+ cols = Schema::TABLES["system_metrics"]
182
+ @d1_update = "UPDATE system_metrics SET #{cols[D1_MAX_PARAMS..].map { |c| "#{c} = NULLIF(?, '')" }.join(', ')} WHERE id = last_insert_rowid()"
183
+ Schema.insert_sql("system_metrics", true, cols[0, D1_MAX_PARAMS])
184
+ end
185
+ [[@d1_insert, row[0, D1_MAX_PARAMS]], [@d1_update, row[D1_MAX_PARAMS..]]]
186
+ end
187
+
171
188
  def host(v, stat, secs)
172
189
  if (cpu = stat[:cpu]) && @cpu_prev
173
190
  total = cpu_total(cpu) - cpu_total(@cpu_prev)