visionary-sdk 1.2.0 → 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: '06876950e4e7e007c2e59fe232268e03b981383062f47c4a1ef2e7c8fe803df2'
4
- data.tar.gz: bb09cbeffec3774e6bec59a5114ecdc5aaa80ebef88c77ff11c3c9f674445dcb
3
+ metadata.gz: dcff3663d4e289c1b84c3d224d0b811595d750769e43cdc7c4fa0a6d6bcf8d42
4
+ data.tar.gz: b8ec5856c74f821d10db7ad40499ab1bf11ca9e3c2e43d61bf895a57fb227538
5
5
  SHA512:
6
- metadata.gz: c220abde41d7f0f77a15385ee8f4ecab2fabea986ff25c9e8a64e61880d65f7370b97561f3983137a8ff77c1165825054ac9141469352ad71c890c120f3bd3e2
7
- data.tar.gz: '0091d167f78f3b35379febe0adcfbfdc8da41ea7c4ca561f62545fad9d1a5dcf49426b7d0faff9691336f7daeeb4d58df6007bf408aee18bfa731ddb16107cf6'
6
+ metadata.gz: c137bb66dd1f1023fac0770e86434eb583f9ef6b7ddd65dac799e9c975b6d8ef3b81650077bf1cce33321bbf0a6e93a2de90bed6394f8ab04e1b5a223ea2f11a
7
+ data.tar.gz: a0d9ebca1ebe887ef1c11ea66383107cd834f023598038757f9c685b1ce698eb5dc3eaffe0155525d1768e8b4bdb8ac0544ee197e30f7ab18337d4bf4757ee5d
data/CHANGELOG.md CHANGED
@@ -2,6 +2,9 @@
2
2
 
3
3
  ## Unreleased
4
4
 
5
+ - Requests on the Settings → Data skip list (`requests.skip`) are handled
6
+ like paths outside `VISIONARY_LOG_PATH_PREFIX`: no `api_logs` row or spans.
7
+
5
8
  - Faster writes: SQL backends get each table's rows as multi-row INSERTs
6
9
  (500 rows a batch, `batch_size` up to 2000), with up to four batches in
7
10
  flight on PostgreSQL and MySQL. SQLite opens in WAL mode with a 5 s busy
data/README.md CHANGED
@@ -652,7 +652,12 @@ ActiveRecord pool, or from `c.db_stats`. On macOS and Windows most host
652
652
  columns are NULL. In a container (or a cgroup with a CPU or memory limit) the
653
653
  `container_*` columns come from the process's own cgroup, v2 or v1: CPU used,
654
654
  limit and throttling, memory used (without inactive page cache) and limit,
655
- OOM kills.
655
+ OOM kills, page faults and `memory.events` high/max hits (v2). Also on Linux:
656
+ pressure stall (`psi_*`) from `/proc/pressure`, or from the cgroup v2
657
+ `*.pressure` files in a container (NULL with cgroup v1 there), page fault,
658
+ swap and reclaim rates from `/proc/vmstat`, and dirty, writeback, slab,
659
+ shmem and committed memory from `/proc/meminfo`. On D1 a row is written as
660
+ an INSERT of the first 100 columns plus an UPDATE with the rest.
656
661
 
657
662
  ### How profiling works
658
663
 
@@ -44,6 +44,7 @@ module Visionary
44
44
  if cfg.log_requests || cfg.spans || cfg.errors || cfg.version || cfg.profile
45
45
  @writer = Writer.new(cfg, @store, @schemas, @logger)
46
46
  end
47
+ @skip = SkipList.new(@store, @logger) if cfg.log_requests && @writer
47
48
  @profiler = Profiler.new(cfg, @instance, @store, @writer, @logger) if cfg.profile && @writer
48
49
  @telemetry = Telemetry::Recorder.new(cfg, @instance, @writer) if cfg.spans || cfg.errors
49
50
  @logs = Logs::Pipeline.new(cfg, @instance, @store, @schemas, @logger) if cfg.logs
@@ -66,7 +67,10 @@ module Visionary
66
67
 
67
68
  def enabled? = !@store.nil? && !@closed && (!@writer.nil? || !@system.nil? || !@logs.nil? || !@metrics.nil?)
68
69
 
69
- def log_path?(path) = @config.log_requests && !@writer.nil? && path.start_with?(@config.path_prefix.to_s)
70
+ # A skipped path is treated exactly like one outside the prefix.
71
+ def log_path?(path)
72
+ @config.log_requests && !@writer.nil? && path.start_with?(@config.path_prefix.to_s) && !@skip&.skip?(path)
73
+ end
70
74
 
71
75
  # Queues a row for the writer; never blocks.
72
76
  def enqueue(item)
@@ -105,6 +109,7 @@ module Visionary
105
109
  end
106
110
  @logs&.start
107
111
  @metrics&.start
112
+ @skip&.start
108
113
  @retention.start
109
114
  @profiler&.start
110
115
  install_hooks
@@ -121,6 +126,7 @@ module Visionary
121
126
  end
122
127
  deadline = mono + timeout
123
128
  @retention&.close
129
+ @skip&.close
124
130
  @profiler&.close
125
131
  idle = @writer ? @writer.close(timeout) : true
126
132
  idle = @logs.close([deadline - mono, 0].max) && idle if @logs
@@ -145,6 +151,7 @@ module Visionary
145
151
  @writer&.after_fork
146
152
  @logs&.after_fork
147
153
  @metrics&.after_fork
154
+ @skip&.after_fork
148
155
  @retention&.after_fork if @retention&.needed?
149
156
  @profiler&.after_fork
150
157
  @logger.info("visionary: restarted in forked process #{@pid}")
@@ -76,7 +76,32 @@ CREATE TABLE IF NOT EXISTS system_metrics (
76
76
  container_cpu_throttled_percent DOUBLE,
77
77
  container_mem_used_mb DOUBLE,
78
78
  container_mem_limit_mb DOUBLE,
79
- container_oom_kills BIGINT
79
+ container_oom_kills BIGINT,
80
+ psi_cpu_some_avg10 DOUBLE,
81
+ psi_cpu_some_avg60 DOUBLE,
82
+ psi_mem_some_avg10 DOUBLE,
83
+ psi_mem_some_avg60 DOUBLE,
84
+ psi_mem_full_avg10 DOUBLE,
85
+ psi_io_some_avg10 DOUBLE,
86
+ psi_io_some_avg60 DOUBLE,
87
+ psi_io_full_avg10 DOUBLE,
88
+ minor_faults_ps DOUBLE,
89
+ major_faults_ps DOUBLE,
90
+ swap_in_ps DOUBLE,
91
+ swap_out_ps DOUBLE,
92
+ pages_scanned_ps DOUBLE,
93
+ pages_stolen_ps DOUBLE,
94
+ host_oom_kills BIGINT,
95
+ mem_dirty_mb DOUBLE,
96
+ mem_writeback_mb DOUBLE,
97
+ mem_slab_mb DOUBLE,
98
+ mem_shmem_mb DOUBLE,
99
+ mem_committed_mb DOUBLE,
100
+ mem_commit_limit_mb DOUBLE,
101
+ container_minor_faults_ps DOUBLE,
102
+ container_major_faults_ps DOUBLE,
103
+ container_mem_high_events BIGINT,
104
+ container_mem_max_events BIGINT
80
105
  ) CHARACTER SET utf8mb4;
81
106
  CREATE INDEX idx_system_metrics_created_at ON system_metrics (created_at);
82
107
  CREATE INDEX idx_system_metrics_instance_created ON system_metrics (instance, created_at);
@@ -76,7 +76,32 @@ CREATE TABLE IF NOT EXISTS system_metrics (
76
76
  container_cpu_throttled_percent DOUBLE PRECISION,
77
77
  container_mem_used_mb DOUBLE PRECISION,
78
78
  container_mem_limit_mb DOUBLE PRECISION,
79
- container_oom_kills BIGINT
79
+ container_oom_kills BIGINT,
80
+ psi_cpu_some_avg10 DOUBLE PRECISION,
81
+ psi_cpu_some_avg60 DOUBLE PRECISION,
82
+ psi_mem_some_avg10 DOUBLE PRECISION,
83
+ psi_mem_some_avg60 DOUBLE PRECISION,
84
+ psi_mem_full_avg10 DOUBLE PRECISION,
85
+ psi_io_some_avg10 DOUBLE PRECISION,
86
+ psi_io_some_avg60 DOUBLE PRECISION,
87
+ psi_io_full_avg10 DOUBLE PRECISION,
88
+ minor_faults_ps DOUBLE PRECISION,
89
+ major_faults_ps DOUBLE PRECISION,
90
+ swap_in_ps DOUBLE PRECISION,
91
+ swap_out_ps DOUBLE PRECISION,
92
+ pages_scanned_ps DOUBLE PRECISION,
93
+ pages_stolen_ps DOUBLE PRECISION,
94
+ host_oom_kills BIGINT,
95
+ mem_dirty_mb DOUBLE PRECISION,
96
+ mem_writeback_mb DOUBLE PRECISION,
97
+ mem_slab_mb DOUBLE PRECISION,
98
+ mem_shmem_mb DOUBLE PRECISION,
99
+ mem_committed_mb DOUBLE PRECISION,
100
+ mem_commit_limit_mb DOUBLE PRECISION,
101
+ container_minor_faults_ps DOUBLE PRECISION,
102
+ container_major_faults_ps DOUBLE PRECISION,
103
+ container_mem_high_events BIGINT,
104
+ container_mem_max_events BIGINT
80
105
  );
81
106
  CREATE INDEX IF NOT EXISTS idx_system_metrics_created_at ON system_metrics (created_at);
82
107
  CREATE INDEX IF NOT EXISTS idx_system_metrics_instance_created ON system_metrics (instance, created_at);
@@ -76,7 +76,32 @@ CREATE TABLE IF NOT EXISTS system_metrics (
76
76
  container_cpu_throttled_percent REAL,
77
77
  container_mem_used_mb REAL,
78
78
  container_mem_limit_mb REAL,
79
- container_oom_kills INTEGER
79
+ container_oom_kills INTEGER,
80
+ psi_cpu_some_avg10 REAL,
81
+ psi_cpu_some_avg60 REAL,
82
+ psi_mem_some_avg10 REAL,
83
+ psi_mem_some_avg60 REAL,
84
+ psi_mem_full_avg10 REAL,
85
+ psi_io_some_avg10 REAL,
86
+ psi_io_some_avg60 REAL,
87
+ psi_io_full_avg10 REAL,
88
+ minor_faults_ps REAL,
89
+ major_faults_ps REAL,
90
+ swap_in_ps REAL,
91
+ swap_out_ps REAL,
92
+ pages_scanned_ps REAL,
93
+ pages_stolen_ps REAL,
94
+ host_oom_kills INTEGER,
95
+ mem_dirty_mb REAL,
96
+ mem_writeback_mb REAL,
97
+ mem_slab_mb REAL,
98
+ mem_shmem_mb REAL,
99
+ mem_committed_mb REAL,
100
+ mem_commit_limit_mb REAL,
101
+ container_minor_faults_ps REAL,
102
+ container_major_faults_ps REAL,
103
+ container_mem_high_events INTEGER,
104
+ container_mem_max_events INTEGER
80
105
  );
81
106
  CREATE INDEX IF NOT EXISTS idx_system_metrics_created_at ON system_metrics (created_at);
82
107
  CREATE INDEX IF NOT EXISTS idx_system_metrics_instance_created ON system_metrics (instance, created_at);
@@ -51,7 +51,16 @@ module Visionary
51
51
  %w[sdk text], %w[runtime text],
52
52
  %w[container_cpu_cores real], %w[container_cpu_limit real],
53
53
  %w[container_cpu_throttled_percent real], %w[container_mem_used_mb real],
54
- %w[container_mem_limit_mb real], %w[container_oom_kills int]
54
+ %w[container_mem_limit_mb real], %w[container_oom_kills int],
55
+ %w[psi_cpu_some_avg10 real], %w[psi_cpu_some_avg60 real],
56
+ %w[psi_mem_some_avg10 real], %w[psi_mem_some_avg60 real], %w[psi_mem_full_avg10 real],
57
+ %w[psi_io_some_avg10 real], %w[psi_io_some_avg60 real], %w[psi_io_full_avg10 real],
58
+ %w[minor_faults_ps real], %w[major_faults_ps real], %w[swap_in_ps real], %w[swap_out_ps real],
59
+ %w[pages_scanned_ps real], %w[pages_stolen_ps real], %w[host_oom_kills int],
60
+ %w[mem_dirty_mb real], %w[mem_writeback_mb real], %w[mem_slab_mb real],
61
+ %w[mem_shmem_mb real], %w[mem_committed_mb real], %w[mem_commit_limit_mb real],
62
+ %w[container_minor_faults_ps real], %w[container_major_faults_ps real],
63
+ %w[container_mem_high_events int], %w[container_mem_max_events int]
55
64
  ].freeze
56
65
 
57
66
  SPAN_COLUMNS = %w[
@@ -163,8 +172,7 @@ module Visionary
163
172
 
164
173
  # D1 takes string params only, so there each placeholder becomes
165
174
  # NULLIF(?, '') and nulls travel as empty strings.
166
- def insert_sql(table, string_params)
167
- cols = TABLES.fetch(table)
175
+ def insert_sql(table, string_params, cols = TABLES.fetch(table))
168
176
  mark = string_params ? "NULLIF(?, '')" : "?"
169
177
  "INSERT INTO #{table} (#{cols.join(', ')}) VALUES (#{([mark] * cols.size).join(', ')})"
170
178
  end
@@ -0,0 +1,86 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+
5
+ module Visionary
6
+ # Settings → Data: requests.skip in visionary_settings, a JSON array of
7
+ # exact paths or prefixes ending in "*". A skipped request is logged like
8
+ # one outside the path prefix. One thread reads it at start and every 30 s;
9
+ # a failed read (no server or table yet) keeps the last list.
10
+ class SkipList
11
+ POLL_S = 30
12
+ SETTING = "requests.skip"
13
+ SETTINGS_SQL = "SELECT name, value FROM visionary_settings WHERE name LIKE 'requests.%'"
14
+
15
+ # Anything but a JSON array of strings skips nothing.
16
+ def self.parse(value)
17
+ list = JSON.parse(value.to_s)
18
+ return [] unless list.is_a?(Array) && list.all?(String)
19
+ list.map(&:strip).reject(&:empty?)
20
+ rescue JSON::ParserError
21
+ []
22
+ end
23
+
24
+ def self.match?(list, path)
25
+ list.any? { |p| p.end_with?("*") ? path.start_with?(p.chomp("*")) : p == path }
26
+ end
27
+
28
+ def initialize(store, logger)
29
+ @store = store
30
+ @logger = logger
31
+ @list = [].freeze
32
+ reset
33
+ end
34
+
35
+ def skip?(path) = SkipList.match?(@list, path)
36
+
37
+ def poll
38
+ list = []
39
+ @store.rows(SETTINGS_SQL).each { |name, value| list = SkipList.parse(value) if name.to_s == SETTING }
40
+ @list = list.freeze
41
+ rescue StandardError
42
+ nil
43
+ end
44
+
45
+ def start
46
+ @thread = Thread.new { run }
47
+ @thread.name = "visionary-skip" if @thread.respond_to?(:name=)
48
+ self
49
+ end
50
+
51
+ def after_fork
52
+ reset
53
+ start
54
+ end
55
+
56
+ def close
57
+ @mutex.synchronize do
58
+ @stopped = true
59
+ @cond.broadcast
60
+ end
61
+ end
62
+
63
+ private
64
+
65
+ def reset
66
+ @mutex = Mutex.new
67
+ @cond = ConditionVariable.new
68
+ @stopped = false
69
+ @thread = nil
70
+ end
71
+
72
+ def run
73
+ Telemetry.internal!
74
+ loop do
75
+ poll
76
+ stopped = @mutex.synchronize do
77
+ @cond.wait(@mutex, POLL_S) unless @stopped
78
+ @stopped
79
+ end
80
+ break if stopped
81
+ end
82
+ rescue StandardError => e
83
+ @logger.error("visionary: skip list stopped: #{e.message}")
84
+ end
85
+ end
86
+ end
@@ -25,9 +25,13 @@ module Visionary
25
25
 
26
26
  attr_reader :cpu_dir, :mem_dir
27
27
 
28
+ def v2? = @v2
29
+
30
+ def in_container? = @in_ctr
31
+
28
32
  def initialize(root, text, env)
29
33
  @root = root
30
- @in_ctr = in_container?(text, env)
34
+ @in_ctr = container?(text, env)
31
35
  @v2 = false
32
36
  @cpu_dir = @cpu_acct = @mem_dir = nil
33
37
  text.strip.each_line do |line|
@@ -55,6 +59,7 @@ module Visionary
55
59
  # The baseline the first sample's deltas start from.
56
60
  def prime
57
61
  @usage, @periods, @throttled, @ooms = counters
62
+ @mem = read_mem
58
63
  @last = mono
59
64
  @primed = true
60
65
  end
@@ -74,6 +79,7 @@ module Visionary
74
79
  v["container_cpu_throttled_percent"] = 0.0
75
80
  end
76
81
  v["container_oom_kills"] = ooms - @ooms if ooms && @ooms && ooms >= @ooms
82
+ sample_mem(v, secs)
77
83
  end
78
84
  @usage, @periods, @throttled, @ooms = usage, periods, throttled, ooms
79
85
  @last = now
@@ -100,9 +106,38 @@ module Visionary
100
106
  File.directory?(dir) ? dir : File.join(@root, mount)
101
107
  end
102
108
 
109
+ # Fault counters from memory.stat and limit hits from memory.events (v2)
110
+ def read_mem
111
+ return {} unless @mem_dir
112
+ m = {}
113
+ m[:faults] = HostStats.parse_cgroup_mem_stat(Sources.read(File.join(@mem_dir, "memory.stat")))
114
+ m[:events] = HostStats.parse_cgroup_mem_events(Sources.read(File.join(@mem_dir, "memory.events"))) if @v2
115
+ m
116
+ end
117
+
118
+ def sample_mem(v, secs)
119
+ now = read_mem
120
+ prev = @mem || {}
121
+ @mem = now
122
+ if now[:faults] && prev[:faults]
123
+ f, maj = now[:faults]
124
+ pf, pmaj = prev[:faults]
125
+ if f >= maj && pf >= pmaj && maj >= pmaj && f - maj >= pf - pmaj
126
+ v["container_minor_faults_ps"] = (((f - maj) - (pf - pmaj)) / secs).round(1)
127
+ v["container_major_faults_ps"] = ((maj - pmaj) / secs).round(1)
128
+ end
129
+ end
130
+ return unless now[:events] && prev[:events]
131
+ h, x = now[:events]
132
+ ph, px = prev[:events]
133
+ return unless h >= ph && x >= px
134
+ v["container_mem_high_events"] = h - ph
135
+ v["container_mem_max_events"] = x - px
136
+ end
137
+
103
138
  # Tells a container from a process on the host by the runtimes' marker
104
139
  # files, the platforms' variables and the cgroup path.
105
- def in_container?(text, env)
140
+ def container?(text, env)
106
141
  File.exist?(path(".dockerenv")) || File.exist?(path("run/.containerenv")) ||
107
142
  CONTAINER_ENV.any? { |k| !env[k].to_s.empty? } || MARKERS.any? { |k| text.include?(k) }
108
143
  end
@@ -0,0 +1,141 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Visionary
4
+ module System
5
+ # Pressure stall information and virtual memory figures from small Linux
6
+ # files, once per sample (sdks/go/hoststats is the reference). In a
7
+ # container pressure comes from the cgroup v2 directory, since
8
+ # /proc/pressure is the whole machine's
9
+ class HostStats
10
+ PRESSURE = [%w[cpu psi_cpu], %w[memory psi_mem], %w[io psi_io]].freeze
11
+
12
+ # root and linux are for tests
13
+ def initialize(container: false, cgroup_dir: nil, root: "/", linux: Sources::LINUX)
14
+ @container = container
15
+ @cgroup_dir = cgroup_dir
16
+ @proc = File.join(root, "proc")
17
+ @linux = linux
18
+ end
19
+
20
+ def prime
21
+ @vm = read_vmstat if @linux
22
+ end
23
+
24
+ # Adds the columns it could read to v. secs is the time since the
25
+ # previous sample, for rates
26
+ def sample(v, secs)
27
+ return unless @linux && secs.positive?
28
+ sample_pressure(v)
29
+ sample_vm(v, secs)
30
+ end
31
+
32
+ def self.parse_pressure(text)
33
+ out = {}
34
+ text.to_s.each_line do |line|
35
+ kind, rest = line.split(" ", 2)
36
+ a10 = pressure_field(rest, "avg10=")
37
+ a60 = pressure_field(rest, "avg60=")
38
+ out[kind] = [a10, a60] if a10 && a60 && %w[some full].include?(kind)
39
+ end
40
+ out["some"] ? out : nil
41
+ end
42
+
43
+ def self.pressure_field(rest, key)
44
+ m = rest.to_s.match(/#{key}(\S+)/) or return nil
45
+ Float(m[1], exception: false)
46
+ end
47
+
48
+ # Sums reclaim over kswapd, direct and khugepaged scans, but not
49
+ # pgscan_anon or pgscan_file, which split the same pages by type
50
+ def self.parse_vmstat(text)
51
+ s = { pgscan: 0, pgsteal: 0 }
52
+ each_kv(text, " ") do |k, n|
53
+ case k
54
+ when "pgfault", "pgmajfault", "pswpin", "pswpout", "oom_kill" then s[k.to_sym] = n
55
+ when "pgscan_direct_throttle" then nil
56
+ when /\Apgscan_(kswapd|direct|khugepaged)/ then s[:pgscan] += n
57
+ when /\Apgsteal_(kswapd|direct|khugepaged)/ then s[:pgsteal] += n
58
+ end
59
+ end
60
+ s[:pgfault] ? s : nil
61
+ end
62
+
63
+ # The /proc/meminfo fields in kB, nil unless Dirty, Committed_AS and
64
+ # CommitLimit are all there
65
+ def self.parse_meminfo(text)
66
+ m = {}
67
+ each_kv(text, ":") { |k, n| m[k] = n }
68
+ %w[Dirty Committed_AS CommitLimit].all? { |k| m.key?(k) } ? m : nil
69
+ end
70
+
71
+ # The fault counters of a cgroup's memory.stat, v2 or v1
72
+ def self.parse_cgroup_mem_stat(text)
73
+ faults = major = nil
74
+ each_kv(text, " ") do |k, n|
75
+ case k
76
+ when "pgfault", "total_pgfault" then faults = n
77
+ when "pgmajfault", "total_pgmajfault" then major = n
78
+ end
79
+ end
80
+ faults ? [faults, major.to_i] : nil
81
+ end
82
+
83
+ # high and max of a v2 memory.events
84
+ def self.parse_cgroup_mem_events(text)
85
+ ev = {}
86
+ each_kv(text, " ") { |k, n| ev[k] = n if %w[high max].include?(k) }
87
+ ev.size == 2 ? [ev["high"], ev["max"]] : nil
88
+ end
89
+
90
+ # "key<sep> value [unit]" lines with an unsigned value
91
+ def self.each_kv(text, sep)
92
+ text.to_s.each_line do |line|
93
+ i = line.index(sep)
94
+ next if i.nil? || i.zero?
95
+ n = line[(i + 1)..].split.first
96
+ yield line[0, i], n.to_i if n&.match?(/\A\d+\z/)
97
+ end
98
+ end
99
+
100
+ private
101
+
102
+ def sample_pressure(v)
103
+ if @container
104
+ return unless @cgroup_dir
105
+ files = PRESSURE.map { |name, _| File.join(@cgroup_dir, "#{name}.pressure") }
106
+ else
107
+ files = PRESSURE.map { |name, _| File.join(@proc, "pressure", name) }
108
+ end
109
+ PRESSURE.each_with_index do |(name, col), i|
110
+ p = self.class.parse_pressure(Sources.read(files[i])) or next
111
+ v["#{col}_some_avg10"], v["#{col}_some_avg60"] = p["some"]
112
+ v["#{col}_full_avg10"] = p["full"][0] if p["full"] && name != "cpu"
113
+ end
114
+ end
115
+
116
+ def read_vmstat = self.class.parse_vmstat(Sources.read(File.join(@proc, "vmstat")))
117
+
118
+ def sample_vm(v, secs)
119
+ now = read_vmstat
120
+ if now && (p = @vm)
121
+ rate = ->(col, a, b) { v[col] = ((a - b) / secs).round(1) if a >= b }
122
+ if now[:pgfault] >= now[:pgmajfault].to_i && p[:pgfault] >= p[:pgmajfault].to_i
123
+ rate.call("minor_faults_ps", now[:pgfault] - now[:pgmajfault].to_i, p[:pgfault] - p[:pgmajfault].to_i)
124
+ end
125
+ rate.call("major_faults_ps", now[:pgmajfault].to_i, p[:pgmajfault].to_i)
126
+ rate.call("swap_in_ps", now[:pswpin].to_i, p[:pswpin].to_i)
127
+ rate.call("swap_out_ps", now[:pswpout].to_i, p[:pswpout].to_i)
128
+ rate.call("pages_scanned_ps", now[:pgscan], p[:pgscan])
129
+ rate.call("pages_stolen_ps", now[:pgsteal], p[:pgsteal])
130
+ v["host_oom_kills"] = now[:oom_kill] - p[:oom_kill] if now[:oom_kill] && p[:oom_kill] && now[:oom_kill] >= p[:oom_kill]
131
+ end
132
+ @vm = now
133
+ m = self.class.parse_meminfo(Sources.read(File.join(@proc, "meminfo"))) or return
134
+ { "Dirty" => "mem_dirty_mb", "Writeback" => "mem_writeback_mb", "Slab" => "mem_slab_mb", "Shmem" => "mem_shmem_mb",
135
+ "Committed_AS" => "mem_committed_mb", "CommitLimit" => "mem_commit_limit_mb" }.each do |k, col|
136
+ v[col] = (m[k].to_i / 1024.0).round(1)
137
+ end
138
+ end
139
+ end
140
+ end
141
+ end
@@ -28,6 +28,8 @@ module Visionary
28
28
  # background thread and writes them four at a time.
29
29
  class Recorder
30
30
  SAMPLES_PER_WRITE = 4
31
+ # D1 binds at most 100 params a statement
32
+ D1_MAX_PARAMS = 100
31
33
  MB = 1024.0 * 1024
32
34
 
33
35
  def initialize(cfg, instance, store, schemas, http, logger)
@@ -91,6 +93,7 @@ module Visionary
91
93
  v["sdk"] = "ruby"
92
94
  v["runtime"] = Telemetry.runtime
93
95
  @cgroup&.sample(v)
96
+ @stats.sample(v, secs)
94
97
  Schema::TABLES["system_metrics"].map { |c| v[c] }
95
98
  end
96
99
 
@@ -122,6 +125,8 @@ module Visionary
122
125
  @prev = read_counters(stat, Sources::LINUX ? Sources.tcp : {})
123
126
  @cgroup = Cgroup.detect # nil off Linux
124
127
  @cgroup&.prime
128
+ @stats = HostStats.new(container: @cgroup&.in_container? || false, cgroup_dir: @cgroup&.v2? ? @cgroup.cpu_dir : nil)
129
+ @stats.prime
125
130
  end
126
131
 
127
132
  def run
@@ -159,7 +164,7 @@ module Visionary
159
164
  return if @buf.empty?
160
165
  @schemas.ensure(:system)
161
166
  begin
162
- @store.batch(@buf.map { |row| [@sql, Store.fit(@store.dialect, "system_metrics", row)] })
167
+ @store.batch(@buf.flat_map { |row| row_stmts(Store.fit(@store.dialect, "system_metrics", row)) })
163
168
  @buf.clear
164
169
  rescue StandardError => e
165
170
  @logger.warn("visionary: system metrics write failed (#{@buf.size} rows kept): #{e.message}")
@@ -168,6 +173,18 @@ module Visionary
168
173
  end
169
174
  end
170
175
 
176
+ # On D1 the columns past the first 100 follow in an UPDATE of the row
177
+ # just inserted, in the same batch
178
+ def row_stmts(row)
179
+ return [[@sql, row]] unless @store.string_params? && row.size > D1_MAX_PARAMS
180
+ @d1_insert ||= begin
181
+ cols = Schema::TABLES["system_metrics"]
182
+ @d1_update = "UPDATE system_metrics SET #{cols[D1_MAX_PARAMS..].map { |c| "#{c} = NULLIF(?, '')" }.join(', ')} WHERE id = last_insert_rowid()"
183
+ Schema.insert_sql("system_metrics", true, cols[0, D1_MAX_PARAMS])
184
+ end
185
+ [[@d1_insert, row[0, D1_MAX_PARAMS]], [@d1_update, row[D1_MAX_PARAMS..]]]
186
+ end
187
+
171
188
  def host(v, stat, secs)
172
189
  if (cpu = stat[:cpu]) && @cpu_prev
173
190
  total = cpu_total(cpu) - cpu_total(@cpu_prev)
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Visionary
4
- VERSION = "1.2.0"
4
+ VERSION = "1.2.1"
5
5
  end
data/lib/visionary.rb CHANGED
@@ -13,7 +13,9 @@ require_relative "visionary/writer"
13
13
  require_relative "visionary/logs"
14
14
  require_relative "visionary/retention"
15
15
  require_relative "visionary/metrics"
16
+ require_relative "visionary/skip_list"
16
17
  require_relative "visionary/system/sources"
18
+ require_relative "visionary/system/host_stats"
17
19
  require_relative "visionary/system/cgroup"
18
20
  require_relative "visionary/system/recorder"
19
21
  require_relative "visionary/telemetry"
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: visionary-sdk
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.0
4
+ version: 1.2.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - styropyr0
@@ -56,11 +56,13 @@ files:
56
56
  - lib/visionary/schema/sqlite/system.sql
57
57
  - lib/visionary/schema/sqlite/telemetry.sql
58
58
  - lib/visionary/schemas.rb
59
+ - lib/visionary/skip_list.rb
59
60
  - lib/visionary/store.rb
60
61
  - lib/visionary/store/active_record.rb
61
62
  - lib/visionary/store/d1.rb
62
63
  - lib/visionary/store/sql.rb
63
64
  - lib/visionary/system/cgroup.rb
65
+ - lib/visionary/system/host_stats.rb
64
66
  - lib/visionary/system/recorder.rb
65
67
  - lib/visionary/system/sources.rb
66
68
  - lib/visionary/telemetry.rb