visionary-sdk 0.4.0

data/lib/visionary/redactor.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Visionary
+  # Scrubs credentials from headers and bodies before anything is stored.
+  # The body patterns are regexes on purpose, so bodies cut off by the
+  # capture cap (invalid JSON) are still scrubbed.
+  class Redactor
+    REDACTED = "[REDACTED]"
+    DEFAULT_HEADERS = %w[Authorization Proxy-Authorization Cookie Set-Cookie X-Api-Key X-Auth-Token].freeze
+    DEFAULT_FIELDS = %w[
+      password passwd secret client_secret token access_token refresh_token id_token
+      api_key apikey otp pin card_number cvv private_key
+    ].freeze
+
+    def initialize(extra_headers = [], extra_fields = [], request_only = [])
+      @headers = (DEFAULT_HEADERS + extra_headers).map { |h| h.strip.downcase }.reject(&:empty?).to_h { |h| [h, true] }
+      both = key_pattern(DEFAULT_FIELDS + extra_fields)
+      req_only = key_pattern(request_only)
+      req_keys = req_only.empty? ? both : "#{both}|#{req_only}"
+      @resp_json = json_secrets(both)
+      @req_json = json_secrets(req_keys)
+      @form = /(^|&)(#{req_keys})=[^&]*/i
+    end
+
+    # Headers is [[name, value], ...]; the result is a JSON object.
+    def headers(pairs)
+      return "" if pairs.empty?
+      out = {}
+      pairs.each { |name, value| out[Visionary.utf8(name)] = @headers[name.downcase] ? REDACTED : Visionary.utf8(value) }
+      JSON.generate(out.sort.to_h)
+    rescue JSON::GeneratorError, EncodingError
+      ""
+    end
+
+    def request_body(body, content_type)
+      return "" if body.empty?
+      if content_type.to_s.include?("application/x-www-form-urlencoded")
+        body.gsub(@form) { "#{Regexp.last_match(1)}#{Regexp.last_match(2)}=#{REDACTED}" }
+      else
+        body.gsub(@req_json) { "#{Regexp.last_match(1)}\"#{REDACTED}\"" }
+      end
+    end
+
+    def response_body(body)
+      return "" if body.empty?
+      body.gsub(@resp_json) { "#{Regexp.last_match(1)}\"#{REDACTED}\"" }
+    end
+
+    # The error code and key from a JSON error response at dotted paths.
+    # Bodies that aren't valid JSON (e.g. truncated) fall back to a pattern
+    # match on the path's last segment.
+    def self.error_fields(body, code_path, key_path)
+      v = begin
+        JSON.parse(body)
+      rescue JSON::ParserError, EncodingError
+        return [match_field(body, code_path), match_field(body, key_path)]
+      end
+      [lookup(v, code_path), lookup(v, key_path)]
+    end
+
+    def self.lookup(v, path)
+      return "" if path.to_s.empty?
+      path.split(".").each do |seg|
+        return "" unless v.is_a?(Hash)
+        v = v[seg]
+      end
+      case v
+      when String then v
+      when Integer then v.to_s
+      when Float then v.finite? && v == v.floor && v.abs < 1e15 ? v.to_i.to_s : v.to_s
+      else ""
+      end
+    end
+
+    def self.match_field(body, path)
+      return "" if path.to_s.empty?
+      last = path.split(".").last
+      m = body.match(/"#{Regexp.escape(last)}"\s*:\s*"([^"]*)"/)
+      m ? m[1] : ""
+    end
+
+    private
+
+    def key_pattern(names)
+      names.map(&:strip).reject(&:empty?).map { |n| Regexp.escape(n) }.join("|")
+    end
+
+    def json_secrets(keys)
+      /("(?:#{keys})"\s*:\s*)("(?:[^"\\]|\\.)*"?|[^,}\]\s]+)/i
+    end
+  end
+end

data/lib/visionary/retention.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Visionary
+  # Deletes expired rows 60 s after start, then every 24 h, in chunks so no
+  # statement runs long or hits D1's limits.
+  class Retention
+    FIRST_RUN = 60
+    EVERY = 24 * 3600
+    MAX_ROUNDS = 200
+
+    # tables: { "api_logs" => days, ... }; 0 days keeps forever.
+    def initialize(store, logger, tables)
+      @store = store
+      @logger = logger
+      @tables = tables.select { |_, days| days.to_i.positive? }
+      reset
+    end
+
+    def needed? = !@tables.empty?
+
+    def start
+      return self unless needed?
+      @thread = Thread.new { run }
+      @thread.name = "visionary-retention" if @thread.respond_to?(:name=)
+      self
+    end
+
+    def after_fork
+      reset
+      start
+    end
+
+    def close
+      @mutex.synchronize do
+        @stopped = true
+        @cond.broadcast
+      end
+    end
+
+    def purge(table, days)
+      cutoff = Visionary.timestamp(Time.now - (days * 86_400))
+      sql = Schema.purge_sql(@store.dialect, table)
+      total = 0
+      MAX_ROUNDS.times do
+        break if stopped?
+        n = begin
+          @store.changes(sql, [cutoff])
+        rescue StandardError => e
+          @logger.error("visionary: retention purge of #{table} failed after #{total} rows: #{e.message}")
+          return total
+        end
+        total += n
+        break if n < Schema::PURGE_CHUNK
+      end
+      @logger.info("visionary: retention purge of #{table} deleted #{total} rows older than #{cutoff}") if total.positive?
+      total
+    end
+
+    private
+
+    def reset
+      @mutex = Mutex.new
+      @cond = ConditionVariable.new
+      @stopped = false
+      @thread = nil
+    end
+
+    def stopped? = @mutex.synchronize { @stopped }
+
+    def run
+      Telemetry.internal!
+      wait = FIRST_RUN
+      loop do
+        @mutex.synchronize do
+          deadline = mono + wait
+          until @stopped || (left = deadline - mono) <= 0
+            @cond.wait(@mutex, left)
+          end
+        end
+        break if stopped?
+        @tables.each { |table, days| purge(table, days) }
+        wait = EVERY
+      end
+    rescue StandardError => e
+      @logger.error("visionary: retention stopped: #{e.message}")
+    end
+
+    def mono = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  end
+end

data/lib/visionary/schema/mysql/requests.sql

@@ -0,0 +1,37 @@
+CREATE TABLE IF NOT EXISTS api_logs (
+    id                      BIGINT       NOT NULL AUTO_INCREMENT PRIMARY KEY,
+    request_id              VARCHAR(128) NOT NULL,
+    created_at              VARCHAR(32)  NOT NULL,
+    method                  VARCHAR(16)  NOT NULL,
+    path                    TEXT         NOT NULL,
+    route                   VARCHAR(255),
+    query                   TEXT,
+    status                  INT          NOT NULL,
+    duration_ms             DOUBLE       NOT NULL,
+    ip                      VARCHAR(64),
+    user_agent              TEXT,
+    user_id                 VARCHAR(128),
+    user_handle             VARCHAR(255),
+    user_role               VARCHAR(64),
+    request_headers         TEXT,
+    request_content_type    VARCHAR(255),
+    request_body            MEDIUMTEXT,
+    request_body_truncated  TINYINT      NOT NULL DEFAULT 0,
+    request_size            BIGINT       NOT NULL DEFAULT 0,
+    response_content_type   VARCHAR(255),
+    response_body           MEDIUMTEXT,
+    response_body_truncated TINYINT      NOT NULL DEFAULT 0,
+    response_size           BIGINT       NOT NULL DEFAULT 0,
+    error_code              VARCHAR(128),
+    error_key               VARCHAR(255)
+    service                 VARCHAR(255),
+    version                 VARCHAR(255),
+    db_count                BIGINT,
+    db_ms                   DOUBLE,
+    http_count              BIGINT,
+    http_ms                 DOUBLE
+) CHARACTER SET utf8mb4;
+CREATE INDEX idx_api_logs_created_at ON api_logs (created_at);
+CREATE INDEX idx_api_logs_user_created ON api_logs (user_id, created_at);
+CREATE INDEX idx_api_logs_route_status ON api_logs (route, status);
+CREATE INDEX idx_api_logs_request_id ON api_logs (request_id);

data/lib/visionary/schema/mysql/system.sql

@@ -0,0 +1,76 @@
+CREATE TABLE IF NOT EXISTS system_metrics (
+    id                 BIGINT       NOT NULL AUTO_INCREMENT PRIMARY KEY,
+    created_at         VARCHAR(32)  NOT NULL,
+    instance           VARCHAR(128) NOT NULL,
+    interval_s         DOUBLE       NOT NULL,
+    host_cpu_percent   DOUBLE,
+    load_1             DOUBLE,
+    load_5             DOUBLE,
+    load_15            DOUBLE,
+    host_mem_used_mb   DOUBLE,
+    host_mem_total_mb  DOUBLE,
+    host_mem_percent   DOUBLE,
+    disk_used_percent  DOUBLE,
+    disk_free_gb       DOUBLE,
+    net_rx_kbps        DOUBLE,
+    net_tx_kbps        DOUBLE,
+    proc_cpu_percent   DOUBLE,
+    proc_rss_mb        DOUBLE,
+    proc_threads       INT,
+    proc_open_fds      INT,
+    goroutines         INT,
+    heap_alloc_mb      DOUBLE,
+    heap_inuse_mb      DOUBLE,
+    heap_objects       BIGINT,
+    gc_count           INT,
+    gc_pause_ms        DOUBLE,
+    gc_cpu_percent     DOUBLE,
+    db_total_conns     INT,
+    db_idle_conns      INT,
+    db_acquired_conns  INT,
+    db_max_conns       INT,
+    db_acquires        BIGINT,
+    db_empty_acquires  BIGINT,
+    db_acquire_wait_ms DOUBLE,
+    http_requests      BIGINT,
+    http_errors        BIGINT,
+    http_rps           DOUBLE,
+    http_in_flight     INT,
+    uptime_s           BIGINT,
+    cpu_count          BIGINT,
+    cpu_user_percent   DOUBLE,
+    cpu_system_percent DOUBLE,
+    cpu_iowait_percent DOUBLE,
+    cpu_steal_percent  DOUBLE,
+    host_mem_available_mb DOUBLE,
+    host_mem_cached_mb DOUBLE,
+    swap_used_mb       DOUBLE,
+    swap_total_mb      DOUBLE,
+    disk_total_gb      DOUBLE,
+    disk_inodes_percent DOUBLE,
+    disk_read_kbps     DOUBLE,
+    disk_write_kbps    DOUBLE,
+    disk_read_iops     DOUBLE,
+    disk_write_iops    DOUBLE,
+    disk_busy_percent  DOUBLE,
+    disk_await_ms      DOUBLE,
+    net_rx_pps         DOUBLE,
+    net_tx_pps         DOUBLE,
+    net_errors         BIGINT,
+    net_drops          BIGINT,
+    tcp_established    BIGINT,
+    tcp_retrans        BIGINT,
+    host_procs         BIGINT,
+    procs_running      BIGINT,
+    procs_blocked      BIGINT,
+    ctx_switches_ps    DOUBLE,
+    proc_read_kbps     DOUBLE,
+    proc_write_kbps    DOUBLE,
+    proc_max_fds       BIGINT
+    service            VARCHAR(255),
+    version            VARCHAR(255),
+    sdk                VARCHAR(255),
+    runtime            VARCHAR(255)
+) CHARACTER SET utf8mb4;
+CREATE INDEX idx_system_metrics_created_at ON system_metrics (created_at);
+CREATE INDEX idx_system_metrics_instance_created ON system_metrics (instance, created_at);

data/lib/visionary/schema/mysql/telemetry.sql

@@ -0,0 +1,52 @@
+CREATE TABLE IF NOT EXISTS spans (
+    id          BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+    created_at  VARCHAR(32) NOT NULL,
+    request_id  VARCHAR(255),
+    service     VARCHAR(255),
+    instance    VARCHAR(255) NOT NULL,
+    kind        VARCHAR(255) NOT NULL,
+    name        VARCHAR(255) NOT NULL,
+    target      VARCHAR(255),
+    detail      TEXT,
+    offset_ms   DOUBLE,
+    duration_ms DOUBLE NOT NULL,
+    status      VARCHAR(255) NOT NULL,
+    error       TEXT,
+    row_count   INT
+) CHARACTER SET utf8mb4;
+CREATE INDEX idx_spans_created_at ON spans (created_at);
+CREATE INDEX idx_spans_request_id ON spans (request_id);
+CREATE INDEX idx_spans_kind_name_created_at ON spans (kind, name, created_at);
+CREATE TABLE IF NOT EXISTS errors (
+    id          BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+    created_at  VARCHAR(32) NOT NULL,
+    service     VARCHAR(255),
+    instance    VARCHAR(255) NOT NULL,
+    version     VARCHAR(255),
+    request_id  VARCHAR(255),
+    method      VARCHAR(255),
+    route       VARCHAR(255),
+    path        TEXT,
+    user_id     VARCHAR(255),
+    type        VARCHAR(255) NOT NULL,
+    message     TEXT,
+    stack       MEDIUMTEXT,
+    fingerprint VARCHAR(255) NOT NULL,
+    handled     INT NOT NULL DEFAULT 0,
+    runtime     VARCHAR(255)
+) CHARACTER SET utf8mb4;
+CREATE INDEX idx_errors_created_at ON errors (created_at);
+CREATE INDEX idx_errors_fingerprint_created_at ON errors (fingerprint, created_at);
+CREATE INDEX idx_errors_request_id ON errors (request_id);
+CREATE TABLE IF NOT EXISTS releases (
+    id          BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+    created_at  VARCHAR(32) NOT NULL,
+    service     VARCHAR(255),
+    instance    VARCHAR(255) NOT NULL,
+    version     VARCHAR(255) NOT NULL,
+    sdk         VARCHAR(255),
+    sdk_version VARCHAR(255),
+    runtime     VARCHAR(255)
+) CHARACTER SET utf8mb4;
+CREATE INDEX idx_releases_created_at ON releases (created_at);
+CREATE INDEX idx_releases_version_created_at ON releases (version, created_at);

data/lib/visionary/schema/postgres/requests.sql

@@ -0,0 +1,37 @@
+CREATE TABLE IF NOT EXISTS api_logs (
+    id                      BIGSERIAL PRIMARY KEY,
+    request_id              TEXT             NOT NULL,
+    created_at              TEXT             NOT NULL,
+    method                  TEXT             NOT NULL,
+    path                    TEXT             NOT NULL,
+    route                   TEXT,
+    query                   TEXT,
+    status                  INTEGER          NOT NULL,
+    duration_ms             DOUBLE PRECISION NOT NULL,
+    ip                      TEXT,
+    user_agent              TEXT,
+    user_id                 TEXT,
+    user_handle             TEXT,
+    user_role               TEXT,
+    request_headers         TEXT,
+    request_content_type    TEXT,
+    request_body            TEXT,
+    request_body_truncated  INTEGER          NOT NULL DEFAULT 0,
+    request_size            BIGINT           NOT NULL DEFAULT 0,
+    response_content_type   TEXT,
+    response_body           TEXT,
+    response_body_truncated INTEGER          NOT NULL DEFAULT 0,
+    response_size           BIGINT           NOT NULL DEFAULT 0,
+    error_code              TEXT,
+    error_key               TEXT
+    service                 TEXT,
+    version                 TEXT,
+    db_count                BIGINT,
+    db_ms                   DOUBLE PRECISION,
+    http_count              BIGINT,
+    http_ms                 DOUBLE PRECISION
+);
+CREATE INDEX IF NOT EXISTS idx_api_logs_created_at ON api_logs (created_at);
+CREATE INDEX IF NOT EXISTS idx_api_logs_user_created ON api_logs (user_id, created_at);
+CREATE INDEX IF NOT EXISTS idx_api_logs_route_status ON api_logs (route, status);
+CREATE INDEX IF NOT EXISTS idx_api_logs_request_id ON api_logs (request_id);

data/lib/visionary/schema/postgres/system.sql

@@ -0,0 +1,76 @@
+CREATE TABLE IF NOT EXISTS system_metrics (
+    id                 BIGSERIAL PRIMARY KEY,
+    created_at         TEXT             NOT NULL,
+    instance           TEXT             NOT NULL,
+    interval_s         DOUBLE PRECISION NOT NULL,
+    host_cpu_percent   DOUBLE PRECISION,
+    load_1             DOUBLE PRECISION,
+    load_5             DOUBLE PRECISION,
+    load_15            DOUBLE PRECISION,
+    host_mem_used_mb   DOUBLE PRECISION,
+    host_mem_total_mb  DOUBLE PRECISION,
+    host_mem_percent   DOUBLE PRECISION,
+    disk_used_percent  DOUBLE PRECISION,
+    disk_free_gb       DOUBLE PRECISION,
+    net_rx_kbps        DOUBLE PRECISION,
+    net_tx_kbps        DOUBLE PRECISION,
+    proc_cpu_percent   DOUBLE PRECISION,
+    proc_rss_mb        DOUBLE PRECISION,
+    proc_threads       INTEGER,
+    proc_open_fds      INTEGER,
+    goroutines         INTEGER,
+    heap_alloc_mb      DOUBLE PRECISION,
+    heap_inuse_mb      DOUBLE PRECISION,
+    heap_objects       BIGINT,
+    gc_count           INTEGER,
+    gc_pause_ms        DOUBLE PRECISION,
+    gc_cpu_percent     DOUBLE PRECISION,
+    db_total_conns     INTEGER,
+    db_idle_conns      INTEGER,
+    db_acquired_conns  INTEGER,
+    db_max_conns       INTEGER,
+    db_acquires        BIGINT,
+    db_empty_acquires  BIGINT,
+    db_acquire_wait_ms DOUBLE PRECISION,
+    http_requests      BIGINT,
+    http_errors        BIGINT,
+    http_rps           DOUBLE PRECISION,
+    http_in_flight     INTEGER,
+    uptime_s           BIGINT,
+    cpu_count          BIGINT,
+    cpu_user_percent   DOUBLE PRECISION,
+    cpu_system_percent DOUBLE PRECISION,
+    cpu_iowait_percent DOUBLE PRECISION,
+    cpu_steal_percent  DOUBLE PRECISION,
+    host_mem_available_mb DOUBLE PRECISION,
+    host_mem_cached_mb DOUBLE PRECISION,
+    swap_used_mb       DOUBLE PRECISION,
+    swap_total_mb      DOUBLE PRECISION,
+    disk_total_gb      DOUBLE PRECISION,
+    disk_inodes_percent DOUBLE PRECISION,
+    disk_read_kbps     DOUBLE PRECISION,
+    disk_write_kbps    DOUBLE PRECISION,
+    disk_read_iops     DOUBLE PRECISION,
+    disk_write_iops    DOUBLE PRECISION,
+    disk_busy_percent  DOUBLE PRECISION,
+    disk_await_ms      DOUBLE PRECISION,
+    net_rx_pps         DOUBLE PRECISION,
+    net_tx_pps         DOUBLE PRECISION,
+    net_errors         BIGINT,
+    net_drops          BIGINT,
+    tcp_established    BIGINT,
+    tcp_retrans        BIGINT,
+    host_procs         BIGINT,
+    procs_running      BIGINT,
+    procs_blocked      BIGINT,
+    ctx_switches_ps    DOUBLE PRECISION,
+    proc_read_kbps     DOUBLE PRECISION,
+    proc_write_kbps    DOUBLE PRECISION,
+    proc_max_fds       BIGINT
+    service            TEXT,
+    version            TEXT,
+    sdk                TEXT,
+    runtime            TEXT
+);
+CREATE INDEX IF NOT EXISTS idx_system_metrics_created_at ON system_metrics (created_at);
+CREATE INDEX IF NOT EXISTS idx_system_metrics_instance_created ON system_metrics (instance, created_at);

data/lib/visionary/schema/postgres/telemetry.sql

@@ -0,0 +1,52 @@
+CREATE TABLE IF NOT EXISTS spans (
+    id          BIGSERIAL PRIMARY KEY,
+    created_at  TEXT NOT NULL,
+    request_id  TEXT,
+    service     TEXT,
+    instance    TEXT NOT NULL,
+    kind        TEXT NOT NULL,
+    name        TEXT NOT NULL,
+    target      TEXT,
+    detail      TEXT,
+    offset_ms   DOUBLE PRECISION,
+    duration_ms DOUBLE PRECISION NOT NULL,
+    status      TEXT NOT NULL,
+    error       TEXT,
+    row_count   INTEGER
+);
+CREATE INDEX IF NOT EXISTS idx_spans_created_at ON spans (created_at);
+CREATE INDEX IF NOT EXISTS idx_spans_request_id ON spans (request_id);
+CREATE INDEX IF NOT EXISTS idx_spans_kind_name_created_at ON spans (kind, name, created_at);
+CREATE TABLE IF NOT EXISTS errors (
+    id          BIGSERIAL PRIMARY KEY,
+    created_at  TEXT NOT NULL,
+    service     TEXT,
+    instance    TEXT NOT NULL,
+    version     TEXT,
+    request_id  TEXT,
+    method      TEXT,
+    route       TEXT,
+    path        TEXT,
+    user_id     TEXT,
+    type        TEXT NOT NULL,
+    message     TEXT,
+    stack       TEXT,
+    fingerprint TEXT NOT NULL,
+    handled     INTEGER NOT NULL DEFAULT 0,
+    runtime     TEXT
+);
+CREATE INDEX IF NOT EXISTS idx_errors_created_at ON errors (created_at);
+CREATE INDEX IF NOT EXISTS idx_errors_fingerprint_created_at ON errors (fingerprint, created_at);
+CREATE INDEX IF NOT EXISTS idx_errors_request_id ON errors (request_id);
+CREATE TABLE IF NOT EXISTS releases (
+    id          BIGSERIAL PRIMARY KEY,
+    created_at  TEXT NOT NULL,
+    service     TEXT,
+    instance    TEXT NOT NULL,
+    version     TEXT NOT NULL,
+    sdk         TEXT,
+    sdk_version TEXT,
+    runtime     TEXT
+);
+CREATE INDEX IF NOT EXISTS idx_releases_created_at ON releases (created_at);
+CREATE INDEX IF NOT EXISTS idx_releases_version_created_at ON releases (version, created_at);

data/lib/visionary/schema/sqlite/requests.sql

@@ -0,0 +1,37 @@
+CREATE TABLE IF NOT EXISTS api_logs (
+    id                      INTEGER PRIMARY KEY AUTOINCREMENT,
+    request_id              TEXT    NOT NULL,
+    created_at              TEXT    NOT NULL,
+    method                  TEXT    NOT NULL,
+    path                    TEXT    NOT NULL,
+    route                   TEXT,
+    query                   TEXT,
+    status                  INTEGER NOT NULL,
+    duration_ms             REAL    NOT NULL,
+    ip                      TEXT,
+    user_agent              TEXT,
+    user_id                 INTEGER,
+    user_handle             TEXT,
+    user_role               TEXT,
+    request_headers         TEXT,
+    request_content_type    TEXT,
+    request_body            TEXT,
+    request_body_truncated  INTEGER NOT NULL DEFAULT 0,
+    request_size            INTEGER NOT NULL DEFAULT 0,
+    response_content_type   TEXT,
+    response_body           TEXT,
+    response_body_truncated INTEGER NOT NULL DEFAULT 0,
+    response_size           INTEGER NOT NULL DEFAULT 0,
+    error_code              TEXT,
+    error_key               TEXT
+    service                 TEXT,
+    version                 TEXT,
+    db_count                INTEGER,
+    db_ms                   REAL,
+    http_count              INTEGER,
+    http_ms                 REAL
+);
+CREATE INDEX IF NOT EXISTS idx_api_logs_created_at ON api_logs (created_at);
+CREATE INDEX IF NOT EXISTS idx_api_logs_user_created ON api_logs (user_id, created_at);
+CREATE INDEX IF NOT EXISTS idx_api_logs_route_status ON api_logs (route, status);
+CREATE INDEX IF NOT EXISTS idx_api_logs_request_id ON api_logs (request_id);

data/lib/visionary/schema/sqlite/system.sql

@@ -0,0 +1,76 @@
+CREATE TABLE IF NOT EXISTS system_metrics (
+    id                 INTEGER PRIMARY KEY AUTOINCREMENT,
+    created_at         TEXT    NOT NULL,
+    instance           TEXT    NOT NULL,
+    interval_s         REAL    NOT NULL,
+    host_cpu_percent   REAL,
+    load_1             REAL,
+    load_5             REAL,
+    load_15            REAL,
+    host_mem_used_mb   REAL,
+    host_mem_total_mb  REAL,
+    host_mem_percent   REAL,
+    disk_used_percent  REAL,
+    disk_free_gb       REAL,
+    net_rx_kbps        REAL,
+    net_tx_kbps        REAL,
+    proc_cpu_percent   REAL,
+    proc_rss_mb        REAL,
+    proc_threads       INTEGER,
+    proc_open_fds      INTEGER,
+    goroutines         INTEGER,
+    heap_alloc_mb      REAL,
+    heap_inuse_mb      REAL,
+    heap_objects       INTEGER,
+    gc_count           INTEGER,
+    gc_pause_ms        REAL,
+    gc_cpu_percent     REAL,
+    db_total_conns     INTEGER,
+    db_idle_conns      INTEGER,
+    db_acquired_conns  INTEGER,
+    db_max_conns       INTEGER,
+    db_acquires        INTEGER,
+    db_empty_acquires  INTEGER,
+    db_acquire_wait_ms REAL,
+    http_requests      INTEGER,
+    http_errors        INTEGER,
+    http_rps           REAL,
+    http_in_flight     INTEGER,
+    uptime_s           INTEGER,
+    cpu_count          INTEGER,
+    cpu_user_percent   REAL,
+    cpu_system_percent REAL,
+    cpu_iowait_percent REAL,
+    cpu_steal_percent  REAL,
+    host_mem_available_mb REAL,
+    host_mem_cached_mb REAL,
+    swap_used_mb       REAL,
+    swap_total_mb      REAL,
+    disk_total_gb      REAL,
+    disk_inodes_percent REAL,
+    disk_read_kbps     REAL,
+    disk_write_kbps    REAL,
+    disk_read_iops     REAL,
+    disk_write_iops    REAL,
+    disk_busy_percent  REAL,
+    disk_await_ms      REAL,
+    net_rx_pps         REAL,
+    net_tx_pps         REAL,
+    net_errors         INTEGER,
+    net_drops          INTEGER,
+    tcp_established    INTEGER,
+    tcp_retrans        INTEGER,
+    host_procs         INTEGER,
+    procs_running      INTEGER,
+    procs_blocked      INTEGER,
+    ctx_switches_ps    REAL,
+    proc_read_kbps     REAL,
+    proc_write_kbps    REAL,
+    proc_max_fds       INTEGER
+    service            TEXT,
+    version            TEXT,
+    sdk                TEXT,
+    runtime            TEXT
+);
+CREATE INDEX IF NOT EXISTS idx_system_metrics_created_at ON system_metrics (created_at);
+CREATE INDEX IF NOT EXISTS idx_system_metrics_instance_created ON system_metrics (instance, created_at);

data/lib/visionary/schema/sqlite/telemetry.sql

@@ -0,0 +1,52 @@
+CREATE TABLE IF NOT EXISTS spans (
+    id          INTEGER PRIMARY KEY AUTOINCREMENT,
+    created_at  TEXT NOT NULL,
+    request_id  TEXT,
+    service     TEXT,
+    instance    TEXT NOT NULL,
+    kind        TEXT NOT NULL,
+    name        TEXT NOT NULL,
+    target      TEXT,
+    detail      TEXT,
+    offset_ms   REAL,
+    duration_ms REAL NOT NULL,
+    status      TEXT NOT NULL,
+    error       TEXT,
+    row_count   INTEGER
+);
+CREATE INDEX IF NOT EXISTS idx_spans_created_at ON spans (created_at);
+CREATE INDEX IF NOT EXISTS idx_spans_request_id ON spans (request_id);
+CREATE INDEX IF NOT EXISTS idx_spans_kind_name_created_at ON spans (kind, name, created_at);
+CREATE TABLE IF NOT EXISTS errors (
+    id          INTEGER PRIMARY KEY AUTOINCREMENT,
+    created_at  TEXT NOT NULL,
+    service     TEXT,
+    instance    TEXT NOT NULL,
+    version     TEXT,
+    request_id  TEXT,
+    method      TEXT,
+    route       TEXT,
+    path        TEXT,
+    user_id     TEXT,
+    type        TEXT NOT NULL,
+    message     TEXT,
+    stack       TEXT,
+    fingerprint TEXT NOT NULL,
+    handled     INTEGER NOT NULL DEFAULT 0,
+    runtime     TEXT
+);
+CREATE INDEX IF NOT EXISTS idx_errors_created_at ON errors (created_at);
+CREATE INDEX IF NOT EXISTS idx_errors_fingerprint_created_at ON errors (fingerprint, created_at);
+CREATE INDEX IF NOT EXISTS idx_errors_request_id ON errors (request_id);
+CREATE TABLE IF NOT EXISTS releases (
+    id          INTEGER PRIMARY KEY AUTOINCREMENT,
+    created_at  TEXT NOT NULL,
+    service     TEXT,
+    instance    TEXT NOT NULL,
+    version     TEXT NOT NULL,
+    sdk         TEXT,
+    sdk_version TEXT,
+    runtime     TEXT
+);
+CREATE INDEX IF NOT EXISTS idx_releases_created_at ON releases (created_at);
+CREATE INDEX IF NOT EXISTS idx_releases_version_created_at ON releases (version, created_at);