@noego/telemetry 0.0.5 → 0.1.0

package/dist/cli/telemetry_cli.js

@@ -66,11 +66,15 @@ var MetricDefinitionContract = class {
     if (!eventSelector || !bracket) {
       return { ok: false, error: `Invalid metric selector ${selector}` };
     }
+    const groupBy = bracket === "*" ? [] : bracket.split(",").map((field) => field.trim());
+    if (groupBy.some((field) => field.length === 0)) {
+      return { ok: false, error: `Invalid metric selector ${selector}` };
+    }
     return {
       ok: true,
       value: {
         selector: eventSelector,
-        groupBy: bracket === "*" ? [] : [bracket],
+        groupBy,
         wildcardUngrouped: bracket === "*"
       }
     };
@@ -1786,18 +1790,194 @@ var MetricGraphRenderer = class {
   }
 };
 
+// src/cli/history_metric_reader.ts
+import { existsSync } from "fs";
+import { dirname as dirname2, join } from "path";
+import BetterSqlite3 from "better-sqlite3";
+var HISTORY_DATABASE_NAME = "history.db";
+var AGGREGATES = /* @__PURE__ */ new Set(["count", "sum", "min", "max", "last", "avg"]);
+function historyDatabasePath(target) {
+  return join(dirname2(target.path), HISTORY_DATABASE_NAME);
+}
+var HistoryMetricReader = class _HistoryMetricReader {
+  constructor(db, definitions) {
+    this.db = db;
+    this.definitions = definitions;
+  }
+  db;
+  definitions;
+  static open(databasePath, definitions) {
+    const db = new BetterSqlite3(databasePath, { readonly: true, fileMustExist: true });
+    db.pragma("busy_timeout = 1000");
+    db.pragma("query_only = ON");
+    return new _HistoryMetricReader(db, definitions);
+  }
+  static exists(databasePath) {
+    return existsSync(databasePath);
+  }
+  metrics(query, targetId) {
+    if (query.toMs <= query.fromMs) {
+      return { ok: false, error: { kind: "invalid-query", message: "--to must be after --from" } };
+    }
+    const invalidMeasurement = query.measurements.find((measurement) => !isMeasurement(measurement));
+    if (invalidMeasurement !== void 0) {
+      return {
+        ok: false,
+        error: {
+          kind: "invalid-query",
+          message: `Unknown measurement ${invalidMeasurement}; use count or <field>.<sum|min|max|last|count|avg>`
+        }
+      };
+    }
+    const versions = this.registeredVersions(query.metric);
+    if (versions.length === 0) {
+      return {
+        ok: false,
+        error: { kind: "invalid-query", message: `Metric ${query.metric} has no projection in history.db` }
+      };
+    }
+    const resolutionMs = this.resolveResolution(query, versions);
+    if (resolutionMs === null) {
+      return {
+        ok: false,
+        error: { kind: "invalid-query", message: `Metric ${query.metric} is not projected at the requested resolution` }
+      };
+    }
+    for (let index = versions.length - 1; index >= 0; index -= 1) {
+      const version = versions[index];
+      const buckets = this.readBuckets(query, version, resolutionMs);
+      if (buckets.length === 0) continue;
+      const olderVersions = versions.slice(0, index);
+      return {
+        ok: true,
+        value: Object.freeze({
+          targetId,
+          metric: query.metric,
+          segments: [Object.freeze({
+            fromMs: query.fromMs,
+            toMs: query.toMs,
+            resolutionMs,
+            isAggregated: true,
+            definitionVersion: version,
+            buckets
+          })],
+          gaps: [],
+          nextCursor: null,
+          partialFailures: olderVersions.map((older) => Object.freeze({
+            targetId,
+            reason: `version ${older} buckets omitted; only version ${version} returned`
+          }))
+        })
+      };
+    }
+    return {
+      ok: true,
+      value: Object.freeze({
+        targetId,
+        metric: query.metric,
+        segments: [],
+        gaps: [Object.freeze({ fromMs: query.fromMs, toMs: query.toMs, reason: "not-materialized" })],
+        nextCursor: null,
+        partialFailures: []
+      })
+    };
+  }
+  close() {
+    if (this.db.open) this.db.close();
+  }
+  registeredVersions(metric) {
+    const rows = this.db.prepare(
+      "SELECT projection_version FROM trace_projection_definitions WHERE projection_name = ? ORDER BY projection_version"
+    ).all(metric);
+    return rows.map((row) => row.projection_version);
+  }
+  resolveResolution(query, versions) {
+    const available = this.definitions.filter((definition) => definition.name === query.metric && versions.includes(definition.version)).flatMap((definition) => [...definition.resolutionsMs]);
+    const resolutions = [...new Set(available.length > 0 ? available : this.storedResolutions(query.metric))].sort((a, b) => a - b);
+    if (resolutions.length === 0) return null;
+    if (query.resolution !== "auto") {
+      return resolutions.includes(query.resolution) ? query.resolution : null;
+    }
+    const span = query.toMs - query.fromMs;
+    return resolutions.find((resolution) => Math.ceil(span / resolution) <= query.limit) ?? resolutions[resolutions.length - 1];
+  }
+  storedResolutions(metric) {
+    const rows = this.db.prepare(
+      "SELECT DISTINCT resolution_ms FROM trace_metric_buckets WHERE projection_name = ?"
+    ).all(metric);
+    return rows.map((row) => row.resolution_ms);
+  }
+  readBuckets(query, version, resolutionMs) {
+    const rows = this.db.prepare(`
+      SELECT projection_version, bucket_start_ms, dimensions_json, count, values_json
+      FROM trace_metric_buckets
+      WHERE projection_name = ? AND projection_version = ? AND resolution_ms = ?
+        AND bucket_start_ms >= ? AND bucket_start_ms < ?
+      ORDER BY bucket_start_ms ${query.order === "desc" ? "DESC" : "ASC"}, dimension_key
+    `).all(query.metric, version, resolutionMs, query.fromMs, query.toMs);
+    const buckets = [];
+    for (const row of rows) {
+      const dimensions = JSON.parse(row.dimensions_json);
+      if (!matchesWhere(dimensions, query.where)) continue;
+      const values = JSON.parse(row.values_json);
+      buckets.push(Object.freeze({
+        startMs: row.bucket_start_ms,
+        endMs: row.bucket_start_ms + resolutionMs,
+        dimensions,
+        values: selectMeasurements(query.measurements, row.count, values)
+      }));
+      if (buckets.length >= query.limit) break;
+    }
+    return buckets;
+  }
+};
+function isMeasurement(measurement) {
+  if (measurement === "count") return true;
+  const dot = measurement.lastIndexOf(".");
+  return dot > 0 && AGGREGATES.has(measurement.slice(dot + 1));
+}
+function selectMeasurements(measurements, bucketCount, values) {
+  const selected = {};
+  for (const measurement of measurements) {
+    if (measurement === "count") {
+      selected.count = bucketCount;
+      continue;
+    }
+    const dot = measurement.lastIndexOf(".");
+    const field = measurement.slice(0, dot);
+    const aggregate = measurement.slice(dot + 1);
+    const state = values[field];
+    if (state === void 0) continue;
+    const value = aggregate === "avg" ? state.count > 0 ? state.sum / state.count : 0 : state[aggregate];
+    if (!Number.isFinite(value)) continue;
+    const existing = selected[field];
+    selected[field] = { ...typeof existing === "object" ? existing : {}, [aggregate]: value };
+  }
+  return Object.freeze(selected);
+}
+function matchesWhere(dimensions, where) {
+  for (const [key, filter] of Object.entries(where)) {
+    const actual = dimensions[key];
+    if (actual === void 0) return false;
+    const accepted = Array.isArray(filter) ? filter : [filter];
+    if (!accepted.some((candidate) => String(candidate) === actual)) return false;
+  }
+  return true;
+}
+
 // src/cli/offline_sqlite_session.ts
 import { randomUUID } from "crypto";
-import { resolve as resolve3 } from "path";
+import { existsSync as existsSync3, statfsSync, statSync } from "fs";
+import { dirname as dirname4, resolve as resolve3 } from "path";
 import { fileURLToPath } from "url";
 import { MigrationRunnerFactory } from "@noego/proper";
-import BetterSqlite3 from "better-sqlite3";
+import BetterSqlite32 from "better-sqlite3";
 import { createSqliteDb } from "sqlstack/adapters";
 
 // src/storage/sqlite_ownership_lease.ts
 import {
   closeSync,
-  existsSync,
+  existsSync as existsSync2,
   mkdirSync,
   openSync,
   readFileSync as readFileSync2,
@@ -1806,13 +1986,13 @@ import {
   rmSync,
   writeFileSync
 } from "fs";
-import { dirname as dirname2, resolve as resolve2 } from "path";
+import { dirname as dirname3, resolve as resolve2 } from "path";
 var SqliteOwnershipLease = class {
   constructor(options) {
     this.options = options;
-    mkdirSync(dirname2(options.databasePath), { recursive: true });
-    const canonicalDirectory = realpathSync(dirname2(resolve2(options.databasePath)));
-    const databaseName = resolve2(options.databasePath).slice(dirname2(resolve2(options.databasePath)).length + 1);
+    mkdirSync(dirname3(options.databasePath), { recursive: true });
+    const canonicalDirectory = realpathSync(dirname3(resolve2(options.databasePath)));
+    const databaseName = resolve2(options.databasePath).slice(dirname3(resolve2(options.databasePath)).length + 1);
     this.canonicalDatabasePath = resolve2(canonicalDirectory, databaseName);
     this.leasePath = `${this.canonicalDatabasePath}.telemetry-owner.json`;
   }
@@ -1856,7 +2036,7 @@ var SqliteOwnershipLease = class {
     });
   }
   release() {
-    if (!this.held || !existsSync(this.leasePath)) {
+    if (!this.held || !existsSync2(this.leasePath)) {
       this.held = false;
       return;
     }
@@ -4483,6 +4663,87 @@ var OfflineSqliteTelemetrySession = class _OfflineSqliteTelemetrySession {
     this.verifySchema();
     return this.migrationStatus();
   }
+  async compact() {
+    this.requireMaintenanceMode();
+    const connection = this.requireConnection();
+    const maintenanceTable = this.requireTableMap().physical("maintenanceRuns");
+    const runId = randomUUID();
+    const startedAtMs = this.now();
+    connection.prepare(
+      `INSERT INTO "${maintenanceTable}" (
+         target_id, run_id, state, stage, started_at_ms,
+         completed_at_ms, affected_rows, affected_pages, detail
+       ) VALUES (?, ?, 'started', 'compaction', ?, NULL, 0, 0, ?)`
+    ).run(this.target.targetId, runId, startedAtMs, "offline-cli; outcome=started");
+    try {
+      const before = this.compactionSnapshot();
+      connection.pragma("wal_checkpoint(TRUNCATE)");
+      const checkpointed = this.compactionSnapshot();
+      this.assertCompactionSpace(checkpointed);
+      connection.pragma("auto_vacuum = INCREMENTAL");
+      connection.exec("VACUUM");
+      connection.pragma("wal_checkpoint(TRUNCATE)");
+      const after = this.compactionSnapshot();
+      if (after.autoVacuum !== 2) {
+        throw new Error("Telemetry compaction did not enable auto_vacuum=INCREMENTAL");
+      }
+      const completedAtMs = this.now();
+      const result = Object.freeze({
+        targetId: this.target.targetId,
+        autoVacuum: "incremental",
+        pageSize: checkpointed.pageSize,
+        pagesBefore: checkpointed.pageCount,
+        pagesAfter: after.pageCount,
+        freePagesBefore: checkpointed.freePages,
+        freePagesAfter: after.freePages,
+        reclaimedPages: Math.max(0, checkpointed.pageCount - after.pageCount),
+        databaseBytesBefore: before.databaseBytes,
+        databaseBytesAfter: after.databaseBytes,
+        walBytesBefore: before.walBytes,
+        walBytesAfter: after.walBytes,
+        reclaimedBytes: Math.max(
+          0,
+          before.databaseBytes + before.walBytes - after.databaseBytes - after.walBytes
+        ),
+        elapsedMs: completedAtMs - startedAtMs
+      });
+      connection.prepare(
+        `UPDATE "${maintenanceTable}"
+         SET state = 'completed', completed_at_ms = ?, affected_pages = ?, detail = ?
+         WHERE target_id = ? AND run_id = ? AND stage = 'compaction'`
+      ).run(
+        completedAtMs,
+        result.reclaimedPages,
+        JSON.stringify({
+          mode: "offline-cli",
+          outcome: "completed",
+          reclaimedBytes: result.reclaimedBytes,
+          walBytesBefore: result.walBytesBefore,
+          walBytesAfter: result.walBytesAfter,
+          autoVacuum: result.autoVacuum
+        }),
+        this.target.targetId,
+        runId
+      );
+      return result;
+    } catch (cause) {
+      const message = cause instanceof Error ? cause.message : String(cause);
+      try {
+        connection.prepare(
+          `UPDATE "${maintenanceTable}"
+           SET state = 'failed', completed_at_ms = ?, detail = ?
+           WHERE target_id = ? AND run_id = ? AND stage = 'compaction'`
+        ).run(
+          this.now(),
+          JSON.stringify({ mode: "offline-cli", outcome: "failed", error: message }),
+          this.target.targetId,
+          runId
+        );
+      } catch {
+      }
+      throw cause;
+    }
+  }
   async close() {
     try {
       if (this.connection?.open) {
@@ -4516,7 +4777,7 @@ var OfflineSqliteTelemetrySession = class _OfflineSqliteTelemetrySession {
         throw new Error(acquired.error);
       }
     }
-    const connection = new BetterSqlite3(this.target.path, {
+    const connection = new BetterSqlite32(this.target.path, {
       readonly: this.mode === "query",
       fileMustExist: true
     });
@@ -4543,6 +4804,32 @@ var OfflineSqliteTelemetrySession = class _OfflineSqliteTelemetrySession {
       }
     }
   }
+  compactionSnapshot() {
+    const connection = this.requireConnection();
+    return Object.freeze({
+      pageSize: Number(connection.pragma("page_size", { simple: true })),
+      pageCount: Number(connection.pragma("page_count", { simple: true })),
+      freePages: Number(connection.pragma("freelist_count", { simple: true })),
+      autoVacuum: Number(connection.pragma("auto_vacuum", { simple: true })),
+      databaseBytes: this.fileBytes(this.target.path),
+      walBytes: this.fileBytes(`${this.target.path}-wal`)
+    });
+  }
+  assertCompactionSpace(snapshot) {
+    const liveBytes = Math.max(0, snapshot.pageCount - snapshot.freePages) * snapshot.pageSize;
+    const reserveBytes = Math.max(64 * 1024 * 1024, Math.ceil(liveBytes * 0.05));
+    const filesystem = statfsSync(dirname4(this.target.path));
+    const availableBytes = Number(filesystem.bavail) * Number(filesystem.bsize);
+    const requiredBytes = liveBytes + reserveBytes;
+    if (availableBytes < requiredBytes) {
+      throw new Error(
+        `Telemetry compaction requires at least ${requiredBytes} free bytes; only ${availableBytes} are available`
+      );
+    }
+  }
+  fileBytes(path) {
+    return existsSync3(path) ? statSync(path).size : 0;
+  }
   verifyManifest() {
     const rows = this.requireConnection().prepare(
       `SELECT target_id, format_version, mapping_json, mapping_checksum,
@@ -4727,7 +5014,8 @@ var ALLOWED_OPTIONS = Object.freeze({
     "metric",
     "where",
     "measurement",
-    "resolution"
+    "resolution",
+    "source"
   ]),
   "metrics-graph": /* @__PURE__ */ new Set([
     ...COMMON_QUERY_OPTIONS,
@@ -4735,6 +5023,7 @@ var ALLOWED_OPTIONS = Object.freeze({
     "where",
     "measurement",
     "resolution",
+    "source",
     "type",
     "output",
     "width",
@@ -4752,6 +5041,7 @@ var ALLOWED_OPTIONS = Object.freeze({
     "limit",
     "max-duration"
   ]),
+  "sqlite-compact": /* @__PURE__ */ new Set(["config", "target", "storage-base", "format"]),
   "sqlite-migration-status": /* @__PURE__ */ new Set(["config", "target", "storage-base", "format"]),
   "sqlite-migration-up": /* @__PURE__ */ new Set(["config", "target", "storage-base", "format"])
 });
@@ -4761,6 +5051,7 @@ var COMMANDS = Object.freeze([
   { tokens: ["metrics", "query"], kind: "metrics-query" },
   { tokens: ["metrics", "graph"], kind: "metrics-graph" },
   { tokens: ["sqlite", "query"], kind: "sqlite-query" },
+  { tokens: ["sqlite", "compact"], kind: "sqlite-compact" },
   { tokens: ["sqlite", "migration", "status"], kind: "sqlite-migration-status" },
   { tokens: ["sqlite", "migration", "up"], kind: "sqlite-migration-up" }
 ]);
@@ -5039,6 +5330,22 @@ var TelemetryCliFormatter = class {
     return `${lines.join("\n")}
 `;
   }
+  compaction(result, format) {
+    if (format === "json") {
+      return this.json(result);
+    }
+    return [
+      `Target: ${result.targetId}`,
+      `Auto-vacuum: ${result.autoVacuum}`,
+      `Pages: ${result.pagesBefore} -> ${result.pagesAfter} (${result.reclaimedPages} reclaimed)`,
+      `Freelist: ${result.freePagesBefore} -> ${result.freePagesAfter}`,
+      `Database bytes: ${result.databaseBytesBefore} -> ${result.databaseBytesAfter}`,
+      `WAL bytes: ${result.walBytesBefore} -> ${result.walBytesAfter}`,
+      `Filesystem bytes reclaimed: ${result.reclaimedBytes}`,
+      `Elapsed ms: ${result.elapsedMs}`,
+      ""
+    ].join("\n");
+  }
   migrations(result, format) {
     if (format === "json") {
       return this.json(result);
@@ -5146,6 +5453,10 @@ var TelemetryCli = class {
       storageBaseDirectory === null ? {} : { storageBaseDirectory }
     );
     const target = this.resolveTarget(configuration, this.optionalOption(command, "target"));
+    if (this.metricSource(command, target) === "history") {
+      await this.executeHistoryMetrics(command, target, configuration);
+      return 0;
+    }
     const mode = this.sessionMode(command);
     const session = await this.createSession(configuration, target, mode);
     this.activeSession = session;
@@ -5159,23 +5470,61 @@ var TelemetryCli = class {
       }
     }
   }
+  /**
+   * Which store answers a metric command. `history` is the projector's
+   * pre-aggregated buckets (history.db, current whenever the app runs with
+   * ingest-time extraction off); `legacy` is the collector's own rollups in
+   * telemetry.db. `auto` prefers history.db when it exists beside the target —
+   * since ingestion moved to the projector the legacy rollups stopped
+   * advancing, so answering from them silently reports stale gaps.
+   */
+  metricSource(command, target) {
+    if (command.kind !== "metrics-query" && command.kind !== "metrics-graph") return "legacy";
+    const requested = this.optionalOption(command, "source") ?? "auto";
+    if (requested === "history" || requested === "legacy") return requested;
+    if (requested !== "auto") {
+      throw new TelemetryCliUsageError("--source must be auto, history, or legacy");
+    }
+    return HistoryMetricReader.exists(historyDatabasePath(target)) ? "history" : "legacy";
+  }
+  async executeHistoryMetrics(command, target, configuration) {
+    const databasePath = historyDatabasePath(target);
+    if (!HistoryMetricReader.exists(databasePath)) {
+      throw new TelemetryCliUsageError(`No projector history store at ${databasePath}`);
+    }
+    const reader = HistoryMetricReader.open(databasePath, configuration.metrics);
+    try {
+      const run = async (query) => reader.metrics(query, target.targetId);
+      if (command.kind === "metrics-graph") {
+        await this.metricGraph(command, target, run);
+      } else {
+        await this.metricQuery(command, target, run);
+      }
+    } finally {
+      reader.close();
+    }
+  }
   async executeCommand(command, target, session) {
     if (command.kind === "timeline-query") {
       await this.timelineQuery(command, target, session);
       return;
     }
     if (command.kind === "metrics-query") {
-      await this.metricQuery(command, target, session);
+      await this.metricQuery(command, target, (query) => session.metrics(query));
       return;
     }
     if (command.kind === "metrics-graph") {
-      await this.metricGraph(command, target, session);
+      await this.metricGraph(command, target, (query) => session.metrics(query));
       return;
     }
     if (command.kind === "sqlite-query") {
       await this.sqliteQuery(command, target, session);
       return;
     }
+    if (command.kind === "sqlite-compact") {
+      await this.sqliteCompact(command, target, session);
+      return;
+    }
     if (command.kind === "sqlite-migration-status" || command.kind === "sqlite-migration-up") {
       await this.sqliteMigration(command, target, session);
       return;
@@ -5205,8 +5554,8 @@ var TelemetryCli = class {
     const output = this.formatter.timeline(result.value, format, this.timezoneOption(command));
     this.writeBounded(output, target.query.maxResponseBytes);
   }
-  async metricQuery(command, target, session) {
-    const result = await session.metrics(this.metricQueryInput(command, target));
+  async metricQuery(command, target, runMetrics) {
+    const result = await runMetrics(this.metricQueryInput(command, target));
     if (!result.ok) {
       throw new Error(`${result.error.kind}: ${result.error.message}`);
     }
@@ -5217,12 +5566,12 @@ var TelemetryCli = class {
     );
     this.writeBounded(output, target.query.maxResponseBytes);
   }
-  async metricGraph(command, target, session) {
+  async metricGraph(command, target, runMetrics) {
     const measurements = this.options(command, "measurement");
     if (measurements.length !== 1) {
       throw new TelemetryCliUsageError("metrics graph requires exactly one --measurement");
     }
-    const result = await session.metrics(this.metricQueryInput(command, target));
+    const result = await runMetrics(this.metricQueryInput(command, target));
     if (!result.ok) {
       throw new Error(`${result.error.kind}: ${result.error.message}`);
     }
@@ -5273,6 +5622,14 @@ var TelemetryCli = class {
     }
     this.writeBounded(this.formatter.migrations(result, format), target.query.maxResponseBytes);
   }
+  async sqliteCompact(command, target, session) {
+    const format = this.optionalOption(command, "format") ?? "table";
+    if (format !== "table" && format !== "json") {
+      throw new TelemetryCliUsageError("Compaction format must be table or json");
+    }
+    const result = await session.compact();
+    this.writeBounded(this.formatter.compaction(result, format), target.query.maxResponseBytes);
+  }
   metricQueryInput(command, target) {
     const measurements = this.options(command, "measurement");
     if (measurements.length === 0) {
@@ -5307,7 +5664,7 @@ var TelemetryCli = class {
     return target;
   }
   sessionMode(command) {
-    return command.kind === "sqlite-migration-status" || command.kind === "sqlite-migration-up" ? "maintenance" : "query";
+    return command.kind === "sqlite-compact" || command.kind === "sqlite-migration-status" || command.kind === "sqlite-migration-up" ? "maintenance" : "query";
   }
   structuredFormat(command) {
     const value = this.optionalOption(command, "format") ?? "table";
@@ -5474,9 +5831,10 @@ var TelemetryCli = class {
   help() {
     return `Usage:
   telemetry timeline query --config FILE --from TIME --to TIME [filters] [--format table|json|jsonl|csv]
-  telemetry metrics query --config FILE --metric NAME --measurement NAME --resolution DURATION|auto --from TIME --to TIME [--where KEY=VALUE]
-  telemetry metrics graph --config FILE --metric NAME --measurement NAME --resolution DURATION|auto --from TIME --to TIME [--type line|bar|stacked-bar] [--format terminal|svg|html]
+  telemetry metrics query --config FILE --metric NAME --measurement NAME --resolution DURATION|auto --from TIME --to TIME [--where KEY=VALUE] [--source auto|history|legacy]
+  telemetry metrics graph --config FILE --metric NAME --measurement NAME --resolution DURATION|auto --from TIME --to TIME [--type line|bar|stacked-bar] [--format terminal|svg|html] [--source auto|history|legacy]
   telemetry sqlite query --config FILE --sql SQL [--param NAME=VALUE] [--format table|json|jsonl|csv]
+  telemetry sqlite compact --config FILE [--format table|json]
   telemetry sqlite migration status --config FILE [--format table|json]
   telemetry sqlite migration up --config FILE [--format table|json]