@tokentop/ttop 0.2.2 → 0.4.0

package/src/storage/repos/usageEvents.ts

@@ -1,5 +1,5 @@
-import { getDatabase } from '../db.ts';
-import type { UsageEventInsert, TimeSeriesPoint, TimeSeriesFilters } from '../types.ts';
+import { getDatabase } from "../db.ts";
+import type { TimeSeriesFilters, TimeSeriesPoint, UsageEventInsert } from "../types.ts";
 
 const INSERT_SQL = `
   INSERT INTO usage_events (
@@ -27,7 +27,7 @@ export function insertUsageEvent(event: UsageEventInsert): number {
     event.cacheWriteTokens,
     event.costUsd,
     event.requestCount,
-    event.pricingSource ?? null
+    event.pricingSource ?? null,
   );
 
   return Number(result.lastInsertRowid);
@@ -55,7 +55,7 @@ export function insertUsageEventBatch(events: UsageEventInsert[]): void {
         e.cacheWriteTokens,
         e.costUsd,
         e.requestCount,
-        e.pricingSource ?? null
+        e.pricingSource ?? null,
       );
     }
   });
@@ -67,31 +67,31 @@ export function queryUsageTimeSeries(
   startMs: number,
   endMs: number,
   bucketMs: number,
-  filters: TimeSeriesFilters = {}
+  filters: TimeSeriesFilters = {},
 ): TimeSeriesPoint[] {
   const db = getDatabase();
 
-  const conditions: string[] = ['timestamp >= ?', 'timestamp <= ?'];
+  const conditions: string[] = ["timestamp >= ?", "timestamp <= ?"];
   const params: (string | number)[] = [startMs, endMs];
 
   if (filters.provider) {
-    conditions.push('provider = ?');
+    conditions.push("provider = ?");
     params.push(filters.provider);
   }
   if (filters.model) {
-    conditions.push('model = ?');
+    conditions.push("model = ?");
     params.push(filters.model);
   }
   if (filters.agentId) {
-    conditions.push('agent_id = ?');
+    conditions.push("agent_id = ?");
     params.push(filters.agentId);
   }
   if (filters.sessionId) {
-    conditions.push('session_id = ?');
+    conditions.push("session_id = ?");
     params.push(filters.sessionId);
   }
   if (filters.projectPath) {
-    conditions.push('project_path = ?');
+    conditions.push("project_path = ?");
     params.push(filters.projectPath);
   }
 
@@ -102,7 +102,7 @@ export function queryUsageTimeSeries(
       COALESCE(SUM(cost_usd), 0) AS cost_usd,
       COALESCE(SUM(request_count), 0) AS request_count
     FROM usage_events
-    WHERE ${conditions.join(' AND ')}
+    WHERE ${conditions.join(" AND ")}
     GROUP BY bucket_start
     ORDER BY bucket_start ASC
   `;
@@ -114,7 +114,7 @@ export function queryUsageTimeSeries(
     request_count: number;
   }>;
 
-  return rows.map(r => ({
+  return rows.map((r) => ({
     bucketStart: r.bucket_start,
     tokens: r.tokens,
     costUsd: r.cost_usd,
@@ -122,17 +122,22 @@ export function queryUsageTimeSeries(
   }));
 }
 
-export function calculateBurnRate(windowMs: number, nowMs = Date.now()): { costPerHour: number; tokensPerMinute: number } {
+export function calculateBurnRate(
+  windowMs: number,
+  nowMs = Date.now(),
+): { costPerHour: number; tokensPerMinute: number } {
   const db = getDatabase();
   const startMs = nowMs - windowMs;
 
-  const row = db.prepare(`
+  const row = db
+    .prepare(`
     SELECT
       COALESCE(SUM(cost_usd), 0) AS cost,
       COALESCE(SUM(input_tokens + output_tokens), 0) AS tokens
     FROM usage_events
     WHERE timestamp >= ? AND timestamp <= ?
-  `).get(startMs, nowMs) as { cost: number; tokens: number };
+  `)
+    .get(startMs, nowMs) as { cost: number; tokens: number };
 
   const hours = windowMs / (1000 * 60 * 60);
   const minutes = windowMs / (1000 * 60);
@@ -143,7 +148,10 @@ export function calculateBurnRate(windowMs: number, nowMs = Date.now()): { costP
   };
 }
 
-export function getTotalUsageInWindow(startMs: number, endMs: number): {
+export function getTotalUsageInWindow(
+  startMs: number,
+  endMs: number,
+): {
   inputTokens: number;
   outputTokens: number;
   costUsd: number;
@@ -151,7 +159,8 @@ export function getTotalUsageInWindow(startMs: number, endMs: number): {
 } {
   const db = getDatabase();
 
-  const row = db.prepare(`
+  const row = db
+    .prepare(`
     SELECT
       COALESCE(SUM(input_tokens), 0) AS input_tokens,
       COALESCE(SUM(output_tokens), 0) AS output_tokens,
@@ -159,7 +168,8 @@ export function getTotalUsageInWindow(startMs: number, endMs: number): {
       COALESCE(SUM(request_count), 0) AS request_count
     FROM usage_events
     WHERE timestamp >= ? AND timestamp <= ?
-  `).get(startMs, endMs) as {
+  `)
+    .get(startMs, endMs) as {
     input_tokens: number;
     output_tokens: number;
     cost_usd: number;
@@ -190,7 +200,7 @@ export interface ProviderDailyCost {
 export function queryProviderDailyCosts(
   startMs: number,
   endMs: number,
-  bucketMs: number
+  bucketMs: number,
 ): ProviderDailyCost[] {
   const db = getDatabase();
 
@@ -216,7 +226,7 @@ export function queryProviderDailyCosts(
     request_count: number;
   }>;
 
-  return rows.map(r => ({
+  return rows.map((r) => ({
     provider: r.provider,
     bucketStart: r.bucket_start,
     costUsd: r.cost_usd,
@@ -236,7 +246,7 @@ export interface ModelDailyCost {
 export function queryModelDailyCosts(
   startMs: number,
   endMs: number,
-  bucketMs: number
+  bucketMs: number,
 ): ModelDailyCost[] {
   const db = getDatabase();
 
@@ -262,7 +272,7 @@ export function queryModelDailyCosts(
     request_count: number;
   }>;
 
-  return rows.map(r => ({
+  return rows.map((r) => ({
     model: r.model,
     bucketStart: r.bucket_start,
     costUsd: r.cost_usd,
@@ -282,7 +292,7 @@ export interface ProjectDailyCost {
 export function queryProjectDailyCosts(
   startMs: number,
   endMs: number,
-  bucketMs: number
+  bucketMs: number,
 ): ProjectDailyCost[] {
   const db = getDatabase();
 
@@ -308,7 +318,7 @@ export function queryProjectDailyCosts(
     request_count: number;
   }>;
 
-  return rows.map(r => ({
+  return rows.map((r) => ({
     projectPath: r.project_path,
     bucketStart: r.bucket_start,
     costUsd: r.cost_usd,
@@ -320,14 +330,16 @@ export function queryProjectDailyCosts(
 export function getSessionActivityTimeline(sessionId: string): SessionActivityPoint[] {
   const db = getDatabase();
 
-  const rows = db.prepare(`
+  const rows = db
+    .prepare(`
     SELECT timestamp, (input_tokens + output_tokens) AS tokens
     FROM usage_events
     WHERE session_id = ?
     ORDER BY timestamp ASC
-  `).all(sessionId) as Array<{ timestamp: number; tokens: number }>;
+  `)
+    .all(sessionId) as Array<{ timestamp: number; tokens: number }>;
 
-  return rows.map(r => ({
+  return rows.map((r) => ({
     timestamp: r.timestamp,
     tokens: r.tokens,
   }));

package/src/storage/retention.ts

@@ -0,0 +1,146 @@
+/**
+ * Data retention policy — prevents unbounded database growth.
+ *
+ * Detailed snapshot data (agent_session_snapshots, agent_session_stream_snapshots,
+ * provider_snapshots) is pruned after a retention window. Aggregated data in
+ * usage_events, hourly_aggregates, and daily_aggregates is kept longer since
+ * those tables grow much more slowly.
+ */
+
+import { getDatabase, isDatabaseInitialized } from "./db.ts";
+
+/** Keep detailed snapshots for 7 days */
+const SNAPSHOT_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
+
+/** Keep provider snapshots for 7 days */
+const PROVIDER_SNAPSHOT_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
+
+/** Keep usage events for 90 days */
+const USAGE_EVENT_RETENTION_MS = 90 * 24 * 60 * 60 * 1000;
+
+/** Maximum rows to delete in a single batch (prevents long-running transactions) */
+const PRUNE_BATCH_SIZE = 10_000;
+
+export interface PruneResult {
+  agentSessionSnapshots: number;
+  providerSnapshots: number;
+  usageEvents: number;
+  durationMs: number;
+}
+
+/**
+ * Prune old data from the database. Deletes in batches to avoid
+ * holding a write lock for too long.
+ */
+export function pruneOldData(nowMs: number = Date.now()): PruneResult {
+  if (!isDatabaseInitialized()) {
+    return { agentSessionSnapshots: 0, providerSnapshots: 0, usageEvents: 0, durationMs: 0 };
+  }
+
+  const db = getDatabase();
+  const start = performance.now();
+
+  let agentSessionSnapshots = 0;
+  let providerSnapshots = 0;
+  let usageEvents = 0;
+
+  const snapshotCutoff = nowMs - SNAPSHOT_RETENTION_MS;
+  const providerCutoff = nowMs - PROVIDER_SNAPSHOT_RETENTION_MS;
+  const usageEventCutoff = nowMs - USAGE_EVENT_RETENTION_MS;
+
+  // Prune agent_session_snapshots in batches.
+  // CASCADE delete handles agent_session_stream_snapshots automatically.
+  {
+    const stmt = db.prepare(`
+      DELETE FROM agent_session_snapshots
+      WHERE id IN (
+        SELECT id FROM agent_session_snapshots
+        WHERE timestamp < ?
+        LIMIT ?
+      )
+    `);
+
+    let deleted: number;
+    do {
+      const result = stmt.run(snapshotCutoff, PRUNE_BATCH_SIZE);
+      deleted = result.changes;
+      agentSessionSnapshots += deleted;
+    } while (deleted === PRUNE_BATCH_SIZE);
+  }
+
+  // Prune old provider_snapshots in batches
+  {
+    const stmt = db.prepare(`
+      DELETE FROM provider_snapshots
+      WHERE id IN (
+        SELECT id FROM provider_snapshots
+        WHERE timestamp < ?
+        LIMIT ?
+      )
+    `);
+
+    let deleted: number;
+    do {
+      const result = stmt.run(providerCutoff, PRUNE_BATCH_SIZE);
+      deleted = result.changes;
+      providerSnapshots += deleted;
+    } while (deleted === PRUNE_BATCH_SIZE);
+  }
+
+  // Prune old usage_events in batches
+  {
+    const stmt = db.prepare(`
+      DELETE FROM usage_events
+      WHERE id IN (
+        SELECT id FROM usage_events
+        WHERE timestamp < ?
+        LIMIT ?
+      )
+    `);
+
+    let deleted: number;
+    do {
+      const result = stmt.run(usageEventCutoff, PRUNE_BATCH_SIZE);
+      deleted = result.changes;
+      usageEvents += deleted;
+    } while (deleted === PRUNE_BATCH_SIZE);
+  }
+
+  const durationMs = Math.round(performance.now() - start);
+
+  return { agentSessionSnapshots, providerSnapshots, usageEvents, durationMs };
+}
+
+/**
+ * Run incremental auto_vacuum if supported (WAL mode).
+ * This reclaims free pages without blocking readers.
+ */
+export function incrementalVacuum(pages = 1000): void {
+  if (!isDatabaseInitialized()) return;
+
+  const db = getDatabase();
+  try {
+    db.exec(`PRAGMA incremental_vacuum(${pages})`);
+  } catch {
+    // Not all builds support incremental vacuum — silently skip
+  }
+}
+
+/**
+ * Get the database file size in bytes. Returns 0 if unable to determine.
+ */
+export function getDatabaseSizeBytes(): number {
+  if (!isDatabaseInitialized()) return 0;
+
+  const db = getDatabase();
+  try {
+    const row = db.prepare("PRAGMA page_count").get() as { page_count: number } | null;
+    const sizeRow = db.prepare("PRAGMA page_size").get() as { page_size: number } | null;
+    if (row && sizeRow) {
+      return row.page_count * sizeRow.page_size;
+    }
+  } catch {
+    // ignore
+  }
+  return 0;
+}

package/src/storage/types.ts

@@ -7,10 +7,10 @@
 // Enums & Literal Types
 // ============================================================================
 
-export type UsageEventSource = 'agent' | 'provider';
-export type SessionStatus = 'active' | 'idle';
-export type CostSource = 'api' | 'estimated';
-export type PricingSource = 'models.dev' | 'fallback' | 'unknown';
+export type UsageEventSource = "agent" | "provider";
+export type SessionStatus = "active" | "idle";
+export type CostSource = "api" | "estimated";
+export type PricingSource = "models.dev" | "fallback" | "unknown";
 
 // ============================================================================
 // App Runs Table
@@ -253,7 +253,7 @@ export interface StreamTotals {
  */
 export function computeStreamDelta(
   current: StreamTotals,
-  previous: StreamTotals | undefined
+  previous: StreamTotals | undefined,
 ): StreamTotals | null {
   if (!previous) {
     // First observation - treat as a delta from zero