@tokentop/ttop 0.2.2

package/src/storage/index.ts

@@ -0,0 +1,66 @@
+export { PATHS } from './paths.ts';
+
+export {
+  initDatabase,
+  getDatabase,
+  closeDatabase,
+  isDatabaseInitialized,
+  getAppRunId,
+} from './db.ts';
+
+export {
+  insertProviderSnapshot,
+  insertProviderSnapshotBatch,
+  queryProviderSnapshots,
+  getLatestProviderSnapshot,
+} from './repos/providerSnapshots.ts';
+
+export {
+  insertUsageEvent,
+  insertUsageEventBatch,
+  queryUsageTimeSeries,
+  queryProviderDailyCosts,
+  queryModelDailyCosts,
+  queryProjectDailyCosts,
+  calculateBurnRate,
+  getTotalUsageInWindow,
+  getSessionActivityTimeline,
+} from './repos/usageEvents.ts';
+
+export type { SessionActivityPoint, ProviderDailyCost, ModelDailyCost, ProjectDailyCost } from './repos/usageEvents.ts';
+
+export {
+  upsertAgentSession,
+  insertAgentSessionSnapshot,
+  getAgentSession,
+  getRecentSessions,
+  getSessionsByProject,
+  getLatestStreamTotalsForAllSessions,
+} from './repos/agentSessions.ts';
+
+export type { LatestStreamTotals } from './repos/agentSessions.ts';
+
+export type {
+  UsageEventSource,
+  SessionStatus,
+  CostSource,
+  PricingSource,
+  AppRunRow,
+  ProviderSnapshotRow,
+  ProviderSnapshotInsert,
+  AgentSessionDim,
+  AgentSessionUpsert,
+  AgentSessionSnapshotRow,
+  AgentSessionSnapshotInsert,
+  AgentSessionStreamSnapshotRow,
+  UsageEventRow,
+  UsageEventInsert,
+  HourlyAggregateRow,
+  DailyAggregateRow,
+  TimeSeriesPoint,
+  TimeSeriesFilters,
+  UsageQueryOptions,
+  StreamTotals,
+} from './types.ts';
+
+export { computeStreamDelta } from './types.ts';

package/src/storage/migrations/index.ts

@@ -0,0 +1,189 @@
+import type { Database } from 'bun:sqlite';
+
+export interface Migration {
+  version: number;
+  up: (db: Database) => void;
+}
+
+export const migrations: Migration[] = [
+  {
+    version: 1,
+    up: (db: Database) => {
+      db.exec(`
+        CREATE TABLE IF NOT EXISTS meta (
+          key TEXT PRIMARY KEY,
+          value TEXT NOT NULL
+        );
+
+        CREATE TABLE IF NOT EXISTS app_runs (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          started_at INTEGER NOT NULL,
+          ended_at INTEGER,
+          app_version TEXT,
+          refresh_interval_ms INTEGER,
+          pid INTEGER,
+          hostname TEXT
+        );
+
+        CREATE TABLE IF NOT EXISTS provider_snapshots (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          timestamp INTEGER NOT NULL,
+          provider TEXT NOT NULL,
+          used_percent REAL,
+          limit_reached INTEGER NOT NULL DEFAULT 0,
+          resets_at INTEGER,
+          tokens_input INTEGER,
+          tokens_output INTEGER,
+          cost_usd REAL,
+          cost_source TEXT,
+          raw_json TEXT,
+          created_at INTEGER NOT NULL DEFAULT (strftime('%s','now') * 1000)
+        );
+
+        CREATE INDEX IF NOT EXISTS idx_provider_snapshots_provider_ts
+          ON provider_snapshots(provider, timestamp);
+
+        CREATE INDEX IF NOT EXISTS idx_provider_snapshots_ts
+          ON provider_snapshots(timestamp);
+
+        CREATE TABLE IF NOT EXISTS agent_sessions (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          agent_id TEXT NOT NULL,
+          session_id TEXT NOT NULL,
+          project_path TEXT,
+          started_at INTEGER,
+          first_seen_at INTEGER NOT NULL,
+          last_seen_at INTEGER NOT NULL,
+          UNIQUE(agent_id, session_id)
+        );
+
+        CREATE INDEX IF NOT EXISTS idx_agent_sessions_last_seen
+          ON agent_sessions(last_seen_at);
+
+        CREATE INDEX IF NOT EXISTS idx_agent_sessions_project
+          ON agent_sessions(project_path);
+
+        CREATE TABLE IF NOT EXISTS agent_session_snapshots (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          timestamp INTEGER NOT NULL,
+          agent_session_id INTEGER NOT NULL REFERENCES agent_sessions(id) ON DELETE CASCADE,
+          last_activity_at INTEGER,
+          status TEXT,
+          total_input_tokens INTEGER NOT NULL DEFAULT 0,
+          total_output_tokens INTEGER NOT NULL DEFAULT 0,
+          total_cache_read_tokens INTEGER NOT NULL DEFAULT 0,
+          total_cache_write_tokens INTEGER NOT NULL DEFAULT 0,
+          total_cost_usd REAL NOT NULL DEFAULT 0,
+          request_count INTEGER NOT NULL DEFAULT 0
+        );
+
+        CREATE INDEX IF NOT EXISTS idx_agent_session_snapshots_session_ts
+          ON agent_session_snapshots(agent_session_id, timestamp);
+
+        CREATE INDEX IF NOT EXISTS idx_agent_session_snapshots_ts
+          ON agent_session_snapshots(timestamp);
+
+        CREATE TABLE IF NOT EXISTS agent_session_stream_snapshots (
+          agent_session_snapshot_id INTEGER NOT NULL REFERENCES agent_session_snapshots(id) ON DELETE CASCADE,
+          provider TEXT NOT NULL,
+          model TEXT NOT NULL,
+          input_tokens INTEGER NOT NULL DEFAULT 0,
+          output_tokens INTEGER NOT NULL DEFAULT 0,
+          cache_read_tokens INTEGER NOT NULL DEFAULT 0,
+          cache_write_tokens INTEGER NOT NULL DEFAULT 0,
+          cost_usd REAL NOT NULL DEFAULT 0,
+          request_count INTEGER NOT NULL DEFAULT 0,
+          pricing_source TEXT,
+          PRIMARY KEY (agent_session_snapshot_id, provider, model)
+        );
+
+        CREATE INDEX IF NOT EXISTS idx_agent_stream_provider_model
+          ON agent_session_stream_snapshots(provider, model);
+
+        CREATE TABLE IF NOT EXISTS usage_events (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          timestamp INTEGER NOT NULL,
+          source TEXT NOT NULL,
+          provider TEXT,
+          model TEXT,
+          agent_id TEXT,
+          session_id TEXT,
+          project_path TEXT,
+          input_tokens INTEGER NOT NULL DEFAULT 0,
+          output_tokens INTEGER NOT NULL DEFAULT 0,
+          cache_read_tokens INTEGER NOT NULL DEFAULT 0,
+          cache_write_tokens INTEGER NOT NULL DEFAULT 0,
+          cost_usd REAL NOT NULL DEFAULT 0,
+          request_count INTEGER NOT NULL DEFAULT 0,
+          pricing_source TEXT,
+          created_at INTEGER NOT NULL DEFAULT (strftime('%s','now') * 1000)
+        );
+
+        CREATE INDEX IF NOT EXISTS idx_usage_events_ts ON usage_events(timestamp);
+        CREATE INDEX IF NOT EXISTS idx_usage_events_provider_ts ON usage_events(provider, timestamp);
+        CREATE INDEX IF NOT EXISTS idx_usage_events_provider_model_ts ON usage_events(provider, model, timestamp);
+        CREATE INDEX IF NOT EXISTS idx_usage_events_agent_ts ON usage_events(agent_id, timestamp);
+        CREATE INDEX IF NOT EXISTS idx_usage_events_session_ts ON usage_events(session_id, timestamp);
+        CREATE INDEX IF NOT EXISTS idx_usage_events_project_ts ON usage_events(project_path, timestamp);
+
+        CREATE TABLE IF NOT EXISTS hourly_aggregates (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          bucket_start INTEGER NOT NULL,
+          provider TEXT NOT NULL DEFAULT '',
+          model TEXT NOT NULL DEFAULT '',
+          agent_id TEXT NOT NULL DEFAULT '',
+          project_path TEXT NOT NULL DEFAULT '',
+          input_tokens INTEGER NOT NULL DEFAULT 0,
+          output_tokens INTEGER NOT NULL DEFAULT 0,
+          cache_read_tokens INTEGER NOT NULL DEFAULT 0,
+          cache_write_tokens INTEGER NOT NULL DEFAULT 0,
+          cost_usd REAL NOT NULL DEFAULT 0,
+          request_count INTEGER NOT NULL DEFAULT 0,
+          UNIQUE (bucket_start, provider, model, agent_id, project_path)
+        );
+
+        CREATE INDEX IF NOT EXISTS idx_hourly_aggregates_bucket ON hourly_aggregates(bucket_start);
+
+        CREATE TABLE IF NOT EXISTS daily_aggregates (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          date TEXT NOT NULL,
+          provider TEXT NOT NULL DEFAULT '',
+          model TEXT NOT NULL DEFAULT '',
+          agent_id TEXT NOT NULL DEFAULT '',
+          project_path TEXT NOT NULL DEFAULT '',
+          input_tokens INTEGER NOT NULL DEFAULT 0,
+          output_tokens INTEGER NOT NULL DEFAULT 0,
+          cache_read_tokens INTEGER NOT NULL DEFAULT 0,
+          cache_write_tokens INTEGER NOT NULL DEFAULT 0,
+          cost_usd REAL NOT NULL DEFAULT 0,
+          request_count INTEGER NOT NULL DEFAULT 0,
+          UNIQUE (date, provider, model, agent_id, project_path)
+        );
+
+        CREATE INDEX IF NOT EXISTS idx_daily_aggregates_date ON daily_aggregates(date);
+      `);
+    },
+  },
+];
+
+export function getCurrentVersion(db: Database): number {
+  const result = db.prepare('PRAGMA user_version').get() as { user_version: number } | null;
+  return result?.user_version ?? 0;
+}
+
+export function setVersion(db: Database, version: number): void {
+  db.exec(`PRAGMA user_version = ${version}`);
+}
+
+export function applyMigrations(db: Database): void {
+  const currentVersion = getCurrentVersion(db);
+
+  for (const migration of migrations) {
+    if (migration.version > currentVersion) {
+      db.transaction(() => {
+        migration.up(db);
+        setVersion(db, migration.version);
+      })();
+    }
+  }
+}

package/src/storage/paths.ts

@@ -0,0 +1,24 @@
+import * as path from 'path';
+import * as os from 'os';
+
+const HOME = os.homedir();
+
+export const PATHS = {
+  config: {
+    dir: path.join(HOME, '.config/tokentop'),
+    file: path.join(HOME, '.config/tokentop/config.json'),
+    fileJsonc: path.join(HOME, '.config/tokentop/config.jsonc'),
+    plugins: path.join(HOME, '.config/tokentop/plugins'),
+  },
+  data: {
+    dir: path.join(HOME, '.local/share/tokentop'),
+    database: path.join(HOME, '.local/share/tokentop/usage.db'),
+    sessions: path.join(HOME, '.local/share/tokentop/sessions'),
+    cache: path.join(HOME, '.local/share/tokentop/cache'),
+    logs: path.join(HOME, '.local/share/tokentop/logs'),
+  },
+  cache: {
+    dir: path.join(HOME, '.cache/tokentop'),
+    nodeModules: path.join(HOME, '.cache/tokentop/node_modules'),
+  },
+} as const;

package/src/storage/repos/agentSessions.ts

@@ -0,0 +1,210 @@
+import { getDatabase } from '../db.ts';
+import type {
+  AgentSessionDim,
+  AgentSessionUpsert,
+  AgentSessionSnapshotInsert,
+  AgentSessionStreamSnapshotRow,
+} from '../types.ts';
+
+const UPSERT_SESSION_SQL = `
+  INSERT INTO agent_sessions (agent_id, session_id, project_path, started_at, first_seen_at, last_seen_at)
+  VALUES (?, ?, ?, ?, ?, ?)
+  ON CONFLICT(agent_id, session_id) DO UPDATE SET
+    project_path = COALESCE(excluded.project_path, agent_sessions.project_path),
+    started_at = COALESCE(excluded.started_at, agent_sessions.started_at),
+    last_seen_at = excluded.last_seen_at
+  RETURNING id
+`;
+
+const INSERT_SNAPSHOT_SQL = `
+  INSERT INTO agent_session_snapshots (
+    timestamp, agent_session_id, last_activity_at, status,
+    total_input_tokens, total_output_tokens, total_cache_read_tokens, total_cache_write_tokens,
+    total_cost_usd, request_count
+  ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+`;
+
+const INSERT_STREAM_SQL = `
+  INSERT INTO agent_session_stream_snapshots (
+    agent_session_snapshot_id, provider, model,
+    input_tokens, output_tokens, cache_read_tokens, cache_write_tokens,
+    cost_usd, request_count, pricing_source
+  ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+`;
+
+export function upsertAgentSession(session: AgentSessionUpsert): number {
+  const db = getDatabase();
+  const now = Date.now();
+
+  const row = db.prepare(UPSERT_SESSION_SQL).get(
+    session.agentId,
+    session.sessionId,
+    session.projectPath ?? null,
+    session.startedAt ?? null,
+    session.firstSeenAt ?? now,
+    session.lastSeenAt
+  ) as { id: number };
+
+  return row.id;
+}
+
+export function insertAgentSessionSnapshot(
+  snapshot: AgentSessionSnapshotInsert,
+  streams: Omit<AgentSessionStreamSnapshotRow, 'agentSessionSnapshotId'>[]
+): number {
+  const db = getDatabase();
+
+  const insertSnapshot = db.prepare(INSERT_SNAPSHOT_SQL);
+  const insertStream = db.prepare(INSERT_STREAM_SQL);
+
+  let snapshotId = 0;
+
+  db.transaction(() => {
+    const result = insertSnapshot.run(
+      snapshot.timestamp,
+      snapshot.agentSessionId,
+      snapshot.lastActivityAt ?? null,
+      snapshot.status ?? null,
+      snapshot.totalInputTokens,
+      snapshot.totalOutputTokens,
+      snapshot.totalCacheReadTokens,
+      snapshot.totalCacheWriteTokens,
+      snapshot.totalCostUsd,
+      snapshot.requestCount
+    );
+
+    snapshotId = Number(result.lastInsertRowid);
+
+    for (const stream of streams) {
+      insertStream.run(
+        snapshotId,
+        stream.provider,
+        stream.model,
+        stream.inputTokens,
+        stream.outputTokens,
+        stream.cacheReadTokens,
+        stream.cacheWriteTokens,
+        stream.costUsd,
+        stream.requestCount,
+        stream.pricingSource ?? null
+      );
+    }
+  })();
+
+  return snapshotId;
+}
+
+export function getAgentSession(agentId: string, sessionId: string): AgentSessionDim | null {
+  const db = getDatabase();
+
+  const row = db.prepare(`
+    SELECT * FROM agent_sessions WHERE agent_id = ? AND session_id = ?
+  `).get(agentId, sessionId) as DbSessionRow | null;
+
+  return row ? mapSessionRow(row) : null;
+}
+
+export function getRecentSessions(limit = 50): AgentSessionDim[] {
+  const db = getDatabase();
+
+  const rows = db.prepare(`
+    SELECT * FROM agent_sessions
+    ORDER BY last_seen_at DESC
+    LIMIT ?
+  `).all(limit) as DbSessionRow[];
+
+  return rows.map(mapSessionRow);
+}
+
+export function getSessionsByProject(projectPath: string, limit = 50): AgentSessionDim[] {
+  const db = getDatabase();
+
+  const rows = db.prepare(`
+    SELECT * FROM agent_sessions
+    WHERE project_path = ?
+    ORDER BY last_seen_at DESC
+    LIMIT ?
+  `).all(projectPath, limit) as DbSessionRow[];
+
+  return rows.map(mapSessionRow);
+}
+
+interface DbSessionRow {
+  id: number;
+  agent_id: string;
+  session_id: string;
+  project_path: string | null;
+  started_at: number | null;
+  first_seen_at: number;
+  last_seen_at: number;
+}
+
+function mapSessionRow(row: DbSessionRow): AgentSessionDim {
+  return {
+    id: row.id,
+    agentId: row.agent_id,
+    sessionId: row.session_id,
+    projectPath: row.project_path,
+    startedAt: row.started_at,
+    firstSeenAt: row.first_seen_at,
+    lastSeenAt: row.last_seen_at,
+  };
+}
+
+export interface LatestStreamTotals {
+  agentId: string;
+  sessionId: string;
+  provider: string;
+  model: string;
+  inputTokens: number;
+  outputTokens: number;
+  cacheReadTokens: number;
+  cacheWriteTokens: number;
+  costUsd: number;
+  requestCount: number;
+}
+
+export function getLatestStreamTotalsForAllSessions(): LatestStreamTotals[] {
+  const db = getDatabase();
+
+  const rows = db.prepare(`
+    SELECT
+      s.agent_id, s.session_id,
+      ss.provider, ss.model,
+      ss.input_tokens, ss.output_tokens,
+      ss.cache_read_tokens, ss.cache_write_tokens,
+      ss.cost_usd, ss.request_count
+    FROM agent_session_stream_snapshots ss
+    JOIN agent_session_snapshots snap ON snap.id = ss.agent_session_snapshot_id
+    JOIN agent_sessions s ON s.id = snap.agent_session_id
+    WHERE snap.id IN (
+      SELECT MAX(snap2.id)
+      FROM agent_session_snapshots snap2
+      GROUP BY snap2.agent_session_id
+    )
+  `).all() as Array<{
+    agent_id: string;
+    session_id: string;
+    provider: string;
+    model: string;
+    input_tokens: number;
+    output_tokens: number;
+    cache_read_tokens: number;
+    cache_write_tokens: number;
+    cost_usd: number;
+    request_count: number;
+  }>;
+
+  return rows.map(r => ({
+    agentId: r.agent_id,
+    sessionId: r.session_id,
+    provider: r.provider,
+    model: r.model,
+    inputTokens: r.input_tokens,
+    outputTokens: r.output_tokens,
+    cacheReadTokens: r.cache_read_tokens,
+    cacheWriteTokens: r.cache_write_tokens,
+    costUsd: r.cost_usd,
+    requestCount: r.request_count,
+  }));
+}

package/src/storage/repos/providerSnapshots.ts

@@ -0,0 +1,116 @@
+import { getDatabase } from '../db.ts';
+import type { ProviderSnapshotInsert, ProviderSnapshotRow, CostSource } from '../types.ts';
+
+const INSERT_SQL = `
+  INSERT INTO provider_snapshots (
+    timestamp, provider, used_percent, limit_reached, resets_at,
+    tokens_input, tokens_output, cost_usd, cost_source, raw_json
+  ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+`;
+
+const QUERY_BY_PROVIDER_SQL = `
+  SELECT * FROM provider_snapshots
+  WHERE provider = ? AND timestamp >= ? AND timestamp <= ?
+  ORDER BY timestamp DESC
+  LIMIT ?
+`;
+
+const QUERY_LATEST_SQL = `
+  SELECT * FROM provider_snapshots
+  WHERE provider = ?
+  ORDER BY timestamp DESC
+  LIMIT 1
+`;
+
+export function insertProviderSnapshot(snapshot: ProviderSnapshotInsert): number {
+  const db = getDatabase();
+  const stmt = db.prepare(INSERT_SQL);
+
+  const result = stmt.run(
+    snapshot.timestamp,
+    snapshot.provider,
+    snapshot.usedPercent ?? null,
+    snapshot.limitReached ? 1 : 0,
+    snapshot.resetsAt ?? null,
+    snapshot.tokensInput ?? null,
+    snapshot.tokensOutput ?? null,
+    snapshot.costUsd ?? null,
+    snapshot.costSource ?? null,
+    snapshot.rawJson ?? null
+  );
+
+  return Number(result.lastInsertRowid);
+}
+
+export function insertProviderSnapshotBatch(snapshots: ProviderSnapshotInsert[]): void {
+  if (snapshots.length === 0) return;
+
+  const db = getDatabase();
+  const stmt = db.prepare(INSERT_SQL);
+
+  const insertMany = db.transaction((items: ProviderSnapshotInsert[]) => {
+    for (const s of items) {
+      stmt.run(
+        s.timestamp,
+        s.provider,
+        s.usedPercent ?? null,
+        s.limitReached ? 1 : 0,
+        s.resetsAt ?? null,
+        s.tokensInput ?? null,
+        s.tokensOutput ?? null,
+        s.costUsd ?? null,
+        s.costSource ?? null,
+        s.rawJson ?? null
+      );
+    }
+  });
+
+  insertMany(snapshots);
+}
+
+export function queryProviderSnapshots(
+  provider: string,
+  startMs: number,
+  endMs: number,
+  limit = 100
+): ProviderSnapshotRow[] {
+  const db = getDatabase();
+  const rows = db.prepare(QUERY_BY_PROVIDER_SQL).all(provider, startMs, endMs, limit) as DbRow[];
+  return rows.map(mapRow);
+}
+
+export function getLatestProviderSnapshot(provider: string): ProviderSnapshotRow | null {
+  const db = getDatabase();
+  const row = db.prepare(QUERY_LATEST_SQL).get(provider) as DbRow | null;
+  return row ? mapRow(row) : null;
+}
+
+interface DbRow {
+  id: number;
+  timestamp: number;
+  provider: string;
+  used_percent: number | null;
+  limit_reached: number;
+  resets_at: number | null;
+  tokens_input: number | null;
+  tokens_output: number | null;
+  cost_usd: number | null;
+  cost_source: string | null;
+  raw_json: string | null;
+}
+
+function mapRow(row: DbRow): ProviderSnapshotRow {
+  return {
+    id: row.id,
+    timestamp: row.timestamp,
+    provider: row.provider,
+    usedPercent: row.used_percent,
+    limitReached: row.limit_reached === 1,
+    resetsAt: row.resets_at,
+    tokensInput: row.tokens_input,
+    tokensOutput: row.tokens_output,
+    costUsd: row.cost_usd,
+    costSource: row.cost_source as CostSource | null,
+    rawJson: row.raw_json,
+  };
+}