@hienlh/ppm 0.23.5 → 0.23.7

package/src/services/compaction-savings.ts

@@ -0,0 +1,127 @@
+/**
+ * What each compaction in a transcript cost and saved.
+ *
+ * Compaction is the largest single thing that happens to a chat and the only one the
+ * transcript records without the UI ever saying so: a session silently loses most of
+ * its history, the next turn's prefix collapses, and the only visible trace is a long
+ * summary message. This reads the figures back out so the divider above that summary
+ * can state them.
+ *
+ * The link between a boundary and the message it introduces is `parentUuid`. Claude
+ * Code writes the `compact_boundary` record with `parentUuid: null` — it is the root
+ * of the post-compaction segment — and the summary immediately after it carries the
+ * boundary's uuid as its parent. Keying the result by that summary's uuid is what
+ * lets both message readers stamp it on without either of them knowing the layout,
+ * since `parseSessionMessage` already uses the record uuid as `ChatMessage.id`.
+ */
+
+import { existsSync } from "node:fs";
+import type { CompactionInfo } from "../types/chat.ts";
+import { readLines } from "./read-lines.ts";
+
+/**
+ * Cheap test for the only two lines per compaction worth parsing.
+ *
+ * A transcript reaches tens of megabytes and `JSON.parse` on every line of one is the
+ * dominant cost of reading it at all — but a compaction leaves two records in a file
+ * that holds thousands, so a substring check keeps this a scan rather than a parse.
+ */
+const BOUNDARY_MARKER = '"compact_boundary"';
+
+/**
+ * Map of compact-summary message uuid → that compaction's figures.
+ *
+ * Answers an empty map for a missing or unreadable file: a transcript that cannot be
+ * read still has messages worth showing, and a divider is not worth failing a history
+ * load over.
+ */
+export async function readCompactions(jsonlPath: string): Promise<Map<string, CompactionInfo>> {
+  const byMessageId = new Map<string, CompactionInfo>();
+  if (!existsSync(jsonlPath)) return byMessageId;
+
+  // The boundary is read one line before the summary that claims it, so it waits here
+  // until a record names it as parent. Held rather than assumed adjacent: the pairing
+  // is by uuid, so a record written between the two cannot mis-attach the figures.
+  let pending: { uuid: string; info: CompactionInfo } | null = null;
+
+  try {
+    for await (const line of readLines(jsonlPath)) {
+      if (line.includes(BOUNDARY_MARKER)) {
+        const info = parseBoundary(line);
+        pending = info ? { uuid: info.uuid, info: info.compaction } : null;
+        continue;
+      }
+      if (!pending) continue;
+      // Only lines that could be the child are parsed — the uuid of the boundary is a
+      // full uuid, so this substring test has no realistic false positive.
+      if (!line.includes(pending.uuid)) continue;
+      const child = parseChild(line);
+      if (child?.parentUuid === pending.uuid) {
+        byMessageId.set(child.uuid, pending.info);
+        pending = null;
+      }
+    }
+  } catch {
+    // Truncated or mid-write — keep whatever pairs were already complete.
+  }
+  return byMessageId;
+}
+
+/** The boundary's own uuid and figures, or null when the record is not one we can use. */
+function parseBoundary(line: string): { uuid: string; compaction: CompactionInfo } | null {
+  let entry: any;
+  try {
+    entry = JSON.parse(line);
+  } catch {
+    return null;
+  }
+  if (entry?.type !== "system" || entry.subtype !== "compact_boundary") return null;
+  const meta = entry.compactMetadata;
+  if (typeof entry.uuid !== "string" || !meta) return null;
+
+  const preTokens = numberOr(meta.preTokens, 0);
+  const postTokens = numberOr(meta.postTokens, 0);
+  // A boundary with no `preTokens` is a record shape this cannot describe — a zero
+  // saving would render as a confident "saved 0", which is worse than no divider.
+  if (preTokens <= 0) return null;
+
+  return {
+    uuid: entry.uuid,
+    compaction: {
+      trigger: meta.trigger === "manual" ? "manual" : "auto",
+      preTokens,
+      postTokens,
+      // Clamped: `postTokens` is absent on older records, and a summary that somehow
+      // measured larger than its input must not report a negative saving.
+      savedTokens: Math.max(0, preTokens - postTokens),
+      ...(typeof meta.durationMs === "number" ? { durationMs: meta.durationMs } : {}),
+    },
+  };
+}
+
+/** The uuid/parentUuid pair of a candidate child record. */
+function parseChild(line: string): { uuid: string; parentUuid: string } | null {
+  try {
+    const entry = JSON.parse(line);
+    if (typeof entry?.uuid !== "string" || typeof entry.parentUuid !== "string") return null;
+    return { uuid: entry.uuid, parentUuid: entry.parentUuid };
+  } catch {
+    return null;
+  }
+}
+
+function numberOr(value: unknown, fallback: number): number {
+  return typeof value === "number" && Number.isFinite(value) ? value : fallback;
+}
+
+/** Stamp each compaction onto the message that opens its segment. */
+export function applyCompactions(
+  messages: { id: string; compaction?: CompactionInfo }[],
+  byMessageId: Map<string, CompactionInfo>,
+): void {
+  if (byMessageId.size === 0) return;
+  for (const msg of messages) {
+    const info = byMessageId.get(msg.id);
+    if (info) msg.compaction = info;
+  }
+}

package/src/services/db.service.ts

@@ -8,7 +8,7 @@ import { backupDbSync } from "./db-backup/db-backup-sync.ts";
 import { CODEX_DEFAULT_MODEL } from "../types/config.ts";
 // Must equal the last `PRAGMA user_version` below: the pre-migration snapshot is skipped for
 // any database already at this version, so a stale value silently drops that backup.
-export const CURRENT_SCHEMA_VERSION = 51;
+export const CURRENT_SCHEMA_VERSION = 54;
 
 let db: Database | null = null;
 let dbProfile: string | null = null;
@@ -36,8 +36,7 @@ export function getDb(): Database {
   const ppmDir = getPpmDir();
   if (!existsSync(ppmDir)) mkdirSync(ppmDir, { recursive: true });
   db = new Database(getDbPath());
-  db.exec("PRAGMA journal_mode = WAL");
-  db.exec("PRAGMA foreign_keys = ON");
+  applyDbPragmas(db);
   backupBeforeMigrations(db);
   runMigrations(db);
   return db;
@@ -68,6 +67,26 @@ function backupBeforeMigrations(database: Database): void {
   }
 }
 
+/**
+ * The pragmas every PPM connection to this database needs.
+ *
+ * `busy_timeout` is the one with a story. SQLite's default is to fail a
+ * contended lock INSTANTLY — measured, 0ms to `SQLiteError: database is
+ * locked` — and PPM routinely has two processes on this file for a moment: a
+ * `systemctl restart` starts the new supervisor while the outgoing server is
+ * still committing and checkpointing, which for a 300MB database is over a
+ * second. That single throw arrived as an unhandled rejection inside
+ * supervisor startup, which then stopped before spawning the edge forwarder —
+ * so PPM ran with nothing listening on its public port. 5s rather than the
+ * query audit's 2s: one snapshot of this database measures ~1.2s, so the wait
+ * has to cover a checkpoint plus whatever writes queued behind it.
+ */
+export function applyDbPragmas(database: Database): void {
+  database.exec("PRAGMA journal_mode = WAL");
+  database.exec("PRAGMA busy_timeout = 5000");
+  database.exec("PRAGMA foreign_keys = ON");
+}
+
 /** Close the DB (for graceful shutdown or tests) */
 export function closeDb(): void {
   if (db) { db.close(); db = null; }
@@ -76,8 +95,7 @@ export function closeDb(): void {
 /** For tests: open an isolated in-memory DB with schema applied */
 export function openTestDb(): Database {
   const testDb = new Database(":memory:");
-  testDb.exec("PRAGMA journal_mode = WAL");
-  testDb.exec("PRAGMA foreign_keys = ON");
+  applyDbPragmas(testDb);
   runMigrations(testDb);
   return testDb;
 }
@@ -1131,10 +1149,33 @@ export function runMigrations(database: Database): void {
     database.exec(`PRAGMA user_version = 48;`);
   }
 
+  // This checkout's prompt-cache work first shipped as migrations 46-48 on turn_usage, before
+  // main used those numbers for accounts (it is 50-52 below now). A database that ran the local
+  // ones sits past 45 without main's columns, so the version gates above skip them for good and
+  // 49 then updates a column that does not exist. Added by schema rather than by version here;
+  // on a database that followed main every statement is a no-op.
+  if (current >= 46 && current < 49) {
+    for (const col of [
+      "granted_at INTEGER",
+      "refresh_expires_at INTEGER",
+      "reauth_required INTEGER NOT NULL DEFAULT 0",
+      "last_refresh_attempt_at INTEGER",
+    ]) {
+      try { database.exec(`ALTER TABLE accounts ADD COLUMN ${col}`); } catch { /* exists */ }
+    }
+    try { database.exec(`ALTER TABLE codex_accounts ADD COLUMN status TEXT NOT NULL DEFAULT 'active'`); } catch { /* exists */ }
+    try { database.exec(`ALTER TABLE codex_accounts ADD COLUMN daily_guard_enabled INTEGER NOT NULL DEFAULT 0`); } catch { /* exists */ }
+  }
+
   if (current < 49) {
     // Daily guard was introduced as an opt-in during development. Ship it enabled so every
     // weekly-only account starts paced; Plus accounts simply do not surface or use it.
-    database.exec(`UPDATE codex_accounts SET daily_guard_enabled = 1`);
+    //
+    // Guarded like every statement around it, and for the same reason: the ladder runs against
+    // databases that do not have this table yet. Unguarded, the throw escapes `runMigrations`
+    // and leaves the database half-migrated at whatever version the previous step wrote —
+    // there is no transaction around the ladder to undo it.
+    try { database.exec(`UPDATE codex_accounts SET daily_guard_enabled = 1`); } catch { /* no accounts table yet */ }
     database.exec(`PRAGMA user_version = 49;`);
   }
 
@@ -1155,6 +1196,33 @@ export function runMigrations(database: Database): void {
     try { database.exec("ALTER TABLE claude_limit_snapshots ADD COLUMN extra_json TEXT"); } catch { /* exists */ }
     database.exec(`PRAGMA user_version = 51;`);
   }
+
+  if (current < 52) {
+    // The idle-cache notice had to live entirely in server memory, because nothing on disk
+    // said when a session last cached or how much. `recorded_at` answers the first; this
+    // column answers the second with a figure the summed token columns cannot give —
+    // see `TurnUsage.contextTokens`. Null on every existing row: unmeasured, not zero.
+    try { database.exec("ALTER TABLE turn_usage ADD COLUMN context_tokens INTEGER"); } catch { /* column exists */ }
+    database.exec(`PRAGMA user_version = 52;`);
+  }
+
+  if (current < 53) {
+    // The cache window was inferred from the credential's shape — an `sk-ant-oat` prefix
+    // meaning a subscription and therefore an hour. The API states it outright in
+    // `usage.cache_creation`, and that answer has to outlive the process the way the rest of
+    // the last turn's facts now do. Null on existing rows: unreported, so the guess stands.
+    try { database.exec("ALTER TABLE turn_usage ADD COLUMN cache_ttl_ms INTEGER"); } catch { /* column exists */ }
+    database.exec(`PRAGMA user_version = 53;`);
+  }
+
+  if (current < 54) {
+    // A compaction makes the cached prefix inapplicable rather than stale, and the window in
+    // which that matters is between the compaction and the user's next message — a window a
+    // restart lands inside often enough to be worth surviving. Null on existing rows: no
+    // compaction is known, which is what every turn before this recorded anyway.
+    try { database.exec("ALTER TABLE turn_usage ADD COLUMN compacted_at INTEGER"); } catch { /* column exists */ }
+    database.exec(`PRAGMA user_version = 54;`);
+  }
 }
 
 // ---------------------------------------------------------------------------
@@ -1729,6 +1797,9 @@ export interface TurnUsageRow {
   cold_reason: string | null;
   account_id: string | null;
   account_label: string | null;
+  context_tokens: number | null;
+  cache_ttl_ms: number | null;
+  compacted_at: number | null;
   recorded_at: string;
 }
 
@@ -1745,13 +1816,16 @@ export function insertTurnUsage(record: {
   coldReason?: string;
   accountId?: string;
   accountLabel?: string;
+  contextTokens?: number;
+  cacheTtlMs?: number;
+  compactedAt?: number;
 }): void {
   getDb().query(
     `INSERT INTO turn_usage
        (session_id, model, input_tokens, output_tokens, cache_read_tokens,
         cache_write_tokens, context_window, cost_usd, cold_start, cold_reason,
-        account_id, account_label)
-     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+        account_id, account_label, context_tokens, cache_ttl_ms, compacted_at)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
   ).run(
     record.sessionId, record.model ?? null,
     record.inputTokens, record.outputTokens,
@@ -1759,9 +1833,50 @@ export function insertTurnUsage(record: {
     record.contextWindow ?? null, record.costUsd ?? null,
     record.coldStart ? 1 : 0, record.coldReason ?? null,
     record.accountId ?? null, record.accountLabel ?? null,
+    record.contextTokens ?? null, record.cacheTtlMs ?? null, record.compactedAt ?? null,
   );
 }
 
+/**
+ * When this session last wrote its prompt cache, and what it would cost to write again.
+ *
+ * The idle notice used to be answerable only from the live `SessionEntry`, which is dropped
+ * five minutes after the last tab disconnects — while the cache it describes lives for an
+ * hour. So the notice appeared only for a session left connected and went silent for exactly
+ * the session reopened the next morning, which is the one it exists for. These three columns
+ * are already written on every turn; reading them back is the whole fix.
+ */
+export function getLastTurnCacheState(sessionId: string): {
+  endedAtMs: number;
+  prefixTokens: number;
+  contextTokens?: number;
+  cacheTtlMs?: number;
+  compactedAt?: number;
+} | null {
+  const row = getDb().query(
+    `SELECT input_tokens, cache_read_tokens, cache_write_tokens, context_tokens, cache_ttl_ms, compacted_at, recorded_at
+       FROM turn_usage WHERE session_id = ? ORDER BY id DESC LIMIT 1`,
+  ).get(sessionId) as Pick<
+    TurnUsageRow,
+    "input_tokens" | "cache_read_tokens" | "cache_write_tokens" | "context_tokens" | "cache_ttl_ms" | "compacted_at" | "recorded_at"
+  > | null;
+  if (!row?.recorded_at) return null;
+
+  // `datetime('now')` is UTC and writes no zone suffix, which `Date` then reads as LOCAL
+  // time — seven hours adrift in Asia/Saigon, enough to make the idle window come out
+  // negative and silence the notice on precisely the machines it was rebuilt for.
+  const endedAtMs = Date.parse(`${row.recorded_at.replace(" ", "T")}Z`);
+  if (!Number.isFinite(endedAtMs)) return null;
+
+  return {
+    endedAtMs,
+    prefixTokens: row.input_tokens + row.cache_read_tokens + row.cache_write_tokens,
+    ...(row.context_tokens != null && { contextTokens: row.context_tokens }),
+    ...(row.cache_ttl_ms != null && { cacheTtlMs: row.cache_ttl_ms }),
+    ...(row.compacted_at != null && { compactedAt: row.compacted_at }),
+  };
+}
+
 /** Most recent turns first — the debug dialog shows the tail of a session. */
 export function listTurnUsage(sessionId: string, limit = 30): TurnUsageRow[] {
   return getDb().query(