@bli-cockpit/cli 0.2.150 → 0.2.155

package/dist/raw-evidence-gc.js

@@ -10,27 +10,33 @@ const RAW_EVIDENCE_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
  * observed on the fleet is minutes, not hours.
  */
 const STAGING_ORPHAN_MS = 6 * 60 * 60 * 1000;
-// GC hashes every file in every old-but-kept dir to confirm uploads. Running
-// that on each 15-min sync would re-read the same gigabytes ~96x/day on
-// machines with unconfirmed evidence, so GC is throttled to once per day.
-const GC_MIN_INTERVAL_MS = 24 * 60 * 60 * 1000;
+/**
+ * The retired once-a-day throttle marker (BLI-11106). The GC runs every tick
+ * now, and deletes this file on sight so it stops being state nobody reads.
+ *
+ * Why every tick is affordable: the GC used to hash every file of every old
+ * pack to confirm it was uploaded, which re-read gigabytes on a machine with
+ * unconfirmed evidence. It now asks the pack's own `manifest.json` first, and a
+ * pack whose manifest names ANY hash the ledger has not committed is skipped
+ * without reading a payload byte. Only a pack the manifest says is fully
+ * committed is hashed, once, on the way to being deleted.
+ */
+export const LEGACY_GC_THROTTLE_MARKER = ".last-raw-evidence-gc";
 export async function runRawEvidenceLocalGc(paths, env = process.env, now = new Date()) {
     if (env["COCKPIT_DISABLE_GC"] === "1") {
         return { skipped: true, removed_dirs: 0, freed_bytes: 0 };
     }
-    const throttleMarker = path.join(paths.state_dir, ".last-raw-evidence-gc");
-    const lastRun = await fs.stat(throttleMarker).catch(() => null);
-    if (lastRun && now.getTime() - lastRun.mtimeMs < GC_MIN_INTERVAL_MS) {
-        return { skipped: true, removed_dirs: 0, freed_bytes: 0 };
-    }
-    await fs.mkdir(paths.state_dir, { recursive: true }).catch(() => undefined);
-    await fs.writeFile(throttleMarker, now.toISOString()).catch(() => undefined);
+    await fs
+        .rm(path.join(paths.state_dir, LEGACY_GC_THROTTLE_MARKER), { force: true })
+        .catch(() => undefined);
     const cursor = await readRawEvidenceCursor(paths);
     const uploadedHashes = new Set(Object.keys(cursor.objects));
     const rawEvidenceRoot = path.join(paths.state_dir, "raw-evidence");
     const entries = await fs.readdir(rawEvidenceRoot, { withFileTypes: true }).catch(() => []);
     let removedDirs = 0;
     let freedBytes = 0;
+    let scannedDirs = 0;
+    let skippedByManifest = 0;
     for (const entry of entries) {
         if (!entry.isDirectory() || !entry.name.startsWith("work-"))
             continue;
@@ -40,6 +46,11 @@ export async function runRawEvidenceLocalGc(paths, env = process.env, now = new
             continue;
         if (now.getTime() - dirStats.mtimeMs < RAW_EVIDENCE_RETENTION_MS)
             continue;
+        scannedDirs += 1;
+        if (await manifestNamesUncommitted(dir, uploadedHashes)) {
+            skippedByManifest += 1;
+            continue;
+        }
         const inspection = await inspectRawEvidenceDirForGc(dir, uploadedHashes);
         if (!inspection.deletable)
             continue;
@@ -50,12 +61,18 @@ export async function runRawEvidenceLocalGc(paths, env = process.env, now = new
         freedBytes += inspection.byteSize;
     }
     // Log capping used to live here, and that is why it did not work: this GC
-    // is throttled to once a day and skipped entirely by COCKPIT_DISABLE_GC,
+    // was throttled to once a day and is skipped entirely by COCKPIT_DISABLE_GC,
     // while sync.log grows ~8 MB an hour on an active machine (measured on the
     // reference Mac, 2026-09-04: 219,866,695 bytes with the last GC 19 hours
-    // earlier). A daily truncate-to-zero of a file that reaches ~190 MB a day is
-    // not a cap, and sync.err.log had no cap at all. One owner now, running in
-    // the tick preamble: log-rotation.ts (BLI-3553).
+    // earlier). One owner now, running in the tick preamble: log-rotation.ts
+    // (BLI-3553).
+    console.error("[raw-evidence] gc sweep", JSON.stringify({
+        reason: removedDirs > 0 ? "gc_removed" : "nothing_to_prune",
+        scanned_dirs: scannedDirs,
+        skipped_by_manifest: skippedByManifest,
+        removed_dirs: removedDirs,
+        freed_bytes: freedBytes,
+    }));
     return {
         skipped: false,
         removed_dirs: removedDirs,
@@ -241,6 +258,29 @@ async function dirByteSize(dir) {
     }
     return total;
 }
+/**
+ * True when the pack's own manifest names a content hash the ledger has not
+ * committed, so the pack cannot be deleted and no payload needs reading. A
+ * missing or unreadable manifest answers false and falls through to the full
+ * hash check, which is the old, conservative path.
+ */
+async function manifestNamesUncommitted(dir, uploadedHashes) {
+    const manifest = await fs
+        .readFile(path.join(dir, "manifest.json"), "utf8")
+        .catch(() => null);
+    if (!manifest)
+        return false;
+    try {
+        const parsed = JSON.parse(manifest);
+        return (parsed.files ?? []).some((file) => typeof file.content_hash_sha256 === "string" &&
+            !uploadedHashes.has(file.content_hash_sha256));
+    }
+    catch {
+        // Not silent in effect: an unparseable manifest falls through to the full
+        // per-file hash check below, which decides on the bytes themselves.
+        return false;
+    }
+}
 async function inspectRawEvidenceDirForGc(dir, uploadedHashes) {
     const files = await listFiles(dir).catch(() => null);
     if (!files)

package/dist/state-dir-audit.js

@@ -0,0 +1,138 @@
+/**
+ * Everything the collector's state directory holds, sorted into who owns it
+ * and what bounds it (BLI-11106).
+ *
+ * `~/.local/state/bli-cockpit` (Windows: `%USERPROFILE%\.local\state\bli-cockpit`,
+ * the same layout with backslashes) is where the collector keeps its state.
+ * By 2026-09-30 it also held a gigabyte of study artifacts written by agents
+ * running benches and experiments (`memory-bench`, `memory-extract-*`,
+ * `chat-model-bench`, …). Those are not the collector's, the collector never
+ * reads them, and counting them as its footprint made `cockpit doctor`
+ * alarm about bytes no prune could ever explain.
+ *
+ * The rule: study artifacts belong in the SIBLING root
+ * `~/.local/state/bli-cockpit-studies` (same parent, same layout on Windows),
+ * see `studiesStateDir`. Existing study folders already under the state dir are
+ * recognised by name, left exactly where they are, and not counted. Anything
+ * the table below does not recognise is counted as `other`, so a new
+ * collector file that nobody registered still shows up in the footprint
+ * rather than hiding.
+ *
+ * Every collector entry carries the one-line reason it is bounded; the
+ * runbook (`docs/runbooks/cockpit-launchd-sync.md`, "What the state
+ * directory holds") prints the same table. Metadata only: entry names are
+ * fixed strings, never a person's path.
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import { directoryBytes } from "./disk-usage-files.js";
+/** The sibling root where agent-written studies live, never under the collector. */
+export const STUDIES_DIR_SUFFIX = "-studies";
+/**
+ * The collector's own entries first (checked before the study prefixes), then
+ * the studies. Order matters only where a study prefix could shadow a
+ * collector name; none does today, and the test pins that.
+ */
+export const STATE_DIR_RULES = [
+    { name: "raw-evidence", match: "exact", class: "staging", bound: "staged copies go the tick the server commits them; 256 MiB cap; uncommitted evidence is never deleted, only counted" },
+    { name: "sync.log", match: "prefix", class: "logs", bound: "rotated every tick at 10 MiB, at most 2 archives (20 MB) per stream" },
+    { name: "sync.err.log", match: "prefix", class: "logs", bound: "rotated every tick at 10 MiB, at most 2 archives (20 MB) per stream" },
+    { name: "spool", match: "exact", class: "spool", bound: "upload queue; each entry is deleted once the ingest door accepts it" },
+    { name: "cursors", match: "exact", class: "state", bound: "raw-evidence ledger and reconcile answers capped at 5,000 rows each; backfill cursors one small file each" },
+    { name: "raw-evidence-staging.json", match: "exact", class: "state", bound: "staged-object index and delivery attempts capped at 5,000 rows each" },
+    { name: "work-contexts", match: "exact", class: "state", bound: "one small file per worktree ever bound with `cockpit start`; overwritten, not pruned (a few KB each)" },
+    { name: "active-work-context.json", match: "exact", class: "state", bound: "one file, overwritten" },
+    { name: "root-work-scan.json", match: "exact", class: "state", bound: "one file, overwritten by each root scan" },
+    { name: "setup-receipt.json", match: "exact", class: "state", bound: "one file, overwritten" },
+    { name: "memory-install-receipt.json", match: "exact", class: "state", bound: "one file, overwritten" },
+    { name: "last-prune.json", match: "exact", class: "state", bound: "one file, overwritten every tick" },
+    { name: "last-turn-trace.json", match: "exact", class: "state", bound: "one file, overwritten" },
+    { name: "sync.lock", match: "exact", class: "state", bound: "lock file, removed when the tick ends" },
+    { name: "backfill.lock", match: "exact", class: "state", bound: "lock file, removed when the backfill ends" },
+    { name: ".last-", match: "prefix", class: "state", bound: "24-byte throttle markers for self-update, autostart repair and the memory install" },
+    { name: "read-cache", match: "exact", class: "state", bound: "one response file per distinct Tower read, overwritten on refresh" },
+    { name: "memory-hook-stats.json", match: "exact", class: "state", bound: "fixed-size histograms, overwritten (BLI Memory hooks)" },
+    { name: "memory-hook-samples", match: "exact", class: "state", bound: "hourly content-free timing samples (BLI Memory hooks); not yet pruned, a few MB a week" },
+    { name: "memory-recall-seen", match: "exact", class: "state", bound: "one file per agent session, deleted after 7 days (BLI Memory hooks)" },
+    { name: "memory-stop-anchors.json", match: "exact", class: "state", bound: "one offset per transcript, overwritten (BLI Memory hooks)" },
+    { name: "memory-daemon-spawn.json", match: "exact", class: "state", bound: "one spawn stamp, overwritten (BLI Memory daemon)" },
+    { name: "memory-daemon.sock", match: "exact", class: "state", bound: "the daemon's socket, zero bytes" },
+    // Studies written by agents and scripts, not by the collector. Counted by
+    // name only, never sized, never deleted.
+    { name: "memory-bench", match: "prefix", class: "study", bound: "agent study; belongs in bli-cockpit-studies" },
+    { name: "memory-extract-", match: "prefix", class: "study", bound: "agent study; belongs in bli-cockpit-studies" },
+    { name: "memory-questions", match: "prefix", class: "study", bound: "agent study; belongs in bli-cockpit-studies" },
+    { name: "memory-hook-context-sweep", match: "prefix", class: "study", bound: "agent study; belongs in bli-cockpit-studies" },
+    { name: "memory-anchor-probe", match: "prefix", class: "study", bound: "agent study; belongs in bli-cockpit-studies" },
+    { name: "memory-dup-threshold", match: "prefix", class: "study", bound: "agent study; belongs in bli-cockpit-studies" },
+    { name: "memory-judge-effort", match: "prefix", class: "study", bound: "agent study; belongs in bli-cockpit-studies" },
+    { name: "memory-recurrence-backfill", match: "prefix", class: "study", bound: "agent study; belongs in bli-cockpit-studies" },
+    { name: "memory-supersede-from-record", match: "prefix", class: "study", bound: "agent study; belongs in bli-cockpit-studies" },
+    { name: "memory-reconcile-precision", match: "prefix", class: "study", bound: "agent study; belongs in bli-cockpit-studies" },
+    { name: "memory-harvest-dedup", match: "prefix", class: "study", bound: "agent study; belongs in bli-cockpit-studies" },
+    { name: "chat-model-bench", match: "prefix", class: "study", bound: "agent study; belongs in bli-cockpit-studies" },
+    { name: "gpt6-bench-", match: "prefix", class: "study", bound: "agent study; belongs in bli-cockpit-studies" },
+    { name: "manual-study-evidence-vault", match: "prefix", class: "study", bound: "hand-made evidence vault; `cockpit clean --all-committed` removes it once every file is committed" },
+    { name: "issue-import", match: "exact", class: "study", bound: "one-off import receipts; belongs in bli-cockpit-studies" },
+    { name: "analysis", match: "exact", class: "study", bound: "local harvest-analysis output; belongs in bli-cockpit-studies" },
+    { name: "task-type-backlog", match: "prefix", class: "study", bound: "local harvest-analysis output; belongs in bli-cockpit-studies" },
+];
+/** Which rule an entry name falls under, or null for `other`. */
+export function classifyStateEntry(name) {
+    for (const rule of STATE_DIR_RULES) {
+        if (rule.match === "exact" ? name === rule.name : name.startsWith(rule.name)) {
+            return rule;
+        }
+    }
+    return null;
+}
+/**
+ * The sibling studies root, on either host family: the state dir's own path
+ * with `-studies` appended, so `/Users/x/.local/state/bli-cockpit` becomes
+ * `/Users/x/.local/state/bli-cockpit-studies` and
+ * `C:\Users\x\.local\state\bli-cockpit` becomes
+ * `C:\Users\x\.local\state\bli-cockpit-studies`. No separator is joined, so
+ * neither layout can be spelled wrong.
+ */
+export function studiesStateDir(paths) {
+    return `${paths.state_dir.replace(/[\\/]+$/u, "")}${STUDIES_DIR_SUFFIX}`;
+}
+/**
+ * Sizes the `state` and `other` entries of the state directory. Staging,
+ * logs and spool are measured by their own readers and skipped here; studies
+ * are counted but never walked, because a gigabyte of somebody's bench run is
+ * not worth a stat per file on every doctor call.
+ */
+export async function readStateDirFootprint(paths) {
+    const footprint = {
+        state_bytes: 0,
+        other_bytes: 0,
+        other_entries: 0,
+        study_entries: 0,
+    };
+    const entries = await fs
+        .readdir(paths.state_dir, { withFileTypes: true })
+        .catch(() => []);
+    for (const entry of entries) {
+        const rule = classifyStateEntry(entry.name);
+        const cls = rule?.class ?? "other";
+        if (cls === "staging" || cls === "logs" || cls === "spool")
+            continue;
+        if (cls === "study") {
+            footprint.study_entries += 1;
+            continue;
+        }
+        const full = path.join(paths.state_dir, entry.name);
+        const bytes = entry.isDirectory()
+            ? await directoryBytes(full)
+            : await fs.stat(full).then((info) => info.size, () => 0);
+        if (cls === "state") {
+            footprint.state_bytes += bytes;
+        }
+        else {
+            footprint.other_bytes += bytes;
+            footprint.other_entries += 1;
+        }
+    }
+    return footprint;
+}

package/dist/upload-envelope-build.js

@@ -14,6 +14,7 @@
  *   `upload-session-reports.ts`, which builds its own provenance without a
  *   full envelope.
  */
+import { priorSnapshotsFromCursor, } from "./adapters/raw-evidence-snapshot-supersede.js";
 import { TelemetryIngestEnvelopeSchema, } from "@bli-cockpit/telemetry-core";
 import path from "node:path";
 import { getCollectorRuntimePaths, LOCAL_COLLECTOR_VERSION, readLocalCollectorConfig, readLocalCollectorSessionFile, readLocalSessionReference, readLocalWorkContextForRepo, } from "./local-state.js";
@@ -120,6 +121,10 @@ export async function buildLocalAmbientEnvelope(options = {}) {
             operatorId: session.operator_id,
             workContextId: uploadContext.work_context_id,
         }),
+        rawEvidencePriorSnapshots: priorSnapshotsFromCursor(transcriptCursorObjects(options.cursorObjects), (objectKey) => rawEvidenceObjectKeyBelongsToWorkContext(objectKey, {
+            operatorId: session.operator_id,
+            workContextId: uploadContext.work_context_id,
+        })),
         rawEvidenceByteBudget: options.rawEvidenceByteBudget,
         rawEvidenceObjectBudget: options.rawEvidenceObjectBudget,
         rawEvidenceBudget: options.rawEvidenceBudget,
@@ -188,6 +193,10 @@ function reusableContentHashes(options, context) {
         .filter(([, entry]) => rawEvidenceObjectKeyBelongsToWorkContext(entry.object_key, context))
         .map(([hash]) => hash));
 }
+/** Cursor entries that remember a transcript source (BLI-11088); the rest are ignored. */
+function transcriptCursorObjects(objects) {
+    return Object.fromEntries(Object.entries(objects ?? {}).filter((pair) => "source_key" in pair[1] && Boolean(pair[1].source_key)));
+}
 export function makeUploadWorkContext(options) {
     const provenance = makeCollectorProvenance({
         context: options.activeContext,

package/dist/upload-sync.js

@@ -1,23 +1,3 @@
-/**
- * `syncLocalAmbientEnvelope`: one ambient sync, end to end.
- *
- * 1. Assemble the envelope — or record why this machine cannot upload at all
- *    and rethrow, so `cockpit status` names the blocker.
- * 2. Deliver the raw evidence objects, skipping anything inside its delivery
- *    backoff window (`upload-evidence-delivery.ts`).
- * 3. Reconcile the envelope with what actually became durable, POST it to
- *    ingest, and refuse to believe a receipt we cannot read
- *    (`upload-ingest-receipt.ts`).
- * 4. Advance the content cursor, report any agent images, and write the ledger
- *    row — a retry when a later sync could still rescue the gap, a named
- *    blocker when it never can, nothing extra when the sync was clean.
- * 5. On any failure, spool a retry that carries its own reason.
- *
- * The whole file is built around one rule from the fleet contract: a sync that
- * did not collect everything must never read green. Objects held by backoff are
- * reported as named failures, permanent rejections are logged and blocked
- * rather than silently retried forever, and the ledger row always says why.
- */
 import { getCollectorRuntimePaths } from "./local-state.js";
 import { uploadRawEvidenceFilesChunked, } from "./evidence-upload-client.js";
 import { markObjectCommitted, readRawEvidenceCursor, writeRawEvidenceCursor, } from "./cursors/raw-evidence-cursor.js";
@@ -80,7 +60,7 @@ export async function syncLocalAmbientEnvelope(options = {}) {
             built,
             envelope: applyRawEvidenceUploadOutcomes(built.envelope, uploadOutcomes),
         });
-        advanceRawEvidenceCursor(cursor, uploadOutcomes, attemptedAt);
+        advanceRawEvidenceCursor(cursor, uploadOutcomes, attemptedAt, built.raw_evidence_upload_files);
         await writeRawEvidenceCursor(paths, cursor);
         await reportAgentImageArtifacts({
             fetchImpl,
@@ -221,17 +201,25 @@ async function buildEnvelopeOrRecordBlocker(paths, options, cursorObjects, attem
  * bytes instead of re-uploading them. A failed upload is deliberately absent:
  * recording it would make the next sync skip a file that never landed.
  */
-function advanceRawEvidenceCursor(cursor, outcomes, attemptedAt) {
+function advanceRawEvidenceCursor(cursor, outcomes, attemptedAt, files = []) {
+    // BLI-11088: a transcript's source key and snapshot facts ride into the
+    // cursor, so the next sync can prove its snapshot rebuilds this one.
+    const snapshotByHash = new Map(files.filter((file) => file.snapshot && file.source_key && file.pointer.content_hash_sha256)
+        .map((file) => [file.pointer.content_hash_sha256, file]));
     for (const outcome of outcomes) {
         if (outcome.upload_state === "upload_failed")
             continue;
         const contentHash = outcome.pointer.content_hash_sha256;
         if (!contentHash)
             continue;
+        const source = snapshotByHash.get(contentHash);
         markObjectCommitted(cursor, contentHash, {
             object_key: outcome.object_key,
             byte_size: outcome.pointer.byte_size ?? 0,
             committed_at: attemptedAt,
+            ...(source
+                ? { source_key: source.source_key, checkpoints: source.snapshot.checkpoints }
+                : {}),
         });
     }
     cursor.updated_at = attemptedAt;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bli-cockpit/cli",
-  "version": "0.2.150",
+  "version": "0.2.155",
   "private": false,
   "type": "module",
   "bin": {
@@ -24,11 +24,11 @@
     "pretypecheck": "node ../../scripts/build-workspace-dep.mjs @bli-cockpit/cli",
     "typecheck": "node -e \"await import('./dist/commands/public-root.js')\"",
     "pretest": "node ../../scripts/build-workspace-dep.mjs @bli-cockpit/cli",
-    "test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-cli-verb-help.mjs && node ../../scripts/assert-public-cli-surface-doors.mjs && node ../../scripts/assert-public-cli-runtime-files.mjs && node ../../scripts/assert-public-cli-exit-contract.mjs && node ../../scripts/assert-public-cli-no-fleet-posts.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
+    "test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-cli-verb-help.mjs && node ../../scripts/assert-public-cli-surface-doors.mjs && node ../../scripts/assert-public-cli-runtime-files.mjs && node ../../scripts/assert-public-cli-exit-contract.mjs && node ../../scripts/assert-public-cli-no-fleet-posts.mjs && node ../../scripts/assert-public-cli-doctor-json.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
   },
   "dependencies": {
-    "@bli-cockpit/memory-mcp": "0.1.39",
-    "@bli-cockpit/mcp": "0.1.73",
-    "@bli-cockpit/telemetry-core": "0.1.51"
+    "@bli-cockpit/memory-mcp": "0.1.41",
+    "@bli-cockpit/mcp": "0.1.75",
+    "@bli-cockpit/telemetry-core": "0.1.53"
   }
 }