@bli-cockpit/cli 0.2.204 → 0.2.206

package/dist/slim/slim-transcript.js

@@ -0,0 +1,133 @@
+/**
+ * The slim transcript (BLI-16977): the transcript the collector uploads when
+ * `IMAGE_HANDLING_MODE` in telemetry-core is `describe_and_drop`. Images are already markers by the
+ * time this runs; this strips the duplicate copies and the bookkeeping that remain.
+ *
+ * The contract, in the order Edward and the skeptic pass set it:
+ *
+ *  1. ONE OUTPUT LINE PER INPUT LINE, same order. Tower counts lines and reads every line's
+ *     timestamp, so a line is never dropped; its payload is. A blank line, a line that is not JSON,
+ *     and a JSON value that is not an object are emitted as they came.
+ *  2. A LINE NO RULE TOUCHES IS EMITTED AS ITS ORIGINAL TEXT, never re-serialised. Only a modified
+ *     line is re-written, and it gains a compact top-level `"_slim":["rule", ...]` naming what was
+ *     stripped, so a reader can tell.
+ *  3. PREFIX STABLE. A line's slim form depends only on that line and the lines before it (no
+ *     look-ahead, no global pass). So the slim form of a growing transcript extends the slim form
+ *     of the shorter one byte for byte, and the BLI-11088 prefix supersede rule keeps working.
+ *     The one honest exception is a last line still being written: a half-written JSON line is
+ *     emitted as it is and becomes a slim line once it is whole.
+ *  4. NEVER FAIL A TRANSCRIPT. A line whose rules throw is emitted as it came and counted; any other
+ *     failure hands back the text it was given (`slimOrKeep`) and says why.
+ *
+ * What each rule keeps and drops: claude-rules.ts and codex-rules.ts. The table for people:
+ * docs/architecture/image-descriptions.md.
+ */
+import { newClaudeSlimState, slimClaudeRecord } from "./claude-rules.js";
+import { newCodexSlimState, slimCodexRecord } from "./codex-rules.js";
+import { LineEdits, SeenBlocks, isJsonObject } from "./slim-support.js";
+/** The text an image marker starts with; a line carrying one is a modified line (an image became a marker). */
+const IMAGE_MARKER_START = "[image sha256:";
+export function createSlimmer(kind, rulesForTests) {
+    const seen = new SeenBlocks();
+    const claude = newClaudeSlimState();
+    const codex = newCodexSlimState();
+    const stats = {
+        lines: 0, linesModified: 0, linesFailed: 0, bytesBefore: 0, bytesAfter: 0, charsByRule: {}, linesByRule: {},
+    };
+    const applyRules = rulesForTests ?? ((record, edits) => {
+        if (kind === "claude_code")
+            slimClaudeRecord(record, claude, seen, edits);
+        else
+            slimCodexRecord(record, codex, seen, edits);
+    });
+    return {
+        slimLine(line) {
+            stats.lines += 1;
+            const before = Buffer.byteLength(line);
+            stats.bytesBefore += before;
+            let out = line;
+            try {
+                out = slimOneLine(line, applyRules, stats);
+            }
+            catch {
+                stats.linesFailed += 1;
+                out = line;
+            }
+            stats.bytesAfter += out === line ? before : Buffer.byteLength(out);
+            return out;
+        },
+        stats: () => stats,
+    };
+}
+function slimOneLine(line, applyRules, stats) {
+    const body = line.endsWith("\r") ? line.slice(0, -1) : line;
+    const first = body.trimStart().charCodeAt(0);
+    if (first !== 0x7b /* { */)
+        return line;
+    let record;
+    try {
+        record = JSON.parse(body);
+    }
+    catch {
+        return line;
+    }
+    if (!isJsonObject(record))
+        return line;
+    const edits = new LineEdits();
+    applyRules(record, edits);
+    const rules = edits.rules;
+    if (line.includes(IMAGE_MARKER_START))
+        rules.push("images");
+    const already = Array.isArray(record["_slim"]) ? record["_slim"] : [];
+    const fresh = rules.filter((rule) => !already.includes(rule));
+    if (fresh.length === 0)
+        return line;
+    record["_slim"] = [...already, ...fresh];
+    for (const [rule, chars] of edits.entries) {
+        stats.charsByRule[rule] = (stats.charsByRule[rule] ?? 0) + chars;
+        stats.linesByRule[rule] = (stats.linesByRule[rule] ?? 0) + 1;
+    }
+    if (fresh.includes("images"))
+        stats.linesByRule["images"] = (stats.linesByRule["images"] ?? 0) + 1;
+    stats.linesModified += 1;
+    return JSON.stringify(record) + (line.endsWith("\r") ? "\r" : "");
+}
+/** The slim form of a whole transcript's text, line for line. */
+export function slimTranscript(text, kind) {
+    const slimmer = createSlimmer(kind);
+    const lines = text.split("\n");
+    const out = new Array(lines.length);
+    for (let index = 0; index < lines.length; index += 1)
+        out[index] = slimmer.slimLine(lines[index]);
+    const stats = slimmer.stats();
+    return { text: stats.linesModified === 0 ? text : out.join("\n"), stats };
+}
+const TAG = "[slim-transcript]";
+/**
+ * `slimTranscript` that can never fail a transcript: any exception returns the text it was given and
+ * logs the reason. Metadata only: counts, sizes, rule names. Never a line, never a field value.
+ */
+export function slimOrKeep(text, kind, label) {
+    const startedAt = Date.now();
+    try {
+        const slim = slimTranscript(text, kind);
+        const stats = slim.stats;
+        console.error(TAG + (stats.linesModified === 0 ? " nothing to strip" : " transcript slimmed"), JSON.stringify({
+            source: kind,
+            session_id: label.sessionId.slice(0, 80),
+            lines: stats.lines,
+            lines_modified: stats.linesModified,
+            lines_failed: stats.linesFailed,
+            bytes_before: stats.bytesBefore,
+            bytes_after: stats.bytesAfter,
+            chars_by_rule: stats.charsByRule,
+            lines_by_rule: stats.linesByRule,
+            elapsed_ms: Date.now() - startedAt,
+        }));
+        return { text: slim.text, stats };
+    }
+    catch (error) {
+        console.error(TAG + " failed, uploading the transcript as it was", JSON.stringify({ reason: "slim_failed", source: kind, error_name: error instanceof Error ? error.name : typeof error, bytes: typeof text === "string" ? text.length : 0 }));
+        return { text, stats: null };
+    }
+}

package/dist/state-dir-audit.js

@@ -43,6 +43,7 @@ export const STATE_DIR_RULES = [
     { name: "spool", match: "exact", class: "spool", bound: "upload queue; each entry is deleted once the ingest door accepts it; at most 20 pending uploads and 100 outbox entries (40 events each), oldest dropped first" },
     { name: "cursors", match: "exact", class: "state", bound: "raw-evidence ledger and reconcile answers capped at 5,000 rows each; backfill cursors one small file each" },
     { name: "raw-evidence-staging.json", match: "exact", class: "state", bound: "staged-object index and delivery attempts capped at 5,000 rows each" },
+    { name: "image-descriptions.json", match: "exact", class: "state", bound: "keys of images already sent to the describe door, capped at 150,000 keys (about 5 MB), oldest dropped first" },
     { name: "work-contexts", match: "exact", class: "state", bound: "one small file per worktree ever bound with `cockpit start`; overwritten, not pruned (a few KB each)" },
     { name: "active-work-context.json", match: "exact", class: "state", bound: "one file, overwritten" },
     { name: "root-work-scan.json", match: "exact", class: "state", bound: "one file, overwritten by each root scan" },

package/dist/upload-envelope-build.js

@@ -21,6 +21,7 @@ import { getCollectorRuntimePaths, LOCAL_COLLECTOR_VERSION, readLocalCollectorCo
 import { runLocalSourceCollectors } from "./adapters/local-sources.js";
 import { defaultCodexSessionDirs, } from "./adapters/codex-attribution.js";
 import { normalizeDashboardUrl } from "./upload-http.js";
+import { openImageStepForSync } from "./image-description-session.js";
 import { describeError, isMissingFileFailure } from "./health-detail.js";
 import { makeSourceScanCompletedEvent, sanitizeRiskFlag, sanitizeSessionReference, sanitizeSourceScanResults, selectedTicketBindingCandidate, } from "./upload-envelope-event.js";
 export class LocalUploadBlockedError extends Error {
@@ -100,6 +101,15 @@ export async function buildLocalAmbientEnvelope(options = {}) {
         repoLabel,
         now,
     });
+    // BLI-16977: describe each distinct image in the transcripts once (and, only when the one switch
+    // in telemetry-core says drop, send a marker instead of the base64). Never throws.
+    const rawEvidenceImages = await openImageStepForSync({
+        budget: options.rawEvidenceBudget,
+        stateDir: paths.state_dir,
+        dashboardUrl: normalizeDashboardUrl(options.dashboardUrl ?? sessionFile.dashboard_url ?? config.dashboard_url),
+        deviceToken: sessionFile.device_token,
+        fetchImpl: options.fetch,
+    });
     const sourceCollection = await runLocalSourceCollectors({
         repoRoot,
         branch: uploadContext.branch,
@@ -129,6 +139,7 @@ export async function buildLocalAmbientEnvelope(options = {}) {
         rawEvidenceByteBudget: options.rawEvidenceByteBudget,
         rawEvidenceObjectBudget: options.rawEvidenceObjectBudget,
         rawEvidenceBudget: options.rawEvidenceBudget,
+        rawEvidenceImages,
         rawEvidenceDeliveryMode: options.evidenceDeliveryMode,
         now,
     });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bli-cockpit/cli",
-  "version": "0.2.204",
+  "version": "0.2.206",
   "private": false,
   "type": "module",
   "bin": {
@@ -27,9 +27,9 @@
     "test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-cli-verb-help.mjs && node ../../scripts/assert-public-cli-surface-doors.mjs && node ../../scripts/assert-public-cli-runtime-files.mjs && node ../../scripts/assert-public-cli-exit-contract.mjs && node ../../scripts/assert-public-cli-no-fleet-posts.mjs && node ../../scripts/assert-public-cli-doctor-json.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
   },
   "dependencies": {
-    "@bli-cockpit/memory-mcp": "0.1.61",
-    "@bli-cockpit/mcp": "0.1.116",
-    "@bli-cockpit/telemetry-core": "0.1.70"
+    "@bli-cockpit/memory-mcp": "0.1.63",
+    "@bli-cockpit/mcp": "0.1.118",
+    "@bli-cockpit/telemetry-core": "0.1.72"
   },
   "repository": {
     "type": "git",