@bli-cockpit/cli 0.2.48 → 0.2.49

package/dist/commands/sync.js

@@ -0,0 +1,381 @@
+import { writeLine } from "./cli-io.js";
+import { attributedSyncRunStatus, cursorStatusLine, displayTicketId, rawEvidenceSyncLine, shortSha, worktreeSyncRow, writeAgentSessionSummary, } from "./collection-report.js";
+import { collectionRootConsentAliases } from "./collection-roots.js";
+import { discoverCommandWorktrees } from "./local-discovery.js";
+import { sendCollectorHeartbeatBestEffort, } from "./heartbeat.js";
+import { classifySyncFailureRecords, classifySyncHealthError, redactedSyncErrorDetail, reportInstallEventsBestEffort, } from "./install-receipts.js";
+import { runAttributedWorktreeSync, } from "./session-sync.js";
+import { runAutostartSelfHealAfterSync, runScheduledSelfUpdateAfterSync, } from "./sync-followups.js";
+import { describeError } from "../health-detail.js";
+import { inspectBackfillLock } from "../backfill-lock.js";
+import { rotateCollectorLogsBestEffort } from "../log-rotation.js";
+import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths, readLocalCollectorConfig, } from "../local-state.js";
+import { CollectionRootRequiredError, } from "../onboarding-roots.js";
+import { rawEvidenceDedupSummary, rawEvidenceGcSummary, runRawEvidenceLocalGc, sweepDuplicateStagedRawEvidence, } from "../raw-evidence-gc.js";
+import { normalizeCollectionRoots } from "../root-normalization.js";
+import { acquireSyncLock } from "../sync-lock.js";
+export async function runSync(command, io) {
+    const paths = getCollectorRuntimePaths(command.homeDir);
+    // BLI-3553, first thing in the tick: cap the scheduler's own logs. Nothing
+    // rotated them before, and two months of a 15-minute tick left 211 MB of
+    // sync.log on the reference Mac. Best-effort by construction — a rotation
+    // problem is its own log line, never a reason collection does not run.
+    await rotateCollectorLogsBestEffort(paths);
+    const config = await readLocalCollectorConfig(paths).catch(() => null);
+    const dashboardUrl = command.dashboardUrl ?? config?.dashboard_url ?? DEFAULT_DASHBOARD_URL;
+    const minCliVersionAtStart = await reportInstallEventsBestEffort({
+        homeDir: command.homeDir,
+        dashboardUrl,
+        command: "sync",
+        events: [{ step: "sync_started", status: "ok" }],
+        json: command.json,
+        io,
+    });
+    try {
+        const result = await runSyncWithHealthReceipt(command, io);
+        // BLI-3551: every tick checks in, including one that collected nothing.
+        // This is the only writer of `last_seen_at` that does not need an envelope,
+        // so it is what separates a quiet machine from a dead one.
+        await sendSyncHeartbeat(command, io, dashboardUrl, result.heartbeat);
+        const minCliVersion = await reportInstallEventsBestEffort({
+            homeDir: command.homeDir,
+            dashboardUrl,
+            command: "sync",
+            events: [result.completion],
+            json: command.json,
+            io,
+        });
+        // BLI-2601: self-update runs only after collection's own outcome above is
+        // already decided and reported, win or lose. See the function doc.
+        await runScheduledSelfUpdateAfterSync(command, io, dashboardUrl, minCliVersion ?? minCliVersionAtStart);
+        await runAutostartSelfHealAfterSync(command, io, dashboardUrl);
+        return result.exitCode;
+    }
+    catch (error) {
+        const errorCode = classifySyncHealthError(error);
+        // A machine whose sync THREW is still alive, and that is worth knowing —
+        // a device that stops checking in entirely is a different problem from one
+        // checking in with a failure every fifteen minutes.
+        await sendSyncHeartbeat(command, io, dashboardUrl, {
+            status: "fail",
+            reason: errorCode,
+        });
+        const minCliVersion = await reportInstallEventsBestEffort({
+            homeDir: command.homeDir,
+            dashboardUrl,
+            command: "sync",
+            events: [
+                {
+                    step: "sync_complete",
+                    status: "fail",
+                    error_code: errorCode,
+                    error_detail: redactedSyncErrorDetail(error),
+                },
+            ],
+            json: command.json,
+            io,
+        });
+        await runScheduledSelfUpdateAfterSync(command, io, dashboardUrl, minCliVersion ?? minCliVersionAtStart);
+        await runAutostartSelfHealAfterSync(command, io, dashboardUrl);
+        throw error;
+    }
+}
+/**
+ * The tick's check-in (BLI-3551).
+ *
+ * Resolving the roots is best-effort on purpose: a machine with NO approved
+ * root is exactly the machine whose silence needs explaining, so it still
+ * checks in — with an empty root list, which is itself the finding.
+ */
+async function sendSyncHeartbeat(command, io, dashboardUrl, facts) {
+    const roots = await resolveSyncCollectionRoots(command).catch(() => []);
+    await sendCollectorHeartbeatBestEffort({
+        homeDir: command.homeDir,
+        dashboardUrl,
+        roots,
+        facts,
+        io,
+    }).catch((error) => {
+        // The sender already swallows everything it knows about; this is the net
+        // for anything it does not, because a heartbeat must never fail a sync.
+        console.error("[heartbeat] the check-in threw and was dropped", JSON.stringify({ reason: "heartbeat_threw", ...describeError(error) }));
+        return false;
+    });
+}
+async function runSyncWithHealthReceipt(command, io) {
+    const backfillLock = await inspectBackfillLock(getCollectorRuntimePaths(command.homeDir));
+    if (backfillLock.held) {
+        if (command.json) {
+            writeLine(io.stdout, JSON.stringify({
+                status: "live_sync_paused_during_backfill",
+                reason: "live sync paused during backfill",
+                held_since: backfillLock.held_since,
+                collection_complete: false,
+                upload_state: "not_uploaded",
+                retryable: true,
+            }, null, 2));
+        }
+        else {
+            writeLine(io.stdout, "Pausing normal sync while it catches up on old sessions.");
+        }
+        return {
+            exitCode: 0,
+            completion: {
+                step: "sync_complete",
+                status: "skipped",
+                error_code: "live_sync_paused_during_backfill",
+            },
+            heartbeat: {
+                status: "skipped",
+                reason: "live_sync_paused_during_backfill",
+            },
+        };
+    }
+    // Single-flight: a launchd timer and a manual sync must not interleave the
+    // cursor read-modify-write. A blocked invocation exits cleanly (B.4 §7).
+    const lock = await acquireSyncLock(getCollectorRuntimePaths(command.homeDir));
+    if (!lock.acquired) {
+        if (command.json) {
+            writeLine(io.stdout, JSON.stringify({
+                status: "sync_already_running",
+                reason: "another sync owns the collection lock",
+                held_since: lock.held_since,
+                collection_complete: false,
+                upload_state: "not_uploaded",
+                retryable: true,
+            }, null, 2));
+        }
+        else {
+            writeLine(io.stdout, "Tower sync already running; skipping this run.");
+        }
+        return {
+            exitCode: 0,
+            completion: {
+                step: "sync_complete",
+                status: "skipped",
+                error_code: "sync_already_running",
+            },
+            heartbeat: { status: "skipped", reason: "sync_already_running" },
+        };
+    }
+    try {
+        const { exitCode, failureReasons, failureRecords, notice, sessionsObserved, sessionsOutsideRoot, } = await runSyncLocked(command, io);
+        const counts = {
+            sessionsObserved,
+            sessionsOutsideRoot,
+        };
+        if (exitCode === 0) {
+            // BLI-3551: an `ok` tick can still have something to say. `nothing_in_root`
+            // is the receipt that separates "this machine is alive and its operator
+            // works outside the approved roots" from "this machine is dead", which
+            // until now looked identical from the dashboard.
+            return {
+                exitCode,
+                completion: {
+                    step: "sync_complete",
+                    status: "ok",
+                    ...(notice ? { error_detail: notice } : {}),
+                },
+                heartbeat: { status: "ok", reason: notice, ...counts },
+            };
+        }
+        // A sync that fails by exit code says exactly as much as one that throws.
+        // It used to say `sync_failed` and nothing else, so 100% of recorded
+        // failure rows carried a null detail and the real reason was reachable only
+        // by running `cockpit status` on the machine itself (BLI-2526).
+        const reasonText = failureReasons.join("; ");
+        // The bucket comes from the records the deciding branches wrote, not from
+        // this sentence (BLI-3551). The sentence is still the detail.
+        const errorCode = classifySyncFailureRecords(failureRecords);
+        return {
+            exitCode,
+            completion: {
+                step: "sync_complete",
+                status: "fail",
+                error_code: errorCode,
+                error_detail: redactedSyncErrorDetail(reasonText),
+            },
+            heartbeat: { status: "fail", reason: errorCode, ...counts },
+        };
+    }
+    finally {
+        await lock.handle.release();
+    }
+}
+/**
+ * Turn a finished run into an exit code and the reasons behind it.
+ *
+ * One place, so a future return path cannot reintroduce a code with no reason.
+ * The reasons come from the run itself — the code that decided `ok` is false is
+ * the only code that knows why.
+ */
+function syncResult(run) {
+    return {
+        exitCode: run.ok ? 0 : 1,
+        failureReasons: run.ok ? [] : run.failure_reasons,
+        failureRecords: run.ok ? [] : run.failure_records,
+        notice: run.notice,
+        sessionsObserved: run.sessions_observed,
+        sessionsOutsideRoot: run.sessions_outside_root,
+    };
+}
+/**
+ * The sync runbook: resolve which roots to collect, dedup staged packs before
+ * touching anything else, discover worktrees, sync them, then report — one of
+ * three shapes depending on how many worktrees came back. The three shapes
+ * share nothing but `dedup`, so each gets its own step function below rather
+ * than one branchy body.
+ */
+async function runSyncLocked(command, io) {
+    const collectionRoots = await resolveSyncCollectionRoots(command);
+    // Before anything is collected: collapse byte-identical staged packs. It runs
+    // first, unconditionally and unthrottled, because a machine that already
+    // holds 559 copies of one rollout needs the disk back before it stages
+    // anything else (BLI-3066). Safe by construction — a duplicate is identical
+    // by content hash to the survivor.
+    const dedup = await sweepDuplicateStagedRawEvidence(getCollectorRuntimePaths(command.homeDir), io.env);
+    if (!dedup.skipped && dedup.removed_dirs > 0) {
+        writeLine(io.stdout, rawEvidenceDedupSummary(dedup));
+    }
+    const worktrees = await discoverCommandWorktrees(collectionRoots, {
+        maxDepth: command.maxDepth,
+        maxRepos: command.maxRepos,
+        homeDir: command.homeDir,
+        allowEmpty: true,
+    }, io);
+    const run = await runAttributedWorktreeSync({
+        homeDir: command.homeDir,
+        dashboardUrl: command.dashboardUrl,
+        collectionRoots,
+        startContexts: false,
+        worktrees,
+        fetchImpl: io.fetch,
+    });
+    if (run.outcomes.length > 1) {
+        return reportMultiRepoSync(command, io, run, dedup);
+    }
+    // Zero worktrees is a legitimate steady state, not a failure: an approved
+    // root can hold no git repos, and sessions upload independently of
+    // worktrees (session-first, BLI-2581). This used to throw "Sync produced no
+    // result", which painted ~90 false-red sync_failed receipts per day on one
+    // fleet machine with a single empty root and taught people to ignore
+    // sync_failed (BLI-2722). A genuinely broken run still fails via run.ok.
+    if (run.outcomes.length === 0) {
+        return reportNoWorktreeSync(command, io, run, dedup);
+    }
+    return reportSingleRepoSync(command, io, run, dedup);
+}
+async function reportMultiRepoSync(command, io, run, dedup) {
+    const rows = run.outcomes.map((outcome) => worktreeSyncRow(outcome, run));
+    const collectionRunStatus = attributedSyncRunStatus(run);
+    const gc = run.ok ? await runSyncRawEvidenceGc(command, io) : null;
+    if (command.json) {
+        writeLine(io.stdout, JSON.stringify({
+            mode: "multi_repo",
+            status: collectionRunStatus,
+            collection_complete: run.ok,
+            results: run.outcomes.map((outcome) => outcome.sync),
+            repos: rows,
+            codex_sessions: run.summary,
+            raw_evidence_gc: gc,
+            raw_evidence_dedup: dedup,
+        }, null, 2));
+        return syncResult(run);
+    }
+    writeLine(run.ok ? io.stdout : io.stderr, `Tower parent sync ${collectionRunStatus} ${run.outcomes.filter((outcome) => outcome.sync.status === "uploaded").length}/${run.outcomes.length} worktree(s).`);
+    for (const outcome of run.outcomes) {
+        const { worktree, sync } = outcome;
+        const uploaded = sync.status === "uploaded";
+        const failureSuffix = sync.status === "spooled" ? ` reason:${sync.failure_reason}` : "";
+        writeLine(uploaded ? io.stdout : io.stderr, `- ${worktree.repo_label}/${worktree.worktree_label} (${worktree.branch}) head:${shortSha(sync.head_sha ?? worktree.head_sha)} ${sync.status} objects:${sync.raw_evidence_uploaded_object_count} chunks:${sync.raw_evidence_uploaded_chunk_count} reused:${sync.raw_evidence_reused_count} failed:${sync.raw_evidence_failed_count} cursor:${sync.cursor_tracked_object_count}${failureSuffix}`);
+    }
+    writeAgentSessionSummary(io, run.summary);
+    if (gc && !gc.skipped)
+        writeLine(io.stdout, rawEvidenceGcSummary(gc));
+    return syncResult(run);
+}
+async function reportNoWorktreeSync(command, io, run, dedup) {
+    const collectionRunStatus = attributedSyncRunStatus(run);
+    const gc = run.ok ? await runSyncRawEvidenceGc(command, io) : null;
+    if (command.json) {
+        writeLine(io.stdout, JSON.stringify({
+            mode: "no_worktrees",
+            status: collectionRunStatus,
+            collection_complete: run.ok,
+            ...(run.notice ? { notice: run.notice } : {}),
+            codex_sessions: run.summary,
+            raw_evidence_gc: gc,
+            raw_evidence_dedup: dedup,
+        }, null, 2));
+        return syncResult(run);
+    }
+    writeLine(run.ok ? io.stdout : io.stderr, `Tower sync ${collectionRunStatus}: no git worktrees under this root; session scan ran.`);
+    if (run.notice) {
+        // Says out loud what the receipt now says to the dashboard: the sessions
+        // this machine ran were all outside the folders it is allowed to look at.
+        writeLine(io.stdout, `Every session seen this run was outside your approved folders (${run.notice}). Nothing was collected, and nothing is broken.`);
+    }
+    writeAgentSessionSummary(io, run.summary);
+    if (gc && !gc.skipped)
+        writeLine(io.stdout, rawEvidenceGcSummary(gc));
+    return syncResult(run);
+}
+async function reportSingleRepoSync(command, io, run, dedup) {
+    const result = run.outcomes[0]?.sync;
+    if (!result) {
+        throw new Error("Sync produced no result for the repo worktree.");
+    }
+    const collectionRunStatus = attributedSyncRunStatus(run);
+    const gc = run.ok ? await runSyncRawEvidenceGc(command, io) : null;
+    if (command.json) {
+        writeLine(io.stdout, JSON.stringify({
+            ...result,
+            status: collectionRunStatus,
+            collection_complete: run.ok,
+            codex_sessions: run.summary,
+            raw_evidence_gc: gc,
+            raw_evidence_dedup: dedup,
+        }, null, 2));
+        return syncResult(run);
+    }
+    if (run.ok) {
+        writeLine(io.stdout, "Tower uploaded this session.");
+        writeLine(io.stdout, `Ticket: ${displayTicketId(result.ticket_id)}`);
+        writeLine(io.stdout, `Context: ${result.work_context_id}`);
+        writeLine(io.stdout, `Head: ${shortSha(result.head_sha)}`);
+        writeLine(io.stdout, `Things recorded: ${result.event_count}`);
+        writeLine(io.stdout, `Risk flags: ${result.risk_flag_count}`);
+        writeLine(io.stdout, `Raw evidence files: ${result.raw_evidence_file_count}`);
+        writeLine(io.stdout, rawEvidenceSyncLine(result));
+        writeAgentSessionSummary(io, run.summary);
+        writeLine(io.stdout, cursorStatusLine(result));
+        if (gc && !gc.skipped)
+            writeLine(io.stdout, rawEvidenceGcSummary(gc));
+        return syncResult(run);
+    }
+    if (result.status === "uploaded") {
+        writeLine(io.stderr, "Tower uploaded, but some sessions did not make it. Run `cockpit sync` again.");
+        writeAgentSessionSummary(io, run.summary);
+        return syncResult(run);
+    }
+    writeLine(io.stderr, "Tower could not upload. It saved a note to retry and will try again on the next sync.");
+    writeLine(io.stderr, `Ticket: ${displayTicketId(result.ticket_id)}`);
+    writeLine(io.stderr, `Failure: ${result.failure_reason}`);
+    writeLine(io.stderr, `Retry: ${result.retry_command}`);
+    return syncResult(run);
+}
+async function resolveSyncCollectionRoots(command) {
+    const explicitRoots = normalizeCollectionRoots(command.repoRoot ? [command.repoRoot] : []);
+    if (explicitRoots.length > 0) {
+        return collectionRootConsentAliases(explicitRoots);
+    }
+    const config = await readLocalCollectorConfig(getCollectorRuntimePaths(command.homeDir)).catch(() => null);
+    const savedRoots = normalizeCollectionRoots(config?.default_repo_paths ?? []);
+    if (savedRoots.length > 0) {
+        return collectionRootConsentAliases(savedRoots);
+    }
+    throw new CollectionRootRequiredError(`no explicit or saved collection root is available.`);
+}
+async function runSyncRawEvidenceGc(command, io) {
+    return runRawEvidenceLocalGc(getCollectorRuntimePaths(command.homeDir), io.env);
+}

package/dist/tower-stream.js

@@ -2,8 +2,9 @@
  * Reading a Tower turn as it happens (BLI-3457).
  *
  * The dashboard's conversation routes speak one JSON object per line
- * (`application/x-ndjson`): `activity` events as tools start and settle, then
- * exactly one `final` event carrying what the plain JSON response used to be.
+ * (`application/x-ndjson`): `activity` events as tools start and settle,
+ * `token` events carrying the answer as it is written (BLI-3517), then exactly
+ * one `final` event carrying what the plain JSON response used to be.
  * That is the same wire format the browser already reads
  * (`apps/dashboard/src/lib/webchat/tool-trace.ts`); this is the terminal's
  * reader for it.
@@ -89,6 +90,8 @@ export async function readTowerTurn(response, options = {}) {
     const notes = [];
     let streamed = true;
     let final = null;
+    let draft = "";
+    let tokenFrames = 0;
     for await (const item of readTowerStream(response)) {
         if (item.kind === "note") {
             if (item.reason === "stream_not_available")
@@ -101,6 +104,13 @@ export async function readTowerTurn(response, options = {}) {
             final = item.event;
             break;
         }
+        if (item.event.type === "token") {
+            const text = item.event.text ?? "";
+            tokenFrames += 1;
+            draft = item.event.reset ? text : draft + text;
+            options.onToken?.(item.event);
+            continue;
+        }
         activity.push(item.event);
         options.onActivity?.(item.event);
     }
@@ -117,6 +127,8 @@ export async function readTowerTurn(response, options = {}) {
         options.log?.(`[tower stream] turn incomplete ${JSON.stringify({
             reason,
             activity_events: activity.length,
+            token_frames: tokenFrames,
+            drafted_chars: draft.length,
             notes: notes.length,
             elapsed_ms: elapsedMs,
         })}`);
@@ -125,10 +137,14 @@ export async function readTowerTurn(response, options = {}) {
     options.log?.(`[tower stream] turn complete ${JSON.stringify({
         streamed,
         activity_events: activity.length,
+        // BLI-3517: zero token frames against a streaming dashboard means the
+        // answer arrived all at once — worth telling apart from a quiet turn.
+        token_frames: tokenFrames,
+        drafted_chars: draft.length,
         notes: notes.length,
         elapsed_ms: elapsedMs,
     })}`);
-    return { ok: true, final, activity, streamed };
+    return { ok: true, final, activity, streamed, draft };
 }
 /** One plain sentence per way a turn can end without an answer. */
 export function streamFailureDetail(reason, detail) {
@@ -207,7 +223,7 @@ function parseLine(line) {
         return { kind: "note", reason: "unexpected_event_type" };
     }
     const type = parsed.type;
-    if (type !== "activity" && type !== "final") {
+    if (type !== "activity" && type !== "token" && type !== "final") {
         return { kind: "note", reason: "unexpected_event_type" };
     }
     return { kind: "event", event: parsed };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bli-cockpit/cli",
-  "version": "0.2.48",
+  "version": "0.2.49",
   "private": false,
   "type": "module",
   "bin": {