@bli-cockpit/cli 0.2.95 → 0.2.97

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,80 @@
1
+ import { describeError } from "../health-detail.js";
2
+ import { getCollectorRuntimePaths, readLocalCollectorSessionFile, recordDeviceTokenExpiry, toSessionReference, } from "../local-state.js";
3
+ /**
4
+ * Read the session and decide whether this tick may check in.
5
+ *
6
+ * The check-in goes out while the machine believes it is EXPIRED, as long as it
7
+ * still holds a device token. That is not a loosening: the SERVER decides
8
+ * whether a token is accepted, and a collector that refused to knock could
9
+ * never be let back in. Uploads are unaffected — they still gate on `valid`
10
+ * (`upload-envelope-build.ts`), so a machine past the grace window spends no
11
+ * bandwidth on evidence that will be refused.
12
+ *
13
+ * Both refusing branches say something, because "the dashboard shows this
14
+ * machine as quiet" has two very different causes.
15
+ */
16
+ export async function readHeartbeatSession(homeDir) {
17
+ const paths = getCollectorRuntimePaths(homeDir);
18
+ const session = await readLocalCollectorSessionFile(paths).catch(() => null);
19
+ const state = session ? toSessionReference(session).session_state : "missing";
20
+ const usable = state === "valid" || state === "expired";
21
+ if (!session ||
22
+ !usable ||
23
+ typeof session.device_token !== "string" ||
24
+ !session.device_token) {
25
+ console.error("[heartbeat] no usable device session on this machine; the dashboard will show it as quiet", JSON.stringify({
26
+ reason: "no_device_session",
27
+ session_state: state,
28
+ next_action: "run `cockpit do-everything` to pair this machine again",
29
+ }));
30
+ return { usable: false };
31
+ }
32
+ if (state === "expired") {
33
+ console.error("[heartbeat] this machine's session looks expired; checking in anyway so the dashboard can renew it", JSON.stringify({
34
+ reason: "session_expired_locally",
35
+ expires_at: session.expires_at ?? null,
36
+ next_action: "the dashboard renews a token that lapsed inside its 30-day grace window on this call",
37
+ }));
38
+ }
39
+ return { usable: true, session, state: state };
40
+ }
41
+ /**
42
+ * Record the `token_expires_at` the heartbeat door answered with.
43
+ *
44
+ * The LOCAL copy is what decides whether this machine even attempts an upload,
45
+ * so a machine the server has just renewed would otherwise go on refusing its
46
+ * own uploads until somebody signed in by hand.
47
+ *
48
+ * A body that is not JSON, or that carries no such field, is not an error and
49
+ * is not logged: an older dashboard answers `{ ok: true }` and a proxy can
50
+ * answer anything. It simply means "this reply said nothing about the expiry",
51
+ * and the machine keeps the date it already holds. Best-effort throughout — a
52
+ * heartbeat never fails a sync.
53
+ */
54
+ export async function recordHeartbeatTokenExpiry(homeDir, response) {
55
+ const expiresAt = await readTokenExpiry(response);
56
+ if (!expiresAt)
57
+ return;
58
+ await recordDeviceTokenExpiry({
59
+ paths: getCollectorRuntimePaths(homeDir),
60
+ expiresAt,
61
+ }).catch((error) => {
62
+ console.error("[heartbeat] the token expiry the dashboard reported was not recorded", JSON.stringify({
63
+ reason: "token_expiry_writeback_threw",
64
+ ...describeError(error),
65
+ }));
66
+ return null;
67
+ });
68
+ }
69
+ async function readTokenExpiry(response) {
70
+ try {
71
+ const body = (await response.json());
72
+ if (!body || typeof body !== "object")
73
+ return null;
74
+ const value = body["token_expires_at"];
75
+ return typeof value === "string" && value ? value : null;
76
+ }
77
+ catch {
78
+ return null;
79
+ }
80
+ }
@@ -28,7 +28,9 @@ import { readCachedSetupReceipt } from "./setup-receipt.js";
28
28
  import { readStagingInventory } from "../disk-usage.js";
29
29
  import { describeError } from "../health-detail.js";
30
30
  import { shouldSuppressFleetReceipts, } from "../dev-build.js";
31
- import { getCollectorRuntimePaths, readLocalCollectorSessionFile, LOCAL_COLLECTOR_VERSION, } from "../local-state.js";
31
+ import { getCollectorRuntimePaths, LOCAL_COLLECTOR_VERSION, } from "../local-state.js";
32
+ import { readHeartbeatSession, recordHeartbeatTokenExpiry, } from "./heartbeat-token.js";
33
+ import { classifySyncHealthError } from "../sync-health-class.js";
32
34
  import { readMemoryReceiptFile } from "./memory-install-receipt.js";
33
35
  import { readMemoryHookCounts } from "./memory-hook-counts.js";
34
36
  import { readMemoryHookPerformance } from "./memory-hook-performance.js";
@@ -232,18 +234,14 @@ export async function sendCollectorHeartbeatBestEffort(options) {
232
234
  }));
233
235
  return false;
234
236
  }
235
- const paths = getCollectorRuntimePaths(options.homeDir);
236
- const session = await readLocalCollectorSessionFile(paths).catch(() => null);
237
- if (!session ||
238
- session.session_state !== "valid" ||
239
- typeof session.device_token !== "string" ||
240
- !session.device_token) {
241
- console.error("[heartbeat] no valid device session on this machine; the dashboard will show it as quiet", JSON.stringify({
242
- reason: "no_device_session",
243
- next_action: "run `cockpit do-everything` to pair this machine again",
244
- }));
237
+ // BLI-4019. Whether this machine may knock at all, and the two log lines
238
+ // that go with it, live in `heartbeat-token.ts` the check-in goes out even
239
+ // when the local copy says `expired`, because that is the door the server
240
+ // renews a lapsed token through.
241
+ const reading = await readHeartbeatSession(options.homeDir);
242
+ if (!reading.usable)
245
243
  return false;
246
- }
244
+ const { session } = reading;
247
245
  const memory = await readHeartbeatMemoryReceipt({
248
246
  ...(options.homeDir ? { homeDir: options.homeDir } : {}),
249
247
  ...(options.now ? { now: options.now } : {}),
@@ -287,13 +285,27 @@ export async function sendCollectorHeartbeatBestEffort(options) {
287
285
  });
288
286
  if (!response.ok) {
289
287
  // A heartbeat is not retried, so the reason has to be said once, here.
288
+ // BLI-4019: the CLASS comes from `sync-health-class.ts`, the one place
289
+ // that decides what a failure is — a 401/403 is `auth_failed` and
290
+ // anything else is not, read off the observed status and never off the
291
+ // words in a message (BLI-3551's rule).
290
292
  console.error("[heartbeat] the dashboard refused this tick's heartbeat", JSON.stringify({
291
293
  reason: "heartbeat_rejected",
294
+ health_class: classifySyncHealthError({ httpStatus: response.status }),
292
295
  http_status: response.status,
293
296
  root_count: heartbeat.roots.length,
297
+ ...(response.status === 401 || response.status === 403
298
+ ? {
299
+ next_action: "past the 30-day grace window this needs `cockpit login` on this machine",
300
+ }
301
+ : {}),
294
302
  }));
295
303
  return false;
296
304
  }
305
+ // BLI-4019. The server slides `token_expires_at` out on every call it
306
+ // authenticates and answers with the result; the sibling records it, so
307
+ // this machine's own copy follows the server rather than drifting.
308
+ await recordHeartbeatTokenExpiry(options.homeDir, response);
297
309
  console.error("[heartbeat] checked in", JSON.stringify({
298
310
  reason: "heartbeat_recorded",
299
311
  http_status: response.status,
@@ -15,7 +15,7 @@ export async function runCockpitCli(argv, io) {
15
15
  }
16
16
 
17
17
  if (command === "--version" || command === "-V" || command === "version") {
18
- writeLine(io?.stdout ?? process.stdout, "0.2.95");
18
+ writeLine(io?.stdout ?? process.stdout, "0.2.97");
19
19
  return 0;
20
20
  }
21
21
 
@@ -13,6 +13,7 @@
13
13
  * without saying why.
14
14
  */
15
15
  import { claudeAttributionReadFailureCount, codexAttributionReadFailureCount, } from "./agent-session-report.js";
16
+ import { isDeliveryHoldReason } from "./session-sync-hold.js";
16
17
  import { sourceScanFailureReason } from "./session-sync-scan.js";
17
18
  /**
18
19
  * The label used when a sync fails and nothing on the way there said why.
@@ -45,6 +46,15 @@ export function createSyncFailureLedger() {
45
46
  * The spooled reason is the most specific thing anyone has, so it leads, and it
46
47
  * names the worktree it belongs to — a fleet failure is usually one repo, and
47
48
  * "which one" is the first question asked.
49
+ *
50
+ * BLI-4022: `delivery_backoff_holding` is the one raw-evidence reason that does
51
+ * NOT come in here. An object inside its own retry window was withheld on
52
+ * purpose and will be offered again; filing that as a failed tick is what
53
+ * painted Ian's Mac mini red on eight consecutive ticks while every session it
54
+ * collected landed. The hold is still reported — it becomes the tick's named
55
+ * hold in `session-sync-hold.ts` and rides the same receipt — and a tick that
56
+ * ALSO failed for a real reason still fails on that reason. Nothing is
57
+ * silenced; only the verdict changed.
48
58
  */
49
59
  export function recordWorktreeDeliveryFailures(ledger, outcomes) {
50
60
  for (const { worktree, sync } of outcomes) {
@@ -61,12 +71,16 @@ export function recordWorktreeDeliveryFailures(ledger, outcomes) {
61
71
  });
62
72
  }
63
73
  for (const reason of sync.raw_evidence_failure_reasons ?? []) {
74
+ if (isDeliveryHoldReason(reason))
75
+ continue;
64
76
  ledger.add({
65
77
  label: "raw_evidence_upload_failed",
66
78
  rendered: `raw_evidence:${reason}`,
67
79
  });
68
80
  }
69
81
  for (const reason of sync.raw_evidence_retry_reasons ?? []) {
82
+ if (isDeliveryHoldReason(reason))
83
+ continue;
70
84
  ledger.add({
71
85
  label: "raw_evidence_retry_required",
72
86
  rendered: `raw_evidence_retry:${reason}`,
@@ -11,6 +11,7 @@
11
11
  */
12
12
  import { createSyncFailureLedger, recordSourceScanFailures, recordUnexplainedFailure, recordUnpostedSessionReportFailure, recordWorktreeDeliveryFailures, } from "./session-sync-failures.js";
13
13
  import { countSessionsNewThisTick, countSessionsPendingUpload, } from "./session-sync-counters.js";
14
+ import { logDeliveryHold, summarizeDeliveryHold } from "./session-sync-hold.js";
14
15
  /**
15
16
  * Decide whether this tick failed, and record every condition that decided it.
16
17
  *
@@ -44,12 +45,20 @@ export function decideSyncHealth(options) {
44
45
  if (nothingInRoot !== null && notice) {
45
46
  announceNothingInRoot(nothingInRoot, options.collectionRootCount);
46
47
  }
48
+ // BLI-4022. Computed after `ok`, and never allowed to change it: a hold is
49
+ // what this tick withheld on purpose, not a verdict on the run. It rides an
50
+ // `ok` receipt as the tick's reason and a failing one beside the reasons that
51
+ // failed it, so the board can say what is waiting in either case.
52
+ const hold = summarizeDeliveryHold(outcomes);
53
+ if (hold && ok)
54
+ logDeliveryHold(hold);
47
55
  const records = ledger.sortedRecords();
48
56
  return {
49
57
  ok,
50
58
  failure_reasons: records.map((record) => record.rendered),
51
59
  failure_records: records,
52
60
  notice,
61
+ hold,
53
62
  sessions_outside_root: sessionsOutsideRoot,
54
63
  sessions_new_this_tick: countSessionsNewThisTick({
55
64
  sessions: options.sessions,
@@ -0,0 +1,90 @@
1
+ /**
2
+ * What this tick is HOLDING, as opposed to what it got wrong (BLI-4022).
3
+ *
4
+ * The sibling `session-sync-failures.ts` is the one place a sync writes down
5
+ * that it FAILED. This is the one place it writes down that it deliberately
6
+ * withheld something and when it will try again — a different question with a
7
+ * different answer, kept in a different file so neither can quietly become the
8
+ * other.
9
+ *
10
+ * Deliberately writes no failure label of any kind: `sync-health-class.test.ts`
11
+ * reads every `session-sync*.ts` sibling and demands that only the failure
12
+ * ledger writes one, and this file is not it.
13
+ */
14
+ import { formatDeliveryHoldNotice, } from "../delivery-hold-notice.js";
15
+ import { DELIVERY_BACKOFF_HOLDING_REASON } from "../raw-evidence-staging.js";
16
+ /**
17
+ * Is this reason a delivery hold rather than a delivery failure?
18
+ *
19
+ * Exact match against the label telemetry-core owns — the same discipline the
20
+ * failure classifier follows. A reason that merely CONTAINS the word "backoff"
21
+ * is not evidence of anything (BLI-3551's lesson, applied to the other side of
22
+ * the same fence).
23
+ */
24
+ export function isDeliveryHoldReason(reason) {
25
+ return reason === DELIVERY_BACKOFF_HOLDING_REASON;
26
+ }
27
+ /**
28
+ * Fold every worktree's held objects into one hold for the tick, or null.
29
+ *
30
+ * The count is summed because a tick can sync several worktrees and each one
31
+ * reports its own held objects. The AGE takes the oldest across all of them and
32
+ * the NEXT ATTEMPT the earliest, because both are promises about the whole
33
+ * machine: the operator asks "how long has anything been stuck" and "when does
34
+ * anything move", not "which worktree".
35
+ *
36
+ * Returns null when nothing is held, so no caller can announce a hold of
37
+ * nothing — a status line that appears on every tick is one people stop
38
+ * reading.
39
+ */
40
+ export function summarizeDeliveryHold(outcomes) {
41
+ let objectCount = 0;
42
+ let oldestFirstFailedAt = null;
43
+ let nextAttemptAt = null;
44
+ let lastReason = null;
45
+ for (const { sync } of outcomes) {
46
+ const held = sync.raw_evidence_delivery_held_count ?? 0;
47
+ if (held <= 0)
48
+ continue;
49
+ objectCount += held;
50
+ const first = sync.raw_evidence_held_oldest_failure_at ?? null;
51
+ if (first && (!oldestFirstFailedAt || first < oldestFirstFailedAt)) {
52
+ oldestFirstFailedAt = first;
53
+ // The reason travels WITH the object it belongs to. A reason taken from
54
+ // one worktree and an age from another would describe no object at all.
55
+ lastReason = sync.raw_evidence_held_last_reason ?? null;
56
+ }
57
+ const next = sync.raw_evidence_held_next_attempt_at ?? null;
58
+ if (next && (!nextAttemptAt || next < nextAttemptAt))
59
+ nextAttemptAt = next;
60
+ }
61
+ if (objectCount <= 0)
62
+ return null;
63
+ const notice = formatDeliveryHoldNotice({
64
+ objectCount,
65
+ oldestFirstFailedAt,
66
+ nextAttemptAt,
67
+ lastReason,
68
+ });
69
+ if (!notice)
70
+ return null;
71
+ return { objectCount, oldestFirstFailedAt, nextAttemptAt, lastReason, notice };
72
+ }
73
+ /**
74
+ * Say out loud, once per tick, that bytes are being withheld on purpose.
75
+ *
76
+ * `logEvidenceHeldByBackoff` already names the objects at the moment they are
77
+ * partitioned off the wire; this is the RUN's line, so the tick that reports
78
+ * itself as held also explains itself on stderr where launchd captures it.
79
+ * Counts, instants and reason labels only.
80
+ */
81
+ export function logDeliveryHold(hold) {
82
+ console.error("[session-sync] raw evidence held by delivery backoff, nothing else failed", JSON.stringify({
83
+ reason: hold.notice,
84
+ object_count: hold.objectCount,
85
+ oldest_first_failed_at: hold.oldestFirstFailedAt,
86
+ next_attempt_at: hold.nextAttemptAt,
87
+ last_reason: hold.lastReason,
88
+ next_action: "nothing until the next attempt is due; run `cockpit doctor` if it is already past",
89
+ }));
90
+ }
@@ -25,6 +25,9 @@
25
25
  * class-lock test reads that file (BLI-3551).
26
26
  * - `session-sync-health.ts` — the verdict, and the notice a quiet-but-working
27
27
  * machine still emits.
28
+ * - `session-sync-hold.ts` — what the tick withheld ON PURPOSE and when it will
29
+ * try again (BLI-4022), which is a different question from what it got wrong
30
+ * and so lives in a different file from the failure ledger.
28
31
  * - `session-sync-counters.ts` — the two honest heartbeat counters beside the
29
32
  * raw scan count: sessions new since the last tick's cursor, and sessions
30
33
  * still pending a durable pointer (BLI-3645).
@@ -49,6 +52,7 @@ export { isLiveSyncCollectableAttributionState, liveSyncCursorEntryRequiresRetry
49
52
  export { sourceScanFailureReason, sourceScanRetryReason, } from "./session-sync-scan.js";
50
53
  export { SYNC_FAILED_WITHOUT_REASON } from "./session-sync-failures.js";
51
54
  export { nothingInRootCount } from "./session-sync-health.js";
55
+ export { isDeliveryHoldReason, summarizeDeliveryHold, } from "./session-sync-hold.js";
52
56
  /**
53
57
  * Shared dual-source sync orchestration for single-repo and parent-folder
54
58
  * modes. Codex AND Claude Code sessions are scanned and attributed once across
@@ -126,6 +130,7 @@ export async function runAttributedWorktreeSync(options) {
126
130
  failure_reasons: health.failure_reasons,
127
131
  failure_records: health.failure_records,
128
132
  notice: health.notice,
133
+ hold: health.hold,
129
134
  sessions_observed: sessions.length,
130
135
  sessions_outside_root: health.sessions_outside_root,
131
136
  sessions_new_this_tick: health.sessions_new_this_tick,
@@ -21,6 +21,20 @@ const FIXES = {
21
21
  "codex.hooks": "Run `cockpit memory install`.",
22
22
  "collector.autostart": "Run `cockpit autostart install`.",
23
23
  };
24
+ /**
25
+ * A fix keyed on the piece's REASON rather than the piece, for the cases where
26
+ * the same gap has two different next actions (BLI-4019).
27
+ *
28
+ * `device` says "run `cockpit login`" for every lapse, and for an expired
29
+ * session that is now the wrong instruction most of the time: the dashboard
30
+ * slides a device token's expiry out on every call it authenticates and renews
31
+ * one that lapsed inside a 30-day grace window, so the ordinary answer is to do
32
+ * nothing and let the next tick fix it. Telling somebody to sign in again on a
33
+ * machine that is about to heal itself is how a receipt stops being believed.
34
+ */
35
+ const REASON_FIXES = {
36
+ session_expired: "Renews on the next sync tick while inside the 30-day grace window; past it, run `cockpit login`.",
37
+ };
24
38
  /** Codex hooks a person switched OFF is a decision, not a fault. */
25
39
  const UNSUPPORTED_FIX = "Switched off in your own config; nothing to do.";
26
40
  /**
@@ -64,6 +78,9 @@ function fixFor(key, piece) {
64
78
  if (piece.status === "skipped") {
65
79
  return "Skipped on purpose; nothing to do.";
66
80
  }
81
+ // The reason wins over the piece when it has its own next action.
82
+ if (piece.reason && REASON_FIXES[piece.reason])
83
+ return REASON_FIXES[piece.reason];
67
84
  return FIXES[key] ?? "Run `cockpit doctor`.";
68
85
  }
69
86
  function reasonSuffix(piece) {
@@ -91,16 +91,25 @@ function standAsideForRunningSync(command, io, heldSince) {
91
91
  * is the receipt that separates "this machine is alive and its operator
92
92
  * works outside the approved roots" from "this machine is dead", which
93
93
  * until now looked identical from the dashboard.
94
+ *
95
+ * BLI-4022 added the second thing an `ok` tick can say: `held_backoff:…`, the
96
+ * objects this tick withheld on purpose and when they may be offered again.
97
+ * The two can never collide — `nothing_in_root` requires that NO worktree
98
+ * synced at all (`nothingInRootCount`), and a hold requires a worktree that
99
+ * did — so one field carries whichever is true, and the notice wins if that
100
+ * ever stops being the case, because "there was nothing to collect" is the
101
+ * larger fact about the tick.
94
102
  */
95
- function collectedReceipt(run) {
103
+ export function collectedReceipt(run) {
104
+ const reason = run.notice ?? run.holdNotice;
96
105
  return {
97
106
  exitCode: run.exitCode,
98
107
  completion: {
99
108
  step: "sync_complete",
100
109
  status: "ok",
101
- ...(run.notice ? { error_detail: run.notice } : {}),
110
+ ...(reason ? { error_detail: reason } : {}),
102
111
  },
103
- heartbeat: { status: "ok", reason: run.notice, ...heartbeatCounts(run) },
112
+ heartbeat: { status: "ok", reason, ...heartbeatCounts(run) },
104
113
  };
105
114
  }
106
115
  /**
@@ -111,8 +120,14 @@ function collectedReceipt(run) {
111
120
  * failure rows carried a null detail and the real reason was reachable only
112
121
  * by running `cockpit status` on the machine itself (BLI-2526).
113
122
  */
114
- function failedReceipt(run) {
115
- const reasonText = run.failureReasons.join("; ");
123
+ export function failedReceipt(run) {
124
+ // BLI-4022: the hold leads on a failing tick. `error_detail` is bounded, and
125
+ // an operator reading a red row still has to be able to see what is merely
126
+ // waiting — otherwise the held objects become invisible the moment anything
127
+ // else goes wrong, which is the opposite of the fix.
128
+ const reasonText = [run.holdNotice, ...run.failureReasons]
129
+ .filter(Boolean)
130
+ .join("; ");
116
131
  // The bucket comes from the records the deciding branches wrote, not from
117
132
  // this sentence (BLI-3551). The sentence is still the detail.
118
133
  const errorCode = classifySyncFailureRecords(run.failureRecords);
@@ -141,6 +141,9 @@ function syncResult(run) {
141
141
  failureReasons: run.ok ? [] : run.failure_reasons,
142
142
  failureRecords: run.ok ? [] : run.failure_records,
143
143
  notice: run.notice,
144
+ // BLI-4022: unlike the reasons, the hold survives a successful run — it is
145
+ // what the tick withheld, not why it failed.
146
+ holdNotice: run.hold?.notice ?? null,
144
147
  sessionsObserved: run.sessions_observed,
145
148
  sessionsOutsideRoot: run.sessions_outside_root,
146
149
  sessionsNewThisTick: run.sessions_new_this_tick,
@@ -0,0 +1,166 @@
1
+ /**
2
+ * A HOLD is not a FAILURE — and this file is the one place that spells it.
3
+ *
4
+ * BLI-4022. `delivery_backoff_holding` means "this object is inside its own
5
+ * retry window, so this tick did not offer it". Until now that reason went
6
+ * into the sync's failure ledger like any other, so a machine collecting
7
+ * perfectly reported `sync_failed` on every tick for as long as the backoff
8
+ * held: Ian's Mac mini was RED on eight consecutive ticks from
9
+ * 2026-09-09T00:11Z with the detail
10
+ * `raw_evidence_retry:delivery_backoff_holding; raw_evidence:delivery_backoff_holding`
11
+ * while its sessions kept landing and its heartbeat kept moving.
12
+ *
13
+ * The honesty is unchanged. A held object IS missing collection and must never
14
+ * read as clean (that is the rule BLI-3066 bought). What changes is the
15
+ * OUTCOME: a tick whose only raw-evidence complaint is held objects reports a
16
+ * named hold instead of a failure, and the hold carries everything an operator
17
+ * needs to decide whether to care — how many objects, how long the oldest has
18
+ * been failing, when the next attempt is due, and the object's own last reason.
19
+ *
20
+ * ## The grammar
21
+ *
22
+ * One token, because it has to survive two narrow channels: the heartbeat's
23
+ * `last_sync_reason` (`^[a-z0-9_:.-]+$`, 120 chars, no spaces) and the receipt's
24
+ * `error_detail`. Fields are positional and colon-separated:
25
+ *
26
+ * held_backoff:<count>:<oldest_first_failed_at>:<next_attempt_at>:<last_reason>
27
+ * held_backoff:3:2026-08-14T17:38Z:2026-09-09T06:11Z:chunk_10_failed_http_500
28
+ *
29
+ * Timestamps are minute precision UTC; `-` stands for a field this machine
30
+ * could not fill. The last reason is bounded so the whole token fits the
31
+ * heartbeat's 120 characters — the count and the ages are what a colour is
32
+ * decided from, so the reason is the field that gives way.
33
+ *
34
+ * The collector WRITES the token and the dashboard's fleet board READS it
35
+ * (`lib/ops/fleet-liveness-device-verdict.ts`, through this package's
36
+ * `./delivery-hold` export). One grammar, one file: a second copy of this
37
+ * string is exactly how the collector's word and the server's word drift
38
+ * apart, which is why `DELIVERY_BACKOFF_HOLDING` itself is aliased from
39
+ * telemetry-core rather than spelled twice.
40
+ */
41
+ /** The token's leading word. Greppable in a receipt, a log line or a DM. */
42
+ export const DELIVERY_HOLD_NOTICE_PREFIX = "held_backoff";
43
+ /** The heartbeat's own ceiling for `last_sync_reason`. The token fits inside it. */
44
+ export const DELIVERY_HOLD_NOTICE_MAX_CHARS = 120;
45
+ /**
46
+ * How long a hold may last before it stops being a wait and becomes a stuck
47
+ * object.
48
+ *
49
+ * The backoff schedule doubles from the scheduler's own 15-minute cadence to a
50
+ * six-hour ceiling (`EVIDENCE_DELIVERY_BACKOFF_MAX_MS`), so an object held
51
+ * longer than this has already been offered at least four more times at the
52
+ * ceiling and been refused every time. Nothing is going to change on its own,
53
+ * and "next retry at T — nothing to do until then" would be a promise the
54
+ * machine cannot keep. 24 hours is also the board's own `SILENT_AFTER_HOURS`,
55
+ * so both clocks agree on when a quiet thing stops being a blip.
56
+ */
57
+ export const DELIVERY_HOLD_STALE_AFTER_HOURS = 24;
58
+ /** The most characters of an object's own reason label the token may carry. */
59
+ const MAX_HOLD_REASON_CHARS = 64;
60
+ /** A field this machine could not fill. Never omitted — the shape is positional. */
61
+ const ABSENT = "-";
62
+ /**
63
+ * Render the hold as the single token the heartbeat and the receipt carry.
64
+ *
65
+ * Returns null for a hold of nothing, so a caller cannot accidentally announce
66
+ * a hold that is not happening.
67
+ */
68
+ export function formatDeliveryHoldNotice(notice) {
69
+ if (notice.objectCount <= 0)
70
+ return null;
71
+ const head = [
72
+ DELIVERY_HOLD_NOTICE_PREFIX,
73
+ String(Math.trunc(notice.objectCount)),
74
+ minuteStamp(notice.oldestFirstFailedAt),
75
+ minuteStamp(notice.nextAttemptAt),
76
+ ].join(":");
77
+ const budget = Math.min(MAX_HOLD_REASON_CHARS, DELIVERY_HOLD_NOTICE_MAX_CHARS - head.length - 1);
78
+ return `${head}:${reasonWord(notice.lastReason, budget)}`;
79
+ }
80
+ /**
81
+ * Find a hold token inside anything a receipt carries, and read it back.
82
+ *
83
+ * Deliberately a SEARCH, not a whole-string match: on a tick that also failed
84
+ * for a real reason the token rides beside those reasons in the same
85
+ * `error_detail`, and the board must still be able to say what is held.
86
+ */
87
+ export function parseDeliveryHoldNotice(text) {
88
+ if (!text)
89
+ return null;
90
+ // The stamp shape is pinned rather than described by a character class: a
91
+ // class containing `:` is greedy across the field separator, and a parser
92
+ // that swallowed one field into the next reported an age of twenty-five
93
+ // years and a next attempt in 2001.
94
+ const stamp = String.raw `(?:-|\d{4}-\d{2}-\d{2}T\d{2}:\d{2}Z)`;
95
+ const match = new RegExp(`${DELIVERY_HOLD_NOTICE_PREFIX}:(\\d+):(${stamp}):(${stamp}):([a-z0-9_-]+)`, "iu").exec(text);
96
+ if (!match)
97
+ return null;
98
+ const objectCount = Number.parseInt(match[1], 10);
99
+ if (!Number.isFinite(objectCount) || objectCount <= 0)
100
+ return null;
101
+ return {
102
+ objectCount,
103
+ oldestFirstFailedAt: readStamp(match[2]),
104
+ nextAttemptAt: readStamp(match[3]),
105
+ lastReason: match[4] === ABSENT ? null : match[4],
106
+ };
107
+ }
108
+ /**
109
+ * How long the longest-held object has been failing, in hours.
110
+ *
111
+ * Null when the token carried no first-failure instant, which is a different
112
+ * fact from "zero hours" — an unknown age is never treated as a fresh hold.
113
+ */
114
+ export function deliveryHoldAgeHours(notice, nowIso) {
115
+ if (!notice.oldestFirstFailedAt)
116
+ return null;
117
+ const from = Date.parse(notice.oldestFirstFailedAt);
118
+ const now = Date.parse(nowIso);
119
+ if (!Number.isFinite(from) || !Number.isFinite(now))
120
+ return null;
121
+ return (now - from) / 3_600_000;
122
+ }
123
+ /**
124
+ * Has this hold stopped being a wait?
125
+ *
126
+ * An age this reader could not compute counts as stale on purpose: the whole
127
+ * point of the amber sentence is the promise "next retry at T", and a hold
128
+ * whose age is unknown is one nobody can make that promise about.
129
+ */
130
+ export function isDeliveryHoldStale(notice, nowIso) {
131
+ const hours = deliveryHoldAgeHours(notice, nowIso);
132
+ if (hours === null)
133
+ return true;
134
+ return hours > DELIVERY_HOLD_STALE_AFTER_HOURS;
135
+ }
136
+ /** `2026-09-09T06:11Z`, or `-` for an instant this machine does not have. */
137
+ function minuteStamp(iso) {
138
+ if (!iso)
139
+ return ABSENT;
140
+ const parsed = Date.parse(iso);
141
+ if (!Number.isFinite(parsed))
142
+ return ABSENT;
143
+ return `${new Date(parsed).toISOString().slice(0, 16)}Z`;
144
+ }
145
+ /** The inverse: a token field back to an ISO instant, or null when absent. */
146
+ function readStamp(field) {
147
+ if (field === ABSENT)
148
+ return null;
149
+ const parsed = Date.parse(field);
150
+ return Number.isFinite(parsed) ? new Date(parsed).toISOString() : null;
151
+ }
152
+ /**
153
+ * A reason label narrowed to what the token's charset and budget allow.
154
+ *
155
+ * Truncation is marked by dropping characters, never by an ellipsis: the
156
+ * heartbeat's charset has no `…`, and a token the door rejects tells nobody
157
+ * anything at all.
158
+ */
159
+ function reasonWord(reason, budget) {
160
+ if (!reason || budget <= 0)
161
+ return ABSENT;
162
+ const safe = reason.replace(/[^a-z0-9_]/giu, "_").replace(/^_+|_+$/gu, "");
163
+ if (safe.length === 0)
164
+ return ABSENT;
165
+ return safe.slice(0, budget);
166
+ }
@@ -7,7 +7,7 @@ import { serverFailureDetail } from "./upload-http.js";
7
7
  import { ensureRuntimeDirectories, getCollectorRuntimePaths, } from "./local-state-paths.js";
8
8
  import { isMissingFileError, writeJsonFile } from "./local-state-files.js";
9
9
  import { defaultDeviceName, LOCAL_COLLECTOR_VERSION, normalizeDashboardUrl, normalizeDeviceName, normalizeOptionalEmail, readLocalCollectorConfig, } from "./local-state-config.js";
10
- import { toSessionReference } from "./local-state-session.js";
10
+ import { readLocalCollectorSessionFile, toSessionReference, } from "./local-state-session.js";
11
11
  /**
12
12
  * `cockpit login` — the whole device-pairing handshake, because a machine can
13
13
  * collect locally without it but can never upload until the dashboard has
@@ -106,6 +106,46 @@ export async function logoutLocalCollector(options = {}) {
106
106
  }
107
107
  return { removed, session_file: paths.session_file };
108
108
  }
109
+ export async function recordDeviceTokenExpiry(options) {
110
+ const parsed = Date.parse(options.expiresAt);
111
+ if (!Number.isFinite(parsed)) {
112
+ console.error("[local-state] the dashboard's token expiry was not a date; this machine keeps the one it has", JSON.stringify({ reason: "token_expiry_unreadable" }));
113
+ return { status: "skipped", reason: "token_expiry_unreadable" };
114
+ }
115
+ let session;
116
+ try {
117
+ session = await readLocalCollectorSessionFile(options.paths);
118
+ }
119
+ catch (error) {
120
+ console.error("[local-state] no readable session file to record the token expiry on", JSON.stringify({
121
+ reason: "session_file_unreadable",
122
+ ...describeError(error),
123
+ }));
124
+ return { status: "skipped", reason: "session_file_unreadable" };
125
+ }
126
+ if (session.expires_at === options.expiresAt) {
127
+ return { status: "unchanged", expires_at: options.expiresAt };
128
+ }
129
+ try {
130
+ await writeJsonFile(options.paths.session_file, {
131
+ ...session,
132
+ expires_at: options.expiresAt,
133
+ });
134
+ }
135
+ catch (error) {
136
+ console.error("[local-state] the renewed token expiry could not be written; this machine still believes the old one", JSON.stringify({
137
+ reason: "session_file_write_failed",
138
+ ...describeError(error),
139
+ }));
140
+ return { status: "skipped", reason: "session_file_write_failed" };
141
+ }
142
+ console.error("[local-state] recorded the token expiry the dashboard reported", JSON.stringify({
143
+ reason: "token_expiry_recorded",
144
+ previous_expires_at: session.expires_at ?? null,
145
+ expires_at: options.expiresAt,
146
+ }));
147
+ return { status: "written", expires_at: options.expiresAt };
148
+ }
109
149
  async function postPairStart(fetchImpl, dashboardUrl, body, accessToken) {
110
150
  const response = await fetchImpl(`${dashboardUrl}/api/ambient/pair/start`, {
111
151
  method: "POST",
@@ -37,8 +37,8 @@
37
37
  */
38
38
  export { getCollectorRuntimePaths } from "./local-state-paths.js";
39
39
  export { DEFAULT_DASHBOARD_URL, ensureLocalCollectorConfig, installLocalCollector, LOCAL_COLLECTOR_VERSION, readLocalCollectorConfig, } from "./local-state-config.js";
40
- export { readLocalCollectorSessionFile, readLocalSessionReference, } from "./local-state-session.js";
41
- export { logoutLocalCollector, pairLocalCollector, } from "./local-state-pairing.js";
40
+ export { readLocalCollectorSessionFile, readLocalSessionReference, toSessionReference, } from "./local-state-session.js";
41
+ export { logoutLocalCollector, pairLocalCollector, recordDeviceTokenExpiry, } from "./local-state-pairing.js";
42
42
  export { PairingCodeError, pairLocalCollectorViaLink, } from "./local-state-pairing-code.js";
43
43
  export { resolveGitBranch } from "./local-state-identity.js";
44
44
  export { readLocalWorkContext, readLocalWorkContextForRepo, startLocalWorkContext, startLocalWorkContextForAttributedTarget, } from "./local-state-work-context.js";
@@ -25,6 +25,10 @@ import { describeError, isMissingFileFailure } from "./health-detail.js";
25
25
  * Metadata only: content hashes, byte sizes, pack ids, pack-relative paths,
26
26
  * reason labels, timestamps. Never content, never an absolute path.
27
27
  */
28
+ // BLI-4022. The stuck/held summary moved to its own sibling so this file
29
+ // stays under the readability band; it is still importable from here,
30
+ // which is the address every caller already knows.
31
+ export { summarizeStuckEvidence, } from "./raw-evidence-stuck-summary.js";
28
32
  export const RAW_EVIDENCE_STAGING_FILENAME = "raw-evidence-staging.json";
29
33
  /**
30
34
  * Backoff schedule for an object whose delivery keeps failing.
@@ -207,33 +211,6 @@ export function evidenceSourceKey(options) {
207
211
  : "unknown";
208
212
  return `${options.kind}:${options.sessionId ?? "none"}:${source}`;
209
213
  }
210
- /**
211
- * What `cockpit status` and the health receipt need in order to be unable to
212
- * read green while an object has been failing for nine days.
213
- */
214
- export function summarizeStuckEvidence(state, now) {
215
- const entries = Object.values(state.delivery_attempts);
216
- const reasons = new Set();
217
- let heldCount = 0;
218
- let maxAttempts = 0;
219
- let oldest = null;
220
- for (const entry of entries) {
221
- reasons.add(entry.last_reason);
222
- if (Date.parse(entry.next_attempt_at) > now.getTime())
223
- heldCount += 1;
224
- maxAttempts = Math.max(maxAttempts, entry.attempts);
225
- if (!oldest || entry.first_failed_at.localeCompare(oldest) < 0) {
226
- oldest = entry.first_failed_at;
227
- }
228
- }
229
- return {
230
- stuck_object_count: entries.length,
231
- held_object_count: heldCount,
232
- max_attempts: maxAttempts,
233
- oldest_first_failed_at: oldest,
234
- reasons: [...reasons].sort(),
235
- };
236
- }
237
214
  /**
238
215
  * Pack id derived from CONTENT, never from the clock.
239
216
  *
@@ -0,0 +1,60 @@
1
+ /**
2
+ * "Is anything stuck, and is anything being held right now?" — the two
3
+ * questions `cockpit status`, the health receipt and the fleet board ask of the
4
+ * delivery-attempt history, split out of `raw-evidence-staging.ts` (BLI-4022)
5
+ * so that file stays under the repo's readability band.
6
+ *
7
+ * The distinction this file exists to keep: an object with a live failure
8
+ * history is STUCK, and an object whose next attempt is still in the future is
9
+ * HELD. Every held object is stuck and most stuck objects are held, but the two
10
+ * answer different questions — "has anything been failing for nine days" versus
11
+ * "what did this tick withhold, and when does it try again" — and BLI-4022
12
+ * needed the second one, which nothing computed.
13
+ *
14
+ * Metadata only, like the rest of this family: content hashes, byte sizes,
15
+ * reason labels, timestamps. Never content, never an absolute path.
16
+ */
17
+ /**
18
+ * What `cockpit status` and the health receipt need in order to be unable to
19
+ * read green while an object has been failing for nine days.
20
+ */
21
+ export function summarizeStuckEvidence(state, now) {
22
+ const entries = Object.values(state.delivery_attempts);
23
+ const reasons = new Set();
24
+ let heldCount = 0;
25
+ let maxAttempts = 0;
26
+ let oldest = null;
27
+ // The held objects are tracked apart from the stuck ones, because a hold is
28
+ // judged on what is being withheld NOW and on when it may next be offered —
29
+ // not on every object that has ever failed here.
30
+ let heldOldest = null;
31
+ let heldNextAttemptAt = null;
32
+ for (const entry of entries) {
33
+ reasons.add(entry.last_reason);
34
+ if (Date.parse(entry.next_attempt_at) > now.getTime()) {
35
+ heldCount += 1;
36
+ if (!heldOldest ||
37
+ entry.first_failed_at.localeCompare(heldOldest.first_failed_at) < 0) {
38
+ heldOldest = entry;
39
+ }
40
+ if (!heldNextAttemptAt ||
41
+ entry.next_attempt_at.localeCompare(heldNextAttemptAt) < 0) {
42
+ heldNextAttemptAt = entry.next_attempt_at;
43
+ }
44
+ }
45
+ maxAttempts = Math.max(maxAttempts, entry.attempts);
46
+ if (!oldest || entry.first_failed_at.localeCompare(oldest) < 0) {
47
+ oldest = entry.first_failed_at;
48
+ }
49
+ }
50
+ return {
51
+ stuck_object_count: entries.length,
52
+ held_object_count: heldCount,
53
+ max_attempts: maxAttempts,
54
+ oldest_first_failed_at: oldest,
55
+ reasons: [...reasons].sort(),
56
+ held_oldest_first_failed_at: heldOldest?.first_failed_at ?? null,
57
+ held_next_attempt_at: heldNextAttemptAt,
58
+ held_last_reason: heldOldest?.last_reason ?? null,
59
+ };
60
+ }
@@ -60,6 +60,13 @@ async function readPairedCollector(paths) {
60
60
  throw new LocalUploadBlockedError("unpaired", "No paired collector session found. Run `cockpit login` or `cockpit pair` before `cockpit sync`.", "cockpit login");
61
61
  });
62
62
  const session = await readLocalSessionReference(paths);
63
+ // BLI-4019 deliberately did NOT loosen this. The heartbeat now knocks while
64
+ // the machine believes it is expired — that is the door the server renews it
65
+ // through — but an UPLOAD stays gated on `valid`, so a machine past the
66
+ // 30-day grace window never spends bandwidth and staging on evidence the
67
+ // server will refuse. One tick after the renewal lands, this reads `valid`
68
+ // again, because the heartbeat writes the server's date back into
69
+ // `session.json` (`recordDeviceTokenExpiry`).
63
70
  if (session.session_state !== "valid") {
64
71
  throw new LocalUploadBlockedError("unpaired", session.session_state === "expired"
65
72
  ? "Collector session expired. Run `cockpit login` or `cockpit pair` again before `cockpit sync`."
@@ -101,6 +101,12 @@ export function hasRetryableEvidenceGap(facts, outcomes) {
101
101
  * backoff is missing collection right now. If it did not land here the sync
102
102
  * would read clean while a transcript sat undelivered for nine days, which is
103
103
  * the exact shape of BLI-3066.
104
+ *
105
+ * BLI-4022 did NOT change that. A hold still queues the retry and still travels
106
+ * to the status output; what changed is downstream, at the ledger in
107
+ * `commands/session-sync-failures.ts`, which no longer files it as a FAILED
108
+ * tick. Retryable and failed are two questions, and answering both with this
109
+ * one set is what painted a collecting machine red for eight ticks.
104
110
  */
105
111
  function isRetryableEvidenceSkipReason(reason) {
106
112
  if (reason === DELIVERY_BACKOFF_HOLDING_REASON)
@@ -201,5 +207,8 @@ export function summarizeRawEvidenceDelivery(built, outcomes, uploadedChunkCount
201
207
  raw_evidence_stuck_object_count: stuck.stuck_object_count,
202
208
  raw_evidence_max_delivery_attempts: stuck.max_attempts,
203
209
  raw_evidence_oldest_delivery_failure_at: stuck.oldest_first_failed_at,
210
+ raw_evidence_held_oldest_failure_at: stuck.held_oldest_first_failed_at,
211
+ raw_evidence_held_next_attempt_at: stuck.held_next_attempt_at,
212
+ raw_evidence_held_last_reason: stuck.held_last_reason,
204
213
  };
205
214
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bli-cockpit/cli",
3
- "version": "0.2.95",
3
+ "version": "0.2.97",
4
4
  "private": false,
5
5
  "type": "module",
6
6
  "bin": {