@bli-cockpit/cli 0.2.95 → 0.2.97
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/heartbeat-token.js +80 -0
- package/dist/commands/heartbeat.js +24 -12
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/session-sync-failures.js +14 -0
- package/dist/commands/session-sync-health.js +9 -0
- package/dist/commands/session-sync-hold.js +90 -0
- package/dist/commands/session-sync.js +5 -0
- package/dist/commands/setup-receipt-lines.js +17 -0
- package/dist/commands/sync-receipt.js +20 -5
- package/dist/commands/sync-report.js +3 -0
- package/dist/delivery-hold-notice.js +166 -0
- package/dist/local-state-pairing.js +41 -1
- package/dist/local-state.js +2 -2
- package/dist/raw-evidence-staging.js +4 -27
- package/dist/raw-evidence-stuck-summary.js +60 -0
- package/dist/upload-envelope-build.js +7 -0
- package/dist/upload-evidence-delivery-summary.js +9 -0
- package/package.json +1 -1
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { describeError } from "../health-detail.js";
|
|
2
|
+
import { getCollectorRuntimePaths, readLocalCollectorSessionFile, recordDeviceTokenExpiry, toSessionReference, } from "../local-state.js";
|
|
3
|
+
/**
|
|
4
|
+
* Read the session and decide whether this tick may check in.
|
|
5
|
+
*
|
|
6
|
+
* The check-in goes out while the machine believes it is EXPIRED, as long as it
|
|
7
|
+
* still holds a device token. That is not a loosening: the SERVER decides
|
|
8
|
+
* whether a token is accepted, and a collector that refused to knock could
|
|
9
|
+
* never be let back in. Uploads are unaffected — they still gate on `valid`
|
|
10
|
+
* (`upload-envelope-build.ts`), so a machine past the grace window spends no
|
|
11
|
+
* bandwidth on evidence that will be refused.
|
|
12
|
+
*
|
|
13
|
+
* Both refusing branches say something, because "the dashboard shows this
|
|
14
|
+
* machine as quiet" has two very different causes.
|
|
15
|
+
*/
|
|
16
|
+
export async function readHeartbeatSession(homeDir) {
|
|
17
|
+
const paths = getCollectorRuntimePaths(homeDir);
|
|
18
|
+
const session = await readLocalCollectorSessionFile(paths).catch(() => null);
|
|
19
|
+
const state = session ? toSessionReference(session).session_state : "missing";
|
|
20
|
+
const usable = state === "valid" || state === "expired";
|
|
21
|
+
if (!session ||
|
|
22
|
+
!usable ||
|
|
23
|
+
typeof session.device_token !== "string" ||
|
|
24
|
+
!session.device_token) {
|
|
25
|
+
console.error("[heartbeat] no usable device session on this machine; the dashboard will show it as quiet", JSON.stringify({
|
|
26
|
+
reason: "no_device_session",
|
|
27
|
+
session_state: state,
|
|
28
|
+
next_action: "run `cockpit do-everything` to pair this machine again",
|
|
29
|
+
}));
|
|
30
|
+
return { usable: false };
|
|
31
|
+
}
|
|
32
|
+
if (state === "expired") {
|
|
33
|
+
console.error("[heartbeat] this machine's session looks expired; checking in anyway so the dashboard can renew it", JSON.stringify({
|
|
34
|
+
reason: "session_expired_locally",
|
|
35
|
+
expires_at: session.expires_at ?? null,
|
|
36
|
+
next_action: "the dashboard renews a token that lapsed inside its 30-day grace window on this call",
|
|
37
|
+
}));
|
|
38
|
+
}
|
|
39
|
+
return { usable: true, session, state: state };
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Record the `token_expires_at` the heartbeat door answered with.
|
|
43
|
+
*
|
|
44
|
+
* The LOCAL copy is what decides whether this machine even attempts an upload,
|
|
45
|
+
* so a machine the server has just renewed would otherwise go on refusing its
|
|
46
|
+
* own uploads until somebody signed in by hand.
|
|
47
|
+
*
|
|
48
|
+
* A body that is not JSON, or that carries no such field, is not an error and
|
|
49
|
+
* is not logged: an older dashboard answers `{ ok: true }` and a proxy can
|
|
50
|
+
* answer anything. It simply means "this reply said nothing about the expiry",
|
|
51
|
+
* and the machine keeps the date it already holds. Best-effort throughout — a
|
|
52
|
+
* heartbeat never fails a sync.
|
|
53
|
+
*/
|
|
54
|
+
export async function recordHeartbeatTokenExpiry(homeDir, response) {
|
|
55
|
+
const expiresAt = await readTokenExpiry(response);
|
|
56
|
+
if (!expiresAt)
|
|
57
|
+
return;
|
|
58
|
+
await recordDeviceTokenExpiry({
|
|
59
|
+
paths: getCollectorRuntimePaths(homeDir),
|
|
60
|
+
expiresAt,
|
|
61
|
+
}).catch((error) => {
|
|
62
|
+
console.error("[heartbeat] the token expiry the dashboard reported was not recorded", JSON.stringify({
|
|
63
|
+
reason: "token_expiry_writeback_threw",
|
|
64
|
+
...describeError(error),
|
|
65
|
+
}));
|
|
66
|
+
return null;
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
async function readTokenExpiry(response) {
|
|
70
|
+
try {
|
|
71
|
+
const body = (await response.json());
|
|
72
|
+
if (!body || typeof body !== "object")
|
|
73
|
+
return null;
|
|
74
|
+
const value = body["token_expires_at"];
|
|
75
|
+
return typeof value === "string" && value ? value : null;
|
|
76
|
+
}
|
|
77
|
+
catch {
|
|
78
|
+
return null;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
@@ -28,7 +28,9 @@ import { readCachedSetupReceipt } from "./setup-receipt.js";
|
|
|
28
28
|
import { readStagingInventory } from "../disk-usage.js";
|
|
29
29
|
import { describeError } from "../health-detail.js";
|
|
30
30
|
import { shouldSuppressFleetReceipts, } from "../dev-build.js";
|
|
31
|
-
import { getCollectorRuntimePaths,
|
|
31
|
+
import { getCollectorRuntimePaths, LOCAL_COLLECTOR_VERSION, } from "../local-state.js";
|
|
32
|
+
import { readHeartbeatSession, recordHeartbeatTokenExpiry, } from "./heartbeat-token.js";
|
|
33
|
+
import { classifySyncHealthError } from "../sync-health-class.js";
|
|
32
34
|
import { readMemoryReceiptFile } from "./memory-install-receipt.js";
|
|
33
35
|
import { readMemoryHookCounts } from "./memory-hook-counts.js";
|
|
34
36
|
import { readMemoryHookPerformance } from "./memory-hook-performance.js";
|
|
@@ -232,18 +234,14 @@ export async function sendCollectorHeartbeatBestEffort(options) {
|
|
|
232
234
|
}));
|
|
233
235
|
return false;
|
|
234
236
|
}
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
console.error("[heartbeat] no valid device session on this machine; the dashboard will show it as quiet", JSON.stringify({
|
|
242
|
-
reason: "no_device_session",
|
|
243
|
-
next_action: "run `cockpit do-everything` to pair this machine again",
|
|
244
|
-
}));
|
|
237
|
+
// BLI-4019. Whether this machine may knock at all, and the two log lines
|
|
238
|
+
// that go with it, live in `heartbeat-token.ts` — the check-in goes out even
|
|
239
|
+
// when the local copy says `expired`, because that is the door the server
|
|
240
|
+
// renews a lapsed token through.
|
|
241
|
+
const reading = await readHeartbeatSession(options.homeDir);
|
|
242
|
+
if (!reading.usable)
|
|
245
243
|
return false;
|
|
246
|
-
}
|
|
244
|
+
const { session } = reading;
|
|
247
245
|
const memory = await readHeartbeatMemoryReceipt({
|
|
248
246
|
...(options.homeDir ? { homeDir: options.homeDir } : {}),
|
|
249
247
|
...(options.now ? { now: options.now } : {}),
|
|
@@ -287,13 +285,27 @@ export async function sendCollectorHeartbeatBestEffort(options) {
|
|
|
287
285
|
});
|
|
288
286
|
if (!response.ok) {
|
|
289
287
|
// A heartbeat is not retried, so the reason has to be said once, here.
|
|
288
|
+
// BLI-4019: the CLASS comes from `sync-health-class.ts`, the one place
|
|
289
|
+
// that decides what a failure is — a 401/403 is `auth_failed` and
|
|
290
|
+
// anything else is not, read off the observed status and never off the
|
|
291
|
+
// words in a message (BLI-3551's rule).
|
|
290
292
|
console.error("[heartbeat] the dashboard refused this tick's heartbeat", JSON.stringify({
|
|
291
293
|
reason: "heartbeat_rejected",
|
|
294
|
+
health_class: classifySyncHealthError({ httpStatus: response.status }),
|
|
292
295
|
http_status: response.status,
|
|
293
296
|
root_count: heartbeat.roots.length,
|
|
297
|
+
...(response.status === 401 || response.status === 403
|
|
298
|
+
? {
|
|
299
|
+
next_action: "past the 30-day grace window this needs `cockpit login` on this machine",
|
|
300
|
+
}
|
|
301
|
+
: {}),
|
|
294
302
|
}));
|
|
295
303
|
return false;
|
|
296
304
|
}
|
|
305
|
+
// BLI-4019. The server slides `token_expires_at` out on every call it
|
|
306
|
+
// authenticates and answers with the result; the sibling records it, so
|
|
307
|
+
// this machine's own copy follows the server rather than drifting.
|
|
308
|
+
await recordHeartbeatTokenExpiry(options.homeDir, response);
|
|
297
309
|
console.error("[heartbeat] checked in", JSON.stringify({
|
|
298
310
|
reason: "heartbeat_recorded",
|
|
299
311
|
http_status: response.status,
|
|
@@ -15,7 +15,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
15
15
|
}
|
|
16
16
|
|
|
17
17
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
18
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
18
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.97");
|
|
19
19
|
return 0;
|
|
20
20
|
}
|
|
21
21
|
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
* without saying why.
|
|
14
14
|
*/
|
|
15
15
|
import { claudeAttributionReadFailureCount, codexAttributionReadFailureCount, } from "./agent-session-report.js";
|
|
16
|
+
import { isDeliveryHoldReason } from "./session-sync-hold.js";
|
|
16
17
|
import { sourceScanFailureReason } from "./session-sync-scan.js";
|
|
17
18
|
/**
|
|
18
19
|
* The label used when a sync fails and nothing on the way there said why.
|
|
@@ -45,6 +46,15 @@ export function createSyncFailureLedger() {
|
|
|
45
46
|
* The spooled reason is the most specific thing anyone has, so it leads, and it
|
|
46
47
|
* names the worktree it belongs to — a fleet failure is usually one repo, and
|
|
47
48
|
* "which one" is the first question asked.
|
|
49
|
+
*
|
|
50
|
+
* BLI-4022: `delivery_backoff_holding` is the one raw-evidence reason that does
|
|
51
|
+
* NOT come in here. An object inside its own retry window was withheld on
|
|
52
|
+
* purpose and will be offered again; filing that as a failed tick is what
|
|
53
|
+
* painted Ian's Mac mini red on eight consecutive ticks while every session it
|
|
54
|
+
* collected landed. The hold is still reported — it becomes the tick's named
|
|
55
|
+
* hold in `session-sync-hold.ts` and rides the same receipt — and a tick that
|
|
56
|
+
* ALSO failed for a real reason still fails on that reason. Nothing is
|
|
57
|
+
* silenced; only the verdict changed.
|
|
48
58
|
*/
|
|
49
59
|
export function recordWorktreeDeliveryFailures(ledger, outcomes) {
|
|
50
60
|
for (const { worktree, sync } of outcomes) {
|
|
@@ -61,12 +71,16 @@ export function recordWorktreeDeliveryFailures(ledger, outcomes) {
|
|
|
61
71
|
});
|
|
62
72
|
}
|
|
63
73
|
for (const reason of sync.raw_evidence_failure_reasons ?? []) {
|
|
74
|
+
if (isDeliveryHoldReason(reason))
|
|
75
|
+
continue;
|
|
64
76
|
ledger.add({
|
|
65
77
|
label: "raw_evidence_upload_failed",
|
|
66
78
|
rendered: `raw_evidence:${reason}`,
|
|
67
79
|
});
|
|
68
80
|
}
|
|
69
81
|
for (const reason of sync.raw_evidence_retry_reasons ?? []) {
|
|
82
|
+
if (isDeliveryHoldReason(reason))
|
|
83
|
+
continue;
|
|
70
84
|
ledger.add({
|
|
71
85
|
label: "raw_evidence_retry_required",
|
|
72
86
|
rendered: `raw_evidence_retry:${reason}`,
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
*/
|
|
12
12
|
import { createSyncFailureLedger, recordSourceScanFailures, recordUnexplainedFailure, recordUnpostedSessionReportFailure, recordWorktreeDeliveryFailures, } from "./session-sync-failures.js";
|
|
13
13
|
import { countSessionsNewThisTick, countSessionsPendingUpload, } from "./session-sync-counters.js";
|
|
14
|
+
import { logDeliveryHold, summarizeDeliveryHold } from "./session-sync-hold.js";
|
|
14
15
|
/**
|
|
15
16
|
* Decide whether this tick failed, and record every condition that decided it.
|
|
16
17
|
*
|
|
@@ -44,12 +45,20 @@ export function decideSyncHealth(options) {
|
|
|
44
45
|
if (nothingInRoot !== null && notice) {
|
|
45
46
|
announceNothingInRoot(nothingInRoot, options.collectionRootCount);
|
|
46
47
|
}
|
|
48
|
+
// BLI-4022. Computed after `ok`, and never allowed to change it: a hold is
|
|
49
|
+
// what this tick withheld on purpose, not a verdict on the run. It rides an
|
|
50
|
+
// `ok` receipt as the tick's reason and a failing one beside the reasons that
|
|
51
|
+
// failed it, so the board can say what is waiting in either case.
|
|
52
|
+
const hold = summarizeDeliveryHold(outcomes);
|
|
53
|
+
if (hold && ok)
|
|
54
|
+
logDeliveryHold(hold);
|
|
47
55
|
const records = ledger.sortedRecords();
|
|
48
56
|
return {
|
|
49
57
|
ok,
|
|
50
58
|
failure_reasons: records.map((record) => record.rendered),
|
|
51
59
|
failure_records: records,
|
|
52
60
|
notice,
|
|
61
|
+
hold,
|
|
53
62
|
sessions_outside_root: sessionsOutsideRoot,
|
|
54
63
|
sessions_new_this_tick: countSessionsNewThisTick({
|
|
55
64
|
sessions: options.sessions,
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What this tick is HOLDING, as opposed to what it got wrong (BLI-4022).
|
|
3
|
+
*
|
|
4
|
+
* The sibling `session-sync-failures.ts` is the one place a sync writes down
|
|
5
|
+
* that it FAILED. This is the one place it writes down that it deliberately
|
|
6
|
+
* withheld something and when it will try again — a different question with a
|
|
7
|
+
* different answer, kept in a different file so neither can quietly become the
|
|
8
|
+
* other.
|
|
9
|
+
*
|
|
10
|
+
* Deliberately writes no failure label of any kind: `sync-health-class.test.ts`
|
|
11
|
+
* reads every `session-sync*.ts` sibling and demands that only the failure
|
|
12
|
+
* ledger writes one, and this file is not it.
|
|
13
|
+
*/
|
|
14
|
+
import { formatDeliveryHoldNotice, } from "../delivery-hold-notice.js";
|
|
15
|
+
import { DELIVERY_BACKOFF_HOLDING_REASON } from "../raw-evidence-staging.js";
|
|
16
|
+
/**
|
|
17
|
+
* Is this reason a delivery hold rather than a delivery failure?
|
|
18
|
+
*
|
|
19
|
+
* Exact match against the label telemetry-core owns — the same discipline the
|
|
20
|
+
* failure classifier follows. A reason that merely CONTAINS the word "backoff"
|
|
21
|
+
* is not evidence of anything (BLI-3551's lesson, applied to the other side of
|
|
22
|
+
* the same fence).
|
|
23
|
+
*/
|
|
24
|
+
export function isDeliveryHoldReason(reason) {
|
|
25
|
+
return reason === DELIVERY_BACKOFF_HOLDING_REASON;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Fold every worktree's held objects into one hold for the tick, or null.
|
|
29
|
+
*
|
|
30
|
+
* The count is summed because a tick can sync several worktrees and each one
|
|
31
|
+
* reports its own held objects. The AGE takes the oldest across all of them and
|
|
32
|
+
* the NEXT ATTEMPT the earliest, because both are promises about the whole
|
|
33
|
+
* machine: the operator asks "how long has anything been stuck" and "when does
|
|
34
|
+
* anything move", not "which worktree".
|
|
35
|
+
*
|
|
36
|
+
* Returns null when nothing is held, so no caller can announce a hold of
|
|
37
|
+
* nothing — a status line that appears on every tick is one people stop
|
|
38
|
+
* reading.
|
|
39
|
+
*/
|
|
40
|
+
export function summarizeDeliveryHold(outcomes) {
|
|
41
|
+
let objectCount = 0;
|
|
42
|
+
let oldestFirstFailedAt = null;
|
|
43
|
+
let nextAttemptAt = null;
|
|
44
|
+
let lastReason = null;
|
|
45
|
+
for (const { sync } of outcomes) {
|
|
46
|
+
const held = sync.raw_evidence_delivery_held_count ?? 0;
|
|
47
|
+
if (held <= 0)
|
|
48
|
+
continue;
|
|
49
|
+
objectCount += held;
|
|
50
|
+
const first = sync.raw_evidence_held_oldest_failure_at ?? null;
|
|
51
|
+
if (first && (!oldestFirstFailedAt || first < oldestFirstFailedAt)) {
|
|
52
|
+
oldestFirstFailedAt = first;
|
|
53
|
+
// The reason travels WITH the object it belongs to. A reason taken from
|
|
54
|
+
// one worktree and an age from another would describe no object at all.
|
|
55
|
+
lastReason = sync.raw_evidence_held_last_reason ?? null;
|
|
56
|
+
}
|
|
57
|
+
const next = sync.raw_evidence_held_next_attempt_at ?? null;
|
|
58
|
+
if (next && (!nextAttemptAt || next < nextAttemptAt))
|
|
59
|
+
nextAttemptAt = next;
|
|
60
|
+
}
|
|
61
|
+
if (objectCount <= 0)
|
|
62
|
+
return null;
|
|
63
|
+
const notice = formatDeliveryHoldNotice({
|
|
64
|
+
objectCount,
|
|
65
|
+
oldestFirstFailedAt,
|
|
66
|
+
nextAttemptAt,
|
|
67
|
+
lastReason,
|
|
68
|
+
});
|
|
69
|
+
if (!notice)
|
|
70
|
+
return null;
|
|
71
|
+
return { objectCount, oldestFirstFailedAt, nextAttemptAt, lastReason, notice };
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Say out loud, once per tick, that bytes are being withheld on purpose.
|
|
75
|
+
*
|
|
76
|
+
* `logEvidenceHeldByBackoff` already names the objects at the moment they are
|
|
77
|
+
* partitioned off the wire; this is the RUN's line, so the tick that reports
|
|
78
|
+
* itself as held also explains itself on stderr where launchd captures it.
|
|
79
|
+
* Counts, instants and reason labels only.
|
|
80
|
+
*/
|
|
81
|
+
export function logDeliveryHold(hold) {
|
|
82
|
+
console.error("[session-sync] raw evidence held by delivery backoff, nothing else failed", JSON.stringify({
|
|
83
|
+
reason: hold.notice,
|
|
84
|
+
object_count: hold.objectCount,
|
|
85
|
+
oldest_first_failed_at: hold.oldestFirstFailedAt,
|
|
86
|
+
next_attempt_at: hold.nextAttemptAt,
|
|
87
|
+
last_reason: hold.lastReason,
|
|
88
|
+
next_action: "nothing until the next attempt is due; run `cockpit doctor` if it is already past",
|
|
89
|
+
}));
|
|
90
|
+
}
|
|
@@ -25,6 +25,9 @@
|
|
|
25
25
|
* class-lock test reads that file (BLI-3551).
|
|
26
26
|
* - `session-sync-health.ts` — the verdict, and the notice a quiet-but-working
|
|
27
27
|
* machine still emits.
|
|
28
|
+
* - `session-sync-hold.ts` — what the tick withheld ON PURPOSE and when it will
|
|
29
|
+
* try again (BLI-4022), which is a different question from what it got wrong
|
|
30
|
+
* and so lives in a different file from the failure ledger.
|
|
28
31
|
* - `session-sync-counters.ts` — the two honest heartbeat counters beside the
|
|
29
32
|
* raw scan count: sessions new since the last tick's cursor, and sessions
|
|
30
33
|
* still pending a durable pointer (BLI-3645).
|
|
@@ -49,6 +52,7 @@ export { isLiveSyncCollectableAttributionState, liveSyncCursorEntryRequiresRetry
|
|
|
49
52
|
export { sourceScanFailureReason, sourceScanRetryReason, } from "./session-sync-scan.js";
|
|
50
53
|
export { SYNC_FAILED_WITHOUT_REASON } from "./session-sync-failures.js";
|
|
51
54
|
export { nothingInRootCount } from "./session-sync-health.js";
|
|
55
|
+
export { isDeliveryHoldReason, summarizeDeliveryHold, } from "./session-sync-hold.js";
|
|
52
56
|
/**
|
|
53
57
|
* Shared dual-source sync orchestration for single-repo and parent-folder
|
|
54
58
|
* modes. Codex AND Claude Code sessions are scanned and attributed once across
|
|
@@ -126,6 +130,7 @@ export async function runAttributedWorktreeSync(options) {
|
|
|
126
130
|
failure_reasons: health.failure_reasons,
|
|
127
131
|
failure_records: health.failure_records,
|
|
128
132
|
notice: health.notice,
|
|
133
|
+
hold: health.hold,
|
|
129
134
|
sessions_observed: sessions.length,
|
|
130
135
|
sessions_outside_root: health.sessions_outside_root,
|
|
131
136
|
sessions_new_this_tick: health.sessions_new_this_tick,
|
|
@@ -21,6 +21,20 @@ const FIXES = {
|
|
|
21
21
|
"codex.hooks": "Run `cockpit memory install`.",
|
|
22
22
|
"collector.autostart": "Run `cockpit autostart install`.",
|
|
23
23
|
};
|
|
24
|
+
/**
|
|
25
|
+
* A fix keyed on the piece's REASON rather than the piece, for the cases where
|
|
26
|
+
* the same gap has two different next actions (BLI-4019).
|
|
27
|
+
*
|
|
28
|
+
* `device` says "run `cockpit login`" for every lapse, and for an expired
|
|
29
|
+
* session that is now the wrong instruction most of the time: the dashboard
|
|
30
|
+
* slides a device token's expiry out on every call it authenticates and renews
|
|
31
|
+
* one that lapsed inside a 30-day grace window, so the ordinary answer is to do
|
|
32
|
+
* nothing and let the next tick fix it. Telling somebody to sign in again on a
|
|
33
|
+
* machine that is about to heal itself is how a receipt stops being believed.
|
|
34
|
+
*/
|
|
35
|
+
const REASON_FIXES = {
|
|
36
|
+
session_expired: "Renews on the next sync tick while inside the 30-day grace window; past it, run `cockpit login`.",
|
|
37
|
+
};
|
|
24
38
|
/** Codex hooks a person switched OFF is a decision, not a fault. */
|
|
25
39
|
const UNSUPPORTED_FIX = "Switched off in your own config; nothing to do.";
|
|
26
40
|
/**
|
|
@@ -64,6 +78,9 @@ function fixFor(key, piece) {
|
|
|
64
78
|
if (piece.status === "skipped") {
|
|
65
79
|
return "Skipped on purpose; nothing to do.";
|
|
66
80
|
}
|
|
81
|
+
// The reason wins over the piece when it has its own next action.
|
|
82
|
+
if (piece.reason && REASON_FIXES[piece.reason])
|
|
83
|
+
return REASON_FIXES[piece.reason];
|
|
67
84
|
return FIXES[key] ?? "Run `cockpit doctor`.";
|
|
68
85
|
}
|
|
69
86
|
function reasonSuffix(piece) {
|
|
@@ -91,16 +91,25 @@ function standAsideForRunningSync(command, io, heldSince) {
|
|
|
91
91
|
* is the receipt that separates "this machine is alive and its operator
|
|
92
92
|
* works outside the approved roots" from "this machine is dead", which
|
|
93
93
|
* until now looked identical from the dashboard.
|
|
94
|
+
*
|
|
95
|
+
* BLI-4022 added the second thing an `ok` tick can say: `held_backoff:…`, the
|
|
96
|
+
* objects this tick withheld on purpose and when they may be offered again.
|
|
97
|
+
* The two can never collide — `nothing_in_root` requires that NO worktree
|
|
98
|
+
* synced at all (`nothingInRootCount`), and a hold requires a worktree that
|
|
99
|
+
* did — so one field carries whichever is true, and the notice wins if that
|
|
100
|
+
* ever stops being the case, because "there was nothing to collect" is the
|
|
101
|
+
* larger fact about the tick.
|
|
94
102
|
*/
|
|
95
|
-
function collectedReceipt(run) {
|
|
103
|
+
export function collectedReceipt(run) {
|
|
104
|
+
const reason = run.notice ?? run.holdNotice;
|
|
96
105
|
return {
|
|
97
106
|
exitCode: run.exitCode,
|
|
98
107
|
completion: {
|
|
99
108
|
step: "sync_complete",
|
|
100
109
|
status: "ok",
|
|
101
|
-
...(
|
|
110
|
+
...(reason ? { error_detail: reason } : {}),
|
|
102
111
|
},
|
|
103
|
-
heartbeat: { status: "ok", reason
|
|
112
|
+
heartbeat: { status: "ok", reason, ...heartbeatCounts(run) },
|
|
104
113
|
};
|
|
105
114
|
}
|
|
106
115
|
/**
|
|
@@ -111,8 +120,14 @@ function collectedReceipt(run) {
|
|
|
111
120
|
* failure rows carried a null detail and the real reason was reachable only
|
|
112
121
|
* by running `cockpit status` on the machine itself (BLI-2526).
|
|
113
122
|
*/
|
|
114
|
-
function failedReceipt(run) {
|
|
115
|
-
|
|
123
|
+
export function failedReceipt(run) {
|
|
124
|
+
// BLI-4022: the hold leads on a failing tick. `error_detail` is bounded, and
|
|
125
|
+
// an operator reading a red row still has to be able to see what is merely
|
|
126
|
+
// waiting — otherwise the held objects become invisible the moment anything
|
|
127
|
+
// else goes wrong, which is the opposite of the fix.
|
|
128
|
+
const reasonText = [run.holdNotice, ...run.failureReasons]
|
|
129
|
+
.filter(Boolean)
|
|
130
|
+
.join("; ");
|
|
116
131
|
// The bucket comes from the records the deciding branches wrote, not from
|
|
117
132
|
// this sentence (BLI-3551). The sentence is still the detail.
|
|
118
133
|
const errorCode = classifySyncFailureRecords(run.failureRecords);
|
|
@@ -141,6 +141,9 @@ function syncResult(run) {
|
|
|
141
141
|
failureReasons: run.ok ? [] : run.failure_reasons,
|
|
142
142
|
failureRecords: run.ok ? [] : run.failure_records,
|
|
143
143
|
notice: run.notice,
|
|
144
|
+
// BLI-4022: unlike the reasons, the hold survives a successful run — it is
|
|
145
|
+
// what the tick withheld, not why it failed.
|
|
146
|
+
holdNotice: run.hold?.notice ?? null,
|
|
144
147
|
sessionsObserved: run.sessions_observed,
|
|
145
148
|
sessionsOutsideRoot: run.sessions_outside_root,
|
|
146
149
|
sessionsNewThisTick: run.sessions_new_this_tick,
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A HOLD is not a FAILURE — and this file is the one place that spells it.
|
|
3
|
+
*
|
|
4
|
+
* BLI-4022. `delivery_backoff_holding` means "this object is inside its own
|
|
5
|
+
* retry window, so this tick did not offer it". Until now that reason went
|
|
6
|
+
* into the sync's failure ledger like any other, so a machine collecting
|
|
7
|
+
* perfectly reported `sync_failed` on every tick for as long as the backoff
|
|
8
|
+
* held: Ian's Mac mini was RED on eight consecutive ticks from
|
|
9
|
+
* 2026-09-09T00:11Z with the detail
|
|
10
|
+
* `raw_evidence_retry:delivery_backoff_holding; raw_evidence:delivery_backoff_holding`
|
|
11
|
+
* while its sessions kept landing and its heartbeat kept moving.
|
|
12
|
+
*
|
|
13
|
+
* The honesty is unchanged. A held object IS missing collection and must never
|
|
14
|
+
* read as clean (that is the rule BLI-3066 bought). What changes is the
|
|
15
|
+
* OUTCOME: a tick whose only raw-evidence complaint is held objects reports a
|
|
16
|
+
* named hold instead of a failure, and the hold carries everything an operator
|
|
17
|
+
* needs to decide whether to care — how many objects, how long the oldest has
|
|
18
|
+
* been failing, when the next attempt is due, and the object's own last reason.
|
|
19
|
+
*
|
|
20
|
+
* ## The grammar
|
|
21
|
+
*
|
|
22
|
+
* One token, because it has to survive two narrow channels: the heartbeat's
|
|
23
|
+
* `last_sync_reason` (`^[a-z0-9_:.-]+$`, 120 chars, no spaces) and the receipt's
|
|
24
|
+
* `error_detail`. Fields are positional and colon-separated:
|
|
25
|
+
*
|
|
26
|
+
* held_backoff:<count>:<oldest_first_failed_at>:<next_attempt_at>:<last_reason>
|
|
27
|
+
* held_backoff:3:2026-08-14T17:38Z:2026-09-09T06:11Z:chunk_10_failed_http_500
|
|
28
|
+
*
|
|
29
|
+
* Timestamps are minute precision UTC; `-` stands for a field this machine
|
|
30
|
+
* could not fill. The last reason is bounded so the whole token fits the
|
|
31
|
+
* heartbeat's 120 characters — the count and the ages are what a colour is
|
|
32
|
+
* decided from, so the reason is the field that gives way.
|
|
33
|
+
*
|
|
34
|
+
* The collector WRITES the token and the dashboard's fleet board READS it
|
|
35
|
+
* (`lib/ops/fleet-liveness-device-verdict.ts`, through this package's
|
|
36
|
+
* `./delivery-hold` export). One grammar, one file: a second copy of this
|
|
37
|
+
* string is exactly how the collector's word and the server's word drift
|
|
38
|
+
* apart, which is why `DELIVERY_BACKOFF_HOLDING` itself is aliased from
|
|
39
|
+
* telemetry-core rather than spelled twice.
|
|
40
|
+
*/
|
|
41
|
+
/** The token's leading word. Greppable in a receipt, a log line or a DM. */
|
|
42
|
+
export const DELIVERY_HOLD_NOTICE_PREFIX = "held_backoff";
|
|
43
|
+
/** The heartbeat's own ceiling for `last_sync_reason`. The token fits inside it. */
|
|
44
|
+
export const DELIVERY_HOLD_NOTICE_MAX_CHARS = 120;
|
|
45
|
+
/**
|
|
46
|
+
* How long a hold may last before it stops being a wait and becomes a stuck
|
|
47
|
+
* object.
|
|
48
|
+
*
|
|
49
|
+
* The backoff schedule doubles from the scheduler's own 15-minute cadence to a
|
|
50
|
+
* six-hour ceiling (`EVIDENCE_DELIVERY_BACKOFF_MAX_MS`), so an object held
|
|
51
|
+
* longer than this has already been offered at least four more times at the
|
|
52
|
+
* ceiling and been refused every time. Nothing is going to change on its own,
|
|
53
|
+
* and "next retry at T — nothing to do until then" would be a promise the
|
|
54
|
+
* machine cannot keep. 24 hours is also the board's own `SILENT_AFTER_HOURS`,
|
|
55
|
+
* so both clocks agree on when a quiet thing stops being a blip.
|
|
56
|
+
*/
|
|
57
|
+
export const DELIVERY_HOLD_STALE_AFTER_HOURS = 24;
|
|
58
|
+
/** The most characters of an object's own reason label the token may carry. */
|
|
59
|
+
const MAX_HOLD_REASON_CHARS = 64;
|
|
60
|
+
/** A field this machine could not fill. Never omitted — the shape is positional. */
|
|
61
|
+
const ABSENT = "-";
|
|
62
|
+
/**
|
|
63
|
+
* Render the hold as the single token the heartbeat and the receipt carry.
|
|
64
|
+
*
|
|
65
|
+
* Returns null for a hold of nothing, so a caller cannot accidentally announce
|
|
66
|
+
* a hold that is not happening.
|
|
67
|
+
*/
|
|
68
|
+
export function formatDeliveryHoldNotice(notice) {
|
|
69
|
+
if (notice.objectCount <= 0)
|
|
70
|
+
return null;
|
|
71
|
+
const head = [
|
|
72
|
+
DELIVERY_HOLD_NOTICE_PREFIX,
|
|
73
|
+
String(Math.trunc(notice.objectCount)),
|
|
74
|
+
minuteStamp(notice.oldestFirstFailedAt),
|
|
75
|
+
minuteStamp(notice.nextAttemptAt),
|
|
76
|
+
].join(":");
|
|
77
|
+
const budget = Math.min(MAX_HOLD_REASON_CHARS, DELIVERY_HOLD_NOTICE_MAX_CHARS - head.length - 1);
|
|
78
|
+
return `${head}:${reasonWord(notice.lastReason, budget)}`;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Find a hold token inside anything a receipt carries, and read it back.
|
|
82
|
+
*
|
|
83
|
+
* Deliberately a SEARCH, not a whole-string match: on a tick that also failed
|
|
84
|
+
* for a real reason the token rides beside those reasons in the same
|
|
85
|
+
* `error_detail`, and the board must still be able to say what is held.
|
|
86
|
+
*/
|
|
87
|
+
export function parseDeliveryHoldNotice(text) {
|
|
88
|
+
if (!text)
|
|
89
|
+
return null;
|
|
90
|
+
// The stamp shape is pinned rather than described by a character class: a
|
|
91
|
+
// class containing `:` is greedy across the field separator, and a parser
|
|
92
|
+
// that swallowed one field into the next reported an age of twenty-five
|
|
93
|
+
// years and a next attempt in 2001.
|
|
94
|
+
const stamp = String.raw `(?:-|\d{4}-\d{2}-\d{2}T\d{2}:\d{2}Z)`;
|
|
95
|
+
const match = new RegExp(`${DELIVERY_HOLD_NOTICE_PREFIX}:(\\d+):(${stamp}):(${stamp}):([a-z0-9_-]+)`, "iu").exec(text);
|
|
96
|
+
if (!match)
|
|
97
|
+
return null;
|
|
98
|
+
const objectCount = Number.parseInt(match[1], 10);
|
|
99
|
+
if (!Number.isFinite(objectCount) || objectCount <= 0)
|
|
100
|
+
return null;
|
|
101
|
+
return {
|
|
102
|
+
objectCount,
|
|
103
|
+
oldestFirstFailedAt: readStamp(match[2]),
|
|
104
|
+
nextAttemptAt: readStamp(match[3]),
|
|
105
|
+
lastReason: match[4] === ABSENT ? null : match[4],
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* How long the longest-held object has been failing, in hours.
|
|
110
|
+
*
|
|
111
|
+
* Null when the token carried no first-failure instant, which is a different
|
|
112
|
+
* fact from "zero hours" — an unknown age is never treated as a fresh hold.
|
|
113
|
+
*/
|
|
114
|
+
export function deliveryHoldAgeHours(notice, nowIso) {
|
|
115
|
+
if (!notice.oldestFirstFailedAt)
|
|
116
|
+
return null;
|
|
117
|
+
const from = Date.parse(notice.oldestFirstFailedAt);
|
|
118
|
+
const now = Date.parse(nowIso);
|
|
119
|
+
if (!Number.isFinite(from) || !Number.isFinite(now))
|
|
120
|
+
return null;
|
|
121
|
+
return (now - from) / 3_600_000;
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Has this hold stopped being a wait?
|
|
125
|
+
*
|
|
126
|
+
* An age this reader could not compute counts as stale on purpose: the whole
|
|
127
|
+
* point of the amber sentence is the promise "next retry at T", and a hold
|
|
128
|
+
* whose age is unknown is one nobody can make that promise about.
|
|
129
|
+
*/
|
|
130
|
+
export function isDeliveryHoldStale(notice, nowIso) {
|
|
131
|
+
const hours = deliveryHoldAgeHours(notice, nowIso);
|
|
132
|
+
if (hours === null)
|
|
133
|
+
return true;
|
|
134
|
+
return hours > DELIVERY_HOLD_STALE_AFTER_HOURS;
|
|
135
|
+
}
|
|
136
|
+
/** `2026-09-09T06:11Z`, or `-` for an instant this machine does not have. */
|
|
137
|
+
function minuteStamp(iso) {
|
|
138
|
+
if (!iso)
|
|
139
|
+
return ABSENT;
|
|
140
|
+
const parsed = Date.parse(iso);
|
|
141
|
+
if (!Number.isFinite(parsed))
|
|
142
|
+
return ABSENT;
|
|
143
|
+
return `${new Date(parsed).toISOString().slice(0, 16)}Z`;
|
|
144
|
+
}
|
|
145
|
+
/** The inverse: a token field back to an ISO instant, or null when absent. */
|
|
146
|
+
function readStamp(field) {
|
|
147
|
+
if (field === ABSENT)
|
|
148
|
+
return null;
|
|
149
|
+
const parsed = Date.parse(field);
|
|
150
|
+
return Number.isFinite(parsed) ? new Date(parsed).toISOString() : null;
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* A reason label narrowed to what the token's charset and budget allow.
|
|
154
|
+
*
|
|
155
|
+
* Truncation is marked by dropping characters, never by an ellipsis: the
|
|
156
|
+
* heartbeat's charset has no `…`, and a token the door rejects tells nobody
|
|
157
|
+
* anything at all.
|
|
158
|
+
*/
|
|
159
|
+
function reasonWord(reason, budget) {
|
|
160
|
+
if (!reason || budget <= 0)
|
|
161
|
+
return ABSENT;
|
|
162
|
+
const safe = reason.replace(/[^a-z0-9_]/giu, "_").replace(/^_+|_+$/gu, "");
|
|
163
|
+
if (safe.length === 0)
|
|
164
|
+
return ABSENT;
|
|
165
|
+
return safe.slice(0, budget);
|
|
166
|
+
}
|
|
@@ -7,7 +7,7 @@ import { serverFailureDetail } from "./upload-http.js";
|
|
|
7
7
|
import { ensureRuntimeDirectories, getCollectorRuntimePaths, } from "./local-state-paths.js";
|
|
8
8
|
import { isMissingFileError, writeJsonFile } from "./local-state-files.js";
|
|
9
9
|
import { defaultDeviceName, LOCAL_COLLECTOR_VERSION, normalizeDashboardUrl, normalizeDeviceName, normalizeOptionalEmail, readLocalCollectorConfig, } from "./local-state-config.js";
|
|
10
|
-
import { toSessionReference } from "./local-state-session.js";
|
|
10
|
+
import { readLocalCollectorSessionFile, toSessionReference, } from "./local-state-session.js";
|
|
11
11
|
/**
|
|
12
12
|
* `cockpit login` — the whole device-pairing handshake, because a machine can
|
|
13
13
|
* collect locally without it but can never upload until the dashboard has
|
|
@@ -106,6 +106,46 @@ export async function logoutLocalCollector(options = {}) {
|
|
|
106
106
|
}
|
|
107
107
|
return { removed, session_file: paths.session_file };
|
|
108
108
|
}
|
|
109
|
+
export async function recordDeviceTokenExpiry(options) {
|
|
110
|
+
const parsed = Date.parse(options.expiresAt);
|
|
111
|
+
if (!Number.isFinite(parsed)) {
|
|
112
|
+
console.error("[local-state] the dashboard's token expiry was not a date; this machine keeps the one it has", JSON.stringify({ reason: "token_expiry_unreadable" }));
|
|
113
|
+
return { status: "skipped", reason: "token_expiry_unreadable" };
|
|
114
|
+
}
|
|
115
|
+
let session;
|
|
116
|
+
try {
|
|
117
|
+
session = await readLocalCollectorSessionFile(options.paths);
|
|
118
|
+
}
|
|
119
|
+
catch (error) {
|
|
120
|
+
console.error("[local-state] no readable session file to record the token expiry on", JSON.stringify({
|
|
121
|
+
reason: "session_file_unreadable",
|
|
122
|
+
...describeError(error),
|
|
123
|
+
}));
|
|
124
|
+
return { status: "skipped", reason: "session_file_unreadable" };
|
|
125
|
+
}
|
|
126
|
+
if (session.expires_at === options.expiresAt) {
|
|
127
|
+
return { status: "unchanged", expires_at: options.expiresAt };
|
|
128
|
+
}
|
|
129
|
+
try {
|
|
130
|
+
await writeJsonFile(options.paths.session_file, {
|
|
131
|
+
...session,
|
|
132
|
+
expires_at: options.expiresAt,
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
catch (error) {
|
|
136
|
+
console.error("[local-state] the renewed token expiry could not be written; this machine still believes the old one", JSON.stringify({
|
|
137
|
+
reason: "session_file_write_failed",
|
|
138
|
+
...describeError(error),
|
|
139
|
+
}));
|
|
140
|
+
return { status: "skipped", reason: "session_file_write_failed" };
|
|
141
|
+
}
|
|
142
|
+
console.error("[local-state] recorded the token expiry the dashboard reported", JSON.stringify({
|
|
143
|
+
reason: "token_expiry_recorded",
|
|
144
|
+
previous_expires_at: session.expires_at ?? null,
|
|
145
|
+
expires_at: options.expiresAt,
|
|
146
|
+
}));
|
|
147
|
+
return { status: "written", expires_at: options.expiresAt };
|
|
148
|
+
}
|
|
109
149
|
async function postPairStart(fetchImpl, dashboardUrl, body, accessToken) {
|
|
110
150
|
const response = await fetchImpl(`${dashboardUrl}/api/ambient/pair/start`, {
|
|
111
151
|
method: "POST",
|
package/dist/local-state.js
CHANGED
|
@@ -37,8 +37,8 @@
|
|
|
37
37
|
*/
|
|
38
38
|
export { getCollectorRuntimePaths } from "./local-state-paths.js";
|
|
39
39
|
export { DEFAULT_DASHBOARD_URL, ensureLocalCollectorConfig, installLocalCollector, LOCAL_COLLECTOR_VERSION, readLocalCollectorConfig, } from "./local-state-config.js";
|
|
40
|
-
export { readLocalCollectorSessionFile, readLocalSessionReference, } from "./local-state-session.js";
|
|
41
|
-
export { logoutLocalCollector, pairLocalCollector, } from "./local-state-pairing.js";
|
|
40
|
+
export { readLocalCollectorSessionFile, readLocalSessionReference, toSessionReference, } from "./local-state-session.js";
|
|
41
|
+
export { logoutLocalCollector, pairLocalCollector, recordDeviceTokenExpiry, } from "./local-state-pairing.js";
|
|
42
42
|
export { PairingCodeError, pairLocalCollectorViaLink, } from "./local-state-pairing-code.js";
|
|
43
43
|
export { resolveGitBranch } from "./local-state-identity.js";
|
|
44
44
|
export { readLocalWorkContext, readLocalWorkContextForRepo, startLocalWorkContext, startLocalWorkContextForAttributedTarget, } from "./local-state-work-context.js";
|
|
@@ -25,6 +25,10 @@ import { describeError, isMissingFileFailure } from "./health-detail.js";
|
|
|
25
25
|
* Metadata only: content hashes, byte sizes, pack ids, pack-relative paths,
|
|
26
26
|
* reason labels, timestamps. Never content, never an absolute path.
|
|
27
27
|
*/
|
|
28
|
+
// BLI-4022. The stuck/held summary moved to its own sibling so this file
|
|
29
|
+
// stays under the readability band; it is still importable from here,
|
|
30
|
+
// which is the address every caller already knows.
|
|
31
|
+
export { summarizeStuckEvidence, } from "./raw-evidence-stuck-summary.js";
|
|
28
32
|
export const RAW_EVIDENCE_STAGING_FILENAME = "raw-evidence-staging.json";
|
|
29
33
|
/**
|
|
30
34
|
* Backoff schedule for an object whose delivery keeps failing.
|
|
@@ -207,33 +211,6 @@ export function evidenceSourceKey(options) {
|
|
|
207
211
|
: "unknown";
|
|
208
212
|
return `${options.kind}:${options.sessionId ?? "none"}:${source}`;
|
|
209
213
|
}
|
|
210
|
-
/**
|
|
211
|
-
* What `cockpit status` and the health receipt need in order to be unable to
|
|
212
|
-
* read green while an object has been failing for nine days.
|
|
213
|
-
*/
|
|
214
|
-
export function summarizeStuckEvidence(state, now) {
|
|
215
|
-
const entries = Object.values(state.delivery_attempts);
|
|
216
|
-
const reasons = new Set();
|
|
217
|
-
let heldCount = 0;
|
|
218
|
-
let maxAttempts = 0;
|
|
219
|
-
let oldest = null;
|
|
220
|
-
for (const entry of entries) {
|
|
221
|
-
reasons.add(entry.last_reason);
|
|
222
|
-
if (Date.parse(entry.next_attempt_at) > now.getTime())
|
|
223
|
-
heldCount += 1;
|
|
224
|
-
maxAttempts = Math.max(maxAttempts, entry.attempts);
|
|
225
|
-
if (!oldest || entry.first_failed_at.localeCompare(oldest) < 0) {
|
|
226
|
-
oldest = entry.first_failed_at;
|
|
227
|
-
}
|
|
228
|
-
}
|
|
229
|
-
return {
|
|
230
|
-
stuck_object_count: entries.length,
|
|
231
|
-
held_object_count: heldCount,
|
|
232
|
-
max_attempts: maxAttempts,
|
|
233
|
-
oldest_first_failed_at: oldest,
|
|
234
|
-
reasons: [...reasons].sort(),
|
|
235
|
-
};
|
|
236
|
-
}
|
|
237
214
|
/**
|
|
238
215
|
* Pack id derived from CONTENT, never from the clock.
|
|
239
216
|
*
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* "Is anything stuck, and is anything being held right now?" — the two
|
|
3
|
+
* questions `cockpit status`, the health receipt and the fleet board ask of the
|
|
4
|
+
* delivery-attempt history, split out of `raw-evidence-staging.ts` (BLI-4022)
|
|
5
|
+
* so that file stays under the repo's readability band.
|
|
6
|
+
*
|
|
7
|
+
* The distinction this file exists to keep: an object with a live failure
|
|
8
|
+
* history is STUCK, and an object whose next attempt is still in the future is
|
|
9
|
+
* HELD. Every held object is stuck and most stuck objects are held, but the two
|
|
10
|
+
* answer different questions — "has anything been failing for nine days" versus
|
|
11
|
+
* "what did this tick withhold, and when does it try again" — and BLI-4022
|
|
12
|
+
* needed the second one, which nothing computed.
|
|
13
|
+
*
|
|
14
|
+
* Metadata only, like the rest of this family: content hashes, byte sizes,
|
|
15
|
+
* reason labels, timestamps. Never content, never an absolute path.
|
|
16
|
+
*/
|
|
17
|
+
/**
|
|
18
|
+
* What `cockpit status` and the health receipt need in order to be unable to
|
|
19
|
+
* read green while an object has been failing for nine days.
|
|
20
|
+
*/
|
|
21
|
+
export function summarizeStuckEvidence(state, now) {
|
|
22
|
+
const entries = Object.values(state.delivery_attempts);
|
|
23
|
+
const reasons = new Set();
|
|
24
|
+
let heldCount = 0;
|
|
25
|
+
let maxAttempts = 0;
|
|
26
|
+
let oldest = null;
|
|
27
|
+
// The held objects are tracked apart from the stuck ones, because a hold is
|
|
28
|
+
// judged on what is being withheld NOW and on when it may next be offered —
|
|
29
|
+
// not on every object that has ever failed here.
|
|
30
|
+
let heldOldest = null;
|
|
31
|
+
let heldNextAttemptAt = null;
|
|
32
|
+
for (const entry of entries) {
|
|
33
|
+
reasons.add(entry.last_reason);
|
|
34
|
+
if (Date.parse(entry.next_attempt_at) > now.getTime()) {
|
|
35
|
+
heldCount += 1;
|
|
36
|
+
if (!heldOldest ||
|
|
37
|
+
entry.first_failed_at.localeCompare(heldOldest.first_failed_at) < 0) {
|
|
38
|
+
heldOldest = entry;
|
|
39
|
+
}
|
|
40
|
+
if (!heldNextAttemptAt ||
|
|
41
|
+
entry.next_attempt_at.localeCompare(heldNextAttemptAt) < 0) {
|
|
42
|
+
heldNextAttemptAt = entry.next_attempt_at;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
maxAttempts = Math.max(maxAttempts, entry.attempts);
|
|
46
|
+
if (!oldest || entry.first_failed_at.localeCompare(oldest) < 0) {
|
|
47
|
+
oldest = entry.first_failed_at;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return {
|
|
51
|
+
stuck_object_count: entries.length,
|
|
52
|
+
held_object_count: heldCount,
|
|
53
|
+
max_attempts: maxAttempts,
|
|
54
|
+
oldest_first_failed_at: oldest,
|
|
55
|
+
reasons: [...reasons].sort(),
|
|
56
|
+
held_oldest_first_failed_at: heldOldest?.first_failed_at ?? null,
|
|
57
|
+
held_next_attempt_at: heldNextAttemptAt,
|
|
58
|
+
held_last_reason: heldOldest?.last_reason ?? null,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
@@ -60,6 +60,13 @@ async function readPairedCollector(paths) {
|
|
|
60
60
|
throw new LocalUploadBlockedError("unpaired", "No paired collector session found. Run `cockpit login` or `cockpit pair` before `cockpit sync`.", "cockpit login");
|
|
61
61
|
});
|
|
62
62
|
const session = await readLocalSessionReference(paths);
|
|
63
|
+
// BLI-4019 deliberately did NOT loosen this. The heartbeat now knocks while
|
|
64
|
+
// the machine believes it is expired — that is the door the server renews it
|
|
65
|
+
// through — but an UPLOAD stays gated on `valid`, so a machine past the
|
|
66
|
+
// 30-day grace window never spends bandwidth and staging on evidence the
|
|
67
|
+
// server will refuse. One tick after the renewal lands, this reads `valid`
|
|
68
|
+
// again, because the heartbeat writes the server's date back into
|
|
69
|
+
// `session.json` (`recordDeviceTokenExpiry`).
|
|
63
70
|
if (session.session_state !== "valid") {
|
|
64
71
|
throw new LocalUploadBlockedError("unpaired", session.session_state === "expired"
|
|
65
72
|
? "Collector session expired. Run `cockpit login` or `cockpit pair` again before `cockpit sync`."
|
|
@@ -101,6 +101,12 @@ export function hasRetryableEvidenceGap(facts, outcomes) {
|
|
|
101
101
|
* backoff is missing collection right now. If it did not land here the sync
|
|
102
102
|
* would read clean while a transcript sat undelivered for nine days, which is
|
|
103
103
|
* the exact shape of BLI-3066.
|
|
104
|
+
*
|
|
105
|
+
* BLI-4022 did NOT change that. A hold still queues the retry and still travels
|
|
106
|
+
* to the status output; what changed is downstream, at the ledger in
|
|
107
|
+
* `commands/session-sync-failures.ts`, which no longer files it as a FAILED
|
|
108
|
+
* tick. Retryable and failed are two questions, and answering both with this
|
|
109
|
+
* one set is what painted a collecting machine red for eight ticks.
|
|
104
110
|
*/
|
|
105
111
|
function isRetryableEvidenceSkipReason(reason) {
|
|
106
112
|
if (reason === DELIVERY_BACKOFF_HOLDING_REASON)
|
|
@@ -201,5 +207,8 @@ export function summarizeRawEvidenceDelivery(built, outcomes, uploadedChunkCount
|
|
|
201
207
|
raw_evidence_stuck_object_count: stuck.stuck_object_count,
|
|
202
208
|
raw_evidence_max_delivery_attempts: stuck.max_attempts,
|
|
203
209
|
raw_evidence_oldest_delivery_failure_at: stuck.oldest_first_failed_at,
|
|
210
|
+
raw_evidence_held_oldest_failure_at: stuck.held_oldest_first_failed_at,
|
|
211
|
+
raw_evidence_held_next_attempt_at: stuck.held_next_attempt_at,
|
|
212
|
+
raw_evidence_held_last_reason: stuck.held_last_reason,
|
|
204
213
|
};
|
|
205
214
|
}
|