@bli-cockpit/cli 0.2.96 → 0.2.97
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/session-sync-failures.js +14 -0
- package/dist/commands/session-sync-health.js +9 -0
- package/dist/commands/session-sync-hold.js +90 -0
- package/dist/commands/session-sync.js +5 -0
- package/dist/commands/sync-receipt.js +20 -5
- package/dist/commands/sync-report.js +3 -0
- package/dist/delivery-hold-notice.js +166 -0
- package/dist/raw-evidence-staging.js +4 -27
- package/dist/raw-evidence-stuck-summary.js +60 -0
- package/dist/upload-evidence-delivery-summary.js +9 -0
- package/package.json +1 -1
|
@@ -15,7 +15,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
15
15
|
}
|
|
16
16
|
|
|
17
17
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
18
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
18
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.97");
|
|
19
19
|
return 0;
|
|
20
20
|
}
|
|
21
21
|
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
* without saying why.
|
|
14
14
|
*/
|
|
15
15
|
import { claudeAttributionReadFailureCount, codexAttributionReadFailureCount, } from "./agent-session-report.js";
|
|
16
|
+
import { isDeliveryHoldReason } from "./session-sync-hold.js";
|
|
16
17
|
import { sourceScanFailureReason } from "./session-sync-scan.js";
|
|
17
18
|
/**
|
|
18
19
|
* The label used when a sync fails and nothing on the way there said why.
|
|
@@ -45,6 +46,15 @@ export function createSyncFailureLedger() {
|
|
|
45
46
|
* The spooled reason is the most specific thing anyone has, so it leads, and it
|
|
46
47
|
* names the worktree it belongs to — a fleet failure is usually one repo, and
|
|
47
48
|
* "which one" is the first question asked.
|
|
49
|
+
*
|
|
50
|
+
* BLI-4022: `delivery_backoff_holding` is the one raw-evidence reason that does
|
|
51
|
+
* NOT come in here. An object inside its own retry window was withheld on
|
|
52
|
+
* purpose and will be offered again; filing that as a failed tick is what
|
|
53
|
+
* painted Ian's Mac mini red on eight consecutive ticks while every session it
|
|
54
|
+
* collected landed. The hold is still reported — it becomes the tick's named
|
|
55
|
+
* hold in `session-sync-hold.ts` and rides the same receipt — and a tick that
|
|
56
|
+
* ALSO failed for a real reason still fails on that reason. Nothing is
|
|
57
|
+
* silenced; only the verdict changed.
|
|
48
58
|
*/
|
|
49
59
|
export function recordWorktreeDeliveryFailures(ledger, outcomes) {
|
|
50
60
|
for (const { worktree, sync } of outcomes) {
|
|
@@ -61,12 +71,16 @@ export function recordWorktreeDeliveryFailures(ledger, outcomes) {
|
|
|
61
71
|
});
|
|
62
72
|
}
|
|
63
73
|
for (const reason of sync.raw_evidence_failure_reasons ?? []) {
|
|
74
|
+
if (isDeliveryHoldReason(reason))
|
|
75
|
+
continue;
|
|
64
76
|
ledger.add({
|
|
65
77
|
label: "raw_evidence_upload_failed",
|
|
66
78
|
rendered: `raw_evidence:${reason}`,
|
|
67
79
|
});
|
|
68
80
|
}
|
|
69
81
|
for (const reason of sync.raw_evidence_retry_reasons ?? []) {
|
|
82
|
+
if (isDeliveryHoldReason(reason))
|
|
83
|
+
continue;
|
|
70
84
|
ledger.add({
|
|
71
85
|
label: "raw_evidence_retry_required",
|
|
72
86
|
rendered: `raw_evidence_retry:${reason}`,
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
*/
|
|
12
12
|
import { createSyncFailureLedger, recordSourceScanFailures, recordUnexplainedFailure, recordUnpostedSessionReportFailure, recordWorktreeDeliveryFailures, } from "./session-sync-failures.js";
|
|
13
13
|
import { countSessionsNewThisTick, countSessionsPendingUpload, } from "./session-sync-counters.js";
|
|
14
|
+
import { logDeliveryHold, summarizeDeliveryHold } from "./session-sync-hold.js";
|
|
14
15
|
/**
|
|
15
16
|
* Decide whether this tick failed, and record every condition that decided it.
|
|
16
17
|
*
|
|
@@ -44,12 +45,20 @@ export function decideSyncHealth(options) {
|
|
|
44
45
|
if (nothingInRoot !== null && notice) {
|
|
45
46
|
announceNothingInRoot(nothingInRoot, options.collectionRootCount);
|
|
46
47
|
}
|
|
48
|
+
// BLI-4022. Computed after `ok`, and never allowed to change it: a hold is
|
|
49
|
+
// what this tick withheld on purpose, not a verdict on the run. It rides an
|
|
50
|
+
// `ok` receipt as the tick's reason and a failing one beside the reasons that
|
|
51
|
+
// failed it, so the board can say what is waiting in either case.
|
|
52
|
+
const hold = summarizeDeliveryHold(outcomes);
|
|
53
|
+
if (hold && ok)
|
|
54
|
+
logDeliveryHold(hold);
|
|
47
55
|
const records = ledger.sortedRecords();
|
|
48
56
|
return {
|
|
49
57
|
ok,
|
|
50
58
|
failure_reasons: records.map((record) => record.rendered),
|
|
51
59
|
failure_records: records,
|
|
52
60
|
notice,
|
|
61
|
+
hold,
|
|
53
62
|
sessions_outside_root: sessionsOutsideRoot,
|
|
54
63
|
sessions_new_this_tick: countSessionsNewThisTick({
|
|
55
64
|
sessions: options.sessions,
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What this tick is HOLDING, as opposed to what it got wrong (BLI-4022).
|
|
3
|
+
*
|
|
4
|
+
* The sibling `session-sync-failures.ts` is the one place a sync writes down
|
|
5
|
+
* that it FAILED. This is the one place it writes down that it deliberately
|
|
6
|
+
* withheld something and when it will try again — a different question with a
|
|
7
|
+
* different answer, kept in a different file so neither can quietly become the
|
|
8
|
+
* other.
|
|
9
|
+
*
|
|
10
|
+
* Deliberately writes no failure label of any kind: `sync-health-class.test.ts`
|
|
11
|
+
* reads every `session-sync*.ts` sibling and demands that only the failure
|
|
12
|
+
* ledger writes one, and this file is not it.
|
|
13
|
+
*/
|
|
14
|
+
import { formatDeliveryHoldNotice, } from "../delivery-hold-notice.js";
|
|
15
|
+
import { DELIVERY_BACKOFF_HOLDING_REASON } from "../raw-evidence-staging.js";
|
|
16
|
+
/**
|
|
17
|
+
* Is this reason a delivery hold rather than a delivery failure?
|
|
18
|
+
*
|
|
19
|
+
* Exact match against the label telemetry-core owns — the same discipline the
|
|
20
|
+
* failure classifier follows. A reason that merely CONTAINS the word "backoff"
|
|
21
|
+
* is not evidence of anything (BLI-3551's lesson, applied to the other side of
|
|
22
|
+
* the same fence).
|
|
23
|
+
*/
|
|
24
|
+
export function isDeliveryHoldReason(reason) {
|
|
25
|
+
return reason === DELIVERY_BACKOFF_HOLDING_REASON;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Fold every worktree's held objects into one hold for the tick, or null.
|
|
29
|
+
*
|
|
30
|
+
* The count is summed because a tick can sync several worktrees and each one
|
|
31
|
+
* reports its own held objects. The AGE takes the oldest across all of them and
|
|
32
|
+
* the NEXT ATTEMPT the earliest, because both are promises about the whole
|
|
33
|
+
* machine: the operator asks "how long has anything been stuck" and "when does
|
|
34
|
+
* anything move", not "which worktree".
|
|
35
|
+
*
|
|
36
|
+
* Returns null when nothing is held, so no caller can announce a hold of
|
|
37
|
+
* nothing — a status line that appears on every tick is one people stop
|
|
38
|
+
* reading.
|
|
39
|
+
*/
|
|
40
|
+
export function summarizeDeliveryHold(outcomes) {
|
|
41
|
+
let objectCount = 0;
|
|
42
|
+
let oldestFirstFailedAt = null;
|
|
43
|
+
let nextAttemptAt = null;
|
|
44
|
+
let lastReason = null;
|
|
45
|
+
for (const { sync } of outcomes) {
|
|
46
|
+
const held = sync.raw_evidence_delivery_held_count ?? 0;
|
|
47
|
+
if (held <= 0)
|
|
48
|
+
continue;
|
|
49
|
+
objectCount += held;
|
|
50
|
+
const first = sync.raw_evidence_held_oldest_failure_at ?? null;
|
|
51
|
+
if (first && (!oldestFirstFailedAt || first < oldestFirstFailedAt)) {
|
|
52
|
+
oldestFirstFailedAt = first;
|
|
53
|
+
// The reason travels WITH the object it belongs to. A reason taken from
|
|
54
|
+
// one worktree and an age from another would describe no object at all.
|
|
55
|
+
lastReason = sync.raw_evidence_held_last_reason ?? null;
|
|
56
|
+
}
|
|
57
|
+
const next = sync.raw_evidence_held_next_attempt_at ?? null;
|
|
58
|
+
if (next && (!nextAttemptAt || next < nextAttemptAt))
|
|
59
|
+
nextAttemptAt = next;
|
|
60
|
+
}
|
|
61
|
+
if (objectCount <= 0)
|
|
62
|
+
return null;
|
|
63
|
+
const notice = formatDeliveryHoldNotice({
|
|
64
|
+
objectCount,
|
|
65
|
+
oldestFirstFailedAt,
|
|
66
|
+
nextAttemptAt,
|
|
67
|
+
lastReason,
|
|
68
|
+
});
|
|
69
|
+
if (!notice)
|
|
70
|
+
return null;
|
|
71
|
+
return { objectCount, oldestFirstFailedAt, nextAttemptAt, lastReason, notice };
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Say out loud, once per tick, that bytes are being withheld on purpose.
|
|
75
|
+
*
|
|
76
|
+
* `logEvidenceHeldByBackoff` already names the objects at the moment they are
|
|
77
|
+
* partitioned off the wire; this is the RUN's line, so the tick that reports
|
|
78
|
+
* itself as held also explains itself on stderr where launchd captures it.
|
|
79
|
+
* Counts, instants and reason labels only.
|
|
80
|
+
*/
|
|
81
|
+
export function logDeliveryHold(hold) {
|
|
82
|
+
console.error("[session-sync] raw evidence held by delivery backoff, nothing else failed", JSON.stringify({
|
|
83
|
+
reason: hold.notice,
|
|
84
|
+
object_count: hold.objectCount,
|
|
85
|
+
oldest_first_failed_at: hold.oldestFirstFailedAt,
|
|
86
|
+
next_attempt_at: hold.nextAttemptAt,
|
|
87
|
+
last_reason: hold.lastReason,
|
|
88
|
+
next_action: "nothing until the next attempt is due; run `cockpit doctor` if it is already past",
|
|
89
|
+
}));
|
|
90
|
+
}
|
|
@@ -25,6 +25,9 @@
|
|
|
25
25
|
* class-lock test reads that file (BLI-3551).
|
|
26
26
|
* - `session-sync-health.ts` — the verdict, and the notice a quiet-but-working
|
|
27
27
|
* machine still emits.
|
|
28
|
+
* - `session-sync-hold.ts` — what the tick withheld ON PURPOSE and when it will
|
|
29
|
+
* try again (BLI-4022), which is a different question from what it got wrong
|
|
30
|
+
* and so lives in a different file from the failure ledger.
|
|
28
31
|
* - `session-sync-counters.ts` — the two honest heartbeat counters beside the
|
|
29
32
|
* raw scan count: sessions new since the last tick's cursor, and sessions
|
|
30
33
|
* still pending a durable pointer (BLI-3645).
|
|
@@ -49,6 +52,7 @@ export { isLiveSyncCollectableAttributionState, liveSyncCursorEntryRequiresRetry
|
|
|
49
52
|
export { sourceScanFailureReason, sourceScanRetryReason, } from "./session-sync-scan.js";
|
|
50
53
|
export { SYNC_FAILED_WITHOUT_REASON } from "./session-sync-failures.js";
|
|
51
54
|
export { nothingInRootCount } from "./session-sync-health.js";
|
|
55
|
+
export { isDeliveryHoldReason, summarizeDeliveryHold, } from "./session-sync-hold.js";
|
|
52
56
|
/**
|
|
53
57
|
* Shared dual-source sync orchestration for single-repo and parent-folder
|
|
54
58
|
* modes. Codex AND Claude Code sessions are scanned and attributed once across
|
|
@@ -126,6 +130,7 @@ export async function runAttributedWorktreeSync(options) {
|
|
|
126
130
|
failure_reasons: health.failure_reasons,
|
|
127
131
|
failure_records: health.failure_records,
|
|
128
132
|
notice: health.notice,
|
|
133
|
+
hold: health.hold,
|
|
129
134
|
sessions_observed: sessions.length,
|
|
130
135
|
sessions_outside_root: health.sessions_outside_root,
|
|
131
136
|
sessions_new_this_tick: health.sessions_new_this_tick,
|
|
@@ -91,16 +91,25 @@ function standAsideForRunningSync(command, io, heldSince) {
|
|
|
91
91
|
* is the receipt that separates "this machine is alive and its operator
|
|
92
92
|
* works outside the approved roots" from "this machine is dead", which
|
|
93
93
|
* until now looked identical from the dashboard.
|
|
94
|
+
*
|
|
95
|
+
* BLI-4022 added the second thing an `ok` tick can say: `held_backoff:…`, the
|
|
96
|
+
* objects this tick withheld on purpose and when they may be offered again.
|
|
97
|
+
* The two can never collide — `nothing_in_root` requires that NO worktree
|
|
98
|
+
* synced at all (`nothingInRootCount`), and a hold requires a worktree that
|
|
99
|
+
* did — so one field carries whichever is true, and the notice wins if that
|
|
100
|
+
* ever stops being the case, because "there was nothing to collect" is the
|
|
101
|
+
* larger fact about the tick.
|
|
94
102
|
*/
|
|
95
|
-
function collectedReceipt(run) {
|
|
103
|
+
export function collectedReceipt(run) {
|
|
104
|
+
const reason = run.notice ?? run.holdNotice;
|
|
96
105
|
return {
|
|
97
106
|
exitCode: run.exitCode,
|
|
98
107
|
completion: {
|
|
99
108
|
step: "sync_complete",
|
|
100
109
|
status: "ok",
|
|
101
|
-
...(
|
|
110
|
+
...(reason ? { error_detail: reason } : {}),
|
|
102
111
|
},
|
|
103
|
-
heartbeat: { status: "ok", reason
|
|
112
|
+
heartbeat: { status: "ok", reason, ...heartbeatCounts(run) },
|
|
104
113
|
};
|
|
105
114
|
}
|
|
106
115
|
/**
|
|
@@ -111,8 +120,14 @@ function collectedReceipt(run) {
|
|
|
111
120
|
* failure rows carried a null detail and the real reason was reachable only
|
|
112
121
|
* by running `cockpit status` on the machine itself (BLI-2526).
|
|
113
122
|
*/
|
|
114
|
-
function failedReceipt(run) {
|
|
115
|
-
|
|
123
|
+
export function failedReceipt(run) {
|
|
124
|
+
// BLI-4022: the hold leads on a failing tick. `error_detail` is bounded, and
|
|
125
|
+
// an operator reading a red row still has to be able to see what is merely
|
|
126
|
+
// waiting — otherwise the held objects become invisible the moment anything
|
|
127
|
+
// else goes wrong, which is the opposite of the fix.
|
|
128
|
+
const reasonText = [run.holdNotice, ...run.failureReasons]
|
|
129
|
+
.filter(Boolean)
|
|
130
|
+
.join("; ");
|
|
116
131
|
// The bucket comes from the records the deciding branches wrote, not from
|
|
117
132
|
// this sentence (BLI-3551). The sentence is still the detail.
|
|
118
133
|
const errorCode = classifySyncFailureRecords(run.failureRecords);
|
|
@@ -141,6 +141,9 @@ function syncResult(run) {
|
|
|
141
141
|
failureReasons: run.ok ? [] : run.failure_reasons,
|
|
142
142
|
failureRecords: run.ok ? [] : run.failure_records,
|
|
143
143
|
notice: run.notice,
|
|
144
|
+
// BLI-4022: unlike the reasons, the hold survives a successful run — it is
|
|
145
|
+
// what the tick withheld, not why it failed.
|
|
146
|
+
holdNotice: run.hold?.notice ?? null,
|
|
144
147
|
sessionsObserved: run.sessions_observed,
|
|
145
148
|
sessionsOutsideRoot: run.sessions_outside_root,
|
|
146
149
|
sessionsNewThisTick: run.sessions_new_this_tick,
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A HOLD is not a FAILURE — and this file is the one place that spells it.
|
|
3
|
+
*
|
|
4
|
+
* BLI-4022. `delivery_backoff_holding` means "this object is inside its own
|
|
5
|
+
* retry window, so this tick did not offer it". Until now that reason went
|
|
6
|
+
* into the sync's failure ledger like any other, so a machine collecting
|
|
7
|
+
* perfectly reported `sync_failed` on every tick for as long as the backoff
|
|
8
|
+
* held: Ian's Mac mini was RED on eight consecutive ticks from
|
|
9
|
+
* 2026-09-09T00:11Z with the detail
|
|
10
|
+
* `raw_evidence_retry:delivery_backoff_holding; raw_evidence:delivery_backoff_holding`
|
|
11
|
+
* while its sessions kept landing and its heartbeat kept moving.
|
|
12
|
+
*
|
|
13
|
+
* The honesty is unchanged. A held object IS missing collection and must never
|
|
14
|
+
* read as clean (that is the rule BLI-3066 bought). What changes is the
|
|
15
|
+
* OUTCOME: a tick whose only raw-evidence complaint is held objects reports a
|
|
16
|
+
* named hold instead of a failure, and the hold carries everything an operator
|
|
17
|
+
* needs to decide whether to care — how many objects, how long the oldest has
|
|
18
|
+
* been failing, when the next attempt is due, and the object's own last reason.
|
|
19
|
+
*
|
|
20
|
+
* ## The grammar
|
|
21
|
+
*
|
|
22
|
+
* One token, because it has to survive two narrow channels: the heartbeat's
|
|
23
|
+
* `last_sync_reason` (`^[a-z0-9_:.-]+$`, 120 chars, no spaces) and the receipt's
|
|
24
|
+
* `error_detail`. Fields are positional and colon-separated:
|
|
25
|
+
*
|
|
26
|
+
* held_backoff:<count>:<oldest_first_failed_at>:<next_attempt_at>:<last_reason>
|
|
27
|
+
* held_backoff:3:2026-08-14T17:38Z:2026-09-09T06:11Z:chunk_10_failed_http_500
|
|
28
|
+
*
|
|
29
|
+
* Timestamps are minute precision UTC; `-` stands for a field this machine
|
|
30
|
+
* could not fill. The last reason is bounded so the whole token fits the
|
|
31
|
+
* heartbeat's 120 characters — the count and the ages are what a colour is
|
|
32
|
+
* decided from, so the reason is the field that gives way.
|
|
33
|
+
*
|
|
34
|
+
* The collector WRITES the token and the dashboard's fleet board READS it
|
|
35
|
+
* (`lib/ops/fleet-liveness-device-verdict.ts`, through this package's
|
|
36
|
+
* `./delivery-hold` export). One grammar, one file: a second copy of this
|
|
37
|
+
* string is exactly how the collector's word and the server's word drift
|
|
38
|
+
* apart, which is why `DELIVERY_BACKOFF_HOLDING` itself is aliased from
|
|
39
|
+
* telemetry-core rather than spelled twice.
|
|
40
|
+
*/
|
|
41
|
+
/** The token's leading word. Greppable in a receipt, a log line or a DM. */
|
|
42
|
+
export const DELIVERY_HOLD_NOTICE_PREFIX = "held_backoff";
|
|
43
|
+
/** The heartbeat's own ceiling for `last_sync_reason`. The token fits inside it. */
|
|
44
|
+
export const DELIVERY_HOLD_NOTICE_MAX_CHARS = 120;
|
|
45
|
+
/**
|
|
46
|
+
* How long a hold may last before it stops being a wait and becomes a stuck
|
|
47
|
+
* object.
|
|
48
|
+
*
|
|
49
|
+
* The backoff schedule doubles from the scheduler's own 15-minute cadence to a
|
|
50
|
+
* six-hour ceiling (`EVIDENCE_DELIVERY_BACKOFF_MAX_MS`), so an object held
|
|
51
|
+
* longer than this has already been offered at least four more times at the
|
|
52
|
+
* ceiling and been refused every time. Nothing is going to change on its own,
|
|
53
|
+
* and "next retry at T — nothing to do until then" would be a promise the
|
|
54
|
+
* machine cannot keep. 24 hours is also the board's own `SILENT_AFTER_HOURS`,
|
|
55
|
+
* so both clocks agree on when a quiet thing stops being a blip.
|
|
56
|
+
*/
|
|
57
|
+
export const DELIVERY_HOLD_STALE_AFTER_HOURS = 24;
|
|
58
|
+
/** The most characters of an object's own reason label the token may carry. */
|
|
59
|
+
const MAX_HOLD_REASON_CHARS = 64;
|
|
60
|
+
/** A field this machine could not fill. Never omitted — the shape is positional. */
|
|
61
|
+
const ABSENT = "-";
|
|
62
|
+
/**
|
|
63
|
+
* Render the hold as the single token the heartbeat and the receipt carry.
|
|
64
|
+
*
|
|
65
|
+
* Returns null for a hold of nothing, so a caller cannot accidentally announce
|
|
66
|
+
* a hold that is not happening.
|
|
67
|
+
*/
|
|
68
|
+
export function formatDeliveryHoldNotice(notice) {
|
|
69
|
+
if (notice.objectCount <= 0)
|
|
70
|
+
return null;
|
|
71
|
+
const head = [
|
|
72
|
+
DELIVERY_HOLD_NOTICE_PREFIX,
|
|
73
|
+
String(Math.trunc(notice.objectCount)),
|
|
74
|
+
minuteStamp(notice.oldestFirstFailedAt),
|
|
75
|
+
minuteStamp(notice.nextAttemptAt),
|
|
76
|
+
].join(":");
|
|
77
|
+
const budget = Math.min(MAX_HOLD_REASON_CHARS, DELIVERY_HOLD_NOTICE_MAX_CHARS - head.length - 1);
|
|
78
|
+
return `${head}:${reasonWord(notice.lastReason, budget)}`;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Find a hold token inside anything a receipt carries, and read it back.
|
|
82
|
+
*
|
|
83
|
+
* Deliberately a SEARCH, not a whole-string match: on a tick that also failed
|
|
84
|
+
* for a real reason the token rides beside those reasons in the same
|
|
85
|
+
* `error_detail`, and the board must still be able to say what is held.
|
|
86
|
+
*/
|
|
87
|
+
export function parseDeliveryHoldNotice(text) {
|
|
88
|
+
if (!text)
|
|
89
|
+
return null;
|
|
90
|
+
// The stamp shape is pinned rather than described by a character class: a
|
|
91
|
+
// class containing `:` is greedy across the field separator, and a parser
|
|
92
|
+
// that swallowed one field into the next reported an age of twenty-five
|
|
93
|
+
// years and a next attempt in 2001.
|
|
94
|
+
const stamp = String.raw `(?:-|\d{4}-\d{2}-\d{2}T\d{2}:\d{2}Z)`;
|
|
95
|
+
const match = new RegExp(`${DELIVERY_HOLD_NOTICE_PREFIX}:(\\d+):(${stamp}):(${stamp}):([a-z0-9_-]+)`, "iu").exec(text);
|
|
96
|
+
if (!match)
|
|
97
|
+
return null;
|
|
98
|
+
const objectCount = Number.parseInt(match[1], 10);
|
|
99
|
+
if (!Number.isFinite(objectCount) || objectCount <= 0)
|
|
100
|
+
return null;
|
|
101
|
+
return {
|
|
102
|
+
objectCount,
|
|
103
|
+
oldestFirstFailedAt: readStamp(match[2]),
|
|
104
|
+
nextAttemptAt: readStamp(match[3]),
|
|
105
|
+
lastReason: match[4] === ABSENT ? null : match[4],
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* How long the longest-held object has been failing, in hours.
|
|
110
|
+
*
|
|
111
|
+
* Null when the token carried no first-failure instant, which is a different
|
|
112
|
+
* fact from "zero hours" — an unknown age is never treated as a fresh hold.
|
|
113
|
+
*/
|
|
114
|
+
export function deliveryHoldAgeHours(notice, nowIso) {
|
|
115
|
+
if (!notice.oldestFirstFailedAt)
|
|
116
|
+
return null;
|
|
117
|
+
const from = Date.parse(notice.oldestFirstFailedAt);
|
|
118
|
+
const now = Date.parse(nowIso);
|
|
119
|
+
if (!Number.isFinite(from) || !Number.isFinite(now))
|
|
120
|
+
return null;
|
|
121
|
+
return (now - from) / 3_600_000;
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Has this hold stopped being a wait?
|
|
125
|
+
*
|
|
126
|
+
* An age this reader could not compute counts as stale on purpose: the whole
|
|
127
|
+
* point of the amber sentence is the promise "next retry at T", and a hold
|
|
128
|
+
* whose age is unknown is one nobody can make that promise about.
|
|
129
|
+
*/
|
|
130
|
+
export function isDeliveryHoldStale(notice, nowIso) {
|
|
131
|
+
const hours = deliveryHoldAgeHours(notice, nowIso);
|
|
132
|
+
if (hours === null)
|
|
133
|
+
return true;
|
|
134
|
+
return hours > DELIVERY_HOLD_STALE_AFTER_HOURS;
|
|
135
|
+
}
|
|
136
|
+
/** `2026-09-09T06:11Z`, or `-` for an instant this machine does not have. */
|
|
137
|
+
function minuteStamp(iso) {
|
|
138
|
+
if (!iso)
|
|
139
|
+
return ABSENT;
|
|
140
|
+
const parsed = Date.parse(iso);
|
|
141
|
+
if (!Number.isFinite(parsed))
|
|
142
|
+
return ABSENT;
|
|
143
|
+
return `${new Date(parsed).toISOString().slice(0, 16)}Z`;
|
|
144
|
+
}
|
|
145
|
+
/** The inverse: a token field back to an ISO instant, or null when absent. */
|
|
146
|
+
function readStamp(field) {
|
|
147
|
+
if (field === ABSENT)
|
|
148
|
+
return null;
|
|
149
|
+
const parsed = Date.parse(field);
|
|
150
|
+
return Number.isFinite(parsed) ? new Date(parsed).toISOString() : null;
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* A reason label narrowed to what the token's charset and budget allow.
|
|
154
|
+
*
|
|
155
|
+
* Truncation is marked by dropping characters, never by an ellipsis: the
|
|
156
|
+
* heartbeat's charset has no `…`, and a token the door rejects tells nobody
|
|
157
|
+
* anything at all.
|
|
158
|
+
*/
|
|
159
|
+
function reasonWord(reason, budget) {
|
|
160
|
+
if (!reason || budget <= 0)
|
|
161
|
+
return ABSENT;
|
|
162
|
+
const safe = reason.replace(/[^a-z0-9_]/giu, "_").replace(/^_+|_+$/gu, "");
|
|
163
|
+
if (safe.length === 0)
|
|
164
|
+
return ABSENT;
|
|
165
|
+
return safe.slice(0, budget);
|
|
166
|
+
}
|
|
@@ -25,6 +25,10 @@ import { describeError, isMissingFileFailure } from "./health-detail.js";
|
|
|
25
25
|
* Metadata only: content hashes, byte sizes, pack ids, pack-relative paths,
|
|
26
26
|
* reason labels, timestamps. Never content, never an absolute path.
|
|
27
27
|
*/
|
|
28
|
+
// BLI-4022. The stuck/held summary moved to its own sibling so this file
|
|
29
|
+
// stays under the readability band; it is still importable from here,
|
|
30
|
+
// which is the address every caller already knows.
|
|
31
|
+
export { summarizeStuckEvidence, } from "./raw-evidence-stuck-summary.js";
|
|
28
32
|
export const RAW_EVIDENCE_STAGING_FILENAME = "raw-evidence-staging.json";
|
|
29
33
|
/**
|
|
30
34
|
* Backoff schedule for an object whose delivery keeps failing.
|
|
@@ -207,33 +211,6 @@ export function evidenceSourceKey(options) {
|
|
|
207
211
|
: "unknown";
|
|
208
212
|
return `${options.kind}:${options.sessionId ?? "none"}:${source}`;
|
|
209
213
|
}
|
|
210
|
-
/**
|
|
211
|
-
* What `cockpit status` and the health receipt need in order to be unable to
|
|
212
|
-
* read green while an object has been failing for nine days.
|
|
213
|
-
*/
|
|
214
|
-
export function summarizeStuckEvidence(state, now) {
|
|
215
|
-
const entries = Object.values(state.delivery_attempts);
|
|
216
|
-
const reasons = new Set();
|
|
217
|
-
let heldCount = 0;
|
|
218
|
-
let maxAttempts = 0;
|
|
219
|
-
let oldest = null;
|
|
220
|
-
for (const entry of entries) {
|
|
221
|
-
reasons.add(entry.last_reason);
|
|
222
|
-
if (Date.parse(entry.next_attempt_at) > now.getTime())
|
|
223
|
-
heldCount += 1;
|
|
224
|
-
maxAttempts = Math.max(maxAttempts, entry.attempts);
|
|
225
|
-
if (!oldest || entry.first_failed_at.localeCompare(oldest) < 0) {
|
|
226
|
-
oldest = entry.first_failed_at;
|
|
227
|
-
}
|
|
228
|
-
}
|
|
229
|
-
return {
|
|
230
|
-
stuck_object_count: entries.length,
|
|
231
|
-
held_object_count: heldCount,
|
|
232
|
-
max_attempts: maxAttempts,
|
|
233
|
-
oldest_first_failed_at: oldest,
|
|
234
|
-
reasons: [...reasons].sort(),
|
|
235
|
-
};
|
|
236
|
-
}
|
|
237
214
|
/**
|
|
238
215
|
* Pack id derived from CONTENT, never from the clock.
|
|
239
216
|
*
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* "Is anything stuck, and is anything being held right now?" — the two
|
|
3
|
+
* questions `cockpit status`, the health receipt and the fleet board ask of the
|
|
4
|
+
* delivery-attempt history, split out of `raw-evidence-staging.ts` (BLI-4022)
|
|
5
|
+
* so that file stays under the repo's readability band.
|
|
6
|
+
*
|
|
7
|
+
* The distinction this file exists to keep: an object with a live failure
|
|
8
|
+
* history is STUCK, and an object whose next attempt is still in the future is
|
|
9
|
+
* HELD. Every held object is stuck and most stuck objects are held, but the two
|
|
10
|
+
* answer different questions — "has anything been failing for nine days" versus
|
|
11
|
+
* "what did this tick withhold, and when does it try again" — and BLI-4022
|
|
12
|
+
* needed the second one, which nothing computed.
|
|
13
|
+
*
|
|
14
|
+
* Metadata only, like the rest of this family: content hashes, byte sizes,
|
|
15
|
+
* reason labels, timestamps. Never content, never an absolute path.
|
|
16
|
+
*/
|
|
17
|
+
/**
|
|
18
|
+
* What `cockpit status` and the health receipt need in order to be unable to
|
|
19
|
+
* read green while an object has been failing for nine days.
|
|
20
|
+
*/
|
|
21
|
+
export function summarizeStuckEvidence(state, now) {
|
|
22
|
+
const entries = Object.values(state.delivery_attempts);
|
|
23
|
+
const reasons = new Set();
|
|
24
|
+
let heldCount = 0;
|
|
25
|
+
let maxAttempts = 0;
|
|
26
|
+
let oldest = null;
|
|
27
|
+
// The held objects are tracked apart from the stuck ones, because a hold is
|
|
28
|
+
// judged on what is being withheld NOW and on when it may next be offered —
|
|
29
|
+
// not on every object that has ever failed here.
|
|
30
|
+
let heldOldest = null;
|
|
31
|
+
let heldNextAttemptAt = null;
|
|
32
|
+
for (const entry of entries) {
|
|
33
|
+
reasons.add(entry.last_reason);
|
|
34
|
+
if (Date.parse(entry.next_attempt_at) > now.getTime()) {
|
|
35
|
+
heldCount += 1;
|
|
36
|
+
if (!heldOldest ||
|
|
37
|
+
entry.first_failed_at.localeCompare(heldOldest.first_failed_at) < 0) {
|
|
38
|
+
heldOldest = entry;
|
|
39
|
+
}
|
|
40
|
+
if (!heldNextAttemptAt ||
|
|
41
|
+
entry.next_attempt_at.localeCompare(heldNextAttemptAt) < 0) {
|
|
42
|
+
heldNextAttemptAt = entry.next_attempt_at;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
maxAttempts = Math.max(maxAttempts, entry.attempts);
|
|
46
|
+
if (!oldest || entry.first_failed_at.localeCompare(oldest) < 0) {
|
|
47
|
+
oldest = entry.first_failed_at;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return {
|
|
51
|
+
stuck_object_count: entries.length,
|
|
52
|
+
held_object_count: heldCount,
|
|
53
|
+
max_attempts: maxAttempts,
|
|
54
|
+
oldest_first_failed_at: oldest,
|
|
55
|
+
reasons: [...reasons].sort(),
|
|
56
|
+
held_oldest_first_failed_at: heldOldest?.first_failed_at ?? null,
|
|
57
|
+
held_next_attempt_at: heldNextAttemptAt,
|
|
58
|
+
held_last_reason: heldOldest?.last_reason ?? null,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
@@ -101,6 +101,12 @@ export function hasRetryableEvidenceGap(facts, outcomes) {
|
|
|
101
101
|
* backoff is missing collection right now. If it did not land here the sync
|
|
102
102
|
* would read clean while a transcript sat undelivered for nine days, which is
|
|
103
103
|
* the exact shape of BLI-3066.
|
|
104
|
+
*
|
|
105
|
+
* BLI-4022 did NOT change that. A hold still queues the retry and still travels
|
|
106
|
+
* to the status output; what changed is downstream, at the ledger in
|
|
107
|
+
* `commands/session-sync-failures.ts`, which no longer files it as a FAILED
|
|
108
|
+
* tick. Retryable and failed are two questions, and answering both with this
|
|
109
|
+
* one set is what painted a collecting machine red for eight ticks.
|
|
104
110
|
*/
|
|
105
111
|
function isRetryableEvidenceSkipReason(reason) {
|
|
106
112
|
if (reason === DELIVERY_BACKOFF_HOLDING_REASON)
|
|
@@ -201,5 +207,8 @@ export function summarizeRawEvidenceDelivery(built, outcomes, uploadedChunkCount
|
|
|
201
207
|
raw_evidence_stuck_object_count: stuck.stuck_object_count,
|
|
202
208
|
raw_evidence_max_delivery_attempts: stuck.max_attempts,
|
|
203
209
|
raw_evidence_oldest_delivery_failure_at: stuck.oldest_first_failed_at,
|
|
210
|
+
raw_evidence_held_oldest_failure_at: stuck.held_oldest_first_failed_at,
|
|
211
|
+
raw_evidence_held_next_attempt_at: stuck.held_next_attempt_at,
|
|
212
|
+
raw_evidence_held_last_reason: stuck.held_last_reason,
|
|
204
213
|
};
|
|
205
214
|
}
|