@bli-cockpit/cli 0.2.12 → 0.2.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/autostart.js +72 -5
- package/dist/commands/backfill.js +23 -3
- package/dist/commands/local.js +34 -9
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/session-sync.js +103 -20
- package/dist/evidence-upload-client.js +27 -1
- package/dist/upload.js +54 -6
- package/package.json +2 -2
package/dist/autostart.js
CHANGED
|
@@ -270,6 +270,13 @@ async function windowsTaskStatus(options) {
|
|
|
270
270
|
const queryMessage = `${query.stderr}\n${query.stdout}`;
|
|
271
271
|
const taskIsKnownAbsent = query.code === 1 &&
|
|
272
272
|
/cannot find|not found|does not exist/iu.test(queryMessage);
|
|
273
|
+
// Metadata only: schtasks stderr can carry the state directory, so the exit
|
|
274
|
+
// code and the classification travel, never the text.
|
|
275
|
+
console.error("[autostart] windows task query failed", JSON.stringify({
|
|
276
|
+
task_name: WINDOWS_AUTOSTART_TASK_NAME,
|
|
277
|
+
exit_code: query.code,
|
|
278
|
+
status: taskIsKnownAbsent ? "absent" : "not_loaded",
|
|
279
|
+
}));
|
|
273
280
|
return {
|
|
274
281
|
status: taskIsKnownAbsent ? "absent" : "not_loaded",
|
|
275
282
|
label: AUTOSTART_LABEL,
|
|
@@ -305,6 +312,23 @@ async function windowsTaskStatus(options) {
|
|
|
305
312
|
registrationProblems.push("sync script does not match the current roots or Cockpit runtime");
|
|
306
313
|
}
|
|
307
314
|
}
|
|
315
|
+
// Both branches log. A line that only fires on failure cannot answer "did
|
|
316
|
+
// background collection validate at all today?", which is the question that
|
|
317
|
+
// would have caught a validator rejecting every healthy Windows task.
|
|
318
|
+
// Problem labels carry no paths, so they are safe to emit verbatim.
|
|
319
|
+
if (registrationProblems.length > 0) {
|
|
320
|
+
console.error("[autostart] windows task needs repair", JSON.stringify({
|
|
321
|
+
task_name: WINDOWS_AUTOSTART_TASK_NAME,
|
|
322
|
+
problem_count: registrationProblems.length,
|
|
323
|
+
problems: registrationProblems,
|
|
324
|
+
}));
|
|
325
|
+
}
|
|
326
|
+
else {
|
|
327
|
+
console.error("[autostart] windows task validated", JSON.stringify({
|
|
328
|
+
task_name: WINDOWS_AUTOSTART_TASK_NAME,
|
|
329
|
+
interval_minutes: intervalMinutes,
|
|
330
|
+
}));
|
|
331
|
+
}
|
|
308
332
|
return {
|
|
309
333
|
status: registrationProblems.length > 0 ? "not_loaded" : "loaded",
|
|
310
334
|
label: AUTOSTART_LABEL,
|
|
@@ -415,7 +439,7 @@ function windowsTaskRegistrationProblems(taskXml, expected) {
|
|
|
415
439
|
const actionArguments = exactExecBlock
|
|
416
440
|
.match(/<Arguments>\s*([^<]*?)\s*<\/Arguments>/iu)?.[1]
|
|
417
441
|
?.trim() ?? "";
|
|
418
|
-
const
|
|
442
|
+
const expectedArgumentFlags = "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File";
|
|
419
443
|
const requirements = [
|
|
420
444
|
[
|
|
421
445
|
/<DisallowStartIfOnBatteries>\s*false\s*<\/DisallowStartIfOnBatteries>/iu,
|
|
@@ -444,8 +468,18 @@ function windowsTaskRegistrationProblems(taskXml, expected) {
|
|
|
444
468
|
if (!/<LogonType>\s*InteractiveToken\s*<\/LogonType>/iu.test(exactPrincipalBlock)) {
|
|
445
469
|
problems.push("logon type is not InteractiveToken");
|
|
446
470
|
}
|
|
447
|
-
|
|
448
|
-
|
|
471
|
+
// Windows omits <RunLevel> entirely when a task runs at the default
|
|
472
|
+
// LeastPrivilege, so `schtasks /Create ... /RL LIMITED` stores a principal
|
|
473
|
+
// block with no RunLevel element at all. Demanding the element asks Windows
|
|
474
|
+
// to state a default it never states, which is why this check failed on
|
|
475
|
+
// every Windows machine in the fleet and passed on none (0 of 4
|
|
476
|
+
// autostart-alive events ok, 2026-08-12). Absent means LeastPrivilege; only
|
|
477
|
+
// an explicit elevated value is a real problem.
|
|
478
|
+
const runLevel = exactPrincipalBlock
|
|
479
|
+
.match(/<RunLevel>\s*([^<]*?)\s*<\/RunLevel>/iu)?.[1]
|
|
480
|
+
?.trim() || "LeastPrivilege";
|
|
481
|
+
if (!/^LeastPrivilege$/iu.test(runLevel)) {
|
|
482
|
+
problems.push(`task does not run with limited privileges (RunLevel=${runLevel})`);
|
|
449
483
|
}
|
|
450
484
|
}
|
|
451
485
|
if (actionContext && actionContext !== principalId) {
|
|
@@ -477,8 +511,9 @@ function windowsTaskRegistrationProblems(taskXml, expected) {
|
|
|
477
511
|
if (!sameWindowsPath(command, expected.powershellPath)) {
|
|
478
512
|
problems.push("task action does not run the expected Windows PowerShell");
|
|
479
513
|
}
|
|
480
|
-
|
|
481
|
-
|
|
514
|
+
const actionArgumentProblem = windowsActionArgumentProblem(actionArguments, expectedArgumentFlags, expected.scriptPath);
|
|
515
|
+
if (actionArgumentProblem) {
|
|
516
|
+
problems.push(actionArgumentProblem);
|
|
482
517
|
}
|
|
483
518
|
if (/<Duration>/iu.test(repetition)) {
|
|
484
519
|
problems.push("repetition has a finite duration");
|
|
@@ -489,6 +524,38 @@ function windowsPowerShellPath(env = process.env) {
|
|
|
489
524
|
const windowsRoot = env.SystemRoot ?? env.WINDIR ?? "C:\\Windows";
|
|
490
525
|
return path.win32.join(windowsRoot, "System32", "WindowsPowerShell", "v1.0", "powershell.exe");
|
|
491
526
|
}
|
|
527
|
+
/**
|
|
528
|
+
* Task Scheduler stores a normalized form of the action it was given, not the
|
|
529
|
+
* string we passed: `schtasks /Create /TR "<powershell> <args>"` is split into
|
|
530
|
+
* <Command> plus <Arguments>, and the quotes around a space-free script path
|
|
531
|
+
* are dropped. Comparing <Arguments> against the exact string we built
|
|
532
|
+
* therefore fails on a task that is registered perfectly — observed on
|
|
533
|
+
* DESKTOP-G2UO1GK (CLI 0.2.13, 2026-08-12), where the stored value differed
|
|
534
|
+
* from the expected one only by those quotes.
|
|
535
|
+
*
|
|
536
|
+
* Compare the flags exactly, because those are ours, and compare the script as
|
|
537
|
+
* a path, because that is what Windows normalizes. Each failure returns its own
|
|
538
|
+
* reason so a repair message says which half is wrong rather than "arguments
|
|
539
|
+
* differ".
|
|
540
|
+
*/
|
|
541
|
+
function windowsActionArgumentProblem(actionArguments, expectedFlags, expectedScriptPath) {
|
|
542
|
+
if (!actionArguments.startsWith(expectedFlags)) {
|
|
543
|
+
return "task action does not run PowerShell with the expected Cockpit sync flags";
|
|
544
|
+
}
|
|
545
|
+
const scriptArgument = unquoteWindowsArgument(actionArguments.slice(expectedFlags.length));
|
|
546
|
+
if (!scriptArgument) {
|
|
547
|
+
return "task action names no sync script to run";
|
|
548
|
+
}
|
|
549
|
+
if (!sameWindowsPath(scriptArgument, expectedScriptPath)) {
|
|
550
|
+
return "task action runs a script other than the Cockpit sync script";
|
|
551
|
+
}
|
|
552
|
+
return null;
|
|
553
|
+
}
|
|
554
|
+
function unquoteWindowsArgument(value) {
|
|
555
|
+
const trimmed = value.trim();
|
|
556
|
+
const quoted = trimmed.match(/^"([\s\S]*)"$/u) ?? trimmed.match(/^'([\s\S]*)'$/u);
|
|
557
|
+
return (quoted?.[1] ?? trimmed).trim();
|
|
558
|
+
}
|
|
492
559
|
function sameWindowsPath(left, right) {
|
|
493
560
|
if (!left || !right)
|
|
494
561
|
return false;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { containsSecretLikeContent, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES } from "@bli-cockpit/telemetry-core";
|
|
1
|
+
import { containsSecretLikeContent, NO_UPLOAD_ATTEMPT_RECORDED, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES } from "@bli-cockpit/telemetry-core";
|
|
2
2
|
import crypto from "node:crypto";
|
|
3
3
|
import fs from "node:fs/promises";
|
|
4
4
|
import os from "node:os";
|
|
@@ -1081,11 +1081,14 @@ async function syncBackfillBatch(options) {
|
|
|
1081
1081
|
}
|
|
1082
1082
|
function buildBackfillSessionReport(options) {
|
|
1083
1083
|
const uploadByKey = new Map();
|
|
1084
|
+
// BLI-2107: an outcome that names a failure but has no pointer used to be
|
|
1085
|
+
// dropped on the floor here, taking its reason with it.
|
|
1086
|
+
const noUploadReasonBySessionId = new Map();
|
|
1084
1087
|
for (const sync of options.syncResults) {
|
|
1085
1088
|
if (sync.status !== "uploaded")
|
|
1086
1089
|
continue;
|
|
1087
1090
|
for (const outcome of sync.raw_evidence_outcomes) {
|
|
1088
|
-
if (!outcome.codex_session_id
|
|
1091
|
+
if (!outcome.codex_session_id)
|
|
1089
1092
|
continue;
|
|
1090
1093
|
const source = outcome.kind === "claude_jsonl"
|
|
1091
1094
|
? "claude_code"
|
|
@@ -1094,9 +1097,16 @@ function buildBackfillSessionReport(options) {
|
|
|
1094
1097
|
: null;
|
|
1095
1098
|
if (!source)
|
|
1096
1099
|
continue;
|
|
1100
|
+
if (!outcome.raw_evidence_pointer_id) {
|
|
1101
|
+
if (outcome.reason) {
|
|
1102
|
+
noUploadReasonBySessionId.set(outcome.codex_session_id, outcome.reason);
|
|
1103
|
+
}
|
|
1104
|
+
continue;
|
|
1105
|
+
}
|
|
1097
1106
|
uploadByKey.set(`${source}:${outcome.codex_session_id}`, {
|
|
1098
1107
|
upload_state: outcome.upload_state,
|
|
1099
1108
|
raw_evidence_pointer_id: outcome.raw_evidence_pointer_id,
|
|
1109
|
+
reason: outcome.reason,
|
|
1100
1110
|
});
|
|
1101
1111
|
}
|
|
1102
1112
|
}
|
|
@@ -1143,9 +1153,19 @@ function buildBackfillSessionReport(options) {
|
|
|
1143
1153
|
? {
|
|
1144
1154
|
raw_evidence_pointer_id: upload.raw_evidence_pointer_id,
|
|
1145
1155
|
upload_state: upload.upload_state,
|
|
1156
|
+
...(upload.upload_state === "upload_failed"
|
|
1157
|
+
? { upload_reason: upload.reason ?? NO_UPLOAD_ATTEMPT_RECORDED }
|
|
1158
|
+
: {}),
|
|
1146
1159
|
}
|
|
1147
1160
|
: isRawEvidenceUploadableAttributionState(candidate.state)
|
|
1148
|
-
? {
|
|
1161
|
+
? {
|
|
1162
|
+
upload_state: "not_uploaded",
|
|
1163
|
+
// BLI-2107: same rule as live sync — an attributed session with
|
|
1164
|
+
// no pointer always says why, even when the answer is that this
|
|
1165
|
+
// path never recorded one.
|
|
1166
|
+
upload_reason: noUploadReasonBySessionId.get(candidate.session_id) ??
|
|
1167
|
+
NO_UPLOAD_ATTEMPT_RECORDED,
|
|
1168
|
+
}
|
|
1149
1169
|
: {}),
|
|
1150
1170
|
};
|
|
1151
1171
|
});
|
package/dist/commands/local.js
CHANGED
|
@@ -2215,13 +2215,25 @@ async function runSyncWithHealthReceipt(command, io) {
|
|
|
2215
2215
|
};
|
|
2216
2216
|
}
|
|
2217
2217
|
try {
|
|
2218
|
-
const exitCode = await runSyncLocked(command, io);
|
|
2218
|
+
const { exitCode, failureReasons } = await runSyncLocked(command, io);
|
|
2219
|
+
if (exitCode === 0) {
|
|
2220
|
+
return {
|
|
2221
|
+
exitCode,
|
|
2222
|
+
completion: { step: "sync_complete", status: "ok" },
|
|
2223
|
+
};
|
|
2224
|
+
}
|
|
2225
|
+
// A sync that fails by exit code says exactly as much as one that throws.
|
|
2226
|
+
// It used to say `sync_failed` and nothing else, so 100% of recorded
|
|
2227
|
+
// failure rows carried a null detail and the real reason was reachable only
|
|
2228
|
+
// by running `cockpit status` on the machine itself (BLI-2526).
|
|
2229
|
+
const reasonText = failureReasons.join("; ");
|
|
2219
2230
|
return {
|
|
2220
2231
|
exitCode,
|
|
2221
2232
|
completion: {
|
|
2222
2233
|
step: "sync_complete",
|
|
2223
|
-
status:
|
|
2224
|
-
|
|
2234
|
+
status: "fail",
|
|
2235
|
+
error_code: classifySyncHealthError(reasonText),
|
|
2236
|
+
error_detail: redactedSyncErrorDetail(reasonText),
|
|
2225
2237
|
},
|
|
2226
2238
|
};
|
|
2227
2239
|
}
|
|
@@ -2264,6 +2276,19 @@ export function redactedSyncErrorDetail(error) {
|
|
|
2264
2276
|
? `${text.slice(0, SYNC_ERROR_DETAIL_MAX_CHARS - 1)}…`
|
|
2265
2277
|
: text;
|
|
2266
2278
|
}
|
|
2279
|
+
/**
|
|
2280
|
+
* Turn a finished run into an exit code and the reasons behind it.
|
|
2281
|
+
*
|
|
2282
|
+
* One place, so a future return path cannot reintroduce a code with no reason.
|
|
2283
|
+
* The reasons come from the run itself — the code that decided `ok` is false is
|
|
2284
|
+
* the only code that knows why.
|
|
2285
|
+
*/
|
|
2286
|
+
function syncResult(run) {
|
|
2287
|
+
return {
|
|
2288
|
+
exitCode: run.ok ? 0 : 1,
|
|
2289
|
+
failureReasons: run.ok ? [] : run.failure_reasons,
|
|
2290
|
+
};
|
|
2291
|
+
}
|
|
2267
2292
|
async function runSyncLocked(command, io) {
|
|
2268
2293
|
const collectionRoots = await resolveSyncCollectionRoots(command);
|
|
2269
2294
|
const worktrees = await discoverCommandWorktrees(collectionRoots, {
|
|
@@ -2294,7 +2319,7 @@ async function runSyncLocked(command, io) {
|
|
|
2294
2319
|
codex_sessions: run.summary,
|
|
2295
2320
|
raw_evidence_gc: gc,
|
|
2296
2321
|
}, null, 2));
|
|
2297
|
-
return run
|
|
2322
|
+
return syncResult(run);
|
|
2298
2323
|
}
|
|
2299
2324
|
writeLine(run.ok ? io.stdout : io.stderr, `Cockpit parent sync ${runStatus} ${run.outcomes.filter((outcome) => outcome.sync.status === "uploaded").length}/${run.outcomes.length} worktree(s).`);
|
|
2300
2325
|
for (const outcome of run.outcomes) {
|
|
@@ -2306,7 +2331,7 @@ async function runSyncLocked(command, io) {
|
|
|
2306
2331
|
writeAgentSessionSummary(io, run.summary);
|
|
2307
2332
|
if (gc && !gc.skipped)
|
|
2308
2333
|
writeLine(io.stdout, rawEvidenceGcSummary(gc));
|
|
2309
|
-
return run
|
|
2334
|
+
return syncResult(run);
|
|
2310
2335
|
}
|
|
2311
2336
|
const result = run.outcomes[0]?.sync;
|
|
2312
2337
|
if (!result) {
|
|
@@ -2322,7 +2347,7 @@ async function runSyncLocked(command, io) {
|
|
|
2322
2347
|
codex_sessions: run.summary,
|
|
2323
2348
|
raw_evidence_gc: gc,
|
|
2324
2349
|
}, null, 2));
|
|
2325
|
-
return run
|
|
2350
|
+
return syncResult(run);
|
|
2326
2351
|
}
|
|
2327
2352
|
if (run.ok) {
|
|
2328
2353
|
writeLine(io.stdout, "Cockpit ambient envelope uploaded.");
|
|
@@ -2337,18 +2362,18 @@ async function runSyncLocked(command, io) {
|
|
|
2337
2362
|
writeLine(io.stdout, cursorStatusLine(result));
|
|
2338
2363
|
if (gc && !gc.skipped)
|
|
2339
2364
|
writeLine(io.stdout, rawEvidenceGcSummary(gc));
|
|
2340
|
-
return
|
|
2365
|
+
return syncResult(run);
|
|
2341
2366
|
}
|
|
2342
2367
|
if (result.status === "uploaded") {
|
|
2343
2368
|
writeLine(io.stderr, "Cockpit ambient upload was accepted, but session collection is partial; retry `cockpit sync`.");
|
|
2344
2369
|
writeAgentSessionSummary(io, run.summary);
|
|
2345
|
-
return
|
|
2370
|
+
return syncResult(run);
|
|
2346
2371
|
}
|
|
2347
2372
|
writeLine(io.stderr, "Cockpit ambient upload failed; safe retry metadata was spooled.");
|
|
2348
2373
|
writeLine(io.stderr, `Ticket: ${displayTicketId(result.ticket_id)}`);
|
|
2349
2374
|
writeLine(io.stderr, `Failure: ${result.failure_reason}`);
|
|
2350
2375
|
writeLine(io.stderr, `Retry: ${result.retry_command}`);
|
|
2351
|
-
return
|
|
2376
|
+
return syncResult(run);
|
|
2352
2377
|
}
|
|
2353
2378
|
async function resolveSyncCollectionRoots(command) {
|
|
2354
2379
|
const explicitRoots = normalizeCollectionRoots(command.repoRoot ? [command.repoRoot] : []);
|
|
@@ -15,7 +15,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
15
15
|
}
|
|
16
16
|
|
|
17
17
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
18
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
18
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.14");
|
|
19
19
|
return 0;
|
|
20
20
|
}
|
|
21
21
|
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
import os from "node:os";
|
|
7
7
|
import path from "node:path";
|
|
8
8
|
import { getCollectorRuntimePaths, readLocalCollectorConfig, startLocalWorkContext, startLocalWorkContextForAttributedTarget, } from "../local-state.js";
|
|
9
|
-
import { CODEX_SESSION_ATTRIBUTION_STATE_RANK, } from "@bli-cockpit/telemetry-core";
|
|
9
|
+
import { CODEX_SESSION_ATTRIBUTION_STATE_RANK, NO_UPLOAD_ATTEMPT_RECORDED, } from "@bli-cockpit/telemetry-core";
|
|
10
10
|
import { flushPendingCodexSessionReports, LocalUploadBlockedError, queueCodexSessionReport, syncLocalAmbientEnvelope, } from "../upload.js";
|
|
11
11
|
import { CODEX_ATTRIBUTION_SCAN_WINDOW_SESSION_LIMIT, CODEX_ATTRIBUTION_SCAN_WINDOW_MINUTES, defaultCodexSessionDirs, scanAndAttributeCodexSessions, } from "../adapters/codex-attribution.js";
|
|
12
12
|
import { scanAndAttributeClaudeSessions } from "../adapters/claude-attribution.js";
|
|
@@ -15,6 +15,14 @@ import { CLAUDE_CURSOR_FILENAME, countStaleSessions, emptyRawEvidenceCursorState
|
|
|
15
15
|
import { normalizeCollectionRoots } from "../root-normalization.js";
|
|
16
16
|
import { clearSourceRetryFailure, readLocalUploadSpoolState, recordSourceRetryFailure, } from "../spool/local-spool.js";
|
|
17
17
|
import { isLiveRawEvidenceSyncAttribution, } from "../raw-evidence-attribution-policy.js";
|
|
18
|
+
/**
|
|
19
|
+
* The label used when a sync fails and nothing on the way there said why.
|
|
20
|
+
*
|
|
21
|
+
* A deliberate sentinel rather than a fallback to `sync_failed`: it means the
|
|
22
|
+
* gate is real but its reason is unrecorded, which is a bug in this file, and
|
|
23
|
+
* it should be visible as one instead of blending into the generic bucket.
|
|
24
|
+
*/
|
|
25
|
+
export const SYNC_FAILED_WITHOUT_REASON = "sync_failed_reason_not_recorded";
|
|
18
26
|
const CLAUDE_FIRST_RUN_BACKFILL_MINUTES = 14 * 24 * 60;
|
|
19
27
|
const CLAUDE_DAMP_GROWTH_BYTES = 256 * 1024;
|
|
20
28
|
const CLAUDE_DAMP_MAX_AGE_MS = 6 * 60 * 60 * 1000;
|
|
@@ -414,20 +422,56 @@ export async function runAttributedWorktreeSync(options) {
|
|
|
414
422
|
growthDamped: dampedClaudePointers.size,
|
|
415
423
|
report,
|
|
416
424
|
});
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
(
|
|
430
|
-
|
|
425
|
+
// Same conditions as before, one per line, each writing down its own reason.
|
|
426
|
+
// The old version was a single boolean chain: correct, and completely mute.
|
|
427
|
+
const failureReasons = new Set();
|
|
428
|
+
const fail = (condition, reason) => {
|
|
429
|
+
if (condition)
|
|
430
|
+
failureReasons.add(reason);
|
|
431
|
+
};
|
|
432
|
+
for (const { worktree, sync } of outcomes) {
|
|
433
|
+
if (sync.status !== "uploaded") {
|
|
434
|
+
// The spooled reason is the most specific thing anyone has, so lead with
|
|
435
|
+
// it and name the worktree it belongs to — a fleet failure is usually one
|
|
436
|
+
// repo, and "which one" is the first question asked.
|
|
437
|
+
failureReasons.add(sync.status === "spooled" && sync.failure_reason
|
|
438
|
+
? `${worktree.worktree_label}:${sync.failure_reason}`
|
|
439
|
+
: `${worktree.worktree_label}:upload_${sync.status}`);
|
|
440
|
+
}
|
|
441
|
+
for (const reason of sync.raw_evidence_failure_reasons ?? []) {
|
|
442
|
+
failureReasons.add(`raw_evidence:${reason}`);
|
|
443
|
+
}
|
|
444
|
+
for (const reason of sync.raw_evidence_retry_reasons ?? []) {
|
|
445
|
+
failureReasons.add(`raw_evidence_retry:${reason}`);
|
|
446
|
+
}
|
|
447
|
+
fail(sync.raw_evidence_deferred_byte_budget > 0, "deferred_byte_budget");
|
|
448
|
+
fail(sync.raw_evidence_deferred_object_budget > 0, "deferred_object_budget");
|
|
449
|
+
}
|
|
450
|
+
fail(codexAttribution.session_limit_applied, "codex_session_limit_applied");
|
|
451
|
+
fail(claudeAttribution.session_limit_applied, "claude_session_limit_applied");
|
|
452
|
+
fail(codexAttributionReadFailureCount(codexAttribution) > 0, "codex_session_read_failed");
|
|
453
|
+
fail(claudeAttributionReadFailureCount(claudeAttribution) > 0, "claude_session_read_failed");
|
|
454
|
+
const codexScanRetry = sourceScanRetryReason("codex", codexAttribution);
|
|
455
|
+
if (codexScanRetry)
|
|
456
|
+
failureReasons.add(`codex_scan:${codexScanRetry}`);
|
|
457
|
+
const claudeScanRetry = sourceScanRetryReason("claude_code", claudeAttribution);
|
|
458
|
+
if (claudeScanRetry)
|
|
459
|
+
failureReasons.add(`claude_scan:${claudeScanRetry}`);
|
|
460
|
+
fail(reportRequired && !report.posted, `session_report_unposted:${report.reason ?? "unknown"}`);
|
|
461
|
+
// `ok` may already be false from the per-worktree loop above; the outcome
|
|
462
|
+
// scan re-derives that, so the two agree by construction.
|
|
463
|
+
ok = ok && failureReasons.size === 0;
|
|
464
|
+
if (!ok && failureReasons.size === 0) {
|
|
465
|
+
failureReasons.add(SYNC_FAILED_WITHOUT_REASON);
|
|
466
|
+
}
|
|
467
|
+
return {
|
|
468
|
+
ok,
|
|
469
|
+
failure_reasons: [...failureReasons].sort(),
|
|
470
|
+
outcomes,
|
|
471
|
+
codexAttribution,
|
|
472
|
+
claudeAttribution,
|
|
473
|
+
summary,
|
|
474
|
+
};
|
|
431
475
|
}
|
|
432
476
|
export const ATTRIBUTION_STATE_RANK = CODEX_SESSION_ATTRIBUTION_STATE_RANK;
|
|
433
477
|
function normalizeCodexResult(result) {
|
|
@@ -474,7 +518,7 @@ function normalizeClaudeResult(result) {
|
|
|
474
518
|
* session's upload state (D3). Damped Claude sessions report `reused_existing`
|
|
475
519
|
* carrying their prior durable pointer.
|
|
476
520
|
*/
|
|
477
|
-
function buildAgentSessionReport(options) {
|
|
521
|
+
export function buildAgentSessionReport(options) {
|
|
478
522
|
const normalized = [
|
|
479
523
|
...options.codexResults.map(normalizeCodexResult),
|
|
480
524
|
...options.claudeResults.map(normalizeClaudeResult),
|
|
@@ -495,11 +539,23 @@ function buildAgentSessionReport(options) {
|
|
|
495
539
|
// Main-file outcomes only (D3): a sidecar making it must never mark a session
|
|
496
540
|
// uploaded when the main did not.
|
|
497
541
|
const uploadByKey = new Map();
|
|
542
|
+
// BLI-2107: why a session got no pointer, keyed the same way. The reasons
|
|
543
|
+
// already existed — file_too_large, deferred_byte_budget, the redaction
|
|
544
|
+
// guards — but only as aggregate skip counts in the health report, so no
|
|
545
|
+
// individual session could say what happened to it.
|
|
546
|
+
const noUploadReasonByKey = new Map();
|
|
547
|
+
// A worktree sync that did not finish carries no per-session outcomes, and
|
|
548
|
+
// WorktreeSyncOutcome does not record which sessions it was going to cover.
|
|
549
|
+
// So this pass knows only that it was degraded, not which session each
|
|
550
|
+
// failure belonged to — and says exactly that rather than picking one.
|
|
551
|
+
let anySyncIncomplete = false;
|
|
498
552
|
for (const outcome of options.outcomes) {
|
|
499
|
-
if (outcome.sync.status !== "uploaded")
|
|
553
|
+
if (outcome.sync.status !== "uploaded") {
|
|
554
|
+
anySyncIncomplete = true;
|
|
500
555
|
continue;
|
|
556
|
+
}
|
|
501
557
|
for (const upload of outcome.sync.raw_evidence_outcomes) {
|
|
502
|
-
if (!upload.codex_session_id
|
|
558
|
+
if (!upload.codex_session_id)
|
|
503
559
|
continue;
|
|
504
560
|
const source = upload.kind === "claude_jsonl"
|
|
505
561
|
? "claude_code"
|
|
@@ -508,7 +564,15 @@ function buildAgentSessionReport(options) {
|
|
|
508
564
|
: null;
|
|
509
565
|
if (!source)
|
|
510
566
|
continue; // sidecars and other kinds do not set session state
|
|
511
|
-
|
|
567
|
+
const key = `${source}:${upload.codex_session_id}`;
|
|
568
|
+
if (!upload.raw_evidence_pointer_id) {
|
|
569
|
+
// An outcome with no pointer is a failure that named itself. Keep the
|
|
570
|
+
// reason even though there is nothing to point at.
|
|
571
|
+
if (upload.reason)
|
|
572
|
+
noUploadReasonByKey.set(key, upload.reason);
|
|
573
|
+
continue;
|
|
574
|
+
}
|
|
575
|
+
uploadByKey.set(key, upload);
|
|
512
576
|
}
|
|
513
577
|
}
|
|
514
578
|
return [...bestByKey.values()].map((result) => {
|
|
@@ -548,6 +612,13 @@ function buildAgentSessionReport(options) {
|
|
|
548
612
|
? {
|
|
549
613
|
raw_evidence_pointer_id: upload.raw_evidence_pointer_id,
|
|
550
614
|
upload_state: upload.upload_state,
|
|
615
|
+
// A failed upload names itself; a successful one has nothing to
|
|
616
|
+
// explain.
|
|
617
|
+
...(upload.upload_state === "upload_failed"
|
|
618
|
+
? {
|
|
619
|
+
upload_reason: upload.reason ?? NO_UPLOAD_ATTEMPT_RECORDED,
|
|
620
|
+
}
|
|
621
|
+
: {}),
|
|
551
622
|
}
|
|
552
623
|
: priorDurablePointer
|
|
553
624
|
? {
|
|
@@ -556,7 +627,19 @@ function buildAgentSessionReport(options) {
|
|
|
556
627
|
}
|
|
557
628
|
: result.state === "attributed" ||
|
|
558
629
|
result.state === "attributed_fallback"
|
|
559
|
-
? {
|
|
630
|
+
? {
|
|
631
|
+
upload_state: "not_uploaded",
|
|
632
|
+
// BLI-2107: `not_uploaded` used to be the branch of last
|
|
633
|
+
// resort, recording that nothing happened and never why. It
|
|
634
|
+
// now always carries a cause, even when the cause is that we
|
|
635
|
+
// have none — a session labelled NO_UPLOAD_ATTEMPT_RECORDED
|
|
636
|
+
// is a path that still needs instrumenting, and saying so is
|
|
637
|
+
// the point.
|
|
638
|
+
upload_reason: noUploadReasonByKey.get(key) ??
|
|
639
|
+
(anySyncIncomplete
|
|
640
|
+
? "sync_incomplete_this_pass"
|
|
641
|
+
: NO_UPLOAD_ATTEMPT_RECORDED),
|
|
642
|
+
}
|
|
560
643
|
: {}),
|
|
561
644
|
};
|
|
562
645
|
});
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { RAW_EVIDENCE_UPLOAD_DEFAULT_CHUNK_BYTES, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, RAW_EVIDENCE_UPLOAD_MAX_OBJECTS_PER_BEGIN, RawEvidenceLegacyUploadResponseSchema, RawEvidenceRedactionMetadataSchema, RawEvidenceUploadBeginResponseSchema, RawEvidenceUploadCommitResponseSchema, } from "@bli-cockpit/telemetry-core";
|
|
1
|
+
import { RAW_EVIDENCE_UPLOAD_DEFAULT_CHUNK_BYTES, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, RAW_EVIDENCE_UPLOAD_MAX_OBJECTS_PER_BEGIN, RawEvidenceLegacyUploadResponseSchema, RawEvidenceRedactionMetadataSchema, RawEvidenceUploadBeginResponseSchema, RawEvidenceUploadCommitResponseSchema, isPermanentUploadFailure, } from "@bli-cockpit/telemetry-core";
|
|
2
2
|
import crypto from "node:crypto";
|
|
3
3
|
import fs from "node:fs/promises";
|
|
4
4
|
const DEFAULT_MAX_ATTEMPTS = 3;
|
|
@@ -214,6 +214,14 @@ async function uploadOneObject(options, entry, disposition, chunkSizeBytes) {
|
|
|
214
214
|
object_key: objectKey,
|
|
215
215
|
});
|
|
216
216
|
if (!commit.ok) {
|
|
217
|
+
// The server can tell us this object will be refused again. Take its reason
|
|
218
|
+
// verbatim so the label names the cause ("storage rejected 116 MB") instead
|
|
219
|
+
// of the transport ("commit_failed_http_500"), which is all the fleet could
|
|
220
|
+
// say for the 57 days of BLI-2528.
|
|
221
|
+
const permanent = permanentCommitRejection(commit.body);
|
|
222
|
+
if (permanent) {
|
|
223
|
+
return failedOutcome(entry.file, permanent, uploadedChunks);
|
|
224
|
+
}
|
|
217
225
|
const detail = safeFailureDetail(commit.body);
|
|
218
226
|
return failedOutcome(entry.file, `commit_failed_http_${commit.status}${detail ? `_${detail}` : ""}`, uploadedChunks);
|
|
219
227
|
}
|
|
@@ -376,6 +384,24 @@ function failedOutcome(file, reason, uploadedChunks = 0) {
|
|
|
376
384
|
uploaded_chunk_count: uploadedChunks,
|
|
377
385
|
};
|
|
378
386
|
}
|
|
387
|
+
/**
|
|
388
|
+
* The server's verdict that repeating this commit is pointless, or null.
|
|
389
|
+
*
|
|
390
|
+
* Two conditions, both required: the response says `retryable: false`, and the
|
|
391
|
+
* reason is one this collector has classified as permanent. The second check is
|
|
392
|
+
* the important one. A newer dashboard could declare a reason this CLI has never
|
|
393
|
+
* heard of, and quietly abandoning an object on a word we cannot interpret is
|
|
394
|
+
* exactly the silent drop the fleet contract forbids — so an unclassified reason
|
|
395
|
+
* falls through to the ordinary retry path and stays visible.
|
|
396
|
+
*/
|
|
397
|
+
function permanentCommitRejection(body) {
|
|
398
|
+
if (!body || typeof body !== "object")
|
|
399
|
+
return null;
|
|
400
|
+
if (body.retryable !== false)
|
|
401
|
+
return null;
|
|
402
|
+
const reason = safeFailureDetail(body);
|
|
403
|
+
return reason && isPermanentUploadFailure(reason) ? reason : null;
|
|
404
|
+
}
|
|
379
405
|
function safeFailureDetail(body) {
|
|
380
406
|
if (!body || typeof body !== "object")
|
|
381
407
|
return null;
|
package/dist/upload.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AgentImageArtifactReportRequestSchema, CODEX_SESSION_REPORT_MAX_SESSIONS, CodexSessionAttributionReportResponseSchema, EvidenceCompletenessPayloadSchema, TelemetryIngestEnvelopeSchema, TelemetryIngestEventDtoSchema, } from "@bli-cockpit/telemetry-core";
|
|
1
|
+
import { AgentImageArtifactReportRequestSchema, CODEX_SESSION_REPORT_MAX_SESSIONS, CodexSessionAttributionReportResponseSchema, EvidenceCompletenessPayloadSchema, TelemetryIngestEnvelopeSchema, TelemetryIngestEventDtoSchema, isPermanentUploadFailure, } from "@bli-cockpit/telemetry-core";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { getCollectorRuntimePaths, LOCAL_COLLECTOR_VERSION, readLocalCollectorConfig, readLocalCollectorSessionFile, readLocalSessionReference, readLocalWorkContextForRepo, } from "./local-state.js";
|
|
4
4
|
import { runLocalSourceCollectors } from "./adapters/local-sources.js";
|
|
@@ -236,6 +236,27 @@ export async function syncLocalAmbientEnvelope(options = {}) {
|
|
|
236
236
|
retry_command: "cockpit sync",
|
|
237
237
|
});
|
|
238
238
|
}
|
|
239
|
+
else {
|
|
240
|
+
// No retry to queue, but a permanently rejected object still has to say
|
|
241
|
+
// so. `recordUploadBlocked` names the reason without spooling a retry —
|
|
242
|
+
// the machine reports the failure instead of promising a fix it cannot
|
|
243
|
+
// deliver, and `cockpit status` stops reading clean.
|
|
244
|
+
const permanentReason = permanentEvidenceFailureReason(uploadOutcomes);
|
|
245
|
+
if (permanentReason) {
|
|
246
|
+
await recordUploadBlocked(paths, {
|
|
247
|
+
attemptedAt,
|
|
248
|
+
reason: permanentReason,
|
|
249
|
+
});
|
|
250
|
+
// stderr, which launchd captures to `sync.err.log`, so an unattended
|
|
251
|
+
// machine leaves a dated record of the objects it gave up on. Reason
|
|
252
|
+
// labels and counts only — never a path or a byte of content.
|
|
253
|
+
console.error("[cockpit-sync] raw evidence permanently rejected", JSON.stringify({
|
|
254
|
+
attempted_at: attemptedAt,
|
|
255
|
+
reason: permanentReason,
|
|
256
|
+
object_count: permanentFailedOutcomes(uploadOutcomes).length,
|
|
257
|
+
}));
|
|
258
|
+
}
|
|
259
|
+
}
|
|
239
260
|
return {
|
|
240
261
|
status: "uploaded",
|
|
241
262
|
dashboard_url: built.dashboard_url,
|
|
@@ -307,8 +328,37 @@ function retrySourcesForFailedSync(options, facts) {
|
|
|
307
328
|
}
|
|
308
329
|
return [...sources];
|
|
309
330
|
}
|
|
331
|
+
/**
|
|
332
|
+
* The failed uploads a later attempt could still rescue.
|
|
333
|
+
*
|
|
334
|
+
* An object storage has already refused on its own terms is not one of them,
|
|
335
|
+
* and counting it as one is what kept Edward's Mac in `retry_pending` through
|
|
336
|
+
* 13 consecutive syncs that were never going to end differently (BLI-2528).
|
|
337
|
+
*/
|
|
338
|
+
function retryableFailedOutcomes(outcomes) {
|
|
339
|
+
return outcomes.filter((outcome) => outcome.upload_state === "upload_failed" &&
|
|
340
|
+
!isPermanentUploadFailure(outcome.reason));
|
|
341
|
+
}
|
|
342
|
+
/** Failed uploads that no retry can rescue, kept so they can still be named. */
|
|
343
|
+
function permanentFailedOutcomes(outcomes) {
|
|
344
|
+
return outcomes.filter((outcome) => outcome.upload_state === "upload_failed" &&
|
|
345
|
+
isPermanentUploadFailure(outcome.reason));
|
|
346
|
+
}
|
|
347
|
+
/**
|
|
348
|
+
* The reason to show for objects that failed for good.
|
|
349
|
+
*
|
|
350
|
+
* Returns null when there are none. These never queue a retry, but they must
|
|
351
|
+
* never disappear either: a machine with a permanently rejected object has
|
|
352
|
+
* missing collection, and a status that reads clean would hide it.
|
|
353
|
+
*/
|
|
354
|
+
function permanentEvidenceFailureReason(outcomes) {
|
|
355
|
+
const reasons = new Set(permanentFailedOutcomes(outcomes).map((outcome) => outcome.reason ?? "unknown"));
|
|
356
|
+
if (reasons.size === 0)
|
|
357
|
+
return null;
|
|
358
|
+
return `raw_evidence_permanently_rejected:${[...reasons].sort().join(",")}`;
|
|
359
|
+
}
|
|
310
360
|
function hasRetryableEvidenceGap(facts, outcomes) {
|
|
311
|
-
if (outcomes
|
|
361
|
+
if (retryableFailedOutcomes(outcomes).length > 0) {
|
|
312
362
|
return true;
|
|
313
363
|
}
|
|
314
364
|
if (!facts)
|
|
@@ -326,10 +376,8 @@ function hasRetryableEvidenceGap(facts, outcomes) {
|
|
|
326
376
|
}
|
|
327
377
|
function retryableEvidenceGapReason(facts, outcomes) {
|
|
328
378
|
const reasons = new Set();
|
|
329
|
-
for (const outcome of outcomes) {
|
|
330
|
-
|
|
331
|
-
reasons.add(outcome.reason ?? "upload_failed");
|
|
332
|
-
}
|
|
379
|
+
for (const outcome of retryableFailedOutcomes(outcomes)) {
|
|
380
|
+
reasons.add(outcome.reason ?? "upload_failed");
|
|
333
381
|
}
|
|
334
382
|
if (facts) {
|
|
335
383
|
if (facts.deferred_byte_budget_count > 0)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bli-cockpit/cli",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.14",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -26,6 +26,6 @@
|
|
|
26
26
|
"test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
|
|
27
27
|
},
|
|
28
28
|
"dependencies": {
|
|
29
|
-
"@bli-cockpit/telemetry-core": "0.1.
|
|
29
|
+
"@bli-cockpit/telemetry-core": "0.1.17"
|
|
30
30
|
}
|
|
31
31
|
}
|