@bli-cockpit/cli 0.2.106 → 0.2.108
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/clean.js +2 -1
- package/dist/commands/collection-report.js +2 -1
- package/dist/commands/doctor-disk-words.js +3 -0
- package/dist/commands/doctor-pipeline-verdicts.js +10 -0
- package/dist/commands/doctor-pipeline.js +9 -7
- package/dist/commands/local-args-tower-usage.js +5 -2
- package/dist/commands/local-discovery.js +7 -12
- package/dist/commands/local-help-commands-tower.js +1 -1
- package/dist/commands/local-help.js +1 -1
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/sync-followups-self-update.js +11 -7
- package/dist/commands/sync-receipt.js +1 -1
- package/dist/commands/sync-report.js +13 -4
- package/dist/commands/sync-run.js +6 -0
- package/dist/commands/sync.js +27 -6
- package/dist/commands/usage.js +4 -0
- package/dist/disk-prune.js +1 -0
- package/dist/disk-retention.js +9 -3
- package/dist/disk-usage-classify.js +22 -16
- package/dist/disk-usage-scan.js +3 -0
- package/dist/disk-usage-totals.js +7 -0
- package/dist/evidence-redelivery.js +13 -6
- package/dist/evidence-upload-object.js +27 -2
- package/dist/evidence-upload-terminal.js +1 -1
- package/dist/raw-evidence-staging.js +17 -4
- package/dist/raw-evidence-stuck-summary.js +6 -4
- package/dist/scheduled-self-update.js +3 -5
- package/dist/upload-evidence-delivery-offer.js +5 -4
- package/dist/upload-evidence-delivery-summary.js +4 -3
- package/package.json +2 -2
package/dist/commands/clean.js
CHANGED
|
@@ -49,7 +49,7 @@ export async function runClean(command, io) {
|
|
|
49
49
|
writeLine(io.stdout, line);
|
|
50
50
|
}
|
|
51
51
|
const footprint = await readDiskFootprint(paths, now);
|
|
52
|
-
const plan = await planFromDisk(paths, env, { allCommitted: command.allCommitted }, now);
|
|
52
|
+
const plan = await planFromDisk(paths, env, { allCommitted: command.allCommitted, reclaimTerminal: true }, now);
|
|
53
53
|
const vaults = command.allCommitted
|
|
54
54
|
? await vaultsSafeToRemove(paths, footprint.vaults)
|
|
55
55
|
: [];
|
|
@@ -69,6 +69,7 @@ export async function runClean(command, io) {
|
|
|
69
69
|
now,
|
|
70
70
|
force: true,
|
|
71
71
|
allCommitted: command.allCommitted,
|
|
72
|
+
reclaimTerminal: true,
|
|
72
73
|
});
|
|
73
74
|
// The two smaller footprints go with it: a rotation pass sweeps the archives
|
|
74
75
|
// outside the keep window, and each removed vault is named on its own line.
|
|
@@ -84,7 +84,7 @@ export function rawEvidenceSyncLine(sync) {
|
|
|
84
84
|
const stuck = sync.raw_evidence_stuck_object_count > 0
|
|
85
85
|
? `, ${sync.raw_evidence_stuck_object_count} stuck (${sync.raw_evidence_max_delivery_attempts} tries since ${sync.raw_evidence_oldest_delivery_failure_at ?? "unknown"})`
|
|
86
86
|
: "";
|
|
87
|
-
return `Files: ${sync.raw_evidence_uploaded_object_count} uploaded, ${sync.raw_evidence_reused_count} already there, ${sync.raw_evidence_failed_count} failed${held}${stuck}${failures}${retries}`;
|
|
87
|
+
return `Files: ${sync.raw_evidence_uploaded_object_count} uploaded, ${sync.raw_evidence_reused_count} already there, ${sync.raw_evidence_failed_count} failed, reconciled_committed_by_409=${sync.reconciled_committed_by_409 ?? 0}${held}${stuck}${failures}${retries}`;
|
|
88
88
|
}
|
|
89
89
|
export function cursorStatusLine(sync) {
|
|
90
90
|
return `Tracked so far: ${sync.cursor_tracked_object_count} uploaded item(s)`;
|
|
@@ -118,6 +118,7 @@ export function worktreeSyncRow(outcome, run) {
|
|
|
118
118
|
raw_evidence_uploaded_object_count: sync.raw_evidence_uploaded_object_count,
|
|
119
119
|
raw_evidence_uploaded_chunk_count: sync.raw_evidence_uploaded_chunk_count,
|
|
120
120
|
raw_evidence_reused_count: sync.raw_evidence_reused_count,
|
|
121
|
+
reconciled_committed_by_409: sync.reconciled_committed_by_409 ?? 0,
|
|
121
122
|
raw_evidence_failed_count: sync.raw_evidence_failed_count,
|
|
122
123
|
raw_evidence_failure_reasons: sync.raw_evidence_failure_reasons,
|
|
123
124
|
raw_evidence_retry_required: sync.raw_evidence_retry_required,
|
|
@@ -38,6 +38,9 @@ export function diskRowMessage(footprint, capBytes) {
|
|
|
38
38
|
// have been here — on the reference Mac the answer was
|
|
39
39
|
// `reconciled_unknown_to_server` on 234 objects whose oldest was 45 days old,
|
|
40
40
|
// which is a different problem from a delivery that failed this morning.
|
|
41
|
+
if (staging.terminal_count) {
|
|
42
|
+
parts.push(`${staging.terminal_count} terminal object(s), ${mib(staging.terminal_bytes ?? 0)} MB excluded from cap; reclaimable with \`cockpit clean\`: ${Object.keys(staging.terminal_reasons ?? {}).join(", ")}`);
|
|
43
|
+
}
|
|
41
44
|
const dominant = staging.dominant_uncommitted_reason;
|
|
42
45
|
if (dominant) {
|
|
43
46
|
parts.push(`oldest undelivered ${days(dominant.oldest_disk_age_ms)}d; biggest reason ${dominant.reason} on ${dominant.count} object(s), ${mib(dominant.bytes)} MB`);
|
|
@@ -235,4 +235,14 @@ function positiveNumberOrZero(value) {
|
|
|
235
235
|
return typeof value === "number" && Number.isFinite(value) && value > 0
|
|
236
236
|
? value
|
|
237
237
|
: 0;
|
|
238
|
+
}
|
|
239
|
+
/** Names only the sync receipt's recorded causes. Missing evidence names itself. */
|
|
240
|
+
export function syncFailureMessage(parsed) {
|
|
241
|
+
const reasons = typeof parsed?.failure_reason === "string"
|
|
242
|
+
? [parsed.failure_reason]
|
|
243
|
+
: Array.isArray(parsed?.failure_reasons)
|
|
244
|
+
? parsed.failure_reasons.filter((value) => typeof value === "string" && value.length > 0)
|
|
245
|
+
: [];
|
|
246
|
+
const reason = reasons.join("; ") || "sync_failure_receipt_missing";
|
|
247
|
+
return /staging_cap/u.test(reason) ? `${reason}; see disk-bounded and run \`cockpit clean --dry-run\`` : reason;
|
|
238
248
|
}
|
|
@@ -13,7 +13,7 @@ import { runRawEvidenceLocalGc, rawEvidenceGcSummary } from "../raw-evidence-gc.
|
|
|
13
13
|
import { runBackfillCommand } from "./backfill.js";
|
|
14
14
|
import { diskRowMessage, mib, redeliveryLine } from "./doctor-disk-words.js";
|
|
15
15
|
import { doctorRoots } from "./doctor-access.js";
|
|
16
|
-
import { backfillCompletionStepState, backfillFixVerdict, jsonField, parseDoctorBackfillJson, parseDoctorSyncJson, syncBacklogDrainingVerdict, syncStandAsideVerdict, } from "./doctor-pipeline-verdicts.js";
|
|
16
|
+
import { backfillCompletionStepState, backfillFixVerdict, jsonField, parseDoctorBackfillJson, parseDoctorSyncJson, syncFailureMessage, syncBacklogDrainingVerdict, syncStandAsideVerdict, } from "./doctor-pipeline-verdicts.js";
|
|
17
17
|
import { asRecord, fail, needsFix, ok, skipped } from "./doctor-report.js";
|
|
18
18
|
/**
|
|
19
19
|
* The `backfill-complete`, `gc-checked`, `disk-bounded`, and `sync-fresh`
|
|
@@ -165,7 +165,7 @@ export async function checkDiskState(context) {
|
|
|
165
165
|
const footprint = await readDiskFootprint(getCollectorRuntimePaths(context.command.homeDir));
|
|
166
166
|
const { capBytes } = retentionOptionsFromEnv(context.io.env);
|
|
167
167
|
const message = diskRowMessage(footprint, capBytes);
|
|
168
|
-
return footprint.staging.total_bytes > capBytes
|
|
168
|
+
return footprint.staging.total_bytes - (footprint.staging.terminal_bytes ?? 0) > capBytes
|
|
169
169
|
? needsFix("disk-bounded", "over_staging_cap", message)
|
|
170
170
|
: ok("disk-bounded", "within_staging_cap", message);
|
|
171
171
|
}
|
|
@@ -198,10 +198,8 @@ export async function fixDiskState(context) {
|
|
|
198
198
|
if (!pruned.cap_blocked_by_uncommitted) {
|
|
199
199
|
return ok("disk-bounded", `freed_${pruned.deleted_files}`, `freed ${mib(pruned.deleted_bytes)} MB; ${message}`);
|
|
200
200
|
}
|
|
201
|
-
//
|
|
202
|
-
|
|
203
|
-
// names the blockage and the one command that goes further.
|
|
204
|
-
return ok("disk-bounded", "staging_cap_blocked_by_uncommitted", `freed ${mib(pruned.deleted_bytes)} MB; ${message}; ${pruned.cap_blocked_count} object(s) the upload ledger cannot vouch for are holding the rest — ${redelivered} — or \`cockpit clean --all-committed\` to drop every accepted copy now`);
|
|
201
|
+
// Unaccepted evidence still occupies the cap. Keep the disk row actionable.
|
|
202
|
+
return needsFix("disk-bounded", "staging_cap_blocked_by_uncommitted", `freed ${mib(pruned.deleted_bytes)} MB; ${message}; ${pruned.cap_blocked_count} object(s) the upload ledger cannot vouch for are holding the rest; ${redelivered}; or \`cockpit clean --all-committed\` to drop every accepted copy now`);
|
|
205
203
|
}
|
|
206
204
|
function capturedIo(io, forward) {
|
|
207
205
|
const stdoutChunks = [];
|
|
@@ -297,7 +295,11 @@ async function runSyncRepair(context) {
|
|
|
297
295
|
`file${draining.remainingObjects === 1 ? "" : "s"} left for later this run); ` +
|
|
298
296
|
"rerun `cockpit sync` to continue");
|
|
299
297
|
}
|
|
300
|
-
return fail("sync-fresh", status ?? "sync_failed",
|
|
298
|
+
return fail("sync-fresh", status ?? "sync_failed", `${repoRoot}: ${syncFailureMessage(parsed)}`);
|
|
299
|
+
}
|
|
300
|
+
if (parsed?.collection_complete === false && typeof parsed.discovery_notice === "string") {
|
|
301
|
+
console.error("[doctor] partial discovery", JSON.stringify({ reason: "discovery_partial" }));
|
|
302
|
+
return needsFix("sync-fresh", "discovery_partial", parsed.discovery_notice);
|
|
301
303
|
}
|
|
302
304
|
if (status !== "uploaded") {
|
|
303
305
|
return fail("sync-fresh", status ?? "sync_unverified", `sync did not return an uploaded receipt for ${repoRoot}`);
|
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
import { optionalNonEmpty, optionalUrl, parseNamedArgs } from "./local-arg-values.js";
|
|
2
2
|
export function parseUsageArgs(args) {
|
|
3
|
-
const values = parseNamedArgs(args, { allowedFlags: ["--by-topic", "--by-repo", "--person", "--all", "--since", "--until", "--include-automated", "--detail", "--home", "--dashboard-url", "--json"], valueFlags: ["--person", "--since", "--until", "--home", "--dashboard-url"] });
|
|
3
|
+
const values = parseNamedArgs(args, { allowedFlags: ["--min-confidence", "--by-topic", "--by-repo", "--person", "--all", "--since", "--until", "--include-automated", "--detail", "--home", "--dashboard-url", "--json"], valueFlags: ["--min-confidence", "--person", "--since", "--until", "--home", "--dashboard-url"] });
|
|
4
4
|
if (values.booleans.has("--by-repo") && values.booleans.has("--by-topic"))
|
|
5
5
|
throw new Error("--by-repo and --by-topic cannot be used together.");
|
|
6
|
+
const minConfidence = values.flags.get("--min-confidence");
|
|
7
|
+
if (minConfidence !== undefined && !["high", "medium", "low"].includes(minConfidence))
|
|
8
|
+
throw new Error("--min-confidence accepts high, medium or low.");
|
|
6
9
|
const action = values.positionals[0] ?? "people";
|
|
7
10
|
if (action !== "people" || values.positionals.length > 1)
|
|
8
11
|
throw new Error("usage takes one verb: people.");
|
|
9
|
-
return { kind: "usage", byTopic: values.booleans.has("--by-topic"), byRepo: values.booleans.has("--by-repo"), person: optionalNonEmpty(values.flags.get("--person")), all: values.booleans.has("--all"), action: "people", since: optionalNonEmpty(values.flags.get("--since")) ?? "30d", until: optionalNonEmpty(values.flags.get("--until")), detail: values.booleans.has("--detail"), includeAutomated: values.booleans.has("--include-automated"), homeDir: optionalNonEmpty(values.flags.get("--home")), dashboardUrl: optionalUrl(values.flags.get("--dashboard-url")), json: values.booleans.has("--json") };
|
|
12
|
+
return { kind: "usage", ...(minConfidence ? { minConfidence: minConfidence } : {}), byTopic: values.booleans.has("--by-topic"), byRepo: values.booleans.has("--by-repo"), person: optionalNonEmpty(values.flags.get("--person")), all: values.booleans.has("--all"), action: "people", since: optionalNonEmpty(values.flags.get("--since")) ?? "30d", until: optionalNonEmpty(values.flags.get("--until")), detail: values.booleans.has("--detail"), includeAutomated: values.booleans.has("--include-automated"), homeDir: optionalNonEmpty(values.flags.get("--home")), dashboardUrl: optionalUrl(values.flags.get("--dashboard-url")), json: values.booleans.has("--json") };
|
|
10
13
|
}
|
|
@@ -20,12 +20,6 @@ export async function discoverCommandWorktrees(repoRoot, discovery = {}, io) {
|
|
|
20
20
|
}
|
|
21
21
|
const worktrees = result.worktrees;
|
|
22
22
|
if (!result.complete) {
|
|
23
|
-
// Sync fails closed here ON PURPOSE, and that is not the bug. Advancing a
|
|
24
|
-
// cursor after a partial scan would mark the run as covering repos it
|
|
25
|
-
// never saw, permanently skipping their sessions — backfill can tolerate
|
|
26
|
-
// partial only because it keeps per-scope completion markers, and sync
|
|
27
|
-
// does not. The bug (BLI-2362) was that the refusal named no roots and
|
|
28
|
-
// gave no runnable command, so a big workspace just stayed red forever.
|
|
29
23
|
const message = incompleteDiscoveryMessage({
|
|
30
24
|
result,
|
|
31
25
|
roots,
|
|
@@ -35,9 +29,10 @@ export async function discoverCommandWorktrees(repoRoot, discovery = {}, io) {
|
|
|
35
29
|
});
|
|
36
30
|
if (io)
|
|
37
31
|
writeLine(io.stderr, message);
|
|
38
|
-
|
|
32
|
+
discovery.onPartial?.(message);
|
|
33
|
+
console.error("[local-discovery] partial collection", JSON.stringify({ reason: result.incomplete_reasons.join(","), roots: roots.length, incomplete_roots: result.incomplete_roots.length, max_depth: limits.maxDepth, found: worktrees.length }));
|
|
39
34
|
}
|
|
40
|
-
if (worktrees.length === 0 && !discovery.allowEmpty) {
|
|
35
|
+
if (worktrees.length === 0 && result.complete && !discovery.allowEmpty) {
|
|
41
36
|
// "No git repos found" is only true when the scan could actually SEE
|
|
42
37
|
// everywhere it looked. With unreadable folders in hand, the sentence sent
|
|
43
38
|
// the operator to the wrong repair — "run from a git repo", when the repos
|
|
@@ -104,17 +99,17 @@ function incompleteDiscoveryMessage(input) {
|
|
|
104
99
|
`--max-repos ${nextRepos}`,
|
|
105
100
|
].join(" ");
|
|
106
101
|
return [
|
|
107
|
-
`
|
|
108
|
-
|
|
102
|
+
`TOWER collection continued with partial repo discovery (${result.incomplete_reasons.join(", ")}).`,
|
|
103
|
+
`Fully scanned ${roots.length - blocked.length} of ${roots.length} roots to depth ${maxDepth}. Sessions under all approved roots remain eligible for collection.`,
|
|
109
104
|
"",
|
|
110
105
|
"Could not fully scan:",
|
|
111
106
|
...blocked.map((root) => ` ${root}`),
|
|
112
107
|
"",
|
|
113
|
-
`Found ${found} repo(s) before
|
|
108
|
+
`Found ${found} repo(s) before the cap, with --max-depth ${maxDepth} and --max-repos ${maxRepos}.`,
|
|
114
109
|
"",
|
|
115
110
|
"Run this to raise the limits and try again:",
|
|
116
111
|
` ${retry}`,
|
|
117
112
|
"",
|
|
118
|
-
"If
|
|
113
|
+
"If the scan is still partial, raise the numbers again, or point --workspace at the specific project folders instead of a parent.",
|
|
119
114
|
].join("\n");
|
|
120
115
|
}
|
|
@@ -143,7 +143,7 @@ export const TOWER_COMMAND_HELP = [
|
|
|
143
143
|
[
|
|
144
144
|
"usage",
|
|
145
145
|
[
|
|
146
|
-
"Usage: cockpit usage people [--by-repo | --by-topic] [--person <email|me>] [--all] [--since <n>d|<n>h|<iso>] [--detail] [--until <iso>] [--include-automated] [--json]",
|
|
146
|
+
"Usage: cockpit usage people [--by-repo | --by-topic] [--min-confidence high|medium|low] [--person <email|me>] [--all] [--since <n>d|<n>h|<iso>] [--detail] [--until <iso>] [--include-automated] [--json]",
|
|
147
147
|
"",
|
|
148
148
|
"--by-topic adds topic rows, including (unlabelled). Choose only one grouping. --by-repo adds project rows under each person (top 10; --all shows every repo). --person me uses your signed-in email.",
|
|
149
149
|
"Claude Code and Codex usage per person: sessions observed and extracted, tokens (total, output,",
|
|
@@ -102,7 +102,7 @@ export function localCommandHelp(command) {
|
|
|
102
102
|
` cockpit search "<words>" [--kind ${SEARCH_KINDS.join(",")}] [--limit <n>] [--dashboard-url <url>] [--json]`,
|
|
103
103
|
" cockpit release [--dry-run] [--skip-checks] [--no-floor] [--tag <tag>] [--access <public|restricted>] [--otp <code>]",
|
|
104
104
|
" cockpit careers [list|show <id>|rescreen <id>] [--role <slug>] [--min-score <n>] [--since <date>] [--json]",
|
|
105
|
-
" cockpit usage people [--by-repo | --by-topic] [--person <email|me>] [--all] [--detail] [--since <n>d|<n>h|<iso>] [--until <iso>] [--include-automated] [--dashboard-url <url>] [--json]",
|
|
105
|
+
" cockpit usage people [--by-repo | --by-topic] [--min-confidence high|medium|low] [--person <email|me>] [--all] [--detail] [--since <n>d|<n>h|<iso>] [--until <iso>] [--include-automated] [--dashboard-url <url>] [--json]",
|
|
106
106
|
"",
|
|
107
107
|
`Default dashboard: ${DEFAULT_DASHBOARD_URL}. Omit --dashboard-url for normal production use; pass it only for staging/custom dashboards or to force a different pairing.`,
|
|
108
108
|
].join("\n");
|
|
@@ -15,7 +15,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
15
15
|
}
|
|
16
16
|
|
|
17
17
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
18
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
18
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.108");
|
|
19
19
|
return 0;
|
|
20
20
|
}
|
|
21
21
|
|
|
@@ -5,17 +5,20 @@ import { envWithNodeRuntimeOnPath, runScheduledSelfUpdate, } from "../scheduled-
|
|
|
5
5
|
/**
|
|
6
6
|
* BLI-2601: the fleet keeps itself current on npm `latest` without anyone
|
|
7
7
|
* re-running `npm i -g @bli-cockpit/cli && cockpit doctor` by hand after day 0. This always
|
|
8
|
-
* runs
|
|
9
|
-
*
|
|
10
|
-
* collection, and a collection failure never blocks the chance to
|
|
11
|
-
* self-update. Every error path here is swallowed on purpose: a failure is
|
|
8
|
+
* runs before collection, so a failing or slow scan cannot delay the floor pull.
|
|
9
|
+
* The replacement process skips this check and continues the same tick. Every error path here is swallowed on purpose: a failure is
|
|
12
10
|
* reported as its own named `update` receipt, never surfaced as a `sync`
|
|
13
11
|
* failure or thrown from this function.
|
|
14
12
|
*/
|
|
15
13
|
export async function runScheduledSelfUpdateAfterSync(command, io, dashboardUrl, minCliVersion) {
|
|
14
|
+
if (io.env?.COCKPIT_SYNC_REEXEC === "1")
|
|
15
|
+
return false;
|
|
16
|
+
let updated = false;
|
|
16
17
|
let event;
|
|
17
18
|
try {
|
|
18
|
-
|
|
19
|
+
const result = await runScheduledSelfUpdateForSync(command, io, minCliVersion);
|
|
20
|
+
updated = result?.reason === "updated";
|
|
21
|
+
event = result ? scheduledSelfUpdateInstallEvent(result) : null;
|
|
19
22
|
}
|
|
20
23
|
catch (error) {
|
|
21
24
|
// The throttle/probe/install machinery below is defensive already; this
|
|
@@ -29,7 +32,7 @@ export async function runScheduledSelfUpdateAfterSync(command, io, dashboardUrl,
|
|
|
29
32
|
};
|
|
30
33
|
}
|
|
31
34
|
if (!event)
|
|
32
|
-
return;
|
|
35
|
+
return false;
|
|
33
36
|
await reportInstallEventsBestEffort({
|
|
34
37
|
homeDir: command.homeDir,
|
|
35
38
|
dashboardUrl,
|
|
@@ -38,6 +41,7 @@ export async function runScheduledSelfUpdateAfterSync(command, io, dashboardUrl,
|
|
|
38
41
|
json: command.json,
|
|
39
42
|
io,
|
|
40
43
|
});
|
|
44
|
+
return updated;
|
|
41
45
|
}
|
|
42
46
|
async function runScheduledSelfUpdateForSync(command, io, minCliVersion) {
|
|
43
47
|
const rawExec = io.exec;
|
|
@@ -60,7 +64,7 @@ async function runScheduledSelfUpdateForSync(command, io, minCliVersion) {
|
|
|
60
64
|
currentVersion: LOCAL_COLLECTOR_VERSION,
|
|
61
65
|
install: (tag) => attemptScheduledSelfUpdateInstall(scheduledIo, tag),
|
|
62
66
|
}, { env: io.env, minVersion: minCliVersion });
|
|
63
|
-
return
|
|
67
|
+
return result;
|
|
64
68
|
}
|
|
65
69
|
async function attemptScheduledSelfUpdateInstall(io, tag) {
|
|
66
70
|
try {
|
|
@@ -125,7 +125,7 @@ export function failedReceipt(run) {
|
|
|
125
125
|
// an operator reading a red row still has to be able to see what is merely
|
|
126
126
|
// waiting — otherwise the held objects become invisible the moment anything
|
|
127
127
|
// else goes wrong, which is the opposite of the fix.
|
|
128
|
-
const reasonText = [run.holdNotice, ...run.failureReasons]
|
|
128
|
+
const reasonText = [run.notice, run.holdNotice, ...run.failureReasons]
|
|
129
129
|
.filter(Boolean)
|
|
130
130
|
.join("; ");
|
|
131
131
|
// The bucket comes from the records the deciding branches wrote, not from
|
|
@@ -16,7 +16,10 @@ export async function reportMultiRepoSync(command, io, run, dedup) {
|
|
|
16
16
|
writeLine(io.stdout, JSON.stringify({
|
|
17
17
|
mode: "multi_repo",
|
|
18
18
|
status: collectionRunStatus,
|
|
19
|
-
collection_complete: run.ok,
|
|
19
|
+
collection_complete: run.ok && !run.discovery_partial,
|
|
20
|
+
discovery_notice: run.notice,
|
|
21
|
+
failure_reasons: run.failure_reasons,
|
|
22
|
+
failure_records: run.failure_records,
|
|
20
23
|
results: run.outcomes.map((outcome) => outcome.sync),
|
|
21
24
|
repos: rows,
|
|
22
25
|
codex_sessions: run.summary,
|
|
@@ -45,7 +48,10 @@ export async function reportNoWorktreeSync(command, io, run, dedup) {
|
|
|
45
48
|
writeLine(io.stdout, JSON.stringify({
|
|
46
49
|
mode: "no_worktrees",
|
|
47
50
|
status: collectionRunStatus,
|
|
48
|
-
collection_complete: run.ok,
|
|
51
|
+
collection_complete: run.ok && !run.discovery_partial,
|
|
52
|
+
discovery_notice: run.notice,
|
|
53
|
+
failure_reasons: run.failure_reasons,
|
|
54
|
+
failure_records: run.failure_records,
|
|
49
55
|
...(run.notice ? { notice: run.notice } : {}),
|
|
50
56
|
codex_sessions: run.summary,
|
|
51
57
|
raw_evidence_gc: gc,
|
|
@@ -54,7 +60,7 @@ export async function reportNoWorktreeSync(command, io, run, dedup) {
|
|
|
54
60
|
return syncResult(run);
|
|
55
61
|
}
|
|
56
62
|
writeLine(run.ok ? io.stdout : io.stderr, `Tower sync ${collectionRunStatus}: no git worktrees under this root; session scan ran.`);
|
|
57
|
-
if (run.notice) {
|
|
63
|
+
if (run.notice?.startsWith("nothing_in_root")) {
|
|
58
64
|
// Says out loud what the receipt now says to the dashboard: the sessions
|
|
59
65
|
// this machine ran were all outside the folders it is allowed to look at.
|
|
60
66
|
writeLine(io.stdout, `Every session seen this run was outside your approved folders (${run.notice}). Nothing was collected, and nothing is broken.`);
|
|
@@ -82,7 +88,10 @@ export async function reportSingleRepoSync(command, io, run, dedup) {
|
|
|
82
88
|
writeLine(io.stdout, JSON.stringify({
|
|
83
89
|
...result,
|
|
84
90
|
status: collectionRunStatus,
|
|
85
|
-
collection_complete: run.ok,
|
|
91
|
+
collection_complete: run.ok && !run.discovery_partial,
|
|
92
|
+
discovery_notice: run.notice,
|
|
93
|
+
failure_reasons: run.failure_reasons,
|
|
94
|
+
failure_records: run.failure_records,
|
|
86
95
|
codex_sessions: run.summary,
|
|
87
96
|
raw_evidence_gc: gc,
|
|
88
97
|
raw_evidence_dedup: dedup,
|
|
@@ -22,11 +22,13 @@ export async function runSyncLocked(command, io) {
|
|
|
22
22
|
if (!dedup.skipped && dedup.removed_dirs > 0) {
|
|
23
23
|
writeLine(io.stdout, rawEvidenceDedupSummary(dedup));
|
|
24
24
|
}
|
|
25
|
+
let discoveryNotice = null;
|
|
25
26
|
const worktrees = await discoverCommandWorktrees(collectionRoots, {
|
|
26
27
|
maxDepth: command.maxDepth,
|
|
27
28
|
maxRepos: command.maxRepos,
|
|
28
29
|
homeDir: command.homeDir,
|
|
29
30
|
allowEmpty: true,
|
|
31
|
+
onPartial: (notice) => { discoveryNotice = notice; },
|
|
30
32
|
}, io);
|
|
31
33
|
const run = await runAttributedWorktreeSync({
|
|
32
34
|
homeDir: command.homeDir,
|
|
@@ -36,6 +38,10 @@ export async function runSyncLocked(command, io) {
|
|
|
36
38
|
worktrees,
|
|
37
39
|
fetchImpl: io.fetch,
|
|
38
40
|
});
|
|
41
|
+
if (discoveryNotice) {
|
|
42
|
+
run.discovery_partial = true;
|
|
43
|
+
run.notice = [discoveryNotice, run.notice].filter(Boolean).join("; ");
|
|
44
|
+
}
|
|
39
45
|
if (run.outcomes.length > 1) {
|
|
40
46
|
return reportMultiRepoSync(command, io, run, dedup);
|
|
41
47
|
}
|
package/dist/commands/sync.js
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
*
|
|
8
8
|
* rotate the scheduler's own logs ../log-rotation.ts
|
|
9
9
|
* say the tick started reportTickStarted, below
|
|
10
|
+
* update and re-exec if needed sync-followups-self-update.ts
|
|
10
11
|
* collect behind both locks sync-receipt.ts
|
|
11
12
|
* check in with the heartbeat door sync-heartbeat.ts
|
|
12
13
|
* say how the tick went reportTickOutcome / reportTickThrew
|
|
@@ -32,6 +33,8 @@
|
|
|
32
33
|
* `scripts/build-public-cli.mjs` `runtimeFiles`, or the repo tests stay green
|
|
33
34
|
* while the packed CLI breaks.
|
|
34
35
|
*/
|
|
36
|
+
import { createInteractiveExecRunner } from "../process-runner.js";
|
|
37
|
+
import { envWithNodeRuntimeOnPath } from "../scheduled-self-update.js";
|
|
35
38
|
import { runMemoryExperienceAfterSync } from "./memory-log.js";
|
|
36
39
|
import { classifySyncHealthError, redactedSyncErrorDetail, reportInstallEventsBestEffort, } from "./install-receipts.js";
|
|
37
40
|
import { sendSyncHeartbeat } from "./sync-heartbeat.js";
|
|
@@ -48,14 +51,33 @@ export async function runSync(command, io) {
|
|
|
48
51
|
await rotateCollectorLogsBestEffort(paths);
|
|
49
52
|
const dashboardUrl = await tickDashboardUrl(command, paths);
|
|
50
53
|
const minCliVersionAtStart = await reportTickStarted(command, io, dashboardUrl);
|
|
54
|
+
const updated = await runScheduledSelfUpdateAfterSync(command, io, dashboardUrl, minCliVersionAtStart);
|
|
55
|
+
if (updated && io.exec) {
|
|
56
|
+
const args = ["sync"];
|
|
57
|
+
if (command.homeDir)
|
|
58
|
+
args.push("--home", command.homeDir);
|
|
59
|
+
if (command.repoRoot)
|
|
60
|
+
args.push("--workspace", command.repoRoot);
|
|
61
|
+
if (command.dashboardUrl)
|
|
62
|
+
args.push("--dashboard-url", command.dashboardUrl);
|
|
63
|
+
if (command.maxDepth)
|
|
64
|
+
args.push("--max-depth", String(command.maxDepth));
|
|
65
|
+
if (command.maxRepos)
|
|
66
|
+
args.push("--max-repos", String(command.maxRepos));
|
|
67
|
+
if (command.json)
|
|
68
|
+
args.push("--json");
|
|
69
|
+
const exec = io.interactiveExec ?? createInteractiveExecRunner();
|
|
70
|
+
const result = await exec("cockpit", args, { env: envWithNodeRuntimeOnPath({ ...process.env, ...io.env, COCKPIT_SYNC_REEXEC: "1" }) });
|
|
71
|
+
return result.code;
|
|
72
|
+
}
|
|
51
73
|
try {
|
|
52
74
|
const result = await runSyncWithHealthReceipt(command, io);
|
|
53
75
|
// BLI-3551: every tick checks in, including one that collected nothing.
|
|
54
76
|
// This is the only writer of `last_seen_at` that does not need an envelope,
|
|
55
77
|
// so it is what separates a quiet machine from a dead one.
|
|
56
78
|
await sendSyncHeartbeat(command, io, dashboardUrl, result.heartbeat);
|
|
57
|
-
|
|
58
|
-
await convergeAfterTick(command, io, dashboardUrl
|
|
79
|
+
await reportTickOutcome(command, io, dashboardUrl, result.completion);
|
|
80
|
+
await convergeAfterTick(command, io, dashboardUrl);
|
|
59
81
|
// BLI-3619: and the disk stops growing without bound — same daily cadence,
|
|
60
82
|
// same rule that a follow-up never blocks or fails collection.
|
|
61
83
|
await runStagingPruneAfterSync(command, io, dashboardUrl);
|
|
@@ -70,8 +92,8 @@ export async function runSync(command, io) {
|
|
|
70
92
|
status: "fail",
|
|
71
93
|
reason: errorCode,
|
|
72
94
|
});
|
|
73
|
-
|
|
74
|
-
await convergeAfterTick(command, io, dashboardUrl
|
|
95
|
+
await reportTickThrew(command, io, dashboardUrl, error, errorCode);
|
|
96
|
+
await convergeAfterTick(command, io, dashboardUrl);
|
|
75
97
|
throw error;
|
|
76
98
|
}
|
|
77
99
|
}
|
|
@@ -137,8 +159,7 @@ async function reportTickThrew(command, io, dashboardUrl, error, errorCode) {
|
|
|
137
159
|
* Memory's registration — each at most once a day, each with its own receipt,
|
|
138
160
|
* and none of them able to block or fail collection (BLI-3580).
|
|
139
161
|
*/
|
|
140
|
-
async function convergeAfterTick(command, io, dashboardUrl
|
|
141
|
-
await runScheduledSelfUpdateAfterSync(command, io, dashboardUrl, minCliVersion);
|
|
162
|
+
async function convergeAfterTick(command, io, dashboardUrl) {
|
|
142
163
|
await runAutostartSelfHealAfterSync(command, io, dashboardUrl);
|
|
143
164
|
await runMemoryInstallAfterSync(command, io, dashboardUrl);
|
|
144
165
|
await runMemoryExperienceAfterSync({ ...command, dashboardUrl }, io);
|
package/dist/commands/usage.js
CHANGED
|
@@ -5,6 +5,8 @@ import { writeLine } from "./cli-io.js";
|
|
|
5
5
|
export async function runUsage(command, io) {
|
|
6
6
|
const door = await openAgentDoor("usage", command, io);
|
|
7
7
|
const query = new URLSearchParams({ since: command.since });
|
|
8
|
+
if (command.minConfidence)
|
|
9
|
+
query.set("minConfidence", command.minConfidence);
|
|
8
10
|
if (command.byTopic)
|
|
9
11
|
query.set("groupBy", "task_type");
|
|
10
12
|
if (command.byRepo)
|
|
@@ -53,6 +55,8 @@ export async function runUsage(command, io) {
|
|
|
53
55
|
if (command.byRepo || command.byTopic) {
|
|
54
56
|
const repos = (command.byTopic ? row.task_types : row.repos) ?? [];
|
|
55
57
|
for (const repo of command.all ? repos : repos.slice(0, 10)) {
|
|
58
|
+
if (command.byTopic && repo.confidence)
|
|
59
|
+
writeLine(io.stdout, ` Confidence: ${repo.confidence.high} high, ${repo.confidence.medium} medium, ${repo.confidence.low} low`);
|
|
56
60
|
printRow([` ${repo.repo_label}`, formatCount(repo.tokens), formatUsageDollars(repo.api_list_price_equivalent_usd), `${repo.extracted_sessions}/${repo.sessions}`, ...(command.detail ? [repo.output, repo.input, repo.cache_read, repo.cache_creation].map(formatCount) : [])]);
|
|
57
61
|
}
|
|
58
62
|
if (!command.all && repos.length > 10)
|
package/dist/disk-prune.js
CHANGED
|
@@ -63,6 +63,7 @@ export async function planFromDisk(paths, env, options, now) {
|
|
|
63
63
|
retentionMs: options.retentionMs ?? fromEnv.retentionMs,
|
|
64
64
|
capBytes: options.capBytes ?? fromEnv.capBytes,
|
|
65
65
|
allCommitted: options.allCommitted,
|
|
66
|
+
reclaimTerminal: options.reclaimTerminal,
|
|
66
67
|
});
|
|
67
68
|
}
|
|
68
69
|
async function applyPlan(paths, plan) {
|
package/dist/disk-retention.js
CHANGED
|
@@ -23,6 +23,11 @@ export function planStagingRetention(inventory, options = {}) {
|
|
|
23
23
|
const committedInWindow = [];
|
|
24
24
|
for (const pack of inventory.packs) {
|
|
25
25
|
for (const object of pack.objects) {
|
|
26
|
+
if (object.state === "terminal") {
|
|
27
|
+
if (options.reclaimTerminal)
|
|
28
|
+
pushDeletion(plan, object, `terminal:${object.reason}`);
|
|
29
|
+
continue;
|
|
30
|
+
}
|
|
26
31
|
if (object.state !== "committed") {
|
|
27
32
|
keepUnvouched(plan, object);
|
|
28
33
|
continue;
|
|
@@ -35,7 +40,8 @@ export function planStagingRetention(inventory, options = {}) {
|
|
|
35
40
|
addToBucket(plan.kept_in_window, object);
|
|
36
41
|
}
|
|
37
42
|
}
|
|
38
|
-
|
|
43
|
+
const retainedTerminalBytes = options.reclaimTerminal ? 0 : (inventory.terminal_bytes ?? 0);
|
|
44
|
+
applyByteCap(plan, committedInWindow, capBytes + retainedTerminalBytes, inventory);
|
|
39
45
|
markEmptyPacks(plan, inventory.packs);
|
|
40
46
|
plan.bytes_after =
|
|
41
47
|
inventory.total_bytes - plan.deleted_bytes - plan.empty_pack_manifest_bytes;
|
|
@@ -44,7 +50,7 @@ export function planStagingRetention(inventory, options = {}) {
|
|
|
44
50
|
// sentence — the remaining manifests alone exceed it — and saying the wrong
|
|
45
51
|
// one would send an operator hunting evidence that is not missing.
|
|
46
52
|
const unvouched = plan.kept_uncommitted.count + plan.kept_unknown.count;
|
|
47
|
-
plan.cap_blocked_by_uncommitted = plan.bytes_after > capBytes && unvouched > 0;
|
|
53
|
+
plan.cap_blocked_by_uncommitted = plan.bytes_after - retainedTerminalBytes > capBytes && unvouched > 0;
|
|
48
54
|
plan.cap_blocked_count = plan.cap_blocked_by_uncommitted ? unvouched : 0;
|
|
49
55
|
return plan;
|
|
50
56
|
}
|
|
@@ -84,7 +90,7 @@ function applyByteCap(plan, committedInWindow, capBytes, inventory) {
|
|
|
84
90
|
function unremovableFloor(inventory) {
|
|
85
91
|
let floor = 0;
|
|
86
92
|
for (const pack of inventory.packs) {
|
|
87
|
-
const unvouched = pack.objects.filter((object) => object.state !== "committed");
|
|
93
|
+
const unvouched = pack.objects.filter((object) => object.state !== "committed" && object.state !== "terminal");
|
|
88
94
|
if (unvouched.length === 0)
|
|
89
95
|
continue;
|
|
90
96
|
floor += pack.manifest_bytes;
|
|
@@ -2,14 +2,13 @@
|
|
|
2
2
|
* The three-state question, asked once per staged file: committed, uncommitted,
|
|
3
3
|
* or unknown — and the reason label that says which record answered.
|
|
4
4
|
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* neither can speak does the object fall to `unknown` with the reason its
|
|
9
|
-
* silence has. Everything downstream — what may be deleted, what is offered
|
|
5
|
+
* A local commit wins unless a newer server reconciliation disputes it.
|
|
6
|
+
* Explicit object refusals remain terminal until a later successful delivery.
|
|
7
|
+
* When neither ledger can speak, the object is unknown and stays on disk. Everything downstream — what may be deleted, what is offered
|
|
10
8
|
* again, what an operator reads — is this label.
|
|
11
9
|
*/
|
|
12
10
|
import path from "node:path";
|
|
11
|
+
import { terminalDeliveryReason } from "./raw-evidence-staging.js";
|
|
13
12
|
import { hashUnnamedFile } from "./disk-usage-files.js";
|
|
14
13
|
export async function classifyObject(dir, packId, relative, size, manifest, options) {
|
|
15
14
|
const hash = manifest?.get(relative) ??
|
|
@@ -23,21 +22,28 @@ export async function classifyObject(dir, packId, relative, size, manifest, opti
|
|
|
23
22
|
staged_on_disk_at: options.stagedOnDiskAt,
|
|
24
23
|
disk_age_ms: ageMs(options.stagedOnDiskAt, options.now),
|
|
25
24
|
};
|
|
26
|
-
|
|
25
|
+
const reconciled = hash ? options.reconciled.get(hash) : undefined;
|
|
26
|
+
const serverDisputesCommit = reconciled && committedAt &&
|
|
27
|
+
Date.parse(reconciled.checked_at) > Date.parse(committedAt) &&
|
|
28
|
+
reconciled.verdict !== "committed";
|
|
29
|
+
if (committedAt && !serverDisputesCommit) {
|
|
27
30
|
return {
|
|
28
|
-
...base,
|
|
29
|
-
|
|
30
|
-
state: "committed",
|
|
31
|
-
reason: "committed_in_ledger",
|
|
32
|
-
decided_at: committedAt,
|
|
31
|
+
...base, content_hash: hash ?? null, state: "committed",
|
|
32
|
+
reason: "committed_in_ledger", decided_at: committedAt,
|
|
33
33
|
age_ms: ageMs(committedAt, options.now),
|
|
34
34
|
};
|
|
35
35
|
}
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
36
|
+
const attempt = hash ? options.deliveryAttempts?.[hash] : undefined;
|
|
37
|
+
const serverConfirmedAfterRefusal = reconciled?.verdict === "committed" &&
|
|
38
|
+
attempt && Date.parse(reconciled.checked_at) >= Date.parse(attempt.last_attempt_at);
|
|
39
|
+
if (attempt && terminalDeliveryReason(attempt.last_reason) && !serverConfirmedAfterRefusal) {
|
|
40
|
+
return {
|
|
41
|
+
...base, content_hash: hash ?? null, state: "terminal", reason: attempt.last_reason,
|
|
42
|
+
decided_at: attempt.last_attempt_at, age_ms: ageMs(attempt.last_attempt_at, options.now),
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
if (hash && reconciled) {
|
|
46
|
+
return classifyFromReconcileAnswer(base, hash, reconciled, options.now);
|
|
41
47
|
}
|
|
42
48
|
const unknownReason = unknownStateReason(hash, manifest, stagedAt, options);
|
|
43
49
|
return {
|
package/dist/disk-usage-scan.js
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
* each pack's own manifest, and a single byte budget shared across the run.
|
|
9
9
|
*/
|
|
10
10
|
import fs from "node:fs/promises";
|
|
11
|
+
import { readRawEvidenceStagingState } from "./raw-evidence-staging.js";
|
|
11
12
|
import path from "node:path";
|
|
12
13
|
import { classifyObject, } from "./disk-usage-classify.js";
|
|
13
14
|
import { RAW_EVIDENCE_DIR, } from "./disk-usage-facts.js";
|
|
@@ -19,6 +20,7 @@ export async function readStagingInventory(paths, now = new Date()) {
|
|
|
19
20
|
const ledger = await readCommitLedger(paths);
|
|
20
21
|
const reconciled = await readReconcileLedger(paths);
|
|
21
22
|
const stagedAt = await readStagedAtIndex(paths);
|
|
23
|
+
const { delivery_attempts: deliveryAttempts } = await readRawEvidenceStagingState(paths.state_dir);
|
|
22
24
|
const entries = await fs
|
|
23
25
|
.readdir(root, { withFileTypes: true })
|
|
24
26
|
.catch(() => []);
|
|
@@ -36,6 +38,7 @@ export async function readStagingInventory(paths, now = new Date()) {
|
|
|
36
38
|
if (!entry.name.startsWith("work-"))
|
|
37
39
|
continue;
|
|
38
40
|
const pack = await readPack(dir, entry.name, {
|
|
41
|
+
deliveryAttempts,
|
|
39
42
|
ledger,
|
|
40
43
|
reconciled,
|
|
41
44
|
stagedAt,
|
|
@@ -26,6 +26,13 @@ export function accumulatePack(inventory, pack) {
|
|
|
26
26
|
for (const object of pack.objects) {
|
|
27
27
|
inventory.object_count += 1;
|
|
28
28
|
inventory.total_bytes += object.byte_size;
|
|
29
|
+
if (object.state === "terminal") {
|
|
30
|
+
inventory.terminal_count = (inventory.terminal_count ?? 0) + 1;
|
|
31
|
+
inventory.terminal_bytes = (inventory.terminal_bytes ?? 0) + object.byte_size;
|
|
32
|
+
const reasons = inventory.terminal_reasons ??= {};
|
|
33
|
+
reasons[object.reason] = (reasons[object.reason] ?? 0) + 1;
|
|
34
|
+
continue;
|
|
35
|
+
}
|
|
29
36
|
if (object.state === "committed") {
|
|
30
37
|
inventory.committed_count += 1;
|
|
31
38
|
inventory.committed_bytes += object.byte_size;
|
|
@@ -28,7 +28,7 @@ import { markObjectCommitted, readRawEvidenceCursor, writeRawEvidenceCursor, } f
|
|
|
28
28
|
import { readStagingInventory } from "./disk-usage.js";
|
|
29
29
|
import { describeError } from "./health-detail.js";
|
|
30
30
|
import { getCollectorRuntimePaths, readLocalCollectorSessionFile, } from "./local-state.js";
|
|
31
|
-
import { clearDeliveryAttempt, readRawEvidenceStagingState, recordDeliveryFailure, writeRawEvidenceStagingState, } from "./raw-evidence-staging.js";
|
|
31
|
+
import { clearDeliveryAttempt, readRawEvidenceStagingState, recordDeliveryFailure, terminalDeliveryReason, writeRawEvidenceStagingState, } from "./raw-evidence-staging.js";
|
|
32
32
|
/**
|
|
33
33
|
* The drain the sync tick runs and `cockpit doctor`'s disk fix reuses.
|
|
34
34
|
* Never throws.
|
|
@@ -46,7 +46,7 @@ export async function runEvidenceRedelivery(options) {
|
|
|
46
46
|
// The steady state, 96 ticks a day. Saying so every time would bury the
|
|
47
47
|
// ticks that matter; `cockpit doctor`'s disk row still says it out loud
|
|
48
48
|
// to a person who asks.
|
|
49
|
-
return emptyResult("nothing_uncommitted", "ok");
|
|
49
|
+
return { ...emptyResult("nothing_uncommitted", "ok"), terminal: inventory.terminal_count ?? 0, terminal_bytes: inventory.terminal_bytes ?? 0 };
|
|
50
50
|
}
|
|
51
51
|
const staging = await readRawEvidenceStagingState(paths.state_dir);
|
|
52
52
|
const plan = await planEvidenceRedelivery({
|
|
@@ -58,7 +58,7 @@ export async function runEvidenceRedelivery(options) {
|
|
|
58
58
|
...(options.maxObjects != null ? { maxObjects: options.maxObjects } : {}),
|
|
59
59
|
});
|
|
60
60
|
if (plan.object_count === 0) {
|
|
61
|
-
const result = emptyPlanResult(plan);
|
|
61
|
+
const result = { ...emptyPlanResult(plan), terminal: inventory.terminal_count ?? 0, terminal_bytes: inventory.terminal_bytes ?? 0 };
|
|
62
62
|
reportRedelivery(result, plan);
|
|
63
63
|
return result;
|
|
64
64
|
}
|
|
@@ -86,14 +86,18 @@ export async function runEvidenceRedelivery(options) {
|
|
|
86
86
|
now,
|
|
87
87
|
});
|
|
88
88
|
await recordOutcomes(paths, staging, outcome.outcomes, now);
|
|
89
|
+
const terminalOutcomes = outcome.outcomes.filter(entry => entry.upload_state === "upload_failed" && terminalDeliveryReason(entry.reason));
|
|
89
90
|
const result = {
|
|
90
91
|
status: outcome.failed > 0 && outcome.uploaded === 0 ? "fail" : "ok",
|
|
91
92
|
reason: "redelivered",
|
|
93
|
+
terminal: (inventory.terminal_count ?? 0) + terminalOutcomes.length,
|
|
94
|
+
terminal_bytes: (inventory.terminal_bytes ?? 0) + terminalOutcomes.reduce((bytes, entry) => bytes + (entry.pointer.byte_size ?? 0), 0),
|
|
92
95
|
offered: plan.object_count,
|
|
93
96
|
offered_bytes: plan.bytes,
|
|
94
97
|
uploaded: outcome.uploaded,
|
|
95
98
|
uploaded_bytes: outcome.uploadedBytes,
|
|
96
99
|
reused: outcome.reused,
|
|
100
|
+
reconciled_committed_by_409: outcome.outcomes.filter(entry => entry.reason === "reconciled_committed_by_409").length,
|
|
97
101
|
failed: outcome.failed,
|
|
98
102
|
held: plan.held_count,
|
|
99
103
|
deferred: plan.deferred_count,
|
|
@@ -229,11 +233,14 @@ function reportRedelivery(result, plan) {
|
|
|
229
233
|
const fields = {
|
|
230
234
|
reason: result.reason,
|
|
231
235
|
status: result.status,
|
|
236
|
+
terminal: result.terminal ?? 0,
|
|
237
|
+
terminal_bytes: result.terminal_bytes ?? 0,
|
|
232
238
|
offered: result.offered,
|
|
233
239
|
offered_bytes: result.offered_bytes,
|
|
234
240
|
uploaded: result.uploaded,
|
|
235
241
|
uploaded_bytes: result.uploaded_bytes,
|
|
236
242
|
reused: result.reused,
|
|
243
|
+
reconciled_committed_by_409: result.reconciled_committed_by_409 ?? 0,
|
|
237
244
|
failed: result.failed,
|
|
238
245
|
held_by_backoff: result.held,
|
|
239
246
|
deferred_to_next_tick: result.deferred,
|
|
@@ -243,7 +250,7 @@ function reportRedelivery(result, plan) {
|
|
|
243
250
|
failure_reasons: result.failure_reasons,
|
|
244
251
|
};
|
|
245
252
|
if (result.status === "fail") {
|
|
246
|
-
console.error("[evidence-redelivery]
|
|
253
|
+
console.error("[evidence-redelivery] staged evidence could not be delivered", JSON.stringify(fields));
|
|
247
254
|
return;
|
|
248
255
|
}
|
|
249
256
|
console.error(result.uploaded > 0
|
|
@@ -251,11 +258,11 @@ function reportRedelivery(result, plan) {
|
|
|
251
258
|
: "[evidence-redelivery] undelivered staged evidence was not re-offered this tick", JSON.stringify(fields));
|
|
252
259
|
}
|
|
253
260
|
function emptyPlanResult(plan) {
|
|
254
|
-
const nothingPlannable = plan.held_count > 0 && plan.deferred_count === 0
|
|
261
|
+
const nothingPlannable = plan.held_count > 0 && plan.deferred_count === 0 && Object.keys(plan.unplannable).length === 0
|
|
255
262
|
? "all_held_by_backoff"
|
|
256
263
|
: "nothing_plannable";
|
|
257
264
|
return {
|
|
258
|
-
...emptyResult(nothingPlannable, "ok"),
|
|
265
|
+
...emptyResult(nothingPlannable, Object.keys(plan.unplannable).length ? "fail" : "ok"),
|
|
259
266
|
held: plan.held_count,
|
|
260
267
|
deferred: plan.deferred_count,
|
|
261
268
|
deferred_bytes: plan.deferred_bytes,
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { COMMIT_CRASHED_PLATFORM, RawEvidenceUploadCommitResponseSchema, isPermanentUploadFailure, } from "@bli-cockpit/telemetry-core";
|
|
2
|
+
import { beginOneObject } from "./evidence-upload-terminal.js";
|
|
2
3
|
import { isNonJsonResponseBody, requestJson, safeFailureDetail, sha256, } from "./evidence-upload-transport.js";
|
|
3
4
|
/**
|
|
4
5
|
* The `chunk` and `commit` stages for one already-`begin`-accepted object:
|
|
5
6
|
* send every chunk the server does not already have, then commit and turn
|
|
6
7
|
* the receipt into a `ChunkedUploadFileOutcome`.
|
|
7
8
|
*/
|
|
8
|
-
export async function uploadOneObject(options, entry, disposition, chunkSizeBytes) {
|
|
9
|
+
export async function uploadOneObject(options, entry, disposition, chunkSizeBytes, restageAttempted = false) {
|
|
9
10
|
const objectKey = entry.file.pointer.object_key ?? "";
|
|
10
11
|
if (disposition.disposition === "already_committed") {
|
|
11
12
|
if (!disposition.upload_id) {
|
|
@@ -18,7 +19,22 @@ export async function uploadOneObject(options, entry, disposition, chunkSizeByte
|
|
|
18
19
|
const chunked = await uploadObjectChunks(options, entry, disposition, chunkSizeBytes, objectKey);
|
|
19
20
|
if (!chunked.ok)
|
|
20
21
|
return chunked.outcome;
|
|
21
|
-
|
|
22
|
+
const committed = await commitUploadedObject(options, entry, disposition, objectKey, chunked.uploadedChunks);
|
|
23
|
+
if (committed.reason !== "commit_failed_http_409_staging_missing_between_passes")
|
|
24
|
+
return committed;
|
|
25
|
+
if (restageAttempted)
|
|
26
|
+
return { ...committed, reason: "staging_missing_after_restage" };
|
|
27
|
+
console.error("[evidence-upload] restaging missing chunks", JSON.stringify({
|
|
28
|
+
reason: "staging_missing_between_passes", upload_id: disposition.upload_id,
|
|
29
|
+
}));
|
|
30
|
+
const restarted = await beginOneObject(options, { ...entry, duplicates: [] }, chunkSizeBytes);
|
|
31
|
+
if (!restarted)
|
|
32
|
+
return failedOutcome(entry.file, "staging_restage_failed:begin_failed", chunked.uploadedChunks);
|
|
33
|
+
const retried = await uploadOneObject(options, entry, restarted, chunkSizeBytes, true);
|
|
34
|
+
if (retried.upload_state === "upload_failed" && retried.reason !== "staging_missing_after_restage") {
|
|
35
|
+
retried.reason = `staging_restage_failed:${retried.reason ?? "unknown"}`;
|
|
36
|
+
}
|
|
37
|
+
return retried;
|
|
22
38
|
}
|
|
23
39
|
async function uploadObjectChunks(options, entry, disposition, chunkSizeBytes, objectKey) {
|
|
24
40
|
const received = new Set(disposition.received_chunk_indexes);
|
|
@@ -47,6 +63,15 @@ async function uploadObjectChunks(options, entry, disposition, chunkSizeBytes, o
|
|
|
47
63
|
// `chunk_3_failed_http_413_object_too_large` and the ledger row names
|
|
48
64
|
// the cause instead of the transport (BLI-3483).
|
|
49
65
|
const chunkDetail = safeFailureDetail(chunkResponse.body);
|
|
66
|
+
if (chunkResponse.status === 409 && chunkDetail === "upload_not_pending:committed") {
|
|
67
|
+
console.error("[evidence-upload] committed receipt reconciled", JSON.stringify({
|
|
68
|
+
reason: "reconciled_committed_by_409", upload_id: disposition.upload_id,
|
|
69
|
+
}));
|
|
70
|
+
return { ok: false, outcome: {
|
|
71
|
+
...failedOutcome(entry.file, "reconciled_committed_by_409", uploadedChunks),
|
|
72
|
+
upload_state: "reused_existing",
|
|
73
|
+
} };
|
|
74
|
+
}
|
|
50
75
|
console.error("[evidence-upload] chunk rejected", JSON.stringify({
|
|
51
76
|
reason: "chunk_rejected",
|
|
52
77
|
upload_id: disposition.upload_id,
|
|
@@ -73,7 +73,7 @@ export async function rekeyConflictedEntry(options, entry, disposition, chunkSiz
|
|
|
73
73
|
* pointer id that is not the one we sent — and the caller then keeps the
|
|
74
74
|
* original conflict rather than acting on a guess.
|
|
75
75
|
*/
|
|
76
|
-
async function beginOneObject(options, entry, chunkSizeBytes) {
|
|
76
|
+
export async function beginOneObject(options, entry, chunkSizeBytes) {
|
|
77
77
|
const objectKey = entry.file.pointer.object_key ?? "";
|
|
78
78
|
const response = await requestJson(options, "/api/ambient/evidence/upload/begin", {
|
|
79
79
|
schema_version: "ambient-raw-evidence-upload-begin.v1",
|
|
@@ -160,6 +160,7 @@ export function recordDeliveryFailure(state, contentHash, options) {
|
|
|
160
160
|
const attemptedAtIso = options.attemptedAt.toISOString();
|
|
161
161
|
const entry = {
|
|
162
162
|
attempts,
|
|
163
|
+
...(terminalDeliveryReason(options.reason) ? { terminal_reason: options.reason } : {}),
|
|
163
164
|
first_failed_at: previous?.first_failed_at ?? attemptedAtIso,
|
|
164
165
|
last_attempt_at: attemptedAtIso,
|
|
165
166
|
last_reason: options.reason,
|
|
@@ -177,13 +178,20 @@ export function clearDeliveryAttempt(state, contentHash) {
|
|
|
177
178
|
delete state.delivery_attempts[contentHash];
|
|
178
179
|
return true;
|
|
179
180
|
}
|
|
181
|
+
export function terminalDeliveryReason(reason) {
|
|
182
|
+
return Boolean(reason && (reason.startsWith("staging_restage_failed:") ||
|
|
183
|
+
["staging_missing_after_restage", "secret_guard_rejected", "storage_rejected_object_too_large", "storage_rejected_media_type"].includes(reason) ||
|
|
184
|
+
/^(?:begin|commit|chunk_\d+)_failed_http_413(?:_|$)/u.test(reason)));
|
|
185
|
+
}
|
|
180
186
|
export function deliveryHold(state, contentHash, now) {
|
|
181
187
|
if (!contentHash)
|
|
182
188
|
return null;
|
|
183
189
|
const entry = state.delivery_attempts[contentHash];
|
|
184
190
|
if (!entry)
|
|
185
191
|
return null;
|
|
186
|
-
|
|
192
|
+
if (terminalDeliveryReason(entry.last_reason))
|
|
193
|
+
return entry;
|
|
194
|
+
return deliveryRetryAtMs(entry) > now.getTime() ? entry : null;
|
|
187
195
|
}
|
|
188
196
|
/**
|
|
189
197
|
* Source keys currently held by backoff.
|
|
@@ -194,10 +202,10 @@ export function deliveryHold(state, contentHash, now) {
|
|
|
194
202
|
*/
|
|
195
203
|
export function heldSourceKeys(state, now) {
|
|
196
204
|
const held = new Set();
|
|
197
|
-
for (const entry of Object.
|
|
198
|
-
if (!entry.source_key)
|
|
205
|
+
for (const [hash, entry] of Object.entries(state.delivery_attempts)) {
|
|
206
|
+
if (!entry.source_key || terminalDeliveryReason(entry.last_reason))
|
|
199
207
|
continue;
|
|
200
|
-
if (
|
|
208
|
+
if (deliveryHold(state, hash, now)) {
|
|
201
209
|
held.add(entry.source_key);
|
|
202
210
|
}
|
|
203
211
|
}
|
|
@@ -286,6 +294,7 @@ function parseAttemptEntry(value) {
|
|
|
286
294
|
return null;
|
|
287
295
|
const attempts = optionalNumber(record["attempts"]);
|
|
288
296
|
return {
|
|
297
|
+
...(terminalDeliveryReason(optionalString(record["last_reason"])) ? { terminal_reason: String(record["last_reason"]) } : {}),
|
|
289
298
|
attempts: attempts > 0 ? attempts : 1,
|
|
290
299
|
first_failed_at: optionalString(record["first_failed_at"]) ?? lastAttemptAt,
|
|
291
300
|
last_attempt_at: lastAttemptAt,
|
|
@@ -307,4 +316,8 @@ function shortHash(value) {
|
|
|
307
316
|
.update(value, "utf8")
|
|
308
317
|
.digest("hex")
|
|
309
318
|
.slice(0, 12);
|
|
319
|
+
}
|
|
320
|
+
/** A persisted clock cannot extend a retry beyond the last attempt's ceiling. */
|
|
321
|
+
export function deliveryRetryAtMs(entry) {
|
|
322
|
+
return Math.min(Date.parse(entry.next_attempt_at), Date.parse(entry.last_attempt_at) + EVIDENCE_DELIVERY_BACKOFF_MAX_MS);
|
|
310
323
|
}
|
|
@@ -14,12 +14,13 @@
|
|
|
14
14
|
* Metadata only, like the rest of this family: content hashes, byte sizes,
|
|
15
15
|
* reason labels, timestamps. Never content, never an absolute path.
|
|
16
16
|
*/
|
|
17
|
+
import { deliveryRetryAtMs, terminalDeliveryReason } from "./raw-evidence-staging.js";
|
|
17
18
|
/**
|
|
18
19
|
* What `cockpit status` and the health receipt need in order to be unable to
|
|
19
20
|
* read green while an object has been failing for nine days.
|
|
20
21
|
*/
|
|
21
22
|
export function summarizeStuckEvidence(state, now) {
|
|
22
|
-
const entries = Object.values(state.delivery_attempts);
|
|
23
|
+
const entries = Object.values(state.delivery_attempts).filter(entry => !terminalDeliveryReason(entry.last_reason));
|
|
23
24
|
const reasons = new Set();
|
|
24
25
|
let heldCount = 0;
|
|
25
26
|
let maxAttempts = 0;
|
|
@@ -31,15 +32,16 @@ export function summarizeStuckEvidence(state, now) {
|
|
|
31
32
|
let heldNextAttemptAt = null;
|
|
32
33
|
for (const entry of entries) {
|
|
33
34
|
reasons.add(entry.last_reason);
|
|
34
|
-
|
|
35
|
+
const retryAt = deliveryRetryAtMs(entry);
|
|
36
|
+
if (retryAt > now.getTime()) {
|
|
35
37
|
heldCount += 1;
|
|
36
38
|
if (!heldOldest ||
|
|
37
39
|
entry.first_failed_at.localeCompare(heldOldest.first_failed_at) < 0) {
|
|
38
40
|
heldOldest = entry;
|
|
39
41
|
}
|
|
40
42
|
if (!heldNextAttemptAt ||
|
|
41
|
-
|
|
42
|
-
heldNextAttemptAt =
|
|
43
|
+
retryAt < Date.parse(heldNextAttemptAt)) {
|
|
44
|
+
heldNextAttemptAt = new Date(retryAt).toISOString();
|
|
43
45
|
}
|
|
44
46
|
}
|
|
45
47
|
maxAttempts = Math.max(maxAttempts, entry.attempts);
|
|
@@ -28,11 +28,9 @@ export function envWithNodeRuntimeOnPath(env, nodeExecutable = process.execPath)
|
|
|
28
28
|
* post-collection tail, so the fleet converges on npm `latest` without
|
|
29
29
|
* anyone re-running `npm i -g @bli-cockpit/cli && cockpit doctor` by hand after day 0.
|
|
30
30
|
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
* `cockpit` process from disk and picks up whatever npm actually installed —
|
|
35
|
-
* that is the verification, not a re-exec here.
|
|
31
|
+
* Installation is verified with npm's installed-version probe. The sync caller
|
|
32
|
+
* re-executes the tick on `updated`, carrying a guard that skips another update
|
|
33
|
+
* attempt in the replacement process.
|
|
36
34
|
*/
|
|
37
35
|
export async function runScheduledSelfUpdate(paths, deps, options = {}) {
|
|
38
36
|
const env = options.env ?? process.env;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { clearDeliveryAttempt, DELIVERY_BACKOFF_BYPASS_REASON, DELIVERY_BACKOFF_HOLDING_REASON, deliveryBackoffApplies, deliveryHold, recordDeliveryFailure, writeRawEvidenceStagingState, } from "./raw-evidence-staging.js";
|
|
1
|
+
import { clearDeliveryAttempt, DELIVERY_BACKOFF_BYPASS_REASON, DELIVERY_BACKOFF_HOLDING_REASON, deliveryBackoffApplies, deliveryHold, recordDeliveryFailure, terminalDeliveryReason, writeRawEvidenceStagingState, } from "./raw-evidence-staging.js";
|
|
2
2
|
/**
|
|
3
3
|
* 1. What may we offer this sync? 2. What just happened to each object?
|
|
4
4
|
*
|
|
@@ -25,7 +25,7 @@ export function partitionHeldEvidenceFiles(files, staging, now, mode) {
|
|
|
25
25
|
const backoffApplies = deliveryBackoffApplies(mode);
|
|
26
26
|
for (const file of files) {
|
|
27
27
|
const hold = deliveryHold(staging, file.pointer.content_hash_sha256, now);
|
|
28
|
-
if (hold && !backoffApplies) {
|
|
28
|
+
if (hold && !backoffApplies && !terminalDeliveryReason(hold.last_reason)) {
|
|
29
29
|
// BLI-3118: a person asked for this one now. Offering it is the whole
|
|
30
30
|
// point of the retry command Cockpit printed, and the bypass is logged
|
|
31
31
|
// rather than assumed.
|
|
@@ -46,7 +46,8 @@ export function partitionHeldEvidenceFiles(files, staging, now, mode) {
|
|
|
46
46
|
? { artifact_metadata: file.artifact_metadata }
|
|
47
47
|
: {}),
|
|
48
48
|
upload_state: "upload_failed",
|
|
49
|
-
reason: DELIVERY_BACKOFF_HOLDING_REASON,
|
|
49
|
+
reason: terminalDeliveryReason(hold.last_reason) ? hold.last_reason : DELIVERY_BACKOFF_HOLDING_REASON,
|
|
50
|
+
delivery_attempted: false,
|
|
50
51
|
uploaded_chunk_count: 0,
|
|
51
52
|
});
|
|
52
53
|
}
|
|
@@ -103,7 +104,7 @@ export async function persistDeliveryAttempts(stateDir, staging, outcomes, attem
|
|
|
103
104
|
const contentHash = outcome.pointer.content_hash_sha256;
|
|
104
105
|
if (!contentHash)
|
|
105
106
|
continue;
|
|
106
|
-
if (outcome.reason === DELIVERY_BACKOFF_HOLDING_REASON)
|
|
107
|
+
if (outcome.reason === DELIVERY_BACKOFF_HOLDING_REASON || outcome.delivery_attempted === false)
|
|
107
108
|
continue;
|
|
108
109
|
if (outcome.upload_state === "upload_failed") {
|
|
109
110
|
const entry = recordDeliveryFailure(staging, contentHash, {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { isPermanentUploadFailure } from "@bli-cockpit/telemetry-core";
|
|
2
|
-
import { DELIVERY_BACKOFF_HOLDING_REASON, summarizeStuckEvidence, } from "./raw-evidence-staging.js";
|
|
2
|
+
import { DELIVERY_BACKOFF_HOLDING_REASON, summarizeStuckEvidence, terminalDeliveryReason, } from "./raw-evidence-staging.js";
|
|
3
3
|
/**
|
|
4
4
|
* Which adapters a queued retry has to re-run.
|
|
5
5
|
*
|
|
@@ -36,12 +36,12 @@ export function retrySourcesForFailedSync(options, facts) {
|
|
|
36
36
|
*/
|
|
37
37
|
function retryableFailedOutcomes(outcomes) {
|
|
38
38
|
return outcomes.filter((outcome) => outcome.upload_state === "upload_failed" &&
|
|
39
|
-
!isPermanentUploadFailure(outcome.reason));
|
|
39
|
+
!isPermanentUploadFailure(outcome.reason) && !terminalDeliveryReason(outcome.reason));
|
|
40
40
|
}
|
|
41
41
|
/** Failed uploads that no retry can rescue, kept so they can still be named. */
|
|
42
42
|
function permanentFailedOutcomes(outcomes) {
|
|
43
43
|
return outcomes.filter((outcome) => outcome.upload_state === "upload_failed" &&
|
|
44
|
-
isPermanentUploadFailure(outcome.reason));
|
|
44
|
+
(isPermanentUploadFailure(outcome.reason) || terminalDeliveryReason(outcome.reason)));
|
|
45
45
|
}
|
|
46
46
|
/**
|
|
47
47
|
* The reason to show for objects that failed for good.
|
|
@@ -174,6 +174,7 @@ export function summarizeRawEvidenceDelivery(built, outcomes, uploadedChunkCount
|
|
|
174
174
|
raw_evidence_uploaded_object_count: outcomes.filter((outcome) => outcome.upload_state === "uploaded").length,
|
|
175
175
|
raw_evidence_uploaded_chunk_count: uploadedChunkCount,
|
|
176
176
|
raw_evidence_reused_count: cursorReused.length + serverReusedCount,
|
|
177
|
+
reconciled_committed_by_409: outcomes.filter((outcome) => outcome.reason === "reconciled_committed_by_409").length,
|
|
177
178
|
raw_evidence_failed_count: failed.length,
|
|
178
179
|
raw_evidence_sanitized_count: built.raw_evidence_facts?.sanitized_count ?? 0,
|
|
179
180
|
raw_evidence_failure_reasons: [
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bli-cockpit/cli",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.108",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
},
|
|
29
29
|
"dependencies": {
|
|
30
30
|
"@bli-cockpit/memory-mcp": "0.1.27",
|
|
31
|
-
"@bli-cockpit/mcp": "0.1.
|
|
31
|
+
"@bli-cockpit/mcp": "0.1.36",
|
|
32
32
|
"@bli-cockpit/telemetry-core": "0.1.43"
|
|
33
33
|
}
|
|
34
34
|
}
|