@bli-cockpit/cli 0.2.46 → 0.2.48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/raw-evidence-git-diff.js +50 -0
- package/dist/adapters/raw-evidence-keys.js +4 -1
- package/dist/adapters/raw-evidence-pack-store.js +13 -4
- package/dist/adapters/raw-evidence.js +43 -1
- package/dist/autostart-node-path.js +141 -0
- package/dist/autostart-self-heal.js +115 -11
- package/dist/autostart.js +213 -41
- package/dist/commands/autostart-heal.js +162 -0
- package/dist/commands/collection-roots.js +4 -4
- package/dist/commands/doctor.js +47 -1
- package/dist/commands/heartbeat.js +193 -0
- package/dist/commands/install-receipts.js +45 -19
- package/dist/commands/install-update.js +3 -3
- package/dist/commands/local-args.js +7 -1
- package/dist/commands/local.js +116 -10
- package/dist/commands/ops-render.js +29 -0
- package/dist/commands/ops.js +6 -0
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/session-sync.js +146 -25
- package/dist/dev-build.js +186 -0
- package/dist/evidence-upload-client.js +112 -4
- package/dist/evidence-upload-rekey.js +40 -0
- package/dist/log-rotation.js +144 -0
- package/dist/onboarding-roots.js +23 -6
- package/dist/raw-evidence-gc.js +9 -23
- package/dist/scheduled-self-update.js +1 -0
- package/dist/second-install.js +160 -0
- package/dist/sync-health-class.js +242 -0
- package/dist/upload.js +2 -0
- package/package.json +3 -3
|
@@ -80,6 +80,8 @@ export function renderOpsStatus(payload, dim) {
|
|
|
80
80
|
lines.push(dim(` scheduled by ${row.configFile} (${row.cron ?? "?"})`));
|
|
81
81
|
}
|
|
82
82
|
}
|
|
83
|
+
if (payload.fleet)
|
|
84
|
+
lines.push(...renderFleet(payload.fleet, dim));
|
|
83
85
|
const slack = payload.skips?.slack;
|
|
84
86
|
const external = payload.skips?.external;
|
|
85
87
|
if (slack || external) {
|
|
@@ -93,6 +95,33 @@ export function renderOpsStatus(payload, dim) {
|
|
|
93
95
|
}
|
|
94
96
|
return lines;
|
|
95
97
|
}
|
|
98
|
+
/**
|
|
99
|
+
* The laptops. Red first, because a person reading this at 9am should not have
|
|
100
|
+
* to scroll past nine healthy machines to find the dead one.
|
|
101
|
+
*/
|
|
102
|
+
export function renderFleet(fleet, dim) {
|
|
103
|
+
const lines = ["", `FLEET ${fleet.summary ?? "(no summary)"}`];
|
|
104
|
+
if (fleet.readError) {
|
|
105
|
+
lines.push(` the fleet could not be read (${fleet.readError}); nothing is known about any machine`);
|
|
106
|
+
return lines;
|
|
107
|
+
}
|
|
108
|
+
const devices = fleet.devices ?? [];
|
|
109
|
+
const rank = (colour) => colour === "red" ? 0 : colour === "amber" ? 1 : 2;
|
|
110
|
+
const ordered = [...devices].sort((left, right) => rank(left.colour) - rank(right.colour));
|
|
111
|
+
if (ordered.length === 0) {
|
|
112
|
+
lines.push(dim(" no live collector device is registered at all"));
|
|
113
|
+
}
|
|
114
|
+
for (const device of ordered) {
|
|
115
|
+
const text = ` ${device.line ?? device.deviceId ?? "(device)"}`;
|
|
116
|
+
// A healthy machine is dimmed, never dropped: "which laptops are fine" is
|
|
117
|
+
// the other half of the question, and a list that only shows failures
|
|
118
|
+
// cannot answer "is everybody else collecting?".
|
|
119
|
+
lines.push(device.colour === "red" || device.colour === "amber" ? text : dim(text));
|
|
120
|
+
}
|
|
121
|
+
if (fleet.noDeviceLine)
|
|
122
|
+
lines.push(dim(` ${fleet.noDeviceLine}`));
|
|
123
|
+
return lines;
|
|
124
|
+
}
|
|
96
125
|
function renderSkipLedger(ledger, dim) {
|
|
97
126
|
const lines = [];
|
|
98
127
|
const name = ledger.relation ?? "skips";
|
package/dist/commands/ops.js
CHANGED
|
@@ -78,6 +78,12 @@ async function runOpsStatus(command, io, tower) {
|
|
|
78
78
|
unhealthy: unhealthy.length,
|
|
79
79
|
unhealthy_ids: unhealthy.map((row) => row.id ?? "?"),
|
|
80
80
|
with_skips: Boolean(command.skips),
|
|
81
|
+
// The laptops (BLI-3550), counted on the same line: a run that shows a
|
|
82
|
+
// green board and says nothing about the fleet cannot answer "was
|
|
83
|
+
// anybody's machine dead this morning?".
|
|
84
|
+
fleet_devices: payload.fleet?.counts?.devices ?? null,
|
|
85
|
+
fleet_red: payload.fleet?.counts?.red ?? null,
|
|
86
|
+
fleet_amber: payload.fleet?.counts?.amber ?? null,
|
|
81
87
|
})}`);
|
|
82
88
|
return unhealthy.length > 0 ? 1 : 0;
|
|
83
89
|
}
|
|
@@ -15,7 +15,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
15
15
|
}
|
|
16
16
|
|
|
17
17
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
18
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
18
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.48");
|
|
19
19
|
return 0;
|
|
20
20
|
}
|
|
21
21
|
|
|
@@ -143,17 +143,34 @@ function claudeAttributionReadFailureCount(scan) {
|
|
|
143
143
|
}
|
|
144
144
|
export function sourceScanRetryReason(source, scan) {
|
|
145
145
|
const reasons = new Set();
|
|
146
|
-
const
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
146
|
+
const failure = sourceScanFailureReason(source, scan);
|
|
147
|
+
if (failure)
|
|
148
|
+
reasons.add(failure);
|
|
149
|
+
// Kept HERE and nowhere else (BLI-3551): a repo that is not on disk is a
|
|
150
|
+
// reason to widen the next scan window, because the transcript fallback can
|
|
151
|
+
// still attribute it. It is not a reason to call this sync failed — see
|
|
152
|
+
// `sourceScanFailureReason`.
|
|
152
153
|
if (scan.results.some((result) => result.reason === "repo_not_on_disk")) {
|
|
153
154
|
reasons.add("repo_not_on_disk");
|
|
154
155
|
}
|
|
155
156
|
return reasons.size > 0 ? [...reasons].sort().join(",") : null;
|
|
156
157
|
}
|
|
158
|
+
/**
|
|
159
|
+
* The part of the scan outcome that is a genuine FAILURE: the session store
|
|
160
|
+
* itself could not be read, so sessions that exist were not seen.
|
|
161
|
+
*
|
|
162
|
+
* Split from {@link sourceScanRetryReason} in BLI-3551. The two used to be one
|
|
163
|
+
* function, so `repo_not_on_disk` — a label the attribution umbrella finding
|
|
164
|
+
* already established is not a defect (nothing was deleted; the transcript
|
|
165
|
+
* names a path git no longer tracks) — failed the sync on every tick for three
|
|
166
|
+
* operators. A retry hint and a failure are different claims.
|
|
167
|
+
*/
|
|
168
|
+
export function sourceScanFailureReason(source, scan) {
|
|
169
|
+
const readFailureCount = source === "codex"
|
|
170
|
+
? codexAttributionReadFailureCount(scan)
|
|
171
|
+
: claudeAttributionReadFailureCount(scan);
|
|
172
|
+
return readFailureCount > 0 ? `${source}_session_store_read_failed` : null;
|
|
173
|
+
}
|
|
157
174
|
async function reconcileSourceScanRetry(options) {
|
|
158
175
|
if (options.reason) {
|
|
159
176
|
await recordSourceRetryFailure(options.paths, {
|
|
@@ -447,25 +464,47 @@ export async function runAttributedWorktreeSync(options) {
|
|
|
447
464
|
});
|
|
448
465
|
// Same conditions as before, one per line, each writing down its own reason.
|
|
449
466
|
// The old version was a single boolean chain: correct, and completely mute.
|
|
450
|
-
|
|
451
|
-
|
|
467
|
+
//
|
|
468
|
+
// Since BLI-3551 each condition also writes down the LABEL it is classified
|
|
469
|
+
// by, beside the rendered string a person reads. The health receipt reads the
|
|
470
|
+
// label; nothing parses the sentence back apart.
|
|
471
|
+
const failureRecords = new Map();
|
|
472
|
+
const add = (record) => {
|
|
473
|
+
if (!failureRecords.has(record.rendered)) {
|
|
474
|
+
failureRecords.set(record.rendered, record);
|
|
475
|
+
}
|
|
476
|
+
};
|
|
477
|
+
const fail = (condition, label, rendered = label) => {
|
|
452
478
|
if (condition)
|
|
453
|
-
|
|
479
|
+
add({ label, rendered });
|
|
454
480
|
};
|
|
455
481
|
for (const { worktree, sync } of outcomes) {
|
|
456
482
|
if (sync.status !== "uploaded") {
|
|
457
483
|
// The spooled reason is the most specific thing anyone has, so lead with
|
|
458
484
|
// it and name the worktree it belongs to — a fleet failure is usually one
|
|
459
485
|
// repo, and "which one" is the first question asked.
|
|
460
|
-
|
|
461
|
-
?
|
|
462
|
-
|
|
486
|
+
add(sync.status === "spooled" && sync.failure_reason
|
|
487
|
+
? {
|
|
488
|
+
label: sync.failure_class,
|
|
489
|
+
rendered: `${worktree.worktree_label}:${sync.failure_reason}`,
|
|
490
|
+
http_status: sync.failure_http_status,
|
|
491
|
+
}
|
|
492
|
+
: {
|
|
493
|
+
label: "upload_not_completed",
|
|
494
|
+
rendered: `${worktree.worktree_label}:upload_${sync.status}`,
|
|
495
|
+
});
|
|
463
496
|
}
|
|
464
497
|
for (const reason of sync.raw_evidence_failure_reasons ?? []) {
|
|
465
|
-
|
|
498
|
+
add({
|
|
499
|
+
label: "raw_evidence_upload_failed",
|
|
500
|
+
rendered: `raw_evidence:${reason}`,
|
|
501
|
+
});
|
|
466
502
|
}
|
|
467
503
|
for (const reason of sync.raw_evidence_retry_reasons ?? []) {
|
|
468
|
-
|
|
504
|
+
add({
|
|
505
|
+
label: "raw_evidence_retry_required",
|
|
506
|
+
rendered: `raw_evidence_retry:${reason}`,
|
|
507
|
+
});
|
|
469
508
|
}
|
|
470
509
|
fail(sync.raw_evidence_deferred_byte_budget > 0, "deferred_byte_budget");
|
|
471
510
|
fail(sync.raw_evidence_deferred_object_budget > 0, "deferred_object_budget");
|
|
@@ -474,28 +513,110 @@ export async function runAttributedWorktreeSync(options) {
|
|
|
474
513
|
fail(claudeAttribution.session_limit_applied, "claude_session_limit_applied");
|
|
475
514
|
fail(codexAttributionReadFailureCount(codexAttribution) > 0, "codex_session_read_failed");
|
|
476
515
|
fail(claudeAttributionReadFailureCount(claudeAttribution) > 0, "claude_session_read_failed");
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
516
|
+
// BLI-3551: the scan's RETRY reason and the scan's FAILURE reason are two
|
|
517
|
+
// different questions, and answering both with one function is what put
|
|
518
|
+
// `claude_scan:repo_not_on_disk` on every tick of three machines. A repo that
|
|
519
|
+
// is not on disk is a label on the session (the attribution umbrella finding:
|
|
520
|
+
// nothing was deleted, the transcript simply names a path git no longer
|
|
521
|
+
// knows). It still widens the next scan window; it is not a failed sync.
|
|
522
|
+
const codexScanFailure = sourceScanFailureReason("codex", codexAttribution);
|
|
523
|
+
if (codexScanFailure) {
|
|
524
|
+
add({ label: "codex_scan_read_failed", rendered: `codex_scan:${codexScanFailure}` });
|
|
525
|
+
}
|
|
526
|
+
const claudeScanFailure = sourceScanFailureReason("claude_code", claudeAttribution);
|
|
527
|
+
if (claudeScanFailure) {
|
|
528
|
+
add({
|
|
529
|
+
label: "claude_scan_read_failed",
|
|
530
|
+
rendered: `claude_scan:${claudeScanFailure}`,
|
|
531
|
+
});
|
|
532
|
+
}
|
|
533
|
+
// BLI-3551: a tick that observed only sessions from outside the operator's
|
|
534
|
+
// approved roots has nothing to post, and that is the consent boundary
|
|
535
|
+
// working — not a failure. It used to fail as
|
|
536
|
+
// `session_report_unposted:no_successful_sync`, whose word "session" then
|
|
537
|
+
// classified as `auth_failed`; one machine reported a broken credential 377
|
|
538
|
+
// times in 38 hours while its token had eleven weeks left. The withhold
|
|
539
|
+
// decision itself is untouched (adapters/attribution-core.ts) — only what it
|
|
540
|
+
// is CALLED.
|
|
541
|
+
const sessionsOutsideRoot = sessions.filter((session) => OUTSIDE_APPROVED_ROOT_REASONS.has(session.attribution_reason)).length;
|
|
542
|
+
const nothingInRoot = nothingInRootCount({
|
|
543
|
+
sessionCount: sessions.length,
|
|
544
|
+
outsideRootCount: sessionsOutsideRoot,
|
|
545
|
+
outcomes,
|
|
546
|
+
reportPosted: report.posted,
|
|
547
|
+
reportReason: report.reason,
|
|
548
|
+
});
|
|
549
|
+
fail(reportRequired && !report.posted && nothingInRoot === null, "session_report_unposted", `session_report_unposted:${report.reason ?? "unknown"}`);
|
|
484
550
|
// `ok` may already be false from the per-worktree loop above; the outcome
|
|
485
551
|
// scan re-derives that, so the two agree by construction.
|
|
486
|
-
ok = ok &&
|
|
487
|
-
if (!ok &&
|
|
488
|
-
|
|
552
|
+
ok = ok && failureRecords.size === 0;
|
|
553
|
+
if (!ok && failureRecords.size === 0) {
|
|
554
|
+
add({
|
|
555
|
+
label: SYNC_FAILED_WITHOUT_REASON,
|
|
556
|
+
rendered: SYNC_FAILED_WITHOUT_REASON,
|
|
557
|
+
});
|
|
558
|
+
}
|
|
559
|
+
const notice = ok && nothingInRoot !== null ? `nothing_in_root:${nothingInRoot}` : null;
|
|
560
|
+
if (notice) {
|
|
561
|
+
// The success branch says something too: this is the receipt that proves a
|
|
562
|
+
// quiet machine is a working machine, and the count is what tells a coach
|
|
563
|
+
// that someone is working entirely outside the approved boundary.
|
|
564
|
+
console.error("[session-sync] nothing to collect inside the approved roots", JSON.stringify({
|
|
565
|
+
reason: "nothing_in_root",
|
|
566
|
+
sessions_outside_root: nothingInRoot,
|
|
567
|
+
collection_root_count: collectionRoots.length,
|
|
568
|
+
next_action: "widen the approved roots (an operator decision) if this machine should be collecting here",
|
|
569
|
+
}));
|
|
489
570
|
}
|
|
571
|
+
const records = [...failureRecords.values()].sort((a, b) => a.rendered.localeCompare(b.rendered));
|
|
490
572
|
return {
|
|
491
573
|
ok,
|
|
492
|
-
failure_reasons:
|
|
574
|
+
failure_reasons: records.map((record) => record.rendered),
|
|
575
|
+
failure_records: records,
|
|
576
|
+
notice,
|
|
577
|
+
sessions_observed: sessions.length,
|
|
578
|
+
sessions_outside_root: sessionsOutsideRoot,
|
|
493
579
|
outcomes,
|
|
494
580
|
codexAttribution,
|
|
495
581
|
claudeAttribution,
|
|
496
582
|
summary,
|
|
497
583
|
};
|
|
498
584
|
}
|
|
585
|
+
/**
|
|
586
|
+
* Reasons attribution gives when a session's working directory is not inside
|
|
587
|
+
* any approved collection root.
|
|
588
|
+
*
|
|
589
|
+
* Exact labels, not a pattern — the same discipline the classifier now follows.
|
|
590
|
+
* `attribution-core.ts` writes both of these and nothing else means
|
|
591
|
+
* "outside the boundary".
|
|
592
|
+
*/
|
|
593
|
+
const OUTSIDE_APPROVED_ROOT_REASONS = new Set([
|
|
594
|
+
"cwd_outside_scanned_worktrees",
|
|
595
|
+
"no_matching_worktree_signals",
|
|
596
|
+
]);
|
|
597
|
+
/**
|
|
598
|
+
* How many observed sessions were outside the approved roots, when that
|
|
599
|
+
* accounts for ALL of them and nothing else went wrong — otherwise `null`.
|
|
600
|
+
*
|
|
601
|
+
* Deliberately narrow. It requires that no worktree was synced at all (so no
|
|
602
|
+
* upload could have succeeded or failed), that every session observed this tick
|
|
603
|
+
* names an outside-the-root reason, and that the unposted report is the
|
|
604
|
+
* `no_successful_sync` shape rather than a spooled report that failed to flush.
|
|
605
|
+
* Anything else keeps its failure.
|
|
606
|
+
*/
|
|
607
|
+
export function nothingInRootCount(options) {
|
|
608
|
+
if (options.reportPosted)
|
|
609
|
+
return null;
|
|
610
|
+
if (options.reportReason !== "no_successful_sync")
|
|
611
|
+
return null;
|
|
612
|
+
if (options.outcomes.length > 0)
|
|
613
|
+
return null;
|
|
614
|
+
if (options.sessionCount === 0)
|
|
615
|
+
return null;
|
|
616
|
+
return options.outsideRootCount === options.sessionCount
|
|
617
|
+
? options.outsideRootCount
|
|
618
|
+
: null;
|
|
619
|
+
}
|
|
499
620
|
export const ATTRIBUTION_STATE_RANK = CODEX_SESSION_ATTRIBUTION_STATE_RANK;
|
|
500
621
|
function normalizeCodexResult(result) {
|
|
501
622
|
return {
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Is this collector a DEV BUILD? One predicate, two callers.
|
|
3
|
+
*
|
|
4
|
+
* BLI-3554. Edward's device row carried 47 `npm_install_eacces` receipts
|
|
5
|
+
* stamped `cli_version` 0.1.50 between 2026-09-02T16:55Z and 2026-09-03T06:25Z.
|
|
6
|
+
* 0.1.50 was never a published install on that machine — it was the workspace
|
|
7
|
+
* version in `packages/cockpit-local-collector/package.json` at the time, so
|
|
8
|
+
* every one of those rows came from an agent or a test running the built
|
|
9
|
+
* collector inside a worktree. They land in the same production
|
|
10
|
+
* `cockpit_install_events` table the fleet reader (BLI-3550,
|
|
11
|
+
* `apps/dashboard/src/lib/ops/fleet-liveness.ts`) alarms on, so a developer's
|
|
12
|
+
* sandbox failure reads as a fleet machine in trouble.
|
|
13
|
+
*
|
|
14
|
+
* The predicate is deliberately NOT a version comparison. A version equal to
|
|
15
|
+
* the workspace version is a symptom: it changes on every release, it is equal
|
|
16
|
+
* for the one machine that legitimately runs the newest published build the
|
|
17
|
+
* day it ships, and it says nothing about where the code was loaded from.
|
|
18
|
+
* What we can actually observe is the FILE the running code was loaded from:
|
|
19
|
+
*
|
|
20
|
+
* - a published install lives under a `node_modules/` prefix
|
|
21
|
+
* (`/opt/homebrew/lib/node_modules/@bli-cockpit/cli/dist/…`,
|
|
22
|
+
* `C:\\Users\\x\\AppData\\Roaming\\npm\\node_modules\\@bli-cockpit\\cli\\dist\\…`),
|
|
23
|
+
* and that is the whole fleet;
|
|
24
|
+
* - a dev build lives inside a checkout of this repo — a
|
|
25
|
+
* `packages/cockpit-local-collector/` (or `packages/cockpit-cli/`) ancestor
|
|
26
|
+
* whose `package.json` names the workspace, with a `.git` marker or a
|
|
27
|
+
* `.claude/worktrees` segment above it.
|
|
28
|
+
*
|
|
29
|
+
* `COCKPIT_DEV=1` forces dev. `COCKPIT_DEV=0` forces the opposite — that is the
|
|
30
|
+
* escape hatch for a deliberate live test from a checkout, documented in
|
|
31
|
+
* `docs/runbooks/cockpit-collector-receipts.md`.
|
|
32
|
+
*
|
|
33
|
+
* Suppression only bites when the target is a PRODUCTION dashboard. Pointing a
|
|
34
|
+
* checkout at `http://localhost:3000` is how the receipt path itself is
|
|
35
|
+
* developed, and refusing to post there would make this predicate the reason
|
|
36
|
+
* the next receipt bug cannot be reproduced.
|
|
37
|
+
*/
|
|
38
|
+
import fs from "node:fs";
|
|
39
|
+
import path from "node:path";
|
|
40
|
+
import { fileURLToPath } from "node:url";
|
|
41
|
+
/** The workspaces whose `dist/` a developer or agent runs from a checkout. */
|
|
42
|
+
const WORKSPACE_PACKAGE_NAMES = new Set([
|
|
43
|
+
"@bli-cockpit/local-collector",
|
|
44
|
+
"@bli-cockpit/cli",
|
|
45
|
+
]);
|
|
46
|
+
const TRUE_VALUES = new Set(["1", "true", "yes", "on"]);
|
|
47
|
+
const FALSE_VALUES = new Set(["0", "false", "no", "off"]);
|
|
48
|
+
/**
|
|
49
|
+
* Walks up from the running module. Pure apart from the two injected readers,
|
|
50
|
+
* so a Windows layout can be asserted from a macOS test run.
|
|
51
|
+
*/
|
|
52
|
+
export function detectDevBuild(probe = {}) {
|
|
53
|
+
const env = probe.env ?? process.env;
|
|
54
|
+
const rawFlag = (env.COCKPIT_DEV ?? "").trim().toLowerCase();
|
|
55
|
+
if (TRUE_VALUES.has(rawFlag)) {
|
|
56
|
+
return { devBuild: true, reason: "cockpit_dev_env" };
|
|
57
|
+
}
|
|
58
|
+
if (FALSE_VALUES.has(rawFlag)) {
|
|
59
|
+
// Deliberate live test from a checkout: the operator has said so out loud,
|
|
60
|
+
// and the path evidence below is not allowed to overrule them.
|
|
61
|
+
return { devBuild: false, reason: "cockpit_dev_env_disabled" };
|
|
62
|
+
}
|
|
63
|
+
const pathApi = probe.pathApi ?? path;
|
|
64
|
+
const modulePath = probe.modulePath ?? currentModulePath();
|
|
65
|
+
if (!modulePath)
|
|
66
|
+
return { devBuild: false, reason: "not_a_checkout" };
|
|
67
|
+
if (hasNodeModulesSegment(modulePath)) {
|
|
68
|
+
// Every published install is under a `node_modules/` prefix, on both host
|
|
69
|
+
// families. This branch is the whole fleet, and it is checked first so a
|
|
70
|
+
// machine that happens to have a repo checkout somewhere above its global
|
|
71
|
+
// npm prefix cannot be mislabelled.
|
|
72
|
+
return { devBuild: false, reason: "published_install" };
|
|
73
|
+
}
|
|
74
|
+
const readPackageName = probe.readPackageName ?? defaultReadPackageName;
|
|
75
|
+
const pathExists = probe.pathExists ?? defaultPathExists;
|
|
76
|
+
const packageRoot = findWorkspacePackageRoot(pathApi.dirname(modulePath), pathApi, readPackageName);
|
|
77
|
+
if (!packageRoot)
|
|
78
|
+
return { devBuild: false, reason: "not_a_checkout" };
|
|
79
|
+
if (!hasCheckoutMarkerAbove(packageRoot, pathApi, pathExists)) {
|
|
80
|
+
return { devBuild: false, reason: "not_a_checkout" };
|
|
81
|
+
}
|
|
82
|
+
return { devBuild: true, reason: "workspace_checkout" };
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Should this process withhold fleet receipts (install events, heartbeat)?
|
|
86
|
+
*
|
|
87
|
+
* Two facts, in this order: is the code a dev build, and is the target the
|
|
88
|
+
* production fleet. A dev build talking to `localhost` still posts, because
|
|
89
|
+
* that is the only way to exercise the receipt path at all.
|
|
90
|
+
*/
|
|
91
|
+
export function shouldSuppressFleetReceipts(options) {
|
|
92
|
+
const verdict = detectDevBuild(options.probe ?? {});
|
|
93
|
+
if (!verdict.devBuild)
|
|
94
|
+
return { suppressed: false, reason: verdict.reason };
|
|
95
|
+
if (isLocalDashboardUrl(options.dashboardUrl)) {
|
|
96
|
+
return { suppressed: false, reason: "local_dashboard" };
|
|
97
|
+
}
|
|
98
|
+
return { suppressed: true, reason: verdict.reason };
|
|
99
|
+
}
|
|
100
|
+
/** A dashboard on this machine — safe for a checkout to post at. */
|
|
101
|
+
export function isLocalDashboardUrl(dashboardUrl) {
|
|
102
|
+
let hostname;
|
|
103
|
+
try {
|
|
104
|
+
hostname = new URL(dashboardUrl).hostname.toLowerCase();
|
|
105
|
+
}
|
|
106
|
+
catch {
|
|
107
|
+
// An unparseable URL is not demonstrably local, and this predicate only
|
|
108
|
+
// ever widens suppression when it says false.
|
|
109
|
+
return false;
|
|
110
|
+
}
|
|
111
|
+
const bare = hostname.replace(/^\[|\]$/gu, "");
|
|
112
|
+
return (bare === "localhost" ||
|
|
113
|
+
bare === "127.0.0.1" ||
|
|
114
|
+
bare === "::1" ||
|
|
115
|
+
bare === "0.0.0.0" ||
|
|
116
|
+
bare.endsWith(".localhost"));
|
|
117
|
+
}
|
|
118
|
+
function currentModulePath() {
|
|
119
|
+
try {
|
|
120
|
+
return fileURLToPath(import.meta.url);
|
|
121
|
+
}
|
|
122
|
+
catch {
|
|
123
|
+
return null;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Case-insensitive on purpose: Windows spells the same folder
|
|
128
|
+
* `Node_Modules` without complaint, and both separators appear on that host
|
|
129
|
+
* because a POSIX-style path survives most Node APIs there.
|
|
130
|
+
*/
|
|
131
|
+
function hasNodeModulesSegment(modulePath) {
|
|
132
|
+
return modulePath
|
|
133
|
+
.split(/[\\/]+/u)
|
|
134
|
+
.some((segment) => segment.toLowerCase() === "node_modules");
|
|
135
|
+
}
|
|
136
|
+
function findWorkspacePackageRoot(startDir, pathApi, readPackageName) {
|
|
137
|
+
let dir = startDir;
|
|
138
|
+
// Bounded: `path.dirname` reaches a fixed point at `/` or `C:\`, and the
|
|
139
|
+
// depth guard keeps a pathological symlink loop from spinning a sync walk.
|
|
140
|
+
for (let depth = 0; depth < 64; depth += 1) {
|
|
141
|
+
const name = readPackageName(pathApi.join(dir, "package.json"));
|
|
142
|
+
if (name && WORKSPACE_PACKAGE_NAMES.has(name))
|
|
143
|
+
return dir;
|
|
144
|
+
const parent = pathApi.dirname(dir);
|
|
145
|
+
if (parent === dir)
|
|
146
|
+
return null;
|
|
147
|
+
dir = parent;
|
|
148
|
+
}
|
|
149
|
+
return null;
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* A repo checkout marker at or above the workspace package: `.git` (a
|
|
153
|
+
* directory in a normal clone, a FILE in a `git worktree`), or the
|
|
154
|
+
* `.claude/worktrees` staging area agents run from.
|
|
155
|
+
*/
|
|
156
|
+
function hasCheckoutMarkerAbove(packageRoot, pathApi, pathExists) {
|
|
157
|
+
let dir = packageRoot;
|
|
158
|
+
for (let depth = 0; depth < 64; depth += 1) {
|
|
159
|
+
if (pathExists(pathApi.join(dir, ".git")))
|
|
160
|
+
return true;
|
|
161
|
+
if (pathExists(pathApi.join(dir, ".claude", "worktrees")))
|
|
162
|
+
return true;
|
|
163
|
+
const parent = pathApi.dirname(dir);
|
|
164
|
+
if (parent === dir)
|
|
165
|
+
return false;
|
|
166
|
+
dir = parent;
|
|
167
|
+
}
|
|
168
|
+
return false;
|
|
169
|
+
}
|
|
170
|
+
function defaultReadPackageName(packageJsonPath) {
|
|
171
|
+
try {
|
|
172
|
+
const parsed = JSON.parse(fs.readFileSync(packageJsonPath, "utf8"));
|
|
173
|
+
return typeof parsed.name === "string" ? parsed.name : null;
|
|
174
|
+
}
|
|
175
|
+
catch {
|
|
176
|
+
return null;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
function defaultPathExists(candidate) {
|
|
180
|
+
try {
|
|
181
|
+
return fs.existsSync(candidate);
|
|
182
|
+
}
|
|
183
|
+
catch {
|
|
184
|
+
return false;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { COMMIT_CRASHED_PLATFORM, RAW_EVIDENCE_UPLOAD_DEFAULT_CHUNK_BYTES, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, RAW_EVIDENCE_UPLOAD_MAX_OBJECTS_PER_BEGIN, RawEvidenceLegacyUploadResponseSchema, RawEvidenceRedactionMetadataSchema, RawEvidenceUploadBeginResponseSchema, RawEvidenceUploadCommitResponseSchema, isPermanentUploadFailure, } from "@bli-cockpit/telemetry-core";
|
|
1
|
+
import { COMMIT_CRASHED_PLATFORM, RAW_EVIDENCE_UPLOAD_DEFAULT_CHUNK_BYTES, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, RAW_EVIDENCE_UPLOAD_MAX_OBJECTS_PER_BEGIN, RawEvidenceLegacyUploadResponseSchema, RawEvidenceRedactionMetadataSchema, RawEvidenceUploadBeginResponseSchema, RawEvidenceUploadCommitResponseSchema, isPermanentUploadFailure, isRekeyableUploadConflict, } from "@bli-cockpit/telemetry-core";
|
|
2
2
|
import crypto from "node:crypto";
|
|
3
3
|
import fs from "node:fs/promises";
|
|
4
|
+
import { rekeyedEvidencePointer } from "./evidence-upload-rekey.js";
|
|
4
5
|
import { describeError } from "./health-detail.js";
|
|
5
6
|
const DEFAULT_MAX_ATTEMPTS = 3;
|
|
6
7
|
const RETRY_DELAY_MS = 250;
|
|
@@ -182,10 +183,18 @@ export async function uploadRawEvidenceFilesChunked(options) {
|
|
|
182
183
|
resolvedEntries.add(entry);
|
|
183
184
|
continue;
|
|
184
185
|
}
|
|
185
|
-
|
|
186
|
-
|
|
186
|
+
// A conflict a different name can settle gets one, here, before the
|
|
187
|
+
// upload is attempted (BLI-3552). Anything else comes back unchanged and
|
|
188
|
+
// fails on its own reason inside `uploadOneObject`.
|
|
189
|
+
const resolved = await rekeyConflictedEntry(options, entry, disposition, chunkSizeBytes);
|
|
190
|
+
const outcome = await uploadOneObject(options, resolved.entry, resolved.disposition, chunkSizeBytes);
|
|
191
|
+
await reportAbandonedUpload(options, resolved.disposition, outcome);
|
|
187
192
|
outcomes.push(outcome);
|
|
188
|
-
|
|
193
|
+
// The duplicates of a re-keyed primary were re-keyed with it: a duplicate
|
|
194
|
+
// is byte-identical and shared the primary's key, so it shares the new
|
|
195
|
+
// one too. Leaving them on the old key would point their evidence refs at
|
|
196
|
+
// somebody else's durable object.
|
|
197
|
+
for (const duplicate of resolved.entry.duplicates) {
|
|
189
198
|
outcomes.push(duplicateOutcome(outcome, duplicate));
|
|
190
199
|
}
|
|
191
200
|
resolvedEntries.add(entry);
|
|
@@ -281,6 +290,105 @@ function conformReasonLabel(reason) {
|
|
|
281
290
|
const conformed = reason.replace(/[^a-z0-9_:.-]/gi, "_").slice(0, 120);
|
|
282
291
|
return conformed.length > 0 ? conformed : "upload_failed_unlabelled";
|
|
283
292
|
}
|
|
293
|
+
/**
|
|
294
|
+
* Settle a re-keyable conflict by asking for a second name (BLI-3552).
|
|
295
|
+
*
|
|
296
|
+
* `hash_mismatch_committed_object` means the key already holds different bytes
|
|
297
|
+
* that are already durable. Re-sending is the loop; deleting the durable object
|
|
298
|
+
* to make room is destroying evidence. The third answer is to offer this
|
|
299
|
+
* content under a name derived from its own hash, which is what this does, once
|
|
300
|
+
* per object per sync.
|
|
301
|
+
*
|
|
302
|
+
* Every path that cannot get there returns the ORIGINAL pair, so the object
|
|
303
|
+
* fails on the conflict reason `begin` actually gave rather than on a label
|
|
304
|
+
* invented here. Nothing is written off permanently either way: the upload
|
|
305
|
+
* cursor only remembers successes, so the next eligible sync offers the bytes
|
|
306
|
+
* again — at delivery-backoff cadence now instead of every 15 minutes.
|
|
307
|
+
*/
|
|
308
|
+
async function rekeyConflictedEntry(options, entry, disposition, chunkSizeBytes) {
|
|
309
|
+
if (disposition.disposition !== "conflict" ||
|
|
310
|
+
!isRekeyableUploadConflict(disposition.reason)) {
|
|
311
|
+
return { entry, disposition };
|
|
312
|
+
}
|
|
313
|
+
const contentHashPrefix = (entry.file.pointer.content_hash_sha256 ?? "none").slice(0, 12);
|
|
314
|
+
const rekeyedPointer = rekeyedEvidencePointer(entry.file.pointer);
|
|
315
|
+
if (!rekeyedPointer) {
|
|
316
|
+
// The key already names this content and the server still says it holds
|
|
317
|
+
// something else. A rename cannot answer that; a person has to.
|
|
318
|
+
console.error("[evidence-rekey] the key already carries this content hash; leaving the conflict for a person", JSON.stringify({
|
|
319
|
+
reason: disposition.reason,
|
|
320
|
+
kind: entry.file.kind ?? "unknown",
|
|
321
|
+
content_hash_prefix: contentHashPrefix,
|
|
322
|
+
byte_size: entry.bytes.byteLength,
|
|
323
|
+
}));
|
|
324
|
+
return { entry, disposition };
|
|
325
|
+
}
|
|
326
|
+
const rekeyedEntry = {
|
|
327
|
+
...entry,
|
|
328
|
+
file: { ...entry.file, pointer: rekeyedPointer },
|
|
329
|
+
duplicates: entry.duplicates.map((duplicate) => ({
|
|
330
|
+
...duplicate,
|
|
331
|
+
pointer: rekeyedEvidencePointer(duplicate.pointer) ?? duplicate.pointer,
|
|
332
|
+
})),
|
|
333
|
+
};
|
|
334
|
+
const rekeyedDisposition = await beginOneObject(options, rekeyedEntry, chunkSizeBytes);
|
|
335
|
+
if (!rekeyedDisposition) {
|
|
336
|
+
return { entry, disposition };
|
|
337
|
+
}
|
|
338
|
+
console.error("[evidence-rekey] committed object holds other content; offering these bytes under their own hash", JSON.stringify({
|
|
339
|
+
reason: disposition.reason,
|
|
340
|
+
kind: entry.file.kind ?? "unknown",
|
|
341
|
+
content_hash_prefix: contentHashPrefix,
|
|
342
|
+
byte_size: entry.bytes.byteLength,
|
|
343
|
+
chunk_count: entry.chunkCount,
|
|
344
|
+
rekeyed_disposition: rekeyedDisposition.disposition,
|
|
345
|
+
rekeyed_reason: rekeyedDisposition.reason ?? "none",
|
|
346
|
+
duplicate_count: entry.duplicates.length,
|
|
347
|
+
}));
|
|
348
|
+
return { entry: rekeyedEntry, disposition: rekeyedDisposition };
|
|
349
|
+
}
|
|
350
|
+
/**
|
|
351
|
+
* `begin` for a single object. Null whenever the answer cannot be trusted —
|
|
352
|
+
* transport failure, an unparseable body, a duplicate or missing key, or a
|
|
353
|
+
* pointer id that is not the one we sent — and the caller then keeps the
|
|
354
|
+
* original conflict rather than acting on a guess.
|
|
355
|
+
*/
|
|
356
|
+
async function beginOneObject(options, entry, chunkSizeBytes) {
|
|
357
|
+
const objectKey = entry.file.pointer.object_key ?? "";
|
|
358
|
+
const response = await requestJson(options, "/api/ambient/evidence/upload/begin", {
|
|
359
|
+
schema_version: "ambient-raw-evidence-upload-begin.v1",
|
|
360
|
+
generated_at: options.generatedAt,
|
|
361
|
+
provenance: options.provenance,
|
|
362
|
+
objects: [
|
|
363
|
+
{
|
|
364
|
+
pointer: entry.file.pointer,
|
|
365
|
+
chunk_size_bytes: chunkSizeBytes,
|
|
366
|
+
chunk_count: entry.chunkCount,
|
|
367
|
+
},
|
|
368
|
+
],
|
|
369
|
+
});
|
|
370
|
+
if (!response.ok) {
|
|
371
|
+
console.error("[evidence-rekey] begin refused the re-keyed object; keeping the original conflict", JSON.stringify({
|
|
372
|
+
reason: "rekey_begin_rejected",
|
|
373
|
+
http_status: response.status,
|
|
374
|
+
server_reason: safeFailureDetail(response.body) ?? "none",
|
|
375
|
+
kind: entry.file.kind ?? "unknown",
|
|
376
|
+
}));
|
|
377
|
+
return null;
|
|
378
|
+
}
|
|
379
|
+
const parsed = RawEvidenceUploadBeginResponseSchema.safeParse(response.body);
|
|
380
|
+
if (!parsed.success)
|
|
381
|
+
return null;
|
|
382
|
+
const dispositions = readBeginDispositions(parsed.data);
|
|
383
|
+
const disposition = dispositions?.get(objectKey);
|
|
384
|
+
if (!disposition)
|
|
385
|
+
return null;
|
|
386
|
+
if (disposition.raw_evidence_pointer_id !==
|
|
387
|
+
entry.file.pointer.raw_evidence_pointer_id) {
|
|
388
|
+
return null;
|
|
389
|
+
}
|
|
390
|
+
return disposition;
|
|
391
|
+
}
|
|
284
392
|
async function uploadOneObject(options, entry, disposition, chunkSizeBytes) {
|
|
285
393
|
const objectKey = entry.file.pointer.object_key ?? "";
|
|
286
394
|
if (disposition.disposition === "already_committed") {
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/** How much of the content hash goes into the name, matching the pack keys. */
|
|
2
|
+
const KEY_HASH_PREFIX_LENGTH = 16;
|
|
3
|
+
/**
|
|
4
|
+
* The same object under a name that carries its own content hash.
|
|
5
|
+
*
|
|
6
|
+
* `null` when a re-key cannot help and must not be attempted:
|
|
7
|
+
* - there is no object key to rewrite;
|
|
8
|
+
* - the last segment already starts with this content's hash prefix, so the
|
|
9
|
+
* re-keyed name would be the identical string and the second `begin` would
|
|
10
|
+
* answer the identical conflict;
|
|
11
|
+
* - the key has no `/`, which the server's namespace rules make impossible, so
|
|
12
|
+
* rewriting it would be inventing a key rather than deriving one.
|
|
13
|
+
*
|
|
14
|
+
* The new segment is `<hash16>-<old segment>`: still inside the operator's
|
|
15
|
+
* readable namespace (so `evidenceObjectKeyBelongsToWorkContext` still passes),
|
|
16
|
+
* still inside the key charset, and still legible to a person doing an incident
|
|
17
|
+
* walk — they can see which pack file it came from.
|
|
18
|
+
*/
|
|
19
|
+
export function rekeyedEvidencePointer(pointer) {
|
|
20
|
+
const objectKey = pointer.object_key;
|
|
21
|
+
if (!objectKey)
|
|
22
|
+
return null;
|
|
23
|
+
const cut = objectKey.lastIndexOf("/");
|
|
24
|
+
if (cut <= 0 || cut === objectKey.length - 1)
|
|
25
|
+
return null;
|
|
26
|
+
const prefix = objectKey.slice(0, cut);
|
|
27
|
+
const segment = objectKey.slice(cut + 1);
|
|
28
|
+
// A pointer with no content hash cannot name itself, and the upload routes
|
|
29
|
+
// reject it anyway; there is nothing to re-key it to.
|
|
30
|
+
const hash = (pointer.content_hash_sha256 ?? "").slice(0, KEY_HASH_PREFIX_LENGTH);
|
|
31
|
+
if (!hash || segment.startsWith(`${hash}-`) || segment.startsWith(hash)) {
|
|
32
|
+
return null;
|
|
33
|
+
}
|
|
34
|
+
const rekeyed = `${prefix}/${hash}-${segment}`;
|
|
35
|
+
return {
|
|
36
|
+
...pointer,
|
|
37
|
+
object_key: rekeyed,
|
|
38
|
+
raw_evidence_pointer_id: rekeyed,
|
|
39
|
+
};
|
|
40
|
+
}
|