@bli-cockpit/cli 0.2.46 → 0.2.48

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -80,6 +80,8 @@ export function renderOpsStatus(payload, dim) {
80
80
  lines.push(dim(` scheduled by ${row.configFile} (${row.cron ?? "?"})`));
81
81
  }
82
82
  }
83
+ if (payload.fleet)
84
+ lines.push(...renderFleet(payload.fleet, dim));
83
85
  const slack = payload.skips?.slack;
84
86
  const external = payload.skips?.external;
85
87
  if (slack || external) {
@@ -93,6 +95,33 @@ export function renderOpsStatus(payload, dim) {
93
95
  }
94
96
  return lines;
95
97
  }
98
+ /**
99
+ * The laptops. Red first, because a person reading this at 9am should not have
100
+ * to scroll past nine healthy machines to find the dead one.
101
+ */
102
+ export function renderFleet(fleet, dim) {
103
+ const lines = ["", `FLEET ${fleet.summary ?? "(no summary)"}`];
104
+ if (fleet.readError) {
105
+ lines.push(` the fleet could not be read (${fleet.readError}); nothing is known about any machine`);
106
+ return lines;
107
+ }
108
+ const devices = fleet.devices ?? [];
109
+ const rank = (colour) => colour === "red" ? 0 : colour === "amber" ? 1 : 2;
110
+ const ordered = [...devices].sort((left, right) => rank(left.colour) - rank(right.colour));
111
+ if (ordered.length === 0) {
112
+ lines.push(dim(" no live collector device is registered at all"));
113
+ }
114
+ for (const device of ordered) {
115
+ const text = ` ${device.line ?? device.deviceId ?? "(device)"}`;
116
+ // A healthy machine is dimmed, never dropped: "which laptops are fine" is
117
+ // the other half of the question, and a list that only shows failures
118
+ // cannot answer "is everybody else collecting?".
119
+ lines.push(device.colour === "red" || device.colour === "amber" ? text : dim(text));
120
+ }
121
+ if (fleet.noDeviceLine)
122
+ lines.push(dim(` ${fleet.noDeviceLine}`));
123
+ return lines;
124
+ }
96
125
  function renderSkipLedger(ledger, dim) {
97
126
  const lines = [];
98
127
  const name = ledger.relation ?? "skips";
@@ -78,6 +78,12 @@ async function runOpsStatus(command, io, tower) {
78
78
  unhealthy: unhealthy.length,
79
79
  unhealthy_ids: unhealthy.map((row) => row.id ?? "?"),
80
80
  with_skips: Boolean(command.skips),
81
+ // The laptops (BLI-3550), counted on the same line: a run that shows a
82
+ // green board and says nothing about the fleet cannot answer "was
83
+ // anybody's machine dead this morning?".
84
+ fleet_devices: payload.fleet?.counts?.devices ?? null,
85
+ fleet_red: payload.fleet?.counts?.red ?? null,
86
+ fleet_amber: payload.fleet?.counts?.amber ?? null,
81
87
  })}`);
82
88
  return unhealthy.length > 0 ? 1 : 0;
83
89
  }
@@ -15,7 +15,7 @@ export async function runCockpitCli(argv, io) {
15
15
  }
16
16
 
17
17
  if (command === "--version" || command === "-V" || command === "version") {
18
- writeLine(io?.stdout ?? process.stdout, "0.2.46");
18
+ writeLine(io?.stdout ?? process.stdout, "0.2.48");
19
19
  return 0;
20
20
  }
21
21
 
@@ -143,17 +143,34 @@ function claudeAttributionReadFailureCount(scan) {
143
143
  }
144
144
  export function sourceScanRetryReason(source, scan) {
145
145
  const reasons = new Set();
146
- const readFailureCount = source === "codex"
147
- ? codexAttributionReadFailureCount(scan)
148
- : claudeAttributionReadFailureCount(scan);
149
- if (readFailureCount > 0) {
150
- reasons.add(`${source}_session_store_read_failed`);
151
- }
146
+ const failure = sourceScanFailureReason(source, scan);
147
+ if (failure)
148
+ reasons.add(failure);
149
+ // Kept HERE and nowhere else (BLI-3551): a repo that is not on disk is a
150
+ // reason to widen the next scan window, because the transcript fallback can
151
+ // still attribute it. It is not a reason to call this sync failed — see
152
+ // `sourceScanFailureReason`.
152
153
  if (scan.results.some((result) => result.reason === "repo_not_on_disk")) {
153
154
  reasons.add("repo_not_on_disk");
154
155
  }
155
156
  return reasons.size > 0 ? [...reasons].sort().join(",") : null;
156
157
  }
158
+ /**
159
+ * The part of the scan outcome that is a genuine FAILURE: the session store
160
+ * itself could not be read, so sessions that exist were not seen.
161
+ *
162
+ * Split from {@link sourceScanRetryReason} in BLI-3551. The two used to be one
163
+ * function, so `repo_not_on_disk` — a label the attribution umbrella finding
164
+ * already established is not a defect (nothing was deleted; the transcript
165
+ * names a path git no longer tracks) — failed the sync on every tick for three
166
+ * operators. A retry hint and a failure are different claims.
167
+ */
168
+ export function sourceScanFailureReason(source, scan) {
169
+ const readFailureCount = source === "codex"
170
+ ? codexAttributionReadFailureCount(scan)
171
+ : claudeAttributionReadFailureCount(scan);
172
+ return readFailureCount > 0 ? `${source}_session_store_read_failed` : null;
173
+ }
157
174
  async function reconcileSourceScanRetry(options) {
158
175
  if (options.reason) {
159
176
  await recordSourceRetryFailure(options.paths, {
@@ -447,25 +464,47 @@ export async function runAttributedWorktreeSync(options) {
447
464
  });
448
465
  // Same conditions as before, one per line, each writing down its own reason.
449
466
  // The old version was a single boolean chain: correct, and completely mute.
450
- const failureReasons = new Set();
451
- const fail = (condition, reason) => {
467
+ //
468
+ // Since BLI-3551 each condition also writes down the LABEL it is classified
469
+ // by, beside the rendered string a person reads. The health receipt reads the
470
+ // label; nothing parses the sentence back apart.
471
+ const failureRecords = new Map();
472
+ const add = (record) => {
473
+ if (!failureRecords.has(record.rendered)) {
474
+ failureRecords.set(record.rendered, record);
475
+ }
476
+ };
477
+ const fail = (condition, label, rendered = label) => {
452
478
  if (condition)
453
- failureReasons.add(reason);
479
+ add({ label, rendered });
454
480
  };
455
481
  for (const { worktree, sync } of outcomes) {
456
482
  if (sync.status !== "uploaded") {
457
483
  // The spooled reason is the most specific thing anyone has, so lead with
458
484
  // it and name the worktree it belongs to — a fleet failure is usually one
459
485
  // repo, and "which one" is the first question asked.
460
- failureReasons.add(sync.status === "spooled" && sync.failure_reason
461
- ? `${worktree.worktree_label}:${sync.failure_reason}`
462
- : `${worktree.worktree_label}:upload_${sync.status}`);
486
+ add(sync.status === "spooled" && sync.failure_reason
487
+ ? {
488
+ label: sync.failure_class,
489
+ rendered: `${worktree.worktree_label}:${sync.failure_reason}`,
490
+ http_status: sync.failure_http_status,
491
+ }
492
+ : {
493
+ label: "upload_not_completed",
494
+ rendered: `${worktree.worktree_label}:upload_${sync.status}`,
495
+ });
463
496
  }
464
497
  for (const reason of sync.raw_evidence_failure_reasons ?? []) {
465
- failureReasons.add(`raw_evidence:${reason}`);
498
+ add({
499
+ label: "raw_evidence_upload_failed",
500
+ rendered: `raw_evidence:${reason}`,
501
+ });
466
502
  }
467
503
  for (const reason of sync.raw_evidence_retry_reasons ?? []) {
468
- failureReasons.add(`raw_evidence_retry:${reason}`);
504
+ add({
505
+ label: "raw_evidence_retry_required",
506
+ rendered: `raw_evidence_retry:${reason}`,
507
+ });
469
508
  }
470
509
  fail(sync.raw_evidence_deferred_byte_budget > 0, "deferred_byte_budget");
471
510
  fail(sync.raw_evidence_deferred_object_budget > 0, "deferred_object_budget");
@@ -474,28 +513,110 @@ export async function runAttributedWorktreeSync(options) {
474
513
  fail(claudeAttribution.session_limit_applied, "claude_session_limit_applied");
475
514
  fail(codexAttributionReadFailureCount(codexAttribution) > 0, "codex_session_read_failed");
476
515
  fail(claudeAttributionReadFailureCount(claudeAttribution) > 0, "claude_session_read_failed");
477
- const codexScanRetry = sourceScanRetryReason("codex", codexAttribution);
478
- if (codexScanRetry)
479
- failureReasons.add(`codex_scan:${codexScanRetry}`);
480
- const claudeScanRetry = sourceScanRetryReason("claude_code", claudeAttribution);
481
- if (claudeScanRetry)
482
- failureReasons.add(`claude_scan:${claudeScanRetry}`);
483
- fail(reportRequired && !report.posted, `session_report_unposted:${report.reason ?? "unknown"}`);
516
+ // BLI-3551: the scan's RETRY reason and the scan's FAILURE reason are two
517
+ // different questions, and answering both with one function is what put
518
+ // `claude_scan:repo_not_on_disk` on every tick of three machines. A repo that
519
+ // is not on disk is a label on the session (the attribution umbrella finding:
520
+ // nothing was deleted, the transcript simply names a path git no longer
521
+ // knows). It still widens the next scan window; it is not a failed sync.
522
+ const codexScanFailure = sourceScanFailureReason("codex", codexAttribution);
523
+ if (codexScanFailure) {
524
+ add({ label: "codex_scan_read_failed", rendered: `codex_scan:${codexScanFailure}` });
525
+ }
526
+ const claudeScanFailure = sourceScanFailureReason("claude_code", claudeAttribution);
527
+ if (claudeScanFailure) {
528
+ add({
529
+ label: "claude_scan_read_failed",
530
+ rendered: `claude_scan:${claudeScanFailure}`,
531
+ });
532
+ }
533
+ // BLI-3551: a tick that observed only sessions from outside the operator's
534
+ // approved roots has nothing to post, and that is the consent boundary
535
+ // working — not a failure. It used to fail as
536
+ // `session_report_unposted:no_successful_sync`, whose word "session" then
537
+ // classified as `auth_failed`; one machine reported a broken credential 377
538
+ // times in 38 hours while its token had eleven weeks left. The withhold
539
+ // decision itself is untouched (adapters/attribution-core.ts) — only what it
540
+ // is CALLED.
541
+ const sessionsOutsideRoot = sessions.filter((session) => OUTSIDE_APPROVED_ROOT_REASONS.has(session.attribution_reason)).length;
542
+ const nothingInRoot = nothingInRootCount({
543
+ sessionCount: sessions.length,
544
+ outsideRootCount: sessionsOutsideRoot,
545
+ outcomes,
546
+ reportPosted: report.posted,
547
+ reportReason: report.reason,
548
+ });
549
+ fail(reportRequired && !report.posted && nothingInRoot === null, "session_report_unposted", `session_report_unposted:${report.reason ?? "unknown"}`);
484
550
  // `ok` may already be false from the per-worktree loop above; the outcome
485
551
  // scan re-derives that, so the two agree by construction.
486
- ok = ok && failureReasons.size === 0;
487
- if (!ok && failureReasons.size === 0) {
488
- failureReasons.add(SYNC_FAILED_WITHOUT_REASON);
552
+ ok = ok && failureRecords.size === 0;
553
+ if (!ok && failureRecords.size === 0) {
554
+ add({
555
+ label: SYNC_FAILED_WITHOUT_REASON,
556
+ rendered: SYNC_FAILED_WITHOUT_REASON,
557
+ });
558
+ }
559
+ const notice = ok && nothingInRoot !== null ? `nothing_in_root:${nothingInRoot}` : null;
560
+ if (notice) {
561
+ // The success branch says something too: this is the receipt that proves a
562
+ // quiet machine is a working machine, and the count is what tells a coach
563
+ // that someone is working entirely outside the approved boundary.
564
+ console.error("[session-sync] nothing to collect inside the approved roots", JSON.stringify({
565
+ reason: "nothing_in_root",
566
+ sessions_outside_root: nothingInRoot,
567
+ collection_root_count: collectionRoots.length,
568
+ next_action: "widen the approved roots (an operator decision) if this machine should be collecting here",
569
+ }));
489
570
  }
571
+ const records = [...failureRecords.values()].sort((a, b) => a.rendered.localeCompare(b.rendered));
490
572
  return {
491
573
  ok,
492
- failure_reasons: [...failureReasons].sort(),
574
+ failure_reasons: records.map((record) => record.rendered),
575
+ failure_records: records,
576
+ notice,
577
+ sessions_observed: sessions.length,
578
+ sessions_outside_root: sessionsOutsideRoot,
493
579
  outcomes,
494
580
  codexAttribution,
495
581
  claudeAttribution,
496
582
  summary,
497
583
  };
498
584
  }
585
+ /**
586
+ * Reasons attribution gives when a session's working directory is not inside
587
+ * any approved collection root.
588
+ *
589
+ * Exact labels, not a pattern — the same discipline the classifier now follows.
590
+ * `attribution-core.ts` writes both of these and nothing else means
591
+ * "outside the boundary".
592
+ */
593
+ const OUTSIDE_APPROVED_ROOT_REASONS = new Set([
594
+ "cwd_outside_scanned_worktrees",
595
+ "no_matching_worktree_signals",
596
+ ]);
597
+ /**
598
+ * How many observed sessions were outside the approved roots, when that
599
+ * accounts for ALL of them and nothing else went wrong — otherwise `null`.
600
+ *
601
+ * Deliberately narrow. It requires that no worktree was synced at all (so no
602
+ * upload could have succeeded or failed), that every session observed this tick
603
+ * names an outside-the-root reason, and that the unposted report is the
604
+ * `no_successful_sync` shape rather than a spooled report that failed to flush.
605
+ * Anything else keeps its failure.
606
+ */
607
+ export function nothingInRootCount(options) {
608
+ if (options.reportPosted)
609
+ return null;
610
+ if (options.reportReason !== "no_successful_sync")
611
+ return null;
612
+ if (options.outcomes.length > 0)
613
+ return null;
614
+ if (options.sessionCount === 0)
615
+ return null;
616
+ return options.outsideRootCount === options.sessionCount
617
+ ? options.outsideRootCount
618
+ : null;
619
+ }
499
620
  export const ATTRIBUTION_STATE_RANK = CODEX_SESSION_ATTRIBUTION_STATE_RANK;
500
621
  function normalizeCodexResult(result) {
501
622
  return {
@@ -0,0 +1,186 @@
1
+ /**
2
+ * Is this collector a DEV BUILD? One predicate, two callers.
3
+ *
4
+ * BLI-3554. Edward's device row carried 47 `npm_install_eacces` receipts
5
+ * stamped `cli_version` 0.1.50 between 2026-09-02T16:55Z and 2026-09-03T06:25Z.
6
+ * 0.1.50 was never a published install on that machine — it was the workspace
7
+ * version in `packages/cockpit-local-collector/package.json` at the time, so
8
+ * every one of those rows came from an agent or a test running the built
9
+ * collector inside a worktree. They land in the same production
10
+ * `cockpit_install_events` table the fleet reader (BLI-3550,
11
+ * `apps/dashboard/src/lib/ops/fleet-liveness.ts`) alarms on, so a developer's
12
+ * sandbox failure reads as a fleet machine in trouble.
13
+ *
14
+ * The predicate is deliberately NOT a version comparison. A version equal to
15
+ * the workspace version is a symptom: it changes on every release, it is equal
16
+ * for the one machine that legitimately runs the newest published build the
17
+ * day it ships, and it says nothing about where the code was loaded from.
18
+ * What we can actually observe is the FILE the running code was loaded from:
19
+ *
20
+ * - a published install lives under a `node_modules/` prefix
21
+ * (`/opt/homebrew/lib/node_modules/@bli-cockpit/cli/dist/…`,
22
+ * `C:\\Users\\x\\AppData\\Roaming\\npm\\node_modules\\@bli-cockpit\\cli\\dist\\…`),
23
+ * and that is the whole fleet;
24
+ * - a dev build lives inside a checkout of this repo — a
25
+ * `packages/cockpit-local-collector/` (or `packages/cockpit-cli/`) ancestor
26
+ * whose `package.json` names the workspace, with a `.git` marker or a
27
+ * `.claude/worktrees` segment above it.
28
+ *
29
+ * `COCKPIT_DEV=1` forces dev. `COCKPIT_DEV=0` forces the opposite — that is the
30
+ * escape hatch for a deliberate live test from a checkout, documented in
31
+ * `docs/runbooks/cockpit-collector-receipts.md`.
32
+ *
33
+ * Suppression only bites when the target is a PRODUCTION dashboard. Pointing a
34
+ * checkout at `http://localhost:3000` is how the receipt path itself is
35
+ * developed, and refusing to post there would make this predicate the reason
36
+ * the next receipt bug cannot be reproduced.
37
+ */
38
+ import fs from "node:fs";
39
+ import path from "node:path";
40
+ import { fileURLToPath } from "node:url";
41
+ /** The workspaces whose `dist/` a developer or agent runs from a checkout. */
42
+ const WORKSPACE_PACKAGE_NAMES = new Set([
43
+ "@bli-cockpit/local-collector",
44
+ "@bli-cockpit/cli",
45
+ ]);
46
+ const TRUE_VALUES = new Set(["1", "true", "yes", "on"]);
47
+ const FALSE_VALUES = new Set(["0", "false", "no", "off"]);
48
+ /**
49
+ * Walks up from the running module. Pure apart from the two injected readers,
50
+ * so a Windows layout can be asserted from a macOS test run.
51
+ */
52
+ export function detectDevBuild(probe = {}) {
53
+ const env = probe.env ?? process.env;
54
+ const rawFlag = (env.COCKPIT_DEV ?? "").trim().toLowerCase();
55
+ if (TRUE_VALUES.has(rawFlag)) {
56
+ return { devBuild: true, reason: "cockpit_dev_env" };
57
+ }
58
+ if (FALSE_VALUES.has(rawFlag)) {
59
+ // Deliberate live test from a checkout: the operator has said so out loud,
60
+ // and the path evidence below is not allowed to overrule them.
61
+ return { devBuild: false, reason: "cockpit_dev_env_disabled" };
62
+ }
63
+ const pathApi = probe.pathApi ?? path;
64
+ const modulePath = probe.modulePath ?? currentModulePath();
65
+ if (!modulePath)
66
+ return { devBuild: false, reason: "not_a_checkout" };
67
+ if (hasNodeModulesSegment(modulePath)) {
68
+ // Every published install is under a `node_modules/` prefix, on both host
69
+ // families. This branch is the whole fleet, and it is checked first so a
70
+ // machine that happens to have a repo checkout somewhere above its global
71
+ // npm prefix cannot be mislabelled.
72
+ return { devBuild: false, reason: "published_install" };
73
+ }
74
+ const readPackageName = probe.readPackageName ?? defaultReadPackageName;
75
+ const pathExists = probe.pathExists ?? defaultPathExists;
76
+ const packageRoot = findWorkspacePackageRoot(pathApi.dirname(modulePath), pathApi, readPackageName);
77
+ if (!packageRoot)
78
+ return { devBuild: false, reason: "not_a_checkout" };
79
+ if (!hasCheckoutMarkerAbove(packageRoot, pathApi, pathExists)) {
80
+ return { devBuild: false, reason: "not_a_checkout" };
81
+ }
82
+ return { devBuild: true, reason: "workspace_checkout" };
83
+ }
84
+ /**
85
+ * Should this process withhold fleet receipts (install events, heartbeat)?
86
+ *
87
+ * Two facts, in this order: is the code a dev build, and is the target the
88
+ * production fleet. A dev build talking to `localhost` still posts, because
89
+ * that is the only way to exercise the receipt path at all.
90
+ */
91
+ export function shouldSuppressFleetReceipts(options) {
92
+ const verdict = detectDevBuild(options.probe ?? {});
93
+ if (!verdict.devBuild)
94
+ return { suppressed: false, reason: verdict.reason };
95
+ if (isLocalDashboardUrl(options.dashboardUrl)) {
96
+ return { suppressed: false, reason: "local_dashboard" };
97
+ }
98
+ return { suppressed: true, reason: verdict.reason };
99
+ }
100
+ /** A dashboard on this machine — safe for a checkout to post at. */
101
+ export function isLocalDashboardUrl(dashboardUrl) {
102
+ let hostname;
103
+ try {
104
+ hostname = new URL(dashboardUrl).hostname.toLowerCase();
105
+ }
106
+ catch {
107
+ // An unparseable URL is not demonstrably local, and this predicate only
108
+ // ever widens suppression when it says false.
109
+ return false;
110
+ }
111
+ const bare = hostname.replace(/^\[|\]$/gu, "");
112
+ return (bare === "localhost" ||
113
+ bare === "127.0.0.1" ||
114
+ bare === "::1" ||
115
+ bare === "0.0.0.0" ||
116
+ bare.endsWith(".localhost"));
117
+ }
118
+ function currentModulePath() {
119
+ try {
120
+ return fileURLToPath(import.meta.url);
121
+ }
122
+ catch {
123
+ return null;
124
+ }
125
+ }
126
+ /**
127
+ * Case-insensitive on purpose: Windows spells the same folder
128
+ * `Node_Modules` without complaint, and both separators appear on that host
129
+ * because a POSIX-style path survives most Node APIs there.
130
+ */
131
+ function hasNodeModulesSegment(modulePath) {
132
+ return modulePath
133
+ .split(/[\\/]+/u)
134
+ .some((segment) => segment.toLowerCase() === "node_modules");
135
+ }
136
+ function findWorkspacePackageRoot(startDir, pathApi, readPackageName) {
137
+ let dir = startDir;
138
+ // Bounded: `path.dirname` reaches a fixed point at `/` or `C:\`, and the
139
+ // depth guard keeps a pathological symlink loop from spinning a sync walk.
140
+ for (let depth = 0; depth < 64; depth += 1) {
141
+ const name = readPackageName(pathApi.join(dir, "package.json"));
142
+ if (name && WORKSPACE_PACKAGE_NAMES.has(name))
143
+ return dir;
144
+ const parent = pathApi.dirname(dir);
145
+ if (parent === dir)
146
+ return null;
147
+ dir = parent;
148
+ }
149
+ return null;
150
+ }
151
+ /**
152
+ * A repo checkout marker at or above the workspace package: `.git` (a
153
+ * directory in a normal clone, a FILE in a `git worktree`), or the
154
+ * `.claude/worktrees` staging area agents run from.
155
+ */
156
+ function hasCheckoutMarkerAbove(packageRoot, pathApi, pathExists) {
157
+ let dir = packageRoot;
158
+ for (let depth = 0; depth < 64; depth += 1) {
159
+ if (pathExists(pathApi.join(dir, ".git")))
160
+ return true;
161
+ if (pathExists(pathApi.join(dir, ".claude", "worktrees")))
162
+ return true;
163
+ const parent = pathApi.dirname(dir);
164
+ if (parent === dir)
165
+ return false;
166
+ dir = parent;
167
+ }
168
+ return false;
169
+ }
170
+ function defaultReadPackageName(packageJsonPath) {
171
+ try {
172
+ const parsed = JSON.parse(fs.readFileSync(packageJsonPath, "utf8"));
173
+ return typeof parsed.name === "string" ? parsed.name : null;
174
+ }
175
+ catch {
176
+ return null;
177
+ }
178
+ }
179
+ function defaultPathExists(candidate) {
180
+ try {
181
+ return fs.existsSync(candidate);
182
+ }
183
+ catch {
184
+ return false;
185
+ }
186
+ }
@@ -1,6 +1,7 @@
1
- import { COMMIT_CRASHED_PLATFORM, RAW_EVIDENCE_UPLOAD_DEFAULT_CHUNK_BYTES, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, RAW_EVIDENCE_UPLOAD_MAX_OBJECTS_PER_BEGIN, RawEvidenceLegacyUploadResponseSchema, RawEvidenceRedactionMetadataSchema, RawEvidenceUploadBeginResponseSchema, RawEvidenceUploadCommitResponseSchema, isPermanentUploadFailure, } from "@bli-cockpit/telemetry-core";
1
+ import { COMMIT_CRASHED_PLATFORM, RAW_EVIDENCE_UPLOAD_DEFAULT_CHUNK_BYTES, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, RAW_EVIDENCE_UPLOAD_MAX_OBJECTS_PER_BEGIN, RawEvidenceLegacyUploadResponseSchema, RawEvidenceRedactionMetadataSchema, RawEvidenceUploadBeginResponseSchema, RawEvidenceUploadCommitResponseSchema, isPermanentUploadFailure, isRekeyableUploadConflict, } from "@bli-cockpit/telemetry-core";
2
2
  import crypto from "node:crypto";
3
3
  import fs from "node:fs/promises";
4
+ import { rekeyedEvidencePointer } from "./evidence-upload-rekey.js";
4
5
  import { describeError } from "./health-detail.js";
5
6
  const DEFAULT_MAX_ATTEMPTS = 3;
6
7
  const RETRY_DELAY_MS = 250;
@@ -182,10 +183,18 @@ export async function uploadRawEvidenceFilesChunked(options) {
182
183
  resolvedEntries.add(entry);
183
184
  continue;
184
185
  }
185
- const outcome = await uploadOneObject(options, entry, disposition, chunkSizeBytes);
186
- await reportAbandonedUpload(options, disposition, outcome);
186
+ // A conflict a different name can settle gets one, here, before the
187
+ // upload is attempted (BLI-3552). Anything else comes back unchanged and
188
+ // fails on its own reason inside `uploadOneObject`.
189
+ const resolved = await rekeyConflictedEntry(options, entry, disposition, chunkSizeBytes);
190
+ const outcome = await uploadOneObject(options, resolved.entry, resolved.disposition, chunkSizeBytes);
191
+ await reportAbandonedUpload(options, resolved.disposition, outcome);
187
192
  outcomes.push(outcome);
188
- for (const duplicate of entry.duplicates) {
193
+ // The duplicates of a re-keyed primary were re-keyed with it: a duplicate
194
+ // is byte-identical and shared the primary's key, so it shares the new
195
+ // one too. Leaving them on the old key would point their evidence refs at
196
+ // somebody else's durable object.
197
+ for (const duplicate of resolved.entry.duplicates) {
189
198
  outcomes.push(duplicateOutcome(outcome, duplicate));
190
199
  }
191
200
  resolvedEntries.add(entry);
@@ -281,6 +290,105 @@ function conformReasonLabel(reason) {
281
290
  const conformed = reason.replace(/[^a-z0-9_:.-]/gi, "_").slice(0, 120);
282
291
  return conformed.length > 0 ? conformed : "upload_failed_unlabelled";
283
292
  }
293
+ /**
294
+ * Settle a re-keyable conflict by asking for a second name (BLI-3552).
295
+ *
296
+ * `hash_mismatch_committed_object` means the key already holds different bytes
297
+ * that are already durable. Re-sending is the loop; deleting the durable object
298
+ * to make room is destroying evidence. The third answer is to offer this
299
+ * content under a name derived from its own hash, which is what this does, once
300
+ * per object per sync.
301
+ *
302
+ * Every path that cannot get there returns the ORIGINAL pair, so the object
303
+ * fails on the conflict reason `begin` actually gave rather than on a label
304
+ * invented here. Nothing is written off permanently either way: the upload
305
+ * cursor only remembers successes, so the next eligible sync offers the bytes
306
+ * again — at delivery-backoff cadence now instead of every 15 minutes.
307
+ */
308
+ async function rekeyConflictedEntry(options, entry, disposition, chunkSizeBytes) {
309
+ if (disposition.disposition !== "conflict" ||
310
+ !isRekeyableUploadConflict(disposition.reason)) {
311
+ return { entry, disposition };
312
+ }
313
+ const contentHashPrefix = (entry.file.pointer.content_hash_sha256 ?? "none").slice(0, 12);
314
+ const rekeyedPointer = rekeyedEvidencePointer(entry.file.pointer);
315
+ if (!rekeyedPointer) {
316
+ // The key already names this content and the server still says it holds
317
+ // something else. A rename cannot answer that; a person has to.
318
+ console.error("[evidence-rekey] the key already carries this content hash; leaving the conflict for a person", JSON.stringify({
319
+ reason: disposition.reason,
320
+ kind: entry.file.kind ?? "unknown",
321
+ content_hash_prefix: contentHashPrefix,
322
+ byte_size: entry.bytes.byteLength,
323
+ }));
324
+ return { entry, disposition };
325
+ }
326
+ const rekeyedEntry = {
327
+ ...entry,
328
+ file: { ...entry.file, pointer: rekeyedPointer },
329
+ duplicates: entry.duplicates.map((duplicate) => ({
330
+ ...duplicate,
331
+ pointer: rekeyedEvidencePointer(duplicate.pointer) ?? duplicate.pointer,
332
+ })),
333
+ };
334
+ const rekeyedDisposition = await beginOneObject(options, rekeyedEntry, chunkSizeBytes);
335
+ if (!rekeyedDisposition) {
336
+ return { entry, disposition };
337
+ }
338
+ console.error("[evidence-rekey] committed object holds other content; offering these bytes under their own hash", JSON.stringify({
339
+ reason: disposition.reason,
340
+ kind: entry.file.kind ?? "unknown",
341
+ content_hash_prefix: contentHashPrefix,
342
+ byte_size: entry.bytes.byteLength,
343
+ chunk_count: entry.chunkCount,
344
+ rekeyed_disposition: rekeyedDisposition.disposition,
345
+ rekeyed_reason: rekeyedDisposition.reason ?? "none",
346
+ duplicate_count: entry.duplicates.length,
347
+ }));
348
+ return { entry: rekeyedEntry, disposition: rekeyedDisposition };
349
+ }
350
+ /**
351
+ * `begin` for a single object. Null whenever the answer cannot be trusted —
352
+ * transport failure, an unparseable body, a duplicate or missing key, or a
353
+ * pointer id that is not the one we sent — and the caller then keeps the
354
+ * original conflict rather than acting on a guess.
355
+ */
356
+ async function beginOneObject(options, entry, chunkSizeBytes) {
357
+ const objectKey = entry.file.pointer.object_key ?? "";
358
+ const response = await requestJson(options, "/api/ambient/evidence/upload/begin", {
359
+ schema_version: "ambient-raw-evidence-upload-begin.v1",
360
+ generated_at: options.generatedAt,
361
+ provenance: options.provenance,
362
+ objects: [
363
+ {
364
+ pointer: entry.file.pointer,
365
+ chunk_size_bytes: chunkSizeBytes,
366
+ chunk_count: entry.chunkCount,
367
+ },
368
+ ],
369
+ });
370
+ if (!response.ok) {
371
+ console.error("[evidence-rekey] begin refused the re-keyed object; keeping the original conflict", JSON.stringify({
372
+ reason: "rekey_begin_rejected",
373
+ http_status: response.status,
374
+ server_reason: safeFailureDetail(response.body) ?? "none",
375
+ kind: entry.file.kind ?? "unknown",
376
+ }));
377
+ return null;
378
+ }
379
+ const parsed = RawEvidenceUploadBeginResponseSchema.safeParse(response.body);
380
+ if (!parsed.success)
381
+ return null;
382
+ const dispositions = readBeginDispositions(parsed.data);
383
+ const disposition = dispositions?.get(objectKey);
384
+ if (!disposition)
385
+ return null;
386
+ if (disposition.raw_evidence_pointer_id !==
387
+ entry.file.pointer.raw_evidence_pointer_id) {
388
+ return null;
389
+ }
390
+ return disposition;
391
+ }
284
392
  async function uploadOneObject(options, entry, disposition, chunkSizeBytes) {
285
393
  const objectKey = entry.file.pointer.object_key ?? "";
286
394
  if (disposition.disposition === "already_committed") {
@@ -0,0 +1,40 @@
1
+ /** How much of the content hash goes into the name, matching the pack keys. */
2
+ const KEY_HASH_PREFIX_LENGTH = 16;
3
+ /**
4
+ * The same object under a name that carries its own content hash.
5
+ *
6
+ * `null` when a re-key cannot help and must not be attempted:
7
+ * - there is no object key to rewrite;
8
+ * - the last segment already starts with this content's hash prefix, so the
9
+ * re-keyed name would be the identical string and the second `begin` would
10
+ * answer the identical conflict;
11
+ * - the key has no `/`, which the server's namespace rules make impossible, so
12
+ * rewriting it would be inventing a key rather than deriving one.
13
+ *
14
+ * The new segment is `<hash16>-<old segment>`: still inside the operator's
15
+ * readable namespace (so `evidenceObjectKeyBelongsToWorkContext` still passes),
16
+ * still inside the key charset, and still legible to a person doing an incident
17
+ * walk — they can see which pack file it came from.
18
+ */
19
+ export function rekeyedEvidencePointer(pointer) {
20
+ const objectKey = pointer.object_key;
21
+ if (!objectKey)
22
+ return null;
23
+ const cut = objectKey.lastIndexOf("/");
24
+ if (cut <= 0 || cut === objectKey.length - 1)
25
+ return null;
26
+ const prefix = objectKey.slice(0, cut);
27
+ const segment = objectKey.slice(cut + 1);
28
+ // A pointer with no content hash cannot name itself, and the upload routes
29
+ // reject it anyway; there is nothing to re-key it to.
30
+ const hash = (pointer.content_hash_sha256 ?? "").slice(0, KEY_HASH_PREFIX_LENGTH);
31
+ if (!hash || segment.startsWith(`${hash}-`) || segment.startsWith(hash)) {
32
+ return null;
33
+ }
34
+ const rekeyed = `${prefix}/${hash}-${segment}`;
35
+ return {
36
+ ...pointer,
37
+ object_key: rekeyed,
38
+ raw_evidence_pointer_id: rekeyed,
39
+ };
40
+ }