@bli-cockpit/cli 0.2.278 → 0.2.279

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -34,7 +34,11 @@ export async function readCodexMetadataSignals(filePath) {
34
34
  const state = makeSignalExtractionState();
35
35
  const stream = createReadStream(filePath);
36
36
  let byteSize = 0;
37
- let lineBuffer = "";
37
+ // The line being assembled, in the pieces the chunks gave and its byte count kept as a running sum. The
38
+ // old `lineBuffer += segment` plus `Buffer.byteLength(lineBuffer)` flattened the whole growing string on
39
+ // every 64 KiB chunk, so a 1 MiB record made about 8 MiB of short-lived copies (BLI-23521).
40
+ let lineParts = [];
41
+ let lineBytes = 0;
38
42
  let discardingOversizedLine = false;
39
43
  for await (const chunk of stream) {
40
44
  const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
@@ -50,25 +54,29 @@ export async function readCodexMetadataSignals(filePath) {
50
54
  discardingOversizedLine = false;
51
55
  continue;
52
56
  }
53
- lineBuffer += segment;
54
- if (Buffer.byteLength(lineBuffer, "utf8") > CODEX_ATTRIBUTION_MAX_LINE_BYTES) {
57
+ lineParts.push(segment);
58
+ lineBytes += Buffer.byteLength(segment, "utf8");
59
+ if (lineBytes > CODEX_ATTRIBUTION_MAX_LINE_BYTES) {
55
60
  state.lineCount += 1;
56
61
  state.parseErrorCount += 1;
57
- lineBuffer = "";
62
+ lineParts = [];
63
+ lineBytes = 0;
58
64
  discardingOversizedLine = !lineEnded;
59
65
  continue;
60
66
  }
61
67
  if (lineEnded) {
62
- absorbCodexSessionLine(state, lineBuffer);
63
- lineBuffer = "";
68
+ absorbCodexSessionLine(state, lineParts.length === 1 ? lineParts[0] : lineParts.join(""));
69
+ lineParts = [];
70
+ lineBytes = 0;
64
71
  }
65
72
  }
66
73
  }
67
74
  const rest = decoder.end();
68
75
  if (rest)
69
- lineBuffer += rest;
70
- if (lineBuffer.trim())
71
- absorbCodexSessionLine(state, lineBuffer);
76
+ lineParts.push(rest);
77
+ const lastLine = lineParts.join("");
78
+ if (lastLine.trim())
79
+ absorbCodexSessionLine(state, lastLine);
72
80
  observeSessionFactsFile(state.facts, { bytes: byteSize, lines: state.lineCount });
73
81
  return {
74
82
  signals: codexSignalsFromState(state),
@@ -269,6 +269,9 @@ function uploadFileFromEntry(entry) {
269
269
  ? { artifact_metadata: entry.artifact_metadata }
270
270
  : {}),
271
271
  ...(entry.snapshot ? { source_key: entry.source_key, snapshot: entry.snapshot } : {}),
272
+ ...(entry.supersedes?.rule === "prefix" && entry.supersedes.base
273
+ ? { extend_base: { object_key: entry.supersedes.pointer_id, ...entry.supersedes.base } }
274
+ : {}),
272
275
  };
273
276
  }
274
277
  function stageReasonLabel(state, priorPackCount) {
@@ -20,7 +20,7 @@
20
20
  * says so: `snapshot_rewritten`, both sizes, and the byte range the first
21
21
  * difference falls in. Never content.
22
22
  */
23
- import { createReadStream } from "node:fs";
23
+ import { chunksOfFileReusingOneBuffer } from "../evidence-upload-staged.js";
24
24
  import { REDACTION_STUB_LABEL } from "@bli-cockpit/telemetry-core";
25
25
  import { firstDifferenceRange, isPrefixSuccessor, prefixCheckpoints, prefixDigestsFromStream, } from "@bli-cockpit/telemetry-core/snapshot-supersede";
26
26
  /**
@@ -73,7 +73,11 @@ function decideClaim(options, isPrefix, differenceRange) {
73
73
  previous_byte_size: prior.byte_size,
74
74
  byte_size: options.byteSize,
75
75
  }));
76
- return { pointer_id: prior.raw_evidence_pointer_id, rule: "prefix" };
76
+ return {
77
+ pointer_id: prior.raw_evidence_pointer_id,
78
+ rule: "prefix",
79
+ base: { content_hash_sha256: prior.content_hash_sha256, byte_size: prior.byte_size },
80
+ };
77
81
  }
78
82
  const range = differenceRange(prior);
79
83
  console.error("[evidence-upload] snapshot rewritten, keeping both", JSON.stringify({
@@ -101,7 +105,7 @@ export async function describeStagedSnapshot(options) {
101
105
  const priorOffsets = wantClaim
102
106
  ? [prior.byte_size, ...Object.keys(prior.checkpoints ?? {}).map(Number)].filter((at) => at > 0 && at <= Math.min(prior.byte_size, byteLength))
103
107
  : [];
104
- const result = await prefixDigestsFromStream(createReadStream(options.filePath, { highWaterMark: 1024 * 1024 }), [...checkpointOffsets, ...priorOffsets]);
108
+ const result = await prefixDigestsFromStream(chunksOfFileReusingOneBuffer(options.filePath, 1024 * 1024), [...checkpointOffsets, ...priorOffsets]);
105
109
  if (result.total_bytes !== byteLength) {
106
110
  // The staged file is not the bytes that were hashed: a claim made on it would be a lie.
107
111
  console.error("[evidence-upload] staged file size differs from what was written; no claim", JSON.stringify({
@@ -28,8 +28,22 @@ export function holdDecodedText(source) {
28
28
  if (!lossless) {
29
29
  console.error("[raw-evidence] transcript is not valid UTF-8; keeping its bytes beside the text so an unchanged file uploads as it is", JSON.stringify({ reason: "invalid_utf8_bytes_kept", byte_size: bytes.byteLength }));
30
30
  }
31
+ if (lossless)
32
+ releaseBackingStore(bytes);
31
33
  return { text, exactBytes: lossless ? null : bytes };
32
34
  }
35
+ /**
36
+ * Free a file-sized buffer now instead of at the next collection (BLI-23521). Dropping the reference only
37
+ * makes it collectable; an ArrayBuffer is not collected until a GC runs, so on a 10 MB transcript the bytes,
38
+ * the decoded text and the encoded output were all resident at once (arrayBuffers 21 MB, RSS +30 MB over a
39
+ * 2.6 MB file). Only a buffer that owns its whole ArrayBuffer is detached: a slice of the shared pool is not ours.
40
+ */
41
+ function releaseBackingStore(bytes) {
42
+ const store = bytes.buffer;
43
+ if (bytes.byteOffset !== 0 || store.byteLength !== bytes.byteLength)
44
+ return;
45
+ store.transfer?.(0);
46
+ }
33
47
  /**
34
48
  * BLI-16977: describe each distinct embedded image once and, when the one switch says so, swap the
35
49
  * base64 for a short marker BEFORE masking, so the masker and the content hash see the final bytes.
@@ -1,6 +1,6 @@
1
1
  /**
2
- * The transcript that is never read whole (BLI-23317): its path goes to the image session's two
3
- * streaming passes, and what comes back is the one slimmed string the sanitizer takes.
2
+ * The transcript that is never read whole (BLI-23317; keep mode too since BLI-23524, where the second pass
3
+ * copies the lines): its path goes to the image session's two streaming passes, and what comes back is the one slimmed string the sanitizer takes.
4
4
  *
5
5
  * Three endings, each named:
6
6
  * - a body: the text is the slim form of the file (valid UTF-8 was checked across every chunk, so
@@ -38,7 +38,7 @@ export async function streamTranscriptBody(session, agent, file) {
38
38
  try {
39
39
  const prepared = await session.prepareTranscriptFile({ filePath: file.filePath, source: agent, sessionId: file.sessionId, maxFileBytes: file.maxFileBytes, stageDir: file.stageDir, omitText: file.stageDir !== undefined, ...(file.provenance ? { provenance: file.provenance } : {}) });
40
40
  const { volatile, pendingImages } = volatilityOf(session.mode, prepared);
41
- console.error(TAG + " transcript streamed through image swap and slim", JSON.stringify({ kind: file.kind, byte_size: prepared.readBytes, changed: prepared.changed, images: prepared.imageCount, volatile, staged_bytes: prepared.staged?.byteLength ?? null }));
41
+ console.error(TAG + " transcript streamed through image swap and slim", JSON.stringify({ kind: file.kind, mode: session.mode, byte_size: prepared.readBytes, changed: prepared.changed, images: prepared.imageCount, volatile, staged_bytes: prepared.staged?.byteLength ?? null }));
42
42
  // BLI-23344: with a spool the text is not read back; the sanitizer reads the spool line by line.
43
43
  return { held: { text: prepared.staged ? null : prepared.text, exactBytes: null, ...(prepared.staged ? { staged: prepared.staged } : {}) }, volatile, pendingImages, readBytes: prepared.readBytes };
44
44
  }
@@ -0,0 +1,32 @@
1
+ import { askAgentDoor, emitAgentDoor, failAgentDoor, openAgentDoor } from "./agent-door.js";
2
+ import { writeLine } from "./cli-io.js";
3
+ import { parseNamedArgs, optionalNonEmpty, optionalUrl } from "./local-arg-values.js";
4
+ const USAGE = "Usage: cockpit ai-spend flow|people|output [--days <1-365>] [--json]";
5
+ export function parseAiSpendArgs(args) {
6
+ const values = parseNamedArgs(args, { allowedFlags: ["--days", "--json", "--home", "--dashboard-url"], valueFlags: ["--days", "--home", "--dashboard-url"] });
7
+ const [action, ...rest] = values.positionals;
8
+ if ((action !== "flow" && action !== "people" && action !== "output") || rest.length)
9
+ throw new Error(USAGE);
10
+ const raw = values.flags.get("--days");
11
+ let days;
12
+ if (raw !== undefined) {
13
+ days = Number(raw);
14
+ if (!Number.isInteger(days) || days < 1 || days > 365)
15
+ throw new Error(USAGE);
16
+ }
17
+ return { kind: "ai-spend", action, days, json: values.booleans.has("--json"), homeDir: optionalNonEmpty(values.flags.get("--home")), dashboardUrl: optionalUrl(values.flags.get("--dashboard-url")) };
18
+ }
19
+ export async function runAiSpend(command, io) {
20
+ const door = await openAgentDoor("ai-spend", command, io);
21
+ const query = new URLSearchParams({ view: command.action });
22
+ if (command.days !== undefined)
23
+ query.set("days", String(command.days));
24
+ const answer = await askAgentDoor(door, { path: `/api/ai-spend?${query}`, method: "GET", label: `ai-spend ${command.action}`, timeoutMs: 30_000 });
25
+ if (!answer.ok)
26
+ return failAgentDoor(door, "[ai-spend cli]", answer.reason, answer.detail);
27
+ if (command.json)
28
+ return emitAgentDoor(door, answer.body);
29
+ for (const line of answer.body.lines)
30
+ writeLine(io.stdout, line);
31
+ return 0;
32
+ }
@@ -57,4 +57,6 @@ export { parseGlossaryArgs } from "./glossary.js";
57
57
  export { parseSpendArgs } from "./spend.js";
58
58
  export { parsePushArgs } from "./push.js";
59
59
  export { parseCiArgs } from "./ci.js";
60
- export { parseDeliveryArgs } from "./delivery.js";
60
+ export { parseDeliveryArgs } from "./delivery.js";
61
+ export { parsePrOutputArgs } from "./pr-output.js";
62
+ export { parseAiSpendArgs } from "./ai-spend.js";
@@ -23,7 +23,7 @@ import { parseSlackPrivacyArgs } from "./slack-privacy-args.js";
23
23
  import { parseMilestoneArgs } from "./local-args-tower-milestone.js";
24
24
  import { SURFACE_ALIASES } from "./local-help.js";
25
25
  import { parseAgentRulesArgs, parseAnalyzeArgs, parseAutostartArgs, parseBackfillArgs, parseCleanArgs, parseDoctorArgs, parseInstallArgs, parseLoginArgs, parseMemoryArgs, parseLogoutArgs, parseOnboardArgs, parseReleaseArgs, parseServeArgs, parseSessionsArgs, parseSourcesArgs, parseStartArgs, parseStatusArgs, parseSyncArgs, parseUpdateArgs, } from "./local-args-collector.js";
26
- import { parseCiArgs, parseDeliveryArgs, parseBriefArgs, parseCorrectArgs, parseDocsArgs, parseIssueArgs, parseJarvisArgs, parseMailArgs, parseCalArgs, parseModelArgs, parseMsgArgs, parseNotesArgs, parseOpsArgs, parseProjectArgs, parseModelsArgs, parseScoutArgs, parseGithubArgs, parseSearchArgs, parseSettingsArgs, parseSlackArgs, parseTeamArgs, parseWorkbookArgs, parseUsageArgs, parseSetupArgs, parseSpeedArgs, parseLogsArgs, parseBrainArgs, parseHitsArgs, parseIntentionsArgs, parseWhoArgs, parsePeopleArgs, parseApprovalsArgs, parseDrillsArgs, parseManagerArgs, parseImageArgs, parseCareersArgs, parseRenameArgs, } from "./local-args-tower.js";
26
+ import { parseCiArgs, parseDeliveryArgs, parsePrOutputArgs, parseAiSpendArgs, parseBriefArgs, parseCorrectArgs, parseDocsArgs, parseIssueArgs, parseJarvisArgs, parseMailArgs, parseCalArgs, parseModelArgs, parseMsgArgs, parseNotesArgs, parseOpsArgs, parseProjectArgs, parseModelsArgs, parseScoutArgs, parseGithubArgs, parseSearchArgs, parseSettingsArgs, parseSlackArgs, parseTeamArgs, parseWorkbookArgs, parseUsageArgs, parseSetupArgs, parseSpeedArgs, parseLogsArgs, parseBrainArgs, parseHitsArgs, parseIntentionsArgs, parseWhoArgs, parsePeopleArgs, parseApprovalsArgs, parseDrillsArgs, parseManagerArgs, parseImageArgs, parseCareersArgs, parseRenameArgs, } from "./local-args-tower.js";
27
27
  // `normalizeUrl` has always been part of this module's surface — `local.ts` and
28
28
  // `local-auth.ts` import it from here — so it stays exported from this address
29
29
  // even though it now lives next door. The same goes for the four names the
@@ -67,6 +67,8 @@ const COMMAND_PARSERS = new Map([
67
67
  ["scout", (_command, rest) => parseScoutArgs(rest)],
68
68
  ["ci", (_command, rest) => parseCiArgs(rest)],
69
69
  ["delivery", (_command, rest) => parseDeliveryArgs(rest)],
70
+ ["pr-output", (_command, rest) => parsePrOutputArgs(rest)],
71
+ ["ai-spend", (_command, rest) => parseAiSpendArgs(rest)],
70
72
  ["ops", (_command, rest) => parseOpsArgs(rest)],
71
73
  // BLI-23244: `slack privacy` has its own flags and its own file.
72
74
  ["slack", (_command, rest) => (rest.includes("privacy") ? parseSlackPrivacyArgs(rest.filter((arg) => arg !== "privacy")) : parseSlackArgs(rest))],
@@ -411,6 +411,17 @@ export const TOWER_OPS_COMMAND_HELP = [
411
411
  "Run `cockpit login` first if this machine is not paired.",
412
412
  ],
413
413
  ],
414
+ [
415
+ "ai-spend",
416
+ [
417
+ "Usage: cockpit ai-spend flow [--days <1-365>] [--json]",
418
+ " cockpit ai-spend people [--days <1-365>] [--json]",
419
+ " cockpit ai-spend output [--days <1-365>] [--json]",
420
+ "flow: where AI spend went, by what each session's work became: Shipped (a linked PR merged), In review (open), Closed (closed unmerged), No PR (the link check ran and found none), Active (still running) and Unlinked (no branch or commit to check), with dollars, tokens, share and the coverage of the PR link check.",
421
+ "people: per account and tool, what Tower estimated at list price from session token counts and what the tool itself reported, each labelled by source (estimated, client_reported); actual (vendor-billed) and flat-rate plan cost are named as not connected. output: list-price dollars per expert hour of the merged PRs sessions link to (an AI estimate of hours, not measured time); says no output scores yet when none exist.",
422
+ "Dollars are API list-price equivalents, not invoices. MCP twins: ai_spend_flow, ai_spend_people, ai_spend_output. Web: TEAM > AI spend.",
423
+ ],
424
+ ],
414
425
  [
415
426
  "ci",
416
427
  ["Usage: cockpit ci report [--repo <owner/name>] [--json]", "Reads the daily stored CI report, as-of age and coverage. MCP twin: ci_report."],
@@ -423,6 +434,17 @@ export const TOWER_OPS_COMMAND_HELP = [
423
434
  "--review-minutes is an assumption for the review-hours line, not a measurement. Counts show their denominators; no verdict is given. MCP twin: delivery_read. Web: TEAM > Delivery.",
424
435
  ],
425
436
  ],
437
+ [
438
+ "pr-output",
439
+ [
440
+ "Usage: cockpit pr-output list [--days <1-365>] [--limit <1-200>] [--cursor <value>] [--json]",
441
+ " cockpit pr-output correct <owner/name#number> --hours <0-2000> [--reason <text>] [--json]",
442
+ " cockpit pr-output estimate [--limit <1-25>] [--days <1-365>] [--json]",
443
+ "list: merged PRs with an AI estimate of the hours an expert engineer would have needed before AI (1 unit = 1 expert hour), a one-line reason, type, component, whether a person corrected it, and the coverage line (N of M merged PRs estimated). It is an estimate, not a measured or validated pre-AI time.",
444
+ "correct: sets your corrected hours on a PR's current estimate; the AI estimate and every earlier correction are kept. estimate: super admin only; estimates merged PRs that have no estimate for their current revision (spends model credit; a re-run on the same revision writes nothing).",
445
+ "MCP twins: pr_output_list, pr_output_correct, pr_output_estimate. Web: TEAM > Output.",
446
+ ],
447
+ ],
426
448
  [
427
449
  "ops",
428
450
  [
@@ -32,6 +32,7 @@ export const SURFACE_ALIASES = new Map([
32
32
  ["calendar", "cal"],
33
33
  ["messages", "msg"],
34
34
  ["meeting-notes", "notes"],
35
+ ["pr", "pr-output"], // BLI-23364: the MCP tools are the `pr_output_*` family
35
36
  ]);
36
37
  export const rootCommandNames = new Set([
37
38
  "roots",
@@ -54,6 +55,8 @@ export const rootCommandNames = new Set([
54
55
  "scout",
55
56
  "ci",
56
57
  "delivery",
58
+ "pr-output",
59
+ "ai-spend",
57
60
  "ops",
58
61
  "slack",
59
62
  "settings",
@@ -123,6 +126,8 @@ const COMMAND_SUMMARY_LINES = [
123
126
  " cockpit scout [board|start|dismiss|undo <experiment-id>] [--days <n>] [--dashboard-url <url>] [--json]",
124
127
  " cockpit ci report [--repo <owner/name>] [--json]",
125
128
  " cockpit delivery report [--days <1-30>] [--end <YYYY-MM-DD>] [--review-minutes <1-240>] [--json]",
129
+ " cockpit pr-output [list [--days <1-365>] [--limit <1-200>] [--cursor <value>] | correct <owner/name#number> --hours <0-2000> [--reason <text>] | estimate [--limit <1-25>] [--days <1-365>]] [--json]",
130
+ " cockpit ai-spend (flow | people | output) [--days <1-365>] [--json]",
126
131
  " cockpit ops [status [--job <id>] [--skips] [--memory] [--models] | recompile --person <email|name|id> [--dry-run]] [--dashboard-url <url>] [--json]",
127
132
  " cockpit slack [connect [--workspace bli|blue_pearl] | disconnect --workspace bli|blue_pearl | coverage [--workspace bli|blue_pearl] [--stale-only] [--list] | read [--person <p>] [--channel <c>] [--query <text> [--word]] [--since <YYYY-MM-DD>] [--until <YYYY-MM-DD>] [--limit <n>] [--format compact|full]] [--json]",
128
133
  " cockpit settings [personal [--chat-model <key>] [--brief-model <key>] | switches [set <key> <value> | set <key> --clear] | models [set --chat <key>] [--memory <id>] | cli-floor [<version>] | env list|set --project <p> --file <f> --content-stdin|delete --id <uuid> [--yes]] [--json]",
@@ -183,6 +188,7 @@ const COMMAND_SUMMARY_LINES = [
183
188
  " cockpit ticket [same verbs and flags as cockpit issue] (the Tickets page; projects: `cockpit project list`)",
184
189
  " cockpit tickets ... same as cockpit ticket",
185
190
  " cockpit work ... same as cockpit ticket (the older name)",
191
+ " cockpit pr ... same as cockpit pr-output (the MCP tools are pr_output_*)",
186
192
  " cockpit calendar [same verbs and flags as cockpit cal]",
187
193
  " cockpit messages [same verbs and flags as cockpit msg]",
188
194
  " cockpit meeting-notes [same verbs and flags as cockpit notes]",
@@ -1,6 +1,8 @@
1
1
  import { runScout } from "./scout.js";
2
2
  import { runCi } from "./ci.js";
3
3
  import { runDelivery } from "./delivery.js";
4
+ import { runPrOutput } from "./pr-output.js";
5
+ import { runAiSpend } from "./ai-spend.js";
4
6
  import { runOps } from "./ops.js";
5
7
  /** Dispatch the operational report commands. */
6
8
  export function runOpsCommand(command, io) {
@@ -8,6 +10,8 @@ export function runOpsCommand(command, io) {
8
10
  case "scout": return runScout(command, io);
9
11
  case "ci": return runCi(command, io);
10
12
  case "delivery": return runDelivery(command, io);
13
+ case "pr-output": return runPrOutput(command, io);
14
+ case "ai-spend": return runAiSpend(command, io);
11
15
  case "ops": return runOps(command, io);
12
16
  }
13
17
  }
@@ -124,6 +124,8 @@ async function runTowerCommand(command, io) {
124
124
  case "scout":
125
125
  case "ci":
126
126
  case "delivery":
127
+ case "pr-output":
128
+ case "ai-spend":
127
129
  case "ops":
128
130
  return await (await import("./local-run-ops.js")).runOpsCommand(command, io);
129
131
  case "slack": return await (await import("./slack.js")).runSlack(command, io);
@@ -0,0 +1,79 @@
1
+ import { askAgentDoor, emitAgentDoor, failAgentDoor, openAgentDoor } from "./agent-door.js";
2
+ import { writeLine } from "./cli-io.js";
3
+ import { parseNamedArgs, optionalNonEmpty, optionalUrl } from "./local-arg-values.js";
4
+ const USAGE = [
5
+ "Usage: cockpit pr-output list [--days <1-365>] [--limit <1-200>] [--cursor <value>] [--json]",
6
+ " cockpit pr-output correct <owner/name#number> --hours <0-2000> [--reason <text>] [--json]",
7
+ " cockpit pr-output estimate [--limit <1-25>] [--days <1-365>] [--json]",
8
+ ].join("\n");
9
+ function bounded(raw, min, max) {
10
+ if (raw === undefined)
11
+ return undefined;
12
+ const n = Number(raw);
13
+ if (!Number.isInteger(n) || n < min || n > max)
14
+ throw new Error(USAGE);
15
+ return n;
16
+ }
17
+ export function parseTarget(raw) {
18
+ const match = /^([A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+)#(\d+)$/.exec(raw ?? "");
19
+ if (!match || Number(match[2]) < 1)
20
+ throw new Error(USAGE);
21
+ return { repo: match[1], number: Number(match[2]) };
22
+ }
23
+ export function parsePrOutputArgs(args) {
24
+ const values = parseNamedArgs(args, {
25
+ allowedFlags: ["--days", "--limit", "--cursor", "--hours", "--reason", "--json", "--home", "--dashboard-url"],
26
+ valueFlags: ["--days", "--limit", "--cursor", "--hours", "--reason", "--home", "--dashboard-url"],
27
+ });
28
+ const [action, target, ...rest] = values.positionals;
29
+ if ((action !== "list" && action !== "correct" && action !== "estimate") || rest.length)
30
+ throw new Error(USAGE);
31
+ const common = { kind: "pr-output", action: action, json: values.booleans.has("--json"), homeDir: optionalNonEmpty(values.flags.get("--home")), dashboardUrl: optionalUrl(values.flags.get("--dashboard-url")) };
32
+ const flags = values.flags;
33
+ if (action === "correct") {
34
+ const hoursRaw = flags.get("--hours");
35
+ const hours = hoursRaw === undefined ? NaN : Number(hoursRaw);
36
+ if (!Number.isFinite(hours) || hours < 0 || hours > 2000 || flags.has("--days") || flags.has("--limit") || flags.has("--cursor"))
37
+ throw new Error(USAGE);
38
+ const reason = optionalNonEmpty(flags.get("--reason"));
39
+ if (reason !== undefined && reason.length > 600)
40
+ throw new Error(USAGE);
41
+ return { ...common, ...parseTarget(target), hours, ...(reason !== undefined ? { reason } : {}) };
42
+ }
43
+ if (target !== undefined || flags.has("--hours") || flags.has("--reason"))
44
+ throw new Error(USAGE);
45
+ if (action === "estimate") {
46
+ if (flags.has("--cursor"))
47
+ throw new Error(USAGE);
48
+ return { ...common, limit: bounded(flags.get("--limit"), 1, 25), days: bounded(flags.get("--days"), 1, 365) };
49
+ }
50
+ return { ...common, days: bounded(flags.get("--days"), 1, 365), limit: bounded(flags.get("--limit"), 1, 200), cursor: optionalNonEmpty(flags.get("--cursor")) };
51
+ }
52
+ export async function runPrOutput(command, io) {
53
+ const door = await openAgentDoor("pr-output", command, io);
54
+ let request;
55
+ if (command.action === "list") {
56
+ const query = new URLSearchParams();
57
+ if (command.days !== undefined)
58
+ query.set("days", String(command.days));
59
+ if (command.limit !== undefined)
60
+ query.set("limit", String(command.limit));
61
+ if (command.cursor !== undefined)
62
+ query.set("cursor", command.cursor);
63
+ request = { path: `/api/pr-output${query.size ? `?${query}` : ""}`, method: "GET" };
64
+ }
65
+ else if (command.action === "correct") {
66
+ request = { path: "/api/pr-output/correct", method: "POST", body: { repo: command.repo, number: command.number, hours: command.hours, ...(command.reason !== undefined ? { reason: command.reason } : {}) } };
67
+ }
68
+ else {
69
+ request = { path: "/api/pr-output/estimate", method: "POST", body: { ...(command.limit !== undefined ? { limit: command.limit } : {}), ...(command.days !== undefined ? { days: command.days } : {}) } };
70
+ }
71
+ const answer = await askAgentDoor(door, { ...request, label: `pr-output ${command.action}`, timeoutMs: command.action === "estimate" ? 300_000 : 30_000 });
72
+ if (!answer.ok)
73
+ return failAgentDoor(door, "[pr-output cli]", answer.reason, answer.detail);
74
+ if (command.json || command.action !== "list")
75
+ return emitAgentDoor(door, answer.body);
76
+ for (const line of answer.body.lines)
77
+ writeLine(io.stdout, line);
78
+ return 0;
79
+ }
@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
21
21
  }
22
22
 
23
23
  if (command === "--version" || command === "-V" || command === "version") {
24
- writeLine(io?.stdout ?? process.stdout, "0.2.278");
24
+ writeLine(io?.stdout ?? process.stdout, "0.2.279");
25
25
  return 0;
26
26
  }
27
27
 
@@ -1,7 +1,7 @@
1
1
  import { encodeChunkForStorage } from "./evidence-upload-encoding.js";
2
2
  import { fetchWithDeadline, isUploadDeadlineError, uploadDeadlineMs } from "./evidence-upload-deadline.js";
3
3
  import { noteSyncProgress } from "./sync-lock.js";
4
- export async function putSignedChunk(options, signedUrl, bytes, uploadId, index, codec = "gzip") {
4
+ export async function putSignedChunk(options, signedUrl, bytes, uploadId, index, codec = "gzip", meter) {
5
5
  // Identical manifest + codec framing to the server chunk route staging write (gzip unless the server listed more).
6
6
  const encoded = encodeChunkForStorage(bytes, codec);
7
7
  console.error("[evidence-upload] chunk encoded for storage", JSON.stringify({
@@ -23,6 +23,8 @@ export async function putSignedChunk(options, signedUrl, bytes, uploadId, index,
23
23
  await response.body?.cancel();
24
24
  if (response.ok) {
25
25
  noteSyncProgress();
26
+ if (meter)
27
+ meter.wireBytes += encoded.length;
26
28
  const elapsedMs = Date.now() - startedAt;
27
29
  log("signed_chunk_uploaded", uploadId, index, attempt, status, {
28
30
  bytes: encoded.length, elapsed_ms: elapsedMs,
@@ -1,10 +1,46 @@
1
1
  import { putSignedChunk } from "./evidence-upload-direct.js";
2
2
  import { compressEvidenceBytes } from "@bli-cockpit/telemetry-core/evidence-storage-encoding";
3
3
  import { chooseUploadEncoding } from "./evidence-upload-encoding.js";
4
- import { COMMIT_CRASHED_PLATFORM, RawEvidenceUploadCommitResponseSchema, isPermanentUploadFailure, } from "@bli-cockpit/telemetry-core";
4
+ import { COMMIT_CRASHED_PLATFORM, EXTEND_BASE_UNAVAILABLE, RawEvidenceUploadCommitResponseSchema, isPermanentUploadFailure, } from "@bli-cockpit/telemetry-core";
5
5
  import { StagedFileError, readVerifiedChunk } from "./evidence-upload-staged.js";
6
6
  import { beginOneObject } from "./evidence-upload-terminal.js";
7
7
  import { isNonJsonResponseBody, requestJson, safeFailureDetail, } from "./evidence-upload-transport.js";
8
+ /**
9
+ * The chunk plan this `begin` answer commits us to (BLI-23345): the tail plan when the server accepted
10
+ * exactly the one we offered, the whole file otherwise. A tail plan the server states differently from
11
+ * ours is not guessed at; the caller re-offers the whole object, which resets the row.
12
+ */
13
+ function planForDisposition(entry, disposition) {
14
+ const accepted = disposition.extend;
15
+ const tail = entry.tail;
16
+ if (!accepted) {
17
+ if (tail) {
18
+ console.error("[evidence-upload] tail upload not used; sending the whole object", JSON.stringify({
19
+ reason: disposition.extend_refused ? `extend_refused:${disposition.extend_refused}` : "extend_not_answered",
20
+ upload_id: disposition.upload_id, byte_size: entry.byteSize,
21
+ }));
22
+ }
23
+ return { plan: "whole", view: entry };
24
+ }
25
+ if (!tail || accepted.base_object_key !== tail.base.object_key || accepted.base_content_hash_sha256 !== tail.base.content_hash_sha256
26
+ || accepted.base_byte_size !== tail.base.byte_size || accepted.tail_offset !== tail.offset
27
+ || accepted.tail_size !== tail.byteSize || accepted.chunk_count !== tail.chunkCount) {
28
+ return { plan: "mismatch" };
29
+ }
30
+ return { plan: "tail", view: { ...entry, byteSize: tail.byteSize, chunkHashes: tail.chunkHashes, chunkCount: tail.chunkCount, readOffset: tail.offset } };
31
+ }
32
+ /**
33
+ * Re-offer the whole object from the same staged file: begin without the tail plan resets the server's
34
+ * row to a whole plan, so nothing is lost and nothing is double-counted.
35
+ */
36
+ async function reofferWhole(options, entry, chunkSizeBytes, restageAttempted, reason, uploadId) {
37
+ console.error("[evidence-upload] re-offering the whole object", JSON.stringify({ reason, upload_id: uploadId, byte_size: entry.byteSize }));
38
+ const wholeEntry = { ...entry, tail: undefined, readOffset: undefined, duplicates: [] };
39
+ const restarted = await beginOneObject(options, wholeEntry, chunkSizeBytes);
40
+ if (!restarted)
41
+ return failedOutcome(entry.file, `${reason}:begin_failed`);
42
+ return uploadOneObject(options, wholeEntry, restarted, chunkSizeBytes, restageAttempted);
43
+ }
8
44
  export async function uploadOneObject(options, entry, disposition, chunkSizeBytes, restageAttempted = false) {
9
45
  const objectKey = entry.file.pointer.object_key ?? "";
10
46
  if (disposition.disposition === "already_committed") {
@@ -15,11 +51,19 @@ export async function uploadOneObject(options, entry, disposition, chunkSizeByte
15
51
  if (disposition.disposition === "conflict" || !disposition.upload_id) {
16
52
  return failedOutcome(entry.file, disposition.reason ?? "upload_conflict");
17
53
  }
54
+ const picked = planForDisposition(entry, disposition);
55
+ if (picked.plan === "mismatch")
56
+ return reofferWhole(options, entry, chunkSizeBytes, restageAttempted, "extend_plan_mismatch", disposition.upload_id);
57
+ const view = picked.view;
18
58
  const directChunks = [];
19
- const chunked = await uploadObjectChunks(options, entry, disposition, chunkSizeBytes, objectKey, directChunks);
59
+ const chunked = await uploadObjectChunks(options, view, disposition, chunkSizeBytes, objectKey, directChunks, picked.plan, entry.byteSize);
20
60
  if (!chunked.ok)
21
61
  return chunked.outcome;
22
- const committed = await commitUploadedObject(options, entry, disposition, objectKey, chunked.uploadedChunks, directChunks);
62
+ const committed = await commitUploadedObject(options, view, disposition, objectKey, chunked.uploadedChunks, directChunks);
63
+ if (picked.plan === "tail" && committed.reason === `commit_failed_http_409_${EXTEND_BASE_UNAVAILABLE}`) {
64
+ // The base was deleted (supersede) between begin and commit. The staged whole file is intact.
65
+ return reofferWhole(options, entry, chunkSizeBytes, restageAttempted, EXTEND_BASE_UNAVAILABLE, disposition.upload_id);
66
+ }
23
67
  if (committed.reason !== "commit_failed_http_409_staging_missing_between_passes")
24
68
  return committed;
25
69
  if (restageAttempted)
@@ -36,9 +80,10 @@ export async function uploadOneObject(options, entry, disposition, chunkSizeByte
36
80
  }
37
81
  return retried;
38
82
  }
39
- async function uploadObjectChunks(options, entry, disposition, chunkSizeBytes, objectKey, directChunks) {
83
+ async function uploadObjectChunks(options, entry, disposition, chunkSizeBytes, objectKey, directChunks, plan, wholeByteSize) {
40
84
  const received = new Set(disposition.received_chunk_indexes);
41
85
  let uploadedChunks = 0;
86
+ const meter = { wireBytes: 0 };
42
87
  if (disposition.disposition === "already_committed" || disposition.commit_ready) {
43
88
  return { ok: true, uploadedChunks };
44
89
  }
@@ -55,7 +100,7 @@ async function uploadObjectChunks(options, entry, disposition, chunkSizeBytes, o
55
100
  try {
56
101
  // Re-read from disk by offset and check against the plan-time hash; the
57
102
  // buffer lives only for this iteration (BLI-23317).
58
- chunk = await readVerifiedChunk(entry.file.local_path, index * chunkSizeBytes, Math.min(chunkSizeBytes, entry.byteSize - index * chunkSizeBytes), entry.chunkHashes[index]);
103
+ chunk = await readVerifiedChunk(entry.file.local_path, (entry.readOffset ?? 0) + index * chunkSizeBytes, Math.min(chunkSizeBytes, entry.byteSize - index * chunkSizeBytes), entry.chunkHashes[index]);
59
104
  }
60
105
  catch (error) {
61
106
  const reason = error instanceof StagedFileError ? error.reason : "file_read_failed";
@@ -67,7 +112,7 @@ async function uploadObjectChunks(options, entry, disposition, chunkSizeBytes, o
67
112
  }
68
113
  const offer = disposition.signed_uploads?.find(upload => upload.chunk_index === index);
69
114
  const direct = offer
70
- ? await putSignedChunk(options, offer.signed_url, chunk, disposition.upload_id, index, encoding.codec)
115
+ ? await putSignedChunk(options, offer.signed_url, chunk, disposition.upload_id, index, encoding.codec, meter)
71
116
  : "refused";
72
117
  if (direct === "deadline") {
73
118
  // BLI-22801: the chunk route would push the same bytes over the same slow link.
@@ -82,19 +127,27 @@ async function uploadObjectChunks(options, entry, disposition, chunkSizeBytes, o
82
127
  uploadedChunks += 1;
83
128
  continue;
84
129
  }
85
- const failure = await uploadChunkThroughEndpoint(options, entry, disposition, objectKey, index, chunk, uploadedChunks, Boolean(offer), encoding.codec);
130
+ const failure = await uploadChunkThroughEndpoint(options, entry, disposition, objectKey, index, chunk, uploadedChunks, Boolean(offer), encoding.codec, meter);
86
131
  if (failure)
87
132
  return failure;
88
133
  uploadedChunks += 1;
89
134
  }
135
+ // The number the ticket is measured by: compressed bytes that actually crossed the wire for this object,
136
+ // against the whole object it replaced (sizes and labels only).
137
+ console.error("[evidence-upload] object bytes sent", JSON.stringify({
138
+ reason: "object_bytes_sent", upload_id: disposition.upload_id, plan,
139
+ whole_byte_size: wholeByteSize, planned_byte_size: entry.byteSize, chunks_sent: uploadedChunks,
140
+ wire_bytes_sent: meter.wireBytes, chunk_encoding: encoding.codec,
141
+ }));
90
142
  return { ok: true, uploadedChunks };
91
143
  }
92
- async function uploadChunkThroughEndpoint(options, entry, disposition, objectKey, index, chunk, uploadedChunks, hadSignedOffer, codec) {
144
+ async function uploadChunkThroughEndpoint(options, entry, disposition, objectKey, index, chunk, uploadedChunks, hadSignedOffer, codec, meter) {
93
145
  console.error("[evidence-upload] using chunk route", JSON.stringify({
94
146
  reason: hadSignedOffer ? "signed_upload_refused_fallback" : "signed_upload_not_offered",
95
147
  upload_id: disposition.upload_id, chunk_index: index,
96
148
  }));
97
149
  const encodedChunk = compressEvidenceBytes(chunk, codec);
150
+ meter.wireBytes += encodedChunk.length;
98
151
  console.error("[evidence-upload] chunk encoded", JSON.stringify({
99
152
  upload_id: disposition.upload_id, chunk_index: index, chunk_encoding: codec,
100
153
  bytes_before: chunk.length, bytes_after: encodedChunk.length,
@@ -4,12 +4,13 @@ import { requestJson, safeFailureDetail } from "./evidence-upload-transport.js";
4
4
  import { failedOutcome, uploadOneObject } from "./evidence-upload-object.js";
5
5
  import { duplicateOutcome, rekeyConflictedEntry, reportAbandonedUpload, } from "./evidence-upload-terminal.js";
6
6
  import { describeError } from "./health-detail.js";
7
- import { StagedFileError, fingerprintStagedFile } from "./evidence-upload-staged.js";
7
+ import { StagedFileError, fingerprintStagedFile, fingerprintStagedTail, } from "./evidence-upload-staged.js";
8
8
  export async function loadPlannedEntries(options, chunkSizeBytes, outcomes) {
9
9
  const loaded = [];
10
10
  const loadedByObjectKey = new Map();
11
11
  for (const file of options.files) {
12
- const measured = await measureStagedUpload(file, chunkSizeBytes, outcomes);
12
+ const base = tailBaseFor(file, chunkSizeBytes);
13
+ const measured = await measureStagedUpload(file, chunkSizeBytes, outcomes, base?.byte_size);
13
14
  if (!measured)
14
15
  continue;
15
16
  // Content-addressed keys collide for byte-identical files (e.g. a resumed
@@ -28,6 +29,7 @@ export async function loadPlannedEntries(options, chunkSizeBytes, outcomes) {
28
29
  objectSha256: measured.objectSha256,
29
30
  chunkHashes: measured.chunkHashes,
30
31
  chunkCount: measured.chunkHashes.length,
32
+ ...(base ? await planTail(file, base, measured, chunkSizeBytes) : {}),
31
33
  duplicates: [],
32
34
  };
33
35
  loaded.push(entry);
@@ -37,7 +39,62 @@ export async function loadPlannedEntries(options, chunkSizeBytes, outcomes) {
37
39
  }
38
40
  return loaded;
39
41
  }
40
- async function measureStagedUpload(file, chunkSizeBytes, outcomes) {
42
+ /** A transcript that extends a committed snapshot by at least one chunk's worth is worth a tail plan. */
43
+ function tailBaseFor(file, chunkSizeBytes) {
44
+ const base = file.extend_base;
45
+ if (!base || (file.pointer.byte_size ?? 0) <= base.byte_size)
46
+ return null;
47
+ if (base.byte_size < chunkSizeBytes)
48
+ return null;
49
+ return base;
50
+ }
51
+ /** The tail plan, or nothing: the base must still be exactly the staged file's first bytes. */
52
+ async function planTail(file, base, measured, chunkSizeBytes) {
53
+ if (measured.prefixSha256 !== base.content_hash_sha256) {
54
+ // The prepare step proved this prefix a moment ago; the file moved since. Whole upload, said out loud.
55
+ console.error("[evidence-upload] tail upload not planned; the staged file no longer starts with its base", JSON.stringify({
56
+ reason: "extend_prefix_changed", kind: file.kind ?? "unknown", byte_size: measured.byteSize, base_byte_size: base.byte_size,
57
+ }));
58
+ return {};
59
+ }
60
+ try {
61
+ const chunkHashes = await fingerprintStagedTail(file.local_path, measured.byteSize, base.byte_size, chunkSizeBytes);
62
+ console.error("[evidence-upload] tail planned", JSON.stringify({
63
+ reason: "extend_planned", kind: file.kind ?? "unknown", byte_size: measured.byteSize,
64
+ base_byte_size: base.byte_size, tail_byte_size: measured.byteSize - base.byte_size, tail_chunk_count: chunkHashes.length,
65
+ }));
66
+ return { tail: { base, offset: base.byte_size, byteSize: measured.byteSize - base.byte_size, chunkHashes, chunkCount: chunkHashes.length } };
67
+ }
68
+ catch (error) {
69
+ console.error("[evidence-upload] tail upload not planned; the staged file could not be read", JSON.stringify({
70
+ reason: "extend_tail_unreadable", byte_size: measured.byteSize,
71
+ detail: error instanceof StagedFileError ? error.reason : "read_error",
72
+ }));
73
+ return {};
74
+ }
75
+ }
76
+ /** One `begin` object. The tail plan rides only where `tailAllowed` (an old server answers 400 to the extra field). */
77
+ export function beginObjectFor(entry, chunkSizeBytes, tailAllowed) {
78
+ const tail = tailAllowed ? entry.tail : undefined;
79
+ return {
80
+ pointer: entry.file.pointer,
81
+ chunk_size_bytes: chunkSizeBytes,
82
+ chunk_count: tail ? tail.chunkCount : entry.chunkCount,
83
+ ...(tail ? { extend: {
84
+ base_object_key: tail.base.object_key,
85
+ base_content_hash_sha256: tail.base.content_hash_sha256,
86
+ base_byte_size: tail.base.byte_size,
87
+ } } : {}),
88
+ // BLI-23329: an image-store object names the session it is staged for.
89
+ ...(entry.file.image_use ? { image_use: entry.file.image_use } : {}),
90
+ };
91
+ }
92
+ /** Runs that learned the server does not know tail plans; the answer holds for the rest of the run. */
93
+ const serverWithoutTailPlans = new WeakSet();
94
+ export function tailPlansAllowed(options) {
95
+ return !serverWithoutTailPlans.has(options);
96
+ }
97
+ async function measureStagedUpload(file, chunkSizeBytes, outcomes, prefixAt) {
41
98
  let stat;
42
99
  try {
43
100
  stat = await fs.stat(file.local_path);
@@ -64,7 +121,7 @@ async function measureStagedUpload(file, chunkSizeBytes, outcomes) {
64
121
  return null;
65
122
  }
66
123
  try {
67
- const fingerprint = await fingerprintStagedFile(file.local_path, stat.size, chunkSizeBytes);
124
+ const fingerprint = await fingerprintStagedFile(file.local_path, stat.size, chunkSizeBytes, undefined, prefixAt);
68
125
  console.error("[evidence-upload] staged object measured", JSON.stringify({
69
126
  byte_size: fingerprint.byteSize,
70
127
  chunk_count: fingerprint.chunkHashes.length,
@@ -108,18 +165,23 @@ export function batches(items, size) {
108
165
  * so each branch stays a guard clause rather than nested conditionals.
109
166
  */
110
167
  export async function resolveBeginBatch(options, totalLoadedCount, batch, chunkSizeBytes, outcomes, resolvedEntries) {
168
+ const offersTails = tailPlansAllowed(options) && batch.some((entry) => entry.tail);
111
169
  const begin = await requestJson(options, "/api/ambient/evidence/upload/begin", {
112
170
  schema_version: "ambient-raw-evidence-upload-begin.v1",
113
171
  generated_at: options.generatedAt,
114
172
  provenance: options.provenance,
115
- objects: batch.map((entry) => ({
116
- pointer: entry.file.pointer,
117
- chunk_size_bytes: chunkSizeBytes,
118
- chunk_count: entry.chunkCount,
119
- // BLI-23329: an image-store object names the session it is staged for.
120
- ...(entry.file.image_use ? { image_use: entry.file.image_use } : {}),
121
- })),
173
+ objects: batch.map((entry) => beginObjectFor(entry, chunkSizeBytes, tailPlansAllowed(options))),
122
174
  });
175
+ if (offersTails && begin.status === 400) {
176
+ // A server from before BLI-23345 rejects the extra field. Ask again as every collector always has,
177
+ // and remember for the run; server deploys first, so this is the rollout window only.
178
+ serverWithoutTailPlans.add(options);
179
+ console.error("[evidence-upload] server does not know tail plans; offering whole objects", JSON.stringify({
180
+ reason: "extend_not_supported", http_status: begin.status, server_reason: safeFailureDetail(begin.body) ?? "none",
181
+ objects_in_batch: batch.length,
182
+ }));
183
+ return resolveBeginBatch(options, totalLoadedCount, batch, chunkSizeBytes, outcomes, resolvedEntries);
184
+ }
123
185
  // 404 means an old dashboard without the chunk routes; a persistent 5xx
124
186
  // (after retries) covers a new dashboard whose ledger migration has not
125
187
  // been applied yet. Both still serve the legacy v1 route.
@@ -243,6 +305,8 @@ export function readBeginDispositions(body) {
243
305
  received_chunk_indexes: entry.received_chunk_indexes,
244
306
  commit_ready: entry.commit_ready === true,
245
307
  signed_uploads: entry.signed_uploads,
308
+ ...(entry.extend ? { extend: entry.extend } : {}),
309
+ ...(entry.extend_refused ? { extend_refused: entry.extend_refused } : {}),
246
310
  reason: entry.reason ?? null,
247
311
  raw_evidence_pointer_id: entry.raw_evidence_pointer_id,
248
312
  ...(entry.accepted_encodings ? { accepted_encodings: entry.accepted_encodings } : {}),
@@ -11,9 +11,14 @@ export class StagedFileError extends Error {
11
11
  this.detail = detail;
12
12
  }
13
13
  }
14
- /** Read exactly `length` bytes at `position`, looping over short reads. */
15
- export async function readExactly(handle, length, position) {
16
- const buffer = Buffer.allocUnsafe(length);
14
+ /**
15
+ * Read exactly `length` bytes at `position`, looping over short reads. With `into` (at least `length` bytes) the
16
+ * read lands in that buffer and the answer is a view of it, so a loop over a big file allocates once instead of
17
+ * once per chunk (BLI-23524: per-chunk buffers are garbage the collector reclaims lazily, and a 100 MB file
18
+ * left tens of MB of it resident).
19
+ */
20
+ export async function readExactly(handle, length, position, into) {
21
+ const buffer = into ? into.subarray(0, length) : Buffer.allocUnsafe(length);
17
22
  let filled = 0;
18
23
  while (filled < length) {
19
24
  const { bytesRead } = await handle.read(buffer, filled, length - filled, position + filled);
@@ -28,25 +33,87 @@ export async function readExactly(handle, length, position) {
28
33
  * one chunk. Throws StagedFileError when the file vanishes or its length
29
34
  * moves while it is being read.
30
35
  */
31
- export async function fingerprintStagedFile(localPath, expectedSize, chunkSizeBytes, openFile = fs.open) {
36
+ export async function fingerprintStagedFile(localPath, expectedSize, chunkSizeBytes, openFile = fs.open, prefixAt) {
32
37
  const handle = await openStaged(localPath, openFile);
33
38
  try {
34
39
  const whole = crypto.createHash("sha256");
35
40
  const chunkHashes = [];
41
+ // One buffer for every chunk: each is hashed before the next read, and none is kept.
42
+ const reused = Buffer.allocUnsafeSlow(Math.min(chunkSizeBytes, Math.max(expectedSize, 1)));
43
+ let prefixSha256;
36
44
  for (let offset = 0; offset < expectedSize; offset += chunkSizeBytes) {
37
45
  const want = Math.min(chunkSizeBytes, expectedSize - offset);
38
- const chunk = await readExactly(handle, want, offset);
46
+ const chunk = await readExactly(handle, want, offset, reused);
39
47
  if (chunk.length !== want) {
40
48
  throw new StagedFileError("staged_file_changed", "file shrank while planning");
41
49
  }
42
- whole.update(chunk);
50
+ if (prefixAt !== undefined && prefixAt > offset && prefixAt <= offset + want) {
51
+ const cut = prefixAt - offset;
52
+ whole.update(chunk.subarray(0, cut));
53
+ prefixSha256 = whole.copy().digest("hex");
54
+ whole.update(chunk.subarray(cut));
55
+ }
56
+ else {
57
+ whole.update(chunk);
58
+ }
43
59
  chunkHashes.push(crypto.createHash("sha256").update(chunk).digest("hex"));
44
60
  }
45
61
  const probe = await readExactly(handle, 1, expectedSize);
46
62
  if (probe.length !== 0) {
47
63
  throw new StagedFileError("staged_file_changed", "file grew while planning");
48
64
  }
49
- return { byteSize: expectedSize, objectSha256: whole.digest("hex"), chunkHashes };
65
+ return {
66
+ byteSize: expectedSize,
67
+ objectSha256: whole.digest("hex"),
68
+ chunkHashes,
69
+ ...(prefixSha256 !== undefined ? { prefixSha256 } : {}),
70
+ };
71
+ }
72
+ finally {
73
+ await handle.close();
74
+ }
75
+ }
76
+ /**
77
+ * The chunk hashes of the TAIL of a staged file, `[start, expectedSize)` cut into chunks that begin at
78
+ * `start` (BLI-23345). One pass over the tail only; memory is one chunk.
79
+ */
80
+ export async function fingerprintStagedTail(localPath, expectedSize, start, chunkSizeBytes, openFile = fs.open) {
81
+ const handle = await openStaged(localPath, openFile);
82
+ try {
83
+ const chunkHashes = [];
84
+ const reused = Buffer.allocUnsafeSlow(Math.min(chunkSizeBytes, Math.max(expectedSize - start, 1)));
85
+ for (let offset = start; offset < expectedSize; offset += chunkSizeBytes) {
86
+ const want = Math.min(chunkSizeBytes, expectedSize - offset);
87
+ const chunk = await readExactly(handle, want, offset, reused);
88
+ if (chunk.length !== want) {
89
+ throw new StagedFileError("staged_file_changed", "file shrank while planning the tail");
90
+ }
91
+ chunkHashes.push(crypto.createHash("sha256").update(chunk).digest("hex"));
92
+ }
93
+ return chunkHashes;
94
+ }
95
+ finally {
96
+ await handle.close();
97
+ }
98
+ }
99
+ /**
100
+ * The file's bytes in order, `chunkBytes` at a time, every piece a view of the SAME buffer: valid until the next
101
+ * piece is asked for, so a consumer must use it (hash it, count it) before looping on. Memory is one chunk
102
+ * whatever the file's size; the descriptor closes when the consumer stops, however it stops. A failed read throws,
103
+ * never ends the stream early (BLI-23524).
104
+ */
105
+ export async function* chunksOfFileReusingOneBuffer(localPath, chunkBytes) {
106
+ const handle = await openStaged(localPath, fs.open);
107
+ try {
108
+ const buffer = Buffer.allocUnsafeSlow(chunkBytes);
109
+ for (let position = 0;; position += chunkBytes) {
110
+ const { bytesRead } = await handle.read(buffer, 0, chunkBytes, position);
111
+ if (bytesRead === 0)
112
+ return;
113
+ yield buffer.subarray(0, bytesRead);
114
+ if (bytesRead < chunkBytes)
115
+ return;
116
+ }
50
117
  }
51
118
  finally {
52
119
  await handle.close();
@@ -1,5 +1,5 @@
1
1
  import { RawEvidenceRedactionMetadataSchema, RawEvidenceUploadBeginResponseSchema, isImageObjectKey, isRekeyableUploadConflict, } from "@bli-cockpit/telemetry-core";
2
- import { readBeginDispositions } from "./evidence-upload-plan.js";
2
+ import { beginObjectFor, readBeginDispositions, tailPlansAllowed } from "./evidence-upload-plan.js";
3
3
  import { rekeyedEvidencePointer } from "./evidence-upload-rekey.js";
4
4
  import { requestJson, safeFailureDetail } from "./evidence-upload-transport.js";
5
5
  /**
@@ -71,14 +71,7 @@ export async function beginOneObject(options, entry, chunkSizeBytes) {
71
71
  schema_version: "ambient-raw-evidence-upload-begin.v1",
72
72
  generated_at: options.generatedAt,
73
73
  provenance: options.provenance,
74
- objects: [
75
- {
76
- pointer: entry.file.pointer,
77
- chunk_size_bytes: chunkSizeBytes,
78
- chunk_count: entry.chunkCount,
79
- ...(entry.file.image_use ? { image_use: entry.file.image_use } : {}),
80
- },
81
- ],
74
+ objects: [beginObjectFor(entry, chunkSizeBytes, tailPlansAllowed(options))],
82
75
  });
83
76
  if (!response.ok) {
84
77
  console.error("[evidence-rekey] begin refused the re-keyed object; keeping the original conflict", JSON.stringify({
@@ -17,6 +17,7 @@ import { addPdfsToFinds, mergeOccurrences, pdfsOfText } from "./pdf-transcript-s
17
17
  import { openPinnedTranscript } from "./adapters/pinned-transcript.js";
18
18
  import { openStagedOutput, readStagedText } from "./adapters/staged-output.js";
19
19
  import { TranscriptNotStreamable } from "./adapters/transcript-text-stream.js";
20
+ import { copyLinesStream } from "./slim/keep-line-copy.js";
20
21
  import { LineBytesSink, slimLineStream } from "./slim/slim-stream.js";
21
22
  import { logSlimmed, slimOrKeep, slimPassOrNull, slimTranscript } from "./slim/slim-transcript.js";
22
23
  import { imageStateKey, readDescribedImageKeys, readFailedImages, writeImageDescriptionState, } from "./image-description-state.js";
@@ -72,7 +73,7 @@ export async function openImageDescriptions(options) {
72
73
  const slim = slimOrKeep(replaced.text, input.source, { sessionId: input.sessionId });
73
74
  return { ...afterImages, text: slim.text, changed: slim.text !== input.text, slim: slim.stats };
74
75
  },
75
- prepareTranscriptFile: modeDropsImages(mode) ? (input) => prepareFileInTwoPasses(run, input) : undefined,
76
+ prepareTranscriptFile: (input) => prepareFileInTwoPasses(run, input),
76
77
  };
77
78
  }
78
79
  /** Files smaller than this are cheaper to read whole; the streamed path pays a second read. */
@@ -139,7 +140,9 @@ async function prepareFileInTwoPasses(run, input) {
139
140
  const census = await censusOfLines(source.lines());
140
141
  // Describe attempts finish and the swap decisions are settled before a single line is swapped, as with the whole text.
141
142
  // BLI-23347: the census also found the PDFs; they are converted here and swapped in the same pass as the images.
142
- const pdfFinds = await addPdfsToFinds(findsOfCensus(census, source), census.pdfs, pdfContext(run, {
143
+ const keeps = !modeDropsImages(run.mode); // BLI-23524: nothing is swapped, so no PDFs and pass two is a copy
144
+ const imageFinds = findsOfCensus(census, source);
145
+ const pdfFinds = keeps ? imageFinds : await addPdfsToFinds(imageFinds, census.pdfs, pdfContext(run, {
143
146
  stateDir: run.options.stateDir,
144
147
  read: (pdf) => source.read(pdf.base64Start, pdf.base64End - pdf.base64Start),
145
148
  changed: () => new TranscriptNotStreamable("file_changed_during_pass"),
@@ -148,14 +151,15 @@ async function prepareFileInTwoPasses(run, input) {
148
151
  // Pass two: swap and slim over exactly the bytes pass one saw, writing the result to staging as it goes.
149
152
  writer = input.stageDir ? await openStagedOutput(input.stageDir) : undefined;
150
153
  const memory = writer ? undefined : new LineBytesSink(source.length);
151
- const slim = await slimLineStream(source.lines(), input.source, mergeOccurrences(census.occurrences, finds.extraOccurrences), replace, writer ?? memory);
154
+ const slim = keeps ? await copyLinesStream(source.lines(), writer ?? memory) : await slimLineStream(source.lines(), input.source, mergeOccurrences(census.occurrences, finds.extraOccurrences), replace, writer ?? memory);
152
155
  if (slim.replacedCount !== afterImages.replacedCount)
153
156
  throw new TranscriptNotStreamable("images_changed_between_passes");
154
157
  await source.verifyUnchanged();
155
158
  const staged = await writer?.finish();
156
159
  // The sanitizer reads the whole text: from the staged file when one was written, else from the sink's buffer.
157
160
  const text = staged ? (input.omitText ? "" : await readStagedText(staged)) : memory.text();
158
- logSlimmed(input.source, input.sessionId, slim.stats, startedAt);
161
+ if (slim.stats)
162
+ logSlimmed(input.source, input.sessionId, slim.stats, startedAt);
159
163
  return { ...afterImages, text, changed: slim.changed, slim: slim.stats, readBytes: source.length, ...(staged ? { staged } : {}) };
160
164
  }
161
165
  catch (error) {
@@ -0,0 +1,34 @@
1
+ /**
2
+ * Pass two of the streamed transcript when the machine keeps its images (`describe_and_keep`, BLI-23524).
3
+ *
4
+ * Keep mode never swaps an image and never slims, so the upload is the file itself. The drop modes' pass two
5
+ * (`slim-stream.ts`) rebuilds each line; this one copies each line, in order, to the same `LineSink`, so the file
6
+ * never sits in memory beside its decoded text and its re-encoded output (a 500 MB cap meant 2 to 3 copies).
7
+ *
8
+ * Byte identity with the whole-file read is by construction: the lines are cut at 0x0a (the same cut as splitting
9
+ * the decoded text at "\n" for valid UTF-8), joined again with "\n", and decoded with the same rules as
10
+ * `Buffer.toString("utf8")` (a BOM stays a character). A file that is not valid UTF-8 is not copied at all: the
11
+ * whole-file path uploads such a file's original bytes, and only a whole read can keep them, so the caller is told
12
+ * `invalid_utf8` and reads it whole. (The census pass has already refused such a file before any image was sent;
13
+ * this check is the same one at the copy, so a file rewritten between the passes cannot slip through.)
14
+ */
15
+ import { TextDecoder } from "node:util";
16
+ import { TranscriptNotStreamable } from "../adapters/transcript-text-stream.js";
17
+ export async function copyLinesStream(lines, sink) {
18
+ const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
19
+ let count = 0;
20
+ for await (const { bytes } of lines) {
21
+ let line;
22
+ try {
23
+ line = decoder.decode(bytes);
24
+ }
25
+ catch {
26
+ throw new TranscriptNotStreamable("invalid_utf8");
27
+ }
28
+ if (count > 0)
29
+ await sink.write("\n");
30
+ count += 1;
31
+ await sink.write(line);
32
+ }
33
+ return { stats: null, replacedCount: 0, charsSaved: 0, changed: false, lines: count };
34
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bli-cockpit/cli",
3
- "version": "0.2.278",
3
+ "version": "0.2.279",
4
4
  "private": false,
5
5
  "type": "module",
6
6
  "bin": {
@@ -27,9 +27,9 @@
27
27
  "test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-cli-verb-help.mjs && node ../../scripts/assert-public-cli-surface-doors.mjs && node ../../scripts/assert-public-cli-runtime-files.mjs && node ../../scripts/assert-public-cli-exit-contract.mjs && node ../../scripts/assert-public-cli-no-fleet-posts.mjs && node ../../scripts/assert-public-cli-doctor-json.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
28
28
  },
29
29
  "dependencies": {
30
- "@bli-cockpit/memory-mcp": "0.1.85",
31
- "@bli-cockpit/mcp": "0.1.169",
32
- "@bli-cockpit/telemetry-core": "0.1.92",
30
+ "@bli-cockpit/memory-mcp": "0.1.86",
31
+ "@bli-cockpit/mcp": "0.1.170",
32
+ "@bli-cockpit/telemetry-core": "0.1.93",
33
33
  "unpdf": "1.8.1"
34
34
  },
35
35
  "repository": {