@bli-cockpit/cli 0.2.270 → 0.2.271

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,7 +2,7 @@ import { containsSecretLikeContent, redactSecretLikeContent, scannedCleanRedacti
2
2
  import { createHash } from "node:crypto";
3
3
  import { WHOLE_FILE_MASK_LINE, fallbackRedactionMetadata, logRedactionCrash, maskSecretBearingLines, } from "./raw-evidence-sanitize.js";
4
4
  /** What `scannedCleanRedactionMetadata` gives for these bytes, from their hash and size (the same record, key for key). */
5
- function scannedCleanFromFacts(facts) {
5
+ export function scannedCleanFromFacts(facts) {
6
6
  return {
7
7
  ...scannedCleanRedactionMetadata(Buffer.alloc(0)),
8
8
  original_content_hash_sha256: facts.sha256,
@@ -0,0 +1,257 @@
1
+ /**
2
+ * Exact line-by-line redaction (BLI-23344): the sanitizer's decision table, run over a transcript that
3
+ * is never held whole.
4
+ *
5
+ * The redactor (`redactSecretLikeContent`, `containsSecretLikeContent`) is the UNCHANGED exported one.
6
+ * It is run on a CHUNK, and a chunk is one physical line, extended across a line boundary only when a
7
+ * secret rule could cross that boundary:
8
+ *
9
+ * - a newline can sit inside a match only in a run of `\s` that follows a name, a quote, `:`, `=`, `-`,
10
+ * `_` or `Bearer` (credential and password assignments, the bearer rule), so a boundary is UNSAFE
11
+ * exactly when the last non-`\s` character so far is in `[A-Za-z0-9_\-"':=]`;
12
+ * - or inside the `[^{]*?` body of a private key block, so a boundary is UNSAFE while a BEGIN header is
13
+ * open: no `{` and no END footer since it.
14
+ *
15
+ * Everywhere else a boundary is safe: no match of any of the 30 patterns contains it, and the lookbehind
16
+ * and `\b` context at the start of the next chunk is what the start of a string gives. So the matches are
17
+ * the same ones, in the same order, with the same counts. Proof, scripts and the field census (3,652
18
+ * real transcripts, 0 unsafe boundaries, longest chunk one line): ticket 23317 `equiv/`; the fixtures of
19
+ * `raw-evidence-sanitize-lines.test.ts` compare old and new byte for byte, metadata JSON included.
20
+ *
21
+ * Two passes with bounded state. Pass one reads the lines and keeps only: whether the content guard fired,
22
+ * the match count over all chunks, the rule counts, the first 200 ranges as ABSOLUTE UTF-16 offsets, the
23
+ * original hash and size, and two residual flags. The residual flag is the whole-file fallback: after
24
+ * line masking (or, for a secret-like file name, after inline replacement) is the text STILL secret-like?
25
+ * It is computed by feeding the masked text, line exactly, through a second chunker, so it needs no copy
26
+ * of the masked text. Pass two writes the decided output a chunk at a time; a clean file needs no pass two
27
+ * (the input already is the output). Memory is a function of the longest chunk, never of the file.
28
+ */
29
+ import { RawEvidenceRedactionMetadataSchema, containsSecretLikeContent, redactSecretLikeContent, } from "@bli-cockpit/telemetry-core";
30
+ import { WHOLE_FILE_MASK_LINE, fallbackRedactionMetadata, maskSecretBearingLines } from "./raw-evidence-sanitize.js";
31
+ import { createHash } from "node:crypto";
32
+ const GUARD_LINE = "[REDACTED_LINE:secret_like_content_guard]\n";
33
+ /** The redactor lists at most this many ranges per call, and so does the whole-file metadata. */
34
+ const RANGE_CAP = 200;
35
+ const PRECURSOR = /[A-Za-z0-9_\-"':=]/;
36
+ const WHITESPACE = /\s/;
37
+ const BEGIN_HEADER = /-----BEGIN [A-Z ]*PRIVATE KEY-----/g;
38
+ const END_FOOTER = /-----END [A-Z ]*PRIVATE KEY-----/g;
39
+ /** Physical lines of an in-memory string, cut exactly where `maskSecretBearingLines` cuts them. */
40
+ export function* linesOfString(text) {
41
+ for (let position = 0; position < text.length;) {
42
+ const newline = text.indexOf("\n", position);
43
+ const end = newline === -1 ? text.length : newline + 1;
44
+ yield text.slice(position, end);
45
+ position = end;
46
+ }
47
+ }
48
+ /** Joins lines into chunks at safe boundaries. */
49
+ export class Chunker {
50
+ parts = [];
51
+ length = 0;
52
+ lastNonSpace = "";
53
+ keyOpen = false;
54
+ maxChunk = 0;
55
+ chunks = 0;
56
+ /** Feed one physical line; returns the chunk it completes when the boundary after it is safe. */
57
+ feed(line) {
58
+ this.parts.push(line);
59
+ this.length += line.length;
60
+ for (let i = line.length - 1; i >= 0; i -= 1) {
61
+ const character = line[i];
62
+ if (!WHITESPACE.test(character)) {
63
+ this.lastNonSpace = character;
64
+ break;
65
+ }
66
+ }
67
+ let lastAt = -1;
68
+ let open = this.keyOpen;
69
+ const brace = line.lastIndexOf("{");
70
+ if (brace !== -1) {
71
+ lastAt = brace;
72
+ open = false;
73
+ }
74
+ for (const match of line.matchAll(BEGIN_HEADER)) {
75
+ if (match.index > lastAt) {
76
+ lastAt = match.index;
77
+ open = true;
78
+ }
79
+ }
80
+ for (const match of line.matchAll(END_FOOTER)) {
81
+ if (match.index > lastAt) {
82
+ lastAt = match.index;
83
+ open = false;
84
+ }
85
+ }
86
+ this.keyOpen = open;
87
+ return !this.keyOpen && !PRECURSOR.test(this.lastNonSpace) ? this.take() : null;
88
+ }
89
+ /** What is held so far (the unfinished last chunk at the end of the input), or null. */
90
+ take() {
91
+ if (this.length === 0)
92
+ return null;
93
+ const chunk = this.parts.length === 1 ? this.parts[0] : this.parts.join("");
94
+ this.parts = [];
95
+ this.length = 0;
96
+ this.maxChunk = Math.max(this.maxChunk, chunk.length);
97
+ this.chunks += 1;
98
+ return chunk;
99
+ }
100
+ }
101
+ export async function scanLines(source, options) {
102
+ const redact = options.redact ?? redactSecretLikeContent;
103
+ const chunker = new Chunker();
104
+ const lineMasked = new Chunker();
105
+ const inlineReplaced = new Chunker();
106
+ const scan = {
107
+ secretLike: false, matchCount: 0, ruleCounts: new Map(), ranges: [], residualAfterLineMask: false, residualAfterInline: false,
108
+ originalSha256: "", originalBytes: 0, textLength: 0, chunks: 0, maxChunkUnits: 0, maxResidualChunkUnits: 0,
109
+ };
110
+ const originalHash = createHash("sha256");
111
+ let offset = 0;
112
+ const feedResidual = (residual, text, flag) => {
113
+ for (const line of linesOfString(text)) {
114
+ const joined = residual.feed(line);
115
+ if (joined !== null && !scan[flag] && containsSecretLikeContent(joined))
116
+ scan[flag] = true;
117
+ }
118
+ };
119
+ const scanChunk = (chunk) => {
120
+ const encoded = Buffer.from(chunk, "utf8");
121
+ originalHash.update(encoded);
122
+ scan.originalBytes += encoded.byteLength;
123
+ if (!scan.secretLike && containsSecretLikeContent(chunk))
124
+ scan.secretLike = true;
125
+ const result = redact(chunk, { appliedBy: "local_collector", redactedFields: options.redactedFields });
126
+ const local = [];
127
+ if (result.redacted && result.metadata) {
128
+ for (const count of result.metadata.rule_counts) {
129
+ const known = scan.ruleCounts.get(count.rule_id) ?? { rule_id: count.rule_id, match_count: 0, redacted_char_count: 0 };
130
+ known.match_count += count.match_count;
131
+ known.redacted_char_count += count.redacted_char_count;
132
+ scan.ruleCounts.set(count.rule_id, known);
133
+ }
134
+ const room = Math.max(0, Math.min(RANGE_CAP - scan.ranges.length, result.metadata.redacted_ranges.length));
135
+ for (let i = 0; i < room; i += 1) {
136
+ const range = result.metadata.redacted_ranges[i];
137
+ local.push(range);
138
+ scan.ranges.push({ start: range.start + offset, end: range.end + offset, rule_id: range.rule_id });
139
+ }
140
+ scan.matchCount += result.metadata.secret_like_match_count;
141
+ }
142
+ feedResidual(lineMasked, local.length > 0 ? maskLocal(chunk, local) : chunk, "residualAfterLineMask");
143
+ if (options.secretLikeFileName)
144
+ feedResidual(inlineReplaced, result.text, "residualAfterInline");
145
+ offset += chunk.length;
146
+ };
147
+ for await (const line of source.lines()) {
148
+ const chunk = chunker.feed(line);
149
+ if (chunk !== null)
150
+ scanChunk(chunk);
151
+ }
152
+ const tail = chunker.take();
153
+ if (tail !== null)
154
+ scanChunk(tail);
155
+ const lineMaskedTail = lineMasked.take();
156
+ if (lineMaskedTail !== null && !scan.residualAfterLineMask && containsSecretLikeContent(lineMaskedTail))
157
+ scan.residualAfterLineMask = true;
158
+ const inlineTail = inlineReplaced.take();
159
+ if (inlineTail !== null && !scan.residualAfterInline && containsSecretLikeContent(inlineTail))
160
+ scan.residualAfterInline = true;
161
+ scan.originalSha256 = originalHash.digest("hex");
162
+ scan.textLength = offset;
163
+ scan.chunks = chunker.chunks;
164
+ scan.maxChunkUnits = chunker.maxChunk;
165
+ scan.maxResidualChunkUnits = Math.max(lineMasked.maxChunk, inlineReplaced.maxChunk);
166
+ return scan;
167
+ }
168
+ /** `maskSecretBearingLines` over ranges local to `chunk`; it reads nothing of the metadata but the ranges. */
169
+ function maskLocal(chunk, ranges) {
170
+ return maskSecretBearingLines(chunk, { redacted_ranges: ranges });
171
+ }
172
+ /** The whole-file metadata the redactor would have returned for the whole text, from the aggregate. */
173
+ function aggregateMetadata(scan, redactedFields) {
174
+ if (scan.matchCount === 0)
175
+ return null;
176
+ return RawEvidenceRedactionMetadataSchema.parse({
177
+ schema_version: "raw-evidence-redaction.v1",
178
+ status: "sanitized",
179
+ mode: "deterministic_text_replacement",
180
+ applied_by: ["local_collector"],
181
+ rule_counts: [...scan.ruleCounts.values()].sort((a, b) => a.rule_id.localeCompare(b.rule_id)),
182
+ secret_like_match_count: scan.matchCount,
183
+ redacted_fields: redactedFields,
184
+ redacted_ranges: scan.ranges,
185
+ });
186
+ }
187
+ /** Same decision table, in the same order, as `sanitizeTextEvidenceForUpload`. */
188
+ export function decideLinePlan(scan, options) {
189
+ const aggregate = aggregateMetadata(scan, options.redactedFields);
190
+ if (options.secretLikeFileName) {
191
+ const metadata = aggregate ?? fallbackRedactionMetadata({
192
+ ruleId: "secret_like_file_name", originalLength: scan.textLength, redactedFields: options.redactedFields, fullContentRedacted: false,
193
+ });
194
+ // Still secret-like after the redactor's own replacement (or, with no match at all, as it came): one marker line.
195
+ const residual = scan.matchCount > 0 ? scan.residualAfterInline : scan.secretLike;
196
+ return residual ? { kind: "whole_mask", label: "secret_like_name_masked", metadata } : { kind: "inline", label: "secret_like_name_masked", metadata };
197
+ }
198
+ if (scan.secretLike) {
199
+ const metadata = aggregate ?? fallbackRedactionMetadata({
200
+ ruleId: "secret_like_content_guard", originalLength: scan.textLength, redactedFields: options.redactedFields,
201
+ });
202
+ if (metadata.redacted_ranges.length === 0)
203
+ return { kind: "guard_line", metadata };
204
+ // The guard fired and the redactor found nothing: the fallback range is the whole text, every line is masked,
205
+ // and a text made only of marker lines holds nothing secret-like.
206
+ if (aggregate === null)
207
+ return { kind: "line_mask", metadata, maskEveryLine: true };
208
+ if (scan.residualAfterLineMask)
209
+ return { kind: "whole_mask", label: "secret_content_masked", metadata };
210
+ return { kind: "line_mask", metadata, maskEveryLine: false };
211
+ }
212
+ if (scan.matchCount > 0)
213
+ return { kind: "inline", label: "secret_content_masked", metadata: aggregate };
214
+ return { kind: "clean" };
215
+ }
216
+ /** The completeness label of a plan that rewrites the file; the content guard's two plans share one label. */
217
+ export function labelOfPlan(plan) {
218
+ return plan.kind === "line_mask" || plan.kind === "guard_line" ? "secret_content_masked" : plan.label;
219
+ }
220
+ /** Pass two: write the decided output a chunk at a time, on the same boundaries pass one used. */
221
+ export async function writeLinePlan(source, plan, options, write) {
222
+ if (plan.kind === "whole_mask")
223
+ return write(WHOLE_FILE_MASK_LINE);
224
+ if (plan.kind === "guard_line")
225
+ return write(GUARD_LINE);
226
+ const redact = options.redact ?? redactSecretLikeContent;
227
+ const ranges = plan.kind === "line_mask" ? plan.metadata.redacted_ranges : [];
228
+ const chunker = new Chunker();
229
+ let offset = 0;
230
+ let next = 0;
231
+ const emit = async (chunk) => {
232
+ if (plan.kind === "inline") {
233
+ await write(redact(chunk, { appliedBy: "local_collector", redactedFields: options.redactedFields }).text);
234
+ }
235
+ else if (plan.maskEveryLine) {
236
+ await write(maskLocal(chunk, [{ start: 0, end: chunk.length, rule_id: plan.metadata.redacted_ranges[0].rule_id }]));
237
+ }
238
+ else {
239
+ const local = [];
240
+ const end = offset + chunk.length;
241
+ for (; next < ranges.length && ranges[next].start < end; next += 1) {
242
+ const range = ranges[next];
243
+ local.push({ start: range.start - offset, end: range.end - offset, rule_id: range.rule_id });
244
+ }
245
+ await write(local.length > 0 ? maskLocal(chunk, local) : chunk);
246
+ }
247
+ offset += chunk.length;
248
+ };
249
+ for await (const line of source.lines()) {
250
+ const chunk = chunker.feed(line);
251
+ if (chunk !== null)
252
+ await emit(chunk);
253
+ }
254
+ const tail = chunker.take();
255
+ if (tail !== null)
256
+ await emit(tail);
257
+ }
@@ -0,0 +1,97 @@
1
+ import { createHash } from "node:crypto";
2
+ import path from "node:path";
3
+ import { fallbackRedactionMetadata, logRedactionCrash } from "./raw-evidence-sanitize.js";
4
+ import { decideLinePlan, labelOfPlan, scanLines, writeLinePlan, } from "./raw-evidence-sanitize-lines.js";
5
+ import { scannedCleanFromFacts } from "./raw-evidence-sanitize-held.js";
6
+ import { linesOfStagedFile } from "./staged-lines.js";
7
+ import { openStagedOutput } from "./staged-output.js";
8
+ import { TranscriptReadError } from "./transcript-text-stream.js";
9
+ const TAG = "[raw-evidence-sanitize-lines]";
10
+ const STUB_LINE = "[REDACTED_LINE:redaction_crashed_stubbed]\n";
11
+ export async function sanitizeStagedTranscript(spool, options) {
12
+ const source = { lines: () => linesOfStagedFile(spool) };
13
+ const scanOptions = { redactedFields: options.redactedFields, secretLikeFileName: options.secretLikeFileName === true, redact: options.redact };
14
+ let output;
15
+ try {
16
+ const scan = await scanLines(source, scanOptions);
17
+ const plan = decideLinePlan(scan, scanOptions);
18
+ if (plan.kind === "clean") {
19
+ console.error(TAG + " transcript is clean, the spool is the upload", JSON.stringify({ reason: "clean", byte_size: spool.byteLength, chunks: scan.chunks, longest_chunk_units: scan.maxChunkUnits }));
20
+ return { status: "clean", staged: spool, redaction: scannedCleanFromFacts(spool) };
21
+ }
22
+ output = await openStagedOutput(path.dirname(spool.path));
23
+ await writeLinePlan(source, plan, scanOptions, (text) => output.write(text));
24
+ const written = await output.finish();
25
+ const redaction = {
26
+ ...plan.metadata,
27
+ original_content_hash_sha256: spool.sha256,
28
+ sanitized_content_hash_sha256: written.sha256,
29
+ original_byte_size: spool.byteLength,
30
+ sanitized_byte_size: written.byteLength,
31
+ };
32
+ console.error(TAG + " transcript rewritten line by line", JSON.stringify({
33
+ reason: plan.kind,
34
+ label: labelOfPlan(plan),
35
+ byte_size: spool.byteLength,
36
+ uploaded_bytes: written.byteLength,
37
+ match_count: scan.matchCount,
38
+ listed_ranges: scan.ranges.length,
39
+ chunks: scan.chunks,
40
+ longest_chunk_units: scan.maxChunkUnits,
41
+ longest_residual_chunk_units: scan.maxResidualChunkUnits,
42
+ }));
43
+ return { status: "redacted", staged: written, redaction, completenessLabel: labelOfPlan(plan) };
44
+ }
45
+ catch (error) {
46
+ await output?.abandon();
47
+ if (error instanceof TranscriptReadError)
48
+ throw error;
49
+ logRedactionCrash(error, spool.byteLength, options.redactedFields.length);
50
+ return stubForCrash(source, spool, options.redactedFields);
51
+ }
52
+ }
53
+ /** The stub the whole-text path builds: its metadata names the text's length in UTF-16 units, which a counting pass gives. */
54
+ async function stubForCrash(source, spool, redactedFields) {
55
+ let units = 0;
56
+ for await (const line of source.lines())
57
+ units += line.length;
58
+ const bytes = Buffer.from(STUB_LINE, "utf8");
59
+ const sanitizedSha = createHash("sha256").update(bytes).digest("hex");
60
+ return {
61
+ status: "redacted",
62
+ bytes,
63
+ redaction: {
64
+ ...fallbackRedactionMetadata({ ruleId: "redaction_crashed_stubbed", originalLength: units, redactedFields }),
65
+ original_content_hash_sha256: spool.sha256,
66
+ sanitized_content_hash_sha256: sanitizedSha,
67
+ original_byte_size: spool.byteLength,
68
+ sanitized_byte_size: bytes.byteLength,
69
+ },
70
+ completenessLabel: "redaction_crashed_stubbed",
71
+ };
72
+ }
73
+ /**
74
+ * Mask secret-like values in a spooled transcript and record that it happened (the twin of
75
+ * `maskSecretsInHeldTranscript`: same ledger entry, same log line).
76
+ */
77
+ export async function maskSecretsInStagedTranscript(collection, spool, options) {
78
+ const sanitized = await sanitizeStagedTranscript(spool, {
79
+ redactedFields: [`${options.kind}.body`],
80
+ secretLikeFileName: options.secretLikeFileName,
81
+ });
82
+ if (sanitized.status === "redacted") {
83
+ collection.redacted.push({
84
+ kind: options.kind,
85
+ label: options.evidenceLabel,
86
+ redaction: sanitized.redaction,
87
+ completenessLabel: sanitized.completenessLabel,
88
+ });
89
+ console.error("[raw-evidence] text evidence sanitized", JSON.stringify({
90
+ kind: options.kind,
91
+ mode: sanitized.completenessLabel,
92
+ original_bytes: sanitized.redaction.original_byte_size,
93
+ uploaded_bytes: sanitized.redaction.sanitized_byte_size,
94
+ }));
95
+ }
96
+ return sanitized;
97
+ }
@@ -22,6 +22,7 @@ import { describeError } from "../health-detail.js";
22
22
  import { DELIVERY_BACKOFF_HOLDING_REASON, evidenceSourceKey, } from "../raw-evidence-staging.js";
23
23
  import { isSecretLikePath, sha256, } from "./raw-evidence-keys.js";
24
24
  import { maskSecretsInHeldTranscript } from "./raw-evidence-sanitize-held.js";
25
+ import { maskSecretsInStagedTranscript } from "./raw-evidence-sanitize-staged.js";
25
26
  import { discardStagedOutput } from "./staged-output.js";
26
27
  import { byteLengthOf, dropStagedBody, stageOneTranscript } from "./raw-evidence-transcript-stage.js";
27
28
  import { agentOfKind, prepareTranscriptBody } from "./raw-evidence-transcript-body.js";
@@ -129,21 +130,20 @@ async function prepareTranscriptForUpload(collection, options, names, source) {
129
130
  const body = await prepareTranscriptBody(collection, { source, kind: options.kind, sessionId: options.sessionId });
130
131
  const redactedBefore = collection.redacted.length;
131
132
  let sanitized;
133
+ const masking = { kind: options.kind, evidenceLabel: names.evidenceLabel, secretLikeFileName: names.secretLikeFileName };
132
134
  try {
133
- sanitized = maskSecretsInHeldTranscript(collection, body.held, {
134
- kind: options.kind,
135
- evidenceLabel: names.evidenceLabel,
136
- secretLikeFileName: names.secretLikeFileName,
137
- });
135
+ // BLI-23344: a spooled transcript is read back a line at a time, never as one string.
136
+ sanitized = body.held.staged && body.held.text === null
137
+ ? await maskSecretsInStagedTranscript(collection, body.held.staged, masking)
138
+ : maskSecretsInHeldTranscript(collection, body.held, masking);
138
139
  }
139
140
  catch (error) {
140
141
  if (body.held.staged)
141
142
  await discardStagedOutput(body.held.staged, "mask_threw");
142
143
  throw error;
143
144
  }
144
- // BLI-23317 stage D: only a clean file IS its staged bytes. Anything the mask rewrote has new bytes, built
145
- // from the sanitizer's text, so the streamed partial is of no use.
146
- if (body.held.staged && !("staged" in sanitized))
145
+ // Only a clean file IS the spool. Anything the mask rewrote has new bytes (a new staged file, or the crash stub's), so the spool is of no use.
146
+ if (body.held.staged && !("staged" in sanitized && sanitized.staged === body.held.staged))
147
147
  await discardStagedOutput(body.held.staged, "redaction_rewrote_the_bytes");
148
148
  // The ledger entry this mask pushed (if any), kept so a later skip can replay it.
149
149
  const redactedEntry = collection.redacted.length > redactedBefore ? collection.redacted[collection.redacted.length - 1] : null;
@@ -35,10 +35,11 @@ export async function streamTranscriptBody(session, agent, file) {
35
35
  return null;
36
36
  }
37
37
  try {
38
- const prepared = await session.prepareTranscriptFile({ filePath: file.filePath, source: agent, sessionId: file.sessionId, maxFileBytes: file.maxFileBytes, stageDir: file.stageDir });
38
+ const prepared = await session.prepareTranscriptFile({ filePath: file.filePath, source: agent, sessionId: file.sessionId, maxFileBytes: file.maxFileBytes, stageDir: file.stageDir, omitText: file.stageDir !== undefined });
39
39
  const { volatile, pendingImages } = volatilityOf(session.mode, prepared);
40
40
  console.error(TAG + " transcript streamed through image swap and slim", JSON.stringify({ kind: file.kind, byte_size: prepared.readBytes, changed: prepared.changed, images: prepared.imageCount, volatile, staged_bytes: prepared.staged?.byteLength ?? null }));
41
- return { held: { text: prepared.text, exactBytes: null, ...(prepared.staged ? { staged: prepared.staged } : {}) }, volatile, pendingImages, readBytes: prepared.readBytes };
41
+ // BLI-23344: with a spool the text is not read back; the sanitizer reads the spool line by line.
42
+ return { held: { text: prepared.staged ? null : prepared.text, exactBytes: null, ...(prepared.staged ? { staged: prepared.staged } : {}) }, volatile, pendingImages, readBytes: prepared.readBytes };
42
43
  }
43
44
  catch (error) {
44
45
  if (error instanceof TranscriptReadError || error instanceof TranscriptTooLargeError)
@@ -0,0 +1,67 @@
1
+ /**
2
+ * A staged file read back as physical lines, a block at a time (BLI-23344).
3
+ *
4
+ * The line-by-line redactor (`raw-evidence-sanitize-lines.ts`) reads the slimmed spool twice and never
5
+ * holds more than the longest chunk. A line ends just after its `\n`; the last one may not. The bytes are
6
+ * decoded with a streaming UTF-8 decoder, so a character split across two blocks is whole in the line, and
7
+ * a byte-order mark stays a character, exactly as `Buffer.toString("utf8")` of the whole file gives it. The
8
+ * file is valid UTF-8 (the streamed pass refuses anything else), so no byte is replaced.
9
+ *
10
+ * A read that fails, or a file shorter than its recorded length, is a `TranscriptReadError`: a local disk
11
+ * problem is reported as one, never mistaken for a redaction crash that would stub a good session.
12
+ */
13
+ import fsp from "node:fs/promises";
14
+ import { StringDecoder } from "node:string_decoder";
15
+ import { describeError } from "../health-detail.js";
16
+ import { TranscriptReadError } from "./transcript-text-stream.js";
17
+ const BLOCK_BYTES = 1024 * 1024;
18
+ export async function* linesOfStagedFile(staged) {
19
+ if (staged.byteLength === 0)
20
+ return;
21
+ let handle;
22
+ try {
23
+ handle = await fsp.open(staged.path, "r");
24
+ }
25
+ catch (error) {
26
+ throw readFailure("open", staged, error);
27
+ }
28
+ try {
29
+ const decoder = new StringDecoder("utf8");
30
+ const block = Buffer.allocUnsafeSlow(BLOCK_BYTES);
31
+ let pending = [];
32
+ for (let position = 0; position < staged.byteLength;) {
33
+ let bytesRead;
34
+ try {
35
+ ({ bytesRead } = await handle.read(block, 0, Math.min(BLOCK_BYTES, staged.byteLength - position), position));
36
+ }
37
+ catch (error) {
38
+ throw readFailure("read", staged, error);
39
+ }
40
+ if (bytesRead === 0)
41
+ throw readFailure("short", staged, new Error("staged_file_shorter_than_written"));
42
+ position += bytesRead;
43
+ const text = decoder.write(block.subarray(0, bytesRead));
44
+ let from = 0;
45
+ for (let newline = text.indexOf("\n", from); newline !== -1; newline = text.indexOf("\n", from)) {
46
+ pending.push(text.slice(from, newline + 1));
47
+ from = newline + 1;
48
+ yield pending.length === 1 ? pending[0] : pending.join("");
49
+ pending = [];
50
+ }
51
+ if (from < text.length)
52
+ pending.push(text.slice(from));
53
+ }
54
+ const rest = decoder.end();
55
+ if (rest.length > 0)
56
+ pending.push(rest);
57
+ if (pending.length > 0)
58
+ yield pending.length === 1 ? pending[0] : pending.join("");
59
+ }
60
+ finally {
61
+ await handle.close().catch(() => undefined);
62
+ }
63
+ }
64
+ function readFailure(stage, staged, error) {
65
+ console.error("[staged-lines] staged file could not be read back", JSON.stringify({ reason: "staged_read_failed", stage, byte_size: staged.byteLength, ...describeError(error) }));
66
+ return new TranscriptReadError(`staged_${stage}`, error);
67
+ }
@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
21
21
  }
22
22
 
23
23
  if (command === "--version" || command === "-V" || command === "version") {
24
- writeLine(io?.stdout ?? process.stdout, "0.2.270");
24
+ writeLine(io?.stdout ?? process.stdout, "0.2.271");
25
25
  return 0;
26
26
  }
27
27
 
@@ -126,7 +126,7 @@ async function prepareFileInTwoPasses(run, input) {
126
126
  await source.verifyUnchanged();
127
127
  const staged = await writer?.finish();
128
128
  // The sanitizer reads the whole text: from the staged file when one was written, else from the sink's buffer.
129
- const text = staged ? await readStagedText(staged) : memory.text();
129
+ const text = staged ? (input.omitText ? "" : await readStagedText(staged)) : memory.text();
130
130
  logSlimmed(input.source, input.sessionId, slim.stats, startedAt);
131
131
  return { ...afterImages, text, changed: slim.changed, slim: slim.stats, readBytes: source.length, ...(staged ? { staged } : {}) };
132
132
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bli-cockpit/cli",
3
- "version": "0.2.270",
3
+ "version": "0.2.271",
4
4
  "private": false,
5
5
  "type": "module",
6
6
  "bin": {