@bli-cockpit/cli 0.2.270 → 0.2.271
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/raw-evidence-sanitize-held.js +1 -1
- package/dist/adapters/raw-evidence-sanitize-lines.js +257 -0
- package/dist/adapters/raw-evidence-sanitize-staged.js +97 -0
- package/dist/adapters/raw-evidence-transcript-file.js +8 -8
- package/dist/adapters/raw-evidence-transcript-stream.js +3 -2
- package/dist/adapters/staged-lines.js +67 -0
- package/dist/commands/public-root.js +1 -1
- package/dist/image-description-session.js +1 -1
- package/package.json +1 -1
|
@@ -2,7 +2,7 @@ import { containsSecretLikeContent, redactSecretLikeContent, scannedCleanRedacti
|
|
|
2
2
|
import { createHash } from "node:crypto";
|
|
3
3
|
import { WHOLE_FILE_MASK_LINE, fallbackRedactionMetadata, logRedactionCrash, maskSecretBearingLines, } from "./raw-evidence-sanitize.js";
|
|
4
4
|
/** What `scannedCleanRedactionMetadata` gives for these bytes, from their hash and size (the same record, key for key). */
|
|
5
|
-
function scannedCleanFromFacts(facts) {
|
|
5
|
+
export function scannedCleanFromFacts(facts) {
|
|
6
6
|
return {
|
|
7
7
|
...scannedCleanRedactionMetadata(Buffer.alloc(0)),
|
|
8
8
|
original_content_hash_sha256: facts.sha256,
|
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Exact line-by-line redaction (BLI-23344): the sanitizer's decision table, run over a transcript that
|
|
3
|
+
* is never held whole.
|
|
4
|
+
*
|
|
5
|
+
* The redactor (`redactSecretLikeContent`, `containsSecretLikeContent`) is the UNCHANGED exported one.
|
|
6
|
+
* It is run on a CHUNK, and a chunk is one physical line, extended across a line boundary only when a
|
|
7
|
+
* secret rule could cross that boundary:
|
|
8
|
+
*
|
|
9
|
+
* - a newline can sit inside a match only in a run of `\s` that follows a name, a quote, `:`, `=`, `-`,
|
|
10
|
+
* `_` or `Bearer` (credential and password assignments, the bearer rule), so a boundary is UNSAFE
|
|
11
|
+
* exactly when the last non-`\s` character so far is in `[A-Za-z0-9_\-"':=]`;
|
|
12
|
+
* - or inside the `[^{]*?` body of a private key block, so a boundary is UNSAFE while a BEGIN header is
|
|
13
|
+
* open: no `{` and no END footer since it.
|
|
14
|
+
*
|
|
15
|
+
* Everywhere else a boundary is safe: no match of any of the 30 patterns contains it, and the lookbehind
|
|
16
|
+
* and `\b` context at the start of the next chunk is what the start of a string gives. So the matches are
|
|
17
|
+
* the same ones, in the same order, with the same counts. Proof, scripts and the field census (3,652
|
|
18
|
+
* real transcripts, 0 unsafe boundaries, longest chunk one line): ticket 23317 `equiv/`; the fixtures of
|
|
19
|
+
* `raw-evidence-sanitize-lines.test.ts` compare old and new byte for byte, metadata JSON included.
|
|
20
|
+
*
|
|
21
|
+
* Two passes with bounded state. Pass one reads the lines and keeps only: whether the content guard fired,
|
|
22
|
+
* the match count over all chunks, the rule counts, the first 200 ranges as ABSOLUTE UTF-16 offsets, the
|
|
23
|
+
* original hash and size, and two residual flags. The residual flag is the whole-file fallback: after
|
|
24
|
+
* line masking (or, for a secret-like file name, after inline replacement) is the text STILL secret-like?
|
|
25
|
+
* It is computed by feeding the masked text, line exactly, through a second chunker, so it needs no copy
|
|
26
|
+
* of the masked text. Pass two writes the decided output a chunk at a time; a clean file needs no pass two
|
|
27
|
+
* (the input already is the output). Memory is a function of the longest chunk, never of the file.
|
|
28
|
+
*/
|
|
29
|
+
import { RawEvidenceRedactionMetadataSchema, containsSecretLikeContent, redactSecretLikeContent, } from "@bli-cockpit/telemetry-core";
|
|
30
|
+
import { WHOLE_FILE_MASK_LINE, fallbackRedactionMetadata, maskSecretBearingLines } from "./raw-evidence-sanitize.js";
|
|
31
|
+
import { createHash } from "node:crypto";
|
|
32
|
+
const GUARD_LINE = "[REDACTED_LINE:secret_like_content_guard]\n";
|
|
33
|
+
/** The redactor lists at most this many ranges per call, and so does the whole-file metadata. */
|
|
34
|
+
const RANGE_CAP = 200;
|
|
35
|
+
const PRECURSOR = /[A-Za-z0-9_\-"':=]/;
|
|
36
|
+
const WHITESPACE = /\s/;
|
|
37
|
+
const BEGIN_HEADER = /-----BEGIN [A-Z ]*PRIVATE KEY-----/g;
|
|
38
|
+
const END_FOOTER = /-----END [A-Z ]*PRIVATE KEY-----/g;
|
|
39
|
+
/** Physical lines of an in-memory string, cut exactly where `maskSecretBearingLines` cuts them. */
|
|
40
|
+
export function* linesOfString(text) {
|
|
41
|
+
for (let position = 0; position < text.length;) {
|
|
42
|
+
const newline = text.indexOf("\n", position);
|
|
43
|
+
const end = newline === -1 ? text.length : newline + 1;
|
|
44
|
+
yield text.slice(position, end);
|
|
45
|
+
position = end;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
/** Joins lines into chunks at safe boundaries. */
|
|
49
|
+
export class Chunker {
|
|
50
|
+
parts = [];
|
|
51
|
+
length = 0;
|
|
52
|
+
lastNonSpace = "";
|
|
53
|
+
keyOpen = false;
|
|
54
|
+
maxChunk = 0;
|
|
55
|
+
chunks = 0;
|
|
56
|
+
/** Feed one physical line; returns the chunk it completes when the boundary after it is safe. */
|
|
57
|
+
feed(line) {
|
|
58
|
+
this.parts.push(line);
|
|
59
|
+
this.length += line.length;
|
|
60
|
+
for (let i = line.length - 1; i >= 0; i -= 1) {
|
|
61
|
+
const character = line[i];
|
|
62
|
+
if (!WHITESPACE.test(character)) {
|
|
63
|
+
this.lastNonSpace = character;
|
|
64
|
+
break;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
let lastAt = -1;
|
|
68
|
+
let open = this.keyOpen;
|
|
69
|
+
const brace = line.lastIndexOf("{");
|
|
70
|
+
if (brace !== -1) {
|
|
71
|
+
lastAt = brace;
|
|
72
|
+
open = false;
|
|
73
|
+
}
|
|
74
|
+
for (const match of line.matchAll(BEGIN_HEADER)) {
|
|
75
|
+
if (match.index > lastAt) {
|
|
76
|
+
lastAt = match.index;
|
|
77
|
+
open = true;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
for (const match of line.matchAll(END_FOOTER)) {
|
|
81
|
+
if (match.index > lastAt) {
|
|
82
|
+
lastAt = match.index;
|
|
83
|
+
open = false;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
this.keyOpen = open;
|
|
87
|
+
return !this.keyOpen && !PRECURSOR.test(this.lastNonSpace) ? this.take() : null;
|
|
88
|
+
}
|
|
89
|
+
/** What is held so far (the unfinished last chunk at the end of the input), or null. */
|
|
90
|
+
take() {
|
|
91
|
+
if (this.length === 0)
|
|
92
|
+
return null;
|
|
93
|
+
const chunk = this.parts.length === 1 ? this.parts[0] : this.parts.join("");
|
|
94
|
+
this.parts = [];
|
|
95
|
+
this.length = 0;
|
|
96
|
+
this.maxChunk = Math.max(this.maxChunk, chunk.length);
|
|
97
|
+
this.chunks += 1;
|
|
98
|
+
return chunk;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
export async function scanLines(source, options) {
|
|
102
|
+
const redact = options.redact ?? redactSecretLikeContent;
|
|
103
|
+
const chunker = new Chunker();
|
|
104
|
+
const lineMasked = new Chunker();
|
|
105
|
+
const inlineReplaced = new Chunker();
|
|
106
|
+
const scan = {
|
|
107
|
+
secretLike: false, matchCount: 0, ruleCounts: new Map(), ranges: [], residualAfterLineMask: false, residualAfterInline: false,
|
|
108
|
+
originalSha256: "", originalBytes: 0, textLength: 0, chunks: 0, maxChunkUnits: 0, maxResidualChunkUnits: 0,
|
|
109
|
+
};
|
|
110
|
+
const originalHash = createHash("sha256");
|
|
111
|
+
let offset = 0;
|
|
112
|
+
const feedResidual = (residual, text, flag) => {
|
|
113
|
+
for (const line of linesOfString(text)) {
|
|
114
|
+
const joined = residual.feed(line);
|
|
115
|
+
if (joined !== null && !scan[flag] && containsSecretLikeContent(joined))
|
|
116
|
+
scan[flag] = true;
|
|
117
|
+
}
|
|
118
|
+
};
|
|
119
|
+
const scanChunk = (chunk) => {
|
|
120
|
+
const encoded = Buffer.from(chunk, "utf8");
|
|
121
|
+
originalHash.update(encoded);
|
|
122
|
+
scan.originalBytes += encoded.byteLength;
|
|
123
|
+
if (!scan.secretLike && containsSecretLikeContent(chunk))
|
|
124
|
+
scan.secretLike = true;
|
|
125
|
+
const result = redact(chunk, { appliedBy: "local_collector", redactedFields: options.redactedFields });
|
|
126
|
+
const local = [];
|
|
127
|
+
if (result.redacted && result.metadata) {
|
|
128
|
+
for (const count of result.metadata.rule_counts) {
|
|
129
|
+
const known = scan.ruleCounts.get(count.rule_id) ?? { rule_id: count.rule_id, match_count: 0, redacted_char_count: 0 };
|
|
130
|
+
known.match_count += count.match_count;
|
|
131
|
+
known.redacted_char_count += count.redacted_char_count;
|
|
132
|
+
scan.ruleCounts.set(count.rule_id, known);
|
|
133
|
+
}
|
|
134
|
+
const room = Math.max(0, Math.min(RANGE_CAP - scan.ranges.length, result.metadata.redacted_ranges.length));
|
|
135
|
+
for (let i = 0; i < room; i += 1) {
|
|
136
|
+
const range = result.metadata.redacted_ranges[i];
|
|
137
|
+
local.push(range);
|
|
138
|
+
scan.ranges.push({ start: range.start + offset, end: range.end + offset, rule_id: range.rule_id });
|
|
139
|
+
}
|
|
140
|
+
scan.matchCount += result.metadata.secret_like_match_count;
|
|
141
|
+
}
|
|
142
|
+
feedResidual(lineMasked, local.length > 0 ? maskLocal(chunk, local) : chunk, "residualAfterLineMask");
|
|
143
|
+
if (options.secretLikeFileName)
|
|
144
|
+
feedResidual(inlineReplaced, result.text, "residualAfterInline");
|
|
145
|
+
offset += chunk.length;
|
|
146
|
+
};
|
|
147
|
+
for await (const line of source.lines()) {
|
|
148
|
+
const chunk = chunker.feed(line);
|
|
149
|
+
if (chunk !== null)
|
|
150
|
+
scanChunk(chunk);
|
|
151
|
+
}
|
|
152
|
+
const tail = chunker.take();
|
|
153
|
+
if (tail !== null)
|
|
154
|
+
scanChunk(tail);
|
|
155
|
+
const lineMaskedTail = lineMasked.take();
|
|
156
|
+
if (lineMaskedTail !== null && !scan.residualAfterLineMask && containsSecretLikeContent(lineMaskedTail))
|
|
157
|
+
scan.residualAfterLineMask = true;
|
|
158
|
+
const inlineTail = inlineReplaced.take();
|
|
159
|
+
if (inlineTail !== null && !scan.residualAfterInline && containsSecretLikeContent(inlineTail))
|
|
160
|
+
scan.residualAfterInline = true;
|
|
161
|
+
scan.originalSha256 = originalHash.digest("hex");
|
|
162
|
+
scan.textLength = offset;
|
|
163
|
+
scan.chunks = chunker.chunks;
|
|
164
|
+
scan.maxChunkUnits = chunker.maxChunk;
|
|
165
|
+
scan.maxResidualChunkUnits = Math.max(lineMasked.maxChunk, inlineReplaced.maxChunk);
|
|
166
|
+
return scan;
|
|
167
|
+
}
|
|
168
|
+
/** `maskSecretBearingLines` over ranges local to `chunk`; it reads nothing of the metadata but the ranges. */
|
|
169
|
+
function maskLocal(chunk, ranges) {
|
|
170
|
+
return maskSecretBearingLines(chunk, { redacted_ranges: ranges });
|
|
171
|
+
}
|
|
172
|
+
/** The whole-file metadata the redactor would have returned for the whole text, from the aggregate. */
|
|
173
|
+
function aggregateMetadata(scan, redactedFields) {
|
|
174
|
+
if (scan.matchCount === 0)
|
|
175
|
+
return null;
|
|
176
|
+
return RawEvidenceRedactionMetadataSchema.parse({
|
|
177
|
+
schema_version: "raw-evidence-redaction.v1",
|
|
178
|
+
status: "sanitized",
|
|
179
|
+
mode: "deterministic_text_replacement",
|
|
180
|
+
applied_by: ["local_collector"],
|
|
181
|
+
rule_counts: [...scan.ruleCounts.values()].sort((a, b) => a.rule_id.localeCompare(b.rule_id)),
|
|
182
|
+
secret_like_match_count: scan.matchCount,
|
|
183
|
+
redacted_fields: redactedFields,
|
|
184
|
+
redacted_ranges: scan.ranges,
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
/** Same decision table, in the same order, as `sanitizeTextEvidenceForUpload`. */
|
|
188
|
+
export function decideLinePlan(scan, options) {
|
|
189
|
+
const aggregate = aggregateMetadata(scan, options.redactedFields);
|
|
190
|
+
if (options.secretLikeFileName) {
|
|
191
|
+
const metadata = aggregate ?? fallbackRedactionMetadata({
|
|
192
|
+
ruleId: "secret_like_file_name", originalLength: scan.textLength, redactedFields: options.redactedFields, fullContentRedacted: false,
|
|
193
|
+
});
|
|
194
|
+
// Still secret-like after the redactor's own replacement (or, with no match at all, as it came): one marker line.
|
|
195
|
+
const residual = scan.matchCount > 0 ? scan.residualAfterInline : scan.secretLike;
|
|
196
|
+
return residual ? { kind: "whole_mask", label: "secret_like_name_masked", metadata } : { kind: "inline", label: "secret_like_name_masked", metadata };
|
|
197
|
+
}
|
|
198
|
+
if (scan.secretLike) {
|
|
199
|
+
const metadata = aggregate ?? fallbackRedactionMetadata({
|
|
200
|
+
ruleId: "secret_like_content_guard", originalLength: scan.textLength, redactedFields: options.redactedFields,
|
|
201
|
+
});
|
|
202
|
+
if (metadata.redacted_ranges.length === 0)
|
|
203
|
+
return { kind: "guard_line", metadata };
|
|
204
|
+
// The guard fired and the redactor found nothing: the fallback range is the whole text, every line is masked,
|
|
205
|
+
// and a text made only of marker lines holds nothing secret-like.
|
|
206
|
+
if (aggregate === null)
|
|
207
|
+
return { kind: "line_mask", metadata, maskEveryLine: true };
|
|
208
|
+
if (scan.residualAfterLineMask)
|
|
209
|
+
return { kind: "whole_mask", label: "secret_content_masked", metadata };
|
|
210
|
+
return { kind: "line_mask", metadata, maskEveryLine: false };
|
|
211
|
+
}
|
|
212
|
+
if (scan.matchCount > 0)
|
|
213
|
+
return { kind: "inline", label: "secret_content_masked", metadata: aggregate };
|
|
214
|
+
return { kind: "clean" };
|
|
215
|
+
}
|
|
216
|
+
/** The completeness label of a plan that rewrites the file; the content guard's two plans share one label. */
|
|
217
|
+
export function labelOfPlan(plan) {
|
|
218
|
+
return plan.kind === "line_mask" || plan.kind === "guard_line" ? "secret_content_masked" : plan.label;
|
|
219
|
+
}
|
|
220
|
+
/** Pass two: write the decided output a chunk at a time, on the same boundaries pass one used. */
|
|
221
|
+
export async function writeLinePlan(source, plan, options, write) {
|
|
222
|
+
if (plan.kind === "whole_mask")
|
|
223
|
+
return write(WHOLE_FILE_MASK_LINE);
|
|
224
|
+
if (plan.kind === "guard_line")
|
|
225
|
+
return write(GUARD_LINE);
|
|
226
|
+
const redact = options.redact ?? redactSecretLikeContent;
|
|
227
|
+
const ranges = plan.kind === "line_mask" ? plan.metadata.redacted_ranges : [];
|
|
228
|
+
const chunker = new Chunker();
|
|
229
|
+
let offset = 0;
|
|
230
|
+
let next = 0;
|
|
231
|
+
const emit = async (chunk) => {
|
|
232
|
+
if (plan.kind === "inline") {
|
|
233
|
+
await write(redact(chunk, { appliedBy: "local_collector", redactedFields: options.redactedFields }).text);
|
|
234
|
+
}
|
|
235
|
+
else if (plan.maskEveryLine) {
|
|
236
|
+
await write(maskLocal(chunk, [{ start: 0, end: chunk.length, rule_id: plan.metadata.redacted_ranges[0].rule_id }]));
|
|
237
|
+
}
|
|
238
|
+
else {
|
|
239
|
+
const local = [];
|
|
240
|
+
const end = offset + chunk.length;
|
|
241
|
+
for (; next < ranges.length && ranges[next].start < end; next += 1) {
|
|
242
|
+
const range = ranges[next];
|
|
243
|
+
local.push({ start: range.start - offset, end: range.end - offset, rule_id: range.rule_id });
|
|
244
|
+
}
|
|
245
|
+
await write(local.length > 0 ? maskLocal(chunk, local) : chunk);
|
|
246
|
+
}
|
|
247
|
+
offset += chunk.length;
|
|
248
|
+
};
|
|
249
|
+
for await (const line of source.lines()) {
|
|
250
|
+
const chunk = chunker.feed(line);
|
|
251
|
+
if (chunk !== null)
|
|
252
|
+
await emit(chunk);
|
|
253
|
+
}
|
|
254
|
+
const tail = chunker.take();
|
|
255
|
+
if (tail !== null)
|
|
256
|
+
await emit(tail);
|
|
257
|
+
}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { fallbackRedactionMetadata, logRedactionCrash } from "./raw-evidence-sanitize.js";
|
|
4
|
+
import { decideLinePlan, labelOfPlan, scanLines, writeLinePlan, } from "./raw-evidence-sanitize-lines.js";
|
|
5
|
+
import { scannedCleanFromFacts } from "./raw-evidence-sanitize-held.js";
|
|
6
|
+
import { linesOfStagedFile } from "./staged-lines.js";
|
|
7
|
+
import { openStagedOutput } from "./staged-output.js";
|
|
8
|
+
import { TranscriptReadError } from "./transcript-text-stream.js";
|
|
9
|
+
const TAG = "[raw-evidence-sanitize-lines]";
|
|
10
|
+
const STUB_LINE = "[REDACTED_LINE:redaction_crashed_stubbed]\n";
|
|
11
|
+
export async function sanitizeStagedTranscript(spool, options) {
|
|
12
|
+
const source = { lines: () => linesOfStagedFile(spool) };
|
|
13
|
+
const scanOptions = { redactedFields: options.redactedFields, secretLikeFileName: options.secretLikeFileName === true, redact: options.redact };
|
|
14
|
+
let output;
|
|
15
|
+
try {
|
|
16
|
+
const scan = await scanLines(source, scanOptions);
|
|
17
|
+
const plan = decideLinePlan(scan, scanOptions);
|
|
18
|
+
if (plan.kind === "clean") {
|
|
19
|
+
console.error(TAG + " transcript is clean, the spool is the upload", JSON.stringify({ reason: "clean", byte_size: spool.byteLength, chunks: scan.chunks, longest_chunk_units: scan.maxChunkUnits }));
|
|
20
|
+
return { status: "clean", staged: spool, redaction: scannedCleanFromFacts(spool) };
|
|
21
|
+
}
|
|
22
|
+
output = await openStagedOutput(path.dirname(spool.path));
|
|
23
|
+
await writeLinePlan(source, plan, scanOptions, (text) => output.write(text));
|
|
24
|
+
const written = await output.finish();
|
|
25
|
+
const redaction = {
|
|
26
|
+
...plan.metadata,
|
|
27
|
+
original_content_hash_sha256: spool.sha256,
|
|
28
|
+
sanitized_content_hash_sha256: written.sha256,
|
|
29
|
+
original_byte_size: spool.byteLength,
|
|
30
|
+
sanitized_byte_size: written.byteLength,
|
|
31
|
+
};
|
|
32
|
+
console.error(TAG + " transcript rewritten line by line", JSON.stringify({
|
|
33
|
+
reason: plan.kind,
|
|
34
|
+
label: labelOfPlan(plan),
|
|
35
|
+
byte_size: spool.byteLength,
|
|
36
|
+
uploaded_bytes: written.byteLength,
|
|
37
|
+
match_count: scan.matchCount,
|
|
38
|
+
listed_ranges: scan.ranges.length,
|
|
39
|
+
chunks: scan.chunks,
|
|
40
|
+
longest_chunk_units: scan.maxChunkUnits,
|
|
41
|
+
longest_residual_chunk_units: scan.maxResidualChunkUnits,
|
|
42
|
+
}));
|
|
43
|
+
return { status: "redacted", staged: written, redaction, completenessLabel: labelOfPlan(plan) };
|
|
44
|
+
}
|
|
45
|
+
catch (error) {
|
|
46
|
+
await output?.abandon();
|
|
47
|
+
if (error instanceof TranscriptReadError)
|
|
48
|
+
throw error;
|
|
49
|
+
logRedactionCrash(error, spool.byteLength, options.redactedFields.length);
|
|
50
|
+
return stubForCrash(source, spool, options.redactedFields);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/** The stub the whole-text path builds: its metadata names the text's length in UTF-16 units, which a counting pass gives. */
|
|
54
|
+
async function stubForCrash(source, spool, redactedFields) {
|
|
55
|
+
let units = 0;
|
|
56
|
+
for await (const line of source.lines())
|
|
57
|
+
units += line.length;
|
|
58
|
+
const bytes = Buffer.from(STUB_LINE, "utf8");
|
|
59
|
+
const sanitizedSha = createHash("sha256").update(bytes).digest("hex");
|
|
60
|
+
return {
|
|
61
|
+
status: "redacted",
|
|
62
|
+
bytes,
|
|
63
|
+
redaction: {
|
|
64
|
+
...fallbackRedactionMetadata({ ruleId: "redaction_crashed_stubbed", originalLength: units, redactedFields }),
|
|
65
|
+
original_content_hash_sha256: spool.sha256,
|
|
66
|
+
sanitized_content_hash_sha256: sanitizedSha,
|
|
67
|
+
original_byte_size: spool.byteLength,
|
|
68
|
+
sanitized_byte_size: bytes.byteLength,
|
|
69
|
+
},
|
|
70
|
+
completenessLabel: "redaction_crashed_stubbed",
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Mask secret-like values in a spooled transcript and record that it happened (the twin of
|
|
75
|
+
* `maskSecretsInHeldTranscript`: same ledger entry, same log line).
|
|
76
|
+
*/
|
|
77
|
+
export async function maskSecretsInStagedTranscript(collection, spool, options) {
|
|
78
|
+
const sanitized = await sanitizeStagedTranscript(spool, {
|
|
79
|
+
redactedFields: [`${options.kind}.body`],
|
|
80
|
+
secretLikeFileName: options.secretLikeFileName,
|
|
81
|
+
});
|
|
82
|
+
if (sanitized.status === "redacted") {
|
|
83
|
+
collection.redacted.push({
|
|
84
|
+
kind: options.kind,
|
|
85
|
+
label: options.evidenceLabel,
|
|
86
|
+
redaction: sanitized.redaction,
|
|
87
|
+
completenessLabel: sanitized.completenessLabel,
|
|
88
|
+
});
|
|
89
|
+
console.error("[raw-evidence] text evidence sanitized", JSON.stringify({
|
|
90
|
+
kind: options.kind,
|
|
91
|
+
mode: sanitized.completenessLabel,
|
|
92
|
+
original_bytes: sanitized.redaction.original_byte_size,
|
|
93
|
+
uploaded_bytes: sanitized.redaction.sanitized_byte_size,
|
|
94
|
+
}));
|
|
95
|
+
}
|
|
96
|
+
return sanitized;
|
|
97
|
+
}
|
|
@@ -22,6 +22,7 @@ import { describeError } from "../health-detail.js";
|
|
|
22
22
|
import { DELIVERY_BACKOFF_HOLDING_REASON, evidenceSourceKey, } from "../raw-evidence-staging.js";
|
|
23
23
|
import { isSecretLikePath, sha256, } from "./raw-evidence-keys.js";
|
|
24
24
|
import { maskSecretsInHeldTranscript } from "./raw-evidence-sanitize-held.js";
|
|
25
|
+
import { maskSecretsInStagedTranscript } from "./raw-evidence-sanitize-staged.js";
|
|
25
26
|
import { discardStagedOutput } from "./staged-output.js";
|
|
26
27
|
import { byteLengthOf, dropStagedBody, stageOneTranscript } from "./raw-evidence-transcript-stage.js";
|
|
27
28
|
import { agentOfKind, prepareTranscriptBody } from "./raw-evidence-transcript-body.js";
|
|
@@ -129,21 +130,20 @@ async function prepareTranscriptForUpload(collection, options, names, source) {
|
|
|
129
130
|
const body = await prepareTranscriptBody(collection, { source, kind: options.kind, sessionId: options.sessionId });
|
|
130
131
|
const redactedBefore = collection.redacted.length;
|
|
131
132
|
let sanitized;
|
|
133
|
+
const masking = { kind: options.kind, evidenceLabel: names.evidenceLabel, secretLikeFileName: names.secretLikeFileName };
|
|
132
134
|
try {
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
});
|
|
135
|
+
// BLI-23344: a spooled transcript is read back a line at a time, never as one string.
|
|
136
|
+
sanitized = body.held.staged && body.held.text === null
|
|
137
|
+
? await maskSecretsInStagedTranscript(collection, body.held.staged, masking)
|
|
138
|
+
: maskSecretsInHeldTranscript(collection, body.held, masking);
|
|
138
139
|
}
|
|
139
140
|
catch (error) {
|
|
140
141
|
if (body.held.staged)
|
|
141
142
|
await discardStagedOutput(body.held.staged, "mask_threw");
|
|
142
143
|
throw error;
|
|
143
144
|
}
|
|
144
|
-
//
|
|
145
|
-
|
|
146
|
-
if (body.held.staged && !("staged" in sanitized))
|
|
145
|
+
// Only a clean file IS the spool. Anything the mask rewrote has new bytes (a new staged file, or the crash stub's), so the spool is of no use.
|
|
146
|
+
if (body.held.staged && !("staged" in sanitized && sanitized.staged === body.held.staged))
|
|
147
147
|
await discardStagedOutput(body.held.staged, "redaction_rewrote_the_bytes");
|
|
148
148
|
// The ledger entry this mask pushed (if any), kept so a later skip can replay it.
|
|
149
149
|
const redactedEntry = collection.redacted.length > redactedBefore ? collection.redacted[collection.redacted.length - 1] : null;
|
|
@@ -35,10 +35,11 @@ export async function streamTranscriptBody(session, agent, file) {
|
|
|
35
35
|
return null;
|
|
36
36
|
}
|
|
37
37
|
try {
|
|
38
|
-
const prepared = await session.prepareTranscriptFile({ filePath: file.filePath, source: agent, sessionId: file.sessionId, maxFileBytes: file.maxFileBytes, stageDir: file.stageDir });
|
|
38
|
+
const prepared = await session.prepareTranscriptFile({ filePath: file.filePath, source: agent, sessionId: file.sessionId, maxFileBytes: file.maxFileBytes, stageDir: file.stageDir, omitText: file.stageDir !== undefined });
|
|
39
39
|
const { volatile, pendingImages } = volatilityOf(session.mode, prepared);
|
|
40
40
|
console.error(TAG + " transcript streamed through image swap and slim", JSON.stringify({ kind: file.kind, byte_size: prepared.readBytes, changed: prepared.changed, images: prepared.imageCount, volatile, staged_bytes: prepared.staged?.byteLength ?? null }));
|
|
41
|
-
|
|
41
|
+
// BLI-23344: with a spool the text is not read back; the sanitizer reads the spool line by line.
|
|
42
|
+
return { held: { text: prepared.staged ? null : prepared.text, exactBytes: null, ...(prepared.staged ? { staged: prepared.staged } : {}) }, volatile, pendingImages, readBytes: prepared.readBytes };
|
|
42
43
|
}
|
|
43
44
|
catch (error) {
|
|
44
45
|
if (error instanceof TranscriptReadError || error instanceof TranscriptTooLargeError)
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A staged file read back as physical lines, a block at a time (BLI-23344).
|
|
3
|
+
*
|
|
4
|
+
* The line-by-line redactor (`raw-evidence-sanitize-lines.ts`) reads the slimmed spool twice and never
|
|
5
|
+
* holds more than the longest chunk. A line ends just after its `\n`; the last one may not. The bytes are
|
|
6
|
+
* decoded with a streaming UTF-8 decoder, so a character split across two blocks is whole in the line, and
|
|
7
|
+
* a byte-order mark stays a character, exactly as `Buffer.toString("utf8")` of the whole file gives it. The
|
|
8
|
+
* file is valid UTF-8 (the streamed pass refuses anything else), so no byte is replaced.
|
|
9
|
+
*
|
|
10
|
+
* A read that fails, or a file shorter than its recorded length, is a `TranscriptReadError`: a local disk
|
|
11
|
+
* problem is reported as one, never mistaken for a redaction crash that would stub a good session.
|
|
12
|
+
*/
|
|
13
|
+
import fsp from "node:fs/promises";
|
|
14
|
+
import { StringDecoder } from "node:string_decoder";
|
|
15
|
+
import { describeError } from "../health-detail.js";
|
|
16
|
+
import { TranscriptReadError } from "./transcript-text-stream.js";
|
|
17
|
+
const BLOCK_BYTES = 1024 * 1024;
|
|
18
|
+
export async function* linesOfStagedFile(staged) {
|
|
19
|
+
if (staged.byteLength === 0)
|
|
20
|
+
return;
|
|
21
|
+
let handle;
|
|
22
|
+
try {
|
|
23
|
+
handle = await fsp.open(staged.path, "r");
|
|
24
|
+
}
|
|
25
|
+
catch (error) {
|
|
26
|
+
throw readFailure("open", staged, error);
|
|
27
|
+
}
|
|
28
|
+
try {
|
|
29
|
+
const decoder = new StringDecoder("utf8");
|
|
30
|
+
const block = Buffer.allocUnsafeSlow(BLOCK_BYTES);
|
|
31
|
+
let pending = [];
|
|
32
|
+
for (let position = 0; position < staged.byteLength;) {
|
|
33
|
+
let bytesRead;
|
|
34
|
+
try {
|
|
35
|
+
({ bytesRead } = await handle.read(block, 0, Math.min(BLOCK_BYTES, staged.byteLength - position), position));
|
|
36
|
+
}
|
|
37
|
+
catch (error) {
|
|
38
|
+
throw readFailure("read", staged, error);
|
|
39
|
+
}
|
|
40
|
+
if (bytesRead === 0)
|
|
41
|
+
throw readFailure("short", staged, new Error("staged_file_shorter_than_written"));
|
|
42
|
+
position += bytesRead;
|
|
43
|
+
const text = decoder.write(block.subarray(0, bytesRead));
|
|
44
|
+
let from = 0;
|
|
45
|
+
for (let newline = text.indexOf("\n", from); newline !== -1; newline = text.indexOf("\n", from)) {
|
|
46
|
+
pending.push(text.slice(from, newline + 1));
|
|
47
|
+
from = newline + 1;
|
|
48
|
+
yield pending.length === 1 ? pending[0] : pending.join("");
|
|
49
|
+
pending = [];
|
|
50
|
+
}
|
|
51
|
+
if (from < text.length)
|
|
52
|
+
pending.push(text.slice(from));
|
|
53
|
+
}
|
|
54
|
+
const rest = decoder.end();
|
|
55
|
+
if (rest.length > 0)
|
|
56
|
+
pending.push(rest);
|
|
57
|
+
if (pending.length > 0)
|
|
58
|
+
yield pending.length === 1 ? pending[0] : pending.join("");
|
|
59
|
+
}
|
|
60
|
+
finally {
|
|
61
|
+
await handle.close().catch(() => undefined);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
function readFailure(stage, staged, error) {
|
|
65
|
+
console.error("[staged-lines] staged file could not be read back", JSON.stringify({ reason: "staged_read_failed", stage, byte_size: staged.byteLength, ...describeError(error) }));
|
|
66
|
+
return new TranscriptReadError(`staged_${stage}`, error);
|
|
67
|
+
}
|
|
@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
24
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
24
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.271");
|
|
25
25
|
return 0;
|
|
26
26
|
}
|
|
27
27
|
|
|
@@ -126,7 +126,7 @@ async function prepareFileInTwoPasses(run, input) {
|
|
|
126
126
|
await source.verifyUnchanged();
|
|
127
127
|
const staged = await writer?.finish();
|
|
128
128
|
// The sanitizer reads the whole text: from the staged file when one was written, else from the sink's buffer.
|
|
129
|
-
const text = staged ? await readStagedText(staged) : memory.text();
|
|
129
|
+
const text = staged ? (input.omitText ? "" : await readStagedText(staged)) : memory.text();
|
|
130
130
|
logSlimmed(input.source, input.sessionId, slim.stats, startedAt);
|
|
131
131
|
return { ...afterImages, text, changed: slim.changed, slim: slim.stats, readBytes: source.length, ...(staged ? { staged } : {}) };
|
|
132
132
|
}
|