@sema-agent/primitives 7.67.2 → 7.68.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -21
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/redaction-marks.d.ts +49 -0
- package/dist/redaction-marks.js +40 -0
- package/dist/secret-scan.d.ts +69 -52
- package/dist/secret-scan.js +210 -98
- package/dist/secret-vocabulary.d.ts +39 -5
- package/dist/secret-vocabulary.js +28 -21
- package/package.json +2 -2
package/dist/secret-scan.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { ENGINE_VOCABULARY, egressPassGroups } from "./secret-vocabulary.js";
|
|
2
2
|
import { percentDecodedOnce, readReadings, readingViews, readsTextReadings } from "./secret-readings.js";
|
|
3
|
+
import { REDACTION_MARKS, eachRedactionMarkerSpan, isRedactionMark, redactionMarkerSpans } from "./redaction-marks.js";
|
|
4
|
+
const OVERSIZED = REDACTION_MARKS.oversized;
|
|
3
5
|
function passZeroMapper(batches) {
|
|
4
6
|
const cursors = batches.map(() => ({ i: 0, delta: 0 }));
|
|
5
7
|
return (pos) => {
|
|
@@ -27,12 +29,23 @@ function passZeroMapper(batches) {
|
|
|
27
29
|
}
|
|
28
30
|
const FORMAT_CHAR_RE = /\p{Cf}/u;
|
|
29
31
|
const MAX_FORMAT_SLOT_SCANS = 64;
|
|
30
|
-
function finding(pass, span, family) {
|
|
31
|
-
const f = { kind: pass.kind, confidence: pass.confidence, span, marker
|
|
32
|
+
function finding(pass, span, family, marker) {
|
|
33
|
+
const f = { kind: pass.kind, confidence: pass.confidence, span, marker, source: pass.source };
|
|
32
34
|
if (family !== undefined)
|
|
33
35
|
f.family = family;
|
|
34
36
|
return f;
|
|
35
37
|
}
|
|
38
|
+
function lastMark(inserted) {
|
|
39
|
+
const spans = redactionMarkerSpans(inserted);
|
|
40
|
+
return spans[spans.length - 1];
|
|
41
|
+
}
|
|
42
|
+
function markOfHit(pass, written) {
|
|
43
|
+
return isRedactionMark(written) ? written : pass.marker;
|
|
44
|
+
}
|
|
45
|
+
function writtenOf(pass, match, groups, inserted) {
|
|
46
|
+
const [a, b] = removedRegion(pass, match, groups, inserted);
|
|
47
|
+
return inserted.slice(a, inserted.length - (match.length - b));
|
|
48
|
+
}
|
|
36
49
|
function insertedFor(pass, match, groups) {
|
|
37
50
|
return typeof pass.replace === "string"
|
|
38
51
|
? pass.replace.replace(/\$(\d)/g, (_m, d) => groups[Number(d) - 1] ?? "")
|
|
@@ -60,9 +73,9 @@ function removedRegion(pass, match, groups, inserted) {
|
|
|
60
73
|
tail = "";
|
|
61
74
|
return [head.length, match.length - tail.length];
|
|
62
75
|
}
|
|
63
|
-
const
|
|
64
|
-
const headStop =
|
|
65
|
-
const tailStop =
|
|
76
|
+
const mark = lastMark(inserted);
|
|
77
|
+
const headStop = mark === undefined ? inserted.length : mark[0];
|
|
78
|
+
const tailStop = mark === undefined ? inserted.length : inserted.length - mark[1];
|
|
66
79
|
let head = 0;
|
|
67
80
|
while (head < match.length && head < headStop && match[head] === inserted[head])
|
|
68
81
|
head++;
|
|
@@ -77,7 +90,7 @@ function appendAll(into, items) {
|
|
|
77
90
|
}
|
|
78
91
|
const SOLO_SCAN_WORK_PER_CHAR = 8;
|
|
79
92
|
const SOLO_SCAN_WORK_FLOOR = 1 << 17;
|
|
80
|
-
function soloHits(input, passes
|
|
93
|
+
function soloHits(input, passes) {
|
|
81
94
|
const hits = [];
|
|
82
95
|
passes.forEach((pass, i) => {
|
|
83
96
|
const re = new RegExp(pass.re.source, pass.re.flags.includes("g") ? pass.re.flags : `${pass.re.flags}g`);
|
|
@@ -89,7 +102,7 @@ function soloHits(input, passes, overStrip) {
|
|
|
89
102
|
if (rescans) {
|
|
90
103
|
work -= m[0].length;
|
|
91
104
|
if (work < 0) {
|
|
92
|
-
hits.push({ start: at, end: input.length, marker:
|
|
105
|
+
hits.push({ start: at, end: input.length, marker: OVERSIZED, pass: i, family: familyAt(), matchStart: at, matchEnd: input.length });
|
|
93
106
|
break;
|
|
94
107
|
}
|
|
95
108
|
re.lastIndex = at + 1;
|
|
@@ -100,7 +113,7 @@ function soloHits(input, passes, overStrip) {
|
|
|
100
113
|
if (inserted === m[0])
|
|
101
114
|
continue;
|
|
102
115
|
const [a, b] = removedRegion(pass, m[0], m.slice(1), inserted);
|
|
103
|
-
if (b <= a)
|
|
116
|
+
if (b <= a || isRedactionMark(m[0].slice(a, b)))
|
|
104
117
|
continue;
|
|
105
118
|
const family = familyAt();
|
|
106
119
|
hits.push({ start: at + a, end: at + b, marker: inserted.slice(a, inserted.length - (m[0].length - b)), pass: i, family, matchStart: at, matchEnd: at + m[0].length });
|
|
@@ -146,8 +159,56 @@ function sequentialRemovals(batches) {
|
|
|
146
159
|
}
|
|
147
160
|
return removals;
|
|
148
161
|
}
|
|
149
|
-
function
|
|
150
|
-
|
|
162
|
+
function firstMarkEndingAfter(marks, pos) {
|
|
163
|
+
let lo = 0;
|
|
164
|
+
let hi = marks.length;
|
|
165
|
+
while (lo < hi) {
|
|
166
|
+
const mid = (lo + hi) >> 1;
|
|
167
|
+
if (marks[mid][1] > pos)
|
|
168
|
+
hi = mid;
|
|
169
|
+
else
|
|
170
|
+
lo = mid + 1;
|
|
171
|
+
}
|
|
172
|
+
return lo;
|
|
173
|
+
}
|
|
174
|
+
function reachesIntoMark(removals, marks) {
|
|
175
|
+
if (marks.length === 0)
|
|
176
|
+
return false;
|
|
177
|
+
return removals.some((r) => {
|
|
178
|
+
if (r.end <= r.start)
|
|
179
|
+
return false;
|
|
180
|
+
const m = marks[firstMarkEndingAfter(marks, r.start)];
|
|
181
|
+
return m !== undefined && m[0] < r.end && (m[0] !== r.start || m[1] !== r.end);
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
function atomicRemovals(input, removals) {
|
|
185
|
+
const marks = redactionMarkerSpans(input);
|
|
186
|
+
const kept = [];
|
|
187
|
+
for (const r of removals) {
|
|
188
|
+
if (r.end <= r.start) {
|
|
189
|
+
if (r.marker === "")
|
|
190
|
+
continue;
|
|
191
|
+
const m = marks.length === 0 ? undefined : marks[firstMarkEndingAfter(marks, r.start)];
|
|
192
|
+
kept.push(m !== undefined && m[0] < r.start ? { ...r, start: m[0], end: m[1] } : r);
|
|
193
|
+
continue;
|
|
194
|
+
}
|
|
195
|
+
if (marks.length === 0) {
|
|
196
|
+
kept.push(r);
|
|
197
|
+
continue;
|
|
198
|
+
}
|
|
199
|
+
if (isRedactionMark(input.slice(r.start, r.end)))
|
|
200
|
+
continue;
|
|
201
|
+
let { start, end } = r;
|
|
202
|
+
for (let k = firstMarkEndingAfter(marks, r.start); k < marks.length && marks[k][0] < r.end; k++) {
|
|
203
|
+
start = Math.min(start, marks[k][0]);
|
|
204
|
+
end = Math.max(end, marks[k][1]);
|
|
205
|
+
}
|
|
206
|
+
kept.push(start === r.start && end === r.end ? r : { ...r, start, end });
|
|
207
|
+
}
|
|
208
|
+
return kept;
|
|
209
|
+
}
|
|
210
|
+
function renderRemovals(input, removals) {
|
|
211
|
+
const sorted = atomicRemovals(input, removals).sort((x, y) => x.start - y.start || y.end - x.end);
|
|
151
212
|
const edits = [];
|
|
152
213
|
let out = "";
|
|
153
214
|
let cursor = 0;
|
|
@@ -155,14 +216,14 @@ function renderRemovals(input, removals, overStrip) {
|
|
|
155
216
|
const start = sorted[i].start;
|
|
156
217
|
let end = sorted[i].end;
|
|
157
218
|
let widest;
|
|
158
|
-
for (; i < sorted.length &&
|
|
219
|
+
for (; i < sorted.length && sorted[i].start <= end; i++) {
|
|
159
220
|
const r = sorted[i];
|
|
160
221
|
if (r.end > end)
|
|
161
222
|
end = r.end;
|
|
162
223
|
if (r.marker !== "" && (widest === undefined || r.end - r.start > widest.end - widest.start || (r.start === widest.start && r.end === widest.end && r.pass > widest.pass)))
|
|
163
224
|
widest = r;
|
|
164
225
|
}
|
|
165
|
-
const marker = widest?.marker ??
|
|
226
|
+
const marker = widest?.marker ?? OVERSIZED;
|
|
166
227
|
out += input.slice(cursor, start) + marker;
|
|
167
228
|
cursor = end;
|
|
168
229
|
edits.push({ at: start, removedLen: end - start, insertedLen: marker.length, inserted: marker, family: widest?.family, head: 0, tail: 0, pass: widest?.pass ?? sorted[i - 1].pass });
|
|
@@ -172,10 +233,12 @@ function renderRemovals(input, removals, overStrip) {
|
|
|
172
233
|
function runPasses(input, passes, findings, toOriginal) {
|
|
173
234
|
const batches = [];
|
|
174
235
|
let cur = input;
|
|
236
|
+
let cutsMark = false;
|
|
175
237
|
passes.forEach((pass, passIndex) => {
|
|
176
238
|
const edits = [];
|
|
177
239
|
const passInput = cur;
|
|
178
240
|
const toPassZero = passZeroMapper(batches.slice());
|
|
241
|
+
let marksOfInput;
|
|
179
242
|
cur = cur.replace(pass.re, (...args) => {
|
|
180
243
|
const match = args[0];
|
|
181
244
|
const offset = args[args.length - 2];
|
|
@@ -183,16 +246,30 @@ function runPasses(input, passes, findings, toOriginal) {
|
|
|
183
246
|
const inserted = insertedFor(pass, match, groups);
|
|
184
247
|
if (inserted === match)
|
|
185
248
|
return inserted;
|
|
249
|
+
const [a, b] = removedRegion(pass, match, groups, inserted);
|
|
250
|
+
const start0 = toPassZero(offset);
|
|
251
|
+
const removedStart0 = toPassZero(offset + a);
|
|
252
|
+
const removedEnd0 = toPassZero(offset + b);
|
|
253
|
+
const end0 = toPassZero(offset + match.length);
|
|
254
|
+
const removed = match.slice(a, b);
|
|
255
|
+
if (isRedactionMark(removed) && input.slice(removedStart0, removedEnd0) === removed)
|
|
256
|
+
return match;
|
|
257
|
+
if (!cutsMark) {
|
|
258
|
+
marksOfInput ??= redactionMarkerSpans(passInput);
|
|
259
|
+
const [from, to] = [offset + a, offset + b];
|
|
260
|
+
for (let k = firstMarkEndingAfter(marksOfInput, from); k < marksOfInput.length && marksOfInput[k][0] < to && !cutsMark; k++) {
|
|
261
|
+
cutsMark = marksOfInput[k][0] < from || marksOfInput[k][1] > to;
|
|
262
|
+
}
|
|
263
|
+
}
|
|
186
264
|
const family = typeof pass.family === "function" ? pass.family(passInput, offset) : pass.family;
|
|
187
265
|
if (findings !== undefined)
|
|
188
|
-
findings.push({ pass: passIndex, f: finding(pass, [toOriginal(
|
|
189
|
-
const [a, b] = removedRegion(pass, match, groups, inserted);
|
|
266
|
+
findings.push({ pass: passIndex, f: finding(pass, [toOriginal(start0), toOriginal(end0)], family, markOfHit(pass, inserted.slice(a, inserted.length - (match.length - b)))) });
|
|
190
267
|
edits.push({ at: offset, removedLen: match.length, insertedLen: inserted.length, inserted, family, head: a, tail: match.length - b, pass: passIndex });
|
|
191
268
|
return inserted;
|
|
192
269
|
});
|
|
193
270
|
batches.push(edits);
|
|
194
271
|
});
|
|
195
|
-
return { out: cur, batches };
|
|
272
|
+
return { out: cur, batches, cutsMark };
|
|
196
273
|
}
|
|
197
274
|
function touching(removals) {
|
|
198
275
|
let reached = -1;
|
|
@@ -208,14 +285,15 @@ function touching(removals) {
|
|
|
208
285
|
}
|
|
209
286
|
return meets;
|
|
210
287
|
}
|
|
211
|
-
function applyPasses(input, passes,
|
|
288
|
+
function applyPasses(input, passes, report, toOriginal, disjoint = false) {
|
|
212
289
|
const found = report === undefined ? undefined : [];
|
|
213
290
|
const plain = runPasses(input, passes, found, toOriginal);
|
|
214
|
-
const solo = soloHits(input, passes
|
|
215
|
-
const
|
|
291
|
+
const solo = soloHits(input, passes);
|
|
292
|
+
const marks = redactionMarkerSpans(input);
|
|
293
|
+
const sequential = solo.length === 0 && marks.length === 0 && !plain.cutsMark ? [] : sequentialRemovals(plain.batches);
|
|
216
294
|
const outer = unremovedHits(solo, sequential);
|
|
217
295
|
const mappable = !disjoint || replacedSpans(plain.batches) !== undefined;
|
|
218
|
-
const whole = outer.length === 0 && mappable && !touching(sequential) ? undefined : renderRemovals(input, [...sequential, ...outer]
|
|
296
|
+
const whole = outer.length === 0 && mappable && !plain.cutsMark && !touching(sequential) && !reachesIntoMark(sequential, marks) ? undefined : renderRemovals(input, [...sequential, ...outer]);
|
|
219
297
|
if (whole === undefined || (whole.out === plain.out && mappable)) {
|
|
220
298
|
if (report !== undefined)
|
|
221
299
|
appendAll(report.findings, found.map((x) => x.f));
|
|
@@ -229,13 +307,28 @@ function applyPasses(input, passes, overStrip, report, toOriginal, disjoint = fa
|
|
|
229
307
|
if (seen.has(key))
|
|
230
308
|
continue;
|
|
231
309
|
seen.add(key);
|
|
232
|
-
found.push({ pass: h.pass, f: finding(passes[h.pass], span, h.family) });
|
|
310
|
+
found.push({ pass: h.pass, f: finding(passes[h.pass], span, h.family, markOfHit(passes[h.pass], h.marker)) });
|
|
233
311
|
}
|
|
234
312
|
found.sort((x, y) => x.pass - y.pass || x.f.span[0] - y.f.span[0]);
|
|
235
313
|
appendAll(report.findings, found.map((x) => x.f));
|
|
236
314
|
}
|
|
237
315
|
return { out: whole.out, batches: [whole.edits] };
|
|
238
316
|
}
|
|
317
|
+
export function renderWithheld(text, spans) {
|
|
318
|
+
if (!Array.isArray(spans))
|
|
319
|
+
fail("the spans are not an array");
|
|
320
|
+
const removals = spans.map((s, i) => {
|
|
321
|
+
if (!isRecord(s))
|
|
322
|
+
return fail(`spans[${i}] is not an object`);
|
|
323
|
+
const { start, end, mark } = s;
|
|
324
|
+
if (typeof start !== "number" || typeof end !== "number" || !Number.isInteger(start) || !Number.isInteger(end) || start < 0 || end < start || end > text.length)
|
|
325
|
+
return fail(`spans[${i}] is not a range of the text`);
|
|
326
|
+
if (typeof mark !== "string" || !isRedactionMark(mark))
|
|
327
|
+
return fail(`spans[${i}].mark is no member of the marks table`);
|
|
328
|
+
return { start, end, marker: mark, pass: i, family: undefined };
|
|
329
|
+
});
|
|
330
|
+
return removals.length === 0 ? text : renderRemovals(text, removals).out;
|
|
331
|
+
}
|
|
239
332
|
function replacedSpans(batches) {
|
|
240
333
|
const spans = [];
|
|
241
334
|
for (let j = 0; j < batches.length; j++) {
|
|
@@ -300,23 +393,23 @@ export function readFormatFree(input) {
|
|
|
300
393
|
return { view, cuts, toOriginal, toOriginalSpan };
|
|
301
394
|
}
|
|
302
395
|
export function runRedactionPasses(input, passes, run) {
|
|
303
|
-
const {
|
|
396
|
+
const { readings, report } = run;
|
|
304
397
|
if (!readsTextReadings(readings))
|
|
305
|
-
return renderOneText(input, passes,
|
|
398
|
+
return renderOneText(input, passes, readings, report).out;
|
|
306
399
|
const { views, unresolved } = readingViews(input, readings);
|
|
307
400
|
if (unresolved && run.onUnresolvedReading === "secret")
|
|
308
|
-
return
|
|
309
|
-
const base = renderOneText(input, passes,
|
|
401
|
+
return OVERSIZED;
|
|
402
|
+
const base = renderOneText(input, passes, readings, report);
|
|
310
403
|
if (views.length === 1)
|
|
311
404
|
return base.out;
|
|
312
405
|
const removals = base.removals();
|
|
313
406
|
if (removals === undefined)
|
|
314
|
-
return
|
|
407
|
+
return OVERSIZED;
|
|
315
408
|
const extra = [];
|
|
316
409
|
for (const view of views.slice(1)) {
|
|
317
|
-
const found = textRegions(view, passes,
|
|
410
|
+
const found = textRegions(view, passes, readings);
|
|
318
411
|
if (found === undefined)
|
|
319
|
-
return
|
|
412
|
+
return OVERSIZED;
|
|
320
413
|
appendAll(extra, found);
|
|
321
414
|
}
|
|
322
415
|
const seen = new Set();
|
|
@@ -328,14 +421,14 @@ export function runRedactionPasses(input, passes, run) {
|
|
|
328
421
|
return base.out;
|
|
329
422
|
if (report !== undefined)
|
|
330
423
|
for (const r of added)
|
|
331
|
-
report.findings.push(finding(passes[r.pass], [r.start, r.end], r.family));
|
|
332
|
-
return renderRemovals(input, [...removals, ...added]
|
|
424
|
+
report.findings.push(finding(passes[r.pass], [r.start, r.end], r.family, markOfHit(passes[r.pass], r.marker)));
|
|
425
|
+
return renderRemovals(input, [...removals, ...added]).out;
|
|
333
426
|
}
|
|
334
|
-
function textRegions(view, passes,
|
|
427
|
+
function textRegions(view, passes, readings) {
|
|
335
428
|
const regions = [];
|
|
336
429
|
const read = (text, back) => {
|
|
337
|
-
const run = applyPasses(text, passes,
|
|
338
|
-
for (const r of [...sequentialRemovals(run.batches), ...soloHits(text, passes
|
|
430
|
+
const run = applyPasses(text, passes, undefined, (pos) => pos);
|
|
431
|
+
for (const r of [...sequentialRemovals(run.batches), ...soloHits(text, passes)]) {
|
|
339
432
|
if (r.end <= r.start)
|
|
340
433
|
continue;
|
|
341
434
|
const inView = back(r.start, r.end);
|
|
@@ -360,15 +453,15 @@ function textRegions(view, passes, overStrip, readings) {
|
|
|
360
453
|
function coveredNot(regions, removals) {
|
|
361
454
|
return unremovedHits(regions, removals);
|
|
362
455
|
}
|
|
363
|
-
function renderOneText(input, passes,
|
|
364
|
-
const whole = { out:
|
|
456
|
+
function renderOneText(input, passes, readings, report) {
|
|
457
|
+
const whole = { out: OVERSIZED, removals: () => undefined };
|
|
365
458
|
if (!readings.formatFree || !FORMAT_CHAR_RE.test(input)) {
|
|
366
|
-
const plain = applyPasses(input, passes,
|
|
459
|
+
const plain = applyPasses(input, passes, report, (pos) => pos);
|
|
367
460
|
return { out: plain.out, removals: () => sequentialRemovals(plain.batches) };
|
|
368
461
|
}
|
|
369
462
|
const reading = readFormatFree(input);
|
|
370
463
|
const { view } = reading;
|
|
371
|
-
const { batches } = applyPasses(view, passes,
|
|
464
|
+
const { batches } = applyPasses(view, passes, report, reading.toOriginal, true);
|
|
372
465
|
if (replacedSpans(batches) === undefined)
|
|
373
466
|
return whole;
|
|
374
467
|
const back = (r, shift) => {
|
|
@@ -381,9 +474,9 @@ function renderOneText(input, passes, overStrip, readings, report) {
|
|
|
381
474
|
return whole;
|
|
382
475
|
const extra = [];
|
|
383
476
|
if (readings.raw)
|
|
384
|
-
appendAll(extra, sequentialRemovals(applyPasses(input, passes,
|
|
477
|
+
appendAll(extra, sequentialRemovals(applyPasses(input, passes, undefined, (pos) => pos, true).batches));
|
|
385
478
|
for (const cut of reading.cuts)
|
|
386
|
-
appendAll(extra, sequentialRemovals(applyPasses(view.slice(cut), passes,
|
|
479
|
+
appendAll(extra, sequentialRemovals(applyPasses(view.slice(cut), passes, undefined, (pos) => pos, true).batches).map((r) => back(r, cut)));
|
|
387
480
|
if (report !== undefined) {
|
|
388
481
|
const gone = [];
|
|
389
482
|
const firstEndingAfter = (pos) => {
|
|
@@ -420,10 +513,10 @@ function renderOneText(input, passes, overStrip, readings, report) {
|
|
|
420
513
|
join(r);
|
|
421
514
|
for (const r of [...extra].sort((a, b) => a.start - b.start))
|
|
422
515
|
if (join(r))
|
|
423
|
-
report.findings.push(finding(passes[r.pass], [r.matchStart, r.matchEnd], r.family));
|
|
516
|
+
report.findings.push(finding(passes[r.pass], [r.matchStart, r.matchEnd], r.family, markOfHit(passes[r.pass], r.marker)));
|
|
424
517
|
}
|
|
425
518
|
const all = [...removals, ...extra];
|
|
426
|
-
return { out: renderRemovals(input, all
|
|
519
|
+
return { out: renderRemovals(input, all).out, removals: () => all };
|
|
427
520
|
}
|
|
428
521
|
const SECRET_GRADES = ["strict", "wide", "fragment"];
|
|
429
522
|
function fail(what) {
|
|
@@ -444,35 +537,46 @@ function readPolicy(policy) {
|
|
|
444
537
|
return { grade, readings: set, urlUserinfo, onUnresolvedReading, key: `${grade} ${set.raw} ${set.formatFree} ${urlUserinfo}` };
|
|
445
538
|
}
|
|
446
539
|
const ROW_IDS = new Set(ENGINE_VOCABULARY.values.map((r) => r.id));
|
|
447
|
-
const
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
540
|
+
const isMarkName = (name) => Object.hasOwn(REDACTION_MARKS, name);
|
|
541
|
+
function rowMark(row, grade) {
|
|
542
|
+
const partial = `${row.mark}:partial`;
|
|
543
|
+
return grade === "fragment" && isMarkName(partial) ? REDACTION_MARKS[partial] : REDACTION_MARKS[row.mark];
|
|
544
|
+
}
|
|
545
|
+
for (const row of ENGINE_VOCABULARY.values) {
|
|
546
|
+
if (row.fragment !== undefined && !isMarkName(`${row.mark}:partial`))
|
|
547
|
+
throw new Error(`secret scan: the "${row.id}" row reads a fragment form, but its mark "${row.mark}" has no :partial member`);
|
|
548
|
+
}
|
|
549
|
+
function admitPasses(passes) {
|
|
550
|
+
if (passes === undefined)
|
|
551
|
+
return [];
|
|
552
|
+
if (!Array.isArray(passes))
|
|
553
|
+
return fail("options.passes must be an array");
|
|
554
|
+
passes.forEach((p, i) => {
|
|
555
|
+
if (!isRecord(p))
|
|
556
|
+
return fail(`options.passes[${i}] is not an object`);
|
|
557
|
+
const { marker, replace } = p;
|
|
558
|
+
if (typeof marker !== "string" || !isRedactionMark(marker))
|
|
559
|
+
fail(`options.passes[${i}].marker is no member of the marks table`);
|
|
560
|
+
if (typeof replace === "string" && replace.replace(/\$\d/g, "") !== marker)
|
|
561
|
+
fail(`options.passes[${i}].replace writes something other than its marker`);
|
|
562
|
+
if (typeof replace !== "string" && typeof replace !== "function")
|
|
563
|
+
fail(`options.passes[${i}].replace is neither a string nor a function`);
|
|
564
|
+
});
|
|
565
|
+
const admitted = passes;
|
|
566
|
+
return admitted.map((pass, i) => {
|
|
567
|
+
const replace = pass.replace;
|
|
568
|
+
if (typeof replace === "string")
|
|
569
|
+
return pass;
|
|
570
|
+
const checked = (match, ...groups) => {
|
|
571
|
+
const inserted = replace(match, ...groups);
|
|
572
|
+
if (typeof inserted !== "string")
|
|
573
|
+
return fail(`options.passes[${i}].replace returned no string`);
|
|
574
|
+
if (inserted !== match && !isRedactionMark(writtenOf(pass, match, groups, inserted)))
|
|
575
|
+
fail(`options.passes[${i}].replace wrote something other than one member of the marks table`);
|
|
576
|
+
return inserted;
|
|
577
|
+
};
|
|
578
|
+
return { ...pass, replace: checked };
|
|
579
|
+
});
|
|
476
580
|
}
|
|
477
581
|
const USERINFO_FAMILY = "url-userinfo";
|
|
478
582
|
if (!ROW_IDS.has(USERINFO_FAMILY))
|
|
@@ -485,8 +589,18 @@ const readsDecodedAsCredential = (username) => {
|
|
|
485
589
|
const decoded = percentDecodedOnce(username);
|
|
486
590
|
return decoded !== username && DECODED_USERNAME_FORMS.some((re) => re.test(decoded));
|
|
487
591
|
};
|
|
592
|
+
function colonOutsideMarks(userinfo) {
|
|
593
|
+
let from = 0;
|
|
594
|
+
for (const [start, end] of eachRedactionMarkerSpan(userinfo)) {
|
|
595
|
+
const colon = userinfo.indexOf(":", from);
|
|
596
|
+
if (colon !== -1 && colon < start)
|
|
597
|
+
return colon;
|
|
598
|
+
from = end;
|
|
599
|
+
}
|
|
600
|
+
return userinfo.indexOf(":", from);
|
|
601
|
+
}
|
|
488
602
|
export function userinfoHolds(userinfo) {
|
|
489
|
-
const colon = userinfo
|
|
603
|
+
const colon = colonOutsideMarks(userinfo);
|
|
490
604
|
return colon === -1 ? userinfo.length > 0 : colon > 0 || colon < userinfo.length - 1;
|
|
491
605
|
}
|
|
492
606
|
function userinfoWithheld(userinfo, urlUserinfo) {
|
|
@@ -496,7 +610,7 @@ function userinfoWithheld(userinfo, urlUserinfo) {
|
|
|
496
610
|
function userinfoCut(userinfo, urlUserinfo) {
|
|
497
611
|
if (urlUserinfo === "whole" || userinfo.includes("@"))
|
|
498
612
|
return [0, userinfo.length];
|
|
499
|
-
const colonIdx = userinfo
|
|
613
|
+
const colonIdx = colonOutsideMarks(userinfo);
|
|
500
614
|
if (readsDecodedAsCredential(colonIdx === -1 ? userinfo : userinfo.slice(0, colonIdx)))
|
|
501
615
|
return [0, userinfo.length];
|
|
502
616
|
if (colonIdx === -1)
|
|
@@ -504,63 +618,61 @@ function userinfoCut(userinfo, urlUserinfo) {
|
|
|
504
618
|
return [colonIdx + 1, userinfo.length];
|
|
505
619
|
}
|
|
506
620
|
const capturesLabel = (re) => new RegExp(`${re.source}|`).exec("").length > 1;
|
|
507
|
-
export function egressPasses(groups,
|
|
621
|
+
export function egressPasses(groups, urlUserinfo) {
|
|
508
622
|
return groups.map((rows) => {
|
|
509
623
|
const head = rows[0];
|
|
510
|
-
const
|
|
511
|
-
const
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
624
|
+
const marks = rows.map((r) => rowMark(r, "wide"));
|
|
625
|
+
const marker = marks[0];
|
|
626
|
+
const mixed = marks.some((m) => m !== marker);
|
|
627
|
+
if (mixed && rows.some((r) => capturesLabel(r.match)))
|
|
628
|
+
fail(`the rows of the "${head.id}" pass write different marks and one captures a label`);
|
|
629
|
+
const re = rows.length === 1 ? head.match : new RegExp(rows.map((r) => (mixed ? `(${r.match.source})` : `(?:${r.match.source})`)).join("|"), head.match.flags);
|
|
515
630
|
const sticky = rows.map((r) => ({ id: r.id, re: new RegExp(r.match.source, r.match.flags.replace("g", "") + "y") }));
|
|
516
631
|
const family = rows.length === 1 ? head.id : (input, at) => sticky.find((s) => ((s.re.lastIndex = at), s.re.test(input)))?.id ?? head.id;
|
|
517
632
|
const replace = head.id === USERINFO_FAMILY
|
|
518
633
|
? (_m, scheme, userinfoWithAt) => {
|
|
519
634
|
const userinfo = userinfoWithAt.slice(0, -1);
|
|
520
635
|
const cut = userinfoWithheld(userinfo, urlUserinfo);
|
|
521
|
-
return cut === undefined ? `${scheme}${userinfo}@` : `${scheme}${userinfo.slice(0, cut[0])}${marker}${userinfo.slice(cut[1])}@`;
|
|
636
|
+
return cut === undefined ? `${scheme}${userinfo}@` : `${scheme}${userinfo.slice(0, cut[0])}${cut[0] === 0 ? marker : REDACTION_MARKS.bare}${userinfo.slice(cut[1])}@`;
|
|
522
637
|
}
|
|
523
|
-
:
|
|
524
|
-
?
|
|
525
|
-
:
|
|
638
|
+
: mixed
|
|
639
|
+
? (_m, ...took) => marks[took.findIndex((g) => g !== undefined)] ?? marker
|
|
640
|
+
: capturesLabel(head.match)
|
|
641
|
+
? `$1${marker}`
|
|
642
|
+
: marker;
|
|
526
643
|
return { kind: head.kind, confidence: head.confidence, marker, source: head.source, re, replace, family };
|
|
527
644
|
});
|
|
528
645
|
}
|
|
529
|
-
function strictPasses(
|
|
646
|
+
function strictPasses() {
|
|
530
647
|
return ENGINE_VOCABULARY.values.flatMap((row) => {
|
|
531
648
|
if (row.strict === undefined)
|
|
532
649
|
return [];
|
|
533
|
-
const marker =
|
|
650
|
+
const marker = rowMark(row, "strict");
|
|
534
651
|
return [{ kind: row.kind, confidence: row.confidence, marker, source: row.source, re: new RegExp(row.strict.source, `${row.strict.flags}g`), replace: marker, family: row.id }];
|
|
535
652
|
});
|
|
536
653
|
}
|
|
537
|
-
function fragmentPasses(
|
|
654
|
+
function fragmentPasses() {
|
|
538
655
|
return ENGINE_VOCABULARY.values.flatMap((row) => {
|
|
539
656
|
if (row.fragment === undefined)
|
|
540
657
|
return [];
|
|
541
|
-
const marker =
|
|
658
|
+
const marker = rowMark(row, "fragment");
|
|
542
659
|
return [{ kind: row.kind, confidence: row.confidence, marker, source: row.source, re: row.fragment, replace: capturesLabel(row.fragment) ? `$1${marker}` : marker, family: row.id }];
|
|
543
660
|
});
|
|
544
661
|
}
|
|
545
662
|
const COMPILED_PASSES = new Map();
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
const key = `${read.key}\u0000${marks.key}`;
|
|
549
|
-
let passes = COMPILED_PASSES.get(key);
|
|
663
|
+
function tablePasses(read) {
|
|
664
|
+
let passes = COMPILED_PASSES.get(read.key);
|
|
550
665
|
if (passes === undefined) {
|
|
551
|
-
passes = Object.freeze(read.grade === "wide" ? egressPasses(egressPassGroups(ENGINE_VOCABULARY),
|
|
552
|
-
|
|
553
|
-
COMPILED_PASSES.clear();
|
|
554
|
-
COMPILED_PASSES.set(key, passes);
|
|
666
|
+
passes = Object.freeze(read.grade === "wide" ? egressPasses(egressPassGroups(ENGINE_VOCABULARY), read.urlUserinfo) : read.grade === "strict" ? strictPasses() : fragmentPasses());
|
|
667
|
+
COMPILED_PASSES.set(read.key, passes);
|
|
555
668
|
}
|
|
556
669
|
return passes;
|
|
557
670
|
}
|
|
558
|
-
export function renderSecrets(text, policy,
|
|
671
|
+
export function renderSecrets(text, policy, options = {}) {
|
|
559
672
|
const read = readPolicy(policy);
|
|
560
|
-
const
|
|
561
|
-
const
|
|
562
|
-
|
|
563
|
-
return runRedactionPasses(text, extra === undefined || extra.length === 0 ? own : [...own, ...extra], { overStrip: marks.overStrip, readings: read.readings, onUnresolvedReading: read.onUnresolvedReading, report });
|
|
673
|
+
const extra = admitPasses(options.passes);
|
|
674
|
+
const own = tablePasses(read);
|
|
675
|
+
return runRedactionPasses(text, extra.length === 0 ? own : [...own, ...extra], { readings: read.readings, onUnresolvedReading: read.onUnresolvedReading, report: options.report });
|
|
564
676
|
}
|
|
565
677
|
const FORMS = {
|
|
566
678
|
strict: ENGINE_VOCABULARY.values.flatMap((row) => (row.strict === undefined ? [] : [{ row, re: new RegExp(row.strict.source, `${row.strict.flags}g`), cut: "whole" }])),
|