@sema-agent/server 7.79.1 → 7.79.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/boot/late-binds.d.ts +35 -0
- package/dist/boot/late-binds.js +20 -0
- package/dist/boot/stage-01-config.d.ts +7 -0
- package/dist/boot/stage-01-config.js +60 -0
- package/dist/boot/stage-02-registry-pull.d.ts +10 -0
- package/dist/boot/stage-02-registry-pull.js +11 -0
- package/dist/boot/stage-03-store-backend.d.ts +32 -0
- package/dist/boot/stage-03-store-backend.js +11 -0
- package/dist/boot/stage-04-stores.d.ts +69 -0
- package/dist/boot/stage-04-stores.js +49 -0
- package/dist/boot/stage-05-execution-env.d.ts +82 -0
- package/dist/boot/stage-05-execution-env.js +20 -0
- package/dist/boot/stage-06-runners.d.ts +143 -0
- package/dist/boot/stage-06-runners.js +207 -0
- package/dist/boot/stage-07-capability-layer.d.ts +157 -0
- package/dist/boot/stage-07-capability-layer.js +162 -0
- package/dist/boot/stage-08-reapers.d.ts +172 -0
- package/dist/boot/stage-08-reapers.js +78 -0
- package/dist/boot/stage-09-leader.d.ts +204 -0
- package/dist/boot/stage-09-leader.js +47 -0
- package/dist/boot/stage-10-http-server.d.ts +204 -0
- package/dist/boot/stage-10-http-server.js +289 -0
- package/dist/config-invariants.d.ts +9 -0
- package/dist/config-invariants.js +7 -1
- package/dist/config-types.d.ts +1 -1
- package/dist/http/dispatch.d.ts +9 -73
- package/dist/http/dispatch.js +12 -281
- package/dist/http/route-table.d.ts +113 -0
- package/dist/http/route-table.js +235 -0
- package/dist/http/routes/admin-memory-optout.d.ts +1 -1
- package/dist/http/routes/memory-compliance.d.ts +1 -1
- package/dist/http/routes/memory-origin.d.ts +1 -1
- package/dist/http/routes/rules.js +4 -4
- package/dist/http/server.d.ts +2 -1
- package/dist/http/server.js +2 -1
- package/dist/main.js +20 -870
- package/dist/service-version.d.ts +5 -0
- package/dist/service-version.js +12 -0
- package/dist/trace/redact.d.ts +16 -6
- package/dist/trace/redact.js +568 -246
- package/package.json +1 -1
package/dist/trace/redact.js
CHANGED
|
@@ -206,7 +206,6 @@ function forgetEscapedSeparatorTails() {
|
|
|
206
206
|
escapedTailsValue = undefined;
|
|
207
207
|
}
|
|
208
208
|
const VALUE_ARM_STICKY = new Map();
|
|
209
|
-
const VALUE_ARMS = new Set();
|
|
210
209
|
const VALUE_ARM_BODIES = [];
|
|
211
210
|
const valueArm = (prefixSrc, valueClass, min, evidence) => {
|
|
212
211
|
const body = `${prefixSrc}${twoStrongAhead(valueClass)}${atLeast(valueClass, min)}`;
|
|
@@ -226,29 +225,145 @@ const valueArm = (prefixSrc, valueClass, min, evidence) => {
|
|
|
226
225
|
}
|
|
227
226
|
}
|
|
228
227
|
})();
|
|
229
|
-
VALUE_ARMS.add(g);
|
|
230
228
|
return g;
|
|
231
229
|
};
|
|
232
230
|
let anyValueArm;
|
|
233
231
|
function anyValueArmRe() {
|
|
234
232
|
return (anyValueArm ??= new RegExp(`(?:${VALUE_ARM_BODIES.join("|")})`, "y"));
|
|
235
233
|
}
|
|
234
|
+
const ARM_RANK = { structural: 0, keyword: 1, shape: 2, fragment: 3 };
|
|
235
|
+
let poolSpanCount = 0;
|
|
236
|
+
let poolArmCount = 0;
|
|
237
|
+
let poolBoundaryArmCount = 0;
|
|
238
|
+
let coveredPositions = 0;
|
|
239
|
+
let headScreenHits = 0;
|
|
240
|
+
let fullYCalls = 0;
|
|
241
|
+
let forensicLevels = 0;
|
|
242
|
+
let ssh2ScanBytes = 0;
|
|
243
|
+
export function armPoolBudgetForTest() {
|
|
244
|
+
return { spanCount: poolSpanCount, coveredPositions, headScreenHits, fullYCalls, armCount: poolArmCount, boundaryArmCount: poolBoundaryArmCount, forensicLevels, ssh2ScanBytes };
|
|
245
|
+
}
|
|
246
|
+
export function resetArmPoolBudgetForTest() {
|
|
247
|
+
poolSpanCount = 0;
|
|
248
|
+
coveredPositions = 0;
|
|
249
|
+
headScreenHits = 0;
|
|
250
|
+
fullYCalls = 0;
|
|
251
|
+
forensicLevels = 0;
|
|
252
|
+
ssh2ScanBytes = 0;
|
|
253
|
+
}
|
|
254
|
+
const unitBit = (t, c) => (t[c >>> 5] >>> (c & 31)) & 1;
|
|
255
|
+
let codeUnitDomain;
|
|
256
|
+
const codeUnitMembersCache = new Map();
|
|
257
|
+
function codeUnitMembers(unitSrc) {
|
|
258
|
+
let table = codeUnitMembersCache.get(unitSrc);
|
|
259
|
+
if (table === undefined) {
|
|
260
|
+
if (codeUnitDomain === undefined) {
|
|
261
|
+
const parts = [];
|
|
262
|
+
for (let c = 0; c < 0x10000; c++)
|
|
263
|
+
parts.push(String.fromCharCode(c));
|
|
264
|
+
codeUnitDomain = parts.join("");
|
|
265
|
+
}
|
|
266
|
+
table = new Uint32Array(0x10000 >>> 5);
|
|
267
|
+
const re = new RegExp(unitSrc, "g");
|
|
268
|
+
let members = 0;
|
|
269
|
+
for (let m = re.exec(codeUnitDomain); m !== null; m = re.exec(codeUnitDomain)) {
|
|
270
|
+
if (m[0].length !== 1)
|
|
271
|
+
throw new Error(`redact: /${unitSrc}/ is not a single-code-unit matcher`);
|
|
272
|
+
table[m.index >>> 5] = table[m.index >>> 5] | (1 << (m.index & 31));
|
|
273
|
+
members += 1;
|
|
274
|
+
re.lastIndex = m.index + 1;
|
|
275
|
+
}
|
|
276
|
+
if (members === 0)
|
|
277
|
+
throw new Error(`redact: /${unitSrc}/ matches no code unit — it cannot serve as a member set`);
|
|
278
|
+
codeUnitMembersCache.set(unitSrc, table);
|
|
279
|
+
}
|
|
280
|
+
return table;
|
|
281
|
+
}
|
|
282
|
+
function unitsDisjoint(a, b) {
|
|
283
|
+
const ta = codeUnitMembers(a);
|
|
284
|
+
const tb = codeUnitMembers(b);
|
|
285
|
+
for (let i = 0; i < ta.length; i++)
|
|
286
|
+
if ((ta[i] & tb[i]) !== 0)
|
|
287
|
+
return false;
|
|
288
|
+
return true;
|
|
289
|
+
}
|
|
290
|
+
function unionMembers(tables) {
|
|
291
|
+
const out = new Uint32Array(0x10000 >>> 5);
|
|
292
|
+
for (const t of tables) {
|
|
293
|
+
if (t === undefined)
|
|
294
|
+
return undefined;
|
|
295
|
+
for (let i = 0; i < out.length; i++)
|
|
296
|
+
out[i] = out[i] | t[i];
|
|
297
|
+
}
|
|
298
|
+
return out;
|
|
299
|
+
}
|
|
236
300
|
const HEAD_CLASS_ESCAPES = new Set(["d", "D", "w", "W", "s", "S"]);
|
|
237
301
|
const HEAD_LITERAL_ESCAPES = new Set(["\\", "/", "-", ".", "+", "*", "?", "(", ")", "[", "]", "{", "}", "|", "^", "$"]);
|
|
302
|
+
function assertWall(atoms, w, src) {
|
|
303
|
+
const wall = atoms[w];
|
|
304
|
+
const fail = (why) => {
|
|
305
|
+
throw new Error(`redact: head derivation cannot bound the unbounded head atom at ${w} in /${src}/ — ${why}`);
|
|
306
|
+
};
|
|
307
|
+
if (wall.unit === undefined)
|
|
308
|
+
fail("the wall is not a single-code-unit atom");
|
|
309
|
+
const wallUnit = wall.unit;
|
|
310
|
+
const firstUnit = atoms[0].unit;
|
|
311
|
+
for (let k = 0; k < w; k++) {
|
|
312
|
+
const a = atoms[k];
|
|
313
|
+
if (a.unit === undefined || a.unbounded)
|
|
314
|
+
fail(`the head atom at ${k} is not a bounded single-code-unit atom`);
|
|
315
|
+
if (!unitsDisjoint(a.unit, wallUnit))
|
|
316
|
+
fail(`the head atom at ${k} can hold the wall's own character`);
|
|
317
|
+
if (k > 0 && (firstUnit === undefined || !unitsDisjoint(a.unit, firstUnit)))
|
|
318
|
+
fail(`the head overlaps itself at ${k}`);
|
|
319
|
+
}
|
|
320
|
+
for (let k = w + 1; k < atoms.length; k++) {
|
|
321
|
+
const a = atoms[k];
|
|
322
|
+
if (a.unit === undefined || !unitsDisjoint(a.unit, wallUnit))
|
|
323
|
+
fail(`the atom at ${k} can hold the wall's own character`);
|
|
324
|
+
}
|
|
325
|
+
}
|
|
238
326
|
function headOfAtoms(atoms, src) {
|
|
239
|
-
const
|
|
240
|
-
|
|
241
|
-
|
|
327
|
+
const tailForm = () => {
|
|
328
|
+
throw new Error(`redact: arm-pool pattern /${src}/ must end in a greedy unbounded single-class value run (optionally followed by a bounded literal tail run disjoint from it)`);
|
|
329
|
+
};
|
|
330
|
+
let vi = atoms.length - 1;
|
|
331
|
+
const last = atoms[vi];
|
|
332
|
+
if (last === undefined)
|
|
333
|
+
tailForm();
|
|
334
|
+
if (last.cls === undefined) {
|
|
335
|
+
const prev = atoms[vi - 1];
|
|
336
|
+
if (last.unit === undefined || prev === undefined || !prev.unbounded || prev.cls === undefined || !unitsDisjoint(last.unit, prev.cls))
|
|
337
|
+
tailForm();
|
|
338
|
+
vi -= 1;
|
|
242
339
|
}
|
|
340
|
+
const value = atoms[vi];
|
|
341
|
+
if (value === undefined || !value.unbounded || value.cls === undefined)
|
|
342
|
+
tailForm();
|
|
343
|
+
const cls = value.cls;
|
|
243
344
|
let pos = 0;
|
|
244
345
|
let head = 0;
|
|
245
|
-
|
|
346
|
+
const parts = [];
|
|
347
|
+
for (let k = 0; k < vi; k++) {
|
|
246
348
|
const a = atoms[k];
|
|
247
|
-
if (a.
|
|
349
|
+
if (a.unbounded) {
|
|
350
|
+
assertWall(atoms, k, src);
|
|
351
|
+
return { len: pos + 1, head: undefined, first: undefined };
|
|
352
|
+
}
|
|
353
|
+
parts.push(a.src);
|
|
354
|
+
if (a.cls !== cls)
|
|
248
355
|
head = pos + a.len;
|
|
249
356
|
pos += a.len;
|
|
250
357
|
}
|
|
251
|
-
return head;
|
|
358
|
+
return { len: head, head: parts.length === 0 ? undefined : parts.join(""), first: firstOfAtoms(atoms.slice(0, vi)) };
|
|
359
|
+
}
|
|
360
|
+
function firstOfAtoms(atoms) {
|
|
361
|
+
for (const a of atoms) {
|
|
362
|
+
if (a.len === 0)
|
|
363
|
+
continue;
|
|
364
|
+
return a.first;
|
|
365
|
+
}
|
|
366
|
+
return undefined;
|
|
252
367
|
}
|
|
253
368
|
const HEAD_GROUP_OPEN = /^\((\?:|\?=|\?!|\?<=|\?<!)/;
|
|
254
369
|
function parseHeadAlternative(src, from) {
|
|
@@ -261,6 +376,7 @@ function parseHeadAlternative(src, from) {
|
|
|
261
376
|
break;
|
|
262
377
|
if (closed)
|
|
263
378
|
throw new Error(`redact: arm-pool pattern /${src}/ has trailing syntax after its value run`);
|
|
379
|
+
const atomFrom = i;
|
|
264
380
|
let base;
|
|
265
381
|
if (c === "(") {
|
|
266
382
|
const open = HEAD_GROUP_OPEN.exec(src.slice(i));
|
|
@@ -278,7 +394,7 @@ function parseHeadAlternative(src, from) {
|
|
|
278
394
|
closed = true;
|
|
279
395
|
continue;
|
|
280
396
|
}
|
|
281
|
-
base = { len: Math.max(...inner.alts.map((alt) => alt.reduce((n, a) => n + a.len, 0))), cls: undefined };
|
|
397
|
+
base = { len: Math.max(...inner.alts.map((alt) => alt.reduce((n, a) => n + a.len, 0))), cls: undefined, unit: undefined, first: unionMembers(inner.alts.map(firstOfAtoms)) };
|
|
282
398
|
}
|
|
283
399
|
else if (c === "[") {
|
|
284
400
|
let k = i + 1;
|
|
@@ -286,7 +402,7 @@ function parseHeadAlternative(src, from) {
|
|
|
286
402
|
k += src[k] === "\\" ? 2 : 1;
|
|
287
403
|
if (k >= src.length)
|
|
288
404
|
throw new Error(`redact: unterminated character class at ${i} in /${src}/`);
|
|
289
|
-
base = { len: 1, cls: src.slice(i, k + 1) };
|
|
405
|
+
base = { len: 1, cls: src.slice(i, k + 1), unit: src.slice(i, k + 1), first: codeUnitMembers(src.slice(i, k + 1)) };
|
|
290
406
|
i = k + 1;
|
|
291
407
|
}
|
|
292
408
|
else if (c === "\\") {
|
|
@@ -294,9 +410,9 @@ function parseHeadAlternative(src, from) {
|
|
|
294
410
|
if (e === undefined)
|
|
295
411
|
throw new Error(`redact: dangling escape at ${i} in /${src}/`);
|
|
296
412
|
if (HEAD_CLASS_ESCAPES.has(e))
|
|
297
|
-
base = { len: 1, cls: `\\${e}` };
|
|
413
|
+
base = { len: 1, cls: `\\${e}`, unit: `\\${e}`, first: codeUnitMembers(`\\${e}`) };
|
|
298
414
|
else if (HEAD_LITERAL_ESCAPES.has(e))
|
|
299
|
-
base = { len: 1, cls: undefined };
|
|
415
|
+
base = { len: 1, cls: undefined, unit: `\\${e}`, first: codeUnitMembers(`\\${e}`) };
|
|
300
416
|
else
|
|
301
417
|
throw new Error(`redact: head derivation cannot bound escape \\${e} at ${i} in /${src}/`);
|
|
302
418
|
i += 2;
|
|
@@ -305,16 +421,17 @@ function parseHeadAlternative(src, from) {
|
|
|
305
421
|
throw new Error(`redact: head derivation cannot bound ${JSON.stringify(c)} at ${i} in /${src}/`);
|
|
306
422
|
}
|
|
307
423
|
else {
|
|
308
|
-
base = { len: 1, cls: undefined };
|
|
424
|
+
base = { len: 1, cls: undefined, unit: c, first: codeUnitMembers(c) };
|
|
309
425
|
i += 1;
|
|
310
426
|
}
|
|
311
|
-
let atom = { ...base, unbounded: false };
|
|
427
|
+
let atom = { ...base, unbounded: false, src: "" };
|
|
312
428
|
const q = src[i];
|
|
313
429
|
if (q === "*" || q === "+") {
|
|
314
|
-
atom = { ...base, unbounded: true };
|
|
430
|
+
atom = { ...base, unbounded: true, src: "" };
|
|
315
431
|
i += 1;
|
|
316
432
|
}
|
|
317
433
|
else if (q === "?") {
|
|
434
|
+
atom = { ...base, unbounded: false, src: "", first: undefined };
|
|
318
435
|
i += 1;
|
|
319
436
|
}
|
|
320
437
|
else if (q === "{") {
|
|
@@ -323,15 +440,14 @@ function parseHeadAlternative(src, from) {
|
|
|
323
440
|
throw new Error(`redact: head derivation cannot bound quantifier at ${i} in /${src}/`);
|
|
324
441
|
const lo = Number(m[1]);
|
|
325
442
|
const hi = m[2] === undefined ? lo : m[3] === "" ? -1 : Number(m[3]);
|
|
326
|
-
atom = hi < 0 ? { ...base, unbounded: true } : { len: base.len * hi, cls: base.cls, unbounded: false };
|
|
443
|
+
atom = hi < 0 ? { ...base, unbounded: true, src: "" } : { len: base.len * hi, cls: base.cls, unit: base.unit, unbounded: false, src: "", first: lo === 0 ? undefined : base.first };
|
|
327
444
|
i += m[0].length;
|
|
328
445
|
}
|
|
329
446
|
if (src[i] === "?")
|
|
330
447
|
throw new Error(`redact: head derivation cannot bound a lazy quantifier at ${i} in /${src}/`);
|
|
331
|
-
|
|
332
|
-
closed = true;
|
|
448
|
+
const withSrc = { ...atom, src: src.slice(atomFrom, i) };
|
|
333
449
|
for (const s of seqs)
|
|
334
|
-
s.push(
|
|
450
|
+
s.push(withSrc);
|
|
335
451
|
}
|
|
336
452
|
return { seqs, next: i };
|
|
337
453
|
}
|
|
@@ -348,39 +464,36 @@ function parseHeadAlternatives(src, from) {
|
|
|
348
464
|
}
|
|
349
465
|
return { alts, next: i };
|
|
350
466
|
}
|
|
351
|
-
function
|
|
467
|
+
function deriveLiteralHead(re) {
|
|
352
468
|
if (/[uv]/.test(re.flags))
|
|
353
469
|
throw new Error(`redact: head derivation does not support the u/v flags on /${re.source}/`);
|
|
354
470
|
const src = re.source;
|
|
355
471
|
const parsed = parseHeadAlternatives(src, 0);
|
|
356
472
|
if (parsed.next !== src.length)
|
|
357
473
|
throw new Error(`redact: head derivation stopped at ${parsed.next} in /${src}/`);
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
474
|
+
const per = parsed.alts.map((alt) => headOfAtoms(alt, src));
|
|
475
|
+
const heads = per.map((p) => p.head);
|
|
476
|
+
return {
|
|
477
|
+
len: Math.max(...per.map((p) => p.len)),
|
|
478
|
+
src: heads.some((h) => h === undefined) ? undefined : `(?:${heads.join("|")})`,
|
|
479
|
+
first: unionMembers(per.map((p) => p.first)),
|
|
480
|
+
};
|
|
362
481
|
}
|
|
363
|
-
function
|
|
364
|
-
|
|
365
|
-
let last = -1;
|
|
366
|
-
for (const e of at) {
|
|
367
|
-
for (let p = Math.max(e - Math.max(maxHead - 1, 0), last + 1, 0); p <= e; p++) {
|
|
368
|
-
out.push({ p, back: e - p });
|
|
369
|
-
last = p;
|
|
370
|
-
}
|
|
371
|
-
}
|
|
372
|
-
return out;
|
|
482
|
+
function scanResumeAfterHit(start, end, back) {
|
|
483
|
+
return Math.max(start + 1, end - back);
|
|
373
484
|
}
|
|
374
|
-
|
|
485
|
+
const spanOffset = (a, m) => (a.outside === 1 ? m[1].length : 0);
|
|
486
|
+
function collectRegexSpans(a, text, out) {
|
|
487
|
+
const g = a.g;
|
|
375
488
|
g.lastIndex = 0;
|
|
376
489
|
for (let m = g.exec(text); m !== null; m = g.exec(text)) {
|
|
377
490
|
const end = m.index + m[0].length;
|
|
378
|
-
out.push({ start: m.index, end, repl, order });
|
|
379
|
-
g.lastIndex = scanResumeAfterHit(m.index, end,
|
|
491
|
+
out.push({ start: m.index + spanOffset(a, m), end, repl: a.repl, rank: a.rank, order: a.order });
|
|
492
|
+
g.lastIndex = scanResumeAfterHit(m.index, end, a.back);
|
|
380
493
|
}
|
|
381
494
|
}
|
|
382
495
|
function resolveSpanOverlaps(spans) {
|
|
383
|
-
spans.sort((a, b) => a.start - b.start || b.end - a.end || a.order - b.order);
|
|
496
|
+
spans.sort((a, b) => a.start - b.start || b.end - a.end || a.rank - b.rank || a.order - b.order);
|
|
384
497
|
const kept = [];
|
|
385
498
|
let covered = 0;
|
|
386
499
|
for (const s of spans) {
|
|
@@ -403,10 +516,11 @@ function applySpans(text, spans) {
|
|
|
403
516
|
parts.push(text.slice(copied));
|
|
404
517
|
return { out: parts.join(""), count: spans.length };
|
|
405
518
|
}
|
|
406
|
-
function collectValueArmSpans(
|
|
519
|
+
function collectValueArmSpans(arm, text, out) {
|
|
520
|
+
const { g, y, back, repl, rank, order } = arm;
|
|
407
521
|
const tails = escapedSeparatorTails(text);
|
|
408
522
|
if (tails === undefined) {
|
|
409
|
-
collectRegexSpans(
|
|
523
|
+
collectRegexSpans(arm, text, out);
|
|
410
524
|
return;
|
|
411
525
|
}
|
|
412
526
|
let pos = 0;
|
|
@@ -451,29 +565,32 @@ function collectValueArmSpans(g, y, head, repl, order, text, out) {
|
|
|
451
565
|
}
|
|
452
566
|
else
|
|
453
567
|
break;
|
|
454
|
-
out.push({ start: idx, end, repl, order });
|
|
455
|
-
pos = scanResumeAfterHit(idx, end,
|
|
568
|
+
out.push({ start: idx, end, repl, rank, order });
|
|
569
|
+
pos = scanResumeAfterHit(idx, end, back);
|
|
456
570
|
}
|
|
457
571
|
}
|
|
458
|
-
function collectStickySpans(
|
|
572
|
+
function collectStickySpans(a, text, at, out) {
|
|
573
|
+
const { y, back, repl, rank, order } = a;
|
|
459
574
|
let pos = 0;
|
|
460
|
-
for (const
|
|
461
|
-
if (back >= head)
|
|
462
|
-
continue;
|
|
575
|
+
for (const p of at) {
|
|
463
576
|
if (p < pos)
|
|
464
577
|
continue;
|
|
465
578
|
y.lastIndex = p;
|
|
579
|
+
fullYCalls += 1;
|
|
466
580
|
const m = y.exec(text);
|
|
467
581
|
if (m === null)
|
|
468
582
|
continue;
|
|
469
583
|
const end = p + m[0].length;
|
|
470
|
-
out.push({ start: p, end, repl, order });
|
|
471
|
-
pos = scanResumeAfterHit(p, end,
|
|
584
|
+
out.push({ start: p + spanOffset(a, m), end, repl, rank, order });
|
|
585
|
+
pos = scanResumeAfterHit(p, end, back);
|
|
472
586
|
}
|
|
473
587
|
}
|
|
474
588
|
const WORD_SEGMENT_CLASS = "[A-Za-z0-9._~+/-]";
|
|
475
|
-
const keywordArm = (keywordSrc, valueClass, min, tailSrc = "") =>
|
|
476
|
-
|
|
589
|
+
const keywordArm = (keywordSrc, valueClass, min, tailSrc = "") => {
|
|
590
|
+
const re = new RegExp(`(?:${keywordSrc}\\s+${atLeast(valueClass, min)}${tailSrc}` +
|
|
591
|
+
`|(?<!${WORD_SEGMENT_CLASS})${keywordSrc}(?:${twoStrongAhead(valueClass)}|${noConnectorAhead(valueClass)})${atLeast(valueClass, min)}${tailSrc})`, "g");
|
|
592
|
+
return re;
|
|
593
|
+
};
|
|
477
594
|
const STRUCTURAL_SECRET_PATTERNS = [
|
|
478
595
|
[/-----BEGIN(?:[^-](?:[^-](?:[^-]|-(?!-))*)?)?PRIVATE KEY(?: BLOCK)?-----[\s\S]{0,8192}?-----END(?:[^-](?:[^-](?:[^-]|-(?!-))*)?)?PRIVATE KEY(?: BLOCK)?-----/g, "«redacted:private-key»"],
|
|
479
596
|
[/---- BEGIN SSH2 ENCRYPTED PRIVATE KEY ----[\s\S]{0,8192}?---- END SSH2 ENCRYPTED PRIVATE KEY ----/g, "«redacted:private-key»"],
|
|
@@ -518,6 +635,8 @@ function skipPemSeps(s, i, limit) {
|
|
|
518
635
|
for (;;) {
|
|
519
636
|
if (i >= limit)
|
|
520
637
|
return i;
|
|
638
|
+
if (opaqueUnitEnd(i) >= 0)
|
|
639
|
+
return i;
|
|
521
640
|
const c = s.charCodeAt(i);
|
|
522
641
|
if (isSepCode(c)) {
|
|
523
642
|
i += 1;
|
|
@@ -560,16 +679,16 @@ function pemInlineSepLen(s, i, limit) {
|
|
|
560
679
|
return 0;
|
|
561
680
|
}
|
|
562
681
|
function probePemHeaderToken(s, p, limit) {
|
|
563
|
-
if (p >= limit || !isIdentStartCode(s.charCodeAt(p)))
|
|
682
|
+
if (p >= limit || opaqueUnitEnd(p) >= 0 || !isIdentStartCode(s.charCodeAt(p)))
|
|
564
683
|
return -1;
|
|
565
684
|
p += 1;
|
|
566
685
|
let identLen = 1;
|
|
567
|
-
while (identLen < PEM_HEADER_IDENT_MAX && p < limit && isIdentCode(s.charCodeAt(p))) {
|
|
686
|
+
while (identLen < PEM_HEADER_IDENT_MAX && p < limit && opaqueUnitEnd(p) < 0 && isIdentCode(s.charCodeAt(p))) {
|
|
568
687
|
p += 1;
|
|
569
688
|
identLen += 1;
|
|
570
689
|
}
|
|
571
690
|
for (;;) {
|
|
572
|
-
if (p >= limit)
|
|
691
|
+
if (p >= limit || opaqueUnitEnd(p) >= 0)
|
|
573
692
|
return -1;
|
|
574
693
|
const c = s.charCodeAt(p);
|
|
575
694
|
if (c === CH_SPACE || c === CH_TAB) {
|
|
@@ -601,6 +720,11 @@ function parsePemHeaderLine(s, i, limit, allowBare, json) {
|
|
|
601
720
|
for (;;) {
|
|
602
721
|
if (p >= limit)
|
|
603
722
|
return setPemHdr(start, p, p, true);
|
|
723
|
+
const unit = opaqueUnitEnd(p);
|
|
724
|
+
if (unit >= 0) {
|
|
725
|
+
p = Math.min(unit, limit);
|
|
726
|
+
continue;
|
|
727
|
+
}
|
|
604
728
|
const c = s.charCodeAt(p);
|
|
605
729
|
if (c !== CH_CR && c !== CH_LF && c !== CH_SPACE && c !== CH_TAB && c !== CH_BACKSLASH && c !== CH_QUOTE) {
|
|
606
730
|
p += 1;
|
|
@@ -626,6 +750,8 @@ function parsePemHeaderLine(s, i, limit, allowBare, json) {
|
|
|
626
750
|
}
|
|
627
751
|
let q = p;
|
|
628
752
|
for (;;) {
|
|
753
|
+
if (opaqueUnitEnd(q) >= 0)
|
|
754
|
+
break;
|
|
629
755
|
const b = pemLineBreakLen(s, q, limit);
|
|
630
756
|
if (b === 1)
|
|
631
757
|
return setPemHdr(start, p, q + 1, false);
|
|
@@ -651,7 +777,7 @@ function parsePemBody(s, i, limit) {
|
|
|
651
777
|
for (;;) {
|
|
652
778
|
const q = skipPemSeps(s, p, limit);
|
|
653
779
|
let r = q;
|
|
654
|
-
while (r < limit && isBase64Code(s.charCodeAt(r)))
|
|
780
|
+
while (r < limit && opaqueUnitEnd(r) < 0 && isBase64Code(s.charCodeAt(r)))
|
|
655
781
|
r += 1;
|
|
656
782
|
if (r - q < PEM_BODY_RUN_MIN)
|
|
657
783
|
break;
|
|
@@ -664,13 +790,15 @@ function rightmostPemRunEnd(s, from, to) {
|
|
|
664
790
|
let best = -1;
|
|
665
791
|
let runStart = -1;
|
|
666
792
|
for (let p = from; p <= to; p++) {
|
|
667
|
-
if (p < to
|
|
668
|
-
const
|
|
669
|
-
if (
|
|
793
|
+
if (p < to) {
|
|
794
|
+
const unit = opaqueUnitEnd(p);
|
|
795
|
+
if (unit > 0) {
|
|
670
796
|
if (runStart >= 0 && p - runStart >= PEM_BODY_RUN_MIN)
|
|
671
797
|
best = p;
|
|
672
798
|
runStart = -1;
|
|
673
|
-
|
|
799
|
+
if (unit > to)
|
|
800
|
+
return best;
|
|
801
|
+
p = unit - 1;
|
|
674
802
|
continue;
|
|
675
803
|
}
|
|
676
804
|
}
|
|
@@ -717,15 +845,74 @@ function findPemFence(text, from) {
|
|
|
717
845
|
if (close < 0)
|
|
718
846
|
return undefined;
|
|
719
847
|
from = labelFrom;
|
|
848
|
+
if (inOpaqueUnit(fence))
|
|
849
|
+
continue;
|
|
720
850
|
if (!isPemFenceLabel(text, labelFrom, close))
|
|
721
851
|
continue;
|
|
722
852
|
return { fence, after: close + PEM_FENCE_CLOSE.length };
|
|
723
853
|
}
|
|
724
854
|
}
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
855
|
+
const SSH2_FENCE_OPEN = "---- BEGIN SSH2 ENCRYPTED PRIVATE KEY ----";
|
|
856
|
+
const SSH2_FENCE_CLOSE = "---- END SSH2 ENCRYPTED PRIVATE KEY ----";
|
|
857
|
+
const SSH2_BODY_MAX = 8192;
|
|
858
|
+
function unitCountWithin(from, to, max) {
|
|
859
|
+
if (to - from <= max)
|
|
860
|
+
return true;
|
|
861
|
+
const st = scanEvidence.starts;
|
|
862
|
+
if (st.length === 0)
|
|
863
|
+
return false;
|
|
864
|
+
const k = firstCredentialSpanFrom(scanEvidence, from);
|
|
865
|
+
if (k >= st.length || st[k] >= to)
|
|
866
|
+
return false;
|
|
867
|
+
let n = 0;
|
|
868
|
+
for (let p = from; p < to;) {
|
|
869
|
+
if (n > max)
|
|
870
|
+
return false;
|
|
871
|
+
const unit = opaqueUnitEnd(p);
|
|
872
|
+
p = unit > 0 && unit <= to ? unit : p + 1;
|
|
873
|
+
n += 1;
|
|
874
|
+
}
|
|
875
|
+
return n <= max;
|
|
876
|
+
}
|
|
877
|
+
function collectSsh2FenceSpans(text, span) {
|
|
878
|
+
let cachedFrom = 0;
|
|
879
|
+
let cachedAt;
|
|
880
|
+
const nextClose = (from) => {
|
|
881
|
+
if (cachedAt !== undefined && cachedFrom <= from && (cachedAt < 0 || cachedAt >= from))
|
|
882
|
+
return cachedAt;
|
|
883
|
+
let p = text.indexOf(SSH2_FENCE_CLOSE, from);
|
|
884
|
+
ssh2ScanBytes += (p < 0 ? text.length : p) - from;
|
|
885
|
+
while (p >= 0 && inOpaqueUnit(p)) {
|
|
886
|
+
const q = text.indexOf(SSH2_FENCE_CLOSE, p + 1);
|
|
887
|
+
ssh2ScanBytes += (q < 0 ? text.length : q) - (p + 1);
|
|
888
|
+
p = q;
|
|
889
|
+
}
|
|
890
|
+
cachedFrom = from;
|
|
891
|
+
cachedAt = p;
|
|
892
|
+
return p;
|
|
893
|
+
};
|
|
894
|
+
for (let from = 0;;) {
|
|
895
|
+
const fence = text.indexOf(SSH2_FENCE_OPEN, from);
|
|
896
|
+
if (fence < 0)
|
|
897
|
+
return;
|
|
898
|
+
if (inOpaqueUnit(fence)) {
|
|
899
|
+
from = fence + 1;
|
|
900
|
+
continue;
|
|
901
|
+
}
|
|
902
|
+
const bodyFrom = fence + SSH2_FENCE_OPEN.length;
|
|
903
|
+
const close = nextClose(bodyFrom);
|
|
904
|
+
if (close < 0)
|
|
905
|
+
return;
|
|
906
|
+
if (!unitCountWithin(bodyFrom, close, SSH2_BODY_MAX)) {
|
|
907
|
+
from = fence + 1;
|
|
908
|
+
continue;
|
|
909
|
+
}
|
|
910
|
+
const end = close + SSH2_FENCE_CLOSE.length;
|
|
911
|
+
span(fence, end);
|
|
912
|
+
from = scanResumeAfterHit(fence, end, SSH2_FENCE_OPEN.length);
|
|
913
|
+
}
|
|
914
|
+
}
|
|
915
|
+
function collectPemFragmentSpans(text, json, span) {
|
|
729
916
|
let cur = findPemFence(text, 0);
|
|
730
917
|
let inherited = false;
|
|
731
918
|
while (cur !== undefined) {
|
|
@@ -752,29 +939,24 @@ function redactPemFragments(text, json) {
|
|
|
752
939
|
let bodyEnd = parsePemBody(text, pos, limit);
|
|
753
940
|
if (bodyEnd < 0)
|
|
754
941
|
bodyEnd = rightmost;
|
|
755
|
-
if (bodyEnd >= 0)
|
|
756
|
-
|
|
757
|
-
out.push(text.slice(copied, cur.fence), PEM_FRAGMENT_TOKEN);
|
|
758
|
-
copied = bodyEnd;
|
|
759
|
-
count += 1;
|
|
760
|
-
}
|
|
942
|
+
if (bodyEnd >= 0)
|
|
943
|
+
span(cur.fence, bodyEnd);
|
|
761
944
|
cur = next;
|
|
762
945
|
}
|
|
763
|
-
if (out === undefined)
|
|
764
|
-
return { out: text, count: 0 };
|
|
765
|
-
out.push(text.slice(copied));
|
|
766
|
-
return { out: out.join(""), count };
|
|
767
946
|
}
|
|
768
|
-
function
|
|
947
|
+
function collectHeadlessPpkSpans(text, span) {
|
|
769
948
|
const KEY = "Private-Lines: ";
|
|
770
949
|
const n = text.length;
|
|
771
950
|
const isB64 = (c) => (c >= 48 && c <= 57) || (c >= 65 && c <= 90) || (c >= 97 && c <= 122) || c === 43 || c === 47 || c === 61;
|
|
772
|
-
const
|
|
951
|
+
const b64At = (i) => opaqueUnitEnd(i) < 0 && isB64(text.charCodeAt(i));
|
|
952
|
+
const skipHs = (i) => { while (i < n && opaqueUnitEnd(i) < 0 && (text.charCodeAt(i) === 32 || text.charCodeAt(i) === 9))
|
|
773
953
|
i += 1; return i; };
|
|
774
954
|
const skipNewlines = (i) => {
|
|
775
955
|
let q = skipHs(i);
|
|
776
956
|
let seen = false;
|
|
777
957
|
for (;;) {
|
|
958
|
+
if (opaqueUnitEnd(q) >= 0)
|
|
959
|
+
break;
|
|
778
960
|
let any = false;
|
|
779
961
|
if (text.charCodeAt(q) === 13) {
|
|
780
962
|
q += 1;
|
|
@@ -806,6 +988,8 @@ function redactHeadlessPpkBodies(text) {
|
|
|
806
988
|
return seen ? q : -1;
|
|
807
989
|
};
|
|
808
990
|
const escapedNewlineLen = (i) => {
|
|
991
|
+
if (opaqueUnitEnd(i) >= 0)
|
|
992
|
+
return 0;
|
|
809
993
|
let k = i;
|
|
810
994
|
while (k < n && text.charCodeAt(k) === 92)
|
|
811
995
|
k += 1;
|
|
@@ -818,20 +1002,23 @@ function redactHeadlessPpkBodies(text) {
|
|
|
818
1002
|
const b = skipHs(i);
|
|
819
1003
|
if (b >= n)
|
|
820
1004
|
return true;
|
|
1005
|
+
if (opaqueUnitEnd(b) >= 0)
|
|
1006
|
+
return false;
|
|
821
1007
|
const c = text.charCodeAt(b);
|
|
822
1008
|
return c === 10 || c === 13 || c === 34 || escapedNewlineLen(b) > 0;
|
|
823
1009
|
};
|
|
824
|
-
let out = "";
|
|
825
|
-
let last = 0;
|
|
826
|
-
let count = 0;
|
|
827
1010
|
let search = 0;
|
|
828
1011
|
for (;;) {
|
|
829
1012
|
const at = text.indexOf(KEY, search);
|
|
830
1013
|
if (at === -1)
|
|
831
1014
|
break;
|
|
1015
|
+
if (inOpaqueUnit(at)) {
|
|
1016
|
+
search = at + KEY.length;
|
|
1017
|
+
continue;
|
|
1018
|
+
}
|
|
832
1019
|
let p = at + KEY.length;
|
|
833
1020
|
const d0 = p;
|
|
834
|
-
while (p < n && text.charCodeAt(p) >= 48 && text.charCodeAt(p) <= 57)
|
|
1021
|
+
while (p < n && opaqueUnitEnd(p) < 0 && text.charCodeAt(p) >= 48 && text.charCodeAt(p) <= 57)
|
|
835
1022
|
p += 1;
|
|
836
1023
|
if (p === d0) {
|
|
837
1024
|
search = at + KEY.length;
|
|
@@ -844,15 +1031,15 @@ function redactHeadlessPpkBodies(text) {
|
|
|
844
1031
|
if (q0 === -1)
|
|
845
1032
|
break;
|
|
846
1033
|
let q = q0;
|
|
847
|
-
const mk = text.charCodeAt(q);
|
|
1034
|
+
const mk = opaqueUnitEnd(q) < 0 ? text.charCodeAt(q) : -1;
|
|
848
1035
|
if (mk === 43 || mk === 45)
|
|
849
1036
|
q = skipHs(q + 1);
|
|
850
1037
|
let r = q;
|
|
851
|
-
while (r < n &&
|
|
1038
|
+
while (r < n && b64At(r))
|
|
852
1039
|
r += 1;
|
|
853
1040
|
if (r - q < 4) {
|
|
854
1041
|
r = q0;
|
|
855
|
-
while (r < n &&
|
|
1042
|
+
while (r < n && b64At(r))
|
|
856
1043
|
r += 1;
|
|
857
1044
|
if (r - q0 < 4)
|
|
858
1045
|
break;
|
|
@@ -866,12 +1053,9 @@ function redactHeadlessPpkBodies(text) {
|
|
|
866
1053
|
search = p;
|
|
867
1054
|
continue;
|
|
868
1055
|
}
|
|
869
|
-
|
|
870
|
-
last = end;
|
|
1056
|
+
span(at, end);
|
|
871
1057
|
search = end;
|
|
872
|
-
count += 1;
|
|
873
1058
|
}
|
|
874
|
-
return count === 0 ? { out: text, count: 0 } : { out: out + text.slice(last), count };
|
|
875
1059
|
}
|
|
876
1060
|
const FRAGMENT_SECRET_PATTERNS = [
|
|
877
1061
|
[valueArm("sk-", "[A-Za-z0-9_-]", 6, "needs-left-boundary"), "«redacted:key:partial»"],
|
|
@@ -885,21 +1069,12 @@ const FRAGMENT_SECRET_PATTERNS = [
|
|
|
885
1069
|
];
|
|
886
1070
|
const REDACTION_TOKENS = new Set([...FULL_ARM_PATTERNS.map(([, r]) => r), PEM_FRAGMENT_TOKEN, ...FRAGMENT_SECRET_PATTERNS.map(([, r]) => r)]
|
|
887
1071
|
.flatMap((r) => r.match(/«[^»]*»/g) ?? []));
|
|
888
|
-
const REDACTION_TOKEN_MAX = Math.max(...[...REDACTION_TOKENS].map((t) => t.length));
|
|
889
1072
|
{
|
|
890
1073
|
const closed = new RegExp(`^${REDACTION_TOKEN_RE_SRC}$`);
|
|
891
1074
|
for (const t of REDACTION_TOKENS)
|
|
892
1075
|
if (!closed.test(t))
|
|
893
1076
|
throw new Error(`redact: token ${t} is outside the closed form ${REDACTION_TOKEN_RE_SRC}`);
|
|
894
1077
|
}
|
|
895
|
-
function redactionTokenEndAt(s, p, limit) {
|
|
896
|
-
const max = Math.min(limit, p + REDACTION_TOKEN_MAX);
|
|
897
|
-
for (let e = p + 2; e <= max; e++) {
|
|
898
|
-
if (s.charCodeAt(e - 1) === 187)
|
|
899
|
-
return REDACTION_TOKENS.has(s.slice(p, e)) ? e : 0;
|
|
900
|
-
}
|
|
901
|
-
return 0;
|
|
902
|
-
}
|
|
903
1078
|
const labelOfToken = (token) => token.replace(/[«»]/g, "").replace(/^redacted:/, "");
|
|
904
1079
|
const FRAGMENT_ARM_TOKENS = new Set(FRAGMENT_SECRET_PATTERNS.map(([, repl]) => repl));
|
|
905
1080
|
{
|
|
@@ -916,42 +1091,118 @@ const FRAGMENT_ARM_TOKENS = new Set(FRAGMENT_SECRET_PATTERNS.map(([, repl]) => r
|
|
|
916
1091
|
}
|
|
917
1092
|
const armOfToken = (repl) => (FRAGMENT_ARM_TOKENS.has(repl) ? "fragment" : "full");
|
|
918
1093
|
const NO_HITS = [];
|
|
919
|
-
const singleHit = (label, arm, r) => r.count > 0 ? { out: r.out, hits: [{ label, count: r.count, arm }] } : { out: r.out, hits: NO_HITS };
|
|
920
|
-
const regexRule = ([re, repl]) => {
|
|
921
|
-
const groupCount = (new RegExp(`${re.source}|`).exec("")?.length ?? 1) - 1;
|
|
922
|
-
const refs = [...repl.matchAll(/\$(.)/g)].map(([, d]) => d ?? "");
|
|
923
|
-
for (const d of refs) {
|
|
924
|
-
if (!/^[1-9]$/.test(d) || Number(d) > groupCount)
|
|
925
|
-
throw new Error(`redact: replacement ${JSON.stringify(repl)} uses unsupported ref $${d}`);
|
|
926
|
-
}
|
|
927
|
-
const expand = refs.length === 0 ? undefined : (args) => repl.replace(/\$([1-9])/g, (_, d) => args[Number(d)] ?? "");
|
|
928
|
-
const arm = armOfToken(repl);
|
|
929
|
-
if (VALUE_ARMS.has(re) || arm === "fragment")
|
|
930
|
-
throw new Error(`redact: rule ${JSON.stringify(repl)} belongs in the arm pool, not a per-arm section`);
|
|
931
|
-
const label = labelOfToken(repl);
|
|
932
|
-
const applyOnce = (text, wantCount) => {
|
|
933
|
-
if (!wantCount)
|
|
934
|
-
return { out: text.replace(re, repl), count: 0 };
|
|
935
|
-
let n = 0;
|
|
936
|
-
const out = text.replace(re, (...args) => {
|
|
937
|
-
n += 1;
|
|
938
|
-
return expand === undefined ? repl : expand(args);
|
|
939
|
-
});
|
|
940
|
-
return { out, count: n };
|
|
941
|
-
};
|
|
942
|
-
return { run: (text, wantCount) => singleHit(label, arm, applyOnce(text, wantCount)) };
|
|
943
|
-
};
|
|
944
1094
|
const URL_USERINFO_PATTERN = (() => {
|
|
945
1095
|
const row = FULL_ARM_PATTERNS.find(([, repl]) => repl.endsWith("»@"));
|
|
946
1096
|
if (row === undefined)
|
|
947
1097
|
throw new Error("redact: URL-userinfo rule missing from FULL_ARM_PATTERNS");
|
|
948
1098
|
return row;
|
|
949
1099
|
})();
|
|
1100
|
+
const SSH2_FENCE_PATTERN = (() => {
|
|
1101
|
+
const row = FULL_ARM_PATTERNS.find(([re]) => re.source.startsWith(SSH2_FENCE_OPEN));
|
|
1102
|
+
if (row === undefined)
|
|
1103
|
+
throw new Error("redact: SSH2 fence rule missing from FULL_ARM_PATTERNS");
|
|
1104
|
+
return row;
|
|
1105
|
+
})();
|
|
950
1106
|
const isRegexSpace = (c) => c === 32 || (c >= 9 && c <= 13) || c === 0xa0 || c === 0x1680 || (c >= 0x2000 && c <= 0x200a) || c === 0x2028 || c === 0x2029 || c === 0x202f || c === 0x205f || c === 0x3000 || c === 0xfeff;
|
|
951
1107
|
const isLineTerminator = (c) => c === 10 || c === 13 || c === 0x2028 || c === 0x2029;
|
|
1108
|
+
const TOKEN_UNIT_Y = new RegExp(REDACTION_TOKEN_RE_SRC, "y");
|
|
1109
|
+
function tokenUnitEnd(text, p) {
|
|
1110
|
+
TOKEN_UNIT_Y.lastIndex = p;
|
|
1111
|
+
return TOKEN_UNIT_Y.test(text) ? TOKEN_UNIT_Y.lastIndex : -1;
|
|
1112
|
+
}
|
|
1113
|
+
const NO_CREDENTIAL_SPANS = { starts: new Int32Array(0), ends: new Int32Array(0) };
|
|
1114
|
+
function credentialSpans(text, spans, withSpans) {
|
|
1115
|
+
const iv = withSpans ? spans.map((sp) => [sp.start, sp.end]) : [];
|
|
1116
|
+
for (let i = text.indexOf("«"); i >= 0; i = text.indexOf("«", i + 1)) {
|
|
1117
|
+
const e = tokenUnitEnd(text, i);
|
|
1118
|
+
if (e > 0) {
|
|
1119
|
+
iv.push([i, e]);
|
|
1120
|
+
i = e - 1;
|
|
1121
|
+
}
|
|
1122
|
+
}
|
|
1123
|
+
if (iv.length === 0)
|
|
1124
|
+
return NO_CREDENTIAL_SPANS;
|
|
1125
|
+
iv.sort((a, b) => a[0] - b[0]);
|
|
1126
|
+
const starts = new Int32Array(iv.length);
|
|
1127
|
+
const ends = new Int32Array(iv.length);
|
|
1128
|
+
let k = 0;
|
|
1129
|
+
let [cs, ce] = iv[0];
|
|
1130
|
+
for (let j = 1; j < iv.length; j++) {
|
|
1131
|
+
const [s2, e2] = iv[j];
|
|
1132
|
+
if (s2 <= ce)
|
|
1133
|
+
ce = Math.max(ce, e2);
|
|
1134
|
+
else {
|
|
1135
|
+
starts[k] = cs;
|
|
1136
|
+
ends[k] = ce;
|
|
1137
|
+
k += 1;
|
|
1138
|
+
cs = s2;
|
|
1139
|
+
ce = e2;
|
|
1140
|
+
}
|
|
1141
|
+
}
|
|
1142
|
+
starts[k] = cs;
|
|
1143
|
+
ends[k] = ce;
|
|
1144
|
+
k += 1;
|
|
1145
|
+
return { starts: starts.subarray(0, k), ends: ends.subarray(0, k) };
|
|
1146
|
+
}
|
|
1147
|
+
function sameCredentialSpans(a, b) {
|
|
1148
|
+
if (a.starts.length !== b.starts.length)
|
|
1149
|
+
return false;
|
|
1150
|
+
for (let i = 0; i < a.starts.length; i++)
|
|
1151
|
+
if (a.starts[i] !== b.starts[i] || a.ends[i] !== b.ends[i])
|
|
1152
|
+
return false;
|
|
1153
|
+
return true;
|
|
1154
|
+
}
|
|
1155
|
+
function firstCredentialSpanFrom(cs, from) {
|
|
1156
|
+
let lo = 0;
|
|
1157
|
+
let hi = cs.starts.length;
|
|
1158
|
+
while (lo < hi) {
|
|
1159
|
+
const mid = (lo + hi) >>> 1;
|
|
1160
|
+
if (cs.starts[mid] < from)
|
|
1161
|
+
lo = mid + 1;
|
|
1162
|
+
else
|
|
1163
|
+
hi = mid;
|
|
1164
|
+
}
|
|
1165
|
+
return lo;
|
|
1166
|
+
}
|
|
1167
|
+
function credentialSpansCover(cs, from, to) {
|
|
1168
|
+
let k = firstCredentialSpanFrom(cs, from + 1) - 1;
|
|
1169
|
+
if (k < 0)
|
|
1170
|
+
k = 0;
|
|
1171
|
+
return k < cs.starts.length && cs.starts[k] <= from && cs.ends[k] >= to;
|
|
1172
|
+
}
|
|
1173
|
+
let scanEvidence = NO_CREDENTIAL_SPANS;
|
|
1174
|
+
let scanEvidenceCursor = 0;
|
|
1175
|
+
function setScanEvidence(cs) {
|
|
1176
|
+
scanEvidence = cs;
|
|
1177
|
+
scanEvidenceCursor = 0;
|
|
1178
|
+
}
|
|
1179
|
+
function inOpaqueUnit(p) {
|
|
1180
|
+
const st = scanEvidence.starts;
|
|
1181
|
+
if (st.length === 0)
|
|
1182
|
+
return false;
|
|
1183
|
+
const k = firstCredentialSpanFrom(scanEvidence, p) - 1;
|
|
1184
|
+
return k >= 0 && st[k] < p && scanEvidence.ends[k] > p;
|
|
1185
|
+
}
|
|
1186
|
+
function opaqueUnitEnd(p) {
|
|
1187
|
+
const st = scanEvidence.starts;
|
|
1188
|
+
if (st.length === 0)
|
|
1189
|
+
return -1;
|
|
1190
|
+
let k = scanEvidenceCursor;
|
|
1191
|
+
while (k < st.length && st[k] < p)
|
|
1192
|
+
k += 1;
|
|
1193
|
+
while (k > 0 && st[k - 1] >= p)
|
|
1194
|
+
k -= 1;
|
|
1195
|
+
scanEvidenceCursor = k;
|
|
1196
|
+
return k < st.length && st[k] === p ? scanEvidence.ends[k] : -1;
|
|
1197
|
+
}
|
|
952
1198
|
function scanUserinfoRun(text, i, kind, json) {
|
|
953
1199
|
const n = text.length;
|
|
954
1200
|
while (i < n) {
|
|
1201
|
+
const unit = opaqueUnitEnd(i);
|
|
1202
|
+
if (unit >= 0) {
|
|
1203
|
+
i = unit;
|
|
1204
|
+
continue;
|
|
1205
|
+
}
|
|
955
1206
|
const c = text.charCodeAt(i);
|
|
956
1207
|
if (json && c === 92) {
|
|
957
1208
|
if (i + 1 < n && !isLineTerminator(text.charCodeAt(i + 1))) {
|
|
@@ -980,48 +1231,19 @@ function scanUserinfoRun(text, i, kind, json) {
|
|
|
980
1231
|
}
|
|
981
1232
|
return i;
|
|
982
1233
|
}
|
|
983
|
-
const REDACTION_TOKEN_Y = new RegExp(REDACTION_TOKEN_RE_SRC, "y");
|
|
984
|
-
const SHAPE_STICKY = SHAPE_SECRET_PATTERNS.map(([re]) => new RegExp(re.source, "y"));
|
|
985
|
-
function keyShapedUsernameEnd(text, p) {
|
|
986
|
-
for (const re of SHAPE_STICKY) {
|
|
987
|
-
re.lastIndex = p;
|
|
988
|
-
if (re.test(text))
|
|
989
|
-
return re.lastIndex;
|
|
990
|
-
}
|
|
991
|
-
return -1;
|
|
992
|
-
}
|
|
993
1234
|
function matchUserinfoAt(text, p, json) {
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
const afterTok = tokEnd >= 0 ? text.charCodeAt(tokEnd) : -1;
|
|
997
|
-
if (afterTok === 64)
|
|
1235
|
+
const userEnd = scanUserinfoRun(text, p, "plain-user", json);
|
|
1236
|
+
if (text.charCodeAt(userEnd) !== 58)
|
|
998
1237
|
return null;
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
if (afterTok === 58) {
|
|
1002
|
-
userEnd = tokEnd;
|
|
1003
|
-
kind = "token-pass";
|
|
1004
|
-
}
|
|
1005
|
-
else if ((() => { const e = keyShapedUsernameEnd(text, p); return e >= 0 && text.charCodeAt(e) === 58 ? ((userEnd = e), true) : false; })()) {
|
|
1006
|
-
kind = "token-pass";
|
|
1007
|
-
}
|
|
1008
|
-
else {
|
|
1009
|
-
userEnd = scanUserinfoRun(text, p, "plain-user", json);
|
|
1010
|
-
if (text.charCodeAt(userEnd) !== 58)
|
|
1011
|
-
return null;
|
|
1012
|
-
kind = "plain-pass";
|
|
1013
|
-
}
|
|
1014
|
-
const passEnd = scanUserinfoRun(text, userEnd + 1, kind, json);
|
|
1238
|
+
const keyId = userEnd > p && credentialSpansCover(scanEvidence, p, userEnd);
|
|
1239
|
+
const passEnd = scanUserinfoRun(text, userEnd + 1, keyId ? "token-pass" : "plain-pass", json);
|
|
1015
1240
|
if (passEnd === userEnd + 1 || text.charCodeAt(passEnd) !== 64)
|
|
1016
1241
|
return null;
|
|
1017
1242
|
return [userEnd, passEnd + 1];
|
|
1018
1243
|
}
|
|
1019
|
-
function
|
|
1020
|
-
const
|
|
1244
|
+
function collectUrlUserinfoSpans(text, json, span) {
|
|
1245
|
+
const head = URL_USERINFO_PATTERN[0];
|
|
1021
1246
|
head.lastIndex = 0;
|
|
1022
|
-
let out = "";
|
|
1023
|
-
let last = 0;
|
|
1024
|
-
let count = 0;
|
|
1025
1247
|
for (let m = head.exec(text); m !== null; m = head.exec(text)) {
|
|
1026
1248
|
const colon = m.index;
|
|
1027
1249
|
const hit = matchUserinfoAt(text, colon + 3, json);
|
|
@@ -1029,18 +1251,14 @@ function redactUrlUserinfo(text, json) {
|
|
|
1029
1251
|
head.lastIndex = colon + 1;
|
|
1030
1252
|
continue;
|
|
1031
1253
|
}
|
|
1032
|
-
|
|
1033
|
-
out += text.slice(last, colon) + repl.replace(/\$1/g, () => user);
|
|
1034
|
-
last = hit[1];
|
|
1254
|
+
span(hit[0], hit[1]);
|
|
1035
1255
|
head.lastIndex = hit[1];
|
|
1036
|
-
count += 1;
|
|
1037
1256
|
}
|
|
1038
|
-
return count === 0 ? { out: text, count: 0 } : { out: out + text.slice(last), count };
|
|
1039
1257
|
}
|
|
1040
1258
|
let ppkScanBytes = 0;
|
|
1041
1259
|
export function ppkScanBytesForTest() { return ppkScanBytes; }
|
|
1042
1260
|
export function resetPpkScanBytesForTest() { ppkScanBytes = 0; }
|
|
1043
|
-
function
|
|
1261
|
+
function collectPpkContainerSpans(text, record, span) {
|
|
1044
1262
|
const HEAD = "PuTTY-User-Key-File-";
|
|
1045
1263
|
const MAC = "Private-MAC: ";
|
|
1046
1264
|
const HASH = "Private-Hash: ";
|
|
@@ -1059,7 +1277,7 @@ function redactPpkContainers(text, record) {
|
|
|
1059
1277
|
const isVersionedHead = (at) => {
|
|
1060
1278
|
let p = at + HEAD.length;
|
|
1061
1279
|
const start = p;
|
|
1062
|
-
while (p < text.length) {
|
|
1280
|
+
while (p < text.length && opaqueUnitEnd(p) < 0) {
|
|
1063
1281
|
const c = text.charCodeAt(p);
|
|
1064
1282
|
if (c >= 48 && c <= 57)
|
|
1065
1283
|
p += 1;
|
|
@@ -1070,13 +1288,13 @@ function redactPpkContainers(text, record) {
|
|
|
1070
1288
|
};
|
|
1071
1289
|
const nextVersionedHead = (from) => {
|
|
1072
1290
|
let p = find(HEAD, from);
|
|
1073
|
-
while (p !== Number.POSITIVE_INFINITY && !isVersionedHead(p))
|
|
1291
|
+
while (p !== Number.POSITIVE_INFINITY && (inOpaqueUnit(p) || !isVersionedHead(p)))
|
|
1074
1292
|
p = find(HEAD, p + HEAD.length);
|
|
1075
1293
|
return p;
|
|
1076
1294
|
};
|
|
1077
1295
|
const hexRunEnd = (from) => {
|
|
1078
1296
|
let e = from;
|
|
1079
|
-
while (e < text.length) {
|
|
1297
|
+
while (e < text.length && opaqueUnitEnd(e) < 0) {
|
|
1080
1298
|
const c = text.charCodeAt(e);
|
|
1081
1299
|
if ((c >= 48 && c <= 57) || (c >= 97 && c <= 102) || (c >= 65 && c <= 70))
|
|
1082
1300
|
e += 1;
|
|
@@ -1085,9 +1303,6 @@ function redactPpkContainers(text, record) {
|
|
|
1085
1303
|
}
|
|
1086
1304
|
return e;
|
|
1087
1305
|
};
|
|
1088
|
-
let out = "";
|
|
1089
|
-
let last = 0;
|
|
1090
|
-
let count = 0;
|
|
1091
1306
|
let at = nextVersionedHead(0);
|
|
1092
1307
|
while (at !== Number.POSITIVE_INFINITY) {
|
|
1093
1308
|
let pos = at + HEAD.length + 1;
|
|
@@ -1114,6 +1329,10 @@ function redactPpkContainers(text, record) {
|
|
|
1114
1329
|
let gapBlank = true;
|
|
1115
1330
|
if (limit !== Number.POSITIVE_INFINITY) {
|
|
1116
1331
|
for (let i = hexEnd; i < limit; i++) {
|
|
1332
|
+
if (opaqueUnitEnd(i) >= 0) {
|
|
1333
|
+
gapBlank = false;
|
|
1334
|
+
break;
|
|
1335
|
+
}
|
|
1117
1336
|
const c = text.charCodeAt(i);
|
|
1118
1337
|
if (c !== 32 && c !== 10 && c !== 13 && c !== 9) {
|
|
1119
1338
|
gapBlank = false;
|
|
@@ -1127,103 +1346,206 @@ function redactPpkContainers(text, record) {
|
|
|
1127
1346
|
}
|
|
1128
1347
|
if (ateToEnd && record)
|
|
1129
1348
|
recordFailOpen("server.redact.ppk-eat-to-end", `from=${at} len=${text.length}`);
|
|
1130
|
-
|
|
1131
|
-
last = end;
|
|
1132
|
-
count += 1;
|
|
1349
|
+
span(at, end);
|
|
1133
1350
|
at = end >= text.length ? Number.POSITIVE_INFINITY : nextVersionedHead(end);
|
|
1134
1351
|
}
|
|
1135
|
-
return count === 0 ? { out: text, count: 0 } : { out: out + text.slice(last), count };
|
|
1136
1352
|
}
|
|
1137
|
-
const
|
|
1138
|
-
const
|
|
1139
|
-
const
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1353
|
+
const armPool = (() => {
|
|
1354
|
+
const spanRepl = (token) => {
|
|
1355
|
+
const body = token.startsWith("$1") ? token.slice(2) : token;
|
|
1356
|
+
if (/\$./.test(body))
|
|
1357
|
+
throw new Error(`redact: arm-pool replacement ${JSON.stringify(token)} may only use a leading $1 (the run kept outside the span)`);
|
|
1358
|
+
return body;
|
|
1359
|
+
};
|
|
1360
|
+
const literalAnchorHead = (re) => {
|
|
1361
|
+
const src = re.source;
|
|
1362
|
+
let i = src.startsWith("(") && !src.startsWith("(?") ? 1 : 0;
|
|
1363
|
+
let out = "";
|
|
1364
|
+
for (; i < src.length; i++) {
|
|
1365
|
+
const c = src[i];
|
|
1366
|
+
if (c === "\\") {
|
|
1367
|
+
const e = src[i + 1];
|
|
1368
|
+
if (e === undefined || !HEAD_LITERAL_ESCAPES.has(e))
|
|
1369
|
+
break;
|
|
1370
|
+
if ("*+?{".includes(src[i + 2] ?? ""))
|
|
1371
|
+
break;
|
|
1372
|
+
out += src.slice(i, i + 2);
|
|
1373
|
+
i += 1;
|
|
1374
|
+
continue;
|
|
1375
|
+
}
|
|
1376
|
+
if ("([{|).^$*+?".includes(c))
|
|
1377
|
+
break;
|
|
1378
|
+
if ("*+?{".includes(src[i + 1] ?? ""))
|
|
1379
|
+
break;
|
|
1380
|
+
out += c;
|
|
1381
|
+
}
|
|
1382
|
+
if (out.length === 0)
|
|
1383
|
+
throw new Error(`redact: structural arm /${src}/ has no literal anchor head — its same-arm rewind amount cannot be derived`);
|
|
1384
|
+
return out;
|
|
1385
|
+
};
|
|
1386
|
+
const regexMember = ([re, token], rank, order, structural) => {
|
|
1387
|
+
const repl = spanRepl(token);
|
|
1388
|
+
const outside = token.startsWith("$1") ? 1 : 0;
|
|
1143
1389
|
const y = VALUE_ARM_STICKY.get(re);
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
const
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1390
|
+
let back;
|
|
1391
|
+
let headSrc;
|
|
1392
|
+
let headFirst;
|
|
1393
|
+
if (structural) {
|
|
1394
|
+
back = literalAnchorHead(re).length;
|
|
1395
|
+
headSrc = undefined;
|
|
1396
|
+
headFirst = undefined;
|
|
1397
|
+
}
|
|
1398
|
+
else {
|
|
1399
|
+
const d = deriveLiteralHead(re);
|
|
1400
|
+
back = Math.max(d.len - 1, 0);
|
|
1401
|
+
headSrc = d.src;
|
|
1402
|
+
headFirst = d.first;
|
|
1403
|
+
if (y !== undefined && (d.len === 0 || headSrc === undefined || headFirst === undefined)) {
|
|
1404
|
+
throw new Error(`redact: boundary arm /${re.source}/ has a zero-length structural head — the bounded head screen would admit every candidate position`);
|
|
1159
1405
|
}
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1406
|
+
}
|
|
1407
|
+
if (y !== undefined && outside === 1)
|
|
1408
|
+
throw new Error(`redact: value arm /${re.source}/ cannot carry an outside-span $1 prefix`);
|
|
1409
|
+
const arm = { g: re, y, back, headSrc, headFirst, repl, rank, order, outside };
|
|
1410
|
+
const boundary = y === undefined || headSrc === undefined || headFirst === undefined ? undefined : { ...arm, y, headSrc, headFirst };
|
|
1411
|
+
return {
|
|
1412
|
+
token,
|
|
1413
|
+
rank,
|
|
1414
|
+
order,
|
|
1415
|
+
phase: 1,
|
|
1416
|
+
collect: boundary === undefined
|
|
1417
|
+
? (text, _json, _observe, out) => collectRegexSpans(arm, text, out)
|
|
1418
|
+
: (text, _json, _observe, out) => collectValueArmSpans(boundary, text, out),
|
|
1419
|
+
retry: boundary === undefined ? undefined : (text, at, out) => collectStickySpans(boundary, text, at, out),
|
|
1420
|
+
headSrc: boundary === undefined ? undefined : headSrc,
|
|
1421
|
+
headFirst: boundary === undefined ? undefined : headFirst,
|
|
1422
|
+
};
|
|
1423
|
+
};
|
|
1424
|
+
const scannerMember = (token, order, phase, scan) => {
|
|
1425
|
+
const repl = spanRepl(token);
|
|
1426
|
+
return {
|
|
1427
|
+
token,
|
|
1428
|
+
rank: ARM_RANK.structural,
|
|
1429
|
+
order,
|
|
1430
|
+
phase,
|
|
1431
|
+
collect: (text, json, observe, out) => scan(text, json, observe, (start, end) => out.push({ start, end, repl, rank: ARM_RANK.structural, order })),
|
|
1432
|
+
retry: undefined,
|
|
1433
|
+
headSrc: undefined,
|
|
1434
|
+
headFirst: undefined,
|
|
1435
|
+
};
|
|
1436
|
+
};
|
|
1437
|
+
const members = [];
|
|
1438
|
+
for (const row of STRUCTURAL_SECRET_PATTERNS) {
|
|
1439
|
+
const order = members.length;
|
|
1440
|
+
members.push(row === URL_USERINFO_PATTERN
|
|
1441
|
+
? scannerMember(row[1], order, 2, (text, json, _observe, span) => collectUrlUserinfoSpans(text, json, span))
|
|
1442
|
+
: row === SSH2_FENCE_PATTERN
|
|
1443
|
+
? scannerMember(row[1], order, 2, (text, _json, _observe, span) => collectSsh2FenceSpans(text, span))
|
|
1444
|
+
: regexMember(row, ARM_RANK.structural, order, true));
|
|
1445
|
+
}
|
|
1446
|
+
members.push(scannerMember(PPK_CONTAINER_TOKEN, members.length, 2, (text, _json, observe, span) => collectPpkContainerSpans(text, observe, span)));
|
|
1447
|
+
members.push(scannerMember(PEM_FRAGMENT_TOKEN, members.length, 2, (text, json, _observe, span) => collectPemFragmentSpans(text, json, span)));
|
|
1448
|
+
members.push(scannerMember(HEADLESS_PPK_TOKEN, members.length, 2, (text, _json, _observe, span) => collectHeadlessPpkSpans(text, span)));
|
|
1449
|
+
for (const row of KEYWORD_SECRET_PATTERNS)
|
|
1450
|
+
members.push(regexMember(row, ARM_RANK.keyword, members.length, false));
|
|
1451
|
+
for (const row of SHAPE_SECRET_PATTERNS)
|
|
1452
|
+
members.push(regexMember(row, ARM_RANK.shape, members.length, false));
|
|
1453
|
+
for (const row of FRAGMENT_SECRET_PATTERNS)
|
|
1454
|
+
members.push(regexMember(row, ARM_RANK.fragment, members.length, false));
|
|
1455
|
+
const retryMembers = members.filter((m) => m.retry !== undefined);
|
|
1456
|
+
const phase1 = members.filter((m) => m.phase === 1);
|
|
1457
|
+
const phase2 = members.filter((m) => m.phase === 2);
|
|
1458
|
+
const headScreen = new RegExp(`(?:${retryMembers.map((m) => m.headSrc).join("|")})`, "y");
|
|
1459
|
+
const headFirstUnit = unionMembers(retryMembers.map((m) => m.headFirst)) ?? new Uint32Array(0x10000 >>> 5).fill(0xffffffff);
|
|
1460
|
+
poolArmCount = members.length;
|
|
1461
|
+
poolBoundaryArmCount = retryMembers.length;
|
|
1462
|
+
return (text, wantCount, json, observe) => {
|
|
1463
|
+
const spans = [];
|
|
1464
|
+
let tried;
|
|
1465
|
+
const freshCovered = (from) => {
|
|
1466
|
+
const at = [];
|
|
1467
|
+
const seen = (tried ??= new Uint8Array(text.length + 1));
|
|
1468
|
+
for (let k = from; k < spans.length; k++) {
|
|
1469
|
+
const s = spans[k];
|
|
1470
|
+
for (let p = s.start; p <= s.end; p++) {
|
|
1471
|
+
if (seen[p] === 1)
|
|
1472
|
+
continue;
|
|
1473
|
+
seen[p] = 1;
|
|
1474
|
+
coveredPositions += 1;
|
|
1475
|
+
if (unitBit(headFirstUnit, text.charCodeAt(p)) !== 1)
|
|
1476
|
+
continue;
|
|
1477
|
+
headScreen.lastIndex = p;
|
|
1478
|
+
if (headScreen.exec(text) === null)
|
|
1166
1479
|
continue;
|
|
1167
|
-
|
|
1168
|
-
at.push(e);
|
|
1480
|
+
at.push(p);
|
|
1169
1481
|
}
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1482
|
+
}
|
|
1483
|
+
headScreenHits += at.length;
|
|
1484
|
+
return at.sort((x, y) => x - y);
|
|
1485
|
+
};
|
|
1486
|
+
const closeCoveragePositions = (from) => {
|
|
1487
|
+
if (retryMembers.length === 0 || spans.length === from)
|
|
1488
|
+
return;
|
|
1489
|
+
for (let at = freshCovered(from); at.length > 0;) {
|
|
1173
1490
|
const before = spans.length;
|
|
1174
|
-
const
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
at = freshEnds(before);
|
|
1491
|
+
for (const m of retryMembers)
|
|
1492
|
+
m.retry(text, at, spans);
|
|
1493
|
+
at = freshCovered(before);
|
|
1178
1494
|
}
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
let
|
|
1186
|
-
let
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
const
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
else
|
|
1198
|
-
cur.count += 1;
|
|
1495
|
+
};
|
|
1496
|
+
setScanEvidence(NO_CREDENTIAL_SPANS);
|
|
1497
|
+
for (const m of phase1)
|
|
1498
|
+
m.collect(text, json, observe, spans);
|
|
1499
|
+
closeCoveragePositions(0);
|
|
1500
|
+
if (phase2.length > 0) {
|
|
1501
|
+
let evidence = credentialSpans(text, spans, false);
|
|
1502
|
+
for (let level = 0;; level += 1) {
|
|
1503
|
+
const before = spans.length;
|
|
1504
|
+
setScanEvidence(evidence);
|
|
1505
|
+
for (const m of phase2)
|
|
1506
|
+
m.collect(text, json, observe && level === 0, spans);
|
|
1507
|
+
closeCoveragePositions(before);
|
|
1508
|
+
forensicLevels += 1;
|
|
1509
|
+
const next = credentialSpans(text, spans, true);
|
|
1510
|
+
if (sameCredentialSpans(next, evidence))
|
|
1511
|
+
break;
|
|
1512
|
+
evidence = next;
|
|
1199
1513
|
}
|
|
1200
|
-
|
|
1201
|
-
}
|
|
1514
|
+
setScanEvidence(NO_CREDENTIAL_SPANS);
|
|
1515
|
+
}
|
|
1516
|
+
poolSpanCount += spans.length;
|
|
1517
|
+
const kept = resolveSpanOverlaps(spans);
|
|
1518
|
+
const out = applySpans(text, kept).out;
|
|
1519
|
+
if (!wantCount || kept.length === 0)
|
|
1520
|
+
return { out, hits: NO_HITS };
|
|
1521
|
+
const tokenOf = (sp) => members[sp.order].token;
|
|
1522
|
+
const strong = spans.filter((sp) => armOfToken(tokenOf(sp)) === "full").sort((a, b) => a.start - b.start);
|
|
1523
|
+
const byArmedToken = new Map();
|
|
1524
|
+
let si = 0;
|
|
1525
|
+
let strongMaxEnd = -1;
|
|
1526
|
+
for (const sp of kept) {
|
|
1527
|
+
while (si < strong.length && strong[si].start < sp.end) {
|
|
1528
|
+
strongMaxEnd = Math.max(strongMaxEnd, strong[si].end);
|
|
1529
|
+
si += 1;
|
|
1530
|
+
}
|
|
1531
|
+
const token = tokenOf(sp);
|
|
1532
|
+
const arm = strongMaxEnd > sp.start ? "full" : armOfToken(token);
|
|
1533
|
+
const key = `${token}\u0000${arm}`;
|
|
1534
|
+
const cur = byArmedToken.get(key);
|
|
1535
|
+
if (cur === undefined)
|
|
1536
|
+
byArmedToken.set(key, { token, arm, count: 1 });
|
|
1537
|
+
else
|
|
1538
|
+
cur.count += 1;
|
|
1539
|
+
}
|
|
1540
|
+
return { out, hits: [...byArmedToken.values()].map(({ token, arm, count }) => ({ label: labelOfToken(token), count, arm })) };
|
|
1202
1541
|
};
|
|
1203
1542
|
})();
|
|
1204
|
-
const SECRET_RULES = [
|
|
1205
|
-
...STRUCTURAL_SECRET_PATTERNS.map((p) => (p === URL_USERINFO_PATTERN ? urlUserinfoRule : regexRule(p))),
|
|
1206
|
-
{ run: (text, _wantCount, _json, observe) => singleHit(labelOfToken(PPK_CONTAINER_TOKEN), armOfToken(PPK_CONTAINER_TOKEN), redactPpkContainers(text, observe)) },
|
|
1207
|
-
{ run: (text, _wantCount, json) => singleHit(labelOfToken(PEM_FRAGMENT_TOKEN), armOfToken(PEM_FRAGMENT_TOKEN), redactPemFragments(text, json)) },
|
|
1208
|
-
{ run: (text) => singleHit(labelOfToken(HEADLESS_PPK_TOKEN), armOfToken(HEADLESS_PPK_TOKEN), redactHeadlessPpkBodies(text)) },
|
|
1209
|
-
...KEYWORD_SECRET_PATTERNS.map((p) => regexRule(p)),
|
|
1210
|
-
armPoolRule,
|
|
1211
|
-
{ run: (text, _wantCount, json) => ({ out: redactUrlUserinfo(text, json).out, hits: NO_HITS }) },
|
|
1212
|
-
];
|
|
1213
1543
|
let onRedaction;
|
|
1214
1544
|
export function setRedactionObserver(fn) {
|
|
1215
1545
|
onRedaction = fn;
|
|
1216
1546
|
}
|
|
1217
1547
|
function applyRules(text, wantCount, json, observe) {
|
|
1218
|
-
|
|
1219
|
-
const hits = [];
|
|
1220
|
-
for (const rule of SECRET_RULES) {
|
|
1221
|
-
const r = rule.run(out, wantCount, json, observe);
|
|
1222
|
-
out = r.out;
|
|
1223
|
-
if (wantCount && r.hits.length > 0)
|
|
1224
|
-
hits.push(...r.hits);
|
|
1225
|
-
}
|
|
1226
|
-
return { out, hits };
|
|
1548
|
+
return armPool(text, wantCount, json, observe);
|
|
1227
1549
|
}
|
|
1228
1550
|
function reportHits(hits) {
|
|
1229
1551
|
if (onRedaction === undefined)
|