@mnemonik/shared 7.154.2 → 7.155.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/dist/ast/grammars.d.ts +4 -2
  2. package/dist/ast/grammars.d.ts.map +1 -1
  3. package/dist/ast/grammars.js +45 -3
  4. package/dist/ast/grammars.js.map +1 -1
  5. package/dist/codeScanner.d.ts +26 -329
  6. package/dist/codeScanner.d.ts.map +1 -1
  7. package/dist/codeScanner.js +65 -1141
  8. package/dist/codeScanner.js.map +1 -1
  9. package/dist/fileSelection.d.ts +451 -0
  10. package/dist/fileSelection.d.ts.map +1 -0
  11. package/dist/fileSelection.js +1256 -0
  12. package/dist/fileSelection.js.map +1 -0
  13. package/dist/hookTrace.d.ts +40 -0
  14. package/dist/hookTrace.d.ts.map +1 -1
  15. package/dist/hookTrace.js +126 -0
  16. package/dist/hookTrace.js.map +1 -1
  17. package/dist/index.d.ts +3 -2
  18. package/dist/index.d.ts.map +1 -1
  19. package/dist/index.js +3 -2
  20. package/dist/index.js.map +1 -1
  21. package/dist/runtimeAssets.d.ts +12 -0
  22. package/dist/runtimeAssets.d.ts.map +1 -0
  23. package/dist/runtimeAssets.js +43 -0
  24. package/dist/runtimeAssets.js.map +1 -0
  25. package/dist/runtimeReader.d.ts +15 -2
  26. package/dist/runtimeReader.d.ts.map +1 -1
  27. package/dist/runtimeReader.js +110 -21
  28. package/dist/runtimeReader.js.map +1 -1
  29. package/dist/sourceRedaction.d.ts +45 -1
  30. package/dist/sourceRedaction.d.ts.map +1 -1
  31. package/dist/sourceRedaction.js +62 -22
  32. package/dist/sourceRedaction.js.map +1 -1
  33. package/dist/sourceRedactionPool.d.ts +198 -0
  34. package/dist/sourceRedactionPool.d.ts.map +1 -0
  35. package/dist/sourceRedactionPool.js +756 -0
  36. package/dist/sourceRedactionPool.js.map +1 -0
  37. package/dist/vueSource.d.ts +0 -2
  38. package/dist/vueSource.d.ts.map +1 -1
  39. package/dist/vueSource.js +8 -19
  40. package/dist/vueSource.js.map +1 -1
  41. package/package.json +6 -2
  42. package/src/ast/grammars.ts +52 -7
  43. package/src/codeScanner.ts +121 -1321
  44. package/src/fileSelection.ts +1440 -0
  45. package/src/hookTrace.ts +143 -0
  46. package/src/index.ts +31 -2
  47. package/src/runtimeAssets.ts +49 -0
  48. package/src/runtimeReader.ts +120 -28
  49. package/src/sourceRedaction.ts +85 -20
  50. package/src/sourceRedactionPool.ts +877 -0
  51. package/src/vueSource.ts +8 -23
  52. package/tests/runtimeVerify.test.ts +166 -0
package/src/hookTrace.ts CHANGED
@@ -38,6 +38,7 @@ import {
38
38
  writeSync,
39
39
  } from 'node:fs';
40
40
  import { dirname, join } from 'node:path';
41
+ import { open as openAsync, stat as statAsync } from 'node:fs/promises';
41
42
  import { mnemonikStateDirectory } from './protectedPaths.js';
42
43
  import { ensureWindowsPrivateDirectorySync } from './runtimeSigners.js';
43
44
  import { scrubSecrets } from './secretPatterns.js';
@@ -586,3 +587,145 @@ export function startHookTrace(options: HookTraceOptions): HookTrace {
586
587
  trace.open();
587
588
  return trace;
588
589
  }
590
+
591
+ /** The trace window a hook reports its recent misses over. */
592
+ export const HOOK_MISS_WINDOW_MS = 30 * 60 * 1000;
593
+ /**
594
+ * How much of the trace file's end is read for that summary, asynchronously
595
+ * and once per hook process (it used to read 256 KB synchronously on every
596
+ * call). About a hundred hook runs: the recent ones are what the count is for.
597
+ */
598
+ const HOOK_MISS_TAIL_BYTES = 64 * 1024;
599
+ /** Sent on the main hook call: `<missed>/<total>` server calls over the window. */
600
+ export const HOOK_MISSES_HEADER = 'x-mnemonik-hook-misses';
601
+
602
+ /**
603
+ * How many of this host's recent calls to the server did not get an answer
604
+ * in time: a timeout, a connection error, a gateway or server error, a call
605
+ * still pending at exit, or a whole run the editor killed at its own hook
606
+ * timeout. Each of these is silent where it happens (the hook fails open, by
607
+ * design), so the main hook call carries the count to the server, and
608
+ * machine health tells the agent when a machine is losing memory this way
609
+ * (limits audit #2-#3: the hook budget chain fails silently).
610
+ *
611
+ * Reads only the end of the trace file, and never throws.
612
+ */
613
+ export async function summarizeRecentHookCalls(
614
+ host: HookTraceHost,
615
+ options: { stateDir?: string; now?: number; windowMs?: number } = {}
616
+ ): Promise<{ missed: number; total: number }> {
617
+ // With the recorder off there is no file, which reads as nothing to report.
618
+ const empty = { missed: 0, total: 0 };
619
+ const now = options.now ?? Date.now();
620
+ const since = now - (options.windowMs ?? HOOK_MISS_WINDOW_MS);
621
+ let text: string;
622
+ try {
623
+ const file = hookTraceFile(host, options.stateDir);
624
+ const size = (await statAsync(file)).size;
625
+ const handle = await openAsync(file, 'r');
626
+ try {
627
+ const length = Math.min(size, HOOK_MISS_TAIL_BYTES);
628
+ const buffer = Buffer.alloc(length);
629
+ await handle.read(buffer, 0, length, size - length);
630
+ text = buffer.toString('utf8');
631
+ } finally {
632
+ await handle.close();
633
+ }
634
+ } catch {
635
+ return empty;
636
+ }
637
+ const started = new Map<string, number>();
638
+ const ended = new Set<string>();
639
+ let missed = 0;
640
+ let total = 0;
641
+ for (const line of text.split('\n')) {
642
+ let record: HookTraceRecord;
643
+ try {
644
+ record = JSON.parse(line) as HookTraceRecord;
645
+ } catch {
646
+ continue; // the first line of the tail is usually cut
647
+ }
648
+ const at = Date.parse(record.ts);
649
+ if (!(at >= since)) continue;
650
+ if (record.phase === 'start') started.set(record.id, at);
651
+ else if (record.phase === 'end') {
652
+ ended.add(record.id);
653
+ // One logical call per path in a run: a hedged request's aborted
654
+ // loser, or a first attempt whose retry was answered, is not a miss.
655
+ // Only a call none of whose requests got an answer is.
656
+ const answeredByPath = new Map<string, boolean>();
657
+ for (const call of record.calls ?? []) {
658
+ const status = call.status ?? 0;
659
+ const answered =
660
+ call.error === undefined &&
661
+ call.pending !== true &&
662
+ status > 0 &&
663
+ status < 500 &&
664
+ status !== 408 &&
665
+ status !== 429;
666
+ answeredByPath.set(call.path, (answeredByPath.get(call.path) ?? false) || answered);
667
+ }
668
+ for (const answered of answeredByPath.values()) {
669
+ total++;
670
+ if (!answered) missed++;
671
+ }
672
+ }
673
+ }
674
+ // A run with a start and no end was ended by the editor at its timeout.
675
+ for (const [id, at] of started) {
676
+ if (ended.has(id) || now - at < 60_000) continue;
677
+ total++;
678
+ missed++;
679
+ }
680
+ return { missed, total };
681
+ }
682
+
683
+ const summaries = new Map<
684
+ string,
685
+ { read: Promise<Record<string, string>>; value?: Record<string, string> }
686
+ >();
687
+
688
+ function summaryFor(
689
+ host: HookTraceHost,
690
+ options: { stateDir?: string; now?: number }
691
+ ): { read: Promise<Record<string, string>>; value?: Record<string, string> } {
692
+ const key = `${host}\0${options.stateDir ?? ''}\0${options.now ?? ''}`;
693
+ const cached = summaries.get(key);
694
+ if (cached) return cached;
695
+ const entry: { read: Promise<Record<string, string>>; value?: Record<string, string> } = {
696
+ read: Promise.resolve({}),
697
+ };
698
+ entry.read = summarizeRecentHookCalls(host, options).then(
699
+ ({ missed, total }): Record<string, string> => {
700
+ const value: Record<string, string> =
701
+ total > 0 ? { [HOOK_MISSES_HEADER]: `${missed}/${total}` } : {};
702
+ entry.value = value;
703
+ return value;
704
+ }
705
+ );
706
+ summaries.set(key, entry);
707
+ return entry;
708
+ }
709
+
710
+ /**
711
+ * The header the main hook call carries, or none when nothing was called
712
+ * recently. Read once per hook process, without blocking it.
713
+ */
714
+ export function hookMissesHeaders(
715
+ host: HookTraceHost,
716
+ options: { stateDir?: string; now?: number } = {}
717
+ ): Promise<Record<string, string>> {
718
+ return summaryFor(host, options).read;
719
+ }
720
+
721
+ /**
722
+ * The header if the read has finished, else none, and never a wait: the
723
+ * count only feeds machine health, so it must never delay a hook request
724
+ * (the hook starts the read when it loads, so it is normally ready).
725
+ */
726
+ export function hookMissesHeadersIfReady(
727
+ host: HookTraceHost,
728
+ options: { stateDir?: string; now?: number } = {}
729
+ ): Record<string, string> {
730
+ return summaryFor(host, options).value ?? {};
731
+ }
package/src/index.ts CHANGED
@@ -20,7 +20,10 @@ export {
20
20
  isGitBoundary,
21
21
  isProjectBoundary,
22
22
  rootProjectId,
23
- SCANNER_MAX_DEPTH,
23
+ SCRUB_VERSION,
24
+ fileOp,
25
+ statTuple,
26
+ type FileStatTuple,
24
27
  isLinkedWorktreeCheckout,
25
28
  FIXTURE_PATH_RE,
26
29
  isFixturePath,
@@ -97,7 +100,33 @@ export {
97
100
  scrubSecrets,
98
101
  redactHighEntropyTokens,
99
102
  } from './secretPatterns.js';
100
- export { scrubSourceSecrets } from './sourceRedaction.js';
103
+ export {
104
+ scrubSourceSecrets,
105
+ scrubSourceSecretsInProcess,
106
+ installSourceRedactor,
107
+ MAX_SOURCE_REDACTION_BYTES,
108
+ type SourceRedactor,
109
+ type SourceRedactionOptions,
110
+ } from './sourceRedaction.js';
111
+ export {
112
+ SourceRedactionPool,
113
+ redactionWorkerThread,
114
+ STRIKE_REASONS,
115
+ type RedactionChild,
116
+ runSourceRedactionWorker,
117
+ redactionHeapMb,
118
+ cgroupMemory,
119
+ containerMemoryProbe,
120
+ scannerMemoryProbe,
121
+ processCpuSeconds,
122
+ CPU_CEILING_SECONDS,
123
+ SCANNER_TOTAL_MEMORY_MB,
124
+ type MemoryProbe,
125
+ type MemoryReading,
126
+ type RedactionPriority,
127
+ LARGE_FILE_BYTES as LARGE_SOURCE_REDACTION_BYTES,
128
+ type SourceRedactionPoolOptions,
129
+ } from './sourceRedactionPool.js';
101
130
  export {
102
131
  redactCredentials,
103
132
  credentialValues,
@@ -0,0 +1,49 @@
1
+ /**
2
+ * Never use bytes from the packaged scanner's extracted runtime unchecked
3
+ * (review of 26 Oct 2026). JavaScript is checked as it loads
4
+ * (packages/scanner/src/verifiedLoader.cts); this covers what is read with fs
5
+ * and then run or parsed as code: the grammar WebAssembly, web-tree-sitter's
6
+ * own runtime WebAssembly and the tag queries. The caller passes the buffer it
7
+ * is about to use, so the bytes checked are the bytes used.
8
+ *
9
+ * The packaged executable's entry point publishes the embedded assets' hashes
10
+ * on globalThis (and runtimeWorker.ts in each worker). Anywhere else, in
11
+ * development, tests and on the server, there is nothing to check against and
12
+ * this does nothing.
13
+ */
14
+ import { createHash } from 'node:crypto';
15
+ import { relative, sep } from 'node:path';
16
+
17
+ interface VerifiedRuntime {
18
+ root: string;
19
+ hashes: ReadonlyMap<string, string>;
20
+ }
21
+
22
+ const RUNTIME = Symbol.for('mnemonik.sea.verifiedLoader');
23
+
24
+ /** Bytes that are not the embedded asset they must be. */
25
+ export class RuntimeAssetError extends Error {}
26
+
27
+ function runtime(): VerifiedRuntime | undefined {
28
+ return (globalThis as { [RUNTIME]?: VerifiedRuntime })[RUNTIME];
29
+ }
30
+
31
+ /** Whether this process runs from the packaged executable's verified runtime. */
32
+ export function inVerifiedRuntime(): boolean {
33
+ return runtime() !== undefined;
34
+ }
35
+
36
+ /**
37
+ * Check bytes read from `path` against the embedded asset they must equal.
38
+ * Throws `mnemonik_asset_integrity` when they differ and
39
+ * `mnemonik_unverified_asset` for a path that is not an embedded asset.
40
+ */
41
+ export function verifyRuntimeAsset(path: string, bytes: Uint8Array): void {
42
+ const verified = runtime();
43
+ if (!verified) return;
44
+ const key = relative(verified.root, path).split(sep).join('/');
45
+ const expected = key.startsWith('../') || key === '..' ? undefined : verified.hashes.get(key);
46
+ if (!expected) throw new RuntimeAssetError(`mnemonik_unverified_asset: ${path}`);
47
+ if (createHash('sha256').update(bytes).digest('hex') !== expected)
48
+ throw new RuntimeAssetError(`mnemonik_asset_integrity: ${key}`);
49
+ }
@@ -90,6 +90,63 @@ export const versionName = (v: string): string => {
90
90
  return v;
91
91
  };
92
92
 
93
+ /**
94
+ * Filesystem operations one runtime verification keeps in flight. Each file
95
+ * costs a few round trips (lstat, open, read, close); awaited one at a time, a
96
+ * 2,000-file hook runtime took 6.6 s to verify (26 Oct 2026). Every check is
97
+ * still made, for every file.
98
+ */
99
+ export const VERIFY_CONCURRENCY = 16;
100
+
101
+ /** Runs `work` with at most `limit` calls in flight; nested calls must not share a limiter. */
102
+ function limiter(limit: number) {
103
+ let active = 0;
104
+ const waiting: Array<() => void> = [];
105
+ return async <T>(work: () => Promise<T>): Promise<T> => {
106
+ if (active >= limit) await new Promise<void>((release) => waiting.push(release));
107
+ active += 1;
108
+ try {
109
+ return await work();
110
+ } finally {
111
+ active -= 1;
112
+ waiting.shift()?.();
113
+ }
114
+ };
115
+ }
116
+
117
+ /**
118
+ * A runtime file's bytes, read from the file that was opened: it must be a
119
+ * regular file (not followed through a link), this user's, with exactly the
120
+ * mode its manifest entry implies. POSIX only.
121
+ */
122
+ async function ownedFileBytes(path: string, mode: number): Promise<Buffer> {
123
+ const fd = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW).catch((error: unknown) => {
124
+ // O_NOFOLLOW refuses a link with ELOOP: a link is not an owned file.
125
+ throw (error as NodeJS.ErrnoException).code === 'ELOOP'
126
+ ? new RuntimeError('permission')
127
+ : error;
128
+ });
129
+ try {
130
+ const stat = await fd.stat();
131
+ if (!stat.isFile() || stat.uid !== process.getuid?.() || (stat.mode & 0o777) !== mode)
132
+ throw new RuntimeError('permission');
133
+ return await fd.readFile();
134
+ } finally {
135
+ await fd.close();
136
+ }
137
+ }
138
+
139
+ /**
140
+ * Waits for every promise, then throws the first rejection in order: nothing a
141
+ * failed verification started is still running when it reports.
142
+ */
143
+ async function settleAll(work: Promise<unknown>[]): Promise<void> {
144
+ const failed = (await Promise.allSettled(work)).find(
145
+ (result): result is PromiseRejectedResult => result.status === 'rejected'
146
+ );
147
+ if (failed) throw failed.reason;
148
+ }
149
+
93
150
  const statIdentity = (stat: Stats): string =>
94
151
  [stat.dev, stat.ino, stat.mode, stat.uid, stat.size, stat.mtimeMs, stat.ctimeMs].join(':');
95
152
 
@@ -237,17 +294,30 @@ export class RuntimeReader {
237
294
  );
238
295
  }
239
296
  }
297
+ /**
298
+ * `proven` (one verification's own set, POSIX only) holds directories this
299
+ * verification has already inspected in full: their components are not
300
+ * inspected again for every file beneath them, and directories proven here
301
+ * are added to it.
302
+ */
240
303
  async inspect(
241
304
  path: string,
242
305
  directory = false,
243
306
  allowMissing = false,
244
- replacing = false
307
+ replacing = false,
308
+ proven?: Set<string>
245
309
  ): Promise<void> {
246
310
  const absolute = this.guardedPath(path);
247
311
  const windowsPaths: string[] = [];
248
- let part = parse(absolute).root;
249
- for (const name of absolute.slice(part.length).split(sep)) {
312
+ const remembered = process.platform === 'win32' ? undefined : proven;
313
+ const checkedDirectories: string[] = [];
314
+ // Directories proven by this verification are skipped; when the parent is
315
+ // proven, so is every directory above it.
316
+ const parent = dirname(absolute);
317
+ let part = remembered?.has(parent) ? parent : parse(absolute).root;
318
+ for (const name of absolute.slice(part.length).split(sep).filter(Boolean)) {
250
319
  part = join(part, name);
320
+ if (part !== absolute && remembered?.has(part)) continue;
251
321
  const stat = await lstat(part).catch((e) => {
252
322
  if (allowMissing && missing(e)) return null;
253
323
  throw e;
@@ -295,7 +365,9 @@ export class RuntimeReader {
295
365
  windowsPaths.push(part);
296
366
  } else if (stat.uid !== process.getuid?.() || (stat.mode & 0o077) !== 0)
297
367
  throw new RuntimeError('permission');
368
+ if (stat.isDirectory()) checkedDirectories.push(part);
298
369
  }
370
+ if (remembered) for (const checked of checkedDirectories) remembered.add(checked);
299
371
  await this.audit(windowsPaths);
300
372
  for (const part of windowsPaths) {
301
373
  const verdict = this.windowsAudit.get(part)?.verdict;
@@ -321,8 +393,8 @@ export class RuntimeReader {
321
393
  this.windowsAudit.delete(absolute);
322
394
  this.windowsStats.delete(absolute);
323
395
  }
324
- async bytes(path: string): Promise<Buffer> {
325
- await this.inspect(path);
396
+ async bytes(path: string, proven?: Set<string>): Promise<Buffer> {
397
+ await this.inspect(path, false, false, false, proven);
326
398
  const fd = await open(
327
399
  path,
328
400
  constants.O_RDONLY | (process.platform === 'win32' ? 0 : constants.O_NOFOLLOW)
@@ -417,25 +489,41 @@ export class RuntimeReader {
417
489
  throw new RuntimeError('unsigned');
418
490
  if (process.platform === 'win32')
419
491
  await this.audit(Object.keys(m.files).map((name) => join(directory, safePath(name))));
492
+ // Every directory and file is inspected (type, owner, private mode) and
493
+ // every file is read and hashed, as before; up to VERIFY_CONCURRENCY at a
494
+ // time, and each directory's own components only once.
420
495
  const actual: string[] = [];
496
+ const proven = new Set<string>();
497
+ // Windows keeps one at a time: its ACL audit shares caches across calls.
498
+ const limit = limiter(process.platform === 'win32' ? 1 : VERIFY_CONCURRENCY);
421
499
  const walk = async (dir: string, prefix = ''): Promise<void> => {
422
- await this.inspect(dir, true);
423
- for (const name of await readdir(dir)) {
424
- const key = prefix + name;
425
- if (!prefix && name === 'manifest.json') continue;
426
- const path = join(dir, name);
427
- if ((await lstat(path)).isDirectory()) await walk(path, key + '/');
428
- else {
429
- await this.inspect(path);
430
- actual.push(key);
431
- }
432
- }
500
+ const entries = await limit(async () => {
501
+ await this.inspect(dir, true, false, false, proven);
502
+ return readdir(dir, { withFileTypes: true });
503
+ });
504
+ await settleAll(
505
+ entries.map(async (entry) => {
506
+ const key = prefix + entry.name;
507
+ if (!prefix && entry.name === 'manifest.json') return;
508
+ const path = join(dir, entry.name);
509
+ if (entry.isDirectory()) await walk(path, key + '/');
510
+ else {
511
+ // POSIX: a link or anything else that is not a regular file is
512
+ // refused here, and every listed file's type, owner and exact mode
513
+ // are checked on the file itself when it is read below. Windows
514
+ // inspects each file for its ACL audit.
515
+ if (process.platform === 'win32')
516
+ await limit(() => this.inspect(path, false, false, false, proven));
517
+ else if (!entry.isFile()) throw new RuntimeError('permission');
518
+ actual.push(key);
519
+ }
520
+ })
521
+ );
433
522
  };
434
523
  await walk(directory);
435
524
  if (actual.sort().join('\n') !== Object.keys(m.files).sort().join('\n'))
436
525
  throw new RuntimeError('digest_mismatch');
437
- let total = 0;
438
- for (const [name, expected] of Object.entries(m.files)) {
526
+ for (const expected of Object.values(m.files))
439
527
  if (
440
528
  !expected ||
441
529
  !/^[a-f0-9]{64}$/.test(expected.sha256) ||
@@ -444,17 +532,21 @@ export class RuntimeReader {
444
532
  typeof expected.executable !== 'boolean'
445
533
  )
446
534
  throw new RuntimeError('digest_mismatch');
447
- const path = join(directory, safePath(name));
448
- const contents = await this.bytes(path);
449
- if (hash(contents) !== expected.sha256 || contents.length !== expected.size)
450
- throw new RuntimeError('digest_mismatch');
451
- if (
452
- process.platform !== 'win32' &&
453
- ((await lstat(path)).mode & 0o777) !== (expected.executable ? 0o700 : 0o600)
535
+ let total = 0;
536
+ await settleAll(
537
+ Object.entries(m.files).map(([name, expected]) =>
538
+ limit(async () => {
539
+ const path = join(directory, safePath(name));
540
+ const contents =
541
+ process.platform === 'win32'
542
+ ? await this.bytes(path, proven)
543
+ : await ownedFileBytes(path, expected.executable ? 0o700 : 0o600);
544
+ if (hash(contents) !== expected.sha256 || contents.length !== expected.size)
545
+ throw new RuntimeError('digest_mismatch');
546
+ total += contents.length;
547
+ })
454
548
  )
455
- throw new RuntimeError('permission');
456
- total += contents.length;
457
- }
549
+ );
458
550
  if (total !== m.totalSize) throw new RuntimeError('digest_mismatch');
459
551
  if (m.signer) {
460
552
  const signer =
@@ -2,7 +2,6 @@ import { parseVueSource, walkVueTemplate, VUE_FIDELITY_MARKER } from './vueSourc
2
2
  import { basename, extname } from 'node:path';
3
3
  import { Parser, type Node, type Tree } from 'web-tree-sitter';
4
4
  import { loadGrammar, objcppSyntaxIssue, type AstLanguageId } from './ast/grammars.js';
5
- import { MAX_AST_PARSE_BYTES } from './ast/astChunker.js';
6
5
  import { commandLineCredentialRanges, redactCredentials } from './redactCredentials.js';
7
6
  import {
8
7
  redactHighEntropyTokens,
@@ -15,8 +14,52 @@ function scrubPayload(text: string): string {
15
14
  return redactCredentials(scrubSecrets(text), { commandLineOnly: true }).text;
16
15
  }
17
16
 
17
+ /**
18
+ * The largest source redacted with its syntax tree: the scanner's own file
19
+ * ceiling (MAX_SCANNED_FILE_BYTES, 10 MB; repeated here because codeScanner.ts
20
+ * imports this module). It was 2 MiB, sized from "the largest file in this
21
+ * repository", and above it every file took the text fallback: PowerShell,
22
+ * CMake, Groovy, Nix, Terraform and the bundled languages were stamped
23
+ * "source-fidelity-unavailable", so connected code refused every such file
24
+ * over 2 MiB, and other languages could keep structural [REDACTED] the
25
+ * extractors refuse (streams F1/F2, 25 October 2026). Memory for a large
26
+ * parse is bounded by running it in a child process sized from the file
27
+ * (installSourceRedactor below; sourceRedactionPool.ts), not by refusing it.
28
+ */
29
+ export const MAX_SOURCE_REDACTION_BYTES = 10 * 1024 * 1024;
30
+
31
+ /**
32
+ * Where a process sends source it should not parse in its own memory: the
33
+ * server sends Swift and every file of a megabyte or more to a Liftoff-only
34
+ * child sized from the file; the scanner sends large files to its own
35
+ * child. Unset (tests, small tools), everything is redacted in process.
36
+ */
37
+ export interface SourceRedactionOptions {
38
+ /** 'interactive' when an agent is waiting on this file (file_push). */
39
+ priority?: 'interactive' | 'background';
40
+ }
41
+ export interface SourceRedactor {
42
+ routes(filePath: string, bytes: number): boolean;
43
+ redact(text: string, filePath: string, options?: SourceRedactionOptions): Promise<string>;
44
+ }
45
+ let outOfProcess: SourceRedactor | null = null;
46
+ export function installSourceRedactor(redactor: SourceRedactor | null): void {
47
+ outOfProcess = redactor;
48
+ }
49
+
18
50
  /** Source syntax stays executable; unsupported or incomplete input uses the conservative scrubber. */
19
- export async function scrubSourceSecrets(text: string, filePath: string): Promise<string> {
51
+ export async function scrubSourceSecrets(
52
+ text: string,
53
+ filePath: string,
54
+ options?: SourceRedactionOptions
55
+ ): Promise<string> {
56
+ if (outOfProcess && text && outOfProcess.routes(filePath, Buffer.byteLength(text, 'utf8')))
57
+ return outOfProcess.redact(text, filePath, options);
58
+ return scrubSourceSecretsInProcess(text, filePath);
59
+ }
60
+
61
+ /** The redaction itself, in this process. The out-of-process worker calls this. */
62
+ export async function scrubSourceSecretsInProcess(text: string, filePath: string): Promise<string> {
20
63
  const extension = extname(filePath).toLowerCase();
21
64
  if (extension === '.vue') return scrubVueSource(text);
22
65
  if (extension === '.svelte') return scrubSvelteSource(text);
@@ -136,7 +179,7 @@ export async function scrubSourceSecrets(text: string, filePath: string): Promis
136
179
  ? marker + clean
137
180
  : clean;
138
181
  };
139
- if (!text || !language || Buffer.byteLength(text, 'utf8') > MAX_AST_PARSE_BYTES) {
182
+ if (!text || !language || Buffer.byteLength(text, 'utf8') > MAX_SOURCE_REDACTION_BYTES) {
140
183
  return fallback();
141
184
  }
142
185
  let parser: Parser | undefined;
@@ -1833,6 +1876,32 @@ function redactNode(node: Node, sensitive = false): string {
1833
1876
  }
1834
1877
 
1835
1878
  /** Sanitize parser-owned payload spans; no SFC boundary is inferred from text matching. */
1879
+ /**
1880
+ * Apply replacement spans to `text` in one pass, or null when two overlap
1881
+ * (or a span is malformed).
1882
+ * The result equals applying them one at a time from the last start to the
1883
+ * first (spans that start together: the one added later lands first), which
1884
+ * is what the Vue scrubber did with a slice-and-concatenate per edit; that
1885
+ * copied the whole file for every edit, about 40 s for a 1 MB component.
1886
+ */
1887
+ export function applySpanEdits(
1888
+ text: string,
1889
+ edits: ReadonlyArray<{ start: number; end: number; clean: string }>
1890
+ ): string | null {
1891
+ // Stable sort, then reversed: ascending start, and among equal starts the
1892
+ // later-added edit first, exactly as the one-at-a-time order placed them.
1893
+ const ordered = [...edits].sort((a, b) => b.start - a.start).reverse();
1894
+ const pieces: string[] = [];
1895
+ let offset = 0;
1896
+ for (const e of ordered) {
1897
+ if (e.start < offset || e.end < e.start || e.end > text.length) return null;
1898
+ pieces.push(text.slice(offset, e.start), e.clean);
1899
+ offset = e.end;
1900
+ }
1901
+ pieces.push(text.slice(offset));
1902
+ return pieces.join('');
1903
+ }
1904
+
1836
1905
  async function scrubVueSource(text: string): Promise<string> {
1837
1906
  if (!text) return text;
1838
1907
  const parsed = await parseVueSource(text, true);
@@ -1973,14 +2042,8 @@ async function scrubVueSource(text: string): Promise<string> {
1973
2042
  const cleanTail = scrubPayload(tail);
1974
2043
  if (cleanTail !== tail) faithful = false;
1975
2044
  edit(gapStart, text.length, cleanTail);
1976
- edits.sort((a, b) => b.start - a.start);
1977
- let clean = text;
1978
- let boundary = text.length;
1979
- for (const e of edits) {
1980
- if (e.end > boundary) return fallback();
1981
- clean = clean.slice(0, e.start) + e.clean + clean.slice(e.end);
1982
- boundary = e.start;
1983
- }
2045
+ const clean = applySpanEdits(text, edits);
2046
+ if (clean === null) return fallback();
1984
2047
  if ('reason' in (await parseVueSource(clean))) faithful = false;
1985
2048
  return faithful || clean.startsWith(VUE_FIDELITY_MARKER) ? clean : VUE_FIDELITY_MARKER + clean;
1986
2049
  } catch {
@@ -1996,29 +2059,33 @@ async function scrubSvelteSource(text: string): Promise<string> {
1996
2059
  const fallback = (): string => (text.startsWith(marker) ? '' : marker) + scrubPayload(text);
1997
2060
  if (!text) return text;
1998
2061
  if (text.includes('mnemonik:svelte-source-fidelity-unavailable')) return fallback();
1999
- if (Buffer.byteLength(text, 'utf8') > MAX_AST_PARSE_BYTES) return fallback();
2062
+ if (Buffer.byteLength(text, 'utf8') > MAX_SOURCE_REDACTION_BYTES) return fallback();
2000
2063
  let parser: Parser | undefined;
2001
2064
  try {
2002
- const deadline = performance.now() + 4_000;
2065
+ // No clock and no node count decide this file: both refused large
2066
+ // components outright (a 4 s deadline and 200,000 nodes). Memory is
2067
+ // bounded by the process the parse runs in (installSourceRedactor).
2003
2068
  const { parse } = await import('svelte/compiler');
2004
2069
  const ast = parse(text, { modern: true });
2005
2070
  const edits: Array<{ start: number; end: number; clean: string }> = [];
2006
2071
  let faithful = true;
2007
- parser = new Parser();
2072
+ // Grammars first: loading one initializes tree-sitter, and a Parser
2073
+ // constructed before that throws. In a fresh process (a redaction child,
2074
+ // or a scanner whose first file is a component) that made every Svelte
2075
+ // file take the unfaithful fallback.
2008
2076
  const js = await loadGrammar('javascript');
2009
2077
  const ts = await loadGrammar('typescript');
2010
2078
  if ('unavailable' in js || 'unavailable' in ts) return fallback();
2079
+ parser = new Parser();
2011
2080
  parser.setLanguage(js.ok.language);
2012
2081
  const activeParser = parser;
2013
2082
  const syntax = (source: string, expression = false, sensitive = false): string => {
2014
2083
  const prefix = expression ? `const ${sensitive ? 'password' : '__mnemonik_svelte'} = (` : '';
2015
2084
  const suffix = expression ? '\n);' : '';
2016
2085
  const full = prefix + source + suffix;
2017
- const tree = activeParser.parse(full, null, {
2018
- progressCallback: () => performance.now() > deadline,
2019
- });
2086
+ const tree = activeParser.parse(full);
2020
2087
  try {
2021
- if (!tree || tree.rootNode.hasError || tree.rootNode.descendantCount > 200_000) {
2088
+ if (!tree || tree.rootNode.hasError) {
2022
2089
  faithful = false;
2023
2090
  return sensitive ? literalPayload(source, true) : scrubPayload(source);
2024
2091
  }
@@ -2062,9 +2129,7 @@ async function scrubSvelteSource(text: string): Promise<string> {
2062
2129
  const pending: Array<{ value: unknown; sensitive: boolean }> = [
2063
2130
  { value: ast.fragment, sensitive: false },
2064
2131
  ];
2065
- let count = 0;
2066
2132
  while (pending.length) {
2067
- if (++count > 200_000 || performance.now() > deadline) return fallback();
2068
2133
  const entry = pending.pop();
2069
2134
  if (!entry || !entry.value || typeof entry.value !== 'object') continue;
2070
2135
  if (Array.isArray(entry.value)) {