@mnemonik/shared 7.154.3 → 7.155.2

package/src/sourceRedactionPool.ts

@@ -0,0 +1,877 @@
+/**
+ * Source redaction in its own processes, without a clock deciding a file's
+ * fate. Used by the server (Swift, and every file of a megabyte or more) and
+ * by the scanner (every file of a megabyte or more), each supplying how to
+ * start its child (`spawn`).
+ *
+ * Why a separate process at all: Swift's tree-sitter grammar, compiled by
+ * TurboFan, allocates 0.5 to 0.7 GB and can take Node down with it, so Swift
+ * is parsed in Liftoff-only children (measured 18 October 2026). What this
+ * replaces forked one child per upload batch, capped its heap at 128 MB and
+ * killed it after 5 seconds (`source_redaction_timeout`). A large Swift file,
+ * or any Swift file on a busy task, then failed redaction and its whole
+ * upload batch failed with it, every retry, with no line saying why: the same
+ * shape as MNEMONIK-CV, where a wall clock and a fixed heap decided a file's
+ * fate (stream F2 fixed that one in its parse pool, whose design this
+ * mirrors: src/agent/codeConnections/parsePool.ts on fix/scale-f2-code).
+ *
+ *  - A persistent child keeps the grammar loaded between files; uploads queue
+ *    for it in arrival order.
+ *  - Its heap is sized from the input: a file of LARGE_FILE_BYTES or more gets
+ *    its own child with 128 MB plus 16 MB per MB of source (110 MB per MB for
+ *    Vue and Svelte components, whose compilers build JavaScript syntax trees),
+ *    and a file that still runs out of memory is tried once more with twice
+ *    that.
+ *  - Memory admission: up to MAX_LARGE_CHILDREN large children run at once,
+ *    each only when the memory it needs fits what the probe reports (the ECS
+ *    task's limit less the container's memory.current on the server; the
+ *    cgroup working set, or the scanner's own total budget less its resident
+ *    memory, on a machine), less a reserve, less what running children may
+ *    still grow into and what admitted-but-waiting work holds. A child that
+ *    would not fit even alone is never started, since running the container
+ *    out of memory ends every request it is serving; the file fails counted
+ *    (`source_redaction_exceeds_memory`).
+ *  - No deadline, but a ceiling on work: a file that has used
+ *    CPU_CEILING_SECONDS of the child's CPU (about ten times the slowest file
+ *    measured) fails (`source_redaction_cpu_ceiling`) and its child is
+ *    replaced, so no file holds a child indefinitely. A child that holds a file
+ *    and has used no CPU for STALL_POLLS polls in a row is presumed hung.
+ *    CPU time comes from /proc on Linux and `ps` elsewhere; where neither
+ *    exists (Windows), only an exit ends a file.
+ *  - Interactive files (an agent's file_push) are served before background
+ *    ingest, in both queues.
+ *  - A failure is counted (`source_redaction.failed`), logged with the path's
+ *    extension and size, and thrown for that file alone: callers store the
+ *    other files and report the failed one as not uploaded, with its reason.
+ *    Redacted text is never replaced by unredacted text.
+ */
+import { execFile } from 'node:child_process';
+import { EventEmitter } from 'node:events';
+import { parentPort, type Worker } from 'node:worker_threads';
+import { readFileSync } from 'node:fs';
+import { scrubSourceSecretsInProcess } from './sourceRedaction.js';
+
+const MB = 1024 * 1024;
+/** At and above this length a file gets a child of its own, sized from it. */
+export const LARGE_FILE_BYTES = 1 * MB;
+/** The shared child's heap: comfortably above any file under LARGE_FILE_BYTES. */
+export const SHARED_HEAP_MB = 256;
+/** A shared child is replaced once its resident memory passes this (WebAssembly memory never shrinks). */
+const SHARED_RECYCLE_RSS_MB = 512;
+export const STALL_POLL_MS = 5_000;
+/** One minute without any CPU while holding a file. */
+export const STALL_POLLS = 12;
+/**
+ * CPU a single file may use. Generous: the slowest files measured (26 October
+ * 2026, loaded host) were a 3.1 MB Svelte component at 23 s and a 9.9 MB Vue
+ * component at 17 s.
+ */
+export const CPU_CEILING_SECONDS = 300;
+/**
+ * CPU a single file may use in the shared small-file child before it is sent
+ * to a child of its own: one tenant's crafted file must not stall small-file
+ * redaction for everyone on the task.
+ */
+export const SHARED_CPU_CEILING_SECONDS = 30;
+/**
+ * Failures that say something about the file itself, which a scanner counts
+ * as strikes against that revision: it ran to the CPU ceiling, ran out of
+ * memory at twice its heap, or could not be redacted. Not counted: a child
+ * that would not fit in memory now, or one that stalled; those are retried.
+ */
+export const STRIKE_REASONS: ReadonlySet<string> = new Set([
+  'source_redaction_cpu_ceiling',
+  'source_redaction_out_of_memory',
+  'source_redaction_invalid_result',
+]);
+/** Large children a pool runs at once, memory permitting. */
+export const MAX_LARGE_CHILDREN = 2;
+/**
+ * A scanner's total memory, itself and its redaction children: the systemd
+ * unit's MemoryMax, and the budget used where no control group limits it.
+ */
+export const SCANNER_TOTAL_MEMORY_MB = 1536;
+
+/**
+ * The heap a file's child may use: 128 MB plus 16 MB per MB of source (as F2's
+ * parse pool), or 110 MB per MB for Vue and Svelte components. Measured 26
+ * October 2026 (Liftoff-only): a 9.9 MB Vue component of about 100,000
+ * template nodes ran out of memory at 900 MB and completed at 1,228 MB (peak
+ * resident 1.13 GB); a 3.1 MB Svelte component completed at 500 MB.
+ */
+export function redactionHeapMb(bytes: number, filePath = ''): number {
+  const perMb = COMPONENT_SOURCE.test(filePath) ? 110 : 16;
+  return Math.ceil(128 + perMb * (bytes / MB));
+}
+const COMPONENT_SOURCE = /\.(?:vue|svelte)$/i;
+/** Resident memory a child uses beyond its heap (code, grammars, buffers). */
+export const CHILD_OVERHEAD_MB = 128;
+/** Memory left free for everything else when a large child is admitted. */
+export const ADMISSION_RESERVE_MB = 256;
+
+/** What a memory probe reports, in MB. */
+export interface MemoryReading {
+  limitMb: number;
+  usedMb: number;
+  /** Whether usedMb already includes this pool's running children. */
+  includesChildren: boolean;
+  source: string;
+}
+export type MemoryProbe = () => MemoryReading | Promise<MemoryReading>;
+
+export interface CgroupMemory {
+  /** null when the group has no limit. */
+  limitMb: number | null;
+  /** memory.current: everything charged, page cache included. */
+  currentMb: number;
+  /** memory.current less inactive file cache: what the kernel cannot easily reclaim. */
+  workingSetMb: number;
+}
+
+type ReadText = (path: string) => string;
+const readText: ReadText = (path) => readFileSync(path, 'utf8');
+
+function numberIn(read: ReadText, path: string): number | null {
+  try {
+    const text = read(path).trim();
+    if (text === 'max') return null;
+    const value = Number(text);
+    return Number.isFinite(value) ? value : null;
+  } catch {
+    return null;
+  }
+}
+
+function statField(read: ReadText, path: string, names: string[]): number {
+  try {
+    const lines = read(path).split('\n');
+    for (const name of names) {
+      const line = lines.find((entry) => entry.startsWith(`${name} `));
+      if (line) return Number(line.slice(name.length + 1)) || 0;
+    }
+  } catch {
+    // No stat file: nothing reclaimable is subtracted.
+  }
+  return 0;
+}
+
+/** This process's control group memory (v2, else v1), or null where there is none. */
+export function cgroupMemory(read: ReadText = readText): CgroupMemory | null {
+  try {
+    const line = read('/proc/self/cgroup')
+      .split('\n')
+      .find((entry) => entry.startsWith('0::'));
+    if (line) {
+      const path = line.slice(3).trim();
+      const dir = `/sys/fs/cgroup${path === '/' ? '' : path}`;
+      const current = numberIn(read, `${dir}/memory.current`);
+      if (current !== null) {
+        const limit = numberIn(read, `${dir}/memory.max`);
+        const inactive = statField(read, `${dir}/memory.stat`, ['inactive_file']);
+        return {
+          limitMb: limit === null ? null : limit / MB,
+          currentMb: current / MB,
+          workingSetMb: Math.max(0, current - inactive) / MB,
+        };
+      }
+    }
+  } catch {
+    // Not Linux, or no v2 hierarchy: try v1.
+  }
+  const usage = numberIn(read, '/sys/fs/cgroup/memory/memory.usage_in_bytes');
+  if (usage === null) return null;
+  const limit = numberIn(read, '/sys/fs/cgroup/memory/memory.limit_in_bytes');
+  const inactive = statField(read, '/sys/fs/cgroup/memory/memory.stat', [
+    'total_inactive_file',
+    'inactive_file',
+  ]);
+  return {
+    limitMb: limit === null || limit >= 2 ** 60 ? null : limit / MB,
+    currentMb: usage / MB,
+    workingSetMb: Math.max(0, usage - inactive) / MB,
+  };
+}
+
+/**
+ * The API server's probe: the task's memory limit (ECS task metadata, else the
+ * cgroup, else the host: the caller supplies it) less the container's working
+ * set (memory.current less inactive file cache, which the kernel reclaims
+ * first). Without a control group, this process's resident memory.
+ */
+export function containerMemoryProbe(
+  limitMb: () => number | Promise<number>,
+  read: ReadText = readText
+): MemoryProbe {
+  return async () => {
+    const group = cgroupMemory(read);
+    return {
+      limitMb: await limitMb(),
+      usedMb: group ? group.workingSetMb : process.memoryUsage().rss / MB,
+      includesChildren: group !== null,
+      source: group ? 'container_working_set' : 'process_resident',
+    };
+  };
+}
+
+/**
+ * A scanner's probe. Under a control group limit (Linux, the systemd unit):
+ * the limit less the working set, since the scanner's file reads leave page
+ * cache the kernel reclaims before it would kill anything. Elsewhere (macOS,
+ * Windows, an unlimited group): the scanner's own total budget less its own
+ * resident memory, never the machine's free memory, which on macOS counts
+ * compressed and cached pages as used and would refuse almost everything.
+ */
+export function scannerMemoryProbe(
+  totalBudgetMb = SCANNER_TOTAL_MEMORY_MB,
+  read: ReadText = readText,
+  /** Children are worker threads, so this process's own resident memory includes them. */
+  childrenInProcess = false
+): MemoryProbe {
+  return () => {
+    const group = cgroupMemory(read);
+    if (group && group.limitMb !== null)
+      return {
+        limitMb: group.limitMb,
+        usedMb: group.workingSetMb,
+        includesChildren: true,
+        source: 'cgroup_working_set',
+      };
+    return {
+      limitMb: totalBudgetMb,
+      usedMb: process.memoryUsage().rss / MB,
+      includesChildren: childrenInProcess,
+      source: 'own_budget',
+    };
+  };
+}
+
+export type RedactionPriority = 'interactive' | 'background';
+
+export interface RedactionInput {
+  text: string;
+  filePath: string;
+  /** An agent waiting on this file (file_push) goes before background ingest. */
+  priority?: RedactionPriority;
+}
+
+type Reply = { id: number; value: string } | { id: number; error: string; rssMb?: number };
+
+const CLOCK_TICKS_PER_SECOND = 100;
+
+function parsePsTime(text: string): number | null {
+  // [[DD-]HH:]MM:SS[.ss]
+  const trimmed = text.trim();
+  if (!trimmed) return null;
+  const [dayPart, rest] = trimmed.includes('-') ? trimmed.split('-', 2) : ['0', trimmed];
+  const parts = (rest ?? '').split(':').map(Number);
+  if (parts.some((part) => !Number.isFinite(part))) return null;
+  let seconds = 0;
+  for (const part of parts) seconds = seconds * 60 + part;
+  return seconds + Number(dayPart) * 86_400;
+}
+
+/** CPU seconds (user + system) a process has used: /proc on Linux, `ps` elsewhere, null on Windows. */
+export async function processCpuSeconds(pid: number | undefined): Promise<number | null> {
+  if (!pid) return null;
+  if (process.platform === 'linux') {
+    try {
+      const fields = readFileSync(`/proc/${pid}/stat`, 'utf8').split(') ')[1]?.split(' ') ?? [];
+      const ticks = Number(fields[11]) + Number(fields[12]);
+      return Number.isFinite(ticks) ? ticks / CLOCK_TICKS_PER_SECOND : null;
+    } catch {
+      return null;
+    }
+  }
+  if (process.platform === 'win32') return null;
+  return new Promise((resolve) => {
+    execFile('ps', ['-o', 'time=', '-p', String(pid)], { timeout: 4_000 }, (error, stdout) =>
+      resolve(error ? null : parsePsTime(String(stdout)))
+    );
+  });
+}
+export { parsePsTime as parsePsTimeForTest };
+
+function residentMb(pid: number | undefined): number | null {
+  if (!pid || process.platform !== 'linux') return null;
+  try {
+    const match = /VmRSS:\s+(\d+) kB/.exec(readFileSync(`/proc/${pid}/status`, 'utf8'));
+    return match ? Number(match[1]) / 1024 : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * What the pool needs from a redaction child: a forked process (the server),
+ * or a worker thread (the scanner: redactionWorkerThread).
+ */
+export interface RedactionChild {
+  readonly pid?: number;
+  readonly channel?: { ref?(): void; unref?(): void } | null;
+  send(message: unknown, callback?: (error: Error | null) => void): boolean;
+  // Listener shapes differ per event (as on ChildProcess and EventEmitter).
+  /* eslint-disable @typescript-eslint/no-explicit-any */
+  on(event: string, listener: (...args: any[]) => void): unknown;
+  once(event: string, listener: (...args: any[]) => void): unknown;
+  /* eslint-enable @typescript-eslint/no-explicit-any */
+  kill(signal?: NodeJS.Signals): boolean;
+  ref?(): void;
+  unref?(): void;
+  /** CPU seconds this child has used, where it can say so itself (a worker thread, on any platform). */
+  cpuSeconds?(): Promise<number | null>;
+  /**
+   * Memory this child is using now, in MB, where it can say so itself: a
+   * worker thread's heap plus its external memory (WebAssembly and Buffers,
+   * which resourceLimits does not bound).
+   */
+  memoryMb?(): Promise<number | null>;
+}
+
+/**
+ * A worker thread as a redaction child. Its heap is the Worker's
+ * resourceLimits (sized per file by the caller), and running out of it ends
+ * the worker, not the process, reported as out of memory so the pool retries
+ * with twice the heap. Its CPU time comes from the Worker itself, on every
+ * platform, even while its own thread is busy (review of 26 Oct 2026: on
+ * Windows there was no CPU ceiling or stall detection).
+ */
+export function redactionWorkerThread(worker: Worker): RedactionChild {
+  const events = new EventEmitter();
+  let outOfMemory = false;
+  let online = false;
+  worker.once('online', () => {
+    online = true;
+  });
+  worker.on('message', (message: unknown) => events.emit('message', message));
+  worker.on('error', (error: NodeJS.ErrnoException) => {
+    if (error.code === 'ERR_WORKER_OUT_OF_MEMORY') outOfMemory = true;
+    else events.emit('error', error);
+  });
+  worker.once('exit', (code: number) => events.emit('exit', code, outOfMemory ? 'SIGABRT' : null));
+  return {
+    channel: null,
+    send(message, callback) {
+      try {
+        worker.postMessage(message);
+        callback?.(null);
+      } catch (error) {
+        callback?.(error as Error);
+      }
+      return true;
+    },
+    on: (event, listener) => events.on(event, listener),
+    once: (event, listener) => events.once(event, listener),
+    kill() {
+      void worker.terminate();
+      return true;
+    },
+    ref: () => worker.ref(),
+    unref: () => worker.unref(),
+    async memoryMb() {
+      if (!online) return null;
+      try {
+        // Worker#getHeapStatistics is not in this @types/node yet.
+        const stats = await (
+          worker as Worker & {
+            getHeapStatistics(): Promise<{ total_heap_size: number; external_memory?: number }>;
+          }
+        ).getHeapStatistics();
+        return (stats.total_heap_size + (stats.external_memory ?? 0)) / MB;
+      } catch {
+        return null;
+      }
+    },
+    async cpuSeconds() {
+      if (!online) return null;
+      try {
+        // Worker#cpuUsage (Node 24.6+) is not in this @types/node yet.
+        const usage = await (
+          worker as Worker & { cpuUsage(): Promise<{ user: number; system: number }> }
+        ).cpuUsage();
+        return (usage.user + usage.system) / 1e6;
+      } catch {
+        return null; // not running (yet or any more)
+      }
+    },
+  };
+}
+
+export interface ProcessTimings {
+  pollMs: number;
+  stallPolls: number;
+  cpuCeilingSeconds: number;
+  cpuSeconds: (pid: number | undefined) => Promise<number | null>;
+}
+
+class RedactionProcess {
+  readonly child: RedactionChild;
+  private nextId = 1;
+  private current: {
+    id: number;
+    resolve: (value: string) => void;
+    reject: (error: Error) => void;
+  } | null = null;
+  private poll: NodeJS.Timeout | undefined;
+  private polling = false;
+  private startCpu: number | null = null;
+  private lastCpu: number | null = null;
+  private stillPolls = 0;
+  rssMb = 0;
+  /** Memory the child last reported using (MB), or null where it cannot say. */
+  usedMb: number | null = null;
+  dead = false;
+
+  constructor(
+    readonly heapMb: number,
+    spawn: (heapMb: number) => RedactionChild,
+    private readonly timings: ProcessTimings,
+    /** CPU a file may use here; past it the file fails with ceilingReason. */
+    private readonly ceiling: { seconds: number; reason: string },
+    /**
+     * Memory this child may use in all (MB): its heap and the overhead around
+     * it. A worker's WebAssembly and Buffer memory are outside its heap limit,
+     * so the pool checks what it reports and ends it past this.
+     */
+    readonly budgetMb: number,
+    private readonly onDeath: (process: RedactionProcess) => void,
+    /** Called each time the child reports what it uses (admission reads it). */
+    private readonly onUsage: () => void = () => undefined
+  ) {
+    this.child = spawn(heapMb);
+    this.child.on('message', (reply: Reply) => this.receive(reply));
+    this.child.once('error', () => this.die('source_redaction_worker_failed'));
+    this.child.once('exit', (code, signal) =>
+      this.die(
+        signal === 'SIGABRT' || code === 134
+          ? 'source_redaction_out_of_memory'
+          : 'source_redaction_worker_exit'
+      )
+    );
+    this.child.unref?.();
+    this.child.channel?.unref?.();
+  }
+
+  get busy(): boolean {
+    return this.current !== null;
+  }
+
+  redact(input: RedactionInput): Promise<string> {
+    if (this.dead) return Promise.reject(new Error('source_redaction_worker_exit'));
+    if (this.current) return Promise.reject(new Error('source_redaction_busy'));
+    const id = this.nextId++;
+    return new Promise<string>((resolve, reject) => {
+      this.current = { id, resolve, reject };
+      // Hold the process open while a file is in flight: an idle child is
+      // unreferenced so it never keeps its parent alive, but a parent whose
+      // only pending work is this reply (a one-shot export) would otherwise
+      // exit mid-await with status 0 and nothing written.
+      this.child.ref?.();
+      this.child.channel?.ref?.();
+      this.startCpu = null;
+      this.lastCpu = null;
+      this.stillPolls = 0;
+      void this.sampleCpu();
+      this.poll = setInterval(() => void this.sampleCpu(), this.timings.pollMs);
+      this.poll.unref();
+      try {
+        this.child.send({ id, text: input.text, filePath: input.filePath }, (error) => {
+          if (error) this.settle(new Error('source_redaction_send_failed'));
+        });
+      } catch {
+        this.settle(new Error('source_redaction_send_failed'));
+      }
+    });
+  }
+
+  private async sampleCpu(): Promise<void> {
+    if (this.polling || !this.current) return;
+    this.polling = true;
+    const holding = this.current;
+    try {
+      const used = (await this.child.memoryMb?.()) ?? null;
+      if (used !== null && this.current === holding) {
+        this.usedMb = used;
+        if (used > this.budgetMb) {
+          // Past its budget in memory its heap limit does not see: out of
+          // memory, as if its heap had run out (retried once with twice the
+          // heap, then a strike), before the scanner itself has to restart.
+          this.settle(new Error('source_redaction_out_of_memory'));
+          this.kill();
+          return;
+        }
+        this.onUsage();
+      }
+      const cpu = this.child.cpuSeconds
+        ? await this.child.cpuSeconds()
+        : await this.timings.cpuSeconds(this.child.pid);
+      if (cpu === null || this.current !== holding) return; // no CPU accounting: only an exit ends the file
+      if (this.startCpu === null) {
+        this.startCpu = cpu;
+        this.lastCpu = cpu;
+        return;
+      }
+      if (cpu - this.startCpu >= this.ceiling.seconds) {
+        this.settle(new Error(this.ceiling.reason));
+        this.kill();
+        return;
+      }
+      this.stillPolls = cpu === this.lastCpu ? this.stillPolls + 1 : 0;
+      this.lastCpu = cpu;
+      if (this.stillPolls >= this.timings.stallPolls) {
+        this.settle(new Error('source_redaction_stalled'));
+        this.kill();
+      }
+    } finally {
+      this.polling = false;
+    }
+  }
+
+  private receive(reply: Reply): void {
+    if (!this.current || reply.id !== this.current.id) return;
+    if ('value' in reply && typeof reply.value === 'string') this.settle(null, reply.value);
+    else this.settle(new Error('source_redaction_invalid_result'));
+    if ('rssMb' in reply && typeof reply.rssMb === 'number') this.rssMb = reply.rssMb;
+  }
+
+  private settle(error: Error | null, value?: string): void {
+    const current = this.current;
+    this.current = null;
+    if (this.poll) clearInterval(this.poll);
+    this.poll = undefined;
+    this.child.unref?.();
+    this.child.channel?.unref?.();
+    if (!current) return;
+    if (error) current.reject(error);
+    else current.resolve(value as string);
+  }
+
+  private die(reason: string): void {
+    if (this.dead) return;
+    this.dead = true;
+    this.settle(new Error(reason));
+    this.onDeath(this);
+  }
+
+  kill(): void {
+    if (this.dead) return;
+    this.child.kill('SIGKILL');
+    this.dead = true;
+    this.settle(new Error('source_redaction_worker_exit'));
+    this.onDeath(this);
+  }
+}
+
+export interface SourceRedactionPoolOptions {
+  /** Start a worker child that runs runSourceRedactionWorker, with this heap. */
+  spawn: (heapMb: number) => RedactionChild;
+  /** Counted, logged and reported failures: never silent. */
+  onFailure: (reason: string, details: { bytes: number; extension: string }) => void;
+  onRetry?: (reason: string, details: { bytes: number; extension: string }) => void;
+  /** Memory a new child is judged against (default: scannerMemoryProbe()). */
+  memory?: MemoryProbe;
+  /** Memory, in MB, held by admitted work not yet redacted (bodies waiting in this task). */
+  heldMb?: () => number;
+  maxLargeChildren?: number;
+  cpuCeilingSeconds?: number;
+  /** CPU a file may use in the shared small-file child before it gets its own. */
+  sharedCpuCeilingSeconds?: number;
+  pollMs?: number;
+  stallPolls?: number;
+  cpuSeconds?: (pid: number | undefined) => Promise<number | null>;
+}
+
+/** Reasons a second, larger attempt cannot help. */
+const SHARED_CEILING_REASON = 'source_redaction_shared_cpu_ceiling';
+
+const FINAL_REASONS = new Set([
+  'source_redaction_invalid_result',
+  'source_redaction_exceeds_memory',
+  'source_redaction_cpu_ceiling',
+]);
+
+interface Job {
+  input: RedactionInput;
+  bytes: number;
+  heapMb: number;
+  large: boolean;
+  interactive: boolean;
+  seq: number;
+  resolve: (value: string) => void;
+  reject: (error: Error) => void;
+}
+
+interface Running {
+  process: RedactionProcess;
+  reservedMb: number;
+}
+
+export class SourceRedactionPool {
+  private shared: RedactionProcess | null = null;
+  private readonly sharedQueue: Job[] = [];
+  private readonly largeQueue: Job[] = [];
+  private readonly running = new Set<Running>();
+  private seq = 0;
+  private admitting = false;
+  private admitAgain = false;
+  /** Large children finished so far (admission re-reads memory when this moves during a read). */
+  private finished = 0;
+  private readonly timings: ProcessTimings;
+
+  constructor(private readonly options: SourceRedactionPoolOptions) {
+    this.timings = {
+      pollMs: options.pollMs ?? STALL_POLL_MS,
+      stallPolls: options.stallPolls ?? STALL_POLLS,
+      cpuCeilingSeconds: options.cpuCeilingSeconds ?? CPU_CEILING_SECONDS,
+      cpuSeconds: options.cpuSeconds ?? processCpuSeconds,
+    };
+  }
+
+  /** Large children running now (tests and diagnostics). */
+  get largeRunning(): number {
+    return this.running.size;
+  }
+
+  /** Interactive before background; arrival order within each. */
+  private enqueue(queue: Job[], job: Job): void {
+    const goesBefore = (other: Job) =>
+      job.interactive !== other.interactive ? job.interactive : job.seq < other.seq;
+    const index = queue.findIndex(goesBefore);
+    queue.splice(index < 0 ? queue.length : index, 0, job);
+  }
+
+  private runShared(): void {
+    if (this.shared?.busy) return;
+    const job = this.sharedQueue.shift();
+    if (!job) return;
+    if (!this.shared || this.shared.dead || this.shared.rssMb > SHARED_RECYCLE_RSS_MB) {
+      this.shared?.kill();
+      const created = new RedactionProcess(
+        SHARED_HEAP_MB,
+        this.options.spawn,
+        this.timings,
+        {
+          seconds: this.options.sharedCpuCeilingSeconds ?? SHARED_CPU_CEILING_SECONDS,
+          reason: SHARED_CEILING_REASON,
+        },
+        SHARED_HEAP_MB + CHILD_OVERHEAD_MB,
+        (dead) => {
+          if (this.shared === dead) this.shared = null;
+        }
+      );
+      this.shared = created;
+    }
+    const process = this.shared;
+    process.redact(job.input).then(
+      (value) => {
+        job.resolve(value);
+        this.runShared();
+      },
+      (error: Error) => {
+        job.reject(error);
+        this.runShared();
+      }
+    );
+  }
+
+  /**
+   * Memory the redaction children use now, in MB, each counted at most up to
+   * its budget: what a scanner's own memory ceiling leaves out when its
+   * children are worker threads in its process.
+   */
+  childrenUsedMb(): number {
+    let total = 0;
+    for (const run of this.running)
+      total += Math.min(run.reservedMb, Math.max(0, run.process.usedMb ?? 0));
+    if (this.shared && !this.shared.dead)
+      total += Math.min(this.shared.budgetMb, Math.max(0, this.shared.usedMb ?? 0));
+    return total;
+  }
+
+  private async admitLarge(): Promise<void> {
+    // A child that finishes while the probe is read calls this again; it must
+    // not be lost, or a waiting file stays queued until the next finish.
+    if (this.admitting) {
+      this.admitAgain = true;
+      return;
+    }
+    this.admitting = true;
+    this.admitAgain = false;
+    try {
+      const limit = this.options.maxLargeChildren ?? MAX_LARGE_CHILDREN;
+      while (this.largeQueue.length > 0 && this.running.size < limit) {
+        const job = this.largeQueue[0] as Job;
+        const needMb = job.heapMb + CHILD_OVERHEAD_MB;
+        const finishedBefore = this.finished;
+        const reading = await (this.options.memory ?? scannerMemoryProbe())();
+        // A child that finished while memory was read makes the reading stale
+        // (it may still count that child): read again before deciding.
+        if (this.finished !== finishedBefore) continue;
+        let outstanding = 0;
+        for (const run of this.running) {
+          // What it may still grow into: its reservation less what it uses
+          // now, as it reports (a worker thread) or as /proc says (a process).
+          const actual = reading.includesChildren
+            ? (run.process.usedMb ?? residentMb(run.process.child.pid) ?? 0)
+            : 0;
+          outstanding += Math.max(0, run.reservedMb - actual);
+        }
+        const available =
+          reading.limitMb -
+          reading.usedMb -
+          ADMISSION_RESERVE_MB -
+          outstanding -
+          (this.options.heldMb?.() ?? 0);
+        if (needMb > available) {
+          // Running children free memory when they finish; with none running,
+          // nothing will, so the file can never fit here.
+          if (this.running.size > 0) return;
+          this.largeQueue.shift();
+          job.reject(new Error('source_redaction_exceeds_memory'));
+          continue;
+        }
+        this.largeQueue.shift();
+        const run: Running = {
+          reservedMb: needMb,
+          process: new RedactionProcess(
+            job.heapMb,
+            this.options.spawn,
+            this.timings,
+            { seconds: this.timings.cpuCeilingSeconds, reason: 'source_redaction_cpu_ceiling' },
+            needMb,
+            () => undefined,
+            // A file waiting for room may fit once this child says what it
+            // uses: until then admission counts its whole reservation.
+            () => {
+              if (this.largeQueue.length > 0) void this.admitLarge();
+            }
+          ),
+        };
+        this.running.add(run);
+        run.process
+          .redact(job.input)
+          .then(job.resolve, job.reject)
+          .finally(() => {
+            run.process.kill();
+            this.running.delete(run);
+            this.finished++;
+            void this.admitLarge();
+          });
+      }
+    } finally {
+      this.admitting = false;
+      if (this.admitAgain) {
+        this.admitAgain = false;
+        void this.admitLarge();
+      }
+    }
+  }
+
+  private submit(
+    input: RedactionInput,
+    bytes: number,
+    heapMb: number,
+    large: boolean,
+    seq: number
+  ) {
+    return new Promise<string>((resolve, reject) => {
+      const job: Job = {
+        input,
+        bytes,
+        heapMb,
+        large,
+        interactive: input.priority === 'interactive',
+        seq,
+        resolve,
+        reject,
+      };
+      if (large) {
+        this.enqueue(this.largeQueue, job);
+        void this.admitLarge();
+      } else {
+        this.enqueue(this.sharedQueue, job);
+        this.runShared();
+      }
+    });
+  }
+
+  /** Redact one source; throws a counted, logged error for this file when it cannot. */
+  async redact(input: RedactionInput): Promise<string> {
+    const bytes = Buffer.byteLength(input.text);
+    const heapMb = Math.max(SHARED_HEAP_MB, redactionHeapMb(bytes, input.filePath));
+    const seq = this.seq++;
+    try {
+      return await this.submit(input, bytes, heapMb, bytes >= LARGE_FILE_BYTES, seq);
+    } catch (first) {
+      const reason = first instanceof Error ? first.message : String(first);
+      if (FINAL_REASONS.has(reason)) return this.fail(input, bytes, reason);
+      if (reason === SHARED_CEILING_REASON) {
+        // Too much CPU for the shared child: a child of its own, with the full
+        // ceiling, so the small files behind it are not held up.
+        try {
+          return await this.submit(input, bytes, heapMb, true, seq);
+        } catch (own) {
+          return this.fail(input, bytes, own instanceof Error ? own.message : String(own));
+        }
+      }
+      // Out of memory, an unexpected exit or a hung child: once more, in its
+      // own child, with twice the heap, keeping its place in line.
+      this.options.onRetry?.(reason, { bytes, extension: extensionOf(input.filePath) });
+      try {
+        return await this.submit(input, bytes, heapMb * 2, true, seq);
+      } catch (second) {
+        return this.fail(input, bytes, second instanceof Error ? second.message : String(second));
+      }
+    }
+  }
+
+  private fail(input: RedactionInput, bytes: number, reason: string): never {
+    this.options.onFailure(reason, { bytes, extension: extensionOf(input.filePath) });
+    throw new Error(reason);
+  }
+
+  /** Tests and shutdown. */
+  close(): void {
+    this.shared?.kill();
+    this.shared = null;
+    for (const run of this.running) run.process.kill();
+  }
+}
+
+function extensionOf(path: string): string {
+  return /\.([A-Za-z0-9]{1,8})$/.exec(path)?.[1]?.toLowerCase() ?? '';
+}
+
+/**
+ * The child side: redact each file sent over IPC, in this process, and
+ * answer with its id and this process's resident memory (so the parent can
+ * recycle a child whose WebAssembly memory has grown).
+ */
+export function runSourceRedactionWorker(): void {
+  // In a worker thread (the scanner) replies go to the parent thread, and
+  // the memory reported is this thread's own heap and external memory (its
+  // WebAssembly memory included); the process's resident memory is shared.
+  const port = parentPort;
+  const reply = port
+    ? (message: unknown) => port.postMessage(message)
+    : (message: unknown) => process.send?.(message);
+  if (!port && !process.send) throw new Error('Source redaction worker requires IPC');
+  const memoryMb = () => {
+    const usage = process.memoryUsage();
+    return Math.round((port ? usage.heapTotal + usage.external : usage.rss) / MB);
+  };
+  const handle = (input: { id: number; text: string; filePath: string }) => {
+    void scrubSourceSecretsInProcess(input.text, input.filePath)
+      .then((value) => {
+        reply({ id: input.id, value, rssMb: memoryMb() });
+      })
+      .catch(() => {
+        reply({ id: input.id, error: 'source_redaction_invalid_result' });
+      });
+  };
+  if (port) {
+    port.on('message', handle);
+    return;
+  }
+  process.on('message', handle);
+  process.once('disconnect', () => process.exit(0));
+}