@mnemonik/shared 7.154.3 → 7.156.0

package/dist/sourceRedactionPool.d.ts

@@ -0,0 +1,198 @@
+import { type Worker } from 'node:worker_threads';
+/** At and above this length a file gets a child of its own, sized from it. */
+export declare const LARGE_FILE_BYTES: number;
+/** The shared child's heap: comfortably above any file under LARGE_FILE_BYTES. */
+export declare const SHARED_HEAP_MB = 256;
+export declare const STALL_POLL_MS = 5000;
+/** One minute without any CPU while holding a file. */
+export declare const STALL_POLLS = 12;
+/**
+ * CPU a single file may use. Generous: the slowest files measured (26 October
+ * 2026, loaded host) were a 3.1 MB Svelte component at 23 s and a 9.9 MB Vue
+ * component at 17 s.
+ */
+export declare const CPU_CEILING_SECONDS = 300;
+/**
+ * CPU a single file may use in the shared small-file child before it is sent
+ * to a child of its own: one tenant's crafted file must not stall small-file
+ * redaction for everyone on the task.
+ */
+export declare const SHARED_CPU_CEILING_SECONDS = 30;
+/**
+ * Failures that say something about the file itself, which a scanner counts
+ * as strikes against that revision: it ran to the CPU ceiling, ran out of
+ * memory at twice its heap, or could not be redacted. Not counted: a child
+ * that would not fit in memory now, or one that stalled; those are retried.
+ */
+export declare const STRIKE_REASONS: ReadonlySet<string>;
+/** Large children a pool runs at once, memory permitting. */
+export declare const MAX_LARGE_CHILDREN = 2;
+/**
+ * A scanner's total memory, itself and its redaction children: the systemd
+ * unit's MemoryMax, and the budget used where no control group limits it.
+ */
+export declare const SCANNER_TOTAL_MEMORY_MB = 1536;
+/**
+ * The heap a file's child may use: 128 MB plus 16 MB per MB of source (as F2's
+ * parse pool), or 110 MB per MB for Vue and Svelte components. Measured 26
+ * October 2026 (Liftoff-only): a 9.9 MB Vue component of about 100,000
+ * template nodes ran out of memory at 900 MB and completed at 1,228 MB (peak
+ * resident 1.13 GB); a 3.1 MB Svelte component completed at 500 MB.
+ */
+export declare function redactionHeapMb(bytes: number, filePath?: string): number;
+/** Resident memory a child uses beyond its heap (code, grammars, buffers). */
+export declare const CHILD_OVERHEAD_MB = 128;
+/** Memory left free for everything else when a large child is admitted. */
+export declare const ADMISSION_RESERVE_MB = 256;
+/** What a memory probe reports, in MB. */
+export interface MemoryReading {
+    limitMb: number;
+    usedMb: number;
+    /** Whether usedMb already includes this pool's running children. */
+    includesChildren: boolean;
+    source: string;
+}
+export type MemoryProbe = () => MemoryReading | Promise<MemoryReading>;
+export interface CgroupMemory {
+    /** null when the group has no limit. */
+    limitMb: number | null;
+    /** memory.current: everything charged, page cache included. */
+    currentMb: number;
+    /** memory.current less inactive file cache: what the kernel cannot easily reclaim. */
+    workingSetMb: number;
+}
+type ReadText = (path: string) => string;
+/** This process's control group memory (v2, else v1), or null where there is none. */
+export declare function cgroupMemory(read?: ReadText): CgroupMemory | null;
+/**
+ * The API server's probe: the task's memory limit (ECS task metadata, else the
+ * cgroup, else the host: the caller supplies it) less the container's working
+ * set (memory.current less inactive file cache, which the kernel reclaims
+ * first). Without a control group, this process's resident memory.
+ */
+export declare function containerMemoryProbe(limitMb: () => number | Promise<number>, read?: ReadText): MemoryProbe;
+/**
+ * A scanner's probe. Under a control group limit (Linux, the systemd unit):
+ * the limit less the working set, since the scanner's file reads leave page
+ * cache the kernel reclaims before it would kill anything. Elsewhere (macOS,
+ * Windows, an unlimited group): the scanner's own total budget less its own
+ * resident memory, never the machine's free memory, which on macOS counts
+ * compressed and cached pages as used and would refuse almost everything.
+ */
+export declare function scannerMemoryProbe(totalBudgetMb?: number, read?: ReadText, 
+/** Children are worker threads, so this process's own resident memory includes them. */
+childrenInProcess?: boolean): MemoryProbe;
+export type RedactionPriority = 'interactive' | 'background';
+export interface RedactionInput {
+    text: string;
+    filePath: string;
+    /** An agent waiting on this file (file_push) goes before background ingest. */
+    priority?: RedactionPriority;
+}
+declare function parsePsTime(text: string): number | null;
+/** CPU seconds (user + system) a process has used: /proc on Linux, `ps` elsewhere, null on Windows. */
+export declare function processCpuSeconds(pid: number | undefined): Promise<number | null>;
+export { parsePsTime as parsePsTimeForTest };
+/**
+ * What the pool needs from a redaction child: a forked process (the server),
+ * or a worker thread (the scanner: redactionWorkerThread).
+ */
+export interface RedactionChild {
+    readonly pid?: number;
+    readonly channel?: {
+        ref?(): void;
+        unref?(): void;
+    } | null;
+    send(message: unknown, callback?: (error: Error | null) => void): boolean;
+    on(event: string, listener: (...args: any[]) => void): unknown;
+    once(event: string, listener: (...args: any[]) => void): unknown;
+    kill(signal?: NodeJS.Signals): boolean;
+    ref?(): void;
+    unref?(): void;
+    /** CPU seconds this child has used, where it can say so itself (a worker thread, on any platform). */
+    cpuSeconds?(): Promise<number | null>;
+    /**
+     * Memory this child is using now, in MB, where it can say so itself: a
+     * worker thread's heap plus its external memory (WebAssembly and Buffers,
+     * which resourceLimits does not bound).
+     */
+    memoryMb?(): Promise<number | null>;
+}
+/**
+ * A worker thread as a redaction child. Its heap is the Worker's
+ * resourceLimits (sized per file by the caller), and running out of it ends
+ * the worker, not the process, reported as out of memory so the pool retries
+ * with twice the heap. Its CPU time comes from the Worker itself, on every
+ * platform, even while its own thread is busy (review of 26 Oct 2026: on
+ * Windows there was no CPU ceiling or stall detection).
+ */
+export declare function redactionWorkerThread(worker: Worker): RedactionChild;
+export interface ProcessTimings {
+    pollMs: number;
+    stallPolls: number;
+    cpuCeilingSeconds: number;
+    cpuSeconds: (pid: number | undefined) => Promise<number | null>;
+}
+export interface SourceRedactionPoolOptions {
+    /** Start a worker child that runs runSourceRedactionWorker, with this heap. */
+    spawn: (heapMb: number) => RedactionChild;
+    /** Counted, logged and reported failures: never silent. */
+    onFailure: (reason: string, details: {
+        bytes: number;
+        extension: string;
+    }) => void;
+    onRetry?: (reason: string, details: {
+        bytes: number;
+        extension: string;
+    }) => void;
+    /** Memory a new child is judged against (default: scannerMemoryProbe()). */
+    memory?: MemoryProbe;
+    /** Memory, in MB, held by admitted work not yet redacted (bodies waiting in this task). */
+    heldMb?: () => number;
+    maxLargeChildren?: number;
+    cpuCeilingSeconds?: number;
+    /** CPU a file may use in the shared small-file child before it gets its own. */
+    sharedCpuCeilingSeconds?: number;
+    pollMs?: number;
+    stallPolls?: number;
+    cpuSeconds?: (pid: number | undefined) => Promise<number | null>;
+}
+export declare class SourceRedactionPool {
+    private readonly options;
+    private shared;
+    private readonly sharedQueue;
+    private readonly largeQueue;
+    private readonly running;
+    private seq;
+    private admitting;
+    private admitAgain;
+    /** Large children finished so far (admission re-reads memory when this moves during a read). */
+    private finished;
+    private readonly timings;
+    constructor(options: SourceRedactionPoolOptions);
+    /** Large children running now (tests and diagnostics). */
+    get largeRunning(): number;
+    /** Interactive before background; arrival order within each. */
+    private enqueue;
+    private runShared;
+    /**
+     * Memory the redaction children use now, in MB, each counted at most up to
+     * its budget: what a scanner's own memory ceiling leaves out when its
+     * children are worker threads in its process.
+     */
+    childrenUsedMb(): number;
+    private admitLarge;
+    private submit;
+    /** Redact one source; throws a counted, logged error for this file when it cannot. */
+    redact(input: RedactionInput): Promise<string>;
+    private fail;
+    /** Tests and shutdown. */
+    close(): void;
+}
+/**
+ * The child side: redact each file sent over IPC, in this process, and
+ * answer with its id and this process's resident memory (so the parent can
+ * recycle a child whose WebAssembly memory has grown).
+ */
+export declare function runSourceRedactionWorker(): void;
+//# sourceMappingURL=sourceRedactionPool.d.ts.map

package/dist/sourceRedactionPool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sourceRedactionPool.d.ts","sourceRoot":"","sources":["../src/sourceRedactionPool.ts"],"names":[],"mappings":"AAiDA,OAAO,EAAc,KAAK,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAK9D,8EAA8E;AAC9E,eAAO,MAAM,gBAAgB,QAAS,CAAC;AACvC,kFAAkF;AAClF,eAAO,MAAM,cAAc,MAAM,CAAC;AAGlC,eAAO,MAAM,aAAa,OAAQ,CAAC;AACnC,uDAAuD;AACvD,eAAO,MAAM,WAAW,KAAK,CAAC;AAC9B;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,MAAM,CAAC;AACvC;;;;GAIG;AACH,eAAO,MAAM,0BAA0B,KAAK,CAAC;AAC7C;;;;;GAKG;AACH,eAAO,MAAM,cAAc,EAAE,WAAW,CAAC,MAAM,CAI7C,CAAC;AACH,6DAA6D;AAC7D,eAAO,MAAM,kBAAkB,IAAI,CAAC;AACpC;;;GAGG;AACH,eAAO,MAAM,uBAAuB,OAAO,CAAC;AAE5C;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,SAAK,GAAG,MAAM,CAGpE;AAED,8EAA8E;AAC9E,eAAO,MAAM,iBAAiB,MAAM,CAAC;AACrC,2EAA2E;AAC3E,eAAO,MAAM,oBAAoB,MAAM,CAAC;AAExC,0CAA0C;AAC1C,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,oEAAoE;IACpE,gBAAgB,EAAE,OAAO,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;CAChB;AACD,MAAM,MAAM,WAAW,GAAG,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;AAEvE,MAAM,WAAW,YAAY;IAC3B,wCAAwC;IACxC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,+DAA+D;IAC/D,SAAS,EAAE,MAAM,CAAC;IAClB,sFAAsF;IACtF,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,KAAK,QAAQ,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;AA2BzC,sFAAsF;AACtF,wBAAgB,YAAY,CAAC,IAAI,GAAE,QAAmB,GAAG,YAAY,GAAG,IAAI,CAkC3E;AAED;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,EACvC,IAAI,GAAE,QAAmB,GACxB,WAAW,CAUb;AAED;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAChC,aAAa,SAA0B,EACvC,IAAI,GAAE,QAAmB;AACzB,wFAAwF;AACxF,iBAAiB,UAAQ,GACxB,WAAW,CAiBb;AAED,MAAM,MAAM,iBAAiB,GAAG,aAAa,GAAG,YAAY,CAAC;AAE7D,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,iBAAiB,CAAC;CAC9B;AAMD,iBAAS,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAUhD;AAED,uGAAuG;AACvG,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAiBvF;AACD,OAAO,EAAE,WAAW,IAAI,kBAAkB,EAAE,CAAC;AAY7C;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,CAAC,EAAE;QAAE,GAAG,CAAC,IAAI,IAAI,CAAC;QAAC,KAAK,CAAC,IAAI,IAAI,CAAA;KAAE,GAAG,IAAI,CAAC;IAC3D,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,KAAK,IAAI,GAAG,OAAO,CAAC;IAG1E,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,GAAG,OAAO,CAAC;IAC/D,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,GAAG,OAAO,CAAC;IAEjE,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC;IACvC,GAAG,CAAC,IAAI,IAAI,CAAC;IACb,KAAK,CAAC,IAAI,IAAI,CAAC;IACf,sGAAsG;IACtG,UAAU,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IACtC;;;;OAIG;IACH,QAAQ,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;CACrC;AAED;;;;;;;GAOG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,MAAM,GAAG,cAAc,CA2DpE;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,UAAU,EAAE,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;CACjE;AAgKD,MAAM,WAAW,0BAA0B;IACzC,+EAA+E;IAC/E,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,CAAC;IAC1C,2DAA2D;IAC3D,SAAS,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IACnF,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IAClF,4EAA4E;IAC5E,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,2FAA2F;IAC3F,MAAM,CAAC,EAAE,MAAM,MAAM,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,gFAAgF;IAChF,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;CAClE;AA2BD,qBAAa,mBAAmB;IAYlB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAXpC,OAAO,CAAC,MAAM,CAAiC;IAC/C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAa;IACzC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAsB;IAC9C,OAAO,CAAC,GAAG,CAAK;IAChB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,UAAU,CAAS;IAC3B,gGAAgG;IAChG,OAAO,CAAC,QAAQ,CAAK;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiB;IAEzC,YAA6B,OAAO,EAAE,0BAA0B,EAO/D;IAED,0DAA0D;IAC1D,IAAI,YAAY,IAAI,MAAM,CAEzB;IAED,gEAAgE;IAChE,OAAO,CAAC,OAAO;IAOf,OAAO,CAAC,SAAS;IAkCjB;;;;OAIG;IACH,cAAc,IAAI,MAAM,CAOvB;YAEa,UAAU;IA+ExB,OAAO,CAAC,MAAM;IA4Bd,sFAAsF;IAChF,MAAM,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,CA2BnD;IAED,OAAO,CAAC,IAAI;IAKZ,0BAA0B;IAC1B,KAAK,IAAI,IAAI,CAIZ;CACF;AAMD;;;;GAIG;AACH,wBAAgB,wBAAwB,IAAI,IAAI,CA4B/C"}