@mnemonik/shared 7.154.3 → 7.156.0

package/src/hookTrace.ts

@@ -38,6 +38,7 @@ import {
   writeSync,
 } from 'node:fs';
 import { dirname, join } from 'node:path';
+import { open as openAsync, stat as statAsync } from 'node:fs/promises';
 import { mnemonikStateDirectory } from './protectedPaths.js';
 import { ensureWindowsPrivateDirectorySync } from './runtimeSigners.js';
 import { scrubSecrets } from './secretPatterns.js';
@@ -586,3 +587,145 @@ export function startHookTrace(options: HookTraceOptions): HookTrace {
   trace.open();
   return trace;
 }
+
+/** The trace window a hook reports its recent misses over. */
+export const HOOK_MISS_WINDOW_MS = 30 * 60 * 1000;
+/**
+ * How much of the trace file's end is read for that summary, asynchronously
+ * and once per hook process (it used to read 256 KB synchronously on every
+ * call). About a hundred hook runs: the recent ones are what the count is for.
+ */
+const HOOK_MISS_TAIL_BYTES = 64 * 1024;
+/** Sent on the main hook call: `<missed>/<total>` server calls over the window. */
+export const HOOK_MISSES_HEADER = 'x-mnemonik-hook-misses';
+
+/**
+ * How many of this host's recent calls to the server did not get an answer
+ * in time: a timeout, a connection error, a gateway or server error, a call
+ * still pending at exit, or a whole run the editor killed at its own hook
+ * timeout. Each of these is silent where it happens (the hook fails open, by
+ * design), so the main hook call carries the count to the server, and
+ * machine health tells the agent when a machine is losing memory this way
+ * (limits audit #2-#3: the hook budget chain fails silently).
+ *
+ * Reads only the end of the trace file, and never throws.
+ */
+export async function summarizeRecentHookCalls(
+  host: HookTraceHost,
+  options: { stateDir?: string; now?: number; windowMs?: number } = {}
+): Promise<{ missed: number; total: number }> {
+  // With the recorder off there is no file, which reads as nothing to report.
+  const empty = { missed: 0, total: 0 };
+  const now = options.now ?? Date.now();
+  const since = now - (options.windowMs ?? HOOK_MISS_WINDOW_MS);
+  let text: string;
+  try {
+    const file = hookTraceFile(host, options.stateDir);
+    const size = (await statAsync(file)).size;
+    const handle = await openAsync(file, 'r');
+    try {
+      const length = Math.min(size, HOOK_MISS_TAIL_BYTES);
+      const buffer = Buffer.alloc(length);
+      await handle.read(buffer, 0, length, size - length);
+      text = buffer.toString('utf8');
+    } finally {
+      await handle.close();
+    }
+  } catch {
+    return empty;
+  }
+  const started = new Map<string, number>();
+  const ended = new Set<string>();
+  let missed = 0;
+  let total = 0;
+  for (const line of text.split('\n')) {
+    let record: HookTraceRecord;
+    try {
+      record = JSON.parse(line) as HookTraceRecord;
+    } catch {
+      continue; // the first line of the tail is usually cut
+    }
+    const at = Date.parse(record.ts);
+    if (!(at >= since)) continue;
+    if (record.phase === 'start') started.set(record.id, at);
+    else if (record.phase === 'end') {
+      ended.add(record.id);
+      // One logical call per path in a run: a hedged request's aborted
+      // loser, or a first attempt whose retry was answered, is not a miss.
+      // Only a call none of whose requests got an answer is.
+      const answeredByPath = new Map<string, boolean>();
+      for (const call of record.calls ?? []) {
+        const status = call.status ?? 0;
+        const answered =
+          call.error === undefined &&
+          call.pending !== true &&
+          status > 0 &&
+          status < 500 &&
+          status !== 408 &&
+          status !== 429;
+        answeredByPath.set(call.path, (answeredByPath.get(call.path) ?? false) || answered);
+      }
+      for (const answered of answeredByPath.values()) {
+        total++;
+        if (!answered) missed++;
+      }
+    }
+  }
+  // A run with a start and no end was ended by the editor at its timeout.
+  for (const [id, at] of started) {
+    if (ended.has(id) || now - at < 60_000) continue;
+    total++;
+    missed++;
+  }
+  return { missed, total };
+}
+
+const summaries = new Map<
+  string,
+  { read: Promise<Record<string, string>>; value?: Record<string, string> }
+>();
+
+function summaryFor(
+  host: HookTraceHost,
+  options: { stateDir?: string; now?: number }
+): { read: Promise<Record<string, string>>; value?: Record<string, string> } {
+  const key = `${host}\0${options.stateDir ?? ''}\0${options.now ?? ''}`;
+  const cached = summaries.get(key);
+  if (cached) return cached;
+  const entry: { read: Promise<Record<string, string>>; value?: Record<string, string> } = {
+    read: Promise.resolve({}),
+  };
+  entry.read = summarizeRecentHookCalls(host, options).then(
+    ({ missed, total }): Record<string, string> => {
+      const value: Record<string, string> =
+        total > 0 ? { [HOOK_MISSES_HEADER]: `${missed}/${total}` } : {};
+      entry.value = value;
+      return value;
+    }
+  );
+  summaries.set(key, entry);
+  return entry;
+}
+
+/**
+ * The header the main hook call carries, or none when nothing was called
+ * recently. Read once per hook process, without blocking it.
+ */
+export function hookMissesHeaders(
+  host: HookTraceHost,
+  options: { stateDir?: string; now?: number } = {}
+): Promise<Record<string, string>> {
+  return summaryFor(host, options).read;
+}
+
+/**
+ * The header if the read has finished, else none, and never a wait: the
+ * count only feeds machine health, so it must never delay a hook request
+ * (the hook starts the read when it loads, so it is normally ready).
+ */
+export function hookMissesHeadersIfReady(
+  host: HookTraceHost,
+  options: { stateDir?: string; now?: number } = {}
+): Record<string, string> {
+  return summaryFor(host, options).value ?? {};
+}

package/src/index.ts

@@ -20,7 +20,10 @@ export {
   isGitBoundary,
   isProjectBoundary,
   rootProjectId,
-  SCANNER_MAX_DEPTH,
+  SCRUB_VERSION,
+  fileOp,
+  statTuple,
+  type FileStatTuple,
   isLinkedWorktreeCheckout,
   FIXTURE_PATH_RE,
   isFixturePath,
@@ -97,7 +100,33 @@ export {
   scrubSecrets,
   redactHighEntropyTokens,
 } from './secretPatterns.js';
-export { scrubSourceSecrets } from './sourceRedaction.js';
+export {
+  scrubSourceSecrets,
+  scrubSourceSecretsInProcess,
+  installSourceRedactor,
+  MAX_SOURCE_REDACTION_BYTES,
+  type SourceRedactor,
+  type SourceRedactionOptions,
+} from './sourceRedaction.js';
+export {
+  SourceRedactionPool,
+  redactionWorkerThread,
+  STRIKE_REASONS,
+  type RedactionChild,
+  runSourceRedactionWorker,
+  redactionHeapMb,
+  cgroupMemory,
+  containerMemoryProbe,
+  scannerMemoryProbe,
+  processCpuSeconds,
+  CPU_CEILING_SECONDS,
+  SCANNER_TOTAL_MEMORY_MB,
+  type MemoryProbe,
+  type MemoryReading,
+  type RedactionPriority,
+  LARGE_FILE_BYTES as LARGE_SOURCE_REDACTION_BYTES,
+  type SourceRedactionPoolOptions,
+} from './sourceRedactionPool.js';
 export {
   redactCredentials,
   credentialValues,

package/src/runtimeAssets.ts

@@ -0,0 +1,49 @@
+/**
+ * Never use bytes from the packaged scanner's extracted runtime unchecked
+ * (review of 26 Oct 2026). JavaScript is checked as it loads
+ * (packages/scanner/src/verifiedLoader.cts); this covers what is read with fs
+ * and then run or parsed as code: the grammar WebAssembly, web-tree-sitter's
+ * own runtime WebAssembly and the tag queries. The caller passes the buffer it
+ * is about to use, so the bytes checked are the bytes used.
+ *
+ * The packaged executable's entry point publishes the embedded assets' hashes
+ * on globalThis (and runtimeWorker.ts in each worker). Anywhere else, in
+ * development, tests and on the server, there is nothing to check against and
+ * this does nothing.
+ */
+import { createHash } from 'node:crypto';
+import { relative, sep } from 'node:path';
+
+interface VerifiedRuntime {
+  root: string;
+  hashes: ReadonlyMap<string, string>;
+}
+
+const RUNTIME = Symbol.for('mnemonik.sea.verifiedLoader');
+
+/** Bytes that are not the embedded asset they must be. */
+export class RuntimeAssetError extends Error {}
+
+function runtime(): VerifiedRuntime | undefined {
+  return (globalThis as { [RUNTIME]?: VerifiedRuntime })[RUNTIME];
+}
+
+/** Whether this process runs from the packaged executable's verified runtime. */
+export function inVerifiedRuntime(): boolean {
+  return runtime() !== undefined;
+}
+
+/**
+ * Check bytes read from `path` against the embedded asset they must equal.
+ * Throws `mnemonik_asset_integrity` when they differ and
+ * `mnemonik_unverified_asset` for a path that is not an embedded asset.
+ */
+export function verifyRuntimeAsset(path: string, bytes: Uint8Array): void {
+  const verified = runtime();
+  if (!verified) return;
+  const key = relative(verified.root, path).split(sep).join('/');
+  const expected = key.startsWith('../') || key === '..' ? undefined : verified.hashes.get(key);
+  if (!expected) throw new RuntimeAssetError(`mnemonik_unverified_asset: ${path}`);
+  if (createHash('sha256').update(bytes).digest('hex') !== expected)
+    throw new RuntimeAssetError(`mnemonik_asset_integrity: ${key}`);
+}

package/src/sourceRedaction.ts

@@ -2,7 +2,6 @@ import { parseVueSource, walkVueTemplate, VUE_FIDELITY_MARKER } from './vueSourc
 import { basename, extname } from 'node:path';
 import { Parser, type Node, type Tree } from 'web-tree-sitter';
 import { loadGrammar, objcppSyntaxIssue, type AstLanguageId } from './ast/grammars.js';
-import { MAX_AST_PARSE_BYTES } from './ast/astChunker.js';
 import { commandLineCredentialRanges, redactCredentials } from './redactCredentials.js';
 import {
   redactHighEntropyTokens,
@@ -15,8 +14,52 @@ function scrubPayload(text: string): string {
   return redactCredentials(scrubSecrets(text), { commandLineOnly: true }).text;
 }
 
+/**
+ * The largest source redacted with its syntax tree: the scanner's own file
+ * ceiling (MAX_SCANNED_FILE_BYTES, 10 MB; repeated here because codeScanner.ts
+ * imports this module). It was 2 MiB, sized from "the largest file in this
+ * repository", and above it every file took the text fallback: PowerShell,
+ * CMake, Groovy, Nix, Terraform and the bundled languages were stamped
+ * "source-fidelity-unavailable", so connected code refused every such file
+ * over 2 MiB, and other languages could keep structural [REDACTED] the
+ * extractors refuse (streams F1/F2, 25 October 2026). Memory for a large
+ * parse is bounded by running it in a child process sized from the file
+ * (installSourceRedactor below; sourceRedactionPool.ts), not by refusing it.
+ */
+export const MAX_SOURCE_REDACTION_BYTES = 10 * 1024 * 1024;
+
+/**
+ * Where a process sends source it should not parse in its own memory: the
+ * server sends Swift and every file of a megabyte or more to a Liftoff-only
+ * child sized from the file; the scanner sends large files to its own
+ * child. Unset (tests, small tools), everything is redacted in process.
+ */
+export interface SourceRedactionOptions {
+  /** 'interactive' when an agent is waiting on this file (file_push). */
+  priority?: 'interactive' | 'background';
+}
+export interface SourceRedactor {
+  routes(filePath: string, bytes: number): boolean;
+  redact(text: string, filePath: string, options?: SourceRedactionOptions): Promise<string>;
+}
+let outOfProcess: SourceRedactor | null = null;
+export function installSourceRedactor(redactor: SourceRedactor | null): void {
+  outOfProcess = redactor;
+}
+
 /** Source syntax stays executable; unsupported or incomplete input uses the conservative scrubber. */
-export async function scrubSourceSecrets(text: string, filePath: string): Promise<string> {
+export async function scrubSourceSecrets(
+  text: string,
+  filePath: string,
+  options?: SourceRedactionOptions
+): Promise<string> {
+  if (outOfProcess && text && outOfProcess.routes(filePath, Buffer.byteLength(text, 'utf8')))
+    return outOfProcess.redact(text, filePath, options);
+  return scrubSourceSecretsInProcess(text, filePath);
+}
+
+/** The redaction itself, in this process. The out-of-process worker calls this. */
+export async function scrubSourceSecretsInProcess(text: string, filePath: string): Promise<string> {
   const extension = extname(filePath).toLowerCase();
   if (extension === '.vue') return scrubVueSource(text);
   if (extension === '.svelte') return scrubSvelteSource(text);
@@ -136,7 +179,7 @@ export async function scrubSourceSecrets(text: string, filePath: string): Promis
       ? marker + clean
       : clean;
   };
-  if (!text || !language || Buffer.byteLength(text, 'utf8') > MAX_AST_PARSE_BYTES) {
+  if (!text || !language || Buffer.byteLength(text, 'utf8') > MAX_SOURCE_REDACTION_BYTES) {
     return fallback();
   }
   let parser: Parser | undefined;
@@ -1833,6 +1876,32 @@ function redactNode(node: Node, sensitive = false): string {
 }
 
 /** Sanitize parser-owned payload spans; no SFC boundary is inferred from text matching. */
+/**
+ * Apply replacement spans to `text` in one pass, or null when two overlap
+ * (or a span is malformed).
+ * The result equals applying them one at a time from the last start to the
+ * first (spans that start together: the one added later lands first), which
+ * is what the Vue scrubber did with a slice-and-concatenate per edit; that
+ * copied the whole file for every edit, about 40 s for a 1 MB component.
+ */
+export function applySpanEdits(
+  text: string,
+  edits: ReadonlyArray<{ start: number; end: number; clean: string }>
+): string | null {
+  // Stable sort, then reversed: ascending start, and among equal starts the
+  // later-added edit first, exactly as the one-at-a-time order placed them.
+  const ordered = [...edits].sort((a, b) => b.start - a.start).reverse();
+  const pieces: string[] = [];
+  let offset = 0;
+  for (const e of ordered) {
+    if (e.start < offset || e.end < e.start || e.end > text.length) return null;
+    pieces.push(text.slice(offset, e.start), e.clean);
+    offset = e.end;
+  }
+  pieces.push(text.slice(offset));
+  return pieces.join('');
+}
+
 async function scrubVueSource(text: string): Promise<string> {
   if (!text) return text;
   const parsed = await parseVueSource(text, true);
@@ -1973,14 +2042,8 @@ async function scrubVueSource(text: string): Promise<string> {
     const cleanTail = scrubPayload(tail);
     if (cleanTail !== tail) faithful = false;
     edit(gapStart, text.length, cleanTail);
-    edits.sort((a, b) => b.start - a.start);
-    let clean = text;
-    let boundary = text.length;
-    for (const e of edits) {
-      if (e.end > boundary) return fallback();
-      clean = clean.slice(0, e.start) + e.clean + clean.slice(e.end);
-      boundary = e.start;
-    }
+    const clean = applySpanEdits(text, edits);
+    if (clean === null) return fallback();
     if ('reason' in (await parseVueSource(clean))) faithful = false;
     return faithful || clean.startsWith(VUE_FIDELITY_MARKER) ? clean : VUE_FIDELITY_MARKER + clean;
   } catch {
@@ -1996,29 +2059,33 @@ async function scrubSvelteSource(text: string): Promise<string> {
   const fallback = (): string => (text.startsWith(marker) ? '' : marker) + scrubPayload(text);
   if (!text) return text;
   if (text.includes('mnemonik:svelte-source-fidelity-unavailable')) return fallback();
-  if (Buffer.byteLength(text, 'utf8') > MAX_AST_PARSE_BYTES) return fallback();
+  if (Buffer.byteLength(text, 'utf8') > MAX_SOURCE_REDACTION_BYTES) return fallback();
   let parser: Parser | undefined;
   try {
-    const deadline = performance.now() + 4_000;
+    // No clock and no node count decide this file: both refused large
+    // components outright (a 4 s deadline and 200,000 nodes). Memory is
+    // bounded by the process the parse runs in (installSourceRedactor).
     const { parse } = await import('svelte/compiler');
     const ast = parse(text, { modern: true });
     const edits: Array<{ start: number; end: number; clean: string }> = [];
     let faithful = true;
-    parser = new Parser();
+    // Grammars first: loading one initializes tree-sitter, and a Parser
+    // constructed before that throws. In a fresh process (a redaction child,
+    // or a scanner whose first file is a component) that made every Svelte
+    // file take the unfaithful fallback.
     const js = await loadGrammar('javascript');
     const ts = await loadGrammar('typescript');
     if ('unavailable' in js || 'unavailable' in ts) return fallback();
+    parser = new Parser();
     parser.setLanguage(js.ok.language);
     const activeParser = parser;
     const syntax = (source: string, expression = false, sensitive = false): string => {
       const prefix = expression ? `const ${sensitive ? 'password' : '__mnemonik_svelte'} = (` : '';
       const suffix = expression ? '\n);' : '';
       const full = prefix + source + suffix;
-      const tree = activeParser.parse(full, null, {
-        progressCallback: () => performance.now() > deadline,
-      });
+      const tree = activeParser.parse(full);
       try {
-        if (!tree || tree.rootNode.hasError || tree.rootNode.descendantCount > 200_000) {
+        if (!tree || tree.rootNode.hasError) {
           faithful = false;
           return sensitive ? literalPayload(source, true) : scrubPayload(source);
         }
@@ -2062,9 +2129,7 @@ async function scrubSvelteSource(text: string): Promise<string> {
     const pending: Array<{ value: unknown; sensitive: boolean }> = [
       { value: ast.fragment, sensitive: false },
     ];
-    let count = 0;
     while (pending.length) {
-      if (++count > 200_000 || performance.now() > deadline) return fallback();
       const entry = pending.pop();
       if (!entry || !entry.value || typeof entry.value !== 'object') continue;
       if (Array.isArray(entry.value)) {