agentic-qe 3.11.2 → 3.11.4

package/dist/shared/llm/providers/ollama-timeout.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Adaptive request-timeout sizing for local Ollama generations (#3).
+ *
+ * Local models vary ~6× in speed (a dense 27B ≈ 15 tok/s vs a 30B MoE ≈ 90
+ * tok/s on the same Mac) and thinking models can spend thousands of tokens
+ * before answering, so one fixed timeout either strands fast models or times
+ * out slow ones mid-generation. We size the timeout from the output cap
+ * (num_predict) and the model's observed throughput, with generous headroom.
+ *
+ * Pure + dependency-free so it is unit-testable in isolation.
+ */
+export declare const OLLAMA_TIMEOUT: {
+    /** Conservative throughput assumed before a model has been observed (dense floor). */
+    readonly DEFAULT_TOK_PER_SEC: 15;
+    /** Multiplier over ideal generation time to absorb variance. */
+    readonly SAFETY_FACTOR: 2.5;
+    /** Fixed overhead for cold model load / prompt eval (~6–9s measured). */
+    readonly LOAD_OVERHEAD_MS: 15000;
+    /** Never time out faster than this. */
+    readonly FLOOR_MS: 30000;
+    /** Never wait longer than this. */
+    readonly CEIL_MS: 600000;
+};
+/** Size a timeout (ms) from the output token cap and observed tok/s. */
+export declare function computeAdaptiveTimeoutMs(numPredict: number, tokPerSec?: number): number;
+//# sourceMappingURL=ollama-timeout.d.ts.map

package/dist/shared/llm/providers/ollama-timeout.js

@@ -0,0 +1,31 @@
+/**
+ * Adaptive request-timeout sizing for local Ollama generations (#3).
+ *
+ * Local models vary ~6× in speed (a dense 27B ≈ 15 tok/s vs a 30B MoE ≈ 90
+ * tok/s on the same Mac) and thinking models can spend thousands of tokens
+ * before answering, so one fixed timeout either strands fast models or times
+ * out slow ones mid-generation. We size the timeout from the output cap
+ * (num_predict) and the model's observed throughput, with generous headroom.
+ *
+ * Pure + dependency-free so it is unit-testable in isolation.
+ */
+export const OLLAMA_TIMEOUT = {
+    /** Conservative throughput assumed before a model has been observed (dense floor). */
+    DEFAULT_TOK_PER_SEC: 15,
+    /** Multiplier over ideal generation time to absorb variance. */
+    SAFETY_FACTOR: 2.5,
+    /** Fixed overhead for cold model load / prompt eval (~6–9s measured). */
+    LOAD_OVERHEAD_MS: 15_000,
+    /** Never time out faster than this. */
+    FLOOR_MS: 30_000,
+    /** Never wait longer than this. */
+    CEIL_MS: 600_000,
+};
+/** Size a timeout (ms) from the output token cap and observed tok/s. */
+export function computeAdaptiveTimeoutMs(numPredict, tokPerSec = OLLAMA_TIMEOUT.DEFAULT_TOK_PER_SEC) {
+    const tps = tokPerSec > 0 ? tokPerSec : OLLAMA_TIMEOUT.DEFAULT_TOK_PER_SEC;
+    const genMs = (Math.max(0, numPredict) / tps) * 1000;
+    const computed = genMs * OLLAMA_TIMEOUT.SAFETY_FACTOR + OLLAMA_TIMEOUT.LOAD_OVERHEAD_MS;
+    return Math.min(OLLAMA_TIMEOUT.CEIL_MS, Math.max(OLLAMA_TIMEOUT.FLOOR_MS, Math.round(computed)));
+}
+//# sourceMappingURL=ollama-timeout.js.map

package/dist/shared/llm/providers/ollama.d.ts

@@ -19,7 +19,17 @@ export declare class OllamaProvider implements LLMProvider {
     private config;
     private requestId;
     private availableModels;
+    /** EMA of observed generation throughput (tok/s) per model, for adaptive timeouts (#3). */
+    private observedTokPerSec;
     constructor(config?: Partial<OllamaConfig>);
+    /**
+     * Adaptive per-request timeout (#3). An explicitly customized config.timeoutMs
+     * (≠ the built-in default) is honored as the caller's intent; otherwise the
+     * timeout is sized from num_predict and the model's observed throughput.
+     */
+    private timeoutFor;
+    /** Fold a completed generation's throughput into the per-model EMA. */
+    private recordThroughput;
     /**
      * Check if Ollama is available and running
      */

package/dist/shared/llm/providers/ollama.js

@@ -8,12 +8,14 @@
 import { createLLMError, } from '../interfaces';
 import { CostTracker } from '../cost-tracker';
 import { TokenMetricsCollector } from '../../../learning/token-tracker.js';
+import { resolveOllamaBaseUrl } from '../ollama-url.js';
+import { computeAdaptiveTimeoutMs } from './ollama-timeout.js';
 /**
  * Default Ollama configuration
  */
 export const DEFAULT_OLLAMA_CONFIG = {
     model: 'llama3.1',
-    baseUrl: 'http://localhost:11434',
+    baseUrl: resolveOllamaBaseUrl('http://localhost:11434'),
     maxTokens: 4096,
     temperature: 0.7,
     timeoutMs: 120000, // Longer timeout for local models
@@ -32,9 +34,32 @@ export class OllamaProvider {
     config;
     requestId = 0;
     availableModels = [];
+    /** EMA of observed generation throughput (tok/s) per model, for adaptive timeouts (#3). */
+    observedTokPerSec = new Map();
     constructor(config = {}) {
         this.config = { ...DEFAULT_OLLAMA_CONFIG, ...config };
     }
+    /**
+     * Adaptive per-request timeout (#3). An explicitly customized config.timeoutMs
+     * (≠ the built-in default) is honored as the caller's intent; otherwise the
+     * timeout is sized from num_predict and the model's observed throughput.
+     */
+    timeoutFor(model, numPredict) {
+        const configured = this.config.timeoutMs;
+        if (configured && configured !== DEFAULT_OLLAMA_CONFIG.timeoutMs)
+            return configured;
+        return computeAdaptiveTimeoutMs(numPredict, this.observedTokPerSec.get(model));
+    }
+    /** Fold a completed generation's throughput into the per-model EMA. */
+    recordThroughput(model, evalCount, evalDurationNs) {
+        if (!evalCount || !evalDurationNs || evalDurationNs <= 0)
+            return;
+        const sample = evalCount / (evalDurationNs / 1e9);
+        if (!Number.isFinite(sample) || sample <= 0)
+            return;
+        const prev = this.observedTokPerSec.get(model);
+        this.observedTokPerSec.set(model, prev ? prev * 0.5 + sample * 0.5 : sample);
+    }
     /**
      * Check if Ollama is available and running
      */
@@ -132,12 +157,13 @@ export class OllamaProvider {
                             stop: options?.stopSequences,
                         },
                     }),
-                }, options?.timeoutMs ?? this.config.timeoutMs ?? 120000);
+                }, options?.timeoutMs ?? this.timeoutFor(model, maxTokens));
                 if (!response.ok) {
                     throw await this.handleApiError(response, model);
                 }
                 const data = await response.json();
                 content = data.message.content;
+                this.recordThroughput(model, data.eval_count, data.eval_duration);
                 promptTokens = data.prompt_eval_count ?? this.estimateTokens(JSON.stringify(messages));
                 completionTokens = data.eval_count ?? this.estimateTokens(content);
             }
@@ -158,12 +184,13 @@ export class OllamaProvider {
                             stop: options?.stopSequences,
                         },
                     }),
-                }, options?.timeoutMs ?? this.config.timeoutMs ?? 120000);
+                }, options?.timeoutMs ?? this.timeoutFor(model, maxTokens));
                 if (!response.ok) {
                     throw await this.handleApiError(response, model);
                 }
                 const data = await response.json();
                 content = data.response;
+                this.recordThroughput(model, data.eval_count, data.eval_duration);
                 promptTokens = data.prompt_eval_count ?? this.estimateTokens(prompt);
                 completionTokens = data.eval_count ?? this.estimateTokens(content);
             }
@@ -321,7 +348,7 @@ export class OllamaProvider {
      * Get base URL
      */
     getBaseUrl() {
-        return (this.config.baseUrl ?? 'http://localhost:11434').replace(/\/$/, '');
+        return (this.config.baseUrl ?? resolveOllamaBaseUrl()).replace(/\/$/, '');
     }
     /**
      * Format input to messages array for chat API

package/dist/shared/llm/providers/openrouter.js

@@ -61,7 +61,7 @@ export const OPENROUTER_PRICING = {
  * OpenRouter LLM provider implementation
  */
 export class OpenRouterProvider {
-    type = 'openai'; // Compatible with OpenAI interface
+    type = 'openrouter';
     name = 'OpenRouter';
     config;
     requestId = 0;
@@ -138,7 +138,7 @@ export class OpenRouterProvider {
     async generate(input, options) {
         const apiKey = this.getApiKey();
         if (!apiKey) {
-            throw createLLMError('OpenRouter API key not configured', 'API_KEY_MISSING', { provider: 'openai', retryable: false });
+            throw createLLMError('OpenRouter API key not configured', 'API_KEY_MISSING', { provider: 'openrouter', retryable: false });
         }
         const messages = this.formatMessages(input, options?.systemPrompt);
         const model = options?.model ?? this.config.model;
@@ -193,7 +193,7 @@ export class OpenRouterProvider {
             return {
                 content,
                 model: data.model,
-                provider: 'openai', // Report as openai for compatibility
+                provider: 'openrouter',
                 usage,
                 cost,
                 latencyMs,
@@ -206,7 +206,7 @@ export class OpenRouterProvider {
             if (error instanceof Error && 'code' in error) {
                 throw error; // Re-throw LLM errors
             }
-            throw createLLMError(error instanceof Error ? error.message : 'Request failed', 'NETWORK_ERROR', { provider: 'openai', model, retryable: true, cause: error });
+            throw createLLMError(error instanceof Error ? error.message : 'Request failed', 'NETWORK_ERROR', { provider: 'openrouter', model, retryable: true, cause: error });
         }
     }
     /**
@@ -216,7 +216,7 @@ export class OpenRouterProvider {
     async *generateStream(input, options) {
         const apiKey = this.getApiKey();
         if (!apiKey) {
-            throw createLLMError('OpenRouter API key not configured', 'API_KEY_MISSING', { provider: 'openai', retryable: false });
+            throw createLLMError('OpenRouter API key not configured', 'API_KEY_MISSING', { provider: 'openrouter', retryable: false });
         }
         const messages = this.formatMessages(input, options?.systemPrompt);
         const model = options?.model ?? this.config.model;
@@ -246,7 +246,7 @@ export class OpenRouterProvider {
             throw this.handleApiError(response.status, errorData, model);
         }
         if (!response.body) {
-            throw createLLMError('No response body for streaming', 'NETWORK_ERROR', { provider: 'openai', model, retryable: true });
+            throw createLLMError('No response body for streaming', 'NETWORK_ERROR', { provider: 'openrouter', model, retryable: true });
         }
         let fullContent = '';
         let finishReason = 'stop';
@@ -311,7 +311,7 @@ export class OpenRouterProvider {
         return {
             content: fullContent,
             model: responseModel,
-            provider: 'openai',
+            provider: 'openrouter',
             usage,
             cost,
             latencyMs,
@@ -327,7 +327,7 @@ export class OpenRouterProvider {
     async embed(text, options) {
         const apiKey = this.getApiKey();
         if (!apiKey) {
-            throw createLLMError('OpenRouter API key not configured', 'API_KEY_MISSING', { provider: 'openai', retryable: false });
+            throw createLLMError('OpenRouter API key not configured', 'API_KEY_MISSING', { provider: 'openrouter', retryable: false });
         }
         const model = options?.model ?? 'openai/text-embedding-3-small';
         const start = Date.now();
@@ -351,7 +351,7 @@ export class OpenRouterProvider {
             return {
                 embedding: data.data[0].embedding,
                 model: data.model,
-                provider: 'openai',
+                provider: 'openrouter',
                 tokenCount: data.usage.total_tokens,
                 latencyMs,
                 cached: false,
@@ -361,7 +361,7 @@ export class OpenRouterProvider {
             if (error instanceof Error && 'code' in error) {
                 throw error;
             }
-            throw createLLMError(error instanceof Error ? error.message : 'Embedding request failed', 'NETWORK_ERROR', { provider: 'openai', model, retryable: true, cause: error });
+            throw createLLMError(error instanceof Error ? error.message : 'Embedding request failed', 'NETWORK_ERROR', { provider: 'openrouter', model, retryable: true, cause: error });
         }
     }
     /**
@@ -378,7 +378,7 @@ export class OpenRouterProvider {
         return {
             completion: response.content,
             model: response.model,
-            provider: 'openai',
+            provider: 'openrouter',
             usage: response.usage,
             latencyMs: response.latencyMs,
             cached: response.cached,
@@ -525,19 +525,19 @@ export class OpenRouterProvider {
         switch (status) {
             case 401:
                 throw createLLMError(message, 'API_KEY_INVALID', {
-                    provider: 'openai',
+                    provider: 'openrouter',
                     model,
                     retryable: false,
                 });
             case 402:
                 throw createLLMError(message, 'COST_LIMIT_EXCEEDED', {
-                    provider: 'openai',
+                    provider: 'openrouter',
                     model,
                     retryable: false,
                 });
             case 429:
                 throw createLLMError(message, 'RATE_LIMITED', {
-                    provider: 'openai',
+                    provider: 'openrouter',
                     model,
                     retryable: true,
                     retryAfterMs: 60000,
@@ -545,19 +545,19 @@ export class OpenRouterProvider {
             case 400:
                 if (errorCode === 'context_length_exceeded' || errorType === 'invalid_request_error') {
                     throw createLLMError(message, 'CONTEXT_LENGTH_EXCEEDED', {
-                        provider: 'openai',
+                        provider: 'openrouter',
                         model,
                         retryable: false,
                     });
                 }
                 throw createLLMError(message, 'UNKNOWN', {
-                    provider: 'openai',
+                    provider: 'openrouter',
                     model,
                     retryable: false,
                 });
             case 404:
                 throw createLLMError(message, 'MODEL_NOT_FOUND', {
-                    provider: 'openai',
+                    provider: 'openrouter',
                     model,
                     retryable: false,
                 });
@@ -565,14 +565,14 @@ export class OpenRouterProvider {
             case 502:
             case 503:
                 throw createLLMError(message, 'PROVIDER_UNAVAILABLE', {
-                    provider: 'openai',
+                    provider: 'openrouter',
                     model,
                     retryable: true,
                     retryAfterMs: 5000,
                 });
             default:
                 throw createLLMError(message, 'UNKNOWN', {
-                    provider: 'openai',
+                    provider: 'openrouter',
                     model,
                     retryable: false,
                 });
@@ -594,7 +594,7 @@ export class OpenRouterProvider {
         catch (error) {
             if (error instanceof Error && error.name === 'AbortError') {
                 throw createLLMError('Request timed out', 'TIMEOUT', {
-                    provider: 'openai',
+                    provider: 'openrouter',
                     retryable: true,
                 });
             }

package/dist/shared/llm/router/hybrid-router.js

@@ -17,6 +17,7 @@ import { applyCyberPin, CYBER_PIN_CHAT_FALLBACK, shouldCyberPin, isOpus47, } fro
 import { createRouterMetricsCollector, createCostMetricsCollector, } from '../metrics';
 import { toError } from '../../error-utils.js';
 import { mapModelId, normalizeModelId, getModelMapping, } from '../model-mapping';
+import { modelSupportsTools } from '../model-registry';
 /**
  * Simple LRU-like decision cache
  */
@@ -472,8 +473,24 @@ export class HybridRouter {
                 executionOrder.push({ provider: entry.provider, model });
             }
         }
+        // Capability-aware guard: when the request needs tools, drop any candidate
+        // whose model is KNOWN to lack tool support (e.g. local llama3, codellama,
+        // mistral) so we don't silently strip tool definitions and return a
+        // degraded answer. Fail-open — if every candidate is tool-less (or
+        // unrecognized) we keep the original order rather than strand the request.
+        let order = executionOrder;
+        if (params.requiresTools) {
+            const toolCapable = executionOrder.filter((e) => modelSupportsTools(e.model));
+            if (toolCapable.length > 0 && toolCapable.length < executionOrder.length) {
+                const skipped = executionOrder
+                    .filter((e) => !modelSupportsTools(e.model))
+                    .map((e) => `${e.provider}:${e.model}`);
+                console.warn(`[llm-router] requiresTools: skipping tool-less model(s) ${skipped.join(', ')}`);
+                order = toolCapable;
+            }
+        }
         // Execute with fallback
-        for (const { provider: providerType, model } of executionOrder) {
+        for (const { provider: providerType, model } of order) {
             if (attempts >= fallbackBehavior.maxAttempts)
                 break;
             const provider = this.providerManager.getProvider(providerType);

package/dist/validation/conservation-guard.d.ts

@@ -0,0 +1,39 @@
+/**
+ * Conservation Guard (Phoenix essay 14 "UI Is a Conservation Layer", ADR-114)
+ *
+ * AQE's human/agent-facing surfaces — CLI commands, output schemas, MCP tools,
+ * the dashboard — are its conservation layer: users and CI pipelines build muscle
+ * memory and scripts around them. Internals may regenerate freely; these surfaces
+ * must change slowly and ADDITIVELY. This module diffs a captured baseline against
+ * the current surface and flags breaking removals/renames that are not in the
+ * deprecation registry.
+ *
+ * Pure functions only — extraction + I/O live in scripts/conservation-guard.ts.
+ */
+export interface SurfaceDiff {
+    surface: string;
+    /** Entries present now but not in the baseline — additive, always allowed. */
+    added: string[];
+    /** Entries removed from the baseline and NOT in the deprecation registry — breaking. */
+    removedBreaking: string[];
+    /** Entries removed but listed as deprecated — an allowed, deliberate removal. */
+    removedDeprecated: string[];
+    /** True when there are no breaking removals. */
+    clean: boolean;
+}
+/**
+ * Diff one surface. Additive changes pass; a removal/rename is breaking unless the
+ * removed entry appears in `deprecated` (the deprecation window has been honored).
+ */
+export declare function diffSurface(surface: string, baseline: string[], current: string[], deprecated?: string[]): SurfaceDiff;
+export interface ConservationReport {
+    diffs: SurfaceDiff[];
+    clean: boolean;
+    /** Total breaking removals across all surfaces. */
+    breakingCount: number;
+}
+/** Aggregate per-surface diffs into an overall verdict. */
+export declare function aggregate(diffs: SurfaceDiff[]): ConservationReport;
+/** Human-readable report (used by the CLI guard and CI logs). */
+export declare function formatReport(report: ConservationReport): string;
+//# sourceMappingURL=conservation-guard.d.ts.map

package/dist/validation/conservation-guard.js

@@ -0,0 +1,54 @@
+/**
+ * Conservation Guard (Phoenix essay 14 "UI Is a Conservation Layer", ADR-114)
+ *
+ * AQE's human/agent-facing surfaces — CLI commands, output schemas, MCP tools,
+ * the dashboard — are its conservation layer: users and CI pipelines build muscle
+ * memory and scripts around them. Internals may regenerate freely; these surfaces
+ * must change slowly and ADDITIVELY. This module diffs a captured baseline against
+ * the current surface and flags breaking removals/renames that are not in the
+ * deprecation registry.
+ *
+ * Pure functions only — extraction + I/O live in scripts/conservation-guard.ts.
+ */
+/**
+ * Diff one surface. Additive changes pass; a removal/rename is breaking unless the
+ * removed entry appears in `deprecated` (the deprecation window has been honored).
+ */
+export function diffSurface(surface, baseline, current, deprecated = []) {
+    const cur = new Set(current);
+    const dep = new Set(deprecated);
+    const added = current.filter((e) => !baseline.includes(e)).sort();
+    const removed = baseline.filter((e) => !cur.has(e));
+    const removedBreaking = removed.filter((e) => !dep.has(e)).sort();
+    const removedDeprecated = removed.filter((e) => dep.has(e)).sort();
+    return {
+        surface,
+        added,
+        removedBreaking,
+        removedDeprecated,
+        clean: removedBreaking.length === 0,
+    };
+}
+/** Aggregate per-surface diffs into an overall verdict. */
+export function aggregate(diffs) {
+    const breakingCount = diffs.reduce((n, d) => n + d.removedBreaking.length, 0);
+    return { diffs, clean: breakingCount === 0, breakingCount };
+}
+/** Human-readable report (used by the CLI guard and CI logs). */
+export function formatReport(report) {
+    const lines = [];
+    for (const d of report.diffs) {
+        const status = d.clean ? '✓' : '✗ BREAKING';
+        lines.push(`${status} ${d.surface}: +${d.added.length} added, -${d.removedBreaking.length} breaking, -${d.removedDeprecated.length} deprecated`);
+        for (const r of d.removedBreaking)
+            lines.push(`    REMOVED (breaking): ${r}`);
+        for (const r of d.removedDeprecated)
+            lines.push(`    removed (deprecated, ok): ${r}`);
+    }
+    lines.push('');
+    lines.push(report.clean
+        ? '✓ conservation guard: no breaking surface changes'
+        : `✗ conservation guard: ${report.breakingCount} breaking surface change(s) — add to the deprecation registry (with a deprecation window) or restore them. To intentionally rebaseline: --update.`);
+    return lines.join('\n');
+}
+//# sourceMappingURL=conservation-guard.js.map

package/dist/validation/oracle-eval.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Agentic QE v3 - Oracle Evaluator for skill evals (Oracle-Evals plan, Phase 0/1)
+ *
+ * Turns a skill eval from a keyword grep into a real ORACLE: it RUNS a
+ * generated test against a reference implementation (must pass) and against
+ * first-order operator mutants of that implementation (each surviving mutant
+ * is a bug the test failed to catch). The mutation kill rate is the eval's
+ * pass signal — a test that asserts nothing scores 0 and fails.
+ *
+ * Reuses the proven qe-arena primitives (ADR-104): the deterministic operator
+ * mutator and the real `node --test` fixture runner. Nothing is simulated.
+ */
+export interface OracleSpec {
+    /** Base name (no extension) for the module under test, e.g. "inRange". */
+    moduleName: string;
+    /** ESM source of the correct implementation, e.g. `export function inRange(...) {...}`. */
+    referenceImpl: string;
+    /** Generated test source. Imports the impl as `../src/<moduleName>.mjs`. */
+    generatedTest: string;
+    /** Cap on mutants evaluated (deterministic prefix of the position-sorted list). */
+    maxMutants?: number;
+    /** Minimum mutation kill rate required to pass. Default 0.5. */
+    threshold?: number;
+    /** Per-`node --test` timeout in ms. Default 30s (evals run many subprocesses). */
+    timeoutMs?: number;
+}
+export interface OracleResult {
+    /** Did the generated test pass against the unmutated reference implementation? */
+    baselinePassed: boolean;
+    mutantsTotal: number;
+    mutantsKilled: number;
+    /** killed / total, rounded to 4dp (0 when there are no mutants). */
+    mutationScore: number;
+    /** Mutant ids the test failed to catch (bugs it would let through). */
+    survivedMutantIds: string[];
+    /** baselinePassed AND mutationScore >= threshold AND at least one mutant existed. */
+    passed: boolean;
+    /** Human-readable explanation of the verdict. */
+    reason: string;
+}
+/**
+ * Evaluate a generated test as an oracle against a reference implementation.
+ * Real `node --test` runs; deterministic mutant enumeration.
+ */
+export declare function evaluateOracle(spec: OracleSpec): OracleResult;
+//# sourceMappingURL=oracle-eval.d.ts.map

package/dist/validation/oracle-eval.js

@@ -0,0 +1,108 @@
+/**
+ * Agentic QE v3 - Oracle Evaluator for skill evals (Oracle-Evals plan, Phase 0/1)
+ *
+ * Turns a skill eval from a keyword grep into a real ORACLE: it RUNS a
+ * generated test against a reference implementation (must pass) and against
+ * first-order operator mutants of that implementation (each surviving mutant
+ * is a bug the test failed to catch). The mutation kill rate is the eval's
+ * pass signal — a test that asserts nothing scores 0 and fails.
+ *
+ * Reuses the proven qe-arena primitives (ADR-104): the deterministic operator
+ * mutator and the real `node --test` fixture runner. Nothing is simulated.
+ */
+import * as fs from 'fs';
+import * as path from 'path';
+import { enumerateMutants, applyMutant } from '../arena/mutator.js';
+import { runNodeTest, makeTmpRoot, cleanupTmpRoot } from '../arena/runner.js';
+/** Write a throwaway { src/<m>.mjs, tests/<m>.test.mjs } workspace and return its dir. */
+function materialize(tmpRoot, moduleName, impl, test) {
+    const dir = fs.mkdtempSync(path.join(tmpRoot, 'oracle-ws-'));
+    fs.mkdirSync(path.join(dir, 'src'), { recursive: true });
+    fs.mkdirSync(path.join(dir, 'tests'), { recursive: true });
+    fs.writeFileSync(path.join(dir, 'src', `${moduleName}.mjs`), impl);
+    fs.writeFileSync(path.join(dir, 'tests', `${moduleName}.test.mjs`), test);
+    return dir;
+}
+/**
+ * Evaluate a generated test as an oracle against a reference implementation.
+ * Real `node --test` runs; deterministic mutant enumeration.
+ */
+export function evaluateOracle(spec) {
+    const threshold = spec.threshold ?? 0.5;
+    const timeoutMs = spec.timeoutMs ?? 30_000;
+    const tmpRoot = makeTmpRoot();
+    try {
+        // 0. Sanity: reject output that is not a runnable, asserting test. An empty
+        // or assertion-less file passes `node --test` (exit 0) but proves nothing —
+        // catch it here so the verdict is honest rather than relying on a 0 kill rate.
+        if (!/assert\s*[.(]|expect\s*\(|\.(toBe|toEqual|toThrow|toMatch)\b/.test(spec.generatedTest)) {
+            return {
+                baselinePassed: false,
+                mutantsTotal: 0,
+                mutantsKilled: 0,
+                mutationScore: 0,
+                survivedMutantIds: [],
+                passed: false,
+                reason: 'generated output contains no assertions — not a runnable test',
+            };
+        }
+        // 1. Baseline: the test must pass against the correct implementation.
+        const baselineDir = materialize(tmpRoot, spec.moduleName, spec.referenceImpl, spec.generatedTest);
+        const baseline = runNodeTest(baselineDir, { timeoutMs });
+        if (!baseline.ok) {
+            return {
+                baselinePassed: false,
+                mutantsTotal: 0,
+                mutantsKilled: 0,
+                mutationScore: 0,
+                survivedMutantIds: [],
+                passed: false,
+                reason: 'generated test does not pass against the reference implementation',
+            };
+        }
+        // 2. Mutants: each one the test fails to fail-on is a missed bug.
+        let mutants = enumerateMutants(spec.referenceImpl, spec.moduleName);
+        if (spec.maxMutants !== undefined && mutants.length > spec.maxMutants) {
+            mutants = mutants.slice(0, spec.maxMutants);
+        }
+        if (mutants.length === 0) {
+            return {
+                baselinePassed: true,
+                mutantsTotal: 0,
+                mutantsKilled: 0,
+                mutationScore: 0,
+                survivedMutantIds: [],
+                passed: false,
+                reason: 'no operator mutants could be generated for the reference implementation (cannot prove the test catches anything)',
+            };
+        }
+        let killed = 0;
+        const survived = [];
+        for (const mutant of mutants) {
+            const dir = materialize(tmpRoot, spec.moduleName, applyMutant(spec.referenceImpl, mutant), spec.generatedTest);
+            const run = runNodeTest(dir, { timeoutMs });
+            if (!run.ok)
+                killed++;
+            else
+                survived.push(mutant.id);
+        }
+        const total = mutants.length;
+        const score = Math.round((killed / total) * 10000) / 10000;
+        const passed = score >= threshold;
+        return {
+            baselinePassed: true,
+            mutantsTotal: total,
+            mutantsKilled: killed,
+            mutationScore: score,
+            survivedMutantIds: survived,
+            passed,
+            reason: passed
+                ? `killed ${killed}/${total} mutants (score ${(score * 100).toFixed(0)}%)`
+                : `mutation score ${(score * 100).toFixed(0)}% < threshold ${(threshold * 100).toFixed(0)}% — ${survived.length} mutant(s) survived`,
+        };
+    }
+    finally {
+        cleanupTmpRoot(tmpRoot);
+    }
+}
+//# sourceMappingURL=oracle-eval.js.map