@sanity/ailf 2.0.0 → 2.0.2

package/dist/pipeline/compiler/__tests__/tool-loop-openai.test.js

@@ -0,0 +1,509 @@
+/**
+ * tool-loop-openai.test.ts — Tests for the OpenAI MCP tool loop.
+ *
+ * Tests both API variants (Chat Completions and Responses) with mocked
+ * fetch to verify tool calling, error handling, token tracking, and
+ * round exhaustion.
+ *
+ * Run: npx tsx --test src/pipeline/compiler/__tests__/tool-loop-openai.test.ts
+ */
+import assert from "node:assert/strict";
+import { afterEach, beforeEach, describe, it } from "node:test";
+import { runOpenAIToolLoop } from "../mode-handlers/mcp-tool-provider/tool-loop-openai.js";
+// ---------------------------------------------------------------------------
+// Test fixtures
+// ---------------------------------------------------------------------------
+const TEST_TOOLS = [
+    {
+        name: "query_documents",
+        description: "Query Sanity documents using GROQ",
+        inputSchema: {
+            type: "object",
+            properties: {
+                query: { type: "string" },
+                projectId: { type: "string" },
+                dataset: { type: "string" },
+            },
+            required: ["query"],
+        },
+    },
+    {
+        name: "get_schema",
+        description: "Get the Sanity schema for a project",
+        inputSchema: {
+            type: "object",
+            properties: { projectId: { type: "string" } },
+        },
+    },
+];
+function makeCallTool(results = {}) {
+    return async (name, _args) => {
+        if (results[name]) {
+            return { content: results[name] };
+        }
+        return { content: `Result from ${name}` };
+    };
+}
+function makeThrowingCallTool(errorMessage) {
+    return async () => {
+        throw new Error(errorMessage);
+    };
+}
+function baseConfig(overrides) {
+    return {
+        prompt: "Query all documents from project abc123",
+        tools: TEST_TOOLS,
+        callTool: makeCallTool(),
+        maxToolRounds: 5,
+        model: "gpt-5.2",
+        temperature: 0.2,
+        maxTokens: 4096,
+        apiKey: "test-api-key",
+        ...overrides,
+    };
+}
+// ---------------------------------------------------------------------------
+// Chat Completions API response builders
+// ---------------------------------------------------------------------------
+function chatResponse(opts) {
+    const message = {
+        role: "assistant",
+        content: opts.content ?? null,
+    };
+    if (opts.toolCalls) {
+        message.tool_calls = opts.toolCalls.map((tc) => ({
+            id: tc.id,
+            type: "function",
+            function: { name: tc.name, arguments: tc.arguments },
+        }));
+    }
+    return {
+        choices: [
+            {
+                message,
+                finish_reason: opts.finishReason ?? (opts.toolCalls ? "tool_calls" : "stop"),
+            },
+        ],
+        usage: {
+            prompt_tokens: opts.promptTokens ?? 100,
+            completion_tokens: opts.completionTokens ?? 50,
+            total_tokens: (opts.promptTokens ?? 100) + (opts.completionTokens ?? 50),
+        },
+    };
+}
+function chatErrorResponse(message) {
+    return { error: { message } };
+}
+// ---------------------------------------------------------------------------
+// Responses API response builders
+// ---------------------------------------------------------------------------
+function responsesResponse(opts) {
+    const output = [];
+    if (opts.functionCalls) {
+        for (const fc of opts.functionCalls) {
+            output.push({
+                type: "function_call",
+                id: `fc_${fc.callId}`,
+                call_id: fc.callId,
+                name: fc.name,
+                arguments: fc.arguments,
+            });
+        }
+    }
+    if (opts.text !== undefined) {
+        output.push({
+            type: "message",
+            id: "msg_001",
+            role: "assistant",
+            content: [{ type: "output_text", text: opts.text }],
+        });
+    }
+    return {
+        id: opts.id ?? "resp_001",
+        status: "completed",
+        output,
+        usage: {
+            input_tokens: opts.inputTokens ?? 100,
+            output_tokens: opts.outputTokens ?? 50,
+            total_tokens: (opts.inputTokens ?? 100) + (opts.outputTokens ?? 50),
+        },
+    };
+}
+function responsesErrorResponse(message) {
+    return { error: { message } };
+}
+// ---------------------------------------------------------------------------
+// Mock fetch helper
+// ---------------------------------------------------------------------------
+let originalFetch;
+let fetchCalls;
+/** Mock fetch that returns JSON responses. All responses have ok: true (API-level errors are in the body). */
+function mockFetch(responses) {
+    let callIndex = 0;
+    fetchCalls = [];
+    globalThis.fetch = (async (url, init) => {
+        const body = init?.body ? JSON.parse(String(init.body)) : undefined;
+        fetchCalls.push({ url: url.toString(), body });
+        const responseData = responses[callIndex] ?? responses[responses.length - 1];
+        callIndex++;
+        return {
+            json: async () => responseData,
+            ok: true,
+            status: 200,
+        };
+    });
+}
+/** Mock fetch that returns an HTTP error (non-JSON body) */
+function mockHttpError(status, body) {
+    fetchCalls = [];
+    globalThis.fetch = (async (url, init) => {
+        const reqBody = init?.body ? JSON.parse(String(init.body)) : undefined;
+        fetchCalls.push({ url: url.toString(), body: reqBody });
+        return new Response(body, { status, statusText: "Error" });
+    });
+}
+// ---------------------------------------------------------------------------
+// Tests: Chat Completions API
+// ---------------------------------------------------------------------------
+describe("runOpenAIToolLoop — Chat Completions API", () => {
+    beforeEach(() => {
+        originalFetch = globalThis.fetch;
+    });
+    afterEach(() => {
+        globalThis.fetch = originalFetch;
+    });
+    it("returns text when model answers without tool calls", async () => {
+        mockFetch([
+            chatResponse({
+                content: "There are 42 documents.",
+                finishReason: "stop",
+            }),
+        ]);
+        const result = await runOpenAIToolLoop(baseConfig({ apiVariant: "chat" }));
+        assert.equal(result.output, "There are 42 documents.");
+        assert.equal(result.toolCallLog.length, 0);
+        assert.equal(result.toolRounds, 0);
+        assert.equal(result.exhaustedRounds, undefined);
+    });
+    it("executes a single tool call and returns final answer", async () => {
+        mockFetch([
+            // Round 0: model calls query_documents
+            chatResponse({
+                toolCalls: [
+                    {
+                        id: "call_1",
+                        name: "query_documents",
+                        arguments: '{"query":"*[_type==\\"post\\"]"}',
+                    },
+                ],
+            }),
+            // Round 1: model synthesizes answer
+            chatResponse({ content: "Found 10 posts.", finishReason: "stop" }),
+        ]);
+        const result = await runOpenAIToolLoop(baseConfig({ apiVariant: "chat" }));
+        assert.equal(result.output, "Found 10 posts.");
+        assert.equal(result.toolCallLog.length, 1);
+        assert.equal(result.toolCallLog[0].name, "query_documents");
+        assert.equal(result.toolRounds, 1);
+    });
+    it("executes multi-turn tool calls", async () => {
+        mockFetch([
+            // Round 0: model calls get_schema
+            chatResponse({
+                toolCalls: [
+                    {
+                        id: "call_1",
+                        name: "get_schema",
+                        arguments: '{"projectId":"abc123"}',
+                    },
+                ],
+            }),
+            // Round 1: model calls query_documents
+            chatResponse({
+                toolCalls: [
+                    { id: "call_2", name: "query_documents", arguments: '{"query":"*"}' },
+                ],
+            }),
+            // Round 2: model synthesizes
+            chatResponse({
+                content: "Schema has 5 types, 100 documents.",
+                finishReason: "stop",
+            }),
+        ]);
+        const result = await runOpenAIToolLoop(baseConfig({ apiVariant: "chat" }));
+        assert.equal(result.toolCallLog.length, 2);
+        assert.equal(result.toolCallLog[0].name, "get_schema");
+        assert.equal(result.toolCallLog[1].name, "query_documents");
+        assert.equal(result.toolRounds, 2);
+    });
+    it("captures tool execution errors in toolCallLog", async () => {
+        mockFetch([
+            chatResponse({
+                toolCalls: [
+                    { id: "call_1", name: "query_documents", arguments: '{"query":"*"}' },
+                ],
+            }),
+            chatResponse({
+                content: "Tool failed, but I'll answer.",
+                finishReason: "stop",
+            }),
+        ]);
+        const result = await runOpenAIToolLoop(baseConfig({
+            apiVariant: "chat",
+            callTool: makeThrowingCallTool("Connection refused"),
+        }));
+        assert.equal(result.toolCallLog.length, 1);
+        assert.equal(result.toolCallLog[0].output, "Error: Connection refused");
+        assert.equal(result.output, "Tool failed, but I'll answer.");
+    });
+    it("handles exhausted rounds", async () => {
+        // Model keeps calling tools for all 3 rounds (maxToolRounds=2 means rounds 0,1,2)
+        mockFetch([
+            chatResponse({
+                toolCalls: [{ id: "call_1", name: "get_schema", arguments: "{}" }],
+            }),
+            chatResponse({
+                toolCalls: [{ id: "call_2", name: "get_schema", arguments: "{}" }],
+            }),
+            // Last round: tool_choice "none" forces text, but model returns nothing useful
+            chatResponse({ content: null, finishReason: "stop" }),
+        ]);
+        const result = await runOpenAIToolLoop(baseConfig({ apiVariant: "chat", maxToolRounds: 2 }));
+        // Round 2 (the last) gets tool_choice: "none", model stops
+        assert.equal(result.toolCallLog.length, 2);
+        assert.equal(result.toolRounds, 2);
+        // The model returned content: null with finishReason: stop on the last round
+        assert.equal(result.output, "");
+    });
+    it("throws on API-level error in JSON body", async () => {
+        mockFetch([chatErrorResponse("Rate limit exceeded")]);
+        await assert.rejects(() => runOpenAIToolLoop(baseConfig({ apiVariant: "chat" })), { message: "Rate limit exceeded" });
+    });
+    it("throws on HTTP error with non-JSON body", async () => {
+        mockHttpError(502, "<html>Bad Gateway</html>");
+        await assert.rejects(() => runOpenAIToolLoop(baseConfig({ apiVariant: "chat" })), (err) => err.message.includes("HTTP 502") && err.message.includes("Bad Gateway"));
+    });
+    it("accumulates token usage across rounds", async () => {
+        mockFetch([
+            chatResponse({
+                toolCalls: [{ id: "call_1", name: "get_schema", arguments: "{}" }],
+                promptTokens: 200,
+                completionTokens: 50,
+            }),
+            chatResponse({
+                content: "Done.",
+                finishReason: "stop",
+                promptTokens: 300,
+                completionTokens: 80,
+            }),
+        ]);
+        const result = await runOpenAIToolLoop(baseConfig({ apiVariant: "chat" }));
+        assert.equal(result.tokenUsage.prompt, 500); // 200 + 300
+        assert.equal(result.tokenUsage.completion, 130); // 50 + 80
+    });
+    it("sends max_completion_tokens for GPT-5.x models", async () => {
+        mockFetch([chatResponse({ content: "Answer.", finishReason: "stop" })]);
+        await runOpenAIToolLoop(baseConfig({ apiVariant: "chat", model: "gpt-5.2" }));
+        assert.equal(fetchCalls.length, 1);
+        const body = fetchCalls[0].body;
+        assert.equal(body.max_completion_tokens, 4096);
+        assert.equal(body.max_tokens, undefined);
+    });
+    it("sends max_tokens for older models", async () => {
+        mockFetch([chatResponse({ content: "Answer.", finishReason: "stop" })]);
+        await runOpenAIToolLoop(baseConfig({ apiVariant: "chat", model: "gpt-4o" }));
+        assert.equal(fetchCalls.length, 1);
+        const body = fetchCalls[0].body;
+        assert.equal(body.max_tokens, 4096);
+        assert.equal(body.max_completion_tokens, undefined);
+    });
+    it("sends tool_choice 'none' on last round", async () => {
+        mockFetch([
+            chatResponse({
+                toolCalls: [{ id: "call_1", name: "get_schema", arguments: "{}" }],
+            }),
+            chatResponse({ content: "Final.", finishReason: "stop" }),
+        ]);
+        await runOpenAIToolLoop(baseConfig({ apiVariant: "chat", maxToolRounds: 1 }));
+        // Round 0: auto, Round 1 (last): none
+        assert.equal(fetchCalls.length, 2);
+        assert.equal(fetchCalls[0].body.tool_choice, "auto");
+        assert.equal(fetchCalls[1].body.tool_choice, "none");
+    });
+});
+// ---------------------------------------------------------------------------
+// Tests: Responses API
+// ---------------------------------------------------------------------------
+describe("runOpenAIToolLoop — Responses API", () => {
+    beforeEach(() => {
+        originalFetch = globalThis.fetch;
+    });
+    afterEach(() => {
+        globalThis.fetch = originalFetch;
+    });
+    it("returns text when model answers without tool calls", async () => {
+        mockFetch([responsesResponse({ text: "42 documents found." })]);
+        const result = await runOpenAIToolLoop(baseConfig({ apiVariant: "responses" }));
+        assert.equal(result.output, "42 documents found.");
+        assert.equal(result.toolCallLog.length, 0);
+        assert.equal(result.toolRounds, 0);
+    });
+    it("executes a single tool call and returns final answer", async () => {
+        mockFetch([
+            // Round 0: model calls query_documents
+            responsesResponse({
+                id: "resp_001",
+                functionCalls: [
+                    {
+                        callId: "call_1",
+                        name: "query_documents",
+                        arguments: '{"query":"*"}',
+                    },
+                ],
+            }),
+            // Round 1: model synthesizes
+            responsesResponse({ id: "resp_002", text: "Found 10 posts." }),
+        ]);
+        const result = await runOpenAIToolLoop(baseConfig({ apiVariant: "responses" }));
+        assert.equal(result.output, "Found 10 posts.");
+        assert.equal(result.toolCallLog.length, 1);
+        assert.equal(result.toolCallLog[0].name, "query_documents");
+        assert.equal(result.toolRounds, 1);
+    });
+    it("chains via previous_response_id", async () => {
+        mockFetch([
+            responsesResponse({
+                id: "resp_001",
+                functionCalls: [
+                    { callId: "call_1", name: "get_schema", arguments: "{}" },
+                ],
+            }),
+            responsesResponse({ id: "resp_002", text: "Schema loaded." }),
+        ]);
+        await runOpenAIToolLoop(baseConfig({ apiVariant: "responses" }));
+        // Second request should chain via previous_response_id
+        assert.equal(fetchCalls.length, 2);
+        const secondBody = fetchCalls[1].body;
+        assert.equal(secondBody.previous_response_id, "resp_001");
+    });
+    it("captures tool execution errors in toolCallLog", async () => {
+        mockFetch([
+            responsesResponse({
+                functionCalls: [
+                    {
+                        callId: "call_1",
+                        name: "query_documents",
+                        arguments: '{"query":"*"}',
+                    },
+                ],
+            }),
+            responsesResponse({ text: "Handled the error." }),
+        ]);
+        const result = await runOpenAIToolLoop(baseConfig({
+            apiVariant: "responses",
+            callTool: makeThrowingCallTool("Server unavailable"),
+        }));
+        assert.equal(result.toolCallLog.length, 1);
+        assert.equal(result.toolCallLog[0].output, "Error: Server unavailable");
+        assert.equal(result.output, "Handled the error.");
+    });
+    it("handles exhausted rounds", async () => {
+        mockFetch([
+            responsesResponse({
+                id: "resp_001",
+                functionCalls: [
+                    { callId: "call_1", name: "get_schema", arguments: "{}" },
+                ],
+            }),
+            responsesResponse({
+                id: "resp_002",
+                functionCalls: [
+                    { callId: "call_2", name: "get_schema", arguments: "{}" },
+                ],
+            }),
+            // Last round with tool_choice: "none" — model must return text
+            // But if it doesn't produce function calls, we get empty output
+            responsesResponse({ id: "resp_003", text: "" }),
+        ]);
+        const result = await runOpenAIToolLoop(baseConfig({ apiVariant: "responses", maxToolRounds: 2 }));
+        assert.equal(result.toolCallLog.length, 2);
+        assert.equal(result.toolRounds, 2);
+    });
+    it("throws on API-level error in JSON body", async () => {
+        mockFetch([responsesErrorResponse("Invalid model")]);
+        await assert.rejects(() => runOpenAIToolLoop(baseConfig({ apiVariant: "responses" })), { message: "Invalid model" });
+    });
+    it("throws on HTTP error with non-JSON body", async () => {
+        mockHttpError(503, "Service Unavailable");
+        await assert.rejects(() => runOpenAIToolLoop(baseConfig({ apiVariant: "responses" })), (err) => err.message.includes("HTTP 503") &&
+            err.message.includes("Service Unavailable"));
+    });
+    it("accumulates token usage across rounds", async () => {
+        mockFetch([
+            responsesResponse({
+                functionCalls: [
+                    { callId: "call_1", name: "get_schema", arguments: "{}" },
+                ],
+                inputTokens: 150,
+                outputTokens: 40,
+            }),
+            responsesResponse({
+                text: "Done.",
+                inputTokens: 250,
+                outputTokens: 60,
+            }),
+        ]);
+        const result = await runOpenAIToolLoop(baseConfig({ apiVariant: "responses" }));
+        assert.equal(result.tokenUsage.prompt, 400); // 150 + 250
+        assert.equal(result.tokenUsage.completion, 100); // 40 + 60
+    });
+    it("sends max_output_tokens (not max_tokens)", async () => {
+        mockFetch([responsesResponse({ text: "Answer." })]);
+        await runOpenAIToolLoop(baseConfig({ apiVariant: "responses" }));
+        const body = fetchCalls[0].body;
+        assert.equal(body.max_output_tokens, 4096);
+        assert.equal(body.max_tokens, undefined);
+        assert.equal(body.max_completion_tokens, undefined);
+    });
+    it("uses correct endpoint URL", async () => {
+        mockFetch([responsesResponse({ text: "Hi." })]);
+        await runOpenAIToolLoop(baseConfig({ apiVariant: "responses" }));
+        assert.ok(fetchCalls[0].url.includes("/v1/responses"));
+    });
+    it("passes reasoning_effort and omits temperature", async () => {
+        mockFetch([responsesResponse({ text: "Thought carefully." })]);
+        await runOpenAIToolLoop(baseConfig({
+            apiVariant: "responses",
+            providerConfig: { reasoning_effort: "medium" },
+        }));
+        const body = fetchCalls[0].body;
+        assert.deepEqual(body.reasoning, { effort: "medium" });
+        assert.equal(body.temperature, undefined);
+    });
+    it("includes temperature when reasoning_effort is not set", async () => {
+        mockFetch([responsesResponse({ text: "Answer." })]);
+        await runOpenAIToolLoop(baseConfig({ apiVariant: "responses" }));
+        const body = fetchCalls[0].body;
+        assert.equal(body.temperature, 0.2);
+        assert.equal(body.reasoning, undefined);
+    });
+});
+// ---------------------------------------------------------------------------
+// Tests: Default routing (no apiVariant)
+// ---------------------------------------------------------------------------
+describe("runOpenAIToolLoop — default routing", () => {
+    beforeEach(() => {
+        originalFetch = globalThis.fetch;
+    });
+    afterEach(() => {
+        globalThis.fetch = originalFetch;
+    });
+    it("defaults to Chat Completions when apiVariant is undefined", async () => {
+        mockFetch([
+            chatResponse({ content: "Default path.", finishReason: "stop" }),
+        ]);
+        const result = await runOpenAIToolLoop(baseConfig());
+        assert.equal(result.output, "Default path.");
+        assert.ok(fetchCalls[0].url.includes("/v1/chat/completions"));
+    });
+});

package/dist/pipeline/compiler/assertion-mapper.d.ts

@@ -12,7 +12,7 @@
  * and normalizes weight fields.
  *
  * @see docs/design-docs/architecture-overhaul/scoring-rubrics-assertions.md
- * @see docs/exec-plans/architecture-overhaul/phase-2-config-compiler.md
+ * @see docs/archive/exec-plans/architecture-overhaul/phase-2-config-compiler.md
  */
 import type { GeneralizedAssertionDefinition } from "../../_vendor/ailf-core/index.d.ts";
 import type { EvalMode } from "../../_vendor/ailf-shared/index.d.ts";

package/dist/pipeline/compiler/assertion-mapper.js

@@ -12,7 +12,7 @@
  * and normalizes weight fields.
  *
  * @see docs/design-docs/architecture-overhaul/scoring-rubrics-assertions.md
- * @see docs/exec-plans/architecture-overhaul/phase-2-config-compiler.md
+ * @see docs/archive/exec-plans/architecture-overhaul/phase-2-config-compiler.md
  */
 // ---------------------------------------------------------------------------
 // Known assertion types and their mode compatibility

package/dist/pipeline/compiler/compiler-to-yaml.d.ts

@@ -1,15 +1,10 @@
 /**
  * compiler-to-yaml.ts — Serialize compiled Promptfoo config to YAML files.
  *
- * This is the bridge between the new compiler pipeline (in-memory
- * CompiledPromptfooConfig) and the existing RunEvalStep which reads
+ * This is the bridge between the compiler pipeline (in-memory
+ * CompiledPromptfooConfig) and the RunEvalStep which reads
  * YAML config files from disk.
  *
- * The output YAML files are identical in structure to what the legacy
- * generate-configs.ts produces, so RunEvalStep, CalculateScoresStep,
- * and all downstream steps work without modification.
- *
- * @see packages/eval/src/pipeline/generate-configs.ts — legacy path
  * @see packages/eval/src/orchestration/steps/run-eval-step.ts — consumer
  */
 import type { Logger, ModeCompileResult } from "../../_vendor/ailf-core/index.d.ts";

package/dist/pipeline/compiler/compiler-to-yaml.js

@@ -1,15 +1,10 @@
 /**
  * compiler-to-yaml.ts — Serialize compiled Promptfoo config to YAML files.
  *
- * This is the bridge between the new compiler pipeline (in-memory
- * CompiledPromptfooConfig) and the existing RunEvalStep which reads
+ * This is the bridge between the compiler pipeline (in-memory
+ * CompiledPromptfooConfig) and the RunEvalStep which reads
  * YAML config files from disk.
  *
- * The output YAML files are identical in structure to what the legacy
- * generate-configs.ts produces, so RunEvalStep, CalculateScoresStep,
- * and all downstream steps work without modification.
- *
- * @see packages/eval/src/pipeline/generate-configs.ts — legacy path
  * @see packages/eval/src/orchestration/steps/run-eval-step.ts — consumer
  */
 import { existsSync, mkdirSync, writeFileSync } from "fs";

package/dist/pipeline/compiler/config-loader.d.ts

@@ -54,3 +54,17 @@ export declare class ConfigNotFoundError extends Error {
     readonly searchDir: string;
     constructor(configName: string, searchDir: string);
 }
+/**
+ * Resolve a package subdirectory, preferring the vendored copy in dist/
+ * when @sanity/ailf-core isn't directly resolvable (i.e., installed outside
+ * the monorepo via npx or npm).
+ *
+ * The build step (bundle-workspace-deps.ts) copies config/ and tasks/ into
+ * dist/ with @sanity/ailf-core imports rewritten to the _vendor/ path.
+ * This function detects that situation and returns the dist/ path instead.
+ *
+ * @param rootDir - Package root directory (packages/eval)
+ * @param subdir - Subdirectory to resolve (e.g., "config", "tasks/literacy")
+ * @returns The resolved subdirectory path — either source or dist/ vendored
+ */
+export declare function resolveVendoredSubdir(rootDir: string, subdir: string): string;

package/dist/pipeline/compiler/config-loader.js

@@ -36,7 +36,8 @@ import { resolve } from "path";
  */
 export function loadConfigFile(name, rootDir, options) {
     const subdir = options?.subdir ?? "config";
-    const basePath = resolve(rootDir, subdir, name);
+    const baseDir = resolveVendoredSubdir(rootDir, subdir);
+    const basePath = resolve(baseDir, name);
     // Priority chain: .ts > .js > .yaml > .yml > .json
     const candidates = [
         { ext: ".ts", format: "ts" },
@@ -60,7 +61,7 @@ export function loadConfigFile(name, rootDir, options) {
         }
     }
     // Always throw ConfigNotFoundError so tryLoadConfigFile can catch it
-    throw new ConfigNotFoundError(name, resolve(rootDir, subdir));
+    throw new ConfigNotFoundError(name, baseDir);
 }
 /**
  * Try to load a config file, returning null if not found.
@@ -88,6 +89,45 @@ export class ConfigNotFoundError extends Error {
     }
 }
 // ---------------------------------------------------------------------------
+// Vendored config detection
+// ---------------------------------------------------------------------------
+/**
+ * Resolve a package subdirectory, preferring the vendored copy in dist/
+ * when @sanity/ailf-core isn't directly resolvable (i.e., installed outside
+ * the monorepo via npx or npm).
+ *
+ * The build step (bundle-workspace-deps.ts) copies config/ and tasks/ into
+ * dist/ with @sanity/ailf-core imports rewritten to the _vendor/ path.
+ * This function detects that situation and returns the dist/ path instead.
+ *
+ * @param rootDir - Package root directory (packages/eval)
+ * @param subdir - Subdirectory to resolve (e.g., "config", "tasks/literacy")
+ * @returns The resolved subdirectory path — either source or dist/ vendored
+ */
+export function resolveVendoredSubdir(rootDir, subdir) {
+    if (shouldUseVendoredConfigs(rootDir)) {
+        const vendoredDir = resolve(rootDir, "dist", subdir);
+        if (existsSync(vendoredDir))
+            return resolve(rootDir, "dist", subdir);
+    }
+    return resolve(rootDir, subdir);
+}
+let _vendoredCache;
+function shouldUseVendoredConfigs(rootDir) {
+    if (_vendoredCache !== undefined)
+        return _vendoredCache;
+    // Check if @sanity/ailf-core resolves (i.e., we're in the monorepo)
+    try {
+        const require = createRequire(resolve(rootDir, "package.json"));
+        require.resolve("@sanity/ailf-core");
+        _vendoredCache = false;
+    }
+    catch {
+        _vendoredCache = true;
+    }
+    return _vendoredCache;
+}
+// ---------------------------------------------------------------------------
 // Format-specific loaders
 // ---------------------------------------------------------------------------
 function loadTsFile(filePath, format) {

package/dist/pipeline/compiler/fixture-resolver.d.ts

@@ -15,7 +15,7 @@
  * as described in the fixtures-artifacts design doc.
  *
  * @see docs/design-docs/architecture-overhaul/fixtures-artifacts.md
- * @see docs/exec-plans/architecture-overhaul/phase-2-config-compiler.md
+ * @see docs/archive/exec-plans/architecture-overhaul/phase-2-config-compiler.md
  */
 import type { GeneralizedTaskDefinition, ResolvedFixture, VariableEnvelope } from "../../_vendor/ailf-core/index.d.ts";
 /** Options for fixture resolution */

package/dist/pipeline/compiler/fixture-resolver.js

@@ -15,7 +15,7 @@
  * as described in the fixtures-artifacts design doc.
  *
  * @see docs/design-docs/architecture-overhaul/fixtures-artifacts.md
- * @see docs/exec-plans/architecture-overhaul/phase-2-config-compiler.md
+ * @see docs/archive/exec-plans/architecture-overhaul/phase-2-config-compiler.md
  */
 import { existsSync, readFileSync } from "fs";
 import { resolve } from "path";

package/dist/pipeline/compiler/ignore-fields.d.ts

@@ -17,7 +17,7 @@
  * ]
  * ```
  *
- * @see docs/exec-plans/architecture-overhaul/phase-4-agent-harness.md
+ * @see docs/archive/exec-plans/architecture-overhaul/phase-4-agent-harness.md
  */
 /**
  * Strip specified fields from an object using dot-notation paths.