@sanity/ailf 2.0.2 → 2.1.0

package/dist/orchestration/steps/fetch-docs-shell.js

@@ -1,30 +0,0 @@
-/**
- * Shell delegation for the fetch-docs step.
- *
- * Isolates the execSync call so it can be replaced when the pipeline
- * fully migrates to the DocFetcher port.
- */
-import { execSync } from "child_process";
-/**
- * Run `pnpm fetch-docs` via shell.
- *
- * Returns a result object instead of throwing so the step can
- * handle the failure uniformly.
- */
-export function runFetchDocsShell(rootDir, source) {
-    try {
-        const sourceArg = source ? ` --source ${source}` : "";
-        execSync(`pnpm fetch-docs${sourceArg}`, {
-            cwd: rootDir,
-            env: process.env,
-            stdio: "inherit",
-        });
-        return { ok: true };
-    }
-    catch (err) {
-        return {
-            ok: false,
-            error: err instanceof Error ? err.message : String(err),
-        };
-    }
-}

package/dist/pipeline/compiler/__tests__/task-bridge.test.d.ts

@@ -1,9 +0,0 @@
-/**
- * task-bridge.test.ts — Unit tests for the TaskDefinition ↔ LiteracyTaskDefinition bridge.
- *
- * Covers round-trip fidelity, edge cases (missing optionals, all optionals),
- * assertion type mapping, and all four CanonicalDocRef / GeneralizedDocRef variants.
- *
- * Run: npx tsx --test src/pipeline/compiler/__tests__/task-bridge.test.ts
- */
-export {};

package/dist/pipeline/compiler/__tests__/task-bridge.test.js

@@ -1,339 +0,0 @@
-/**
- * task-bridge.test.ts — Unit tests for the TaskDefinition ↔ LiteracyTaskDefinition bridge.
- *
- * Covers round-trip fidelity, edge cases (missing optionals, all optionals),
- * assertion type mapping, and all four CanonicalDocRef / GeneralizedDocRef variants.
- *
- * Run: npx tsx --test src/pipeline/compiler/__tests__/task-bridge.test.ts
- */
-import assert from "node:assert/strict";
-import { describe, it } from "node:test";
-import { toGeneralized, toLiteracyTask } from "../task-bridge.js";
-// ---------------------------------------------------------------------------
-// Fixtures
-// ---------------------------------------------------------------------------
-/** Minimal old-style task — only required fields, no optionals */
-const minimalOldTask = {
-    id: "groq-filter-basic",
-    description: "Filter documents with GROQ",
-    featureArea: "groq",
-    taskPrompt: "Write a GROQ query that filters by _type",
-    canonicalDocs: [],
-    referenceSolution: "solutions/groq-filter-basic.ts",
-    docCoverage: false,
-    assertions: [],
-};
-/** Fully-populated old-style task — every optional filled */
-const fullOldTask = {
-    id: "mutations-create-advanced",
-    description: "Create documents with mutations API",
-    featureArea: "mutations",
-    taskPrompt: "Use the mutations API to create a document with references",
-    canonicalDocs: [
-        { slug: "mutations-overview", reason: "Primary mutations guide" },
-        { path: "/docs/mutations/create", reason: "Create-specific docs" },
-        {
-            id: "doc-123",
-            reason: "Imported draft",
-            slug: "draft-slug",
-            path: "/docs/draft",
-        },
-        { perspective: "release-v3", reason: "V3 release content" },
-    ],
-    referenceSolution: "solutions/mutations-create-advanced.ts",
-    docCoverage: true,
-    assertions: [
-        {
-            type: "llm-rubric",
-            template: "code-quality",
-            criteria: ["correct", "idiomatic"],
-            weight: 2,
-        },
-        { type: "contains", value: "createIfNotExists" },
-        { type: "javascript", value: "output.includes('mutation')", weight: 1 },
-    ],
-    baseline: { enabled: true, rubric: "full" },
-    tags: ["mutations", "advanced", "references"],
-    status: "active",
-    extraVars: { customHint: "Use createIfNotExists", maxRetries: 3 },
-};
-/** Minimal new-style literacy task — only required/mode fields */
-const minimalNewTask = {
-    mode: "literacy",
-    id: "studio-config-basic",
-    title: "Configure a Sanity Studio",
-};
-/** Fully-populated new-style literacy task (only fields that round-trip through old type) */
-const fullNewTask = {
-    mode: "literacy",
-    id: "groq-projection-advanced",
-    title: "Advanced GROQ projections",
-    area: "groq",
-    tags: ["groq", "projections"],
-    status: "active",
-    assertions: [
-        {
-            type: "llm-rubric",
-            template: "completeness",
-            criteria: ["covers edge cases"],
-            weight: 3,
-        },
-        { type: "contains", value: "coalesce" },
-    ],
-    prompt: {
-        text: "Write a GROQ query using projections with coalesce",
-        vars: { difficulty: "advanced", topic: "projections" },
-    },
-    context: {
-        docs: [
-            { slug: "groq-projections", reason: "Projection docs" },
-            { path: "/docs/groq/projections", reason: "Path-based ref" },
-            { id: "groq-doc-456", reason: "By ID" },
-            { perspective: "release-groq-v2", reason: "GROQ v2 release" },
-        ],
-    },
-    referenceSolution: "solutions/groq-projection-advanced.ts",
-    docCoverage: true,
-    baseline: { enabled: false, rubric: "abbreviated" },
-};
-// ---------------------------------------------------------------------------
-// toGeneralized — old TaskDefinition → LiteracyTaskDefinition
-// ---------------------------------------------------------------------------
-describe("toGeneralized", () => {
-    it("converts a minimal old task to a LiteracyTaskDefinition", () => {
-        const result = toGeneralized(minimalOldTask);
-        assert.equal(result.mode, "literacy");
-        assert.equal(result.id, "groq-filter-basic");
-        assert.equal(result.title, "Filter documents with GROQ");
-        assert.equal(result.area, "groq");
-        assert.equal(result.referenceSolution, "solutions/groq-filter-basic.ts");
-        assert.equal(result.docCoverage, false);
-        assert.deepEqual(result.assertions, []);
-        assert.deepEqual(result.context?.docs, []);
-        assert.equal(result.prompt?.text, "Write a GROQ query that filters by _type");
-    });
-    it("converts a fully-populated old task preserving all fields", () => {
-        const result = toGeneralized(fullOldTask);
-        assert.equal(result.mode, "literacy");
-        assert.equal(result.id, "mutations-create-advanced");
-        assert.equal(result.title, "Create documents with mutations API");
-        assert.equal(result.area, "mutations");
-        assert.equal(result.referenceSolution, "solutions/mutations-create-advanced.ts");
-        assert.equal(result.docCoverage, true);
-        assert.deepEqual(result.baseline, { enabled: true, rubric: "full" });
-        assert.deepEqual(result.tags, ["mutations", "advanced", "references"]);
-        assert.equal(result.status, "active");
-        assert.deepEqual(result.prompt?.vars, {
-            customHint: "Use createIfNotExists",
-            maxRetries: 3,
-        });
-        assert.equal(result.prompt?.text, "Use the mutations API to create a document with references");
-    });
-    it("does not set optional fields when absent in old task", () => {
-        const result = toGeneralized(minimalOldTask);
-        assert.equal(result.baseline, undefined);
-        assert.equal(result.tags, undefined);
-        assert.equal(result.status, undefined);
-        assert.equal(result.prompt?.vars, undefined);
-    });
-});
-// ---------------------------------------------------------------------------
-// toLiteracyTask — LiteracyTaskDefinition → old TaskDefinition
-// ---------------------------------------------------------------------------
-describe("toLiteracyTask", () => {
-    it("converts a minimal new task to a TaskDefinition", () => {
-        const result = toLiteracyTask(minimalNewTask);
-        assert.equal(result.id, "studio-config-basic");
-        assert.equal(result.description, "Configure a Sanity Studio");
-        assert.equal(result.featureArea, "");
-        assert.equal(result.taskPrompt, "");
-        assert.deepEqual(result.canonicalDocs, []);
-        assert.equal(result.referenceSolution, "");
-        assert.equal(result.docCoverage, false);
-        assert.deepEqual(result.assertions, []);
-    });
-    it("converts a fully-populated new task preserving all mappable fields", () => {
-        const result = toLiteracyTask(fullNewTask);
-        assert.equal(result.id, "groq-projection-advanced");
-        assert.equal(result.description, "Advanced GROQ projections");
-        assert.equal(result.featureArea, "groq");
-        assert.equal(result.taskPrompt, "Write a GROQ query using projections with coalesce");
-        assert.equal(result.referenceSolution, "solutions/groq-projection-advanced.ts");
-        assert.equal(result.docCoverage, true);
-        assert.deepEqual(result.baseline, { enabled: false, rubric: "abbreviated" });
-        assert.deepEqual(result.tags, ["groq", "projections"]);
-        assert.equal(result.status, "active");
-        assert.deepEqual(result.extraVars, {
-            difficulty: "advanced",
-            topic: "projections",
-        });
-    });
-    it("uses prompt.template as fallback when prompt.text is absent", () => {
-        const task = {
-            mode: "literacy",
-            id: "template-task",
-            title: "Template-based task",
-            prompt: { template: "my-named-template" },
-        };
-        const result = toLiteracyTask(task);
-        assert.equal(result.taskPrompt, "my-named-template");
-    });
-    it("does not set optional fields when absent in new task", () => {
-        const result = toLiteracyTask(minimalNewTask);
-        assert.equal(result.baseline, undefined);
-        assert.equal(result.tags, undefined);
-        assert.equal(result.status, undefined);
-        assert.equal(result.extraVars, undefined);
-    });
-});
-// ---------------------------------------------------------------------------
-// Round-trip: old → new → old (must be lossless)
-// ---------------------------------------------------------------------------
-describe("round-trip: toLiteracyTask(toGeneralized(oldTask))", () => {
-    it("preserves all fields of a minimal old task", () => {
-        const roundTripped = toLiteracyTask(toGeneralized(minimalOldTask));
-        assert.deepEqual(roundTripped, minimalOldTask);
-    });
-    it("preserves all fields of a fully-populated old task", () => {
-        const roundTripped = toLiteracyTask(toGeneralized(fullOldTask));
-        assert.deepEqual(roundTripped, fullOldTask);
-    });
-});
-// ---------------------------------------------------------------------------
-// Round-trip: new → old → new (lossless for mappable fields)
-// ---------------------------------------------------------------------------
-describe("round-trip: toGeneralized(toLiteracyTask(newTask))", () => {
-    it("preserves all fields of a minimal new task", () => {
-        const roundTripped = toGeneralized(toLiteracyTask(minimalNewTask));
-        assert.equal(roundTripped.mode, "literacy");
-        assert.equal(roundTripped.id, minimalNewTask.id);
-        assert.equal(roundTripped.title, minimalNewTask.title);
-    });
-    it("preserves all mappable fields of a fully-populated new task", () => {
-        const roundTripped = toGeneralized(toLiteracyTask(fullNewTask));
-        assert.equal(roundTripped.mode, "literacy");
-        assert.equal(roundTripped.id, fullNewTask.id);
-        assert.equal(roundTripped.title, fullNewTask.title);
-        assert.equal(roundTripped.area, fullNewTask.area);
-        assert.deepEqual(roundTripped.tags, fullNewTask.tags);
-        assert.equal(roundTripped.status, fullNewTask.status);
-        assert.deepEqual(roundTripped.assertions, fullNewTask.assertions);
-        assert.equal(roundTripped.prompt?.text, fullNewTask.prompt?.text);
-        assert.deepEqual(roundTripped.prompt?.vars, fullNewTask.prompt?.vars);
-        assert.deepEqual(roundTripped.context?.docs, fullNewTask.context?.docs);
-        assert.equal(roundTripped.referenceSolution, fullNewTask.referenceSolution);
-        assert.equal(roundTripped.docCoverage, fullNewTask.docCoverage);
-        assert.deepEqual(roundTripped.baseline, fullNewTask.baseline);
-    });
-});
-// ---------------------------------------------------------------------------
-// CanonicalDocRef ↔ GeneralizedDocRef mapping (all 4 variants)
-// ---------------------------------------------------------------------------
-describe("doc ref mapping", () => {
-    const slugRef = {
-        slug: "my-article",
-        reason: "testing slug",
-    };
-    const pathRef = {
-        path: "/docs/my-article",
-        reason: "testing path",
-    };
-    const idRef = {
-        id: "abc-123",
-        reason: "testing id",
-        slug: "annotated-slug",
-        path: "/docs/annotated",
-    };
-    const perspectiveRef = {
-        perspective: "release-v4",
-        reason: "testing perspective",
-    };
-    it("preserves slug ref through round-trip", () => {
-        const task = { ...minimalOldTask, canonicalDocs: [slugRef] };
-        const roundTripped = toLiteracyTask(toGeneralized(task));
-        assert.deepEqual(roundTripped.canonicalDocs, [slugRef]);
-    });
-    it("preserves path ref through round-trip", () => {
-        const task = { ...minimalOldTask, canonicalDocs: [pathRef] };
-        const roundTripped = toLiteracyTask(toGeneralized(task));
-        assert.deepEqual(roundTripped.canonicalDocs, [pathRef]);
-    });
-    it("preserves id ref (with optional slug/path annotations) through round-trip", () => {
-        const task = { ...minimalOldTask, canonicalDocs: [idRef] };
-        const roundTripped = toLiteracyTask(toGeneralized(task));
-        assert.deepEqual(roundTripped.canonicalDocs, [idRef]);
-    });
-    it("preserves perspective ref through round-trip", () => {
-        const task = {
-            ...minimalOldTask,
-            canonicalDocs: [perspectiveRef],
-        };
-        const roundTripped = toLiteracyTask(toGeneralized(task));
-        assert.deepEqual(roundTripped.canonicalDocs, [perspectiveRef]);
-    });
-    it("preserves all 4 ref variants together through round-trip", () => {
-        const allRefs = [slugRef, pathRef, idRef, perspectiveRef];
-        const task = { ...minimalOldTask, canonicalDocs: allRefs };
-        const roundTripped = toLiteracyTask(toGeneralized(task));
-        assert.deepEqual(roundTripped.canonicalDocs, allRefs);
-    });
-});
-// ---------------------------------------------------------------------------
-// Assertion type mapping
-// ---------------------------------------------------------------------------
-describe("assertion type mapping", () => {
-    const templatedAssertion = {
-        type: "llm-rubric",
-        template: "code-quality",
-        criteria: ["correct", "idiomatic", "secure"],
-        weight: 2,
-    };
-    const valueAssertion = {
-        type: "contains",
-        value: "createDocument",
-    };
-    const jsAssertion = {
-        type: "javascript",
-        value: "output.includes('done')",
-        weight: 1,
-    };
-    it("preserves templated assertions through old→new→old round-trip", () => {
-        const task = {
-            ...minimalOldTask,
-            assertions: [templatedAssertion],
-        };
-        const roundTripped = toLiteracyTask(toGeneralized(task));
-        assert.deepEqual(roundTripped.assertions, [templatedAssertion]);
-    });
-    it("preserves value assertions through old→new→old round-trip", () => {
-        const task = {
-            ...minimalOldTask,
-            assertions: [valueAssertion],
-        };
-        const roundTripped = toLiteracyTask(toGeneralized(task));
-        assert.deepEqual(roundTripped.assertions, [valueAssertion]);
-    });
-    it("preserves mixed assertion types through old→new→old round-trip", () => {
-        const mixed = [templatedAssertion, valueAssertion, jsAssertion];
-        const task = { ...minimalOldTask, assertions: mixed };
-        const roundTripped = toLiteracyTask(toGeneralized(task));
-        assert.deepEqual(roundTripped.assertions, mixed);
-    });
-    it("preserves assertions through new→old→new round-trip", () => {
-        const genAssertions = [
-            {
-                type: "llm-rubric",
-                template: "completeness",
-                criteria: ["thorough"],
-                weight: 1,
-            },
-            { type: "regex", value: "^import.*sanity" },
-        ];
-        const task = {
-            ...minimalNewTask,
-            assertions: genAssertions,
-        };
-        const roundTripped = toGeneralized(toLiteracyTask(task));
-        assert.deepEqual(roundTripped.assertions, genAssertions);
-    });
-});

package/dist/pipeline/compiler/mode-handlers/agent-harness-handler.d.ts

@@ -1,70 +0,0 @@
-/**
- * AgentHarnessModeHandler — compilation rules for `agent-harness` mode.
- *
- * Maps agent harness task definitions to Promptfoo configuration with:
- * - Claude Agent SDK / OpenAI Codex SDK providers
- * - Tool permission configuration (preset/allowed/disallowed)
- * - Sandbox setup/teardown via Promptfoo extensions
- * - Fixture provisioning into sandbox working directory
- *
- * @see docs/exec-plans/architecture-overhaul/phase-4-agent-harness.md
- * @see packages/core/src/types/generalized-task.ts — AgentHarnessTaskDefinition
- */
-import type { AgentHarnessTaskDefinition, ModeHandler, PromptTemplate } from "../../../_vendor/ailf-core/index.d.ts";
-import type { PromptfooPrompt, PromptfooProvider, PromptfooTestCase } from "../promptfoo-compiler.js";
-import type { SandboxType } from "../sandbox/sandbox-strategy.js";
-export declare const AGENT_HARNESS_PROMPT_TEMPLATES: Record<string, PromptTemplate>;
-/** Options for compiling an agent harness task */
-export interface AgentHarnessCompileOptions {
-    /** Grader provider for LLM-graded assertions */
-    graderProvider?: string;
-    /** Root directory for fixture resolution */
-    rootDir?: string;
-}
-/** Result of compiling a single agent harness task */
-export interface AgentHarnessCompileResult {
-    /** Promptfoo provider config */
-    providers: PromptfooProvider[];
-    /** Compiled test cases */
-    tests: PromptfooTestCase[];
-    /** Prompts for evaluation */
-    prompts: PromptfooPrompt[];
-    /** Promptfoo extensions for sandbox lifecycle */
-    extensions: PromptfooExtension[];
-    /** Sandbox configuration metadata */
-    sandboxConfig: SandboxConfigMeta;
-    /** Warnings generated during compilation */
-    warnings: string[];
-}
-/** Promptfoo extension hook */
-export interface PromptfooExtension {
-    type: "afterEach" | "beforeEach";
-    /** JavaScript code or module path for the hook */
-    code: string;
-}
-/** Metadata about sandbox configuration for this task */
-export interface SandboxConfigMeta {
-    type: SandboxType;
-    image?: string;
-    fixtures: string[];
-    limits?: {
-        cpus?: number;
-        memoryBytes?: number;
-        networkAccess?: boolean;
-    };
-}
-/** Validation errors for agent harness task definitions */
-export interface AgentHarnessValidationError {
-    field: string;
-    message: string;
-}
-/**
- * Validate that an agent harness task definition has all required fields.
- */
-export declare function validateAgentHarnessTask(task: AgentHarnessTaskDefinition): AgentHarnessValidationError[];
-/**
- * Compile an agent harness task definition into Promptfoo configuration.
- */
-export declare function compileAgentHarnessTask(task: AgentHarnessTaskDefinition, options?: AgentHarnessCompileOptions): AgentHarnessCompileResult;
-/** ModeHandler-conformant export for the agent-harness evaluation mode. */
-export declare const handler: ModeHandler;