@sanity/ailf 4.5.0 → 5.0.0

package/dist/adapters/llm/fake-llm-client.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Fake LLMClient for tests.
+ *
+ * Returns canned responses in insertion order and records each call so
+ * consumers' tests can assert on prompts, models, and telemetry context.
+ *
+ * Exported alongside the real adapters so consuming-feature tests can
+ * stub the LLM cleanly without a network round-trip.
+ */
+import type { LLMClient, LLMCompleteArgs, LLMCompleteStructuredArgs, LLMCompletion, LLMStructuredCompletion } from "../../_vendor/ailf-core/index.d.ts";
+export interface FakeCallRecord {
+    kind: "complete" | "completeStructured";
+    model: string;
+    prompt: string;
+    context?: {
+        feature: string;
+        runId?: string;
+        cardId?: string;
+    };
+}
+export interface FakeCompletionResponse {
+    text: string;
+    promptTokens?: number;
+    completionTokens?: number;
+    cost?: number;
+}
+export interface FakeStructuredResponse {
+    /**
+     * Value returned to the consumer. Run through the consumer-supplied Zod
+     * schema in `completeStructured` so consumers exercise their parse-don't-
+     * validate path even with the fake; supply a value that matches the
+     * schema's shape (or supply a deliberately malformed one to test failure).
+     */
+    value: unknown;
+    promptTokens?: number;
+    completionTokens?: number;
+    cost?: number;
+}
+export declare class FakeLLMClient implements LLMClient {
+    readonly calls: FakeCallRecord[];
+    private readonly completeQueue;
+    private readonly structuredQueue;
+    constructor(args?: {
+        completeResponses?: FakeCompletionResponse[];
+        structuredResponses?: FakeStructuredResponse[];
+    });
+    complete(args: LLMCompleteArgs): Promise<LLMCompletion>;
+    completeStructured<T>(args: LLMCompleteStructuredArgs<T>): Promise<LLMStructuredCompletion<T>>;
+}

package/dist/adapters/llm/fake-llm-client.js

@@ -0,0 +1,63 @@
+/**
+ * Fake LLMClient for tests.
+ *
+ * Returns canned responses in insertion order and records each call so
+ * consumers' tests can assert on prompts, models, and telemetry context.
+ *
+ * Exported alongside the real adapters so consuming-feature tests can
+ * stub the LLM cleanly without a network round-trip.
+ */
+export class FakeLLMClient {
+    calls = [];
+    completeQueue;
+    structuredQueue;
+    constructor(args = {}) {
+        this.completeQueue = [...(args.completeResponses ?? [])];
+        this.structuredQueue = [...(args.structuredResponses ?? [])];
+    }
+    async complete(args) {
+        this.calls.push({
+            kind: "complete",
+            model: args.model,
+            prompt: args.prompt,
+            ...(args.context ? { context: args.context } : {}),
+        });
+        const next = this.completeQueue.shift();
+        if (!next) {
+            throw new Error("FakeLLMClient: no more queued complete responses (call exceeded queue)");
+        }
+        return {
+            text: next.text,
+            usage: {
+                promptTokens: next.promptTokens ?? 0,
+                completionTokens: next.completionTokens ?? 0,
+            },
+            cost: next.cost ?? 0,
+            model: args.model,
+        };
+    }
+    async completeStructured(args) {
+        this.calls.push({
+            kind: "completeStructured",
+            model: args.model,
+            prompt: args.prompt,
+            ...(args.context ? { context: args.context } : {}),
+        });
+        const next = this.structuredQueue.shift();
+        if (!next) {
+            throw new Error("FakeLLMClient: no more queued structured responses (call exceeded queue)");
+        }
+        // Run through the consumer-supplied schema so consumers exercise the
+        // parse-don't-validate path even with the fake.
+        const value = args.schema.parse(next.value);
+        return {
+            value,
+            usage: {
+                promptTokens: next.promptTokens ?? 0,
+                completionTokens: next.completionTokens ?? 0,
+            },
+            cost: next.cost ?? 0,
+            model: args.model,
+        };
+    }
+}

package/dist/adapters/llm/index.d.ts

@@ -0,0 +1,9 @@
+export { AnthropicLLMClient } from "./anthropic-llm-client.js";
+export type { AnthropicLLMClientOptions } from "./anthropic-llm-client.js";
+export { FakeLLMClient } from "./fake-llm-client.js";
+export type { FakeCallRecord, FakeCompletionResponse, FakeStructuredResponse, } from "./fake-llm-client.js";
+export { OpenAILLMClient } from "./openai-llm-client.js";
+export type { OpenAILLMClientOptions } from "./openai-llm-client.js";
+export type { ModelPricing } from "./pricing.js";
+export { DEFAULT_RETRY_POLICY, LLMHttpError, isRetryableStatus, parseRetryAfterSeconds, runWithRetry, } from "./retry.js";
+export type { RetryPolicy } from "./retry.js";

package/dist/adapters/llm/index.js

@@ -0,0 +1,4 @@
+export { AnthropicLLMClient } from "./anthropic-llm-client.js";
+export { FakeLLMClient } from "./fake-llm-client.js";
+export { OpenAILLMClient } from "./openai-llm-client.js";
+export { DEFAULT_RETRY_POLICY, LLMHttpError, isRetryableStatus, parseRetryAfterSeconds, runWithRetry, } from "./retry.js";

package/dist/adapters/llm/openai-llm-client.d.ts

@@ -0,0 +1,44 @@
+/**
+ * OpenAI adapter for the LLMClient port.
+ *
+ * Uses fetch() against the Chat Completions API — same transport pattern as
+ * the existing grader (`packages/eval/src/pipeline/grader-api.ts`). No SDK
+ * dependency. Centralizes retry, rate-limit handling, cost accounting, and
+ * per-call telemetry tagging via `context.feature`.
+ *
+ * Per `.claude/rules/eval-pipeline.md` and `.claude/rules/typescript.md`:
+ * the adapter never reads `process.env`. The composition root maps env vars
+ * to typed constructor args.
+ */
+import { type LLMClient, type LLMCompleteArgs, type LLMCompleteStructuredArgs, type LLMCompletion, type LLMStructuredCompletion, type Logger } from "../../_vendor/ailf-core/index.d.ts";
+import type { ModelPricing } from "./pricing.js";
+import { type RetryPolicy } from "./retry.js";
+export interface OpenAILLMClientOptions {
+    /** OpenAI API key. */
+    apiKey: string;
+    /** Optional override of the chat-completions endpoint. */
+    baseUrl?: string;
+    /** Pricing table keyed by canonical model id (without the `openai:` prefix). */
+    pricing?: Record<string, ModelPricing>;
+    retryPolicy?: Partial<RetryPolicy>;
+    logger?: Logger;
+    /** Sleep injectable for tests. */
+    sleep?: (ms: number) => Promise<void>;
+    /** Random source for jitter — injectable for tests. */
+    rng?: () => number;
+}
+export declare class OpenAILLMClient implements LLMClient {
+    private readonly apiKey;
+    private readonly baseUrl;
+    private readonly pricing;
+    private readonly retryPolicy;
+    private readonly logger?;
+    private readonly sleep?;
+    private readonly rng?;
+    constructor(options: OpenAILLMClientOptions);
+    complete(args: LLMCompleteArgs): Promise<LLMCompletion>;
+    completeStructured<T>(args: LLMCompleteStructuredArgs<T>): Promise<LLMStructuredCompletion<T>>;
+    private callApi;
+    private computeCost;
+    private logTelemetry;
+}

package/dist/adapters/llm/openai-llm-client.js

@@ -0,0 +1,168 @@
+/**
+ * OpenAI adapter for the LLMClient port.
+ *
+ * Uses fetch() against the Chat Completions API — same transport pattern as
+ * the existing grader (`packages/eval/src/pipeline/grader-api.ts`). No SDK
+ * dependency. Centralizes retry, rate-limit handling, cost accounting, and
+ * per-call telemetry tagging via `context.feature`.
+ *
+ * Per `.claude/rules/eval-pipeline.md` and `.claude/rules/typescript.md`:
+ * the adapter never reads `process.env`. The composition root maps env vars
+ * to typed constructor args.
+ */
+import { OpenAIChatResponseSchema, splitModelId, } from "../../_vendor/ailf-core/index.js";
+import { DEFAULT_RETRY_POLICY, parseRetryAfterSeconds, runWithRetry, } from "./retry.js";
+const DEFAULT_BASE_URL = "https://api.openai.com/v1/chat/completions";
+/**
+ * Conservative defaults for the models in `packages/eval/config/models.ts`.
+ * Update when models are added or vendor pricing changes. Unknown models
+ * fall through to `cost: 0` with a warning logged.
+ *
+ * Pricing reference: https://openai.com/api/pricing
+ */
+const DEFAULT_PRICING = {
+    "gpt-5.2": { inputPer1k: 0.005, outputPer1k: 0.015 },
+    "gpt-5": { inputPer1k: 0.005, outputPer1k: 0.015 },
+    "gpt-5.4": { inputPer1k: 0.005, outputPer1k: 0.015 },
+};
+export class OpenAILLMClient {
+    apiKey;
+    baseUrl;
+    pricing;
+    retryPolicy;
+    logger;
+    sleep;
+    rng;
+    constructor(options) {
+        this.apiKey = options.apiKey;
+        this.baseUrl = options.baseUrl ?? DEFAULT_BASE_URL;
+        this.pricing = { ...DEFAULT_PRICING, ...(options.pricing ?? {}) };
+        this.retryPolicy = {
+            ...DEFAULT_RETRY_POLICY,
+            ...(options.retryPolicy ?? {}),
+        };
+        if (options.logger)
+            this.logger = options.logger;
+        if (options.sleep)
+            this.sleep = options.sleep;
+        if (options.rng)
+            this.rng = options.rng;
+    }
+    async complete(args) {
+        const { modelName } = splitModelId(args.model);
+        const body = buildBody(modelName, args.prompt, {
+            temperature: args.temperature,
+            maxTokens: args.maxTokens,
+            stop: args.stop,
+        });
+        const data = await this.callApi(body);
+        const text = data.choices?.[0]?.message?.content;
+        if (text == null || text === "") {
+            throw new Error(`OpenAI returned empty completion for model ${args.model}`);
+        }
+        const usage = extractUsage(data.usage);
+        const cost = this.computeCost(modelName, usage);
+        this.logTelemetry(args.context, args.model, usage, cost);
+        return { text, usage, cost, model: args.model };
+    }
+    async completeStructured(args) {
+        const { modelName } = splitModelId(args.model);
+        const body = buildBody(modelName, args.prompt, {
+            temperature: args.temperature,
+            maxTokens: args.maxTokens,
+            responseFormat: { type: "json_object" },
+        });
+        const data = await this.callApi(body);
+        const raw = data.choices?.[0]?.message?.content;
+        if (raw == null || raw === "") {
+            throw new Error(`OpenAI returned empty structured completion for model ${args.model}`);
+        }
+        let parsed;
+        try {
+            parsed = JSON.parse(raw);
+        }
+        catch (err) {
+            throw new Error(`OpenAI structured completion returned invalid JSON for model ${args.model}: ${err instanceof Error ? err.message : String(err)}`, { cause: err });
+        }
+        const value = args.schema.parse(parsed);
+        const usage = extractUsage(data.usage);
+        const cost = this.computeCost(modelName, usage);
+        this.logTelemetry(args.context, args.model, usage, cost);
+        return { value, usage, cost, model: args.model };
+    }
+    async callApi(body) {
+        return runWithRetry({
+            policy: this.retryPolicy,
+            ...(this.sleep ? { sleep: this.sleep } : {}),
+            ...(this.rng ? { rng: this.rng } : {}),
+            attempt: async () => {
+                const response = await fetch(this.baseUrl, {
+                    method: "POST",
+                    headers: {
+                        Authorization: `Bearer ${this.apiKey}`,
+                        "Content-Type": "application/json",
+                    },
+                    body: JSON.stringify(body),
+                });
+                if (!response.ok) {
+                    const text = await response.text();
+                    const retryAfter = parseRetryAfterSeconds(response.headers.get("retry-after"));
+                    return {
+                        ok: false,
+                        status: response.status,
+                        body: text,
+                        ...(retryAfter !== undefined
+                            ? { retryAfterSeconds: retryAfter }
+                            : {}),
+                    };
+                }
+                const json = await response.json();
+                const data = OpenAIChatResponseSchema.parse(json);
+                if (data.error?.message) {
+                    throw new Error(`OpenAI API error: ${data.error.message}`);
+                }
+                return { ok: true, value: data };
+            },
+        });
+    }
+    computeCost(modelName, usage) {
+        const price = this.pricing[modelName];
+        if (!price) {
+            this.logger?.warn(`OpenAI cost unknown for model "${modelName}" — recording cost=0. Add it to OpenAILLMClientOptions.pricing.`);
+            return 0;
+        }
+        return ((usage.promptTokens / 1000) * price.inputPer1k +
+            (usage.completionTokens / 1000) * price.outputPer1k);
+    }
+    logTelemetry(context, model, usage, cost) {
+        if (!this.logger)
+            return;
+        const tag = context ? ` feature=${context.feature}` : "";
+        const runTag = context?.runId ? ` runId=${context.runId}` : "";
+        const cardTag = context?.cardId ? ` cardId=${context.cardId}` : "";
+        this.logger.debug(`LLM call (openai)${tag}${runTag}${cardTag} model=${model} ` +
+            `prompt_tokens=${usage.promptTokens} completion_tokens=${usage.completionTokens} ` +
+            `cost_usd=${cost.toFixed(6)}`);
+    }
+}
+function buildBody(modelName, prompt, opts) {
+    const body = {
+        model: modelName,
+        messages: [{ role: "user", content: prompt }],
+    };
+    if (opts.temperature !== undefined)
+        body.temperature = opts.temperature;
+    if (opts.maxTokens !== undefined)
+        body.max_tokens = opts.maxTokens;
+    if (opts.stop && opts.stop.length > 0)
+        body.stop = opts.stop;
+    if (opts.responseFormat)
+        body.response_format = opts.responseFormat;
+    return body;
+}
+function extractUsage(usage) {
+    return {
+        promptTokens: usage?.prompt_tokens ?? 0,
+        completionTokens: usage?.completion_tokens ?? 0,
+    };
+}

package/dist/adapters/llm/pricing.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Per-model pricing types shared by LLM adapters.
+ *
+ * Hard-coded vendor pricing drifts; treat the in-adapter defaults as a
+ * sensible starting point and override via constructor options when the
+ * vendor changes their rate card.
+ */
+/** USD per 1K tokens. */
+export interface ModelPricing {
+    inputPer1k: number;
+    outputPer1k: number;
+}

package/dist/adapters/llm/pricing.js

@@ -0,0 +1,8 @@
+/**
+ * Per-model pricing types shared by LLM adapters.
+ *
+ * Hard-coded vendor pricing drifts; treat the in-adapter defaults as a
+ * sensible starting point and override via constructor options when the
+ * vendor changes their rate card.
+ */
+export {};

package/dist/adapters/llm/retry.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Shared retry helper for LLMClient adapters.
+ *
+ * Bounded exponential backoff with optional `Retry-After` honoring and
+ * symmetric jitter. Treats 429 / 5xx as retryable and any other HTTP error
+ * as terminal.
+ *
+ * Errors carry the full response body on the instance for callers that need
+ * to inspect it; the message is intentionally short and body-free so it's
+ * safe to include in user-facing logs and stack traces.
+ */
+export interface RetryPolicy {
+    /** Total attempts including the initial call. Default 3. */
+    maxAttempts: number;
+    /** Initial backoff in ms. Default 500. */
+    baseDelayMs: number;
+    /** Multiplier per attempt. Default 2. */
+    backoffFactor: number;
+    /** Cap on a single delay in ms. Default 10_000. */
+    maxDelayMs: number;
+    /**
+     * Symmetric jitter as a fraction of the computed delay, in `[0, 1)`. The
+     * actual delay is `delay * (1 + (rng() - 0.5) * 2 * jitter)`. Default 0.3.
+     * Set to 0 to disable.
+     */
+    jitter: number;
+}
+export declare const DEFAULT_RETRY_POLICY: RetryPolicy;
+export declare class LLMHttpError extends Error {
+    readonly status: number;
+    readonly attempts: number;
+    /** Full upstream response body (kept on the instance, NOT in `message`). */
+    readonly body: string;
+    constructor(status: number, body: string, attempts: number);
+}
+export declare function isRetryableStatus(status: number): boolean;
+export interface RunWithRetryArgs<T> {
+    policy: RetryPolicy;
+    /** Per-attempt callable. Resolves to {result} on success, or returns ok:false to fail. */
+    attempt: () => Promise<{
+        ok: true;
+        value: T;
+    } | {
+        ok: false;
+        status: number;
+        body: string;
+        retryAfterSeconds?: number;
+    }>;
+    /** Sleeps for `ms`. Injectable for tests. */
+    sleep?: (ms: number) => Promise<void>;
+    /** Random source in `[0, 1)`. Injectable for tests. Defaults to `Math.random`. */
+    rng?: () => number;
+}
+export declare function runWithRetry<T>(args: RunWithRetryArgs<T>): Promise<T>;
+/** Parses a `Retry-After` header (seconds-only form). */
+export declare function parseRetryAfterSeconds(header: null | string): number | undefined;

package/dist/adapters/llm/retry.js

@@ -0,0 +1,66 @@
+/**
+ * Shared retry helper for LLMClient adapters.
+ *
+ * Bounded exponential backoff with optional `Retry-After` honoring and
+ * symmetric jitter. Treats 429 / 5xx as retryable and any other HTTP error
+ * as terminal.
+ *
+ * Errors carry the full response body on the instance for callers that need
+ * to inspect it; the message is intentionally short and body-free so it's
+ * safe to include in user-facing logs and stack traces.
+ */
+export const DEFAULT_RETRY_POLICY = {
+    maxAttempts: 3,
+    baseDelayMs: 500,
+    backoffFactor: 2,
+    maxDelayMs: 10_000,
+    jitter: 0.3,
+};
+export class LLMHttpError extends Error {
+    status;
+    attempts;
+    /** Full upstream response body (kept on the instance, NOT in `message`). */
+    body;
+    constructor(status, body, attempts) {
+        super(`LLM request failed with status ${status} after ${attempts} attempt(s)`);
+        this.status = status;
+        this.attempts = attempts;
+        this.name = "LLMHttpError";
+        this.body = body;
+    }
+}
+export function isRetryableStatus(status) {
+    return status === 429 || (status >= 500 && status < 600);
+}
+const defaultSleep = (ms) => new Promise((r) => setTimeout(r, ms));
+export async function runWithRetry(args) {
+    const { policy, attempt, sleep = defaultSleep, rng = Math.random } = args;
+    for (let i = 1; i <= policy.maxAttempts; i++) {
+        const res = await attempt();
+        if (res.ok)
+            return res.value;
+        const canRetry = i < policy.maxAttempts && isRetryableStatus(res.status);
+        if (!canRetry) {
+            throw new LLMHttpError(res.status, res.body, i);
+        }
+        const exp = policy.baseDelayMs * Math.pow(policy.backoffFactor, i - 1);
+        const base = res.retryAfterSeconds ? res.retryAfterSeconds * 1000 : exp;
+        const capped = Math.min(base, policy.maxDelayMs);
+        const jittered = policy.jitter > 0
+            ? capped * (1 + (rng() - 0.5) * 2 * policy.jitter)
+            : capped;
+        await sleep(Math.max(0, Math.round(jittered)));
+    }
+    // Unreachable: the canRetry branch always throws on the final attempt.
+    // Defensive throw so the type checker sees a definite return.
+    throw new LLMHttpError(0, "no error body", policy.maxAttempts);
+}
+/** Parses a `Retry-After` header (seconds-only form). */
+export function parseRetryAfterSeconds(header) {
+    if (!header)
+        return undefined;
+    const n = Number(header);
+    if (Number.isFinite(n) && n >= 0)
+        return n;
+    return undefined;
+}

package/dist/adapters/task-sources/content-lake-task-source.d.ts

@@ -55,9 +55,13 @@ interface ContentLakeCanonicalDoc {
     sectionSlug?: string;
     slug?: string;
 }
+interface ContentLakeCriterion {
+    id?: string;
+    text?: string;
+}
 /** Assertion shape from the Content Lake (mirrors the Studio schema). */
 interface ContentLakeAssertion {
-    criteria?: string[];
+    criteria?: ContentLakeCriterion[];
     template?: string;
     threshold?: number;
     type?: string;

package/dist/adapters/task-sources/content-lake-task-source.js

@@ -73,7 +73,13 @@ const TASKS_QUERY = /* groq */ `
     perspective,
     reason
   },
-  "assertions": coalesce(assertions, assert),
+  "assertions": coalesce(assertions, assert)[] {
+    type, template, weight, value, threshold,
+    "criteria": criteria[] {
+      "id": coalesce(id.current, _key),
+      "text": coalesce(text, @)
+    }
+  },
   rawAssert,
   baseline,
   tags,
@@ -256,8 +262,28 @@ function mapAssertions(raw) {
         .filter((a) => !!a.type)
         .map((a) => {
         if (a.type === "llm-rubric" && a.template && a.criteria) {
+            // Tighten the runtime contract: the GROQ projection's
+            // `coalesce(text, @)` falls through to the entire criterion
+            // element when `text` is missing, so a partial legacy criterion
+            // like `{_key: "abc"}` arrives here as `{ id: "abc", text: {...} }`
+            // — `text` set to the whole `@` object. Explicit type checks
+            // drop those with a diagnostic, instead of letting the non-string
+            // `text` propagate until the outer ContentLakeAuthorableTaskSchema
+            // parse fails deep inside the assertions array (noisy diagnostic).
             return {
-                criteria: a.criteria,
+                criteria: a.criteria
+                    .filter((c) => {
+                    if (!c)
+                        return false;
+                    const idOk = typeof c.id === "string" && c.id.length > 0;
+                    const textOk = typeof c.text === "string" && c.text.length > 0;
+                    if (!idOk || !textOk) {
+                        console.warn(`[ContentLakeTaskSource] dropping malformed criterion: ${JSON.stringify(c).slice(0, 100)}`);
+                        return false;
+                    }
+                    return true;
+                })
+                    .map((c) => ({ id: c.id, text: c.text })),
                 template: a.template,
                 type: "llm-rubric",
                 ...(a.weight !== undefined ? { weight: a.weight } : {}),