@sanity/ailf 4.5.0 → 4.6.0

package/dist/adapters/llm/pricing.js

@@ -0,0 +1,8 @@
+/**
+ * Per-model pricing types shared by LLM adapters.
+ *
+ * Hard-coded vendor pricing drifts; treat the in-adapter defaults as a
+ * sensible starting point and override via constructor options when the
+ * vendor changes their rate card.
+ */
+export {};

package/dist/adapters/llm/retry.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Shared retry helper for LLMClient adapters.
+ *
+ * Bounded exponential backoff with optional `Retry-After` honoring and
+ * symmetric jitter. Treats 429 / 5xx as retryable and any other HTTP error
+ * as terminal.
+ *
+ * Errors carry the full response body on the instance for callers that need
+ * to inspect it; the message is intentionally short and body-free so it's
+ * safe to include in user-facing logs and stack traces.
+ */
+export interface RetryPolicy {
+    /** Total attempts including the initial call. Default 3. */
+    maxAttempts: number;
+    /** Initial backoff in ms. Default 500. */
+    baseDelayMs: number;
+    /** Multiplier per attempt. Default 2. */
+    backoffFactor: number;
+    /** Cap on a single delay in ms. Default 10_000. */
+    maxDelayMs: number;
+    /**
+     * Symmetric jitter as a fraction of the computed delay, in `[0, 1)`. The
+     * actual delay is `delay * (1 + (rng() - 0.5) * 2 * jitter)`. Default 0.3.
+     * Set to 0 to disable.
+     */
+    jitter: number;
+}
+export declare const DEFAULT_RETRY_POLICY: RetryPolicy;
+export declare class LLMHttpError extends Error {
+    readonly status: number;
+    readonly attempts: number;
+    /** Full upstream response body (kept on the instance, NOT in `message`). */
+    readonly body: string;
+    constructor(status: number, body: string, attempts: number);
+}
+export declare function isRetryableStatus(status: number): boolean;
+export interface RunWithRetryArgs<T> {
+    policy: RetryPolicy;
+    /** Per-attempt callable. Resolves to {result} on success, or returns ok:false to fail. */
+    attempt: () => Promise<{
+        ok: true;
+        value: T;
+    } | {
+        ok: false;
+        status: number;
+        body: string;
+        retryAfterSeconds?: number;
+    }>;
+    /** Sleeps for `ms`. Injectable for tests. */
+    sleep?: (ms: number) => Promise<void>;
+    /** Random source in `[0, 1)`. Injectable for tests. Defaults to `Math.random`. */
+    rng?: () => number;
+}
+export declare function runWithRetry<T>(args: RunWithRetryArgs<T>): Promise<T>;
+/** Parses a `Retry-After` header (seconds-only form). */
+export declare function parseRetryAfterSeconds(header: null | string): number | undefined;

package/dist/adapters/llm/retry.js

@@ -0,0 +1,66 @@
+/**
+ * Shared retry helper for LLMClient adapters.
+ *
+ * Bounded exponential backoff with optional `Retry-After` honoring and
+ * symmetric jitter. Treats 429 / 5xx as retryable and any other HTTP error
+ * as terminal.
+ *
+ * Errors carry the full response body on the instance for callers that need
+ * to inspect it; the message is intentionally short and body-free so it's
+ * safe to include in user-facing logs and stack traces.
+ */
+export const DEFAULT_RETRY_POLICY = {
+    maxAttempts: 3,
+    baseDelayMs: 500,
+    backoffFactor: 2,
+    maxDelayMs: 10_000,
+    jitter: 0.3,
+};
+export class LLMHttpError extends Error {
+    status;
+    attempts;
+    /** Full upstream response body (kept on the instance, NOT in `message`). */
+    body;
+    constructor(status, body, attempts) {
+        super(`LLM request failed with status ${status} after ${attempts} attempt(s)`);
+        this.status = status;
+        this.attempts = attempts;
+        this.name = "LLMHttpError";
+        this.body = body;
+    }
+}
+export function isRetryableStatus(status) {
+    return status === 429 || (status >= 500 && status < 600);
+}
+const defaultSleep = (ms) => new Promise((r) => setTimeout(r, ms));
+export async function runWithRetry(args) {
+    const { policy, attempt, sleep = defaultSleep, rng = Math.random } = args;
+    for (let i = 1; i <= policy.maxAttempts; i++) {
+        const res = await attempt();
+        if (res.ok)
+            return res.value;
+        const canRetry = i < policy.maxAttempts && isRetryableStatus(res.status);
+        if (!canRetry) {
+            throw new LLMHttpError(res.status, res.body, i);
+        }
+        const exp = policy.baseDelayMs * Math.pow(policy.backoffFactor, i - 1);
+        const base = res.retryAfterSeconds ? res.retryAfterSeconds * 1000 : exp;
+        const capped = Math.min(base, policy.maxDelayMs);
+        const jittered = policy.jitter > 0
+            ? capped * (1 + (rng() - 0.5) * 2 * policy.jitter)
+            : capped;
+        await sleep(Math.max(0, Math.round(jittered)));
+    }
+    // Unreachable: the canRetry branch always throws on the final attempt.
+    // Defensive throw so the type checker sees a definite return.
+    throw new LLMHttpError(0, "no error body", policy.maxAttempts);
+}
+/** Parses a `Retry-After` header (seconds-only form). */
+export function parseRetryAfterSeconds(header) {
+    if (!header)
+        return undefined;
+    const n = Number(header);
+    if (Number.isFinite(n) && n >= 0)
+        return n;
+    return undefined;
+}

package/dist/adapters/task-sources/repo-schemas.d.ts

@@ -77,6 +77,7 @@ export declare const CanonicalTaskSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
         weight: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodEnum<{
+            cost: "cost";
             "llm-rubric": "llm-rubric";
             contains: "contains";
             "contains-any": "contains-any";
@@ -87,7 +88,6 @@ export declare const CanonicalTaskSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
             regex: "regex";
             javascript: "javascript";
             similar: "similar";
-            cost: "cost";
             latency: "latency";
             "file-exists": "file-exists";
             "file-contains": "file-contains";
@@ -191,6 +191,7 @@ export declare const CanonicalTaskSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
         weight: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodEnum<{
+            cost: "cost";
             "llm-rubric": "llm-rubric";
             contains: "contains";
             "contains-any": "contains-any";
@@ -201,7 +202,6 @@ export declare const CanonicalTaskSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
             regex: "regex";
             javascript: "javascript";
             similar: "similar";
-            cost: "cost";
             latency: "latency";
             "file-exists": "file-exists";
             "file-contains": "file-contains";
@@ -345,6 +345,7 @@ export declare const CanonicalTaskSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
         weight: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodEnum<{
+            cost: "cost";
             "llm-rubric": "llm-rubric";
             contains: "contains";
             "contains-any": "contains-any";
@@ -355,7 +356,6 @@ export declare const CanonicalTaskSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
             regex: "regex";
             javascript: "javascript";
             similar: "similar";
-            cost: "cost";
             latency: "latency";
             "file-exists": "file-exists";
             "file-contains": "file-contains";
@@ -476,6 +476,7 @@ export declare const CanonicalTaskSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
         weight: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodEnum<{
+            cost: "cost";
             "llm-rubric": "llm-rubric";
             contains: "contains";
             "contains-any": "contains-any";
@@ -486,7 +487,6 @@ export declare const CanonicalTaskSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
             regex: "regex";
             javascript: "javascript";
             similar: "similar";
-            cost: "cost";
             latency: "latency";
             "file-exists": "file-exists";
             "file-contains": "file-contains";
@@ -595,6 +595,7 @@ export declare const CanonicalTaskSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
         weight: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodEnum<{
+            cost: "cost";
             "llm-rubric": "llm-rubric";
             contains: "contains";
             "contains-any": "contains-any";
@@ -605,7 +606,6 @@ export declare const CanonicalTaskSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
             regex: "regex";
             javascript: "javascript";
             similar: "similar";
-            cost: "cost";
             latency: "latency";
             "file-exists": "file-exists";
             "file-contains": "file-contains";
@@ -703,6 +703,7 @@ export declare const ContentLakeAuthorableTaskSchema: z.ZodObject<{
         weight: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodEnum<{
+            cost: "cost";
             "llm-rubric": "llm-rubric";
             contains: "contains";
             "contains-any": "contains-any";
@@ -713,7 +714,6 @@ export declare const ContentLakeAuthorableTaskSchema: z.ZodObject<{
             regex: "regex";
             javascript: "javascript";
             similar: "similar";
-            cost: "cost";
             latency: "latency";
             "file-exists": "file-exists";
             "file-contains": "file-contains";
@@ -823,6 +823,7 @@ export declare const CanonicalTaskFileSchema: z.ZodArray<z.ZodDiscriminatedUnion
         weight: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodEnum<{
+            cost: "cost";
             "llm-rubric": "llm-rubric";
             contains: "contains";
             "contains-any": "contains-any";
@@ -833,7 +834,6 @@ export declare const CanonicalTaskFileSchema: z.ZodArray<z.ZodDiscriminatedUnion
             regex: "regex";
             javascript: "javascript";
             similar: "similar";
-            cost: "cost";
             latency: "latency";
             "file-exists": "file-exists";
             "file-contains": "file-contains";
@@ -937,6 +937,7 @@ export declare const CanonicalTaskFileSchema: z.ZodArray<z.ZodDiscriminatedUnion
         weight: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodEnum<{
+            cost: "cost";
             "llm-rubric": "llm-rubric";
             contains: "contains";
             "contains-any": "contains-any";
@@ -947,7 +948,6 @@ export declare const CanonicalTaskFileSchema: z.ZodArray<z.ZodDiscriminatedUnion
             regex: "regex";
             javascript: "javascript";
             similar: "similar";
-            cost: "cost";
             latency: "latency";
             "file-exists": "file-exists";
             "file-contains": "file-contains";
@@ -1091,6 +1091,7 @@ export declare const CanonicalTaskFileSchema: z.ZodArray<z.ZodDiscriminatedUnion
         weight: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodEnum<{
+            cost: "cost";
             "llm-rubric": "llm-rubric";
             contains: "contains";
             "contains-any": "contains-any";
@@ -1101,7 +1102,6 @@ export declare const CanonicalTaskFileSchema: z.ZodArray<z.ZodDiscriminatedUnion
             regex: "regex";
             javascript: "javascript";
             similar: "similar";
-            cost: "cost";
             latency: "latency";
             "file-exists": "file-exists";
             "file-contains": "file-contains";
@@ -1222,6 +1222,7 @@ export declare const CanonicalTaskFileSchema: z.ZodArray<z.ZodDiscriminatedUnion
         weight: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodEnum<{
+            cost: "cost";
             "llm-rubric": "llm-rubric";
             contains: "contains";
             "contains-any": "contains-any";
@@ -1232,7 +1233,6 @@ export declare const CanonicalTaskFileSchema: z.ZodArray<z.ZodDiscriminatedUnion
             regex: "regex";
             javascript: "javascript";
             similar: "similar";
-            cost: "cost";
             latency: "latency";
             "file-exists": "file-exists";
             "file-contains": "file-contains";
@@ -1341,6 +1341,7 @@ export declare const CanonicalTaskFileSchema: z.ZodArray<z.ZodDiscriminatedUnion
         weight: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodEnum<{
+            cost: "cost";
             "llm-rubric": "llm-rubric";
             contains: "contains";
             "contains-any": "contains-any";
@@ -1351,7 +1352,6 @@ export declare const CanonicalTaskFileSchema: z.ZodArray<z.ZodDiscriminatedUnion
             regex: "regex";
             javascript: "javascript";
             similar: "similar";
-            cost: "cost";
             latency: "latency";
             "file-exists": "file-exists";
             "file-contains": "file-contains";

package/dist/artifact-capture/api-gateway-artifact-writer.js

@@ -27,7 +27,7 @@
  * @see docs/decisions/D0032-run-anchored-artifact-store.md
  * @see docs/decisions/D0033-unified-run-anchored-artifact-capture.md
  */
-import { ARTIFACT_REGISTRY, NotImplementedError, } from "../_vendor/ailf-core/index.js";
+import { ARTIFACT_REGISTRY, assertWritePolicyMatches, NotImplementedError, } from "../_vendor/ailf-core/index.js";
 import { prepareUploadBody } from "./prepare-upload-body.js";
 import { NO_OP_UPLOAD_METRICS, } from "./upload-metrics.js";
 export class ApiGatewayArtifactWriter {
@@ -40,6 +40,7 @@ export class ApiGatewayArtifactWriter {
     // ---- Canonical W0049 API ------------------------------------------------
     async emit(type, association, payload) {
         const descriptor = ARTIFACT_REGISTRY[type];
+        assertWritePolicyMatches("pipeline", descriptor);
         const runId = association.run;
         if (!runId) {
             console.warn(`  ⚠️  emit("${type}"): association.run is required, skipping`);

package/dist/artifact-capture/batching-api-gateway-artifact-writer.js

@@ -25,7 +25,7 @@
  * does this writer. Traces flow through the GCS-direct writer when ADC
  * credentials are present.
  */
-import { ARTIFACT_REGISTRY, BULK_ENTRY_KEY, NotImplementedError, } from "../_vendor/ailf-core/index.js";
+import { ARTIFACT_REGISTRY, assertWritePolicyMatches, BULK_ENTRY_KEY, NotImplementedError, } from "../_vendor/ailf-core/index.js";
 import { prepareUploadBody } from "./prepare-upload-body.js";
 import { NO_OP_UPLOAD_METRICS, } from "./upload-metrics.js";
 /**
@@ -64,6 +64,7 @@ export class BatchingApiGatewayArtifactWriter {
     // ---- ArtifactWriter surface --------------------------------------------
     async emit(type, association, payload) {
         const descriptor = ARTIFACT_REGISTRY[type];
+        assertWritePolicyMatches("pipeline", descriptor);
         const runId = association.run;
         if (!runId) {
             console.warn(`  ⚠️  emit("${type}"): association.run is required, skipping`);

package/dist/artifact-capture/gcs-artifact-writer.js

@@ -28,7 +28,7 @@
  * @see docs/decisions/D0033-unified-run-anchored-artifact-capture.md
  */
 import { Storage } from "@google-cloud/storage";
-import { ARTIFACT_REGISTRY, buildManifestPreview, } from "../_vendor/ailf-core/index.js";
+import { ARTIFACT_REGISTRY, assertWritePolicyMatches, buildManifestPreview, } from "../_vendor/ailf-core/index.js";
 import { resolveUploadConcurrency } from "./parallel-emit.js";
 import { prepareUploadBody } from "./prepare-upload-body.js";
 import { redactArtifactData } from "./redact-artifact.js";
@@ -79,6 +79,7 @@ export class GcsArtifactWriter {
     // ---- Canonical W0049 API ------------------------------------------------
     async emit(type, association, payload) {
         const descriptor = ARTIFACT_REGISTRY[type];
+        assertWritePolicyMatches("pipeline", descriptor);
         const runId = association.run;
         if (!runId) {
             console.warn(`  ⚠️  emit("${type}"): association.run is required, skipping`);
@@ -132,6 +133,7 @@ export class GcsArtifactWriter {
     }
     async appendNdjson(type, association, rows) {
         const descriptor = ARTIFACT_REGISTRY[type];
+        assertWritePolicyMatches("pipeline", descriptor);
         if (descriptor.mime !== "application/x-ndjson") {
             console.warn(`  ⚠️  appendNdjson("${type}"): descriptor mime is ${descriptor.mime}, not application/x-ndjson — skipping`);
             return null;

package/dist/artifact-capture/local-fs-artifact-writer.js

@@ -38,7 +38,7 @@
  */
 import { promises as fs } from "node:fs";
 import path from "node:path";
-import { ARTIFACT_REGISTRY, buildManifestPreview, } from "../_vendor/ailf-core/index.js";
+import { ARTIFACT_REGISTRY, assertWritePolicyMatches, buildManifestPreview, } from "../_vendor/ailf-core/index.js";
 import { redactArtifactData } from "./redact-artifact.js";
 // ---------------------------------------------------------------------------
 // Implementation
@@ -66,6 +66,7 @@ export class LocalFilesystemArtifactWriter {
         if (this.excludeSet.has(type))
             return null;
         const descriptor = ARTIFACT_REGISTRY[type];
+        assertWritePolicyMatches("pipeline", descriptor);
         const runId = association.run;
         if (!runId) {
             console.warn(`  ⚠️  emit("${type}"): association.run is required, skipping`);
@@ -127,6 +128,7 @@ export class LocalFilesystemArtifactWriter {
         if (this.excludeSet.has(type))
             return null;
         const descriptor = ARTIFACT_REGISTRY[type];
+        assertWritePolicyMatches("pipeline", descriptor);
         if (descriptor.mime !== "application/x-ndjson") {
             console.warn(`  ⚠️  appendNdjson("${type}"): descriptor mime is ${descriptor.mime}, not application/x-ndjson — skipping`);
             return null;

package/dist/composition-root.d.ts

@@ -15,7 +15,7 @@
  * @see packages/core/src/ports/context.ts — AppContext interface
  * @see docs/archive/exec-plans/ports-and-adapters/phase-7-composition-root.md
  */
-import { type AppContext, type ArtifactWriter, type ArtifactWriterProgressOptions, type AssertionRegistration, type Logger, type ResolvedConfig } from "./_vendor/ailf-core/index.d.ts";
+import { type AppContext, type ArtifactWriter, type ArtifactWriterProgressOptions, type AssertionRegistration, type LLMClient, type Logger, type ResolvedConfig } from "./_vendor/ailf-core/index.d.ts";
 import { CompositeTaskSource, ContentLakeTaskSource, RepoTaskSource } from "./adapters/task-sources/index.js";
 /**
  * Create a fully wired AppContext from resolved configuration.
@@ -24,6 +24,28 @@ import { CompositeTaskSource, ContentLakeTaskSource, RepoTaskSource } from "./ad
  * Swapping an adapter is a one-line change in this function.
  */
 export declare function createAppContext(config: ResolvedConfig): AppContext;
+/**
+ * Typed key bag passed to `createLLMClient`. The composition root reads
+ * env once and supplies values here; the factory stays pure so tests don't
+ * have to mutate `process.env`.
+ */
+export interface LLMClientKeys {
+    anthropicApiKey?: string;
+    openaiApiKey?: string;
+}
+/**
+ * Select the LLMClient adapter based on `config.llmProvider` and the
+ * supplied API keys. Returns `undefined` when no usable credential is
+ * present — `AppContext.llmClient` stays unset and consumers handle that
+ * explicitly.
+ *
+ * Adapters never read `process.env` themselves (per
+ * `.claude/rules/typescript.md`); env mapping happens at the call site
+ * (typically `createAppContext`).
+ *
+ * Exported for unit-test access; not part of the public package API.
+ */
+export declare function createLLMClient(config: ResolvedConfig, keys: LLMClientKeys, logger: Logger): LLMClient | undefined;
 /**
  * Selects the `ArtifactWriter` wiring per D0033 M4:
  *

package/dist/composition-root.js

@@ -26,6 +26,7 @@ import { LocalFilesystemArtifactWriter } from "./artifact-capture/local-fs-artif
 import { resolveUploadConcurrency, setDefaultUploadConcurrency, } from "./artifact-capture/parallel-emit.js";
 import { UploadMetrics } from "./artifact-capture/upload-metrics.js";
 import { ContentLakeCacheAdapter } from "./adapters/cache/content-lake-cache.js";
+import { AnthropicLLMClient, OpenAILLMClient } from "./adapters/llm/index.js";
 import { loadExternalPresets } from "./pipeline/compiler/preset-loader.js";
 import { FilesystemCache } from "./adapters/cache/filesystem-cache.js";
 import { PromptfooEvalAdapter } from "./adapters/eval-runners/promptfoo-eval-adapter.js";
@@ -91,12 +92,20 @@ export function createAppContext(config) {
     // from the context (D0032).
     const runId = generateRunId();
     logger.debug(`Pipeline runId: ${runId}`);
+    // LLM client (D0051) — wired when an API key is present. The grader path
+    // does NOT consume this; D0051 defers grader migration as a follow-up.
+    // Env mapping happens here so `createLLMClient` stays pure and testable.
+    const llmClient = createLLMClient(config, {
+        anthropicApiKey: process.env.ANTHROPIC_API_KEY,
+        openaiApiKey: process.env.OPENAI_API_KEY,
+    }, logger);
     return {
         artifactWriter,
         cache,
         config,
         docFetcher,
         evalRunner,
+        ...(llmClient ? { llmClient } : {}),
         logger,
         packageSurfaceResolver,
         progress,
@@ -107,6 +116,44 @@ export function createAppContext(config) {
         taskSource,
     };
 }
+/**
+ * Select the LLMClient adapter based on `config.llmProvider` and the
+ * supplied API keys. Returns `undefined` when no usable credential is
+ * present — `AppContext.llmClient` stays unset and consumers handle that
+ * explicitly.
+ *
+ * Adapters never read `process.env` themselves (per
+ * `.claude/rules/typescript.md`); env mapping happens at the call site
+ * (typically `createAppContext`).
+ *
+ * Exported for unit-test access; not part of the public package API.
+ */
+export function createLLMClient(config, keys, logger) {
+    const explicit = config.llmProvider;
+    const anthropicKey = keys.anthropicApiKey;
+    const openaiKey = keys.openaiApiKey;
+    // Auto-select: prefer Anthropic when both are present (matches the
+    // current grader's default model in `config/models.ts`).
+    const provider = explicit ?? (anthropicKey ? "anthropic" : openaiKey ? "openai" : undefined);
+    if (!provider) {
+        logger.debug("LLM client: not wired — no Anthropic or OpenAI API key supplied");
+        return undefined;
+    }
+    if (provider === "anthropic") {
+        if (!anthropicKey) {
+            logger.warn('llmProvider="anthropic" but no Anthropic API key supplied — LLMClient not wired');
+            return undefined;
+        }
+        logger.debug("LLM client: AnthropicLLMClient");
+        return new AnthropicLLMClient({ apiKey: anthropicKey, logger });
+    }
+    if (!openaiKey) {
+        logger.warn('llmProvider="openai" but no OpenAI API key supplied — LLMClient not wired');
+        return undefined;
+    }
+    logger.debug("LLM client: OpenAILLMClient");
+    return new OpenAILLMClient({ apiKey: openaiKey, logger });
+}
 // ---------------------------------------------------------------------------
 // Sub-factories (extracted to keep createAppContext readable)
 // ---------------------------------------------------------------------------

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/ailf",
-  "version": "4.5.0",
+  "version": "4.6.0",
   "private": false,
   "publishConfig": {
     "access": "public"