@sanity/ailf 4.1.0 → 4.3.0

package/dist/adapters/package-surface/dts-package-surface.js

@@ -0,0 +1,173 @@
+/**
+ * DtsPackageSurface — `PackageSurfaceResolver` adapter that reads installed
+ * package `.d.ts` files from `node_modules`.
+ *
+ * Resolution flow:
+ *   1. Find the package's root directory via the configured resolver
+ *      (default uses `createRequire` against the working directory).
+ *   2. Read its `package.json` to capture the resolved `version` and the
+ *      `.d.ts` entry path (`types`, `typings`, or `exports["."].types`).
+ *   3. Parse the entry `.d.ts` for top-level export declarations.
+ *   4. Follow ONE hop of `export * from "./relative"` re-exports — direct
+ *      siblings only, no transitive walking.
+ *   5. Cache the result by package name for the resolver's lifetime.
+ *
+ * Throws `PackageSurfaceResolverError` with a typed `reason` when the
+ * package isn't installed or its types entry can't be resolved. Callers
+ * (the W0198 preflight) catch the error and convert it into per-binding
+ * `unresolved` findings rather than `missing` deductions.
+ */
+import { existsSync, readFileSync } from "node:fs";
+import { createRequire } from "node:module";
+import { dirname, isAbsolute, join, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import { PackageSurfaceResolverError, } from "../../_vendor/ailf-core/index.js";
+import { parseDtsExports } from "./parse-dts-exports.js";
+export class DtsPackageSurface {
+    resolveRoot;
+    cache = new Map();
+    constructor(opts = {}) {
+        this.resolveRoot =
+            opts.resolvePackageRoot ??
+                makeDefaultPackageRootResolver(opts.resolveFromDir ?? process.cwd());
+    }
+    async resolveExports(pkg) {
+        const cached = this.cache.get(pkg);
+        if (cached)
+            return cached;
+        const root = this.resolveRoot(pkg);
+        if (!root) {
+            throw new PackageSurfaceResolverError("package-not-installed", pkg, `Package "${pkg}" was not resolvable from the configured lookup path.`);
+        }
+        const pkgJsonPath = join(root, "package.json");
+        if (!existsSync(pkgJsonPath)) {
+            throw new PackageSurfaceResolverError("package-not-installed", pkg, `Package "${pkg}" has no package.json at "${pkgJsonPath}".`);
+        }
+        const pkgJson = readPackageJson(pkgJsonPath, pkg);
+        const version = pkgJson.version ?? "0.0.0";
+        const typesEntry = resolveTypesEntry(pkgJson, root);
+        if (!typesEntry) {
+            throw new PackageSurfaceResolverError("types-entry-missing", pkg, `Package "${pkg}@${version}" does not declare a \`types\` entry ` +
+                `(checked package.json \`types\`, \`typings\`, and \`exports["."].types\`).`);
+        }
+        const symbols = readSurface(pkg, version, typesEntry);
+        const surface = {
+            pkg,
+            version,
+            symbols,
+        };
+        this.cache.set(pkg, surface);
+        return surface;
+    }
+}
+function readPackageJson(path, pkg) {
+    try {
+        return JSON.parse(readFileSync(path, "utf-8"));
+    }
+    catch (err) {
+        throw new PackageSurfaceResolverError("parse-failed", pkg, `Failed to parse "${path}": ${err instanceof Error ? err.message : String(err)}`);
+    }
+}
+/**
+ * Pick the `.d.ts` entry the resolver should parse. Order:
+ *   1. `package.json#types`
+ *   2. `package.json#typings`
+ *   3. `exports["."].types` (string or object form)
+ */
+function resolveTypesEntry(pkgJson, root) {
+    const candidates = [];
+    if (typeof pkgJson.types === "string")
+        candidates.push(pkgJson.types);
+    if (typeof pkgJson.typings === "string")
+        candidates.push(pkgJson.typings);
+    const dotExport = pkgJson.exports && typeof pkgJson.exports === "object"
+        ? pkgJson.exports["."]
+        : undefined;
+    if (typeof dotExport === "string") {
+        if (dotExport.endsWith(".d.ts"))
+            candidates.push(dotExport);
+    }
+    else if (dotExport && typeof dotExport === "object") {
+        const typesField = dotExport.types;
+        if (typeof typesField === "string")
+            candidates.push(typesField);
+        else if (typesField && typeof typesField === "object") {
+            // Conditional `types` entry (rare) — pick any string leaf.
+            for (const v of Object.values(typesField)) {
+                if (typeof v === "string") {
+                    candidates.push(v);
+                    break;
+                }
+            }
+        }
+    }
+    for (const candidate of candidates) {
+        const abs = isAbsolute(candidate) ? candidate : join(root, candidate);
+        if (existsSync(abs))
+            return abs;
+    }
+    return null;
+}
+function readSurface(pkg, version, entryPath) {
+    const names = new Set();
+    const visited = new Set();
+    const parseFile = (path, hops) => {
+        if (visited.has(path))
+            return;
+        visited.add(path);
+        let src;
+        try {
+            src = readFileSync(path, "utf-8");
+        }
+        catch (err) {
+            throw new PackageSurfaceResolverError("parse-failed", pkg, `Failed to read "${path}" for "${pkg}@${version}": ${err instanceof Error ? err.message : String(err)}`);
+        }
+        const parsed = parseDtsExports(src);
+        for (const name of parsed.names)
+            names.add(name);
+        if (hops <= 0)
+            return;
+        const baseDir = dirname(path);
+        for (const spec of parsed.reExports) {
+            if (!spec.startsWith("."))
+                continue; // bare specifier — out of scope
+            const resolved = resolveRelativeDts(baseDir, spec);
+            if (resolved)
+                parseFile(resolved, hops - 1);
+        }
+    };
+    parseFile(entryPath, /* hops */ 1);
+    return [...names].sort().map((name) => ({ name, source: "types" }));
+}
+/**
+ * Resolve a relative re-export specifier to an existing `.d.ts` file.
+ * Tries `<spec>.d.ts`, `<spec>/index.d.ts`, and `<spec>` literally.
+ */
+function resolveRelativeDts(baseDir, spec) {
+    const base = resolve(baseDir, spec);
+    const candidates = [
+        base.endsWith(".d.ts") ? base : null,
+        `${base}.d.ts`,
+        join(base, "index.d.ts"),
+    ].filter((p) => p !== null);
+    for (const path of candidates) {
+        if (existsSync(path))
+            return path;
+    }
+    return null;
+}
+function makeDefaultPackageRootResolver(fromDir) {
+    // `createRequire` needs a file URL or path that ends with a slash so
+    // it knows it's a directory, not a file.
+    const anchor = fromDir.endsWith("/") ? fromDir : `${fromDir}/`;
+    const req = createRequire(pathToFileURL(anchor));
+    return (pkg) => {
+        try {
+            const pkgJsonPath = req.resolve(`${pkg}/package.json`);
+            return dirname(pkgJsonPath);
+        }
+        catch {
+            return null;
+        }
+    };
+}

package/dist/adapters/package-surface/in-memory-package-surface.d.ts

@@ -0,0 +1,15 @@
+/**
+ * InMemoryPackageSurface — `PackageSurfaceResolver` test double.
+ *
+ * Backed by a plain `Map<string, PackageSurface>`; calls for unknown
+ * packages throw the same `package-not-installed` error the
+ * `DtsPackageSurface` adapter throws, so test scenarios for the
+ * `unresolved` path need no special handling.
+ */
+import { type PackageSurface, type PackageSurfaceResolver } from "../../_vendor/ailf-core/index.d.ts";
+export declare class InMemoryPackageSurface implements PackageSurfaceResolver {
+    private readonly surfaces;
+    constructor(surfaces?: Iterable<PackageSurface>);
+    set(surface: PackageSurface): void;
+    resolveExports(pkg: string): Promise<PackageSurface>;
+}

package/dist/adapters/package-surface/in-memory-package-surface.js

@@ -0,0 +1,28 @@
+/**
+ * InMemoryPackageSurface — `PackageSurfaceResolver` test double.
+ *
+ * Backed by a plain `Map<string, PackageSurface>`; calls for unknown
+ * packages throw the same `package-not-installed` error the
+ * `DtsPackageSurface` adapter throws, so test scenarios for the
+ * `unresolved` path need no special handling.
+ */
+import { PackageSurfaceResolverError, } from "../../_vendor/ailf-core/index.js";
+export class InMemoryPackageSurface {
+    surfaces;
+    constructor(surfaces = []) {
+        this.surfaces = new Map();
+        for (const surface of surfaces) {
+            this.surfaces.set(surface.pkg, surface);
+        }
+    }
+    set(surface) {
+        this.surfaces.set(surface.pkg, surface);
+    }
+    async resolveExports(pkg) {
+        const surface = this.surfaces.get(pkg);
+        if (!surface) {
+            throw new PackageSurfaceResolverError("package-not-installed", pkg, `InMemoryPackageSurface has no entry for "${pkg}".`);
+        }
+        return surface;
+    }
+}

package/dist/adapters/package-surface/index.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Package-surface resolver adapters.
+ *
+ * @see packages/core/src/ports/package-surface-resolver.ts
+ */
+export { DtsPackageSurface, type DtsPackageSurfaceOptions, type PackageRootResolver, } from "./dts-package-surface.js";
+export { InMemoryPackageSurface } from "./in-memory-package-surface.js";
+export { parseDtsExports } from "./parse-dts-exports.js";
+export type { ParsedDtsExports } from "./parse-dts-exports.js";

package/dist/adapters/package-surface/index.js

@@ -0,0 +1,8 @@
+/**
+ * Package-surface resolver adapters.
+ *
+ * @see packages/core/src/ports/package-surface-resolver.ts
+ */
+export { DtsPackageSurface, } from "./dts-package-surface.js";
+export { InMemoryPackageSurface } from "./in-memory-package-surface.js";
+export { parseDtsExports } from "./parse-dts-exports.js";

package/dist/adapters/package-surface/parse-dts-exports.d.ts

@@ -0,0 +1,31 @@
+/**
+ * parse-dts-exports — pure function that extracts the public surface of a
+ * single `.d.ts` file as a list of top-level exported binding names plus
+ * any `export * from "./relative"` re-export specifiers.
+ *
+ * Implementation: delegates to `oxc-parser`'s `staticExports` view, which
+ * already decomposes each export statement into entries with `importName` /
+ * `exportName` / `moduleRequest` discriminators. We translate that view
+ * into the two outputs the W0198 preflight cares about — bare names and
+ * wildcard re-export specifiers — and drop default exports per the
+ * design's "named-bindings only" rule.
+ *
+ * Why oxc-parser instead of regex: top-level `.d.ts` syntax has enough TS
+ * surface area (declaration merging, conditional `exports` map types,
+ * ambient namespace augmentation) that a real AST is cheaper to maintain
+ * than a regex with the same coverage. Why oxc-parser instead of
+ * `typescript`: typescript isn't in `@sanity/ailf`'s runtime install graph
+ * and adding it adds ~50MB; oxc-parser is a few-MB native binary aligned
+ * with our existing `oxlint` / `oxfmt` toolchain.
+ */
+export interface ParsedDtsExports {
+    /** Bare exported identifier names found in this file. */
+    readonly names: readonly string[];
+    /**
+     * Specifiers from `export * from "<spec>"` declarations. Only relative
+     * specifiers (starting with `.`) are useful for one-hop following; the
+     * caller decides which to resolve.
+     */
+    readonly reExports: readonly string[];
+}
+export declare function parseDtsExports(src: string): ParsedDtsExports;

package/dist/adapters/package-surface/parse-dts-exports.js

@@ -0,0 +1,54 @@
+/**
+ * parse-dts-exports — pure function that extracts the public surface of a
+ * single `.d.ts` file as a list of top-level exported binding names plus
+ * any `export * from "./relative"` re-export specifiers.
+ *
+ * Implementation: delegates to `oxc-parser`'s `staticExports` view, which
+ * already decomposes each export statement into entries with `importName` /
+ * `exportName` / `moduleRequest` discriminators. We translate that view
+ * into the two outputs the W0198 preflight cares about — bare names and
+ * wildcard re-export specifiers — and drop default exports per the
+ * design's "named-bindings only" rule.
+ *
+ * Why oxc-parser instead of regex: top-level `.d.ts` syntax has enough TS
+ * surface area (declaration merging, conditional `exports` map types,
+ * ambient namespace augmentation) that a real AST is cheaper to maintain
+ * than a regex with the same coverage. Why oxc-parser instead of
+ * `typescript`: typescript isn't in `@sanity/ailf`'s runtime install graph
+ * and adding it adds ~50MB; oxc-parser is a few-MB native binary aligned
+ * with our existing `oxlint` / `oxfmt` toolchain.
+ */
+import { parseSync } from "oxc-parser";
+export function parseDtsExports(src) {
+    // Filename hint drives the parser's grammar — `.d.ts` enables the
+    // ambient-only forms we want and disables expression-context grammar
+    // we'd otherwise have to ignore.
+    const result = parseSync("input.d.ts", src, { lang: "dts" });
+    const names = new Set();
+    const reExports = [];
+    for (const exportStmt of result.module.staticExports) {
+        for (const entry of exportStmt.entries) {
+            // Wildcard re-export: `export * from "./other"`. The namespace form
+            // `export * as ns from "./other"` falls into the named-export branch
+            // below because it does expose a binding (`ns`) at the top level.
+            if (entry.importName.kind === "AllButDefault" &&
+                entry.moduleRequest !== null) {
+                reExports.push(entry.moduleRequest.value);
+                continue;
+            }
+            // Anything that produces a stable named binding visible to consumers.
+            // Covers own declarations, local re-exports (`export { x as y }`),
+            // module re-exports (`export { x } from "./y"`), and namespace
+            // re-exports (`export * as ns from "./y"`). `export default ...`
+            // lands in `exportName.kind === "Default"` and is intentionally
+            // skipped — the W0198 preflight only judges named bindings.
+            if (entry.exportName.kind === "Name" && entry.exportName.name) {
+                names.add(entry.exportName.name);
+            }
+        }
+    }
+    return {
+        names: [...names].sort(),
+        reExports,
+    };
+}

package/dist/adapters/task-sources/repo-schemas.d.ts

@@ -1425,6 +1425,22 @@ export declare function parseCanonicalTaskFile(raw: unknown, filename: string):
  * GeneralizedTaskDefinition shape.
  */
 export declare function detectLegacyFieldNames(raw: unknown, filename: string): string[];
+interface MigrationResult {
+    migrated: unknown;
+    warnings: string[];
+}
+/**
+ * Pre-process legacy `prompt.vars.{task,docs,__featureArea}` into the
+ * canonical shape. Backwards-compatible: legacy-shape tasks continue to
+ * load, but a deprecation warning is emitted per affected task.
+ *
+ *   Legacy:    prompt: { vars: { task: "...", docs: "file://..." } }
+ *   Canonical: prompt: { text: "..." }
+ *
+ * Applies to every task regardless of mode. Per-task dedup: at most one
+ * warning per task per call, listing every reserved key that was present.
+ */
+export declare function migratePromptShape(raw: unknown, filename: string): MigrationResult;
 /**
  * Zod schema for .ailf/config.yaml — controls documentation source,
  * report destination, and trigger behavior for evaluations from an
@@ -1455,6 +1471,12 @@ export declare const RepoConfigSchema: z.ZodObject<{
         gapAnalysis: z.ZodOptional<z.ZodBoolean>;
         apiUrl: z.ZodOptional<z.ZodString>;
     }, z.core.$strip>>;
+    grader: z.ZodOptional<z.ZodObject<{
+        context: z.ZodOptional<z.ZodEnum<{
+            "rubric-only": "rubric-only";
+            "with-docs": "with-docs";
+        }>>;
+    }, z.core.$strip>>;
     output: z.ZodOptional<z.ZodObject<{
         dir: z.ZodOptional<z.ZodString>;
     }, z.core.$strip>>;

package/dist/adapters/task-sources/repo-schemas.js

@@ -141,11 +141,30 @@ const AssertionSchema = z.union([
 // ---------------------------------------------------------------------------
 // Shared field schemas — building blocks reused across mode variants
 // ---------------------------------------------------------------------------
+/**
+ * Variable keys reserved by the AILF compilers — populated automatically
+ * from canonical task fields (`prompt.text`, `context.docs`, `area`).
+ * Mirrors `ReservedPromptVarKey` in `@sanity/ailf-core`; the `satisfies`
+ * clause makes drift a build error.
+ */
+const RESERVED_PROMPT_VAR_KEYS = [
+    "task",
+    "docs",
+    "__featureArea",
+];
 const TaskPromptSchema = z.object({
     template: z.string().optional(),
     text: z.string().optional(),
     systemMessage: z.string().optional(),
-    vars: z.record(z.string(), z.unknown()).optional(),
+    vars: z
+        .record(z.string(), z.unknown())
+        .refine((vars) => !RESERVED_PROMPT_VAR_KEYS.some((key) => key in vars), {
+        message: `prompt.vars contains a reserved key. Reserved keys: ` +
+            RESERVED_PROMPT_VAR_KEYS.join(", ") +
+            `. Use prompt.text for the prompt body and context.docs for ` +
+            `documentation references.`,
+    })
+        .optional(),
 });
 const RubricRefSchema = z.union([
     z.object({ ref: z.string().min(1) }),
@@ -416,6 +435,64 @@ export function detectLegacyFieldNames(raw, filename) {
     }
     return warnings;
 }
+/**
+ * Pre-process legacy `prompt.vars.{task,docs,__featureArea}` into the
+ * canonical shape. Backwards-compatible: legacy-shape tasks continue to
+ * load, but a deprecation warning is emitted per affected task.
+ *
+ *   Legacy:    prompt: { vars: { task: "...", docs: "file://..." } }
+ *   Canonical: prompt: { text: "..." }
+ *
+ * Applies to every task regardless of mode. Per-task dedup: at most one
+ * warning per task per call, listing every reserved key that was present.
+ */
+export function migratePromptShape(raw, filename) {
+    if (!Array.isArray(raw))
+        return { migrated: raw, warnings: [] };
+    const warnings = [];
+    const migrated = raw.map((entry, i) => {
+        if (typeof entry !== "object" || entry === null)
+            return entry;
+        const obj = entry;
+        const prompt = obj.prompt;
+        if (typeof prompt !== "object" || prompt === null)
+            return entry;
+        const promptObj = prompt;
+        const vars = promptObj.vars;
+        if (typeof vars !== "object" || vars === null)
+            return entry;
+        const varsObj = vars;
+        // Detect which reserved keys are present
+        const presentReserved = RESERVED_PROMPT_VAR_KEYS.filter((key) => key in varsObj);
+        if (presentReserved.length === 0)
+            return entry;
+        const taskId = typeof obj.id === "string" ? obj.id : `task[${i}]`;
+        // Build migrated prompt + vars
+        const newPrompt = { ...promptObj };
+        const newVars = { ...varsObj };
+        for (const key of presentReserved) {
+            if (key === "task" && newPrompt.text === undefined) {
+                // Move the prompt body to prompt.text only if the canonical slot
+                // is unset; an explicit prompt.text always wins.
+                newPrompt.text = newVars.task;
+            }
+            delete newVars[key];
+        }
+        // Drop empty vars to keep the migrated shape minimal
+        if (Object.keys(newVars).length === 0) {
+            delete newPrompt.vars;
+        }
+        else {
+            newPrompt.vars = newVars;
+        }
+        warnings.push(`[${filename}] ${taskId}: deprecated prompt.vars keys ` +
+            `(${presentReserved.join(", ")}) — migrated to canonical shape ` +
+            `(prompt.text + context.docs). Update the task source to silence ` +
+            `this warning.`);
+        return { ...obj, prompt: newPrompt };
+    });
+    return { migrated, warnings };
+}
 // ---------------------------------------------------------------------------
 // Config schemas — specific to the eval pipeline
 // ---------------------------------------------------------------------------
@@ -489,6 +566,20 @@ const ExecutionConfigSchema = z
     apiUrl: z.string().url().optional(),
 })
     .optional();
+/**
+ * Grader configuration.
+ *
+ * - `context: "rubric-only"` — grader sees only the rubric template +
+ *   criteria + candidate response.
+ * - `context: "with-docs"` — canonical reference content is injected into
+ *   the assertion's `rubricPrompt` so the grader has authoritative ground
+ *   truth.
+ */
+const GraderConfigSchema = z
+    .object({
+    context: z.enum(["rubric-only", "with-docs"]).optional(),
+})
+    .optional();
 /**
  * Task-source configuration (W0077 Phase 6h). Replaces the retired
  * `--task-source` and `--repo-tasks-path` CLI flags on `ailf run`.
@@ -581,6 +672,7 @@ export const RepoConfigSchema = z.object({
     reportStore: ReportStoreConfigSchema,
     publish: PublishConfigSchema,
     execution: ExecutionConfigSchema,
+    grader: GraderConfigSchema,
     output: OutputConfigSchema,
     owner: OwnerConfigSchema,
     agentic: AgenticConfigSchema,

package/dist/adapters/task-sources/repo-task-source.js

@@ -22,7 +22,7 @@ import { existsSync, readdirSync, readFileSync } from "fs";
 import { resolve } from "path";
 import { load } from "js-yaml";
 import { CANONICAL_EVAL_MODES } from "../../_vendor/ailf-shared/index.js";
-import { detectLegacyFieldNames, parseCanonicalTaskFile, } from "./repo-schemas.js";
+import { detectLegacyFieldNames, migratePromptShape, parseCanonicalTaskFile, } from "./repo-schemas.js";
 import { discoverTsTaskFiles, loadTsTaskFile } from "./task-file-loader.js";
 /** Set of canonical mode names for O(1) lookup */
 const KNOWN_MODES = new Set(CANONICAL_EVAL_MODES);
@@ -69,10 +69,19 @@ export class RepoTaskSource {
                     legacyWarnings.join("\n") +
                     "\n\nSee contributing-tasks.md for the canonical task format.");
             }
+            // W0193: pre-migrate legacy prompt.vars.{task,docs,__featureArea}
+            // to the canonical prompt.text + context.docs shape. Mode-agnostic —
+            // every mode's TaskPromptSchema rejects reserved keys, so the shim
+            // unblocks legacy tasks regardless of mode. Per-task deprecation
+            // warning fires on stderr.
+            const { migrated, warnings: deprecationWarnings } = migratePromptShape(parsed, file);
+            for (const warning of deprecationWarnings) {
+                console.warn(warning);
+            }
             // Validate through canonical Zod schema
             let validated;
             try {
-                validated = parseCanonicalTaskFile(parsed, file);
+                validated = parseCanonicalTaskFile(migrated, file);
             }
             catch (err) {
                 const msg = err instanceof Error ? err.message : String(err);

package/dist/commands/pipeline-action.d.ts

@@ -27,6 +27,8 @@ export interface ResolvedOptions {
     dryRun: boolean;
     gapAnalysisEnabled: boolean;
     graderReplications?: number;
+    /** Grader context policy from `.ailf/config.yaml` `grader.context` */
+    graderContext?: "rubric-only" | "with-docs";
     headerArgs: string[];
     impactSummary?: ImpactSummary;
     mode: EvalMode;

package/dist/commands/pipeline-action.js

@@ -249,6 +249,17 @@ export function computeResolvedOptions(opts) {
     const concurrency = repoConfig?.execution?.concurrency;
     const graderReplications = repoConfig?.execution?.graderReplications;
     const gapAnalysisEnabled = repoConfig?.execution?.gapAnalysis ?? true;
+    // Grader context policy. Cascade: env var > .ailf/config.yaml > unset
+    // (defaults to rubric-only at the EvalConfig boundary). The env var is the
+    // operational lever for one-shot comparison runs without editing the config file.
+    const rawGraderContext = process.env.AILF_GRADER_CONTEXT ?? repoConfig?.grader?.context;
+    const graderContext = rawGraderContext === "with-docs" || rawGraderContext === "rubric-only"
+        ? rawGraderContext
+        : undefined;
+    if (rawGraderContext && graderContext === undefined) {
+        console.error(`❌ Invalid grader.context "${rawGraderContext}". Must be "rubric-only" or "with-docs".`);
+        process.exit(1);
+    }
     // Remote mode
     const remote = opts.remote || process.env.AILF_REMOTE === "1";
     const apiUrl = process.env.AILF_API_URL ??
@@ -274,6 +285,7 @@ export function computeResolvedOptions(opts) {
         dryRun: opts.dryRun,
         gapAnalysisEnabled,
         graderReplications,
+        graderContext,
         headerArgs,
         impactSummary,
         mode,

package/dist/commands/remote-pipeline.js

@@ -90,12 +90,19 @@ export async function runRemotePipeline(opts, rootDir) {
         console.error(formatJobError(job));
         process.exit(1);
     }
-    // 7. Fetch and write output artifacts
-    await writeRemoteResults(client, job, {
+    // 7. Fetch and write output artifacts. A `completed` job that carries
+    //    `job.error` is a degraded completion (DOC-2121 RC-3): a configured
+    //    optional step failed end-to-end. Artifacts still write so the caller
+    //    keeps useful local state, but the CLI exits non-zero so external
+    //    `--remote` consumers don't mistake the placeholder for success.
+    const outcome = await writeRemoteResults(client, job, {
         outputDir: opts.outputDir,
         outputPath: opts.outputPath,
         apiUrl: opts.apiUrl,
     });
+    if (outcome.degraded) {
+        process.exit(1);
+    }
 }
 // ---------------------------------------------------------------------------
 // Helpers

package/dist/commands/remote-results.d.ts

@@ -21,6 +21,11 @@ export interface WriteResultsOptions {
     /** API base URL (for metadata). */
     apiUrl: string;
 }
+/** Outcome flags so the caller can decide the process exit code. */
+export interface WriteResultsOutcome {
+    /** True when `job.error` was set on a completed job (DOC-2121 RC-3). */
+    degraded: boolean;
+}
 /**
  * Fetch report artifacts from the API and write them to disk.
  *
@@ -29,5 +34,11 @@ export interface WriteResultsOptions {
  * - `<outputDir>/report.md` — full markdown report (if reportId present)
  * - `<outputDir>/job-metadata.json` — job tracking info
  * - `--output` path — markdown report (if specified)
+ *
+ * Returns an outcome the caller uses to choose an exit code: a `completed`
+ * job that carries `job.error` is treated as a *degraded* completion (a
+ * configured optional step failed end-to-end; see DOC-2121 RC-3) and the
+ * caller should exit non-zero so external `--remote` consumers don't read
+ * a clean completion as success.
  */
-export declare function writeRemoteResults(client: ApiClient, job: JobResponse, options: WriteResultsOptions): Promise<void>;
+export declare function writeRemoteResults(client: ApiClient, job: JobResponse, options: WriteResultsOptions): Promise<WriteResultsOutcome>;

package/dist/commands/remote-results.js

@@ -12,9 +12,6 @@
  */
 import { mkdirSync, writeFileSync } from "fs";
 import { resolve } from "path";
-// ---------------------------------------------------------------------------
-// Public API
-// ---------------------------------------------------------------------------
 /**
  * Fetch report artifacts from the API and write them to disk.
  *
@@ -23,6 +20,12 @@ import { resolve } from "path";
  * - `<outputDir>/report.md` — full markdown report (if reportId present)
  * - `<outputDir>/job-metadata.json` — job tracking info
  * - `--output` path — markdown report (if specified)
+ *
+ * Returns an outcome the caller uses to choose an exit code: a `completed`
+ * job that carries `job.error` is treated as a *degraded* completion (a
+ * configured optional step failed end-to-end; see DOC-2121 RC-3) and the
+ * caller should exit non-zero so external `--remote` consumers don't read
+ * a clean completion as success.
  */
 export async function writeRemoteResults(client, job, options) {
     const resultsDir = options.outputDir;
@@ -55,11 +58,20 @@ export async function writeRemoteResults(client, job, options) {
         reportId: job.reportId ?? null,
         reportUrl: job.reportUrl ?? null,
         execution: job.execution ?? null,
+        error: job.error ?? null,
         apiUrl: options.apiUrl,
     }, null, 2));
-    // 4. Print summary
+    // 4. Print summary. A completed job with `job.error` set means a
+    // configured optional step failed end-to-end — print the diagnostic
+    // and signal the caller to exit non-zero.
+    const degraded = Boolean(job.error);
     console.log("");
-    console.log(`✅ Evaluation completed`);
+    if (degraded) {
+        console.log(`⚠️  Evaluation completed with errors`);
+    }
+    else {
+        console.log(`✅ Evaluation completed`);
+    }
     console.log(`   📊 Results: ${resolve(resultsDir, "score-summary.json")}`);
     if (reportWritten) {
         console.log(`   📝 Report:  ${resolve(resultsDir, "report.md")}`);
@@ -71,6 +83,14 @@ export async function writeRemoteResults(client, job, options) {
         console.log(`   🔗 Studio:  ${job.reportUrl}`);
     }
     console.log(`   🏷️  Job ID:  ${job.jobId}`);
+    if (job.error) {
+        console.error("");
+        console.error(`   ❌ Step "${job.error.step ?? "<unknown>"}" failed: ${job.error.message}`);
+        if (!job.reportId) {
+            console.error("   No report was published. See the API gateway run page for details.");
+        }
+    }
+    return { degraded };
 }
 // ---------------------------------------------------------------------------
 // Helpers