@sanity/ailf 2.0.1 → 2.0.2

package/dist/orchestration/steps/run-eval-step.js

@@ -29,7 +29,7 @@ export class RunEvalStep {
         const start = Date.now();
         const { rootDir, debug, concurrency, noCache } = ctx.config;
         // Precondition: config file exists
-        const configIssues = checkGeneratedConfigsExist(rootDir);
+        const configIssues = checkGeneratedConfigsExist(rootDir, this.mode);
         const configErrors = configIssues.filter((i) => i.severity === "error");
         if (configErrors.length > 0) {
             return {

package/dist/pipeline/checks.d.ts

@@ -23,10 +23,15 @@ export declare function checkContextsExist(rootDir: string, areas: string[]): Va
  */
 export declare function checkEnvironment(rootDir: string): ValidationIssue[];
 /**
- * Check that the baseline `promptfooconfig.yaml` exists. Optionally check
- * for `promptfooconfig.observed.yaml` and `promptfooconfig.agentic.yaml`.
+ * Check that the generated promptfoo config for a given mode exists.
+ *
+ * When `mode` is provided, checks only for that mode's config file
+ * (e.g. `promptfooconfig.agent-harness.yaml` for mode `"agent-harness"`).
+ *
+ * When `mode` is omitted, falls back to the legacy literacy check:
+ * baseline `promptfooconfig.yaml` (required) plus optional observed/agentic.
  */
-export declare function checkGeneratedConfigsExist(rootDir: string): ValidationIssue[];
+export declare function checkGeneratedConfigsExist(rootDir: string, mode?: string): ValidationIssue[];
 /**
  * Check that the eval results JSON file exists, is valid JSON, and contains
  * a `results` array.

package/dist/pipeline/checks.js

@@ -8,6 +8,7 @@
 import { config as loadEnv } from "dotenv";
 import { existsSync, readFileSync, statSync } from "fs";
 import { join, resolve } from "path";
+import { configFileForMode } from "./eval-constants.js";
 // ---------------------------------------------------------------------------
 // Precondition: contexts exist for each feature area
 // ---------------------------------------------------------------------------
@@ -109,11 +110,30 @@ export function checkEnvironment(rootDir) {
 // Postcondition: score summary is valid
 // ---------------------------------------------------------------------------
 /**
- * Check that the baseline `promptfooconfig.yaml` exists. Optionally check
- * for `promptfooconfig.observed.yaml` and `promptfooconfig.agentic.yaml`.
+ * Check that the generated promptfoo config for a given mode exists.
+ *
+ * When `mode` is provided, checks only for that mode's config file
+ * (e.g. `promptfooconfig.agent-harness.yaml` for mode `"agent-harness"`).
+ *
+ * When `mode` is omitted, falls back to the legacy literacy check:
+ * baseline `promptfooconfig.yaml` (required) plus optional observed/agentic.
  */
-export function checkGeneratedConfigsExist(rootDir) {
+export function checkGeneratedConfigsExist(rootDir, mode) {
     const issues = [];
+    if (mode) {
+        const configName = configFileForMode(mode);
+        const configPath = resolve(rootDir, configName);
+        if (!existsSync(configPath)) {
+            issues.push({
+                message: `Config '${configName}' not found for mode '${mode}'. Run the pipeline to generate it.`,
+                path: configPath,
+                severity: "error",
+                source: "checkGeneratedConfigsExist",
+            });
+        }
+        return issues;
+    }
+    // Legacy literacy check: baseline required, observed/agentic optional
     const baselinePath = resolve(rootDir, "promptfooconfig.yaml");
     if (!existsSync(baselinePath)) {
         issues.push({

package/package.json

@@ -1,9 +1,9 @@
 {
   "name": "@sanity/ailf",
-  "version": "2.0.1",
+  "version": "2.0.2",
   "private": false,
   "publishConfig": {
-    "access": "restricted"
+    "access": "public"
   },
   "license": "MIT",
   "repository": {