@sanity/ailf 3.5.0 → 3.5.1

package/bin/ailf.js

@@ -33,9 +33,24 @@ const callerCwd = process.cwd()
 // ---------------------------------------------------------------------------
 if (existsSync(tsSrc)) {
   try {
+    // Enable the `ailf-source` export condition so @sanity/ailf-shared and
+    // @sanity/ailf-core resolve to their `src/index.ts` entrypoints rather
+    // than whatever happens to be in their `dist/` directories. Without
+    // this, running `ailf …` against a freshly pulled monorepo (or any
+    // workspace with a stale dist) fails at import time whenever the
+    // source introduces a new export that the dist hasn't caught up with.
+    const existingNodeOptions = process.env.NODE_OPTIONS ?? ""
+    const conditionFlag = "--conditions=ailf-source"
+    const nodeOptions = existingNodeOptions.includes(conditionFlag)
+      ? existingNodeOptions
+      : `${existingNodeOptions} ${conditionFlag}`.trim()
     execFileSync("npx", ["tsx", tsSrc, ...args], {
       cwd: ROOT,
-      env: { ...process.env, AILF_CALLER_CWD: callerCwd },
+      env: {
+        ...process.env,
+        AILF_CALLER_CWD: callerCwd,
+        NODE_OPTIONS: nodeOptions,
+      },
       stdio: "inherit",
     })
     process.exit(0)

package/config/bigquery/README.md

@@ -39,26 +39,55 @@ from `docs/design-docs/report-store/bigquery.md`.
 ### 1. Create the raw dataset (Airbyte writes here)
 
 ```bash
-bq mk --dataset data-platform-302218:ailf_raw
+bq --project_id=data-platform-302218 --location=EU mk --dataset ailf_raw
 ```
 
 ### 2. Create the analytics dataset (views live here)
 
 ```bash
-bq mk --dataset data-platform-302218:ailf
+bq --project_id=data-platform-302218 --location=EU mk --dataset ailf
 ```
 
 ### 3. Create the views
 
+**Important ordering (learned 2026-04-23):** Airbyte must be redeployed with the
+current manifest **before** you run these view SQLs. Each view binds to specific
+columns on `ailf_raw.reports`; if the raw table is missing columns the Airbyte
+projection expects, the `CREATE VIEW` statement fails with
+`Unrecognized name: <column>`.
+
+If your Airbyte destination has **schema evolution enabled** ("Propagate column
+changes" in the UI), new columns appear automatically on the next incremental
+sync. If not, flip it on, trigger a resync, and confirm the expected columns
+exist before creating views:
+
+```bash
+bq --project_id=data-platform-302218 --location=EU query --use_legacy_sql=false \
+  "SELECT column_name FROM ailf_raw.INFORMATION_SCHEMA.COLUMNS WHERE table_name = 'reports' ORDER BY column_name"
+```
+
+If propagation is disabled and you can't flip it quickly, manually
+`ALTER TABLE ailf_raw.reports ADD COLUMN IF NOT EXISTS …` for each missing
+column as a stop-gap. Values will be `NULL` until Airbyte writes to them on the
+next sync.
+
+Once the raw table has the expected columns:
+
 ```bash
-bq query --use_legacy_sql=false < views/reports.sql
-bq query --use_legacy_sql=false < views/area_scores.sql
-bq query --use_legacy_sql=false < views/official_runs.sql
-bq query --use_legacy_sql=false < views/official_area_scores.sql
+cd packages/eval/config/bigquery
+bq --project_id=data-platform-302218 --location=EU query --use_legacy_sql=false < views/reports.sql
+bq --project_id=data-platform-302218 --location=EU query --use_legacy_sql=false < views/area_scores.sql
+bq --project_id=data-platform-302218 --location=EU query --use_legacy_sql=false < views/official_runs.sql
+bq --project_id=data-platform-302218 --location=EU query --use_legacy_sql=false < views/official_area_scores.sql
 # per-team views are optional — copy views/team_runs_template.sql,
 # fill in the slug, and run.
 ```
 
+> `--project_id` / `--location=EU` are required because `bq` needs an explicit
+> billing project and the `ailf*` datasets live in the EU multi-region. If you
+> run `bq query` from this repo regularly, consider setting the default with
+> `gcloud config set project data-platform-302218`.
+
 ## Naming conventions
 
 - **`ailf_raw.*`** — raw Airbyte-loaded tables (nested JSON, Airbyte metadata

package/dist/adapters/api-client/build-request.d.ts

@@ -13,6 +13,16 @@
  * @see packages/eval/src/adapters/task-sources/repo-task-source.ts
  */
 import { type PipelineRequest } from "../../_vendor/ailf-core/index.d.ts";
+/**
+ * Thrown when `buildRemoteRequest` can't find any runnable tasks.
+ *
+ * The CLI catches this separately from ZodError so it can print the
+ * message without an accompanying stack trace — the message is already
+ * the whole story for the user.
+ */
+export declare class NoRunnableTasksError extends Error {
+    readonly name = "NoRunnableTasksError";
+}
 /** Options for building a remote pipeline request. */
 export interface BuildRequestOptions {
     /** Path to .ailf/tasks/ directory. */
@@ -27,6 +37,7 @@ export interface BuildRequestOptions {
  */
 export interface RemoteConfigSlice {
     mode?: string;
+    variant?: string;
     debug?: {
         enabled?: boolean;
         firstN?: number;

package/dist/adapters/api-client/build-request.js

@@ -16,7 +16,6 @@ import { existsSync } from "fs";
 import { resolve } from "path";
 import { PipelineRequestSchema, } from "../../_vendor/ailf-core/index.js";
 import { LEGACY_EVAL_MODE_ALIASES, isRunClassification, } from "../../_vendor/ailf-shared/index.js";
-import { LiteracyVariant } from "../../pipeline/normalize-mode.js";
 import { RepoTaskSource } from "../task-sources/repo-task-source.js";
 const LEGACY_LITERACY_VARIANT_SET = new Set(LEGACY_EVAL_MODE_ALIASES);
 /**
@@ -27,6 +26,16 @@ const LEGACY_LITERACY_VARIANT_SET = new Set(LEGACY_EVAL_MODE_ALIASES);
 function resolveCanonicalTaskMode(configMode) {
     return LEGACY_LITERACY_VARIANT_SET.has(configMode) ? "literacy" : configMode;
 }
+/**
+ * Thrown when `buildRemoteRequest` can't find any runnable tasks.
+ *
+ * The CLI catches this separately from ZodError so it can print the
+ * message without an accompanying stack trace — the message is already
+ * the whole story for the user.
+ */
+export class NoRunnableTasksError extends Error {
+    name = "NoRunnableTasksError";
+}
 // ---------------------------------------------------------------------------
 // Public API
 // ---------------------------------------------------------------------------
@@ -56,11 +65,13 @@ export async function buildRemoteRequest(options) {
         ? allTasks.filter((t) => t.mode === taskModeFilter)
         : allTasks;
     if (tasks.length === 0) {
-        throw new Error("No tasks found after applying filters.\n" +
-            `  Tasks directory: ${tasksDir}\n` +
-            (config.areas ? `  Area filter: ${config.areas.join(", ")}\n` : "") +
-            (config.tasks ? `  Task filter: ${config.tasks.join(", ")}\n` : "") +
-            "  Check that your .ailf/tasks/ YAML files define tasks matching these filters.");
+        throw await emptyTasksError({
+            taskSource,
+            tasksDir,
+            config,
+            filterOptions,
+            taskModeFilter,
+        });
     }
     // 2. Convert tasks to inline format
     const inlineTasks = tasks.map(taskToInlineFormat);
@@ -69,10 +80,14 @@ export async function buildRemoteRequest(options) {
         taskMode: "inline",
         inlineTasks,
     };
-    // Mode
-    if (config.mode && config.mode !== LiteracyVariant.FULL) {
+    // Mode + variant — send both when set so the server sees the caller's
+    // canonical intent. Legacy aliases ("full", "baseline", …) are accepted
+    // by `PipelineRequestSchema.mode` for back-compat but the CLI now emits
+    // the canonical form (`mode: "literacy"` + explicit `variant`).
+    if (config.mode)
         raw.mode = config.mode;
-    }
+    if (config.variant)
+        raw.variant = config.variant;
     // Debug
     if (config.debug?.enabled) {
         raw.debug = config.debug;
@@ -206,6 +221,88 @@ function taskToInlineFormat(task) {
     }
     return inline;
 }
+/**
+ * Build a descriptive error when the task list is empty after filtering.
+ *
+ * Loads the full task list a second time with `includeDrafts: true` so we
+ * can distinguish the two common failure modes:
+ *
+ * 1. Every discovered task is non-active (`status: "draft"` from
+ *    `ailf init` scaffolding, or `status: "paused"`). Tell the user how
+ *    to opt a task in.
+ * 2. The tasks directory is genuinely empty for this filter combination.
+ *    Echo the filters back so the mismatch is obvious.
+ *
+ * The directory-missing and file-missing cases are already surfaced
+ * earlier by `RepoTaskSource.loadTasks()`, so we never reach this helper
+ * for those.
+ */
+async function emptyTasksError(args) {
+    const { taskSource, tasksDir, config, filterOptions, taskModeFilter } = args;
+    // Re-load without the status gate to categorize what got filtered.
+    let relaxed = [];
+    try {
+        relaxed = await taskSource.loadTasks({
+            ...(filterOptions ?? {}),
+            includeDrafts: true,
+        });
+    }
+    catch {
+        // Fall through to the generic message if re-loading fails for any
+        // reason (e.g. directory removed mid-run).
+    }
+    const modeMatched = taskModeFilter
+        ? relaxed.filter((t) => t.mode === taskModeFilter)
+        : relaxed;
+    const drafts = modeMatched.filter((t) => (t.status ?? "active") === "draft");
+    const paused = modeMatched.filter((t) => t.status === "paused");
+    const filtersBlock = (config.areas?.length
+        ? `  Area filter: ${config.areas.join(", ")}\n`
+        : "") +
+        (config.tasks?.length
+            ? `  Task filter: ${config.tasks.join(", ")}\n`
+            : "") +
+        (config.tags?.length ? `  Tag filter: ${config.tags.join(", ")}\n` : "") +
+        (taskModeFilter ? `  Mode filter: ${taskModeFilter}\n` : "");
+    if (modeMatched.length === 0) {
+        return new NoRunnableTasksError("No tasks matched your filters.\n" +
+            `  Tasks directory: ${tasksDir}\n` +
+            filtersBlock +
+            "  Check that your .ailf/tasks/ YAML or .task.ts files define tasks\n" +
+            "  matching these filters.");
+    }
+    // All matched tasks were excluded by the status gate.
+    const draftIds = drafts.map((t) => t.id);
+    const pausedIds = paused.map((t) => t.id);
+    const draftSample = draftIds.slice(0, 3).join(", ");
+    const draftMore = draftIds.length > 3 ? `, +${draftIds.length - 3} more` : "";
+    const pausedSample = pausedIds.slice(0, 3).join(", ");
+    const pausedMore = pausedIds.length > 3 ? `, +${pausedIds.length - 3} more` : "";
+    const lines = [];
+    lines.push("No runnable tasks after applying filters.");
+    lines.push(`  Tasks directory: ${tasksDir}`);
+    if (filtersBlock)
+        lines.push(filtersBlock.trimEnd());
+    if (drafts.length > 0) {
+        lines.push(`  ${drafts.length} task(s) skipped because status: "draft": ${draftSample}${draftMore}`);
+    }
+    if (paused.length > 0) {
+        lines.push(`  ${paused.length} task(s) skipped because status: "paused": ${pausedSample}${pausedMore}`);
+    }
+    lines.push("");
+    lines.push("  To run one of these anyway, either:");
+    if (drafts.length > 0) {
+        lines.push(`    • Change the task's status field from "draft" to "active", or`);
+        lines.push(`    • Target it explicitly: --task ${drafts[0]?.id ?? "<id>"}`);
+    }
+    else if (paused.length > 0) {
+        lines.push(`    • Target it explicitly by id: --task ${paused[0]?.id ?? "<id>"}, or`);
+        lines.push(`    • Flip its status from "paused" to "active"`);
+    }
+    lines.push("  Tasks scaffolded by `ailf init` ship as drafts so you can edit");
+    lines.push("  them before they start contributing to your literacy score.");
+    return new NoRunnableTasksError(lines.join("\n"));
+}
 function buildFilterOptions(config) {
     const areas = config.areas?.length ? config.areas : undefined;
     const taskIds = config.tasks?.length ? config.tasks : undefined;

package/dist/adapters/api-client/index.d.ts

@@ -5,7 +5,7 @@
  *   import { ApiClient, buildRemoteRequest, resolveTasksDir } from "./adapters/api-client/index.js"
  */
 export { ApiClient } from "./api-client.js";
-export { buildCallerEnvelope, buildRemoteRequest, resolveTasksDir, type BuildRequestOptions, type RemoteConfigSlice, } from "./build-request.js";
+export { buildCallerEnvelope, buildRemoteRequest, NoRunnableTasksError, resolveTasksDir, type BuildRequestOptions, type RemoteConfigSlice, } from "./build-request.js";
 export { ApiAuthError, ApiConnectionError, ApiError, ApiTimeoutError, } from "./errors.js";
 export { formatJobError } from "./format-error.js";
 export { createProgressDisplay } from "./progress.js";

package/dist/adapters/api-client/index.js

@@ -5,7 +5,7 @@
  *   import { ApiClient, buildRemoteRequest, resolveTasksDir } from "./adapters/api-client/index.js"
  */
 export { ApiClient } from "./api-client.js";
-export { buildCallerEnvelope, buildRemoteRequest, resolveTasksDir, } from "./build-request.js";
+export { buildCallerEnvelope, buildRemoteRequest, NoRunnableTasksError, resolveTasksDir, } from "./build-request.js";
 export { ApiAuthError, ApiConnectionError, ApiError, ApiTimeoutError, } from "./errors.js";
 export { formatJobError } from "./format-error.js";
 export { createProgressDisplay } from "./progress.js";

package/dist/commands/pipeline-action.js

@@ -14,7 +14,7 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
 import { dirname, resolve } from "path";
 import { fileURLToPath } from "url";
 import { classifyUrls } from "../pipeline/classify-url.js";
-import { normalizeMode } from "../pipeline/normalize-mode.js";
+import { LiteracyVariant, normalizeMode } from "../pipeline/normalize-mode.js";
 import { assessImpact, buildReverseMapping, } from "../pipeline/reverse-mapping.js";
 import { buildAppContext, parseArtifactUploadEnv, } from "../orchestration/build-app-context.js";
 import { buildStepSequence } from "../orchestration/build-step-sequence.js";
@@ -47,6 +47,13 @@ export function computeResolvedOptions(opts) {
         mode = normalized.mode;
         // Explicit --variant flag takes precedence over what normalizeMode inferred
         variant = opts.variant ?? normalized.variant;
+        // Canonical mode "literacy" with no variant defaults to the full variant
+        // (standard + agentic). This preserves the pre-canonical CLI behavior
+        // where `--mode full` was the default, without emitting the legacy alias
+        // deprecation warning for users who pass no flags at all.
+        if (mode === "literacy" && !variant) {
+            variant = LiteracyVariant.FULL;
+        }
     }
     catch (err) {
         console.error(`❌ ${err instanceof Error ? err.message : String(err)}`);

package/dist/commands/pipeline.js

@@ -8,12 +8,11 @@
  * @see docs/cli.md for the full flag reference.
  */
 import { Command } from "commander";
-import { LiteracyVariant } from "../pipeline/normalize-mode.js";
 import { addAgenticOptions, addDebugOptions, addSanitySourceOptions, } from "./shared/options.js";
 export function createPipelineCommand() {
     const cmd = new Command("pipeline")
         .description("Run the full evaluation pipeline")
-        .option("-m, --mode <mode>", "Evaluation mode: literacy (default), mcp-server, agent-harness, knowledge-probe, custom. Legacy aliases (baseline, agentic, observed, full) are accepted and normalized to literacy + variant.", LiteracyVariant.FULL)
+        .option("-m, --mode <mode>", "Evaluation mode: literacy (default), mcp-server, agent-harness, knowledge-probe, custom. Legacy aliases (baseline, agentic, observed, full) are accepted and normalized to literacy + variant.", "literacy")
         .option("--variant <variant>", "Literacy variant: full (default — standard + agentic), baseline (standard only), agentic (agentic only), observed. Only applies to --mode literacy.")
         .option("-s, --source <name>", "Documentation source name (from sources.yaml)")
         .option("-n, --dry-run", "Validate configuration only, no execution", false)

package/dist/commands/remote-pipeline.js

@@ -14,7 +14,7 @@
  * @see docs/design-docs/cli-as-api-client.md — design doc
  */
 import { ZodError } from "zod";
-import { ApiClient, buildRemoteRequest, createProgressDisplay, formatJobError, resolveTasksDir, } from "../adapters/api-client/index.js";
+import { ApiClient, buildRemoteRequest, createProgressDisplay, formatJobError, NoRunnableTasksError, resolveTasksDir, } from "../adapters/api-client/index.js";
 import { writeRemoteResults } from "./remote-results.js";
 // ---------------------------------------------------------------------------
 // Public API
@@ -66,6 +66,10 @@ export async function runRemotePipeline(opts, rootDir) {
             console.error("💡 Fix the issues above in your .ailf/tasks/ YAML files.");
             process.exit(2);
         }
+        if (err instanceof NoRunnableTasksError) {
+            console.error(`❌ ${err.message}`);
+            process.exit(2);
+        }
         throw err;
     }
     console.log(`📦 Found ${taskCount} task(s) in ${tasksDir}`);
@@ -103,6 +107,7 @@ export async function runRemotePipeline(opts, rootDir) {
 function toConfigSlice(opts) {
     return {
         mode: opts.mode,
+        variant: opts.variant,
         debug: opts.debug,
         areas: opts.areaOption
             ?.split(",")

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/ailf",
-  "version": "3.5.0",
+  "version": "3.5.1",
   "private": false,
   "publishConfig": {
     "access": "public"