@sanity/ailf 3.8.0 → 3.8.1

package/dist/adapters/config-sources/file-config-adapter.js

@@ -47,7 +47,7 @@ export class FileConfigAdapter {
             return this.validateAndMap(result.value, ext);
         }
         // YAML / JSON files — load via fs
-        const raw = readConfigFile(this.filePath);
+        const raw = await readConfigFile(this.filePath);
         return this.validateAndMap(raw, ext);
     }
     /**
@@ -69,13 +69,12 @@ export class FileConfigAdapter {
 // ---------------------------------------------------------------------------
 // Helpers
 // ---------------------------------------------------------------------------
-function readConfigFile(filePath) {
+async function readConfigFile(filePath) {
     const content = readFileSync(filePath, "utf-8");
     const ext = extname(filePath).toLowerCase();
     if (ext === ".yaml" || ext === ".yml") {
-        // Dynamic import for yaml parser — only needed when reading YAML configs
-        // eslint-disable-next-line @typescript-eslint/no-require-imports
-        const { parse } = require("yaml");
+        // Dynamic ESM import — only loaded when reading YAML configs.
+        const { parse } = await import("yaml");
         return parse(content);
     }
     return JSON.parse(content);

package/dist/adapters/task-sources/repo-schemas.d.ts

@@ -147,8 +147,8 @@ export declare const CanonicalTaskSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
     baseline: z.ZodOptional<z.ZodObject<{
         enabled: z.ZodOptional<z.ZodBoolean>;
         rubric: z.ZodOptional<z.ZodEnum<{
-            full: "full";
             abbreviated: "abbreviated";
+            full: "full";
             none: "none";
         }>>;
     }, z.core.$strip>>;
@@ -773,8 +773,8 @@ export declare const ContentLakeAuthorableTaskSchema: z.ZodObject<{
     baseline: z.ZodOptional<z.ZodObject<{
         enabled: z.ZodOptional<z.ZodBoolean>;
         rubric: z.ZodOptional<z.ZodEnum<{
-            full: "full";
             abbreviated: "abbreviated";
+            full: "full";
             none: "none";
         }>>;
     }, z.core.$strip>>;
@@ -893,8 +893,8 @@ export declare const CanonicalTaskFileSchema: z.ZodArray<z.ZodDiscriminatedUnion
     baseline: z.ZodOptional<z.ZodObject<{
         enabled: z.ZodOptional<z.ZodBoolean>;
         rubric: z.ZodOptional<z.ZodEnum<{
-            full: "full";
             abbreviated: "abbreviated";
+            full: "full";
             none: "none";
         }>>;
     }, z.core.$strip>>;

package/dist/cli-program.d.ts

@@ -0,0 +1,39 @@
+/**
+ * cli-program.ts — pure factory for the AILF Commander program.
+ *
+ * Splits the program construction out of cli.ts so the CLI is testable
+ * in-process. cli.ts owns bootstrap side effects (dotenv loading,
+ * retired-flag/env/cmd checks, AILF_LOG_LEVEL pre-scan, parseAsync); this
+ * module owns command wiring.
+ *
+ * The W0078 M4 black-box harness imports `buildCliProgram()` directly so
+ * tests can construct a fresh program, attach `exitOverride()`, capture
+ * stdout/stderr, and parse a synthetic argv — all without spawning a
+ * subprocess.
+ *
+ * @see packages/eval/src/__tests__/cli-harness/run-cli.ts
+ */
+import { Command } from "commander";
+/**
+ * Options for `buildCliProgram`.
+ */
+export interface BuildCliProgramOptions {
+    /**
+     * Path to the eval package root (the directory containing package.json).
+     * Used to resolve the version string and as the root passed to the
+     * `--explain` handler.
+     */
+    evalRoot: string;
+}
+/**
+ * Construct the Commander program with every subcommand registered.
+ *
+ * Pure factory — no I/O beyond reading package.json for the version, no
+ * `process.exit()`, no `process.argv` access. Tests can call this and
+ * attach `program.exitOverride()` before parsing to capture exit codes
+ * instead of terminating the process.
+ *
+ * Registration order determines group display order in `--help`. Commands
+ * within a group appear in the order they're added.
+ */
+export declare function buildCliProgram(opts: BuildCliProgramOptions): Command;

package/dist/cli-program.js

@@ -0,0 +1,137 @@
+/**
+ * cli-program.ts — pure factory for the AILF Commander program.
+ *
+ * Splits the program construction out of cli.ts so the CLI is testable
+ * in-process. cli.ts owns bootstrap side effects (dotenv loading,
+ * retired-flag/env/cmd checks, AILF_LOG_LEVEL pre-scan, parseAsync); this
+ * module owns command wiring.
+ *
+ * The W0078 M4 black-box harness imports `buildCliProgram()` directly so
+ * tests can construct a fresh program, attach `exitOverride()`, capture
+ * stdout/stderr, and parse a synthetic argv — all without spawning a
+ * subprocess.
+ *
+ * @see packages/eval/src/__tests__/cli-harness/run-cli.ts
+ */
+import { Command } from "commander";
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { CommandGroup, configureProgram } from "./commands/shared/help.js";
+import { createAgentReportCommand } from "./commands/agent-report.js";
+import { createBaselineCommand } from "./commands/baseline.js";
+import { createCacheCommand } from "./commands/cache.js";
+import { createCalculateScoresCommand } from "./commands/calculate-scores.js";
+import { createCheckStalenessCommand } from "./commands/check-staleness.js";
+import { createChronicFailuresCommand } from "./commands/chronic-failures.js";
+import { createCompareCommand } from "./commands/compare.js";
+import { createCompletionCommand } from "./commands/completion.js";
+import { createCoverageAuditCommand } from "./commands/coverage-audit.js";
+import { createDiscoveryReportCommand } from "./commands/discovery-report.js";
+import { createEvalCommand } from "./commands/eval.js";
+import { createFetchDocsCommand } from "./commands/fetch-docs.js";
+import { createGenerateConfigsCommand } from "./commands/generate-configs.js";
+import { createGraderCommand } from "./commands/grader/index.js";
+import { createInitCommand } from "./commands/init.js";
+import { createInteractiveCommand } from "./commands/interactive.js";
+import { createLookupDocCommand } from "./commands/lookup-doc.js";
+import { createMeasureRetrievalCommand } from "./commands/measure-retrieval.js";
+import { createPrCommentCommand } from "./commands/pr-comment.js";
+import { createPublishCommand } from "./commands/publish.js";
+import { createReadinessReportCommand } from "./commands/readiness-report.js";
+import { createRunCommand } from "./commands/run.js";
+import { createRunsCommand } from "./commands/runs.js";
+import { createValidateConfigCommand } from "./commands/validate.js";
+import { createValidateTasksCommand } from "./commands/validate-tasks.js";
+import { createWebhookServerCommand } from "./commands/webhook-server.js";
+import { createWeeklyDigestCommand } from "./commands/weekly-digest.js";
+/**
+ * Construct the Commander program with every subcommand registered.
+ *
+ * Pure factory — no I/O beyond reading package.json for the version, no
+ * `process.exit()`, no `process.argv` access. Tests can call this and
+ * attach `program.exitOverride()` before parsing to capture exit codes
+ * instead of terminating the process.
+ *
+ * Registration order determines group display order in `--help`. Commands
+ * within a group appear in the order they're added.
+ */
+export function buildCliProgram(opts) {
+    const { evalRoot } = opts;
+    const pkgPath = resolve(evalRoot, "package.json");
+    const pkg = JSON.parse(readFileSync(pkgPath, "utf-8"));
+    const program = new Command()
+        .name("ailf")
+        .description("AI Literacy Framework — evaluate how well docs enable AI coding tools\n\nMeasure whether AI coding agents can find the right documentation\nand produce correct implementations of your product features.")
+        .version(pkg.version)
+        .option("-v, --verbose", "Increase log output")
+        .option("-q, --quiet", "Suppress non-error output")
+        .option("--dotenv <path>", "Override default .env file path")
+        .option("--explain", "Show execution plan without running")
+        .option("--format <fmt>", "Output format for --explain (console, json)", "console")
+        .option("-y, --yes", "With --explain: show plan then prompt to confirm execution");
+    configureProgram(program);
+    // Global --explain hook — intercepts any command before execution
+    program.hook("preAction", async (thisCommand, actionCommand) => {
+        const globalOpts = thisCommand.opts();
+        if (!globalOpts.explain)
+            return;
+        const { handleExplain } = await import("./commands/explain-handler.js");
+        try {
+            await handleExplain(actionCommand, globalOpts.yes ?? false, evalRoot);
+            process.exit(0);
+        }
+        catch (err) {
+            // Sentinel from --yes confirmation: user wants to proceed
+            if (err !== null &&
+                typeof err === "object" &&
+                "__proceedArgv" in err) {
+                const filteredArgv = err.__proceedArgv;
+                console.log("\n  ▸ Proceeding with execution…\n");
+                await program.parseAsync(filteredArgv);
+                return;
+            }
+            throw err;
+        }
+    });
+    // ── Core Workflow ──────────────────────────────────────────────────────
+    program.addCommand(createRunCommand().helpGroup(CommandGroup.CoreWorkflow));
+    program.addCommand(createCompareCommand().helpGroup(CommandGroup.CoreWorkflow));
+    program.addCommand(createBaselineCommand().helpGroup(CommandGroup.CoreWorkflow));
+    program.addCommand(createPublishCommand().helpGroup(CommandGroup.CoreWorkflow));
+    program.addCommand(createRunsCommand().helpGroup(CommandGroup.CoreWorkflow));
+    // ── Analysis & Reports ────────────────────────────────────────────────
+    const reportCommand = new Command("report")
+        .description("Generate analysis and reporting outputs from evaluation runs")
+        .addCommand(createReadinessReportCommand())
+        .addCommand(createChronicFailuresCommand())
+        .addCommand(createCoverageAuditCommand())
+        .addCommand(createDiscoveryReportCommand())
+        .addCommand(createAgentReportCommand())
+        .addCommand(createWeeklyDigestCommand())
+        .addCommand(createCheckStalenessCommand());
+    program.addCommand(reportCommand.helpGroup(CommandGroup.AnalysisReports));
+    // ── Grader Reliability ────────────────────────────────────────────────
+    program.addCommand(createGraderCommand().helpGroup(CommandGroup.GraderReliability));
+    // ── Setup & Configuration ─────────────────────────────────────────────
+    program.addCommand(createInitCommand().helpGroup(CommandGroup.SetupConfig));
+    const validateCommand = new Command("validate")
+        .description("Validate AILF configuration and task files")
+        .addCommand(createValidateConfigCommand())
+        .addCommand(createValidateTasksCommand());
+    program.addCommand(validateCommand.helpGroup(CommandGroup.SetupConfig));
+    program.addCommand(createFetchDocsCommand().helpGroup(CommandGroup.SetupConfig));
+    program.addCommand(createCacheCommand().helpGroup(CommandGroup.SetupConfig));
+    // ── Pipeline Internals ────────────────────────────────────────────────
+    program.addCommand(createEvalCommand().helpGroup(CommandGroup.PipelineInternals));
+    program.addCommand(createCalculateScoresCommand().helpGroup(CommandGroup.PipelineInternals));
+    program.addCommand(createPrCommentCommand().helpGroup(CommandGroup.PipelineInternals));
+    program.addCommand(createGenerateConfigsCommand().helpGroup(CommandGroup.PipelineInternals));
+    program.addCommand(createMeasureRetrievalCommand().helpGroup(CommandGroup.PipelineInternals));
+    program.addCommand(createLookupDocCommand().helpGroup(CommandGroup.PipelineInternals));
+    program.addCommand(createWebhookServerCommand().helpGroup(CommandGroup.PipelineInternals));
+    // ── Developer Tools ───────────────────────────────────────────────────
+    program.addCommand(createInteractiveCommand().helpGroup(CommandGroup.DeveloperTools));
+    // Shell completion — must be registered last (needs full program tree)
+    program.addCommand(createCompletionCommand(program).helpGroup(CommandGroup.DeveloperTools));
+    return program;
+}

package/dist/cli.d.ts

@@ -25,5 +25,11 @@
  *
  * Dev mode (without building):
  *   tsx src/cli.ts run --debug
+ *
+ * Module split: this file owns *bootstrap side effects* (dotenv,
+ * retired-flag/env/cmd checks, AILF_LOG_LEVEL pre-scan, parseAsync).
+ * The Commander wiring lives in ./cli-program.ts so the W0078 M4 black-box
+ * harness can build the program in-process without firing those side
+ * effects.
  */
 export {};

package/dist/cli.js

@@ -1,6 +1,4 @@
 #!/usr/bin/env node
-/* oxlint-disable import/first -- imports are intentionally interleaved with
-   command registration for readability and lazy loading */
 /**
  * cli.ts — AILF CLI entry point.
  *
@@ -27,15 +25,20 @@
  *
  * Dev mode (without building):
  *   tsx src/cli.ts run --debug
+ *
+ * Module split: this file owns *bootstrap side effects* (dotenv,
+ * retired-flag/env/cmd checks, AILF_LOG_LEVEL pre-scan, parseAsync).
+ * The Commander wiring lives in ./cli-program.ts so the W0078 M4 black-box
+ * harness can build the program in-process without firing those side
+ * effects.
  */
 import { config as dotenvConfig } from "dotenv";
-import { existsSync, readFileSync } from "fs";
+import { existsSync } from "fs";
 import { dirname, resolve } from "path";
 import { fileURLToPath } from "url";
+import { buildCliProgram } from "./cli-program.js";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const ROOT = resolve(__dirname, "..");
-/** Path to the eval package root (packages/eval). Used by --explain. */
-const EVAL_ROOT = ROOT;
 // ---------------------------------------------------------------------------
 // Load .env — must happen before Commander parses so that .env()
 // fallbacks resolve correctly.
@@ -220,127 +223,14 @@ if (retiredFlag || retiredCommand || retiredEnv) {
     process.exit(2);
 }
 // ---------------------------------------------------------------------------
-// Build CLI program
-// ---------------------------------------------------------------------------
-import { Command } from "commander";
-import { CommandGroup, configureProgram } from "./commands/shared/help.js";
-// Read version from package.json
-const pkgPath = resolve(ROOT, "package.json");
-const pkg = JSON.parse(readFileSync(pkgPath, "utf-8"));
-const program = new Command()
-    .name("ailf")
-    .description("AI Literacy Framework — evaluate how well docs enable AI coding tools\n\nMeasure whether AI coding agents can find the right documentation\nand produce correct implementations of your product features.")
-    .version(pkg.version)
-    .option("-v, --verbose", "Increase log output")
-    .option("-q, --quiet", "Suppress non-error output")
-    .option("--dotenv <path>", "Override default .env file path")
-    .option("--explain", "Show execution plan without running")
-    .option("--format <fmt>", "Output format for --explain (console, json)", "console")
-    .option("-y, --yes", "With --explain: show plan then prompt to confirm execution");
-configureProgram(program);
-// ---------------------------------------------------------------------------
-// Global --explain hook — intercepts any command before execution
+// Build CLI program (delegates Commander wiring to ./cli-program.ts)
 // ---------------------------------------------------------------------------
-program.hook("preAction", async (thisCommand, actionCommand) => {
-    const globalOpts = thisCommand.opts();
-    if (!globalOpts.explain)
-        return;
-    const { handleExplain } = await import("./commands/explain-handler.js");
-    try {
-        await handleExplain(actionCommand, globalOpts.yes ?? false, EVAL_ROOT);
-        process.exit(0);
-    }
-    catch (err) {
-        // Sentinel from --yes confirmation: user wants to proceed
-        if (err !== null &&
-            typeof err === "object" &&
-            "__proceedArgv" in err) {
-            const filteredArgv = err.__proceedArgv;
-            console.log("\n  ▸ Proceeding with execution…\n");
-            await program.parseAsync(filteredArgv);
-            return;
-        }
-        throw err;
-    }
-});
+const program = buildCliProgram({ evalRoot: ROOT });
 // ---------------------------------------------------------------------------
-// Register commands
+// Parse and run — default to showing help when no arguments given.
 //
-// Registration order determines group display order in --help.
-// Within each group, commands appear in the order they are added.
-// ---------------------------------------------------------------------------
-// ── Core Workflow ──────────────────────────────────────────────────────
-import { createRunCommand } from "./commands/run.js";
-program.addCommand(createRunCommand().helpGroup(CommandGroup.CoreWorkflow));
-import { createCompareCommand } from "./commands/compare.js";
-program.addCommand(createCompareCommand().helpGroup(CommandGroup.CoreWorkflow));
-import { createBaselineCommand } from "./commands/baseline.js";
-program.addCommand(createBaselineCommand().helpGroup(CommandGroup.CoreWorkflow));
-import { createPublishCommand } from "./commands/publish.js";
-program.addCommand(createPublishCommand().helpGroup(CommandGroup.CoreWorkflow));
-import { createRunsCommand } from "./commands/runs.js";
-program.addCommand(createRunsCommand().helpGroup(CommandGroup.CoreWorkflow));
-// ── Analysis & Reports ────────────────────────────────────────────────
-import { createReadinessReportCommand } from "./commands/readiness-report.js";
-import { createChronicFailuresCommand } from "./commands/chronic-failures.js";
-import { createCoverageAuditCommand } from "./commands/coverage-audit.js";
-import { createDiscoveryReportCommand } from "./commands/discovery-report.js";
-import { createAgentReportCommand } from "./commands/agent-report.js";
-import { createWeeklyDigestCommand } from "./commands/weekly-digest.js";
-import { createCheckStalenessCommand } from "./commands/check-staleness.js";
-const reportCommand = new Command("report")
-    .description("Generate analysis and reporting outputs from evaluation runs")
-    .addCommand(createReadinessReportCommand())
-    .addCommand(createChronicFailuresCommand())
-    .addCommand(createCoverageAuditCommand())
-    .addCommand(createDiscoveryReportCommand())
-    .addCommand(createAgentReportCommand())
-    .addCommand(createWeeklyDigestCommand())
-    .addCommand(createCheckStalenessCommand());
-program.addCommand(reportCommand.helpGroup(CommandGroup.AnalysisReports));
-// ── Grader Reliability ────────────────────────────────────────────────
-import { createGraderCommand } from "./commands/grader/index.js";
-program.addCommand(createGraderCommand().helpGroup(CommandGroup.GraderReliability));
-// ── Setup & Configuration ─────────────────────────────────────────────
-import { createInitCommand } from "./commands/init.js";
-program.addCommand(createInitCommand().helpGroup(CommandGroup.SetupConfig));
-import { createValidateConfigCommand } from "./commands/validate.js";
-import { createValidateTasksCommand } from "./commands/validate-tasks.js";
-const validateCommand = new Command("validate")
-    .description("Validate AILF configuration and task files")
-    .addCommand(createValidateConfigCommand())
-    .addCommand(createValidateTasksCommand());
-program.addCommand(validateCommand.helpGroup(CommandGroup.SetupConfig));
-import { createFetchDocsCommand } from "./commands/fetch-docs.js";
-program.addCommand(createFetchDocsCommand().helpGroup(CommandGroup.SetupConfig));
-import { createCacheCommand } from "./commands/cache.js";
-program.addCommand(createCacheCommand().helpGroup(CommandGroup.SetupConfig));
-// ── Pipeline Internals ────────────────────────────────────────────────
-import { createEvalCommand } from "./commands/eval.js";
-program.addCommand(createEvalCommand().helpGroup(CommandGroup.PipelineInternals));
-import { createCalculateScoresCommand } from "./commands/calculate-scores.js";
-program.addCommand(createCalculateScoresCommand().helpGroup(CommandGroup.PipelineInternals));
-import { createPrCommentCommand } from "./commands/pr-comment.js";
-program.addCommand(createPrCommentCommand().helpGroup(CommandGroup.PipelineInternals));
-import { createGenerateConfigsCommand } from "./commands/generate-configs.js";
-program.addCommand(createGenerateConfigsCommand().helpGroup(CommandGroup.PipelineInternals));
-import { createMeasureRetrievalCommand } from "./commands/measure-retrieval.js";
-program.addCommand(createMeasureRetrievalCommand().helpGroup(CommandGroup.PipelineInternals));
-import { createLookupDocCommand } from "./commands/lookup-doc.js";
-program.addCommand(createLookupDocCommand().helpGroup(CommandGroup.PipelineInternals));
-import { createWebhookServerCommand } from "./commands/webhook-server.js";
-program.addCommand(createWebhookServerCommand().helpGroup(CommandGroup.PipelineInternals));
-// ── Developer Tools ───────────────────────────────────────────────────
-import { createInteractiveCommand } from "./commands/interactive.js";
-program.addCommand(createInteractiveCommand().helpGroup(CommandGroup.DeveloperTools));
-// Shell completion — must be registered last (needs full program tree)
-import { createCompletionCommand } from "./commands/completion.js";
-program.addCommand(createCompletionCommand(program).helpGroup(CommandGroup.DeveloperTools));
-// ---------------------------------------------------------------------------
-// Parse and run — default to showing help when no arguments given
-// ---------------------------------------------------------------------------
-// If no command is specified (just `ailf`), show help.
 // The interactive wizard is still available via `ailf interactive`.
+// ---------------------------------------------------------------------------
 if (process.argv.length <= 2) {
     program.outputHelp();
 }

package/dist/pipeline/mirror-repo-tasks.d.ts

@@ -107,7 +107,7 @@ export declare function buildMirrorDocument(task: LiteracyTaskDefinition, opts:
     slugToDocId: Map<string, string>;
 }): {
     baseline?: {
-        rubric?: "full" | "abbreviated" | "none" | undefined;
+        rubric?: "abbreviated" | "full" | "none" | undefined;
         enabled?: boolean | undefined;
     } | undefined;
     _id: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/ailf",
-  "version": "3.8.0",
+  "version": "3.8.1",
   "private": false,
   "publishConfig": {
     "access": "public"
@@ -50,6 +50,7 @@
     "@anthropic-ai/claude-agent-sdk": "^0.2.105",
     "@types/js-yaml": "^4.0.9",
     "@types/node": "^22.13.1",
+    "nock": "^14.0.13",
     "tsx": "^4.19.2",
     "typescript": "^5.7.3",
     "@sanity/ailf-core": "0.1.0",
@@ -73,9 +74,10 @@
     "cli": "tsx src/cli.ts",
     "pipeline": "tsx src/cli.ts pipeline",
     "validate": "tsx src/cli.ts validate config",
-    "test": "tsx --test src/__tests__/*.test.ts",
+    "test": "tsx --test src/__tests__/*.test.ts src/adapters/**/__tests__/*.adapter.test.ts",
     "test:e2e": "AILF_E2E=1 tsx --test src/__tests__/e2e/*.e2e.test.ts",
-    "test:all": "AILF_E2E=1 tsx --test src/__tests__/*.test.ts src/pipeline/compiler/__tests__/*.test.ts src/__tests__/e2e/*.e2e.test.ts",
+    "test:e2e:adapters": "AILF_E2E=1 tsx --test src/adapters/**/__tests__/*.adapter.test.ts",
+    "test:all": "AILF_E2E=1 tsx --test src/__tests__/*.test.ts src/pipeline/compiler/__tests__/*.test.ts src/__tests__/e2e/*.e2e.test.ts src/adapters/**/__tests__/*.adapter.test.ts",
     "pr-comment": "tsx src/cli.ts pr-comment",
     "coverage-audit": "tsx src/cli.ts report coverage",
     "readiness-report": "tsx src/cli.ts report readiness",

package/dist/pipeline/compiler/__tests__/agent-harness-handler.test.d.ts

@@ -1,10 +0,0 @@
-/**
- * agent-harness-handler.test.ts — Tests for agent harness mode compilation.
- *
- * Tests validation, provider assembly, tool permission resolution,
- * assertion mapping, sandbox config, lifecycle extensions, and
- * end-to-end compilation of example tasks.
- *
- * Run: npx tsx --test src/pipeline/compiler/__tests__/agent-harness-handler.test.ts
- */
-export {};