@sanity/ailf 3.7.0 → 3.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/config/airbyte/ai_literacy_framework.connector.yaml +1 -1
  2. package/config/thresholds.ts +3 -3
  3. package/dist/_vendor/ailf-core/examples/index.d.ts +2 -2
  4. package/dist/_vendor/ailf-core/examples/index.js +2 -2
  5. package/dist/_vendor/ailf-core/ports/context.d.ts +0 -4
  6. package/dist/_vendor/ailf-core/schemas/eval-config.d.ts +38 -12
  7. package/dist/_vendor/ailf-core/schemas/eval-config.js +102 -22
  8. package/dist/_vendor/ailf-core/schemas/pipeline-request.d.ts +4 -6
  9. package/dist/_vendor/ailf-core/schemas/pipeline-request.js +1 -3
  10. package/dist/_vendor/ailf-core/schemas/schedules.d.ts +2 -2
  11. package/dist/_vendor/ailf-shared/run-classification.d.ts +2 -2
  12. package/dist/_vendor/ailf-shared/run-classification.js +1 -1
  13. package/dist/_vendor/ailf-shared/run-context.d.ts +1 -1
  14. package/dist/adapters/api-client/build-request.d.ts +0 -2
  15. package/dist/adapters/api-client/build-request.js +2 -6
  16. package/dist/adapters/config-sources/cli-config-adapter.d.ts +1 -1
  17. package/dist/adapters/config-sources/file-config-adapter.d.ts +1 -1
  18. package/dist/adapters/config-sources/file-config-adapter.js +42 -17
  19. package/dist/adapters/task-sources/repo-schemas.d.ts +41 -3
  20. package/dist/adapters/task-sources/repo-schemas.js +127 -0
  21. package/dist/cli-program.d.ts +39 -0
  22. package/dist/cli-program.js +137 -0
  23. package/dist/cli.d.ts +8 -2
  24. package/dist/cli.js +128 -142
  25. package/dist/commands/agent-report.js +1 -1
  26. package/dist/commands/calculate-scores.js +0 -2
  27. package/dist/commands/check-staleness.js +1 -1
  28. package/dist/commands/chronic-failures.js +4 -4
  29. package/dist/commands/coverage-audit.js +6 -7
  30. package/dist/commands/discovery-report.js +16 -4
  31. package/dist/commands/eval.d.ts +1 -1
  32. package/dist/commands/eval.js +1 -1
  33. package/dist/commands/explain-handler.d.ts +1 -1
  34. package/dist/commands/explain-handler.js +13 -44
  35. package/dist/commands/fetch-docs.js +0 -2
  36. package/dist/commands/generate-configs.js +0 -2
  37. package/dist/commands/grader/index.js +3 -3
  38. package/dist/commands/init.d.ts +2 -2
  39. package/dist/commands/init.js +10 -9
  40. package/dist/commands/interactive.d.ts +1 -1
  41. package/dist/commands/interactive.js +8 -8
  42. package/dist/commands/pipeline-action.d.ts +1 -3
  43. package/dist/commands/pipeline-action.js +174 -140
  44. package/dist/commands/pr-comment.js +1 -3
  45. package/dist/commands/publish.d.ts +1 -1
  46. package/dist/commands/publish.js +2 -4
  47. package/dist/commands/readiness-report.js +17 -8
  48. package/dist/commands/remote-pipeline.d.ts +1 -1
  49. package/dist/commands/remote-pipeline.js +1 -3
  50. package/dist/commands/run.d.ts +64 -0
  51. package/dist/commands/{pipeline.js → run.js} +19 -30
  52. package/dist/commands/shared/help.js +4 -4
  53. package/dist/commands/shared/options.d.ts +29 -3
  54. package/dist/commands/shared/options.js +37 -13
  55. package/dist/commands/validate-tasks.js +1 -1
  56. package/dist/commands/validate.d.ts +1 -1
  57. package/dist/commands/validate.js +2 -2
  58. package/dist/commands/weekly-digest.js +3 -3
  59. package/dist/config/thresholds.ts +3 -3
  60. package/dist/orchestration/build-app-context.js +0 -2
  61. package/dist/orchestration/build-step-sequence.js +1 -11
  62. package/dist/orchestration/steps/fetch-docs-step.js +1 -1
  63. package/dist/orchestration/steps/index.d.ts +0 -2
  64. package/dist/orchestration/steps/index.js +0 -2
  65. package/dist/orchestration/steps/run-eval-step.js +1 -1
  66. package/dist/pipeline/cache.d.ts +1 -1
  67. package/dist/pipeline/map-request-to-config.js +0 -2
  68. package/dist/pipeline/mirror-repo-tasks.d.ts +1 -1
  69. package/dist/pipeline/plan.d.ts +2 -4
  70. package/dist/pipeline/plan.js +4 -32
  71. package/dist/pipeline/run-context.d.ts +1 -1
  72. package/dist/pipeline/run-context.js +4 -4
  73. package/dist/pipeline/validate.d.ts +1 -1
  74. package/dist/pipeline/validate.js +1 -1
  75. package/package.json +11 -9
  76. package/dist/commands/pipeline.d.ts +0 -77
  77. package/dist/orchestration/steps/discovery-report-step.d.ts +0 -13
  78. package/dist/orchestration/steps/discovery-report-step.js +0 -62
  79. package/dist/orchestration/steps/readiness-step.d.ts +0 -13
  80. package/dist/orchestration/steps/readiness-step.js +0 -98
  81. package/dist/pipeline/compiler/__tests__/agent-harness-handler.test.d.ts +0 -10
  82. package/dist/pipeline/compiler/__tests__/agent-harness-handler.test.js +0 -366
  83. package/dist/pipeline/compiler/__tests__/assertion-mapper.test.d.ts +0 -9
  84. package/dist/pipeline/compiler/__tests__/assertion-mapper.test.js +0 -145
  85. package/dist/pipeline/compiler/__tests__/knowledge-probe-handler.test.d.ts +0 -10
  86. package/dist/pipeline/compiler/__tests__/knowledge-probe-handler.test.js +0 -314
  87. package/dist/pipeline/compiler/__tests__/literacy-handler.test.d.ts +0 -10
  88. package/dist/pipeline/compiler/__tests__/literacy-handler.test.js +0 -486
  89. package/dist/pipeline/compiler/__tests__/mcp-server-handler.test.d.ts +0 -10
  90. package/dist/pipeline/compiler/__tests__/mcp-server-handler.test.js +0 -425
  91. package/dist/pipeline/compiler/__tests__/promptfoo-compiler.test.d.ts +0 -9
  92. package/dist/pipeline/compiler/__tests__/promptfoo-compiler.test.js +0 -332
  93. package/dist/pipeline/compiler/__tests__/sandbox-and-fixtures.test.d.ts +0 -12
  94. package/dist/pipeline/compiler/__tests__/sandbox-and-fixtures.test.js +0 -210
  95. package/dist/pipeline/compiler/__tests__/scoring-and-presets.test.d.ts +0 -7
  96. package/dist/pipeline/compiler/__tests__/scoring-and-presets.test.js +0 -404
  97. package/dist/pipeline/compiler/__tests__/scoring-bridge.test.d.ts +0 -10
  98. package/dist/pipeline/compiler/__tests__/scoring-bridge.test.js +0 -184
  99. package/dist/pipeline/compiler/__tests__/task-graph-builder.test.d.ts +0 -8
  100. package/dist/pipeline/compiler/__tests__/task-graph-builder.test.js +0 -301
  101. package/dist/pipeline/compiler/__tests__/telemetry.test.d.ts +0 -9
  102. package/dist/pipeline/compiler/__tests__/telemetry.test.js +0 -503
  103. package/dist/pipeline/compiler/__tests__/tool-loop-openai.test.d.ts +0 -10
  104. package/dist/pipeline/compiler/__tests__/tool-loop-openai.test.js +0 -509
@@ -4,7 +4,7 @@
4
4
  *
5
5
  * Reads a score-summary.json (defaulting to results/latest/score-summary.json),
6
6
  * builds provenance, writes the report to Sanity, and fans out to configured
7
- * sinks — exactly the same as the publish step in `ailf pipeline`, but
7
+ * sinks — exactly the same as the publish step in `ailf run`, but
8
8
  * standalone.
9
9
  *
10
10
  * Uses createAppContext() (composition root) for all infrastructure access.
@@ -4,7 +4,7 @@
4
4
  *
5
5
  * Reads a score-summary.json (defaulting to results/latest/score-summary.json),
6
6
  * builds provenance, writes the report to Sanity, and fans out to configured
7
- * sinks — exactly the same as the publish step in `ailf pipeline`, but
7
+ * sinks — exactly the same as the publish step in `ailf run`, but
8
8
  * standalone.
9
9
  *
10
10
  * Uses createAppContext() (composition root) for all infrastructure access.
@@ -88,7 +88,6 @@ async function runPublishCommand(summaryPath, outputDir, opts) {
88
88
  // Wire up infrastructure via composition root
89
89
  const ctx = createAppContext({
90
90
  compareEnabled: false,
91
- discoveryReportEnabled: false,
92
91
  gapAnalysisEnabled: false,
93
92
  mode: "literacy",
94
93
  noAutoScope: false,
@@ -97,7 +96,6 @@ async function runPublishCommand(summaryPath, outputDir, opts) {
97
96
  outputDir,
98
97
  publishEnabled: true,
99
98
  publishTag: opts.tag,
100
- readinessEnabled: false,
101
99
  rootDir: ROOT,
102
100
  searchMode: "open",
103
101
  skipEval: true,
@@ -117,7 +115,7 @@ async function runPublishCommand(summaryPath, outputDir, opts) {
117
115
  if (!existsSync(resolvedPath)) {
118
116
  console.error(` ✖ File not found: ${resolvedPath}`);
119
117
  console.error();
120
- console.error(" Hint: Run `ailf pipeline` first to generate results,");
118
+ console.error(" Hint: Run `ailf run` first to generate results,");
121
119
  console.error(" or provide a path to an existing score-summary.json.");
122
120
  process.exit(1);
123
121
  }
@@ -15,23 +15,32 @@ import { formatReadinessMarkdown, generateReadinessReport, } from "../pipeline/r
15
15
  import { ThresholdConfigSchema, } from "../pipeline/schemas.js";
16
16
  const __dirname = dirname(fileURLToPath(import.meta.url));
17
17
  const ROOT = resolve(__dirname, "..", "..");
18
- const SCORE_SUMMARY_PATH = join(ROOT, "results", "latest", "score-summary.json");
19
- const GAP_ANALYSIS_PATH = join(ROOT, "results", "latest", "gap-analysis.json");
18
+ const DEFAULT_RESULTS_DIR = join(ROOT, "results", "latest");
20
19
  // thresholds loaded via loadConfigFile below
21
20
  const BASELINES_DIR = join(ROOT, "results", "baselines");
21
+ /** Resolve `--from-run` to an absolute results directory. */
22
+ function resolveFromRun(value) {
23
+ if (value === "latest")
24
+ return DEFAULT_RESULTS_DIR;
25
+ return resolve(value);
26
+ }
22
27
  export function createReadinessReportCommand() {
23
- return new Command("readiness-report")
28
+ return new Command("readiness")
24
29
  .description("Generate launch readiness report for a feature area")
25
30
  .requiredOption("-a, --area <area>", "Feature area to evaluate (required)")
31
+ .option("--from-run <path>", "Results directory to read from (`latest` or a path to a results directory containing score-summary.json)", "latest")
26
32
  .option("-H, --history", "Include historical progress from baselines", false)
27
33
  .option("-o, --output <path>", "Write markdown to file instead of stdout")
28
34
  .action(async (opts) => {
35
+ const resultsDir = resolveFromRun(opts.fromRun);
36
+ const scoreSummaryPath = join(resultsDir, "score-summary.json");
37
+ const gapAnalysisPath = join(resultsDir, "gap-analysis.json");
29
38
  // Load score summary
30
- if (!existsSync(SCORE_SUMMARY_PATH)) {
31
- console.error(`❌ Score summary not found at ${SCORE_SUMMARY_PATH}. Run \`pnpm pipeline\` first.`);
39
+ if (!existsSync(scoreSummaryPath)) {
40
+ console.error(`❌ Score summary not found at ${scoreSummaryPath}. Run \`ailf run\` first or pass --from-run <path>.`);
32
41
  process.exit(1);
33
42
  }
34
- const scoreSummary = JSON.parse(readFileSync(SCORE_SUMMARY_PATH, "utf-8"));
43
+ const scoreSummary = JSON.parse(readFileSync(scoreSummaryPath, "utf-8"));
35
44
  // Load threshold config
36
45
  let parsedThresholds;
37
46
  try {
@@ -57,8 +66,8 @@ export function createReadinessReportCommand() {
57
66
  const thresholdConfig = thresholdResult.data;
58
67
  // Load gap analysis (optional)
59
68
  let gapAnalysis;
60
- if (existsSync(GAP_ANALYSIS_PATH)) {
61
- gapAnalysis = JSON.parse(readFileSync(GAP_ANALYSIS_PATH, "utf-8"));
69
+ if (existsSync(gapAnalysisPath)) {
70
+ gapAnalysis = JSON.parse(readFileSync(gapAnalysisPath, "utf-8"));
62
71
  }
63
72
  const history = [];
64
73
  if (opts.history && existsSync(BASELINES_DIR)) {
@@ -1,5 +1,5 @@
1
1
  /**
2
- * remote-pipeline.ts — Remote execution flow for `ailf pipeline --remote`.
2
+ * remote-pipeline.ts — Remote execution flow for `ailf run --remote`.
3
3
  *
4
4
  * Reads local `.ailf/tasks/` YAML, validates locally via Zod schemas,
5
5
  * submits to the AILF API, polls for completion, and writes the same
@@ -1,5 +1,5 @@
1
1
  /**
2
- * remote-pipeline.ts — Remote execution flow for `ailf pipeline --remote`.
2
+ * remote-pipeline.ts — Remote execution flow for `ailf run --remote`.
3
3
  *
4
4
  * Reads local `.ailf/tasks/` YAML, validates locally via Zod schemas,
5
5
  * submits to the AILF API, polls for completion, and writes the same
@@ -135,8 +135,6 @@ function toConfigSlice(opts) {
135
135
  perspectiveOverride: opts.perspectiveOverride,
136
136
  graderReplications: opts.graderReplications,
137
137
  gapAnalysisEnabled: opts.gapAnalysisEnabled,
138
- readinessEnabled: opts.readinessEnabled,
139
- discoveryReportEnabled: opts.discoveryReportEnabled,
140
138
  noRemoteCache: opts.noRemoteCache,
141
139
  // D0037 / W0069 caller envelope overrides — flags override env vars
142
140
  // inside buildCallerEnvelope(), which also merges AILF_* defaults.
@@ -0,0 +1,64 @@
1
+ /**
2
+ * run command — the main evaluation entry point.
3
+ *
4
+ * Renamed from `ailf pipeline` to `ailf run` per W0077 Phase 1
5
+ * (see docs/design-docs/pipeline-command-surface.md §3). The command drives
6
+ * the evaluation pipeline; the orchestrator internals retain the "pipeline"
7
+ * name because they describe a multi-step process, not the CLI verb.
8
+ *
9
+ * Defines all 44+ CLI flags via Commander, resolves them into a typed
10
+ * options object, bridges to process.env for downstream modules, and
11
+ * delegates to runPipeline().
12
+ *
13
+ * @see docs/guides/cli-guide.md for per-flag behavior notes, or
14
+ * docs/references/cli-reference.md for the auto-generated flag matrix.
15
+ */
16
+ import { Command } from "commander";
17
+ /**
18
+ * Raw CLI options as parsed by Commander.
19
+ * Field names follow Commander's camelCase convention for kebab-case flags.
20
+ */
21
+ export interface PipelineCliOptions {
22
+ area?: string;
23
+ autoScope: boolean;
24
+ /** `--before-source <name>` — swap the doc source to a "before" state for impact evaluation. */
25
+ beforeSource?: string;
26
+ cache: boolean;
27
+ changedDocs?: string;
28
+ /**
29
+ * `--compare [baseline]` — Commander optional argument.
30
+ * undefined → flag not passed
31
+ * true → bare `--compare` (compare against latest baseline)
32
+ * string → `--compare path/to/baseline.json` (pin to a file)
33
+ */
34
+ compare?: boolean | string;
35
+ config?: string;
36
+ debug: boolean;
37
+ filterFirstN?: number;
38
+ filterPattern?: string;
39
+ filterSample?: number;
40
+ dryRun: boolean;
41
+ eval: boolean;
42
+ fetch: boolean;
43
+ mode: string;
44
+ variant?: string;
45
+ output?: string;
46
+ promptfooUrl?: string;
47
+ publish?: boolean;
48
+ publishTag?: string;
49
+ remoteCache?: boolean;
50
+ sanityDocument: string[];
51
+ sanityPerspective?: string;
52
+ search?: string;
53
+ source?: string;
54
+ remote: boolean;
55
+ task?: string;
56
+ tag: string[];
57
+ threshold?: number;
58
+ url: string[];
59
+ artifactsWrite: boolean;
60
+ classification?: string;
61
+ purpose?: string;
62
+ label: string[];
63
+ }
64
+ export declare function createRunCommand(): Command;
@@ -1,23 +1,29 @@
1
1
  /**
2
- * pipeline command — the main evaluation pipeline orchestrator.
2
+ * run command — the main evaluation entry point.
3
3
  *
4
- * Defines all 36+ CLI flags via Commander, resolves them into a typed
4
+ * Renamed from `ailf pipeline` to `ailf run` per W0077 Phase 1
5
+ * (see docs/design-docs/pipeline-command-surface.md §3). The command drives
6
+ * the evaluation pipeline; the orchestrator internals retain the "pipeline"
7
+ * name because they describe a multi-step process, not the CLI verb.
8
+ *
9
+ * Defines all 44+ CLI flags via Commander, resolves them into a typed
5
10
  * options object, bridges to process.env for downstream modules, and
6
11
  * delegates to runPipeline().
7
12
  *
8
- * @see docs/cli.md for the full flag reference.
13
+ * @see docs/guides/cli-guide.md for per-flag behavior notes, or
14
+ * docs/references/cli-reference.md for the auto-generated flag matrix.
9
15
  */
10
16
  import { Command } from "commander";
11
- import { addAgenticOptions, addDebugOptions, addSanitySourceOptions, } from "./shared/options.js";
12
- export function createPipelineCommand() {
13
- const cmd = new Command("pipeline")
17
+ import { addAgenticOptions, addDebugOptions, addSanityScopeOptions, } from "./shared/options.js";
18
+ export function createRunCommand() {
19
+ const cmd = new Command("run")
14
20
  .description("Run the full evaluation pipeline")
15
21
  .option("-m, --mode <mode>", "Evaluation mode: literacy (default), mcp-server, agent-harness, knowledge-probe, custom. Legacy aliases (baseline, agentic, observed, full) are accepted and normalized to literacy + variant.", "literacy")
16
22
  .option("--variant <variant>", "Literacy variant: full (default — standard + agentic), baseline (standard only), agentic (agentic only), observed. Only applies to --mode literacy.")
17
23
  .option("-s, --source <name>", "Documentation source name (from sources.yaml)")
18
24
  .option("-n, --dry-run", "Validate configuration only, no execution", false)
19
- .option("--skip-fetch", "Reuse cached documentation contexts", false)
20
- .option("--skip-eval", "Recalculate from existing eval results", false)
25
+ .option("--no-fetch", "Reuse cached documentation contexts")
26
+ .option("--no-eval", "Recalculate from existing eval results")
21
27
  .option("--no-cache", "Bypass all pipeline-level caching")
22
28
  .option("--no-remote-cache", "Disable Content Lake cache lookup (local cache still active)")
23
29
  .option("--no-auto-scope", "Disable release-aware auto-scoping (evaluate all tasks even when a perspective is set)")
@@ -31,39 +37,22 @@ export function createPipelineCommand() {
31
37
  .filter(Boolean),
32
38
  ], [])
33
39
  .option("--changed-docs <slugs>", "Auto-scope to tasks affected by these document slugs")
34
- .option("-j, --concurrency <n>", "Max parallel API calls during evaluation", parseInt)
35
- .option("--grader-replications <n>", "Grader consistency replications", parseInt)
36
- .option("--before <source>", "Before-state for impact evaluation")
37
- .option("-c, --compare", "Compare scores against latest baseline", false)
38
- .option("--compare-baseline <path>", "Specific baseline file to compare")
40
+ .option("--before-source <name>", "Swap the doc source to a `before` state and run a paired evaluation for impact analysis. Pairs with `ailf baseline` and `--compare`. Distinct from `--compare <baseline>`, which compares scores against a saved snapshot.")
41
+ .option("-c, --compare [baseline]", "Compare scores against the latest baseline. Pass a path to pin a specific baseline file (e.g. --compare results/baselines/2026-04-22.json).")
39
42
  .option("--threshold <n>", "Noise threshold for comparison (default: 2)", parseFloat)
40
- .option("--no-gap-analysis", "Skip failure mode + impact analysis")
41
- .option("--readiness", "Generate launch readiness checklist", false)
42
- .option("--discovery-report", "Generate agent discoverability report", false)
43
43
  .option("-p, --publish", "Write report to Sanity + fan out to sinks (auto-enabled for full runs when report store is configured)")
44
44
  .option("--no-publish", "Suppress auto-publishing")
45
45
  .option("--publish-tag <tag>", "Label for published report")
46
- .option("--report-dataset <name>", "Sanity dataset for report store")
47
- .option("--report-project <id>", "Sanity project ID for report store")
48
46
  .option("--config <path>", "Load pipeline config from a TS/JS/YAML/JSON file (overrides most CLI flags)")
49
47
  .option("-o, --output <path>", "Write PR comment markdown to file")
50
- .option("--output-dir <path>", "Base directory for pipeline output artifacts (default: inferred from execution context)")
51
48
  .option("--promptfoo-url <url>", "Promptfoo share URL for report")
52
- .option("--task-source <type>", "Task definition source: content-lake (default — Sanity Content Lake), repo (repo tasks only, no Content Lake merge)", "content-lake")
53
- .option("--repo-tasks-path <path>", "Path to repo-based task definitions. Defaults to ./.ailf/tasks/ when --task-source=repo.")
54
49
  .option("--remote", "Submit evaluation to the AILF API instead of running locally", false)
55
- .option("--api-url <url>", "AILF API base URL (default: https://ailf-api.sanity.build)")
56
- .option("--no-artifacts", "Disable all artifact writers (D0033). Overrides --artifacts-dir.")
57
- .option("--artifacts-dir <path>", "Root directory for local artifact output (D0033; default: .ailf/results/captures/)")
58
- .option("--artifacts-dry-run", "Run artifact writers in dry-run mode — log intended writes, touch no storage", false)
59
- .option("--artifacts-exclude <types>", "Comma-separated artifact types to skip (e.g. traces,graderPrompts)")
50
+ .option("--no-artifacts-write", "Run artifact writers in dry-run mode — log intended writes, touch no storage")
60
51
  // D0037 caller envelope (W0069) — threads through --remote so the
61
52
  // server-side pipeline attributes provenance to the caller, not the
62
53
  // API gateway runner. All env-var equivalents are honored too;
63
54
  // explicit flags win over env vars.
64
- .option("--classification <value>", "Run classification for provenance: official | ad-hoc | experimental | test | external. Overrides AILF_CLASSIFICATION. See D0037.")
65
- .option("--owner-team <slug>", "Team slug this run is attributable to. Overrides AILF_OWNER_TEAM.")
66
- .option("--owner-individual <slug>", "Individual (GH actor / user ID) this run is attributable to. Overrides AILF_OWNER_INDIVIDUAL.")
55
+ .option("--classification <value>", "Run classification for provenance: official | adhoc | experimental | test | external. Overrides AILF_CLASSIFICATION. See D0037.")
67
56
  .option("--purpose <text>", 'Free-text "why I ran this" attached to provenance. Overrides AILF_PURPOSE.')
68
57
  .option("--label <value>", "Free-form searchable label (repeatable). Appends to any AILF_LABELS env value.", (val, prev) => [
69
58
  ...prev,
@@ -78,7 +67,7 @@ export function createPipelineCommand() {
78
67
  });
79
68
  // Add shared option groups
80
69
  addDebugOptions(cmd);
81
- addSanitySourceOptions(cmd);
70
+ addSanityScopeOptions(cmd);
82
71
  addAgenticOptions(cmd);
83
72
  return cmd;
84
73
  }
@@ -67,14 +67,14 @@ function hasColorSupport() {
67
67
  // ---------------------------------------------------------------------------
68
68
  const afterHelpText = `
69
69
  Quick Start:
70
- $ ailf pipeline --debug Run a quick evaluation (first 2 tests)
71
- $ ailf pipeline --area groq Evaluate a specific feature area
72
- $ ailf pipeline --explain Preview the execution plan
70
+ $ ailf run --debug Run a quick evaluation (first 2 tests)
71
+ $ ailf run --area groq Evaluate a specific feature area
72
+ $ ailf run --explain Preview the execution plan
73
73
  $ ailf init Set up AILF in a new project
74
74
 
75
75
  Documentation:
76
76
  Repository https://github.com/sanity-io/ai-literacy-framework
77
- CLI Guide https://github.com/sanity-io/ai-literacy-framework/blob/main/docs/cli.md
77
+ CLI Guide https://github.com/sanity-io/ai-literacy-framework/blob/main/docs/guides/cli-guide.md
78
78
  Getting Started https://github.com/sanity-io/ai-literacy-framework/blob/main/docs/getting-started.md
79
79
 
80
80
  Run ailf <command> --help for detailed usage of any command.`;
@@ -7,11 +7,22 @@
7
7
  */
8
8
  import type { Command } from "commander";
9
9
  /**
10
- * Add agentic options: --url, --header, --allowed-origin, --search
10
+ * Add agentic options on `ailf run`: `--url` and `--search`. Both are
11
+ * per-run overrides.
12
+ *
13
+ * **`--header` and `--allowed-origin` retired in W0077 Phase 6f** —
14
+ * configure them in `.ailf/config.yaml`'s `agentic` block instead. The
15
+ * `DOC_HEADERS` and `DOC_ALLOWED_ORIGIN(S)` env vars still apply.
11
16
  */
12
17
  export declare function addAgenticOptions(cmd: Command): Command;
13
18
  /**
14
- * Add debug options: --debug, --debug-n, --debug-pattern, --debug-sample
19
+ * Add debug + filter options.
20
+ *
21
+ * `-d, --debug` is a shortcut that runs only the first 2 tests for fast
22
+ * feedback. `--filter-first-n`, `--filter-pattern`, and `--filter-sample`
23
+ * narrow the test set explicitly and mirror the same-named flags on
24
+ * `ailf eval` so the pipeline surface matches the underlying Promptfoo
25
+ * passthrough.
15
26
  */
16
27
  export declare function addDebugOptions(cmd: Command): Command;
17
28
  /**
@@ -27,9 +38,24 @@ export declare function addOutputOptions(cmd: Command): Command;
27
38
  */
28
39
  export declare function addOutputDirOption(cmd: Command): Command;
29
40
  /**
30
- * Add Sanity source options: --sanity-dataset, --sanity-project, etc.
41
+ * Add the full Sanity-source CLI surface — dataset, project, perspective,
42
+ * studio origin, document. Used by `ailf fetch-docs`, which is invoked
43
+ * directly with explicit Sanity targeting per run.
44
+ *
45
+ * **Not used by `ailf run`** post-W0077 Phase 6d. The dataset, project, and
46
+ * studio-origin trio moved to `.ailf/config.yaml`'s `source` block (with
47
+ * `SANITY_DATASET` / `SANITY_PROJECT_ID` / `SANITY_STUDIO_ORIGIN` env-var
48
+ * fallbacks). `ailf run` uses `addSanityScopeOptions` instead, which keeps
49
+ * only the per-run flags (`--sanity-perspective`, `--sanity-document`).
31
50
  */
32
51
  export declare function addSanitySourceOptions(cmd: Command): Command;
52
+ /**
53
+ * Add the per-run Sanity-scope CLI surface — `--sanity-perspective` and
54
+ * `--sanity-document`. Used by `ailf run`. The per-environment trio
55
+ * (dataset, project, studio origin) lives in `.ailf/config.yaml`'s
56
+ * `source` block instead (W0077 Phase 6d).
57
+ */
58
+ export declare function addSanityScopeOptions(cmd: Command): Command;
33
59
  /**
34
60
  * Collect repeatable string options into an array.
35
61
  * Used as a Commander argParser for options like --url, --header, --allowed-origin.
@@ -6,27 +6,33 @@
6
6
  * (e.g., debug options, Sanity source options, output options).
7
7
  */
8
8
  /**
9
- * Add agentic options: --url, --header, --allowed-origin, --search
9
+ * Add agentic options on `ailf run`: `--url` and `--search`. Both are
10
+ * per-run overrides.
11
+ *
12
+ * **`--header` and `--allowed-origin` retired in W0077 Phase 6f** —
13
+ * configure them in `.ailf/config.yaml`'s `agentic` block instead. The
14
+ * `DOC_HEADERS` and `DOC_ALLOWED_ORIGIN(S)` env vars still apply.
10
15
  */
11
16
  export function addAgenticOptions(cmd) {
12
17
  return cmd
13
18
  .option("--url <url>", "Documentation URL (repeatable)", collect, [])
14
- .option("--urls <url>", "Alias for --url (repeatable)", collect, [])
15
- .option("--header <header>", 'Custom HTTP header "Key: Value" (repeatable)', collect, [])
16
- .option("--headers <header>", "Alias for --header (repeatable)", collect, [])
17
- .option("--allowed-origin <origin>", "Agent origin sandbox (repeatable, supports globs)", collect, [])
18
- .option("--allowed-origins <origin>", "Alias for --allowed-origin (repeatable)", collect, [])
19
19
  .option("-S, --search <mode>", "Web search mode: open, origin-only, off");
20
20
  }
21
21
  /**
22
- * Add debug options: --debug, --debug-n, --debug-pattern, --debug-sample
22
+ * Add debug + filter options.
23
+ *
24
+ * `-d, --debug` is a shortcut that runs only the first 2 tests for fast
25
+ * feedback. `--filter-first-n`, `--filter-pattern`, and `--filter-sample`
26
+ * narrow the test set explicitly and mirror the same-named flags on
27
+ * `ailf eval` so the pipeline surface matches the underlying Promptfoo
28
+ * passthrough.
23
29
  */
24
30
  export function addDebugOptions(cmd) {
25
31
  return cmd
26
32
  .option("-d, --debug", "Run subset of tests for fast feedback", false)
27
- .option("--debug-n <n>", "First N tests", parseInt)
28
- .option("--debug-pattern <regex>", "Filter tests by description regex")
29
- .option("--debug-sample <n>", "Random sample of N tests", parseInt);
33
+ .option("--filter-first-n <n>", "Run only first N tests", parseInt)
34
+ .option("--filter-pattern <regex>", "Filter tests by description regex")
35
+ .option("--filter-sample <n>", "Random sample of N tests", parseInt);
30
36
  }
31
37
  /**
32
38
  * Add output options: --output, --format
@@ -47,7 +53,15 @@ export function addOutputDirOption(cmd) {
47
53
  return cmd.option("--output-dir <path>", "Base directory for output artifacts (default: .ailf/results/latest/)");
48
54
  }
49
55
  /**
50
- * Add Sanity source options: --sanity-dataset, --sanity-project, etc.
56
+ * Add the full Sanity-source CLI surface — dataset, project, perspective,
57
+ * studio origin, document. Used by `ailf fetch-docs`, which is invoked
58
+ * directly with explicit Sanity targeting per run.
59
+ *
60
+ * **Not used by `ailf run`** post-W0077 Phase 6d. The dataset, project, and
61
+ * studio-origin trio moved to `.ailf/config.yaml`'s `source` block (with
62
+ * `SANITY_DATASET` / `SANITY_PROJECT_ID` / `SANITY_STUDIO_ORIGIN` env-var
63
+ * fallbacks). `ailf run` uses `addSanityScopeOptions` instead, which keeps
64
+ * only the per-run flags (`--sanity-perspective`, `--sanity-document`).
51
65
  */
52
66
  export function addSanitySourceOptions(cmd) {
53
67
  return cmd
@@ -55,8 +69,18 @@ export function addSanitySourceOptions(cmd) {
55
69
  .option("--sanity-project <id>", "Override Sanity project ID")
56
70
  .option("--sanity-perspective <id>", "Sanity release perspective ID")
57
71
  .option("--sanity-studio-origin <url>", "Sanity Studio base URL")
58
- .option("--sanity-document <id>", "Evaluate specific Sanity document(s) (repeatable)", collect, [])
59
- .option("--sanity-documents <id>", "Alias for --sanity-document (repeatable)", collect, []);
72
+ .option("--sanity-document <id>", "Evaluate specific Sanity document(s) (repeatable)", collect, []);
73
+ }
74
+ /**
75
+ * Add the per-run Sanity-scope CLI surface — `--sanity-perspective` and
76
+ * `--sanity-document`. Used by `ailf run`. The per-environment trio
77
+ * (dataset, project, studio origin) lives in `.ailf/config.yaml`'s
78
+ * `source` block instead (W0077 Phase 6d).
79
+ */
80
+ export function addSanityScopeOptions(cmd) {
81
+ return cmd
82
+ .option("--sanity-perspective <id>", "Sanity release perspective ID")
83
+ .option("--sanity-document <id>", "Evaluate specific Sanity document(s) (repeatable)", collect, []);
60
84
  }
61
85
  /**
62
86
  * Collect repeatable string options into an array.
@@ -21,7 +21,7 @@ import { detectLegacyFieldNames, parseCanonicalTaskFile, } from "../adapters/tas
21
21
  import { validateCanonicalTasks, formatValidationResult, } from "../adapters/task-sources/repo-validation.js";
22
22
  import { discoverTsTaskFiles, loadTsTaskFile, } from "../adapters/task-sources/task-file-loader.js";
23
23
  export function createValidateTasksCommand() {
24
- return new Command("validate-tasks")
24
+ return new Command("tasks")
25
25
  .description("Validate task files (YAML and TypeScript) in .ailf/tasks/ against the canonical schema")
26
26
  .argument("[path]", "Path to tasks directory (default: .ailf/tasks/)", ".ailf/tasks")
27
27
  .option("--strict", "Treat warnings as errors", false)
@@ -6,4 +6,4 @@
6
6
  *
7
7
  */
8
8
  import { Command } from "commander";
9
- export declare function createValidateCommand(): Command;
9
+ export declare function createValidateConfigCommand(): Command;
@@ -10,8 +10,8 @@ import { dirname, resolve } from "path";
10
10
  import { fileURLToPath } from "url";
11
11
  const __dirname = dirname(fileURLToPath(import.meta.url));
12
12
  const ROOT = resolve(__dirname, "..", "..");
13
- export function createValidateCommand() {
14
- return new Command("validate")
13
+ export function createValidateConfigCommand() {
14
+ return new Command("config")
15
15
  .description("Validate pipeline configuration")
16
16
  .option("--strict", "Treat warnings as errors", false)
17
17
  .option("--contexts", "Check that context files exist", false)
@@ -8,11 +8,11 @@
8
8
  */
9
9
  import { Command } from "commander";
10
10
  export function createWeeklyDigestCommand() {
11
- return new Command("weekly-digest")
11
+ return new Command("digest")
12
12
  .description("Generate and deliver a weekly evaluation digest")
13
13
  .option("-n, --dry-run", "Print to stdout only, do not send to Slack", false)
14
14
  .option("--lookback <days>", "Lookback window in days (default: from config or 7)", parseInt)
15
- .option("--json", "Output raw JSON digest data", false)
15
+ .option("-f, --format <fmt>", "Output format: console or json", "console")
16
16
  .action(async (opts) => {
17
17
  const { generateDigest } = await import("../schedules/digest.js");
18
18
  const { getDigestConfig } = await import("../schedules/loader.js");
@@ -45,7 +45,7 @@ export function createWeeklyDigestCommand() {
45
45
  console.log(` Regressed: ${digest.regressed.join(", ") || "none"}`);
46
46
  console.log(` Stable: ${digest.stable.join(", ") || "none"}`);
47
47
  console.log();
48
- if (opts.json) {
48
+ if (opts.format === "json") {
49
49
  console.log(JSON.stringify(digest, null, 2));
50
50
  return;
51
51
  }
@@ -2,9 +2,9 @@
2
2
  * thresholds.ts — Quality thresholds for readiness gates and regression alerts.
3
3
  *
4
4
  * Used by:
5
- * - `npx @sanity/ailf pipeline --readiness` (launch readiness checklist)
6
- * - `npx @sanity/ailf pipeline --publish` (severity-aware sink routing)
7
- * - `npx @sanity/ailf pipeline --compare` (regression alerting)
5
+ * - `npx @sanity/ailf report readiness` (launch readiness checklist)
6
+ * - `npx @sanity/ailf run --publish` (severity-aware sink routing)
7
+ * - `npx @sanity/ailf run --compare` (regression alerting)
8
8
  *
9
9
  * @see docs/archive/exec-plans/scenario-matrix-implementation/phase-5-readiness-thresholds.md
10
10
  */
@@ -44,8 +44,6 @@ export function mapToResolvedConfig(opts, rootDir) {
44
44
  compareThreshold: opts.compareThreshold,
45
45
  compareBaseline: opts.compareBaseline,
46
46
  gapAnalysisEnabled: opts.gapAnalysisEnabled,
47
- readinessEnabled: opts.readinessEnabled,
48
- discoveryReportEnabled: opts.discoveryReportEnabled,
49
47
  publishEnabled: opts.publishEnabled,
50
48
  publishTag: opts.publishTag,
51
49
  noCache: opts.noCache,
@@ -9,14 +9,12 @@ import { LiteracyVariant } from "../pipeline/normalize-mode.js";
9
9
  import { CallbackStep } from "./steps/callback-step.js";
10
10
  import { CalculateScoresStep } from "./steps/calculate-scores-step.js";
11
11
  import { CompareStep } from "./steps/compare-step.js";
12
- import { DiscoveryReportStep } from "./steps/discovery-report-step.js";
13
12
  import { FetchDocsStep } from "./steps/fetch-docs-step.js";
14
13
  import { FinalizeRunStep } from "./steps/finalize-run-step.js";
15
14
  import { GapAnalysisStep } from "./steps/gap-analysis-step.js";
16
15
  import { GenerateConfigsStep } from "./steps/generate-configs-step.js";
17
16
  import { GraderConsistencyStep } from "./steps/grader-consistency-step.js";
18
17
  import { PublishReportStep } from "./steps/publish-report-step.js";
19
- import { ReadinessStep } from "./steps/readiness-step.js";
20
18
  import { ReportStep } from "./steps/report-step.js";
21
19
  import { RunEvalStep } from "./steps/run-eval-step.js";
22
20
  import { MirrorRepoTasksStep } from "./steps/mirror-repo-tasks-step.js";
@@ -93,15 +91,7 @@ export function buildStepSequence(ctx, pipelineStart = Date.now()) {
93
91
  if (config.compareEnabled) {
94
92
  steps.push(new CompareStep());
95
93
  }
96
- // Step 6b: Readiness report (optional)
97
- if (config.readinessEnabled) {
98
- steps.push(new ReadinessStep());
99
- }
100
- // Step 6c: Discovery report (optional)
101
- if (config.discoveryReportEnabled) {
102
- steps.push(new DiscoveryReportStep());
103
- }
104
- // Step 7: Callback delivery (optional, API-triggered evaluations)
94
+ // Step 6: Callback delivery (optional, API-triggered evaluations)
105
95
  if (config.callback?.url) {
106
96
  steps.push(new CallbackStep(config.callback, config.jobId));
107
97
  }
@@ -27,7 +27,7 @@ export class FetchDocsStep {
27
27
  }
28
28
  async execute(ctx, state) {
29
29
  if (ctx.config.skipFetch) {
30
- return { status: "skipped", reason: "--skip-fetch" };
30
+ return { status: "skipped", reason: "--no-fetch" };
31
31
  }
32
32
  const start = Date.now();
33
33
  // Load tasks — use the same source as GenerateConfigsStep to avoid
@@ -6,14 +6,12 @@
6
6
  */
7
7
  export { CalculateScoresStep } from "./calculate-scores-step.js";
8
8
  export { CompareStep } from "./compare-step.js";
9
- export { DiscoveryReportStep } from "./discovery-report-step.js";
10
9
  export { FetchDocsStep } from "./fetch-docs-step.js";
11
10
  export { GapAnalysisStep } from "./gap-analysis-step.js";
12
11
  export { GenerateConfigsStep } from "./generate-configs-step.js";
13
12
  export { MirrorRepoTasksStep } from "./mirror-repo-tasks-step.js";
14
13
  export { GraderConsistencyStep } from "./grader-consistency-step.js";
15
14
  export { PublishReportStep } from "./publish-report-step.js";
16
- export { ReadinessStep } from "./readiness-step.js";
17
15
  export { ReportStep } from "./report-step.js";
18
16
  export { RunEvalStep } from "./run-eval-step.js";
19
17
  export { ValidateStep } from "./validate-step.js";
@@ -6,14 +6,12 @@
6
6
  */
7
7
  export { CalculateScoresStep } from "./calculate-scores-step.js";
8
8
  export { CompareStep } from "./compare-step.js";
9
- export { DiscoveryReportStep } from "./discovery-report-step.js";
10
9
  export { FetchDocsStep } from "./fetch-docs-step.js";
11
10
  export { GapAnalysisStep } from "./gap-analysis-step.js";
12
11
  export { GenerateConfigsStep } from "./generate-configs-step.js";
13
12
  export { MirrorRepoTasksStep } from "./mirror-repo-tasks-step.js";
14
13
  export { GraderConsistencyStep } from "./grader-consistency-step.js";
15
14
  export { PublishReportStep } from "./publish-report-step.js";
16
- export { ReadinessStep } from "./readiness-step.js";
17
15
  export { ReportStep } from "./report-step.js";
18
16
  export { RunEvalStep } from "./run-eval-step.js";
19
17
  export { ValidateStep } from "./validate-step.js";
@@ -25,7 +25,7 @@ export class RunEvalStep {
25
25
  }
26
26
  async execute(ctx, state) {
27
27
  if (ctx.config.skipEval) {
28
- return { status: "skipped", reason: "--skip-eval" };
28
+ return { status: "skipped", reason: "--no-eval" };
29
29
  }
30
30
  const start = Date.now();
31
31
  const { rootDir, debug, concurrency, noCache } = ctx.config;
@@ -48,7 +48,7 @@ export interface CacheStats {
48
48
  hits: number;
49
49
  /** Steps where cache was missed (executed normally) */
50
50
  misses: number;
51
- /** Steps that were skipped for other reasons (--skip-fetch, etc.) */
51
+ /** Steps that were skipped for other reasons (--no-fetch, etc.) */
52
52
  skipped: number;
53
53
  /** Per-step detail */
54
54
  steps: Record<string, "disabled" | "hit" | "miss" | "skipped">;
@@ -47,8 +47,6 @@ export function mapRequestToConfig(request, rootDir) {
47
47
  compareThreshold: request.compareThreshold,
48
48
  compareBaseline: request.compareBaseline,
49
49
  gapAnalysisEnabled: request.gapAnalysis ?? true,
50
- readinessEnabled: request.readiness ?? false,
51
- discoveryReportEnabled: request.discoveryReport ?? false,
52
50
  publishEnabled: request.publish ?? publishDefault,
53
51
  publishTag: request.publishTag,
54
52
  noAutoScope: request.noAutoScope ?? false,
@@ -107,7 +107,7 @@ export declare function buildMirrorDocument(task: LiteracyTaskDefinition, opts:
107
107
  slugToDocId: Map<string, string>;
108
108
  }): {
109
109
  baseline?: {
110
- rubric?: "full" | "abbreviated" | "none" | undefined;
110
+ rubric?: "abbreviated" | "full" | "none" | undefined;
111
111
  enabled?: boolean | undefined;
112
112
  } | undefined;
113
113
  _id: string;