@sanity/ailf 0.5.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (288) hide show
  1. package/config/features.ts +23 -0
  2. package/config/models.ts +83 -0
  3. package/config/prompts.ts +16 -0
  4. package/config/rubrics.ts +225 -0
  5. package/config/schedules.ts +47 -0
  6. package/config/sinks.ts +37 -0
  7. package/config/sources.ts +21 -0
  8. package/config/thresholds.ts +61 -0
  9. package/dist/_vendor/ailf-core/config-helpers.d.ts +174 -0
  10. package/dist/_vendor/ailf-core/config-helpers.js +150 -0
  11. package/dist/_vendor/ailf-core/env-helper.d.ts +35 -0
  12. package/dist/_vendor/ailf-core/env-helper.js +45 -0
  13. package/dist/_vendor/ailf-core/index.d.ts +3 -0
  14. package/dist/_vendor/ailf-core/index.js +5 -0
  15. package/dist/_vendor/ailf-core/ports/context.d.ts +15 -2
  16. package/dist/_vendor/ailf-core/ports/doc-fetcher.d.ts +2 -2
  17. package/dist/_vendor/ailf-core/ports/index.d.ts +2 -1
  18. package/dist/_vendor/ailf-core/ports/mode-handler.d.ts +129 -0
  19. package/dist/_vendor/ailf-core/ports/mode-handler.js +19 -0
  20. package/dist/_vendor/ailf-core/ports/task-source.d.ts +16 -122
  21. package/dist/_vendor/ailf-core/ports/task-source.js +7 -7
  22. package/dist/_vendor/ailf-core/schemas/eval-config.d.ts +7 -2
  23. package/dist/_vendor/ailf-core/schemas/eval-config.js +7 -2
  24. package/dist/_vendor/ailf-core/schemas/pipeline-request.d.ts +8 -3
  25. package/dist/_vendor/ailf-core/schemas/pipeline-request.js +6 -1
  26. package/dist/_vendor/ailf-core/schemas/pipeline.d.ts +14 -29
  27. package/dist/_vendor/ailf-core/schemas/pipeline.js +17 -8
  28. package/dist/_vendor/ailf-core/schemas/schedules.d.ts +14 -4
  29. package/dist/_vendor/ailf-core/schemas/schedules.js +6 -2
  30. package/dist/_vendor/ailf-core/schemas/sinks.d.ts +1 -1
  31. package/dist/_vendor/ailf-core/services/comparison-formatters.js +57 -19
  32. package/dist/_vendor/ailf-core/services/index.d.ts +2 -1
  33. package/dist/_vendor/ailf-core/services/index.js +2 -1
  34. package/dist/_vendor/ailf-core/services/scoring-engine.d.ts +153 -0
  35. package/dist/_vendor/ailf-core/services/scoring-engine.js +237 -0
  36. package/dist/_vendor/ailf-core/services/scoring.d.ts +15 -2
  37. package/dist/_vendor/ailf-core/services/scoring.js +25 -15
  38. package/dist/_vendor/ailf-core/types/branded-ids.d.ts +137 -0
  39. package/dist/_vendor/ailf-core/types/branded-ids.js +136 -0
  40. package/dist/_vendor/ailf-core/types/eval-mode-config.d.ts +150 -0
  41. package/dist/_vendor/ailf-core/types/eval-mode-config.js +24 -0
  42. package/dist/_vendor/ailf-core/types/generalized-task.d.ts +319 -0
  43. package/dist/_vendor/ailf-core/types/generalized-task.js +13 -0
  44. package/dist/_vendor/ailf-core/types/index.d.ts +45 -81
  45. package/dist/_vendor/ailf-core/types/index.js +8 -1
  46. package/dist/_vendor/ailf-core/types/plugin-registry.d.ts +202 -0
  47. package/dist/_vendor/ailf-core/types/plugin-registry.js +132 -0
  48. package/dist/_vendor/ailf-core/types/storage-schema.d.ts +199 -0
  49. package/dist/_vendor/ailf-core/types/storage-schema.js +39 -0
  50. package/dist/_vendor/ailf-core/types/task-graph.d.ts +86 -0
  51. package/dist/_vendor/ailf-core/types/task-graph.js +20 -0
  52. package/dist/_vendor/ailf-core/types/trace.d.ts +118 -0
  53. package/dist/_vendor/ailf-core/types/trace.js +18 -0
  54. package/dist/_vendor/ailf-core/types/variable-envelope.d.ts +80 -0
  55. package/dist/_vendor/ailf-core/types/variable-envelope.js +16 -0
  56. package/dist/_vendor/ailf-shared/dimension-names.d.ts +5 -18
  57. package/dist/_vendor/ailf-shared/dimension-names.js +6 -24
  58. package/dist/_vendor/ailf-shared/eval-modes.d.ts +38 -6
  59. package/dist/_vendor/ailf-shared/eval-modes.js +26 -2
  60. package/dist/_vendor/ailf-shared/index.d.ts +0 -1
  61. package/dist/_vendor/ailf-shared/index.js +0 -1
  62. package/dist/adapters/api-client/build-request.js +14 -13
  63. package/dist/adapters/config-sources/file-config-adapter.d.ts +20 -11
  64. package/dist/adapters/config-sources/file-config-adapter.js +38 -12
  65. package/dist/adapters/config-sources/index.d.ts +2 -0
  66. package/dist/adapters/config-sources/index.js +1 -0
  67. package/dist/adapters/config-sources/ts-config-loader.d.ts +59 -0
  68. package/dist/adapters/config-sources/ts-config-loader.js +133 -0
  69. package/dist/adapters/doc-fetchers/sanity-doc-fetcher.d.ts +3 -2
  70. package/dist/adapters/doc-fetchers/sanity-doc-fetcher.js +7 -2
  71. package/dist/adapters/task-sources/composite-task-source.d.ts +3 -3
  72. package/dist/adapters/task-sources/composite-task-source.js +1 -1
  73. package/dist/adapters/task-sources/content-lake-task-source.d.ts +7 -6
  74. package/dist/adapters/task-sources/content-lake-task-source.js +22 -23
  75. package/dist/adapters/task-sources/index.d.ts +1 -0
  76. package/dist/adapters/task-sources/index.js +1 -0
  77. package/dist/adapters/task-sources/repo-task-source.d.ts +4 -4
  78. package/dist/adapters/task-sources/repo-task-source.js +69 -16
  79. package/dist/adapters/task-sources/task-file-loader.d.ts +64 -0
  80. package/dist/adapters/task-sources/task-file-loader.js +83 -0
  81. package/dist/adapters/task-sources/yaml-task-source.d.ts +6 -6
  82. package/dist/adapters/task-sources/yaml-task-source.js +19 -16
  83. package/dist/cli.js +0 -2
  84. package/dist/commands/baseline.js +4 -1
  85. package/dist/commands/calculate-scores.js +1 -1
  86. package/dist/commands/coverage-audit.js +7 -1
  87. package/dist/commands/explain-handler.js +25 -23
  88. package/dist/commands/fetch-docs.js +3 -2
  89. package/dist/commands/generate-configs.js +1 -1
  90. package/dist/commands/interactive.js +11 -7
  91. package/dist/commands/pipeline-action.d.ts +2 -0
  92. package/dist/commands/pipeline-action.js +16 -6
  93. package/dist/commands/pipeline.d.ts +1 -0
  94. package/dist/commands/pipeline.js +4 -2
  95. package/dist/commands/pr-comment.js +1 -1
  96. package/dist/commands/publish.js +2 -2
  97. package/dist/commands/readiness-report.js +13 -6
  98. package/dist/composition-root.d.ts +1 -1
  99. package/dist/composition-root.js +67 -4
  100. package/dist/orchestration/build-app-context.js +1 -0
  101. package/dist/orchestration/build-step-sequence.js +24 -6
  102. package/dist/orchestration/steps/calculate-scores-step.js +24 -11
  103. package/dist/orchestration/steps/fetch-docs-step.js +6 -4
  104. package/dist/orchestration/steps/gap-analysis-step.js +8 -7
  105. package/dist/orchestration/steps/generate-configs-step.d.ts +16 -3
  106. package/dist/orchestration/steps/generate-configs-step.js +245 -51
  107. package/dist/orchestration/steps/grader-consistency-step.js +7 -4
  108. package/dist/orchestration/steps/mirror-repo-tasks-step.js +1 -1
  109. package/dist/orchestration/steps/readiness-step.js +5 -6
  110. package/dist/orchestration/steps/run-eval-step.d.ts +1 -2
  111. package/dist/orchestration/steps/run-eval-step.js +8 -7
  112. package/dist/pipeline/cache.d.ts +1 -1
  113. package/dist/pipeline/cache.js +36 -8
  114. package/dist/pipeline/calculate-scores.d.ts +2 -4
  115. package/dist/pipeline/calculate-scores.js +43 -113
  116. package/dist/pipeline/checks.js +2 -2
  117. package/dist/pipeline/compare.js +8 -8
  118. package/dist/pipeline/compiler/__tests__/agent-harness-handler.test.d.ts +10 -0
  119. package/dist/pipeline/compiler/__tests__/agent-harness-handler.test.js +288 -0
  120. package/dist/pipeline/compiler/__tests__/assertion-mapper.test.d.ts +9 -0
  121. package/dist/pipeline/compiler/__tests__/assertion-mapper.test.js +145 -0
  122. package/dist/pipeline/compiler/__tests__/knowledge-probe-handler.test.d.ts +10 -0
  123. package/dist/pipeline/compiler/__tests__/knowledge-probe-handler.test.js +314 -0
  124. package/dist/pipeline/compiler/__tests__/literacy-handler.test.d.ts +10 -0
  125. package/dist/pipeline/compiler/__tests__/literacy-handler.test.js +486 -0
  126. package/dist/pipeline/compiler/__tests__/mcp-server-handler.test.d.ts +10 -0
  127. package/dist/pipeline/compiler/__tests__/mcp-server-handler.test.js +355 -0
  128. package/dist/pipeline/compiler/__tests__/promptfoo-compiler.test.d.ts +9 -0
  129. package/dist/pipeline/compiler/__tests__/promptfoo-compiler.test.js +333 -0
  130. package/dist/pipeline/compiler/__tests__/sandbox-and-fixtures.test.d.ts +12 -0
  131. package/dist/pipeline/compiler/__tests__/sandbox-and-fixtures.test.js +210 -0
  132. package/dist/pipeline/compiler/__tests__/scoring-and-presets.test.d.ts +7 -0
  133. package/dist/pipeline/compiler/__tests__/scoring-and-presets.test.js +471 -0
  134. package/dist/pipeline/compiler/__tests__/scoring-bridge.test.d.ts +10 -0
  135. package/dist/pipeline/compiler/__tests__/scoring-bridge.test.js +184 -0
  136. package/dist/pipeline/compiler/__tests__/task-graph-builder.test.d.ts +8 -0
  137. package/dist/pipeline/compiler/__tests__/task-graph-builder.test.js +301 -0
  138. package/dist/pipeline/compiler/__tests__/telemetry.test.d.ts +9 -0
  139. package/dist/pipeline/compiler/__tests__/telemetry.test.js +503 -0
  140. package/dist/pipeline/compiler/assertion-mapper.d.ts +58 -0
  141. package/dist/pipeline/compiler/assertion-mapper.js +175 -0
  142. package/dist/pipeline/compiler/compiler-to-yaml.d.ts +51 -0
  143. package/dist/pipeline/compiler/compiler-to-yaml.js +222 -0
  144. package/dist/pipeline/compiler/config-loader.d.ts +56 -0
  145. package/dist/pipeline/compiler/config-loader.js +111 -0
  146. package/dist/pipeline/compiler/fixture-resolver.d.ts +41 -0
  147. package/dist/pipeline/compiler/fixture-resolver.js +113 -0
  148. package/dist/pipeline/compiler/hash.d.ts +11 -0
  149. package/dist/pipeline/compiler/hash.js +18 -0
  150. package/dist/pipeline/compiler/ignore-fields.d.ts +53 -0
  151. package/dist/pipeline/compiler/ignore-fields.js +113 -0
  152. package/dist/pipeline/compiler/index.d.ts +29 -0
  153. package/dist/pipeline/compiler/index.js +45 -0
  154. package/dist/pipeline/compiler/literacy-bridge.d.ts +102 -0
  155. package/dist/pipeline/compiler/literacy-bridge.js +172 -0
  156. package/dist/pipeline/compiler/mode-handlers/__fixtures__/agent-harness-example-tasks.d.ts +14 -0
  157. package/dist/pipeline/compiler/mode-handlers/__fixtures__/agent-harness-example-tasks.js +152 -0
  158. package/dist/pipeline/compiler/mode-handlers/__fixtures__/knowledge-probe-example-tasks.d.ts +32 -0
  159. package/dist/pipeline/compiler/mode-handlers/__fixtures__/knowledge-probe-example-tasks.js +176 -0
  160. package/dist/pipeline/compiler/mode-handlers/__fixtures__/mcp-example-tasks.d.ts +49 -0
  161. package/dist/pipeline/compiler/mode-handlers/__fixtures__/mcp-example-tasks.js +259 -0
  162. package/dist/pipeline/compiler/mode-handlers/agent-harness-handler.d.ts +70 -0
  163. package/dist/pipeline/compiler/mode-handlers/agent-harness-handler.js +485 -0
  164. package/dist/pipeline/compiler/mode-handlers/index.d.ts +16 -0
  165. package/dist/pipeline/compiler/mode-handlers/index.js +21 -0
  166. package/dist/pipeline/compiler/mode-handlers/knowledge-probe-handler.d.ts +76 -0
  167. package/dist/pipeline/compiler/mode-handlers/knowledge-probe-handler.js +245 -0
  168. package/dist/pipeline/compiler/mode-handlers/literacy-handler.d.ts +89 -0
  169. package/dist/pipeline/compiler/mode-handlers/literacy-handler.js +379 -0
  170. package/dist/pipeline/compiler/mode-handlers/mcp-assertions.d.ts +50 -0
  171. package/dist/pipeline/compiler/mode-handlers/mcp-assertions.js +277 -0
  172. package/dist/pipeline/compiler/mode-handlers/mcp-server-handler.d.ts +67 -0
  173. package/dist/pipeline/compiler/mode-handlers/mcp-server-handler.js +309 -0
  174. package/dist/pipeline/compiler/presets/index.d.ts +9 -0
  175. package/dist/pipeline/compiler/presets/index.js +8 -0
  176. package/dist/pipeline/compiler/presets/sanity-literacy.d.ts +45 -0
  177. package/dist/pipeline/compiler/presets/sanity-literacy.js +354 -0
  178. package/dist/pipeline/compiler/promptfoo-compiler.d.ts +96 -0
  179. package/dist/pipeline/compiler/promptfoo-compiler.js +230 -0
  180. package/dist/pipeline/compiler/provider-assembler.d.ts +39 -0
  181. package/dist/pipeline/compiler/provider-assembler.js +137 -0
  182. package/dist/pipeline/compiler/sandbox/docker-sandbox.d.ts +21 -0
  183. package/dist/pipeline/compiler/sandbox/docker-sandbox.js +136 -0
  184. package/dist/pipeline/compiler/sandbox/fixture-provisioner.d.ts +69 -0
  185. package/dist/pipeline/compiler/sandbox/fixture-provisioner.js +189 -0
  186. package/dist/pipeline/compiler/sandbox/git-worktree-sandbox.d.ts +20 -0
  187. package/dist/pipeline/compiler/sandbox/git-worktree-sandbox.js +114 -0
  188. package/dist/pipeline/compiler/sandbox/index.d.ts +10 -0
  189. package/dist/pipeline/compiler/sandbox/index.js +11 -0
  190. package/dist/pipeline/compiler/sandbox/sandbox-selector.d.ts +35 -0
  191. package/dist/pipeline/compiler/sandbox/sandbox-selector.js +86 -0
  192. package/dist/pipeline/compiler/sandbox/sandbox-strategy.d.ts +81 -0
  193. package/dist/pipeline/compiler/sandbox/sandbox-strategy.js +15 -0
  194. package/dist/pipeline/compiler/sandbox/tempdir-sandbox.d.ts +20 -0
  195. package/dist/pipeline/compiler/sandbox/tempdir-sandbox.js +74 -0
  196. package/dist/pipeline/compiler/scoring-bridge.d.ts +49 -0
  197. package/dist/pipeline/compiler/scoring-bridge.js +114 -0
  198. package/dist/pipeline/compiler/task-graph-builder.d.ts +54 -0
  199. package/dist/pipeline/compiler/task-graph-builder.js +291 -0
  200. package/dist/pipeline/compiler/telemetry/cost-tracker.d.ts +90 -0
  201. package/dist/pipeline/compiler/telemetry/cost-tracker.js +146 -0
  202. package/dist/pipeline/compiler/telemetry/index.d.ts +14 -0
  203. package/dist/pipeline/compiler/telemetry/index.js +19 -0
  204. package/dist/pipeline/compiler/telemetry/redactor.d.ts +58 -0
  205. package/dist/pipeline/compiler/telemetry/redactor.js +222 -0
  206. package/dist/pipeline/compiler/telemetry/tool-classifier.d.ts +32 -0
  207. package/dist/pipeline/compiler/telemetry/tool-classifier.js +120 -0
  208. package/dist/pipeline/compiler/telemetry/trace-collector.d.ts +75 -0
  209. package/dist/pipeline/compiler/telemetry/trace-collector.js +297 -0
  210. package/dist/pipeline/compiler/telemetry/trace-store.d.ts +78 -0
  211. package/dist/pipeline/compiler/telemetry/trace-store.js +85 -0
  212. package/dist/pipeline/compiler/variable-resolver.d.ts +46 -0
  213. package/dist/pipeline/compiler/variable-resolver.js +115 -0
  214. package/dist/pipeline/coverage-audit.d.ts +15 -5
  215. package/dist/pipeline/coverage-audit.js +41 -22
  216. package/dist/pipeline/eval-constants.d.ts +16 -6
  217. package/dist/pipeline/eval-constants.js +25 -4
  218. package/dist/pipeline/eval-fingerprint.d.ts +2 -2
  219. package/dist/pipeline/eval-fingerprint.js +8 -9
  220. package/dist/pipeline/expand-tasks.d.ts +19 -10
  221. package/dist/pipeline/expand-tasks.js +34 -28
  222. package/dist/pipeline/gap-analysis.d.ts +1 -1
  223. package/dist/pipeline/gap-analysis.js +2 -2
  224. package/dist/pipeline/generate-configs.d.ts +22 -4
  225. package/dist/pipeline/generate-configs.js +53 -24
  226. package/dist/pipeline/grader-api.d.ts +3 -3
  227. package/dist/pipeline/grader-api.js +5 -12
  228. package/dist/pipeline/grader-compare-runner.js +20 -27
  229. package/dist/pipeline/grader-comparison.d.ts +4 -8
  230. package/dist/pipeline/grader-comparison.js +11 -17
  231. package/dist/pipeline/grader-consistency-runner.d.ts +2 -3
  232. package/dist/pipeline/grader-consistency-runner.js +16 -20
  233. package/dist/pipeline/grader-consistency.d.ts +6 -10
  234. package/dist/pipeline/grader-consistency.js +13 -32
  235. package/dist/pipeline/grader-sensitivity-runner.js +7 -5
  236. package/dist/pipeline/grader-sensitivity.d.ts +2 -6
  237. package/dist/pipeline/grader-sensitivity.js +10 -10
  238. package/dist/pipeline/grader-validate-runner.js +7 -5
  239. package/dist/pipeline/grader-validation.d.ts +2 -6
  240. package/dist/pipeline/grader-validation.js +14 -22
  241. package/dist/pipeline/map-request-to-config.js +6 -1
  242. package/dist/pipeline/mirror-repo-tasks.d.ts +6 -6
  243. package/dist/pipeline/mirror-repo-tasks.js +16 -15
  244. package/dist/pipeline/normalize-mode.d.ts +49 -0
  245. package/dist/pipeline/normalize-mode.js +64 -0
  246. package/dist/pipeline/plan.d.ts +5 -2
  247. package/dist/pipeline/plan.js +134 -78
  248. package/dist/pipeline/pr-comment.js +2 -0
  249. package/dist/pipeline/profile-resolution.d.ts +22 -14
  250. package/dist/pipeline/profile-resolution.js +41 -19
  251. package/dist/pipeline/provenance.d.ts +2 -2
  252. package/dist/pipeline/provenance.js +12 -17
  253. package/dist/pipeline/release-report.js +4 -4
  254. package/dist/pipeline/repo-threshold-evaluator.d.ts +1 -1
  255. package/dist/pipeline/repo-threshold-evaluator.js +1 -1
  256. package/dist/pipeline/rubric-loader.d.ts +20 -0
  257. package/dist/pipeline/rubric-loader.js +37 -0
  258. package/dist/pipeline/validate.d.ts +4 -4
  259. package/dist/pipeline/validate.js +64 -53
  260. package/dist/schedules/loader.js +18 -8
  261. package/dist/scripts/migrate-task-mode.d.ts +24 -0
  262. package/dist/scripts/migrate-task-mode.js +85 -0
  263. package/dist/scripts/migrate-tasks-to-content-lake.js +11 -10
  264. package/dist/scripts/validate-task-sources.d.ts +1 -1
  265. package/dist/scripts/validate-task-sources.js +15 -15
  266. package/dist/sinks/loader.js +5 -7
  267. package/dist/sources.d.ts +7 -7
  268. package/dist/sources.js +22 -24
  269. package/dist/webhook/dispatch.js +2 -1
  270. package/package.json +6 -3
  271. package/tasks/knowledge-probe/define-type-api.task.ts +55 -0
  272. package/tasks/knowledge-probe/groq-projections.task.ts +59 -0
  273. package/tasks/literacy/frameworks.task.ts +128 -0
  274. package/tasks/literacy/functions.task.ts +69 -0
  275. package/tasks/literacy/groq.task.ts +258 -0
  276. package/tasks/literacy/nextjs-live.task.ts +75 -0
  277. package/tasks/literacy/studio-setup.task.ts +131 -0
  278. package/tasks/literacy/visual-editing.task.ts +146 -0
  279. package/config/features.yaml +0 -116
  280. package/config/models.yaml +0 -116
  281. package/config/prompts.yaml +0 -75
  282. package/config/rubrics.yaml +0 -81
  283. package/config/schedules.yaml +0 -43
  284. package/config/sinks.yaml +0 -54
  285. package/config/sources.yaml +0 -51
  286. package/config/thresholds.yaml +0 -49
  287. package/dist/agent-observer/test-imports.d.ts +0 -7
  288. package/dist/agent-observer/test-imports.js +0 -185
@@ -0,0 +1,174 @@
1
+ /**
2
+ * define* identity helpers — typed configuration authoring functions.
3
+ *
4
+ * Each function is a **pure identity function** that returns its input
5
+ * unchanged. The value is purely at the type level: TypeScript narrows
6
+ * the parameter type so authors get full IDE autocomplete and compile-
7
+ * time validation when writing configuration files.
8
+ *
9
+ * Runtime validation is NOT performed here — it happens later in the
10
+ * pipeline via Zod schemas. This separation keeps the authoring
11
+ * experience fast (no schema import overhead at edit time) and lets
12
+ * the pipeline control when and how validation errors are surfaced.
13
+ *
14
+ * Usage:
15
+ * ```typescript
16
+ * import { defineTask, env } from "@sanity/ailf"
17
+ *
18
+ * export default defineTask({
19
+ * id: "groq-projection-basics",
20
+ * mode: "literacy",
21
+ * title: "GROQ Projection Basics",
22
+ * // ... full autocomplete here
23
+ * })
24
+ * ```
25
+ *
26
+ * @see docs/design-docs/architecture-overhaul/typescript-configuration.md (canonical)
27
+ */
28
+ import type { EvalConfig } from "./schemas/eval-config.js";
29
+ import type { FeatureRegistry, RubricConfig, ThresholdConfig } from "./schemas/pipeline.js";
30
+ import type { SchedulesFile } from "./schemas/schedules.js";
31
+ import type { SinksFile } from "./schemas/sinks.js";
32
+ import type { ModelsConfig } from "./types/index.js";
33
+ import type { GeneralizedTaskDefinition } from "./types/generalized-task.js";
34
+ /**
35
+ * Define an AILF evaluation configuration.
36
+ *
37
+ * Used in `ailf.config.ts` files for typed configuration authoring.
38
+ */
39
+ export declare function defineConfig(config: EvalConfig): EvalConfig;
40
+ /**
41
+ * Define an evaluation task with full type narrowing by mode.
42
+ *
43
+ * Setting `mode: "literacy"` narrows available fields to literacy-specific
44
+ * options; `mode: "mcp-server"` narrows to MCP-specific options; etc.
45
+ *
46
+ * The returned value is the input unchanged — this is a type-only helper.
47
+ */
48
+ export declare function defineTask(task: GeneralizedTaskDefinition): GeneralizedTaskDefinition;
49
+ /**
50
+ * Define the model registry (models to evaluate and grader model).
51
+ *
52
+ * Used in `config/models.ts` for typed model configuration.
53
+ */
54
+ export declare function defineModels(models: ModelsConfig): ModelsConfig;
55
+ /**
56
+ * Define rubric templates and scoring profiles.
57
+ *
58
+ * Used in `config/rubrics.ts` for typed rubric configuration.
59
+ */
60
+ export declare function defineRubrics(rubrics: RubricConfig): RubricConfig;
61
+ /**
62
+ * Define quality thresholds for readiness gates and regression alerts.
63
+ *
64
+ * Used in `config/thresholds.ts` for typed threshold configuration.
65
+ */
66
+ export declare function defineThresholds(thresholds: ThresholdConfig): ThresholdConfig;
67
+ /**
68
+ * Define the product feature coverage registry.
69
+ *
70
+ * Used in `config/features.ts` for typed feature tracking.
71
+ */
72
+ export declare function defineFeatures(features: FeatureRegistry): FeatureRegistry;
73
+ /**
74
+ * Define report delivery sinks (Slack, BigQuery, webhooks).
75
+ *
76
+ * Used in `config/sinks.ts` for typed sink configuration.
77
+ */
78
+ export declare function defineSinks(sinks: SinksFile): SinksFile;
79
+ /**
80
+ * Define scheduled evaluation cron jobs and digest config.
81
+ *
82
+ * Used in `config/schedules.ts` for typed schedule configuration.
83
+ */
84
+ export declare function defineSchedules(schedules: SchedulesFile): SchedulesFile;
85
+ /**
86
+ * Source configuration — typed inline until a dedicated schema exists.
87
+ *
88
+ * Sources define where documentation is fetched from (production, branch,
89
+ * local). The shape matches config/sources.yaml entries.
90
+ */
91
+ export interface SourceEntry {
92
+ /** Source name (e.g., "production", "branch", "local") */
93
+ name: string;
94
+ /** Base URL for the documentation site */
95
+ baseUrl: string;
96
+ /** Sanity project ID */
97
+ projectId?: string;
98
+ /** Sanity dataset */
99
+ dataset?: string;
100
+ /** Content release perspective */
101
+ perspective?: string;
102
+ /** Whether this is the default source */
103
+ default?: boolean;
104
+ }
105
+ /**
106
+ * Define documentation source configurations.
107
+ *
108
+ * Used in `config/sources.ts` for typed source configuration.
109
+ */
110
+ export declare function defineSources(sources: SourceEntry[]): SourceEntry[];
111
+ /**
112
+ * Prompt template configuration — typed inline until a dedicated schema exists.
113
+ */
114
+ export interface PromptEntry {
115
+ /** Unique identifier (used in Promptfoo config references) */
116
+ id?: string;
117
+ /** Human-readable label (shown in results display) */
118
+ label?: string;
119
+ /** Prompt template name */
120
+ name: string;
121
+ /** Template string with `{{variable}}` placeholders */
122
+ template: string;
123
+ /** System message (for chat-format prompts) */
124
+ systemMessage?: string;
125
+ /** Supported modes for this prompt */
126
+ modes?: string[];
127
+ }
128
+ /**
129
+ * Define prompt templates.
130
+ *
131
+ * Used in `config/prompts.ts` for typed prompt configuration.
132
+ */
133
+ export declare function definePrompts(prompts: PromptEntry[]): PromptEntry[];
134
+ /**
135
+ * A preset is a named bundle of configuration that can be referenced
136
+ * by tasks or suites to apply shared settings.
137
+ */
138
+ export interface PresetConfig {
139
+ /** Preset name */
140
+ name: string;
141
+ /** Default providers for tasks using this preset */
142
+ providers?: string[];
143
+ /** Default rubric template */
144
+ rubric?: string;
145
+ /** Default assertion templates */
146
+ assertions?: unknown[];
147
+ /** Default options */
148
+ options?: Record<string, unknown>;
149
+ }
150
+ /**
151
+ * Define reusable configuration presets.
152
+ *
153
+ * Used in `config/presets.ts` for typed preset configuration.
154
+ */
155
+ export declare function definePreset(preset: PresetConfig): PresetConfig;
156
+ /**
157
+ * A pricing table entry for cost estimation.
158
+ */
159
+ export interface PricingEntry {
160
+ /** Model ID (e.g., "openai:gpt-4o") */
161
+ modelId: string;
162
+ /** Cost per 1K input tokens in USD */
163
+ inputPer1k: number;
164
+ /** Cost per 1K output tokens in USD */
165
+ outputPer1k: number;
166
+ /** Optional label */
167
+ label?: string;
168
+ }
169
+ /**
170
+ * Define a pricing table for cost estimation.
171
+ *
172
+ * Used in `config/pricing.ts` for typed pricing configuration.
173
+ */
174
+ export declare function definePricingTable(entries: PricingEntry[]): PricingEntry[];
@@ -0,0 +1,150 @@
1
+ /**
2
+ * define* identity helpers — typed configuration authoring functions.
3
+ *
4
+ * Each function is a **pure identity function** that returns its input
5
+ * unchanged. The value is purely at the type level: TypeScript narrows
6
+ * the parameter type so authors get full IDE autocomplete and compile-
7
+ * time validation when writing configuration files.
8
+ *
9
+ * Runtime validation is NOT performed here — it happens later in the
10
+ * pipeline via Zod schemas. This separation keeps the authoring
11
+ * experience fast (no schema import overhead at edit time) and lets
12
+ * the pipeline control when and how validation errors are surfaced.
13
+ *
14
+ * Usage:
15
+ * ```typescript
16
+ * import { defineTask, env } from "@sanity/ailf"
17
+ *
18
+ * export default defineTask({
19
+ * id: "groq-projection-basics",
20
+ * mode: "literacy",
21
+ * title: "GROQ Projection Basics",
22
+ * // ... full autocomplete here
23
+ * })
24
+ * ```
25
+ *
26
+ * @see docs/design-docs/architecture-overhaul/typescript-configuration.md (canonical)
27
+ */
28
+ // ---------------------------------------------------------------------------
29
+ // Config-level helpers
30
+ // ---------------------------------------------------------------------------
31
+ /**
32
+ * Define an AILF evaluation configuration.
33
+ *
34
+ * Used in `ailf.config.ts` files for typed configuration authoring.
35
+ */
36
+ export function defineConfig(config) {
37
+ return config;
38
+ }
39
+ // ---------------------------------------------------------------------------
40
+ // Task-level helpers
41
+ // ---------------------------------------------------------------------------
42
+ /**
43
+ * Define an evaluation task with full type narrowing by mode.
44
+ *
45
+ * Setting `mode: "literacy"` narrows available fields to literacy-specific
46
+ * options; `mode: "mcp-server"` narrows to MCP-specific options; etc.
47
+ *
48
+ * The returned value is the input unchanged — this is a type-only helper.
49
+ */
50
+ export function defineTask(task) {
51
+ return task;
52
+ }
53
+ // ---------------------------------------------------------------------------
54
+ // Model registry helpers
55
+ // ---------------------------------------------------------------------------
56
+ /**
57
+ * Define the model registry (models to evaluate and grader model).
58
+ *
59
+ * Used in `config/models.ts` for typed model configuration.
60
+ */
61
+ export function defineModels(models) {
62
+ return models;
63
+ }
64
+ // ---------------------------------------------------------------------------
65
+ // Rubric helpers
66
+ // ---------------------------------------------------------------------------
67
+ /**
68
+ * Define rubric templates and scoring profiles.
69
+ *
70
+ * Used in `config/rubrics.ts` for typed rubric configuration.
71
+ */
72
+ export function defineRubrics(rubrics) {
73
+ return rubrics;
74
+ }
75
+ // ---------------------------------------------------------------------------
76
+ // Threshold helpers
77
+ // ---------------------------------------------------------------------------
78
+ /**
79
+ * Define quality thresholds for readiness gates and regression alerts.
80
+ *
81
+ * Used in `config/thresholds.ts` for typed threshold configuration.
82
+ */
83
+ export function defineThresholds(thresholds) {
84
+ return thresholds;
85
+ }
86
+ // ---------------------------------------------------------------------------
87
+ // Feature registry helpers
88
+ // ---------------------------------------------------------------------------
89
+ /**
90
+ * Define the product feature coverage registry.
91
+ *
92
+ * Used in `config/features.ts` for typed feature tracking.
93
+ */
94
+ export function defineFeatures(features) {
95
+ return features;
96
+ }
97
+ // ---------------------------------------------------------------------------
98
+ // Sink helpers
99
+ // ---------------------------------------------------------------------------
100
+ /**
101
+ * Define report delivery sinks (Slack, BigQuery, webhooks).
102
+ *
103
+ * Used in `config/sinks.ts` for typed sink configuration.
104
+ */
105
+ export function defineSinks(sinks) {
106
+ return sinks;
107
+ }
108
+ // ---------------------------------------------------------------------------
109
+ // Schedule helpers
110
+ // ---------------------------------------------------------------------------
111
+ /**
112
+ * Define scheduled evaluation cron jobs and digest config.
113
+ *
114
+ * Used in `config/schedules.ts` for typed schedule configuration.
115
+ */
116
+ export function defineSchedules(schedules) {
117
+ return schedules;
118
+ }
119
+ /**
120
+ * Define documentation source configurations.
121
+ *
122
+ * Used in `config/sources.ts` for typed source configuration.
123
+ */
124
+ export function defineSources(sources) {
125
+ return sources;
126
+ }
127
+ /**
128
+ * Define prompt templates.
129
+ *
130
+ * Used in `config/prompts.ts` for typed prompt configuration.
131
+ */
132
+ export function definePrompts(prompts) {
133
+ return prompts;
134
+ }
135
+ /**
136
+ * Define reusable configuration presets.
137
+ *
138
+ * Used in `config/presets.ts` for typed preset configuration.
139
+ */
140
+ export function definePreset(preset) {
141
+ return preset;
142
+ }
143
+ /**
144
+ * Define a pricing table for cost estimation.
145
+ *
146
+ * Used in `config/pricing.ts` for typed pricing configuration.
147
+ */
148
+ export function definePricingTable(entries) {
149
+ return entries;
150
+ }
@@ -0,0 +1,35 @@
1
+ /**
2
+ * env() — Typed environment variable access for configuration files.
3
+ *
4
+ * Provides a statically-analyzable alternative to raw `process.env` access.
5
+ * Tooling can detect which environment variables a config file depends on
6
+ * by scanning for `env("VAR_NAME")` calls.
7
+ *
8
+ * Usage:
9
+ * ```typescript
10
+ * import { env, defineConfig } from "@sanity/ailf"
11
+ *
12
+ * export default defineConfig({
13
+ * source: {
14
+ * projectId: env("SANITY_PROJECT_ID", "3do82whm"),
15
+ * dataset: env("SANITY_DATASET", "next"),
16
+ * apiToken: env("SANITY_API_TOKEN"), // throws if missing
17
+ * },
18
+ * })
19
+ * ```
20
+ *
21
+ * YAML configs continue using the existing `${{ VAR | default }}` syntax.
22
+ * The `env()` helper is for TypeScript configuration files only.
23
+ *
24
+ * @see docs/design-docs/architecture-overhaul/typescript-configuration.md
25
+ */
26
+ /**
27
+ * Read an environment variable, with an optional default.
28
+ *
29
+ * @param name - The environment variable name (e.g., `"SANITY_API_TOKEN"`)
30
+ * @param defaultValue - Fallback value if the variable is not set.
31
+ * When omitted, a missing variable throws an error.
32
+ * @returns The environment variable's value, or the default.
33
+ * @throws {Error} When the variable is not set and no default is provided.
34
+ */
35
+ export declare function env(name: string, defaultValue?: string): string;
@@ -0,0 +1,45 @@
1
+ /**
2
+ * env() — Typed environment variable access for configuration files.
3
+ *
4
+ * Provides a statically-analyzable alternative to raw `process.env` access.
5
+ * Tooling can detect which environment variables a config file depends on
6
+ * by scanning for `env("VAR_NAME")` calls.
7
+ *
8
+ * Usage:
9
+ * ```typescript
10
+ * import { env, defineConfig } from "@sanity/ailf"
11
+ *
12
+ * export default defineConfig({
13
+ * source: {
14
+ * projectId: env("SANITY_PROJECT_ID", "3do82whm"),
15
+ * dataset: env("SANITY_DATASET", "next"),
16
+ * apiToken: env("SANITY_API_TOKEN"), // throws if missing
17
+ * },
18
+ * })
19
+ * ```
20
+ *
21
+ * YAML configs continue using the existing `${{ VAR | default }}` syntax.
22
+ * The `env()` helper is for TypeScript configuration files only.
23
+ *
24
+ * @see docs/design-docs/architecture-overhaul/typescript-configuration.md
25
+ */
26
+ /**
27
+ * Read an environment variable, with an optional default.
28
+ *
29
+ * @param name - The environment variable name (e.g., `"SANITY_API_TOKEN"`)
30
+ * @param defaultValue - Fallback value if the variable is not set.
31
+ * When omitted, a missing variable throws an error.
32
+ * @returns The environment variable's value, or the default.
33
+ * @throws {Error} When the variable is not set and no default is provided.
34
+ */
35
+ export function env(name, defaultValue) {
36
+ const value = process.env[name];
37
+ if (value !== undefined && value !== "") {
38
+ return value;
39
+ }
40
+ if (defaultValue !== undefined) {
41
+ return defaultValue;
42
+ }
43
+ throw new Error(`Missing required environment variable: ${name}\n` +
44
+ `Set it in your shell, .env file, or provide a default: env("${name}", "fallback")`);
45
+ }
@@ -15,3 +15,6 @@ export * from "./schemas/index.js";
15
15
  export * from "./ports/index.js";
16
16
  export * from "./services/index.js";
17
17
  export * from "./examples/index.js";
18
+ export { defineConfig, defineFeatures, defineModels, definePricingTable, definePreset, definePrompts, defineRubrics, defineSchedules, defineSinks, defineSources, defineTask, defineThresholds, } from "./config-helpers.js";
19
+ export type { PresetConfig, PricingEntry, PromptEntry, SourceEntry, } from "./config-helpers.js";
20
+ export { env } from "./env-helper.js";
@@ -15,3 +15,8 @@ export * from "./schemas/index.js";
15
15
  export * from "./ports/index.js";
16
16
  export * from "./services/index.js";
17
17
  export * from "./examples/index.js";
18
+ // ---------------------------------------------------------------------------
19
+ // Architecture overhaul — Phase 0 helpers
20
+ // ---------------------------------------------------------------------------
21
+ export { defineConfig, defineFeatures, defineModels, definePricingTable, definePreset, definePrompts, defineRubrics, defineSchedules, defineSinks, defineSources, defineTask, defineThresholds, } from "./config-helpers.js";
22
+ export { env } from "./env-helper.js";
@@ -11,7 +11,7 @@
11
11
  * Fields marked optional are transitional — they will become required
12
12
  * as downstream consumers are converted to use them.
13
13
  */
14
- import type { DebugOptions, EvalMode } from "../types/index.js";
14
+ import type { DebugOptions, EvalMode, PluginRegistry } from "../types/index.js";
15
15
  import type { CacheStore } from "./cache-store.js";
16
16
  import type { DocFetcher } from "./doc-fetcher.js";
17
17
  import type { EvalRunner } from "./eval-runner.js";
@@ -27,8 +27,19 @@ import type { TaskSource } from "./task-source.js";
27
27
  export interface ResolvedConfig {
28
28
  /** Eval package root directory */
29
29
  rootDir: string;
30
- /** Evaluation mode */
30
+ /** Evaluation mode — canonical name (e.g., "literacy", "knowledge-probe") */
31
31
  mode: EvalMode;
32
+ /**
33
+ * Literacy variant — only meaningful when mode is "literacy".
34
+ *
35
+ * When a user passes `--mode baseline`, the CLI normalizes this to
36
+ * `mode: "literacy", variant: "baseline"`. This keeps the pipeline
37
+ * mode-agnostic while preserving literacy's multi-variant behavior.
38
+ *
39
+ * Values: "baseline" | "agentic" | "observed" | "full" | undefined
40
+ * Undefined means "use the default variant for the mode" (baseline for literacy).
41
+ */
42
+ variant?: string;
32
43
  /** Debug options */
33
44
  debug?: DebugOptions;
34
45
  /** Feature area filter */
@@ -153,6 +164,8 @@ export interface AppContext {
153
164
  readonly evalRunner: EvalRunner;
154
165
  /** Structured logger */
155
166
  readonly logger: Logger;
167
+ /** Plugin registry — mode handlers, assertions, rubric templates, etc. */
168
+ readonly registry: PluginRegistry;
156
169
  /**
157
170
  * Persistent report store (Sanity Content Lake).
158
171
  * Optional — not all commands need it. Commands that publish or
@@ -9,7 +9,7 @@
9
9
  * The pipeline orchestrator and all downstream steps work with
10
10
  * FetchResult regardless of where the documentation came from.
11
11
  */
12
- import type { TaskDefinition } from "./task-source.js";
12
+ import type { GeneralizedTaskDefinition } from "../types/generalized-task.js";
13
13
  /**
14
14
  * A fetched documentation context ready for injection into prompts.
15
15
  *
@@ -127,5 +127,5 @@ export interface DocFetcher {
127
127
  * @param source — Where to fetch documentation from
128
128
  * @returns Fetched doc contexts + optional metadata
129
129
  */
130
- fetch(tasks: TaskDefinition[], source?: DocSourceConfig): Promise<FetchResult>;
130
+ fetch(tasks: GeneralizedTaskDefinition[], source?: DocSourceConfig): Promise<FetchResult>;
131
131
  }
@@ -9,7 +9,8 @@ export type { ConfigSource } from "./config-source.js";
9
9
  export type { AppContext, ReportSinkPort, ReportStorePort, ResolvedConfig, } from "./context.js";
10
10
  export type { DocContext, DocFetcher, DocSourceConfig, DocumentManifestEntry, DocumentOverlaySummary, FetchMetadata, FetchResult, ReleaseImpact, UrlFetchEntry, UrlFetchSummary, } from "./doc-fetcher.js";
11
11
  export type { EvalRunConfig, EvalRunner } from "./eval-runner.js";
12
+ export type { CompilationContext, CompileResultAssertion, CompileResultPrompt, CompileResultProvider, CompileResultTestCase, ModeCompileResult, ModeHandler, ModeProviderEntry, ModeRubricConfig, PromptTemplate, } from "./mode-handler.js";
12
13
  export type { Logger } from "./logger.js";
13
14
  export type { PipelineStep } from "./pipeline-step.js";
14
- export type { AssertionDefinition, BaselineConfig, CanonicalDocRef, IdDocRef, PathDocRef, PerspectiveDocRef, SlugDocRef, TaskDefinition, TaskSource, TemplatedAssertion, ValueAssertion, } from "./task-source.js";
15
+ export type { TaskSource } from "./task-source.js";
15
16
  export { canonicalDocRefLabel, isIdRef, isPathRef, isPerspectiveRef, isSlugRef, isTemplatedAssertion, } from "./task-source.js";
@@ -0,0 +1,129 @@
1
+ /**
2
+ * ModeHandler — the common interface every evaluation mode implements.
3
+ *
4
+ * The pipeline dispatches to mode handlers through the PluginRegistry:
5
+ * 1. Look up the mode: `ctx.registry.getMode(mode)`
6
+ * 2. Import the handler module: `import(registration.handlerModule)`
7
+ * 3. Call: `module.handler.compileTask(task, ctx)`
8
+ *
9
+ * Each handler file exports a `handler` object conforming to this interface.
10
+ * The handler narrows the GeneralizedTaskDefinition to its mode-specific
11
+ * variant and produces a ModeCompileResult.
12
+ *
13
+ * Types here are minimal structural contracts — the eval package's Promptfoo
14
+ * types satisfy them via TypeScript structural compatibility.
15
+ *
16
+ * @see docs/design-docs/architecture-overhaul/extensibility-plugins.md
17
+ * @see packages/eval/src/pipeline/compiler/mode-handlers/
18
+ */
19
+ import type { GeneralizedTaskDefinition } from "../types/generalized-task.js";
20
+ /**
21
+ * A prompt template owned by a mode handler.
22
+ *
23
+ * Mode handlers can return these via `getPrompts()` to override the global
24
+ * config/prompts.ts templates. This lets non-literacy modes define their
25
+ * own prompt structures without polluting the global config.
26
+ */
27
+ export interface PromptTemplate {
28
+ /** Unique identifier (e.g. "with-docs", "agentic") */
29
+ id: string;
30
+ /** Human-readable label for display */
31
+ label: string;
32
+ /** The prompt template string with {{variable}} placeholders */
33
+ template: string;
34
+ /** Variable names used by this template — for documentation only */
35
+ variables?: string[];
36
+ }
37
+ /** Compilation context — shared state the pipeline provides to every handler */
38
+ export interface CompilationContext {
39
+ /** Eval package root directory (for resolving file paths) */
40
+ rootDir: string;
41
+ /** Grader provider ID for LLM-graded assertions */
42
+ graderProvider?: string;
43
+ /** Model providers to include in the evaluation */
44
+ models?: ModeProviderEntry[];
45
+ /** Rubric config (templates, weights) — loaded from config/rubrics */
46
+ rubricConfig?: ModeRubricConfig;
47
+ }
48
+ /** A model provider entry for compilation */
49
+ export interface ModeProviderEntry {
50
+ id: string;
51
+ label: string;
52
+ config?: Record<string, unknown>;
53
+ }
54
+ /** Minimal rubric config needed by mode handlers */
55
+ export interface ModeRubricConfig {
56
+ templates: Record<string, {
57
+ dimension?: string;
58
+ header: string;
59
+ scale: string[];
60
+ criteria_label?: string;
61
+ }>;
62
+ }
63
+ /** A provider entry in the compile result */
64
+ export interface CompileResultProvider {
65
+ id: string;
66
+ label?: string;
67
+ config?: Record<string, unknown>;
68
+ }
69
+ /** A prompt entry in the compile result */
70
+ export interface CompileResultPrompt {
71
+ id: string;
72
+ label: string;
73
+ raw: string;
74
+ }
75
+ /** A test case entry in the compile result */
76
+ export interface CompileResultTestCase {
77
+ description: string;
78
+ vars: Record<string, unknown>;
79
+ assert?: CompileResultAssertion[];
80
+ prompts?: string[];
81
+ }
82
+ /** An assertion entry in a test case */
83
+ export interface CompileResultAssertion {
84
+ type: string;
85
+ value?: unknown;
86
+ weight?: number;
87
+ provider?: string;
88
+ metadata?: Record<string, unknown>;
89
+ }
90
+ /**
91
+ * ModeCompileResult — the common output every mode handler produces.
92
+ *
93
+ * All four fields (providers, tests, prompts, warnings) are present in
94
+ * every handler's result type. Mode-specific extras (sandbox config,
95
+ * metadata, extensions) go in the `extras` bag.
96
+ */
97
+ export interface ModeCompileResult {
98
+ /** Provider configurations for Promptfoo */
99
+ providers: CompileResultProvider[];
100
+ /** Compiled test cases */
101
+ tests: CompileResultTestCase[];
102
+ /** Prompt templates */
103
+ prompts: CompileResultPrompt[];
104
+ /** Warnings generated during compilation */
105
+ warnings: string[];
106
+ /** Mode-specific extras (extensions, sandboxConfig, metadata, etc.) */
107
+ extras?: Record<string, unknown>;
108
+ }
109
+ /**
110
+ * ModeHandler — the interface every evaluation mode handler exports.
111
+ *
112
+ * Handler modules are referenced by `ModeRegistration.handlerModule` in the
113
+ * plugin registry. The pipeline imports the module and calls `handler.compileTask()`.
114
+ *
115
+ * Each handler file should export:
116
+ * export const handler: ModeHandler = { ... }
117
+ */
118
+ export interface ModeHandler {
119
+ /** Compile a task definition into evaluation configuration */
120
+ compileTask(task: GeneralizedTaskDefinition, ctx: CompilationContext): ModeCompileResult;
121
+ /**
122
+ * Return prompt templates owned by this mode.
123
+ *
124
+ * When defined, the compiler uses these instead of global config/prompts.ts.
125
+ * Keys are prompt IDs (e.g. "with-docs", "agentic"). Returning undefined
126
+ * or omitting the method falls back to global prompts.
127
+ */
128
+ getPrompts?(): Record<string, PromptTemplate>;
129
+ }
@@ -0,0 +1,19 @@
1
+ /**
2
+ * ModeHandler — the common interface every evaluation mode implements.
3
+ *
4
+ * The pipeline dispatches to mode handlers through the PluginRegistry:
5
+ * 1. Look up the mode: `ctx.registry.getMode(mode)`
6
+ * 2. Import the handler module: `import(registration.handlerModule)`
7
+ * 3. Call: `module.handler.compileTask(task, ctx)`
8
+ *
9
+ * Each handler file exports a `handler` object conforming to this interface.
10
+ * The handler narrows the GeneralizedTaskDefinition to its mode-specific
11
+ * variant and produces a ModeCompileResult.
12
+ *
13
+ * Types here are minimal structural contracts — the eval package's Promptfoo
14
+ * types satisfy them via TypeScript structural compatibility.
15
+ *
16
+ * @see docs/design-docs/architecture-overhaul/extensibility-plugins.md
17
+ * @see packages/eval/src/pipeline/compiler/mode-handlers/
18
+ */
19
+ export {};