@sanity/ailf 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (530) hide show
  1. package/README.md +89 -0
  2. package/bin/ailf.js +64 -0
  3. package/canonical/grader-references/README.md +88 -0
  4. package/canonical/grader-references/groq.yaml +234 -0
  5. package/canonical/grader-references/studio-setup.yaml +275 -0
  6. package/canonical/reference-solutions/.gitkeep +1 -0
  7. package/canonical/reference-solutions/frameworks/nuxt.ts +119 -0
  8. package/canonical/reference-solutions/frameworks/remix.tsx +100 -0
  9. package/canonical/reference-solutions/functions/publish-webhook.ts +60 -0
  10. package/canonical/reference-solutions/groq/advanced-filtering.ts +379 -0
  11. package/canonical/reference-solutions/groq/blog-queries.ts +137 -0
  12. package/canonical/reference-solutions/groq/joins-references.ts +300 -0
  13. package/canonical/reference-solutions/nextjs/app-router-integration.tsx +128 -0
  14. package/canonical/reference-solutions/studio-setup/blog-schema.ts +143 -0
  15. package/canonical/reference-solutions/studio-setup/custom-tool.tsx +78 -0
  16. package/canonical/reference-solutions/visual-editing/live-preview.tsx +137 -0
  17. package/canonical/reference-solutions/visual-editing/presentation-nextjs.tsx +130 -0
  18. package/config/airbyte/ai_literacy_framework.connector.yaml +639 -0
  19. package/config/bigquery/README.md +74 -0
  20. package/config/bigquery/views/area_scores.sql +87 -0
  21. package/config/bigquery/views/reports.sql +49 -0
  22. package/config/features.yaml +116 -0
  23. package/config/models.yaml +115 -0
  24. package/config/prompts.yaml +75 -0
  25. package/config/rubrics.yaml +62 -0
  26. package/config/schedules.yaml +43 -0
  27. package/config/sinks.yaml +54 -0
  28. package/config/sources.yaml +51 -0
  29. package/config/thresholds.yaml +49 -0
  30. package/dist/_vendor/ailf-core/examples/index.d.ts +190 -0
  31. package/dist/_vendor/ailf-core/examples/index.js +285 -0
  32. package/dist/_vendor/ailf-core/index.d.ts +17 -0
  33. package/dist/_vendor/ailf-core/index.js +17 -0
  34. package/dist/_vendor/ailf-core/ports/cache-store.d.ts +72 -0
  35. package/dist/_vendor/ailf-core/ports/cache-store.js +17 -0
  36. package/dist/_vendor/ailf-core/ports/config-source.d.ts +33 -0
  37. package/dist/_vendor/ailf-core/ports/config-source.js +15 -0
  38. package/dist/_vendor/ailf-core/ports/context.d.ts +172 -0
  39. package/dist/_vendor/ailf-core/ports/context.js +14 -0
  40. package/dist/_vendor/ailf-core/ports/doc-fetcher.d.ts +131 -0
  41. package/dist/_vendor/ailf-core/ports/doc-fetcher.js +12 -0
  42. package/dist/_vendor/ailf-core/ports/eval-runner.d.ts +24 -0
  43. package/dist/_vendor/ailf-core/ports/eval-runner.js +8 -0
  44. package/dist/_vendor/ailf-core/ports/index.d.ts +15 -0
  45. package/dist/_vendor/ailf-core/ports/index.js +7 -0
  46. package/dist/_vendor/ailf-core/ports/logger.d.ts +36 -0
  47. package/dist/_vendor/ailf-core/ports/logger.js +11 -0
  48. package/dist/_vendor/ailf-core/ports/pipeline-step.d.ts +46 -0
  49. package/dist/_vendor/ailf-core/ports/pipeline-step.js +8 -0
  50. package/dist/_vendor/ailf-core/ports/task-source.d.ts +159 -0
  51. package/dist/_vendor/ailf-core/ports/task-source.js +72 -0
  52. package/dist/_vendor/ailf-core/schemas/callback-payload.d.ts +24 -0
  53. package/dist/_vendor/ailf-core/schemas/callback-payload.js +29 -0
  54. package/dist/_vendor/ailf-core/schemas/eval-config.d.ts +55 -0
  55. package/dist/_vendor/ailf-core/schemas/eval-config.js +78 -0
  56. package/dist/_vendor/ailf-core/schemas/index.d.ts +16 -0
  57. package/dist/_vendor/ailf-core/schemas/index.js +16 -0
  58. package/dist/_vendor/ailf-core/schemas/pipeline-request.d.ts +125 -0
  59. package/dist/_vendor/ailf-core/schemas/pipeline-request.js +67 -0
  60. package/dist/_vendor/ailf-core/schemas/pipeline.d.ts +531 -0
  61. package/dist/_vendor/ailf-core/schemas/pipeline.js +318 -0
  62. package/dist/_vendor/ailf-core/schemas/schedules.d.ts +68 -0
  63. package/dist/_vendor/ailf-core/schemas/schedules.js +74 -0
  64. package/dist/_vendor/ailf-core/schemas/sinks.d.ts +207 -0
  65. package/dist/_vendor/ailf-core/schemas/sinks.js +108 -0
  66. package/dist/_vendor/ailf-core/services/comparison-formatters.d.ts +18 -0
  67. package/dist/_vendor/ailf-core/services/comparison-formatters.js +189 -0
  68. package/dist/_vendor/ailf-core/services/config-helpers.d.ts +41 -0
  69. package/dist/_vendor/ailf-core/services/config-helpers.js +86 -0
  70. package/dist/_vendor/ailf-core/services/index.d.ts +12 -0
  71. package/dist/_vendor/ailf-core/services/index.js +12 -0
  72. package/dist/_vendor/ailf-core/services/scoring.d.ts +49 -0
  73. package/dist/_vendor/ailf-core/services/scoring.js +222 -0
  74. package/dist/_vendor/ailf-core/types/index.d.ts +1082 -0
  75. package/dist/_vendor/ailf-core/types/index.js +21 -0
  76. package/dist/_vendor/ailf-core/types/scoring-input.d.ts +54 -0
  77. package/dist/_vendor/ailf-core/types/scoring-input.js +9 -0
  78. package/dist/_vendor/ailf-shared/dimension-names.d.ts +21 -0
  79. package/dist/_vendor/ailf-shared/dimension-names.js +27 -0
  80. package/dist/_vendor/ailf-shared/document-ref.d.ts +29 -0
  81. package/dist/_vendor/ailf-shared/document-ref.js +1 -0
  82. package/dist/_vendor/ailf-shared/eval-modes.d.ts +12 -0
  83. package/dist/_vendor/ailf-shared/eval-modes.js +8 -0
  84. package/dist/_vendor/ailf-shared/index.d.ts +16 -0
  85. package/dist/_vendor/ailf-shared/index.js +16 -0
  86. package/dist/_vendor/ailf-shared/noise-threshold.d.ts +9 -0
  87. package/dist/_vendor/ailf-shared/noise-threshold.js +9 -0
  88. package/dist/_vendor/ailf-shared/score-grades.d.ts +17 -0
  89. package/dist/_vendor/ailf-shared/score-grades.js +23 -0
  90. package/dist/adapters/cache/content-lake-cache.d.ts +24 -0
  91. package/dist/adapters/cache/content-lake-cache.js +59 -0
  92. package/dist/adapters/cache/filesystem-cache.d.ts +18 -0
  93. package/dist/adapters/cache/filesystem-cache.js +54 -0
  94. package/dist/adapters/cache/index.d.ts +2 -0
  95. package/dist/adapters/cache/index.js +2 -0
  96. package/dist/adapters/config-sources/cli-config-adapter.d.ts +17 -0
  97. package/dist/adapters/config-sources/cli-config-adapter.js +23 -0
  98. package/dist/adapters/config-sources/file-config-adapter.d.ts +26 -0
  99. package/dist/adapters/config-sources/file-config-adapter.js +96 -0
  100. package/dist/adapters/config-sources/index.d.ts +2 -0
  101. package/dist/adapters/config-sources/index.js +2 -0
  102. package/dist/adapters/doc-fetchers/index.d.ts +1 -0
  103. package/dist/adapters/doc-fetchers/index.js +1 -0
  104. package/dist/adapters/doc-fetchers/sanity-doc-fetcher.d.ts +76 -0
  105. package/dist/adapters/doc-fetchers/sanity-doc-fetcher.js +620 -0
  106. package/dist/adapters/eval-runners/index.d.ts +1 -0
  107. package/dist/adapters/eval-runners/index.js +1 -0
  108. package/dist/adapters/eval-runners/promptfoo-eval-adapter.d.ts +14 -0
  109. package/dist/adapters/eval-runners/promptfoo-eval-adapter.js +63 -0
  110. package/dist/adapters/index.d.ts +12 -0
  111. package/dist/adapters/index.js +12 -0
  112. package/dist/adapters/loggers/console-logger.d.ts +22 -0
  113. package/dist/adapters/loggers/console-logger.js +54 -0
  114. package/dist/adapters/loggers/index.d.ts +9 -0
  115. package/dist/adapters/loggers/index.js +9 -0
  116. package/dist/adapters/loggers/json-logger.d.ts +18 -0
  117. package/dist/adapters/loggers/json-logger.js +33 -0
  118. package/dist/adapters/loggers/quiet-logger.d.ts +16 -0
  119. package/dist/adapters/loggers/quiet-logger.js +30 -0
  120. package/dist/adapters/task-sources/composite-task-source.d.ts +20 -0
  121. package/dist/adapters/task-sources/composite-task-source.js +59 -0
  122. package/dist/adapters/task-sources/content-lake-task-source.d.ts +20 -0
  123. package/dist/adapters/task-sources/content-lake-task-source.js +219 -0
  124. package/dist/adapters/task-sources/index.d.ts +7 -0
  125. package/dist/adapters/task-sources/index.js +7 -0
  126. package/dist/adapters/task-sources/repo-schemas.d.ts +245 -0
  127. package/dist/adapters/task-sources/repo-schemas.js +234 -0
  128. package/dist/adapters/task-sources/repo-task-source.d.ts +22 -0
  129. package/dist/adapters/task-sources/repo-task-source.js +104 -0
  130. package/dist/adapters/task-sources/repo-trigger.d.ts +52 -0
  131. package/dist/adapters/task-sources/repo-trigger.js +153 -0
  132. package/dist/adapters/task-sources/repo-validation.d.ts +49 -0
  133. package/dist/adapters/task-sources/repo-validation.js +164 -0
  134. package/dist/adapters/task-sources/yaml-task-source.d.ts +18 -0
  135. package/dist/adapters/task-sources/yaml-task-source.js +136 -0
  136. package/dist/agent-observer/agentic-provider.d.ts +132 -0
  137. package/dist/agent-observer/agentic-provider.js +983 -0
  138. package/dist/agent-observer/classifier.d.ts +62 -0
  139. package/dist/agent-observer/classifier.js +269 -0
  140. package/dist/agent-observer/index.d.ts +7 -0
  141. package/dist/agent-observer/index.js +4 -0
  142. package/dist/agent-observer/pricing.d.ts +35 -0
  143. package/dist/agent-observer/pricing.js +82 -0
  144. package/dist/agent-observer/provider.d.ts +77 -0
  145. package/dist/agent-observer/provider.js +151 -0
  146. package/dist/agent-observer/proxy.d.ts +91 -0
  147. package/dist/agent-observer/proxy.js +321 -0
  148. package/dist/agent-observer/test-imports.d.ts +7 -0
  149. package/dist/agent-observer/test-imports.js +185 -0
  150. package/dist/agent-observer/types.d.ts +137 -0
  151. package/dist/agent-observer/types.js +16 -0
  152. package/dist/assertions/source-isolation.d.ts +72 -0
  153. package/dist/assertions/source-isolation.js +117 -0
  154. package/dist/cli.d.ts +24 -0
  155. package/dist/cli.js +199 -0
  156. package/dist/commands/agent-report.d.ts +5 -0
  157. package/dist/commands/agent-report.js +69 -0
  158. package/dist/commands/baseline.d.ts +9 -0
  159. package/dist/commands/baseline.js +141 -0
  160. package/dist/commands/cache.d.ts +13 -0
  161. package/dist/commands/cache.js +135 -0
  162. package/dist/commands/calculate-scores.d.ts +8 -0
  163. package/dist/commands/calculate-scores.js +48 -0
  164. package/dist/commands/compare.d.ts +8 -0
  165. package/dist/commands/compare.js +120 -0
  166. package/dist/commands/completion.d.ts +18 -0
  167. package/dist/commands/completion.js +260 -0
  168. package/dist/commands/coverage-audit.d.ts +7 -0
  169. package/dist/commands/coverage-audit.js +40 -0
  170. package/dist/commands/discovery-report.d.ts +10 -0
  171. package/dist/commands/discovery-report.js +44 -0
  172. package/dist/commands/eval.d.ts +9 -0
  173. package/dist/commands/eval.js +35 -0
  174. package/dist/commands/explain-handler.d.ts +34 -0
  175. package/dist/commands/explain-handler.js +719 -0
  176. package/dist/commands/fetch-docs.d.ts +8 -0
  177. package/dist/commands/fetch-docs.js +128 -0
  178. package/dist/commands/generate-configs.d.ts +8 -0
  179. package/dist/commands/generate-configs.js +46 -0
  180. package/dist/commands/grader/index.d.ts +11 -0
  181. package/dist/commands/grader/index.js +118 -0
  182. package/dist/commands/init.d.ts +19 -0
  183. package/dist/commands/init.js +150 -0
  184. package/dist/commands/interactive.d.ts +12 -0
  185. package/dist/commands/interactive.js +238 -0
  186. package/dist/commands/lookup-doc.d.ts +15 -0
  187. package/dist/commands/lookup-doc.js +84 -0
  188. package/dist/commands/measure-retrieval.d.ts +5 -0
  189. package/dist/commands/measure-retrieval.js +65 -0
  190. package/dist/commands/pipeline-action.d.ts +71 -0
  191. package/dist/commands/pipeline-action.js +305 -0
  192. package/dist/commands/pipeline.d.ts +62 -0
  193. package/dist/commands/pipeline.js +53 -0
  194. package/dist/commands/pr-comment.d.ts +8 -0
  195. package/dist/commands/pr-comment.js +47 -0
  196. package/dist/commands/publish.d.ts +26 -0
  197. package/dist/commands/publish.js +253 -0
  198. package/dist/commands/readiness-report.d.ts +10 -0
  199. package/dist/commands/readiness-report.js +104 -0
  200. package/dist/commands/shared/options.d.ts +29 -0
  201. package/dist/commands/shared/options.js +57 -0
  202. package/dist/commands/update-quality-scores.d.ts +5 -0
  203. package/dist/commands/update-quality-scores.js +20 -0
  204. package/dist/commands/validate-tasks.d.ts +16 -0
  205. package/dist/commands/validate-tasks.js +93 -0
  206. package/dist/commands/validate.d.ts +9 -0
  207. package/dist/commands/validate.js +73 -0
  208. package/dist/commands/webhook-server.d.ts +5 -0
  209. package/dist/commands/webhook-server.js +30 -0
  210. package/dist/commands/weekly-digest.d.ts +10 -0
  211. package/dist/commands/weekly-digest.js +104 -0
  212. package/dist/composition-root.d.ts +26 -0
  213. package/dist/composition-root.js +107 -0
  214. package/dist/interpolate.d.ts +26 -0
  215. package/dist/interpolate.js +70 -0
  216. package/dist/job-store.d.ts +104 -0
  217. package/dist/job-store.js +188 -0
  218. package/dist/lib/agent-behavior-report.d.ts +8 -0
  219. package/dist/lib/agent-behavior-report.js +185 -0
  220. package/dist/lib/baseline.d.ts +19 -0
  221. package/dist/lib/baseline.js +153 -0
  222. package/dist/lib/calculate-scores.d.ts +23 -0
  223. package/dist/lib/calculate-scores.js +42 -0
  224. package/dist/lib/compare.d.ts +18 -0
  225. package/dist/lib/compare.js +170 -0
  226. package/dist/lib/coverage-audit.d.ts +4 -0
  227. package/dist/lib/coverage-audit.js +42 -0
  228. package/dist/lib/discovery-report.d.ts +13 -0
  229. package/dist/lib/discovery-report.js +57 -0
  230. package/dist/lib/fetch-docs.d.ts +30 -0
  231. package/dist/lib/fetch-docs.js +171 -0
  232. package/dist/lib/generate-configs.d.ts +25 -0
  233. package/dist/lib/generate-configs.js +42 -0
  234. package/dist/lib/grader-api.d.ts +21 -0
  235. package/dist/lib/grader-api.js +34 -0
  236. package/dist/lib/grader-compare.d.ts +19 -0
  237. package/dist/lib/grader-compare.js +91 -0
  238. package/dist/lib/grader-consistency.d.ts +27 -0
  239. package/dist/lib/grader-consistency.js +79 -0
  240. package/dist/lib/grader-sensitivity.d.ts +19 -0
  241. package/dist/lib/grader-sensitivity.js +75 -0
  242. package/dist/lib/grader-validate.d.ts +19 -0
  243. package/dist/lib/grader-validate.js +78 -0
  244. package/dist/lib/measure-retrieval.d.ts +14 -0
  245. package/dist/lib/measure-retrieval.js +71 -0
  246. package/dist/lib/pr-comment.d.ts +16 -0
  247. package/dist/lib/pr-comment.js +28 -0
  248. package/dist/lib/readiness-report.d.ts +13 -0
  249. package/dist/lib/readiness-report.js +108 -0
  250. package/dist/lib/webhook-server.d.ts +11 -0
  251. package/dist/lib/webhook-server.js +24 -0
  252. package/dist/lib/weekly-digest.d.ts +24 -0
  253. package/dist/lib/weekly-digest.js +148 -0
  254. package/dist/orchestration/build-app-context.d.ts +27 -0
  255. package/dist/orchestration/build-app-context.js +81 -0
  256. package/dist/orchestration/build-step-sequence.d.ts +15 -0
  257. package/dist/orchestration/build-step-sequence.js +84 -0
  258. package/dist/orchestration/config-to-source-overrides.d.ts +9 -0
  259. package/dist/orchestration/config-to-source-overrides.js +28 -0
  260. package/dist/orchestration/env-bridge.d.ts +21 -0
  261. package/dist/orchestration/env-bridge.js +66 -0
  262. package/dist/orchestration/index.d.ts +11 -0
  263. package/dist/orchestration/index.js +11 -0
  264. package/dist/orchestration/pipeline-orchestrator.d.ts +24 -0
  265. package/dist/orchestration/pipeline-orchestrator.js +153 -0
  266. package/dist/orchestration/step-runner.d.ts +20 -0
  267. package/dist/orchestration/step-runner.js +88 -0
  268. package/dist/orchestration/steps/calculate-scores-step.d.ts +13 -0
  269. package/dist/orchestration/steps/calculate-scores-step.js +95 -0
  270. package/dist/orchestration/steps/callback-step.d.ts +24 -0
  271. package/dist/orchestration/steps/callback-step.js +76 -0
  272. package/dist/orchestration/steps/compare-step.d.ts +14 -0
  273. package/dist/orchestration/steps/compare-step.js +92 -0
  274. package/dist/orchestration/steps/discovery-report-step.d.ts +13 -0
  275. package/dist/orchestration/steps/discovery-report-step.js +55 -0
  276. package/dist/orchestration/steps/fetch-docs-shell.d.ts +17 -0
  277. package/dist/orchestration/steps/fetch-docs-shell.js +30 -0
  278. package/dist/orchestration/steps/fetch-docs-step.d.ts +14 -0
  279. package/dist/orchestration/steps/fetch-docs-step.js +135 -0
  280. package/dist/orchestration/steps/gap-analysis-step.d.ts +16 -0
  281. package/dist/orchestration/steps/gap-analysis-step.js +136 -0
  282. package/dist/orchestration/steps/generate-configs-step.d.ts +14 -0
  283. package/dist/orchestration/steps/generate-configs-step.js +85 -0
  284. package/dist/orchestration/steps/grader-consistency-step.d.ts +13 -0
  285. package/dist/orchestration/steps/grader-consistency-step.js +64 -0
  286. package/dist/orchestration/steps/index.d.ts +19 -0
  287. package/dist/orchestration/steps/index.js +19 -0
  288. package/dist/orchestration/steps/mirror-repo-tasks-step.d.ts +21 -0
  289. package/dist/orchestration/steps/mirror-repo-tasks-step.js +94 -0
  290. package/dist/orchestration/steps/publish-report-step.d.ts +26 -0
  291. package/dist/orchestration/steps/publish-report-step.js +216 -0
  292. package/dist/orchestration/steps/readiness-step.d.ts +13 -0
  293. package/dist/orchestration/steps/readiness-step.js +91 -0
  294. package/dist/orchestration/steps/report-step.d.ts +12 -0
  295. package/dist/orchestration/steps/report-step.js +49 -0
  296. package/dist/orchestration/steps/run-eval-step.d.ts +17 -0
  297. package/dist/orchestration/steps/run-eval-step.js +195 -0
  298. package/dist/orchestration/steps/validate-step.d.ts +12 -0
  299. package/dist/orchestration/steps/validate-step.js +41 -0
  300. package/dist/pipeline/agent-behavior-report.d.ts +53 -0
  301. package/dist/pipeline/agent-behavior-report.js +132 -0
  302. package/dist/pipeline/attribution.d.ts +47 -0
  303. package/dist/pipeline/attribution.js +226 -0
  304. package/dist/pipeline/baseline.d.ts +37 -0
  305. package/dist/pipeline/baseline.js +141 -0
  306. package/dist/pipeline/cache.d.ts +101 -0
  307. package/dist/pipeline/cache.js +283 -0
  308. package/dist/pipeline/calculate-scores.d.ts +102 -0
  309. package/dist/pipeline/calculate-scores.js +1128 -0
  310. package/dist/pipeline/callback-delivery.d.ts +50 -0
  311. package/dist/pipeline/callback-delivery.js +89 -0
  312. package/dist/pipeline/checks.d.ts +39 -0
  313. package/dist/pipeline/checks.js +280 -0
  314. package/dist/pipeline/classify-url.d.ts +61 -0
  315. package/dist/pipeline/classify-url.js +93 -0
  316. package/dist/pipeline/compare.d.ts +31 -0
  317. package/dist/pipeline/compare.js +208 -0
  318. package/dist/pipeline/coverage-audit.d.ts +39 -0
  319. package/dist/pipeline/coverage-audit.js +165 -0
  320. package/dist/pipeline/degradations.d.ts +85 -0
  321. package/dist/pipeline/degradations.js +242 -0
  322. package/dist/pipeline/discovery-report.d.ts +55 -0
  323. package/dist/pipeline/discovery-report.js +178 -0
  324. package/dist/pipeline/eval-constants.d.ts +68 -0
  325. package/dist/pipeline/eval-constants.js +111 -0
  326. package/dist/pipeline/eval-fingerprint.d.ts +66 -0
  327. package/dist/pipeline/eval-fingerprint.js +175 -0
  328. package/dist/pipeline/expand-tasks.d.ts +220 -0
  329. package/dist/pipeline/expand-tasks.js +421 -0
  330. package/dist/pipeline/failure-modes.d.ts +46 -0
  331. package/dist/pipeline/failure-modes.js +348 -0
  332. package/dist/pipeline/fetch-url-content.d.ts +44 -0
  333. package/dist/pipeline/fetch-url-content.js +93 -0
  334. package/dist/pipeline/gap-analysis.d.ts +48 -0
  335. package/dist/pipeline/gap-analysis.js +231 -0
  336. package/dist/pipeline/generate-configs.d.ts +72 -0
  337. package/dist/pipeline/generate-configs.js +395 -0
  338. package/dist/pipeline/grader-api.d.ts +49 -0
  339. package/dist/pipeline/grader-api.js +200 -0
  340. package/dist/pipeline/grader-compare-runner.d.ts +44 -0
  341. package/dist/pipeline/grader-compare-runner.js +301 -0
  342. package/dist/pipeline/grader-comparison.d.ts +111 -0
  343. package/dist/pipeline/grader-comparison.js +161 -0
  344. package/dist/pipeline/grader-consistency-runner.d.ts +60 -0
  345. package/dist/pipeline/grader-consistency-runner.js +270 -0
  346. package/dist/pipeline/grader-consistency.d.ts +103 -0
  347. package/dist/pipeline/grader-consistency.js +146 -0
  348. package/dist/pipeline/grader-sensitivity-runner.d.ts +40 -0
  349. package/dist/pipeline/grader-sensitivity-runner.js +282 -0
  350. package/dist/pipeline/grader-sensitivity.d.ts +94 -0
  351. package/dist/pipeline/grader-sensitivity.js +144 -0
  352. package/dist/pipeline/grader-validate-runner.d.ts +38 -0
  353. package/dist/pipeline/grader-validate-runner.js +229 -0
  354. package/dist/pipeline/grader-validation.d.ts +107 -0
  355. package/dist/pipeline/grader-validation.js +169 -0
  356. package/dist/pipeline/map-request-to-config.d.ts +19 -0
  357. package/dist/pipeline/map-request-to-config.js +80 -0
  358. package/dist/pipeline/measure-retrieval.d.ts +59 -0
  359. package/dist/pipeline/measure-retrieval.js +111 -0
  360. package/dist/pipeline/mirror-repo-tasks.d.ts +86 -0
  361. package/dist/pipeline/mirror-repo-tasks.js +350 -0
  362. package/dist/pipeline/plan-format.d.ts +33 -0
  363. package/dist/pipeline/plan-format.js +202 -0
  364. package/dist/pipeline/plan.d.ts +169 -0
  365. package/dist/pipeline/plan.js +708 -0
  366. package/dist/pipeline/pr-comment.d.ts +19 -0
  367. package/dist/pipeline/pr-comment.js +502 -0
  368. package/dist/pipeline/probe.d.ts +52 -0
  369. package/dist/pipeline/probe.js +390 -0
  370. package/dist/pipeline/provenance.d.ts +47 -0
  371. package/dist/pipeline/provenance.js +146 -0
  372. package/dist/pipeline/readiness-report.d.ts +87 -0
  373. package/dist/pipeline/readiness-report.js +205 -0
  374. package/dist/pipeline/release-classification.d.ts +54 -0
  375. package/dist/pipeline/release-classification.js +238 -0
  376. package/dist/pipeline/release-report.d.ts +37 -0
  377. package/dist/pipeline/release-report.js +222 -0
  378. package/dist/pipeline/repo-eval-comment.d.ts +37 -0
  379. package/dist/pipeline/repo-eval-comment.js +165 -0
  380. package/dist/pipeline/repo-threshold-evaluator.d.ts +89 -0
  381. package/dist/pipeline/repo-threshold-evaluator.js +162 -0
  382. package/dist/pipeline/resolve-mappings.d.ts +35 -0
  383. package/dist/pipeline/resolve-mappings.js +72 -0
  384. package/dist/pipeline/retrieval-metrics.d.ts +39 -0
  385. package/dist/pipeline/retrieval-metrics.js +136 -0
  386. package/dist/pipeline/reverse-mapping.d.ts +67 -0
  387. package/dist/pipeline/reverse-mapping.js +88 -0
  388. package/dist/pipeline/schemas.d.ts +9 -0
  389. package/dist/pipeline/schemas.js +9 -0
  390. package/dist/pipeline/steps/calculate-scores-step.d.ts +11 -0
  391. package/dist/pipeline/steps/calculate-scores-step.js +89 -0
  392. package/dist/pipeline/steps/compare-step.d.ts +18 -0
  393. package/dist/pipeline/steps/compare-step.js +90 -0
  394. package/dist/pipeline/steps/eval-step.d.ts +53 -0
  395. package/dist/pipeline/steps/eval-step.js +347 -0
  396. package/dist/pipeline/steps/fetch-docs-step.d.ts +11 -0
  397. package/dist/pipeline/steps/fetch-docs-step.js +84 -0
  398. package/dist/pipeline/steps/generate-configs-step.d.ts +11 -0
  399. package/dist/pipeline/steps/generate-configs-step.js +98 -0
  400. package/dist/pipeline/steps/grader-consistency-step.d.ts +21 -0
  401. package/dist/pipeline/steps/grader-consistency-step.js +74 -0
  402. package/dist/pipeline/steps/publish-report-step.d.ts +57 -0
  403. package/dist/pipeline/steps/publish-report-step.js +243 -0
  404. package/dist/pipeline/steps/report-step.d.ts +13 -0
  405. package/dist/pipeline/steps/report-step.js +56 -0
  406. package/dist/pipeline/steps/update-scores-step.d.ts +11 -0
  407. package/dist/pipeline/steps/update-scores-step.js +42 -0
  408. package/dist/pipeline/targeted-loo.d.ts +88 -0
  409. package/dist/pipeline/targeted-loo.js +203 -0
  410. package/dist/pipeline/thresholds.d.ts +27 -0
  411. package/dist/pipeline/thresholds.js +245 -0
  412. package/dist/pipeline/types.d.ts +10 -0
  413. package/dist/pipeline/types.js +10 -0
  414. package/dist/pipeline/validate.d.ts +67 -0
  415. package/dist/pipeline/validate.js +406 -0
  416. package/dist/pipeline/webhook-server.d.ts +37 -0
  417. package/dist/pipeline/webhook-server.js +133 -0
  418. package/dist/report-store.d.ts +84 -0
  419. package/dist/report-store.js +208 -0
  420. package/dist/sanity/client.d.ts +38 -0
  421. package/dist/sanity/client.js +86 -0
  422. package/dist/sanity/portable-text.d.ts +11 -0
  423. package/dist/sanity/portable-text.js +211 -0
  424. package/dist/sanity/queries.d.ts +133 -0
  425. package/dist/sanity/queries.js +300 -0
  426. package/dist/schedules/digest.d.ts +116 -0
  427. package/dist/schedules/digest.js +156 -0
  428. package/dist/schedules/index.d.ts +12 -0
  429. package/dist/schedules/index.js +10 -0
  430. package/dist/schedules/loader.d.ts +31 -0
  431. package/dist/schedules/loader.js +73 -0
  432. package/dist/schedules/schema.d.ts +9 -0
  433. package/dist/schedules/schema.js +9 -0
  434. package/dist/scripts/agent-behavior-report.d.ts +19 -0
  435. package/dist/scripts/agent-behavior-report.js +315 -0
  436. package/dist/scripts/baseline.d.ts +43 -0
  437. package/dist/scripts/baseline.js +267 -0
  438. package/dist/scripts/calculate-scores.d.ts +166 -0
  439. package/dist/scripts/calculate-scores.js +1296 -0
  440. package/dist/scripts/compare.d.ts +22 -0
  441. package/dist/scripts/compare.js +334 -0
  442. package/dist/scripts/coverage-audit.d.ts +44 -0
  443. package/dist/scripts/coverage-audit.js +209 -0
  444. package/dist/scripts/debug-eval.d.ts +19 -0
  445. package/dist/scripts/debug-eval.js +73 -0
  446. package/dist/scripts/discovery-report.d.ts +58 -0
  447. package/dist/scripts/discovery-report.js +250 -0
  448. package/dist/scripts/fetch-docs.d.ts +35 -0
  449. package/dist/scripts/fetch-docs.js +472 -0
  450. package/dist/scripts/generate-configs.d.ts +66 -0
  451. package/dist/scripts/generate-configs.js +459 -0
  452. package/dist/scripts/grader-api.d.ts +27 -0
  453. package/dist/scripts/grader-api.js +206 -0
  454. package/dist/scripts/grader-compare.d.ts +22 -0
  455. package/dist/scripts/grader-compare.js +368 -0
  456. package/dist/scripts/grader-consistency.d.ts +20 -0
  457. package/dist/scripts/grader-consistency.js +313 -0
  458. package/dist/scripts/grader-sensitivity.d.ts +22 -0
  459. package/dist/scripts/grader-sensitivity.js +354 -0
  460. package/dist/scripts/grader-validate.d.ts +19 -0
  461. package/dist/scripts/grader-validate.js +267 -0
  462. package/dist/scripts/measure-retrieval.d.ts +10 -0
  463. package/dist/scripts/measure-retrieval.js +145 -0
  464. package/dist/scripts/migrate-tasks-to-content-lake.d.ts +24 -0
  465. package/dist/scripts/migrate-tasks-to-content-lake.js +327 -0
  466. package/dist/scripts/pipeline.d.ts +76 -0
  467. package/dist/scripts/pipeline.js +1031 -0
  468. package/dist/scripts/pr-comment.d.ts +10 -0
  469. package/dist/scripts/pr-comment.js +510 -0
  470. package/dist/scripts/readiness-report.d.ts +88 -0
  471. package/dist/scripts/readiness-report.js +342 -0
  472. package/dist/scripts/update-quality-scores.d.ts +15 -0
  473. package/dist/scripts/update-quality-scores.js +184 -0
  474. package/dist/scripts/validate-task-sources.d.ts +21 -0
  475. package/dist/scripts/validate-task-sources.js +210 -0
  476. package/dist/scripts/validate.d.ts +13 -0
  477. package/dist/scripts/validate.js +79 -0
  478. package/dist/scripts/webhook-server.d.ts +26 -0
  479. package/dist/scripts/webhook-server.js +147 -0
  480. package/dist/scripts/weekly-digest.d.ts +24 -0
  481. package/dist/scripts/weekly-digest.js +144 -0
  482. package/dist/sinks/bigquery/index.d.ts +131 -0
  483. package/dist/sinks/bigquery/index.js +222 -0
  484. package/dist/sinks/format-slack.d.ts +64 -0
  485. package/dist/sinks/format-slack.js +306 -0
  486. package/dist/sinks/index.d.ts +23 -0
  487. package/dist/sinks/index.js +18 -0
  488. package/dist/sinks/loader.d.ts +18 -0
  489. package/dist/sinks/loader.js +82 -0
  490. package/dist/sinks/retry.d.ts +24 -0
  491. package/dist/sinks/retry.js +52 -0
  492. package/dist/sinks/schema.d.ts +9 -0
  493. package/dist/sinks/schema.js +9 -0
  494. package/dist/sinks/slack/format.d.ts +65 -0
  495. package/dist/sinks/slack/format.js +327 -0
  496. package/dist/sinks/slack/index.d.ts +27 -0
  497. package/dist/sinks/slack/index.js +78 -0
  498. package/dist/sinks/slack-sink.d.ts +27 -0
  499. package/dist/sinks/slack-sink.js +78 -0
  500. package/dist/sinks/types.d.ts +59 -0
  501. package/dist/sinks/types.js +44 -0
  502. package/dist/sinks/webhook/index.d.ts +19 -0
  503. package/dist/sinks/webhook/index.js +50 -0
  504. package/dist/sinks/webhook-sink.d.ts +19 -0
  505. package/dist/sinks/webhook-sink.js +50 -0
  506. package/dist/sources.d.ts +104 -0
  507. package/dist/sources.js +292 -0
  508. package/dist/webhook/budget.d.ts +42 -0
  509. package/dist/webhook/budget.js +60 -0
  510. package/dist/webhook/debounce.d.ts +67 -0
  511. package/dist/webhook/debounce.js +76 -0
  512. package/dist/webhook/dispatch.d.ts +45 -0
  513. package/dist/webhook/dispatch.js +84 -0
  514. package/dist/webhook/eval-request-handler.d.ts +87 -0
  515. package/dist/webhook/eval-request-handler.js +181 -0
  516. package/dist/webhook/handler.d.ts +88 -0
  517. package/dist/webhook/handler.js +203 -0
  518. package/dist/webhook/index.d.ts +17 -0
  519. package/dist/webhook/index.js +12 -0
  520. package/dist/webhook/types.d.ts +109 -0
  521. package/dist/webhook/types.js +10 -0
  522. package/package.json +72 -0
  523. package/tasks/.expanded.agentic.yaml +51 -0
  524. package/tasks/.expanded.yaml +66 -0
  525. package/tasks/frameworks.yaml +98 -0
  526. package/tasks/functions.yaml +51 -0
  527. package/tasks/groq.yaml +216 -0
  528. package/tasks/nextjs-live.yaml +62 -0
  529. package/tasks/studio-setup.yaml +111 -0
  530. package/tasks/visual-editing.yaml +120 -0
@@ -0,0 +1,24 @@
1
+ /**
2
+ * lib/webhook-server.ts — DEPRECATED re-export shim.
3
+ *
4
+ * The real implementation has moved to pipeline/webhook-server.ts.
5
+ * This shim preserves backward compatibility for direct CLI invocation.
6
+ *
7
+ * TODO: Update all importers to use pipeline/webhook-server.ts, then delete this file.
8
+ *
9
+ * @deprecated Import from ../pipeline/webhook-server.js instead.
10
+ */
11
+ import { dirname, resolve } from "path";
12
+ import { fileURLToPath } from "url";
13
+ import { startWebhookServer } from "../pipeline/webhook-server.js";
14
+ export { startWebhookServer, } from "../pipeline/webhook-server.js";
15
+ const __dirname = dirname(fileURLToPath(import.meta.url));
16
+ const ROOT = resolve(__dirname, "..", "..");
17
+ // When imported directly (e.g., `tsx src/lib/webhook-server.ts`), start the server
18
+ startWebhookServer({
19
+ dailyBudget: parseInt(process.env.WEBHOOK_DAILY_BUDGET ?? "20", 10),
20
+ debounceMs: parseInt(process.env.WEBHOOK_DEBOUNCE_MS ?? "10000", 10),
21
+ githubToken: process.env.GITHUB_TOKEN ?? "",
22
+ port: parseInt(process.env.WEBHOOK_PORT ?? "3333", 10),
23
+ rootDir: ROOT,
24
+ });
@@ -0,0 +1,24 @@
1
+ /**
2
+ * weekly-digest.ts
3
+ *
4
+ * CLI script to generate and deliver a weekly evaluation digest.
5
+ *
6
+ * Queries the Sanity Content Lake for all reports within the configured
7
+ * lookback window, computes trend analysis, and delivers the digest
8
+ * via configured channels (Slack, stdout, or both).
9
+ *
10
+ * Usage:
11
+ * pnpm weekly-digest # send to configured Slack webhook
12
+ * pnpm weekly-digest --dry-run # print to stdout only
13
+ * pnpm weekly-digest --lookback 14 # 14-day lookback window
14
+ * pnpm weekly-digest --json # output raw JSON
15
+ *
16
+ * Environment variables:
17
+ * SLACK_WEBHOOK_URL — Slack incoming webhook URL
18
+ * SANITY_API_TOKEN — Sanity read token
19
+ * AILF_TRIGGER_TYPE — set to "scheduled" by the cron workflow
20
+ * AILF_SCHEDULE — the schedule name (e.g., "weekly-digest")
21
+ *
22
+ * @see docs/design-docs/report-store/implementation.md — Phase 5
23
+ */
24
+ export declare function main(): Promise<void>;
@@ -0,0 +1,148 @@
1
+ /**
2
+ * weekly-digest.ts
3
+ *
4
+ * CLI script to generate and deliver a weekly evaluation digest.
5
+ *
6
+ * Queries the Sanity Content Lake for all reports within the configured
7
+ * lookback window, computes trend analysis, and delivers the digest
8
+ * via configured channels (Slack, stdout, or both).
9
+ *
10
+ * Usage:
11
+ * pnpm weekly-digest # send to configured Slack webhook
12
+ * pnpm weekly-digest --dry-run # print to stdout only
13
+ * pnpm weekly-digest --lookback 14 # 14-day lookback window
14
+ * pnpm weekly-digest --json # output raw JSON
15
+ *
16
+ * Environment variables:
17
+ * SLACK_WEBHOOK_URL — Slack incoming webhook URL
18
+ * SANITY_API_TOKEN — Sanity read token
19
+ * AILF_TRIGGER_TYPE — set to "scheduled" by the cron workflow
20
+ * AILF_SCHEDULE — the schedule name (e.g., "weekly-digest")
21
+ *
22
+ * @see docs/design-docs/report-store/implementation.md — Phase 5
23
+ */
24
+ import { config as dotenvConfig } from "dotenv";
25
+ import { existsSync } from "fs";
26
+ import { dirname, resolve } from "path";
27
+ import { fileURLToPath } from "url";
28
+ import { generateDigest } from "../schedules/digest.js";
29
+ import { getDigestConfig } from "../schedules/loader.js";
30
+ import { formatWeeklyDigest } from "../sinks/slack/format.js";
31
+ // Load root .env (same override behavior as pipeline.ts)
32
+ const __dirname = dirname(fileURLToPath(import.meta.url));
33
+ const envPath = resolve(__dirname, "..", "..", "..", "..", ".env");
34
+ if (existsSync(envPath)) {
35
+ dotenvConfig({ override: true, path: envPath });
36
+ }
37
+ // ---------------------------------------------------------------------------
38
+ // CLI argument parsing
39
+ // ---------------------------------------------------------------------------
40
+ const args = process.argv.slice(2);
41
+ function getOption(name) {
42
+ const idx = args.indexOf(`--${name}`);
43
+ return idx >= 0 && idx + 1 < args.length ? args[idx + 1] : undefined;
44
+ }
45
+ function hasFlag(name) {
46
+ return args.includes(`--${name}`);
47
+ }
48
+ const DRY_RUN = hasFlag("dry-run");
49
+ const JSON_OUTPUT = hasFlag("json");
50
+ const lookbackOverride = getOption("lookback");
51
+ // ---------------------------------------------------------------------------
52
+ // Main
53
+ // ---------------------------------------------------------------------------
54
+ export async function main() {
55
+ console.log();
56
+ console.log("=== AI Literacy Weekly Digest ===");
57
+ console.log();
58
+ // Load digest config
59
+ const digestConfig = getDigestConfig();
60
+ const lookbackDays = lookbackOverride
61
+ ? parseInt(lookbackOverride, 10)
62
+ : (digestConfig?.lookbackDays ?? 7);
63
+ console.log(` Lookback: ${lookbackDays} days`);
64
+ console.log(` Mode: ${DRY_RUN ? "dry run (stdout only)" : "live"}`);
65
+ console.log();
66
+ // Generate digest — uses AILF_REPORT_* env vars for report store access,
67
+ // independent of SANITY_DATASET/SANITY_PROJECT_ID (which control doc evaluation)
68
+ const digest = await generateDigest({
69
+ dataset: process.env.AILF_REPORT_DATASET,
70
+ lookbackDays,
71
+ projectId: process.env.AILF_REPORT_PROJECT_ID,
72
+ token: process.env.AILF_REPORT_SANITY_API_TOKEN ?? process.env.SANITY_API_TOKEN,
73
+ });
74
+ if (!digest) {
75
+ console.log(" No reports found in the lookback window. Nothing to send.");
76
+ process.exit(0);
77
+ }
78
+ // Output
79
+ console.log(` Reports found: ${digest.reportCount}`);
80
+ console.log(` Overall: ${Math.round(digest.overallLatest)} (${digest.overallTrend})`);
81
+ console.log(` Improved: ${digest.improved.join(", ") || "none"}`);
82
+ console.log(` Regressed: ${digest.regressed.join(", ") || "none"}`);
83
+ console.log(` Stable: ${digest.stable.join(", ") || "none"}`);
84
+ console.log();
85
+ if (JSON_OUTPUT) {
86
+ console.log(JSON.stringify(digest, null, 2));
87
+ return;
88
+ }
89
+ // Format for Slack
90
+ const message = formatWeeklyDigest(digest);
91
+ if (DRY_RUN) {
92
+ console.log(" --- Slack Message Preview ---");
93
+ console.log(` Text: ${message.text}`);
94
+ console.log();
95
+ for (const block of message.blocks) {
96
+ if (block.text) {
97
+ console.log(` [${block.type}] ${block.text.text}`);
98
+ }
99
+ if (block.fields) {
100
+ for (const field of block.fields) {
101
+ console.log(` [field] ${field.text}`);
102
+ }
103
+ }
104
+ if (block.elements) {
105
+ for (const el of block.elements) {
106
+ console.log(` [element] ${el.text}`);
107
+ }
108
+ }
109
+ }
110
+ console.log();
111
+ return;
112
+ }
113
+ // Deliver via Slack
114
+ const webhookUrl = digestConfig?.slackWebhookUrl ?? process.env.SLACK_WEBHOOK_URL;
115
+ if (!webhookUrl) {
116
+ console.warn(" ⚠️ No Slack webhook URL configured. Set SLACK_WEBHOOK_URL or configure in schedules.yaml");
117
+ console.log(" Printing digest to stdout instead:");
118
+ console.log();
119
+ console.log(` ${message.text}`);
120
+ return;
121
+ }
122
+ console.log(" Sending to Slack...");
123
+ try {
124
+ const response = await fetch(webhookUrl, {
125
+ body: JSON.stringify(message),
126
+ headers: { "Content-Type": "application/json" },
127
+ method: "POST",
128
+ });
129
+ if (response.ok) {
130
+ console.log(" ✅ Digest sent successfully");
131
+ }
132
+ else {
133
+ const text = await response.text();
134
+ console.warn(` ⚠️ Slack delivery failed: ${response.status} ${text}`);
135
+ }
136
+ }
137
+ catch (error) {
138
+ console.warn(` ⚠️ Slack delivery error: ${error instanceof Error ? error.message : String(error)}`);
139
+ }
140
+ }
141
+ // Only run when invoked directly (not when imported)
142
+ if (process.argv[1]?.endsWith("weekly-digest.ts") ||
143
+ process.argv[1]?.endsWith("weekly-digest.js")) {
144
+ main().catch((error) => {
145
+ console.error("Fatal error:", error);
146
+ process.exit(1);
147
+ });
148
+ }
@@ -0,0 +1,27 @@
1
+ /**
2
+ * Bridge from legacy ResolvedOptions to the new AppContext.
3
+ *
4
+ * This mapping function exists during the Phase 3→7 migration. It converts
5
+ * the old ResolvedOptions shape to ResolvedConfig, then delegates to
6
+ * createAppContext() (the composition root).
7
+ *
8
+ * Once all commands construct ResolvedConfig directly (or use --config),
9
+ * this bridge can be deleted.
10
+ */
11
+ import type { AppContext, ResolvedConfig } from "../_vendor/ailf-core/index.d.ts";
12
+ import type { ResolvedOptions } from "../commands/pipeline-action.js";
13
+ /**
14
+ * Map legacy ResolvedOptions to ResolvedConfig.
15
+ *
16
+ * This is a mechanical mapping — every field in ResolvedConfig has a
17
+ * corresponding field in ResolvedOptions. Fields that don't map directly
18
+ * are derived (e.g., areas from areaOption).
19
+ */
20
+ export declare function mapToResolvedConfig(opts: ResolvedOptions, rootDir: string): ResolvedConfig;
21
+ /**
22
+ * Build an AppContext from legacy ResolvedOptions.
23
+ *
24
+ * Delegates to createAppContext() (composition root) after mapping
25
+ * ResolvedOptions → ResolvedConfig.
26
+ */
27
+ export declare function buildAppContext(opts: ResolvedOptions, rootDir: string): AppContext;
@@ -0,0 +1,81 @@
1
+ /**
2
+ * Bridge from legacy ResolvedOptions to the new AppContext.
3
+ *
4
+ * This mapping function exists during the Phase 3→7 migration. It converts
5
+ * the old ResolvedOptions shape to ResolvedConfig, then delegates to
6
+ * createAppContext() (the composition root).
7
+ *
8
+ * Once all commands construct ResolvedConfig directly (or use --config),
9
+ * this bridge can be deleted.
10
+ */
11
+ import { createAppContext } from "../composition-root.js";
12
+ /**
13
+ * Map legacy ResolvedOptions to ResolvedConfig.
14
+ *
15
+ * This is a mechanical mapping — every field in ResolvedConfig has a
16
+ * corresponding field in ResolvedOptions. Fields that don't map directly
17
+ * are derived (e.g., areas from areaOption).
18
+ */
19
+ export function mapToResolvedConfig(opts, rootDir) {
20
+ return {
21
+ rootDir,
22
+ mode: opts.mode,
23
+ debug: opts.debug,
24
+ areas: opts.areaOption
25
+ ?.split(",")
26
+ .map((s) => s.trim())
27
+ .filter(Boolean),
28
+ tasks: opts.taskOption
29
+ ?.split(",")
30
+ .map((s) => s.trim())
31
+ .filter(Boolean),
32
+ changedDocs: opts.changedDocsOption
33
+ ?.split(",")
34
+ .map((s) => s.trim())
35
+ .filter(Boolean),
36
+ source: opts.source,
37
+ skipFetch: opts.skipFetch,
38
+ skipEval: opts.skipEval,
39
+ compareEnabled: opts.compareEnabled,
40
+ compareThreshold: opts.compareThreshold,
41
+ compareBaseline: opts.compareBaseline,
42
+ gapAnalysisEnabled: opts.gapAnalysisEnabled,
43
+ readinessEnabled: opts.readinessEnabled,
44
+ discoveryReportEnabled: opts.discoveryReportEnabled,
45
+ publishEnabled: opts.publishEnabled,
46
+ publishTag: opts.publishTag,
47
+ noCache: opts.noCache,
48
+ noRemoteCache: opts.noRemoteCache,
49
+ graderReplications: opts.graderReplications,
50
+ outputPath: opts.outputPath,
51
+ urls: opts.urlArgs.length > 0 ? opts.urlArgs : undefined,
52
+ headers: opts.headerArgs.length > 0
53
+ ? Object.fromEntries(opts.headerArgs.map((h) => {
54
+ const [key, ...rest] = h.split(":");
55
+ return [key.trim(), rest.join(":").trim()];
56
+ }))
57
+ : undefined,
58
+ allowedOrigins: opts.allowedOriginArgs.length > 0 ? opts.allowedOriginArgs : undefined,
59
+ searchMode: opts.searchMode ?? "open",
60
+ concurrency: opts.concurrency,
61
+ promptfooUrl: opts.promptfooUrl,
62
+ datasetOverride: opts.datasetOverride,
63
+ projectIdOverride: opts.projectIdOverride,
64
+ perspectiveOverride: opts.perspectiveOverride,
65
+ studioOriginOverride: opts.studioOriginOverride,
66
+ sanityDocumentArgs: opts.sanityDocumentArgs.length > 0 ? opts.sanityDocumentArgs : undefined,
67
+ beforeOption: opts.beforeOption,
68
+ taskSourceType: opts.taskSourceType,
69
+ repoTasksPath: opts.repoTasksPath,
70
+ };
71
+ }
72
+ /**
73
+ * Build an AppContext from legacy ResolvedOptions.
74
+ *
75
+ * Delegates to createAppContext() (composition root) after mapping
76
+ * ResolvedOptions → ResolvedConfig.
77
+ */
78
+ export function buildAppContext(opts, rootDir) {
79
+ const config = mapToResolvedConfig(opts, rootDir);
80
+ return createAppContext(config);
81
+ }
@@ -0,0 +1,15 @@
1
+ /**
2
+ * Build the step sequence based on resolved config.
3
+ *
4
+ * This is the declarative description of the pipeline — a list of
5
+ * PipelineStep objects determined by config flags like skipFetch,
6
+ * skipEval, compareEnabled, etc.
7
+ */
8
+ import type { AppContext, PipelineStep } from "../_vendor/ailf-core/index.d.ts";
9
+ /**
10
+ * Build the pipeline step sequence based on config.
11
+ *
12
+ * @param ctx - The application context (used for config)
13
+ * @param pipelineStart - Pipeline start timestamp (for publish step)
14
+ */
15
+ export declare function buildStepSequence(ctx: AppContext, pipelineStart?: number): PipelineStep[];
@@ -0,0 +1,84 @@
1
+ /**
2
+ * Build the step sequence based on resolved config.
3
+ *
4
+ * This is the declarative description of the pipeline — a list of
5
+ * PipelineStep objects determined by config flags like skipFetch,
6
+ * skipEval, compareEnabled, etc.
7
+ */
8
+ import { FULL_MODE_SUBMODES } from "../_vendor/ailf-shared/index.js";
9
+ import { CallbackStep } from "./steps/callback-step.js";
10
+ import { CalculateScoresStep } from "./steps/calculate-scores-step.js";
11
+ import { CompareStep } from "./steps/compare-step.js";
12
+ import { DiscoveryReportStep } from "./steps/discovery-report-step.js";
13
+ import { FetchDocsStep } from "./steps/fetch-docs-step.js";
14
+ import { GapAnalysisStep } from "./steps/gap-analysis-step.js";
15
+ import { GenerateConfigsStep } from "./steps/generate-configs-step.js";
16
+ import { GraderConsistencyStep } from "./steps/grader-consistency-step.js";
17
+ import { PublishReportStep } from "./steps/publish-report-step.js";
18
+ import { ReadinessStep } from "./steps/readiness-step.js";
19
+ import { ReportStep } from "./steps/report-step.js";
20
+ import { RunEvalStep } from "./steps/run-eval-step.js";
21
+ import { MirrorRepoTasksStep } from "./steps/mirror-repo-tasks-step.js";
22
+ import { ValidateStep } from "./steps/validate-step.js";
23
+ /**
24
+ * Build the pipeline step sequence based on config.
25
+ *
26
+ * @param ctx - The application context (used for config)
27
+ * @param pipelineStart - Pipeline start timestamp (for publish step)
28
+ */
29
+ export function buildStepSequence(ctx, pipelineStart = Date.now()) {
30
+ const config = ctx.config;
31
+ const steps = [];
32
+ // Step 0: Validate configuration
33
+ steps.push(new ValidateStep());
34
+ // Step 0b: Mirror repo tasks to Content Lake (optional, when --repo-tasks-path)
35
+ if (config.repoTasksPath) {
36
+ steps.push(new MirrorRepoTasksStep());
37
+ }
38
+ // Step 1: Fetch documentation (step handles --skip-fetch internally)
39
+ steps.push(new FetchDocsStep());
40
+ // Step 2: Generate Promptfoo configs
41
+ steps.push(new GenerateConfigsStep());
42
+ // Step 3: Run evaluation (steps handle --skip-eval internally)
43
+ const modes = config.mode === "full"
44
+ ? [...FULL_MODE_SUBMODES]
45
+ : [config.mode];
46
+ for (const mode of modes) {
47
+ steps.push(new RunEvalStep(mode));
48
+ }
49
+ // Step 3c: Grader consistency (optional, conditional)
50
+ if (config.graderReplications) {
51
+ steps.push(new GraderConsistencyStep());
52
+ }
53
+ // Step 4: Calculate scores
54
+ steps.push(new CalculateScoresStep());
55
+ // Step 4b: Gap analysis (on by default, opt-out via --no-gap-analysis)
56
+ if (config.gapAnalysisEnabled) {
57
+ steps.push(new GapAnalysisStep());
58
+ }
59
+ // Step 4b: Publish report (optional, when token is configured)
60
+ if (config.publishEnabled) {
61
+ steps.push(new PublishReportStep(pipelineStart, {
62
+ publishTag: config.publishTag,
63
+ }));
64
+ }
65
+ // Step 5: Generate report
66
+ steps.push(new ReportStep());
67
+ // Step 5b: Compare against baseline (optional)
68
+ if (config.compareEnabled) {
69
+ steps.push(new CompareStep());
70
+ }
71
+ // Step 6b: Readiness report (optional)
72
+ if (config.readinessEnabled) {
73
+ steps.push(new ReadinessStep());
74
+ }
75
+ // Step 6c: Discovery report (optional)
76
+ if (config.discoveryReportEnabled) {
77
+ steps.push(new DiscoveryReportStep());
78
+ }
79
+ // Step 7: Callback delivery (optional, API-triggered evaluations)
80
+ if (config.callback?.url) {
81
+ steps.push(new CallbackStep(config.callback, config.jobId));
82
+ }
83
+ return steps;
84
+ }
@@ -0,0 +1,9 @@
1
+ /**
2
+ * Extract source-relevant overrides from ResolvedConfig.
3
+ *
4
+ * Used by orchestration steps and command handlers to call loadSource()
5
+ * with typed overrides instead of relying on process.env.
6
+ */
7
+ import type { ResolvedConfig } from "../_vendor/ailf-core/index.d.ts";
8
+ import type { SourceOverrides } from "../sources.js";
9
+ export declare function configToSourceOverrides(config: ResolvedConfig): SourceOverrides;
@@ -0,0 +1,28 @@
1
+ /**
2
+ * Extract source-relevant overrides from ResolvedConfig.
3
+ *
4
+ * Used by orchestration steps and command handlers to call loadSource()
5
+ * with typed overrides instead of relying on process.env.
6
+ */
7
+ export function configToSourceOverrides(config) {
8
+ return {
9
+ ...(config.allowedOrigins?.length
10
+ ? { allowedOrigins: config.allowedOrigins }
11
+ : {}),
12
+ ...(config.urls?.[0] ? { baseUrl: config.urls[0] } : {}),
13
+ ...(config.datasetOverride ? { dataset: config.datasetOverride } : {}),
14
+ ...(config.sanityDocumentArgs?.length
15
+ ? { documentIds: config.sanityDocumentArgs }
16
+ : {}),
17
+ ...(config.headers ? { headers: config.headers } : {}),
18
+ ...(config.perspectiveOverride
19
+ ? { perspective: config.perspectiveOverride }
20
+ : {}),
21
+ ...(config.projectIdOverride
22
+ ? { projectId: config.projectIdOverride }
23
+ : {}),
24
+ ...(config.studioOriginOverride
25
+ ? { studioOrigin: config.studioOriginOverride }
26
+ : {}),
27
+ };
28
+ }
@@ -0,0 +1,21 @@
1
+ /**
2
+ * Environment variable bridge — writes ResolvedConfig values to process.env
3
+ * so that lib/*.ts modules (which still read process.env) work correctly.
4
+ *
5
+ * This replaces the former global applyEnvironment() with an explicit
6
+ * per-step bridge. Each orchestration step calls this before invoking
7
+ * its lib/*.ts main() function.
8
+ *
9
+ * Phase 9 will eliminate this file entirely by giving lib/*.ts main()
10
+ * functions typed option parameters.
11
+ *
12
+ * @see docs/exec-plans/active/ports-and-adapters/phase-8-delete-legacy-step-layer.md
13
+ */
14
+ import type { ResolvedConfig } from "../_vendor/ailf-core/index.d.ts";
15
+ /**
16
+ * Bridge ResolvedConfig values to process.env.
17
+ *
18
+ * Idempotent — safe to call multiple times. Only sets env vars for
19
+ * config values that are defined (never deletes or resets).
20
+ */
21
+ export declare function bridgeConfigToEnv(config: ResolvedConfig): void;
@@ -0,0 +1,66 @@
1
+ /**
2
+ * Environment variable bridge — writes ResolvedConfig values to process.env
3
+ * so that lib/*.ts modules (which still read process.env) work correctly.
4
+ *
5
+ * This replaces the former global applyEnvironment() with an explicit
6
+ * per-step bridge. Each orchestration step calls this before invoking
7
+ * its lib/*.ts main() function.
8
+ *
9
+ * Phase 9 will eliminate this file entirely by giving lib/*.ts main()
10
+ * functions typed option parameters.
11
+ *
12
+ * @see docs/exec-plans/active/ports-and-adapters/phase-8-delete-legacy-step-layer.md
13
+ */
14
+ /**
15
+ * Bridge ResolvedConfig values to process.env.
16
+ *
17
+ * Idempotent — safe to call multiple times. Only sets env vars for
18
+ * config values that are defined (never deletes or resets).
19
+ */
20
+ export function bridgeConfigToEnv(config) {
21
+ // Mode
22
+ process.env.EVAL_MODE = config.mode;
23
+ // Search mode
24
+ if (config.searchMode !== "open") {
25
+ process.env.EVAL_SEARCH_MODE = config.searchMode;
26
+ }
27
+ // Source
28
+ if (config.source) {
29
+ process.env.DOC_SOURCE = config.source;
30
+ }
31
+ // URL-derived overrides
32
+ if (config.urls?.[0]) {
33
+ process.env.DOC_BASE_URL = config.urls[0];
34
+ }
35
+ // Sanity overrides
36
+ if (config.datasetOverride) {
37
+ process.env.SANITY_DATASET = config.datasetOverride;
38
+ }
39
+ if (config.projectIdOverride) {
40
+ process.env.SANITY_PROJECT_ID = config.projectIdOverride;
41
+ }
42
+ if (config.perspectiveOverride) {
43
+ process.env.SANITY_PERSPECTIVE = config.perspectiveOverride;
44
+ }
45
+ if (config.studioOriginOverride) {
46
+ process.env.SANITY_STUDIO_ORIGIN = config.studioOriginOverride;
47
+ }
48
+ if (config.sanityDocumentArgs?.length) {
49
+ process.env.SANITY_DOCUMENT_IDS = config.sanityDocumentArgs.join(",");
50
+ }
51
+ // Custom headers
52
+ if (config.headers) {
53
+ process.env.DOC_HEADERS = JSON.stringify(config.headers);
54
+ }
55
+ // Allowed origins
56
+ if (config.allowedOrigins?.length) {
57
+ process.env.DOC_ALLOWED_ORIGINS = config.allowedOrigins.join(",");
58
+ }
59
+ // Scoping filters
60
+ if (config.areas) {
61
+ process.env.EVAL_FILTER_AREAS = config.areas.join(",");
62
+ }
63
+ if (config.tasks) {
64
+ process.env.EVAL_FILTER_TASKS = config.tasks.join(",");
65
+ }
66
+ }
@@ -0,0 +1,11 @@
1
+ /**
2
+ * Orchestration barrel export.
3
+ *
4
+ * The new pipeline execution model: composable steps, uniform middleware,
5
+ * and declarative sequencing.
6
+ */
7
+ export { buildAppContext, mapToResolvedConfig } from "./build-app-context.js";
8
+ export { buildStepSequence } from "./build-step-sequence.js";
9
+ export { orchestratePipeline } from "./pipeline-orchestrator.js";
10
+ export { runStep } from "./step-runner.js";
11
+ export * from "./steps/index.js";
@@ -0,0 +1,11 @@
1
+ /**
2
+ * Orchestration barrel export.
3
+ *
4
+ * The new pipeline execution model: composable steps, uniform middleware,
5
+ * and declarative sequencing.
6
+ */
7
+ export { buildAppContext, mapToResolvedConfig } from "./build-app-context.js";
8
+ export { buildStepSequence } from "./build-step-sequence.js";
9
+ export { orchestratePipeline } from "./pipeline-orchestrator.js";
10
+ export { runStep } from "./step-runner.js";
11
+ export * from "./steps/index.js";
@@ -0,0 +1,24 @@
1
+ /**
2
+ * Pipeline orchestrator — sequences steps and short-circuits on failure.
3
+ *
4
+ * This replaces the 1,672-line executePipeline() in pipeline-action.ts
5
+ * with declarative step sequencing. Each step is run through the
6
+ * StepRunner which provides uniform timing, precondition checking,
7
+ * error handling, and logging.
8
+ *
9
+ * When a jobId is present in the config (API-triggered evaluations),
10
+ * the orchestrator emits progress updates to the Content Lake after
11
+ * each step completes. This enables the GET /v1/jobs/:jobId polling
12
+ * endpoint to show real-time progress.
13
+ */
14
+ import type { AppContext, PipelineResult, PipelineStep } from "../_vendor/ailf-core/index.d.ts";
15
+ /**
16
+ * Run a sequence of pipeline steps, short-circuiting on required step failure.
17
+ *
18
+ * Optional steps (step.optional === true) may fail without stopping
19
+ * the pipeline. Required step failures cause an immediate abort.
20
+ *
21
+ * When ctx.config.jobId is set, emits progress updates to the Content
22
+ * Lake after each step completes.
23
+ */
24
+ export declare function orchestratePipeline(ctx: AppContext, steps: PipelineStep[]): Promise<PipelineResult>;