@sanity/ailf 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (530) hide show
  1. package/README.md +89 -0
  2. package/bin/ailf.js +64 -0
  3. package/canonical/grader-references/README.md +88 -0
  4. package/canonical/grader-references/groq.yaml +234 -0
  5. package/canonical/grader-references/studio-setup.yaml +275 -0
  6. package/canonical/reference-solutions/.gitkeep +1 -0
  7. package/canonical/reference-solutions/frameworks/nuxt.ts +119 -0
  8. package/canonical/reference-solutions/frameworks/remix.tsx +100 -0
  9. package/canonical/reference-solutions/functions/publish-webhook.ts +60 -0
  10. package/canonical/reference-solutions/groq/advanced-filtering.ts +379 -0
  11. package/canonical/reference-solutions/groq/blog-queries.ts +137 -0
  12. package/canonical/reference-solutions/groq/joins-references.ts +300 -0
  13. package/canonical/reference-solutions/nextjs/app-router-integration.tsx +128 -0
  14. package/canonical/reference-solutions/studio-setup/blog-schema.ts +143 -0
  15. package/canonical/reference-solutions/studio-setup/custom-tool.tsx +78 -0
  16. package/canonical/reference-solutions/visual-editing/live-preview.tsx +137 -0
  17. package/canonical/reference-solutions/visual-editing/presentation-nextjs.tsx +130 -0
  18. package/config/airbyte/ai_literacy_framework.connector.yaml +639 -0
  19. package/config/bigquery/README.md +74 -0
  20. package/config/bigquery/views/area_scores.sql +87 -0
  21. package/config/bigquery/views/reports.sql +49 -0
  22. package/config/features.yaml +116 -0
  23. package/config/models.yaml +115 -0
  24. package/config/prompts.yaml +75 -0
  25. package/config/rubrics.yaml +62 -0
  26. package/config/schedules.yaml +43 -0
  27. package/config/sinks.yaml +54 -0
  28. package/config/sources.yaml +51 -0
  29. package/config/thresholds.yaml +49 -0
  30. package/dist/_vendor/ailf-core/examples/index.d.ts +190 -0
  31. package/dist/_vendor/ailf-core/examples/index.js +285 -0
  32. package/dist/_vendor/ailf-core/index.d.ts +17 -0
  33. package/dist/_vendor/ailf-core/index.js +17 -0
  34. package/dist/_vendor/ailf-core/ports/cache-store.d.ts +72 -0
  35. package/dist/_vendor/ailf-core/ports/cache-store.js +17 -0
  36. package/dist/_vendor/ailf-core/ports/config-source.d.ts +33 -0
  37. package/dist/_vendor/ailf-core/ports/config-source.js +15 -0
  38. package/dist/_vendor/ailf-core/ports/context.d.ts +172 -0
  39. package/dist/_vendor/ailf-core/ports/context.js +14 -0
  40. package/dist/_vendor/ailf-core/ports/doc-fetcher.d.ts +131 -0
  41. package/dist/_vendor/ailf-core/ports/doc-fetcher.js +12 -0
  42. package/dist/_vendor/ailf-core/ports/eval-runner.d.ts +24 -0
  43. package/dist/_vendor/ailf-core/ports/eval-runner.js +8 -0
  44. package/dist/_vendor/ailf-core/ports/index.d.ts +15 -0
  45. package/dist/_vendor/ailf-core/ports/index.js +7 -0
  46. package/dist/_vendor/ailf-core/ports/logger.d.ts +36 -0
  47. package/dist/_vendor/ailf-core/ports/logger.js +11 -0
  48. package/dist/_vendor/ailf-core/ports/pipeline-step.d.ts +46 -0
  49. package/dist/_vendor/ailf-core/ports/pipeline-step.js +8 -0
  50. package/dist/_vendor/ailf-core/ports/task-source.d.ts +159 -0
  51. package/dist/_vendor/ailf-core/ports/task-source.js +72 -0
  52. package/dist/_vendor/ailf-core/schemas/callback-payload.d.ts +24 -0
  53. package/dist/_vendor/ailf-core/schemas/callback-payload.js +29 -0
  54. package/dist/_vendor/ailf-core/schemas/eval-config.d.ts +55 -0
  55. package/dist/_vendor/ailf-core/schemas/eval-config.js +78 -0
  56. package/dist/_vendor/ailf-core/schemas/index.d.ts +16 -0
  57. package/dist/_vendor/ailf-core/schemas/index.js +16 -0
  58. package/dist/_vendor/ailf-core/schemas/pipeline-request.d.ts +125 -0
  59. package/dist/_vendor/ailf-core/schemas/pipeline-request.js +67 -0
  60. package/dist/_vendor/ailf-core/schemas/pipeline.d.ts +531 -0
  61. package/dist/_vendor/ailf-core/schemas/pipeline.js +318 -0
  62. package/dist/_vendor/ailf-core/schemas/schedules.d.ts +68 -0
  63. package/dist/_vendor/ailf-core/schemas/schedules.js +74 -0
  64. package/dist/_vendor/ailf-core/schemas/sinks.d.ts +207 -0
  65. package/dist/_vendor/ailf-core/schemas/sinks.js +108 -0
  66. package/dist/_vendor/ailf-core/services/comparison-formatters.d.ts +18 -0
  67. package/dist/_vendor/ailf-core/services/comparison-formatters.js +189 -0
  68. package/dist/_vendor/ailf-core/services/config-helpers.d.ts +41 -0
  69. package/dist/_vendor/ailf-core/services/config-helpers.js +86 -0
  70. package/dist/_vendor/ailf-core/services/index.d.ts +12 -0
  71. package/dist/_vendor/ailf-core/services/index.js +12 -0
  72. package/dist/_vendor/ailf-core/services/scoring.d.ts +49 -0
  73. package/dist/_vendor/ailf-core/services/scoring.js +222 -0
  74. package/dist/_vendor/ailf-core/types/index.d.ts +1082 -0
  75. package/dist/_vendor/ailf-core/types/index.js +21 -0
  76. package/dist/_vendor/ailf-core/types/scoring-input.d.ts +54 -0
  77. package/dist/_vendor/ailf-core/types/scoring-input.js +9 -0
  78. package/dist/_vendor/ailf-shared/dimension-names.d.ts +21 -0
  79. package/dist/_vendor/ailf-shared/dimension-names.js +27 -0
  80. package/dist/_vendor/ailf-shared/document-ref.d.ts +29 -0
  81. package/dist/_vendor/ailf-shared/document-ref.js +1 -0
  82. package/dist/_vendor/ailf-shared/eval-modes.d.ts +12 -0
  83. package/dist/_vendor/ailf-shared/eval-modes.js +8 -0
  84. package/dist/_vendor/ailf-shared/index.d.ts +16 -0
  85. package/dist/_vendor/ailf-shared/index.js +16 -0
  86. package/dist/_vendor/ailf-shared/noise-threshold.d.ts +9 -0
  87. package/dist/_vendor/ailf-shared/noise-threshold.js +9 -0
  88. package/dist/_vendor/ailf-shared/score-grades.d.ts +17 -0
  89. package/dist/_vendor/ailf-shared/score-grades.js +23 -0
  90. package/dist/adapters/cache/content-lake-cache.d.ts +24 -0
  91. package/dist/adapters/cache/content-lake-cache.js +59 -0
  92. package/dist/adapters/cache/filesystem-cache.d.ts +18 -0
  93. package/dist/adapters/cache/filesystem-cache.js +54 -0
  94. package/dist/adapters/cache/index.d.ts +2 -0
  95. package/dist/adapters/cache/index.js +2 -0
  96. package/dist/adapters/config-sources/cli-config-adapter.d.ts +17 -0
  97. package/dist/adapters/config-sources/cli-config-adapter.js +23 -0
  98. package/dist/adapters/config-sources/file-config-adapter.d.ts +26 -0
  99. package/dist/adapters/config-sources/file-config-adapter.js +96 -0
  100. package/dist/adapters/config-sources/index.d.ts +2 -0
  101. package/dist/adapters/config-sources/index.js +2 -0
  102. package/dist/adapters/doc-fetchers/index.d.ts +1 -0
  103. package/dist/adapters/doc-fetchers/index.js +1 -0
  104. package/dist/adapters/doc-fetchers/sanity-doc-fetcher.d.ts +76 -0
  105. package/dist/adapters/doc-fetchers/sanity-doc-fetcher.js +620 -0
  106. package/dist/adapters/eval-runners/index.d.ts +1 -0
  107. package/dist/adapters/eval-runners/index.js +1 -0
  108. package/dist/adapters/eval-runners/promptfoo-eval-adapter.d.ts +14 -0
  109. package/dist/adapters/eval-runners/promptfoo-eval-adapter.js +63 -0
  110. package/dist/adapters/index.d.ts +12 -0
  111. package/dist/adapters/index.js +12 -0
  112. package/dist/adapters/loggers/console-logger.d.ts +22 -0
  113. package/dist/adapters/loggers/console-logger.js +54 -0
  114. package/dist/adapters/loggers/index.d.ts +9 -0
  115. package/dist/adapters/loggers/index.js +9 -0
  116. package/dist/adapters/loggers/json-logger.d.ts +18 -0
  117. package/dist/adapters/loggers/json-logger.js +33 -0
  118. package/dist/adapters/loggers/quiet-logger.d.ts +16 -0
  119. package/dist/adapters/loggers/quiet-logger.js +30 -0
  120. package/dist/adapters/task-sources/composite-task-source.d.ts +20 -0
  121. package/dist/adapters/task-sources/composite-task-source.js +59 -0
  122. package/dist/adapters/task-sources/content-lake-task-source.d.ts +20 -0
  123. package/dist/adapters/task-sources/content-lake-task-source.js +219 -0
  124. package/dist/adapters/task-sources/index.d.ts +7 -0
  125. package/dist/adapters/task-sources/index.js +7 -0
  126. package/dist/adapters/task-sources/repo-schemas.d.ts +245 -0
  127. package/dist/adapters/task-sources/repo-schemas.js +234 -0
  128. package/dist/adapters/task-sources/repo-task-source.d.ts +22 -0
  129. package/dist/adapters/task-sources/repo-task-source.js +104 -0
  130. package/dist/adapters/task-sources/repo-trigger.d.ts +52 -0
  131. package/dist/adapters/task-sources/repo-trigger.js +153 -0
  132. package/dist/adapters/task-sources/repo-validation.d.ts +49 -0
  133. package/dist/adapters/task-sources/repo-validation.js +164 -0
  134. package/dist/adapters/task-sources/yaml-task-source.d.ts +18 -0
  135. package/dist/adapters/task-sources/yaml-task-source.js +136 -0
  136. package/dist/agent-observer/agentic-provider.d.ts +132 -0
  137. package/dist/agent-observer/agentic-provider.js +983 -0
  138. package/dist/agent-observer/classifier.d.ts +62 -0
  139. package/dist/agent-observer/classifier.js +269 -0
  140. package/dist/agent-observer/index.d.ts +7 -0
  141. package/dist/agent-observer/index.js +4 -0
  142. package/dist/agent-observer/pricing.d.ts +35 -0
  143. package/dist/agent-observer/pricing.js +82 -0
  144. package/dist/agent-observer/provider.d.ts +77 -0
  145. package/dist/agent-observer/provider.js +151 -0
  146. package/dist/agent-observer/proxy.d.ts +91 -0
  147. package/dist/agent-observer/proxy.js +321 -0
  148. package/dist/agent-observer/test-imports.d.ts +7 -0
  149. package/dist/agent-observer/test-imports.js +185 -0
  150. package/dist/agent-observer/types.d.ts +137 -0
  151. package/dist/agent-observer/types.js +16 -0
  152. package/dist/assertions/source-isolation.d.ts +72 -0
  153. package/dist/assertions/source-isolation.js +117 -0
  154. package/dist/cli.d.ts +24 -0
  155. package/dist/cli.js +199 -0
  156. package/dist/commands/agent-report.d.ts +5 -0
  157. package/dist/commands/agent-report.js +69 -0
  158. package/dist/commands/baseline.d.ts +9 -0
  159. package/dist/commands/baseline.js +141 -0
  160. package/dist/commands/cache.d.ts +13 -0
  161. package/dist/commands/cache.js +135 -0
  162. package/dist/commands/calculate-scores.d.ts +8 -0
  163. package/dist/commands/calculate-scores.js +48 -0
  164. package/dist/commands/compare.d.ts +8 -0
  165. package/dist/commands/compare.js +120 -0
  166. package/dist/commands/completion.d.ts +18 -0
  167. package/dist/commands/completion.js +260 -0
  168. package/dist/commands/coverage-audit.d.ts +7 -0
  169. package/dist/commands/coverage-audit.js +40 -0
  170. package/dist/commands/discovery-report.d.ts +10 -0
  171. package/dist/commands/discovery-report.js +44 -0
  172. package/dist/commands/eval.d.ts +9 -0
  173. package/dist/commands/eval.js +35 -0
  174. package/dist/commands/explain-handler.d.ts +34 -0
  175. package/dist/commands/explain-handler.js +719 -0
  176. package/dist/commands/fetch-docs.d.ts +8 -0
  177. package/dist/commands/fetch-docs.js +128 -0
  178. package/dist/commands/generate-configs.d.ts +8 -0
  179. package/dist/commands/generate-configs.js +46 -0
  180. package/dist/commands/grader/index.d.ts +11 -0
  181. package/dist/commands/grader/index.js +118 -0
  182. package/dist/commands/init.d.ts +19 -0
  183. package/dist/commands/init.js +150 -0
  184. package/dist/commands/interactive.d.ts +12 -0
  185. package/dist/commands/interactive.js +238 -0
  186. package/dist/commands/lookup-doc.d.ts +15 -0
  187. package/dist/commands/lookup-doc.js +84 -0
  188. package/dist/commands/measure-retrieval.d.ts +5 -0
  189. package/dist/commands/measure-retrieval.js +65 -0
  190. package/dist/commands/pipeline-action.d.ts +71 -0
  191. package/dist/commands/pipeline-action.js +305 -0
  192. package/dist/commands/pipeline.d.ts +62 -0
  193. package/dist/commands/pipeline.js +53 -0
  194. package/dist/commands/pr-comment.d.ts +8 -0
  195. package/dist/commands/pr-comment.js +47 -0
  196. package/dist/commands/publish.d.ts +26 -0
  197. package/dist/commands/publish.js +253 -0
  198. package/dist/commands/readiness-report.d.ts +10 -0
  199. package/dist/commands/readiness-report.js +104 -0
  200. package/dist/commands/shared/options.d.ts +29 -0
  201. package/dist/commands/shared/options.js +57 -0
  202. package/dist/commands/update-quality-scores.d.ts +5 -0
  203. package/dist/commands/update-quality-scores.js +20 -0
  204. package/dist/commands/validate-tasks.d.ts +16 -0
  205. package/dist/commands/validate-tasks.js +93 -0
  206. package/dist/commands/validate.d.ts +9 -0
  207. package/dist/commands/validate.js +73 -0
  208. package/dist/commands/webhook-server.d.ts +5 -0
  209. package/dist/commands/webhook-server.js +30 -0
  210. package/dist/commands/weekly-digest.d.ts +10 -0
  211. package/dist/commands/weekly-digest.js +104 -0
  212. package/dist/composition-root.d.ts +26 -0
  213. package/dist/composition-root.js +107 -0
  214. package/dist/interpolate.d.ts +26 -0
  215. package/dist/interpolate.js +70 -0
  216. package/dist/job-store.d.ts +104 -0
  217. package/dist/job-store.js +188 -0
  218. package/dist/lib/agent-behavior-report.d.ts +8 -0
  219. package/dist/lib/agent-behavior-report.js +185 -0
  220. package/dist/lib/baseline.d.ts +19 -0
  221. package/dist/lib/baseline.js +153 -0
  222. package/dist/lib/calculate-scores.d.ts +23 -0
  223. package/dist/lib/calculate-scores.js +42 -0
  224. package/dist/lib/compare.d.ts +18 -0
  225. package/dist/lib/compare.js +170 -0
  226. package/dist/lib/coverage-audit.d.ts +4 -0
  227. package/dist/lib/coverage-audit.js +42 -0
  228. package/dist/lib/discovery-report.d.ts +13 -0
  229. package/dist/lib/discovery-report.js +57 -0
  230. package/dist/lib/fetch-docs.d.ts +30 -0
  231. package/dist/lib/fetch-docs.js +171 -0
  232. package/dist/lib/generate-configs.d.ts +25 -0
  233. package/dist/lib/generate-configs.js +42 -0
  234. package/dist/lib/grader-api.d.ts +21 -0
  235. package/dist/lib/grader-api.js +34 -0
  236. package/dist/lib/grader-compare.d.ts +19 -0
  237. package/dist/lib/grader-compare.js +91 -0
  238. package/dist/lib/grader-consistency.d.ts +27 -0
  239. package/dist/lib/grader-consistency.js +79 -0
  240. package/dist/lib/grader-sensitivity.d.ts +19 -0
  241. package/dist/lib/grader-sensitivity.js +75 -0
  242. package/dist/lib/grader-validate.d.ts +19 -0
  243. package/dist/lib/grader-validate.js +78 -0
  244. package/dist/lib/measure-retrieval.d.ts +14 -0
  245. package/dist/lib/measure-retrieval.js +71 -0
  246. package/dist/lib/pr-comment.d.ts +16 -0
  247. package/dist/lib/pr-comment.js +28 -0
  248. package/dist/lib/readiness-report.d.ts +13 -0
  249. package/dist/lib/readiness-report.js +108 -0
  250. package/dist/lib/webhook-server.d.ts +11 -0
  251. package/dist/lib/webhook-server.js +24 -0
  252. package/dist/lib/weekly-digest.d.ts +24 -0
  253. package/dist/lib/weekly-digest.js +148 -0
  254. package/dist/orchestration/build-app-context.d.ts +27 -0
  255. package/dist/orchestration/build-app-context.js +81 -0
  256. package/dist/orchestration/build-step-sequence.d.ts +15 -0
  257. package/dist/orchestration/build-step-sequence.js +84 -0
  258. package/dist/orchestration/config-to-source-overrides.d.ts +9 -0
  259. package/dist/orchestration/config-to-source-overrides.js +28 -0
  260. package/dist/orchestration/env-bridge.d.ts +21 -0
  261. package/dist/orchestration/env-bridge.js +66 -0
  262. package/dist/orchestration/index.d.ts +11 -0
  263. package/dist/orchestration/index.js +11 -0
  264. package/dist/orchestration/pipeline-orchestrator.d.ts +24 -0
  265. package/dist/orchestration/pipeline-orchestrator.js +153 -0
  266. package/dist/orchestration/step-runner.d.ts +20 -0
  267. package/dist/orchestration/step-runner.js +88 -0
  268. package/dist/orchestration/steps/calculate-scores-step.d.ts +13 -0
  269. package/dist/orchestration/steps/calculate-scores-step.js +95 -0
  270. package/dist/orchestration/steps/callback-step.d.ts +24 -0
  271. package/dist/orchestration/steps/callback-step.js +76 -0
  272. package/dist/orchestration/steps/compare-step.d.ts +14 -0
  273. package/dist/orchestration/steps/compare-step.js +92 -0
  274. package/dist/orchestration/steps/discovery-report-step.d.ts +13 -0
  275. package/dist/orchestration/steps/discovery-report-step.js +55 -0
  276. package/dist/orchestration/steps/fetch-docs-shell.d.ts +17 -0
  277. package/dist/orchestration/steps/fetch-docs-shell.js +30 -0
  278. package/dist/orchestration/steps/fetch-docs-step.d.ts +14 -0
  279. package/dist/orchestration/steps/fetch-docs-step.js +135 -0
  280. package/dist/orchestration/steps/gap-analysis-step.d.ts +16 -0
  281. package/dist/orchestration/steps/gap-analysis-step.js +136 -0
  282. package/dist/orchestration/steps/generate-configs-step.d.ts +14 -0
  283. package/dist/orchestration/steps/generate-configs-step.js +85 -0
  284. package/dist/orchestration/steps/grader-consistency-step.d.ts +13 -0
  285. package/dist/orchestration/steps/grader-consistency-step.js +64 -0
  286. package/dist/orchestration/steps/index.d.ts +19 -0
  287. package/dist/orchestration/steps/index.js +19 -0
  288. package/dist/orchestration/steps/mirror-repo-tasks-step.d.ts +21 -0
  289. package/dist/orchestration/steps/mirror-repo-tasks-step.js +94 -0
  290. package/dist/orchestration/steps/publish-report-step.d.ts +26 -0
  291. package/dist/orchestration/steps/publish-report-step.js +216 -0
  292. package/dist/orchestration/steps/readiness-step.d.ts +13 -0
  293. package/dist/orchestration/steps/readiness-step.js +91 -0
  294. package/dist/orchestration/steps/report-step.d.ts +12 -0
  295. package/dist/orchestration/steps/report-step.js +49 -0
  296. package/dist/orchestration/steps/run-eval-step.d.ts +17 -0
  297. package/dist/orchestration/steps/run-eval-step.js +195 -0
  298. package/dist/orchestration/steps/validate-step.d.ts +12 -0
  299. package/dist/orchestration/steps/validate-step.js +41 -0
  300. package/dist/pipeline/agent-behavior-report.d.ts +53 -0
  301. package/dist/pipeline/agent-behavior-report.js +132 -0
  302. package/dist/pipeline/attribution.d.ts +47 -0
  303. package/dist/pipeline/attribution.js +226 -0
  304. package/dist/pipeline/baseline.d.ts +37 -0
  305. package/dist/pipeline/baseline.js +141 -0
  306. package/dist/pipeline/cache.d.ts +101 -0
  307. package/dist/pipeline/cache.js +283 -0
  308. package/dist/pipeline/calculate-scores.d.ts +102 -0
  309. package/dist/pipeline/calculate-scores.js +1128 -0
  310. package/dist/pipeline/callback-delivery.d.ts +50 -0
  311. package/dist/pipeline/callback-delivery.js +89 -0
  312. package/dist/pipeline/checks.d.ts +39 -0
  313. package/dist/pipeline/checks.js +280 -0
  314. package/dist/pipeline/classify-url.d.ts +61 -0
  315. package/dist/pipeline/classify-url.js +93 -0
  316. package/dist/pipeline/compare.d.ts +31 -0
  317. package/dist/pipeline/compare.js +208 -0
  318. package/dist/pipeline/coverage-audit.d.ts +39 -0
  319. package/dist/pipeline/coverage-audit.js +165 -0
  320. package/dist/pipeline/degradations.d.ts +85 -0
  321. package/dist/pipeline/degradations.js +242 -0
  322. package/dist/pipeline/discovery-report.d.ts +55 -0
  323. package/dist/pipeline/discovery-report.js +178 -0
  324. package/dist/pipeline/eval-constants.d.ts +68 -0
  325. package/dist/pipeline/eval-constants.js +111 -0
  326. package/dist/pipeline/eval-fingerprint.d.ts +66 -0
  327. package/dist/pipeline/eval-fingerprint.js +175 -0
  328. package/dist/pipeline/expand-tasks.d.ts +220 -0
  329. package/dist/pipeline/expand-tasks.js +421 -0
  330. package/dist/pipeline/failure-modes.d.ts +46 -0
  331. package/dist/pipeline/failure-modes.js +348 -0
  332. package/dist/pipeline/fetch-url-content.d.ts +44 -0
  333. package/dist/pipeline/fetch-url-content.js +93 -0
  334. package/dist/pipeline/gap-analysis.d.ts +48 -0
  335. package/dist/pipeline/gap-analysis.js +231 -0
  336. package/dist/pipeline/generate-configs.d.ts +72 -0
  337. package/dist/pipeline/generate-configs.js +395 -0
  338. package/dist/pipeline/grader-api.d.ts +49 -0
  339. package/dist/pipeline/grader-api.js +200 -0
  340. package/dist/pipeline/grader-compare-runner.d.ts +44 -0
  341. package/dist/pipeline/grader-compare-runner.js +301 -0
  342. package/dist/pipeline/grader-comparison.d.ts +111 -0
  343. package/dist/pipeline/grader-comparison.js +161 -0
  344. package/dist/pipeline/grader-consistency-runner.d.ts +60 -0
  345. package/dist/pipeline/grader-consistency-runner.js +270 -0
  346. package/dist/pipeline/grader-consistency.d.ts +103 -0
  347. package/dist/pipeline/grader-consistency.js +146 -0
  348. package/dist/pipeline/grader-sensitivity-runner.d.ts +40 -0
  349. package/dist/pipeline/grader-sensitivity-runner.js +282 -0
  350. package/dist/pipeline/grader-sensitivity.d.ts +94 -0
  351. package/dist/pipeline/grader-sensitivity.js +144 -0
  352. package/dist/pipeline/grader-validate-runner.d.ts +38 -0
  353. package/dist/pipeline/grader-validate-runner.js +229 -0
  354. package/dist/pipeline/grader-validation.d.ts +107 -0
  355. package/dist/pipeline/grader-validation.js +169 -0
  356. package/dist/pipeline/map-request-to-config.d.ts +19 -0
  357. package/dist/pipeline/map-request-to-config.js +80 -0
  358. package/dist/pipeline/measure-retrieval.d.ts +59 -0
  359. package/dist/pipeline/measure-retrieval.js +111 -0
  360. package/dist/pipeline/mirror-repo-tasks.d.ts +86 -0
  361. package/dist/pipeline/mirror-repo-tasks.js +350 -0
  362. package/dist/pipeline/plan-format.d.ts +33 -0
  363. package/dist/pipeline/plan-format.js +202 -0
  364. package/dist/pipeline/plan.d.ts +169 -0
  365. package/dist/pipeline/plan.js +708 -0
  366. package/dist/pipeline/pr-comment.d.ts +19 -0
  367. package/dist/pipeline/pr-comment.js +502 -0
  368. package/dist/pipeline/probe.d.ts +52 -0
  369. package/dist/pipeline/probe.js +390 -0
  370. package/dist/pipeline/provenance.d.ts +47 -0
  371. package/dist/pipeline/provenance.js +146 -0
  372. package/dist/pipeline/readiness-report.d.ts +87 -0
  373. package/dist/pipeline/readiness-report.js +205 -0
  374. package/dist/pipeline/release-classification.d.ts +54 -0
  375. package/dist/pipeline/release-classification.js +238 -0
  376. package/dist/pipeline/release-report.d.ts +37 -0
  377. package/dist/pipeline/release-report.js +222 -0
  378. package/dist/pipeline/repo-eval-comment.d.ts +37 -0
  379. package/dist/pipeline/repo-eval-comment.js +165 -0
  380. package/dist/pipeline/repo-threshold-evaluator.d.ts +89 -0
  381. package/dist/pipeline/repo-threshold-evaluator.js +162 -0
  382. package/dist/pipeline/resolve-mappings.d.ts +35 -0
  383. package/dist/pipeline/resolve-mappings.js +72 -0
  384. package/dist/pipeline/retrieval-metrics.d.ts +39 -0
  385. package/dist/pipeline/retrieval-metrics.js +136 -0
  386. package/dist/pipeline/reverse-mapping.d.ts +67 -0
  387. package/dist/pipeline/reverse-mapping.js +88 -0
  388. package/dist/pipeline/schemas.d.ts +9 -0
  389. package/dist/pipeline/schemas.js +9 -0
  390. package/dist/pipeline/steps/calculate-scores-step.d.ts +11 -0
  391. package/dist/pipeline/steps/calculate-scores-step.js +89 -0
  392. package/dist/pipeline/steps/compare-step.d.ts +18 -0
  393. package/dist/pipeline/steps/compare-step.js +90 -0
  394. package/dist/pipeline/steps/eval-step.d.ts +53 -0
  395. package/dist/pipeline/steps/eval-step.js +347 -0
  396. package/dist/pipeline/steps/fetch-docs-step.d.ts +11 -0
  397. package/dist/pipeline/steps/fetch-docs-step.js +84 -0
  398. package/dist/pipeline/steps/generate-configs-step.d.ts +11 -0
  399. package/dist/pipeline/steps/generate-configs-step.js +98 -0
  400. package/dist/pipeline/steps/grader-consistency-step.d.ts +21 -0
  401. package/dist/pipeline/steps/grader-consistency-step.js +74 -0
  402. package/dist/pipeline/steps/publish-report-step.d.ts +57 -0
  403. package/dist/pipeline/steps/publish-report-step.js +243 -0
  404. package/dist/pipeline/steps/report-step.d.ts +13 -0
  405. package/dist/pipeline/steps/report-step.js +56 -0
  406. package/dist/pipeline/steps/update-scores-step.d.ts +11 -0
  407. package/dist/pipeline/steps/update-scores-step.js +42 -0
  408. package/dist/pipeline/targeted-loo.d.ts +88 -0
  409. package/dist/pipeline/targeted-loo.js +203 -0
  410. package/dist/pipeline/thresholds.d.ts +27 -0
  411. package/dist/pipeline/thresholds.js +245 -0
  412. package/dist/pipeline/types.d.ts +10 -0
  413. package/dist/pipeline/types.js +10 -0
  414. package/dist/pipeline/validate.d.ts +67 -0
  415. package/dist/pipeline/validate.js +406 -0
  416. package/dist/pipeline/webhook-server.d.ts +37 -0
  417. package/dist/pipeline/webhook-server.js +133 -0
  418. package/dist/report-store.d.ts +84 -0
  419. package/dist/report-store.js +208 -0
  420. package/dist/sanity/client.d.ts +38 -0
  421. package/dist/sanity/client.js +86 -0
  422. package/dist/sanity/portable-text.d.ts +11 -0
  423. package/dist/sanity/portable-text.js +211 -0
  424. package/dist/sanity/queries.d.ts +133 -0
  425. package/dist/sanity/queries.js +300 -0
  426. package/dist/schedules/digest.d.ts +116 -0
  427. package/dist/schedules/digest.js +156 -0
  428. package/dist/schedules/index.d.ts +12 -0
  429. package/dist/schedules/index.js +10 -0
  430. package/dist/schedules/loader.d.ts +31 -0
  431. package/dist/schedules/loader.js +73 -0
  432. package/dist/schedules/schema.d.ts +9 -0
  433. package/dist/schedules/schema.js +9 -0
  434. package/dist/scripts/agent-behavior-report.d.ts +19 -0
  435. package/dist/scripts/agent-behavior-report.js +315 -0
  436. package/dist/scripts/baseline.d.ts +43 -0
  437. package/dist/scripts/baseline.js +267 -0
  438. package/dist/scripts/calculate-scores.d.ts +166 -0
  439. package/dist/scripts/calculate-scores.js +1296 -0
  440. package/dist/scripts/compare.d.ts +22 -0
  441. package/dist/scripts/compare.js +334 -0
  442. package/dist/scripts/coverage-audit.d.ts +44 -0
  443. package/dist/scripts/coverage-audit.js +209 -0
  444. package/dist/scripts/debug-eval.d.ts +19 -0
  445. package/dist/scripts/debug-eval.js +73 -0
  446. package/dist/scripts/discovery-report.d.ts +58 -0
  447. package/dist/scripts/discovery-report.js +250 -0
  448. package/dist/scripts/fetch-docs.d.ts +35 -0
  449. package/dist/scripts/fetch-docs.js +472 -0
  450. package/dist/scripts/generate-configs.d.ts +66 -0
  451. package/dist/scripts/generate-configs.js +459 -0
  452. package/dist/scripts/grader-api.d.ts +27 -0
  453. package/dist/scripts/grader-api.js +206 -0
  454. package/dist/scripts/grader-compare.d.ts +22 -0
  455. package/dist/scripts/grader-compare.js +368 -0
  456. package/dist/scripts/grader-consistency.d.ts +20 -0
  457. package/dist/scripts/grader-consistency.js +313 -0
  458. package/dist/scripts/grader-sensitivity.d.ts +22 -0
  459. package/dist/scripts/grader-sensitivity.js +354 -0
  460. package/dist/scripts/grader-validate.d.ts +19 -0
  461. package/dist/scripts/grader-validate.js +267 -0
  462. package/dist/scripts/measure-retrieval.d.ts +10 -0
  463. package/dist/scripts/measure-retrieval.js +145 -0
  464. package/dist/scripts/migrate-tasks-to-content-lake.d.ts +24 -0
  465. package/dist/scripts/migrate-tasks-to-content-lake.js +327 -0
  466. package/dist/scripts/pipeline.d.ts +76 -0
  467. package/dist/scripts/pipeline.js +1031 -0
  468. package/dist/scripts/pr-comment.d.ts +10 -0
  469. package/dist/scripts/pr-comment.js +510 -0
  470. package/dist/scripts/readiness-report.d.ts +88 -0
  471. package/dist/scripts/readiness-report.js +342 -0
  472. package/dist/scripts/update-quality-scores.d.ts +15 -0
  473. package/dist/scripts/update-quality-scores.js +184 -0
  474. package/dist/scripts/validate-task-sources.d.ts +21 -0
  475. package/dist/scripts/validate-task-sources.js +210 -0
  476. package/dist/scripts/validate.d.ts +13 -0
  477. package/dist/scripts/validate.js +79 -0
  478. package/dist/scripts/webhook-server.d.ts +26 -0
  479. package/dist/scripts/webhook-server.js +147 -0
  480. package/dist/scripts/weekly-digest.d.ts +24 -0
  481. package/dist/scripts/weekly-digest.js +144 -0
  482. package/dist/sinks/bigquery/index.d.ts +131 -0
  483. package/dist/sinks/bigquery/index.js +222 -0
  484. package/dist/sinks/format-slack.d.ts +64 -0
  485. package/dist/sinks/format-slack.js +306 -0
  486. package/dist/sinks/index.d.ts +23 -0
  487. package/dist/sinks/index.js +18 -0
  488. package/dist/sinks/loader.d.ts +18 -0
  489. package/dist/sinks/loader.js +82 -0
  490. package/dist/sinks/retry.d.ts +24 -0
  491. package/dist/sinks/retry.js +52 -0
  492. package/dist/sinks/schema.d.ts +9 -0
  493. package/dist/sinks/schema.js +9 -0
  494. package/dist/sinks/slack/format.d.ts +65 -0
  495. package/dist/sinks/slack/format.js +327 -0
  496. package/dist/sinks/slack/index.d.ts +27 -0
  497. package/dist/sinks/slack/index.js +78 -0
  498. package/dist/sinks/slack-sink.d.ts +27 -0
  499. package/dist/sinks/slack-sink.js +78 -0
  500. package/dist/sinks/types.d.ts +59 -0
  501. package/dist/sinks/types.js +44 -0
  502. package/dist/sinks/webhook/index.d.ts +19 -0
  503. package/dist/sinks/webhook/index.js +50 -0
  504. package/dist/sinks/webhook-sink.d.ts +19 -0
  505. package/dist/sinks/webhook-sink.js +50 -0
  506. package/dist/sources.d.ts +104 -0
  507. package/dist/sources.js +292 -0
  508. package/dist/webhook/budget.d.ts +42 -0
  509. package/dist/webhook/budget.js +60 -0
  510. package/dist/webhook/debounce.d.ts +67 -0
  511. package/dist/webhook/debounce.js +76 -0
  512. package/dist/webhook/dispatch.d.ts +45 -0
  513. package/dist/webhook/dispatch.js +84 -0
  514. package/dist/webhook/eval-request-handler.d.ts +87 -0
  515. package/dist/webhook/eval-request-handler.js +181 -0
  516. package/dist/webhook/handler.d.ts +88 -0
  517. package/dist/webhook/handler.js +203 -0
  518. package/dist/webhook/index.d.ts +17 -0
  519. package/dist/webhook/index.js +12 -0
  520. package/dist/webhook/types.d.ts +109 -0
  521. package/dist/webhook/types.js +10 -0
  522. package/package.json +72 -0
  523. package/tasks/.expanded.agentic.yaml +51 -0
  524. package/tasks/.expanded.yaml +66 -0
  525. package/tasks/frameworks.yaml +98 -0
  526. package/tasks/functions.yaml +51 -0
  527. package/tasks/groq.yaml +216 -0
  528. package/tasks/nextjs-live.yaml +62 -0
  529. package/tasks/studio-setup.yaml +111 -0
  530. package/tasks/visual-editing.yaml +120 -0
@@ -0,0 +1,318 @@
1
+ /**
2
+ * @sanity/ailf-core — Pipeline schemas
3
+ *
4
+ * Zod schemas for runtime validation of task YAML files and related configs.
5
+ * These schemas are the single source of truth for what constitutes a valid
6
+ * task definition — the TypeScript interfaces in types.ts are derived from
7
+ * (or should match) these schemas.
8
+ *
9
+ * Extracted from packages/eval/src/pipeline/schemas.ts (Phase 0d).
10
+ */
11
+ import { z } from "zod";
12
+ // ---------------------------------------------------------------------------
13
+ // Rubric template schemas — validates config/rubrics.yaml
14
+ // ---------------------------------------------------------------------------
15
+ /**
16
+ * Schema for a single rubric template (e.g., task-completion, code-correctness).
17
+ *
18
+ * The optional `dimension` field tags which scoring dimension this template
19
+ * belongs to. When present, it propagates into assertion metadata so the
20
+ * scoring engine can classify rubrics structurally instead of heuristically.
21
+ * See docs/design-docs/structured-dimensions.md.
22
+ */
23
+ export const RubricTemplateSchema = z.object({
24
+ criteria_label: z
25
+ .string()
26
+ .min(1, "criteria_label must be a non-empty string")
27
+ .nullish(),
28
+ dimension: z.string().min(1).optional(),
29
+ header: z.string().min(1, "header must be a non-empty string"),
30
+ scale: z
31
+ .array(z.string().min(1))
32
+ .min(1, "scale must have at least one entry"),
33
+ });
34
+ /**
35
+ * Schema for the full config/rubrics.yaml config file.
36
+ *
37
+ * Each dimension is scored on a uniform 0–100 scale. The `weights` section
38
+ * defines how dimensions are combined into a composite score (must sum to 1.0).
39
+ */
40
+ export const RubricConfigSchema = z.object({
41
+ footer: z.string().min(1, "footer must be a non-empty string"),
42
+ templates: z
43
+ .record(z.string(), RubricTemplateSchema)
44
+ .refine((t) => Object.keys(t).length > 0, {
45
+ message: "templates must have at least one entry",
46
+ }),
47
+ weights: z.record(z.string(), z.number().min(0).max(1)).refine((w) => {
48
+ const sum = Object.values(w).reduce((s, v) => s + v, 0);
49
+ return Math.abs(sum - 1.0) < 0.001;
50
+ }, { message: "weights must sum to 1.0" }),
51
+ });
52
+ // ---------------------------------------------------------------------------
53
+ // Feature registry schema — validates config/features.yaml (Phase 3c)
54
+ // ---------------------------------------------------------------------------
55
+ /**
56
+ * Schema for a single product feature in the feature registry.
57
+ */
58
+ export const FeatureSchema = z.object({
59
+ area: z.string().min(1).optional(),
60
+ id: z
61
+ .string()
62
+ .min(1)
63
+ .regex(/^[a-z0-9][a-z0-9-]*$/, "id must be lowercase alphanumeric with hyphens"),
64
+ name: z.string().min(1),
65
+ priority: z.enum(["critical", "high", "medium", "low"]),
66
+ sections: z.array(z.string().min(1)).min(1),
67
+ status: z.enum(["covered", "uncovered", "planned", "out-of-scope"]),
68
+ taskCount: z.number().int().min(0).optional(),
69
+ });
70
+ /**
71
+ * Schema for the full config/features.yaml config file.
72
+ */
73
+ export const FeatureRegistrySchema = z.object({
74
+ features: z.array(FeatureSchema).min(1),
75
+ });
76
+ // ---------------------------------------------------------------------------
77
+ // Assertion schemas — one per Promptfoo assertion type
78
+ // ---------------------------------------------------------------------------
79
+ /**
80
+ * Templated llm-rubric assertion — references a rubric template by key
81
+ * and provides task-specific criteria.
82
+ */
83
+ const TemplatedLlmRubricAssertSchema = z.object({
84
+ criteria: z
85
+ .array(z.string().min(1))
86
+ .min(1, "criteria must have at least one entry"),
87
+ template: z.string().min(1, "template must be a non-empty string"),
88
+ type: z.literal("llm-rubric"),
89
+ weight: z.number().optional(),
90
+ });
91
+ const ContainsAssertSchema = z.object({
92
+ type: z.literal("contains"),
93
+ value: z.string().min(1, "contains must have a non-empty value"),
94
+ weight: z.number().optional(),
95
+ });
96
+ const ContainsAnyAssertSchema = z.object({
97
+ type: z.literal("contains-any"),
98
+ value: z
99
+ .array(z.string().min(1))
100
+ .min(1, "contains-any must have at least one value"),
101
+ weight: z.number().optional(),
102
+ });
103
+ const NotContainsAssertSchema = z.object({
104
+ type: z.literal("not-contains"),
105
+ value: z.string().min(1, "not-contains must have a non-empty value"),
106
+ weight: z.number().optional(),
107
+ });
108
+ const JavascriptAssertSchema = z.object({
109
+ type: z.literal("javascript"),
110
+ value: z.string().min(1, "javascript must have a non-empty value"),
111
+ weight: z.number().optional(),
112
+ });
113
+ /**
114
+ * Union of all supported Promptfoo assertion types.
115
+ *
116
+ * llm-rubric assertions use the templated format (template + criteria).
117
+ * Other assertion types use their standard value-based format.
118
+ *
119
+ * Note: we use z.union instead of z.discriminatedUnion because multiple
120
+ * schemas share the "llm-rubric" type literal (the old value-based format
121
+ * is gone, but union is more flexible for future additions).
122
+ */
123
+ export const AssertionSchema = z.union([
124
+ TemplatedLlmRubricAssertSchema,
125
+ ContainsAssertSchema,
126
+ ContainsAnyAssertSchema,
127
+ NotContainsAssertSchema,
128
+ JavascriptAssertSchema,
129
+ ]);
130
+ // ---------------------------------------------------------------------------
131
+ // Task variable schemas
132
+ // ---------------------------------------------------------------------------
133
+ const TaskVarsSchema = z
134
+ .object({
135
+ docs: z.string({
136
+ message: "vars.docs must be a string (use empty string for baseline)",
137
+ }),
138
+ task: z.string().min(1, "vars.task must be a non-empty string"),
139
+ })
140
+ .passthrough();
141
+ // ---------------------------------------------------------------------------
142
+ // Canonical doc mapping schema (inline in task files)
143
+ // ---------------------------------------------------------------------------
144
+ /**
145
+ * Polymorphic canonical doc reference — discriminated by key presence.
146
+ *
147
+ * Exactly one resolution key must be present: slug, path, id, or perspective.
148
+ * All carry an optional reason. The union uses Zod discriminatedUnion-like
149
+ * semantics but keyed on field presence rather than a literal discriminator.
150
+ *
151
+ * @see docs/design-docs/canonical-doc-resolution.md
152
+ */
153
+ const SlugDocRefSchema = z.object({
154
+ reason: z.string().optional().default(""),
155
+ slug: z.string().min(1, "slug must be a non-empty string"),
156
+ });
157
+ const PathDocRefSchema = z.object({
158
+ path: z.string().min(1, "path must be a non-empty string"),
159
+ reason: z.string().optional().default(""),
160
+ });
161
+ const IdDocRefSchema = z.object({
162
+ id: z.string().min(1, "id must be a non-empty string"),
163
+ reason: z.string().optional().default(""),
164
+ });
165
+ const PerspectiveDocRefSchema = z.object({
166
+ perspective: z.string().min(1, "perspective must be a non-empty string"),
167
+ reason: z.string().optional().default(""),
168
+ });
169
+ const CanonicalDocSchema = z.union([
170
+ SlugDocRefSchema,
171
+ PathDocRefSchema,
172
+ IdDocRefSchema,
173
+ PerspectiveDocRefSchema,
174
+ ]);
175
+ // ---------------------------------------------------------------------------
176
+ // Baseline options schema
177
+ // ---------------------------------------------------------------------------
178
+ const BaselineOptionsSchema = z.object({
179
+ enabled: z.boolean().optional(),
180
+ rubric: z.enum(["abbreviated", "full", "none"]).optional(),
181
+ });
182
+ // ---------------------------------------------------------------------------
183
+ // Task entry schemas — new (single-definition) and legacy formats
184
+ // ---------------------------------------------------------------------------
185
+ /**
186
+ * Schema for the new single-definition task format.
187
+ * Each task is defined once with an id field; the pipeline auto-generates
188
+ * gold + baseline variants.
189
+ */
190
+ export const SingleTaskSchema = z.object({
191
+ assert: z
192
+ .array(AssertionSchema)
193
+ .min(1, "assert must have at least one entry"),
194
+ baseline: BaselineOptionsSchema.optional(),
195
+ /** Canonical docs — which documentation articles are gold-standard for this task. */
196
+ canonical_docs: z
197
+ .array(CanonicalDocSchema)
198
+ .min(1, "canonical_docs must have at least one entry"),
199
+ description: z.string().min(1, "description must be a non-empty string"),
200
+ /** Opt-in: auto-generate a documentation coverage rubric for the gold variant. */
201
+ doc_coverage: z.boolean().optional(),
202
+ id: z
203
+ .string()
204
+ .min(1, "id must be a non-empty string")
205
+ .regex(/^[a-z0-9][a-z0-9-]*$/, "id must be lowercase alphanumeric with hyphens (e.g., 'groq-blog-queries')"),
206
+ /** Path to the reference solution file, relative to canonical/. */
207
+ reference_solution: z
208
+ .string()
209
+ .min(1, "reference_solution must be a non-empty path"),
210
+ vars: TaskVarsSchema,
211
+ });
212
+ /**
213
+ * Schema for legacy task entries (the old paired gold/baseline format).
214
+ * These have description and vars but no id field.
215
+ */
216
+ export const LegacyTaskSchema = z.object({
217
+ assert: z.array(AssertionSchema).optional(),
218
+ description: z.string().min(1, "description must be a non-empty string"),
219
+ transform: z.string().optional(),
220
+ vars: z
221
+ .object({
222
+ task: z.string().min(1, "vars.task must be a non-empty string"),
223
+ })
224
+ .passthrough(),
225
+ });
226
+ // ---------------------------------------------------------------------------
227
+ // Task entry — union of both formats
228
+ // ---------------------------------------------------------------------------
229
+ /**
230
+ * A single task entry that can be either format.
231
+ * Entries with an id field are validated as SingleTask;
232
+ * entries without are validated as LegacyTask.
233
+ */
234
+ export const TaskEntrySchema = z.union([SingleTaskSchema, LegacyTaskSchema]);
235
+ // ---------------------------------------------------------------------------
236
+ // Task file schema — the top-level array
237
+ // ---------------------------------------------------------------------------
238
+ /**
239
+ * Schema for an entire task YAML file: an array of task entries.
240
+ */
241
+ export const TaskFileSchema = z
242
+ .array(TaskEntrySchema)
243
+ .min(1, "Task file must contain at least one task entry");
244
+ // ---------------------------------------------------------------------------
245
+ // Threshold config schemas — validates config/thresholds.yaml (Phase 5a)
246
+ // ---------------------------------------------------------------------------
247
+ /**
248
+ * Schema for per-dimension threshold values.
249
+ * Keys use kebab-case to match YAML convention; the threshold engine
250
+ * normalizes to camelCase for comparison against FeatureScore fields.
251
+ */
252
+ export const ThresholdDimensionsSchema = z.object({
253
+ "code-correctness": z.number().min(0).max(100).optional(),
254
+ "doc-coverage": z.number().min(0).max(100).optional(),
255
+ "task-completion": z.number().min(0).max(100).optional(),
256
+ });
257
+ /**
258
+ * Schema for threshold defaults (and per-area overrides).
259
+ * All fields are optional in per-area overrides; defaults must have composite.
260
+ */
261
+ export const ThresholdDefaultsSchema = z.object({
262
+ ceiling: z.number().min(0).max(100).optional(),
263
+ composite: z.number().min(0).max(100),
264
+ dimensions: ThresholdDimensionsSchema.optional(),
265
+ "doc-lift": z.number().optional(),
266
+ });
267
+ /**
268
+ * Schema for regression thresholds (all values should be negative or zero).
269
+ */
270
+ export const RegressionThresholdsSchema = z.object({
271
+ composite: z.number().max(0),
272
+ "per-area": z.number().max(0),
273
+ "per-dimension": z.number().max(0),
274
+ });
275
+ /**
276
+ * Schema for severity classification rules.
277
+ */
278
+ export const SeverityConfigSchema = z.object({
279
+ critical: z
280
+ .object({
281
+ "composite-below": z.number().min(0).max(100).optional(),
282
+ "negative-doc-lift": z.boolean().optional(),
283
+ })
284
+ .optional(),
285
+ info: z
286
+ .object({
287
+ "composite-below": z.number().min(0).max(100).optional(),
288
+ })
289
+ .optional(),
290
+ warning: z
291
+ .object({
292
+ "composite-below": z.number().min(0).max(100).optional(),
293
+ "regression-exceeds": z.number().optional(),
294
+ })
295
+ .optional(),
296
+ });
297
+ /**
298
+ * Schema for the complete config/thresholds.yaml file.
299
+ */
300
+ export const ThresholdConfigSchema = z.object({
301
+ areas: z.record(z.string(), ThresholdDefaultsSchema.partial()).optional(),
302
+ defaults: ThresholdDefaultsSchema,
303
+ regression: RegressionThresholdsSchema.optional(),
304
+ severity: SeverityConfigSchema.optional(),
305
+ });
306
+ // ---------------------------------------------------------------------------
307
+ // Error formatting
308
+ // ---------------------------------------------------------------------------
309
+ /**
310
+ * Format a Zod error into human-readable lines suitable for CLI output.
311
+ * Each line includes the path (e.g., "[2].vars.task") and the error message.
312
+ */
313
+ export function formatZodErrors(error) {
314
+ return error.issues.map((issue) => {
315
+ const path = issue.path.length > 0 ? issue.path.join(".") : "(root)";
316
+ return " [" + path + "]: " + issue.message;
317
+ });
318
+ }
@@ -0,0 +1,68 @@
1
+ /**
2
+ * @sanity/ailf-core — Schedule schemas
3
+ *
4
+ * Zod schemas for runtime validation of config/schedules.yaml — the
5
+ * configuration file for scheduled evaluation runs and digests.
6
+ *
7
+ * Each schedule defines a recurring pipeline run with source, mode,
8
+ * area/task filters, and publishing options. The digest section controls
9
+ * periodic summary notifications.
10
+ *
11
+ * @see docs/design-docs/report-store/implementation.md — Phase 5
12
+ */
13
+ import { z } from "zod";
14
+ /** Schema for a single scheduled evaluation run. */
15
+ export declare const ScheduleEntrySchema: z.ZodObject<{
16
+ areas: z.ZodOptional<z.ZodArray<z.ZodString>>;
17
+ compare: z.ZodDefault<z.ZodBoolean>;
18
+ cron: z.ZodString;
19
+ enabled: z.ZodDefault<z.ZodBoolean>;
20
+ mode: z.ZodDefault<z.ZodEnum<{
21
+ agentic: "agentic";
22
+ baseline: "baseline";
23
+ full: "full";
24
+ observed: "observed";
25
+ }>>;
26
+ name: z.ZodString;
27
+ publish: z.ZodDefault<z.ZodBoolean>;
28
+ source: z.ZodDefault<z.ZodString>;
29
+ tasks: z.ZodOptional<z.ZodArray<z.ZodString>>;
30
+ }, z.core.$strip>;
31
+ /** Inferred TypeScript type for a schedule entry. */
32
+ export type ScheduleEntry = z.infer<typeof ScheduleEntrySchema>;
33
+ /** Schema for the weekly digest section. */
34
+ export declare const DigestConfigSchema: z.ZodObject<{
35
+ cron: z.ZodOptional<z.ZodString>;
36
+ enabled: z.ZodDefault<z.ZodBoolean>;
37
+ lookbackDays: z.ZodDefault<z.ZodNumber>;
38
+ slackWebhookUrl: z.ZodOptional<z.ZodString>;
39
+ }, z.core.$strip>;
40
+ /** Inferred TypeScript type for the digest config. */
41
+ export type DigestConfig = z.infer<typeof DigestConfigSchema>;
42
+ /** Schema for the complete config/schedules.yaml file. */
43
+ export declare const SchedulesFileSchema: z.ZodObject<{
44
+ digest: z.ZodOptional<z.ZodObject<{
45
+ cron: z.ZodOptional<z.ZodString>;
46
+ enabled: z.ZodDefault<z.ZodBoolean>;
47
+ lookbackDays: z.ZodDefault<z.ZodNumber>;
48
+ slackWebhookUrl: z.ZodOptional<z.ZodString>;
49
+ }, z.core.$strip>>;
50
+ schedules: z.ZodDefault<z.ZodArray<z.ZodObject<{
51
+ areas: z.ZodOptional<z.ZodArray<z.ZodString>>;
52
+ compare: z.ZodDefault<z.ZodBoolean>;
53
+ cron: z.ZodString;
54
+ enabled: z.ZodDefault<z.ZodBoolean>;
55
+ mode: z.ZodDefault<z.ZodEnum<{
56
+ agentic: "agentic";
57
+ baseline: "baseline";
58
+ full: "full";
59
+ observed: "observed";
60
+ }>>;
61
+ name: z.ZodString;
62
+ publish: z.ZodDefault<z.ZodBoolean>;
63
+ source: z.ZodDefault<z.ZodString>;
64
+ tasks: z.ZodOptional<z.ZodArray<z.ZodString>>;
65
+ }, z.core.$strip>>>;
66
+ }, z.core.$strip>;
67
+ /** Inferred TypeScript type for the full schedules config file. */
68
+ export type SchedulesFile = z.infer<typeof SchedulesFileSchema>;
@@ -0,0 +1,74 @@
1
+ /**
2
+ * @sanity/ailf-core — Schedule schemas
3
+ *
4
+ * Zod schemas for runtime validation of config/schedules.yaml — the
5
+ * configuration file for scheduled evaluation runs and digests.
6
+ *
7
+ * Each schedule defines a recurring pipeline run with source, mode,
8
+ * area/task filters, and publishing options. The digest section controls
9
+ * periodic summary notifications.
10
+ *
11
+ * @see docs/design-docs/report-store/implementation.md — Phase 5
12
+ */
13
+ import { z } from "zod";
14
+ // ---------------------------------------------------------------------------
15
+ // Cron expression validation
16
+ // ---------------------------------------------------------------------------
17
+ /**
18
+ * Validates a basic cron expression (5 fields: min hour dom month dow).
19
+ * Accepts standard GitHub Actions cron syntax.
20
+ */
21
+ const CronSchema = z
22
+ .string()
23
+ .regex(/^[\d*,/-]+\s+[\d*,/-]+\s+[\d*,/-]+\s+[\d*,/-]+\s+[\d*,/-]+$/, "Invalid cron expression — expected 5 space-separated fields");
24
+ // ---------------------------------------------------------------------------
25
+ // Schedule entry
26
+ // ---------------------------------------------------------------------------
27
+ /** Schema for a single scheduled evaluation run. */
28
+ export const ScheduleEntrySchema = z.object({
29
+ /** Feature areas to evaluate (blank = all areas) */
30
+ areas: z.array(z.string()).optional(),
31
+ /** Whether to auto-compare against the most recent baseline */
32
+ compare: z.boolean().default(false),
33
+ /** Cron expression (UTC) for when this schedule runs */
34
+ cron: CronSchema,
35
+ /** Whether this schedule is active */
36
+ enabled: z.boolean().default(true),
37
+ /** Evaluation mode */
38
+ mode: z.enum(["agentic", "baseline", "full", "observed"]).default("baseline"),
39
+ /** Human-readable schedule name (used as report tag) */
40
+ name: z
41
+ .string()
42
+ .min(1)
43
+ .regex(/^[a-z][a-z0-9-]*$/, "Schedule name must be lowercase kebab-case"),
44
+ /** Whether to publish the report to the Sanity store + sinks */
45
+ publish: z.boolean().default(true),
46
+ /** Documentation source to evaluate */
47
+ source: z.string().default("production"),
48
+ /** Specific task IDs to evaluate (blank = all tasks) */
49
+ tasks: z.array(z.string()).optional(),
50
+ });
51
+ // ---------------------------------------------------------------------------
52
+ // Digest configuration
53
+ // ---------------------------------------------------------------------------
54
+ /** Schema for the weekly digest section. */
55
+ export const DigestConfigSchema = z.object({
56
+ /** Cron expression for when the digest is sent */
57
+ cron: CronSchema.optional(),
58
+ /** Whether the digest is enabled */
59
+ enabled: z.boolean().default(false),
60
+ /** How many days of reports to include in the digest */
61
+ lookbackDays: z.number().int().min(1).max(90).default(7),
62
+ /** Slack webhook URL (supports env var interpolation) */
63
+ slackWebhookUrl: z.string().optional(),
64
+ });
65
+ // ---------------------------------------------------------------------------
66
+ // Top-level file schema (config/schedules.yaml)
67
+ // ---------------------------------------------------------------------------
68
+ /** Schema for the complete config/schedules.yaml file. */
69
+ export const SchedulesFileSchema = z.object({
70
+ /** Digest aggregation settings */
71
+ digest: DigestConfigSchema.optional(),
72
+ /** Scheduled evaluation runs */
73
+ schedules: z.array(ScheduleEntrySchema).default([]),
74
+ });
@@ -0,0 +1,207 @@
1
+ /**
2
+ * @sanity/ailf-core — Sink schemas
3
+ *
4
+ * Zod schemas for runtime validation of config/sinks.yaml — the configuration
5
+ * file for report delivery sinks.
6
+ *
7
+ * Each sink type has its own config schema, and the top-level SinksFileSchema
8
+ * validates the complete YAML structure. The discriminated union on `type`
9
+ * ensures type-safe parsing with clear error messages for unknown sink types.
10
+ *
11
+ * Usage:
12
+ * import { SinksFileSchema } from "./schema.js"
13
+ * const parsed = SinksFileSchema.parse(rawYaml)
14
+ *
15
+ * @see docs/design-docs/report-store/sink-architecture.md
16
+ */
17
+ import { z } from "zod";
18
+ /** All supported sink types as a Zod union. */
19
+ export declare const SinkTypeSchema: z.ZodEnum<{
20
+ webhook: "webhook";
21
+ bigquery: "bigquery";
22
+ "github-comment": "github-comment";
23
+ slack: "slack";
24
+ }>;
25
+ /** Supported sink type string literal union. */
26
+ export type SinkType = z.infer<typeof SinkTypeSchema>;
27
+ /**
28
+ * Schema for sink routing rules.
29
+ * Controls which messages are delivered based on severity and type.
30
+ * When omitted, the sink receives all reports (backward compatible).
31
+ */
32
+ export declare const SinkRoutingSchema: z.ZodObject<{
33
+ critical: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
34
+ digest: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
35
+ info: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
36
+ regression: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
37
+ warning: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
38
+ }, z.core.$strip>;
39
+ /** Inferred TypeScript type for sink routing rules. */
40
+ /** Config schema for a generic webhook sink. */
41
+ export declare const WebhookSinkConfigSchema: z.ZodObject<{
42
+ enabled: z.ZodDefault<z.ZodBoolean>;
43
+ headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
44
+ routing: z.ZodOptional<z.ZodObject<{
45
+ critical: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
46
+ digest: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
47
+ info: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
48
+ regression: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
49
+ warning: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
50
+ }, z.core.$strip>>;
51
+ type: z.ZodLiteral<"webhook">;
52
+ url: z.ZodString;
53
+ }, z.core.$strip>;
54
+ export type SinkRouting = z.infer<typeof SinkRoutingSchema>;
55
+ /** Config schema for a Slack notification sink. */
56
+ export declare const SlackSinkConfigSchema: z.ZodObject<{
57
+ channel: z.ZodOptional<z.ZodString>;
58
+ enabled: z.ZodDefault<z.ZodBoolean>;
59
+ routing: z.ZodOptional<z.ZodObject<{
60
+ critical: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
61
+ digest: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
62
+ info: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
63
+ regression: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
64
+ warning: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
65
+ }, z.core.$strip>>;
66
+ type: z.ZodLiteral<"slack">;
67
+ webhookUrl: z.ZodString;
68
+ }, z.core.$strip>;
69
+ /** Config schema for a GitHub PR comment sink. */
70
+ export declare const GitHubCommentSinkConfigSchema: z.ZodObject<{
71
+ enabled: z.ZodDefault<z.ZodBoolean>;
72
+ routing: z.ZodOptional<z.ZodObject<{
73
+ critical: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
74
+ digest: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
75
+ info: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
76
+ regression: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
77
+ warning: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
78
+ }, z.core.$strip>>;
79
+ token: z.ZodOptional<z.ZodString>;
80
+ type: z.ZodLiteral<"github-comment">;
81
+ }, z.core.$strip>;
82
+ /** Config schema for a BigQuery sink. */
83
+ export declare const BigQuerySinkConfigSchema: z.ZodObject<{
84
+ credentials: z.ZodOptional<z.ZodString>;
85
+ dataset: z.ZodString;
86
+ enabled: z.ZodDefault<z.ZodBoolean>;
87
+ project: z.ZodString;
88
+ routing: z.ZodOptional<z.ZodObject<{
89
+ critical: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
90
+ digest: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
91
+ info: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
92
+ regression: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
93
+ warning: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
94
+ }, z.core.$strip>>;
95
+ type: z.ZodLiteral<"bigquery">;
96
+ }, z.core.$strip>;
97
+ /**
98
+ * A single sink configuration — discriminated on the `type` field.
99
+ *
100
+ * Zod v4 discriminatedUnion provides clear errors when the `type` value
101
+ * doesn't match any known sink.
102
+ */
103
+ export declare const SinkConfigSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
104
+ credentials: z.ZodOptional<z.ZodString>;
105
+ dataset: z.ZodString;
106
+ enabled: z.ZodDefault<z.ZodBoolean>;
107
+ project: z.ZodString;
108
+ routing: z.ZodOptional<z.ZodObject<{
109
+ critical: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
110
+ digest: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
111
+ info: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
112
+ regression: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
113
+ warning: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
114
+ }, z.core.$strip>>;
115
+ type: z.ZodLiteral<"bigquery">;
116
+ }, z.core.$strip>, z.ZodObject<{
117
+ enabled: z.ZodDefault<z.ZodBoolean>;
118
+ routing: z.ZodOptional<z.ZodObject<{
119
+ critical: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
120
+ digest: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
121
+ info: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
122
+ regression: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
123
+ warning: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
124
+ }, z.core.$strip>>;
125
+ token: z.ZodOptional<z.ZodString>;
126
+ type: z.ZodLiteral<"github-comment">;
127
+ }, z.core.$strip>, z.ZodObject<{
128
+ channel: z.ZodOptional<z.ZodString>;
129
+ enabled: z.ZodDefault<z.ZodBoolean>;
130
+ routing: z.ZodOptional<z.ZodObject<{
131
+ critical: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
132
+ digest: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
133
+ info: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
134
+ regression: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
135
+ warning: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
136
+ }, z.core.$strip>>;
137
+ type: z.ZodLiteral<"slack">;
138
+ webhookUrl: z.ZodString;
139
+ }, z.core.$strip>, z.ZodObject<{
140
+ enabled: z.ZodDefault<z.ZodBoolean>;
141
+ headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
142
+ routing: z.ZodOptional<z.ZodObject<{
143
+ critical: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
144
+ digest: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
145
+ info: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
146
+ regression: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
147
+ warning: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
148
+ }, z.core.$strip>>;
149
+ type: z.ZodLiteral<"webhook">;
150
+ url: z.ZodString;
151
+ }, z.core.$strip>], "type">;
152
+ /** Inferred TypeScript type for a single sink config. */
153
+ export type SinkConfig = z.infer<typeof SinkConfigSchema>;
154
+ /** Schema for the complete config/sinks.yaml file. */
155
+ export declare const SinksFileSchema: z.ZodObject<{
156
+ sinks: z.ZodDefault<z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
157
+ credentials: z.ZodOptional<z.ZodString>;
158
+ dataset: z.ZodString;
159
+ enabled: z.ZodDefault<z.ZodBoolean>;
160
+ project: z.ZodString;
161
+ routing: z.ZodOptional<z.ZodObject<{
162
+ critical: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
163
+ digest: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
164
+ info: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
165
+ regression: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
166
+ warning: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
167
+ }, z.core.$strip>>;
168
+ type: z.ZodLiteral<"bigquery">;
169
+ }, z.core.$strip>, z.ZodObject<{
170
+ enabled: z.ZodDefault<z.ZodBoolean>;
171
+ routing: z.ZodOptional<z.ZodObject<{
172
+ critical: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
173
+ digest: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
174
+ info: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
175
+ regression: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
176
+ warning: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
177
+ }, z.core.$strip>>;
178
+ token: z.ZodOptional<z.ZodString>;
179
+ type: z.ZodLiteral<"github-comment">;
180
+ }, z.core.$strip>, z.ZodObject<{
181
+ channel: z.ZodOptional<z.ZodString>;
182
+ enabled: z.ZodDefault<z.ZodBoolean>;
183
+ routing: z.ZodOptional<z.ZodObject<{
184
+ critical: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
185
+ digest: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
186
+ info: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
187
+ regression: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
188
+ warning: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
189
+ }, z.core.$strip>>;
190
+ type: z.ZodLiteral<"slack">;
191
+ webhookUrl: z.ZodString;
192
+ }, z.core.$strip>, z.ZodObject<{
193
+ enabled: z.ZodDefault<z.ZodBoolean>;
194
+ headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
195
+ routing: z.ZodOptional<z.ZodObject<{
196
+ critical: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
197
+ digest: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
198
+ info: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
199
+ regression: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
200
+ warning: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
201
+ }, z.core.$strip>>;
202
+ type: z.ZodLiteral<"webhook">;
203
+ url: z.ZodString;
204
+ }, z.core.$strip>], "type">>>;
205
+ }, z.core.$strip>;
206
+ /** Inferred TypeScript type for the full sinks config file. */
207
+ export type SinksFile = z.infer<typeof SinksFileSchema>;