@sanity/ailf 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (530) hide show
  1. package/README.md +89 -0
  2. package/bin/ailf.js +64 -0
  3. package/canonical/grader-references/README.md +88 -0
  4. package/canonical/grader-references/groq.yaml +234 -0
  5. package/canonical/grader-references/studio-setup.yaml +275 -0
  6. package/canonical/reference-solutions/.gitkeep +1 -0
  7. package/canonical/reference-solutions/frameworks/nuxt.ts +119 -0
  8. package/canonical/reference-solutions/frameworks/remix.tsx +100 -0
  9. package/canonical/reference-solutions/functions/publish-webhook.ts +60 -0
  10. package/canonical/reference-solutions/groq/advanced-filtering.ts +379 -0
  11. package/canonical/reference-solutions/groq/blog-queries.ts +137 -0
  12. package/canonical/reference-solutions/groq/joins-references.ts +300 -0
  13. package/canonical/reference-solutions/nextjs/app-router-integration.tsx +128 -0
  14. package/canonical/reference-solutions/studio-setup/blog-schema.ts +143 -0
  15. package/canonical/reference-solutions/studio-setup/custom-tool.tsx +78 -0
  16. package/canonical/reference-solutions/visual-editing/live-preview.tsx +137 -0
  17. package/canonical/reference-solutions/visual-editing/presentation-nextjs.tsx +130 -0
  18. package/config/airbyte/ai_literacy_framework.connector.yaml +639 -0
  19. package/config/bigquery/README.md +74 -0
  20. package/config/bigquery/views/area_scores.sql +87 -0
  21. package/config/bigquery/views/reports.sql +49 -0
  22. package/config/features.yaml +116 -0
  23. package/config/models.yaml +115 -0
  24. package/config/prompts.yaml +75 -0
  25. package/config/rubrics.yaml +62 -0
  26. package/config/schedules.yaml +43 -0
  27. package/config/sinks.yaml +54 -0
  28. package/config/sources.yaml +51 -0
  29. package/config/thresholds.yaml +49 -0
  30. package/dist/_vendor/ailf-core/examples/index.d.ts +190 -0
  31. package/dist/_vendor/ailf-core/examples/index.js +285 -0
  32. package/dist/_vendor/ailf-core/index.d.ts +17 -0
  33. package/dist/_vendor/ailf-core/index.js +17 -0
  34. package/dist/_vendor/ailf-core/ports/cache-store.d.ts +72 -0
  35. package/dist/_vendor/ailf-core/ports/cache-store.js +17 -0
  36. package/dist/_vendor/ailf-core/ports/config-source.d.ts +33 -0
  37. package/dist/_vendor/ailf-core/ports/config-source.js +15 -0
  38. package/dist/_vendor/ailf-core/ports/context.d.ts +172 -0
  39. package/dist/_vendor/ailf-core/ports/context.js +14 -0
  40. package/dist/_vendor/ailf-core/ports/doc-fetcher.d.ts +131 -0
  41. package/dist/_vendor/ailf-core/ports/doc-fetcher.js +12 -0
  42. package/dist/_vendor/ailf-core/ports/eval-runner.d.ts +24 -0
  43. package/dist/_vendor/ailf-core/ports/eval-runner.js +8 -0
  44. package/dist/_vendor/ailf-core/ports/index.d.ts +15 -0
  45. package/dist/_vendor/ailf-core/ports/index.js +7 -0
  46. package/dist/_vendor/ailf-core/ports/logger.d.ts +36 -0
  47. package/dist/_vendor/ailf-core/ports/logger.js +11 -0
  48. package/dist/_vendor/ailf-core/ports/pipeline-step.d.ts +46 -0
  49. package/dist/_vendor/ailf-core/ports/pipeline-step.js +8 -0
  50. package/dist/_vendor/ailf-core/ports/task-source.d.ts +159 -0
  51. package/dist/_vendor/ailf-core/ports/task-source.js +72 -0
  52. package/dist/_vendor/ailf-core/schemas/callback-payload.d.ts +24 -0
  53. package/dist/_vendor/ailf-core/schemas/callback-payload.js +29 -0
  54. package/dist/_vendor/ailf-core/schemas/eval-config.d.ts +55 -0
  55. package/dist/_vendor/ailf-core/schemas/eval-config.js +78 -0
  56. package/dist/_vendor/ailf-core/schemas/index.d.ts +16 -0
  57. package/dist/_vendor/ailf-core/schemas/index.js +16 -0
  58. package/dist/_vendor/ailf-core/schemas/pipeline-request.d.ts +125 -0
  59. package/dist/_vendor/ailf-core/schemas/pipeline-request.js +67 -0
  60. package/dist/_vendor/ailf-core/schemas/pipeline.d.ts +531 -0
  61. package/dist/_vendor/ailf-core/schemas/pipeline.js +318 -0
  62. package/dist/_vendor/ailf-core/schemas/schedules.d.ts +68 -0
  63. package/dist/_vendor/ailf-core/schemas/schedules.js +74 -0
  64. package/dist/_vendor/ailf-core/schemas/sinks.d.ts +207 -0
  65. package/dist/_vendor/ailf-core/schemas/sinks.js +108 -0
  66. package/dist/_vendor/ailf-core/services/comparison-formatters.d.ts +18 -0
  67. package/dist/_vendor/ailf-core/services/comparison-formatters.js +189 -0
  68. package/dist/_vendor/ailf-core/services/config-helpers.d.ts +41 -0
  69. package/dist/_vendor/ailf-core/services/config-helpers.js +86 -0
  70. package/dist/_vendor/ailf-core/services/index.d.ts +12 -0
  71. package/dist/_vendor/ailf-core/services/index.js +12 -0
  72. package/dist/_vendor/ailf-core/services/scoring.d.ts +49 -0
  73. package/dist/_vendor/ailf-core/services/scoring.js +222 -0
  74. package/dist/_vendor/ailf-core/types/index.d.ts +1082 -0
  75. package/dist/_vendor/ailf-core/types/index.js +21 -0
  76. package/dist/_vendor/ailf-core/types/scoring-input.d.ts +54 -0
  77. package/dist/_vendor/ailf-core/types/scoring-input.js +9 -0
  78. package/dist/_vendor/ailf-shared/dimension-names.d.ts +21 -0
  79. package/dist/_vendor/ailf-shared/dimension-names.js +27 -0
  80. package/dist/_vendor/ailf-shared/document-ref.d.ts +29 -0
  81. package/dist/_vendor/ailf-shared/document-ref.js +1 -0
  82. package/dist/_vendor/ailf-shared/eval-modes.d.ts +12 -0
  83. package/dist/_vendor/ailf-shared/eval-modes.js +8 -0
  84. package/dist/_vendor/ailf-shared/index.d.ts +16 -0
  85. package/dist/_vendor/ailf-shared/index.js +16 -0
  86. package/dist/_vendor/ailf-shared/noise-threshold.d.ts +9 -0
  87. package/dist/_vendor/ailf-shared/noise-threshold.js +9 -0
  88. package/dist/_vendor/ailf-shared/score-grades.d.ts +17 -0
  89. package/dist/_vendor/ailf-shared/score-grades.js +23 -0
  90. package/dist/adapters/cache/content-lake-cache.d.ts +24 -0
  91. package/dist/adapters/cache/content-lake-cache.js +59 -0
  92. package/dist/adapters/cache/filesystem-cache.d.ts +18 -0
  93. package/dist/adapters/cache/filesystem-cache.js +54 -0
  94. package/dist/adapters/cache/index.d.ts +2 -0
  95. package/dist/adapters/cache/index.js +2 -0
  96. package/dist/adapters/config-sources/cli-config-adapter.d.ts +17 -0
  97. package/dist/adapters/config-sources/cli-config-adapter.js +23 -0
  98. package/dist/adapters/config-sources/file-config-adapter.d.ts +26 -0
  99. package/dist/adapters/config-sources/file-config-adapter.js +96 -0
  100. package/dist/adapters/config-sources/index.d.ts +2 -0
  101. package/dist/adapters/config-sources/index.js +2 -0
  102. package/dist/adapters/doc-fetchers/index.d.ts +1 -0
  103. package/dist/adapters/doc-fetchers/index.js +1 -0
  104. package/dist/adapters/doc-fetchers/sanity-doc-fetcher.d.ts +76 -0
  105. package/dist/adapters/doc-fetchers/sanity-doc-fetcher.js +620 -0
  106. package/dist/adapters/eval-runners/index.d.ts +1 -0
  107. package/dist/adapters/eval-runners/index.js +1 -0
  108. package/dist/adapters/eval-runners/promptfoo-eval-adapter.d.ts +14 -0
  109. package/dist/adapters/eval-runners/promptfoo-eval-adapter.js +63 -0
  110. package/dist/adapters/index.d.ts +12 -0
  111. package/dist/adapters/index.js +12 -0
  112. package/dist/adapters/loggers/console-logger.d.ts +22 -0
  113. package/dist/adapters/loggers/console-logger.js +54 -0
  114. package/dist/adapters/loggers/index.d.ts +9 -0
  115. package/dist/adapters/loggers/index.js +9 -0
  116. package/dist/adapters/loggers/json-logger.d.ts +18 -0
  117. package/dist/adapters/loggers/json-logger.js +33 -0
  118. package/dist/adapters/loggers/quiet-logger.d.ts +16 -0
  119. package/dist/adapters/loggers/quiet-logger.js +30 -0
  120. package/dist/adapters/task-sources/composite-task-source.d.ts +20 -0
  121. package/dist/adapters/task-sources/composite-task-source.js +59 -0
  122. package/dist/adapters/task-sources/content-lake-task-source.d.ts +20 -0
  123. package/dist/adapters/task-sources/content-lake-task-source.js +219 -0
  124. package/dist/adapters/task-sources/index.d.ts +7 -0
  125. package/dist/adapters/task-sources/index.js +7 -0
  126. package/dist/adapters/task-sources/repo-schemas.d.ts +245 -0
  127. package/dist/adapters/task-sources/repo-schemas.js +234 -0
  128. package/dist/adapters/task-sources/repo-task-source.d.ts +22 -0
  129. package/dist/adapters/task-sources/repo-task-source.js +104 -0
  130. package/dist/adapters/task-sources/repo-trigger.d.ts +52 -0
  131. package/dist/adapters/task-sources/repo-trigger.js +153 -0
  132. package/dist/adapters/task-sources/repo-validation.d.ts +49 -0
  133. package/dist/adapters/task-sources/repo-validation.js +164 -0
  134. package/dist/adapters/task-sources/yaml-task-source.d.ts +18 -0
  135. package/dist/adapters/task-sources/yaml-task-source.js +136 -0
  136. package/dist/agent-observer/agentic-provider.d.ts +132 -0
  137. package/dist/agent-observer/agentic-provider.js +983 -0
  138. package/dist/agent-observer/classifier.d.ts +62 -0
  139. package/dist/agent-observer/classifier.js +269 -0
  140. package/dist/agent-observer/index.d.ts +7 -0
  141. package/dist/agent-observer/index.js +4 -0
  142. package/dist/agent-observer/pricing.d.ts +35 -0
  143. package/dist/agent-observer/pricing.js +82 -0
  144. package/dist/agent-observer/provider.d.ts +77 -0
  145. package/dist/agent-observer/provider.js +151 -0
  146. package/dist/agent-observer/proxy.d.ts +91 -0
  147. package/dist/agent-observer/proxy.js +321 -0
  148. package/dist/agent-observer/test-imports.d.ts +7 -0
  149. package/dist/agent-observer/test-imports.js +185 -0
  150. package/dist/agent-observer/types.d.ts +137 -0
  151. package/dist/agent-observer/types.js +16 -0
  152. package/dist/assertions/source-isolation.d.ts +72 -0
  153. package/dist/assertions/source-isolation.js +117 -0
  154. package/dist/cli.d.ts +24 -0
  155. package/dist/cli.js +199 -0
  156. package/dist/commands/agent-report.d.ts +5 -0
  157. package/dist/commands/agent-report.js +69 -0
  158. package/dist/commands/baseline.d.ts +9 -0
  159. package/dist/commands/baseline.js +141 -0
  160. package/dist/commands/cache.d.ts +13 -0
  161. package/dist/commands/cache.js +135 -0
  162. package/dist/commands/calculate-scores.d.ts +8 -0
  163. package/dist/commands/calculate-scores.js +48 -0
  164. package/dist/commands/compare.d.ts +8 -0
  165. package/dist/commands/compare.js +120 -0
  166. package/dist/commands/completion.d.ts +18 -0
  167. package/dist/commands/completion.js +260 -0
  168. package/dist/commands/coverage-audit.d.ts +7 -0
  169. package/dist/commands/coverage-audit.js +40 -0
  170. package/dist/commands/discovery-report.d.ts +10 -0
  171. package/dist/commands/discovery-report.js +44 -0
  172. package/dist/commands/eval.d.ts +9 -0
  173. package/dist/commands/eval.js +35 -0
  174. package/dist/commands/explain-handler.d.ts +34 -0
  175. package/dist/commands/explain-handler.js +719 -0
  176. package/dist/commands/fetch-docs.d.ts +8 -0
  177. package/dist/commands/fetch-docs.js +128 -0
  178. package/dist/commands/generate-configs.d.ts +8 -0
  179. package/dist/commands/generate-configs.js +46 -0
  180. package/dist/commands/grader/index.d.ts +11 -0
  181. package/dist/commands/grader/index.js +118 -0
  182. package/dist/commands/init.d.ts +19 -0
  183. package/dist/commands/init.js +150 -0
  184. package/dist/commands/interactive.d.ts +12 -0
  185. package/dist/commands/interactive.js +238 -0
  186. package/dist/commands/lookup-doc.d.ts +15 -0
  187. package/dist/commands/lookup-doc.js +84 -0
  188. package/dist/commands/measure-retrieval.d.ts +5 -0
  189. package/dist/commands/measure-retrieval.js +65 -0
  190. package/dist/commands/pipeline-action.d.ts +71 -0
  191. package/dist/commands/pipeline-action.js +305 -0
  192. package/dist/commands/pipeline.d.ts +62 -0
  193. package/dist/commands/pipeline.js +53 -0
  194. package/dist/commands/pr-comment.d.ts +8 -0
  195. package/dist/commands/pr-comment.js +47 -0
  196. package/dist/commands/publish.d.ts +26 -0
  197. package/dist/commands/publish.js +253 -0
  198. package/dist/commands/readiness-report.d.ts +10 -0
  199. package/dist/commands/readiness-report.js +104 -0
  200. package/dist/commands/shared/options.d.ts +29 -0
  201. package/dist/commands/shared/options.js +57 -0
  202. package/dist/commands/update-quality-scores.d.ts +5 -0
  203. package/dist/commands/update-quality-scores.js +20 -0
  204. package/dist/commands/validate-tasks.d.ts +16 -0
  205. package/dist/commands/validate-tasks.js +93 -0
  206. package/dist/commands/validate.d.ts +9 -0
  207. package/dist/commands/validate.js +73 -0
  208. package/dist/commands/webhook-server.d.ts +5 -0
  209. package/dist/commands/webhook-server.js +30 -0
  210. package/dist/commands/weekly-digest.d.ts +10 -0
  211. package/dist/commands/weekly-digest.js +104 -0
  212. package/dist/composition-root.d.ts +26 -0
  213. package/dist/composition-root.js +107 -0
  214. package/dist/interpolate.d.ts +26 -0
  215. package/dist/interpolate.js +70 -0
  216. package/dist/job-store.d.ts +104 -0
  217. package/dist/job-store.js +188 -0
  218. package/dist/lib/agent-behavior-report.d.ts +8 -0
  219. package/dist/lib/agent-behavior-report.js +185 -0
  220. package/dist/lib/baseline.d.ts +19 -0
  221. package/dist/lib/baseline.js +153 -0
  222. package/dist/lib/calculate-scores.d.ts +23 -0
  223. package/dist/lib/calculate-scores.js +42 -0
  224. package/dist/lib/compare.d.ts +18 -0
  225. package/dist/lib/compare.js +170 -0
  226. package/dist/lib/coverage-audit.d.ts +4 -0
  227. package/dist/lib/coverage-audit.js +42 -0
  228. package/dist/lib/discovery-report.d.ts +13 -0
  229. package/dist/lib/discovery-report.js +57 -0
  230. package/dist/lib/fetch-docs.d.ts +30 -0
  231. package/dist/lib/fetch-docs.js +171 -0
  232. package/dist/lib/generate-configs.d.ts +25 -0
  233. package/dist/lib/generate-configs.js +42 -0
  234. package/dist/lib/grader-api.d.ts +21 -0
  235. package/dist/lib/grader-api.js +34 -0
  236. package/dist/lib/grader-compare.d.ts +19 -0
  237. package/dist/lib/grader-compare.js +91 -0
  238. package/dist/lib/grader-consistency.d.ts +27 -0
  239. package/dist/lib/grader-consistency.js +79 -0
  240. package/dist/lib/grader-sensitivity.d.ts +19 -0
  241. package/dist/lib/grader-sensitivity.js +75 -0
  242. package/dist/lib/grader-validate.d.ts +19 -0
  243. package/dist/lib/grader-validate.js +78 -0
  244. package/dist/lib/measure-retrieval.d.ts +14 -0
  245. package/dist/lib/measure-retrieval.js +71 -0
  246. package/dist/lib/pr-comment.d.ts +16 -0
  247. package/dist/lib/pr-comment.js +28 -0
  248. package/dist/lib/readiness-report.d.ts +13 -0
  249. package/dist/lib/readiness-report.js +108 -0
  250. package/dist/lib/webhook-server.d.ts +11 -0
  251. package/dist/lib/webhook-server.js +24 -0
  252. package/dist/lib/weekly-digest.d.ts +24 -0
  253. package/dist/lib/weekly-digest.js +148 -0
  254. package/dist/orchestration/build-app-context.d.ts +27 -0
  255. package/dist/orchestration/build-app-context.js +81 -0
  256. package/dist/orchestration/build-step-sequence.d.ts +15 -0
  257. package/dist/orchestration/build-step-sequence.js +84 -0
  258. package/dist/orchestration/config-to-source-overrides.d.ts +9 -0
  259. package/dist/orchestration/config-to-source-overrides.js +28 -0
  260. package/dist/orchestration/env-bridge.d.ts +21 -0
  261. package/dist/orchestration/env-bridge.js +66 -0
  262. package/dist/orchestration/index.d.ts +11 -0
  263. package/dist/orchestration/index.js +11 -0
  264. package/dist/orchestration/pipeline-orchestrator.d.ts +24 -0
  265. package/dist/orchestration/pipeline-orchestrator.js +153 -0
  266. package/dist/orchestration/step-runner.d.ts +20 -0
  267. package/dist/orchestration/step-runner.js +88 -0
  268. package/dist/orchestration/steps/calculate-scores-step.d.ts +13 -0
  269. package/dist/orchestration/steps/calculate-scores-step.js +95 -0
  270. package/dist/orchestration/steps/callback-step.d.ts +24 -0
  271. package/dist/orchestration/steps/callback-step.js +76 -0
  272. package/dist/orchestration/steps/compare-step.d.ts +14 -0
  273. package/dist/orchestration/steps/compare-step.js +92 -0
  274. package/dist/orchestration/steps/discovery-report-step.d.ts +13 -0
  275. package/dist/orchestration/steps/discovery-report-step.js +55 -0
  276. package/dist/orchestration/steps/fetch-docs-shell.d.ts +17 -0
  277. package/dist/orchestration/steps/fetch-docs-shell.js +30 -0
  278. package/dist/orchestration/steps/fetch-docs-step.d.ts +14 -0
  279. package/dist/orchestration/steps/fetch-docs-step.js +135 -0
  280. package/dist/orchestration/steps/gap-analysis-step.d.ts +16 -0
  281. package/dist/orchestration/steps/gap-analysis-step.js +136 -0
  282. package/dist/orchestration/steps/generate-configs-step.d.ts +14 -0
  283. package/dist/orchestration/steps/generate-configs-step.js +85 -0
  284. package/dist/orchestration/steps/grader-consistency-step.d.ts +13 -0
  285. package/dist/orchestration/steps/grader-consistency-step.js +64 -0
  286. package/dist/orchestration/steps/index.d.ts +19 -0
  287. package/dist/orchestration/steps/index.js +19 -0
  288. package/dist/orchestration/steps/mirror-repo-tasks-step.d.ts +21 -0
  289. package/dist/orchestration/steps/mirror-repo-tasks-step.js +94 -0
  290. package/dist/orchestration/steps/publish-report-step.d.ts +26 -0
  291. package/dist/orchestration/steps/publish-report-step.js +216 -0
  292. package/dist/orchestration/steps/readiness-step.d.ts +13 -0
  293. package/dist/orchestration/steps/readiness-step.js +91 -0
  294. package/dist/orchestration/steps/report-step.d.ts +12 -0
  295. package/dist/orchestration/steps/report-step.js +49 -0
  296. package/dist/orchestration/steps/run-eval-step.d.ts +17 -0
  297. package/dist/orchestration/steps/run-eval-step.js +195 -0
  298. package/dist/orchestration/steps/validate-step.d.ts +12 -0
  299. package/dist/orchestration/steps/validate-step.js +41 -0
  300. package/dist/pipeline/agent-behavior-report.d.ts +53 -0
  301. package/dist/pipeline/agent-behavior-report.js +132 -0
  302. package/dist/pipeline/attribution.d.ts +47 -0
  303. package/dist/pipeline/attribution.js +226 -0
  304. package/dist/pipeline/baseline.d.ts +37 -0
  305. package/dist/pipeline/baseline.js +141 -0
  306. package/dist/pipeline/cache.d.ts +101 -0
  307. package/dist/pipeline/cache.js +283 -0
  308. package/dist/pipeline/calculate-scores.d.ts +102 -0
  309. package/dist/pipeline/calculate-scores.js +1128 -0
  310. package/dist/pipeline/callback-delivery.d.ts +50 -0
  311. package/dist/pipeline/callback-delivery.js +89 -0
  312. package/dist/pipeline/checks.d.ts +39 -0
  313. package/dist/pipeline/checks.js +280 -0
  314. package/dist/pipeline/classify-url.d.ts +61 -0
  315. package/dist/pipeline/classify-url.js +93 -0
  316. package/dist/pipeline/compare.d.ts +31 -0
  317. package/dist/pipeline/compare.js +208 -0
  318. package/dist/pipeline/coverage-audit.d.ts +39 -0
  319. package/dist/pipeline/coverage-audit.js +165 -0
  320. package/dist/pipeline/degradations.d.ts +85 -0
  321. package/dist/pipeline/degradations.js +242 -0
  322. package/dist/pipeline/discovery-report.d.ts +55 -0
  323. package/dist/pipeline/discovery-report.js +178 -0
  324. package/dist/pipeline/eval-constants.d.ts +68 -0
  325. package/dist/pipeline/eval-constants.js +111 -0
  326. package/dist/pipeline/eval-fingerprint.d.ts +66 -0
  327. package/dist/pipeline/eval-fingerprint.js +175 -0
  328. package/dist/pipeline/expand-tasks.d.ts +220 -0
  329. package/dist/pipeline/expand-tasks.js +421 -0
  330. package/dist/pipeline/failure-modes.d.ts +46 -0
  331. package/dist/pipeline/failure-modes.js +348 -0
  332. package/dist/pipeline/fetch-url-content.d.ts +44 -0
  333. package/dist/pipeline/fetch-url-content.js +93 -0
  334. package/dist/pipeline/gap-analysis.d.ts +48 -0
  335. package/dist/pipeline/gap-analysis.js +231 -0
  336. package/dist/pipeline/generate-configs.d.ts +72 -0
  337. package/dist/pipeline/generate-configs.js +395 -0
  338. package/dist/pipeline/grader-api.d.ts +49 -0
  339. package/dist/pipeline/grader-api.js +200 -0
  340. package/dist/pipeline/grader-compare-runner.d.ts +44 -0
  341. package/dist/pipeline/grader-compare-runner.js +301 -0
  342. package/dist/pipeline/grader-comparison.d.ts +111 -0
  343. package/dist/pipeline/grader-comparison.js +161 -0
  344. package/dist/pipeline/grader-consistency-runner.d.ts +60 -0
  345. package/dist/pipeline/grader-consistency-runner.js +270 -0
  346. package/dist/pipeline/grader-consistency.d.ts +103 -0
  347. package/dist/pipeline/grader-consistency.js +146 -0
  348. package/dist/pipeline/grader-sensitivity-runner.d.ts +40 -0
  349. package/dist/pipeline/grader-sensitivity-runner.js +282 -0
  350. package/dist/pipeline/grader-sensitivity.d.ts +94 -0
  351. package/dist/pipeline/grader-sensitivity.js +144 -0
  352. package/dist/pipeline/grader-validate-runner.d.ts +38 -0
  353. package/dist/pipeline/grader-validate-runner.js +229 -0
  354. package/dist/pipeline/grader-validation.d.ts +107 -0
  355. package/dist/pipeline/grader-validation.js +169 -0
  356. package/dist/pipeline/map-request-to-config.d.ts +19 -0
  357. package/dist/pipeline/map-request-to-config.js +80 -0
  358. package/dist/pipeline/measure-retrieval.d.ts +59 -0
  359. package/dist/pipeline/measure-retrieval.js +111 -0
  360. package/dist/pipeline/mirror-repo-tasks.d.ts +86 -0
  361. package/dist/pipeline/mirror-repo-tasks.js +350 -0
  362. package/dist/pipeline/plan-format.d.ts +33 -0
  363. package/dist/pipeline/plan-format.js +202 -0
  364. package/dist/pipeline/plan.d.ts +169 -0
  365. package/dist/pipeline/plan.js +708 -0
  366. package/dist/pipeline/pr-comment.d.ts +19 -0
  367. package/dist/pipeline/pr-comment.js +502 -0
  368. package/dist/pipeline/probe.d.ts +52 -0
  369. package/dist/pipeline/probe.js +390 -0
  370. package/dist/pipeline/provenance.d.ts +47 -0
  371. package/dist/pipeline/provenance.js +146 -0
  372. package/dist/pipeline/readiness-report.d.ts +87 -0
  373. package/dist/pipeline/readiness-report.js +205 -0
  374. package/dist/pipeline/release-classification.d.ts +54 -0
  375. package/dist/pipeline/release-classification.js +238 -0
  376. package/dist/pipeline/release-report.d.ts +37 -0
  377. package/dist/pipeline/release-report.js +222 -0
  378. package/dist/pipeline/repo-eval-comment.d.ts +37 -0
  379. package/dist/pipeline/repo-eval-comment.js +165 -0
  380. package/dist/pipeline/repo-threshold-evaluator.d.ts +89 -0
  381. package/dist/pipeline/repo-threshold-evaluator.js +162 -0
  382. package/dist/pipeline/resolve-mappings.d.ts +35 -0
  383. package/dist/pipeline/resolve-mappings.js +72 -0
  384. package/dist/pipeline/retrieval-metrics.d.ts +39 -0
  385. package/dist/pipeline/retrieval-metrics.js +136 -0
  386. package/dist/pipeline/reverse-mapping.d.ts +67 -0
  387. package/dist/pipeline/reverse-mapping.js +88 -0
  388. package/dist/pipeline/schemas.d.ts +9 -0
  389. package/dist/pipeline/schemas.js +9 -0
  390. package/dist/pipeline/steps/calculate-scores-step.d.ts +11 -0
  391. package/dist/pipeline/steps/calculate-scores-step.js +89 -0
  392. package/dist/pipeline/steps/compare-step.d.ts +18 -0
  393. package/dist/pipeline/steps/compare-step.js +90 -0
  394. package/dist/pipeline/steps/eval-step.d.ts +53 -0
  395. package/dist/pipeline/steps/eval-step.js +347 -0
  396. package/dist/pipeline/steps/fetch-docs-step.d.ts +11 -0
  397. package/dist/pipeline/steps/fetch-docs-step.js +84 -0
  398. package/dist/pipeline/steps/generate-configs-step.d.ts +11 -0
  399. package/dist/pipeline/steps/generate-configs-step.js +98 -0
  400. package/dist/pipeline/steps/grader-consistency-step.d.ts +21 -0
  401. package/dist/pipeline/steps/grader-consistency-step.js +74 -0
  402. package/dist/pipeline/steps/publish-report-step.d.ts +57 -0
  403. package/dist/pipeline/steps/publish-report-step.js +243 -0
  404. package/dist/pipeline/steps/report-step.d.ts +13 -0
  405. package/dist/pipeline/steps/report-step.js +56 -0
  406. package/dist/pipeline/steps/update-scores-step.d.ts +11 -0
  407. package/dist/pipeline/steps/update-scores-step.js +42 -0
  408. package/dist/pipeline/targeted-loo.d.ts +88 -0
  409. package/dist/pipeline/targeted-loo.js +203 -0
  410. package/dist/pipeline/thresholds.d.ts +27 -0
  411. package/dist/pipeline/thresholds.js +245 -0
  412. package/dist/pipeline/types.d.ts +10 -0
  413. package/dist/pipeline/types.js +10 -0
  414. package/dist/pipeline/validate.d.ts +67 -0
  415. package/dist/pipeline/validate.js +406 -0
  416. package/dist/pipeline/webhook-server.d.ts +37 -0
  417. package/dist/pipeline/webhook-server.js +133 -0
  418. package/dist/report-store.d.ts +84 -0
  419. package/dist/report-store.js +208 -0
  420. package/dist/sanity/client.d.ts +38 -0
  421. package/dist/sanity/client.js +86 -0
  422. package/dist/sanity/portable-text.d.ts +11 -0
  423. package/dist/sanity/portable-text.js +211 -0
  424. package/dist/sanity/queries.d.ts +133 -0
  425. package/dist/sanity/queries.js +300 -0
  426. package/dist/schedules/digest.d.ts +116 -0
  427. package/dist/schedules/digest.js +156 -0
  428. package/dist/schedules/index.d.ts +12 -0
  429. package/dist/schedules/index.js +10 -0
  430. package/dist/schedules/loader.d.ts +31 -0
  431. package/dist/schedules/loader.js +73 -0
  432. package/dist/schedules/schema.d.ts +9 -0
  433. package/dist/schedules/schema.js +9 -0
  434. package/dist/scripts/agent-behavior-report.d.ts +19 -0
  435. package/dist/scripts/agent-behavior-report.js +315 -0
  436. package/dist/scripts/baseline.d.ts +43 -0
  437. package/dist/scripts/baseline.js +267 -0
  438. package/dist/scripts/calculate-scores.d.ts +166 -0
  439. package/dist/scripts/calculate-scores.js +1296 -0
  440. package/dist/scripts/compare.d.ts +22 -0
  441. package/dist/scripts/compare.js +334 -0
  442. package/dist/scripts/coverage-audit.d.ts +44 -0
  443. package/dist/scripts/coverage-audit.js +209 -0
  444. package/dist/scripts/debug-eval.d.ts +19 -0
  445. package/dist/scripts/debug-eval.js +73 -0
  446. package/dist/scripts/discovery-report.d.ts +58 -0
  447. package/dist/scripts/discovery-report.js +250 -0
  448. package/dist/scripts/fetch-docs.d.ts +35 -0
  449. package/dist/scripts/fetch-docs.js +472 -0
  450. package/dist/scripts/generate-configs.d.ts +66 -0
  451. package/dist/scripts/generate-configs.js +459 -0
  452. package/dist/scripts/grader-api.d.ts +27 -0
  453. package/dist/scripts/grader-api.js +206 -0
  454. package/dist/scripts/grader-compare.d.ts +22 -0
  455. package/dist/scripts/grader-compare.js +368 -0
  456. package/dist/scripts/grader-consistency.d.ts +20 -0
  457. package/dist/scripts/grader-consistency.js +313 -0
  458. package/dist/scripts/grader-sensitivity.d.ts +22 -0
  459. package/dist/scripts/grader-sensitivity.js +354 -0
  460. package/dist/scripts/grader-validate.d.ts +19 -0
  461. package/dist/scripts/grader-validate.js +267 -0
  462. package/dist/scripts/measure-retrieval.d.ts +10 -0
  463. package/dist/scripts/measure-retrieval.js +145 -0
  464. package/dist/scripts/migrate-tasks-to-content-lake.d.ts +24 -0
  465. package/dist/scripts/migrate-tasks-to-content-lake.js +327 -0
  466. package/dist/scripts/pipeline.d.ts +76 -0
  467. package/dist/scripts/pipeline.js +1031 -0
  468. package/dist/scripts/pr-comment.d.ts +10 -0
  469. package/dist/scripts/pr-comment.js +510 -0
  470. package/dist/scripts/readiness-report.d.ts +88 -0
  471. package/dist/scripts/readiness-report.js +342 -0
  472. package/dist/scripts/update-quality-scores.d.ts +15 -0
  473. package/dist/scripts/update-quality-scores.js +184 -0
  474. package/dist/scripts/validate-task-sources.d.ts +21 -0
  475. package/dist/scripts/validate-task-sources.js +210 -0
  476. package/dist/scripts/validate.d.ts +13 -0
  477. package/dist/scripts/validate.js +79 -0
  478. package/dist/scripts/webhook-server.d.ts +26 -0
  479. package/dist/scripts/webhook-server.js +147 -0
  480. package/dist/scripts/weekly-digest.d.ts +24 -0
  481. package/dist/scripts/weekly-digest.js +144 -0
  482. package/dist/sinks/bigquery/index.d.ts +131 -0
  483. package/dist/sinks/bigquery/index.js +222 -0
  484. package/dist/sinks/format-slack.d.ts +64 -0
  485. package/dist/sinks/format-slack.js +306 -0
  486. package/dist/sinks/index.d.ts +23 -0
  487. package/dist/sinks/index.js +18 -0
  488. package/dist/sinks/loader.d.ts +18 -0
  489. package/dist/sinks/loader.js +82 -0
  490. package/dist/sinks/retry.d.ts +24 -0
  491. package/dist/sinks/retry.js +52 -0
  492. package/dist/sinks/schema.d.ts +9 -0
  493. package/dist/sinks/schema.js +9 -0
  494. package/dist/sinks/slack/format.d.ts +65 -0
  495. package/dist/sinks/slack/format.js +327 -0
  496. package/dist/sinks/slack/index.d.ts +27 -0
  497. package/dist/sinks/slack/index.js +78 -0
  498. package/dist/sinks/slack-sink.d.ts +27 -0
  499. package/dist/sinks/slack-sink.js +78 -0
  500. package/dist/sinks/types.d.ts +59 -0
  501. package/dist/sinks/types.js +44 -0
  502. package/dist/sinks/webhook/index.d.ts +19 -0
  503. package/dist/sinks/webhook/index.js +50 -0
  504. package/dist/sinks/webhook-sink.d.ts +19 -0
  505. package/dist/sinks/webhook-sink.js +50 -0
  506. package/dist/sources.d.ts +104 -0
  507. package/dist/sources.js +292 -0
  508. package/dist/webhook/budget.d.ts +42 -0
  509. package/dist/webhook/budget.js +60 -0
  510. package/dist/webhook/debounce.d.ts +67 -0
  511. package/dist/webhook/debounce.js +76 -0
  512. package/dist/webhook/dispatch.d.ts +45 -0
  513. package/dist/webhook/dispatch.js +84 -0
  514. package/dist/webhook/eval-request-handler.d.ts +87 -0
  515. package/dist/webhook/eval-request-handler.js +181 -0
  516. package/dist/webhook/handler.d.ts +88 -0
  517. package/dist/webhook/handler.js +203 -0
  518. package/dist/webhook/index.d.ts +17 -0
  519. package/dist/webhook/index.js +12 -0
  520. package/dist/webhook/types.d.ts +109 -0
  521. package/dist/webhook/types.js +10 -0
  522. package/package.json +72 -0
  523. package/tasks/.expanded.agentic.yaml +51 -0
  524. package/tasks/.expanded.yaml +66 -0
  525. package/tasks/frameworks.yaml +98 -0
  526. package/tasks/functions.yaml +51 -0
  527. package/tasks/groq.yaml +216 -0
  528. package/tasks/nextjs-live.yaml +62 -0
  529. package/tasks/studio-setup.yaml +111 -0
  530. package/tasks/visual-editing.yaml +120 -0
@@ -0,0 +1,21 @@
1
+ /**
2
+ * @sanity/ailf-core — Domain types
3
+ *
4
+ * Core types for the AI Literacy Framework evaluation pipeline.
5
+ * These define the contracts between pipeline steps, domain services,
6
+ * and port adapters.
7
+ *
8
+ * Extracted from packages/eval/src/pipeline/types.ts during the
9
+ * Ports & Adapters migration (Phase 0c). The original file is now a
10
+ * re-export barrel that preserves backward compatibility.
11
+ */
12
+ export { FULL_MODE_SUBMODES } from "../../ailf-shared/index.js";
13
+ // ---------------------------------------------------------------------------
14
+ // Comparison (Approach 2: structured comparison output)
15
+ // ---------------------------------------------------------------------------
16
+ /**
17
+ * Default noise threshold — re-exported from @sanity/ailf-shared.
18
+ * Named DEFAULT_NOISE_THRESHOLD here for backward compatibility with eval code
19
+ * that imports it from @sanity/ailf-core.
20
+ */
21
+ export { NOISE_THRESHOLD as DEFAULT_NOISE_THRESHOLD } from "../../ailf-shared/index.js";
@@ -0,0 +1,54 @@
1
+ /**
2
+ * Scoring input types — normalized evaluation result shapes consumed by
3
+ * pure scoring functions in core/services/scoring.ts.
4
+ *
5
+ * These represent the domain model for evaluation results after
6
+ * Promptfoo-specific normalization. The raw Promptfoo shapes
7
+ * (RawTestResult, RawPromptfooFile) remain in packages/eval.
8
+ */
9
+ /** A single grading component result from the evaluation harness */
10
+ export interface ComponentResult {
11
+ assertion?: {
12
+ type: string;
13
+ value?: string;
14
+ /** Structured metadata propagated from rubric templates (Approach 5). */
15
+ metadata?: Record<string, unknown>;
16
+ };
17
+ pass: boolean;
18
+ reason?: string;
19
+ score?: number;
20
+ }
21
+ /** Normalized test result used by scoring functions */
22
+ export interface TestResult {
23
+ cost: number;
24
+ description: string;
25
+ gradingResult: {
26
+ componentResults: ComponentResult[];
27
+ pass: boolean;
28
+ };
29
+ metadata?: Record<string, unknown>;
30
+ /** Provider identifier (e.g., "openai:gpt-4o") */
31
+ providerId?: string;
32
+ /** Provider label (e.g., "GPT-4o") */
33
+ providerLabel?: string;
34
+ response: {
35
+ output: string;
36
+ };
37
+ vars: Record<string, string>;
38
+ }
39
+ /** Extracted URL metadata from evaluation results */
40
+ export interface UrlMetadata {
41
+ otherUrls: string[];
42
+ sanityUrlCount: number;
43
+ sanityUrls: string[];
44
+ totalUrlCount: number;
45
+ }
46
+ /** Agentic scoring entry — per-area actual scores from tool-based retrieval */
47
+ export interface ActualScoreEntry {
48
+ actualScore: number;
49
+ codeCorrectness: number;
50
+ docCoverage: number;
51
+ taskCompletion: number;
52
+ testCount: number;
53
+ totalCost: number;
54
+ }
@@ -0,0 +1,9 @@
1
+ /**
2
+ * Scoring input types — normalized evaluation result shapes consumed by
3
+ * pure scoring functions in core/services/scoring.ts.
4
+ *
5
+ * These represent the domain model for evaluation results after
6
+ * Promptfoo-specific normalization. The raw Promptfoo shapes
7
+ * (RawTestResult, RawPromptfooFile) remain in packages/eval.
8
+ */
9
+ export {};
@@ -0,0 +1,21 @@
1
+ /**
2
+ * Grading dimension types and labels.
3
+ *
4
+ * Shared between eval (scoring engine) and studio (display components).
5
+ * Adding a dimension here ensures both packages show it correctly.
6
+ */
7
+ /** The three grading dimensions */
8
+ export type DimensionName = "codeCorrectness" | "docCoverage" | "taskCompletion";
9
+ /**
10
+ * Human-readable display labels for dimensions.
11
+ *
12
+ * Note: The Sanity Content Lake stores dimension names in kebab-case
13
+ * (e.g., "task-completion") while TypeScript uses camelCase. Both
14
+ * mappings are provided for cross-system compatibility.
15
+ */
16
+ export declare const DIMENSION_LABELS: Record<DimensionName, string>;
17
+ /**
18
+ * Kebab-case dimension labels as stored in Sanity Content Lake.
19
+ * Used by the studio package when rendering judgment data from GROQ.
20
+ */
21
+ export declare const DIMENSION_LABELS_KEBAB: Record<string, string>;
@@ -0,0 +1,27 @@
1
+ /**
2
+ * Grading dimension types and labels.
3
+ *
4
+ * Shared between eval (scoring engine) and studio (display components).
5
+ * Adding a dimension here ensures both packages show it correctly.
6
+ */
7
+ /**
8
+ * Human-readable display labels for dimensions.
9
+ *
10
+ * Note: The Sanity Content Lake stores dimension names in kebab-case
11
+ * (e.g., "task-completion") while TypeScript uses camelCase. Both
12
+ * mappings are provided for cross-system compatibility.
13
+ */
14
+ export const DIMENSION_LABELS = {
15
+ codeCorrectness: "Code Correctness",
16
+ docCoverage: "Doc Coverage",
17
+ taskCompletion: "Task Completion",
18
+ };
19
+ /**
20
+ * Kebab-case dimension labels as stored in Sanity Content Lake.
21
+ * Used by the studio package when rendering judgment data from GROQ.
22
+ */
23
+ export const DIMENSION_LABELS_KEBAB = {
24
+ "code-correctness": "Code Correctness",
25
+ "doc-coverage": "Doc Coverage",
26
+ "task-completion": "Task Completion",
27
+ };
@@ -0,0 +1,29 @@
1
+ /**
2
+ * A reference to a Sanity document used during evaluation.
3
+ *
4
+ * Shared between eval (producer) and studio (consumer) packages.
5
+ * The Sanity Content Lake's `ailf.report` schema is the enforcement
6
+ * point — both sides must agree on this shape.
7
+ *
8
+ * Attachable at every level of the report hierarchy:
9
+ * - ScoreSummary.documentManifest — all docs used in the evaluation
10
+ * - FeatureScore.documents — docs used for a specific area
11
+ * - StoredJudgment.canonicalDocs — docs expected for a specific task
12
+ */
13
+ export interface DocumentRef {
14
+ /**
15
+ * Sanity document ID.
16
+ * Named `documentId` (not `_id`) because Sanity Content Lake reserves
17
+ * underscore-prefixed field names for system fields.
18
+ */
19
+ documentId: string;
20
+ /**
21
+ * Content revision at evaluation time (enables staleness detection).
22
+ * Named `revision` (not `_rev`) for the same Sanity reserved-name reason.
23
+ */
24
+ revision?: string;
25
+ /** URL-path identifier (e.g., "groq-introduction") */
26
+ slug: string;
27
+ /** Human-readable document title */
28
+ title: string;
29
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,12 @@
1
+ /**
2
+ * Evaluation mode types.
3
+ *
4
+ * Shared between eval (pipeline orchestrator) and studio (GROQ filters, UI).
5
+ * A new mode added here is immediately visible to both packages.
6
+ */
7
+ /** The evaluation modes supported by the pipeline */
8
+ export type EvalMode = "agentic" | "baseline" | "full" | "observed";
9
+ /** Concrete modes that map 1:1 to a Promptfoo config file */
10
+ export type ConcreteEvalMode = Exclude<EvalMode, "full">;
11
+ /** The sub-modes that compose a `full` evaluation run */
12
+ export declare const FULL_MODE_SUBMODES: ConcreteEvalMode[];
@@ -0,0 +1,8 @@
1
+ /**
2
+ * Evaluation mode types.
3
+ *
4
+ * Shared between eval (pipeline orchestrator) and studio (GROQ filters, UI).
5
+ * A new mode added here is immediately visible to both packages.
6
+ */
7
+ /** The sub-modes that compose a `full` evaluation run */
8
+ export const FULL_MODE_SUBMODES = ["baseline", "agentic"];
@@ -0,0 +1,16 @@
1
+ /**
2
+ * @sanity/ailf-shared — Cross-package contract types and constants.
3
+ *
4
+ * This package contains types and constants that both the eval and studio
5
+ * packages must agree on. It replaces convention-only sharing where both
6
+ * packages independently defined the same types.
7
+ *
8
+ * Design rule: this package has ZERO runtime dependencies and ZERO imports
9
+ * from @sanity/ailf-core, @sanity/ailf, or
10
+ * @sanity/ailf-studio. It is the leaf of the dependency graph.
11
+ */
12
+ export * from "./document-ref.js";
13
+ export * from "./score-grades.js";
14
+ export * from "./noise-threshold.js";
15
+ export * from "./eval-modes.js";
16
+ export * from "./dimension-names.js";
@@ -0,0 +1,16 @@
1
+ /**
2
+ * @sanity/ailf-shared — Cross-package contract types and constants.
3
+ *
4
+ * This package contains types and constants that both the eval and studio
5
+ * packages must agree on. It replaces convention-only sharing where both
6
+ * packages independently defined the same types.
7
+ *
8
+ * Design rule: this package has ZERO runtime dependencies and ZERO imports
9
+ * from @sanity/ailf-core, @sanity/ailf, or
10
+ * @sanity/ailf-studio. It is the leaf of the dependency graph.
11
+ */
12
+ export * from "./document-ref.js";
13
+ export * from "./score-grades.js";
14
+ export * from "./noise-threshold.js";
15
+ export * from "./eval-modes.js";
16
+ export * from "./dimension-names.js";
@@ -0,0 +1,9 @@
1
+ /**
2
+ * Default noise threshold for score comparison.
3
+ *
4
+ * Deltas within ±NOISE_THRESHOLD are classified as "unchanged."
5
+ * Can be overridden by --threshold flag or empirical grader consistency data.
6
+ *
7
+ * Shared between eval (comparison engine) and studio (comparison view).
8
+ */
9
+ export declare const NOISE_THRESHOLD = 2;
@@ -0,0 +1,9 @@
1
+ /**
2
+ * Default noise threshold for score comparison.
3
+ *
4
+ * Deltas within ±NOISE_THRESHOLD are classified as "unchanged."
5
+ * Can be overridden by --threshold flag or empirical grader consistency data.
6
+ *
7
+ * Shared between eval (comparison engine) and studio (comparison view).
8
+ */
9
+ export const NOISE_THRESHOLD = 2;
@@ -0,0 +1,17 @@
1
+ /**
2
+ * Score grade boundaries and classification.
3
+ *
4
+ * The single source of truth for the 80 / 70 / 50 threshold boundaries.
5
+ * Both the eval pipeline (scoring) and studio dashboard (display) must
6
+ * agree on these boundaries — a change here updates both.
7
+ */
8
+ /** Score grade boundaries — the single source of truth. */
9
+ export declare const GRADE_BOUNDARIES: {
10
+ readonly good: 80;
11
+ readonly warning: 70;
12
+ readonly critical: 50;
13
+ };
14
+ /** Score grade classification */
15
+ export type ScoreGrade = "critical" | "good" | "needs-work" | "warning";
16
+ /** Classify a numeric score into a grade. */
17
+ export declare function scoreGrade(score: number): ScoreGrade;
@@ -0,0 +1,23 @@
1
+ /**
2
+ * Score grade boundaries and classification.
3
+ *
4
+ * The single source of truth for the 80 / 70 / 50 threshold boundaries.
5
+ * Both the eval pipeline (scoring) and studio dashboard (display) must
6
+ * agree on these boundaries — a change here updates both.
7
+ */
8
+ /** Score grade boundaries — the single source of truth. */
9
+ export const GRADE_BOUNDARIES = {
10
+ good: 80,
11
+ warning: 70,
12
+ critical: 50,
13
+ };
14
+ /** Classify a numeric score into a grade. */
15
+ export function scoreGrade(score) {
16
+ if (score >= GRADE_BOUNDARIES.good)
17
+ return "good";
18
+ if (score >= GRADE_BOUNDARIES.warning)
19
+ return "warning";
20
+ if (score >= GRADE_BOUNDARIES.critical)
21
+ return "needs-work";
22
+ return "critical";
23
+ }
@@ -0,0 +1,24 @@
1
+ /**
2
+ * ContentLakeCacheAdapter — decorator that adds remote fingerprint lookup
3
+ * on local cache miss.
4
+ *
5
+ * On local miss, queries the Content Lake for a report with a matching
6
+ * eval fingerprint. If found, the eval + calculate-scores steps can be
7
+ * skipped entirely (the cached score-summary.json is reused).
8
+ *
9
+ * This decorator wraps any CacheStore (typically FilesystemCache) and
10
+ * only intercepts `lookup()` — all other operations delegate directly.
11
+ *
12
+ * @see docs/design-docs/content-lake-eval-caching.md
13
+ */
14
+ import type { CacheKey, CacheLookupResult, CacheRecordInput, CacheStore } from "../../_vendor/ailf-core/index.d.ts";
15
+ import type { ReportStore } from "../../report-store.js";
16
+ export declare class ContentLakeCacheAdapter implements CacheStore {
17
+ private readonly local;
18
+ private readonly reportStore;
19
+ constructor(local: CacheStore, reportStore: ReportStore);
20
+ computeKey(inputPaths: string[], context?: string[]): Promise<CacheKey>;
21
+ lookup(step: string, key: CacheKey): Promise<CacheLookupResult>;
22
+ record(step: string, key: CacheKey, metadata: CacheRecordInput): Promise<void>;
23
+ clear(): Promise<void>;
24
+ }
@@ -0,0 +1,59 @@
1
+ /**
2
+ * ContentLakeCacheAdapter — decorator that adds remote fingerprint lookup
3
+ * on local cache miss.
4
+ *
5
+ * On local miss, queries the Content Lake for a report with a matching
6
+ * eval fingerprint. If found, the eval + calculate-scores steps can be
7
+ * skipped entirely (the cached score-summary.json is reused).
8
+ *
9
+ * This decorator wraps any CacheStore (typically FilesystemCache) and
10
+ * only intercepts `lookup()` — all other operations delegate directly.
11
+ *
12
+ * @see docs/design-docs/content-lake-eval-caching.md
13
+ */
14
+ export class ContentLakeCacheAdapter {
15
+ local;
16
+ reportStore;
17
+ constructor(local, reportStore) {
18
+ this.local = local;
19
+ this.reportStore = reportStore;
20
+ }
21
+ async computeKey(inputPaths, context) {
22
+ return this.local.computeKey(inputPaths, context);
23
+ }
24
+ async lookup(step, key) {
25
+ // Try local first
26
+ const local = await this.local.lookup(step, key);
27
+ if (local.hit)
28
+ return local;
29
+ // Try remote — only for eval steps (the expensive ones)
30
+ if (!step.startsWith("eval")) {
31
+ return { hit: false };
32
+ }
33
+ try {
34
+ const report = await this.reportStore.findByFingerprint(key);
35
+ if (report) {
36
+ return {
37
+ entry: {
38
+ durationMs: 0,
39
+ outputPaths: [],
40
+ summary: `Remote cache hit — reusing report from ${report.completedAt ?? "unknown date"}`,
41
+ timestamp: new Date().toISOString(),
42
+ },
43
+ hit: true,
44
+ };
45
+ }
46
+ }
47
+ catch {
48
+ // Remote lookup failure is non-fatal — fall through to miss
49
+ }
50
+ return { hit: false };
51
+ }
52
+ async record(step, key, metadata) {
53
+ // Only record locally — remote recording happens via the publish step
54
+ await this.local.record(step, key, metadata);
55
+ }
56
+ async clear() {
57
+ await this.local.clear();
58
+ }
59
+ }
@@ -0,0 +1,18 @@
1
+ /**
2
+ * FilesystemCache — wraps the existing pipeline/cache.ts behind the CacheStore port.
3
+ *
4
+ * This adapter delegates to the battle-tested hashing, manifest read/write,
5
+ * and output-existence checks in pipeline/cache.ts. It adds no new logic —
6
+ * just adapts the function-based API to the CacheStore interface.
7
+ *
8
+ * @see packages/eval/src/pipeline/cache.ts — underlying implementation
9
+ */
10
+ import type { CacheKey, CacheLookupResult, CacheRecordInput, CacheStore } from "../../_vendor/ailf-core/index.d.ts";
11
+ export declare class FilesystemCache implements CacheStore {
12
+ private readonly rootDir;
13
+ constructor(rootDir: string);
14
+ computeKey(inputPaths: string[], context?: string[]): Promise<CacheKey>;
15
+ lookup(step: string, key: CacheKey): Promise<CacheLookupResult>;
16
+ record(step: string, key: CacheKey, metadata: CacheRecordInput): Promise<void>;
17
+ clear(): Promise<void>;
18
+ }
@@ -0,0 +1,54 @@
1
+ /**
2
+ * FilesystemCache — wraps the existing pipeline/cache.ts behind the CacheStore port.
3
+ *
4
+ * This adapter delegates to the battle-tested hashing, manifest read/write,
5
+ * and output-existence checks in pipeline/cache.ts. It adds no new logic —
6
+ * just adapts the function-based API to the CacheStore interface.
7
+ *
8
+ * @see packages/eval/src/pipeline/cache.ts — underlying implementation
9
+ */
10
+ import { existsSync, rmSync } from "fs";
11
+ import { resolve } from "path";
12
+ import { hashFiles, readManifest, recordCache } from "../../pipeline/cache.js";
13
+ export class FilesystemCache {
14
+ rootDir;
15
+ constructor(rootDir) {
16
+ this.rootDir = rootDir;
17
+ }
18
+ async computeKey(inputPaths, context) {
19
+ return hashFiles(inputPaths, context);
20
+ }
21
+ async lookup(step, key) {
22
+ const manifest = readManifest(this.rootDir, step);
23
+ if (!manifest?.entry) {
24
+ return { hit: false };
25
+ }
26
+ // Hash must match
27
+ if (manifest.entry.inputHash !== key) {
28
+ return { hit: false };
29
+ }
30
+ // All output files must still exist (guard against manual deletion)
31
+ const outputsExist = manifest.entry.outputPaths.every((p) => existsSync(resolve(this.rootDir, p)));
32
+ if (!outputsExist) {
33
+ return { hit: false };
34
+ }
35
+ return {
36
+ entry: {
37
+ durationMs: manifest.entry.durationMs,
38
+ outputPaths: manifest.entry.outputPaths,
39
+ summary: manifest.entry.summary,
40
+ timestamp: manifest.entry.timestamp,
41
+ },
42
+ hit: true,
43
+ };
44
+ }
45
+ async record(step, key, metadata) {
46
+ recordCache(this.rootDir, step, key, metadata.summary, metadata.durationMs, metadata.outputPaths);
47
+ }
48
+ async clear() {
49
+ const cacheDir = resolve(this.rootDir, "results", "cache");
50
+ if (existsSync(cacheDir)) {
51
+ rmSync(cacheDir, { recursive: true, force: true });
52
+ }
53
+ }
54
+ }
@@ -0,0 +1,2 @@
1
+ export { ContentLakeCacheAdapter } from "./content-lake-cache.js";
2
+ export { FilesystemCache } from "./filesystem-cache.js";
@@ -0,0 +1,2 @@
1
+ export { ContentLakeCacheAdapter } from "./content-lake-cache.js";
2
+ export { FilesystemCache } from "./filesystem-cache.js";
@@ -0,0 +1,17 @@
1
+ /**
2
+ * CliConfigAdapter — resolves pipeline config from Commander CLI flags.
3
+ *
4
+ * This is the default adapter — it wraps the existing option resolution
5
+ * pipeline: PipelineCliOptions → computeResolvedOptions → mapToResolvedConfig.
6
+ *
7
+ * @see packages/eval/src/commands/pipeline-action.ts — underlying implementation
8
+ */
9
+ import type { ConfigSource, ResolvedConfig } from "../../_vendor/ailf-core/index.d.ts";
10
+ import type { PipelineCliOptions } from "../../commands/pipeline.js";
11
+ export declare class CliConfigAdapter implements ConfigSource {
12
+ private readonly cliOpts;
13
+ private readonly rootDir;
14
+ readonly label = "CLI flags";
15
+ constructor(cliOpts: PipelineCliOptions, rootDir: string);
16
+ resolve(): Promise<ResolvedConfig>;
17
+ }
@@ -0,0 +1,23 @@
1
+ /**
2
+ * CliConfigAdapter — resolves pipeline config from Commander CLI flags.
3
+ *
4
+ * This is the default adapter — it wraps the existing option resolution
5
+ * pipeline: PipelineCliOptions → computeResolvedOptions → mapToResolvedConfig.
6
+ *
7
+ * @see packages/eval/src/commands/pipeline-action.ts — underlying implementation
8
+ */
9
+ import { computeResolvedOptions } from "../../commands/pipeline-action.js";
10
+ import { mapToResolvedConfig } from "../../orchestration/build-app-context.js";
11
+ export class CliConfigAdapter {
12
+ cliOpts;
13
+ rootDir;
14
+ label = "CLI flags";
15
+ constructor(cliOpts, rootDir) {
16
+ this.cliOpts = cliOpts;
17
+ this.rootDir = rootDir;
18
+ }
19
+ async resolve() {
20
+ const resolved = computeResolvedOptions(this.cliOpts);
21
+ return mapToResolvedConfig(resolved, this.rootDir);
22
+ }
23
+ }
@@ -0,0 +1,26 @@
1
+ /**
2
+ * FileConfigAdapter — resolves pipeline config from a local JSON/YAML file.
3
+ *
4
+ * Enables `ailf pipeline --config config.json` to load all pipeline
5
+ * options from a file instead of CLI flags. Supports two schemas:
6
+ *
7
+ * 1. PipelineRequestSchema (JSON) — the universal API contract.
8
+ * Used by: API gateway payloads, GitHub Actions dispatch.
9
+ * Tried first for .json files.
10
+ *
11
+ * 2. EvalConfigSchema (JSON/YAML) — the developer-facing config.
12
+ * Used by: local config files, manual --config usage.
13
+ * Fallback for .json files; primary for .yaml/.yml files.
14
+ *
15
+ * @see packages/core/src/schemas/pipeline-request.ts — PipelineRequestSchema
16
+ * @see packages/core/src/schemas/eval-config.ts — EvalConfigSchema
17
+ */
18
+ import { type ResolvedConfig } from "../../_vendor/ailf-core/index.d.ts";
19
+ import type { ConfigSource } from "../../_vendor/ailf-core/index.d.ts";
20
+ export declare class FileConfigAdapter implements ConfigSource {
21
+ private readonly filePath;
22
+ private readonly rootDir;
23
+ readonly label: string;
24
+ constructor(filePath: string, rootDir: string);
25
+ resolve(): Promise<ResolvedConfig>;
26
+ }
@@ -0,0 +1,96 @@
1
+ /**
2
+ * FileConfigAdapter — resolves pipeline config from a local JSON/YAML file.
3
+ *
4
+ * Enables `ailf pipeline --config config.json` to load all pipeline
5
+ * options from a file instead of CLI flags. Supports two schemas:
6
+ *
7
+ * 1. PipelineRequestSchema (JSON) — the universal API contract.
8
+ * Used by: API gateway payloads, GitHub Actions dispatch.
9
+ * Tried first for .json files.
10
+ *
11
+ * 2. EvalConfigSchema (JSON/YAML) — the developer-facing config.
12
+ * Used by: local config files, manual --config usage.
13
+ * Fallback for .json files; primary for .yaml/.yml files.
14
+ *
15
+ * @see packages/core/src/schemas/pipeline-request.ts — PipelineRequestSchema
16
+ * @see packages/core/src/schemas/eval-config.ts — EvalConfigSchema
17
+ */
18
+ import { readFileSync } from "fs";
19
+ import { extname } from "path";
20
+ import { EvalConfigSchema, PipelineRequestSchema, } from "../../_vendor/ailf-core/index.js";
21
+ import { mapRequestToConfig } from "../../pipeline/map-request-to-config.js";
22
+ export class FileConfigAdapter {
23
+ filePath;
24
+ rootDir;
25
+ label;
26
+ constructor(filePath, rootDir) {
27
+ this.filePath = filePath;
28
+ this.rootDir = rootDir;
29
+ this.label = `config file: ${filePath}`;
30
+ }
31
+ async resolve() {
32
+ const raw = readConfigFile(this.filePath);
33
+ const ext = extname(this.filePath).toLowerCase();
34
+ // For JSON files, try PipelineRequestSchema first (API contract)
35
+ if (ext === ".json") {
36
+ const pipelineResult = PipelineRequestSchema.safeParse(raw);
37
+ if (pipelineResult.success) {
38
+ return mapRequestToConfig(pipelineResult.data, this.rootDir);
39
+ }
40
+ }
41
+ // Fall back to EvalConfigSchema (developer config)
42
+ const validated = EvalConfigSchema.parse(raw);
43
+ return mapEvalConfigToResolvedConfig(validated, this.rootDir);
44
+ }
45
+ }
46
+ // ---------------------------------------------------------------------------
47
+ // Helpers
48
+ // ---------------------------------------------------------------------------
49
+ function readConfigFile(filePath) {
50
+ const content = readFileSync(filePath, "utf-8");
51
+ const ext = extname(filePath).toLowerCase();
52
+ if (ext === ".yaml" || ext === ".yml") {
53
+ // Dynamic import for yaml parser — only needed when reading YAML configs
54
+ // eslint-disable-next-line @typescript-eslint/no-require-imports
55
+ const { parse } = require("yaml");
56
+ return parse(content);
57
+ }
58
+ return JSON.parse(content);
59
+ }
60
+ function mapEvalConfigToResolvedConfig(config, rootDir) {
61
+ // Normalize debug option
62
+ let debug;
63
+ if (config.debug === true) {
64
+ debug = { enabled: true };
65
+ }
66
+ else if (typeof config.debug === "object") {
67
+ debug = { enabled: config.debug.enabled ?? true, ...config.debug };
68
+ }
69
+ return {
70
+ rootDir,
71
+ mode: config.mode ?? "full",
72
+ debug,
73
+ areas: config.areas,
74
+ tasks: config.tasks,
75
+ changedDocs: config.changedDocs,
76
+ source: config.source,
77
+ skipFetch: config.skipFetch ?? false,
78
+ skipEval: config.skipEval ?? false,
79
+ compareEnabled: config.compare ?? false,
80
+ compareThreshold: config.compareThreshold,
81
+ compareBaseline: config.compareBaseline,
82
+ gapAnalysisEnabled: config.gapAnalysis ?? true,
83
+ readinessEnabled: config.readiness ?? false,
84
+ discoveryReportEnabled: config.discoveryReport ?? false,
85
+ publishEnabled: config.publish ?? false,
86
+ publishTag: config.publishTag,
87
+ noCache: config.noCache ?? false,
88
+ noRemoteCache: config.noRemoteCache ?? false,
89
+ graderReplications: config.graderReplications,
90
+ urls: config.urls,
91
+ headers: config.headers,
92
+ allowedOrigins: config.allowedOrigins,
93
+ searchMode: config.searchMode ?? "open",
94
+ concurrency: config.concurrency,
95
+ };
96
+ }
@@ -0,0 +1,2 @@
1
+ export { CliConfigAdapter } from "./cli-config-adapter.js";
2
+ export { FileConfigAdapter } from "./file-config-adapter.js";
@@ -0,0 +1,2 @@
1
+ export { CliConfigAdapter } from "./cli-config-adapter.js";
2
+ export { FileConfigAdapter } from "./file-config-adapter.js";
@@ -0,0 +1 @@
1
+ export { SanityDocFetcher } from "./sanity-doc-fetcher.js";
@@ -0,0 +1 @@
1
+ export { SanityDocFetcher } from "./sanity-doc-fetcher.js";