@sanity/ailf 2.0.0 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (442) hide show
  1. package/canonical/grader-references/README.md +2 -2
  2. package/canonical/reference-solutions/content-lake/mutations.ts +160 -0
  3. package/canonical/reference-solutions/content-lake/realtime.ts +187 -0
  4. package/canonical/reference-solutions/image-handling/asset-pipeline.tsx +166 -0
  5. package/canonical/reference-solutions/portable-text/custom-blocks.ts +204 -0
  6. package/canonical/reference-solutions/portable-text/rendering.tsx +163 -0
  7. package/config/features.ts +1 -1
  8. package/config/models.ts +28 -23
  9. package/config/sources.ts +1 -1
  10. package/config/thresholds.ts +1 -1
  11. package/dist/_vendor/ailf-core/__tests__/comparison-formatters.test.d.ts +10 -0
  12. package/dist/_vendor/ailf-core/__tests__/comparison-formatters.test.js +185 -0
  13. package/dist/_vendor/ailf-core/artifact-capture/__tests__/noop-collector.test.d.ts +6 -0
  14. package/dist/_vendor/ailf-core/artifact-capture/__tests__/noop-collector.test.js +42 -0
  15. package/dist/_vendor/ailf-core/artifact-capture/noop-collector.d.ts +14 -0
  16. package/dist/_vendor/ailf-core/artifact-capture/noop-collector.js +25 -0
  17. package/dist/_vendor/ailf-core/config-helpers.d.ts +6 -0
  18. package/dist/_vendor/ailf-core/config-helpers.js +29 -0
  19. package/dist/_vendor/ailf-core/examples/index.d.ts +164 -94
  20. package/dist/_vendor/ailf-core/examples/index.js +208 -114
  21. package/dist/_vendor/ailf-core/index.d.ts +1 -0
  22. package/dist/_vendor/ailf-core/index.js +1 -0
  23. package/dist/_vendor/ailf-core/ports/artifact-collector.d.ts +94 -0
  24. package/dist/_vendor/ailf-core/ports/artifact-collector.js +13 -0
  25. package/dist/_vendor/ailf-core/ports/capture-comparator.d.ts +138 -0
  26. package/dist/_vendor/ailf-core/ports/capture-comparator.js +10 -0
  27. package/dist/_vendor/ailf-core/ports/context.d.ts +20 -1
  28. package/dist/_vendor/ailf-core/ports/eval-runner.d.ts +6 -0
  29. package/dist/_vendor/ailf-core/ports/index.d.ts +2 -0
  30. package/dist/_vendor/ailf-core/ports/pipeline-step.d.ts +11 -0
  31. package/dist/_vendor/ailf-core/ports/task-source.d.ts +3 -3
  32. package/dist/_vendor/ailf-core/ports/task-source.js +3 -3
  33. package/dist/_vendor/ailf-core/schemas/pipeline-request.d.ts +6 -1
  34. package/dist/_vendor/ailf-core/schemas/pipeline-request.js +14 -2
  35. package/dist/_vendor/ailf-core/services/config-helpers.d.ts +16 -1
  36. package/dist/_vendor/ailf-core/services/config-helpers.js +21 -0
  37. package/dist/_vendor/ailf-core/services/index.d.ts +1 -1
  38. package/dist/_vendor/ailf-core/services/index.js +1 -1
  39. package/dist/_vendor/ailf-core/services/scoring.js +9 -0
  40. package/dist/_vendor/ailf-core/types/generalized-task.d.ts +12 -1
  41. package/dist/_vendor/ailf-core/types/generalized-task.js +1 -1
  42. package/dist/_vendor/ailf-core/types/index.d.ts +47 -4
  43. package/dist/_vendor/ailf-core/types/plugin-registry.d.ts +27 -0
  44. package/dist/_vendor/ailf-shared/eval-modes.d.ts +15 -0
  45. package/dist/_vendor/ailf-shared/eval-modes.js +18 -0
  46. package/dist/_vendor/ailf-tasks/cli.d.ts +8 -0
  47. package/dist/_vendor/ailf-tasks/cli.js +61 -0
  48. package/dist/_vendor/ailf-tasks/index.d.ts +13 -0
  49. package/dist/_vendor/ailf-tasks/index.js +16 -0
  50. package/dist/_vendor/ailf-tasks/parser.d.ts +27 -0
  51. package/dist/_vendor/ailf-tasks/parser.js +73 -0
  52. package/dist/_vendor/ailf-tasks/schemas.d.ts +198 -0
  53. package/dist/_vendor/ailf-tasks/schemas.js +180 -0
  54. package/dist/_vendor/ailf-tasks/validation.d.ts +47 -0
  55. package/dist/_vendor/ailf-tasks/validation.js +162 -0
  56. package/dist/adapters/api-client/remediation.js +2 -2
  57. package/dist/adapters/config-sources/file-config-adapter.js +6 -1
  58. package/dist/adapters/eval-runners/promptfoo-eval-adapter.js +8 -2
  59. package/dist/adapters/index.d.ts +0 -1
  60. package/dist/adapters/index.js +0 -1
  61. package/dist/adapters/task-sources/composite-task-source.d.ts +1 -1
  62. package/dist/adapters/task-sources/composite-task-source.js +1 -1
  63. package/dist/adapters/task-sources/content-lake-task-source.d.ts +4 -6
  64. package/dist/adapters/task-sources/content-lake-task-source.js +4 -6
  65. package/dist/adapters/task-sources/index.d.ts +1 -2
  66. package/dist/adapters/task-sources/index.js +1 -2
  67. package/dist/adapters/task-sources/repo-schemas.d.ts +1 -1
  68. package/dist/adapters/task-sources/repo-schemas.js +2 -2
  69. package/dist/adapters/task-sources/repo-task-source.js +1 -1
  70. package/dist/adapters/task-sources/repo-trigger.d.ts +1 -1
  71. package/dist/adapters/task-sources/repo-trigger.js +1 -1
  72. package/dist/adapters/task-sources/task-file-loader.d.ts +9 -6
  73. package/dist/adapters/task-sources/task-file-loader.js +20 -6
  74. package/dist/agent-observer/test-imports.d.ts +7 -0
  75. package/dist/agent-observer/test-imports.js +185 -0
  76. package/dist/artifact-capture/comparator.d.ts +22 -0
  77. package/dist/artifact-capture/comparator.js +493 -0
  78. package/dist/artifact-capture/filesystem-collector.d.ts +42 -0
  79. package/dist/artifact-capture/filesystem-collector.js +237 -0
  80. package/dist/artifact-capture/redact-artifact.d.ts +20 -0
  81. package/dist/artifact-capture/redact-artifact.js +115 -0
  82. package/dist/assertions/source-isolation.d.ts +1 -1
  83. package/dist/assertions/source-isolation.js +1 -1
  84. package/dist/cli.js +4 -0
  85. package/dist/commands/calculate-scores.js +1 -0
  86. package/dist/commands/capture-compare.d.ts +15 -0
  87. package/dist/commands/capture-compare.js +253 -0
  88. package/dist/commands/capture-list.d.ts +12 -0
  89. package/dist/commands/capture-list.js +147 -0
  90. package/dist/commands/capture.d.ts +9 -0
  91. package/dist/commands/capture.js +16 -0
  92. package/dist/commands/chronic-failures.d.ts +8 -0
  93. package/dist/commands/chronic-failures.js +33 -0
  94. package/dist/commands/explain-handler.d.ts +1 -1
  95. package/dist/commands/explain-handler.js +37 -8
  96. package/dist/commands/fetch-docs.js +1 -0
  97. package/dist/commands/generate-configs.d.ts +3 -3
  98. package/dist/commands/generate-configs.js +20 -8
  99. package/dist/commands/init.d.ts +2 -3
  100. package/dist/commands/init.js +56 -170
  101. package/dist/commands/pipeline-action.d.ts +7 -1
  102. package/dist/commands/pipeline-action.js +43 -19
  103. package/dist/commands/pipeline.d.ts +6 -1
  104. package/dist/commands/pipeline.js +7 -2
  105. package/dist/commands/pr-comment.js +1 -0
  106. package/dist/commands/publish.js +1 -0
  107. package/dist/commands/shared/help.js +2 -2
  108. package/dist/commands/update-quality-scores.d.ts +5 -0
  109. package/dist/commands/update-quality-scores.js +20 -0
  110. package/dist/composition-root.d.ts +2 -3
  111. package/dist/composition-root.js +27 -14
  112. package/dist/config/features.ts +23 -0
  113. package/dist/config/models.ts +100 -0
  114. package/dist/config/prompts.ts +16 -0
  115. package/dist/config/rubrics.ts +225 -0
  116. package/dist/config/schedules.ts +47 -0
  117. package/dist/config/sinks.ts +37 -0
  118. package/dist/config/sources.ts +21 -0
  119. package/dist/config/thresholds.ts +61 -0
  120. package/dist/lib/agent-behavior-report.d.ts +8 -0
  121. package/dist/lib/agent-behavior-report.js +185 -0
  122. package/dist/lib/baseline.d.ts +19 -0
  123. package/dist/lib/baseline.js +153 -0
  124. package/dist/lib/calculate-scores.d.ts +23 -0
  125. package/dist/lib/calculate-scores.js +42 -0
  126. package/dist/lib/compare.d.ts +18 -0
  127. package/dist/lib/compare.js +170 -0
  128. package/dist/lib/coverage-audit.d.ts +4 -0
  129. package/dist/lib/coverage-audit.js +42 -0
  130. package/dist/lib/discovery-report.d.ts +13 -0
  131. package/dist/lib/discovery-report.js +57 -0
  132. package/dist/lib/fetch-docs.d.ts +30 -0
  133. package/dist/lib/fetch-docs.js +171 -0
  134. package/dist/lib/generate-configs.d.ts +25 -0
  135. package/dist/lib/generate-configs.js +42 -0
  136. package/dist/lib/grader-api.d.ts +21 -0
  137. package/dist/lib/grader-api.js +34 -0
  138. package/dist/lib/grader-compare.d.ts +19 -0
  139. package/dist/lib/grader-compare.js +91 -0
  140. package/dist/lib/grader-consistency.d.ts +27 -0
  141. package/dist/lib/grader-consistency.js +79 -0
  142. package/dist/lib/grader-sensitivity.d.ts +19 -0
  143. package/dist/lib/grader-sensitivity.js +75 -0
  144. package/dist/lib/grader-validate.d.ts +19 -0
  145. package/dist/lib/grader-validate.js +78 -0
  146. package/dist/lib/measure-retrieval.d.ts +14 -0
  147. package/dist/lib/measure-retrieval.js +71 -0
  148. package/dist/lib/pr-comment.d.ts +16 -0
  149. package/dist/lib/pr-comment.js +28 -0
  150. package/dist/lib/readiness-report.d.ts +13 -0
  151. package/dist/lib/readiness-report.js +108 -0
  152. package/dist/lib/webhook-server.d.ts +11 -0
  153. package/dist/lib/webhook-server.js +24 -0
  154. package/dist/lib/weekly-digest.d.ts +24 -0
  155. package/dist/lib/weekly-digest.js +148 -0
  156. package/dist/orchestration/build-app-context.js +13 -0
  157. package/dist/orchestration/cache-context.d.ts +23 -0
  158. package/dist/orchestration/cache-context.js +43 -0
  159. package/dist/orchestration/env-bridge.d.ts +21 -0
  160. package/dist/orchestration/env-bridge.js +66 -0
  161. package/dist/orchestration/load-pipeline-tasks.d.ts +34 -0
  162. package/dist/orchestration/load-pipeline-tasks.js +52 -0
  163. package/dist/orchestration/pipeline-orchestrator.js +75 -5
  164. package/dist/orchestration/step-runner.js +5 -1
  165. package/dist/orchestration/steps/calculate-scores-step.d.ts +1 -0
  166. package/dist/orchestration/steps/calculate-scores-step.js +13 -0
  167. package/dist/orchestration/steps/callback-step.js +10 -1
  168. package/dist/orchestration/steps/compare-step.js +6 -3
  169. package/dist/orchestration/steps/discovery-report-step.js +6 -2
  170. package/dist/orchestration/steps/fetch-docs-shell.d.ts +17 -0
  171. package/dist/orchestration/steps/fetch-docs-shell.js +30 -0
  172. package/dist/orchestration/steps/fetch-docs-step.d.ts +1 -0
  173. package/dist/orchestration/steps/fetch-docs-step.js +30 -16
  174. package/dist/orchestration/steps/gap-analysis-step.js +13 -2
  175. package/dist/orchestration/steps/generate-configs-step.d.ts +1 -0
  176. package/dist/orchestration/steps/generate-configs-step.js +50 -15
  177. package/dist/orchestration/steps/mirror-repo-tasks-step.d.ts +1 -1
  178. package/dist/orchestration/steps/mirror-repo-tasks-step.js +1 -1
  179. package/dist/orchestration/steps/publish-report-step.js +19 -0
  180. package/dist/orchestration/steps/readiness-step.js +8 -3
  181. package/dist/orchestration/steps/report-step.js +17 -4
  182. package/dist/orchestration/steps/run-eval-step.d.ts +1 -0
  183. package/dist/orchestration/steps/run-eval-step.js +51 -31
  184. package/dist/pipeline/agent-behavior-report.js +6 -0
  185. package/dist/pipeline/attribution.d.ts +1 -1
  186. package/dist/pipeline/attribution.js +1 -1
  187. package/dist/pipeline/cache.js +29 -15
  188. package/dist/pipeline/calculate-scores.d.ts +2 -0
  189. package/dist/pipeline/calculate-scores.js +70 -33
  190. package/dist/pipeline/chronic-failures.d.ts +55 -0
  191. package/dist/pipeline/chronic-failures.js +110 -0
  192. package/dist/pipeline/compiler/__tests__/mcp-server-handler.test.js +33 -0
  193. package/dist/pipeline/compiler/__tests__/promptfoo-compiler.test.js +2 -3
  194. package/dist/pipeline/compiler/__tests__/task-bridge.test.d.ts +9 -0
  195. package/dist/pipeline/compiler/__tests__/task-bridge.test.js +339 -0
  196. package/dist/pipeline/compiler/__tests__/tool-loop-openai.test.d.ts +10 -0
  197. package/dist/pipeline/compiler/__tests__/tool-loop-openai.test.js +509 -0
  198. package/dist/pipeline/compiler/assertion-mapper.d.ts +1 -1
  199. package/dist/pipeline/compiler/assertion-mapper.js +1 -1
  200. package/dist/pipeline/compiler/compiler-to-yaml.d.ts +2 -7
  201. package/dist/pipeline/compiler/compiler-to-yaml.js +2 -7
  202. package/dist/pipeline/compiler/config-loader.d.ts +14 -0
  203. package/dist/pipeline/compiler/config-loader.js +42 -2
  204. package/dist/pipeline/compiler/fixture-resolver.d.ts +1 -1
  205. package/dist/pipeline/compiler/fixture-resolver.js +1 -1
  206. package/dist/pipeline/compiler/ignore-fields.d.ts +1 -1
  207. package/dist/pipeline/compiler/ignore-fields.js +1 -1
  208. package/dist/pipeline/compiler/index.d.ts +2 -5
  209. package/dist/pipeline/compiler/index.js +2 -5
  210. package/dist/pipeline/compiler/literacy-bridge.d.ts +1 -1
  211. package/dist/pipeline/compiler/literacy-bridge.js +1 -1
  212. package/dist/pipeline/compiler/mode-bases/agent-harness.d.ts +1 -1
  213. package/dist/pipeline/compiler/mode-bases/agent-harness.js +1 -1
  214. package/dist/pipeline/compiler/mode-bases/knowledge-probe.d.ts +1 -1
  215. package/dist/pipeline/compiler/mode-bases/knowledge-probe.js +1 -1
  216. package/dist/pipeline/compiler/mode-bases/literacy.d.ts +13 -2
  217. package/dist/pipeline/compiler/mode-bases/literacy.js +55 -1
  218. package/dist/pipeline/compiler/mode-bases/mcp-server.d.ts +1 -1
  219. package/dist/pipeline/compiler/mode-bases/mcp-server.js +1 -1
  220. package/dist/pipeline/compiler/mode-handlers/agent-harness/index.d.ts +1 -1
  221. package/dist/pipeline/compiler/mode-handlers/agent-harness/index.js +1 -1
  222. package/dist/pipeline/compiler/mode-handlers/agent-harness-handler.d.ts +70 -0
  223. package/dist/pipeline/compiler/mode-handlers/agent-harness-handler.js +485 -0
  224. package/dist/pipeline/compiler/mode-handlers/index.d.ts +2 -2
  225. package/dist/pipeline/compiler/mode-handlers/index.js +2 -2
  226. package/dist/pipeline/compiler/mode-handlers/knowledge-probe/index.d.ts +1 -1
  227. package/dist/pipeline/compiler/mode-handlers/knowledge-probe/index.js +1 -1
  228. package/dist/pipeline/compiler/mode-handlers/knowledge-probe-handler.d.ts +76 -0
  229. package/dist/pipeline/compiler/mode-handlers/knowledge-probe-handler.js +245 -0
  230. package/dist/pipeline/compiler/mode-handlers/literacy/index.d.ts +1 -1
  231. package/dist/pipeline/compiler/mode-handlers/literacy/index.js +1 -1
  232. package/dist/pipeline/compiler/mode-handlers/literacy-handler.d.ts +89 -0
  233. package/dist/pipeline/compiler/mode-handlers/literacy-handler.js +379 -0
  234. package/dist/pipeline/compiler/mode-handlers/mcp-assertions.d.ts +50 -0
  235. package/dist/pipeline/compiler/mode-handlers/mcp-assertions.js +334 -0
  236. package/dist/pipeline/compiler/mode-handlers/mcp-server/assertions.d.ts +1 -1
  237. package/dist/pipeline/compiler/mode-handlers/mcp-server/assertions.js +1 -1
  238. package/dist/pipeline/compiler/mode-handlers/mcp-server/index.d.ts +1 -1
  239. package/dist/pipeline/compiler/mode-handlers/mcp-server/index.js +1 -1
  240. package/dist/pipeline/compiler/mode-handlers/mcp-server/provider-config.js +4 -0
  241. package/dist/pipeline/compiler/mode-handlers/mcp-server-handler.d.ts +69 -0
  242. package/dist/pipeline/compiler/mode-handlers/mcp-server-handler.js +307 -0
  243. package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/index.js +22 -5
  244. package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/mcp-connection.js +6 -0
  245. package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/tool-loop-openai.d.ts +10 -5
  246. package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/tool-loop-openai.js +314 -7
  247. package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/types.d.ts +10 -0
  248. package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider.d.ts +65 -0
  249. package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider.js +368 -0
  250. package/dist/pipeline/compiler/presets/sanity-literacy.d.ts +1 -1
  251. package/dist/pipeline/compiler/presets/sanity-literacy.js +1 -1
  252. package/dist/pipeline/compiler/promptfoo-compiler.d.ts +1 -4
  253. package/dist/pipeline/compiler/promptfoo-compiler.js +3 -12
  254. package/dist/pipeline/compiler/provider-assembler.js +13 -7
  255. package/dist/pipeline/compiler/sandbox/docker-sandbox.d.ts +1 -1
  256. package/dist/pipeline/compiler/sandbox/docker-sandbox.js +1 -1
  257. package/dist/pipeline/compiler/sandbox/fixture-provisioner.d.ts +1 -1
  258. package/dist/pipeline/compiler/sandbox/fixture-provisioner.js +1 -1
  259. package/dist/pipeline/compiler/sandbox/git-worktree-sandbox.d.ts +1 -1
  260. package/dist/pipeline/compiler/sandbox/git-worktree-sandbox.js +1 -1
  261. package/dist/pipeline/compiler/sandbox/index.d.ts +1 -1
  262. package/dist/pipeline/compiler/sandbox/index.js +1 -1
  263. package/dist/pipeline/compiler/sandbox/sandbox-selector.d.ts +1 -1
  264. package/dist/pipeline/compiler/sandbox/sandbox-selector.js +1 -1
  265. package/dist/pipeline/compiler/sandbox/sandbox-strategy.d.ts +1 -1
  266. package/dist/pipeline/compiler/sandbox/sandbox-strategy.js +1 -1
  267. package/dist/pipeline/compiler/sandbox/tempdir-sandbox.d.ts +1 -1
  268. package/dist/pipeline/compiler/sandbox/tempdir-sandbox.js +1 -1
  269. package/dist/pipeline/compiler/scoring-bridge.d.ts +1 -1
  270. package/dist/pipeline/compiler/scoring-bridge.js +1 -1
  271. package/dist/pipeline/compiler/task-bridge.d.ts +41 -0
  272. package/dist/pipeline/compiler/task-bridge.js +92 -0
  273. package/dist/pipeline/compiler/task-graph-builder.d.ts +1 -4
  274. package/dist/pipeline/compiler/task-graph-builder.js +1 -4
  275. package/dist/pipeline/compiler/telemetry/index.d.ts +1 -1
  276. package/dist/pipeline/compiler/telemetry/index.js +1 -1
  277. package/dist/pipeline/compiler/variable-resolver.d.ts +1 -1
  278. package/dist/pipeline/compiler/variable-resolver.js +1 -1
  279. package/dist/pipeline/coverage-audit.d.ts +1 -1
  280. package/dist/pipeline/coverage-audit.js +1 -1
  281. package/dist/pipeline/degradations.d.ts +1 -1
  282. package/dist/pipeline/degradations.js +1 -1
  283. package/dist/pipeline/failure-modes.d.ts +1 -1
  284. package/dist/pipeline/failure-modes.js +13 -1
  285. package/dist/pipeline/gap-analysis.d.ts +1 -1
  286. package/dist/pipeline/gap-analysis.js +3 -1
  287. package/dist/pipeline/generate-configs.d.ts +2 -2
  288. package/dist/pipeline/generate-configs.js +15 -8
  289. package/dist/pipeline/grader-compare-runner.d.ts +1 -1
  290. package/dist/pipeline/grader-compare-runner.js +7 -1
  291. package/dist/pipeline/grader-comparison.d.ts +1 -1
  292. package/dist/pipeline/grader-comparison.js +1 -1
  293. package/dist/pipeline/grader-consistency-runner.d.ts +1 -1
  294. package/dist/pipeline/grader-consistency-runner.js +7 -1
  295. package/dist/pipeline/grader-consistency.d.ts +1 -1
  296. package/dist/pipeline/grader-consistency.js +1 -1
  297. package/dist/pipeline/grader-sensitivity-runner.d.ts +1 -1
  298. package/dist/pipeline/grader-sensitivity-runner.js +1 -1
  299. package/dist/pipeline/grader-sensitivity.d.ts +1 -1
  300. package/dist/pipeline/grader-sensitivity.js +1 -1
  301. package/dist/pipeline/grader-validate-runner.d.ts +1 -1
  302. package/dist/pipeline/grader-validate-runner.js +2 -2
  303. package/dist/pipeline/grader-validation.d.ts +1 -1
  304. package/dist/pipeline/grader-validation.js +1 -1
  305. package/dist/pipeline/map-request-to-config.js +15 -2
  306. package/dist/pipeline/mirror-repo-tasks.d.ts +1 -1
  307. package/dist/pipeline/mirror-repo-tasks.js +1 -1
  308. package/dist/pipeline/plan-format.d.ts +1 -1
  309. package/dist/pipeline/plan-format.js +1 -1
  310. package/dist/pipeline/plan.d.ts +1 -1
  311. package/dist/pipeline/plan.js +67 -29
  312. package/dist/pipeline/probe.d.ts +1 -1
  313. package/dist/pipeline/probe.js +1 -1
  314. package/dist/pipeline/readiness-report.d.ts +2 -2
  315. package/dist/pipeline/readiness-report.js +2 -2
  316. package/dist/pipeline/release-classification.d.ts +1 -1
  317. package/dist/pipeline/release-classification.js +1 -1
  318. package/dist/pipeline/release-report.d.ts +1 -1
  319. package/dist/pipeline/release-report.js +1 -1
  320. package/dist/pipeline/repo-eval-comment.d.ts +1 -1
  321. package/dist/pipeline/repo-eval-comment.js +1 -1
  322. package/dist/pipeline/repo-threshold-evaluator.d.ts +1 -1
  323. package/dist/pipeline/repo-threshold-evaluator.js +1 -1
  324. package/dist/pipeline/resolve-mappings.d.ts +6 -6
  325. package/dist/pipeline/resolve-mappings.js +44 -44
  326. package/dist/pipeline/retrieval-metrics.d.ts +3 -3
  327. package/dist/pipeline/retrieval-metrics.js +28 -20
  328. package/dist/pipeline/steps/calculate-scores-step.d.ts +11 -0
  329. package/dist/pipeline/steps/calculate-scores-step.js +89 -0
  330. package/dist/pipeline/steps/compare-step.d.ts +18 -0
  331. package/dist/pipeline/steps/compare-step.js +90 -0
  332. package/dist/pipeline/steps/eval-step.d.ts +53 -0
  333. package/dist/pipeline/steps/eval-step.js +347 -0
  334. package/dist/pipeline/steps/fetch-docs-step.d.ts +11 -0
  335. package/dist/pipeline/steps/fetch-docs-step.js +84 -0
  336. package/dist/pipeline/steps/generate-configs-step.d.ts +11 -0
  337. package/dist/pipeline/steps/generate-configs-step.js +98 -0
  338. package/dist/pipeline/steps/grader-consistency-step.d.ts +21 -0
  339. package/dist/pipeline/steps/grader-consistency-step.js +74 -0
  340. package/dist/pipeline/steps/publish-report-step.d.ts +57 -0
  341. package/dist/pipeline/steps/publish-report-step.js +243 -0
  342. package/dist/pipeline/steps/report-step.d.ts +13 -0
  343. package/dist/pipeline/steps/report-step.js +56 -0
  344. package/dist/pipeline/steps/update-scores-step.d.ts +11 -0
  345. package/dist/pipeline/steps/update-scores-step.js +42 -0
  346. package/dist/pipeline/targeted-loo.d.ts +1 -1
  347. package/dist/pipeline/targeted-loo.js +1 -1
  348. package/dist/pipeline/thresholds.d.ts +1 -1
  349. package/dist/pipeline/thresholds.js +1 -1
  350. package/dist/pipeline/validate.js +13 -0
  351. package/dist/report-store.d.ts +17 -0
  352. package/dist/report-store.js +24 -0
  353. package/dist/scripts/agent-behavior-report.d.ts +19 -0
  354. package/dist/scripts/agent-behavior-report.js +315 -0
  355. package/dist/scripts/baseline.d.ts +43 -0
  356. package/dist/scripts/baseline.js +267 -0
  357. package/dist/scripts/calculate-scores.d.ts +166 -0
  358. package/dist/scripts/calculate-scores.js +1296 -0
  359. package/dist/scripts/compare.d.ts +22 -0
  360. package/dist/scripts/compare.js +334 -0
  361. package/dist/scripts/coverage-audit.d.ts +44 -0
  362. package/dist/scripts/coverage-audit.js +209 -0
  363. package/dist/scripts/debug-eval.d.ts +19 -0
  364. package/dist/scripts/debug-eval.js +73 -0
  365. package/dist/scripts/discovery-report.d.ts +58 -0
  366. package/dist/scripts/discovery-report.js +250 -0
  367. package/dist/scripts/fetch-docs.d.ts +35 -0
  368. package/dist/scripts/fetch-docs.js +472 -0
  369. package/dist/scripts/generate-configs.d.ts +66 -0
  370. package/dist/scripts/generate-configs.js +459 -0
  371. package/dist/scripts/grader-api.d.ts +27 -0
  372. package/dist/scripts/grader-api.js +206 -0
  373. package/dist/scripts/grader-compare.d.ts +22 -0
  374. package/dist/scripts/grader-compare.js +368 -0
  375. package/dist/scripts/grader-consistency.d.ts +20 -0
  376. package/dist/scripts/grader-consistency.js +313 -0
  377. package/dist/scripts/grader-sensitivity.d.ts +22 -0
  378. package/dist/scripts/grader-sensitivity.js +354 -0
  379. package/dist/scripts/grader-validate.d.ts +19 -0
  380. package/dist/scripts/grader-validate.js +267 -0
  381. package/dist/scripts/measure-retrieval.d.ts +10 -0
  382. package/dist/scripts/measure-retrieval.js +145 -0
  383. package/dist/scripts/migrate-task-mode.d.ts +1 -1
  384. package/dist/scripts/migrate-task-mode.js +1 -1
  385. package/dist/scripts/migrate-tasks-to-content-lake.d.ts +1 -1
  386. package/dist/scripts/migrate-tasks-to-content-lake.js +1 -1
  387. package/dist/scripts/pipeline.d.ts +76 -0
  388. package/dist/scripts/pipeline.js +1031 -0
  389. package/dist/scripts/pr-comment.d.ts +10 -0
  390. package/dist/scripts/pr-comment.js +510 -0
  391. package/dist/scripts/readiness-report.d.ts +88 -0
  392. package/dist/scripts/readiness-report.js +342 -0
  393. package/dist/scripts/update-quality-scores.d.ts +15 -0
  394. package/dist/scripts/update-quality-scores.js +184 -0
  395. package/dist/scripts/validate-task-sources.d.ts +1 -1
  396. package/dist/scripts/validate-task-sources.js +1 -1
  397. package/dist/scripts/validate.d.ts +13 -0
  398. package/dist/scripts/validate.js +79 -0
  399. package/dist/scripts/webhook-server.d.ts +26 -0
  400. package/dist/scripts/webhook-server.js +147 -0
  401. package/dist/scripts/weekly-digest.d.ts +24 -0
  402. package/dist/scripts/weekly-digest.js +144 -0
  403. package/dist/sinks/format-slack.d.ts +64 -0
  404. package/dist/sinks/format-slack.js +306 -0
  405. package/dist/sinks/slack-sink.d.ts +27 -0
  406. package/dist/sinks/slack-sink.js +78 -0
  407. package/dist/sinks/types.d.ts +1 -1
  408. package/dist/sinks/types.js +1 -1
  409. package/dist/sinks/webhook-sink.d.ts +19 -0
  410. package/dist/sinks/webhook-sink.js +50 -0
  411. package/dist/tasks/knowledge-probe/define-type-api.task.ts +66 -0
  412. package/dist/tasks/knowledge-probe/groq-projections.task.ts +62 -0
  413. package/dist/tasks/literacy/content-lake.task.ts +181 -0
  414. package/dist/tasks/literacy/frameworks.task.ts +129 -0
  415. package/dist/tasks/literacy/functions.task.ts +70 -0
  416. package/dist/tasks/literacy/groq.task.ts +259 -0
  417. package/dist/tasks/literacy/image-handling.task.ts +95 -0
  418. package/dist/tasks/literacy/nextjs-live.task.ts +76 -0
  419. package/dist/tasks/literacy/portable-text.task.ts +169 -0
  420. package/dist/tasks/literacy/studio-setup.task.ts +134 -0
  421. package/dist/tasks/literacy/visual-editing.task.ts +147 -0
  422. package/package.json +24 -24
  423. package/tasks/.expanded.agentic.yaml +280 -0
  424. package/tasks/.expanded.yaml +565 -0
  425. package/tasks/knowledge-probe/define-type-api.task.ts +11 -0
  426. package/tasks/knowledge-probe/groq-projections.task.ts +3 -0
  427. package/tasks/literacy/content-lake.task.ts +181 -0
  428. package/tasks/literacy/frameworks.task.ts +1 -0
  429. package/tasks/literacy/functions.task.ts +1 -0
  430. package/tasks/literacy/groq.task.ts +1 -0
  431. package/tasks/literacy/image-handling.task.ts +95 -0
  432. package/tasks/literacy/nextjs-live.task.ts +2 -1
  433. package/tasks/literacy/portable-text.task.ts +169 -0
  434. package/tasks/literacy/studio-setup.task.ts +5 -2
  435. package/tasks/literacy/visual-editing.task.ts +1 -0
  436. package/LICENSE +0 -21
  437. package/tasks/frameworks.yaml +0 -98
  438. package/tasks/functions.yaml +0 -51
  439. package/tasks/groq.yaml +0 -216
  440. package/tasks/nextjs-live.yaml +0 -62
  441. package/tasks/studio-setup.yaml +0 -111
  442. package/tasks/visual-editing.yaml +0 -120
@@ -83,6 +83,6 @@ discrimination power across the full scale.
83
83
 
84
84
  ## Related
85
85
 
86
- - [Grader Reliability Plan](../../../docs/exec-plans/grader-reliability.md)
87
- Phase 2
86
+ - [Grader Reliability Plan](../../../docs/archive/exec-plans/grader-reliability.md)
87
+ Phase 2
88
88
  - [Rubric Templates](../../config/rubrics.yaml) — the rubrics used for grading
@@ -0,0 +1,160 @@
1
+ /**
2
+ * Reference Solution: Content Lake CRUD Operations
3
+ *
4
+ * Demonstrates:
5
+ * - Creating documents with client.create()
6
+ * - Patching documents with set, unset, inc
7
+ * - Deleting documents with client.delete()
8
+ * - Atomic transactions for multiple mutations
9
+ * - TypeScript types for document shapes
10
+ */
11
+
12
+ import { createClient, type SanityDocument } from "@sanity/client"
13
+
14
+ // === Client Setup ===
15
+
16
+ const client = createClient({
17
+ projectId: "your-project-id",
18
+ dataset: "production",
19
+ apiVersion: "2024-01-01",
20
+ token: process.env.SANITY_API_TOKEN,
21
+ useCdn: false, // Mutations require the write API, not CDN
22
+ })
23
+
24
+ // === Types ===
25
+
26
+ interface Post {
27
+ _type: "post"
28
+ title: string
29
+ slug: { _type: "slug"; current: string }
30
+ publishedAt: string
31
+ viewCount: number
32
+ author: { _type: "reference"; _ref: string }
33
+ categories?: Array<{ _type: "reference"; _ref: string }>
34
+ }
35
+
36
+ type PostDocument = SanityDocument & Post
37
+
38
+ // === 1. Create a Document ===
39
+
40
+ export async function createPost(data: {
41
+ title: string
42
+ slug: string
43
+ authorId: string
44
+ }): Promise<PostDocument> {
45
+ return client.create<Post>({
46
+ _type: "post",
47
+ title: data.title,
48
+ slug: { _type: "slug", current: data.slug },
49
+ publishedAt: new Date().toISOString(),
50
+ viewCount: 0,
51
+ author: { _type: "reference", _ref: data.authorId },
52
+ })
53
+ }
54
+
55
+ // === 2. Patch a Document ===
56
+
57
+ // Set and unset fields
58
+ export async function updatePostTitle(
59
+ postId: string,
60
+ title: string,
61
+ ): Promise<PostDocument> {
62
+ return client
63
+ .patch(postId)
64
+ .set({ title })
65
+ .unset(["subtitle"]) // Remove a field
66
+ .commit<PostDocument>()
67
+ }
68
+
69
+ // Increment a numeric field
70
+ export async function incrementViewCount(
71
+ postId: string,
72
+ ): Promise<PostDocument> {
73
+ return client
74
+ .patch(postId)
75
+ .inc({ viewCount: 1 })
76
+ .commit<PostDocument>()
77
+ }
78
+
79
+ // Insert into an array
80
+ export async function addCategory(
81
+ postId: string,
82
+ categoryId: string,
83
+ ): Promise<PostDocument> {
84
+ return client
85
+ .patch(postId)
86
+ .setIfMissing({ categories: [] })
87
+ .insert("after", "categories[-1]", [
88
+ { _type: "reference", _ref: categoryId },
89
+ ])
90
+ .commit<PostDocument>()
91
+ }
92
+
93
+ // === 3. Delete a Document ===
94
+
95
+ export async function deletePost(postId: string): Promise<void> {
96
+ await client.delete(postId)
97
+ }
98
+
99
+ // Delete with a GROQ filter
100
+ export async function deleteDraftPosts(): Promise<void> {
101
+ // Delete all drafts of posts (draft IDs start with "drafts.")
102
+ await client.delete({
103
+ query: '*[_type == "post" && _id in path("drafts.**")]',
104
+ })
105
+ }
106
+
107
+ // === 4. Transactions (Atomic Multi-Mutation) ===
108
+
109
+ export async function publishPostWithCategories(data: {
110
+ title: string
111
+ slug: string
112
+ authorId: string
113
+ categoryNames: string[]
114
+ }): Promise<void> {
115
+ const transaction = client.transaction()
116
+
117
+ // Create category documents
118
+ const categoryIds: string[] = []
119
+ for (const name of data.categoryNames) {
120
+ const id = `category-${name.toLowerCase().replace(/\s+/g, "-")}`
121
+ categoryIds.push(id)
122
+
123
+ transaction.createIfNotExists({
124
+ _id: id,
125
+ _type: "category",
126
+ title: name,
127
+ })
128
+ }
129
+
130
+ // Create the post with references to categories
131
+ transaction.create({
132
+ _type: "post",
133
+ title: data.title,
134
+ slug: { _type: "slug", current: data.slug },
135
+ publishedAt: new Date().toISOString(),
136
+ viewCount: 0,
137
+ author: { _type: "reference", _ref: data.authorId },
138
+ categories: categoryIds.map((id) => ({
139
+ _type: "reference",
140
+ _ref: id,
141
+ })),
142
+ })
143
+
144
+ // All mutations happen atomically
145
+ await transaction.commit()
146
+ }
147
+
148
+ // === 5. createOrReplace — Idempotent upsert ===
149
+
150
+ export async function upsertSettings(settings: {
151
+ siteName: string
152
+ description: string
153
+ }): Promise<SanityDocument> {
154
+ return client.createOrReplace({
155
+ _id: "siteSettings",
156
+ _type: "siteSettings",
157
+ siteName: settings.siteName,
158
+ description: settings.description,
159
+ })
160
+ }
@@ -0,0 +1,187 @@
1
+ /**
2
+ * Reference Solution: Real-Time Content Listeners
3
+ *
4
+ * Demonstrates:
5
+ * - Setting up client.listen() with GROQ filters
6
+ * - Handling mutation events (create, update, delete)
7
+ * - Maintaining an in-memory cache synchronized with the Content Lake
8
+ * - Subscription cleanup and error handling
9
+ * - TypeScript types for listener events
10
+ */
11
+
12
+ import { createClient, type SanityDocument } from "@sanity/client"
13
+ import type { Subscription } from "rxjs"
14
+
15
+ // === Client Setup ===
16
+
17
+ const client = createClient({
18
+ projectId: "your-project-id",
19
+ dataset: "production",
20
+ apiVersion: "2024-01-01",
21
+ token: process.env.SANITY_API_TOKEN,
22
+ useCdn: false, // Listeners require the real-time API
23
+ })
24
+
25
+ // === Types ===
26
+
27
+ interface Post extends SanityDocument {
28
+ _type: "post"
29
+ title: string
30
+ slug: { current: string }
31
+ publishedAt: string
32
+ }
33
+
34
+ interface MutationEvent {
35
+ type: "mutation"
36
+ documentId: string
37
+ transition: "update" | "appear" | "disappear"
38
+ result?: SanityDocument
39
+ previousRev?: string
40
+ resultRev?: string
41
+ }
42
+
43
+ interface ReconnectEvent {
44
+ type: "reconnect"
45
+ }
46
+
47
+ type ListenerEvent = MutationEvent | ReconnectEvent
48
+
49
+ // === In-Memory Cache ===
50
+
51
+ class ContentCache<T extends SanityDocument> {
52
+ private documents = new Map<string, T>()
53
+
54
+ getAll(): T[] {
55
+ return Array.from(this.documents.values())
56
+ }
57
+
58
+ get(id: string): T | undefined {
59
+ return this.documents.get(id)
60
+ }
61
+
62
+ set(id: string, doc: T): void {
63
+ this.documents.set(id, doc)
64
+ }
65
+
66
+ delete(id: string): boolean {
67
+ return this.documents.delete(id)
68
+ }
69
+
70
+ get size(): number {
71
+ return this.documents.size
72
+ }
73
+ }
74
+
75
+ // === Listener Setup ===
76
+
77
+ /**
78
+ * Creates a real-time listener that keeps a local cache in sync with
79
+ * documents matching a GROQ filter.
80
+ *
81
+ * Returns an object with the cache and an unsubscribe function.
82
+ */
83
+ export async function createRealtimeSync<T extends SanityDocument>(options: {
84
+ filter: string
85
+ params?: Record<string, unknown>
86
+ onUpdate?: (event: MutationEvent) => void
87
+ onError?: (error: Error) => void
88
+ }): Promise<{
89
+ cache: ContentCache<T>
90
+ unsubscribe: () => void
91
+ }> {
92
+ const cache = new ContentCache<T>()
93
+
94
+ // 1. Initial fetch — populate cache with current data
95
+ const initialDocs = await client.fetch<T[]>(
96
+ `*[${options.filter}]`,
97
+ options.params || {},
98
+ )
99
+
100
+ for (const doc of initialDocs) {
101
+ cache.set(doc._id, doc)
102
+ }
103
+
104
+ // 2. Start listener — watch for mutations matching the filter
105
+ const subscription: Subscription = client
106
+ .listen<T>(
107
+ `*[${options.filter}]`,
108
+ options.params || {},
109
+ {
110
+ includeResult: true,
111
+ includePreviousRevision: false,
112
+ visibility: "query",
113
+ },
114
+ )
115
+ .subscribe({
116
+ next: (event) => {
117
+ if (event.type === "reconnect") {
118
+ // The listener reconnected — could refetch to ensure consistency
119
+ return
120
+ }
121
+
122
+ if (event.type === "mutation") {
123
+ const mutation = event as unknown as MutationEvent
124
+
125
+ switch (mutation.transition) {
126
+ case "appear": {
127
+ // New document matches the filter
128
+ if (mutation.result) {
129
+ cache.set(mutation.documentId, mutation.result as T)
130
+ }
131
+ break
132
+ }
133
+ case "update": {
134
+ // Existing document was modified
135
+ if (mutation.result) {
136
+ cache.set(mutation.documentId, mutation.result as T)
137
+ }
138
+ break
139
+ }
140
+ case "disappear": {
141
+ // Document no longer matches the filter (deleted or updated)
142
+ cache.delete(mutation.documentId)
143
+ break
144
+ }
145
+ }
146
+
147
+ options.onUpdate?.(mutation)
148
+ }
149
+ },
150
+ error: (err) => {
151
+ options.onError?.(
152
+ err instanceof Error ? err : new Error(String(err)),
153
+ )
154
+ },
155
+ })
156
+
157
+ // 3. Return cache and cleanup function
158
+ return {
159
+ cache,
160
+ unsubscribe: () => subscription.unsubscribe(),
161
+ }
162
+ }
163
+
164
+ // === Usage Example ===
165
+
166
+ async function main() {
167
+ const { cache, unsubscribe } = await createRealtimeSync<Post>({
168
+ filter: '_type == "post"',
169
+ onUpdate: (event) => {
170
+ console.log(
171
+ `Post ${event.documentId} ${event.transition}:`,
172
+ event.result?.title,
173
+ )
174
+ },
175
+ onError: (error) => {
176
+ console.error("Listener error:", error.message)
177
+ },
178
+ })
179
+
180
+ console.log(`Watching ${cache.size} posts for changes...`)
181
+
182
+ // Clean up when done (e.g., on process exit)
183
+ process.on("SIGINT", () => {
184
+ unsubscribe()
185
+ process.exit(0)
186
+ })
187
+ }
@@ -0,0 +1,166 @@
1
+ /**
2
+ * Reference Solution: Image Asset Pipeline with Responsive Images
3
+ *
4
+ * Demonstrates:
5
+ * - Setting up @sanity/image-url builder
6
+ * - Hotspot and crop support
7
+ * - Responsive srcSet generation for multiple widths
8
+ * - WebP format with auto-format
9
+ * - Reusable React component with TypeScript props
10
+ */
11
+
12
+ import imageUrlBuilder from "@sanity/image-url"
13
+ import { createClient } from "@sanity/client"
14
+
15
+ // === Client & Image Builder Setup ===
16
+
17
+ const client = createClient({
18
+ projectId: "your-project-id",
19
+ dataset: "production",
20
+ apiVersion: "2024-01-01",
21
+ useCdn: true,
22
+ })
23
+
24
+ const builder = imageUrlBuilder(client)
25
+
26
+ // === Types ===
27
+
28
+ interface SanityImageCrop {
29
+ top: number
30
+ bottom: number
31
+ left: number
32
+ right: number
33
+ }
34
+
35
+ interface SanityImageHotspot {
36
+ x: number
37
+ y: number
38
+ height: number
39
+ width: number
40
+ }
41
+
42
+ interface SanityImageSource {
43
+ _type: "image"
44
+ asset: {
45
+ _ref: string
46
+ _type: "reference"
47
+ }
48
+ crop?: SanityImageCrop
49
+ hotspot?: SanityImageHotspot
50
+ alt?: string
51
+ }
52
+
53
+ interface SanityImageProps {
54
+ /** The image field value from a Sanity document */
55
+ image: SanityImageSource
56
+ /** Alt text override (falls back to image.alt) */
57
+ alt?: string
58
+ /** Max display width in pixels */
59
+ maxWidth?: number
60
+ /** CSS sizes attribute for responsive images */
61
+ sizes?: string
62
+ /** Additional CSS class names */
63
+ className?: string
64
+ /** Loading strategy */
65
+ loading?: "lazy" | "eager"
66
+ /** Priority hint for LCP images */
67
+ priority?: boolean
68
+ }
69
+
70
+ // === Image URL Helper ===
71
+
72
+ /**
73
+ * Generate a URL for a Sanity image with transforms applied.
74
+ * The builder automatically respects hotspot and crop metadata.
75
+ */
76
+ function urlFor(source: SanityImageSource) {
77
+ return builder.image(source)
78
+ }
79
+
80
+ // === Responsive Breakpoints ===
81
+
82
+ const DEFAULT_WIDTHS = [320, 480, 640, 768, 1024, 1280, 1536, 1920]
83
+ const DEFAULT_MAX_WIDTH = 1920
84
+ const DEFAULT_SIZES = "(max-width: 768px) 100vw, (max-width: 1200px) 50vw, 33vw"
85
+
86
+ // === Generate srcSet ===
87
+
88
+ function generateSrcSet(
89
+ image: SanityImageSource,
90
+ widths: number[] = DEFAULT_WIDTHS,
91
+ ): string {
92
+ return widths
93
+ .map((w) => {
94
+ const url = urlFor(image)
95
+ .width(w)
96
+ .auto("format") // Serves WebP to browsers that support it
97
+ .fit("crop") // Respects hotspot and crop data
98
+ .url()
99
+
100
+ return `${url} ${w}w`
101
+ })
102
+ .join(", ")
103
+ }
104
+
105
+ // === React Component ===
106
+
107
+ export function SanityImage({
108
+ image,
109
+ alt,
110
+ maxWidth = DEFAULT_MAX_WIDTH,
111
+ sizes = DEFAULT_SIZES,
112
+ className,
113
+ loading = "lazy",
114
+ priority = false,
115
+ }: SanityImageProps) {
116
+ if (!image?.asset) return null
117
+
118
+ // Filter widths to only include those up to maxWidth
119
+ const widths = DEFAULT_WIDTHS.filter((w) => w <= maxWidth)
120
+ if (widths[widths.length - 1] !== maxWidth) {
121
+ widths.push(maxWidth)
122
+ }
123
+
124
+ // Default src at maxWidth for non-srcset browsers
125
+ const src = urlFor(image)
126
+ .width(maxWidth)
127
+ .auto("format")
128
+ .fit("crop")
129
+ .url()
130
+
131
+ const srcSet = generateSrcSet(image, widths)
132
+
133
+ return (
134
+ <img
135
+ src={src}
136
+ srcSet={srcSet}
137
+ sizes={sizes}
138
+ alt={alt || image.alt || ""}
139
+ className={className}
140
+ loading={priority ? "eager" : loading}
141
+ decoding={priority ? "sync" : "async"}
142
+ style={{ maxWidth: "100%", height: "auto" }}
143
+ />
144
+ )
145
+ }
146
+
147
+ // === Low-Level Helpers (for non-React usage) ===
148
+
149
+ /**
150
+ * Get a single image URL with transforms applied.
151
+ *
152
+ * @example
153
+ * const url = getImageUrl(post.mainImage, { width: 800, height: 600 })
154
+ */
155
+ export function getImageUrl(
156
+ source: SanityImageSource,
157
+ options: { width?: number; height?: number; quality?: number } = {},
158
+ ): string {
159
+ let img = urlFor(source).auto("format").fit("crop")
160
+
161
+ if (options.width) img = img.width(options.width)
162
+ if (options.height) img = img.height(options.height)
163
+ if (options.quality) img = img.quality(options.quality)
164
+
165
+ return img.url()
166
+ }