@sanity/ailf 0.5.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (377) hide show
  1. package/README.md +0 -1
  2. package/config/features.ts +23 -0
  3. package/config/models.ts +95 -0
  4. package/config/prompts.ts +16 -0
  5. package/config/rubrics.ts +225 -0
  6. package/config/schedules.ts +47 -0
  7. package/config/sinks.ts +37 -0
  8. package/config/sources.ts +21 -0
  9. package/config/thresholds.ts +61 -0
  10. package/dist/_vendor/ailf-core/config-helpers.d.ts +171 -0
  11. package/dist/_vendor/ailf-core/config-helpers.js +170 -0
  12. package/dist/_vendor/ailf-core/env-helper.d.ts +35 -0
  13. package/dist/_vendor/ailf-core/env-helper.js +45 -0
  14. package/dist/_vendor/ailf-core/examples/index.d.ts +16 -0
  15. package/dist/_vendor/ailf-core/examples/index.js +25 -0
  16. package/dist/_vendor/ailf-core/index.d.ts +3 -0
  17. package/dist/_vendor/ailf-core/index.js +5 -0
  18. package/dist/_vendor/ailf-core/ports/context.d.ts +17 -2
  19. package/dist/_vendor/ailf-core/ports/doc-fetcher.d.ts +2 -2
  20. package/dist/_vendor/ailf-core/ports/index.d.ts +2 -1
  21. package/dist/_vendor/ailf-core/ports/mode-handler.d.ts +129 -0
  22. package/dist/_vendor/ailf-core/ports/mode-handler.js +19 -0
  23. package/dist/_vendor/ailf-core/ports/task-source.d.ts +16 -122
  24. package/dist/_vendor/ailf-core/ports/task-source.js +7 -7
  25. package/dist/_vendor/ailf-core/schemas/eval-config.d.ts +8 -2
  26. package/dist/_vendor/ailf-core/schemas/eval-config.js +17 -2
  27. package/dist/_vendor/ailf-core/schemas/pipeline-request.d.ts +9 -3
  28. package/dist/_vendor/ailf-core/schemas/pipeline-request.js +8 -1
  29. package/dist/_vendor/ailf-core/schemas/pipeline.d.ts +14 -31
  30. package/dist/_vendor/ailf-core/schemas/pipeline.js +17 -9
  31. package/dist/_vendor/ailf-core/schemas/schedules.d.ts +14 -4
  32. package/dist/_vendor/ailf-core/schemas/schedules.js +6 -2
  33. package/dist/_vendor/ailf-core/schemas/sinks.d.ts +1 -1
  34. package/dist/_vendor/ailf-core/services/comparison-formatters.js +57 -19
  35. package/dist/_vendor/ailf-core/services/index.d.ts +2 -1
  36. package/dist/_vendor/ailf-core/services/index.js +2 -1
  37. package/dist/_vendor/ailf-core/services/scoring-engine.d.ts +153 -0
  38. package/dist/_vendor/ailf-core/services/scoring-engine.js +237 -0
  39. package/dist/_vendor/ailf-core/services/scoring.d.ts +15 -2
  40. package/dist/_vendor/ailf-core/services/scoring.js +25 -15
  41. package/dist/_vendor/ailf-core/types/branded-ids.d.ts +137 -0
  42. package/dist/_vendor/ailf-core/types/branded-ids.js +136 -0
  43. package/dist/_vendor/ailf-core/types/eval-mode-config.d.ts +150 -0
  44. package/dist/_vendor/ailf-core/types/eval-mode-config.js +24 -0
  45. package/dist/_vendor/ailf-core/types/generalized-task.d.ts +332 -0
  46. package/dist/_vendor/ailf-core/types/generalized-task.js +13 -0
  47. package/dist/_vendor/ailf-core/types/index.d.ts +45 -83
  48. package/dist/_vendor/ailf-core/types/index.js +8 -1
  49. package/dist/_vendor/ailf-core/types/plugin-registry.d.ts +257 -0
  50. package/dist/_vendor/ailf-core/types/plugin-registry.js +185 -0
  51. package/dist/_vendor/ailf-core/types/storage-schema.d.ts +199 -0
  52. package/dist/_vendor/ailf-core/types/storage-schema.js +39 -0
  53. package/dist/_vendor/ailf-core/types/task-graph.d.ts +86 -0
  54. package/dist/_vendor/ailf-core/types/task-graph.js +20 -0
  55. package/dist/_vendor/ailf-core/types/trace.d.ts +118 -0
  56. package/dist/_vendor/ailf-core/types/trace.js +18 -0
  57. package/dist/_vendor/ailf-core/types/variable-envelope.d.ts +80 -0
  58. package/dist/_vendor/ailf-core/types/variable-envelope.js +16 -0
  59. package/dist/_vendor/ailf-shared/dimension-names.d.ts +5 -18
  60. package/dist/_vendor/ailf-shared/dimension-names.js +6 -24
  61. package/dist/_vendor/ailf-shared/eval-modes.d.ts +38 -6
  62. package/dist/_vendor/ailf-shared/eval-modes.js +26 -2
  63. package/dist/_vendor/ailf-shared/index.d.ts +0 -1
  64. package/dist/_vendor/ailf-shared/index.js +0 -1
  65. package/dist/adapters/api-client/build-request.js +14 -13
  66. package/dist/adapters/config-sources/file-config-adapter.d.ts +20 -11
  67. package/dist/adapters/config-sources/file-config-adapter.js +39 -12
  68. package/dist/adapters/config-sources/index.d.ts +2 -0
  69. package/dist/adapters/config-sources/index.js +1 -0
  70. package/dist/adapters/config-sources/ts-config-loader.d.ts +59 -0
  71. package/dist/adapters/config-sources/ts-config-loader.js +141 -0
  72. package/dist/adapters/doc-fetchers/sanity-doc-fetcher.d.ts +3 -2
  73. package/dist/adapters/doc-fetchers/sanity-doc-fetcher.js +7 -2
  74. package/dist/adapters/task-sources/composite-task-source.d.ts +3 -3
  75. package/dist/adapters/task-sources/composite-task-source.js +1 -1
  76. package/dist/adapters/task-sources/content-lake-task-source.d.ts +7 -6
  77. package/dist/adapters/task-sources/content-lake-task-source.js +35 -39
  78. package/dist/adapters/task-sources/index.d.ts +3 -2
  79. package/dist/adapters/task-sources/index.js +3 -2
  80. package/dist/adapters/task-sources/repo-schemas.d.ts +218 -16
  81. package/dist/adapters/task-sources/repo-schemas.js +227 -19
  82. package/dist/adapters/task-sources/repo-task-source.d.ts +16 -12
  83. package/dist/adapters/task-sources/repo-task-source.js +92 -80
  84. package/dist/adapters/task-sources/repo-validation.d.ts +36 -5
  85. package/dist/adapters/task-sources/repo-validation.js +126 -5
  86. package/dist/adapters/task-sources/task-file-loader.d.ts +64 -0
  87. package/dist/adapters/task-sources/task-file-loader.js +83 -0
  88. package/dist/adapters/task-sources/yaml-task-source.d.ts +6 -6
  89. package/dist/adapters/task-sources/yaml-task-source.js +19 -16
  90. package/dist/cli.js +0 -2
  91. package/dist/commands/baseline.js +4 -1
  92. package/dist/commands/calculate-scores.js +1 -1
  93. package/dist/commands/coverage-audit.js +9 -1
  94. package/dist/commands/explain-handler.js +25 -23
  95. package/dist/commands/fetch-docs.js +3 -2
  96. package/dist/commands/generate-configs.js +1 -1
  97. package/dist/commands/init.d.ts +6 -4
  98. package/dist/commands/init.js +302 -23
  99. package/dist/commands/interactive.js +11 -7
  100. package/dist/commands/pipeline-action.d.ts +2 -0
  101. package/dist/commands/pipeline-action.js +16 -6
  102. package/dist/commands/pipeline.d.ts +1 -0
  103. package/dist/commands/pipeline.js +4 -2
  104. package/dist/commands/pr-comment.js +1 -1
  105. package/dist/commands/publish.js +2 -2
  106. package/dist/commands/readiness-report.js +13 -6
  107. package/dist/commands/validate-tasks.d.ts +2 -2
  108. package/dist/commands/validate-tasks.js +26 -15
  109. package/dist/composition-root.d.ts +13 -1
  110. package/dist/composition-root.js +99 -4
  111. package/dist/index.d.ts +41 -0
  112. package/dist/index.js +48 -0
  113. package/dist/orchestration/build-app-context.js +1 -0
  114. package/dist/orchestration/build-step-sequence.js +28 -8
  115. package/dist/orchestration/steps/calculate-scores-step.js +24 -11
  116. package/dist/orchestration/steps/fetch-docs-step.js +8 -7
  117. package/dist/orchestration/steps/gap-analysis-step.js +8 -7
  118. package/dist/orchestration/steps/generate-configs-step.d.ts +16 -3
  119. package/dist/orchestration/steps/generate-configs-step.js +261 -51
  120. package/dist/orchestration/steps/grader-consistency-step.js +7 -4
  121. package/dist/orchestration/steps/mirror-repo-tasks-step.js +1 -1
  122. package/dist/orchestration/steps/readiness-step.js +5 -6
  123. package/dist/orchestration/steps/run-eval-step.d.ts +1 -2
  124. package/dist/orchestration/steps/run-eval-step.js +8 -7
  125. package/dist/pipeline/cache.d.ts +1 -1
  126. package/dist/pipeline/cache.js +36 -8
  127. package/dist/pipeline/calculate-scores.d.ts +2 -4
  128. package/dist/pipeline/calculate-scores.js +43 -113
  129. package/dist/pipeline/checks.js +2 -2
  130. package/dist/pipeline/compare.js +8 -8
  131. package/dist/pipeline/compiler/__tests__/agent-harness-handler.test.d.ts +10 -0
  132. package/dist/pipeline/compiler/__tests__/agent-harness-handler.test.js +288 -0
  133. package/dist/pipeline/compiler/__tests__/assertion-mapper.test.d.ts +9 -0
  134. package/dist/pipeline/compiler/__tests__/assertion-mapper.test.js +145 -0
  135. package/dist/pipeline/compiler/__tests__/knowledge-probe-handler.test.d.ts +10 -0
  136. package/dist/pipeline/compiler/__tests__/knowledge-probe-handler.test.js +314 -0
  137. package/dist/pipeline/compiler/__tests__/literacy-handler.test.d.ts +10 -0
  138. package/dist/pipeline/compiler/__tests__/literacy-handler.test.js +486 -0
  139. package/dist/pipeline/compiler/__tests__/mcp-server-handler.test.d.ts +10 -0
  140. package/dist/pipeline/compiler/__tests__/mcp-server-handler.test.js +392 -0
  141. package/dist/pipeline/compiler/__tests__/promptfoo-compiler.test.d.ts +9 -0
  142. package/dist/pipeline/compiler/__tests__/promptfoo-compiler.test.js +333 -0
  143. package/dist/pipeline/compiler/__tests__/sandbox-and-fixtures.test.d.ts +12 -0
  144. package/dist/pipeline/compiler/__tests__/sandbox-and-fixtures.test.js +210 -0
  145. package/dist/pipeline/compiler/__tests__/scoring-and-presets.test.d.ts +7 -0
  146. package/dist/pipeline/compiler/__tests__/scoring-and-presets.test.js +404 -0
  147. package/dist/pipeline/compiler/__tests__/scoring-bridge.test.d.ts +10 -0
  148. package/dist/pipeline/compiler/__tests__/scoring-bridge.test.js +184 -0
  149. package/dist/pipeline/compiler/__tests__/task-graph-builder.test.d.ts +8 -0
  150. package/dist/pipeline/compiler/__tests__/task-graph-builder.test.js +301 -0
  151. package/dist/pipeline/compiler/__tests__/telemetry.test.d.ts +9 -0
  152. package/dist/pipeline/compiler/__tests__/telemetry.test.js +503 -0
  153. package/dist/pipeline/compiler/assertion-mapper.d.ts +58 -0
  154. package/dist/pipeline/compiler/assertion-mapper.js +175 -0
  155. package/dist/pipeline/compiler/compiler-to-yaml.d.ts +51 -0
  156. package/dist/pipeline/compiler/compiler-to-yaml.js +222 -0
  157. package/dist/pipeline/compiler/config-loader.d.ts +56 -0
  158. package/dist/pipeline/compiler/config-loader.js +111 -0
  159. package/dist/pipeline/compiler/fixture-resolver.d.ts +41 -0
  160. package/dist/pipeline/compiler/fixture-resolver.js +113 -0
  161. package/dist/pipeline/compiler/hash.d.ts +11 -0
  162. package/dist/pipeline/compiler/hash.js +18 -0
  163. package/dist/pipeline/compiler/ignore-fields.d.ts +53 -0
  164. package/dist/pipeline/compiler/ignore-fields.js +113 -0
  165. package/dist/pipeline/compiler/index.d.ts +29 -0
  166. package/dist/pipeline/compiler/index.js +45 -0
  167. package/dist/pipeline/compiler/literacy-bridge.d.ts +102 -0
  168. package/dist/pipeline/compiler/literacy-bridge.js +172 -0
  169. package/dist/pipeline/compiler/mode-bases/agent-harness.d.ts +10 -0
  170. package/dist/pipeline/compiler/mode-bases/agent-harness.js +21 -0
  171. package/dist/pipeline/compiler/mode-bases/index.d.ts +4 -0
  172. package/dist/pipeline/compiler/mode-bases/index.js +4 -0
  173. package/dist/pipeline/compiler/mode-bases/knowledge-probe.d.ts +10 -0
  174. package/dist/pipeline/compiler/mode-bases/knowledge-probe.js +22 -0
  175. package/dist/pipeline/compiler/mode-bases/literacy.d.ts +12 -0
  176. package/dist/pipeline/compiler/mode-bases/literacy.js +78 -0
  177. package/dist/pipeline/compiler/mode-bases/mcp-server.d.ts +10 -0
  178. package/dist/pipeline/compiler/mode-bases/mcp-server.js +70 -0
  179. package/dist/pipeline/compiler/mode-handlers/__fixtures__/agent-harness-example-tasks.d.ts +14 -0
  180. package/dist/pipeline/compiler/mode-handlers/__fixtures__/agent-harness-example-tasks.js +152 -0
  181. package/dist/pipeline/compiler/mode-handlers/__fixtures__/knowledge-probe-example-tasks.d.ts +32 -0
  182. package/dist/pipeline/compiler/mode-handlers/__fixtures__/knowledge-probe-example-tasks.js +176 -0
  183. package/dist/pipeline/compiler/mode-handlers/__fixtures__/mcp-example-tasks.d.ts +49 -0
  184. package/dist/pipeline/compiler/mode-handlers/__fixtures__/mcp-example-tasks.js +259 -0
  185. package/dist/pipeline/compiler/mode-handlers/agent-harness/assertions.d.ts +43 -0
  186. package/dist/pipeline/compiler/mode-handlers/agent-harness/assertions.js +187 -0
  187. package/dist/pipeline/compiler/mode-handlers/agent-harness/compiler.d.ts +19 -0
  188. package/dist/pipeline/compiler/mode-handlers/agent-harness/compiler.js +138 -0
  189. package/dist/pipeline/compiler/mode-handlers/agent-harness/index.d.ts +16 -0
  190. package/dist/pipeline/compiler/mode-handlers/agent-harness/index.js +43 -0
  191. package/dist/pipeline/compiler/mode-handlers/agent-harness/prompts.d.ts +9 -0
  192. package/dist/pipeline/compiler/mode-handlers/agent-harness/prompts.js +29 -0
  193. package/dist/pipeline/compiler/mode-handlers/agent-harness/sandbox.d.ts +12 -0
  194. package/dist/pipeline/compiler/mode-handlers/agent-harness/sandbox.js +82 -0
  195. package/dist/pipeline/compiler/mode-handlers/agent-harness/tool-presets.d.ts +4 -0
  196. package/dist/pipeline/compiler/mode-handlers/agent-harness/tool-presets.js +19 -0
  197. package/dist/pipeline/compiler/mode-handlers/agent-harness/types.d.ts +49 -0
  198. package/dist/pipeline/compiler/mode-handlers/agent-harness/types.js +4 -0
  199. package/dist/pipeline/compiler/mode-handlers/agent-harness/validation.d.ts +9 -0
  200. package/dist/pipeline/compiler/mode-handlers/agent-harness/validation.js +16 -0
  201. package/dist/pipeline/compiler/mode-handlers/index.d.ts +15 -0
  202. package/dist/pipeline/compiler/mode-handlers/index.js +19 -0
  203. package/dist/pipeline/compiler/mode-handlers/knowledge-probe/assertions.d.ts +16 -0
  204. package/dist/pipeline/compiler/mode-handlers/knowledge-probe/assertions.js +61 -0
  205. package/dist/pipeline/compiler/mode-handlers/knowledge-probe/compiler.d.ts +18 -0
  206. package/dist/pipeline/compiler/mode-handlers/knowledge-probe/compiler.js +112 -0
  207. package/dist/pipeline/compiler/mode-handlers/knowledge-probe/index.d.ts +26 -0
  208. package/dist/pipeline/compiler/mode-handlers/knowledge-probe/index.js +49 -0
  209. package/dist/pipeline/compiler/mode-handlers/knowledge-probe/prompts.d.ts +9 -0
  210. package/dist/pipeline/compiler/mode-handlers/knowledge-probe/prompts.js +28 -0
  211. package/dist/pipeline/compiler/mode-handlers/knowledge-probe/types.d.ts +44 -0
  212. package/dist/pipeline/compiler/mode-handlers/knowledge-probe/types.js +4 -0
  213. package/dist/pipeline/compiler/mode-handlers/knowledge-probe/validation.d.ts +9 -0
  214. package/dist/pipeline/compiler/mode-handlers/knowledge-probe/validation.js +24 -0
  215. package/dist/pipeline/compiler/mode-handlers/literacy/assertions.d.ts +18 -0
  216. package/dist/pipeline/compiler/mode-handlers/literacy/assertions.js +118 -0
  217. package/dist/pipeline/compiler/mode-handlers/literacy/compiler.d.ts +14 -0
  218. package/dist/pipeline/compiler/mode-handlers/literacy/compiler.js +105 -0
  219. package/dist/pipeline/compiler/mode-handlers/literacy/index.d.ts +11 -0
  220. package/dist/pipeline/compiler/mode-handlers/literacy/index.js +38 -0
  221. package/dist/pipeline/compiler/mode-handlers/literacy/prompts.d.ts +9 -0
  222. package/dist/pipeline/compiler/mode-handlers/literacy/prompts.js +74 -0
  223. package/dist/pipeline/compiler/mode-handlers/literacy/types.d.ts +41 -0
  224. package/dist/pipeline/compiler/mode-handlers/literacy/types.js +4 -0
  225. package/dist/pipeline/compiler/mode-handlers/literacy/validation.d.ts +12 -0
  226. package/dist/pipeline/compiler/mode-handlers/literacy/validation.js +28 -0
  227. package/dist/pipeline/compiler/mode-handlers/mcp-server/assertions.d.ts +42 -0
  228. package/dist/pipeline/compiler/mode-handlers/mcp-server/assertions.js +334 -0
  229. package/dist/pipeline/compiler/mode-handlers/mcp-server/compiler.d.ts +19 -0
  230. package/dist/pipeline/compiler/mode-handlers/mcp-server/compiler.js +100 -0
  231. package/dist/pipeline/compiler/mode-handlers/mcp-server/index.d.ts +27 -0
  232. package/dist/pipeline/compiler/mode-handlers/mcp-server/index.js +54 -0
  233. package/dist/pipeline/compiler/mode-handlers/mcp-server/prompts.d.ts +8 -0
  234. package/dist/pipeline/compiler/mode-handlers/mcp-server/prompts.js +28 -0
  235. package/dist/pipeline/compiler/mode-handlers/mcp-server/provider-config.d.ts +28 -0
  236. package/dist/pipeline/compiler/mode-handlers/mcp-server/provider-config.js +104 -0
  237. package/dist/pipeline/compiler/mode-handlers/mcp-server/types.d.ts +37 -0
  238. package/dist/pipeline/compiler/mode-handlers/mcp-server/types.js +4 -0
  239. package/dist/pipeline/compiler/mode-handlers/mcp-server/validation.d.ts +9 -0
  240. package/dist/pipeline/compiler/mode-handlers/mcp-server/validation.js +43 -0
  241. package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/index.d.ts +33 -0
  242. package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/index.js +174 -0
  243. package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/mcp-connection.d.ts +19 -0
  244. package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/mcp-connection.js +95 -0
  245. package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/tool-loop-anthropic.d.ts +19 -0
  246. package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/tool-loop-anthropic.js +172 -0
  247. package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/tool-loop-openai.d.ts +14 -0
  248. package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/tool-loop-openai.js +16 -0
  249. package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/types.d.ts +93 -0
  250. package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/types.js +4 -0
  251. package/dist/pipeline/compiler/preset-loader.d.ts +22 -0
  252. package/dist/pipeline/compiler/preset-loader.js +99 -0
  253. package/dist/pipeline/compiler/presets/index.d.ts +9 -0
  254. package/dist/pipeline/compiler/presets/index.js +8 -0
  255. package/dist/pipeline/compiler/presets/sanity-literacy.d.ts +42 -0
  256. package/dist/pipeline/compiler/presets/sanity-literacy.js +208 -0
  257. package/dist/pipeline/compiler/promptfoo-compiler.d.ts +96 -0
  258. package/dist/pipeline/compiler/promptfoo-compiler.js +230 -0
  259. package/dist/pipeline/compiler/provider-assembler.d.ts +39 -0
  260. package/dist/pipeline/compiler/provider-assembler.js +137 -0
  261. package/dist/pipeline/compiler/sandbox/docker-sandbox.d.ts +21 -0
  262. package/dist/pipeline/compiler/sandbox/docker-sandbox.js +136 -0
  263. package/dist/pipeline/compiler/sandbox/fixture-provisioner.d.ts +69 -0
  264. package/dist/pipeline/compiler/sandbox/fixture-provisioner.js +189 -0
  265. package/dist/pipeline/compiler/sandbox/git-worktree-sandbox.d.ts +20 -0
  266. package/dist/pipeline/compiler/sandbox/git-worktree-sandbox.js +114 -0
  267. package/dist/pipeline/compiler/sandbox/index.d.ts +10 -0
  268. package/dist/pipeline/compiler/sandbox/index.js +11 -0
  269. package/dist/pipeline/compiler/sandbox/sandbox-selector.d.ts +35 -0
  270. package/dist/pipeline/compiler/sandbox/sandbox-selector.js +86 -0
  271. package/dist/pipeline/compiler/sandbox/sandbox-strategy.d.ts +81 -0
  272. package/dist/pipeline/compiler/sandbox/sandbox-strategy.js +15 -0
  273. package/dist/pipeline/compiler/sandbox/tempdir-sandbox.d.ts +20 -0
  274. package/dist/pipeline/compiler/sandbox/tempdir-sandbox.js +74 -0
  275. package/dist/pipeline/compiler/scoring-bridge.d.ts +49 -0
  276. package/dist/pipeline/compiler/scoring-bridge.js +114 -0
  277. package/dist/pipeline/compiler/task-graph-builder.d.ts +54 -0
  278. package/dist/pipeline/compiler/task-graph-builder.js +291 -0
  279. package/dist/pipeline/compiler/telemetry/cost-tracker.d.ts +90 -0
  280. package/dist/pipeline/compiler/telemetry/cost-tracker.js +146 -0
  281. package/dist/pipeline/compiler/telemetry/index.d.ts +14 -0
  282. package/dist/pipeline/compiler/telemetry/index.js +19 -0
  283. package/dist/pipeline/compiler/telemetry/redactor.d.ts +58 -0
  284. package/dist/pipeline/compiler/telemetry/redactor.js +222 -0
  285. package/dist/pipeline/compiler/telemetry/tool-classifier.d.ts +32 -0
  286. package/dist/pipeline/compiler/telemetry/tool-classifier.js +120 -0
  287. package/dist/pipeline/compiler/telemetry/trace-collector.d.ts +75 -0
  288. package/dist/pipeline/compiler/telemetry/trace-collector.js +297 -0
  289. package/dist/pipeline/compiler/telemetry/trace-store.d.ts +78 -0
  290. package/dist/pipeline/compiler/telemetry/trace-store.js +85 -0
  291. package/dist/pipeline/compiler/variable-resolver.d.ts +46 -0
  292. package/dist/pipeline/compiler/variable-resolver.js +115 -0
  293. package/dist/pipeline/coverage-audit.d.ts +15 -5
  294. package/dist/pipeline/coverage-audit.js +41 -22
  295. package/dist/pipeline/eval-constants.d.ts +16 -6
  296. package/dist/pipeline/eval-constants.js +25 -4
  297. package/dist/pipeline/eval-fingerprint.d.ts +2 -2
  298. package/dist/pipeline/eval-fingerprint.js +8 -9
  299. package/dist/pipeline/expand-tasks.d.ts +19 -10
  300. package/dist/pipeline/expand-tasks.js +34 -28
  301. package/dist/pipeline/gap-analysis.d.ts +1 -1
  302. package/dist/pipeline/gap-analysis.js +2 -2
  303. package/dist/pipeline/generate-configs.d.ts +22 -4
  304. package/dist/pipeline/generate-configs.js +53 -24
  305. package/dist/pipeline/grader-api.d.ts +3 -3
  306. package/dist/pipeline/grader-api.js +5 -12
  307. package/dist/pipeline/grader-compare-runner.js +20 -27
  308. package/dist/pipeline/grader-comparison.d.ts +4 -8
  309. package/dist/pipeline/grader-comparison.js +11 -17
  310. package/dist/pipeline/grader-consistency-runner.d.ts +2 -3
  311. package/dist/pipeline/grader-consistency-runner.js +16 -20
  312. package/dist/pipeline/grader-consistency.d.ts +6 -10
  313. package/dist/pipeline/grader-consistency.js +13 -32
  314. package/dist/pipeline/grader-sensitivity-runner.js +7 -5
  315. package/dist/pipeline/grader-sensitivity.d.ts +2 -6
  316. package/dist/pipeline/grader-sensitivity.js +10 -10
  317. package/dist/pipeline/grader-validate-runner.js +7 -5
  318. package/dist/pipeline/grader-validation.d.ts +2 -6
  319. package/dist/pipeline/grader-validation.js +14 -22
  320. package/dist/pipeline/map-request-to-config.js +7 -1
  321. package/dist/pipeline/mirror-repo-tasks.d.ts +13 -13
  322. package/dist/pipeline/mirror-repo-tasks.js +22 -21
  323. package/dist/pipeline/normalize-mode.d.ts +49 -0
  324. package/dist/pipeline/normalize-mode.js +64 -0
  325. package/dist/pipeline/plan.d.ts +5 -2
  326. package/dist/pipeline/plan.js +134 -78
  327. package/dist/pipeline/pr-comment.js +2 -0
  328. package/dist/pipeline/profile-resolution.d.ts +22 -14
  329. package/dist/pipeline/profile-resolution.js +41 -19
  330. package/dist/pipeline/provenance.d.ts +2 -2
  331. package/dist/pipeline/provenance.js +12 -17
  332. package/dist/pipeline/release-report.js +4 -4
  333. package/dist/pipeline/repo-threshold-evaluator.d.ts +1 -1
  334. package/dist/pipeline/repo-threshold-evaluator.js +1 -1
  335. package/dist/pipeline/rubric-loader.d.ts +20 -0
  336. package/dist/pipeline/rubric-loader.js +37 -0
  337. package/dist/pipeline/validate.d.ts +4 -4
  338. package/dist/pipeline/validate.js +64 -53
  339. package/dist/schedules/loader.js +18 -8
  340. package/dist/scripts/migrate-task-mode.d.ts +24 -0
  341. package/dist/scripts/migrate-task-mode.js +85 -0
  342. package/dist/scripts/migrate-tasks-to-content-lake.js +11 -10
  343. package/dist/scripts/validate-task-sources.d.ts +1 -1
  344. package/dist/scripts/validate-task-sources.js +15 -15
  345. package/dist/sinks/loader.js +5 -7
  346. package/dist/sources.d.ts +7 -7
  347. package/dist/sources.js +22 -24
  348. package/dist/webhook/dispatch.js +2 -1
  349. package/package.json +15 -4
  350. package/tasks/knowledge-probe/define-type-api.task.ts +55 -0
  351. package/tasks/knowledge-probe/groq-projections.task.ts +59 -0
  352. package/tasks/literacy/frameworks.task.ts +128 -0
  353. package/tasks/literacy/functions.task.ts +69 -0
  354. package/tasks/literacy/groq.task.ts +258 -0
  355. package/tasks/literacy/nextjs-live.task.ts +75 -0
  356. package/tasks/literacy/studio-setup.task.ts +131 -0
  357. package/tasks/literacy/visual-editing.task.ts +146 -0
  358. package/config/features.yaml +0 -116
  359. package/config/models.yaml +0 -116
  360. package/config/prompts.yaml +0 -75
  361. package/config/rubrics.yaml +0 -81
  362. package/config/schedules.yaml +0 -43
  363. package/config/sinks.yaml +0 -54
  364. package/config/sources.yaml +0 -51
  365. package/config/thresholds.yaml +0 -49
  366. package/dist/_vendor/ailf-tasks/cli.d.ts +0 -8
  367. package/dist/_vendor/ailf-tasks/cli.js +0 -61
  368. package/dist/_vendor/ailf-tasks/index.d.ts +0 -13
  369. package/dist/_vendor/ailf-tasks/index.js +0 -16
  370. package/dist/_vendor/ailf-tasks/parser.d.ts +0 -27
  371. package/dist/_vendor/ailf-tasks/parser.js +0 -73
  372. package/dist/_vendor/ailf-tasks/schemas.d.ts +0 -198
  373. package/dist/_vendor/ailf-tasks/schemas.js +0 -180
  374. package/dist/_vendor/ailf-tasks/validation.d.ts +0 -47
  375. package/dist/_vendor/ailf-tasks/validation.js +0 -162
  376. package/dist/agent-observer/test-imports.d.ts +0 -7
  377. package/dist/agent-observer/test-imports.js +0 -185
@@ -0,0 +1,258 @@
1
+ import { defineTask } from "@sanity/ailf-core"
2
+
3
+ export default [
4
+ defineTask({
5
+ id: "groq-blog-queries",
6
+ mode: "literacy",
7
+ title: "GROQ - Blog queries with filtering and pagination",
8
+ description: "GROQ - Blog queries with filtering and pagination",
9
+ area: "groq",
10
+ docCoverage: true,
11
+ context: {
12
+ docs: [
13
+ {
14
+ slug: "groq-introduction",
15
+ reason: "GROQ overview — syntax, filters, projections, core concepts",
16
+ },
17
+ {
18
+ slug: "how-queries-work",
19
+ reason: "Detailed walkthrough of GROQ query mechanics",
20
+ },
21
+ {
22
+ slug: "query-cheat-sheet",
23
+ reason: "Ready-made query patterns for common use cases",
24
+ },
25
+ {
26
+ slug: "paginating-with-groq",
27
+ reason: "Pagination patterns with slice syntax",
28
+ },
29
+ ],
30
+ },
31
+ referenceSolution: "reference-solutions/groq/blog-queries.ts",
32
+ prompt: {
33
+ vars: {
34
+ task: `Write GROQ queries for a Sanity blog application:
35
+
36
+ 1. Fetch all published blog posts ordered by publishedAt descending,
37
+ with a projection that includes: _id, title, slug (from slug.current),
38
+ publishedAt, excerpt, and the author's name (resolved from a reference)
39
+ 2. Add pagination to return only the first 10 results
40
+ 3. Fetch a single post by its slug parameter, including the full body
41
+ content and resolved author and category references
42
+ 4. Fetch posts published after a specific date
43
+ 5. Fetch posts that belong to a specific category (where categories
44
+ is an array of references)
45
+
46
+ Use @sanity/client with client.fetch() for all queries. Include
47
+ TypeScript types for the query results.`,
48
+ docs: "file://contexts/canonical/groq-blog-queries.md",
49
+ },
50
+ },
51
+ assertions: [
52
+ {
53
+ type: "llm-rubric",
54
+ template: "task-completion",
55
+ criteria: [
56
+ 'GROQ filter with _type == "post"',
57
+ 'Projection with aliased slug field ("slug": slug.current)',
58
+ "Reference resolution with -> for author",
59
+ "Ordering with | order(publishedAt desc)",
60
+ "Slice/pagination syntax [0...10] or [0..9]",
61
+ "Parameterized query with $slug for single post fetch",
62
+ "Date filtering with dateTime() or string comparison",
63
+ "Category filtering using references or array contains",
64
+ ],
65
+ },
66
+ {
67
+ type: "llm-rubric",
68
+ template: "code-correctness",
69
+ criteria: [
70
+ "Valid GROQ syntax (proper filter brackets, projection braces)",
71
+ "Uses @sanity/client createClient + client.fetch()",
72
+ "Correct parameter passing syntax ($param)",
73
+ "Proper reference dereference with ->",
74
+ "No deprecated patterns",
75
+ ],
76
+ },
77
+ {
78
+ type: "contains-any",
79
+ value: ["client.fetch", "createClient"],
80
+ weight: 1,
81
+ },
82
+ {
83
+ type: "contains-any",
84
+ value: ["order(publishedAt", "order(_createdAt", "| order("],
85
+ weight: 1,
86
+ },
87
+ {
88
+ type: "contains-any",
89
+ value: ["[0...10]", "[0..9]", "[0..."],
90
+ weight: 1,
91
+ },
92
+ ],
93
+ }),
94
+
95
+ defineTask({
96
+ id: "groq-joins-references",
97
+ mode: "literacy",
98
+ title: "GROQ - Joins and reference resolution",
99
+ description: "GROQ - Joins and reference resolution",
100
+ area: "groq",
101
+ docCoverage: true,
102
+ context: {
103
+ docs: [
104
+ {
105
+ slug: "groq-joins",
106
+ reason: "Join patterns — ->, []-> , subqueries, parent scope (^)",
107
+ },
108
+ {
109
+ slug: "how-queries-work",
110
+ reason: "Core query mechanics including reference traversal",
111
+ },
112
+ {
113
+ slug: "groq-introduction",
114
+ reason: "Overview including reference resolution basics",
115
+ },
116
+ ],
117
+ },
118
+ referenceSolution: "reference-solutions/groq/joins-references.ts",
119
+ prompt: {
120
+ vars: {
121
+ task: `Write GROQ queries that demonstrate join patterns in Sanity:
122
+
123
+ 1. Follow a single reference to resolve an author's full profile
124
+ from a post (post.author -> author document with name, bio, image)
125
+ 2. Resolve an array of category references from a post
126
+ (post.categories[]-> with title and slug)
127
+ 3. Write a reverse reference query: given an author's ID, find all
128
+ posts by that author using a subquery and the parent scope operator (^)
129
+ 4. Create a nested join: for each author, include their 5 most recent
130
+ posts as a nested array
131
+ 5. Use the references() function to find all documents that reference
132
+ a specific document ID
133
+
134
+ Use @sanity/client with client.fetch(). Include TypeScript types.`,
135
+ docs: "file://contexts/canonical/groq-joins-references.md",
136
+ },
137
+ },
138
+ assertions: [
139
+ {
140
+ type: "llm-rubric",
141
+ template: "task-completion",
142
+ criteria: [
143
+ "Single reference follow with -> operator",
144
+ "Array reference resolution with []->",
145
+ "Reverse reference / subquery using *[references(^._id)]",
146
+ "Nested join pattern with parent scope (^)",
147
+ "The references() function",
148
+ ],
149
+ },
150
+ {
151
+ type: "llm-rubric",
152
+ template: "code-correctness",
153
+ criteria: [
154
+ "Correct -> dereference syntax",
155
+ "Valid []-> array dereference",
156
+ "Proper use of ^ parent scope operator",
157
+ "Valid references() function usage",
158
+ "No made-up syntax",
159
+ ],
160
+ },
161
+ { type: "contains", value: "->", weight: 1 },
162
+ {
163
+ type: "contains-any",
164
+ value: ["references(", "references(^"],
165
+ weight: 1,
166
+ },
167
+ ],
168
+ }),
169
+
170
+ defineTask({
171
+ id: "groq-advanced-filtering",
172
+ mode: "literacy",
173
+ title: "GROQ - Advanced filtering and projections",
174
+ description: "GROQ - Advanced filtering and projections",
175
+ area: "groq",
176
+ docCoverage: true,
177
+ context: {
178
+ docs: [
179
+ {
180
+ slug: "groq-functions",
181
+ reason:
182
+ "GROQ function reference — select(), coalesce(), count(), defined()",
183
+ },
184
+ {
185
+ slug: "query-cheat-sheet",
186
+ reason: "Common filtering patterns and match operator examples",
187
+ },
188
+ {
189
+ slug: "groq-operators",
190
+ reason: "Operator reference — match, comparison, logical operators",
191
+ },
192
+ {
193
+ slug: "how-queries-work",
194
+ reason: "Query mechanics including projections and ordering",
195
+ },
196
+ ],
197
+ },
198
+ referenceSolution: "reference-solutions/groq/advanced-filtering.ts",
199
+ prompt: {
200
+ vars: {
201
+ task: `Write GROQ queries demonstrating advanced filtering and projection patterns:
202
+
203
+ 1. Use select() for conditional projections — return different fields
204
+ based on the document's _type (e.g., posts get excerpt, events get
205
+ date and venue)
206
+ 2. Use coalesce() for fallback values — e.g., use seoTitle if it
207
+ exists, otherwise fall back to title
208
+ 3. Use the match operator for full-text search in titles
209
+ 4. Use count() to count documents matching a filter and to count
210
+ items within an array field
211
+ 5. Use defined() to filter for documents that have a specific field set
212
+ 6. Filter items within an array using [condition] syntax
213
+ 7. Order results by multiple fields (e.g., featured status first,
214
+ then by publishedAt)
215
+
216
+ Use @sanity/client with client.fetch(). Include TypeScript types.`,
217
+ docs: "file://contexts/canonical/groq-advanced-filtering.md",
218
+ },
219
+ },
220
+ assertions: [
221
+ {
222
+ type: "llm-rubric",
223
+ template: "task-completion",
224
+ criteria: [
225
+ "select() for conditional projections",
226
+ "coalesce() for fallback values",
227
+ "match operator for text search",
228
+ "count() function usage",
229
+ "defined() function for existence checks",
230
+ "Array filtering with [condition]",
231
+ "Multi-field ordering",
232
+ ],
233
+ },
234
+ {
235
+ type: "llm-rubric",
236
+ template: "code-correctness",
237
+ criteria: [
238
+ "Valid select() syntax with => arrow notation",
239
+ "Correct coalesce() usage",
240
+ "Proper match operator usage (on text fields)",
241
+ "Valid count() and defined() function calls",
242
+ "Correct array filter syntax",
243
+ ],
244
+ },
245
+ {
246
+ type: "contains-any",
247
+ value: ["select(", "coalesce("],
248
+ weight: 1,
249
+ },
250
+ {
251
+ type: "contains-any",
252
+ value: ["count(", "defined("],
253
+ weight: 1,
254
+ },
255
+ { type: "contains-any", value: ["match"], weight: 1 },
256
+ ],
257
+ }),
258
+ ]
@@ -0,0 +1,75 @@
1
+ import { defineTask } from "@sanity/ailf-core"
2
+
3
+ export default [
4
+ defineTask({
5
+ id: "nextjs-app-router-integration",
6
+ mode: "literacy",
7
+ title: "Next.js - App Router integration with TypeScript",
8
+ description: "Next.js - App Router integration with TypeScript",
9
+ area: "nextjs",
10
+ docCoverage: true,
11
+ context: {
12
+ docs: [
13
+ {
14
+ slug: "next-js-app-router-quickstart",
15
+ reason: "Primary Next.js App Router quickstart guide",
16
+ },
17
+ {
18
+ slug: "diplaying-content-in-next-js",
19
+ reason: "Fetching and displaying Sanity content in Next.js",
20
+ },
21
+ {
22
+ slug: "sanity-typegen",
23
+ reason: "TypeGen — generating TypeScript types from GROQ queries",
24
+ },
25
+ {
26
+ slug: "groq-introduction",
27
+ reason: "GROQ query language introduction",
28
+ },
29
+ ],
30
+ },
31
+ referenceSolution: "reference-solutions/nextjs/app-router-integration.tsx",
32
+ prompt: {
33
+ vars: {
34
+ task: `Integrate Sanity into a Next.js 16 App Router application
35
+ with full TypeScript support:
36
+
37
+ 1. Set up the Sanity client with proper configuration
38
+ 2. Create a typed GROQ query for fetching blog posts
39
+ 3. Build a server component that fetches and renders posts
40
+ 4. Generate TypeScript types using sanity typegen
41
+
42
+ Provide all files needed for a working integration.`,
43
+ docs: "file://contexts/canonical/nextjs-app-router-integration.md",
44
+ },
45
+ },
46
+ assertions: [
47
+ {
48
+ type: "llm-rubric",
49
+ template: "task-completion",
50
+ criteria: [
51
+ "Sanity client configuration (createClient)",
52
+ "GROQ query with proper syntax",
53
+ "Next.js App Router server component using async/await",
54
+ "TypeScript type definitions or typegen setup",
55
+ ],
56
+ },
57
+ {
58
+ type: "llm-rubric",
59
+ template: "code-correctness",
60
+ criteria: [
61
+ "Uses @sanity/client (not deprecated packages)",
62
+ "App Router patterns (not Pages Router)",
63
+ "Proper server component data fetching (no useEffect)",
64
+ "Valid GROQ query syntax",
65
+ ],
66
+ },
67
+ { type: "contains", value: "createClient", weight: 1 },
68
+ {
69
+ type: "contains-any",
70
+ value: ["groq", "GROQ", "*["],
71
+ weight: 1,
72
+ },
73
+ ],
74
+ }),
75
+ ]
@@ -0,0 +1,131 @@
1
+ import { defineTask } from "@sanity/ailf-core"
2
+
3
+ export default [
4
+ defineTask({
5
+ id: "studio-blog-schema",
6
+ mode: "literacy",
7
+ title: "Studio Setup - Blog schema with posts, authors, categories",
8
+ description: "Studio Setup - Blog schema with posts, authors, categories",
9
+ area: "studio",
10
+ docCoverage: true,
11
+ context: {
12
+ docs: [
13
+ {
14
+ slug: "schemas-and-forms",
15
+ reason: "High-level overview of schemas and the form builder",
16
+ },
17
+ {
18
+ slug: "introduction-to-schemas",
19
+ reason: "Introduces schema concepts, defineType/defineField",
20
+ },
21
+ {
22
+ slug: "config-api-reference",
23
+ reason:
24
+ "Configuration API — defineConfig, plugins, schema registration",
25
+ },
26
+ {
27
+ slug: "reference-type",
28
+ reason: "Reference field type for author/category relationships",
29
+ },
30
+ ],
31
+ },
32
+ referenceSolution: "reference-solutions/studio-setup/blog-schema.ts",
33
+ prompt: {
34
+ vars: {
35
+ task: `Set up a new Sanity Studio with a custom schema for a blog:
36
+
37
+ 1. Create document types for: posts, authors, categories
38
+ 2. Posts should have: title, slug, body (portable text), author reference, categories array
39
+ 3. Authors should have: name, bio, image
40
+ 4. Categories should have: title, description
41
+
42
+ Include the schema definitions and sanity.config.ts setup.`,
43
+ docs: "file://contexts/canonical/studio-blog-schema.md",
44
+ },
45
+ },
46
+ assertions: [
47
+ {
48
+ type: "llm-rubric",
49
+ template: "task-completion",
50
+ criteria: [
51
+ "Three document types (post, author, category)",
52
+ "Post with all required fields including portable text body",
53
+ "Reference from post to author",
54
+ "Array of references from post to categories",
55
+ "sanity.config.ts with schema registration",
56
+ ],
57
+ },
58
+ {
59
+ type: "llm-rubric",
60
+ template: "code-correctness",
61
+ criteria: [
62
+ "Uses defineConfig, defineType, defineField (v3 syntax)",
63
+ "Does NOT use createSchema (deprecated v2)",
64
+ "Proper reference syntax with 'to' array",
65
+ "Correct portable text array definition",
66
+ ],
67
+ },
68
+ { type: "contains", value: "defineConfig", weight: 1 },
69
+ { type: "contains", value: "defineType", weight: 1 },
70
+ { type: "contains", value: "defineField", weight: 1 },
71
+ { type: "not-contains", value: "createSchema", weight: 1 },
72
+ { type: "not-contains", value: "import Schema from", weight: 1 },
73
+ ],
74
+ }),
75
+
76
+ defineTask({
77
+ id: "studio-custom-tool",
78
+ mode: "literacy",
79
+ title: "Studio Setup - Custom tool in sidebar",
80
+ description: "Studio Setup - Custom tool in sidebar",
81
+ area: "studio",
82
+ context: {
83
+ docs: [
84
+ {
85
+ slug: "studio-tools",
86
+ reason: "Overview of Studio tools system",
87
+ },
88
+ {
89
+ slug: "tool-api-reference",
90
+ reason: "Tool API — name, title, icon, component properties",
91
+ },
92
+ {
93
+ slug: "custom-studio-tool",
94
+ reason: "Step-by-step guide for creating a custom tool",
95
+ },
96
+ ],
97
+ },
98
+ referenceSolution: "reference-solutions/studio-setup/custom-tool.tsx",
99
+ prompt: {
100
+ vars: {
101
+ task: `Add a custom tool to the Sanity Studio sidebar that displays
102
+ a dashboard. The tool should:
103
+
104
+ 1. Appear in the studio navigation with a custom icon
105
+ 2. Have a title and name
106
+ 3. Render a React component showing a "Dashboard" heading
107
+
108
+ Provide the tool definition and sanity.config.ts registration.`,
109
+ docs: "file://contexts/canonical/studio-custom-tool.md",
110
+ },
111
+ },
112
+ assertions: [
113
+ {
114
+ type: "llm-rubric",
115
+ template: "task-completion",
116
+ criteria: [
117
+ "Tool object with name, title, icon, component",
118
+ "React component for the tool UI",
119
+ "Registration in sanity.config.ts tools array",
120
+ ],
121
+ },
122
+ { type: "contains", value: "tools", weight: 1 },
123
+ {
124
+ type: "javascript",
125
+ value: `return output.includes('name:') &&
126
+ output.includes('component') &&
127
+ (output.includes('icon:') || output.includes('Icon'))`,
128
+ },
129
+ ],
130
+ }),
131
+ ]
@@ -0,0 +1,146 @@
1
+ import { defineTask } from "@sanity/ailf-core"
2
+
3
+ export default [
4
+ defineTask({
5
+ id: "visual-editing-presentation",
6
+ mode: "literacy",
7
+ title: "Visual Editing - Presentation tool with click-to-edit",
8
+ description: "Visual Editing - Presentation tool with click-to-edit",
9
+ area: "visual-editing",
10
+ docCoverage: true,
11
+ context: {
12
+ docs: [
13
+ {
14
+ slug: "configuring-the-presentation-tool",
15
+ reason: "Core presentationTool configuration and setup",
16
+ },
17
+ {
18
+ slug: "introduction-to-visual-editing",
19
+ reason: "Visual Editing concepts — stega, overlays, data attributes",
20
+ },
21
+ {
22
+ slug: "visual-editing-with-next-js-app-router",
23
+ reason: "Next.js App Router-specific visual editing guide",
24
+ },
25
+ {
26
+ slug: "stega",
27
+ reason: "Stega encoding for click-to-edit data attributes",
28
+ },
29
+ ],
30
+ },
31
+ referenceSolution:
32
+ "reference-solutions/visual-editing/presentation-nextjs.tsx",
33
+ prompt: {
34
+ vars: {
35
+ task: `Set up the Presentation tool with a Next.js 14 (App Router) frontend
36
+ and implement click-to-edit functionality:
37
+
38
+ 1. Configure the Presentation tool in sanity.config.ts
39
+ 2. Set up the Next.js app to work with Visual Editing
40
+ 3. Implement data attributes so clicking content in the preview
41
+ opens the corresponding field in Studio
42
+
43
+ Provide all necessary code for both Studio and Next.js sides.`,
44
+ docs: "file://contexts/canonical/visual-editing-presentation.md",
45
+ },
46
+ },
47
+ assertions: [
48
+ {
49
+ type: "llm-rubric",
50
+ template: "task-completion",
51
+ criteria: [
52
+ "presentationTool configured in sanity.config.ts",
53
+ "previewUrl or equivalent configured",
54
+ "Data attributes for click-to-edit (createDataAttribute or stega)",
55
+ "Next.js App Router patterns used correctly",
56
+ ],
57
+ },
58
+ {
59
+ type: "llm-rubric",
60
+ template: "code-correctness",
61
+ criteria: [
62
+ "Uses @sanity/presentation (not deprecated packages)",
63
+ "Uses createDataAttribute or stega encoding correctly",
64
+ "Proper Next.js App Router patterns (not Pages Router)",
65
+ "No mixing of deprecated and current APIs",
66
+ ],
67
+ },
68
+ { type: "contains", value: "presentationTool", weight: 1 },
69
+ {
70
+ type: "contains-any",
71
+ value: [
72
+ "createDataAttribute",
73
+ "data-sanity",
74
+ "encodeDataAttribute",
75
+ "stega",
76
+ ],
77
+ weight: 1,
78
+ },
79
+ { type: "not-contains", value: "@sanity/preview-kit", weight: 1 },
80
+ ],
81
+ }),
82
+
83
+ defineTask({
84
+ id: "visual-editing-live-preview",
85
+ mode: "literacy",
86
+ title: "Visual Editing - Live preview with draft content",
87
+ description: "Visual Editing - Live preview with draft content",
88
+ area: "visual-editing",
89
+ context: {
90
+ docs: [
91
+ {
92
+ slug: "live-content-api",
93
+ reason: "Live Content API — defineLive, real-time subscriptions",
94
+ },
95
+ {
96
+ slug: "perspectives",
97
+ reason: "Draft vs published perspectives",
98
+ },
99
+ {
100
+ slug: "fetching-content-for-visual-editing",
101
+ reason: "Data fetching patterns for visual editing contexts",
102
+ },
103
+ ],
104
+ },
105
+ referenceSolution: "reference-solutions/visual-editing/live-preview.tsx",
106
+ prompt: {
107
+ vars: {
108
+ task: `Implement live preview in a Next.js app that shows draft content
109
+ from Sanity in real-time as editors make changes in the Studio.
110
+
111
+ Requirements:
112
+ - Use the Live Content API approach
113
+ - Handle draft vs published perspectives correctly
114
+ - Show real-time updates without page refresh
115
+
116
+ Provide a complete implementation.`,
117
+ docs: "file://contexts/canonical/visual-editing-live-preview.md",
118
+ },
119
+ },
120
+ assertions: [
121
+ {
122
+ type: "llm-rubric",
123
+ template: "task-completion",
124
+ criteria: [
125
+ "Live Content API usage (defineLive, useLiveQuery, or sanityFetch with live option)",
126
+ "Draft perspective configuration",
127
+ "Real-time subscription/update mechanism",
128
+ ],
129
+ },
130
+ {
131
+ type: "llm-rubric",
132
+ template: "code-correctness",
133
+ criteria: [
134
+ "Modern API usage (not deprecated preview packages)",
135
+ "Proper perspective handling",
136
+ "Correct subscription lifecycle management",
137
+ ],
138
+ },
139
+ {
140
+ type: "contains-any",
141
+ value: ["useLiveQuery", "defineLive", "live:", "perspective"],
142
+ weight: 1,
143
+ },
144
+ ],
145
+ }),
146
+ ]