@sanity/ailf 4.6.0 → 5.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/canonical/grader-references/agent-harness-tools.yaml +42 -0
  2. package/canonical/grader-references/knowledge-probe-recall.yaml +36 -0
  3. package/canonical/grader-references/mcp-server-spec.yaml +51 -0
  4. package/canonical/grader-references/portable-text.yaml +48 -0
  5. package/config/rubrics.ts +38 -2
  6. package/dist/_vendor/ailf-core/artifact-registry.d.ts +60 -2
  7. package/dist/_vendor/ailf-core/artifact-registry.js +288 -7
  8. package/dist/_vendor/ailf-core/examples/index.d.ts +125 -26
  9. package/dist/_vendor/ailf-core/examples/index.js +146 -47
  10. package/dist/_vendor/ailf-core/ports/context.d.ts +8 -0
  11. package/dist/_vendor/ailf-core/ports/mode-handler.d.ts +15 -0
  12. package/dist/_vendor/ailf-core/schemas/branded-string.d.ts +40 -0
  13. package/dist/_vendor/ailf-core/schemas/branded-string.js +45 -0
  14. package/dist/_vendor/ailf-core/schemas/confidence-schema.d.ts +36 -0
  15. package/dist/_vendor/ailf-core/schemas/confidence-schema.js +32 -0
  16. package/dist/_vendor/ailf-core/schemas/eval-config.d.ts +1 -0
  17. package/dist/_vendor/ailf-core/schemas/eval-config.js +8 -4
  18. package/dist/_vendor/ailf-core/schemas/index.d.ts +2 -0
  19. package/dist/_vendor/ailf-core/schemas/index.js +9 -0
  20. package/dist/_vendor/ailf-core/schemas/pipeline-request.d.ts +1 -0
  21. package/dist/_vendor/ailf-core/schemas/pipeline-request.js +1 -0
  22. package/dist/_vendor/ailf-core/schemas/pipeline.d.ts +34 -8
  23. package/dist/_vendor/ailf-core/schemas/pipeline.js +23 -1
  24. package/dist/_vendor/ailf-core/services/diagnosis/registry.d.ts +40 -0
  25. package/dist/_vendor/ailf-core/services/diagnosis/registry.js +25 -0
  26. package/dist/_vendor/ailf-core/services/diagnosis-runner.d.ts +19 -0
  27. package/dist/_vendor/ailf-core/services/diagnosis-runner.js +19 -0
  28. package/dist/_vendor/ailf-core/services/index.d.ts +2 -0
  29. package/dist/_vendor/ailf-core/services/index.js +5 -0
  30. package/dist/_vendor/ailf-core/services/report-to-markdown.js +3 -2
  31. package/dist/_vendor/ailf-core/types/attribution.d.ts +82 -0
  32. package/dist/_vendor/ailf-core/types/attribution.js +18 -0
  33. package/dist/_vendor/ailf-core/types/branded-ids.d.ts +26 -1
  34. package/dist/_vendor/ailf-core/types/branded-ids.js +80 -4
  35. package/dist/_vendor/ailf-core/types/confidence.d.ts +1 -1
  36. package/dist/_vendor/ailf-core/types/confidence.js +7 -0
  37. package/dist/_vendor/ailf-core/types/diagnosis.d.ts +169 -0
  38. package/dist/_vendor/ailf-core/types/diagnosis.js +17 -0
  39. package/dist/_vendor/ailf-core/types/generalized-task.d.ts +16 -1
  40. package/dist/_vendor/ailf-core/types/grader-judgment.d.ts +125 -0
  41. package/dist/_vendor/ailf-core/types/grader-judgment.js +30 -0
  42. package/dist/_vendor/ailf-core/types/index.d.ts +80 -29
  43. package/dist/_vendor/ailf-core/types/index.js +15 -1
  44. package/dist/_vendor/ailf-core/types/legacy-grader-judgment.d.ts +55 -0
  45. package/dist/_vendor/ailf-core/types/legacy-grader-judgment.js +30 -0
  46. package/dist/_vendor/ailf-core/types/pipeline-request.d.ts +1 -0
  47. package/dist/_vendor/ailf-core/types/repo-config.d.ts +8 -0
  48. package/dist/_vendor/ailf-shared/document-ref.d.ts +1 -1
  49. package/dist/adapters/api-client/build-request.d.ts +1 -0
  50. package/dist/adapters/api-client/build-request.js +3 -0
  51. package/dist/adapters/attribution/attribution-meta-writer.d.ts +35 -0
  52. package/dist/adapters/attribution/attribution-meta-writer.js +34 -0
  53. package/dist/adapters/attribution/index.d.ts +9 -0
  54. package/dist/adapters/attribution/index.js +8 -0
  55. package/dist/adapters/attribution/per-entry-attribution-writer.d.ts +56 -0
  56. package/dist/adapters/attribution/per-entry-attribution-writer.js +49 -0
  57. package/dist/adapters/config-sources/file-config-adapter.js +1 -0
  58. package/dist/adapters/grader-outputs/index.d.ts +10 -0
  59. package/dist/adapters/grader-outputs/index.js +8 -0
  60. package/dist/adapters/grader-outputs/legacy/index.d.ts +11 -0
  61. package/dist/adapters/grader-outputs/legacy/index.js +10 -0
  62. package/dist/adapters/grader-outputs/legacy/promptfoo-grader-output-legacy.d.ts +49 -0
  63. package/dist/adapters/grader-outputs/legacy/promptfoo-grader-output-legacy.js +48 -0
  64. package/dist/adapters/grader-outputs/promptfoo-grader-output.d.ts +102 -0
  65. package/dist/adapters/grader-outputs/promptfoo-grader-output.js +93 -0
  66. package/dist/adapters/index.d.ts +3 -0
  67. package/dist/adapters/index.js +4 -0
  68. package/dist/adapters/task-sources/content-lake-task-source.d.ts +5 -1
  69. package/dist/adapters/task-sources/content-lake-task-source.js +28 -2
  70. package/dist/adapters/task-sources/repo-schemas.d.ts +79 -11
  71. package/dist/adapters/task-sources/repo-schemas.js +19 -2
  72. package/dist/commands/calculate-scores.js +1 -1
  73. package/dist/commands/explain-handler.js +1 -1
  74. package/dist/commands/lookup-doc.d.ts +1 -1
  75. package/dist/commands/lookup-doc.js +3 -3
  76. package/dist/commands/pipeline-action.d.ts +6 -0
  77. package/dist/commands/pipeline-action.js +2 -0
  78. package/dist/commands/remote-pipeline.js +1 -0
  79. package/dist/composition-root.d.ts +36 -0
  80. package/dist/composition-root.js +48 -0
  81. package/dist/config/rubrics.ts +38 -2
  82. package/dist/grader/agent-harness.d.ts +14 -0
  83. package/dist/grader/agent-harness.js +17 -0
  84. package/dist/grader/common.d.ts +17 -0
  85. package/dist/grader/common.js +21 -0
  86. package/dist/grader/index.d.ts +38 -0
  87. package/dist/grader/index.js +75 -0
  88. package/dist/grader/knowledge-probe.d.ts +14 -0
  89. package/dist/grader/knowledge-probe.js +18 -0
  90. package/dist/grader/literacy.d.ts +13 -0
  91. package/dist/grader/literacy.js +17 -0
  92. package/dist/grader/mcp.d.ts +14 -0
  93. package/dist/grader/mcp.js +18 -0
  94. package/dist/orchestration/build-app-context.js +1 -0
  95. package/dist/orchestration/build-step-sequence.js +5 -0
  96. package/dist/orchestration/steps/calculate-scores-step.js +23 -1
  97. package/dist/orchestration/steps/compute-attribution-step.d.ts +44 -0
  98. package/dist/orchestration/steps/compute-attribution-step.js +279 -0
  99. package/dist/orchestration/steps/gap-analysis-step.js +35 -7
  100. package/dist/orchestration/steps/index.d.ts +1 -0
  101. package/dist/orchestration/steps/index.js +1 -0
  102. package/dist/pipeline/attribution.d.ts +15 -0
  103. package/dist/pipeline/attribution.js +18 -9
  104. package/dist/pipeline/borderline-consensus-runner.d.ts +63 -0
  105. package/dist/pipeline/borderline-consensus-runner.js +124 -0
  106. package/dist/pipeline/borderline-detector.d.ts +24 -0
  107. package/dist/pipeline/borderline-detector.js +26 -0
  108. package/dist/pipeline/calculate-scores.d.ts +114 -3
  109. package/dist/pipeline/calculate-scores.js +426 -24
  110. package/dist/pipeline/compiler/literacy-bridge.d.ts +1 -1
  111. package/dist/pipeline/compiler/literacy-bridge.js +35 -17
  112. package/dist/pipeline/compiler/rubric-resolution.d.ts +15 -0
  113. package/dist/pipeline/compiler/rubric-resolution.js +9 -1
  114. package/dist/pipeline/compute-attribution.d.ts +80 -0
  115. package/dist/pipeline/compute-attribution.js +196 -0
  116. package/dist/pipeline/failure-modes.d.ts +52 -17
  117. package/dist/pipeline/failure-modes.js +178 -117
  118. package/dist/pipeline/map-request-to-config.js +1 -0
  119. package/package.json +6 -4
@@ -185,11 +185,26 @@ export declare const exampleGroqBlogListingData: readonly [{
185
185
  readonly assertions: readonly [{
186
186
  readonly type: "llm-rubric";
187
187
  readonly template: "task-completion";
188
- readonly criteria: readonly ["Uses the groq tagged template literal", "Fetches blog posts with title, slug, and publishedAt fields", "Orders results by publishedAt in descending order"];
188
+ readonly criteria: readonly [{
189
+ readonly id: "uses-groq-template-literal";
190
+ readonly text: "Uses the groq tagged template literal";
191
+ }, {
192
+ readonly id: "fetches-blog-fields";
193
+ readonly text: "Fetches blog posts with title, slug, and publishedAt fields";
194
+ }, {
195
+ readonly id: "orders-by-published-at";
196
+ readonly text: "Orders results by publishedAt in descending order";
197
+ }];
189
198
  }, {
190
199
  readonly type: "llm-rubric";
191
200
  readonly template: "code-correctness";
192
- readonly criteria: readonly ["Uses createClient from @sanity/client or next-sanity", "Exports a valid Next.js page component"];
201
+ readonly criteria: readonly [{
202
+ readonly id: "uses-create-client";
203
+ readonly text: "Uses createClient from @sanity/client or next-sanity";
204
+ }, {
205
+ readonly id: "exports-nextjs-page";
206
+ readonly text: "Exports a valid Next.js page component";
207
+ }];
193
208
  }];
194
209
  readonly baseline: {
195
210
  readonly enabled: true;
@@ -198,9 +213,9 @@ export declare const exampleGroqBlogListingData: readonly [{
198
213
  readonly status: "draft";
199
214
  }];
200
215
  /** TypeScript task template for example-groq-blog-listing */
201
- export declare const exampleGroqBlogListingTs = "/**\n * Example Task: Blog listing with GROQ queries.\n *\n * This is a starter template \u2014 edit it for your own documentation.\n * Each task evaluates whether an AI coding agent can implement a feature\n * using your docs as context. Delete this file or replace it entirely.\n *\n * This example task ships as a DRAFT so it does not run in production\n * evaluations automatically. To activate it, change status to \"active\"\n * or remove the status field entirely (defaults to active).\n *\n * Full field reference:\n * https://github.com/sanity-labs/ai-literacy-framework/blob/main/docs/contributing-tasks.md\n */\n\nimport { defineTask } from \"@sanity/ailf\"\n\nexport default defineTask({\n // \u2500\u2500 Mode \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n // \"literacy\" tests whether AI coding tools can implement features\n // using your docs as context. Other modes: \"mcp-server\",\n // \"knowledge-probe\", \"agent-harness\", \"custom\".\n mode: \"literacy\",\n\n // \u2500\u2500 Identity \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n // Unique identifier \u2014 lowercase alphanumeric with hyphens.\n // Must be unique across all task files in .ailf/tasks/.\n id: \"example-groq-blog-listing\",\n title: \"Blog listing with GROQ queries\",\n description: \"Example \u2014 tests GROQ blog listing implementation\",\n\n // Feature area this task belongs to. Tasks with the same area are\n // grouped together in score summaries.\n area: \"groq\",\n\n // \u2500\u2500 Documentation context \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n // Canonical doc references for this task. The pipeline fetches these\n // from Sanity and injects them into the prompt for baseline evaluation.\n //\n // This example uses slug-based references \u2014 the simplest form.\n // See the other example tasks for path, id, and perspective references.\n context: {\n docs: [\n {\n slug: \"groq-introduction\",\n reason: \"Core GROQ syntax and query language reference\",\n },\n {\n slug: \"how-queries-work\",\n reason: \"Query execution model and best practices\",\n },\n ],\n },\n\n // When true, the pipeline auto-generates an additional rubric that\n // checks whether the LLM's response actually used the provided docs.\n docCoverage: true,\n\n // Path to a gold-standard implementation, relative to canonical/.\n // The grader uses this as a reference when scoring code correctness.\n referenceSolution: \"canonical/example-groq-blog-listing.ts\",\n\n // \u2500\u2500 Prompt \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n // prompt.text \u2014 the implementation prompt given to the LLM.\n // Write this as if you're asking a developer to build the feature.\n // Be specific about requirements so the grader can evaluate clearly.\n prompt: {\n text: `Create a Next.js page component that lists blog posts from Sanity\nusing GROQ. The page should display the title, slug, and published\ndate for each post, sorted by most recent first. Use the Sanity\nclient to fetch data.`,\n },\n\n // \u2500\u2500 Assertions \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n // Grading assertions \u2014 how the LLM's response is scored.\n //\n // \"llm-rubric\" assertions use a grader LLM to score against criteria.\n // The \"template\" references a rubric template (e.g. task-completion).\n //\n // Available templates:\n // task-completion \u2014 did the LLM implement the feature? (weight: 0.50)\n // code-correctness \u2014 is the code idiomatic and correct? (weight: 0.25)\n //\n // You can also use value-based assertions:\n // { type: \"contains\", value: \"client.fetch\" }\n // { type: \"contains-any\", value: [\"createClient\", \"sanityClient\"] }\n assertions: [\n {\n type: \"llm-rubric\",\n template: \"task-completion\",\n criteria: [\n \"Uses the groq tagged template literal\",\n \"Fetches blog posts with title, slug, and publishedAt fields\",\n \"Orders results by publishedAt in descending order\",\n ],\n },\n {\n type: \"llm-rubric\",\n template: \"code-correctness\",\n criteria: [\n \"Uses createClient from @sanity/client or next-sanity\",\n \"Exports a valid Next.js page component\",\n ],\n },\n ],\n\n // \u2500\u2500 Baseline variant \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n // enabled \u2014 set to false to skip this task entirely\n // rubric \u2014 \"full\" (default), \"abbreviated\" (faster), or \"none\"\n baseline: {\n enabled: true,\n rubric: \"full\",\n },\n\n // Example tasks ship as drafts so they don't run in production evals.\n // Change to \"active\" (or remove this field) to activate.\n status: \"draft\",\n})\n";
216
+ export declare const exampleGroqBlogListingTs = "/**\n * Example Task: Blog listing with GROQ queries.\n *\n * This is a starter template \u2014 edit it for your own documentation.\n * Each task evaluates whether an AI coding agent can implement a feature\n * using your docs as context. Delete this file or replace it entirely.\n *\n * This example task ships as a DRAFT so it does not run in production\n * evaluations automatically. To activate it, change status to \"active\"\n * or remove the status field entirely (defaults to active).\n *\n * Full field reference:\n * https://github.com/sanity-labs/ai-literacy-framework/blob/main/docs/contributing-tasks.md\n */\n\nimport { defineTask } from \"@sanity/ailf\"\n\nexport default defineTask({\n // \u2500\u2500 Mode \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n // \"literacy\" tests whether AI coding tools can implement features\n // using your docs as context. Other modes: \"mcp-server\",\n // \"knowledge-probe\", \"agent-harness\", \"custom\".\n mode: \"literacy\",\n\n // \u2500\u2500 Identity \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n // Unique identifier \u2014 lowercase alphanumeric with hyphens.\n // Must be unique across all task files in .ailf/tasks/.\n id: \"example-groq-blog-listing\",\n title: \"Blog listing with GROQ queries\",\n description: \"Example \u2014 tests GROQ blog listing implementation\",\n\n // Feature area this task belongs to. Tasks with the same area are\n // grouped together in score summaries.\n area: \"groq\",\n\n // \u2500\u2500 Documentation context \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n // Canonical doc references for this task. The pipeline fetches these\n // from Sanity and injects them into the prompt for baseline evaluation.\n //\n // This example uses slug-based references \u2014 the simplest form.\n // See the other example tasks for path, id, and perspective references.\n context: {\n docs: [\n {\n slug: \"groq-introduction\",\n reason: \"Core GROQ syntax and query language reference\",\n },\n {\n slug: \"how-queries-work\",\n reason: \"Query execution model and best practices\",\n },\n ],\n },\n\n // When true, the pipeline auto-generates an additional rubric that\n // checks whether the LLM's response actually used the provided docs.\n docCoverage: true,\n\n // Path to a gold-standard implementation, relative to canonical/.\n // The grader uses this as a reference when scoring code correctness.\n referenceSolution: \"canonical/example-groq-blog-listing.ts\",\n\n // \u2500\u2500 Prompt \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n // prompt.text \u2014 the implementation prompt given to the LLM.\n // Write this as if you're asking a developer to build the feature.\n // Be specific about requirements so the grader can evaluate clearly.\n prompt: {\n text: `Create a Next.js page component that lists blog posts from Sanity\nusing GROQ. The page should display the title, slug, and published\ndate for each post, sorted by most recent first. Use the Sanity\nclient to fetch data.`,\n },\n\n // \u2500\u2500 Assertions \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n // Grading assertions \u2014 how the LLM's response is scored.\n //\n // \"llm-rubric\" assertions use a grader LLM to score against criteria.\n // The \"template\" references a rubric template (e.g. task-completion).\n //\n // Available templates:\n // task-completion \u2014 did the LLM implement the feature? (weight: 0.50)\n // code-correctness \u2014 is the code idiomatic and correct? (weight: 0.25)\n //\n // You can also use value-based assertions:\n // { type: \"contains\", value: \"client.fetch\" }\n // { type: \"contains-any\", value: [\"createClient\", \"sanityClient\"] }\n assertions: [\n {\n type: \"llm-rubric\",\n template: \"task-completion\",\n criteria: [\n {\n id: \"uses-groq-template-literal\",\n text: \"Uses the groq tagged template literal\",\n },\n {\n id: \"fetches-blog-fields\",\n text: \"Fetches blog posts with title, slug, and publishedAt fields\",\n },\n {\n id: \"orders-by-published-at\",\n text: \"Orders results by publishedAt in descending order\",\n },\n ],\n },\n {\n type: \"llm-rubric\",\n template: \"code-correctness\",\n criteria: [\n {\n id: \"uses-create-client\",\n text: \"Uses createClient from @sanity/client or next-sanity\",\n },\n {\n id: \"exports-nextjs-page\",\n text: \"Exports a valid Next.js page component\",\n },\n ],\n },\n ],\n\n // \u2500\u2500 Baseline variant \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n // enabled \u2014 set to false to skip this task entirely\n // rubric \u2014 \"full\" (default), \"abbreviated\" (faster), or \"none\"\n baseline: {\n enabled: true,\n rubric: \"full\",\n },\n\n // Example tasks ship as drafts so they don't run in production evals.\n // Change to \"active\" (or remove this field) to activate.\n status: \"draft\",\n})\n";
202
217
  /** Generated YAML for example-groq-blog-listing (from parsed TS data) */
203
- export declare const exampleGroqBlogListingYaml = "- mode: literacy\n id: example-groq-blog-listing\n title: Blog listing with GROQ queries\n description: Example \u2014 tests GROQ blog listing implementation\n area: groq\n context:\n docs:\n - slug: groq-introduction\n reason: Core GROQ syntax and query language reference\n - slug: how-queries-work\n reason: Query execution model and best practices\n docCoverage: true\n referenceSolution: canonical/example-groq-blog-listing.ts\n prompt:\n text: |-\n Create a Next.js page component that lists blog posts from Sanity\n using GROQ. The page should display the title, slug, and published\n date for each post, sorted by most recent first. Use the Sanity\n client to fetch data.\n assertions:\n - type: llm-rubric\n template: task-completion\n criteria:\n - Uses the groq tagged template literal\n - Fetches blog posts with title, slug, and publishedAt fields\n - Orders results by publishedAt in descending order\n - type: llm-rubric\n template: code-correctness\n criteria:\n - Uses createClient from @sanity/client or next-sanity\n - Exports a valid Next.js page component\n baseline:\n enabled: true\n rubric: full\n status: draft\n";
218
+ export declare const exampleGroqBlogListingYaml = "- mode: literacy\n id: example-groq-blog-listing\n title: Blog listing with GROQ queries\n description: Example \u2014 tests GROQ blog listing implementation\n area: groq\n context:\n docs:\n - slug: groq-introduction\n reason: Core GROQ syntax and query language reference\n - slug: how-queries-work\n reason: Query execution model and best practices\n docCoverage: true\n referenceSolution: canonical/example-groq-blog-listing.ts\n prompt:\n text: |-\n Create a Next.js page component that lists blog posts from Sanity\n using GROQ. The page should display the title, slug, and published\n date for each post, sorted by most recent first. Use the Sanity\n client to fetch data.\n assertions:\n - type: llm-rubric\n template: task-completion\n criteria:\n - id: uses-groq-template-literal\n text: Uses the groq tagged template literal\n - id: fetches-blog-fields\n text: Fetches blog posts with title, slug, and publishedAt fields\n - id: orders-by-published-at\n text: Orders results by publishedAt in descending order\n - type: llm-rubric\n template: code-correctness\n criteria:\n - id: uses-create-client\n text: Uses createClient from @sanity/client or next-sanity\n - id: exports-nextjs-page\n text: Exports a valid Next.js page component\n baseline:\n enabled: true\n rubric: full\n status: draft\n";
204
219
  /** Parsed task data for example-id-based-ref (JSON-safe) */
205
220
  export declare const exampleIdBasedRefData: readonly [{
206
221
  readonly mode: "literacy";
@@ -226,11 +241,26 @@ export declare const exampleIdBasedRefData: readonly [{
226
241
  readonly assertions: readonly [{
227
242
  readonly type: "llm-rubric";
228
243
  readonly template: "task-completion";
229
- readonly criteria: readonly ["Explains GROQ availability across different Sanity contexts", "Describes custom GROQ function creation and usage", "Notes differences in GROQ support between contexts"];
244
+ readonly criteria: readonly [{
245
+ readonly id: "groq-availability";
246
+ readonly text: "Explains GROQ availability across different Sanity contexts";
247
+ }, {
248
+ readonly id: "custom-groq-functions";
249
+ readonly text: "Describes custom GROQ function creation and usage";
250
+ }, {
251
+ readonly id: "groq-context-differences";
252
+ readonly text: "Notes differences in GROQ support between contexts";
253
+ }];
230
254
  }, {
231
255
  readonly type: "llm-rubric";
232
256
  readonly template: "code-correctness";
233
- readonly criteria: readonly ["GROQ examples use valid syntax", "Custom function examples follow the correct API pattern"];
257
+ readonly criteria: readonly [{
258
+ readonly id: "valid-groq-syntax";
259
+ readonly text: "GROQ examples use valid syntax";
260
+ }, {
261
+ readonly id: "correct-function-api";
262
+ readonly text: "Custom function examples follow the correct API pattern";
263
+ }];
234
264
  }];
235
265
  readonly baseline: {
236
266
  readonly enabled: true;
@@ -239,9 +269,9 @@ export declare const exampleIdBasedRefData: readonly [{
239
269
  readonly status: "draft";
240
270
  }];
241
271
  /** TypeScript task template for example-id-based-ref */
242
- export declare const exampleIdBasedRefTs = "/**\n * Example Task: Document ID-based canonical doc references.\n *\n * Demonstrates using `id` to reference canonical documentation by\n * Sanity document `_id`. This is useful for:\n * - Draft documents that don't have a stable slug yet\n * - Programmatic references from imports or migrations\n * - Documents where you know the _id but not the slug\n *\n * The `id` ref type can also carry optional `slug` and `path` fields\n * as human-readable annotations \u2014 these are NOT used for resolution,\n * only for display in logs and reports.\n *\n * This example task ships as a DRAFT so it does not run in production\n * evaluations automatically. To activate it, change status to \"active\"\n * or remove the status field entirely (defaults to active).\n *\n * @see docs/design-docs/canonical-doc-resolution.md\n */\n\nimport { defineTask } from \"@sanity/ailf\"\n\nexport default defineTask({\n mode: \"literacy\",\n id: \"example-id-based-ref\",\n title: \"GROQ feature support (ID-based doc references)\",\n description: \"Example \u2014 demonstrates ID-based canonical doc references\",\n\n area: \"groq\",\n\n // ID-based canonical doc references.\n //\n // Use the Sanity document _id to reference articles directly.\n // Optional slug/path annotations help humans reading the file\n // but are NOT used for resolution \u2014 only the `id` field matters.\n //\n // These IDs reference real articles in the Sanity docs (next dataset):\n // 0ba88f1b... = \"GROQ feature support across Sanity\"\n // 5b9c2863... = \"Custom GROQ functions\"\n context: {\n docs: [\n {\n id: \"0ba88f1b-d1a7-418a-9267-2e343d01886a\",\n slug: \"groq-feature-support-by-context\", // annotation only\n reason: \"GROQ feature support across different Sanity contexts\",\n },\n {\n id: \"5b9c2863-ef01-4565-af8e-ee54e081ee74\",\n slug: \"custom-groq-functions\", // annotation only\n reason: \"Custom GROQ functions and pipelines\",\n },\n ],\n },\n\n docCoverage: true,\n\n prompt: {\n text: `Explain how GROQ is used across different Sanity contexts.\nCover the following:\n1. Which GROQ features are available in each context (API queries,\n webhooks, custom functions, access control)\n2. How to create and use custom GROQ functions\n3. Any differences in GROQ support between contexts\nProvide examples demonstrating context-specific GROQ patterns.`,\n },\n\n assertions: [\n {\n type: \"llm-rubric\",\n template: \"task-completion\",\n criteria: [\n \"Explains GROQ availability across different Sanity contexts\",\n \"Describes custom GROQ function creation and usage\",\n \"Notes differences in GROQ support between contexts\",\n ],\n },\n {\n type: \"llm-rubric\",\n template: \"code-correctness\",\n criteria: [\n \"GROQ examples use valid syntax\",\n \"Custom function examples follow the correct API pattern\",\n ],\n },\n ],\n\n baseline: { enabled: true, rubric: \"full\" },\n status: \"draft\",\n})\n";
272
+ export declare const exampleIdBasedRefTs = "/**\n * Example Task: Document ID-based canonical doc references.\n *\n * Demonstrates using `id` to reference canonical documentation by\n * Sanity document `_id`. This is useful for:\n * - Draft documents that don't have a stable slug yet\n * - Programmatic references from imports or migrations\n * - Documents where you know the _id but not the slug\n *\n * The `id` ref type can also carry optional `slug` and `path` fields\n * as human-readable annotations \u2014 these are NOT used for resolution,\n * only for display in logs and reports.\n *\n * This example task ships as a DRAFT so it does not run in production\n * evaluations automatically. To activate it, change status to \"active\"\n * or remove the status field entirely (defaults to active).\n *\n * @see docs/design-docs/canonical-doc-resolution.md\n */\n\nimport { defineTask } from \"@sanity/ailf\"\n\nexport default defineTask({\n mode: \"literacy\",\n id: \"example-id-based-ref\",\n title: \"GROQ feature support (ID-based doc references)\",\n description: \"Example \u2014 demonstrates ID-based canonical doc references\",\n\n area: \"groq\",\n\n // ID-based canonical doc references.\n //\n // Use the Sanity document _id to reference articles directly.\n // Optional slug/path annotations help humans reading the file\n // but are NOT used for resolution \u2014 only the `id` field matters.\n //\n // These IDs reference real articles in the Sanity docs (next dataset):\n // 0ba88f1b... = \"GROQ feature support across Sanity\"\n // 5b9c2863... = \"Custom GROQ functions\"\n context: {\n docs: [\n {\n id: \"0ba88f1b-d1a7-418a-9267-2e343d01886a\",\n slug: \"groq-feature-support-by-context\", // annotation only\n reason: \"GROQ feature support across different Sanity contexts\",\n },\n {\n id: \"5b9c2863-ef01-4565-af8e-ee54e081ee74\",\n slug: \"custom-groq-functions\", // annotation only\n reason: \"Custom GROQ functions and pipelines\",\n },\n ],\n },\n\n docCoverage: true,\n\n prompt: {\n text: `Explain how GROQ is used across different Sanity contexts.\nCover the following:\n1. Which GROQ features are available in each context (API queries,\n webhooks, custom functions, access control)\n2. How to create and use custom GROQ functions\n3. Any differences in GROQ support between contexts\nProvide examples demonstrating context-specific GROQ patterns.`,\n },\n\n assertions: [\n {\n type: \"llm-rubric\",\n template: \"task-completion\",\n criteria: [\n {\n id: \"groq-availability\",\n text: \"Explains GROQ availability across different Sanity contexts\",\n },\n {\n id: \"custom-groq-functions\",\n text: \"Describes custom GROQ function creation and usage\",\n },\n {\n id: \"groq-context-differences\",\n text: \"Notes differences in GROQ support between contexts\",\n },\n ],\n },\n {\n type: \"llm-rubric\",\n template: \"code-correctness\",\n criteria: [\n { id: \"valid-groq-syntax\", text: \"GROQ examples use valid syntax\" },\n {\n id: \"correct-function-api\",\n text: \"Custom function examples follow the correct API pattern\",\n },\n ],\n },\n ],\n\n baseline: { enabled: true, rubric: \"full\" },\n status: \"draft\",\n})\n";
243
273
  /** Generated YAML for example-id-based-ref (from parsed TS data) */
244
- export declare const exampleIdBasedRefYaml = "- mode: literacy\n id: example-id-based-ref\n title: GROQ feature support (ID-based doc references)\n description: Example \u2014 demonstrates ID-based canonical doc references\n area: groq\n context:\n docs:\n - id: 0ba88f1b-d1a7-418a-9267-2e343d01886a\n slug: groq-feature-support-by-context\n reason: GROQ feature support across different Sanity contexts\n - id: 5b9c2863-ef01-4565-af8e-ee54e081ee74\n slug: custom-groq-functions\n reason: Custom GROQ functions and pipelines\n docCoverage: true\n prompt:\n text: |-\n Explain how GROQ is used across different Sanity contexts.\n Cover the following:\n 1. Which GROQ features are available in each context (API queries,\n webhooks, custom functions, access control)\n 2. How to create and use custom GROQ functions\n 3. Any differences in GROQ support between contexts\n Provide examples demonstrating context-specific GROQ patterns.\n assertions:\n - type: llm-rubric\n template: task-completion\n criteria:\n - Explains GROQ availability across different Sanity contexts\n - Describes custom GROQ function creation and usage\n - Notes differences in GROQ support between contexts\n - type: llm-rubric\n template: code-correctness\n criteria:\n - GROQ examples use valid syntax\n - Custom function examples follow the correct API pattern\n baseline:\n enabled: true\n rubric: full\n status: draft\n";
274
+ export declare const exampleIdBasedRefYaml = "- mode: literacy\n id: example-id-based-ref\n title: GROQ feature support (ID-based doc references)\n description: Example \u2014 demonstrates ID-based canonical doc references\n area: groq\n context:\n docs:\n - id: 0ba88f1b-d1a7-418a-9267-2e343d01886a\n slug: groq-feature-support-by-context\n reason: GROQ feature support across different Sanity contexts\n - id: 5b9c2863-ef01-4565-af8e-ee54e081ee74\n slug: custom-groq-functions\n reason: Custom GROQ functions and pipelines\n docCoverage: true\n prompt:\n text: |-\n Explain how GROQ is used across different Sanity contexts.\n Cover the following:\n 1. Which GROQ features are available in each context (API queries,\n webhooks, custom functions, access control)\n 2. How to create and use custom GROQ functions\n 3. Any differences in GROQ support between contexts\n Provide examples demonstrating context-specific GROQ patterns.\n assertions:\n - type: llm-rubric\n template: task-completion\n criteria:\n - id: groq-availability\n text: Explains GROQ availability across different Sanity contexts\n - id: custom-groq-functions\n text: Describes custom GROQ function creation and usage\n - id: groq-context-differences\n text: Notes differences in GROQ support between contexts\n - type: llm-rubric\n template: code-correctness\n criteria:\n - id: valid-groq-syntax\n text: GROQ examples use valid syntax\n - id: correct-function-api\n text: Custom function examples follow the correct API pattern\n baseline:\n enabled: true\n rubric: full\n status: draft\n";
245
275
  /** Parsed task data for example-knowledge-probe (JSON-safe) */
246
276
  export declare const exampleKnowledgeProbeData: readonly [{
247
277
  readonly mode: "knowledge-probe";
@@ -255,14 +285,23 @@ export declare const exampleKnowledgeProbeData: readonly [{
255
285
  readonly assertions: readonly [{
256
286
  readonly type: "llm-rubric";
257
287
  readonly template: "task-completion";
258
- readonly criteria: readonly ["Demonstrates understanding of GROQ query syntax", "Shows filtering and projection patterns", "Code examples use valid GROQ syntax"];
288
+ readonly criteria: readonly [{
289
+ readonly id: "understands-groq-syntax";
290
+ readonly text: "Demonstrates understanding of GROQ query syntax";
291
+ }, {
292
+ readonly id: "filter-projection-patterns";
293
+ readonly text: "Shows filtering and projection patterns";
294
+ }, {
295
+ readonly id: "valid-groq-examples";
296
+ readonly text: "Code examples use valid GROQ syntax";
297
+ }];
259
298
  }];
260
299
  readonly status: "draft";
261
300
  }];
262
301
  /** TypeScript task template for example-knowledge-probe */
263
- export declare const exampleKnowledgeProbeTs = "/**\n * Example Task: Knowledge probe baseline (DRAFT).\n *\n * Tests what the model knows about a topic without providing documentation.\n * Used to establish a baseline for comparison with literacy evaluations.\n * This task is a DRAFT \u2014 it won't run unless activated or explicitly targeted.\n *\n * To activate: change status to \"active\" or remove the status field.\n */\n\nimport { defineTask } from \"@sanity/ailf\"\n\nexport default defineTask({\n mode: \"knowledge-probe\",\n id: \"example-knowledge-probe\",\n title: \"Model knowledge of GROQ syntax\",\n description: \"Example \u2014 probes baseline model knowledge (draft)\",\n area: \"groq\",\n\n prompt: {\n text: `Explain the GROQ query language used by Sanity. Cover:\n1. Basic query syntax and projections\n2. How to filter and sort results\n3. Common patterns for fetching related documents\nProvide working code examples.`,\n },\n\n assertions: [\n {\n type: \"llm-rubric\",\n template: \"task-completion\",\n criteria: [\n \"Demonstrates understanding of GROQ query syntax\",\n \"Shows filtering and projection patterns\",\n \"Code examples use valid GROQ syntax\",\n ],\n },\n ],\n\n status: \"draft\",\n})\n";
302
+ export declare const exampleKnowledgeProbeTs = "/**\n * Example Task: Knowledge probe baseline (DRAFT).\n *\n * Tests what the model knows about a topic without providing documentation.\n * Used to establish a baseline for comparison with literacy evaluations.\n * This task is a DRAFT \u2014 it won't run unless activated or explicitly targeted.\n *\n * To activate: change status to \"active\" or remove the status field.\n */\n\nimport { defineTask } from \"@sanity/ailf\"\n\nexport default defineTask({\n mode: \"knowledge-probe\",\n id: \"example-knowledge-probe\",\n title: \"Model knowledge of GROQ syntax\",\n description: \"Example \u2014 probes baseline model knowledge (draft)\",\n area: \"groq\",\n\n prompt: {\n text: `Explain the GROQ query language used by Sanity. Cover:\n1. Basic query syntax and projections\n2. How to filter and sort results\n3. Common patterns for fetching related documents\nProvide working code examples.`,\n },\n\n assertions: [\n {\n type: \"llm-rubric\",\n template: \"task-completion\",\n criteria: [\n {\n id: \"understands-groq-syntax\",\n text: \"Demonstrates understanding of GROQ query syntax\",\n },\n {\n id: \"filter-projection-patterns\",\n text: \"Shows filtering and projection patterns\",\n },\n {\n id: \"valid-groq-examples\",\n text: \"Code examples use valid GROQ syntax\",\n },\n ],\n },\n ],\n\n status: \"draft\",\n})\n";
264
303
  /** Generated YAML for example-knowledge-probe (from parsed TS data) */
265
- export declare const exampleKnowledgeProbeYaml = "- mode: knowledge-probe\n id: example-knowledge-probe\n title: Model knowledge of GROQ syntax\n description: Example \u2014 probes baseline model knowledge (draft)\n area: groq\n prompt:\n text: |-\n Explain the GROQ query language used by Sanity. Cover:\n 1. Basic query syntax and projections\n 2. How to filter and sort results\n 3. Common patterns for fetching related documents\n Provide working code examples.\n assertions:\n - type: llm-rubric\n template: task-completion\n criteria:\n - Demonstrates understanding of GROQ query syntax\n - Shows filtering and projection patterns\n - Code examples use valid GROQ syntax\n status: draft\n";
304
+ export declare const exampleKnowledgeProbeYaml = "- mode: knowledge-probe\n id: example-knowledge-probe\n title: Model knowledge of GROQ syntax\n description: Example \u2014 probes baseline model knowledge (draft)\n area: groq\n prompt:\n text: |-\n Explain the GROQ query language used by Sanity. Cover:\n 1. Basic query syntax and projections\n 2. How to filter and sort results\n 3. Common patterns for fetching related documents\n Provide working code examples.\n assertions:\n - type: llm-rubric\n template: task-completion\n criteria:\n - id: understands-groq-syntax\n text: Demonstrates understanding of GROQ query syntax\n - id: filter-projection-patterns\n text: Shows filtering and projection patterns\n - id: valid-groq-examples\n text: Code examples use valid GROQ syntax\n status: draft\n";
266
305
  /** Parsed task data for example-mcp-tool-usage (JSON-safe) */
267
306
  export declare const exampleMcpToolUsageData: readonly [{
268
307
  readonly mode: "mcp-server";
@@ -283,14 +322,23 @@ export declare const exampleMcpToolUsageData: readonly [{
283
322
  readonly assertions: readonly [{
284
323
  readonly type: "llm-rubric";
285
324
  readonly template: "mcp-output-correctness";
286
- readonly criteria: readonly ["Correctly discovers and selects the appropriate tool", "Passes valid arguments to the tool", "Interprets the tool response coherently"];
325
+ readonly criteria: readonly [{
326
+ readonly id: "discovers-correct-tool";
327
+ readonly text: "Correctly discovers and selects the appropriate tool";
328
+ }, {
329
+ readonly id: "passes-valid-arguments";
330
+ readonly text: "Passes valid arguments to the tool";
331
+ }, {
332
+ readonly id: "interprets-response";
333
+ readonly text: "Interprets the tool response coherently";
334
+ }];
287
335
  }];
288
336
  readonly status: "draft";
289
337
  }];
290
338
  /** TypeScript task template for example-mcp-tool-usage */
291
- export declare const exampleMcpToolUsageTs = "/**\n * Example Task: MCP Server tool-use evaluation (DRAFT).\n *\n * Tests whether an LLM can correctly discover and invoke tools from\n * an MCP server. Replace the placeholder serverConfig with your own\n * MCP server's URL and authentication details.\n *\n * Transports:\n * - \"streamable-http\" / \"sse\" \u2014 remote servers (set url + optional headers)\n * - \"stdio\" \u2014 local process (set command instead of url)\n *\n * Authentication:\n * - `headers` \u2014 send arbitrary HTTP headers (e.g., Authorization)\n * - `auth` \u2014 structured auth config (bearer, basic, api_key, oauth)\n * Values support {{env.VAR}} syntax so secrets stay out of source control.\n *\n * This task is a DRAFT \u2014 it won't run unless activated or explicitly targeted.\n * To activate: change status to \"active\" or remove the status field.\n */\n\nimport { defineTask } from \"@sanity/ailf\"\n\nexport default defineTask({\n mode: \"mcp-server\",\n id: \"example-mcp-tool-usage\",\n title: \"MCP tool discovery and invocation\",\n description: \"Example \u2014 tests MCP server tool-use (draft)\",\n area: \"mcp\",\n\n // \u2500\u2500 Server configuration \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n // Replace the URL and headers below with your MCP server's details.\n //\n // For a local stdio server, use:\n // transport: \"stdio\",\n // command: \"node dist/my-mcp-server.js\",\n serverConfig: {\n transport: \"streamable-http\",\n url: \"https://your-mcp-server.example.com\",\n headers: {\n Authorization: \"Bearer {{env.MCP_AUTH_TOKEN}}\",\n },\n },\n\n // \u2500\u2500 Capabilities \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n // Restrict which MCP tools the model can call. If omitted, all\n // tools discovered from the server are available.\n // capabilities: [\"tool_a\", \"tool_b\"],\n\n prompt: {\n text: `Use the available MCP tools to complete the task.\nReplace this prompt with instructions specific to your MCP server.`,\n },\n\n assertions: [\n {\n type: \"llm-rubric\",\n template: \"mcp-output-correctness\",\n criteria: [\n \"Correctly discovers and selects the appropriate tool\",\n \"Passes valid arguments to the tool\",\n \"Interprets the tool response coherently\",\n ],\n },\n ],\n\n status: \"draft\",\n})\n";
339
+ export declare const exampleMcpToolUsageTs = "/**\n * Example Task: MCP Server tool-use evaluation (DRAFT).\n *\n * Tests whether an LLM can correctly discover and invoke tools from\n * an MCP server. Replace the placeholder serverConfig with your own\n * MCP server's URL and authentication details.\n *\n * Transports:\n * - \"streamable-http\" / \"sse\" \u2014 remote servers (set url + optional headers)\n * - \"stdio\" \u2014 local process (set command instead of url)\n *\n * Authentication:\n * - `headers` \u2014 send arbitrary HTTP headers (e.g., Authorization)\n * - `auth` \u2014 structured auth config (bearer, basic, api_key, oauth)\n * Values support {{env.VAR}} syntax so secrets stay out of source control.\n *\n * This task is a DRAFT \u2014 it won't run unless activated or explicitly targeted.\n * To activate: change status to \"active\" or remove the status field.\n */\n\nimport { defineTask } from \"@sanity/ailf\"\n\nexport default defineTask({\n mode: \"mcp-server\",\n id: \"example-mcp-tool-usage\",\n title: \"MCP tool discovery and invocation\",\n description: \"Example \u2014 tests MCP server tool-use (draft)\",\n area: \"mcp\",\n\n // \u2500\u2500 Server configuration \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n // Replace the URL and headers below with your MCP server's details.\n //\n // For a local stdio server, use:\n // transport: \"stdio\",\n // command: \"node dist/my-mcp-server.js\",\n serverConfig: {\n transport: \"streamable-http\",\n url: \"https://your-mcp-server.example.com\",\n headers: {\n Authorization: \"Bearer {{env.MCP_AUTH_TOKEN}}\",\n },\n },\n\n // \u2500\u2500 Capabilities \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n // Restrict which MCP tools the model can call. If omitted, all\n // tools discovered from the server are available.\n // capabilities: [\"tool_a\", \"tool_b\"],\n\n prompt: {\n text: `Use the available MCP tools to complete the task.\nReplace this prompt with instructions specific to your MCP server.`,\n },\n\n assertions: [\n {\n type: \"llm-rubric\",\n template: \"mcp-output-correctness\",\n criteria: [\n {\n id: \"discovers-correct-tool\",\n text: \"Correctly discovers and selects the appropriate tool\",\n },\n {\n id: \"passes-valid-arguments\",\n text: \"Passes valid arguments to the tool\",\n },\n {\n id: \"interprets-response\",\n text: \"Interprets the tool response coherently\",\n },\n ],\n },\n ],\n\n status: \"draft\",\n})\n";
292
340
  /** Generated YAML for example-mcp-tool-usage (from parsed TS data) */
293
- export declare const exampleMcpToolUsageYaml = "- mode: mcp-server\n id: example-mcp-tool-usage\n title: MCP tool discovery and invocation\n description: Example \u2014 tests MCP server tool-use (draft)\n area: mcp\n serverConfig:\n transport: streamable-http\n url: https://your-mcp-server.example.com\n headers:\n Authorization: Bearer {{env.MCP_AUTH_TOKEN}}\n prompt:\n text: |-\n Use the available MCP tools to complete the task.\n Replace this prompt with instructions specific to your MCP server.\n assertions:\n - type: llm-rubric\n template: mcp-output-correctness\n criteria:\n - Correctly discovers and selects the appropriate tool\n - Passes valid arguments to the tool\n - Interprets the tool response coherently\n status: draft\n";
341
+ export declare const exampleMcpToolUsageYaml = "- mode: mcp-server\n id: example-mcp-tool-usage\n title: MCP tool discovery and invocation\n description: Example \u2014 tests MCP server tool-use (draft)\n area: mcp\n serverConfig:\n transport: streamable-http\n url: https://your-mcp-server.example.com\n headers:\n Authorization: Bearer {{env.MCP_AUTH_TOKEN}}\n prompt:\n text: |-\n Use the available MCP tools to complete the task.\n Replace this prompt with instructions specific to your MCP server.\n assertions:\n - type: llm-rubric\n template: mcp-output-correctness\n criteria:\n - id: discovers-correct-tool\n text: Correctly discovers and selects the appropriate tool\n - id: passes-valid-arguments\n text: Passes valid arguments to the tool\n - id: interprets-response\n text: Interprets the tool response coherently\n status: draft\n";
294
342
  /** Parsed task data for example-path-based-ref (JSON-safe) */
295
343
  export declare const examplePathBasedRefData: readonly [{
296
344
  readonly mode: "literacy";
@@ -314,11 +362,29 @@ export declare const examplePathBasedRefData: readonly [{
314
362
  readonly assertions: readonly [{
315
363
  readonly type: "llm-rubric";
316
364
  readonly template: "task-completion";
317
- readonly criteria: readonly ["Explains create, createOrReplace, patch, and delete mutations", "Describes required document fields (_id, _type)", "Shows patch operations for field-level updates", "Includes practical code examples"];
365
+ readonly criteria: readonly [{
366
+ readonly id: "explains-mutations";
367
+ readonly text: "Explains create, createOrReplace, patch, and delete mutations";
368
+ }, {
369
+ readonly id: "required-document-fields";
370
+ readonly text: "Describes required document fields (_id, _type)";
371
+ }, {
372
+ readonly id: "patch-operations";
373
+ readonly text: "Shows patch operations for field-level updates";
374
+ }, {
375
+ readonly id: "practical-code-examples";
376
+ readonly text: "Includes practical code examples";
377
+ }];
318
378
  }, {
319
379
  readonly type: "llm-rubric";
320
380
  readonly template: "code-correctness";
321
- readonly criteria: readonly ["Uses correct @sanity/client mutation API", "Patch operations use valid set/unset/inc syntax"];
381
+ readonly criteria: readonly [{
382
+ readonly id: "correct-mutation-api";
383
+ readonly text: "Uses correct @sanity/client mutation API";
384
+ }, {
385
+ readonly id: "valid-patch-syntax";
386
+ readonly text: "Patch operations use valid set/unset/inc syntax";
387
+ }];
322
388
  }];
323
389
  readonly baseline: {
324
390
  readonly enabled: true;
@@ -327,9 +393,9 @@ export declare const examplePathBasedRefData: readonly [{
327
393
  readonly status: "draft";
328
394
  }];
329
395
  /** TypeScript task template for example-path-based-ref */
330
- export declare const examplePathBasedRefTs = "/**\n * Example Task: Path-based canonical doc references.\n *\n * Demonstrates using `path` to reference canonical documentation.\n * Paths are the preferred reference type because they uniquely identify\n * an article across sections (unlike slugs, which can collide).\n *\n * Path format:\n * - Simple: \"webhooks\" \u2192 resolves by slug lookup\n * - Sectioned: \"content-lake/webhooks\" \u2192 disambiguates by section + slug\n *\n * This example demonstrates why paths matter: the slug \"documents\"\n * exists in both the \"content-lake\" and \"cli-reference\" sections.\n * Using \"content-lake/documents\" ensures we get the right one.\n *\n * This example task ships as a DRAFT so it does not run in production\n * evaluations automatically. To activate it, change status to \"active\"\n * or remove the status field entirely (defaults to active).\n *\n * @see docs/design-docs/canonical-doc-resolution.md\n */\n\nimport { defineTask } from \"@sanity/ailf\"\n\nexport default defineTask({\n mode: \"literacy\",\n id: \"example-path-based-ref\",\n title: \"GROQ mutations (path-based doc references)\",\n description: \"Example \u2014 demonstrates path-based canonical doc references\",\n\n area: \"groq\",\n\n // Path-based canonical doc references.\n //\n // Use \"section/slug\" format to uniquely identify articles:\n // - \"content-lake/mutations-introduction\" \u2192 the mutations article\n // - \"content-lake/documents\" \u2192 the documents article in Content Lake\n // (not the CLI \"documents\" article in cli-reference section)\n //\n // The \"documents\" slug exists in two sections \u2014 this is exactly why\n // path-based references are preferred over slug-based references.\n context: {\n docs: [\n {\n path: \"content-lake/mutations-introduction\",\n reason: \"Introduction to document mutations in the Content Lake\",\n },\n {\n path: \"content-lake/documents\",\n reason:\n \"Document structure and types (Content Lake, not CLI reference)\",\n },\n ],\n },\n\n docCoverage: true,\n\n prompt: {\n text: `Explain how to create, update, and delete documents in Sanity's\nContent Lake using mutations. Cover:\n1. The different mutation types (create, createOrReplace, patch, delete)\n2. Document structure and required fields (_id, _type)\n3. How to use patch operations to update specific fields\n4. Best practices for mutation patterns\nProvide working code examples using @sanity/client.`,\n },\n\n assertions: [\n {\n type: \"llm-rubric\",\n template: \"task-completion\",\n criteria: [\n \"Explains create, createOrReplace, patch, and delete mutations\",\n \"Describes required document fields (_id, _type)\",\n \"Shows patch operations for field-level updates\",\n \"Includes practical code examples\",\n ],\n },\n {\n type: \"llm-rubric\",\n template: \"code-correctness\",\n criteria: [\n \"Uses correct @sanity/client mutation API\",\n \"Patch operations use valid set/unset/inc syntax\",\n ],\n },\n ],\n\n baseline: { enabled: true, rubric: \"full\" },\n status: \"draft\",\n})\n";
396
+ export declare const examplePathBasedRefTs = "/**\n * Example Task: Path-based canonical doc references.\n *\n * Demonstrates using `path` to reference canonical documentation.\n * Paths are the preferred reference type because they uniquely identify\n * an article across sections (unlike slugs, which can collide).\n *\n * Path format:\n * - Simple: \"webhooks\" \u2192 resolves by slug lookup\n * - Sectioned: \"content-lake/webhooks\" \u2192 disambiguates by section + slug\n *\n * This example demonstrates why paths matter: the slug \"documents\"\n * exists in both the \"content-lake\" and \"cli-reference\" sections.\n * Using \"content-lake/documents\" ensures we get the right one.\n *\n * This example task ships as a DRAFT so it does not run in production\n * evaluations automatically. To activate it, change status to \"active\"\n * or remove the status field entirely (defaults to active).\n *\n * @see docs/design-docs/canonical-doc-resolution.md\n */\n\nimport { defineTask } from \"@sanity/ailf\"\n\nexport default defineTask({\n mode: \"literacy\",\n id: \"example-path-based-ref\",\n title: \"GROQ mutations (path-based doc references)\",\n description: \"Example \u2014 demonstrates path-based canonical doc references\",\n\n area: \"groq\",\n\n // Path-based canonical doc references.\n //\n // Use \"section/slug\" format to uniquely identify articles:\n // - \"content-lake/mutations-introduction\" \u2192 the mutations article\n // - \"content-lake/documents\" \u2192 the documents article in Content Lake\n // (not the CLI \"documents\" article in cli-reference section)\n //\n // The \"documents\" slug exists in two sections \u2014 this is exactly why\n // path-based references are preferred over slug-based references.\n context: {\n docs: [\n {\n path: \"content-lake/mutations-introduction\",\n reason: \"Introduction to document mutations in the Content Lake\",\n },\n {\n path: \"content-lake/documents\",\n reason:\n \"Document structure and types (Content Lake, not CLI reference)\",\n },\n ],\n },\n\n docCoverage: true,\n\n prompt: {\n text: `Explain how to create, update, and delete documents in Sanity's\nContent Lake using mutations. Cover:\n1. The different mutation types (create, createOrReplace, patch, delete)\n2. Document structure and required fields (_id, _type)\n3. How to use patch operations to update specific fields\n4. Best practices for mutation patterns\nProvide working code examples using @sanity/client.`,\n },\n\n assertions: [\n {\n type: \"llm-rubric\",\n template: \"task-completion\",\n criteria: [\n {\n id: \"explains-mutations\",\n text: \"Explains create, createOrReplace, patch, and delete mutations\",\n },\n {\n id: \"required-document-fields\",\n text: \"Describes required document fields (_id, _type)\",\n },\n {\n id: \"patch-operations\",\n text: \"Shows patch operations for field-level updates\",\n },\n {\n id: \"practical-code-examples\",\n text: \"Includes practical code examples\",\n },\n ],\n },\n {\n type: \"llm-rubric\",\n template: \"code-correctness\",\n criteria: [\n {\n id: \"correct-mutation-api\",\n text: \"Uses correct @sanity/client mutation API\",\n },\n {\n id: \"valid-patch-syntax\",\n text: \"Patch operations use valid set/unset/inc syntax\",\n },\n ],\n },\n ],\n\n baseline: { enabled: true, rubric: \"full\" },\n status: \"draft\",\n})\n";
331
397
  /** Generated YAML for example-path-based-ref (from parsed TS data) */
332
- export declare const examplePathBasedRefYaml = "- mode: literacy\n id: example-path-based-ref\n title: GROQ mutations (path-based doc references)\n description: Example \u2014 demonstrates path-based canonical doc references\n area: groq\n context:\n docs:\n - path: content-lake/mutations-introduction\n reason: Introduction to document mutations in the Content Lake\n - path: content-lake/documents\n reason: Document structure and types (Content Lake, not CLI reference)\n docCoverage: true\n prompt:\n text: |-\n Explain how to create, update, and delete documents in Sanity's\n Content Lake using mutations. Cover:\n 1. The different mutation types (create, createOrReplace, patch, delete)\n 2. Document structure and required fields (_id, _type)\n 3. How to use patch operations to update specific fields\n 4. Best practices for mutation patterns\n Provide working code examples using @sanity/client.\n assertions:\n - type: llm-rubric\n template: task-completion\n criteria:\n - Explains create, createOrReplace, patch, and delete mutations\n - Describes required document fields (_id, _type)\n - Shows patch operations for field-level updates\n - Includes practical code examples\n - type: llm-rubric\n template: code-correctness\n criteria:\n - Uses correct @sanity/client mutation API\n - Patch operations use valid set/unset/inc syntax\n baseline:\n enabled: true\n rubric: full\n status: draft\n";
398
+ export declare const examplePathBasedRefYaml = "- mode: literacy\n id: example-path-based-ref\n title: GROQ mutations (path-based doc references)\n description: Example \u2014 demonstrates path-based canonical doc references\n area: groq\n context:\n docs:\n - path: content-lake/mutations-introduction\n reason: Introduction to document mutations in the Content Lake\n - path: content-lake/documents\n reason: Document structure and types (Content Lake, not CLI reference)\n docCoverage: true\n prompt:\n text: |-\n Explain how to create, update, and delete documents in Sanity's\n Content Lake using mutations. Cover:\n 1. The different mutation types (create, createOrReplace, patch, delete)\n 2. Document structure and required fields (_id, _type)\n 3. How to use patch operations to update specific fields\n 4. Best practices for mutation patterns\n Provide working code examples using @sanity/client.\n assertions:\n - type: llm-rubric\n template: task-completion\n criteria:\n - id: explains-mutations\n text: Explains create, createOrReplace, patch, and delete mutations\n - id: required-document-fields\n text: Describes required document fields (_id, _type)\n - id: patch-operations\n text: Shows patch operations for field-level updates\n - id: practical-code-examples\n text: Includes practical code examples\n - type: llm-rubric\n template: code-correctness\n criteria:\n - id: correct-mutation-api\n text: Uses correct @sanity/client mutation API\n - id: valid-patch-syntax\n text: Patch operations use valid set/unset/inc syntax\n baseline:\n enabled: true\n rubric: full\n status: draft\n";
333
399
  /** Parsed task data for example-perspective-ref (JSON-safe) */
334
400
  export declare const examplePerspectiveRefData: readonly [{
335
401
  readonly mode: "literacy";
@@ -353,11 +419,26 @@ export declare const examplePerspectiveRefData: readonly [{
353
419
  readonly assertions: readonly [{
354
420
  readonly type: "llm-rubric";
355
421
  readonly template: "task-completion";
356
- readonly criteria: readonly ["Demonstrates GROQ join syntax for cross-document queries", "Shows GROQ filter patterns for webhook configuration", "Includes practical query examples from cheat sheet patterns"];
422
+ readonly criteria: readonly [{
423
+ readonly id: "groq-join-syntax";
424
+ readonly text: "Demonstrates GROQ join syntax for cross-document queries";
425
+ }, {
426
+ readonly id: "groq-filter-patterns";
427
+ readonly text: "Shows GROQ filter patterns for webhook configuration";
428
+ }, {
429
+ readonly id: "practical-query-examples";
430
+ readonly text: "Includes practical query examples from cheat sheet patterns";
431
+ }];
357
432
  }, {
358
433
  readonly type: "llm-rubric";
359
434
  readonly template: "code-correctness";
360
- readonly criteria: readonly ["All GROQ queries use valid syntax", "Reference joins use correct dereference operator (->)"];
435
+ readonly criteria: readonly [{
436
+ readonly id: "valid-groq-syntax";
437
+ readonly text: "All GROQ queries use valid syntax";
438
+ }, {
439
+ readonly id: "correct-dereference-operator";
440
+ readonly text: "Reference joins use correct dereference operator (->)";
441
+ }];
361
442
  }];
362
443
  readonly baseline: {
363
444
  readonly enabled: true;
@@ -366,9 +447,9 @@ export declare const examplePerspectiveRefData: readonly [{
366
447
  readonly status: "draft";
367
448
  }];
368
449
  /** TypeScript task template for example-perspective-ref */
369
- export declare const examplePerspectiveRefTs = "/**\n * Example Task: Perspective / content release doc references.\n *\n * Demonstrates using `perspective` to reference all documentation\n * articles within a content release. This is the key capability for\n * evaluating NEW feature documentation before it's published.\n *\n * How it works:\n * - A perspective ref is one-to-many: the doc fetcher queries the\n * named release and expands it to ALL articles versioned within it.\n * - Downstream consumers see the same flat DocContext[] regardless\n * of how docs were resolved.\n * - When the release is published, the perspective entry becomes a\n * no-op (articles are now in published). Migrate to explicit path\n * or slug refs at your convenience.\n *\n * This example task ships as a DRAFT so it does not run in production\n * evaluations automatically. To activate it, change status to \"active\"\n * or remove the status field entirely (defaults to active).\n *\n * @see docs/design-docs/canonical-doc-resolution.md\n */\n\nimport { defineTask } from \"@sanity/ailf\"\n\nexport default defineTask({\n mode: \"literacy\",\n id: \"example-perspective-ref\",\n title:\n \"GROQ features from content release (perspective-based doc references)\",\n description:\n \"Example \u2014 demonstrates perspective-based canonical doc references\",\n\n area: \"groq\",\n\n // Perspective-based canonical doc reference.\n //\n // The perspective ID references a content release in the Sanity\n // Content Lake. At evaluation time, the doc fetcher auto-discovers\n // all articles versioned in this release and includes them as\n // canonical documentation context.\n //\n // Release rE9TSJvR4 contains:\n // - \"GROQ-powered webhooks\" (webhooks)\n // - \"Query Cheat Sheet - GROQ\" (query-cheat-sheet)\n // - \"GROQ joins\" (groq-joins)\n //\n // You can combine perspective refs with explicit slug/path/id refs\n // to include foundational published docs alongside release content.\n // Here we add groq-data-types as a complementary published reference.\n context: {\n docs: [\n {\n perspective: \"rE9TSJvR4\",\n reason: \"All GROQ documentation updates in the test content release\",\n },\n {\n slug: \"groq-data-types\",\n reason: \"GROQ data type reference (published, stable)\",\n },\n ],\n },\n\n docCoverage: true,\n\n prompt: {\n text: `Using GROQ, demonstrate advanced query patterns including:\n1. Joining data across document types using references\n2. Filtering webhook payloads with GROQ projections\n3. Using the query cheat sheet patterns for common operations\n4. Working with different GROQ data types in filters\nProvide working GROQ query examples for each pattern.`,\n },\n\n assertions: [\n {\n type: \"llm-rubric\",\n template: \"task-completion\",\n criteria: [\n \"Demonstrates GROQ join syntax for cross-document queries\",\n \"Shows GROQ filter patterns for webhook configuration\",\n \"Includes practical query examples from cheat sheet patterns\",\n ],\n },\n {\n type: \"llm-rubric\",\n template: \"code-correctness\",\n criteria: [\n \"All GROQ queries use valid syntax\",\n \"Reference joins use correct dereference operator (->)\",\n ],\n },\n ],\n\n baseline: { enabled: true, rubric: \"full\" },\n status: \"draft\",\n})\n";
450
+ export declare const examplePerspectiveRefTs = "/**\n * Example Task: Perspective / content release doc references.\n *\n * Demonstrates using `perspective` to reference all documentation\n * articles within a content release. This is the key capability for\n * evaluating NEW feature documentation before it's published.\n *\n * How it works:\n * - A perspective ref is one-to-many: the doc fetcher queries the\n * named release and expands it to ALL articles versioned within it.\n * - Downstream consumers see the same flat DocContext[] regardless\n * of how docs were resolved.\n * - When the release is published, the perspective entry becomes a\n * no-op (articles are now in published). Migrate to explicit path\n * or slug refs at your convenience.\n *\n * This example task ships as a DRAFT so it does not run in production\n * evaluations automatically. To activate it, change status to \"active\"\n * or remove the status field entirely (defaults to active).\n *\n * @see docs/design-docs/canonical-doc-resolution.md\n */\n\nimport { defineTask } from \"@sanity/ailf\"\n\nexport default defineTask({\n mode: \"literacy\",\n id: \"example-perspective-ref\",\n title:\n \"GROQ features from content release (perspective-based doc references)\",\n description:\n \"Example \u2014 demonstrates perspective-based canonical doc references\",\n\n area: \"groq\",\n\n // Perspective-based canonical doc reference.\n //\n // The perspective ID references a content release in the Sanity\n // Content Lake. At evaluation time, the doc fetcher auto-discovers\n // all articles versioned in this release and includes them as\n // canonical documentation context.\n //\n // Release rE9TSJvR4 contains:\n // - \"GROQ-powered webhooks\" (webhooks)\n // - \"Query Cheat Sheet - GROQ\" (query-cheat-sheet)\n // - \"GROQ joins\" (groq-joins)\n //\n // You can combine perspective refs with explicit slug/path/id refs\n // to include foundational published docs alongside release content.\n // Here we add groq-data-types as a complementary published reference.\n context: {\n docs: [\n {\n perspective: \"rE9TSJvR4\",\n reason: \"All GROQ documentation updates in the test content release\",\n },\n {\n slug: \"groq-data-types\",\n reason: \"GROQ data type reference (published, stable)\",\n },\n ],\n },\n\n docCoverage: true,\n\n prompt: {\n text: `Using GROQ, demonstrate advanced query patterns including:\n1. Joining data across document types using references\n2. Filtering webhook payloads with GROQ projections\n3. Using the query cheat sheet patterns for common operations\n4. Working with different GROQ data types in filters\nProvide working GROQ query examples for each pattern.`,\n },\n\n assertions: [\n {\n type: \"llm-rubric\",\n template: \"task-completion\",\n criteria: [\n {\n id: \"groq-join-syntax\",\n text: \"Demonstrates GROQ join syntax for cross-document queries\",\n },\n {\n id: \"groq-filter-patterns\",\n text: \"Shows GROQ filter patterns for webhook configuration\",\n },\n {\n id: \"practical-query-examples\",\n text: \"Includes practical query examples from cheat sheet patterns\",\n },\n ],\n },\n {\n type: \"llm-rubric\",\n template: \"code-correctness\",\n criteria: [\n { id: \"valid-groq-syntax\", text: \"All GROQ queries use valid syntax\" },\n {\n id: \"correct-dereference-operator\",\n text: \"Reference joins use correct dereference operator (->)\",\n },\n ],\n },\n ],\n\n baseline: { enabled: true, rubric: \"full\" },\n status: \"draft\",\n})\n";
370
451
  /** Generated YAML for example-perspective-ref (from parsed TS data) */
371
- export declare const examplePerspectiveRefYaml = "- mode: literacy\n id: example-perspective-ref\n title: GROQ features from content release (perspective-based doc references)\n description: Example \u2014 demonstrates perspective-based canonical doc references\n area: groq\n context:\n docs:\n - perspective: rE9TSJvR4\n reason: All GROQ documentation updates in the test content release\n - slug: groq-data-types\n reason: GROQ data type reference (published, stable)\n docCoverage: true\n prompt:\n text: |-\n Using GROQ, demonstrate advanced query patterns including:\n 1. Joining data across document types using references\n 2. Filtering webhook payloads with GROQ projections\n 3. Using the query cheat sheet patterns for common operations\n 4. Working with different GROQ data types in filters\n Provide working GROQ query examples for each pattern.\n assertions:\n - type: llm-rubric\n template: task-completion\n criteria:\n - Demonstrates GROQ join syntax for cross-document queries\n - Shows GROQ filter patterns for webhook configuration\n - Includes practical query examples from cheat sheet patterns\n - type: llm-rubric\n template: code-correctness\n criteria:\n - All GROQ queries use valid syntax\n - Reference joins use correct dereference operator (->)\n baseline:\n enabled: true\n rubric: full\n status: draft\n";
452
+ export declare const examplePerspectiveRefYaml = "- mode: literacy\n id: example-perspective-ref\n title: GROQ features from content release (perspective-based doc references)\n description: Example \u2014 demonstrates perspective-based canonical doc references\n area: groq\n context:\n docs:\n - perspective: rE9TSJvR4\n reason: All GROQ documentation updates in the test content release\n - slug: groq-data-types\n reason: GROQ data type reference (published, stable)\n docCoverage: true\n prompt:\n text: |-\n Using GROQ, demonstrate advanced query patterns including:\n 1. Joining data across document types using references\n 2. Filtering webhook payloads with GROQ projections\n 3. Using the query cheat sheet patterns for common operations\n 4. Working with different GROQ data types in filters\n Provide working GROQ query examples for each pattern.\n assertions:\n - type: llm-rubric\n template: task-completion\n criteria:\n - id: groq-join-syntax\n text: Demonstrates GROQ join syntax for cross-document queries\n - id: groq-filter-patterns\n text: Shows GROQ filter patterns for webhook configuration\n - id: practical-query-examples\n text: Includes practical query examples from cheat sheet patterns\n - type: llm-rubric\n template: code-correctness\n criteria:\n - id: valid-groq-syntax\n text: All GROQ queries use valid syntax\n - id: correct-dereference-operator\n text: Reference joins use correct dereference operator (->)\n baseline:\n enabled: true\n rubric: full\n status: draft\n";
372
453
  /** Parsed task data for example-studio-custom-input (JSON-safe) */
373
454
  export declare const exampleStudioCustomInputData: readonly [{
374
455
  readonly mode: "literacy";
@@ -393,11 +474,29 @@ export declare const exampleStudioCustomInputData: readonly [{
393
474
  readonly assertions: readonly [{
394
475
  readonly type: "llm-rubric";
395
476
  readonly template: "task-completion";
396
- readonly criteria: readonly ["Implements a React component that renders a text input", "Displays a live character count", "Reads maxLength from schema options", "Shows a visual warning when limit is exceeded"];
477
+ readonly criteria: readonly [{
478
+ readonly id: "renders-text-input";
479
+ readonly text: "Implements a React component that renders a text input";
480
+ }, {
481
+ readonly id: "displays-character-count";
482
+ readonly text: "Displays a live character count";
483
+ }, {
484
+ readonly id: "reads-max-length";
485
+ readonly text: "Reads maxLength from schema options";
486
+ }, {
487
+ readonly id: "shows-visual-warning";
488
+ readonly text: "Shows a visual warning when limit is exceeded";
489
+ }];
397
490
  }, {
398
491
  readonly type: "llm-rubric";
399
492
  readonly template: "code-correctness";
400
- readonly criteria: readonly ["Uses the Sanity UI library for styling", "Calls onChange with patch operations"];
493
+ readonly criteria: readonly [{
494
+ readonly id: "uses-sanity-ui";
495
+ readonly text: "Uses the Sanity UI library for styling";
496
+ }, {
497
+ readonly id: "calls-onchange-patches";
498
+ readonly text: "Calls onChange with patch operations";
499
+ }];
401
500
  }];
402
501
  readonly baseline: {
403
502
  readonly enabled: true;
@@ -406,9 +505,9 @@ export declare const exampleStudioCustomInputData: readonly [{
406
505
  readonly status: "draft";
407
506
  }];
408
507
  /** TypeScript task template for example-studio-custom-input */
409
- export declare const exampleStudioCustomInputTs = "/**\n * Example Task: Custom input component in Sanity Studio.\n *\n * This is a starter template \u2014 edit it for your own documentation.\n * Delete this file or replace it with your own tasks.\n *\n * This example task ships as a DRAFT so it does not run in production\n * evaluations automatically. To activate it, change status to \"active\"\n * or remove the status field entirely (defaults to active).\n */\n\nimport { defineTask } from \"@sanity/ailf\"\n\nexport default defineTask({\n mode: \"literacy\",\n id: \"example-studio-custom-input\",\n title: \"Custom input component in Sanity Studio\",\n description: \"Example \u2014 tests Studio custom input implementation\",\n\n area: \"studio\",\n\n context: {\n docs: [\n {\n slug: \"custom-input-widgets\",\n reason: \"Guide for building custom form inputs in Sanity Studio\",\n },\n {\n slug: \"form-components\",\n reason: \"Form component API and customization patterns\",\n },\n ],\n },\n\n docCoverage: true,\n referenceSolution: \"canonical/example-studio-custom-input.ts\",\n\n prompt: {\n text: `Build a custom string input component for Sanity Studio that shows\na character count below the input field. The component should accept\na maxLength option from the field schema and display a warning when\nthe text exceeds the limit.`,\n },\n\n assertions: [\n {\n type: \"llm-rubric\",\n template: \"task-completion\",\n criteria: [\n \"Implements a React component that renders a text input\",\n \"Displays a live character count\",\n \"Reads maxLength from schema options\",\n \"Shows a visual warning when limit is exceeded\",\n ],\n },\n {\n type: \"llm-rubric\",\n template: \"code-correctness\",\n criteria: [\n \"Uses the Sanity UI library for styling\",\n \"Calls onChange with patch operations\",\n ],\n },\n ],\n\n baseline: { enabled: true, rubric: \"full\" },\n status: \"draft\",\n})\n";
508
+ export declare const exampleStudioCustomInputTs = "/**\n * Example Task: Custom input component in Sanity Studio.\n *\n * This is a starter template \u2014 edit it for your own documentation.\n * Delete this file or replace it with your own tasks.\n *\n * This example task ships as a DRAFT so it does not run in production\n * evaluations automatically. To activate it, change status to \"active\"\n * or remove the status field entirely (defaults to active).\n */\n\nimport { defineTask } from \"@sanity/ailf\"\n\nexport default defineTask({\n mode: \"literacy\",\n id: \"example-studio-custom-input\",\n title: \"Custom input component in Sanity Studio\",\n description: \"Example \u2014 tests Studio custom input implementation\",\n\n area: \"studio\",\n\n context: {\n docs: [\n {\n slug: \"custom-input-widgets\",\n reason: \"Guide for building custom form inputs in Sanity Studio\",\n },\n {\n slug: \"form-components\",\n reason: \"Form component API and customization patterns\",\n },\n ],\n },\n\n docCoverage: true,\n referenceSolution: \"canonical/example-studio-custom-input.ts\",\n\n prompt: {\n text: `Build a custom string input component for Sanity Studio that shows\na character count below the input field. The component should accept\na maxLength option from the field schema and display a warning when\nthe text exceeds the limit.`,\n },\n\n assertions: [\n {\n type: \"llm-rubric\",\n template: \"task-completion\",\n criteria: [\n {\n id: \"renders-text-input\",\n text: \"Implements a React component that renders a text input\",\n },\n {\n id: \"displays-character-count\",\n text: \"Displays a live character count\",\n },\n { id: \"reads-max-length\", text: \"Reads maxLength from schema options\" },\n {\n id: \"shows-visual-warning\",\n text: \"Shows a visual warning when limit is exceeded\",\n },\n ],\n },\n {\n type: \"llm-rubric\",\n template: \"code-correctness\",\n criteria: [\n {\n id: \"uses-sanity-ui\",\n text: \"Uses the Sanity UI library for styling\",\n },\n {\n id: \"calls-onchange-patches\",\n text: \"Calls onChange with patch operations\",\n },\n ],\n },\n ],\n\n baseline: { enabled: true, rubric: \"full\" },\n status: \"draft\",\n})\n";
410
509
  /** Generated YAML for example-studio-custom-input (from parsed TS data) */
411
- export declare const exampleStudioCustomInputYaml = "- mode: literacy\n id: example-studio-custom-input\n title: Custom input component in Sanity Studio\n description: Example \u2014 tests Studio custom input implementation\n area: studio\n context:\n docs:\n - slug: custom-input-widgets\n reason: Guide for building custom form inputs in Sanity Studio\n - slug: form-components\n reason: Form component API and customization patterns\n docCoverage: true\n referenceSolution: canonical/example-studio-custom-input.ts\n prompt:\n text: |-\n Build a custom string input component for Sanity Studio that shows\n a character count below the input field. The component should accept\n a maxLength option from the field schema and display a warning when\n the text exceeds the limit.\n assertions:\n - type: llm-rubric\n template: task-completion\n criteria:\n - Implements a React component that renders a text input\n - Displays a live character count\n - Reads maxLength from schema options\n - Shows a visual warning when limit is exceeded\n - type: llm-rubric\n template: code-correctness\n criteria:\n - Uses the Sanity UI library for styling\n - Calls onChange with patch operations\n baseline:\n enabled: true\n rubric: full\n status: draft\n";
510
+ export declare const exampleStudioCustomInputYaml = "- mode: literacy\n id: example-studio-custom-input\n title: Custom input component in Sanity Studio\n description: Example \u2014 tests Studio custom input implementation\n area: studio\n context:\n docs:\n - slug: custom-input-widgets\n reason: Guide for building custom form inputs in Sanity Studio\n - slug: form-components\n reason: Form component API and customization patterns\n docCoverage: true\n referenceSolution: canonical/example-studio-custom-input.ts\n prompt:\n text: |-\n Build a custom string input component for Sanity Studio that shows\n a character count below the input field. The component should accept\n a maxLength option from the field schema and display a warning when\n the text exceeds the limit.\n assertions:\n - type: llm-rubric\n template: task-completion\n criteria:\n - id: renders-text-input\n text: Implements a React component that renders a text input\n - id: displays-character-count\n text: Displays a live character count\n - id: reads-max-length\n text: Reads maxLength from schema options\n - id: shows-visual-warning\n text: Shows a visual warning when limit is exceeded\n - type: llm-rubric\n template: code-correctness\n criteria:\n - id: uses-sanity-ui\n text: Uses the Sanity UI library for styling\n - id: calls-onchange-patches\n text: Calls onChange with patch operations\n baseline:\n enabled: true\n rubric: full\n status: draft\n";
412
511
  /** All task example data as a flat array (JSON-safe) */
413
512
  export declare const allTaskData: readonly unknown[];
414
513
  /** Map of task ID (filename stem) → raw TypeScript source */