@sanity/ailf 0.1.18 → 0.1.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_vendor/ailf-core/examples/index.d.ts +29 -69
- package/dist/_vendor/ailf-core/examples/index.js +37 -98
- package/dist/_vendor/ailf-core/services/scoring.js +3 -3
- package/dist/pipeline/agent-behavior-report.js +2 -0
- package/dist/pipeline/calculate-scores.js +4 -4
- package/dist/pipeline/expand-tasks.d.ts +2 -0
- package/dist/pipeline/expand-tasks.js +3 -1
- package/dist/pipeline/grader-compare-runner.js +2 -2
- package/dist/pipeline/grader-consistency-runner.js +2 -2
- package/package.json +1 -1
|
@@ -151,30 +151,30 @@ export declare const exampleGroqBlogListingYaml = "# \u2500\u2500\u2500\u2500\u2
|
|
|
151
151
|
/** Parsed task data for example-id-based-ref (JSON-safe) */
|
|
152
152
|
export declare const exampleIdBasedRefData: readonly [{
|
|
153
153
|
readonly id: "example-id-based-ref";
|
|
154
|
-
readonly description: "Example — GROQ
|
|
154
|
+
readonly description: "Example — GROQ feature support (ID-based doc references)";
|
|
155
155
|
readonly featureArea: "groq";
|
|
156
156
|
readonly canonicalDocs: readonly [{
|
|
157
|
-
readonly id: "
|
|
158
|
-
readonly slug: "
|
|
159
|
-
readonly reason: "GROQ
|
|
157
|
+
readonly id: "0ba88f1b-d1a7-418a-9267-2e343d01886a";
|
|
158
|
+
readonly slug: "groq-feature-support-by-context";
|
|
159
|
+
readonly reason: "GROQ feature support across different Sanity contexts";
|
|
160
160
|
}, {
|
|
161
|
-
readonly id: "
|
|
162
|
-
readonly slug: "groq-
|
|
163
|
-
readonly reason: "
|
|
161
|
+
readonly id: "5b9c2863-ef01-4565-af8e-ee54e081ee74";
|
|
162
|
+
readonly slug: "custom-groq-functions";
|
|
163
|
+
readonly reason: "Custom GROQ functions and pipelines";
|
|
164
164
|
}];
|
|
165
165
|
readonly docCoverage: true;
|
|
166
166
|
readonly vars: {
|
|
167
|
-
readonly task: "
|
|
167
|
+
readonly task: "Explain how GROQ is used across different Sanity contexts.\nCover the following:\n1. Which GROQ features are available in each context (API queries,\n webhooks, custom functions, access control)\n2. How to create and use custom GROQ functions\n3. Any differences in GROQ support between contexts\nProvide examples demonstrating context-specific GROQ patterns.\n";
|
|
168
168
|
readonly docs: "";
|
|
169
169
|
};
|
|
170
170
|
readonly assert: readonly [{
|
|
171
171
|
readonly type: "llm-rubric";
|
|
172
172
|
readonly template: "task-completion";
|
|
173
|
-
readonly criteria: readonly ["
|
|
173
|
+
readonly criteria: readonly ["Explains GROQ availability across different Sanity contexts", "Describes custom GROQ function creation and usage", "Notes differences in GROQ support between contexts"];
|
|
174
174
|
}, {
|
|
175
175
|
readonly type: "llm-rubric";
|
|
176
176
|
readonly template: "code-correctness";
|
|
177
|
-
readonly criteria: readonly ["
|
|
177
|
+
readonly criteria: readonly ["GROQ examples use valid syntax", "Custom function examples follow the correct API pattern"];
|
|
178
178
|
}];
|
|
179
179
|
readonly baseline: {
|
|
180
180
|
readonly enabled: true;
|
|
@@ -185,75 +185,32 @@ export declare const exampleIdBasedRefData: readonly [{
|
|
|
185
185
|
};
|
|
186
186
|
}];
|
|
187
187
|
/** Raw YAML string for example-id-based-ref (preserves comments) */
|
|
188
|
-
export declare const exampleIdBasedRefYaml = "# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n# Example Task: Document ID-based canonical doc references\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n#\n# Demonstrates using `id` to reference canonical documentation by\n# Sanity document `_id`. This is useful for:\n# - Draft documents that don't have a stable slug yet\n# - Programmatic references from imports or migrations\n# - Documents where you know the _id but not the slug\n#\n# The `id` ref type can also carry optional `slug` and `path` fields\n# as human-readable annotations \u2014 these are NOT used for resolution,\n# only for display in logs and reports.\n#\n# This example task is DISABLED by default. To enable it, either:\n# 1. Remove the execution.enabled: false line below, or\n# 2. Set execution.enabled: true\n#\n# @see docs/design-docs/canonical-doc-resolution.md\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n- id: example-id-based-ref\n description: \"Example \u2014 GROQ
|
|
189
|
-
/** Parsed task data for example-mixed-ref-types (JSON-safe) */
|
|
190
|
-
export declare const exampleMixedRefTypesData: readonly [{
|
|
191
|
-
readonly id: "example-mixed-ref-types";
|
|
192
|
-
readonly description: "Example — Mixed canonical doc reference types (slug + path + id + perspective)";
|
|
193
|
-
readonly featureArea: "groq";
|
|
194
|
-
readonly canonicalDocs: readonly [{
|
|
195
|
-
readonly slug: "groq-introduction";
|
|
196
|
-
readonly reason: "Core GROQ language overview";
|
|
197
|
-
}, {
|
|
198
|
-
readonly path: "content-lake/how-queries-work";
|
|
199
|
-
readonly reason: "Query execution model (disambiguated by section)";
|
|
200
|
-
}, {
|
|
201
|
-
readonly id: "81b839a4-2fc1-4769-941a-ec4de9276492";
|
|
202
|
-
readonly slug: "query-cheat-sheet";
|
|
203
|
-
readonly reason: "GROQ query patterns cheat sheet (referenced by document ID)";
|
|
204
|
-
}, {
|
|
205
|
-
readonly perspective: "rE9TSJvR4";
|
|
206
|
-
readonly reason: "Additional GROQ docs from the test content release";
|
|
207
|
-
}];
|
|
208
|
-
readonly docCoverage: true;
|
|
209
|
-
readonly vars: {
|
|
210
|
-
readonly task: "Create a comprehensive GROQ query guide that covers:\n1. Basic query structure and filtering by document type\n2. Projections and field selection\n3. Ordering, slicing, and pagination\n4. Reference joins and nested data\n5. Webhook GROQ filter configuration\nInclude working examples for each concept and explain the\nquery execution model.\n";
|
|
211
|
-
readonly docs: "";
|
|
212
|
-
};
|
|
213
|
-
readonly assert: readonly [{
|
|
214
|
-
readonly type: "llm-rubric";
|
|
215
|
-
readonly template: "task-completion";
|
|
216
|
-
readonly criteria: readonly ["Covers basic GROQ query structure with _type filtering", "Demonstrates projections with field aliasing", "Shows ordering with order() and slice syntax", "Includes reference joins with the dereference operator", "Explains webhook GROQ filter patterns"];
|
|
217
|
-
}, {
|
|
218
|
-
readonly type: "llm-rubric";
|
|
219
|
-
readonly template: "code-correctness";
|
|
220
|
-
readonly criteria: readonly ["All GROQ queries use valid syntax", "Examples are practical and self-contained"];
|
|
221
|
-
}];
|
|
222
|
-
readonly baseline: {
|
|
223
|
-
readonly enabled: true;
|
|
224
|
-
readonly rubric: "abbreviated";
|
|
225
|
-
};
|
|
226
|
-
readonly execution: {
|
|
227
|
-
readonly enabled: false;
|
|
228
|
-
};
|
|
229
|
-
}];
|
|
230
|
-
/** Raw YAML string for example-mixed-ref-types (preserves comments) */
|
|
231
|
-
export declare const exampleMixedRefTypesYaml = "# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n# Example Task: All canonical doc reference types in one task\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n#\n# Demonstrates combining all four canonical doc reference strategies\n# in a single task's canonicalDocs array:\n#\n# slug \u2014 by article slug (simplest, may not be unique)\n# path \u2014 by section/slug path (unique, preferred)\n# id \u2014 by Sanity document _id (for drafts, imports)\n# perspective \u2014 by content release (one-to-many expansion)\n#\n# Each reference type resolves through a different strategy, but the\n# pipeline merges them into a single flat documentation context. The\n# LLM sees the same combined markdown regardless of how docs were found.\n#\n# This example task is DISABLED by default. To enable it, either:\n# 1. Remove the execution.enabled: false line below, or\n# 2. Set execution.enabled: true\n#\n# @see docs/design-docs/canonical-doc-resolution.md\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n- id: example-mixed-ref-types\n description:\n \"Example \u2014 Mixed canonical doc reference types (slug + path + id +\n perspective)\"\n\n featureArea: groq\n\n # All four canonical doc reference types demonstrated together.\n #\n # The pipeline resolves each ref independently:\n # 1. slug refs \u2192 GROQ query by slug.current\n # 2. path refs \u2192 GROQ query by section + slug (or slug fallback)\n # 3. id refs \u2192 GROQ query by _id\n # 4. perspective refs \u2192 expand to all articles in the release\n #\n # After resolution, all entries become a flat list of document slugs.\n # The fetched markdown is concatenated into a single context string.\n canonicalDocs:\n # Slug reference \u2014 simplest form, resolves by article slug\n - slug: groq-introduction\n reason: \"Core GROQ language overview\"\n\n # Path reference \u2014 section-qualified, uniquely identifies the article\n - path: content-lake/how-queries-work\n reason: \"Query execution model (disambiguated by section)\"\n\n # ID reference \u2014 resolves by Sanity document _id\n # Optional slug annotation helps humans reading the YAML\n - id: \"81b839a4-2fc1-4769-941a-ec4de9276492\"\n slug: query-cheat-sheet\n reason: \"GROQ query patterns cheat sheet (referenced by document ID)\"\n\n # Perspective reference \u2014 expands to all articles in the release\n # This adds webhooks, query-cheat-sheet, and groq-joins from release rE9TSJvR4\n # Note: query-cheat-sheet appears via both the id ref and the perspective ref,\n # but the pipeline deduplicates slugs automatically.\n - perspective: rE9TSJvR4\n reason: \"Additional GROQ docs from the test content release\"\n\n docCoverage: true\n\n vars:\n task: |\n Create a comprehensive GROQ query guide that covers:\n 1. Basic query structure and filtering by document type\n 2. Projections and field selection\n 3. Ordering, slicing, and pagination\n 4. Reference joins and nested data\n 5. Webhook GROQ filter configuration\n Include working examples for each concept and explain the\n query execution model.\n docs: \"\"\n\n assert:\n - type: llm-rubric\n template: task-completion\n criteria:\n - \"Covers basic GROQ query structure with _type filtering\"\n - \"Demonstrates projections with field aliasing\"\n - \"Shows ordering with order() and slice syntax\"\n - \"Includes reference joins with the dereference operator\"\n - \"Explains webhook GROQ filter patterns\"\n\n - type: llm-rubric\n template: code-correctness\n criteria:\n - \"All GROQ queries use valid syntax\"\n - \"Examples are practical and self-contained\"\n\n baseline:\n enabled: true\n rubric: abbreviated\n\n # Example tasks ship disabled so they don't run automatically.\n # Set enabled: true (or remove this block) to activate.\n execution:\n enabled: false\n";
|
|
188
|
+
export declare const exampleIdBasedRefYaml = "# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n# Example Task: Document ID-based canonical doc references\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n#\n# Demonstrates using `id` to reference canonical documentation by\n# Sanity document `_id`. This is useful for:\n# - Draft documents that don't have a stable slug yet\n# - Programmatic references from imports or migrations\n# - Documents where you know the _id but not the slug\n#\n# The `id` ref type can also carry optional `slug` and `path` fields\n# as human-readable annotations \u2014 these are NOT used for resolution,\n# only for display in logs and reports.\n#\n# This example task is DISABLED by default. To enable it, either:\n# 1. Remove the execution.enabled: false line below, or\n# 2. Set execution.enabled: true\n#\n# @see docs/design-docs/canonical-doc-resolution.md\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n- id: example-id-based-ref\n description: \"Example \u2014 GROQ feature support (ID-based doc references)\"\n\n featureArea: groq\n\n # ID-based canonical doc references.\n #\n # Use the Sanity document _id to reference articles directly.\n # Optional slug/path annotations help humans reading the YAML\n # but are NOT used for resolution \u2014 only the `id` field matters.\n #\n # These IDs reference real articles in the Sanity docs (next dataset):\n # 0ba88f1b... = \"GROQ feature support across Sanity\"\n # 5b9c2863... = \"Custom GROQ functions\"\n canonicalDocs:\n - id: \"0ba88f1b-d1a7-418a-9267-2e343d01886a\"\n slug: groq-feature-support-by-context # annotation only \u2014 not used for resolution\n reason: \"GROQ feature support across different Sanity contexts\"\n - id: \"5b9c2863-ef01-4565-af8e-ee54e081ee74\"\n slug: custom-groq-functions # annotation only \u2014 not used for resolution\n reason: \"Custom GROQ functions and pipelines\"\n\n docCoverage: true\n\n vars:\n task: |\n Explain how GROQ is used across different Sanity contexts.\n Cover the following:\n 1. Which GROQ features are available in each context (API queries,\n webhooks, custom functions, access control)\n 2. How to create and use custom GROQ functions\n 3. Any differences in GROQ support between contexts\n Provide examples demonstrating context-specific GROQ patterns.\n docs: \"\"\n\n assert:\n - type: llm-rubric\n template: task-completion\n criteria:\n - \"Explains GROQ availability across different Sanity contexts\"\n - \"Describes custom GROQ function creation and usage\"\n - \"Notes differences in GROQ support between contexts\"\n\n - type: llm-rubric\n template: code-correctness\n criteria:\n - \"GROQ examples use valid syntax\"\n - \"Custom function examples follow the correct API pattern\"\n\n baseline:\n enabled: true\n rubric: abbreviated\n\n # Example tasks ship disabled so they don't run automatically.\n # Set enabled: true (or remove this block) to activate.\n execution:\n enabled: false\n";
|
|
232
189
|
/** Parsed task data for example-path-based-ref (JSON-safe) */
|
|
233
190
|
export declare const examplePathBasedRefData: readonly [{
|
|
234
191
|
readonly id: "example-path-based-ref";
|
|
235
|
-
readonly description: "Example — GROQ
|
|
192
|
+
readonly description: "Example — GROQ mutations (path-based doc references)";
|
|
236
193
|
readonly featureArea: "groq";
|
|
237
194
|
readonly canonicalDocs: readonly [{
|
|
238
|
-
readonly path: "content-lake/
|
|
239
|
-
readonly reason: "
|
|
195
|
+
readonly path: "content-lake/mutations-introduction";
|
|
196
|
+
readonly reason: "Introduction to document mutations in the Content Lake";
|
|
240
197
|
}, {
|
|
241
|
-
readonly path: "content-lake/
|
|
242
|
-
readonly reason: "
|
|
198
|
+
readonly path: "content-lake/documents";
|
|
199
|
+
readonly reason: "Document structure and types (Content Lake, not CLI reference)";
|
|
243
200
|
}];
|
|
244
201
|
readonly docCoverage: true;
|
|
245
202
|
readonly vars: {
|
|
246
|
-
readonly task: "
|
|
203
|
+
readonly task: "Explain how to create, update, and delete documents in Sanity's\nContent Lake using mutations. Cover:\n1. The different mutation types (create, createOrReplace, patch, delete)\n2. Document structure and required fields (_id, _type)\n3. How to use patch operations to update specific fields\n4. Best practices for mutation patterns\nProvide working code examples using @sanity/client.\n";
|
|
247
204
|
readonly docs: "";
|
|
248
205
|
};
|
|
249
206
|
readonly assert: readonly [{
|
|
250
207
|
readonly type: "llm-rubric";
|
|
251
208
|
readonly template: "task-completion";
|
|
252
|
-
readonly criteria: readonly ["
|
|
209
|
+
readonly criteria: readonly ["Explains create, createOrReplace, patch, and delete mutations", "Describes required document fields (_id, _type)", "Shows patch operations for field-level updates", "Includes practical code examples"];
|
|
253
210
|
}, {
|
|
254
211
|
readonly type: "llm-rubric";
|
|
255
212
|
readonly template: "code-correctness";
|
|
256
|
-
readonly criteria: readonly ["
|
|
213
|
+
readonly criteria: readonly ["Uses correct @sanity/client mutation API", "Patch operations use valid set/unset/inc syntax"];
|
|
257
214
|
}];
|
|
258
215
|
readonly baseline: {
|
|
259
216
|
readonly enabled: true;
|
|
@@ -264,7 +221,7 @@ export declare const examplePathBasedRefData: readonly [{
|
|
|
264
221
|
};
|
|
265
222
|
}];
|
|
266
223
|
/** Raw YAML string for example-path-based-ref (preserves comments) */
|
|
267
|
-
export declare const examplePathBasedRefYaml = "# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n# Example Task: Path-based canonical doc references\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n#\n# Demonstrates using `path` to reference canonical documentation.\n# Paths are the preferred reference type because they uniquely identify\n# an article across sections (unlike slugs, which can collide).\n#\n# Path format:\n# - Simple: \"webhooks\" \u2192 resolves by slug lookup\n# - Sectioned: \"content-lake/webhooks\" \u2192 disambiguates by section + slug\n#\n# This example task is DISABLED by default. To enable it, either:\n# 1. Remove the execution.enabled: false line below, or\n# 2. Set execution.enabled: true\n#\n# @see docs/design-docs/canonical-doc-resolution.md\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n- id: example-path-based-ref\n description: \"Example \u2014 GROQ
|
|
224
|
+
export declare const examplePathBasedRefYaml = "# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n# Example Task: Path-based canonical doc references\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n#\n# Demonstrates using `path` to reference canonical documentation.\n# Paths are the preferred reference type because they uniquely identify\n# an article across sections (unlike slugs, which can collide).\n#\n# Path format:\n# - Simple: \"webhooks\" \u2192 resolves by slug lookup\n# - Sectioned: \"content-lake/webhooks\" \u2192 disambiguates by section + slug\n#\n# This example demonstrates why paths matter: the slug \"documents\"\n# exists in both the \"content-lake\" and \"cli-reference\" sections.\n# Using \"content-lake/documents\" ensures we get the right one.\n#\n# This example task is DISABLED by default. To enable it, either:\n# 1. Remove the execution.enabled: false line below, or\n# 2. Set execution.enabled: true\n#\n# @see docs/design-docs/canonical-doc-resolution.md\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n- id: example-path-based-ref\n description: \"Example \u2014 GROQ mutations (path-based doc references)\"\n\n featureArea: groq\n\n # Path-based canonical doc references.\n #\n # Use \"section/slug\" format to uniquely identify articles:\n # - \"content-lake/mutations-introduction\" \u2192 the mutations article\n # - \"content-lake/documents\" \u2192 the documents article in Content Lake\n # (not the CLI \"documents\" article in cli-reference section)\n #\n # The \"documents\" slug exists in two sections \u2014 this is exactly why\n # path-based references are preferred over slug-based references.\n canonicalDocs:\n - path: content-lake/mutations-introduction\n reason: \"Introduction to document mutations in the Content Lake\"\n - path: content-lake/documents\n reason: \"Document structure and types (Content Lake, not CLI reference)\"\n\n docCoverage: true\n\n vars:\n task: |\n Explain how to create, update, and delete documents in Sanity's\n Content Lake using mutations. Cover:\n 1. The different mutation types (create, createOrReplace, patch, delete)\n 2. Document structure and required fields (_id, _type)\n 3. How to use patch operations to update specific fields\n 4. Best practices for mutation patterns\n Provide working code examples using @sanity/client.\n docs: \"\"\n\n assert:\n - type: llm-rubric\n template: task-completion\n criteria:\n - \"Explains create, createOrReplace, patch, and delete mutations\"\n - \"Describes required document fields (_id, _type)\"\n - \"Shows patch operations for field-level updates\"\n - \"Includes practical code examples\"\n\n - type: llm-rubric\n template: code-correctness\n criteria:\n - \"Uses correct @sanity/client mutation API\"\n - \"Patch operations use valid set/unset/inc syntax\"\n\n baseline:\n enabled: true\n rubric: abbreviated\n\n # Example tasks ship disabled so they don't run automatically.\n # Set enabled: true (or remove this block) to activate.\n execution:\n enabled: false\n";
|
|
268
225
|
/** Parsed task data for example-perspective-ref (JSON-safe) */
|
|
269
226
|
export declare const examplePerspectiveRefData: readonly [{
|
|
270
227
|
readonly id: "example-perspective-ref";
|
|
@@ -274,12 +231,12 @@ export declare const examplePerspectiveRefData: readonly [{
|
|
|
274
231
|
readonly perspective: "rE9TSJvR4";
|
|
275
232
|
readonly reason: "All GROQ documentation updates in the test content release";
|
|
276
233
|
}, {
|
|
277
|
-
readonly slug: "groq-
|
|
278
|
-
readonly reason: "
|
|
234
|
+
readonly slug: "groq-data-types";
|
|
235
|
+
readonly reason: "GROQ data type reference (published, stable)";
|
|
279
236
|
}];
|
|
280
237
|
readonly docCoverage: true;
|
|
281
238
|
readonly vars: {
|
|
282
|
-
readonly task: "Using GROQ, demonstrate advanced query patterns including:\n1. Joining data across document types using references\n2. Filtering webhook payloads with GROQ projections\n3. Using the query cheat sheet patterns for common operations\nProvide working GROQ query examples for each pattern.\n";
|
|
239
|
+
readonly task: "Using GROQ, demonstrate advanced query patterns including:\n1. Joining data across document types using references\n2. Filtering webhook payloads with GROQ projections\n3. Using the query cheat sheet patterns for common operations\n4. Working with different GROQ data types in filters\nProvide working GROQ query examples for each pattern.\n";
|
|
283
240
|
readonly docs: "";
|
|
284
241
|
};
|
|
285
242
|
readonly assert: readonly [{
|
|
@@ -300,7 +257,7 @@ export declare const examplePerspectiveRefData: readonly [{
|
|
|
300
257
|
};
|
|
301
258
|
}];
|
|
302
259
|
/** Raw YAML string for example-perspective-ref (preserves comments) */
|
|
303
|
-
export declare const examplePerspectiveRefYaml = "# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n# Example Task: Perspective / content release doc references\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n#\n# Demonstrates using `perspective` to reference all documentation\n# articles within a content release. This is the key capability for\n# evaluating NEW feature documentation before it's published.\n#\n# How it works:\n# - A perspective ref is one-to-many: the doc fetcher queries the\n# named release and expands it to ALL articles versioned within it.\n# - Downstream consumers see the same flat DocContext[] regardless\n# of how docs were resolved.\n# - When the release is published, the perspective entry becomes a\n# no-op (articles are now in published). Migrate to explicit path\n# or slug refs at your convenience.\n#\n# This example task is DISABLED by default. To enable it, either:\n# 1. Remove the execution.enabled: false line below, or\n# 2. Set execution.enabled: true\n#\n# @see docs/design-docs/canonical-doc-resolution.md\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n- id: example-perspective-ref\n description:\n \"Example \u2014 GROQ features from content release (perspective-based doc\n references)\"\n\n featureArea: groq\n\n # Perspective-based canonical doc reference.\n #\n # The perspective ID references a content release in the Sanity\n # Content Lake. At evaluation time, the doc fetcher auto-discovers\n # all articles versioned in this release and includes them as\n # canonical documentation context.\n #\n # Release rE9TSJvR4 contains:\n # - \"GROQ-powered webhooks\" (webhooks)\n # - \"Query Cheat Sheet - GROQ\" (query-cheat-sheet)\n # - \"GROQ joins\" (groq-joins)\n #\n # You can combine perspective refs with explicit slug/path/id refs\n # to include foundational published docs alongside release content.\n canonicalDocs:\n - perspective: rE9TSJvR4\n reason: \"All GROQ documentation updates in the test content release\"\n - slug: groq-
|
|
260
|
+
export declare const examplePerspectiveRefYaml = "# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n# Example Task: Perspective / content release doc references\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n#\n# Demonstrates using `perspective` to reference all documentation\n# articles within a content release. This is the key capability for\n# evaluating NEW feature documentation before it's published.\n#\n# How it works:\n# - A perspective ref is one-to-many: the doc fetcher queries the\n# named release and expands it to ALL articles versioned within it.\n# - Downstream consumers see the same flat DocContext[] regardless\n# of how docs were resolved.\n# - When the release is published, the perspective entry becomes a\n# no-op (articles are now in published). Migrate to explicit path\n# or slug refs at your convenience.\n#\n# This example task is DISABLED by default. To enable it, either:\n# 1. Remove the execution.enabled: false line below, or\n# 2. Set execution.enabled: true\n#\n# @see docs/design-docs/canonical-doc-resolution.md\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n- id: example-perspective-ref\n description:\n \"Example \u2014 GROQ features from content release (perspective-based doc\n references)\"\n\n featureArea: groq\n\n # Perspective-based canonical doc reference.\n #\n # The perspective ID references a content release in the Sanity\n # Content Lake. At evaluation time, the doc fetcher auto-discovers\n # all articles versioned in this release and includes them as\n # canonical documentation context.\n #\n # Release rE9TSJvR4 contains:\n # - \"GROQ-powered webhooks\" (webhooks)\n # - \"Query Cheat Sheet - GROQ\" (query-cheat-sheet)\n # - \"GROQ joins\" (groq-joins)\n #\n # You can combine perspective refs with explicit slug/path/id refs\n # to include foundational published docs alongside release content.\n # Here we add groq-data-types as a complementary published reference.\n canonicalDocs:\n - perspective: rE9TSJvR4\n reason: \"All GROQ documentation updates in the test content release\"\n - slug: groq-data-types\n reason: \"GROQ data type reference (published, stable)\"\n\n docCoverage: true\n\n vars:\n task: |\n Using GROQ, demonstrate advanced query patterns including:\n 1. Joining data across document types using references\n 2. Filtering webhook payloads with GROQ projections\n 3. Using the query cheat sheet patterns for common operations\n 4. Working with different GROQ data types in filters\n Provide working GROQ query examples for each pattern.\n docs: \"\"\n\n assert:\n - type: llm-rubric\n template: task-completion\n criteria:\n - \"Demonstrates GROQ join syntax for cross-document queries\"\n - \"Shows GROQ filter patterns for webhook configuration\"\n - \"Includes practical query examples from cheat sheet patterns\"\n\n - type: llm-rubric\n template: code-correctness\n criteria:\n - \"All GROQ queries use valid syntax\"\n - \"Reference joins use correct dereference operator (->)\"\n\n baseline:\n enabled: true\n rubric: abbreviated\n\n # Example tasks ship disabled so they don't run automatically.\n # Set enabled: true (or remove this block) to activate.\n execution:\n enabled: false\n";
|
|
304
261
|
/** Parsed task data for example-studio-custom-input (JSON-safe) */
|
|
305
262
|
export declare const exampleStudioCustomInputData: readonly [{
|
|
306
263
|
readonly id: "example-studio-custom-input";
|
|
@@ -309,6 +266,9 @@ export declare const exampleStudioCustomInputData: readonly [{
|
|
|
309
266
|
readonly canonicalDocs: readonly [{
|
|
310
267
|
readonly slug: "custom-input-widgets";
|
|
311
268
|
readonly reason: "Guide for building custom form inputs in Sanity Studio";
|
|
269
|
+
}, {
|
|
270
|
+
readonly slug: "form-components";
|
|
271
|
+
readonly reason: "Form component API and customization patterns";
|
|
312
272
|
}];
|
|
313
273
|
readonly docCoverage: true;
|
|
314
274
|
readonly referenceSolution: "canonical/example-studio-custom-input.ts";
|
|
@@ -334,13 +294,13 @@ export declare const exampleStudioCustomInputData: readonly [{
|
|
|
334
294
|
};
|
|
335
295
|
}];
|
|
336
296
|
/** Raw YAML string for example-studio-custom-input (preserves comments) */
|
|
337
|
-
export declare const exampleStudioCustomInputYaml = "# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n# Example Task: Custom input component in Sanity Studio\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n#\n# This is a starter template \u2014 edit it for your own documentation.\n# Delete this file or replace it with your own tasks.\n#\n# This example task is DISABLED by default. To enable it, either:\n# 1. Remove the execution.enabled: false line below, or\n# 2. Set execution.enabled: true\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n- id: example-studio-custom-input\n description: \"Example \u2014 Custom input component in Sanity Studio\"\n\n featureArea: studio\n\n # Slug-based canonical doc
|
|
297
|
+
export declare const exampleStudioCustomInputYaml = "# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n# Example Task: Custom input component in Sanity Studio\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n#\n# This is a starter template \u2014 edit it for your own documentation.\n# Delete this file or replace it with your own tasks.\n#\n# This example task is DISABLED by default. To enable it, either:\n# 1. Remove the execution.enabled: false line below, or\n# 2. Set execution.enabled: true\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n- id: example-studio-custom-input\n description: \"Example \u2014 Custom input component in Sanity Studio\"\n\n featureArea: studio\n\n # Slug-based canonical doc references.\n canonicalDocs:\n - slug: custom-input-widgets\n reason: \"Guide for building custom form inputs in Sanity Studio\"\n - slug: form-components\n reason: \"Form component API and customization patterns\"\n\n docCoverage: true\n referenceSolution: canonical/example-studio-custom-input.ts\n\n vars:\n task: |\n Build a custom string input component for Sanity Studio that shows\n a character count below the input field. The component should accept\n a maxLength option from the field schema and display a warning when\n the text exceeds the limit.\n docs: \"\"\n\n assert:\n - type: llm-rubric\n template: task-completion\n criteria:\n - \"Implements a React component that renders a text input\"\n - \"Displays a live character count\"\n - \"Reads maxLength from schema options\"\n - \"Shows a visual warning when limit is exceeded\"\n\n - type: llm-rubric\n template: code-correctness\n criteria:\n - \"Uses the Sanity UI library for styling\"\n - \"Calls onChange with patch operations\"\n\n baseline:\n enabled: true\n rubric: abbreviated\n\n # Example tasks ship disabled so they don't run automatically.\n # Set enabled: true (or remove this block) to activate.\n execution:\n enabled: false\n";
|
|
338
298
|
/** All task example data as a flat array (JSON-safe) */
|
|
339
299
|
export declare const allTaskData: readonly unknown[];
|
|
340
300
|
/** Map of task ID (filename stem) → raw YAML string (preserves comments) */
|
|
341
301
|
export declare const taskYamlFiles: Record<string, string>;
|
|
342
302
|
/** List of task file stems, in alphabetical order */
|
|
343
|
-
export declare const TASK_FILE_NAMES: readonly ["example-groq-blog-listing", "example-id-based-ref", "example-
|
|
303
|
+
export declare const TASK_FILE_NAMES: readonly ["example-groq-blog-listing", "example-id-based-ref", "example-path-based-ref", "example-perspective-ref", "example-studio-custom-input"];
|
|
344
304
|
export type ExampleType = "config" | "source" | "rubric" | "threshold" | "ailf-config" | "task";
|
|
345
305
|
export declare const EXAMPLE_TYPES: readonly ExampleType[];
|
|
346
306
|
export interface ExampleRecord {
|
|
@@ -198,23 +198,23 @@ export const exampleGroqBlogListingYaml = "# ───────────
|
|
|
198
198
|
export const exampleIdBasedRefData = [
|
|
199
199
|
{
|
|
200
200
|
"id": "example-id-based-ref",
|
|
201
|
-
"description": "Example — GROQ
|
|
201
|
+
"description": "Example — GROQ feature support (ID-based doc references)",
|
|
202
202
|
"featureArea": "groq",
|
|
203
203
|
"canonicalDocs": [
|
|
204
204
|
{
|
|
205
|
-
"id": "
|
|
206
|
-
"slug": "
|
|
207
|
-
"reason": "GROQ
|
|
205
|
+
"id": "0ba88f1b-d1a7-418a-9267-2e343d01886a",
|
|
206
|
+
"slug": "groq-feature-support-by-context",
|
|
207
|
+
"reason": "GROQ feature support across different Sanity contexts"
|
|
208
208
|
},
|
|
209
209
|
{
|
|
210
|
-
"id": "
|
|
211
|
-
"slug": "groq-
|
|
212
|
-
"reason": "
|
|
210
|
+
"id": "5b9c2863-ef01-4565-af8e-ee54e081ee74",
|
|
211
|
+
"slug": "custom-groq-functions",
|
|
212
|
+
"reason": "Custom GROQ functions and pipelines"
|
|
213
213
|
}
|
|
214
214
|
],
|
|
215
215
|
"docCoverage": true,
|
|
216
216
|
"vars": {
|
|
217
|
-
"task": "
|
|
217
|
+
"task": "Explain how GROQ is used across different Sanity contexts.\nCover the following:\n1. Which GROQ features are available in each context (API queries,\n webhooks, custom functions, access control)\n2. How to create and use custom GROQ functions\n3. Any differences in GROQ support between contexts\nProvide examples demonstrating context-specific GROQ patterns.\n",
|
|
218
218
|
"docs": ""
|
|
219
219
|
},
|
|
220
220
|
"assert": [
|
|
@@ -222,18 +222,17 @@ export const exampleIdBasedRefData = [
|
|
|
222
222
|
"type": "llm-rubric",
|
|
223
223
|
"template": "task-completion",
|
|
224
224
|
"criteria": [
|
|
225
|
-
"
|
|
226
|
-
"
|
|
227
|
-
"
|
|
228
|
-
"Demonstrates reference expansion (dereferencing with ->)"
|
|
225
|
+
"Explains GROQ availability across different Sanity contexts",
|
|
226
|
+
"Describes custom GROQ function creation and usage",
|
|
227
|
+
"Notes differences in GROQ support between contexts"
|
|
229
228
|
]
|
|
230
229
|
},
|
|
231
230
|
{
|
|
232
231
|
"type": "llm-rubric",
|
|
233
232
|
"template": "code-correctness",
|
|
234
233
|
"criteria": [
|
|
235
|
-
"
|
|
236
|
-
"
|
|
234
|
+
"GROQ examples use valid syntax",
|
|
235
|
+
"Custom function examples follow the correct API pattern"
|
|
237
236
|
]
|
|
238
237
|
}
|
|
239
238
|
],
|
|
@@ -247,88 +246,26 @@ export const exampleIdBasedRefData = [
|
|
|
247
246
|
}
|
|
248
247
|
];
|
|
249
248
|
/** Raw YAML string for example-id-based-ref (preserves comments) */
|
|
250
|
-
export const exampleIdBasedRefYaml = "# ──────────────────────────────────────────────────────────────────────\n# Example Task: Document ID-based canonical doc references\n# ──────────────────────────────────────────────────────────────────────\n#\n# Demonstrates using `id` to reference canonical documentation by\n# Sanity document `_id`. This is useful for:\n# - Draft documents that don't have a stable slug yet\n# - Programmatic references from imports or migrations\n# - Documents where you know the _id but not the slug\n#\n# The `id` ref type can also carry optional `slug` and `path` fields\n# as human-readable annotations — these are NOT used for resolution,\n# only for display in logs and reports.\n#\n# This example task is DISABLED by default. To enable it, either:\n# 1. Remove the execution.enabled: false line below, or\n# 2. Set execution.enabled: true\n#\n# @see docs/design-docs/canonical-doc-resolution.md\n# ──────────────────────────────────────────────────────────────────────\n\n- id: example-id-based-ref\n description: \"Example — GROQ
|
|
251
|
-
/** Parsed task data for example-mixed-ref-types (JSON-safe) */
|
|
252
|
-
export const exampleMixedRefTypesData = [
|
|
253
|
-
{
|
|
254
|
-
"id": "example-mixed-ref-types",
|
|
255
|
-
"description": "Example — Mixed canonical doc reference types (slug + path + id + perspective)",
|
|
256
|
-
"featureArea": "groq",
|
|
257
|
-
"canonicalDocs": [
|
|
258
|
-
{
|
|
259
|
-
"slug": "groq-introduction",
|
|
260
|
-
"reason": "Core GROQ language overview"
|
|
261
|
-
},
|
|
262
|
-
{
|
|
263
|
-
"path": "content-lake/how-queries-work",
|
|
264
|
-
"reason": "Query execution model (disambiguated by section)"
|
|
265
|
-
},
|
|
266
|
-
{
|
|
267
|
-
"id": "81b839a4-2fc1-4769-941a-ec4de9276492",
|
|
268
|
-
"slug": "query-cheat-sheet",
|
|
269
|
-
"reason": "GROQ query patterns cheat sheet (referenced by document ID)"
|
|
270
|
-
},
|
|
271
|
-
{
|
|
272
|
-
"perspective": "rE9TSJvR4",
|
|
273
|
-
"reason": "Additional GROQ docs from the test content release"
|
|
274
|
-
}
|
|
275
|
-
],
|
|
276
|
-
"docCoverage": true,
|
|
277
|
-
"vars": {
|
|
278
|
-
"task": "Create a comprehensive GROQ query guide that covers:\n1. Basic query structure and filtering by document type\n2. Projections and field selection\n3. Ordering, slicing, and pagination\n4. Reference joins and nested data\n5. Webhook GROQ filter configuration\nInclude working examples for each concept and explain the\nquery execution model.\n",
|
|
279
|
-
"docs": ""
|
|
280
|
-
},
|
|
281
|
-
"assert": [
|
|
282
|
-
{
|
|
283
|
-
"type": "llm-rubric",
|
|
284
|
-
"template": "task-completion",
|
|
285
|
-
"criteria": [
|
|
286
|
-
"Covers basic GROQ query structure with _type filtering",
|
|
287
|
-
"Demonstrates projections with field aliasing",
|
|
288
|
-
"Shows ordering with order() and slice syntax",
|
|
289
|
-
"Includes reference joins with the dereference operator",
|
|
290
|
-
"Explains webhook GROQ filter patterns"
|
|
291
|
-
]
|
|
292
|
-
},
|
|
293
|
-
{
|
|
294
|
-
"type": "llm-rubric",
|
|
295
|
-
"template": "code-correctness",
|
|
296
|
-
"criteria": [
|
|
297
|
-
"All GROQ queries use valid syntax",
|
|
298
|
-
"Examples are practical and self-contained"
|
|
299
|
-
]
|
|
300
|
-
}
|
|
301
|
-
],
|
|
302
|
-
"baseline": {
|
|
303
|
-
"enabled": true,
|
|
304
|
-
"rubric": "abbreviated"
|
|
305
|
-
},
|
|
306
|
-
"execution": {
|
|
307
|
-
"enabled": false
|
|
308
|
-
}
|
|
309
|
-
}
|
|
310
|
-
];
|
|
311
|
-
/** Raw YAML string for example-mixed-ref-types (preserves comments) */
|
|
312
|
-
export const exampleMixedRefTypesYaml = "# ──────────────────────────────────────────────────────────────────────\n# Example Task: All canonical doc reference types in one task\n# ──────────────────────────────────────────────────────────────────────\n#\n# Demonstrates combining all four canonical doc reference strategies\n# in a single task's canonicalDocs array:\n#\n# slug — by article slug (simplest, may not be unique)\n# path — by section/slug path (unique, preferred)\n# id — by Sanity document _id (for drafts, imports)\n# perspective — by content release (one-to-many expansion)\n#\n# Each reference type resolves through a different strategy, but the\n# pipeline merges them into a single flat documentation context. The\n# LLM sees the same combined markdown regardless of how docs were found.\n#\n# This example task is DISABLED by default. To enable it, either:\n# 1. Remove the execution.enabled: false line below, or\n# 2. Set execution.enabled: true\n#\n# @see docs/design-docs/canonical-doc-resolution.md\n# ──────────────────────────────────────────────────────────────────────\n\n- id: example-mixed-ref-types\n description:\n \"Example — Mixed canonical doc reference types (slug + path + id +\n perspective)\"\n\n featureArea: groq\n\n # All four canonical doc reference types demonstrated together.\n #\n # The pipeline resolves each ref independently:\n # 1. slug refs → GROQ query by slug.current\n # 2. path refs → GROQ query by section + slug (or slug fallback)\n # 3. id refs → GROQ query by _id\n # 4. perspective refs → expand to all articles in the release\n #\n # After resolution, all entries become a flat list of document slugs.\n # The fetched markdown is concatenated into a single context string.\n canonicalDocs:\n # Slug reference — simplest form, resolves by article slug\n - slug: groq-introduction\n reason: \"Core GROQ language overview\"\n\n # Path reference — section-qualified, uniquely identifies the article\n - path: content-lake/how-queries-work\n reason: \"Query execution model (disambiguated by section)\"\n\n # ID reference — resolves by Sanity document _id\n # Optional slug annotation helps humans reading the YAML\n - id: \"81b839a4-2fc1-4769-941a-ec4de9276492\"\n slug: query-cheat-sheet\n reason: \"GROQ query patterns cheat sheet (referenced by document ID)\"\n\n # Perspective reference — expands to all articles in the release\n # This adds webhooks, query-cheat-sheet, and groq-joins from release rE9TSJvR4\n # Note: query-cheat-sheet appears via both the id ref and the perspective ref,\n # but the pipeline deduplicates slugs automatically.\n - perspective: rE9TSJvR4\n reason: \"Additional GROQ docs from the test content release\"\n\n docCoverage: true\n\n vars:\n task: |\n Create a comprehensive GROQ query guide that covers:\n 1. Basic query structure and filtering by document type\n 2. Projections and field selection\n 3. Ordering, slicing, and pagination\n 4. Reference joins and nested data\n 5. Webhook GROQ filter configuration\n Include working examples for each concept and explain the\n query execution model.\n docs: \"\"\n\n assert:\n - type: llm-rubric\n template: task-completion\n criteria:\n - \"Covers basic GROQ query structure with _type filtering\"\n - \"Demonstrates projections with field aliasing\"\n - \"Shows ordering with order() and slice syntax\"\n - \"Includes reference joins with the dereference operator\"\n - \"Explains webhook GROQ filter patterns\"\n\n - type: llm-rubric\n template: code-correctness\n criteria:\n - \"All GROQ queries use valid syntax\"\n - \"Examples are practical and self-contained\"\n\n baseline:\n enabled: true\n rubric: abbreviated\n\n # Example tasks ship disabled so they don't run automatically.\n # Set enabled: true (or remove this block) to activate.\n execution:\n enabled: false\n";
|
|
249
|
+
export const exampleIdBasedRefYaml = "# ──────────────────────────────────────────────────────────────────────\n# Example Task: Document ID-based canonical doc references\n# ──────────────────────────────────────────────────────────────────────\n#\n# Demonstrates using `id` to reference canonical documentation by\n# Sanity document `_id`. This is useful for:\n# - Draft documents that don't have a stable slug yet\n# - Programmatic references from imports or migrations\n# - Documents where you know the _id but not the slug\n#\n# The `id` ref type can also carry optional `slug` and `path` fields\n# as human-readable annotations — these are NOT used for resolution,\n# only for display in logs and reports.\n#\n# This example task is DISABLED by default. To enable it, either:\n# 1. Remove the execution.enabled: false line below, or\n# 2. Set execution.enabled: true\n#\n# @see docs/design-docs/canonical-doc-resolution.md\n# ──────────────────────────────────────────────────────────────────────\n\n- id: example-id-based-ref\n description: \"Example — GROQ feature support (ID-based doc references)\"\n\n featureArea: groq\n\n # ID-based canonical doc references.\n #\n # Use the Sanity document _id to reference articles directly.\n # Optional slug/path annotations help humans reading the YAML\n # but are NOT used for resolution — only the `id` field matters.\n #\n # These IDs reference real articles in the Sanity docs (next dataset):\n # 0ba88f1b... = \"GROQ feature support across Sanity\"\n # 5b9c2863... = \"Custom GROQ functions\"\n canonicalDocs:\n - id: \"0ba88f1b-d1a7-418a-9267-2e343d01886a\"\n slug: groq-feature-support-by-context # annotation only — not used for resolution\n reason: \"GROQ feature support across different Sanity contexts\"\n - id: \"5b9c2863-ef01-4565-af8e-ee54e081ee74\"\n slug: custom-groq-functions # annotation only — not used for resolution\n reason: \"Custom GROQ functions and pipelines\"\n\n docCoverage: true\n\n vars:\n task: |\n Explain how GROQ is used across different Sanity contexts.\n Cover the following:\n 1. Which GROQ features are available in each context (API queries,\n webhooks, custom functions, access control)\n 2. How to create and use custom GROQ functions\n 3. Any differences in GROQ support between contexts\n Provide examples demonstrating context-specific GROQ patterns.\n docs: \"\"\n\n assert:\n - type: llm-rubric\n template: task-completion\n criteria:\n - \"Explains GROQ availability across different Sanity contexts\"\n - \"Describes custom GROQ function creation and usage\"\n - \"Notes differences in GROQ support between contexts\"\n\n - type: llm-rubric\n template: code-correctness\n criteria:\n - \"GROQ examples use valid syntax\"\n - \"Custom function examples follow the correct API pattern\"\n\n baseline:\n enabled: true\n rubric: abbreviated\n\n # Example tasks ship disabled so they don't run automatically.\n # Set enabled: true (or remove this block) to activate.\n execution:\n enabled: false\n";
|
|
313
250
|
/** Parsed task data for example-path-based-ref (JSON-safe) */
|
|
314
251
|
export const examplePathBasedRefData = [
|
|
315
252
|
{
|
|
316
253
|
"id": "example-path-based-ref",
|
|
317
|
-
"description": "Example — GROQ
|
|
254
|
+
"description": "Example — GROQ mutations (path-based doc references)",
|
|
318
255
|
"featureArea": "groq",
|
|
319
256
|
"canonicalDocs": [
|
|
320
257
|
{
|
|
321
|
-
"path": "content-lake/
|
|
322
|
-
"reason": "
|
|
258
|
+
"path": "content-lake/mutations-introduction",
|
|
259
|
+
"reason": "Introduction to document mutations in the Content Lake"
|
|
323
260
|
},
|
|
324
261
|
{
|
|
325
|
-
"path": "content-lake/
|
|
326
|
-
"reason": "
|
|
262
|
+
"path": "content-lake/documents",
|
|
263
|
+
"reason": "Document structure and types (Content Lake, not CLI reference)"
|
|
327
264
|
}
|
|
328
265
|
],
|
|
329
266
|
"docCoverage": true,
|
|
330
267
|
"vars": {
|
|
331
|
-
"task": "
|
|
268
|
+
"task": "Explain how to create, update, and delete documents in Sanity's\nContent Lake using mutations. Cover:\n1. The different mutation types (create, createOrReplace, patch, delete)\n2. Document structure and required fields (_id, _type)\n3. How to use patch operations to update specific fields\n4. Best practices for mutation patterns\nProvide working code examples using @sanity/client.\n",
|
|
332
269
|
"docs": ""
|
|
333
270
|
},
|
|
334
271
|
"assert": [
|
|
@@ -336,18 +273,18 @@ export const examplePathBasedRefData = [
|
|
|
336
273
|
"type": "llm-rubric",
|
|
337
274
|
"template": "task-completion",
|
|
338
275
|
"criteria": [
|
|
339
|
-
"
|
|
340
|
-
"
|
|
341
|
-
"
|
|
342
|
-
"
|
|
276
|
+
"Explains create, createOrReplace, patch, and delete mutations",
|
|
277
|
+
"Describes required document fields (_id, _type)",
|
|
278
|
+
"Shows patch operations for field-level updates",
|
|
279
|
+
"Includes practical code examples"
|
|
343
280
|
]
|
|
344
281
|
},
|
|
345
282
|
{
|
|
346
283
|
"type": "llm-rubric",
|
|
347
284
|
"template": "code-correctness",
|
|
348
285
|
"criteria": [
|
|
349
|
-
"
|
|
350
|
-
"
|
|
286
|
+
"Uses correct @sanity/client mutation API",
|
|
287
|
+
"Patch operations use valid set/unset/inc syntax"
|
|
351
288
|
]
|
|
352
289
|
}
|
|
353
290
|
],
|
|
@@ -361,7 +298,7 @@ export const examplePathBasedRefData = [
|
|
|
361
298
|
}
|
|
362
299
|
];
|
|
363
300
|
/** Raw YAML string for example-path-based-ref (preserves comments) */
|
|
364
|
-
export const examplePathBasedRefYaml = "# ──────────────────────────────────────────────────────────────────────\n# Example Task: Path-based canonical doc references\n# ──────────────────────────────────────────────────────────────────────\n#\n# Demonstrates using `path` to reference canonical documentation.\n# Paths are the preferred reference type because they uniquely identify\n# an article across sections (unlike slugs, which can collide).\n#\n# Path format:\n# - Simple: \"webhooks\" → resolves by slug lookup\n# - Sectioned: \"content-lake/webhooks\" → disambiguates by section + slug\n#\n# This example task is DISABLED by default. To enable it, either:\n# 1. Remove the execution.enabled: false line below, or\n# 2. Set execution.enabled: true\n#\n# @see docs/design-docs/canonical-doc-resolution.md\n# ──────────────────────────────────────────────────────────────────────\n\n- id: example-path-based-ref\n description: \"Example — GROQ
|
|
301
|
+
export const examplePathBasedRefYaml = "# ──────────────────────────────────────────────────────────────────────\n# Example Task: Path-based canonical doc references\n# ──────────────────────────────────────────────────────────────────────\n#\n# Demonstrates using `path` to reference canonical documentation.\n# Paths are the preferred reference type because they uniquely identify\n# an article across sections (unlike slugs, which can collide).\n#\n# Path format:\n# - Simple: \"webhooks\" → resolves by slug lookup\n# - Sectioned: \"content-lake/webhooks\" → disambiguates by section + slug\n#\n# This example demonstrates why paths matter: the slug \"documents\"\n# exists in both the \"content-lake\" and \"cli-reference\" sections.\n# Using \"content-lake/documents\" ensures we get the right one.\n#\n# This example task is DISABLED by default. To enable it, either:\n# 1. Remove the execution.enabled: false line below, or\n# 2. Set execution.enabled: true\n#\n# @see docs/design-docs/canonical-doc-resolution.md\n# ──────────────────────────────────────────────────────────────────────\n\n- id: example-path-based-ref\n description: \"Example — GROQ mutations (path-based doc references)\"\n\n featureArea: groq\n\n # Path-based canonical doc references.\n #\n # Use \"section/slug\" format to uniquely identify articles:\n # - \"content-lake/mutations-introduction\" → the mutations article\n # - \"content-lake/documents\" → the documents article in Content Lake\n # (not the CLI \"documents\" article in cli-reference section)\n #\n # The \"documents\" slug exists in two sections — this is exactly why\n # path-based references are preferred over slug-based references.\n canonicalDocs:\n - path: content-lake/mutations-introduction\n reason: \"Introduction to document mutations in the Content Lake\"\n - path: content-lake/documents\n reason: \"Document structure and types (Content Lake, not CLI reference)\"\n\n docCoverage: true\n\n vars:\n task: |\n Explain how to create, update, and delete documents in Sanity's\n Content Lake using mutations. Cover:\n 1. The different mutation types (create, createOrReplace, patch, delete)\n 2. Document structure and required fields (_id, _type)\n 3. How to use patch operations to update specific fields\n 4. Best practices for mutation patterns\n Provide working code examples using @sanity/client.\n docs: \"\"\n\n assert:\n - type: llm-rubric\n template: task-completion\n criteria:\n - \"Explains create, createOrReplace, patch, and delete mutations\"\n - \"Describes required document fields (_id, _type)\"\n - \"Shows patch operations for field-level updates\"\n - \"Includes practical code examples\"\n\n - type: llm-rubric\n template: code-correctness\n criteria:\n - \"Uses correct @sanity/client mutation API\"\n - \"Patch operations use valid set/unset/inc syntax\"\n\n baseline:\n enabled: true\n rubric: abbreviated\n\n # Example tasks ship disabled so they don't run automatically.\n # Set enabled: true (or remove this block) to activate.\n execution:\n enabled: false\n";
|
|
365
302
|
/** Parsed task data for example-perspective-ref (JSON-safe) */
|
|
366
303
|
export const examplePerspectiveRefData = [
|
|
367
304
|
{
|
|
@@ -374,13 +311,13 @@ export const examplePerspectiveRefData = [
|
|
|
374
311
|
"reason": "All GROQ documentation updates in the test content release"
|
|
375
312
|
},
|
|
376
313
|
{
|
|
377
|
-
"slug": "groq-
|
|
378
|
-
"reason": "
|
|
314
|
+
"slug": "groq-data-types",
|
|
315
|
+
"reason": "GROQ data type reference (published, stable)"
|
|
379
316
|
}
|
|
380
317
|
],
|
|
381
318
|
"docCoverage": true,
|
|
382
319
|
"vars": {
|
|
383
|
-
"task": "Using GROQ, demonstrate advanced query patterns including:\n1. Joining data across document types using references\n2. Filtering webhook payloads with GROQ projections\n3. Using the query cheat sheet patterns for common operations\nProvide working GROQ query examples for each pattern.\n",
|
|
320
|
+
"task": "Using GROQ, demonstrate advanced query patterns including:\n1. Joining data across document types using references\n2. Filtering webhook payloads with GROQ projections\n3. Using the query cheat sheet patterns for common operations\n4. Working with different GROQ data types in filters\nProvide working GROQ query examples for each pattern.\n",
|
|
384
321
|
"docs": ""
|
|
385
322
|
},
|
|
386
323
|
"assert": [
|
|
@@ -412,7 +349,7 @@ export const examplePerspectiveRefData = [
|
|
|
412
349
|
}
|
|
413
350
|
];
|
|
414
351
|
/** Raw YAML string for example-perspective-ref (preserves comments) */
|
|
415
|
-
export const examplePerspectiveRefYaml = "# ──────────────────────────────────────────────────────────────────────\n# Example Task: Perspective / content release doc references\n# ──────────────────────────────────────────────────────────────────────\n#\n# Demonstrates using `perspective` to reference all documentation\n# articles within a content release. This is the key capability for\n# evaluating NEW feature documentation before it's published.\n#\n# How it works:\n# - A perspective ref is one-to-many: the doc fetcher queries the\n# named release and expands it to ALL articles versioned within it.\n# - Downstream consumers see the same flat DocContext[] regardless\n# of how docs were resolved.\n# - When the release is published, the perspective entry becomes a\n# no-op (articles are now in published). Migrate to explicit path\n# or slug refs at your convenience.\n#\n# This example task is DISABLED by default. To enable it, either:\n# 1. Remove the execution.enabled: false line below, or\n# 2. Set execution.enabled: true\n#\n# @see docs/design-docs/canonical-doc-resolution.md\n# ──────────────────────────────────────────────────────────────────────\n\n- id: example-perspective-ref\n description:\n \"Example — GROQ features from content release (perspective-based doc\n references)\"\n\n featureArea: groq\n\n # Perspective-based canonical doc reference.\n #\n # The perspective ID references a content release in the Sanity\n # Content Lake. At evaluation time, the doc fetcher auto-discovers\n # all articles versioned in this release and includes them as\n # canonical documentation context.\n #\n # Release rE9TSJvR4 contains:\n # - \"GROQ-powered webhooks\" (webhooks)\n # - \"Query Cheat Sheet - GROQ\" (query-cheat-sheet)\n # - \"GROQ joins\" (groq-joins)\n #\n # You can combine perspective refs with explicit slug/path/id refs\n # to include foundational published docs alongside release content.\n canonicalDocs:\n - perspective: rE9TSJvR4\n reason: \"All GROQ documentation updates in the test content release\"\n - slug: groq-
|
|
352
|
+
export const examplePerspectiveRefYaml = "# ──────────────────────────────────────────────────────────────────────\n# Example Task: Perspective / content release doc references\n# ──────────────────────────────────────────────────────────────────────\n#\n# Demonstrates using `perspective` to reference all documentation\n# articles within a content release. This is the key capability for\n# evaluating NEW feature documentation before it's published.\n#\n# How it works:\n# - A perspective ref is one-to-many: the doc fetcher queries the\n# named release and expands it to ALL articles versioned within it.\n# - Downstream consumers see the same flat DocContext[] regardless\n# of how docs were resolved.\n# - When the release is published, the perspective entry becomes a\n# no-op (articles are now in published). Migrate to explicit path\n# or slug refs at your convenience.\n#\n# This example task is DISABLED by default. To enable it, either:\n# 1. Remove the execution.enabled: false line below, or\n# 2. Set execution.enabled: true\n#\n# @see docs/design-docs/canonical-doc-resolution.md\n# ──────────────────────────────────────────────────────────────────────\n\n- id: example-perspective-ref\n description:\n \"Example — GROQ features from content release (perspective-based doc\n references)\"\n\n featureArea: groq\n\n # Perspective-based canonical doc reference.\n #\n # The perspective ID references a content release in the Sanity\n # Content Lake. At evaluation time, the doc fetcher auto-discovers\n # all articles versioned in this release and includes them as\n # canonical documentation context.\n #\n # Release rE9TSJvR4 contains:\n # - \"GROQ-powered webhooks\" (webhooks)\n # - \"Query Cheat Sheet - GROQ\" (query-cheat-sheet)\n # - \"GROQ joins\" (groq-joins)\n #\n # You can combine perspective refs with explicit slug/path/id refs\n # to include foundational published docs alongside release content.\n # Here we add groq-data-types as a complementary published reference.\n canonicalDocs:\n - perspective: rE9TSJvR4\n reason: \"All GROQ documentation updates in the test content release\"\n - slug: groq-data-types\n reason: \"GROQ data type reference (published, stable)\"\n\n docCoverage: true\n\n vars:\n task: |\n Using GROQ, demonstrate advanced query patterns including:\n 1. Joining data across document types using references\n 2. Filtering webhook payloads with GROQ projections\n 3. Using the query cheat sheet patterns for common operations\n 4. Working with different GROQ data types in filters\n Provide working GROQ query examples for each pattern.\n docs: \"\"\n\n assert:\n - type: llm-rubric\n template: task-completion\n criteria:\n - \"Demonstrates GROQ join syntax for cross-document queries\"\n - \"Shows GROQ filter patterns for webhook configuration\"\n - \"Includes practical query examples from cheat sheet patterns\"\n\n - type: llm-rubric\n template: code-correctness\n criteria:\n - \"All GROQ queries use valid syntax\"\n - \"Reference joins use correct dereference operator (->)\"\n\n baseline:\n enabled: true\n rubric: abbreviated\n\n # Example tasks ship disabled so they don't run automatically.\n # Set enabled: true (or remove this block) to activate.\n execution:\n enabled: false\n";
|
|
416
353
|
/** Parsed task data for example-studio-custom-input (JSON-safe) */
|
|
417
354
|
export const exampleStudioCustomInputData = [
|
|
418
355
|
{
|
|
@@ -423,6 +360,10 @@ export const exampleStudioCustomInputData = [
|
|
|
423
360
|
{
|
|
424
361
|
"slug": "custom-input-widgets",
|
|
425
362
|
"reason": "Guide for building custom form inputs in Sanity Studio"
|
|
363
|
+
},
|
|
364
|
+
{
|
|
365
|
+
"slug": "form-components",
|
|
366
|
+
"reason": "Form component API and customization patterns"
|
|
426
367
|
}
|
|
427
368
|
],
|
|
428
369
|
"docCoverage": true,
|
|
@@ -461,7 +402,7 @@ export const exampleStudioCustomInputData = [
|
|
|
461
402
|
}
|
|
462
403
|
];
|
|
463
404
|
/** Raw YAML string for example-studio-custom-input (preserves comments) */
|
|
464
|
-
export const exampleStudioCustomInputYaml = "# ──────────────────────────────────────────────────────────────────────\n# Example Task: Custom input component in Sanity Studio\n# ──────────────────────────────────────────────────────────────────────\n#\n# This is a starter template — edit it for your own documentation.\n# Delete this file or replace it with your own tasks.\n#\n# This example task is DISABLED by default. To enable it, either:\n# 1. Remove the execution.enabled: false line below, or\n# 2. Set execution.enabled: true\n# ──────────────────────────────────────────────────────────────────────\n\n- id: example-studio-custom-input\n description: \"Example — Custom input component in Sanity Studio\"\n\n featureArea: studio\n\n # Slug-based canonical doc
|
|
405
|
+
export const exampleStudioCustomInputYaml = "# ──────────────────────────────────────────────────────────────────────\n# Example Task: Custom input component in Sanity Studio\n# ──────────────────────────────────────────────────────────────────────\n#\n# This is a starter template — edit it for your own documentation.\n# Delete this file or replace it with your own tasks.\n#\n# This example task is DISABLED by default. To enable it, either:\n# 1. Remove the execution.enabled: false line below, or\n# 2. Set execution.enabled: true\n# ──────────────────────────────────────────────────────────────────────\n\n- id: example-studio-custom-input\n description: \"Example — Custom input component in Sanity Studio\"\n\n featureArea: studio\n\n # Slug-based canonical doc references.\n canonicalDocs:\n - slug: custom-input-widgets\n reason: \"Guide for building custom form inputs in Sanity Studio\"\n - slug: form-components\n reason: \"Form component API and customization patterns\"\n\n docCoverage: true\n referenceSolution: canonical/example-studio-custom-input.ts\n\n vars:\n task: |\n Build a custom string input component for Sanity Studio that shows\n a character count below the input field. The component should accept\n a maxLength option from the field schema and display a warning when\n the text exceeds the limit.\n docs: \"\"\n\n assert:\n - type: llm-rubric\n template: task-completion\n criteria:\n - \"Implements a React component that renders a text input\"\n - \"Displays a live character count\"\n - \"Reads maxLength from schema options\"\n - \"Shows a visual warning when limit is exceeded\"\n\n - type: llm-rubric\n template: code-correctness\n criteria:\n - \"Uses the Sanity UI library for styling\"\n - \"Calls onChange with patch operations\"\n\n baseline:\n enabled: true\n rubric: abbreviated\n\n # Example tasks ship disabled so they don't run automatically.\n # Set enabled: true (or remove this block) to activate.\n execution:\n enabled: false\n";
|
|
465
406
|
// ---------------------------------------------------------------------------
|
|
466
407
|
// Aggregate task exports
|
|
467
408
|
// ---------------------------------------------------------------------------
|
|
@@ -469,7 +410,6 @@ export const exampleStudioCustomInputYaml = "# ───────────
|
|
|
469
410
|
export const allTaskData = [
|
|
470
411
|
...exampleGroqBlogListingData,
|
|
471
412
|
...exampleIdBasedRefData,
|
|
472
|
-
...exampleMixedRefTypesData,
|
|
473
413
|
...examplePathBasedRefData,
|
|
474
414
|
...examplePerspectiveRefData,
|
|
475
415
|
...exampleStudioCustomInputData,
|
|
@@ -478,13 +418,12 @@ export const allTaskData = [
|
|
|
478
418
|
export const taskYamlFiles = {
|
|
479
419
|
"example-groq-blog-listing": exampleGroqBlogListingYaml,
|
|
480
420
|
"example-id-based-ref": exampleIdBasedRefYaml,
|
|
481
|
-
"example-mixed-ref-types": exampleMixedRefTypesYaml,
|
|
482
421
|
"example-path-based-ref": examplePathBasedRefYaml,
|
|
483
422
|
"example-perspective-ref": examplePerspectiveRefYaml,
|
|
484
423
|
"example-studio-custom-input": exampleStudioCustomInputYaml,
|
|
485
424
|
};
|
|
486
425
|
/** List of task file stems, in alphabetical order */
|
|
487
|
-
export const TASK_FILE_NAMES = ["example-groq-blog-listing", "example-id-based-ref", "example-
|
|
426
|
+
export const TASK_FILE_NAMES = ["example-groq-blog-listing", "example-id-based-ref", "example-path-based-ref", "example-perspective-ref", "example-studio-custom-input"];
|
|
488
427
|
export const EXAMPLE_TYPES = ["config", "source", "rubric", "threshold", "ailf-config", "task"];
|
|
489
428
|
export const EXAMPLES = {
|
|
490
429
|
"config": {
|
|
@@ -63,12 +63,12 @@ export function detectFeatureArea(description) {
|
|
|
63
63
|
desc.includes("live preview")) {
|
|
64
64
|
return "visual-editing";
|
|
65
65
|
}
|
|
66
|
+
if (desc.includes("groq")) {
|
|
67
|
+
return "groq";
|
|
68
|
+
}
|
|
66
69
|
if (desc.includes("function") || desc.includes("webhook")) {
|
|
67
70
|
return "functions";
|
|
68
71
|
}
|
|
69
|
-
if (desc.startsWith("groq")) {
|
|
70
|
-
return "groq";
|
|
71
|
-
}
|
|
72
72
|
if (desc.includes("next") || desc.includes("app router")) {
|
|
73
73
|
return "nextjs-live";
|
|
74
74
|
}
|
|
@@ -53,6 +53,8 @@ export function detectFeatureArea(description) {
|
|
|
53
53
|
desc.includes("presentation") ||
|
|
54
54
|
desc.includes("live preview"))
|
|
55
55
|
return "visual-editing";
|
|
56
|
+
if (desc.includes("groq"))
|
|
57
|
+
return "groq";
|
|
56
58
|
if (desc.includes("function") || desc.includes("webhook"))
|
|
57
59
|
return "functions";
|
|
58
60
|
if (desc.includes("next") || desc.includes("app router"))
|
|
@@ -163,7 +163,7 @@ function aggregateAgentBehavior(resultsPath) {
|
|
|
163
163
|
const byFeature = {};
|
|
164
164
|
let hasBehaviorData = false;
|
|
165
165
|
for (const result of results) {
|
|
166
|
-
const feature = detectFeatureArea(result.description);
|
|
166
|
+
const feature = result.vars.__featureArea || detectFeatureArea(result.description);
|
|
167
167
|
const behavior = extractAgentBehavior(result);
|
|
168
168
|
if (!behavior) {
|
|
169
169
|
continue;
|
|
@@ -241,7 +241,7 @@ function aggregateUrlReferences(resultsPath) {
|
|
|
241
241
|
const results = readAndNormalizeResults(resultsPath);
|
|
242
242
|
const byFeature = {};
|
|
243
243
|
for (const result of results) {
|
|
244
|
-
const feature = detectFeatureArea(result.description);
|
|
244
|
+
const feature = result.vars.__featureArea || detectFeatureArea(result.description);
|
|
245
245
|
if (!byFeature[feature]) {
|
|
246
246
|
byFeature[feature] = {
|
|
247
247
|
baseline: { testCount: 0, urls: {} },
|
|
@@ -481,7 +481,7 @@ function scoreResults(results, weights, modelId) {
|
|
|
481
481
|
// Group by feature + docs/no-docs
|
|
482
482
|
const byFeature = {};
|
|
483
483
|
for (const result of results) {
|
|
484
|
-
const feature = detectFeatureArea(result.description);
|
|
484
|
+
const feature = result.vars.__featureArea || detectFeatureArea(result.description);
|
|
485
485
|
if (!byFeature[feature]) {
|
|
486
486
|
byFeature[feature] = { withDocs: [], withoutDocs: [] };
|
|
487
487
|
}
|
|
@@ -580,7 +580,7 @@ export function scoreAgenticResults(resultsPath, weights) {
|
|
|
580
580
|
// Group by feature area
|
|
581
581
|
const byFeature = {};
|
|
582
582
|
for (const result of results) {
|
|
583
|
-
const feature = detectFeatureArea(result.description);
|
|
583
|
+
const feature = result.vars.__featureArea || detectFeatureArea(result.description);
|
|
584
584
|
if (!byFeature[feature]) {
|
|
585
585
|
byFeature[feature] = [];
|
|
586
586
|
}
|
|
@@ -85,6 +85,8 @@ export interface SingleTaskDefinition {
|
|
|
85
85
|
description: string;
|
|
86
86
|
/** Opt-in: auto-generate a documentation coverage rubric for gold. */
|
|
87
87
|
doc_coverage?: boolean;
|
|
88
|
+
/** Feature area this task belongs to (flows through to scoring). */
|
|
89
|
+
featureArea?: string;
|
|
88
90
|
/** Explicit task ID — determines the canonical context filename. */
|
|
89
91
|
id: string;
|
|
90
92
|
/** Template variables: task prompt and docs path. */
|
|
@@ -168,7 +168,7 @@ export function expandTask(task, rubricConfig, mode = "baseline") {
|
|
|
168
168
|
assert: [...resolvedAsserts],
|
|
169
169
|
description: `${task.description} (gold)`,
|
|
170
170
|
...(mode === "baseline" ? { prompts: ["with-docs"] } : {}),
|
|
171
|
-
vars: { ...task.vars },
|
|
171
|
+
vars: { ...task.vars, __featureArea: task.featureArea ?? "" },
|
|
172
172
|
});
|
|
173
173
|
// Baseline entry — floor measurement (no docs, parametric knowledge only).
|
|
174
174
|
// Skipped entirely in agentic mode: the agentic prompt doesn't reference
|
|
@@ -188,6 +188,7 @@ export function expandTask(task, rubricConfig, mode = "baseline") {
|
|
|
188
188
|
vars: {
|
|
189
189
|
...task.vars,
|
|
190
190
|
docs: "",
|
|
191
|
+
__featureArea: task.featureArea ?? "",
|
|
191
192
|
},
|
|
192
193
|
...(baselineAsserts.length > 0 ? { assert: baselineAsserts } : {}),
|
|
193
194
|
});
|
|
@@ -204,6 +205,7 @@ function taskDefinitionToSingle(task) {
|
|
|
204
205
|
baseline: task.baseline,
|
|
205
206
|
description: task.description,
|
|
206
207
|
doc_coverage: task.docCoverage,
|
|
208
|
+
featureArea: task.featureArea,
|
|
207
209
|
id: task.id,
|
|
208
210
|
vars: {
|
|
209
211
|
docs: `file://contexts/canonical/${task.id}.md`,
|
|
@@ -52,10 +52,10 @@ function detectFeatureArea(description) {
|
|
|
52
52
|
desc.includes("presentation") ||
|
|
53
53
|
desc.includes("live preview"))
|
|
54
54
|
return "visual-editing";
|
|
55
|
+
if (desc.includes("groq"))
|
|
56
|
+
return "groq";
|
|
55
57
|
if (desc.includes("function") || desc.includes("webhook"))
|
|
56
58
|
return "functions";
|
|
57
|
-
if (desc.startsWith("groq"))
|
|
58
|
-
return "groq";
|
|
59
59
|
if (desc.includes("next") || desc.includes("app router"))
|
|
60
60
|
return "nextjs-live";
|
|
61
61
|
if (desc.includes("remix") ||
|
|
@@ -57,10 +57,10 @@ function detectFeatureArea(description) {
|
|
|
57
57
|
desc.includes("presentation") ||
|
|
58
58
|
desc.includes("live preview"))
|
|
59
59
|
return "visual-editing";
|
|
60
|
+
if (desc.includes("groq"))
|
|
61
|
+
return "groq";
|
|
60
62
|
if (desc.includes("function") || desc.includes("webhook"))
|
|
61
63
|
return "functions";
|
|
62
|
-
if (desc.startsWith("groq"))
|
|
63
|
-
return "groq";
|
|
64
64
|
if (desc.includes("next") || desc.includes("app router"))
|
|
65
65
|
return "nextjs-live";
|
|
66
66
|
if (desc.includes("remix") ||
|