@sanity/ailf 4.0.7 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/ailf.js +6 -1
- package/dist/_vendor/ailf-core/schemas/external-providers.d.ts +136 -0
- package/dist/_vendor/ailf-core/schemas/external-providers.js +136 -0
- package/dist/_vendor/ailf-core/schemas/index.d.ts +2 -0
- package/dist/_vendor/ailf-core/schemas/index.js +2 -0
- package/dist/_vendor/ailf-core/schemas/pipeline-request.d.ts +2 -3
- package/dist/_vendor/ailf-core/schemas/report.d.ts +251 -0
- package/dist/_vendor/ailf-core/schemas/report.js +235 -0
- package/dist/_vendor/ailf-core/services/index.d.ts +1 -0
- package/dist/_vendor/ailf-core/services/index.js +1 -0
- package/dist/_vendor/ailf-core/services/report-to-markdown.d.ts +38 -0
- package/dist/_vendor/ailf-core/services/report-to-markdown.js +696 -0
- package/dist/_vendor/ailf-core/types/api-requests.d.ts +159 -0
- package/dist/_vendor/ailf-core/types/api-requests.js +27 -0
- package/dist/_vendor/ailf-core/types/generalized-task.d.ts +20 -3
- package/dist/_vendor/ailf-core/types/index.d.ts +4 -1
- package/dist/_vendor/ailf-core/types/pipeline-request.d.ts +112 -0
- package/dist/_vendor/ailf-core/types/pipeline-request.js +18 -0
- package/dist/_vendor/ailf-core/types/repo-config.d.ts +146 -0
- package/dist/_vendor/ailf-core/types/repo-config.js +18 -0
- package/dist/_vendor/ailf-shared/index.d.ts +7 -5
- package/dist/_vendor/ailf-shared/index.js +7 -5
- package/dist/adapters/api-client/types.d.ts +2 -5
- package/dist/adapters/doc-fetchers/sanity-doc-fetcher.d.ts +21 -5
- package/dist/adapters/doc-fetchers/sanity-doc-fetcher.js +129 -25
- package/dist/adapters/task-sources/content-lake-task-source.d.ts +58 -1
- package/dist/adapters/task-sources/content-lake-task-source.js +1 -1
- package/dist/adapters/task-sources/index.d.ts +1 -1
- package/dist/adapters/task-sources/index.js +1 -1
- package/dist/adapters/task-sources/repo-schemas.d.ts +19 -2
- package/dist/adapters/task-sources/repo-schemas.js +81 -2
- package/dist/adapters/task-sources/repo-task-source.js +11 -2
- package/dist/adapters/task-sources/repo-validation.d.ts +6 -6
- package/dist/adapters/task-sources/repo-validation.js +1 -1
- package/dist/agent-observer/agentic-provider.d.ts +1 -0
- package/dist/agent-observer/agentic-provider.js +43 -36
- package/dist/agent-observer/config-schemas.d.ts +61 -0
- package/dist/agent-observer/config-schemas.js +65 -0
- package/dist/agent-observer/provider.d.ts +1 -0
- package/dist/agent-observer/provider.js +19 -17
- package/dist/cli.js +4 -4
- package/dist/commands/validate-tasks.js +10 -4
- package/dist/composition-root.js +4 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/job-store.js +2 -2
- package/dist/lib/dotenv-resolution.d.ts +21 -0
- package/dist/lib/dotenv-resolution.js +30 -0
- package/dist/orchestration/steps/mirror-repo-tasks-step.js +14 -3
- package/dist/orchestration/steps/run-eval-step.js +21 -3
- package/dist/pipeline/agent-behavior-report.d.ts +2 -8
- package/dist/pipeline/cache.d.ts +2 -2
- package/dist/pipeline/checks.d.ts +10 -2
- package/dist/pipeline/checks.js +14 -4
- package/dist/pipeline/compiler/literacy-bridge.js +2 -2
- package/dist/pipeline/compiler/mode-handlers/__fixtures__/agent-harness-example-tasks.js +0 -12
- package/dist/pipeline/compiler/mode-handlers/__fixtures__/knowledge-probe-example-tasks.js +0 -12
- package/dist/pipeline/compiler/mode-handlers/agent-harness/types.d.ts +2 -2
- package/dist/pipeline/compiler/mode-handlers/index.d.ts +1 -1
- package/dist/pipeline/compiler/mode-handlers/knowledge-probe/types.d.ts +2 -2
- package/dist/pipeline/compiler/mode-handlers/literacy/compiler.js +44 -5
- package/dist/pipeline/compiler/mode-handlers/literacy/index.d.ts +1 -1
- package/dist/pipeline/compiler/mode-handlers/literacy/types.d.ts +3 -3
- package/dist/pipeline/compiler/promptfoo-compiler.js +7 -11
- package/dist/pipeline/compiler/provider-assembler.js +33 -3
- package/dist/pipeline/compiler/rubric-resolution.d.ts +2 -2
- package/dist/pipeline/mirror-repo-tasks.d.ts +13 -5
- package/dist/pipeline/mirror-repo-tasks.js +16 -8
- package/dist/pipeline/pr-comment.d.ts +22 -9
- package/dist/pipeline/pr-comment.js +52 -472
- package/dist/pipeline/resolve-mappings.d.ts +8 -3
- package/dist/promptfoo-providers/mock-path.d.ts +12 -0
- package/dist/promptfoo-providers/mock-path.js +15 -0
- package/dist/report-store.d.ts +63 -1
- package/dist/report-store.js +111 -31
- package/dist/sanity/client.d.ts +58 -0
- package/dist/sanity/client.js +106 -0
- package/dist/sanity/document-renderers.d.ts +68 -0
- package/dist/sanity/document-renderers.js +221 -0
- package/dist/sanity/queries.d.ts +21 -0
- package/dist/sanity/queries.js +71 -0
- package/dist/tasks/knowledge-probe/define-type-api.task.ts +2 -6
- package/dist/tasks/knowledge-probe/groq-projections.task.ts +0 -5
- package/dist/tasks/literacy/content-lake.task.ts +4 -10
- package/dist/tasks/literacy/frameworks.task.ts +2 -8
- package/dist/tasks/literacy/functions.task.ts +1 -4
- package/dist/tasks/literacy/groq.task.ts +3 -12
- package/dist/tasks/literacy/image-handling.task.ts +1 -4
- package/dist/tasks/literacy/nextjs-live.task.ts +1 -4
- package/dist/tasks/literacy/portable-text.task.ts +2 -8
- package/dist/tasks/literacy/studio-setup.task.ts +2 -8
- package/dist/tasks/literacy/visual-editing.task.ts +2 -8
- package/package.json +8 -7
- package/tasks/knowledge-probe/define-type-api.task.ts +2 -6
- package/tasks/knowledge-probe/groq-projections.task.ts +0 -5
- package/tasks/literacy/content-lake.task.ts +4 -10
- package/tasks/literacy/frameworks.task.ts +2 -8
- package/tasks/literacy/functions.task.ts +1 -4
- package/tasks/literacy/groq.task.ts +3 -12
- package/tasks/literacy/image-handling.task.ts +1 -4
- package/tasks/literacy/nextjs-live.task.ts +1 -4
- package/tasks/literacy/portable-text.task.ts +2 -8
- package/tasks/literacy/studio-setup.task.ts +2 -8
- package/tasks/literacy/visual-editing.task.ts +2 -8
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* document-renderers.ts
|
|
3
|
+
*
|
|
4
|
+
* Renderer registry that turns a Sanity document fetched by `_id` into
|
|
5
|
+
* Markdown for inclusion in a literacy task's grader context.
|
|
6
|
+
*
|
|
7
|
+
* The resolver fetches docs without an `_type` filter and dispatches here.
|
|
8
|
+
* Two tiers of fidelity:
|
|
9
|
+
*
|
|
10
|
+
* 1. Registered renderers (high fidelity) — `article`, `typesReference`.
|
|
11
|
+
* Hand-written for the document shapes we care about most.
|
|
12
|
+
* 2. Default renderer (best effort) — walks the doc, flattens portable-text
|
|
13
|
+
* fields, surfaces top-level scalars, follows references one level deep,
|
|
14
|
+
* skips framework-internal fields. Lets pinning a `marketingPage`,
|
|
15
|
+
* `glossaryEntry`, etc. work without AILF code changes.
|
|
16
|
+
*
|
|
17
|
+
* Adding a new high-fidelity renderer: implement a `DocumentRenderer` and
|
|
18
|
+
* register it in `BUILT_IN_RENDERERS` keyed by `_type`.
|
|
19
|
+
*/
|
|
20
|
+
import { toMarkdown } from "./portable-text.js";
|
|
21
|
+
// ---------------------------------------------------------------------------
|
|
22
|
+
// Helpers
|
|
23
|
+
// ---------------------------------------------------------------------------
|
|
24
|
+
const DEFAULT_MAX_BYTES = 30_000;
|
|
25
|
+
const SKIP_FIELDS = new Set([
|
|
26
|
+
"_createdAt",
|
|
27
|
+
"_id",
|
|
28
|
+
"_rev",
|
|
29
|
+
"_system",
|
|
30
|
+
"_type",
|
|
31
|
+
"_updatedAt",
|
|
32
|
+
"_key",
|
|
33
|
+
]);
|
|
34
|
+
function isPortableTextArray(value) {
|
|
35
|
+
return (Array.isArray(value) &&
|
|
36
|
+
value.length > 0 &&
|
|
37
|
+
typeof value[0] === "object" &&
|
|
38
|
+
value[0] !== null &&
|
|
39
|
+
"_type" in value[0]);
|
|
40
|
+
}
|
|
41
|
+
function truncate(content, max) {
|
|
42
|
+
if (content.length <= max)
|
|
43
|
+
return content;
|
|
44
|
+
return (content.slice(0, max) +
|
|
45
|
+
`\n\n*[content truncated — ${content.length - max} more bytes omitted]*`);
|
|
46
|
+
}
|
|
47
|
+
function slugForDoc(doc) {
|
|
48
|
+
const slugField = doc.slug;
|
|
49
|
+
if (typeof slugField === "object" &&
|
|
50
|
+
slugField !== null &&
|
|
51
|
+
"current" in slugField &&
|
|
52
|
+
typeof slugField.current === "string") {
|
|
53
|
+
return slugField.current;
|
|
54
|
+
}
|
|
55
|
+
if (typeof slugField === "string")
|
|
56
|
+
return slugField;
|
|
57
|
+
return `${doc._type}:${doc._id}`;
|
|
58
|
+
}
|
|
59
|
+
function articleRenderer(doc) {
|
|
60
|
+
const title = doc.title ?? "(untitled)";
|
|
61
|
+
const description = doc.description;
|
|
62
|
+
const section = doc.section;
|
|
63
|
+
const content = doc.content;
|
|
64
|
+
const sectionLabel = section?.title ? `Section: ${section.title}\n` : "";
|
|
65
|
+
const desc = description ? `${description}\n\n` : "";
|
|
66
|
+
const markdown = toMarkdown(content ?? []);
|
|
67
|
+
return {
|
|
68
|
+
content: `## ${title}\n\n${sectionLabel}${desc}${markdown}`,
|
|
69
|
+
fidelity: "high",
|
|
70
|
+
slug: slugForDoc(doc),
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
async function typesReferenceRenderer(doc, ctx) {
|
|
74
|
+
const title = doc.title ?? "(untitled)";
|
|
75
|
+
const slug = slugForDoc(doc);
|
|
76
|
+
const library = doc.library;
|
|
77
|
+
const latestVersion = doc.latestVersion;
|
|
78
|
+
const asset = latestVersion?.attachment?.asset;
|
|
79
|
+
const max = ctx.maxBytes ?? DEFAULT_MAX_BYTES;
|
|
80
|
+
const lines = [];
|
|
81
|
+
lines.push(`## ${title}`, "");
|
|
82
|
+
lines.push(`Slug: \`${slug}\``);
|
|
83
|
+
if (library?.npmName)
|
|
84
|
+
lines.push(`Package: \`${library.npmName}\``);
|
|
85
|
+
if (latestVersion?.semver)
|
|
86
|
+
lines.push(`Version: \`${latestVersion.semver}\``);
|
|
87
|
+
if (latestVersion?.date)
|
|
88
|
+
lines.push(`Released: ${latestVersion.date}`);
|
|
89
|
+
lines.push("");
|
|
90
|
+
if (!asset?.url) {
|
|
91
|
+
lines.push("*[no attached type definitions found — resolver could not locate `latestVersion.attachment.asset.url`]*");
|
|
92
|
+
return { content: lines.join("\n"), fidelity: "high", slug };
|
|
93
|
+
}
|
|
94
|
+
if (!ctx.fetchUrl) {
|
|
95
|
+
lines.push(`*[type definitions live at ${asset.url} — fetcher not configured to retrieve attachments]*`);
|
|
96
|
+
return { content: lines.join("\n"), fidelity: "high", slug };
|
|
97
|
+
}
|
|
98
|
+
const body = await ctx.fetchUrl(asset.url);
|
|
99
|
+
if (body === null) {
|
|
100
|
+
lines.push(`*[failed to fetch type definitions from ${asset.url}]*`);
|
|
101
|
+
return { content: lines.join("\n"), fidelity: "high", slug };
|
|
102
|
+
}
|
|
103
|
+
const filename = asset.originalFilename ?? "types.json";
|
|
104
|
+
const lang = filename.endsWith(".json") ? "json" : "";
|
|
105
|
+
lines.push(`### Type definitions (${filename})`, "");
|
|
106
|
+
lines.push("```" + lang);
|
|
107
|
+
// Reserve ~200 bytes for the fence + trailing notice
|
|
108
|
+
lines.push(truncate(body, Math.max(0, max - 200)));
|
|
109
|
+
lines.push("```");
|
|
110
|
+
return { content: lines.join("\n"), fidelity: "high", slug };
|
|
111
|
+
}
|
|
112
|
+
// ---------------------------------------------------------------------------
|
|
113
|
+
// formatDefault — generic walker for any unknown `_type`.
|
|
114
|
+
//
|
|
115
|
+
// Walks top-level fields, renders portable-text arrays as Markdown, surfaces
|
|
116
|
+
// scalars as labeled key/value lines, and follows resolved references one
|
|
117
|
+
// level deep. The output is "best effort" — not as tight as a hand-written
|
|
118
|
+
// renderer, but feeds the LLM grader real content for any pinned doc type.
|
|
119
|
+
// ---------------------------------------------------------------------------
|
|
120
|
+
function renderScalar(value) {
|
|
121
|
+
if (typeof value === "string")
|
|
122
|
+
return value;
|
|
123
|
+
if (typeof value === "number" || typeof value === "boolean") {
|
|
124
|
+
return String(value);
|
|
125
|
+
}
|
|
126
|
+
return null;
|
|
127
|
+
}
|
|
128
|
+
function renderField(key, value, depth = 0) {
|
|
129
|
+
if (value === null || value === undefined)
|
|
130
|
+
return null;
|
|
131
|
+
if (SKIP_FIELDS.has(key))
|
|
132
|
+
return null;
|
|
133
|
+
const indent = " ".repeat(depth);
|
|
134
|
+
if (isPortableTextArray(value)) {
|
|
135
|
+
const md = toMarkdown(value);
|
|
136
|
+
if (!md.trim())
|
|
137
|
+
return null;
|
|
138
|
+
return `${indent}**${key}:**\n\n${md}`;
|
|
139
|
+
}
|
|
140
|
+
// Sanity slug field: { _type: "slug", current: "…" }
|
|
141
|
+
if (typeof value === "object" &&
|
|
142
|
+
value !== null &&
|
|
143
|
+
!Array.isArray(value) &&
|
|
144
|
+
value._type === "slug") {
|
|
145
|
+
const current = value.current;
|
|
146
|
+
if (typeof current === "string")
|
|
147
|
+
return `${indent}**${key}:** \`${current}\``;
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
150
|
+
const scalar = renderScalar(value);
|
|
151
|
+
if (scalar !== null)
|
|
152
|
+
return `${indent}**${key}:** ${scalar}`;
|
|
153
|
+
// Single-level deref: a previously-resolved reference shows up as an
|
|
154
|
+
// object with its own `_id`/`_type` (the projection joined it in). Walk
|
|
155
|
+
// its fields one level deep.
|
|
156
|
+
if (typeof value === "object" &&
|
|
157
|
+
!Array.isArray(value) &&
|
|
158
|
+
depth === 0 &&
|
|
159
|
+
value._id !== undefined) {
|
|
160
|
+
const inner = Object.entries(value)
|
|
161
|
+
.map(([k, v]) => renderField(k, v, depth + 1))
|
|
162
|
+
.filter((line) => line !== null);
|
|
163
|
+
if (inner.length === 0)
|
|
164
|
+
return null;
|
|
165
|
+
return `${indent}**${key}:**\n${inner.join("\n")}`;
|
|
166
|
+
}
|
|
167
|
+
if (Array.isArray(value)) {
|
|
168
|
+
const items = value
|
|
169
|
+
.map((item, i) => renderField(String(i), item, depth + 1))
|
|
170
|
+
.filter((line) => line !== null);
|
|
171
|
+
if (items.length === 0)
|
|
172
|
+
return null;
|
|
173
|
+
return `${indent}**${key}:**\n${items.join("\n")}`;
|
|
174
|
+
}
|
|
175
|
+
return null;
|
|
176
|
+
}
|
|
177
|
+
function defaultRenderer(doc) {
|
|
178
|
+
const title = doc.title ?? `(${doc._type})`;
|
|
179
|
+
const slug = slugForDoc(doc);
|
|
180
|
+
const lines = [`## ${title}`, "", `Type: \`${doc._type}\``];
|
|
181
|
+
if (slug !== `${doc._type}:${doc._id}`)
|
|
182
|
+
lines.push(`Slug: \`${slug}\``);
|
|
183
|
+
lines.push("");
|
|
184
|
+
const fieldLines = [];
|
|
185
|
+
for (const [key, value] of Object.entries(doc)) {
|
|
186
|
+
if (key === "title")
|
|
187
|
+
continue;
|
|
188
|
+
if (key === "slug" && slug !== `${doc._type}:${doc._id}`)
|
|
189
|
+
continue;
|
|
190
|
+
const rendered = renderField(key, value);
|
|
191
|
+
if (rendered !== null)
|
|
192
|
+
fieldLines.push(rendered);
|
|
193
|
+
}
|
|
194
|
+
if (fieldLines.length > 0) {
|
|
195
|
+
lines.push(...fieldLines);
|
|
196
|
+
}
|
|
197
|
+
else {
|
|
198
|
+
lines.push("*[no renderable content — all fields were null, framework metadata, or unrecognized shapes]*");
|
|
199
|
+
}
|
|
200
|
+
return { content: lines.join("\n"), fidelity: "default", slug };
|
|
201
|
+
}
|
|
202
|
+
// ---------------------------------------------------------------------------
|
|
203
|
+
// Registry
|
|
204
|
+
// ---------------------------------------------------------------------------
|
|
205
|
+
const BUILT_IN_RENDERERS = {
|
|
206
|
+
article: articleRenderer,
|
|
207
|
+
typesReference: typesReferenceRenderer,
|
|
208
|
+
};
|
|
209
|
+
/**
|
|
210
|
+
* Render a document using the registered renderer for its `_type`, falling
|
|
211
|
+
* back to the default walker. The returned `fidelity` flag tells callers
|
|
212
|
+
* whether to emit the "info: dedicated renderer would help" log.
|
|
213
|
+
*/
|
|
214
|
+
export async function renderDocument(doc, ctx = {}) {
|
|
215
|
+
const renderer = BUILT_IN_RENDERERS[doc._type];
|
|
216
|
+
if (renderer)
|
|
217
|
+
return renderer(doc, ctx);
|
|
218
|
+
return defaultRenderer(doc);
|
|
219
|
+
}
|
|
220
|
+
/** Exported for tests and consumers that want the registered set. */
|
|
221
|
+
export const REGISTERED_RENDERER_TYPES = Object.keys(BUILT_IN_RENDERERS).sort();
|
package/dist/sanity/queries.d.ts
CHANGED
|
@@ -87,6 +87,27 @@ export declare const ARTICLE_BY_SLUG_WITH_PERSPECTIVE_QUERY = "\n *[_type == \"
|
|
|
87
87
|
* @param $ids — array of document ID strings
|
|
88
88
|
*/
|
|
89
89
|
export declare const ARTICLES_BY_IDS_QUERY = "\n *[_type == \"article\"\n && _id in $ids\n ] {\n title,\n description,\n \"slug\": slug.current,\n \"section\": primarySection->{ \"slug\": slug.current, title },\n \"content\": content[] {\n // Pass through all standard blocks unchanged\n _type != \"docsCardCollection\" => { ... },\n // Resolve references inside card collections so we get titles/slugs\n _type == \"docsCardCollection\" => {\n ...,\n cards[] {\n ...,\n \"resolvedTitle\": reference->title,\n \"resolvedSlug\": reference->slug.current\n }\n }\n }\n}\n";
|
|
90
|
+
/**
|
|
91
|
+
* Fetch arbitrary documents by their `_id` — no `_type` filter.
|
|
92
|
+
*
|
|
93
|
+
* Used by id-ref resolution so authors can pin any document type
|
|
94
|
+
* (`article`, `typesReference`, `marketingPage`, `glossaryEntry`, …),
|
|
95
|
+
* not just articles. The projection branches on `_type`:
|
|
96
|
+
*
|
|
97
|
+
* - `article` → reuses ARTICLE_PROJECTION's flatten + card-deref shape.
|
|
98
|
+
* - `typesReference` → derefs `library` and `latestVersion`, including
|
|
99
|
+
* the resolved file asset URL so the renderer can fetch typedoc JSON
|
|
100
|
+
* without a follow-up round-trip.
|
|
101
|
+
* - default `{...}` → returns the raw document fields, which the
|
|
102
|
+
* default walker in `document-renderers.ts` flattens to Markdown.
|
|
103
|
+
*
|
|
104
|
+
* Does NOT include the `!(_id in path("drafts.**"))` filter — id-ref
|
|
105
|
+
* resolution is intentional, drafts are queryable when perspective is
|
|
106
|
+
* draft-enabled, and authors are expected to pin published `_id`s.
|
|
107
|
+
*
|
|
108
|
+
* @param $ids — array of document ID strings
|
|
109
|
+
*/
|
|
110
|
+
export declare const DOCS_BY_IDS_QUERY = "\n *[_id in $ids] {\n _id,\n _type,\n _system,\n _createdAt,\n _updatedAt,\n _rev,\n _type == \"article\" => {\n title,\n description,\n \"slug\": slug.current,\n \"section\": primarySection->{ \"slug\": slug.current, title },\n \"content\": content[] {\n _type != \"docsCardCollection\" => { ... },\n _type == \"docsCardCollection\" => {\n ...,\n cards[] {\n ...,\n \"resolvedTitle\": reference->title,\n \"resolvedSlug\": reference->slug.current\n }\n }\n }\n },\n _type == \"typesReference\" => {\n title,\n autoPublish,\n \"slug\": slug,\n \"library\": library->{ _id, _type, title, npmName },\n \"latestVersion\": latestVersion->{\n _id,\n _type,\n semver,\n date,\n \"attachment\": {\n \"asset\": attachment.asset->{\n _id,\n _type,\n url,\n originalFilename,\n mimeType,\n size,\n extension\n }\n }\n }\n },\n !(_type in [\"article\", \"typesReference\"]) => { ... }\n }\n";
|
|
90
111
|
/**
|
|
91
112
|
* Fetch a single article by its document ID.
|
|
92
113
|
*
|
package/dist/sanity/queries.js
CHANGED
|
@@ -229,6 +229,77 @@ export const ARTICLES_BY_IDS_QUERY = `
|
|
|
229
229
|
&& _id in $ids
|
|
230
230
|
] ${ARTICLE_PROJECTION}
|
|
231
231
|
`;
|
|
232
|
+
/**
|
|
233
|
+
* Fetch arbitrary documents by their `_id` — no `_type` filter.
|
|
234
|
+
*
|
|
235
|
+
* Used by id-ref resolution so authors can pin any document type
|
|
236
|
+
* (`article`, `typesReference`, `marketingPage`, `glossaryEntry`, …),
|
|
237
|
+
* not just articles. The projection branches on `_type`:
|
|
238
|
+
*
|
|
239
|
+
* - `article` → reuses ARTICLE_PROJECTION's flatten + card-deref shape.
|
|
240
|
+
* - `typesReference` → derefs `library` and `latestVersion`, including
|
|
241
|
+
* the resolved file asset URL so the renderer can fetch typedoc JSON
|
|
242
|
+
* without a follow-up round-trip.
|
|
243
|
+
* - default `{...}` → returns the raw document fields, which the
|
|
244
|
+
* default walker in `document-renderers.ts` flattens to Markdown.
|
|
245
|
+
*
|
|
246
|
+
* Does NOT include the `!(_id in path("drafts.**"))` filter — id-ref
|
|
247
|
+
* resolution is intentional, drafts are queryable when perspective is
|
|
248
|
+
* draft-enabled, and authors are expected to pin published `_id`s.
|
|
249
|
+
*
|
|
250
|
+
* @param $ids — array of document ID strings
|
|
251
|
+
*/
|
|
252
|
+
export const DOCS_BY_IDS_QUERY = `
|
|
253
|
+
*[_id in $ids] {
|
|
254
|
+
_id,
|
|
255
|
+
_type,
|
|
256
|
+
_system,
|
|
257
|
+
_createdAt,
|
|
258
|
+
_updatedAt,
|
|
259
|
+
_rev,
|
|
260
|
+
_type == "article" => {
|
|
261
|
+
title,
|
|
262
|
+
description,
|
|
263
|
+
"slug": slug.current,
|
|
264
|
+
"section": primarySection->{ "slug": slug.current, title },
|
|
265
|
+
"content": content[] {
|
|
266
|
+
_type != "docsCardCollection" => { ... },
|
|
267
|
+
_type == "docsCardCollection" => {
|
|
268
|
+
...,
|
|
269
|
+
cards[] {
|
|
270
|
+
...,
|
|
271
|
+
"resolvedTitle": reference->title,
|
|
272
|
+
"resolvedSlug": reference->slug.current
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
},
|
|
277
|
+
_type == "typesReference" => {
|
|
278
|
+
title,
|
|
279
|
+
autoPublish,
|
|
280
|
+
"slug": slug,
|
|
281
|
+
"library": library->{ _id, _type, title, npmName },
|
|
282
|
+
"latestVersion": latestVersion->{
|
|
283
|
+
_id,
|
|
284
|
+
_type,
|
|
285
|
+
semver,
|
|
286
|
+
date,
|
|
287
|
+
"attachment": {
|
|
288
|
+
"asset": attachment.asset->{
|
|
289
|
+
_id,
|
|
290
|
+
_type,
|
|
291
|
+
url,
|
|
292
|
+
originalFilename,
|
|
293
|
+
mimeType,
|
|
294
|
+
size,
|
|
295
|
+
extension
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
},
|
|
300
|
+
!(_type in ["article", "typesReference"]) => { ... }
|
|
301
|
+
}
|
|
302
|
+
`;
|
|
232
303
|
/**
|
|
233
304
|
* Fetch a single article by its document ID.
|
|
234
305
|
*
|
|
@@ -26,7 +26,8 @@ export default defineTask({
|
|
|
26
26
|
// Controls how the probe explores knowledge: "breadth-first" covers many topics, "depth-first" drills deep
|
|
27
27
|
probeStrategy: "breadth-first",
|
|
28
28
|
prompt: {
|
|
29
|
-
// Direct prompt text sent to the model
|
|
29
|
+
// Direct prompt text sent to the model. The compiler treats `prompt.text`
|
|
30
|
+
// as the canonical prompt body across every mode.
|
|
30
31
|
text:
|
|
31
32
|
"Explain Sanity's schema definition API:\n\n" +
|
|
32
33
|
"1. What is `defineType` and how do you use it?\n" +
|
|
@@ -34,11 +35,6 @@ export default defineTask({
|
|
|
34
35
|
"3. Why were these typed helpers introduced? What did they replace?\n" +
|
|
35
36
|
"4. Show a complete example of a document schema with various field types\n" +
|
|
36
37
|
"5. How do you add validation rules using the typed API?",
|
|
37
|
-
vars: {
|
|
38
|
-
task:
|
|
39
|
-
"Explain Sanity's defineType/defineField schema API with examples, " +
|
|
40
|
-
"motivation, and validation rules.",
|
|
41
|
-
},
|
|
42
38
|
},
|
|
43
39
|
assertions: [
|
|
44
40
|
{ type: "contains", value: "defineType" },
|
|
@@ -32,11 +32,6 @@ export default defineTask({
|
|
|
32
32
|
"5. Array slicing with `[0..5]` and `[0...5]`\n" +
|
|
33
33
|
"6. Conditional projections using `select()`\n\n" +
|
|
34
34
|
"Provide working code examples for each.",
|
|
35
|
-
vars: {
|
|
36
|
-
task:
|
|
37
|
-
"Explain GROQ projection syntax with working code examples " +
|
|
38
|
-
"covering projections, spread, dereference, slicing, and select().",
|
|
39
|
-
},
|
|
40
35
|
},
|
|
41
36
|
assertions: [
|
|
42
37
|
{ type: "contains", value: "->" },
|
|
@@ -33,10 +33,10 @@ export default [
|
|
|
33
33
|
},
|
|
34
34
|
// Path (relative to eval package root) to a gold-standard implementation the grader compares against
|
|
35
35
|
referenceSolution: "reference-solutions/content-lake/mutations.ts",
|
|
36
|
+
// The instruction the model under evaluation sees. The compiler auto-derives
|
|
37
|
+
// the docs context (file://contexts/canonical/{task.id}.md) from context.docs.
|
|
36
38
|
prompt: {
|
|
37
|
-
|
|
38
|
-
// The instruction the model under evaluation sees (interpolated into the prompt template)
|
|
39
|
-
task: `Implement a content management service using @sanity/client that
|
|
39
|
+
text: `Implement a content management service using @sanity/client that
|
|
40
40
|
performs CRUD operations on Sanity documents.
|
|
41
41
|
|
|
42
42
|
Requirements:
|
|
@@ -47,9 +47,6 @@ Requirements:
|
|
|
47
47
|
5. Include proper TypeScript types
|
|
48
48
|
|
|
49
49
|
Provide a complete, reusable implementation.`,
|
|
50
|
-
// file:// URI resolved at runtime to a generated canonical context file (from npx @sanity/ailf fetch-docs)
|
|
51
|
-
docs: "file://contexts/canonical/content-lake-mutations.md",
|
|
52
|
-
},
|
|
53
50
|
},
|
|
54
51
|
// Grading criteria applied to the model's response; each assertion produces a pass/fail contributing to the task score
|
|
55
52
|
assertions: [
|
|
@@ -130,8 +127,7 @@ Provide a complete, reusable implementation.`,
|
|
|
130
127
|
},
|
|
131
128
|
referenceSolution: "reference-solutions/content-lake/realtime.ts",
|
|
132
129
|
prompt: {
|
|
133
|
-
|
|
134
|
-
task: `Implement real-time content synchronization using Sanity's listener API.
|
|
130
|
+
text: `Implement real-time content synchronization using Sanity's listener API.
|
|
135
131
|
|
|
136
132
|
Requirements:
|
|
137
133
|
1. Set up a listener that watches for changes to documents of a specific type
|
|
@@ -141,8 +137,6 @@ Requirements:
|
|
|
141
137
|
5. Provide a way to unsubscribe/clean up the listener
|
|
142
138
|
|
|
143
139
|
Provide a complete implementation with TypeScript types.`,
|
|
144
|
-
docs: "file://contexts/canonical/content-lake-realtime.md",
|
|
145
|
-
},
|
|
146
140
|
},
|
|
147
141
|
assertions: [
|
|
148
142
|
{
|
|
@@ -27,8 +27,7 @@ export default [
|
|
|
27
27
|
},
|
|
28
28
|
referenceSolution: "reference-solutions/frameworks/remix.tsx",
|
|
29
29
|
prompt: {
|
|
30
|
-
|
|
31
|
-
task: `Integrate Sanity into a Remix application:
|
|
30
|
+
text: `Integrate Sanity into a Remix application:
|
|
32
31
|
|
|
33
32
|
1. Set up the Sanity client
|
|
34
33
|
2. Create a loader that fetches blog posts using GROQ
|
|
@@ -36,8 +35,6 @@ export default [
|
|
|
36
35
|
4. Handle loading and error states properly
|
|
37
36
|
|
|
38
37
|
Provide all necessary files for a working Remix + Sanity integration.`,
|
|
39
|
-
docs: "file://contexts/canonical/remix-integration.md",
|
|
40
|
-
},
|
|
41
38
|
},
|
|
42
39
|
assertions: [
|
|
43
40
|
{
|
|
@@ -89,16 +86,13 @@ Provide all necessary files for a working Remix + Sanity integration.`,
|
|
|
89
86
|
},
|
|
90
87
|
referenceSolution: "reference-solutions/frameworks/nuxt.ts",
|
|
91
88
|
prompt: {
|
|
92
|
-
|
|
93
|
-
task: `Integrate Sanity into a Nuxt 4 application:
|
|
89
|
+
text: `Integrate Sanity into a Nuxt 4 application:
|
|
94
90
|
|
|
95
91
|
1. Install and configure the @nuxtjs/sanity module
|
|
96
92
|
2. Create a page that fetches and displays blog posts
|
|
97
93
|
3. Use Nuxt composables for data fetching
|
|
98
94
|
|
|
99
95
|
Provide all necessary configuration and component code.`,
|
|
100
|
-
docs: "file://contexts/canonical/nuxt-integration.md",
|
|
101
|
-
},
|
|
102
96
|
},
|
|
103
97
|
assertions: [
|
|
104
98
|
{
|
|
@@ -31,8 +31,7 @@ export default [
|
|
|
31
31
|
},
|
|
32
32
|
referenceSolution: "reference-solutions/functions/publish-webhook.ts",
|
|
33
33
|
prompt: {
|
|
34
|
-
|
|
35
|
-
task: `Deploy a Sanity function that triggers when a document is published
|
|
34
|
+
text: `Deploy a Sanity function that triggers when a document is published
|
|
36
35
|
and sends a webhook notification to an external endpoint.
|
|
37
36
|
|
|
38
37
|
Requirements:
|
|
@@ -42,8 +41,6 @@ Requirements:
|
|
|
42
41
|
4. Handle errors gracefully
|
|
43
42
|
|
|
44
43
|
Provide a complete implementation including any configuration.`,
|
|
45
|
-
docs: "file://contexts/canonical/functions-webhook.md",
|
|
46
|
-
},
|
|
47
44
|
},
|
|
48
45
|
assertions: [
|
|
49
46
|
{
|
|
@@ -31,8 +31,7 @@ export default [
|
|
|
31
31
|
},
|
|
32
32
|
referenceSolution: "reference-solutions/groq/blog-queries.ts",
|
|
33
33
|
prompt: {
|
|
34
|
-
|
|
35
|
-
task: `Write GROQ queries for a Sanity blog application:
|
|
34
|
+
text: `Write GROQ queries for a Sanity blog application:
|
|
36
35
|
|
|
37
36
|
1. Fetch all published blog posts ordered by publishedAt descending,
|
|
38
37
|
with a projection that includes: _id, title, slug (from slug.current),
|
|
@@ -46,8 +45,6 @@ export default [
|
|
|
46
45
|
|
|
47
46
|
Use @sanity/client with client.fetch() for all queries. Include
|
|
48
47
|
TypeScript types for the query results.`,
|
|
49
|
-
docs: "file://contexts/canonical/groq-blog-queries.md",
|
|
50
|
-
},
|
|
51
48
|
},
|
|
52
49
|
assertions: [
|
|
53
50
|
{
|
|
@@ -118,8 +115,7 @@ TypeScript types for the query results.`,
|
|
|
118
115
|
},
|
|
119
116
|
referenceSolution: "reference-solutions/groq/joins-references.ts",
|
|
120
117
|
prompt: {
|
|
121
|
-
|
|
122
|
-
task: `Write GROQ queries that demonstrate join patterns in Sanity:
|
|
118
|
+
text: `Write GROQ queries that demonstrate join patterns in Sanity:
|
|
123
119
|
|
|
124
120
|
1. Follow a single reference to resolve an author's full profile
|
|
125
121
|
from a post (post.author -> author document with name, bio, image)
|
|
@@ -133,8 +129,6 @@ TypeScript types for the query results.`,
|
|
|
133
129
|
a specific document ID
|
|
134
130
|
|
|
135
131
|
Use @sanity/client with client.fetch(). Include TypeScript types.`,
|
|
136
|
-
docs: "file://contexts/canonical/groq-joins-references.md",
|
|
137
|
-
},
|
|
138
132
|
},
|
|
139
133
|
assertions: [
|
|
140
134
|
{
|
|
@@ -198,8 +192,7 @@ Use @sanity/client with client.fetch(). Include TypeScript types.`,
|
|
|
198
192
|
},
|
|
199
193
|
referenceSolution: "reference-solutions/groq/advanced-filtering.ts",
|
|
200
194
|
prompt: {
|
|
201
|
-
|
|
202
|
-
task: `Write GROQ queries demonstrating advanced filtering and projection patterns:
|
|
195
|
+
text: `Write GROQ queries demonstrating advanced filtering and projection patterns:
|
|
203
196
|
|
|
204
197
|
1. Use select() for conditional projections — return different fields
|
|
205
198
|
based on the document's _type (e.g., posts get excerpt, events get
|
|
@@ -215,8 +208,6 @@ Use @sanity/client with client.fetch(). Include TypeScript types.`,
|
|
|
215
208
|
then by publishedAt)
|
|
216
209
|
|
|
217
210
|
Use @sanity/client with client.fetch(). Include TypeScript types.`,
|
|
218
|
-
docs: "file://contexts/canonical/groq-advanced-filtering.md",
|
|
219
|
-
},
|
|
220
211
|
},
|
|
221
212
|
assertions: [
|
|
222
213
|
{
|
|
@@ -33,8 +33,7 @@ export default [
|
|
|
33
33
|
},
|
|
34
34
|
referenceSolution: "reference-solutions/image-handling/asset-pipeline.tsx",
|
|
35
35
|
prompt: {
|
|
36
|
-
|
|
37
|
-
task: `Build an image component for a Next.js app that renders Sanity images
|
|
36
|
+
text: `Build an image component for a Next.js app that renders Sanity images
|
|
38
37
|
with proper transforms, hotspot/crop support, and responsive sizing.
|
|
39
38
|
|
|
40
39
|
Requirements:
|
|
@@ -45,8 +44,6 @@ Requirements:
|
|
|
45
44
|
5. Create a reusable React component with TypeScript props
|
|
46
45
|
|
|
47
46
|
Provide a complete implementation.`,
|
|
48
|
-
docs: "file://contexts/canonical/image-asset-pipeline.md",
|
|
49
|
-
},
|
|
50
47
|
},
|
|
51
48
|
assertions: [
|
|
52
49
|
{
|
|
@@ -31,8 +31,7 @@ export default [
|
|
|
31
31
|
},
|
|
32
32
|
referenceSolution: "reference-solutions/nextjs/app-router-integration.tsx",
|
|
33
33
|
prompt: {
|
|
34
|
-
|
|
35
|
-
task: `Integrate Sanity into a Next.js 16 App Router application
|
|
34
|
+
text: `Integrate Sanity into a Next.js 16 App Router application
|
|
36
35
|
with full TypeScript support:
|
|
37
36
|
|
|
38
37
|
1. Set up the Sanity client with proper configuration
|
|
@@ -41,8 +40,6 @@ with full TypeScript support:
|
|
|
41
40
|
4. Generate TypeScript types using sanity typegen
|
|
42
41
|
|
|
43
42
|
Provide all files needed for a working integration.`,
|
|
44
|
-
docs: "file://contexts/canonical/nextjs-app-router-integration.md",
|
|
45
|
-
},
|
|
46
43
|
},
|
|
47
44
|
assertions: [
|
|
48
45
|
{
|
|
@@ -33,8 +33,7 @@ export default [
|
|
|
33
33
|
},
|
|
34
34
|
referenceSolution: "reference-solutions/portable-text/rendering.tsx",
|
|
35
35
|
prompt: {
|
|
36
|
-
|
|
37
|
-
task: `Render Portable Text content from Sanity in a React application using
|
|
36
|
+
text: `Render Portable Text content from Sanity in a React application using
|
|
38
37
|
@portabletext/react.
|
|
39
38
|
|
|
40
39
|
Requirements:
|
|
@@ -45,8 +44,6 @@ Requirements:
|
|
|
45
44
|
5. Provide TypeScript types for the component props
|
|
46
45
|
|
|
47
46
|
Provide a complete, reusable implementation.`,
|
|
48
|
-
docs: "file://contexts/canonical/portable-text-rendering.md",
|
|
49
|
-
},
|
|
50
47
|
},
|
|
51
48
|
assertions: [
|
|
52
49
|
{
|
|
@@ -119,8 +116,7 @@ Provide a complete, reusable implementation.`,
|
|
|
119
116
|
},
|
|
120
117
|
referenceSolution: "reference-solutions/portable-text/custom-blocks.ts",
|
|
121
118
|
prompt: {
|
|
122
|
-
|
|
123
|
-
task: `Define custom block types for a Portable Text field and render them
|
|
119
|
+
text: `Define custom block types for a Portable Text field and render them
|
|
124
120
|
in a React frontend.
|
|
125
121
|
|
|
126
122
|
Requirements:
|
|
@@ -130,8 +126,6 @@ Requirements:
|
|
|
130
126
|
4. Show how to render these custom blocks with @portabletext/react
|
|
131
127
|
|
|
132
128
|
Provide both the schema definition and the frontend rendering code.`,
|
|
133
|
-
docs: "file://contexts/canonical/portable-text-custom-blocks.md",
|
|
134
|
-
},
|
|
135
129
|
},
|
|
136
130
|
assertions: [
|
|
137
131
|
{
|
|
@@ -32,8 +32,7 @@ export default [
|
|
|
32
32
|
},
|
|
33
33
|
referenceSolution: "reference-solutions/studio-setup/blog-schema.ts",
|
|
34
34
|
prompt: {
|
|
35
|
-
|
|
36
|
-
task: `Set up a new Sanity Studio with a custom schema for a blog:
|
|
35
|
+
text: `Set up a new Sanity Studio with a custom schema for a blog:
|
|
37
36
|
|
|
38
37
|
1. Create document types for: posts, authors, categories
|
|
39
38
|
2. Posts should have: title, slug, body (portable text), author reference, categories array
|
|
@@ -41,8 +40,6 @@ export default [
|
|
|
41
40
|
4. Categories should have: title, description
|
|
42
41
|
|
|
43
42
|
Include the schema definitions and sanity.config.ts setup.`,
|
|
44
|
-
docs: "file://contexts/canonical/studio-blog-schema.md",
|
|
45
|
-
},
|
|
46
43
|
},
|
|
47
44
|
assertions: [
|
|
48
45
|
{
|
|
@@ -99,8 +96,7 @@ Include the schema definitions and sanity.config.ts setup.`,
|
|
|
99
96
|
},
|
|
100
97
|
referenceSolution: "reference-solutions/studio-setup/custom-tool.tsx",
|
|
101
98
|
prompt: {
|
|
102
|
-
|
|
103
|
-
task: `Add a custom tool to the Sanity Studio sidebar that displays
|
|
99
|
+
text: `Add a custom tool to the Sanity Studio sidebar that displays
|
|
104
100
|
a dashboard. The tool should:
|
|
105
101
|
|
|
106
102
|
1. Appear in the studio navigation with a custom icon
|
|
@@ -108,8 +104,6 @@ a dashboard. The tool should:
|
|
|
108
104
|
3. Render a React component showing a "Dashboard" heading
|
|
109
105
|
|
|
110
106
|
Provide the tool definition and sanity.config.ts registration.`,
|
|
111
|
-
docs: "file://contexts/canonical/studio-custom-tool.md",
|
|
112
|
-
},
|
|
113
107
|
},
|
|
114
108
|
assertions: [
|
|
115
109
|
{
|
|
@@ -32,8 +32,7 @@ export default [
|
|
|
32
32
|
referenceSolution:
|
|
33
33
|
"reference-solutions/visual-editing/presentation-nextjs.tsx",
|
|
34
34
|
prompt: {
|
|
35
|
-
|
|
36
|
-
task: `Set up the Presentation tool with a Next.js 14 (App Router) frontend
|
|
35
|
+
text: `Set up the Presentation tool with a Next.js 14 (App Router) frontend
|
|
37
36
|
and implement click-to-edit functionality:
|
|
38
37
|
|
|
39
38
|
1. Configure the Presentation tool in sanity.config.ts
|
|
@@ -42,8 +41,6 @@ and implement click-to-edit functionality:
|
|
|
42
41
|
opens the corresponding field in Studio
|
|
43
42
|
|
|
44
43
|
Provide all necessary code for both Studio and Next.js sides.`,
|
|
45
|
-
docs: "file://contexts/canonical/visual-editing-presentation.md",
|
|
46
|
-
},
|
|
47
44
|
},
|
|
48
45
|
assertions: [
|
|
49
46
|
{
|
|
@@ -105,8 +102,7 @@ Provide all necessary code for both Studio and Next.js sides.`,
|
|
|
105
102
|
},
|
|
106
103
|
referenceSolution: "reference-solutions/visual-editing/live-preview.tsx",
|
|
107
104
|
prompt: {
|
|
108
|
-
|
|
109
|
-
task: `Implement live preview in a Next.js app that shows draft content
|
|
105
|
+
text: `Implement live preview in a Next.js app that shows draft content
|
|
110
106
|
from Sanity in real-time as editors make changes in the Studio.
|
|
111
107
|
|
|
112
108
|
Requirements:
|
|
@@ -115,8 +111,6 @@ Requirements:
|
|
|
115
111
|
- Show real-time updates without page refresh
|
|
116
112
|
|
|
117
113
|
Provide a complete implementation.`,
|
|
118
|
-
docs: "file://contexts/canonical/visual-editing-live-preview.md",
|
|
119
|
-
},
|
|
120
114
|
},
|
|
121
115
|
assertions: [
|
|
122
116
|
{
|