@sanity/ailf 4.0.7 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/ailf.js +6 -1
- package/dist/_vendor/ailf-core/schemas/external-providers.d.ts +136 -0
- package/dist/_vendor/ailf-core/schemas/external-providers.js +136 -0
- package/dist/_vendor/ailf-core/schemas/index.d.ts +2 -0
- package/dist/_vendor/ailf-core/schemas/index.js +2 -0
- package/dist/_vendor/ailf-core/schemas/pipeline-request.d.ts +2 -3
- package/dist/_vendor/ailf-core/schemas/report.d.ts +251 -0
- package/dist/_vendor/ailf-core/schemas/report.js +235 -0
- package/dist/_vendor/ailf-core/services/index.d.ts +1 -0
- package/dist/_vendor/ailf-core/services/index.js +1 -0
- package/dist/_vendor/ailf-core/services/report-to-markdown.d.ts +38 -0
- package/dist/_vendor/ailf-core/services/report-to-markdown.js +696 -0
- package/dist/_vendor/ailf-core/types/api-requests.d.ts +159 -0
- package/dist/_vendor/ailf-core/types/api-requests.js +27 -0
- package/dist/_vendor/ailf-core/types/generalized-task.d.ts +20 -3
- package/dist/_vendor/ailf-core/types/index.d.ts +4 -1
- package/dist/_vendor/ailf-core/types/pipeline-request.d.ts +112 -0
- package/dist/_vendor/ailf-core/types/pipeline-request.js +18 -0
- package/dist/_vendor/ailf-core/types/repo-config.d.ts +146 -0
- package/dist/_vendor/ailf-core/types/repo-config.js +18 -0
- package/dist/_vendor/ailf-shared/index.d.ts +7 -5
- package/dist/_vendor/ailf-shared/index.js +7 -5
- package/dist/adapters/api-client/types.d.ts +2 -5
- package/dist/adapters/doc-fetchers/sanity-doc-fetcher.d.ts +21 -5
- package/dist/adapters/doc-fetchers/sanity-doc-fetcher.js +129 -25
- package/dist/adapters/task-sources/content-lake-task-source.d.ts +58 -1
- package/dist/adapters/task-sources/content-lake-task-source.js +1 -1
- package/dist/adapters/task-sources/index.d.ts +1 -1
- package/dist/adapters/task-sources/index.js +1 -1
- package/dist/adapters/task-sources/repo-schemas.d.ts +19 -2
- package/dist/adapters/task-sources/repo-schemas.js +81 -2
- package/dist/adapters/task-sources/repo-task-source.js +11 -2
- package/dist/adapters/task-sources/repo-validation.d.ts +6 -6
- package/dist/adapters/task-sources/repo-validation.js +1 -1
- package/dist/agent-observer/agentic-provider.d.ts +1 -0
- package/dist/agent-observer/agentic-provider.js +43 -36
- package/dist/agent-observer/config-schemas.d.ts +61 -0
- package/dist/agent-observer/config-schemas.js +65 -0
- package/dist/agent-observer/provider.d.ts +1 -0
- package/dist/agent-observer/provider.js +19 -17
- package/dist/cli.js +4 -4
- package/dist/commands/validate-tasks.js +10 -4
- package/dist/composition-root.js +4 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/job-store.js +2 -2
- package/dist/lib/dotenv-resolution.d.ts +21 -0
- package/dist/lib/dotenv-resolution.js +30 -0
- package/dist/orchestration/steps/mirror-repo-tasks-step.js +14 -3
- package/dist/orchestration/steps/run-eval-step.js +21 -3
- package/dist/pipeline/agent-behavior-report.d.ts +2 -8
- package/dist/pipeline/cache.d.ts +2 -2
- package/dist/pipeline/checks.d.ts +10 -2
- package/dist/pipeline/checks.js +14 -4
- package/dist/pipeline/compiler/literacy-bridge.js +2 -2
- package/dist/pipeline/compiler/mode-handlers/__fixtures__/agent-harness-example-tasks.js +0 -12
- package/dist/pipeline/compiler/mode-handlers/__fixtures__/knowledge-probe-example-tasks.js +0 -12
- package/dist/pipeline/compiler/mode-handlers/agent-harness/types.d.ts +2 -2
- package/dist/pipeline/compiler/mode-handlers/index.d.ts +1 -1
- package/dist/pipeline/compiler/mode-handlers/knowledge-probe/types.d.ts +2 -2
- package/dist/pipeline/compiler/mode-handlers/literacy/compiler.js +44 -5
- package/dist/pipeline/compiler/mode-handlers/literacy/index.d.ts +1 -1
- package/dist/pipeline/compiler/mode-handlers/literacy/types.d.ts +3 -3
- package/dist/pipeline/compiler/promptfoo-compiler.js +7 -11
- package/dist/pipeline/compiler/provider-assembler.js +33 -3
- package/dist/pipeline/compiler/rubric-resolution.d.ts +2 -2
- package/dist/pipeline/mirror-repo-tasks.d.ts +13 -5
- package/dist/pipeline/mirror-repo-tasks.js +16 -8
- package/dist/pipeline/pr-comment.d.ts +22 -9
- package/dist/pipeline/pr-comment.js +52 -472
- package/dist/pipeline/resolve-mappings.d.ts +8 -3
- package/dist/promptfoo-providers/mock-path.d.ts +12 -0
- package/dist/promptfoo-providers/mock-path.js +15 -0
- package/dist/report-store.d.ts +63 -1
- package/dist/report-store.js +111 -31
- package/dist/sanity/client.d.ts +58 -0
- package/dist/sanity/client.js +106 -0
- package/dist/sanity/document-renderers.d.ts +68 -0
- package/dist/sanity/document-renderers.js +221 -0
- package/dist/sanity/queries.d.ts +21 -0
- package/dist/sanity/queries.js +71 -0
- package/dist/tasks/knowledge-probe/define-type-api.task.ts +2 -6
- package/dist/tasks/knowledge-probe/groq-projections.task.ts +0 -5
- package/dist/tasks/literacy/content-lake.task.ts +4 -10
- package/dist/tasks/literacy/frameworks.task.ts +2 -8
- package/dist/tasks/literacy/functions.task.ts +1 -4
- package/dist/tasks/literacy/groq.task.ts +3 -12
- package/dist/tasks/literacy/image-handling.task.ts +1 -4
- package/dist/tasks/literacy/nextjs-live.task.ts +1 -4
- package/dist/tasks/literacy/portable-text.task.ts +2 -8
- package/dist/tasks/literacy/studio-setup.task.ts +2 -8
- package/dist/tasks/literacy/visual-editing.task.ts +2 -8
- package/package.json +8 -7
- package/tasks/knowledge-probe/define-type-api.task.ts +2 -6
- package/tasks/knowledge-probe/groq-projections.task.ts +0 -5
- package/tasks/literacy/content-lake.task.ts +4 -10
- package/tasks/literacy/frameworks.task.ts +2 -8
- package/tasks/literacy/functions.task.ts +1 -4
- package/tasks/literacy/groq.task.ts +3 -12
- package/tasks/literacy/image-handling.task.ts +1 -4
- package/tasks/literacy/nextjs-live.task.ts +1 -4
- package/tasks/literacy/portable-text.task.ts +2 -8
- package/tasks/literacy/studio-setup.task.ts +2 -8
- package/tasks/literacy/visual-editing.task.ts +2 -8
|
@@ -18,8 +18,9 @@ import { join } from "path";
|
|
|
18
18
|
import { canonicalDocRefLabel, isIdRef, isPathRef, isPerspectiveRef, isSlugRef, } from "../../_vendor/ailf-core/index.js";
|
|
19
19
|
import { fetchUrlContent, } from "../../pipeline/fetch-url-content.js";
|
|
20
20
|
import { createPerspectiveClient, createPublishedClient, getSanityClient, } from "../../sanity/client.js";
|
|
21
|
+
import { renderDocument, } from "../../sanity/document-renderers.js";
|
|
21
22
|
import { toMarkdown } from "../../sanity/portable-text.js";
|
|
22
|
-
import { ALL_ARTICLES_QUERY, ALL_FEATURE_AREAS, ARTICLE_BY_ID_QUERY, ARTICLE_BY_SLUG_QUERY, ARTICLE_BY_SLUG_WITH_PERSPECTIVE_QUERY, ARTICLE_SLUG_BY_PATH_QUERY, ARTICLE_SLUG_BY_SECTION_PATH_QUERY, ARTICLES_IN_RELEASE_QUERY, ARTICLES_METADATA_BY_SLUGS_QUERY, FEATURE_AREA_QUERIES, } from "../../sanity/queries.js";
|
|
23
|
+
import { ALL_ARTICLES_QUERY, ALL_FEATURE_AREAS, ARTICLE_BY_ID_QUERY, ARTICLE_BY_SLUG_QUERY, ARTICLE_BY_SLUG_WITH_PERSPECTIVE_QUERY, ARTICLE_SLUG_BY_PATH_QUERY, ARTICLE_SLUG_BY_SECTION_PATH_QUERY, ARTICLES_IN_RELEASE_QUERY, ARTICLES_METADATA_BY_SLUGS_QUERY, DOCS_BY_IDS_QUERY, FEATURE_AREA_QUERIES, } from "../../sanity/queries.js";
|
|
23
24
|
// ---------------------------------------------------------------------------
|
|
24
25
|
// Helpers
|
|
25
26
|
// ---------------------------------------------------------------------------
|
|
@@ -99,9 +100,11 @@ export class SanityDocFetcher {
|
|
|
99
100
|
}
|
|
100
101
|
}
|
|
101
102
|
}
|
|
102
|
-
// Resolve ID refs
|
|
103
|
-
|
|
104
|
-
|
|
103
|
+
// Resolve ID refs. Articles get added to allSlugs (legacy slug-flow takes
|
|
104
|
+
// over for fetch + render). Non-articles render directly via the renderer
|
|
105
|
+
// registry and bypass the slug-keyed content map.
|
|
106
|
+
const idResolution = await this.resolveIdRefs(idRefs, source);
|
|
107
|
+
for (const slug of idResolution.idToSlug.values()) {
|
|
105
108
|
allSlugs.add(slug);
|
|
106
109
|
}
|
|
107
110
|
// Resolve path refs → slugs
|
|
@@ -118,8 +121,14 @@ export class SanityDocFetcher {
|
|
|
118
121
|
}
|
|
119
122
|
}
|
|
120
123
|
const metadata = {};
|
|
121
|
-
// 2. Fetch document manifest (traceability metadata)
|
|
124
|
+
// 2. Fetch document manifest (traceability metadata). Articles flow
|
|
125
|
+
// through fetchManifest; non-article id-refs contribute their own
|
|
126
|
+
// entries from the resolution we already did above.
|
|
122
127
|
const manifest = await this.fetchManifest(allSlugs, source);
|
|
128
|
+
for (const extra of idResolution.extraManifest) {
|
|
129
|
+
manifest.push(extra);
|
|
130
|
+
}
|
|
131
|
+
manifest.sort((a, b) => a.slug.localeCompare(b.slug));
|
|
123
132
|
if (manifest.length > 0) {
|
|
124
133
|
metadata.manifest = manifest;
|
|
125
134
|
}
|
|
@@ -232,12 +241,22 @@ export class SanityDocFetcher {
|
|
|
232
241
|
}
|
|
233
242
|
continue;
|
|
234
243
|
}
|
|
244
|
+
// Non-article id-refs bypass the slug-keyed content map: their
|
|
245
|
+
// rendered Markdown was produced by `resolveIdRefs` and is cached
|
|
246
|
+
// by document `_id`. Article id-refs continue to go through the
|
|
247
|
+
// slug-flow below (consistent with overlay/perspective handling).
|
|
248
|
+
if (isIdRef(ref) && idResolution.contentById.has(ref.id)) {
|
|
249
|
+
const entry = idResolution.contentById.get(ref.id);
|
|
250
|
+
parts.push(entry.content);
|
|
251
|
+
slugs.push(entry.slug);
|
|
252
|
+
continue;
|
|
253
|
+
}
|
|
235
254
|
let slug;
|
|
236
255
|
if (isSlugRef(ref)) {
|
|
237
256
|
slug = ref.slug;
|
|
238
257
|
}
|
|
239
258
|
else if (isIdRef(ref)) {
|
|
240
|
-
slug = idToSlug.get(ref.id);
|
|
259
|
+
slug = idResolution.idToSlug.get(ref.id);
|
|
241
260
|
}
|
|
242
261
|
else if (isPathRef(ref)) {
|
|
243
262
|
slug = pathToSlug.get(ref.path);
|
|
@@ -289,40 +308,125 @@ export class SanityDocFetcher {
|
|
|
289
308
|
.sort((a, b) => a.slug.localeCompare(b.slug));
|
|
290
309
|
}
|
|
291
310
|
// -----------------------------------------------------------------------
|
|
292
|
-
// Private: Resolve ID refs
|
|
311
|
+
// Private: Resolve ID refs (type-agnostic)
|
|
293
312
|
// -----------------------------------------------------------------------
|
|
294
313
|
/**
|
|
295
|
-
* Batch-resolve document ID refs
|
|
314
|
+
* Batch-resolve document ID refs without filtering on `_type`.
|
|
315
|
+
*
|
|
316
|
+
* Articles are routed back into the slug-flow (so manifest, perspective
|
|
317
|
+
* diffing, and overlay handling continue to work unchanged). Non-articles
|
|
318
|
+
* are rendered eagerly via the renderer registry — `typesReference` gets
|
|
319
|
+
* a high-fidelity formatter; anything else falls through to the default
|
|
320
|
+
* walker so authors can pin marketing pages, glossary entries, etc.
|
|
321
|
+
* without an AILF code change.
|
|
296
322
|
*
|
|
297
|
-
*
|
|
298
|
-
*
|
|
299
|
-
*
|
|
323
|
+
* Three log shapes:
|
|
324
|
+
* - `info` — doc rendered with the default formatter (suggests a
|
|
325
|
+
* dedicated renderer would improve fidelity)
|
|
326
|
+
* - `warn` — registered renderer produced empty content (W0195 AC#3)
|
|
327
|
+
* - `warn` — id had no matching document (existed/wrong tenant/typo)
|
|
300
328
|
*/
|
|
301
|
-
async
|
|
302
|
-
const
|
|
329
|
+
async resolveIdRefs(idRefs, source) {
|
|
330
|
+
const idToSlug = new Map();
|
|
331
|
+
const contentById = new Map();
|
|
332
|
+
const extraManifest = [];
|
|
303
333
|
if (idRefs.length === 0)
|
|
304
|
-
return
|
|
334
|
+
return { idToSlug, contentById, extraManifest };
|
|
305
335
|
const uniqueIds = [...new Set(idRefs.map((r) => r.id))];
|
|
336
|
+
const taskByRef = new Map();
|
|
337
|
+
for (const { id, taskId } of idRefs) {
|
|
338
|
+
const existing = taskByRef.get(id) ?? [];
|
|
339
|
+
existing.push(taskId);
|
|
340
|
+
taskByRef.set(id, existing);
|
|
341
|
+
}
|
|
306
342
|
const client = source?.perspective
|
|
307
343
|
? createPerspectiveClient(source.perspective, source)
|
|
308
344
|
: getSanityClient(toSanityOverrides(source));
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
345
|
+
const docs = await client.fetch(DOCS_BY_IDS_QUERY, {
|
|
346
|
+
ids: uniqueIds,
|
|
347
|
+
});
|
|
348
|
+
const docsById = new Map(docs.map((d) => [d._id, d]));
|
|
349
|
+
let articleCount = 0;
|
|
350
|
+
let highFidelity = 0;
|
|
351
|
+
let defaultFidelity = 0;
|
|
352
|
+
for (const id of uniqueIds) {
|
|
353
|
+
const taskIds = taskByRef.get(id) ?? [];
|
|
354
|
+
const doc = docsById.get(id);
|
|
355
|
+
if (!doc) {
|
|
356
|
+
console.warn(` [warn] doc "${id}" not found (referenced by task(s): ${taskIds.join(", ")})`);
|
|
357
|
+
continue;
|
|
358
|
+
}
|
|
359
|
+
// Article id-refs flow back into the slug-keyed pipeline so manifest,
|
|
360
|
+
// perspective diff, and document overlay continue to work. The slug
|
|
361
|
+
// projection in DOCS_BY_IDS_QUERY mirrors ARTICLE_PROJECTION.
|
|
362
|
+
if (doc._type === "article") {
|
|
363
|
+
const slug = doc.slug;
|
|
364
|
+
if (typeof slug === "string" && slug.length > 0) {
|
|
365
|
+
idToSlug.set(id, slug);
|
|
366
|
+
articleCount += 1;
|
|
367
|
+
}
|
|
368
|
+
else {
|
|
369
|
+
console.warn(` [warn] article "${id}" has no slug (referenced by task(s): ${taskIds.join(", ")})`);
|
|
370
|
+
}
|
|
371
|
+
continue;
|
|
372
|
+
}
|
|
373
|
+
const rendered = await renderDocument(doc, {
|
|
374
|
+
fetchUrl: this.fetchAttachmentBody,
|
|
375
|
+
});
|
|
376
|
+
if (!rendered.content) {
|
|
377
|
+
console.warn(` [warn] doc "${id}" resolved as "${doc._type}" but produced empty context (referenced by task(s): ${taskIds.join(", ")})`);
|
|
378
|
+
continue;
|
|
379
|
+
}
|
|
380
|
+
contentById.set(id, {
|
|
381
|
+
content: rendered.content,
|
|
382
|
+
slug: rendered.slug,
|
|
383
|
+
type: doc._type,
|
|
384
|
+
});
|
|
385
|
+
const _rev = doc._rev;
|
|
386
|
+
const title = doc.title;
|
|
387
|
+
extraManifest.push({
|
|
388
|
+
_id: doc._id,
|
|
389
|
+
_rev: typeof _rev === "string" ? _rev : "",
|
|
390
|
+
slug: rendered.slug,
|
|
391
|
+
title: typeof title === "string" ? title : `(${doc._type})`,
|
|
392
|
+
});
|
|
393
|
+
if (rendered.fidelity === "default") {
|
|
394
|
+
console.log(` [info] doc "${id}" rendered with default formatter — a dedicated renderer for "${doc._type}" would likely improve grader fidelity`);
|
|
395
|
+
defaultFidelity += 1;
|
|
316
396
|
}
|
|
317
397
|
else {
|
|
318
|
-
|
|
398
|
+
highFidelity += 1;
|
|
319
399
|
}
|
|
320
400
|
}
|
|
321
|
-
if (
|
|
322
|
-
console.log(` Resolved ${
|
|
401
|
+
if (articleCount > 0) {
|
|
402
|
+
console.log(` Resolved ${articleCount} article id ref(s) to slugs`);
|
|
323
403
|
}
|
|
324
|
-
|
|
404
|
+
if (highFidelity + defaultFidelity > 0) {
|
|
405
|
+
console.log(` Resolved ${highFidelity + defaultFidelity} non-article id ref(s) (${highFidelity} high-fidelity, ${defaultFidelity} default)`);
|
|
406
|
+
}
|
|
407
|
+
return { idToSlug, contentById, extraManifest };
|
|
325
408
|
}
|
|
409
|
+
/**
|
|
410
|
+
* Fetch the raw body of a Sanity file asset URL, used by the
|
|
411
|
+
* `typesReference` renderer to inline typedoc JSON. Returns `null`
|
|
412
|
+
* on any HTTP/network failure rather than throwing — the renderer
|
|
413
|
+
* surfaces a placeholder so the rest of the context still renders.
|
|
414
|
+
*/
|
|
415
|
+
fetchAttachmentBody = async (url) => {
|
|
416
|
+
try {
|
|
417
|
+
const response = await fetch(url);
|
|
418
|
+
if (!response.ok) {
|
|
419
|
+
console.warn(` [warn] attachment fetch failed for ${url}: HTTP ${response.status}`);
|
|
420
|
+
return null;
|
|
421
|
+
}
|
|
422
|
+
return await response.text();
|
|
423
|
+
}
|
|
424
|
+
catch (err) {
|
|
425
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
426
|
+
console.warn(` [warn] attachment fetch failed for ${url}: ${msg}`);
|
|
427
|
+
return null;
|
|
428
|
+
}
|
|
429
|
+
};
|
|
326
430
|
// -----------------------------------------------------------------------
|
|
327
431
|
// Private: Resolve path refs to slugs
|
|
328
432
|
// -----------------------------------------------------------------------
|
|
@@ -16,9 +16,66 @@
|
|
|
16
16
|
* @see docs/decisions/D0038-content-lake-authorable-task-modes.md
|
|
17
17
|
*/
|
|
18
18
|
import type { SanityClient } from "@sanity/client";
|
|
19
|
-
import type { FilterOptions, GeneralizedTaskDefinition, TaskSource } from "../../_vendor/ailf-core/index.d.ts";
|
|
19
|
+
import type { ContentLakeAuthorableTask, FilterOptions, GeneralizedTaskDefinition, TaskSource, TaskStatus } from "../../_vendor/ailf-core/index.d.ts";
|
|
20
|
+
/** Shape of a single ailf.task document as returned by the GROQ query. */
|
|
21
|
+
export interface ContentLakeTask {
|
|
22
|
+
areaId?: string;
|
|
23
|
+
assertions?: ContentLakeAssertion[];
|
|
24
|
+
baseline?: {
|
|
25
|
+
enabled?: boolean;
|
|
26
|
+
rubric?: string;
|
|
27
|
+
};
|
|
28
|
+
contextDocs?: ContentLakeCanonicalDoc[];
|
|
29
|
+
description?: string;
|
|
30
|
+
docCoverage?: boolean;
|
|
31
|
+
promptText?: string;
|
|
32
|
+
rawAssert?: {
|
|
33
|
+
threshold?: number;
|
|
34
|
+
type?: string;
|
|
35
|
+
value?: string;
|
|
36
|
+
}[];
|
|
37
|
+
referenceSolutionTitle?: string;
|
|
38
|
+
status?: TaskStatus;
|
|
39
|
+
tags?: string[];
|
|
40
|
+
taskId?: string;
|
|
41
|
+
title?: string;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Context doc ref shape from the Content Lake.
|
|
45
|
+
* The GROQ query projects refType + all possible value fields.
|
|
46
|
+
* Only the field matching refType will have a value.
|
|
47
|
+
*/
|
|
48
|
+
interface ContentLakeCanonicalDoc {
|
|
49
|
+
docId?: string;
|
|
50
|
+
docRefId?: string;
|
|
51
|
+
path?: string;
|
|
52
|
+
perspective?: string;
|
|
53
|
+
reason?: string;
|
|
54
|
+
refType?: string;
|
|
55
|
+
sectionSlug?: string;
|
|
56
|
+
slug?: string;
|
|
57
|
+
}
|
|
58
|
+
/** Assertion shape from the Content Lake (mirrors the Studio schema). */
|
|
59
|
+
interface ContentLakeAssertion {
|
|
60
|
+
criteria?: string[];
|
|
61
|
+
template?: string;
|
|
62
|
+
threshold?: number;
|
|
63
|
+
type?: string;
|
|
64
|
+
value?: string;
|
|
65
|
+
weight?: number;
|
|
66
|
+
}
|
|
20
67
|
export declare class ContentLakeTaskSource implements TaskSource {
|
|
21
68
|
private readonly client;
|
|
22
69
|
constructor(client: SanityClient);
|
|
23
70
|
loadTasks(filter?: FilterOptions): Promise<GeneralizedTaskDefinition[]>;
|
|
24
71
|
}
|
|
72
|
+
/**
|
|
73
|
+
* Map a Content Lake ailf.task document to a `ContentLakeAuthorableTask`.
|
|
74
|
+
*
|
|
75
|
+
* Returns null if the document is missing required fields (taskId,
|
|
76
|
+
* title, areaId, promptText). These are required by the
|
|
77
|
+
* Studio schema, but defensive coding handles edge cases (drafts,
|
|
78
|
+
* partially-created documents, etc.).
|
|
79
|
+
*/
|
|
80
|
+
export declare function mapToAuthorableTask(raw: ContentLakeTask): ContentLakeAuthorableTask | null;
|
|
81
|
+
export {};
|
|
@@ -148,7 +148,7 @@ function buildGroqParams(filter) {
|
|
|
148
148
|
* Studio schema, but defensive coding handles edge cases (drafts,
|
|
149
149
|
* partially-created documents, etc.).
|
|
150
150
|
*/
|
|
151
|
-
function mapToAuthorableTask(raw) {
|
|
151
|
+
export function mapToAuthorableTask(raw) {
|
|
152
152
|
// Required fields — skip malformed documents
|
|
153
153
|
if (!raw.taskId || !raw.title || !raw.areaId || !raw.promptText) {
|
|
154
154
|
return null;
|
|
@@ -3,5 +3,5 @@ export { ContentLakeTaskSource } from "./content-lake-task-source.js";
|
|
|
3
3
|
export { AilfEvalWorkflowSchema, CanonicalTaskFileSchema, CanonicalTaskSchema, ContentLakeAuthorableTaskSchema, CURATED_ASSERTION_TYPES, detectLegacyFieldNames, parseAilfEvalWorkflow, parseCanonicalTaskFile, parseRepoConfig, RepoConfigSchema, RUBRIC_TEMPLATE_NAMES, type AilfEvalWorkflow, type CanonicalTask, type ContentLakeAuthorableTaskParsed, type CuratedAssertionType, type RepoConfig, type RubricTemplateName, } from "./repo-schemas.js";
|
|
4
4
|
export { RepoTaskSource } from "./repo-task-source.js";
|
|
5
5
|
export { detectTriggerContext, resolveTrigger, type ResolvedTrigger, type TriggerContext, } from "./repo-trigger.js";
|
|
6
|
-
export {
|
|
6
|
+
export { formatRepoValidationResult, validateCanonicalTasks, type RepoValidationMessage, type RepoValidationResult, } from "./repo-validation.js";
|
|
7
7
|
export { discoverTsTaskFiles, loadAllTsTaskFiles, loadTsTaskFile, loadTsTaskFileSync, } from "./task-file-loader.js";
|
|
@@ -3,5 +3,5 @@ export { ContentLakeTaskSource } from "./content-lake-task-source.js";
|
|
|
3
3
|
export { AilfEvalWorkflowSchema, CanonicalTaskFileSchema, CanonicalTaskSchema, ContentLakeAuthorableTaskSchema, CURATED_ASSERTION_TYPES, detectLegacyFieldNames, parseAilfEvalWorkflow, parseCanonicalTaskFile, parseRepoConfig, RepoConfigSchema, RUBRIC_TEMPLATE_NAMES, } from "./repo-schemas.js";
|
|
4
4
|
export { RepoTaskSource } from "./repo-task-source.js";
|
|
5
5
|
export { detectTriggerContext, resolveTrigger, } from "./repo-trigger.js";
|
|
6
|
-
export {
|
|
6
|
+
export { formatRepoValidationResult, validateCanonicalTasks, } from "./repo-validation.js";
|
|
7
7
|
export { discoverTsTaskFiles, loadAllTsTaskFiles, loadTsTaskFile, loadTsTaskFileSync, } from "./task-file-loader.js";
|
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
* @see docs/archive/exec-plans/tasks-as-content/phase-4-repo-based-tasks.md
|
|
17
17
|
*/
|
|
18
18
|
import { z } from "zod";
|
|
19
|
+
import type { AilfEvalWorkflow, RepoConfig } from "../../_vendor/ailf-core/index.d.ts";
|
|
19
20
|
/**
|
|
20
21
|
* The set of assertion types allowed in task files.
|
|
21
22
|
*
|
|
@@ -1424,6 +1425,22 @@ export declare function parseCanonicalTaskFile(raw: unknown, filename: string):
|
|
|
1424
1425
|
* GeneralizedTaskDefinition shape.
|
|
1425
1426
|
*/
|
|
1426
1427
|
export declare function detectLegacyFieldNames(raw: unknown, filename: string): string[];
|
|
1428
|
+
interface MigrationResult {
|
|
1429
|
+
migrated: unknown;
|
|
1430
|
+
warnings: string[];
|
|
1431
|
+
}
|
|
1432
|
+
/**
|
|
1433
|
+
* Pre-process legacy `prompt.vars.{task,docs,__featureArea}` into the
|
|
1434
|
+
* canonical shape. Backwards-compatible: legacy-shape tasks continue to
|
|
1435
|
+
* load, but a deprecation warning is emitted per affected task.
|
|
1436
|
+
*
|
|
1437
|
+
* Legacy: prompt: { vars: { task: "...", docs: "file://..." } }
|
|
1438
|
+
* Canonical: prompt: { text: "..." }
|
|
1439
|
+
*
|
|
1440
|
+
* Applies to every task regardless of mode. Per-task dedup: at most one
|
|
1441
|
+
* warning per task per call, listing every reserved key that was present.
|
|
1442
|
+
*/
|
|
1443
|
+
export declare function migratePromptShape(raw: unknown, filename: string): MigrationResult;
|
|
1427
1444
|
/**
|
|
1428
1445
|
* Zod schema for .ailf/config.yaml — controls documentation source,
|
|
1429
1446
|
* report destination, and trigger behavior for evaluations from an
|
|
@@ -1521,7 +1538,7 @@ export declare const RepoConfigSchema: z.ZodObject<{
|
|
|
1521
1538
|
}, z.core.$strip>>;
|
|
1522
1539
|
}, z.core.$strip>>;
|
|
1523
1540
|
}, z.core.$strip>;
|
|
1524
|
-
export type RepoConfig
|
|
1541
|
+
export type { RepoConfig } from "../../_vendor/ailf-core/index.d.ts";
|
|
1525
1542
|
/**
|
|
1526
1543
|
* Parse and validate .ailf/config.yaml content. Returns typed config or throws.
|
|
1527
1544
|
*/
|
|
@@ -1551,7 +1568,7 @@ export declare const AilfEvalWorkflowSchema: z.ZodObject<{
|
|
|
1551
1568
|
}, z.core.$loose>>;
|
|
1552
1569
|
}, z.core.$loose>>;
|
|
1553
1570
|
}, z.core.$loose>;
|
|
1554
|
-
export type AilfEvalWorkflow
|
|
1571
|
+
export type { AilfEvalWorkflow } from "../../_vendor/ailf-core/index.d.ts";
|
|
1555
1572
|
/**
|
|
1556
1573
|
* Parse and validate a `.github/workflows/ailf-eval.yml` payload (already
|
|
1557
1574
|
* loaded from YAML). Throws with a Zod-formatted message on failure.
|
|
@@ -141,11 +141,30 @@ const AssertionSchema = z.union([
|
|
|
141
141
|
// ---------------------------------------------------------------------------
|
|
142
142
|
// Shared field schemas — building blocks reused across mode variants
|
|
143
143
|
// ---------------------------------------------------------------------------
|
|
144
|
+
/**
|
|
145
|
+
* Variable keys reserved by the AILF compilers — populated automatically
|
|
146
|
+
* from canonical task fields (`prompt.text`, `context.docs`, `area`).
|
|
147
|
+
* Mirrors `ReservedPromptVarKey` in `@sanity/ailf-core`; the `satisfies`
|
|
148
|
+
* clause makes drift a build error.
|
|
149
|
+
*/
|
|
150
|
+
const RESERVED_PROMPT_VAR_KEYS = [
|
|
151
|
+
"task",
|
|
152
|
+
"docs",
|
|
153
|
+
"__featureArea",
|
|
154
|
+
];
|
|
144
155
|
const TaskPromptSchema = z.object({
|
|
145
156
|
template: z.string().optional(),
|
|
146
157
|
text: z.string().optional(),
|
|
147
158
|
systemMessage: z.string().optional(),
|
|
148
|
-
vars: z
|
|
159
|
+
vars: z
|
|
160
|
+
.record(z.string(), z.unknown())
|
|
161
|
+
.refine((vars) => !RESERVED_PROMPT_VAR_KEYS.some((key) => key in vars), {
|
|
162
|
+
message: `prompt.vars contains a reserved key. Reserved keys: ` +
|
|
163
|
+
RESERVED_PROMPT_VAR_KEYS.join(", ") +
|
|
164
|
+
`. Use prompt.text for the prompt body and context.docs for ` +
|
|
165
|
+
`documentation references.`,
|
|
166
|
+
})
|
|
167
|
+
.optional(),
|
|
149
168
|
});
|
|
150
169
|
const RubricRefSchema = z.union([
|
|
151
170
|
z.object({ ref: z.string().min(1) }),
|
|
@@ -334,7 +353,9 @@ export const ContentLakeAuthorableTaskSchema = LiteracyTaskSchema;
|
|
|
334
353
|
* Schema for an array of canonical tasks — what a single .ailf/tasks/*.yaml
|
|
335
354
|
* file contains. Each file must define at least one task.
|
|
336
355
|
*/
|
|
337
|
-
export const CanonicalTaskFileSchema = z
|
|
356
|
+
export const CanonicalTaskFileSchema = z
|
|
357
|
+
.array(CanonicalTaskSchema)
|
|
358
|
+
.min(1);
|
|
338
359
|
/**
|
|
339
360
|
* Pre-process raw task entries before discriminated-union parsing: when
|
|
340
361
|
* `mode` is missing, default it to `"literacy"`. Zod cannot default a
|
|
@@ -414,6 +435,64 @@ export function detectLegacyFieldNames(raw, filename) {
|
|
|
414
435
|
}
|
|
415
436
|
return warnings;
|
|
416
437
|
}
|
|
438
|
+
/**
|
|
439
|
+
* Pre-process legacy `prompt.vars.{task,docs,__featureArea}` into the
|
|
440
|
+
* canonical shape. Backwards-compatible: legacy-shape tasks continue to
|
|
441
|
+
* load, but a deprecation warning is emitted per affected task.
|
|
442
|
+
*
|
|
443
|
+
* Legacy: prompt: { vars: { task: "...", docs: "file://..." } }
|
|
444
|
+
* Canonical: prompt: { text: "..." }
|
|
445
|
+
*
|
|
446
|
+
* Applies to every task regardless of mode. Per-task dedup: at most one
|
|
447
|
+
* warning per task per call, listing every reserved key that was present.
|
|
448
|
+
*/
|
|
449
|
+
export function migratePromptShape(raw, filename) {
|
|
450
|
+
if (!Array.isArray(raw))
|
|
451
|
+
return { migrated: raw, warnings: [] };
|
|
452
|
+
const warnings = [];
|
|
453
|
+
const migrated = raw.map((entry, i) => {
|
|
454
|
+
if (typeof entry !== "object" || entry === null)
|
|
455
|
+
return entry;
|
|
456
|
+
const obj = entry;
|
|
457
|
+
const prompt = obj.prompt;
|
|
458
|
+
if (typeof prompt !== "object" || prompt === null)
|
|
459
|
+
return entry;
|
|
460
|
+
const promptObj = prompt;
|
|
461
|
+
const vars = promptObj.vars;
|
|
462
|
+
if (typeof vars !== "object" || vars === null)
|
|
463
|
+
return entry;
|
|
464
|
+
const varsObj = vars;
|
|
465
|
+
// Detect which reserved keys are present
|
|
466
|
+
const presentReserved = RESERVED_PROMPT_VAR_KEYS.filter((key) => key in varsObj);
|
|
467
|
+
if (presentReserved.length === 0)
|
|
468
|
+
return entry;
|
|
469
|
+
const taskId = typeof obj.id === "string" ? obj.id : `task[${i}]`;
|
|
470
|
+
// Build migrated prompt + vars
|
|
471
|
+
const newPrompt = { ...promptObj };
|
|
472
|
+
const newVars = { ...varsObj };
|
|
473
|
+
for (const key of presentReserved) {
|
|
474
|
+
if (key === "task" && newPrompt.text === undefined) {
|
|
475
|
+
// Move the prompt body to prompt.text only if the canonical slot
|
|
476
|
+
// is unset; an explicit prompt.text always wins.
|
|
477
|
+
newPrompt.text = newVars.task;
|
|
478
|
+
}
|
|
479
|
+
delete newVars[key];
|
|
480
|
+
}
|
|
481
|
+
// Drop empty vars to keep the migrated shape minimal
|
|
482
|
+
if (Object.keys(newVars).length === 0) {
|
|
483
|
+
delete newPrompt.vars;
|
|
484
|
+
}
|
|
485
|
+
else {
|
|
486
|
+
newPrompt.vars = newVars;
|
|
487
|
+
}
|
|
488
|
+
warnings.push(`[${filename}] ${taskId}: deprecated prompt.vars keys ` +
|
|
489
|
+
`(${presentReserved.join(", ")}) — migrated to canonical shape ` +
|
|
490
|
+
`(prompt.text + context.docs). Update the task source to silence ` +
|
|
491
|
+
`this warning.`);
|
|
492
|
+
return { ...obj, prompt: newPrompt };
|
|
493
|
+
});
|
|
494
|
+
return { migrated, warnings };
|
|
495
|
+
}
|
|
417
496
|
// ---------------------------------------------------------------------------
|
|
418
497
|
// Config schemas — specific to the eval pipeline
|
|
419
498
|
// ---------------------------------------------------------------------------
|
|
@@ -22,7 +22,7 @@ import { existsSync, readdirSync, readFileSync } from "fs";
|
|
|
22
22
|
import { resolve } from "path";
|
|
23
23
|
import { load } from "js-yaml";
|
|
24
24
|
import { CANONICAL_EVAL_MODES } from "../../_vendor/ailf-shared/index.js";
|
|
25
|
-
import { detectLegacyFieldNames, parseCanonicalTaskFile, } from "./repo-schemas.js";
|
|
25
|
+
import { detectLegacyFieldNames, migratePromptShape, parseCanonicalTaskFile, } from "./repo-schemas.js";
|
|
26
26
|
import { discoverTsTaskFiles, loadTsTaskFile } from "./task-file-loader.js";
|
|
27
27
|
/** Set of canonical mode names for O(1) lookup */
|
|
28
28
|
const KNOWN_MODES = new Set(CANONICAL_EVAL_MODES);
|
|
@@ -69,10 +69,19 @@ export class RepoTaskSource {
|
|
|
69
69
|
legacyWarnings.join("\n") +
|
|
70
70
|
"\n\nSee contributing-tasks.md for the canonical task format.");
|
|
71
71
|
}
|
|
72
|
+
// W0193: pre-migrate legacy prompt.vars.{task,docs,__featureArea}
|
|
73
|
+
// to the canonical prompt.text + context.docs shape. Mode-agnostic —
|
|
74
|
+
// every mode's TaskPromptSchema rejects reserved keys, so the shim
|
|
75
|
+
// unblocks legacy tasks regardless of mode. Per-task deprecation
|
|
76
|
+
// warning fires on stderr.
|
|
77
|
+
const { migrated, warnings: deprecationWarnings } = migratePromptShape(parsed, file);
|
|
78
|
+
for (const warning of deprecationWarnings) {
|
|
79
|
+
console.warn(warning);
|
|
80
|
+
}
|
|
72
81
|
// Validate through canonical Zod schema
|
|
73
82
|
let validated;
|
|
74
83
|
try {
|
|
75
|
-
validated = parseCanonicalTaskFile(
|
|
84
|
+
validated = parseCanonicalTaskFile(migrated, file);
|
|
76
85
|
}
|
|
77
86
|
catch (err) {
|
|
78
87
|
const msg = err instanceof Error ? err.message : String(err);
|
|
@@ -15,12 +15,12 @@
|
|
|
15
15
|
* has been eliminated — all validation logic now lives here.
|
|
16
16
|
*/
|
|
17
17
|
import { type CanonicalTask } from "./repo-schemas.js";
|
|
18
|
-
export interface
|
|
18
|
+
export interface RepoValidationResult {
|
|
19
19
|
valid: boolean;
|
|
20
|
-
errors:
|
|
21
|
-
warnings:
|
|
20
|
+
errors: RepoValidationMessage[];
|
|
21
|
+
warnings: RepoValidationMessage[];
|
|
22
22
|
}
|
|
23
|
-
export interface
|
|
23
|
+
export interface RepoValidationMessage {
|
|
24
24
|
taskId: string;
|
|
25
25
|
field: string;
|
|
26
26
|
message: string;
|
|
@@ -32,8 +32,8 @@ export interface ValidationMessage {
|
|
|
32
32
|
* areas, unresolved slugs) and errors for issues that would cause pipeline
|
|
33
33
|
* failures (completely missing required fields — though Zod catches most).
|
|
34
34
|
*/
|
|
35
|
-
export declare function validateCanonicalTasks(tasks: CanonicalTask[]):
|
|
35
|
+
export declare function validateCanonicalTasks(tasks: CanonicalTask[]): RepoValidationResult;
|
|
36
36
|
/**
|
|
37
37
|
* Format validation results for console output.
|
|
38
38
|
*/
|
|
39
|
-
export declare function
|
|
39
|
+
export declare function formatRepoValidationResult(result: RepoValidationResult): string;
|
|
@@ -110,7 +110,7 @@ export function validateCanonicalTasks(tasks) {
|
|
|
110
110
|
/**
|
|
111
111
|
* Format validation results for console output.
|
|
112
112
|
*/
|
|
113
|
-
export function
|
|
113
|
+
export function formatRepoValidationResult(result) {
|
|
114
114
|
const lines = [];
|
|
115
115
|
if (result.errors.length > 0) {
|
|
116
116
|
lines.push("Errors:");
|