@sanity/ailf 4.0.7 → 4.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/bin/ailf.js +6 -1
  2. package/dist/_vendor/ailf-core/schemas/external-providers.d.ts +136 -0
  3. package/dist/_vendor/ailf-core/schemas/external-providers.js +136 -0
  4. package/dist/_vendor/ailf-core/schemas/index.d.ts +2 -0
  5. package/dist/_vendor/ailf-core/schemas/index.js +2 -0
  6. package/dist/_vendor/ailf-core/schemas/pipeline-request.d.ts +2 -3
  7. package/dist/_vendor/ailf-core/schemas/report.d.ts +251 -0
  8. package/dist/_vendor/ailf-core/schemas/report.js +235 -0
  9. package/dist/_vendor/ailf-core/services/index.d.ts +1 -0
  10. package/dist/_vendor/ailf-core/services/index.js +1 -0
  11. package/dist/_vendor/ailf-core/services/report-to-markdown.d.ts +38 -0
  12. package/dist/_vendor/ailf-core/services/report-to-markdown.js +696 -0
  13. package/dist/_vendor/ailf-core/types/api-requests.d.ts +159 -0
  14. package/dist/_vendor/ailf-core/types/api-requests.js +27 -0
  15. package/dist/_vendor/ailf-core/types/generalized-task.d.ts +20 -3
  16. package/dist/_vendor/ailf-core/types/index.d.ts +4 -1
  17. package/dist/_vendor/ailf-core/types/pipeline-request.d.ts +112 -0
  18. package/dist/_vendor/ailf-core/types/pipeline-request.js +18 -0
  19. package/dist/_vendor/ailf-core/types/repo-config.d.ts +146 -0
  20. package/dist/_vendor/ailf-core/types/repo-config.js +18 -0
  21. package/dist/_vendor/ailf-shared/index.d.ts +7 -5
  22. package/dist/_vendor/ailf-shared/index.js +7 -5
  23. package/dist/adapters/api-client/types.d.ts +2 -5
  24. package/dist/adapters/doc-fetchers/sanity-doc-fetcher.d.ts +21 -5
  25. package/dist/adapters/doc-fetchers/sanity-doc-fetcher.js +129 -25
  26. package/dist/adapters/task-sources/content-lake-task-source.d.ts +58 -1
  27. package/dist/adapters/task-sources/content-lake-task-source.js +1 -1
  28. package/dist/adapters/task-sources/index.d.ts +1 -1
  29. package/dist/adapters/task-sources/index.js +1 -1
  30. package/dist/adapters/task-sources/repo-schemas.d.ts +19 -2
  31. package/dist/adapters/task-sources/repo-schemas.js +81 -2
  32. package/dist/adapters/task-sources/repo-task-source.js +11 -2
  33. package/dist/adapters/task-sources/repo-validation.d.ts +6 -6
  34. package/dist/adapters/task-sources/repo-validation.js +1 -1
  35. package/dist/agent-observer/agentic-provider.d.ts +1 -0
  36. package/dist/agent-observer/agentic-provider.js +43 -36
  37. package/dist/agent-observer/config-schemas.d.ts +61 -0
  38. package/dist/agent-observer/config-schemas.js +65 -0
  39. package/dist/agent-observer/provider.d.ts +1 -0
  40. package/dist/agent-observer/provider.js +19 -17
  41. package/dist/cli.js +4 -4
  42. package/dist/commands/validate-tasks.js +10 -4
  43. package/dist/composition-root.js +4 -2
  44. package/dist/index.d.ts +1 -1
  45. package/dist/index.js +1 -1
  46. package/dist/job-store.js +2 -2
  47. package/dist/lib/dotenv-resolution.d.ts +21 -0
  48. package/dist/lib/dotenv-resolution.js +30 -0
  49. package/dist/orchestration/steps/mirror-repo-tasks-step.js +14 -3
  50. package/dist/orchestration/steps/run-eval-step.js +21 -3
  51. package/dist/pipeline/agent-behavior-report.d.ts +2 -8
  52. package/dist/pipeline/cache.d.ts +2 -2
  53. package/dist/pipeline/checks.d.ts +10 -2
  54. package/dist/pipeline/checks.js +14 -4
  55. package/dist/pipeline/compiler/literacy-bridge.js +2 -2
  56. package/dist/pipeline/compiler/mode-handlers/__fixtures__/agent-harness-example-tasks.js +0 -12
  57. package/dist/pipeline/compiler/mode-handlers/__fixtures__/knowledge-probe-example-tasks.js +0 -12
  58. package/dist/pipeline/compiler/mode-handlers/agent-harness/types.d.ts +2 -2
  59. package/dist/pipeline/compiler/mode-handlers/index.d.ts +1 -1
  60. package/dist/pipeline/compiler/mode-handlers/knowledge-probe/types.d.ts +2 -2
  61. package/dist/pipeline/compiler/mode-handlers/literacy/compiler.js +44 -5
  62. package/dist/pipeline/compiler/mode-handlers/literacy/index.d.ts +1 -1
  63. package/dist/pipeline/compiler/mode-handlers/literacy/types.d.ts +3 -3
  64. package/dist/pipeline/compiler/promptfoo-compiler.js +7 -11
  65. package/dist/pipeline/compiler/provider-assembler.js +33 -3
  66. package/dist/pipeline/compiler/rubric-resolution.d.ts +2 -2
  67. package/dist/pipeline/mirror-repo-tasks.d.ts +13 -5
  68. package/dist/pipeline/mirror-repo-tasks.js +16 -8
  69. package/dist/pipeline/pr-comment.d.ts +22 -9
  70. package/dist/pipeline/pr-comment.js +52 -472
  71. package/dist/pipeline/resolve-mappings.d.ts +8 -3
  72. package/dist/promptfoo-providers/mock-path.d.ts +12 -0
  73. package/dist/promptfoo-providers/mock-path.js +15 -0
  74. package/dist/report-store.d.ts +63 -1
  75. package/dist/report-store.js +111 -31
  76. package/dist/sanity/client.d.ts +58 -0
  77. package/dist/sanity/client.js +106 -0
  78. package/dist/sanity/document-renderers.d.ts +68 -0
  79. package/dist/sanity/document-renderers.js +221 -0
  80. package/dist/sanity/queries.d.ts +21 -0
  81. package/dist/sanity/queries.js +71 -0
  82. package/dist/tasks/knowledge-probe/define-type-api.task.ts +2 -6
  83. package/dist/tasks/knowledge-probe/groq-projections.task.ts +0 -5
  84. package/dist/tasks/literacy/content-lake.task.ts +4 -10
  85. package/dist/tasks/literacy/frameworks.task.ts +2 -8
  86. package/dist/tasks/literacy/functions.task.ts +1 -4
  87. package/dist/tasks/literacy/groq.task.ts +3 -12
  88. package/dist/tasks/literacy/image-handling.task.ts +1 -4
  89. package/dist/tasks/literacy/nextjs-live.task.ts +1 -4
  90. package/dist/tasks/literacy/portable-text.task.ts +2 -8
  91. package/dist/tasks/literacy/studio-setup.task.ts +2 -8
  92. package/dist/tasks/literacy/visual-editing.task.ts +2 -8
  93. package/package.json +8 -7
  94. package/tasks/knowledge-probe/define-type-api.task.ts +2 -6
  95. package/tasks/knowledge-probe/groq-projections.task.ts +0 -5
  96. package/tasks/literacy/content-lake.task.ts +4 -10
  97. package/tasks/literacy/frameworks.task.ts +2 -8
  98. package/tasks/literacy/functions.task.ts +1 -4
  99. package/tasks/literacy/groq.task.ts +3 -12
  100. package/tasks/literacy/image-handling.task.ts +1 -4
  101. package/tasks/literacy/nextjs-live.task.ts +1 -4
  102. package/tasks/literacy/portable-text.task.ts +2 -8
  103. package/tasks/literacy/studio-setup.task.ts +2 -8
  104. package/tasks/literacy/visual-editing.task.ts +2 -8
@@ -18,8 +18,9 @@ import { join } from "path";
18
18
  import { canonicalDocRefLabel, isIdRef, isPathRef, isPerspectiveRef, isSlugRef, } from "../../_vendor/ailf-core/index.js";
19
19
  import { fetchUrlContent, } from "../../pipeline/fetch-url-content.js";
20
20
  import { createPerspectiveClient, createPublishedClient, getSanityClient, } from "../../sanity/client.js";
21
+ import { renderDocument, } from "../../sanity/document-renderers.js";
21
22
  import { toMarkdown } from "../../sanity/portable-text.js";
22
- import { ALL_ARTICLES_QUERY, ALL_FEATURE_AREAS, ARTICLE_BY_ID_QUERY, ARTICLE_BY_SLUG_QUERY, ARTICLE_BY_SLUG_WITH_PERSPECTIVE_QUERY, ARTICLE_SLUG_BY_PATH_QUERY, ARTICLE_SLUG_BY_SECTION_PATH_QUERY, ARTICLES_IN_RELEASE_QUERY, ARTICLES_METADATA_BY_SLUGS_QUERY, FEATURE_AREA_QUERIES, } from "../../sanity/queries.js";
23
+ import { ALL_ARTICLES_QUERY, ALL_FEATURE_AREAS, ARTICLE_BY_ID_QUERY, ARTICLE_BY_SLUG_QUERY, ARTICLE_BY_SLUG_WITH_PERSPECTIVE_QUERY, ARTICLE_SLUG_BY_PATH_QUERY, ARTICLE_SLUG_BY_SECTION_PATH_QUERY, ARTICLES_IN_RELEASE_QUERY, ARTICLES_METADATA_BY_SLUGS_QUERY, DOCS_BY_IDS_QUERY, FEATURE_AREA_QUERIES, } from "../../sanity/queries.js";
23
24
  // ---------------------------------------------------------------------------
24
25
  // Helpers
25
26
  // ---------------------------------------------------------------------------
@@ -99,9 +100,11 @@ export class SanityDocFetcher {
99
100
  }
100
101
  }
101
102
  }
102
- // Resolve ID refs slugs via batch query
103
- const idToSlug = await this.resolveIdRefsToSlugs(idRefs, source);
104
- for (const slug of idToSlug.values()) {
103
+ // Resolve ID refs. Articles get added to allSlugs (legacy slug-flow takes
104
+ // over for fetch + render). Non-articles render directly via the renderer
105
+ // registry and bypass the slug-keyed content map.
106
+ const idResolution = await this.resolveIdRefs(idRefs, source);
107
+ for (const slug of idResolution.idToSlug.values()) {
105
108
  allSlugs.add(slug);
106
109
  }
107
110
  // Resolve path refs → slugs
@@ -118,8 +121,14 @@ export class SanityDocFetcher {
118
121
  }
119
122
  }
120
123
  const metadata = {};
121
- // 2. Fetch document manifest (traceability metadata)
124
+ // 2. Fetch document manifest (traceability metadata). Articles flow
125
+ // through fetchManifest; non-article id-refs contribute their own
126
+ // entries from the resolution we already did above.
122
127
  const manifest = await this.fetchManifest(allSlugs, source);
128
+ for (const extra of idResolution.extraManifest) {
129
+ manifest.push(extra);
130
+ }
131
+ manifest.sort((a, b) => a.slug.localeCompare(b.slug));
123
132
  if (manifest.length > 0) {
124
133
  metadata.manifest = manifest;
125
134
  }
@@ -232,12 +241,22 @@ export class SanityDocFetcher {
232
241
  }
233
242
  continue;
234
243
  }
244
+ // Non-article id-refs bypass the slug-keyed content map: their
245
+ // rendered Markdown was produced by `resolveIdRefs` and is cached
246
+ // by document `_id`. Article id-refs continue to go through the
247
+ // slug-flow below (consistent with overlay/perspective handling).
248
+ if (isIdRef(ref) && idResolution.contentById.has(ref.id)) {
249
+ const entry = idResolution.contentById.get(ref.id);
250
+ parts.push(entry.content);
251
+ slugs.push(entry.slug);
252
+ continue;
253
+ }
235
254
  let slug;
236
255
  if (isSlugRef(ref)) {
237
256
  slug = ref.slug;
238
257
  }
239
258
  else if (isIdRef(ref)) {
240
- slug = idToSlug.get(ref.id);
259
+ slug = idResolution.idToSlug.get(ref.id);
241
260
  }
242
261
  else if (isPathRef(ref)) {
243
262
  slug = pathToSlug.get(ref.path);
@@ -289,40 +308,125 @@ export class SanityDocFetcher {
289
308
  .sort((a, b) => a.slug.localeCompare(b.slug));
290
309
  }
291
310
  // -----------------------------------------------------------------------
292
- // Private: Resolve ID refs to slugs
311
+ // Private: Resolve ID refs (type-agnostic)
293
312
  // -----------------------------------------------------------------------
294
313
  /**
295
- * Batch-resolve document ID refs to their article slugs.
314
+ * Batch-resolve document ID refs without filtering on `_type`.
315
+ *
316
+ * Articles are routed back into the slug-flow (so manifest, perspective
317
+ * diffing, and overlay handling continue to work unchanged). Non-articles
318
+ * are rendered eagerly via the renderer registry — `typesReference` gets
319
+ * a high-fidelity formatter; anything else falls through to the default
320
+ * walker so authors can pin marketing pages, glossary entries, etc.
321
+ * without an AILF code change.
296
322
  *
297
- * This bridges IdDocRef entries into the slug-based fetch pipeline.
298
- * Articles are queried by _id and their slugs are returned for use
299
- * in the existing slug-based content map.
323
+ * Three log shapes:
324
+ * - `info` doc rendered with the default formatter (suggests a
325
+ * dedicated renderer would improve fidelity)
326
+ * - `warn` — registered renderer produced empty content (W0195 AC#3)
327
+ * - `warn` — id had no matching document (existed/wrong tenant/typo)
300
328
  */
301
- async resolveIdRefsToSlugs(idRefs, source) {
302
- const result = new Map();
329
+ async resolveIdRefs(idRefs, source) {
330
+ const idToSlug = new Map();
331
+ const contentById = new Map();
332
+ const extraManifest = [];
303
333
  if (idRefs.length === 0)
304
- return result;
334
+ return { idToSlug, contentById, extraManifest };
305
335
  const uniqueIds = [...new Set(idRefs.map((r) => r.id))];
336
+ const taskByRef = new Map();
337
+ for (const { id, taskId } of idRefs) {
338
+ const existing = taskByRef.get(id) ?? [];
339
+ existing.push(taskId);
340
+ taskByRef.set(id, existing);
341
+ }
306
342
  const client = source?.perspective
307
343
  ? createPerspectiveClient(source.perspective, source)
308
344
  : getSanityClient(toSanityOverrides(source));
309
- // Batch query: fetch slug for each document ID
310
- const articles = await client.fetch(`*[_type == "article" && _id in $ids] { _id, "slug": slug.current }`, { ids: uniqueIds });
311
- const idToSlugMap = new Map(articles.map((a) => [a._id, a.slug]));
312
- for (const { id, taskId } of idRefs) {
313
- const slug = idToSlugMap.get(id);
314
- if (slug) {
315
- result.set(id, slug);
345
+ const docs = await client.fetch(DOCS_BY_IDS_QUERY, {
346
+ ids: uniqueIds,
347
+ });
348
+ const docsById = new Map(docs.map((d) => [d._id, d]));
349
+ let articleCount = 0;
350
+ let highFidelity = 0;
351
+ let defaultFidelity = 0;
352
+ for (const id of uniqueIds) {
353
+ const taskIds = taskByRef.get(id) ?? [];
354
+ const doc = docsById.get(id);
355
+ if (!doc) {
356
+ console.warn(` [warn] doc "${id}" not found (referenced by task(s): ${taskIds.join(", ")})`);
357
+ continue;
358
+ }
359
+ // Article id-refs flow back into the slug-keyed pipeline so manifest,
360
+ // perspective diff, and document overlay continue to work. The slug
361
+ // projection in DOCS_BY_IDS_QUERY mirrors ARTICLE_PROJECTION.
362
+ if (doc._type === "article") {
363
+ const slug = doc.slug;
364
+ if (typeof slug === "string" && slug.length > 0) {
365
+ idToSlug.set(id, slug);
366
+ articleCount += 1;
367
+ }
368
+ else {
369
+ console.warn(` [warn] article "${id}" has no slug (referenced by task(s): ${taskIds.join(", ")})`);
370
+ }
371
+ continue;
372
+ }
373
+ const rendered = await renderDocument(doc, {
374
+ fetchUrl: this.fetchAttachmentBody,
375
+ });
376
+ if (!rendered.content) {
377
+ console.warn(` [warn] doc "${id}" resolved as "${doc._type}" but produced empty context (referenced by task(s): ${taskIds.join(", ")})`);
378
+ continue;
379
+ }
380
+ contentById.set(id, {
381
+ content: rendered.content,
382
+ slug: rendered.slug,
383
+ type: doc._type,
384
+ });
385
+ const _rev = doc._rev;
386
+ const title = doc.title;
387
+ extraManifest.push({
388
+ _id: doc._id,
389
+ _rev: typeof _rev === "string" ? _rev : "",
390
+ slug: rendered.slug,
391
+ title: typeof title === "string" ? title : `(${doc._type})`,
392
+ });
393
+ if (rendered.fidelity === "default") {
394
+ console.log(` [info] doc "${id}" rendered with default formatter — a dedicated renderer for "${doc._type}" would likely improve grader fidelity`);
395
+ defaultFidelity += 1;
316
396
  }
317
397
  else {
318
- console.warn(` [warn] No article found for document ID "${id}" (referenced by task "${taskId}")`);
398
+ highFidelity += 1;
319
399
  }
320
400
  }
321
- if (result.size > 0) {
322
- console.log(` Resolved ${result.size} document ID ref(s) to slugs`);
401
+ if (articleCount > 0) {
402
+ console.log(` Resolved ${articleCount} article id ref(s) to slugs`);
323
403
  }
324
- return result;
404
+ if (highFidelity + defaultFidelity > 0) {
405
+ console.log(` Resolved ${highFidelity + defaultFidelity} non-article id ref(s) (${highFidelity} high-fidelity, ${defaultFidelity} default)`);
406
+ }
407
+ return { idToSlug, contentById, extraManifest };
325
408
  }
409
+ /**
410
+ * Fetch the raw body of a Sanity file asset URL, used by the
411
+ * `typesReference` renderer to inline typedoc JSON. Returns `null`
412
+ * on any HTTP/network failure rather than throwing — the renderer
413
+ * surfaces a placeholder so the rest of the context still renders.
414
+ */
415
+ fetchAttachmentBody = async (url) => {
416
+ try {
417
+ const response = await fetch(url);
418
+ if (!response.ok) {
419
+ console.warn(` [warn] attachment fetch failed for ${url}: HTTP ${response.status}`);
420
+ return null;
421
+ }
422
+ return await response.text();
423
+ }
424
+ catch (err) {
425
+ const msg = err instanceof Error ? err.message : String(err);
426
+ console.warn(` [warn] attachment fetch failed for ${url}: ${msg}`);
427
+ return null;
428
+ }
429
+ };
326
430
  // -----------------------------------------------------------------------
327
431
  // Private: Resolve path refs to slugs
328
432
  // -----------------------------------------------------------------------
@@ -16,9 +16,66 @@
16
16
  * @see docs/decisions/D0038-content-lake-authorable-task-modes.md
17
17
  */
18
18
  import type { SanityClient } from "@sanity/client";
19
- import type { FilterOptions, GeneralizedTaskDefinition, TaskSource } from "../../_vendor/ailf-core/index.d.ts";
19
+ import type { ContentLakeAuthorableTask, FilterOptions, GeneralizedTaskDefinition, TaskSource, TaskStatus } from "../../_vendor/ailf-core/index.d.ts";
20
+ /** Shape of a single ailf.task document as returned by the GROQ query. */
21
+ export interface ContentLakeTask {
22
+ areaId?: string;
23
+ assertions?: ContentLakeAssertion[];
24
+ baseline?: {
25
+ enabled?: boolean;
26
+ rubric?: string;
27
+ };
28
+ contextDocs?: ContentLakeCanonicalDoc[];
29
+ description?: string;
30
+ docCoverage?: boolean;
31
+ promptText?: string;
32
+ rawAssert?: {
33
+ threshold?: number;
34
+ type?: string;
35
+ value?: string;
36
+ }[];
37
+ referenceSolutionTitle?: string;
38
+ status?: TaskStatus;
39
+ tags?: string[];
40
+ taskId?: string;
41
+ title?: string;
42
+ }
43
+ /**
44
+ * Context doc ref shape from the Content Lake.
45
+ * The GROQ query projects refType + all possible value fields.
46
+ * Only the field matching refType will have a value.
47
+ */
48
+ interface ContentLakeCanonicalDoc {
49
+ docId?: string;
50
+ docRefId?: string;
51
+ path?: string;
52
+ perspective?: string;
53
+ reason?: string;
54
+ refType?: string;
55
+ sectionSlug?: string;
56
+ slug?: string;
57
+ }
58
+ /** Assertion shape from the Content Lake (mirrors the Studio schema). */
59
+ interface ContentLakeAssertion {
60
+ criteria?: string[];
61
+ template?: string;
62
+ threshold?: number;
63
+ type?: string;
64
+ value?: string;
65
+ weight?: number;
66
+ }
20
67
  export declare class ContentLakeTaskSource implements TaskSource {
21
68
  private readonly client;
22
69
  constructor(client: SanityClient);
23
70
  loadTasks(filter?: FilterOptions): Promise<GeneralizedTaskDefinition[]>;
24
71
  }
72
+ /**
73
+ * Map a Content Lake ailf.task document to a `ContentLakeAuthorableTask`.
74
+ *
75
+ * Returns null if the document is missing required fields (taskId,
76
+ * title, areaId, promptText). These are required by the
77
+ * Studio schema, but defensive coding handles edge cases (drafts,
78
+ * partially-created documents, etc.).
79
+ */
80
+ export declare function mapToAuthorableTask(raw: ContentLakeTask): ContentLakeAuthorableTask | null;
81
+ export {};
@@ -148,7 +148,7 @@ function buildGroqParams(filter) {
148
148
  * Studio schema, but defensive coding handles edge cases (drafts,
149
149
  * partially-created documents, etc.).
150
150
  */
151
- function mapToAuthorableTask(raw) {
151
+ export function mapToAuthorableTask(raw) {
152
152
  // Required fields — skip malformed documents
153
153
  if (!raw.taskId || !raw.title || !raw.areaId || !raw.promptText) {
154
154
  return null;
@@ -3,5 +3,5 @@ export { ContentLakeTaskSource } from "./content-lake-task-source.js";
3
3
  export { AilfEvalWorkflowSchema, CanonicalTaskFileSchema, CanonicalTaskSchema, ContentLakeAuthorableTaskSchema, CURATED_ASSERTION_TYPES, detectLegacyFieldNames, parseAilfEvalWorkflow, parseCanonicalTaskFile, parseRepoConfig, RepoConfigSchema, RUBRIC_TEMPLATE_NAMES, type AilfEvalWorkflow, type CanonicalTask, type ContentLakeAuthorableTaskParsed, type CuratedAssertionType, type RepoConfig, type RubricTemplateName, } from "./repo-schemas.js";
4
4
  export { RepoTaskSource } from "./repo-task-source.js";
5
5
  export { detectTriggerContext, resolveTrigger, type ResolvedTrigger, type TriggerContext, } from "./repo-trigger.js";
6
- export { formatValidationResult, validateCanonicalTasks, type ValidationMessage, type ValidationResult, } from "./repo-validation.js";
6
+ export { formatRepoValidationResult, validateCanonicalTasks, type RepoValidationMessage, type RepoValidationResult, } from "./repo-validation.js";
7
7
  export { discoverTsTaskFiles, loadAllTsTaskFiles, loadTsTaskFile, loadTsTaskFileSync, } from "./task-file-loader.js";
@@ -3,5 +3,5 @@ export { ContentLakeTaskSource } from "./content-lake-task-source.js";
3
3
  export { AilfEvalWorkflowSchema, CanonicalTaskFileSchema, CanonicalTaskSchema, ContentLakeAuthorableTaskSchema, CURATED_ASSERTION_TYPES, detectLegacyFieldNames, parseAilfEvalWorkflow, parseCanonicalTaskFile, parseRepoConfig, RepoConfigSchema, RUBRIC_TEMPLATE_NAMES, } from "./repo-schemas.js";
4
4
  export { RepoTaskSource } from "./repo-task-source.js";
5
5
  export { detectTriggerContext, resolveTrigger, } from "./repo-trigger.js";
6
- export { formatValidationResult, validateCanonicalTasks, } from "./repo-validation.js";
6
+ export { formatRepoValidationResult, validateCanonicalTasks, } from "./repo-validation.js";
7
7
  export { discoverTsTaskFiles, loadAllTsTaskFiles, loadTsTaskFile, loadTsTaskFileSync, } from "./task-file-loader.js";
@@ -16,6 +16,7 @@
16
16
  * @see docs/archive/exec-plans/tasks-as-content/phase-4-repo-based-tasks.md
17
17
  */
18
18
  import { z } from "zod";
19
+ import type { AilfEvalWorkflow, RepoConfig } from "../../_vendor/ailf-core/index.d.ts";
19
20
  /**
20
21
  * The set of assertion types allowed in task files.
21
22
  *
@@ -1424,6 +1425,22 @@ export declare function parseCanonicalTaskFile(raw: unknown, filename: string):
1424
1425
  * GeneralizedTaskDefinition shape.
1425
1426
  */
1426
1427
  export declare function detectLegacyFieldNames(raw: unknown, filename: string): string[];
1428
+ interface MigrationResult {
1429
+ migrated: unknown;
1430
+ warnings: string[];
1431
+ }
1432
+ /**
1433
+ * Pre-process legacy `prompt.vars.{task,docs,__featureArea}` into the
1434
+ * canonical shape. Backwards-compatible: legacy-shape tasks continue to
1435
+ * load, but a deprecation warning is emitted per affected task.
1436
+ *
1437
+ * Legacy: prompt: { vars: { task: "...", docs: "file://..." } }
1438
+ * Canonical: prompt: { text: "..." }
1439
+ *
1440
+ * Applies to every task regardless of mode. Per-task dedup: at most one
1441
+ * warning per task per call, listing every reserved key that was present.
1442
+ */
1443
+ export declare function migratePromptShape(raw: unknown, filename: string): MigrationResult;
1427
1444
  /**
1428
1445
  * Zod schema for .ailf/config.yaml — controls documentation source,
1429
1446
  * report destination, and trigger behavior for evaluations from an
@@ -1521,7 +1538,7 @@ export declare const RepoConfigSchema: z.ZodObject<{
1521
1538
  }, z.core.$strip>>;
1522
1539
  }, z.core.$strip>>;
1523
1540
  }, z.core.$strip>;
1524
- export type RepoConfig = z.infer<typeof RepoConfigSchema>;
1541
+ export type { RepoConfig } from "../../_vendor/ailf-core/index.d.ts";
1525
1542
  /**
1526
1543
  * Parse and validate .ailf/config.yaml content. Returns typed config or throws.
1527
1544
  */
@@ -1551,7 +1568,7 @@ export declare const AilfEvalWorkflowSchema: z.ZodObject<{
1551
1568
  }, z.core.$loose>>;
1552
1569
  }, z.core.$loose>>;
1553
1570
  }, z.core.$loose>;
1554
- export type AilfEvalWorkflow = z.infer<typeof AilfEvalWorkflowSchema>;
1571
+ export type { AilfEvalWorkflow } from "../../_vendor/ailf-core/index.d.ts";
1555
1572
  /**
1556
1573
  * Parse and validate a `.github/workflows/ailf-eval.yml` payload (already
1557
1574
  * loaded from YAML). Throws with a Zod-formatted message on failure.
@@ -141,11 +141,30 @@ const AssertionSchema = z.union([
141
141
  // ---------------------------------------------------------------------------
142
142
  // Shared field schemas — building blocks reused across mode variants
143
143
  // ---------------------------------------------------------------------------
144
+ /**
145
+ * Variable keys reserved by the AILF compilers — populated automatically
146
+ * from canonical task fields (`prompt.text`, `context.docs`, `area`).
147
+ * Mirrors `ReservedPromptVarKey` in `@sanity/ailf-core`; the `satisfies`
148
+ * clause makes drift a build error.
149
+ */
150
+ const RESERVED_PROMPT_VAR_KEYS = [
151
+ "task",
152
+ "docs",
153
+ "__featureArea",
154
+ ];
144
155
  const TaskPromptSchema = z.object({
145
156
  template: z.string().optional(),
146
157
  text: z.string().optional(),
147
158
  systemMessage: z.string().optional(),
148
- vars: z.record(z.string(), z.unknown()).optional(),
159
+ vars: z
160
+ .record(z.string(), z.unknown())
161
+ .refine((vars) => !RESERVED_PROMPT_VAR_KEYS.some((key) => key in vars), {
162
+ message: `prompt.vars contains a reserved key. Reserved keys: ` +
163
+ RESERVED_PROMPT_VAR_KEYS.join(", ") +
164
+ `. Use prompt.text for the prompt body and context.docs for ` +
165
+ `documentation references.`,
166
+ })
167
+ .optional(),
149
168
  });
150
169
  const RubricRefSchema = z.union([
151
170
  z.object({ ref: z.string().min(1) }),
@@ -334,7 +353,9 @@ export const ContentLakeAuthorableTaskSchema = LiteracyTaskSchema;
334
353
  * Schema for an array of canonical tasks — what a single .ailf/tasks/*.yaml
335
354
  * file contains. Each file must define at least one task.
336
355
  */
337
- export const CanonicalTaskFileSchema = z.array(CanonicalTaskSchema).min(1);
356
+ export const CanonicalTaskFileSchema = z
357
+ .array(CanonicalTaskSchema)
358
+ .min(1);
338
359
  /**
339
360
  * Pre-process raw task entries before discriminated-union parsing: when
340
361
  * `mode` is missing, default it to `"literacy"`. Zod cannot default a
@@ -414,6 +435,64 @@ export function detectLegacyFieldNames(raw, filename) {
414
435
  }
415
436
  return warnings;
416
437
  }
438
+ /**
439
+ * Pre-process legacy `prompt.vars.{task,docs,__featureArea}` into the
440
+ * canonical shape. Backwards-compatible: legacy-shape tasks continue to
441
+ * load, but a deprecation warning is emitted per affected task.
442
+ *
443
+ * Legacy: prompt: { vars: { task: "...", docs: "file://..." } }
444
+ * Canonical: prompt: { text: "..." }
445
+ *
446
+ * Applies to every task regardless of mode. Per-task dedup: at most one
447
+ * warning per task per call, listing every reserved key that was present.
448
+ */
449
+ export function migratePromptShape(raw, filename) {
450
+ if (!Array.isArray(raw))
451
+ return { migrated: raw, warnings: [] };
452
+ const warnings = [];
453
+ const migrated = raw.map((entry, i) => {
454
+ if (typeof entry !== "object" || entry === null)
455
+ return entry;
456
+ const obj = entry;
457
+ const prompt = obj.prompt;
458
+ if (typeof prompt !== "object" || prompt === null)
459
+ return entry;
460
+ const promptObj = prompt;
461
+ const vars = promptObj.vars;
462
+ if (typeof vars !== "object" || vars === null)
463
+ return entry;
464
+ const varsObj = vars;
465
+ // Detect which reserved keys are present
466
+ const presentReserved = RESERVED_PROMPT_VAR_KEYS.filter((key) => key in varsObj);
467
+ if (presentReserved.length === 0)
468
+ return entry;
469
+ const taskId = typeof obj.id === "string" ? obj.id : `task[${i}]`;
470
+ // Build migrated prompt + vars
471
+ const newPrompt = { ...promptObj };
472
+ const newVars = { ...varsObj };
473
+ for (const key of presentReserved) {
474
+ if (key === "task" && newPrompt.text === undefined) {
475
+ // Move the prompt body to prompt.text only if the canonical slot
476
+ // is unset; an explicit prompt.text always wins.
477
+ newPrompt.text = newVars.task;
478
+ }
479
+ delete newVars[key];
480
+ }
481
+ // Drop empty vars to keep the migrated shape minimal
482
+ if (Object.keys(newVars).length === 0) {
483
+ delete newPrompt.vars;
484
+ }
485
+ else {
486
+ newPrompt.vars = newVars;
487
+ }
488
+ warnings.push(`[${filename}] ${taskId}: deprecated prompt.vars keys ` +
489
+ `(${presentReserved.join(", ")}) — migrated to canonical shape ` +
490
+ `(prompt.text + context.docs). Update the task source to silence ` +
491
+ `this warning.`);
492
+ return { ...obj, prompt: newPrompt };
493
+ });
494
+ return { migrated, warnings };
495
+ }
417
496
  // ---------------------------------------------------------------------------
418
497
  // Config schemas — specific to the eval pipeline
419
498
  // ---------------------------------------------------------------------------
@@ -22,7 +22,7 @@ import { existsSync, readdirSync, readFileSync } from "fs";
22
22
  import { resolve } from "path";
23
23
  import { load } from "js-yaml";
24
24
  import { CANONICAL_EVAL_MODES } from "../../_vendor/ailf-shared/index.js";
25
- import { detectLegacyFieldNames, parseCanonicalTaskFile, } from "./repo-schemas.js";
25
+ import { detectLegacyFieldNames, migratePromptShape, parseCanonicalTaskFile, } from "./repo-schemas.js";
26
26
  import { discoverTsTaskFiles, loadTsTaskFile } from "./task-file-loader.js";
27
27
  /** Set of canonical mode names for O(1) lookup */
28
28
  const KNOWN_MODES = new Set(CANONICAL_EVAL_MODES);
@@ -69,10 +69,19 @@ export class RepoTaskSource {
69
69
  legacyWarnings.join("\n") +
70
70
  "\n\nSee contributing-tasks.md for the canonical task format.");
71
71
  }
72
+ // W0193: pre-migrate legacy prompt.vars.{task,docs,__featureArea}
73
+ // to the canonical prompt.text + context.docs shape. Mode-agnostic —
74
+ // every mode's TaskPromptSchema rejects reserved keys, so the shim
75
+ // unblocks legacy tasks regardless of mode. Per-task deprecation
76
+ // warning fires on stderr.
77
+ const { migrated, warnings: deprecationWarnings } = migratePromptShape(parsed, file);
78
+ for (const warning of deprecationWarnings) {
79
+ console.warn(warning);
80
+ }
72
81
  // Validate through canonical Zod schema
73
82
  let validated;
74
83
  try {
75
- validated = parseCanonicalTaskFile(parsed, file);
84
+ validated = parseCanonicalTaskFile(migrated, file);
76
85
  }
77
86
  catch (err) {
78
87
  const msg = err instanceof Error ? err.message : String(err);
@@ -15,12 +15,12 @@
15
15
  * has been eliminated — all validation logic now lives here.
16
16
  */
17
17
  import { type CanonicalTask } from "./repo-schemas.js";
18
- export interface ValidationResult {
18
+ export interface RepoValidationResult {
19
19
  valid: boolean;
20
- errors: ValidationMessage[];
21
- warnings: ValidationMessage[];
20
+ errors: RepoValidationMessage[];
21
+ warnings: RepoValidationMessage[];
22
22
  }
23
- export interface ValidationMessage {
23
+ export interface RepoValidationMessage {
24
24
  taskId: string;
25
25
  field: string;
26
26
  message: string;
@@ -32,8 +32,8 @@ export interface ValidationMessage {
32
32
  * areas, unresolved slugs) and errors for issues that would cause pipeline
33
33
  * failures (completely missing required fields — though Zod catches most).
34
34
  */
35
- export declare function validateCanonicalTasks(tasks: CanonicalTask[]): ValidationResult;
35
+ export declare function validateCanonicalTasks(tasks: CanonicalTask[]): RepoValidationResult;
36
36
  /**
37
37
  * Format validation results for console output.
38
38
  */
39
- export declare function formatValidationResult(result: ValidationResult): string;
39
+ export declare function formatRepoValidationResult(result: RepoValidationResult): string;
@@ -110,7 +110,7 @@ export function validateCanonicalTasks(tasks) {
110
110
  /**
111
111
  * Format validation results for console output.
112
112
  */
113
- export function formatValidationResult(result) {
113
+ export function formatRepoValidationResult(result) {
114
114
  const lines = [];
115
115
  if (result.errors.length > 0) {
116
116
  lines.push("Errors:");
@@ -63,6 +63,7 @@ export default class AgenticProvider {
63
63
  private docBaseUrl;
64
64
  private docsUrlPattern;
65
65
  private llmsTxtUrl;
66
+ private parsedConfig;
66
67
  private priorityDomain;
67
68
  private recorder;
68
69
  private searchMode;