@sanity/ailf 0.1.6 → 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/dist/_vendor/ailf-core/examples/index.d.ts +1 -1
  2. package/dist/_vendor/ailf-core/examples/index.js +1 -1
  3. package/dist/_vendor/ailf-core/ports/context.d.ts +6 -0
  4. package/dist/_vendor/ailf-tasks/cli.d.ts +8 -0
  5. package/dist/_vendor/ailf-tasks/cli.js +61 -0
  6. package/dist/_vendor/ailf-tasks/index.d.ts +13 -0
  7. package/dist/_vendor/ailf-tasks/index.js +16 -0
  8. package/dist/_vendor/ailf-tasks/parser.d.ts +27 -0
  9. package/dist/_vendor/ailf-tasks/parser.js +73 -0
  10. package/dist/_vendor/ailf-tasks/schemas.d.ts +186 -0
  11. package/dist/_vendor/ailf-tasks/schemas.js +176 -0
  12. package/dist/_vendor/ailf-tasks/validation.d.ts +47 -0
  13. package/dist/_vendor/ailf-tasks/validation.js +162 -0
  14. package/dist/adapters/api-client/api-client.d.ts +75 -0
  15. package/dist/adapters/api-client/api-client.js +201 -0
  16. package/dist/adapters/api-client/build-request.d.ts +75 -0
  17. package/dist/adapters/api-client/build-request.js +176 -0
  18. package/dist/adapters/api-client/errors.d.ts +43 -0
  19. package/dist/adapters/api-client/errors.js +68 -0
  20. package/dist/adapters/api-client/format-error.d.ts +22 -0
  21. package/dist/adapters/api-client/format-error.js +48 -0
  22. package/dist/adapters/api-client/index.d.ts +13 -0
  23. package/dist/adapters/api-client/index.js +12 -0
  24. package/dist/adapters/api-client/progress.d.ts +26 -0
  25. package/dist/adapters/api-client/progress.js +69 -0
  26. package/dist/adapters/api-client/remediation.d.ts +19 -0
  27. package/dist/adapters/api-client/remediation.js +76 -0
  28. package/dist/adapters/api-client/types.d.ts +98 -0
  29. package/dist/adapters/api-client/types.js +14 -0
  30. package/dist/adapters/config-sources/file-config-adapter.js +2 -0
  31. package/dist/adapters/task-sources/repo-schemas.d.ts +16 -181
  32. package/dist/adapters/task-sources/repo-schemas.js +27 -184
  33. package/dist/adapters/task-sources/repo-validation.d.ts +5 -46
  34. package/dist/adapters/task-sources/repo-validation.js +5 -161
  35. package/dist/commands/calculate-scores.js +2 -0
  36. package/dist/commands/explain-handler.js +6 -0
  37. package/dist/commands/fetch-docs.js +2 -0
  38. package/dist/commands/generate-configs.js +2 -0
  39. package/dist/commands/init.js +9 -9
  40. package/dist/commands/pipeline-action.d.ts +3 -0
  41. package/dist/commands/pipeline-action.js +13 -0
  42. package/dist/commands/pipeline.d.ts +2 -0
  43. package/dist/commands/pipeline.js +2 -0
  44. package/dist/commands/pr-comment.js +2 -0
  45. package/dist/commands/publish.js +2 -0
  46. package/dist/commands/remote-pipeline.d.ts +27 -0
  47. package/dist/commands/remote-pipeline.js +133 -0
  48. package/dist/commands/remote-results.d.ts +33 -0
  49. package/dist/commands/remote-results.js +97 -0
  50. package/dist/orchestration/build-app-context.js +3 -0
  51. package/dist/pipeline/map-request-to-config.js +2 -0
  52. package/package.json +2 -1
@@ -0,0 +1,98 @@
1
+ /**
2
+ * api-client/types.ts — Response types for the AILF API.
3
+ *
4
+ * These are the client-side types derived from the API's standard response
5
+ * envelope pattern (Stripe-style: `object`, `createdAt`, `apiVersion` on
6
+ * every response). They mirror the shapes returned by the API routes but
7
+ * are defined independently — the client doesn't import from the API
8
+ * package.
9
+ *
10
+ * @see packages/api/src/lib/envelope.ts — server-side envelope builders
11
+ * @see packages/api/src/routes/pipeline.ts — POST /v1/pipeline response
12
+ * @see packages/api/src/routes/jobs.ts — GET /v1/jobs/:id response
13
+ */
14
+ /** Standard envelope fields present on every API response. */
15
+ export interface ApiEnvelope {
16
+ object: string;
17
+ createdAt: string;
18
+ apiVersion: string;
19
+ }
20
+ /** Possible terminal and non-terminal job statuses. */
21
+ export type JobStatus = "received" | "queued" | "running" | "completed" | "failed" | "timed-out";
22
+ /** Structured error detail attached to failed/timed-out jobs. */
23
+ export interface JobError {
24
+ message: string;
25
+ step?: string;
26
+ }
27
+ /** Step-level progress info for running jobs. */
28
+ export interface JobProgress {
29
+ step: string;
30
+ current: number;
31
+ total: number;
32
+ }
33
+ /** Execution backend info. */
34
+ export interface JobExecution {
35
+ backend: string;
36
+ runId?: string | null;
37
+ runUrl?: string | null;
38
+ }
39
+ /** Full job response from GET /v1/jobs/:id. */
40
+ export interface JobResponse extends ApiEnvelope {
41
+ object: "job";
42
+ jobId: string;
43
+ status: JobStatus;
44
+ startedAt?: string | null;
45
+ completedAt?: string | null;
46
+ progress?: JobProgress | null;
47
+ execution?: JobExecution | null;
48
+ reportId?: string | null;
49
+ reportUrl?: string | null;
50
+ error?: JobError | null;
51
+ }
52
+ /** Response from POST /v1/pipeline. */
53
+ export interface SubmitResponse extends ApiEnvelope {
54
+ object: "job";
55
+ jobId: string;
56
+ status: string;
57
+ statusUrl: string;
58
+ backend: string;
59
+ estimatedDurationMs?: number;
60
+ runUrl?: string;
61
+ }
62
+ export interface ValidationIssue {
63
+ severity: string;
64
+ source: string;
65
+ message: string;
66
+ }
67
+ export interface ValidationResponse extends ApiEnvelope {
68
+ object: "validation";
69
+ valid: boolean;
70
+ issues?: ValidationIssue[];
71
+ }
72
+ export interface ApiErrorDetail {
73
+ code: string;
74
+ message: string;
75
+ param?: string;
76
+ issues?: ValidationIssue[];
77
+ retryAfter?: string;
78
+ }
79
+ export interface ApiErrorResponse extends ApiEnvelope {
80
+ object: "error";
81
+ error: ApiErrorDetail;
82
+ }
83
+ /** Configuration for creating an ApiClient instance. */
84
+ export interface ApiClientOptions {
85
+ /** AILF API key (starts with ailf_live_sk_ or ailf_test_sk_). */
86
+ apiKey: string;
87
+ /** Base URL for the AILF API. Defaults to https://ailf-api.sanity.build. */
88
+ baseUrl?: string;
89
+ }
90
+ /** Options for the waitForCompletion() polling method. */
91
+ export interface WaitOptions {
92
+ /** Maximum time to wait in milliseconds (default: 20 minutes). */
93
+ timeoutMs?: number;
94
+ /** Called after each poll with the current job state. */
95
+ onProgress?: (job: JobResponse) => void;
96
+ /** Long-poll wait time in seconds sent via Prefer header (default: 25, max: 25). */
97
+ longPollSeconds?: number;
98
+ }
@@ -0,0 +1,14 @@
1
+ /**
2
+ * api-client/types.ts — Response types for the AILF API.
3
+ *
4
+ * These are the client-side types derived from the API's standard response
5
+ * envelope pattern (Stripe-style: `object`, `createdAt`, `apiVersion` on
6
+ * every response). They mirror the shapes returned by the API routes but
7
+ * are defined independently — the client doesn't import from the API
8
+ * package.
9
+ *
10
+ * @see packages/api/src/lib/envelope.ts — server-side envelope builders
11
+ * @see packages/api/src/routes/pipeline.ts — POST /v1/pipeline response
12
+ * @see packages/api/src/routes/jobs.ts — GET /v1/jobs/:id response
13
+ */
14
+ export {};
@@ -92,5 +92,7 @@ function mapEvalConfigToResolvedConfig(config, rootDir) {
92
92
  allowedOrigins: config.allowedOrigins,
93
93
  searchMode: config.searchMode ?? "open",
94
94
  concurrency: config.concurrency,
95
+ remote: false,
96
+ apiUrl: "https://ailf-api.sanity.build",
95
97
  };
96
98
  }
@@ -1,189 +1,29 @@
1
1
  /**
2
- * repo-schemas.ts — Zod schemas for repo-based task definitions.
2
+ * repo-schemas.ts — Re-exports task schemas + defines config schemas.
3
3
  *
4
- * Validates .ailf/tasks/*.yaml task files and .ailf/config.yaml trigger
5
- * configuration from external repositories (Phase 4 of Tasks as Content).
4
+ * Task schemas (RepoTaskSchema, assertions, etc.) are the single source
5
+ * of truth in @sanity/ailf-tasks. This file re-exports them so existing
6
+ * importers within the eval package don't need to change their paths.
6
7
  *
7
- * These schemas are the contract between external repos and the AILF eval
8
- * pipeline they define exactly what fields are accepted, with friendly
9
- * error messages for authors writing task YAML by hand.
8
+ * Config schemas (RepoConfigSchema, trigger config, etc.) remain here
9
+ * because they are specific to the eval pipeline and not needed by
10
+ * external tools that only validate task YAML.
10
11
  *
12
+ * @see packages/tasks/src/schemas.ts — task schema source of truth
11
13
  * @see docs/exec-plans/completed/tasks-as-content/phase-4-repo-based-tasks.md
12
- * @see packages/eval/src/adapters/task-sources/repo-task-source.ts
13
14
  */
15
+ import { RepoTaskFileSchema as _Schema } from "../../_vendor/ailf-tasks/index.d.ts";
14
16
  import { z } from "zod";
17
+ export { CURATED_ASSERTION_TYPES, RepoTaskFileSchema, RepoTaskSchema, RUBRIC_TEMPLATE_NAMES, type CuratedAssertionType, type RepoTask, type RubricTemplateName, } from "../../_vendor/ailf-tasks/index.d.ts";
18
+ export { loadTaskDir, parseTaskFile } from "../../_vendor/ailf-tasks/index.d.ts";
15
19
  /**
16
- * The set of assertion types allowed in repo-based task files.
17
- *
18
- * This is a curated subset of Promptfoo assertion types — we expose only the
19
- * types that are stable, well-documented, and useful for external authors.
20
- */
21
- export declare const CURATED_ASSERTION_TYPES: readonly ["llm-rubric", "contains", "contains-any", "contains-all", "not-contains", "icontains", "icontains-any", "regex", "javascript", "similar", "cost", "latency"];
22
- export type CuratedAssertionType = (typeof CURATED_ASSERTION_TYPES)[number];
23
- /**
24
- * Valid rubric template names — must match keys in config/rubrics.yaml.
25
- */
26
- export declare const RUBRIC_TEMPLATE_NAMES: readonly ["task-completion", "code-correctness", "doc-coverage"];
27
- export type RubricTemplateName = (typeof RUBRIC_TEMPLATE_NAMES)[number];
28
- /**
29
- * Zod schema for a single repo-based task definition.
20
+ * Parse and validate a repo task file's content. Returns typed tasks or throws
21
+ * with a user-friendly Zod error message.
30
22
  *
31
- * This is the external-author-facing contract. Field names are camelCase
32
- * to match the Content Lake document schema (ailf.task).
33
- */
34
- export declare const RepoTaskSchema: z.ZodObject<{
35
- id: z.ZodString;
36
- description: z.ZodString;
37
- featureArea: z.ZodString;
38
- tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
39
- canonicalDocs: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
40
- id: z.ZodString;
41
- reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
42
- slug: z.ZodOptional<z.ZodString>;
43
- path: z.ZodOptional<z.ZodString>;
44
- }, z.core.$strip>, z.ZodObject<{
45
- slug: z.ZodString;
46
- reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
47
- }, z.core.$strip>, z.ZodObject<{
48
- path: z.ZodString;
49
- reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
50
- }, z.core.$strip>, z.ZodObject<{
51
- perspective: z.ZodString;
52
- reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
53
- }, z.core.$strip>]>>>>;
54
- vars: z.ZodOptional<z.ZodObject<{
55
- task: z.ZodString;
56
- }, z.core.$loose>>;
57
- assert: z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
58
- type: z.ZodLiteral<"llm-rubric">;
59
- template: z.ZodEnum<{
60
- "task-completion": "task-completion";
61
- "code-correctness": "code-correctness";
62
- "doc-coverage": "doc-coverage";
63
- }>;
64
- criteria: z.ZodArray<z.ZodString>;
65
- weight: z.ZodOptional<z.ZodNumber>;
66
- }, z.core.$strip>, z.ZodObject<{
67
- type: z.ZodEnum<{
68
- "llm-rubric": "llm-rubric";
69
- contains: "contains";
70
- "contains-any": "contains-any";
71
- "contains-all": "contains-all";
72
- "not-contains": "not-contains";
73
- icontains: "icontains";
74
- "icontains-any": "icontains-any";
75
- regex: "regex";
76
- javascript: "javascript";
77
- similar: "similar";
78
- cost: "cost";
79
- latency: "latency";
80
- }>;
81
- value: z.ZodOptional<z.ZodUnknown>;
82
- threshold: z.ZodOptional<z.ZodNumber>;
83
- weight: z.ZodOptional<z.ZodNumber>;
84
- }, z.core.$loose>]>>;
85
- baseline: z.ZodOptional<z.ZodObject<{
86
- enabled: z.ZodOptional<z.ZodBoolean>;
87
- rubric: z.ZodOptional<z.ZodEnum<{
88
- full: "full";
89
- abbreviated: "abbreviated";
90
- none: "none";
91
- }>>;
92
- }, z.core.$strip>>;
93
- docCoverage: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
94
- referenceSolution: z.ZodOptional<z.ZodString>;
95
- execution: z.ZodOptional<z.ZodObject<{
96
- enabled: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
97
- blocking: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
98
- threshold: z.ZodOptional<z.ZodObject<{
99
- score: z.ZodOptional<z.ZodNumber>;
100
- }, z.core.$strip>>;
101
- trigger: z.ZodOptional<z.ZodObject<{
102
- branches: z.ZodOptional<z.ZodArray<z.ZodString>>;
103
- paths: z.ZodOptional<z.ZodArray<z.ZodString>>;
104
- }, z.core.$strip>>;
105
- source: z.ZodOptional<z.ZodString>;
106
- }, z.core.$strip>>;
107
- }, z.core.$strip>;
108
- export type RepoTask = z.infer<typeof RepoTaskSchema>;
109
- /**
110
- * Schema for an array of repo tasks — what a single .ailf/tasks/*.yaml file
111
- * contains. Each file must define at least one task.
23
+ * NOTE: This accepts pre-parsed YAML data (unknown), not a raw string.
24
+ * For raw YAML strings, use `parseTaskFile()` from @sanity/ailf-tasks.
112
25
  */
113
- export declare const RepoTaskFileSchema: z.ZodArray<z.ZodObject<{
114
- id: z.ZodString;
115
- description: z.ZodString;
116
- featureArea: z.ZodString;
117
- tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
118
- canonicalDocs: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
119
- id: z.ZodString;
120
- reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
121
- slug: z.ZodOptional<z.ZodString>;
122
- path: z.ZodOptional<z.ZodString>;
123
- }, z.core.$strip>, z.ZodObject<{
124
- slug: z.ZodString;
125
- reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
126
- }, z.core.$strip>, z.ZodObject<{
127
- path: z.ZodString;
128
- reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
129
- }, z.core.$strip>, z.ZodObject<{
130
- perspective: z.ZodString;
131
- reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
132
- }, z.core.$strip>]>>>>;
133
- vars: z.ZodOptional<z.ZodObject<{
134
- task: z.ZodString;
135
- }, z.core.$loose>>;
136
- assert: z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
137
- type: z.ZodLiteral<"llm-rubric">;
138
- template: z.ZodEnum<{
139
- "task-completion": "task-completion";
140
- "code-correctness": "code-correctness";
141
- "doc-coverage": "doc-coverage";
142
- }>;
143
- criteria: z.ZodArray<z.ZodString>;
144
- weight: z.ZodOptional<z.ZodNumber>;
145
- }, z.core.$strip>, z.ZodObject<{
146
- type: z.ZodEnum<{
147
- "llm-rubric": "llm-rubric";
148
- contains: "contains";
149
- "contains-any": "contains-any";
150
- "contains-all": "contains-all";
151
- "not-contains": "not-contains";
152
- icontains: "icontains";
153
- "icontains-any": "icontains-any";
154
- regex: "regex";
155
- javascript: "javascript";
156
- similar: "similar";
157
- cost: "cost";
158
- latency: "latency";
159
- }>;
160
- value: z.ZodOptional<z.ZodUnknown>;
161
- threshold: z.ZodOptional<z.ZodNumber>;
162
- weight: z.ZodOptional<z.ZodNumber>;
163
- }, z.core.$loose>]>>;
164
- baseline: z.ZodOptional<z.ZodObject<{
165
- enabled: z.ZodOptional<z.ZodBoolean>;
166
- rubric: z.ZodOptional<z.ZodEnum<{
167
- full: "full";
168
- abbreviated: "abbreviated";
169
- none: "none";
170
- }>>;
171
- }, z.core.$strip>>;
172
- docCoverage: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
173
- referenceSolution: z.ZodOptional<z.ZodString>;
174
- execution: z.ZodOptional<z.ZodObject<{
175
- enabled: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
176
- blocking: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
177
- threshold: z.ZodOptional<z.ZodObject<{
178
- score: z.ZodOptional<z.ZodNumber>;
179
- }, z.core.$strip>>;
180
- trigger: z.ZodOptional<z.ZodObject<{
181
- branches: z.ZodOptional<z.ZodArray<z.ZodString>>;
182
- paths: z.ZodOptional<z.ZodArray<z.ZodString>>;
183
- }, z.core.$strip>>;
184
- source: z.ZodOptional<z.ZodString>;
185
- }, z.core.$strip>>;
186
- }, z.core.$strip>>;
26
+ export declare function parseRepoTaskFile(raw: unknown, filename: string): z.infer<typeof _Schema>;
187
27
  /**
188
28
  * Zod schema for .ailf/config.yaml — controls documentation source,
189
29
  * report destination, and trigger behavior for evaluations from an
@@ -244,11 +84,6 @@ export declare const RepoConfigSchema: z.ZodObject<{
244
84
  }, z.core.$strip>>;
245
85
  }, z.core.$strip>;
246
86
  export type RepoConfig = z.infer<typeof RepoConfigSchema>;
247
- /**
248
- * Parse and validate a repo task file's content. Returns typed tasks or throws
249
- * with a user-friendly Zod error message.
250
- */
251
- export declare function parseRepoTaskFile(raw: unknown, filename: string): RepoTask[];
252
87
  /**
253
88
  * Parse and validate .ailf/config.yaml content. Returns typed config or throws.
254
89
  */
@@ -1,181 +1,43 @@
1
1
  /**
2
- * repo-schemas.ts — Zod schemas for repo-based task definitions.
2
+ * repo-schemas.ts — Re-exports task schemas + defines config schemas.
3
3
  *
4
- * Validates .ailf/tasks/*.yaml task files and .ailf/config.yaml trigger
5
- * configuration from external repositories (Phase 4 of Tasks as Content).
4
+ * Task schemas (RepoTaskSchema, assertions, etc.) are the single source
5
+ * of truth in @sanity/ailf-tasks. This file re-exports them so existing
6
+ * importers within the eval package don't need to change their paths.
6
7
  *
7
- * These schemas are the contract between external repos and the AILF eval
8
- * pipeline they define exactly what fields are accepted, with friendly
9
- * error messages for authors writing task YAML by hand.
8
+ * Config schemas (RepoConfigSchema, trigger config, etc.) remain here
9
+ * because they are specific to the eval pipeline and not needed by
10
+ * external tools that only validate task YAML.
10
11
  *
12
+ * @see packages/tasks/src/schemas.ts — task schema source of truth
11
13
  * @see docs/exec-plans/completed/tasks-as-content/phase-4-repo-based-tasks.md
12
- * @see packages/eval/src/adapters/task-sources/repo-task-source.ts
13
14
  */
15
+ import { RepoTaskFileSchema as _Schema } from "../../_vendor/ailf-tasks/index.js";
14
16
  import { z } from "zod";
15
17
  // ---------------------------------------------------------------------------
16
- // Constants curated assertion types and rubric template names
18
+ // Re-exports from @sanity/ailf-tasks (task schemas + validation)
17
19
  // ---------------------------------------------------------------------------
20
+ export { CURATED_ASSERTION_TYPES, RepoTaskFileSchema, RepoTaskSchema, RUBRIC_TEMPLATE_NAMES, } from "../../_vendor/ailf-tasks/index.js";
21
+ export { loadTaskDir, parseTaskFile } from "../../_vendor/ailf-tasks/index.js";
18
22
  /**
19
- * The set of assertion types allowed in repo-based task files.
20
- *
21
- * This is a curated subset of Promptfoo assertion types — we expose only the
22
- * types that are stable, well-documented, and useful for external authors.
23
- */
24
- export const CURATED_ASSERTION_TYPES = [
25
- "llm-rubric",
26
- "contains",
27
- "contains-any",
28
- "contains-all",
29
- "not-contains",
30
- "icontains",
31
- "icontains-any",
32
- "regex",
33
- "javascript",
34
- "similar",
35
- "cost",
36
- "latency",
37
- ];
38
- /**
39
- * Valid rubric template names — must match keys in config/rubrics.yaml.
40
- */
41
- export const RUBRIC_TEMPLATE_NAMES = [
42
- "task-completion",
43
- "code-correctness",
44
- "doc-coverage",
45
- ];
46
- // ---------------------------------------------------------------------------
47
- // Assertion schemas
48
- // ---------------------------------------------------------------------------
49
- /**
50
- * Polymorphic canonical doc reference — discriminated by key presence.
51
- * Exactly one resolution key (slug, path, id, or perspective) must be present.
52
- *
53
- * @see docs/design-docs/canonical-doc-resolution.md
54
- */
55
- const SlugDocRefSchema = z.object({
56
- slug: z.string().min(1),
57
- reason: z.string().optional().default(""),
58
- });
59
- const PathDocRefSchema = z.object({
60
- path: z.string().min(1),
61
- reason: z.string().optional().default(""),
62
- });
63
- const IdDocRefSchema = z.object({
64
- id: z.string().min(1),
65
- reason: z.string().optional().default(""),
66
- /** Human-readable slug annotation (not used for resolution) */
67
- slug: z.string().optional(),
68
- /** Human-readable path annotation (not used for resolution) */
69
- path: z.string().optional(),
70
- });
71
- const PerspectiveDocRefSchema = z.object({
72
- perspective: z.string().min(1),
73
- reason: z.string().optional().default(""),
74
- });
75
- // Order matters: IdDocRefSchema first because it may also carry `slug`
76
- // and `path` as optional annotations. Zod tries schemas in order, so
77
- // entries like `{ id: "...", slug: "..." }` must match IdDocRefSchema
78
- // (not SlugDocRefSchema).
79
- const CanonicalDocRefSchema = z.union([
80
- IdDocRefSchema,
81
- SlugDocRefSchema,
82
- PathDocRefSchema,
83
- PerspectiveDocRefSchema,
84
- ]);
85
- /**
86
- * A templated LLM-rubric assertion — uses one of the predefined rubric
87
- * templates with author-supplied criteria.
88
- */
89
- const TemplatedAssertionSchema = z.object({
90
- type: z.literal("llm-rubric"),
91
- template: z.enum(RUBRIC_TEMPLATE_NAMES),
92
- criteria: z.array(z.string().min(1)).min(1),
93
- weight: z.number().optional(),
94
- });
95
- /**
96
- * A value-based assertion (contains, regex, cost, etc.). Uses .passthrough()
97
- * to allow extra fields for future extension without schema breakage.
98
- */
99
- const ValueAssertionSchema = z
100
- .object({
101
- type: z.enum(CURATED_ASSERTION_TYPES),
102
- value: z.unknown().optional(),
103
- threshold: z.number().optional(),
104
- weight: z.number().optional(),
105
- })
106
- .passthrough();
107
- /** Union of all supported assertion shapes. */
108
- const AssertionSchema = z.union([
109
- TemplatedAssertionSchema,
110
- ValueAssertionSchema,
111
- ]);
112
- // ---------------------------------------------------------------------------
113
- // Nested config schemas
114
- // ---------------------------------------------------------------------------
115
- const BaselineConfigSchema = z
116
- .object({
117
- enabled: z.boolean().optional(),
118
- rubric: z.enum(["abbreviated", "full", "none"]).optional(),
119
- })
120
- .optional();
121
- const ExecutionConfigSchema = z
122
- .object({
123
- enabled: z.boolean().optional().default(true),
124
- blocking: z.boolean().optional().default(false),
125
- threshold: z
126
- .object({
127
- score: z.number().min(0).max(100).optional(),
128
- })
129
- .optional(),
130
- trigger: z
131
- .object({
132
- branches: z.array(z.string()).optional(),
133
- paths: z.array(z.string()).optional(),
134
- })
135
- .optional(),
136
- source: z.string().optional(),
137
- })
138
- .optional();
139
- // ---------------------------------------------------------------------------
140
- // RepoTaskSchema — a single task definition from .ailf/tasks/*.yaml
141
- // ---------------------------------------------------------------------------
142
- /**
143
- * Zod schema for a single repo-based task definition.
23
+ * Parse and validate a repo task file's content. Returns typed tasks or throws
24
+ * with a user-friendly Zod error message.
144
25
  *
145
- * This is the external-author-facing contract. Field names are camelCase
146
- * to match the Content Lake document schema (ailf.task).
26
+ * NOTE: This accepts pre-parsed YAML data (unknown), not a raw string.
27
+ * For raw YAML strings, use `parseTaskFile()` from @sanity/ailf-tasks.
147
28
  */
148
- export const RepoTaskSchema = z.object({
149
- id: z
150
- .string()
151
- .min(1)
152
- .regex(/^[a-z0-9][a-z0-9-]*$/, "Task ID must be lowercase alphanumeric with hyphens"),
153
- description: z.string().min(1),
154
- featureArea: z
155
- .string()
156
- .min(1)
157
- .regex(/^[a-z0-9][a-z0-9-]*$/, "Feature area must be lowercase alphanumeric with hyphens"),
158
- tags: z.array(z.string()).optional(),
159
- canonicalDocs: z.array(CanonicalDocRefSchema).optional().default([]),
160
- vars: z
161
- .object({
162
- task: z.string().min(1),
163
- })
164
- .passthrough()
165
- .optional(),
166
- assert: z.array(AssertionSchema).min(1),
167
- baseline: BaselineConfigSchema,
168
- docCoverage: z.boolean().optional().default(false),
169
- referenceSolution: z.string().optional(),
170
- execution: ExecutionConfigSchema,
171
- });
172
- /**
173
- * Schema for an array of repo tasks — what a single .ailf/tasks/*.yaml file
174
- * contains. Each file must define at least one task.
175
- */
176
- export const RepoTaskFileSchema = z.array(RepoTaskSchema).min(1);
29
+ export function parseRepoTaskFile(raw, filename) {
30
+ const result = _Schema.safeParse(raw);
31
+ if (!result.success) {
32
+ const messages = result.error.issues
33
+ .map((i) => ` [${i.path.join(".")}]: ${i.message}`)
34
+ .join("\n");
35
+ throw new Error(`Invalid repo task file "${filename}":\n${messages}`);
36
+ }
37
+ return result.data;
38
+ }
177
39
  // ---------------------------------------------------------------------------
178
- // RepoConfigSchema.ailf/config.yaml
40
+ // Config schemas specific to the eval pipeline
179
41
  // ---------------------------------------------------------------------------
180
42
  const TriggerModeSchema = z.enum(["validate-only", "eval"]);
181
43
  const TriggerConfigSchema = z.object({
@@ -202,8 +64,6 @@ const SourceConfigSchema = z
202
64
  /**
203
65
  * Report store configuration.
204
66
  * Defines which Sanity project receives `ailf.report` documents.
205
- * This should match the project/dataset configured in the user's Studio.
206
- * The API token comes from the AILF_REPORT_SANITY_API_TOKEN env var.
207
67
  */
208
68
  const ReportStoreConfigSchema = z
209
69
  .object({
@@ -228,23 +88,6 @@ export const RepoConfigSchema = z.object({
228
88
  })
229
89
  .optional(),
230
90
  });
231
- // ---------------------------------------------------------------------------
232
- // Validation helpers
233
- // ---------------------------------------------------------------------------
234
- /**
235
- * Parse and validate a repo task file's content. Returns typed tasks or throws
236
- * with a user-friendly Zod error message.
237
- */
238
- export function parseRepoTaskFile(raw, filename) {
239
- const result = RepoTaskFileSchema.safeParse(raw);
240
- if (!result.success) {
241
- const messages = result.error.issues
242
- .map((i) => ` [${i.path.join(".")}]: ${i.message}`)
243
- .join("\n");
244
- throw new Error(`Invalid repo task file "${filename}":\n${messages}`);
245
- }
246
- return result.data;
247
- }
248
91
  /**
249
92
  * Parse and validate .ailf/config.yaml content. Returns typed config or throws.
250
93
  */
@@ -1,49 +1,8 @@
1
1
  /**
2
- * Semantic validation for repo-based tasks.
2
+ * repo-validation.ts — Re-exports semantic validation from @sanity/ailf-tasks.
3
3
  *
4
- * Checks that go beyond Zod schema parsing:
5
- * - Assertion types are in the curated set
6
- * - Rubric template names resolve to known templates
7
- * - Feature area strings are well-formed
8
- * - Canonical doc slugs look reasonable (slugs, not URLs)
9
- *
10
- * These produce warnings, not errors — the pipeline can still run
11
- * with imperfect tasks. Only structural failures (caught by Zod) block.
12
- *
13
- * @see packages/eval/src/adapters/task-sources/repo-schemas.ts
14
- */
15
- import { type RepoTask } from "./repo-schemas.js";
16
- export interface ValidationResult {
17
- valid: boolean;
18
- errors: ValidationMessage[];
19
- warnings: ValidationMessage[];
20
- }
21
- export interface ValidationMessage {
22
- taskId: string;
23
- field: string;
24
- message: string;
25
- }
26
- /**
27
- * Run semantic validation on an array of parsed repo tasks.
28
- *
29
- * Returns warnings for issues that don't block execution (unknown feature
30
- * areas, unresolved slugs) and errors for issues that would cause pipeline
31
- * failures (completely missing required fields — though Zod catches most).
32
- */
33
- export declare function validateRepoTasks(tasks: RepoTask[]): ValidationResult;
34
- /**
35
- * Format validation results for console output.
36
- */
37
- export declare function formatValidationResult(result: ValidationResult): string;
38
- /**
39
- * Detect snake_case field names in raw task YAML data.
40
- *
41
- * This runs BEFORE Zod parsing to provide a user-friendly error message
42
- * when authors use framework-internal snake_case names instead of the
43
- * camelCase names expected in repo task files.
44
- *
45
- * @param raw - Raw parsed YAML (before Zod validation)
46
- * @param filename - Source filename for error messages
47
- * @returns Array of warning messages (empty if no issues)
4
+ * The validation logic is the single source of truth in @sanity/ailf-tasks.
5
+ * This file re-exports so existing eval-package importers don't need
6
+ * to change their import paths.
48
7
  */
49
- export declare function detectSnakeCaseFields(raw: unknown, filename: string): string[];
8
+ export { detectSnakeCaseFields, formatValidationResult, validateRepoTasks, type ValidationMessage, type ValidationResult, } from "../../_vendor/ailf-tasks/index.d.ts";