@sanity/ailf 0.1.6 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_vendor/ailf-core/examples/index.d.ts +1 -1
- package/dist/_vendor/ailf-core/examples/index.js +1 -1
- package/dist/_vendor/ailf-core/ports/context.d.ts +6 -0
- package/dist/_vendor/ailf-tasks/cli.d.ts +8 -0
- package/dist/_vendor/ailf-tasks/cli.js +61 -0
- package/dist/_vendor/ailf-tasks/index.d.ts +13 -0
- package/dist/_vendor/ailf-tasks/index.js +16 -0
- package/dist/_vendor/ailf-tasks/parser.d.ts +27 -0
- package/dist/_vendor/ailf-tasks/parser.js +73 -0
- package/dist/_vendor/ailf-tasks/schemas.d.ts +186 -0
- package/dist/_vendor/ailf-tasks/schemas.js +176 -0
- package/dist/_vendor/ailf-tasks/validation.d.ts +47 -0
- package/dist/_vendor/ailf-tasks/validation.js +162 -0
- package/dist/adapters/api-client/api-client.d.ts +75 -0
- package/dist/adapters/api-client/api-client.js +201 -0
- package/dist/adapters/api-client/build-request.d.ts +75 -0
- package/dist/adapters/api-client/build-request.js +176 -0
- package/dist/adapters/api-client/errors.d.ts +43 -0
- package/dist/adapters/api-client/errors.js +68 -0
- package/dist/adapters/api-client/format-error.d.ts +22 -0
- package/dist/adapters/api-client/format-error.js +48 -0
- package/dist/adapters/api-client/index.d.ts +13 -0
- package/dist/adapters/api-client/index.js +12 -0
- package/dist/adapters/api-client/progress.d.ts +26 -0
- package/dist/adapters/api-client/progress.js +69 -0
- package/dist/adapters/api-client/remediation.d.ts +19 -0
- package/dist/adapters/api-client/remediation.js +76 -0
- package/dist/adapters/api-client/types.d.ts +98 -0
- package/dist/adapters/api-client/types.js +14 -0
- package/dist/adapters/config-sources/file-config-adapter.js +2 -0
- package/dist/adapters/task-sources/repo-schemas.d.ts +16 -181
- package/dist/adapters/task-sources/repo-schemas.js +27 -184
- package/dist/adapters/task-sources/repo-validation.d.ts +5 -46
- package/dist/adapters/task-sources/repo-validation.js +5 -161
- package/dist/commands/calculate-scores.js +2 -0
- package/dist/commands/explain-handler.js +6 -0
- package/dist/commands/fetch-docs.js +2 -0
- package/dist/commands/generate-configs.js +2 -0
- package/dist/commands/init.js +9 -9
- package/dist/commands/pipeline-action.d.ts +3 -0
- package/dist/commands/pipeline-action.js +13 -0
- package/dist/commands/pipeline.d.ts +2 -0
- package/dist/commands/pipeline.js +2 -0
- package/dist/commands/pr-comment.js +2 -0
- package/dist/commands/publish.js +2 -0
- package/dist/commands/remote-pipeline.d.ts +27 -0
- package/dist/commands/remote-pipeline.js +133 -0
- package/dist/commands/remote-results.d.ts +33 -0
- package/dist/commands/remote-results.js +97 -0
- package/dist/orchestration/build-app-context.js +3 -0
- package/dist/pipeline/map-request-to-config.js +2 -0
- package/package.json +2 -1
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* api-client/types.ts — Response types for the AILF API.
|
|
3
|
+
*
|
|
4
|
+
* These are the client-side types derived from the API's standard response
|
|
5
|
+
* envelope pattern (Stripe-style: `object`, `createdAt`, `apiVersion` on
|
|
6
|
+
* every response). They mirror the shapes returned by the API routes but
|
|
7
|
+
* are defined independently — the client doesn't import from the API
|
|
8
|
+
* package.
|
|
9
|
+
*
|
|
10
|
+
* @see packages/api/src/lib/envelope.ts — server-side envelope builders
|
|
11
|
+
* @see packages/api/src/routes/pipeline.ts — POST /v1/pipeline response
|
|
12
|
+
* @see packages/api/src/routes/jobs.ts — GET /v1/jobs/:id response
|
|
13
|
+
*/
|
|
14
|
+
/** Standard envelope fields present on every API response. */
|
|
15
|
+
export interface ApiEnvelope {
|
|
16
|
+
object: string;
|
|
17
|
+
createdAt: string;
|
|
18
|
+
apiVersion: string;
|
|
19
|
+
}
|
|
20
|
+
/** Possible terminal and non-terminal job statuses. */
|
|
21
|
+
export type JobStatus = "received" | "queued" | "running" | "completed" | "failed" | "timed-out";
|
|
22
|
+
/** Structured error detail attached to failed/timed-out jobs. */
|
|
23
|
+
export interface JobError {
|
|
24
|
+
message: string;
|
|
25
|
+
step?: string;
|
|
26
|
+
}
|
|
27
|
+
/** Step-level progress info for running jobs. */
|
|
28
|
+
export interface JobProgress {
|
|
29
|
+
step: string;
|
|
30
|
+
current: number;
|
|
31
|
+
total: number;
|
|
32
|
+
}
|
|
33
|
+
/** Execution backend info. */
|
|
34
|
+
export interface JobExecution {
|
|
35
|
+
backend: string;
|
|
36
|
+
runId?: string | null;
|
|
37
|
+
runUrl?: string | null;
|
|
38
|
+
}
|
|
39
|
+
/** Full job response from GET /v1/jobs/:id. */
|
|
40
|
+
export interface JobResponse extends ApiEnvelope {
|
|
41
|
+
object: "job";
|
|
42
|
+
jobId: string;
|
|
43
|
+
status: JobStatus;
|
|
44
|
+
startedAt?: string | null;
|
|
45
|
+
completedAt?: string | null;
|
|
46
|
+
progress?: JobProgress | null;
|
|
47
|
+
execution?: JobExecution | null;
|
|
48
|
+
reportId?: string | null;
|
|
49
|
+
reportUrl?: string | null;
|
|
50
|
+
error?: JobError | null;
|
|
51
|
+
}
|
|
52
|
+
/** Response from POST /v1/pipeline. */
|
|
53
|
+
export interface SubmitResponse extends ApiEnvelope {
|
|
54
|
+
object: "job";
|
|
55
|
+
jobId: string;
|
|
56
|
+
status: string;
|
|
57
|
+
statusUrl: string;
|
|
58
|
+
backend: string;
|
|
59
|
+
estimatedDurationMs?: number;
|
|
60
|
+
runUrl?: string;
|
|
61
|
+
}
|
|
62
|
+
export interface ValidationIssue {
|
|
63
|
+
severity: string;
|
|
64
|
+
source: string;
|
|
65
|
+
message: string;
|
|
66
|
+
}
|
|
67
|
+
export interface ValidationResponse extends ApiEnvelope {
|
|
68
|
+
object: "validation";
|
|
69
|
+
valid: boolean;
|
|
70
|
+
issues?: ValidationIssue[];
|
|
71
|
+
}
|
|
72
|
+
export interface ApiErrorDetail {
|
|
73
|
+
code: string;
|
|
74
|
+
message: string;
|
|
75
|
+
param?: string;
|
|
76
|
+
issues?: ValidationIssue[];
|
|
77
|
+
retryAfter?: string;
|
|
78
|
+
}
|
|
79
|
+
export interface ApiErrorResponse extends ApiEnvelope {
|
|
80
|
+
object: "error";
|
|
81
|
+
error: ApiErrorDetail;
|
|
82
|
+
}
|
|
83
|
+
/** Configuration for creating an ApiClient instance. */
|
|
84
|
+
export interface ApiClientOptions {
|
|
85
|
+
/** AILF API key (starts with ailf_live_sk_ or ailf_test_sk_). */
|
|
86
|
+
apiKey: string;
|
|
87
|
+
/** Base URL for the AILF API. Defaults to https://ailf-api.sanity.build. */
|
|
88
|
+
baseUrl?: string;
|
|
89
|
+
}
|
|
90
|
+
/** Options for the waitForCompletion() polling method. */
|
|
91
|
+
export interface WaitOptions {
|
|
92
|
+
/** Maximum time to wait in milliseconds (default: 20 minutes). */
|
|
93
|
+
timeoutMs?: number;
|
|
94
|
+
/** Called after each poll with the current job state. */
|
|
95
|
+
onProgress?: (job: JobResponse) => void;
|
|
96
|
+
/** Long-poll wait time in seconds sent via Prefer header (default: 25, max: 25). */
|
|
97
|
+
longPollSeconds?: number;
|
|
98
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* api-client/types.ts — Response types for the AILF API.
|
|
3
|
+
*
|
|
4
|
+
* These are the client-side types derived from the API's standard response
|
|
5
|
+
* envelope pattern (Stripe-style: `object`, `createdAt`, `apiVersion` on
|
|
6
|
+
* every response). They mirror the shapes returned by the API routes but
|
|
7
|
+
* are defined independently — the client doesn't import from the API
|
|
8
|
+
* package.
|
|
9
|
+
*
|
|
10
|
+
* @see packages/api/src/lib/envelope.ts — server-side envelope builders
|
|
11
|
+
* @see packages/api/src/routes/pipeline.ts — POST /v1/pipeline response
|
|
12
|
+
* @see packages/api/src/routes/jobs.ts — GET /v1/jobs/:id response
|
|
13
|
+
*/
|
|
14
|
+
export {};
|
|
@@ -1,189 +1,29 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* repo-schemas.ts —
|
|
2
|
+
* repo-schemas.ts — Re-exports task schemas + defines config schemas.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
4
|
+
* Task schemas (RepoTaskSchema, assertions, etc.) are the single source
|
|
5
|
+
* of truth in @sanity/ailf-tasks. This file re-exports them so existing
|
|
6
|
+
* importers within the eval package don't need to change their paths.
|
|
6
7
|
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
8
|
+
* Config schemas (RepoConfigSchema, trigger config, etc.) remain here
|
|
9
|
+
* because they are specific to the eval pipeline and not needed by
|
|
10
|
+
* external tools that only validate task YAML.
|
|
10
11
|
*
|
|
12
|
+
* @see packages/tasks/src/schemas.ts — task schema source of truth
|
|
11
13
|
* @see docs/exec-plans/completed/tasks-as-content/phase-4-repo-based-tasks.md
|
|
12
|
-
* @see packages/eval/src/adapters/task-sources/repo-task-source.ts
|
|
13
14
|
*/
|
|
15
|
+
import { RepoTaskFileSchema as _Schema } from "../../_vendor/ailf-tasks/index.d.ts";
|
|
14
16
|
import { z } from "zod";
|
|
17
|
+
export { CURATED_ASSERTION_TYPES, RepoTaskFileSchema, RepoTaskSchema, RUBRIC_TEMPLATE_NAMES, type CuratedAssertionType, type RepoTask, type RubricTemplateName, } from "../../_vendor/ailf-tasks/index.d.ts";
|
|
18
|
+
export { loadTaskDir, parseTaskFile } from "../../_vendor/ailf-tasks/index.d.ts";
|
|
15
19
|
/**
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
* This is a curated subset of Promptfoo assertion types — we expose only the
|
|
19
|
-
* types that are stable, well-documented, and useful for external authors.
|
|
20
|
-
*/
|
|
21
|
-
export declare const CURATED_ASSERTION_TYPES: readonly ["llm-rubric", "contains", "contains-any", "contains-all", "not-contains", "icontains", "icontains-any", "regex", "javascript", "similar", "cost", "latency"];
|
|
22
|
-
export type CuratedAssertionType = (typeof CURATED_ASSERTION_TYPES)[number];
|
|
23
|
-
/**
|
|
24
|
-
* Valid rubric template names — must match keys in config/rubrics.yaml.
|
|
25
|
-
*/
|
|
26
|
-
export declare const RUBRIC_TEMPLATE_NAMES: readonly ["task-completion", "code-correctness", "doc-coverage"];
|
|
27
|
-
export type RubricTemplateName = (typeof RUBRIC_TEMPLATE_NAMES)[number];
|
|
28
|
-
/**
|
|
29
|
-
* Zod schema for a single repo-based task definition.
|
|
20
|
+
* Parse and validate a repo task file's content. Returns typed tasks or throws
|
|
21
|
+
* with a user-friendly Zod error message.
|
|
30
22
|
*
|
|
31
|
-
* This
|
|
32
|
-
*
|
|
33
|
-
*/
|
|
34
|
-
export declare const RepoTaskSchema: z.ZodObject<{
|
|
35
|
-
id: z.ZodString;
|
|
36
|
-
description: z.ZodString;
|
|
37
|
-
featureArea: z.ZodString;
|
|
38
|
-
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
39
|
-
canonicalDocs: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
|
|
40
|
-
id: z.ZodString;
|
|
41
|
-
reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
42
|
-
slug: z.ZodOptional<z.ZodString>;
|
|
43
|
-
path: z.ZodOptional<z.ZodString>;
|
|
44
|
-
}, z.core.$strip>, z.ZodObject<{
|
|
45
|
-
slug: z.ZodString;
|
|
46
|
-
reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
47
|
-
}, z.core.$strip>, z.ZodObject<{
|
|
48
|
-
path: z.ZodString;
|
|
49
|
-
reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
50
|
-
}, z.core.$strip>, z.ZodObject<{
|
|
51
|
-
perspective: z.ZodString;
|
|
52
|
-
reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
53
|
-
}, z.core.$strip>]>>>>;
|
|
54
|
-
vars: z.ZodOptional<z.ZodObject<{
|
|
55
|
-
task: z.ZodString;
|
|
56
|
-
}, z.core.$loose>>;
|
|
57
|
-
assert: z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
|
|
58
|
-
type: z.ZodLiteral<"llm-rubric">;
|
|
59
|
-
template: z.ZodEnum<{
|
|
60
|
-
"task-completion": "task-completion";
|
|
61
|
-
"code-correctness": "code-correctness";
|
|
62
|
-
"doc-coverage": "doc-coverage";
|
|
63
|
-
}>;
|
|
64
|
-
criteria: z.ZodArray<z.ZodString>;
|
|
65
|
-
weight: z.ZodOptional<z.ZodNumber>;
|
|
66
|
-
}, z.core.$strip>, z.ZodObject<{
|
|
67
|
-
type: z.ZodEnum<{
|
|
68
|
-
"llm-rubric": "llm-rubric";
|
|
69
|
-
contains: "contains";
|
|
70
|
-
"contains-any": "contains-any";
|
|
71
|
-
"contains-all": "contains-all";
|
|
72
|
-
"not-contains": "not-contains";
|
|
73
|
-
icontains: "icontains";
|
|
74
|
-
"icontains-any": "icontains-any";
|
|
75
|
-
regex: "regex";
|
|
76
|
-
javascript: "javascript";
|
|
77
|
-
similar: "similar";
|
|
78
|
-
cost: "cost";
|
|
79
|
-
latency: "latency";
|
|
80
|
-
}>;
|
|
81
|
-
value: z.ZodOptional<z.ZodUnknown>;
|
|
82
|
-
threshold: z.ZodOptional<z.ZodNumber>;
|
|
83
|
-
weight: z.ZodOptional<z.ZodNumber>;
|
|
84
|
-
}, z.core.$loose>]>>;
|
|
85
|
-
baseline: z.ZodOptional<z.ZodObject<{
|
|
86
|
-
enabled: z.ZodOptional<z.ZodBoolean>;
|
|
87
|
-
rubric: z.ZodOptional<z.ZodEnum<{
|
|
88
|
-
full: "full";
|
|
89
|
-
abbreviated: "abbreviated";
|
|
90
|
-
none: "none";
|
|
91
|
-
}>>;
|
|
92
|
-
}, z.core.$strip>>;
|
|
93
|
-
docCoverage: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
|
|
94
|
-
referenceSolution: z.ZodOptional<z.ZodString>;
|
|
95
|
-
execution: z.ZodOptional<z.ZodObject<{
|
|
96
|
-
enabled: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
|
|
97
|
-
blocking: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
|
|
98
|
-
threshold: z.ZodOptional<z.ZodObject<{
|
|
99
|
-
score: z.ZodOptional<z.ZodNumber>;
|
|
100
|
-
}, z.core.$strip>>;
|
|
101
|
-
trigger: z.ZodOptional<z.ZodObject<{
|
|
102
|
-
branches: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
103
|
-
paths: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
104
|
-
}, z.core.$strip>>;
|
|
105
|
-
source: z.ZodOptional<z.ZodString>;
|
|
106
|
-
}, z.core.$strip>>;
|
|
107
|
-
}, z.core.$strip>;
|
|
108
|
-
export type RepoTask = z.infer<typeof RepoTaskSchema>;
|
|
109
|
-
/**
|
|
110
|
-
* Schema for an array of repo tasks — what a single .ailf/tasks/*.yaml file
|
|
111
|
-
* contains. Each file must define at least one task.
|
|
23
|
+
* NOTE: This accepts pre-parsed YAML data (unknown), not a raw string.
|
|
24
|
+
* For raw YAML strings, use `parseTaskFile()` from @sanity/ailf-tasks.
|
|
112
25
|
*/
|
|
113
|
-
export declare
|
|
114
|
-
id: z.ZodString;
|
|
115
|
-
description: z.ZodString;
|
|
116
|
-
featureArea: z.ZodString;
|
|
117
|
-
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
118
|
-
canonicalDocs: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
|
|
119
|
-
id: z.ZodString;
|
|
120
|
-
reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
121
|
-
slug: z.ZodOptional<z.ZodString>;
|
|
122
|
-
path: z.ZodOptional<z.ZodString>;
|
|
123
|
-
}, z.core.$strip>, z.ZodObject<{
|
|
124
|
-
slug: z.ZodString;
|
|
125
|
-
reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
126
|
-
}, z.core.$strip>, z.ZodObject<{
|
|
127
|
-
path: z.ZodString;
|
|
128
|
-
reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
129
|
-
}, z.core.$strip>, z.ZodObject<{
|
|
130
|
-
perspective: z.ZodString;
|
|
131
|
-
reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
132
|
-
}, z.core.$strip>]>>>>;
|
|
133
|
-
vars: z.ZodOptional<z.ZodObject<{
|
|
134
|
-
task: z.ZodString;
|
|
135
|
-
}, z.core.$loose>>;
|
|
136
|
-
assert: z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
|
|
137
|
-
type: z.ZodLiteral<"llm-rubric">;
|
|
138
|
-
template: z.ZodEnum<{
|
|
139
|
-
"task-completion": "task-completion";
|
|
140
|
-
"code-correctness": "code-correctness";
|
|
141
|
-
"doc-coverage": "doc-coverage";
|
|
142
|
-
}>;
|
|
143
|
-
criteria: z.ZodArray<z.ZodString>;
|
|
144
|
-
weight: z.ZodOptional<z.ZodNumber>;
|
|
145
|
-
}, z.core.$strip>, z.ZodObject<{
|
|
146
|
-
type: z.ZodEnum<{
|
|
147
|
-
"llm-rubric": "llm-rubric";
|
|
148
|
-
contains: "contains";
|
|
149
|
-
"contains-any": "contains-any";
|
|
150
|
-
"contains-all": "contains-all";
|
|
151
|
-
"not-contains": "not-contains";
|
|
152
|
-
icontains: "icontains";
|
|
153
|
-
"icontains-any": "icontains-any";
|
|
154
|
-
regex: "regex";
|
|
155
|
-
javascript: "javascript";
|
|
156
|
-
similar: "similar";
|
|
157
|
-
cost: "cost";
|
|
158
|
-
latency: "latency";
|
|
159
|
-
}>;
|
|
160
|
-
value: z.ZodOptional<z.ZodUnknown>;
|
|
161
|
-
threshold: z.ZodOptional<z.ZodNumber>;
|
|
162
|
-
weight: z.ZodOptional<z.ZodNumber>;
|
|
163
|
-
}, z.core.$loose>]>>;
|
|
164
|
-
baseline: z.ZodOptional<z.ZodObject<{
|
|
165
|
-
enabled: z.ZodOptional<z.ZodBoolean>;
|
|
166
|
-
rubric: z.ZodOptional<z.ZodEnum<{
|
|
167
|
-
full: "full";
|
|
168
|
-
abbreviated: "abbreviated";
|
|
169
|
-
none: "none";
|
|
170
|
-
}>>;
|
|
171
|
-
}, z.core.$strip>>;
|
|
172
|
-
docCoverage: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
|
|
173
|
-
referenceSolution: z.ZodOptional<z.ZodString>;
|
|
174
|
-
execution: z.ZodOptional<z.ZodObject<{
|
|
175
|
-
enabled: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
|
|
176
|
-
blocking: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
|
|
177
|
-
threshold: z.ZodOptional<z.ZodObject<{
|
|
178
|
-
score: z.ZodOptional<z.ZodNumber>;
|
|
179
|
-
}, z.core.$strip>>;
|
|
180
|
-
trigger: z.ZodOptional<z.ZodObject<{
|
|
181
|
-
branches: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
182
|
-
paths: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
183
|
-
}, z.core.$strip>>;
|
|
184
|
-
source: z.ZodOptional<z.ZodString>;
|
|
185
|
-
}, z.core.$strip>>;
|
|
186
|
-
}, z.core.$strip>>;
|
|
26
|
+
export declare function parseRepoTaskFile(raw: unknown, filename: string): z.infer<typeof _Schema>;
|
|
187
27
|
/**
|
|
188
28
|
* Zod schema for .ailf/config.yaml — controls documentation source,
|
|
189
29
|
* report destination, and trigger behavior for evaluations from an
|
|
@@ -244,11 +84,6 @@ export declare const RepoConfigSchema: z.ZodObject<{
|
|
|
244
84
|
}, z.core.$strip>>;
|
|
245
85
|
}, z.core.$strip>;
|
|
246
86
|
export type RepoConfig = z.infer<typeof RepoConfigSchema>;
|
|
247
|
-
/**
|
|
248
|
-
* Parse and validate a repo task file's content. Returns typed tasks or throws
|
|
249
|
-
* with a user-friendly Zod error message.
|
|
250
|
-
*/
|
|
251
|
-
export declare function parseRepoTaskFile(raw: unknown, filename: string): RepoTask[];
|
|
252
87
|
/**
|
|
253
88
|
* Parse and validate .ailf/config.yaml content. Returns typed config or throws.
|
|
254
89
|
*/
|
|
@@ -1,181 +1,43 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* repo-schemas.ts —
|
|
2
|
+
* repo-schemas.ts — Re-exports task schemas + defines config schemas.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
4
|
+
* Task schemas (RepoTaskSchema, assertions, etc.) are the single source
|
|
5
|
+
* of truth in @sanity/ailf-tasks. This file re-exports them so existing
|
|
6
|
+
* importers within the eval package don't need to change their paths.
|
|
6
7
|
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
8
|
+
* Config schemas (RepoConfigSchema, trigger config, etc.) remain here
|
|
9
|
+
* because they are specific to the eval pipeline and not needed by
|
|
10
|
+
* external tools that only validate task YAML.
|
|
10
11
|
*
|
|
12
|
+
* @see packages/tasks/src/schemas.ts — task schema source of truth
|
|
11
13
|
* @see docs/exec-plans/completed/tasks-as-content/phase-4-repo-based-tasks.md
|
|
12
|
-
* @see packages/eval/src/adapters/task-sources/repo-task-source.ts
|
|
13
14
|
*/
|
|
15
|
+
import { RepoTaskFileSchema as _Schema } from "../../_vendor/ailf-tasks/index.js";
|
|
14
16
|
import { z } from "zod";
|
|
15
17
|
// ---------------------------------------------------------------------------
|
|
16
|
-
//
|
|
18
|
+
// Re-exports from @sanity/ailf-tasks (task schemas + validation)
|
|
17
19
|
// ---------------------------------------------------------------------------
|
|
20
|
+
export { CURATED_ASSERTION_TYPES, RepoTaskFileSchema, RepoTaskSchema, RUBRIC_TEMPLATE_NAMES, } from "../../_vendor/ailf-tasks/index.js";
|
|
21
|
+
export { loadTaskDir, parseTaskFile } from "../../_vendor/ailf-tasks/index.js";
|
|
18
22
|
/**
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
* This is a curated subset of Promptfoo assertion types — we expose only the
|
|
22
|
-
* types that are stable, well-documented, and useful for external authors.
|
|
23
|
-
*/
|
|
24
|
-
export const CURATED_ASSERTION_TYPES = [
|
|
25
|
-
"llm-rubric",
|
|
26
|
-
"contains",
|
|
27
|
-
"contains-any",
|
|
28
|
-
"contains-all",
|
|
29
|
-
"not-contains",
|
|
30
|
-
"icontains",
|
|
31
|
-
"icontains-any",
|
|
32
|
-
"regex",
|
|
33
|
-
"javascript",
|
|
34
|
-
"similar",
|
|
35
|
-
"cost",
|
|
36
|
-
"latency",
|
|
37
|
-
];
|
|
38
|
-
/**
|
|
39
|
-
* Valid rubric template names — must match keys in config/rubrics.yaml.
|
|
40
|
-
*/
|
|
41
|
-
export const RUBRIC_TEMPLATE_NAMES = [
|
|
42
|
-
"task-completion",
|
|
43
|
-
"code-correctness",
|
|
44
|
-
"doc-coverage",
|
|
45
|
-
];
|
|
46
|
-
// ---------------------------------------------------------------------------
|
|
47
|
-
// Assertion schemas
|
|
48
|
-
// ---------------------------------------------------------------------------
|
|
49
|
-
/**
|
|
50
|
-
* Polymorphic canonical doc reference — discriminated by key presence.
|
|
51
|
-
* Exactly one resolution key (slug, path, id, or perspective) must be present.
|
|
52
|
-
*
|
|
53
|
-
* @see docs/design-docs/canonical-doc-resolution.md
|
|
54
|
-
*/
|
|
55
|
-
const SlugDocRefSchema = z.object({
|
|
56
|
-
slug: z.string().min(1),
|
|
57
|
-
reason: z.string().optional().default(""),
|
|
58
|
-
});
|
|
59
|
-
const PathDocRefSchema = z.object({
|
|
60
|
-
path: z.string().min(1),
|
|
61
|
-
reason: z.string().optional().default(""),
|
|
62
|
-
});
|
|
63
|
-
const IdDocRefSchema = z.object({
|
|
64
|
-
id: z.string().min(1),
|
|
65
|
-
reason: z.string().optional().default(""),
|
|
66
|
-
/** Human-readable slug annotation (not used for resolution) */
|
|
67
|
-
slug: z.string().optional(),
|
|
68
|
-
/** Human-readable path annotation (not used for resolution) */
|
|
69
|
-
path: z.string().optional(),
|
|
70
|
-
});
|
|
71
|
-
const PerspectiveDocRefSchema = z.object({
|
|
72
|
-
perspective: z.string().min(1),
|
|
73
|
-
reason: z.string().optional().default(""),
|
|
74
|
-
});
|
|
75
|
-
// Order matters: IdDocRefSchema first because it may also carry `slug`
|
|
76
|
-
// and `path` as optional annotations. Zod tries schemas in order, so
|
|
77
|
-
// entries like `{ id: "...", slug: "..." }` must match IdDocRefSchema
|
|
78
|
-
// (not SlugDocRefSchema).
|
|
79
|
-
const CanonicalDocRefSchema = z.union([
|
|
80
|
-
IdDocRefSchema,
|
|
81
|
-
SlugDocRefSchema,
|
|
82
|
-
PathDocRefSchema,
|
|
83
|
-
PerspectiveDocRefSchema,
|
|
84
|
-
]);
|
|
85
|
-
/**
|
|
86
|
-
* A templated LLM-rubric assertion — uses one of the predefined rubric
|
|
87
|
-
* templates with author-supplied criteria.
|
|
88
|
-
*/
|
|
89
|
-
const TemplatedAssertionSchema = z.object({
|
|
90
|
-
type: z.literal("llm-rubric"),
|
|
91
|
-
template: z.enum(RUBRIC_TEMPLATE_NAMES),
|
|
92
|
-
criteria: z.array(z.string().min(1)).min(1),
|
|
93
|
-
weight: z.number().optional(),
|
|
94
|
-
});
|
|
95
|
-
/**
|
|
96
|
-
* A value-based assertion (contains, regex, cost, etc.). Uses .passthrough()
|
|
97
|
-
* to allow extra fields for future extension without schema breakage.
|
|
98
|
-
*/
|
|
99
|
-
const ValueAssertionSchema = z
|
|
100
|
-
.object({
|
|
101
|
-
type: z.enum(CURATED_ASSERTION_TYPES),
|
|
102
|
-
value: z.unknown().optional(),
|
|
103
|
-
threshold: z.number().optional(),
|
|
104
|
-
weight: z.number().optional(),
|
|
105
|
-
})
|
|
106
|
-
.passthrough();
|
|
107
|
-
/** Union of all supported assertion shapes. */
|
|
108
|
-
const AssertionSchema = z.union([
|
|
109
|
-
TemplatedAssertionSchema,
|
|
110
|
-
ValueAssertionSchema,
|
|
111
|
-
]);
|
|
112
|
-
// ---------------------------------------------------------------------------
|
|
113
|
-
// Nested config schemas
|
|
114
|
-
// ---------------------------------------------------------------------------
|
|
115
|
-
const BaselineConfigSchema = z
|
|
116
|
-
.object({
|
|
117
|
-
enabled: z.boolean().optional(),
|
|
118
|
-
rubric: z.enum(["abbreviated", "full", "none"]).optional(),
|
|
119
|
-
})
|
|
120
|
-
.optional();
|
|
121
|
-
const ExecutionConfigSchema = z
|
|
122
|
-
.object({
|
|
123
|
-
enabled: z.boolean().optional().default(true),
|
|
124
|
-
blocking: z.boolean().optional().default(false),
|
|
125
|
-
threshold: z
|
|
126
|
-
.object({
|
|
127
|
-
score: z.number().min(0).max(100).optional(),
|
|
128
|
-
})
|
|
129
|
-
.optional(),
|
|
130
|
-
trigger: z
|
|
131
|
-
.object({
|
|
132
|
-
branches: z.array(z.string()).optional(),
|
|
133
|
-
paths: z.array(z.string()).optional(),
|
|
134
|
-
})
|
|
135
|
-
.optional(),
|
|
136
|
-
source: z.string().optional(),
|
|
137
|
-
})
|
|
138
|
-
.optional();
|
|
139
|
-
// ---------------------------------------------------------------------------
|
|
140
|
-
// RepoTaskSchema — a single task definition from .ailf/tasks/*.yaml
|
|
141
|
-
// ---------------------------------------------------------------------------
|
|
142
|
-
/**
|
|
143
|
-
* Zod schema for a single repo-based task definition.
|
|
23
|
+
* Parse and validate a repo task file's content. Returns typed tasks or throws
|
|
24
|
+
* with a user-friendly Zod error message.
|
|
144
25
|
*
|
|
145
|
-
* This
|
|
146
|
-
*
|
|
26
|
+
* NOTE: This accepts pre-parsed YAML data (unknown), not a raw string.
|
|
27
|
+
* For raw YAML strings, use `parseTaskFile()` from @sanity/ailf-tasks.
|
|
147
28
|
*/
|
|
148
|
-
export
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
.
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
tags: z.array(z.string()).optional(),
|
|
159
|
-
canonicalDocs: z.array(CanonicalDocRefSchema).optional().default([]),
|
|
160
|
-
vars: z
|
|
161
|
-
.object({
|
|
162
|
-
task: z.string().min(1),
|
|
163
|
-
})
|
|
164
|
-
.passthrough()
|
|
165
|
-
.optional(),
|
|
166
|
-
assert: z.array(AssertionSchema).min(1),
|
|
167
|
-
baseline: BaselineConfigSchema,
|
|
168
|
-
docCoverage: z.boolean().optional().default(false),
|
|
169
|
-
referenceSolution: z.string().optional(),
|
|
170
|
-
execution: ExecutionConfigSchema,
|
|
171
|
-
});
|
|
172
|
-
/**
|
|
173
|
-
* Schema for an array of repo tasks — what a single .ailf/tasks/*.yaml file
|
|
174
|
-
* contains. Each file must define at least one task.
|
|
175
|
-
*/
|
|
176
|
-
export const RepoTaskFileSchema = z.array(RepoTaskSchema).min(1);
|
|
29
|
+
export function parseRepoTaskFile(raw, filename) {
|
|
30
|
+
const result = _Schema.safeParse(raw);
|
|
31
|
+
if (!result.success) {
|
|
32
|
+
const messages = result.error.issues
|
|
33
|
+
.map((i) => ` [${i.path.join(".")}]: ${i.message}`)
|
|
34
|
+
.join("\n");
|
|
35
|
+
throw new Error(`Invalid repo task file "${filename}":\n${messages}`);
|
|
36
|
+
}
|
|
37
|
+
return result.data;
|
|
38
|
+
}
|
|
177
39
|
// ---------------------------------------------------------------------------
|
|
178
|
-
//
|
|
40
|
+
// Config schemas — specific to the eval pipeline
|
|
179
41
|
// ---------------------------------------------------------------------------
|
|
180
42
|
const TriggerModeSchema = z.enum(["validate-only", "eval"]);
|
|
181
43
|
const TriggerConfigSchema = z.object({
|
|
@@ -202,8 +64,6 @@ const SourceConfigSchema = z
|
|
|
202
64
|
/**
|
|
203
65
|
* Report store configuration.
|
|
204
66
|
* Defines which Sanity project receives `ailf.report` documents.
|
|
205
|
-
* This should match the project/dataset configured in the user's Studio.
|
|
206
|
-
* The API token comes from the AILF_REPORT_SANITY_API_TOKEN env var.
|
|
207
67
|
*/
|
|
208
68
|
const ReportStoreConfigSchema = z
|
|
209
69
|
.object({
|
|
@@ -228,23 +88,6 @@ export const RepoConfigSchema = z.object({
|
|
|
228
88
|
})
|
|
229
89
|
.optional(),
|
|
230
90
|
});
|
|
231
|
-
// ---------------------------------------------------------------------------
|
|
232
|
-
// Validation helpers
|
|
233
|
-
// ---------------------------------------------------------------------------
|
|
234
|
-
/**
|
|
235
|
-
* Parse and validate a repo task file's content. Returns typed tasks or throws
|
|
236
|
-
* with a user-friendly Zod error message.
|
|
237
|
-
*/
|
|
238
|
-
export function parseRepoTaskFile(raw, filename) {
|
|
239
|
-
const result = RepoTaskFileSchema.safeParse(raw);
|
|
240
|
-
if (!result.success) {
|
|
241
|
-
const messages = result.error.issues
|
|
242
|
-
.map((i) => ` [${i.path.join(".")}]: ${i.message}`)
|
|
243
|
-
.join("\n");
|
|
244
|
-
throw new Error(`Invalid repo task file "${filename}":\n${messages}`);
|
|
245
|
-
}
|
|
246
|
-
return result.data;
|
|
247
|
-
}
|
|
248
91
|
/**
|
|
249
92
|
* Parse and validate .ailf/config.yaml content. Returns typed config or throws.
|
|
250
93
|
*/
|
|
@@ -1,49 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* repo-validation.ts — Re-exports semantic validation from @sanity/ailf-tasks.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* -
|
|
6
|
-
*
|
|
7
|
-
* - Feature area strings are well-formed
|
|
8
|
-
* - Canonical doc slugs look reasonable (slugs, not URLs)
|
|
9
|
-
*
|
|
10
|
-
* These produce warnings, not errors — the pipeline can still run
|
|
11
|
-
* with imperfect tasks. Only structural failures (caught by Zod) block.
|
|
12
|
-
*
|
|
13
|
-
* @see packages/eval/src/adapters/task-sources/repo-schemas.ts
|
|
14
|
-
*/
|
|
15
|
-
import { type RepoTask } from "./repo-schemas.js";
|
|
16
|
-
export interface ValidationResult {
|
|
17
|
-
valid: boolean;
|
|
18
|
-
errors: ValidationMessage[];
|
|
19
|
-
warnings: ValidationMessage[];
|
|
20
|
-
}
|
|
21
|
-
export interface ValidationMessage {
|
|
22
|
-
taskId: string;
|
|
23
|
-
field: string;
|
|
24
|
-
message: string;
|
|
25
|
-
}
|
|
26
|
-
/**
|
|
27
|
-
* Run semantic validation on an array of parsed repo tasks.
|
|
28
|
-
*
|
|
29
|
-
* Returns warnings for issues that don't block execution (unknown feature
|
|
30
|
-
* areas, unresolved slugs) and errors for issues that would cause pipeline
|
|
31
|
-
* failures (completely missing required fields — though Zod catches most).
|
|
32
|
-
*/
|
|
33
|
-
export declare function validateRepoTasks(tasks: RepoTask[]): ValidationResult;
|
|
34
|
-
/**
|
|
35
|
-
* Format validation results for console output.
|
|
36
|
-
*/
|
|
37
|
-
export declare function formatValidationResult(result: ValidationResult): string;
|
|
38
|
-
/**
|
|
39
|
-
* Detect snake_case field names in raw task YAML data.
|
|
40
|
-
*
|
|
41
|
-
* This runs BEFORE Zod parsing to provide a user-friendly error message
|
|
42
|
-
* when authors use framework-internal snake_case names instead of the
|
|
43
|
-
* camelCase names expected in repo task files.
|
|
44
|
-
*
|
|
45
|
-
* @param raw - Raw parsed YAML (before Zod validation)
|
|
46
|
-
* @param filename - Source filename for error messages
|
|
47
|
-
* @returns Array of warning messages (empty if no issues)
|
|
4
|
+
* The validation logic is the single source of truth in @sanity/ailf-tasks.
|
|
5
|
+
* This file re-exports so existing eval-package importers don't need
|
|
6
|
+
* to change their import paths.
|
|
48
7
|
*/
|
|
49
|
-
export
|
|
8
|
+
export { detectSnakeCaseFields, formatValidationResult, validateRepoTasks, type ValidationMessage, type ValidationResult, } from "../../_vendor/ailf-tasks/index.d.ts";
|