@sanity/ailf 4.0.7 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/ailf.js +6 -1
- package/dist/_vendor/ailf-core/schemas/external-providers.d.ts +136 -0
- package/dist/_vendor/ailf-core/schemas/external-providers.js +136 -0
- package/dist/_vendor/ailf-core/schemas/index.d.ts +2 -0
- package/dist/_vendor/ailf-core/schemas/index.js +2 -0
- package/dist/_vendor/ailf-core/schemas/pipeline-request.d.ts +2 -3
- package/dist/_vendor/ailf-core/schemas/report.d.ts +251 -0
- package/dist/_vendor/ailf-core/schemas/report.js +235 -0
- package/dist/_vendor/ailf-core/services/index.d.ts +1 -0
- package/dist/_vendor/ailf-core/services/index.js +1 -0
- package/dist/_vendor/ailf-core/services/report-to-markdown.d.ts +38 -0
- package/dist/_vendor/ailf-core/services/report-to-markdown.js +696 -0
- package/dist/_vendor/ailf-core/types/api-requests.d.ts +159 -0
- package/dist/_vendor/ailf-core/types/api-requests.js +27 -0
- package/dist/_vendor/ailf-core/types/index.d.ts +3 -0
- package/dist/_vendor/ailf-core/types/pipeline-request.d.ts +112 -0
- package/dist/_vendor/ailf-core/types/pipeline-request.js +18 -0
- package/dist/_vendor/ailf-core/types/repo-config.d.ts +146 -0
- package/dist/_vendor/ailf-core/types/repo-config.js +18 -0
- package/dist/_vendor/ailf-shared/index.d.ts +7 -5
- package/dist/_vendor/ailf-shared/index.js +7 -5
- package/dist/adapters/api-client/types.d.ts +2 -5
- package/dist/adapters/task-sources/content-lake-task-source.d.ts +58 -1
- package/dist/adapters/task-sources/content-lake-task-source.js +1 -1
- package/dist/adapters/task-sources/index.d.ts +1 -1
- package/dist/adapters/task-sources/index.js +1 -1
- package/dist/adapters/task-sources/repo-schemas.d.ts +3 -2
- package/dist/adapters/task-sources/repo-schemas.js +3 -1
- package/dist/adapters/task-sources/repo-validation.d.ts +6 -6
- package/dist/adapters/task-sources/repo-validation.js +1 -1
- package/dist/agent-observer/agentic-provider.d.ts +1 -0
- package/dist/agent-observer/agentic-provider.js +43 -36
- package/dist/agent-observer/config-schemas.d.ts +61 -0
- package/dist/agent-observer/config-schemas.js +65 -0
- package/dist/agent-observer/provider.d.ts +1 -0
- package/dist/agent-observer/provider.js +19 -17
- package/dist/cli.js +4 -4
- package/dist/commands/validate-tasks.js +2 -2
- package/dist/composition-root.js +4 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/job-store.js +2 -2
- package/dist/lib/dotenv-resolution.d.ts +21 -0
- package/dist/lib/dotenv-resolution.js +30 -0
- package/dist/orchestration/steps/mirror-repo-tasks-step.js +14 -3
- package/dist/orchestration/steps/run-eval-step.js +21 -3
- package/dist/pipeline/agent-behavior-report.d.ts +2 -8
- package/dist/pipeline/cache.d.ts +2 -2
- package/dist/pipeline/checks.d.ts +10 -2
- package/dist/pipeline/checks.js +14 -4
- package/dist/pipeline/compiler/literacy-bridge.js +2 -2
- package/dist/pipeline/compiler/mode-handlers/agent-harness/types.d.ts +2 -2
- package/dist/pipeline/compiler/mode-handlers/index.d.ts +1 -1
- package/dist/pipeline/compiler/mode-handlers/knowledge-probe/types.d.ts +2 -2
- package/dist/pipeline/compiler/mode-handlers/literacy/index.d.ts +1 -1
- package/dist/pipeline/compiler/mode-handlers/literacy/types.d.ts +3 -3
- package/dist/pipeline/compiler/promptfoo-compiler.js +7 -11
- package/dist/pipeline/compiler/provider-assembler.js +33 -3
- package/dist/pipeline/compiler/rubric-resolution.d.ts +2 -2
- package/dist/pipeline/mirror-repo-tasks.d.ts +13 -5
- package/dist/pipeline/mirror-repo-tasks.js +16 -8
- package/dist/pipeline/pr-comment.d.ts +22 -9
- package/dist/pipeline/pr-comment.js +52 -472
- package/dist/pipeline/resolve-mappings.d.ts +8 -3
- package/dist/promptfoo-providers/mock-path.d.ts +12 -0
- package/dist/promptfoo-providers/mock-path.js +15 -0
- package/dist/report-store.d.ts +63 -1
- package/dist/report-store.js +111 -31
- package/dist/sanity/client.d.ts +58 -0
- package/dist/sanity/client.js +106 -0
- package/package.json +8 -7
package/bin/ailf.js
CHANGED
|
@@ -28,10 +28,15 @@ const args = process.argv.slice(2)
|
|
|
28
28
|
// operate on the user's cwd, not the eval package root.
|
|
29
29
|
const callerCwd = process.cwd()
|
|
30
30
|
|
|
31
|
+
// Test harnesses (and any caller that knows the dist is fresh) can opt out
|
|
32
|
+
// of the tsx dev path with AILF_PREFER_DIST=1. This avoids the npx + tsx
|
|
33
|
+
// cold-start cost (~1–2s per invocation) on every `spawnCli(...)` call.
|
|
34
|
+
const preferDist = process.env.AILF_PREFER_DIST === "1" && existsSync(jsDist)
|
|
35
|
+
|
|
31
36
|
// ---------------------------------------------------------------------------
|
|
32
37
|
// Mode 1: Development — tsx + TypeScript source (no build step needed)
|
|
33
38
|
// ---------------------------------------------------------------------------
|
|
34
|
-
if (existsSync(tsSrc)) {
|
|
39
|
+
if (existsSync(tsSrc) && !preferDist) {
|
|
35
40
|
try {
|
|
36
41
|
// Enable the `ailf-source` export condition so @sanity/ailf-shared and
|
|
37
42
|
// @sanity/ailf-core resolve to their `src/index.ts` entrypoints rather
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @sanity/ailf-core — External provider response schemas
|
|
3
|
+
*
|
|
4
|
+
* Zod schemas for HTTP responses from third-party AI providers and search
|
|
5
|
+
* APIs that the agent-observer providers parse. Replaces inline `as`
|
|
6
|
+
* casts on `await response.json()` with parse-don't-validate boundaries.
|
|
7
|
+
*
|
|
8
|
+
* Schemas are intentionally permissive on optional fields (real APIs add
|
|
9
|
+
* fields over time) but strict on shape — a malformed payload surfaces
|
|
10
|
+
* as a clear ZodError at the call site instead of a cryptic
|
|
11
|
+
* `Cannot read properties of undefined` later.
|
|
12
|
+
*
|
|
13
|
+
* @see docs/design-docs/parse-dont-validate.md
|
|
14
|
+
* @see docs/work-items/W0004.json
|
|
15
|
+
*/
|
|
16
|
+
import { z } from "zod";
|
|
17
|
+
export declare const OpenAIChatResponseSchema: z.ZodObject<{
|
|
18
|
+
choices: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
19
|
+
finish_reason: z.ZodOptional<z.ZodString>;
|
|
20
|
+
message: z.ZodOptional<z.ZodObject<{
|
|
21
|
+
content: z.ZodUnion<readonly [z.ZodString, z.ZodNull]>;
|
|
22
|
+
role: z.ZodEnum<{
|
|
23
|
+
user: "user";
|
|
24
|
+
assistant: "assistant";
|
|
25
|
+
system: "system";
|
|
26
|
+
tool: "tool";
|
|
27
|
+
}>;
|
|
28
|
+
tool_call_id: z.ZodOptional<z.ZodString>;
|
|
29
|
+
tool_calls: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
30
|
+
function: z.ZodObject<{
|
|
31
|
+
arguments: z.ZodString;
|
|
32
|
+
name: z.ZodString;
|
|
33
|
+
}, z.core.$strip>;
|
|
34
|
+
id: z.ZodString;
|
|
35
|
+
type: z.ZodLiteral<"function">;
|
|
36
|
+
}, z.core.$strip>>>;
|
|
37
|
+
}, z.core.$strip>>;
|
|
38
|
+
}, z.core.$strip>>>;
|
|
39
|
+
error: z.ZodOptional<z.ZodObject<{
|
|
40
|
+
message: z.ZodOptional<z.ZodString>;
|
|
41
|
+
}, z.core.$strip>>;
|
|
42
|
+
usage: z.ZodOptional<z.ZodObject<{
|
|
43
|
+
completion_tokens: z.ZodOptional<z.ZodNumber>;
|
|
44
|
+
prompt_tokens: z.ZodOptional<z.ZodNumber>;
|
|
45
|
+
total_tokens: z.ZodOptional<z.ZodNumber>;
|
|
46
|
+
}, z.core.$strip>>;
|
|
47
|
+
}, z.core.$strip>;
|
|
48
|
+
export type OpenAIChatResponse = z.infer<typeof OpenAIChatResponseSchema>;
|
|
49
|
+
export declare const OpenAIResponsesResponseSchema: z.ZodObject<{
|
|
50
|
+
error: z.ZodOptional<z.ZodObject<{
|
|
51
|
+
message: z.ZodOptional<z.ZodString>;
|
|
52
|
+
}, z.core.$strip>>;
|
|
53
|
+
output: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
54
|
+
content: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
55
|
+
text: z.ZodOptional<z.ZodString>;
|
|
56
|
+
type: z.ZodOptional<z.ZodString>;
|
|
57
|
+
}, z.core.$strip>>>;
|
|
58
|
+
role: z.ZodOptional<z.ZodString>;
|
|
59
|
+
type: z.ZodOptional<z.ZodString>;
|
|
60
|
+
}, z.core.$strip>>>;
|
|
61
|
+
usage: z.ZodOptional<z.ZodObject<{
|
|
62
|
+
input_tokens: z.ZodOptional<z.ZodNumber>;
|
|
63
|
+
output_tokens: z.ZodOptional<z.ZodNumber>;
|
|
64
|
+
total_tokens: z.ZodOptional<z.ZodNumber>;
|
|
65
|
+
}, z.core.$strip>>;
|
|
66
|
+
}, z.core.$strip>;
|
|
67
|
+
export type OpenAIResponsesResponse = z.infer<typeof OpenAIResponsesResponseSchema>;
|
|
68
|
+
declare const AnthropicTextBlockSchema: z.ZodObject<{
|
|
69
|
+
text: z.ZodString;
|
|
70
|
+
type: z.ZodLiteral<"text">;
|
|
71
|
+
}, z.core.$strip>;
|
|
72
|
+
declare const AnthropicToolUseBlockSchema: z.ZodObject<{
|
|
73
|
+
id: z.ZodString;
|
|
74
|
+
input: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
75
|
+
name: z.ZodString;
|
|
76
|
+
type: z.ZodLiteral<"tool_use">;
|
|
77
|
+
}, z.core.$strip>;
|
|
78
|
+
declare const AnthropicContentBlockSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
79
|
+
text: z.ZodString;
|
|
80
|
+
type: z.ZodLiteral<"text">;
|
|
81
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
82
|
+
id: z.ZodString;
|
|
83
|
+
input: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
84
|
+
name: z.ZodString;
|
|
85
|
+
type: z.ZodLiteral<"tool_use">;
|
|
86
|
+
}, z.core.$strip>], "type">;
|
|
87
|
+
export declare const AnthropicResponseSchema: z.ZodObject<{
|
|
88
|
+
content: z.ZodOptional<z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
89
|
+
text: z.ZodString;
|
|
90
|
+
type: z.ZodLiteral<"text">;
|
|
91
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
92
|
+
id: z.ZodString;
|
|
93
|
+
input: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
94
|
+
name: z.ZodString;
|
|
95
|
+
type: z.ZodLiteral<"tool_use">;
|
|
96
|
+
}, z.core.$strip>], "type">>>;
|
|
97
|
+
error: z.ZodOptional<z.ZodObject<{
|
|
98
|
+
message: z.ZodOptional<z.ZodString>;
|
|
99
|
+
type: z.ZodOptional<z.ZodString>;
|
|
100
|
+
}, z.core.$strip>>;
|
|
101
|
+
model: z.ZodOptional<z.ZodString>;
|
|
102
|
+
stop_reason: z.ZodOptional<z.ZodEnum<{
|
|
103
|
+
tool_use: "tool_use";
|
|
104
|
+
end_turn: "end_turn";
|
|
105
|
+
max_tokens: "max_tokens";
|
|
106
|
+
}>>;
|
|
107
|
+
usage: z.ZodOptional<z.ZodObject<{
|
|
108
|
+
input_tokens: z.ZodOptional<z.ZodNumber>;
|
|
109
|
+
output_tokens: z.ZodOptional<z.ZodNumber>;
|
|
110
|
+
}, z.core.$strip>>;
|
|
111
|
+
}, z.core.$strip>;
|
|
112
|
+
export type AnthropicResponse = z.infer<typeof AnthropicResponseSchema>;
|
|
113
|
+
export type AnthropicTextBlock = z.infer<typeof AnthropicTextBlockSchema>;
|
|
114
|
+
export type AnthropicToolUseBlock = z.infer<typeof AnthropicToolUseBlockSchema>;
|
|
115
|
+
export type AnthropicContentBlock = z.infer<typeof AnthropicContentBlockSchema>;
|
|
116
|
+
export declare const GoogleSearchResponseSchema: z.ZodObject<{
|
|
117
|
+
items: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
118
|
+
link: z.ZodString;
|
|
119
|
+
snippet: z.ZodString;
|
|
120
|
+
title: z.ZodString;
|
|
121
|
+
}, z.core.$strip>>>;
|
|
122
|
+
}, z.core.$strip>;
|
|
123
|
+
export type GoogleSearchResponse = z.infer<typeof GoogleSearchResponseSchema>;
|
|
124
|
+
export declare const FetchPageToolArgsSchema: z.ZodObject<{
|
|
125
|
+
url: z.ZodString;
|
|
126
|
+
}, z.core.$strip>;
|
|
127
|
+
export declare const ListDocsToolArgsSchema: z.ZodObject<{
|
|
128
|
+
site: z.ZodString;
|
|
129
|
+
}, z.core.$strip>;
|
|
130
|
+
export declare const WebSearchToolArgsSchema: z.ZodObject<{
|
|
131
|
+
query: z.ZodString;
|
|
132
|
+
}, z.core.$strip>;
|
|
133
|
+
export type FetchPageToolArgs = z.infer<typeof FetchPageToolArgsSchema>;
|
|
134
|
+
export type ListDocsToolArgs = z.infer<typeof ListDocsToolArgsSchema>;
|
|
135
|
+
export type WebSearchToolArgs = z.infer<typeof WebSearchToolArgsSchema>;
|
|
136
|
+
export {};
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @sanity/ailf-core — External provider response schemas
|
|
3
|
+
*
|
|
4
|
+
* Zod schemas for HTTP responses from third-party AI providers and search
|
|
5
|
+
* APIs that the agent-observer providers parse. Replaces inline `as`
|
|
6
|
+
* casts on `await response.json()` with parse-don't-validate boundaries.
|
|
7
|
+
*
|
|
8
|
+
* Schemas are intentionally permissive on optional fields (real APIs add
|
|
9
|
+
* fields over time) but strict on shape — a malformed payload surfaces
|
|
10
|
+
* as a clear ZodError at the call site instead of a cryptic
|
|
11
|
+
* `Cannot read properties of undefined` later.
|
|
12
|
+
*
|
|
13
|
+
* @see docs/design-docs/parse-dont-validate.md
|
|
14
|
+
* @see docs/work-items/W0004.json
|
|
15
|
+
*/
|
|
16
|
+
import { z } from "zod";
|
|
17
|
+
// ---------------------------------------------------------------------------
|
|
18
|
+
// OpenAI Chat Completions response
|
|
19
|
+
//
|
|
20
|
+
// Covers both the minimal shape used by the InstrumentedObserverProvider
|
|
21
|
+
// (`provider.ts`) and the richer tool-calling shape used by
|
|
22
|
+
// `AgenticObserverProvider` (`agentic-provider.ts`).
|
|
23
|
+
// ---------------------------------------------------------------------------
|
|
24
|
+
const OpenAIToolCallSchema = z.object({
|
|
25
|
+
function: z.object({
|
|
26
|
+
arguments: z.string(),
|
|
27
|
+
name: z.string(),
|
|
28
|
+
}),
|
|
29
|
+
id: z.string(),
|
|
30
|
+
type: z.literal("function"),
|
|
31
|
+
});
|
|
32
|
+
const OpenAIChatMessageSchema = z.object({
|
|
33
|
+
content: z.union([z.string(), z.null()]),
|
|
34
|
+
role: z.enum(["assistant", "system", "tool", "user"]),
|
|
35
|
+
tool_call_id: z.string().optional(),
|
|
36
|
+
tool_calls: z.array(OpenAIToolCallSchema).optional(),
|
|
37
|
+
});
|
|
38
|
+
const OpenAIUsageSchema = z.object({
|
|
39
|
+
completion_tokens: z.number().optional(),
|
|
40
|
+
prompt_tokens: z.number().optional(),
|
|
41
|
+
total_tokens: z.number().optional(),
|
|
42
|
+
});
|
|
43
|
+
const OpenAIErrorSchema = z.object({
|
|
44
|
+
message: z.string().optional(),
|
|
45
|
+
});
|
|
46
|
+
export const OpenAIChatResponseSchema = z.object({
|
|
47
|
+
choices: z
|
|
48
|
+
.array(z.object({
|
|
49
|
+
finish_reason: z.string().optional(),
|
|
50
|
+
message: OpenAIChatMessageSchema.optional(),
|
|
51
|
+
}))
|
|
52
|
+
.optional(),
|
|
53
|
+
error: OpenAIErrorSchema.optional(),
|
|
54
|
+
usage: OpenAIUsageSchema.optional(),
|
|
55
|
+
});
|
|
56
|
+
// ---------------------------------------------------------------------------
|
|
57
|
+
// OpenAI Responses API response (reasoning models)
|
|
58
|
+
// ---------------------------------------------------------------------------
|
|
59
|
+
const OpenAIResponsesUsageSchema = z.object({
|
|
60
|
+
input_tokens: z.number().optional(),
|
|
61
|
+
output_tokens: z.number().optional(),
|
|
62
|
+
total_tokens: z.number().optional(),
|
|
63
|
+
});
|
|
64
|
+
const OpenAIResponsesContentSchema = z.object({
|
|
65
|
+
text: z.string().optional(),
|
|
66
|
+
type: z.string().optional(),
|
|
67
|
+
});
|
|
68
|
+
const OpenAIResponsesOutputItemSchema = z.object({
|
|
69
|
+
content: z.array(OpenAIResponsesContentSchema).optional(),
|
|
70
|
+
role: z.string().optional(),
|
|
71
|
+
type: z.string().optional(),
|
|
72
|
+
});
|
|
73
|
+
export const OpenAIResponsesResponseSchema = z.object({
|
|
74
|
+
error: OpenAIErrorSchema.optional(),
|
|
75
|
+
output: z.array(OpenAIResponsesOutputItemSchema).optional(),
|
|
76
|
+
usage: OpenAIResponsesUsageSchema.optional(),
|
|
77
|
+
});
|
|
78
|
+
// ---------------------------------------------------------------------------
|
|
79
|
+
// Anthropic Messages API response
|
|
80
|
+
// ---------------------------------------------------------------------------
|
|
81
|
+
const AnthropicTextBlockSchema = z.object({
|
|
82
|
+
text: z.string(),
|
|
83
|
+
type: z.literal("text"),
|
|
84
|
+
});
|
|
85
|
+
const AnthropicToolUseBlockSchema = z.object({
|
|
86
|
+
id: z.string(),
|
|
87
|
+
input: z.record(z.string(), z.unknown()),
|
|
88
|
+
name: z.string(),
|
|
89
|
+
type: z.literal("tool_use"),
|
|
90
|
+
});
|
|
91
|
+
const AnthropicContentBlockSchema = z.discriminatedUnion("type", [
|
|
92
|
+
AnthropicTextBlockSchema,
|
|
93
|
+
AnthropicToolUseBlockSchema,
|
|
94
|
+
]);
|
|
95
|
+
const AnthropicUsageSchema = z.object({
|
|
96
|
+
input_tokens: z.number().optional(),
|
|
97
|
+
output_tokens: z.number().optional(),
|
|
98
|
+
});
|
|
99
|
+
const AnthropicErrorSchema = z.object({
|
|
100
|
+
message: z.string().optional(),
|
|
101
|
+
type: z.string().optional(),
|
|
102
|
+
});
|
|
103
|
+
export const AnthropicResponseSchema = z.object({
|
|
104
|
+
content: z.array(AnthropicContentBlockSchema).optional(),
|
|
105
|
+
error: AnthropicErrorSchema.optional(),
|
|
106
|
+
model: z.string().optional(),
|
|
107
|
+
stop_reason: z.enum(["end_turn", "max_tokens", "tool_use"]).optional(),
|
|
108
|
+
usage: AnthropicUsageSchema.optional(),
|
|
109
|
+
});
|
|
110
|
+
// ---------------------------------------------------------------------------
|
|
111
|
+
// Google Custom Search API response
|
|
112
|
+
// ---------------------------------------------------------------------------
|
|
113
|
+
export const GoogleSearchResponseSchema = z.object({
|
|
114
|
+
items: z
|
|
115
|
+
.array(z.object({
|
|
116
|
+
link: z.string(),
|
|
117
|
+
snippet: z.string(),
|
|
118
|
+
title: z.string(),
|
|
119
|
+
}))
|
|
120
|
+
.optional(),
|
|
121
|
+
});
|
|
122
|
+
// ---------------------------------------------------------------------------
|
|
123
|
+
// Agent tool-call arguments
|
|
124
|
+
//
|
|
125
|
+
// The model emits these as JSON strings via tool-calling; we parse them
|
|
126
|
+
// before dispatching. Per-tool schemas keep the contract explicit.
|
|
127
|
+
// ---------------------------------------------------------------------------
|
|
128
|
+
export const FetchPageToolArgsSchema = z.object({
|
|
129
|
+
url: z.string(),
|
|
130
|
+
});
|
|
131
|
+
export const ListDocsToolArgsSchema = z.object({
|
|
132
|
+
site: z.string(),
|
|
133
|
+
});
|
|
134
|
+
export const WebSearchToolArgsSchema = z.object({
|
|
135
|
+
query: z.string(),
|
|
136
|
+
});
|
|
@@ -11,8 +11,10 @@
|
|
|
11
11
|
export * from "./callback-payload.js";
|
|
12
12
|
export * from "./canary-tasks.js";
|
|
13
13
|
export * from "./eval-config.js";
|
|
14
|
+
export * from "./external-providers.js";
|
|
14
15
|
export * from "./pipeline-request.js";
|
|
15
16
|
export * from "./pipeline.js";
|
|
17
|
+
export * from "./report.js";
|
|
16
18
|
export * from "./schedules.js";
|
|
17
19
|
export * from "./sinks.js";
|
|
18
20
|
export * from "./test-budgets.js";
|
|
@@ -11,8 +11,10 @@
|
|
|
11
11
|
export * from "./callback-payload.js";
|
|
12
12
|
export * from "./canary-tasks.js";
|
|
13
13
|
export * from "./eval-config.js";
|
|
14
|
+
export * from "./external-providers.js";
|
|
14
15
|
export * from "./pipeline-request.js";
|
|
15
16
|
export * from "./pipeline.js";
|
|
17
|
+
export * from "./report.js";
|
|
16
18
|
export * from "./schedules.js";
|
|
17
19
|
export * from "./sinks.js";
|
|
18
20
|
export * from "./test-budgets.js";
|
|
@@ -16,8 +16,8 @@ import { z } from "zod";
|
|
|
16
16
|
export declare const PipelineRequestSchema: z.ZodObject<{
|
|
17
17
|
allowedOrigins: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
18
18
|
backend: z.ZodOptional<z.ZodEnum<{
|
|
19
|
-
"github-actions": "github-actions";
|
|
20
19
|
"cloud-run": "cloud-run";
|
|
20
|
+
"github-actions": "github-actions";
|
|
21
21
|
}>>;
|
|
22
22
|
areas: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
23
23
|
callerGit: z.ZodOptional<z.ZodObject<{
|
|
@@ -116,5 +116,4 @@ export declare const PipelineRequestSchema: z.ZodObject<{
|
|
|
116
116
|
purpose: z.ZodOptional<z.ZodString>;
|
|
117
117
|
labels: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
118
118
|
}, z.core.$strip>;
|
|
119
|
-
|
|
120
|
-
export type PipelineRequest = z.infer<typeof PipelineRequestSchema>;
|
|
119
|
+
export type { PipelineRequest } from "../types/pipeline-request.js";
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @sanity/ailf-core — Report schema (W0191)
|
|
3
|
+
*
|
|
4
|
+
* Runtime Zod gate for `ailf.report` documents at the ReportStore
|
|
5
|
+
* read/write boundary. Mirrors the W0073 pattern that
|
|
6
|
+
* `ContentLakeAuthorableTaskSchema` applies to `ailf.task` — turn silent
|
|
7
|
+
* shape drift into a loud parse failure at the Content Lake boundary.
|
|
8
|
+
*
|
|
9
|
+
* **Scope A** (this file): top-level shape is `passthrough()` so that
|
|
10
|
+
* adding a new top-level Report field on the TS side does not silently
|
|
11
|
+
* fail validation before the schema author updates this mirror.
|
|
12
|
+
* `provenance` (and nested `provenance.lineage`) are `strict()` because
|
|
13
|
+
* they are fully mirrored — unknown keys there signal real drift.
|
|
14
|
+
* `summary`, `comparison`, and `artifactManifest` are modeled as
|
|
15
|
+
* non-empty objects with passthrough; their deep contents
|
|
16
|
+
* (`ScoreSummary`, `FeatureScore`, the W0051 slim shapes, etc.) are
|
|
17
|
+
* intentionally out of scope and may grow into a Scope B follow-up.
|
|
18
|
+
*
|
|
19
|
+
* @see packages/eval/src/adapters/task-sources/content-lake-task-source.ts
|
|
20
|
+
* — sibling W0073 gate
|
|
21
|
+
* @see packages/core/src/types/index.ts — `Report`, `ReportProvenance`,
|
|
22
|
+
* `ReportLineage`
|
|
23
|
+
* @see packages/shared/src/run-context.ts — `RunContext`
|
|
24
|
+
* @see docs/work-items/W0191-report-store-schema-gate.json
|
|
25
|
+
*/
|
|
26
|
+
import { z } from "zod";
|
|
27
|
+
export declare const ReportLineageSchema: z.ZodObject<{
|
|
28
|
+
rerunOf: z.ZodOptional<z.ZodString>;
|
|
29
|
+
comparedAgainst: z.ZodOptional<z.ZodString>;
|
|
30
|
+
parentJobId: z.ZodOptional<z.ZodString>;
|
|
31
|
+
}, z.core.$strict>;
|
|
32
|
+
export declare const ReportProvenanceSchema: z.ZodObject<{
|
|
33
|
+
areas: z.ZodArray<z.ZodString>;
|
|
34
|
+
classification: z.ZodEnum<{
|
|
35
|
+
external: "external";
|
|
36
|
+
official: "official";
|
|
37
|
+
adhoc: "adhoc";
|
|
38
|
+
experimental: "experimental";
|
|
39
|
+
test: "test";
|
|
40
|
+
}>;
|
|
41
|
+
evalFingerprint: z.ZodOptional<z.ZodString>;
|
|
42
|
+
executor: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
43
|
+
type: z.ZodLiteral<"user">;
|
|
44
|
+
name: z.ZodOptional<z.ZodString>;
|
|
45
|
+
email: z.ZodOptional<z.ZodString>;
|
|
46
|
+
surface: z.ZodEnum<{
|
|
47
|
+
cli: "cli";
|
|
48
|
+
studio: "studio";
|
|
49
|
+
api: "api";
|
|
50
|
+
}>;
|
|
51
|
+
githubActor: z.ZodOptional<z.ZodString>;
|
|
52
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
53
|
+
type: z.ZodLiteral<"system">;
|
|
54
|
+
name: z.ZodString;
|
|
55
|
+
workflow: z.ZodOptional<z.ZodString>;
|
|
56
|
+
runId: z.ZodOptional<z.ZodString>;
|
|
57
|
+
}, z.core.$strict>], "type">;
|
|
58
|
+
git: z.ZodOptional<z.ZodObject<{
|
|
59
|
+
branch: z.ZodString;
|
|
60
|
+
prNumber: z.ZodOptional<z.ZodNumber>;
|
|
61
|
+
repo: z.ZodString;
|
|
62
|
+
sha: z.ZodString;
|
|
63
|
+
}, z.core.$strict>>;
|
|
64
|
+
graderModel: z.ZodString;
|
|
65
|
+
host: z.ZodOptional<z.ZodObject<{
|
|
66
|
+
platform: z.ZodString;
|
|
67
|
+
arch: z.ZodString;
|
|
68
|
+
ci: z.ZodOptional<z.ZodString>;
|
|
69
|
+
}, z.core.$strict>>;
|
|
70
|
+
labels: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
71
|
+
lineage: z.ZodOptional<z.ZodObject<{
|
|
72
|
+
rerunOf: z.ZodOptional<z.ZodString>;
|
|
73
|
+
comparedAgainst: z.ZodOptional<z.ZodString>;
|
|
74
|
+
parentJobId: z.ZodOptional<z.ZodString>;
|
|
75
|
+
}, z.core.$strict>>;
|
|
76
|
+
mode: z.ZodString;
|
|
77
|
+
models: z.ZodArray<z.ZodObject<{
|
|
78
|
+
id: z.ZodString;
|
|
79
|
+
label: z.ZodString;
|
|
80
|
+
}, z.core.$strict>>;
|
|
81
|
+
owner: z.ZodObject<{
|
|
82
|
+
team: z.ZodString;
|
|
83
|
+
individual: z.ZodOptional<z.ZodString>;
|
|
84
|
+
}, z.core.$strict>;
|
|
85
|
+
purpose: z.ZodOptional<z.ZodString>;
|
|
86
|
+
source: z.ZodObject<{
|
|
87
|
+
baseUrl: z.ZodString;
|
|
88
|
+
dataset: z.ZodOptional<z.ZodString>;
|
|
89
|
+
name: z.ZodString;
|
|
90
|
+
perspective: z.ZodOptional<z.ZodString>;
|
|
91
|
+
projectId: z.ZodOptional<z.ZodString>;
|
|
92
|
+
}, z.core.$strict>;
|
|
93
|
+
taskIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
94
|
+
tool: z.ZodOptional<z.ZodObject<{
|
|
95
|
+
ailfVersion: z.ZodString;
|
|
96
|
+
nodeVersion: z.ZodString;
|
|
97
|
+
}, z.core.$strict>>;
|
|
98
|
+
trigger: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
99
|
+
type: z.ZodLiteral<"ci">;
|
|
100
|
+
runId: z.ZodString;
|
|
101
|
+
workflow: z.ZodString;
|
|
102
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
103
|
+
type: z.ZodLiteral<"cross-repo">;
|
|
104
|
+
callerRef: z.ZodOptional<z.ZodString>;
|
|
105
|
+
callerRepo: z.ZodString;
|
|
106
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
107
|
+
type: z.ZodLiteral<"manual">;
|
|
108
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
109
|
+
type: z.ZodLiteral<"scheduled">;
|
|
110
|
+
schedule: z.ZodString;
|
|
111
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
112
|
+
type: z.ZodLiteral<"webhook">;
|
|
113
|
+
documentId: z.ZodOptional<z.ZodString>;
|
|
114
|
+
source: z.ZodString;
|
|
115
|
+
}, z.core.$strict>], "type">;
|
|
116
|
+
autoScope: z.ZodOptional<z.ZodObject<{
|
|
117
|
+
enabled: z.ZodBoolean;
|
|
118
|
+
affectedTaskIds: z.ZodArray<z.ZodString>;
|
|
119
|
+
skippedTaskIds: z.ZodArray<z.ZodString>;
|
|
120
|
+
perspective: z.ZodString;
|
|
121
|
+
impactSummary: z.ZodObject<{
|
|
122
|
+
added: z.ZodNumber;
|
|
123
|
+
modified: z.ZodNumber;
|
|
124
|
+
removed: z.ZodNumber;
|
|
125
|
+
}, z.core.$strict>;
|
|
126
|
+
}, z.core.$strict>>;
|
|
127
|
+
contextHash: z.ZodOptional<z.ZodString>;
|
|
128
|
+
promptfooUrl: z.ZodOptional<z.ZodString>;
|
|
129
|
+
promptfooUrls: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
130
|
+
mode: z.ZodString;
|
|
131
|
+
url: z.ZodString;
|
|
132
|
+
}, z.core.$strict>>>;
|
|
133
|
+
runId: z.ZodString;
|
|
134
|
+
targetDocuments: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
135
|
+
}, z.core.$strict>;
|
|
136
|
+
export declare const ReportSchema: z.ZodObject<{
|
|
137
|
+
id: z.ZodString;
|
|
138
|
+
completedAt: z.ZodISODateTime;
|
|
139
|
+
durationMs: z.ZodNumber;
|
|
140
|
+
summary: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
141
|
+
provenance: z.ZodObject<{
|
|
142
|
+
areas: z.ZodArray<z.ZodString>;
|
|
143
|
+
classification: z.ZodEnum<{
|
|
144
|
+
external: "external";
|
|
145
|
+
official: "official";
|
|
146
|
+
adhoc: "adhoc";
|
|
147
|
+
experimental: "experimental";
|
|
148
|
+
test: "test";
|
|
149
|
+
}>;
|
|
150
|
+
evalFingerprint: z.ZodOptional<z.ZodString>;
|
|
151
|
+
executor: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
152
|
+
type: z.ZodLiteral<"user">;
|
|
153
|
+
name: z.ZodOptional<z.ZodString>;
|
|
154
|
+
email: z.ZodOptional<z.ZodString>;
|
|
155
|
+
surface: z.ZodEnum<{
|
|
156
|
+
cli: "cli";
|
|
157
|
+
studio: "studio";
|
|
158
|
+
api: "api";
|
|
159
|
+
}>;
|
|
160
|
+
githubActor: z.ZodOptional<z.ZodString>;
|
|
161
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
162
|
+
type: z.ZodLiteral<"system">;
|
|
163
|
+
name: z.ZodString;
|
|
164
|
+
workflow: z.ZodOptional<z.ZodString>;
|
|
165
|
+
runId: z.ZodOptional<z.ZodString>;
|
|
166
|
+
}, z.core.$strict>], "type">;
|
|
167
|
+
git: z.ZodOptional<z.ZodObject<{
|
|
168
|
+
branch: z.ZodString;
|
|
169
|
+
prNumber: z.ZodOptional<z.ZodNumber>;
|
|
170
|
+
repo: z.ZodString;
|
|
171
|
+
sha: z.ZodString;
|
|
172
|
+
}, z.core.$strict>>;
|
|
173
|
+
graderModel: z.ZodString;
|
|
174
|
+
host: z.ZodOptional<z.ZodObject<{
|
|
175
|
+
platform: z.ZodString;
|
|
176
|
+
arch: z.ZodString;
|
|
177
|
+
ci: z.ZodOptional<z.ZodString>;
|
|
178
|
+
}, z.core.$strict>>;
|
|
179
|
+
labels: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
180
|
+
lineage: z.ZodOptional<z.ZodObject<{
|
|
181
|
+
rerunOf: z.ZodOptional<z.ZodString>;
|
|
182
|
+
comparedAgainst: z.ZodOptional<z.ZodString>;
|
|
183
|
+
parentJobId: z.ZodOptional<z.ZodString>;
|
|
184
|
+
}, z.core.$strict>>;
|
|
185
|
+
mode: z.ZodString;
|
|
186
|
+
models: z.ZodArray<z.ZodObject<{
|
|
187
|
+
id: z.ZodString;
|
|
188
|
+
label: z.ZodString;
|
|
189
|
+
}, z.core.$strict>>;
|
|
190
|
+
owner: z.ZodObject<{
|
|
191
|
+
team: z.ZodString;
|
|
192
|
+
individual: z.ZodOptional<z.ZodString>;
|
|
193
|
+
}, z.core.$strict>;
|
|
194
|
+
purpose: z.ZodOptional<z.ZodString>;
|
|
195
|
+
source: z.ZodObject<{
|
|
196
|
+
baseUrl: z.ZodString;
|
|
197
|
+
dataset: z.ZodOptional<z.ZodString>;
|
|
198
|
+
name: z.ZodString;
|
|
199
|
+
perspective: z.ZodOptional<z.ZodString>;
|
|
200
|
+
projectId: z.ZodOptional<z.ZodString>;
|
|
201
|
+
}, z.core.$strict>;
|
|
202
|
+
taskIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
203
|
+
tool: z.ZodOptional<z.ZodObject<{
|
|
204
|
+
ailfVersion: z.ZodString;
|
|
205
|
+
nodeVersion: z.ZodString;
|
|
206
|
+
}, z.core.$strict>>;
|
|
207
|
+
trigger: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
208
|
+
type: z.ZodLiteral<"ci">;
|
|
209
|
+
runId: z.ZodString;
|
|
210
|
+
workflow: z.ZodString;
|
|
211
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
212
|
+
type: z.ZodLiteral<"cross-repo">;
|
|
213
|
+
callerRef: z.ZodOptional<z.ZodString>;
|
|
214
|
+
callerRepo: z.ZodString;
|
|
215
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
216
|
+
type: z.ZodLiteral<"manual">;
|
|
217
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
218
|
+
type: z.ZodLiteral<"scheduled">;
|
|
219
|
+
schedule: z.ZodString;
|
|
220
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
221
|
+
type: z.ZodLiteral<"webhook">;
|
|
222
|
+
documentId: z.ZodOptional<z.ZodString>;
|
|
223
|
+
source: z.ZodString;
|
|
224
|
+
}, z.core.$strict>], "type">;
|
|
225
|
+
autoScope: z.ZodOptional<z.ZodObject<{
|
|
226
|
+
enabled: z.ZodBoolean;
|
|
227
|
+
affectedTaskIds: z.ZodArray<z.ZodString>;
|
|
228
|
+
skippedTaskIds: z.ZodArray<z.ZodString>;
|
|
229
|
+
perspective: z.ZodString;
|
|
230
|
+
impactSummary: z.ZodObject<{
|
|
231
|
+
added: z.ZodNumber;
|
|
232
|
+
modified: z.ZodNumber;
|
|
233
|
+
removed: z.ZodNumber;
|
|
234
|
+
}, z.core.$strict>;
|
|
235
|
+
}, z.core.$strict>>;
|
|
236
|
+
contextHash: z.ZodOptional<z.ZodString>;
|
|
237
|
+
promptfooUrl: z.ZodOptional<z.ZodString>;
|
|
238
|
+
promptfooUrls: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
239
|
+
mode: z.ZodString;
|
|
240
|
+
url: z.ZodString;
|
|
241
|
+
}, z.core.$strict>>>;
|
|
242
|
+
runId: z.ZodString;
|
|
243
|
+
targetDocuments: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
244
|
+
}, z.core.$strict>;
|
|
245
|
+
comparison: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
|
|
246
|
+
artifactManifest: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
247
|
+
tag: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
248
|
+
title: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
249
|
+
}, z.core.$loose>;
|
|
250
|
+
export type ReportSchemaInput = z.input<typeof ReportSchema>;
|
|
251
|
+
export type ReportSchemaOutput = z.infer<typeof ReportSchema>;
|