anpord 0.1.11 → 0.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -78
- package/dist/bin.cjs +211 -204
- package/dist/bin.mjs +211 -204
- package/dist/cli-runtime.cjs +2 -2
- package/dist/cli-runtime.d.cts +1 -1
- package/dist/cli-runtime.d.mts +1 -1
- package/dist/cli-runtime.mjs +2 -2
- package/dist/client-BfdjFtlV.mjs +305 -0
- package/dist/{client-C9m0uLCz.d.cts → client-BgRs8JvW.d.mts} +2150 -174
- package/dist/{client-TFCLuMv8.d.mts → client-BirFO3jz.d.cts} +2150 -174
- package/dist/client-C-P2I00z.cjs +346 -0
- package/dist/{compiler-DXlpU_In.mjs → compiler-D96wI7UP.mjs} +108 -75
- package/dist/{compiler-Ap8DM-JU.cjs → compiler-uwb1pkdK.cjs} +116 -83
- package/dist/config.cjs +1 -1
- package/dist/config.d.cts +1 -1
- package/dist/config.d.mts +1 -1
- package/dist/config.mjs +1 -1
- package/dist/{errors-C9bQUcA3.mjs → errors-B0YknR5V.mjs} +3 -19
- package/dist/{errors-Boum34f5.cjs → errors-BX1wry8K.cjs} +3 -19
- package/dist/{errors-BK3f7Rdr.d.cts → errors-aGrd21jy.d.cts} +0 -3
- package/dist/{errors-BK3f7Rdr.d.mts → errors-aGrd21jy.d.mts} +0 -3
- package/dist/eval-judges-BkmiVLLq.cjs +59 -0
- package/dist/eval-judges-CwTvTq7N.mjs +42 -0
- package/dist/eval-judges-DPPUftbh.d.cts +48 -0
- package/dist/eval-judges-DPPUftbh.d.mts +48 -0
- package/dist/eval-validations-BRdoZrTQ.mjs +113 -0
- package/dist/eval-validations-_MlnQ_uv.cjs +148 -0
- package/dist/eval.cjs +1 -1
- package/dist/eval.d.cts +60 -6
- package/dist/eval.d.mts +60 -6
- package/dist/eval.mjs +1 -1
- package/dist/evals-B6aT3xgH.d.cts +1364 -0
- package/dist/evals-B6aT3xgH.d.mts +1364 -0
- package/dist/evals-Dcc_x6jW.mjs +582 -0
- package/dist/evals-DouMsrsM.cjs +737 -0
- package/dist/index.cjs +23 -99
- package/dist/index.d.cts +138 -34
- package/dist/index.d.mts +138 -34
- package/dist/index.mjs +23 -99
- package/dist/source.d.cts +1 -1
- package/dist/source.d.mts +1 -1
- package/dist/{types-oaMF6-E1.d.mts → types-Codjaq-o.d.mts} +146 -45
- package/dist/{types-H9L450Iu.d.cts → types-ISsteq95.d.cts} +146 -45
- package/dist/validator-runtime.cjs +238 -0
- package/dist/validator-runtime.d.cts +9 -0
- package/dist/validator-runtime.d.mts +9 -0
- package/dist/validator-runtime.mjs +237 -0
- package/dist/validators.cjs +10 -0
- package/dist/validators.d.cts +8 -0
- package/dist/validators.d.mts +8 -0
- package/dist/validators.mjs +9 -0
- package/package.json +11 -1
- package/dist/client-5K3dMI5Q.mjs +0 -849
- package/dist/client-Ci0woWLW.cjs +0 -890
- package/dist/evals-BEgEmfiO.d.cts +0 -603
- package/dist/evals-BEgEmfiO.d.mts +0 -603
- package/dist/harness-profile-BMjsN460.mjs +0 -43
- package/dist/harness-profile-CDQpPLPd.cjs +0 -78
package/dist/client-5K3dMI5Q.mjs
DELETED
|
@@ -1,849 +0,0 @@
|
|
|
1
|
-
import { n as HarnessProfile, o as profileFitsHarness, r as PROFILE_HARNESS_RULE } from "./harness-profile-BMjsN460.mjs";
|
|
2
|
-
import { FetchHttpClient, HttpApi, HttpApiClient, HttpApiEndpoint, HttpApiGroup, HttpApiMiddleware, HttpApiSchema, HttpApiSecurity, HttpClient, HttpClientRequest, OpenApi } from "@effect/platform";
|
|
3
|
-
import { Context, Effect, Layer, Redacted, Schema } from "effect";
|
|
4
|
-
//#region ../schema/src/domain/prompts.ts
|
|
5
|
-
const ChannelName = Schema.String.pipe(Schema.minLength(1), Schema.maxLength(36), Schema.pattern(/^[a-z0-9][a-z0-9_-]*$/, { message: () => "Channel must be lowercase alphanumeric, optionally with - or _" }), Schema.brand("ChannelName")).annotations({ description: "Addresses a version. `latest` is derived from the highest version rather than stored, so it cannot drift from the version table." });
|
|
6
|
-
ChannelName.make("production");
|
|
7
|
-
ChannelName.make("latest");
|
|
8
|
-
const PromptId = Schema.String.pipe(Schema.minLength(1), Schema.maxLength(255), Schema.pattern(/^[a-z0-9][a-z0-9/_-]*$/, { message: () => "Prompt id must be lowercase alphanumeric, optionally with / _ or -" }), Schema.brand("PromptId"));
|
|
9
|
-
const PromptName = Schema.String.pipe(Schema.minLength(1), Schema.maxLength(255), Schema.brand("PromptName"));
|
|
10
|
-
const VersionNumber = Schema.Int.pipe(Schema.positive(), Schema.brand("VersionNumber"));
|
|
11
|
-
Schema.NumberFromString.pipe(Schema.int(), Schema.positive(), Schema.brand("VersionNumber"));
|
|
12
|
-
Schema.NumberFromString.pipe(Schema.int(), Schema.positive(), Schema.lessThanOrEqualTo(100));
|
|
13
|
-
const Timestamp = Schema.Union(Schema.DateFromSelf, Schema.Date);
|
|
14
|
-
const CommitMessage = Schema.String.pipe(Schema.maxLength(500));
|
|
15
|
-
const PromptContent = Schema.String.pipe(Schema.minLength(1), Schema.maxLength(262144, { message: () => "Prompt content is longer than 256 KB" }));
|
|
16
|
-
const PromptDescription = Schema.String.pipe(Schema.maxLength(2e3));
|
|
17
|
-
const PromptConfig = Schema.Record({
|
|
18
|
-
key: Schema.String,
|
|
19
|
-
value: Schema.Unknown
|
|
20
|
-
});
|
|
21
|
-
const Author = Schema.Struct({
|
|
22
|
-
image: Schema.NullOr(Schema.String),
|
|
23
|
-
name: Schema.String
|
|
24
|
-
});
|
|
25
|
-
Schema.Struct({
|
|
26
|
-
author: Schema.NullOr(Author),
|
|
27
|
-
channel: Schema.NullOr(ChannelName),
|
|
28
|
-
commitMessage: Schema.NullOr(CommitMessage),
|
|
29
|
-
config: PromptConfig,
|
|
30
|
-
content: Schema.String,
|
|
31
|
-
createdAt: Timestamp,
|
|
32
|
-
id: PromptId,
|
|
33
|
-
name: PromptName,
|
|
34
|
-
version: VersionNumber,
|
|
35
|
-
versionId: Schema.String
|
|
36
|
-
});
|
|
37
|
-
Schema.Struct({
|
|
38
|
-
channel: ChannelName,
|
|
39
|
-
updatedAt: Timestamp,
|
|
40
|
-
updatedBy: Schema.NullOr(Author),
|
|
41
|
-
version: VersionNumber
|
|
42
|
-
});
|
|
43
|
-
const PromptSummary = Schema.Struct({
|
|
44
|
-
author: Schema.NullOr(Author),
|
|
45
|
-
description: Schema.NullOr(Schema.String),
|
|
46
|
-
id: PromptId,
|
|
47
|
-
latestVersion: Schema.NullOr(VersionNumber),
|
|
48
|
-
name: PromptName,
|
|
49
|
-
productionVersion: Schema.NullOr(VersionNumber),
|
|
50
|
-
updatedAt: Timestamp
|
|
51
|
-
});
|
|
52
|
-
Schema.Struct({
|
|
53
|
-
id: PromptId,
|
|
54
|
-
updatedAt: Timestamp
|
|
55
|
-
});
|
|
56
|
-
Schema.Literal("all", "draft", "live");
|
|
57
|
-
Schema.Literal("name", "updated");
|
|
58
|
-
Schema.Struct({
|
|
59
|
-
items: Schema.Array(PromptSummary),
|
|
60
|
-
/** Opaque to callers, and null once the last page has been read. */
|
|
61
|
-
nextCursor: Schema.NullOr(Schema.String)
|
|
62
|
-
});
|
|
63
|
-
Schema.Struct({
|
|
64
|
-
commitMessage: Schema.optional(CommitMessage),
|
|
65
|
-
config: Schema.optional(PromptConfig),
|
|
66
|
-
content: PromptContent,
|
|
67
|
-
description: Schema.optional(PromptDescription),
|
|
68
|
-
id: PromptId,
|
|
69
|
-
name: PromptName,
|
|
70
|
-
publish: Schema.optional(Schema.Boolean)
|
|
71
|
-
});
|
|
72
|
-
Schema.Struct({
|
|
73
|
-
commitMessage: Schema.optional(CommitMessage),
|
|
74
|
-
config: Schema.optional(PromptConfig),
|
|
75
|
-
content: PromptContent,
|
|
76
|
-
publish: Schema.optional(Schema.Boolean)
|
|
77
|
-
});
|
|
78
|
-
Schema.Struct({
|
|
79
|
-
commitMessage: Schema.optional(CommitMessage),
|
|
80
|
-
config: Schema.optional(PromptConfig),
|
|
81
|
-
content: PromptContent
|
|
82
|
-
});
|
|
83
|
-
Schema.Struct({
|
|
84
|
-
description: Schema.optional(PromptDescription),
|
|
85
|
-
id: Schema.optional(PromptId),
|
|
86
|
-
name: Schema.optional(PromptName)
|
|
87
|
-
});
|
|
88
|
-
Schema.Struct({
|
|
89
|
-
channel: ChannelName,
|
|
90
|
-
version: VersionNumber
|
|
91
|
-
});
|
|
92
|
-
Schema.Struct({
|
|
93
|
-
channel: Schema.optional(ChannelName),
|
|
94
|
-
version: Schema.optional(VersionNumber)
|
|
95
|
-
});
|
|
96
|
-
//#endregion
|
|
97
|
-
//#region ../schema/src/domain/errors.ts
|
|
98
|
-
var NotFound = class extends Schema.TaggedError()("NotFound", { message: Schema.String }, HttpApiSchema.annotations({ status: 404 })) {};
|
|
99
|
-
var Conflict = class extends Schema.TaggedError()("Conflict", { message: Schema.String }, HttpApiSchema.annotations({ status: 409 })) {};
|
|
100
|
-
var BadRequest = class extends Schema.TaggedError()("BadRequest", { message: Schema.String }, HttpApiSchema.annotations({ status: 400 })) {};
|
|
101
|
-
var Unauthorized = class extends Schema.TaggedError()("Unauthorized", { message: Schema.String }, HttpApiSchema.annotations({ status: 401 })) {};
|
|
102
|
-
/** Distinct from Unauthorized: the caller is who they say they are, and the
|
|
103
|
-
* answer is still no. Signing in again cannot help, so the client must not
|
|
104
|
-
* treat this as an expired session. */
|
|
105
|
-
var Forbidden = class extends Schema.TaggedError()("Forbidden", { message: Schema.String }, HttpApiSchema.annotations({ status: 403 })) {};
|
|
106
|
-
Schema.TaggedError()("InternalError", { message: Schema.String }, HttpApiSchema.annotations({ status: 500 }));
|
|
107
|
-
//#endregion
|
|
108
|
-
//#region ../schema/src/domain/eval-limits.ts
|
|
109
|
-
/**
|
|
110
|
-
* How long a string a trial may carry to the sandbox.
|
|
111
|
-
*
|
|
112
|
-
* Every one of these is `shellQuote`d into a single command line and handed to
|
|
113
|
-
* `sh -c` as one argv element, and Linux caps one argument at `MAX_ARG_STRLEN`
|
|
114
|
-
* -- 128 KiB. Past that the trial dies as an opaque `E2BIG` inside a VM that
|
|
115
|
-
* is already billing, which reads as a broken provider rather than as a
|
|
116
|
-
* request nobody should have accepted.
|
|
117
|
-
*
|
|
118
|
-
* The budget is in characters against a byte ceiling, so it holds for text
|
|
119
|
-
* that is not ASCII: 128 KiB is 32768 characters even where every one of them
|
|
120
|
-
* costs the four bytes UTF-8 allows. The prompt takes half of that and the
|
|
121
|
-
* verifier a quarter, leaving the rest for the harness's own flags, the
|
|
122
|
-
* workspace path, the model name, the profile arguments, and the escaping
|
|
123
|
-
* `shellQuote` adds -- which turns each apostrophe into four characters.
|
|
124
|
-
*
|
|
125
|
-
* These are ceilings on what the wire accepts, not targets. The prompts people
|
|
126
|
-
* actually send are hundreds of characters and the longest verifier stored is
|
|
127
|
-
* under three thousand.
|
|
128
|
-
*/
|
|
129
|
-
const PROMPT_LIMIT = 16384;
|
|
130
|
-
const VERIFY_LIMIT = 8192;
|
|
131
|
-
const NAME_LIMIT = 200;
|
|
132
|
-
const VARIABLE_VALUE_LIMIT = 2048;
|
|
133
|
-
const EvalPrompt = Schema.String.pipe(Schema.maxLength(PROMPT_LIMIT), Schema.annotations({
|
|
134
|
-
description: "What the agent is asked to do.",
|
|
135
|
-
identifier: "EvalPrompt",
|
|
136
|
-
message: () => `A prompt must be at most ${PROMPT_LIMIT} characters, because it is quoted into one sandbox command line.`
|
|
137
|
-
}));
|
|
138
|
-
const EvalVerify = Schema.String.pipe(Schema.maxLength(VERIFY_LIMIT), Schema.annotations({
|
|
139
|
-
description: "The shell command that decides whether a trial passed.",
|
|
140
|
-
identifier: "EvalVerify",
|
|
141
|
-
message: () => `A verifier must be at most ${VERIFY_LIMIT} characters, because it is quoted into one sandbox command line.`
|
|
142
|
-
}));
|
|
143
|
-
const EvalCaseName = Schema.String.pipe(Schema.maxLength(NAME_LIMIT), Schema.annotations({
|
|
144
|
-
description: "What a case is called.",
|
|
145
|
-
identifier: "EvalCaseName",
|
|
146
|
-
message: () => `A case name must be at most ${NAME_LIMIT} characters.`
|
|
147
|
-
}));
|
|
148
|
-
const EvalVariableValue = Schema.String.pipe(Schema.maxLength(VARIABLE_VALUE_LIMIT), Schema.annotations({
|
|
149
|
-
identifier: "EvalVariableValue",
|
|
150
|
-
message: () => `A variable value must be at most ${VARIABLE_VALUE_LIMIT} characters, because it is substituted into the prompt before it is quoted.`
|
|
151
|
-
}));
|
|
152
|
-
//#endregion
|
|
153
|
-
//#region ../schema/src/domain/credentials.ts
|
|
154
|
-
const CredentialScope = Schema.Literal("organization", "personal");
|
|
155
|
-
const CredentialStatus = Schema.Literal("active", "invalid");
|
|
156
|
-
const CredentialField = Schema.Struct({
|
|
157
|
-
/** The shape of the value, shown as a placeholder beside the label. Absent
|
|
158
|
-
* where the label already says it, as a bare API key does. */
|
|
159
|
-
hint: Schema.optional(Schema.String),
|
|
160
|
-
label: Schema.String,
|
|
161
|
-
name: Schema.String,
|
|
162
|
-
required: Schema.Boolean,
|
|
163
|
-
secret: Schema.Boolean
|
|
164
|
-
});
|
|
165
|
-
const CredentialAuthMethod = Schema.Struct({
|
|
166
|
-
fields: Schema.Array(CredentialField),
|
|
167
|
-
id: Schema.String,
|
|
168
|
-
kind: Schema.Literal("secret", "device", "env"),
|
|
169
|
-
label: Schema.String
|
|
170
|
-
});
|
|
171
|
-
Schema.Struct({
|
|
172
|
-
authMethods: Schema.Array(CredentialAuthMethod),
|
|
173
|
-
category: Schema.Literal("harness", "sandbox"),
|
|
174
|
-
id: Schema.String,
|
|
175
|
-
label: Schema.String
|
|
176
|
-
});
|
|
177
|
-
Schema.Struct({
|
|
178
|
-
integrationId: Schema.String,
|
|
179
|
-
/** Display names of the members holding one, so the reader knows who to
|
|
180
|
-
* ask. Never ids: this is read by everyone in the organization. */
|
|
181
|
-
owners: Schema.Array(Schema.String)
|
|
182
|
-
}).annotations({
|
|
183
|
-
description: "Who else in the organization has an integration connected.",
|
|
184
|
-
identifier: "IntegrationAwareness"
|
|
185
|
-
});
|
|
186
|
-
Schema.Struct({
|
|
187
|
-
authMethodId: Schema.String,
|
|
188
|
-
createdAt: Schema.DateTimeUtc,
|
|
189
|
-
id: Schema.String,
|
|
190
|
-
integrationId: Schema.String,
|
|
191
|
-
isDefault: Schema.Boolean,
|
|
192
|
-
lastUsedAt: Schema.NullOr(Schema.DateTimeUtc),
|
|
193
|
-
name: Schema.String,
|
|
194
|
-
scope: CredentialScope,
|
|
195
|
-
status: CredentialStatus
|
|
196
|
-
});
|
|
197
|
-
const CredentialValues = Schema.Record({
|
|
198
|
-
key: Schema.String,
|
|
199
|
-
value: Schema.String
|
|
200
|
-
});
|
|
201
|
-
Schema.Struct({
|
|
202
|
-
authMethodId: Schema.String,
|
|
203
|
-
connectionId: Schema.String,
|
|
204
|
-
integrationId: Schema.String,
|
|
205
|
-
revision: Schema.Int,
|
|
206
|
-
values: CredentialValues
|
|
207
|
-
});
|
|
208
|
-
Schema.Struct({
|
|
209
|
-
authMethodId: Schema.String,
|
|
210
|
-
integrationId: Schema.String,
|
|
211
|
-
isDefault: Schema.optionalWith(Schema.Boolean, { default: () => false }),
|
|
212
|
-
name: Schema.String.pipe(Schema.minLength(1), Schema.maxLength(80)),
|
|
213
|
-
scope: CredentialScope,
|
|
214
|
-
values: CredentialValues
|
|
215
|
-
});
|
|
216
|
-
const CredentialBindings = Schema.Struct({
|
|
217
|
-
harnessConnectionId: Schema.optional(Schema.String),
|
|
218
|
-
sandboxConnectionId: Schema.optional(Schema.String)
|
|
219
|
-
});
|
|
220
|
-
const CredentialSelections = Schema.Record({
|
|
221
|
-
key: Schema.String,
|
|
222
|
-
value: Schema.String
|
|
223
|
-
});
|
|
224
|
-
Schema.Struct({ values: CredentialValues });
|
|
225
|
-
Schema.Struct({
|
|
226
|
-
integrationId: Schema.Literal("codex"),
|
|
227
|
-
name: Schema.String.pipe(Schema.minLength(1), Schema.maxLength(80)),
|
|
228
|
-
scope: CredentialScope
|
|
229
|
-
});
|
|
230
|
-
Schema.Struct({
|
|
231
|
-
attemptId: Schema.String,
|
|
232
|
-
code: Schema.String,
|
|
233
|
-
expiresAt: Schema.DateTimeUtc,
|
|
234
|
-
verificationUrl: Schema.String
|
|
235
|
-
});
|
|
236
|
-
Schema.Struct({
|
|
237
|
-
connectionId: Schema.NullOr(Schema.String),
|
|
238
|
-
status: Schema.Literal("pending", "complete", "failed", "expired")
|
|
239
|
-
});
|
|
240
|
-
//#endregion
|
|
241
|
-
//#region ../schema/src/domain/evals.ts
|
|
242
|
-
const EvalProvider = Schema.Literal("daytona", "e2b", "upstash", "modal", "cloudflare", "vercel");
|
|
243
|
-
EvalProvider.literals;
|
|
244
|
-
const EvalHarness = Schema.Literal("codex", "opencode", "pi", "fx", "claude", "gemini", "qwen", "cursor", "command");
|
|
245
|
-
const EvalTrialStatus = Schema.Literal("queued", "running", "passed", "failed", "void");
|
|
246
|
-
const EvalRunStatus = Schema.Literal("running", "finished", "failed");
|
|
247
|
-
const EvalSource = Schema.Union(Schema.Struct({ kind: Schema.Literal("empty") }), Schema.Struct({
|
|
248
|
-
kind: Schema.Literal("repo"),
|
|
249
|
-
ref: Schema.NullOr(Schema.String),
|
|
250
|
-
url: Schema.String.pipe(Schema.minLength(1), Schema.annotations({ message: () => "Give the repository a URL." }))
|
|
251
|
-
}), Schema.Struct({
|
|
252
|
-
files: Schema.Record({
|
|
253
|
-
key: Schema.String,
|
|
254
|
-
value: Schema.String
|
|
255
|
-
}),
|
|
256
|
-
kind: Schema.Literal("files")
|
|
257
|
-
})).annotations({
|
|
258
|
-
description: "The workspace available to the harness before setup runs.",
|
|
259
|
-
identifier: "EvalSource"
|
|
260
|
-
});
|
|
261
|
-
const EvalValidator = Schema.Struct({
|
|
262
|
-
name: Schema.String.pipe(Schema.minLength(1), Schema.maxLength(100)),
|
|
263
|
-
source: Schema.String.pipe(Schema.minLength(1), Schema.maxLength(1e6))
|
|
264
|
-
}).annotations({
|
|
265
|
-
description: "A bundled TypeScript validator and its exported function name.",
|
|
266
|
-
identifier: "EvalValidator"
|
|
267
|
-
});
|
|
268
|
-
const EvalPrepare = Schema.Struct({
|
|
269
|
-
name: Schema.String.pipe(Schema.minLength(1), Schema.maxLength(100)),
|
|
270
|
-
source: Schema.String.pipe(Schema.minLength(1), Schema.maxLength(1e6))
|
|
271
|
-
}).annotations({
|
|
272
|
-
description: "A bundled TypeScript workspace setup and its exported function name.",
|
|
273
|
-
identifier: "EvalPrepare"
|
|
274
|
-
});
|
|
275
|
-
const EvalPrepareValue = Schema.Record({
|
|
276
|
-
key: Schema.String,
|
|
277
|
-
value: Schema.Unknown
|
|
278
|
-
}).annotations({
|
|
279
|
-
description: "What a workspace setup returned, handed to the validator.",
|
|
280
|
-
identifier: "EvalPrepareValue"
|
|
281
|
-
});
|
|
282
|
-
const EvalVariables = Schema.Record({
|
|
283
|
-
key: Schema.String,
|
|
284
|
-
value: EvalVariableValue
|
|
285
|
-
}).annotations({
|
|
286
|
-
description: "Values for the placeholders the run prompt names, such as {{task}}.",
|
|
287
|
-
identifier: "EvalVariables"
|
|
288
|
-
});
|
|
289
|
-
/**
|
|
290
|
-
* A directory worth keeping between runs of a case.
|
|
291
|
-
*
|
|
292
|
-
* Declared on the case rather than reported by its prepare, because a restore
|
|
293
|
-
* happens before the prepare runs and so cannot be told where to look by it.
|
|
294
|
-
* The shape CI caches use, for the same reason.
|
|
295
|
-
*/
|
|
296
|
-
const CaseCache = Schema.Struct({
|
|
297
|
-
key: Schema.String.pipe(Schema.minLength(1)),
|
|
298
|
-
path: Schema.String.pipe(Schema.minLength(1), Schema.filter((value) => !(value.startsWith("/") || value.split("/").includes("..")), { message: () => "a cache path must stay inside the workspace" }))
|
|
299
|
-
}).annotations({ identifier: "CaseCache" });
|
|
300
|
-
const EvalCase = Schema.Struct({
|
|
301
|
-
cache: Schema.optional(CaseCache),
|
|
302
|
-
name: EvalCaseName,
|
|
303
|
-
prepare: Schema.NullOr(EvalPrepare),
|
|
304
|
-
source: EvalSource,
|
|
305
|
-
variables: Schema.optionalWith(EvalVariables, { default: () => ({}) }),
|
|
306
|
-
validator: Schema.optionalWith(Schema.NullOr(EvalValidator), { default: () => null }),
|
|
307
|
-
verify: Schema.NullOr(EvalVerify)
|
|
308
|
-
});
|
|
309
|
-
const EvalTaskProfile = Schema.Struct({
|
|
310
|
-
name: Schema.String,
|
|
311
|
-
version: Schema.String
|
|
312
|
-
}).annotations({
|
|
313
|
-
description: "The profile a cell's harness ran under, by name and version.",
|
|
314
|
-
identifier: "EvalTaskProfile"
|
|
315
|
-
});
|
|
316
|
-
const EvalTask = Schema.Struct({
|
|
317
|
-
harness: EvalHarness,
|
|
318
|
-
harnessVersion: Schema.String,
|
|
319
|
-
model: Schema.String,
|
|
320
|
-
profile: Schema.optional(Schema.NullOr(EvalTaskProfile)),
|
|
321
|
-
provider: EvalProvider
|
|
322
|
-
}).annotations({
|
|
323
|
-
description: "The harness, installed version, profile, model, and sandbox for a cell.",
|
|
324
|
-
identifier: "EvalTask"
|
|
325
|
-
});
|
|
326
|
-
const EvalTaskRequest = Schema.Struct({
|
|
327
|
-
credentials: Schema.optional(CredentialBindings),
|
|
328
|
-
harness: EvalHarness,
|
|
329
|
-
model: Schema.String,
|
|
330
|
-
profile: Schema.optional(HarnessProfile),
|
|
331
|
-
provider: EvalProvider
|
|
332
|
-
}).pipe(Schema.filter(profileFitsHarness, { message: () => PROFILE_HARNESS_RULE }));
|
|
333
|
-
const EvalName = Schema.String.pipe(Schema.minLength(1), Schema.maxLength(100));
|
|
334
|
-
Schema.Struct({
|
|
335
|
-
cases: Schema.Array(EvalCase).pipe(Schema.minItems(1), Schema.maxItems(100)),
|
|
336
|
-
name: Schema.optional(EvalName),
|
|
337
|
-
prompt: EvalPrompt,
|
|
338
|
-
tasks: Schema.Array(EvalTaskRequest).pipe(Schema.minItems(1), Schema.maxItems(20)),
|
|
339
|
-
trials: Schema.Int.pipe(Schema.between(1, 10))
|
|
340
|
-
});
|
|
341
|
-
const OccurredAtMillis = Schema.NullOr(Schema.Number);
|
|
342
|
-
const EvalUsage = Schema.Struct({
|
|
343
|
-
cacheReadTokens: Schema.Int,
|
|
344
|
-
cacheWriteTokens: Schema.Int,
|
|
345
|
-
costUsd: Schema.optional(Schema.NullOr(Schema.Number)),
|
|
346
|
-
inputTokens: Schema.Int,
|
|
347
|
-
outputTokens: Schema.Int,
|
|
348
|
-
totalTokens: Schema.Int
|
|
349
|
-
}).annotations({
|
|
350
|
-
description: "Token usage reported by the harness.",
|
|
351
|
-
identifier: "EvalUsage"
|
|
352
|
-
});
|
|
353
|
-
const EvalJournalEntry = Schema.Union(Schema.Struct({
|
|
354
|
-
_tag: Schema.Literal("command"),
|
|
355
|
-
command: Schema.String,
|
|
356
|
-
exitCode: Schema.NullOr(Schema.Int),
|
|
357
|
-
finishedAtMillis: OccurredAtMillis,
|
|
358
|
-
output: Schema.String,
|
|
359
|
-
startedAtMillis: OccurredAtMillis
|
|
360
|
-
}), Schema.Struct({
|
|
361
|
-
_tag: Schema.Literal("message"),
|
|
362
|
-
finishedAtMillis: OccurredAtMillis,
|
|
363
|
-
text: Schema.String,
|
|
364
|
-
usage: Schema.optional(Schema.NullOr(EvalUsage))
|
|
365
|
-
}), Schema.Struct({
|
|
366
|
-
_tag: Schema.Literal("toolCall"),
|
|
367
|
-
finishedAtMillis: OccurredAtMillis,
|
|
368
|
-
name: Schema.String,
|
|
369
|
-
startedAtMillis: Schema.optional(OccurredAtMillis),
|
|
370
|
-
status: Schema.NullOr(Schema.String)
|
|
371
|
-
}), Schema.Struct({
|
|
372
|
-
_tag: Schema.Literal("fileChange"),
|
|
373
|
-
finishedAtMillis: OccurredAtMillis,
|
|
374
|
-
paths: Schema.Array(Schema.String)
|
|
375
|
-
})).annotations({
|
|
376
|
-
description: "A normalized event recorded from the harness trajectory.",
|
|
377
|
-
identifier: "EvalJournalEntry"
|
|
378
|
-
});
|
|
379
|
-
const EvalVerifyStep = Schema.Struct({
|
|
380
|
-
command: Schema.String,
|
|
381
|
-
exitCode: Schema.Int
|
|
382
|
-
}).annotations({
|
|
383
|
-
description: "One condition of the verifier, and how it exited. Only the steps that ran are listed: the script stops at the first failure.",
|
|
384
|
-
identifier: "EvalVerifyStep"
|
|
385
|
-
});
|
|
386
|
-
/**
|
|
387
|
-
* How much of a cost is known, and on what basis.
|
|
388
|
-
*
|
|
389
|
-
* The distinction is the point: a public-rate calculation is not an invoice, a
|
|
390
|
-
* subscription's marginal price is not zero, and a cost the platform absorbs
|
|
391
|
-
* is not one the customer paid. Collapsing any of those into a number produces
|
|
392
|
-
* a total that reads as authoritative and is not.
|
|
393
|
-
*/
|
|
394
|
-
const CostClassification = Schema.Literal("actual", "allocated", "estimate", "included", "managed", "unknown");
|
|
395
|
-
const CostComponentName = Schema.Literal("harness", "model", "platform", "sandbox");
|
|
396
|
-
const EvalCostComponent = Schema.Struct({
|
|
397
|
-
classification: CostClassification,
|
|
398
|
-
component: CostComponentName,
|
|
399
|
-
detail: Schema.Record({
|
|
400
|
-
key: Schema.String,
|
|
401
|
-
value: Schema.Unknown
|
|
402
|
-
}),
|
|
403
|
-
explanation: Schema.String,
|
|
404
|
-
source: Schema.String,
|
|
405
|
-
usd: Schema.NullOr(Schema.Number)
|
|
406
|
-
}).annotations({
|
|
407
|
-
description: "What one layer of a trial cost, and how far that is known.",
|
|
408
|
-
identifier: "EvalCostComponent"
|
|
409
|
-
});
|
|
410
|
-
/**
|
|
411
|
-
* What a run, case, or trial cost, kept apart by how it is known.
|
|
412
|
-
*
|
|
413
|
-
* No single total, deliberately. Adding an estimate to an actual charge and an
|
|
414
|
-
* allocated share produces a number that means none of the three.
|
|
415
|
-
*/
|
|
416
|
-
const EvalCosts = Schema.Struct({
|
|
417
|
-
allocatedUsd: Schema.Number,
|
|
418
|
-
components: Schema.Array(EvalCostComponent),
|
|
419
|
-
estimatedEquivalentUsd: Schema.Number,
|
|
420
|
-
incomplete: Schema.Boolean,
|
|
421
|
-
knownActualUsd: Schema.Number
|
|
422
|
-
}).annotations({
|
|
423
|
-
description: "Cost by component, kept apart by classification.",
|
|
424
|
-
identifier: "EvalCosts"
|
|
425
|
-
});
|
|
426
|
-
const EvalTrial = Schema.Struct({
|
|
427
|
-
commands: Schema.Int,
|
|
428
|
-
costs: Schema.NullOr(EvalCosts),
|
|
429
|
-
prepared: Schema.NullOr(EvalPrepareValue),
|
|
430
|
-
exitCode: Schema.Int,
|
|
431
|
-
failedCommands: Schema.Int,
|
|
432
|
-
filesChanged: Schema.Array(Schema.String),
|
|
433
|
-
modelMs: Schema.Int,
|
|
434
|
-
ordinal: Schema.Int,
|
|
435
|
-
passed: Schema.Boolean,
|
|
436
|
-
sandboxId: Schema.NullOr(Schema.String),
|
|
437
|
-
sandboxMs: Schema.Int,
|
|
438
|
-
status: EvalTrialStatus,
|
|
439
|
-
timed: Schema.Boolean,
|
|
440
|
-
trajectory: Schema.Array(EvalJournalEntry),
|
|
441
|
-
usage: Schema.NullOr(EvalUsage),
|
|
442
|
-
verifySteps: Schema.Array(EvalVerifyStep),
|
|
443
|
-
voidFields: Schema.Array(Schema.String)
|
|
444
|
-
}).annotations({
|
|
445
|
-
description: "One sandbox attempt for a grid cell.",
|
|
446
|
-
identifier: "EvalTrial"
|
|
447
|
-
});
|
|
448
|
-
const EvalDistribution = Schema.Struct({
|
|
449
|
-
commandMax: Schema.Int,
|
|
450
|
-
commandMedian: Schema.Number,
|
|
451
|
-
commandMin: Schema.Int,
|
|
452
|
-
deterministic: Schema.Boolean,
|
|
453
|
-
failed: Schema.Int,
|
|
454
|
-
passRate: Schema.Number,
|
|
455
|
-
passed: Schema.Int,
|
|
456
|
-
scored: Schema.Int,
|
|
457
|
-
trials: Schema.Int,
|
|
458
|
-
voided: Schema.Int
|
|
459
|
-
}).annotations({
|
|
460
|
-
description: "The scored outcome across all trials in a cell.",
|
|
461
|
-
identifier: "EvalDistribution"
|
|
462
|
-
});
|
|
463
|
-
const EvalVerdict = Schema.Literal("improved", "incomparable", "regressed", "unchanged");
|
|
464
|
-
const EvalComparison = Schema.Struct({
|
|
465
|
-
baselineHarnessVersion: Schema.String,
|
|
466
|
-
baselinePassRate: Schema.Number,
|
|
467
|
-
baselineProfileVersion: Schema.NullOr(Schema.String),
|
|
468
|
-
candidateHarnessVersion: Schema.String,
|
|
469
|
-
candidatePassRate: Schema.Number,
|
|
470
|
-
candidateProfileVersion: Schema.NullOr(Schema.String),
|
|
471
|
-
delta: Schema.Number,
|
|
472
|
-
determinismLost: Schema.Boolean,
|
|
473
|
-
reason: Schema.NullOr(Schema.String),
|
|
474
|
-
verdict: EvalVerdict
|
|
475
|
-
}).annotations({
|
|
476
|
-
description: "The cell result compared with its most recent baseline.",
|
|
477
|
-
identifier: "EvalComparison"
|
|
478
|
-
});
|
|
479
|
-
const EvalSetup = Schema.Struct({
|
|
480
|
-
prompt: Schema.String,
|
|
481
|
-
repoRef: Schema.NullOr(Schema.String),
|
|
482
|
-
repoUrl: Schema.NullOr(Schema.String),
|
|
483
|
-
prepareName: Schema.NullOr(Schema.String),
|
|
484
|
-
validatorName: Schema.NullOr(Schema.String),
|
|
485
|
-
verifyCommand: Schema.NullOr(Schema.String),
|
|
486
|
-
workspace: Schema.String
|
|
487
|
-
}).annotations({
|
|
488
|
-
description: "The prompt, workspace, setup, and verifier used by a cell.",
|
|
489
|
-
identifier: "EvalSetup"
|
|
490
|
-
});
|
|
491
|
-
const EvalCell = Schema.Struct({
|
|
492
|
-
caseName: Schema.String,
|
|
493
|
-
cellKey: Schema.NullOr(Schema.String),
|
|
494
|
-
costs: Schema.NullOr(EvalCosts),
|
|
495
|
-
comparison: Schema.NullOr(EvalComparison),
|
|
496
|
-
distribution: Schema.NullOr(EvalDistribution),
|
|
497
|
-
internalId: Schema.NullOr(Schema.String),
|
|
498
|
-
setup: Schema.NullOr(EvalSetup),
|
|
499
|
-
status: EvalRunStatus,
|
|
500
|
-
taskIndex: Schema.Int,
|
|
501
|
-
trials: Schema.Array(EvalTrial)
|
|
502
|
-
}).annotations({
|
|
503
|
-
description: "One case and task combination in an eval grid.",
|
|
504
|
-
identifier: "EvalCell"
|
|
505
|
-
});
|
|
506
|
-
const EvalTimestamp = Schema.DateTimeUtc.annotations({
|
|
507
|
-
description: "An ISO-8601 timestamp in UTC.",
|
|
508
|
-
identifier: "EvalTimestamp",
|
|
509
|
-
jsonSchema: { format: "date-time" }
|
|
510
|
-
});
|
|
511
|
-
const EvalRun = Schema.Struct({
|
|
512
|
-
cases: Schema.Array(Schema.String),
|
|
513
|
-
cells: Schema.Array(EvalCell),
|
|
514
|
-
costs: Schema.NullOr(EvalCosts),
|
|
515
|
-
failure: Schema.NullOr(Schema.String),
|
|
516
|
-
finishedAt: Schema.NullOr(EvalTimestamp),
|
|
517
|
-
id: Schema.String,
|
|
518
|
-
name: Schema.NullOr(EvalName),
|
|
519
|
-
startedAt: EvalTimestamp,
|
|
520
|
-
status: EvalRunStatus,
|
|
521
|
-
tasks: Schema.Array(EvalTask)
|
|
522
|
-
}).annotations({
|
|
523
|
-
description: "A complete eval run with its cells and trials.",
|
|
524
|
-
identifier: "EvalRun"
|
|
525
|
-
});
|
|
526
|
-
const EvalRunSummary = Schema.Struct({
|
|
527
|
-
caseCount: Schema.Int,
|
|
528
|
-
columns: Schema.Array(EvalTask),
|
|
529
|
-
commandMax: Schema.NullOr(Schema.Int),
|
|
530
|
-
commandMin: Schema.NullOr(Schema.Int),
|
|
531
|
-
failure: Schema.NullOr(Schema.String),
|
|
532
|
-
finishedAt: Schema.NullOr(EvalTimestamp),
|
|
533
|
-
firstCaseName: Schema.NullOr(Schema.String),
|
|
534
|
-
id: Schema.String,
|
|
535
|
-
name: Schema.NullOr(Schema.String),
|
|
536
|
-
passed: Schema.Int,
|
|
537
|
-
scored: Schema.Int,
|
|
538
|
-
startedAt: EvalTimestamp,
|
|
539
|
-
status: EvalRunStatus,
|
|
540
|
-
taskCount: Schema.Int,
|
|
541
|
-
voided: Schema.Int
|
|
542
|
-
}).annotations({
|
|
543
|
-
description: "A compact eval run returned by the run list.",
|
|
544
|
-
identifier: "EvalRunSummary"
|
|
545
|
-
});
|
|
546
|
-
const EvalPageCursor = Schema.Struct({
|
|
547
|
-
id: Schema.String,
|
|
548
|
-
startedAtMillis: Schema.Int
|
|
549
|
-
});
|
|
550
|
-
/** One page of runs. `next` is null at the end rather than an empty cursor, so
|
|
551
|
-
* a caller stops because there is nothing more rather than because a fetch
|
|
552
|
-
* came back empty. */
|
|
553
|
-
const EvalRunPage = Schema.Struct({
|
|
554
|
-
next: Schema.NullOr(EvalPageCursor),
|
|
555
|
-
runs: Schema.Array(EvalRunSummary),
|
|
556
|
-
/** Every run the organization has, so a listing can say how far it goes
|
|
557
|
-
* rather than only whether there is more. */
|
|
558
|
-
total: Schema.Int
|
|
559
|
-
});
|
|
560
|
-
const EvalCellHistoryEntry = Schema.Struct({
|
|
561
|
-
distribution: EvalDistribution,
|
|
562
|
-
finishedAt: Schema.NullOr(EvalTimestamp),
|
|
563
|
-
harnessVersion: Schema.String,
|
|
564
|
-
internalId: Schema.String,
|
|
565
|
-
profileVersion: Schema.NullOr(Schema.String),
|
|
566
|
-
runId: Schema.String,
|
|
567
|
-
trials: Schema.Array(EvalTrial)
|
|
568
|
-
}).annotations({
|
|
569
|
-
description: "A previous scored result for the same cell identity.",
|
|
570
|
-
identifier: "EvalCellHistoryEntry"
|
|
571
|
-
});
|
|
572
|
-
const PlaygroundCaseView = Schema.Struct({
|
|
573
|
-
name: Schema.String,
|
|
574
|
-
setup: Schema.NullOr(Schema.String),
|
|
575
|
-
source: EvalSource,
|
|
576
|
-
variables: Schema.optionalWith(EvalVariables, { default: () => ({}) }),
|
|
577
|
-
verify: Schema.NullOr(Schema.String)
|
|
578
|
-
});
|
|
579
|
-
const PlaygroundColumnView = Schema.Struct({
|
|
580
|
-
harness: EvalHarness,
|
|
581
|
-
model: Schema.String,
|
|
582
|
-
provider: EvalProvider
|
|
583
|
-
});
|
|
584
|
-
const PlaygroundConfigView = Schema.Struct({
|
|
585
|
-
cases: Schema.Array(PlaygroundCaseView),
|
|
586
|
-
columns: Schema.Array(PlaygroundColumnView),
|
|
587
|
-
connections: Schema.optionalWith(CredentialSelections, { default: () => ({}) }),
|
|
588
|
-
prompt: Schema.String,
|
|
589
|
-
trials: Schema.Int.pipe(Schema.between(1, 10))
|
|
590
|
-
});
|
|
591
|
-
const EvalDraftCase = Schema.Struct({
|
|
592
|
-
name: Schema.String.pipe(Schema.minLength(1), Schema.annotations({ message: () => "Name this case." })),
|
|
593
|
-
setup: Schema.NullOr(Schema.String),
|
|
594
|
-
source: EvalSource,
|
|
595
|
-
variables: EvalVariables.pipe(Schema.filter((values) => Object.values(values).some((value) => value.trim() !== ""), { message: () => "Say what the agent should do." })),
|
|
596
|
-
verify: Schema.NullOr(Schema.String)
|
|
597
|
-
});
|
|
598
|
-
const EvalAgent = Schema.Struct({
|
|
599
|
-
harness: EvalHarness,
|
|
600
|
-
model: Schema.String
|
|
601
|
-
});
|
|
602
|
-
Schema.Struct({
|
|
603
|
-
agents: Schema.mutable(Schema.Array(EvalAgent)).pipe(Schema.minItems(1), Schema.annotations({ message: () => "Choose at least one agent." })),
|
|
604
|
-
cases: Schema.mutable(Schema.Array(EvalDraftCase)).pipe(Schema.minItems(1), Schema.annotations({ message: () => "Add at least one case." })),
|
|
605
|
-
connections: CredentialSelections,
|
|
606
|
-
name: Schema.String,
|
|
607
|
-
prompt: Schema.String,
|
|
608
|
-
providers: Schema.mutable(Schema.Array(EvalProvider)).pipe(Schema.minItems(1), Schema.annotations({ message: () => "Choose at least one sandbox." })),
|
|
609
|
-
trials: Schema.Int.pipe(Schema.between(1, 10), Schema.annotations({ message: () => "Run between 1 and 10 trials." }))
|
|
610
|
-
});
|
|
611
|
-
Schema.Struct({
|
|
612
|
-
config: PlaygroundConfigView,
|
|
613
|
-
id: Schema.String,
|
|
614
|
-
lastRunId: Schema.NullOr(Schema.String),
|
|
615
|
-
name: Schema.String,
|
|
616
|
-
problems: Schema.Array(Schema.String),
|
|
617
|
-
ungated: Schema.Array(Schema.String),
|
|
618
|
-
updatedAt: Schema.DateTimeUtc
|
|
619
|
-
});
|
|
620
|
-
Schema.Struct({ name: Schema.String.pipe(Schema.minLength(1)) });
|
|
621
|
-
Schema.Struct({
|
|
622
|
-
config: PlaygroundConfigView,
|
|
623
|
-
name: Schema.String.pipe(Schema.minLength(1))
|
|
624
|
-
});
|
|
625
|
-
const StartedEval = Schema.Struct({ id: Schema.String }).annotations({
|
|
626
|
-
description: "The id of an eval run accepted for background execution.",
|
|
627
|
-
identifier: "StartedEval"
|
|
628
|
-
});
|
|
629
|
-
const RerunCellRequest = Schema.Struct({ trials: Schema.Int.pipe(Schema.between(1, 10)) });
|
|
630
|
-
const CatalogueModel = Schema.Struct({
|
|
631
|
-
displayName: Schema.String,
|
|
632
|
-
id: Schema.String,
|
|
633
|
-
summary: Schema.NullOr(Schema.String),
|
|
634
|
-
vendor: Schema.NullOr(Schema.String)
|
|
635
|
-
}).annotations({
|
|
636
|
-
description: "A model available to the installed harness.",
|
|
637
|
-
identifier: "CatalogueModel"
|
|
638
|
-
});
|
|
639
|
-
const ModelCatalogue = Schema.Struct({
|
|
640
|
-
harness: EvalHarness,
|
|
641
|
-
models: Schema.Array(CatalogueModel),
|
|
642
|
-
total: Schema.Int
|
|
643
|
-
}).annotations({
|
|
644
|
-
description: "Models available to the installed harness.",
|
|
645
|
-
identifier: "ModelCatalogue"
|
|
646
|
-
});
|
|
647
|
-
//#endregion
|
|
648
|
-
//#region ../schema/src/internal/authentication.ts
|
|
649
|
-
var CurrentActor = class extends Context.Tag("@anpord/schema/CurrentActor")() {};
|
|
650
|
-
HttpApiMiddleware.Tag()("@anpord/schema/Authentication", {
|
|
651
|
-
failure: Unauthorized,
|
|
652
|
-
provides: CurrentActor,
|
|
653
|
-
security: { session: HttpApiSecurity.apiKey({
|
|
654
|
-
in: "cookie",
|
|
655
|
-
key: "anpord.session_token"
|
|
656
|
-
}) }
|
|
657
|
-
});
|
|
658
|
-
//#endregion
|
|
659
|
-
//#region ../schema/src/public/authentication.ts
|
|
660
|
-
var ApiKeyAuthentication = class extends HttpApiMiddleware.Tag()("@anpord/schema/ApiKeyAuthentication", {
|
|
661
|
-
failure: Unauthorized,
|
|
662
|
-
provides: CurrentActor,
|
|
663
|
-
security: { bearer: HttpApiSecurity.bearer }
|
|
664
|
-
}) {};
|
|
665
|
-
//#endregion
|
|
666
|
-
//#region ../schema/src/public/evals-api.ts
|
|
667
|
-
const EvalRunRequest = Schema.Struct({ id: Schema.String }).annotations({
|
|
668
|
-
description: "Select an eval run by id.",
|
|
669
|
-
identifier: "EvalRunRequest"
|
|
670
|
-
});
|
|
671
|
-
const EvalCellRequest = Schema.Struct({ cellKey: Schema.String }).annotations({
|
|
672
|
-
description: "Select an eval cell by its stable key.",
|
|
673
|
-
identifier: "EvalCellRequest"
|
|
674
|
-
});
|
|
675
|
-
const EvalModelsRequest = Schema.Struct({
|
|
676
|
-
harness: EvalHarness,
|
|
677
|
-
/** Narrows to models whose id or name contains this. */
|
|
678
|
-
q: Schema.optional(Schema.String)
|
|
679
|
-
}).annotations({
|
|
680
|
-
description: "Select a harness whose available models should be listed.",
|
|
681
|
-
identifier: "EvalModelsRequest"
|
|
682
|
-
});
|
|
683
|
-
const PublicEvalProvider = EvalProvider.annotations({
|
|
684
|
-
description: "A hosted sandbox provider.",
|
|
685
|
-
identifier: "PublicEvalProvider"
|
|
686
|
-
});
|
|
687
|
-
const ListEvalsRequest = Schema.Struct({
|
|
688
|
-
cursor: Schema.optional(Schema.NullOr(EvalPageCursor)),
|
|
689
|
-
limit: Schema.optional(Schema.Int)
|
|
690
|
-
}).annotations({
|
|
691
|
-
description: "Where to read from, and how much.",
|
|
692
|
-
identifier: "ListEvalsRequest"
|
|
693
|
-
});
|
|
694
|
-
const PublicEvalCase = Schema.Struct({
|
|
695
|
-
cache: Schema.optional(CaseCache),
|
|
696
|
-
name: EvalCaseName,
|
|
697
|
-
prepare: Schema.optional(Schema.NullOr(EvalPrepare)),
|
|
698
|
-
source: Schema.optional(EvalSource),
|
|
699
|
-
validator: Schema.optional(Schema.NullOr(EvalValidator)),
|
|
700
|
-
variables: Schema.optional(EvalVariables),
|
|
701
|
-
verify: Schema.NullOr(EvalVerify)
|
|
702
|
-
}).pipe(Schema.filter(({ validator, verify }) => !(validator !== void 0 && validator !== null && verify !== null), { message: () => "Use either validator or verify, not both." })).annotations({
|
|
703
|
-
description: "A task, workspace source, setup command, and verifier.",
|
|
704
|
-
identifier: "StartEvalCase"
|
|
705
|
-
});
|
|
706
|
-
const PublicEvalTask = Schema.Struct({
|
|
707
|
-
harness: EvalHarness,
|
|
708
|
-
model: Schema.String.pipe(Schema.minLength(1)),
|
|
709
|
-
profile: Schema.optional(HarnessProfile),
|
|
710
|
-
provider: PublicEvalProvider
|
|
711
|
-
}).pipe(Schema.filter(profileFitsHarness, { message: () => PROFILE_HARNESS_RULE })).annotations({
|
|
712
|
-
description: `A harness, model, and hosted sandbox combination, with an optional profile layered on the harness. ${PROFILE_HARNESS_RULE}`,
|
|
713
|
-
identifier: "StartEvalTask"
|
|
714
|
-
});
|
|
715
|
-
const PublicStartEvalRequest = Schema.Struct({
|
|
716
|
-
cases: Schema.Array(PublicEvalCase).pipe(Schema.minItems(1), Schema.maxItems(100)),
|
|
717
|
-
name: Schema.optional(EvalName),
|
|
718
|
-
prompt: EvalPrompt,
|
|
719
|
-
tasks: Schema.Array(PublicEvalTask).pipe(Schema.minItems(1), Schema.maxItems(20)),
|
|
720
|
-
trials: Schema.Int.pipe(Schema.between(1, 10))
|
|
721
|
-
}).annotations({
|
|
722
|
-
description: `Start a grid with at most 100 total case, task, and trial combinations.`,
|
|
723
|
-
identifier: "StartEvalRequest"
|
|
724
|
-
});
|
|
725
|
-
var PublicEvalsGroup = class extends HttpApiGroup.make("evals").add(HttpApiEndpoint.post("list", "/evals.list").setPayload(ListEvalsRequest).addSuccess(EvalRunPage).annotate(OpenApi.Summary, "List eval runs").annotate(OpenApi.Description, "Newest first. Pass the `next` cursor from a response to read the page after it; a null `next` means there are no more.")).add(HttpApiEndpoint.post("start", "/evals.start").setPayload(PublicStartEvalRequest).addSuccess(StartedEval).annotate(OpenApi.Summary, "Start an eval run").annotate(OpenApi.Description, `Starts the grid and returns its id while trials continue in the background. ${PROFILE_HARNESS_RULE}`)).add(HttpApiEndpoint.post("get", "/evals.get").setPayload(EvalRunRequest).addSuccess(EvalRun).annotate(OpenApi.Summary, "Get an eval run")).add(HttpApiEndpoint.post("cellHistory", "/evals.cellHistory").setPayload(EvalCellRequest).addSuccess(Schema.Array(EvalCellHistoryEntry)).annotate(OpenApi.Summary, "List a cell's history").annotate(OpenApi.Description, "Returns the 20 most recent results.")).add(HttpApiEndpoint.post("rerunCell", "/evals.rerunCell").setPayload(Schema.extend(EvalRunRequest, Schema.extend(EvalCellRequest, RerunCellRequest))).addSuccess(StartedEval).annotate(OpenApi.Summary, "Rerun one cell")).add(HttpApiEndpoint.post("models", "/evals.models").setPayload(EvalModelsRequest).addSuccess(ModelCatalogue).annotate(OpenApi.Summary, "List models available to the harness").annotate(OpenApi.Description, "The command harness has no catalogue of its own, so its list is empty: the model is whatever the profile's run command reads from ANPORD_MODEL.")).addError(BadRequest).addError(Conflict).addError(Forbidden).addError(NotFound).middleware(ApiKeyAuthentication).annotate(OpenApi.Title, "Evals").annotate(OpenApi.Description, "Run cases across harness, model, and sandbox combinations and compare the results with their baselines.") {};
|
|
726
|
-
//#endregion
|
|
727
|
-
//#region ../schema/src/public/requests.ts
|
|
728
|
-
const GetPromptRequest = Schema.Struct({
|
|
729
|
-
channel: Schema.optional(ChannelName),
|
|
730
|
-
id: PromptId,
|
|
731
|
-
includeVersions: Schema.optional(Schema.Boolean),
|
|
732
|
-
version: Schema.optional(VersionNumber)
|
|
733
|
-
}).annotations({
|
|
734
|
-
description: "Resolve a prompt. Give a version to pin one, a channel to follow one, or neither for the organization's default channel.",
|
|
735
|
-
identifier: "GetPromptRequest"
|
|
736
|
-
});
|
|
737
|
-
Schema.Struct({ id: PromptId }).annotations({
|
|
738
|
-
description: "Show a prompt's version history.",
|
|
739
|
-
identifier: "ListVersionsRequest"
|
|
740
|
-
});
|
|
741
|
-
const ListPromptsRequest = Schema.Record({
|
|
742
|
-
key: Schema.String,
|
|
743
|
-
value: Schema.Never
|
|
744
|
-
}).annotations({
|
|
745
|
-
description: "No parameters; returns up to 100 prompts in the organization.",
|
|
746
|
-
identifier: "ListPromptsRequest"
|
|
747
|
-
});
|
|
748
|
-
const CreatePromptRequest = Schema.Struct({
|
|
749
|
-
config: Schema.optional(PromptConfig),
|
|
750
|
-
content: PromptContent,
|
|
751
|
-
description: Schema.optional(Schema.String),
|
|
752
|
-
id: PromptId,
|
|
753
|
-
message: Schema.optional(CommitMessage),
|
|
754
|
-
name: PromptName
|
|
755
|
-
}).annotations({
|
|
756
|
-
description: "Create a prompt and publish its first version to the default channel.",
|
|
757
|
-
identifier: "CreatePromptRequest"
|
|
758
|
-
});
|
|
759
|
-
const UpdatePromptRequest = Schema.Struct({
|
|
760
|
-
config: Schema.optional(PromptConfig),
|
|
761
|
-
content: PromptContent,
|
|
762
|
-
id: PromptId,
|
|
763
|
-
message: Schema.optional(CommitMessage)
|
|
764
|
-
}).annotations({
|
|
765
|
-
description: "Add a version to a prompt. The new version becomes latest, and omitted config is stored as an empty object.",
|
|
766
|
-
identifier: "UpdatePromptRequest"
|
|
767
|
-
});
|
|
768
|
-
const PromotePromptRequest = Schema.Struct({
|
|
769
|
-
channel: ChannelName,
|
|
770
|
-
id: PromptId,
|
|
771
|
-
version: VersionNumber
|
|
772
|
-
}).annotations({
|
|
773
|
-
description: "Point a channel at a version.",
|
|
774
|
-
identifier: "PromotePromptRequest"
|
|
775
|
-
});
|
|
776
|
-
Schema.Struct({ id: PromptId }).annotations({
|
|
777
|
-
description: "Archive a prompt. Existing selectors stay readable.",
|
|
778
|
-
identifier: "ArchivePromptRequest"
|
|
779
|
-
});
|
|
780
|
-
const Ok = Schema.Struct({ ok: Schema.Literal(true) }).annotations({
|
|
781
|
-
description: "The operation succeeded.",
|
|
782
|
-
identifier: "Ok"
|
|
783
|
-
});
|
|
784
|
-
//#endregion
|
|
785
|
-
//#region ../schema/src/public/shapes.ts
|
|
786
|
-
const Instant = Schema.DateTimeUtc.annotations({
|
|
787
|
-
description: "An ISO-8601 timestamp in UTC.",
|
|
788
|
-
identifier: "Instant",
|
|
789
|
-
jsonSchema: { format: "date-time" }
|
|
790
|
-
});
|
|
791
|
-
const PublicPrompt = Schema.Struct({
|
|
792
|
-
channel: Schema.NullOr(ChannelName),
|
|
793
|
-
config: PromptConfig,
|
|
794
|
-
content: Schema.String,
|
|
795
|
-
createdAt: Instant,
|
|
796
|
-
id: PromptId,
|
|
797
|
-
message: Schema.NullOr(CommitMessage),
|
|
798
|
-
name: PromptName,
|
|
799
|
-
version: VersionNumber
|
|
800
|
-
}).annotations({
|
|
801
|
-
description: "A prompt resolved at a specific version.",
|
|
802
|
-
identifier: "ResolvedPrompt"
|
|
803
|
-
});
|
|
804
|
-
const PublicVersion = Schema.Struct({
|
|
805
|
-
createdAt: Instant,
|
|
806
|
-
message: Schema.NullOr(CommitMessage),
|
|
807
|
-
version: VersionNumber
|
|
808
|
-
}).annotations({
|
|
809
|
-
description: "One entry in a prompt's history.",
|
|
810
|
-
identifier: "Version"
|
|
811
|
-
});
|
|
812
|
-
const PublicPromptWithVersions = Schema.extend(PublicPrompt, Schema.Struct({ versions: Schema.optional(Schema.Array(PublicVersion)) })).annotations({
|
|
813
|
-
description: "A prompt, with its version history when `includeVersions` is set.",
|
|
814
|
-
identifier: "Prompt"
|
|
815
|
-
});
|
|
816
|
-
const PublicPromptSummary = Schema.Struct({
|
|
817
|
-
id: PromptId,
|
|
818
|
-
latestVersion: Schema.NullOr(VersionNumber),
|
|
819
|
-
name: PromptName,
|
|
820
|
-
productionVersion: Schema.NullOr(VersionNumber),
|
|
821
|
-
updatedAt: Instant
|
|
822
|
-
}).annotations({
|
|
823
|
-
description: "A prompt without its content.",
|
|
824
|
-
identifier: "PromptSummary"
|
|
825
|
-
});
|
|
826
|
-
const PromptList = Schema.Struct({ data: Schema.Array(PublicPromptSummary) }).annotations({
|
|
827
|
-
description: "Every prompt in the organization.",
|
|
828
|
-
identifier: "PromptList"
|
|
829
|
-
});
|
|
830
|
-
//#endregion
|
|
831
|
-
//#region ../schema/src/public/prompts-api.ts
|
|
832
|
-
var PublicPromptsGroup = class extends HttpApiGroup.make("prompts").add(HttpApiEndpoint.post("get", "/prompts.get").setPayload(GetPromptRequest).addSuccess(PublicPromptWithVersions).annotate(OpenApi.Summary, "Resolve a prompt").annotate(OpenApi.Description, "Returns the content a caller should send to a model. With no selector this follows the organization's default channel.")).add(HttpApiEndpoint.post("list", "/prompts.list").setPayload(ListPromptsRequest).addSuccess(PromptList).annotate(OpenApi.Summary, "List prompts").annotate(OpenApi.Description, "Up to 100 prompts in the organization, without content.")).add(HttpApiEndpoint.post("create", "/prompts.create").setPayload(CreatePromptRequest).addSuccess(PublicPromptWithVersions).annotate(OpenApi.Summary, "Create a prompt").annotate(OpenApi.Description, "Creates the prompt and its first version in one call.")).add(HttpApiEndpoint.post("update", "/prompts.update").setPayload(UpdatePromptRequest).addSuccess(PublicPromptWithVersions).annotate(OpenApi.Summary, "Add a version").annotate(OpenApi.Description, "Content is versioned, so updating a prompt appends a version rather than overwriting one. Earlier versions stay readable.")).add(HttpApiEndpoint.post("promote", "/prompts.promote").setPayload(PromotePromptRequest).addSuccess(Ok).annotate(OpenApi.Summary, "Promote a version to a channel").annotate(OpenApi.Description, "Points a channel, such as production, at a version. This is how a version goes live without callers changing anything.")).addError(BadRequest).addError(Forbidden).addError(Conflict).addError(NotFound).middleware(ApiKeyAuthentication).annotate(OpenApi.Title, "Prompts").annotate(OpenApi.Description, "Read and write prompts. Content is versioned, so writes append rather than overwrite, and channels decide which version callers receive.") {};
|
|
833
|
-
//#endregion
|
|
834
|
-
//#region ../schema/src/public/api.ts
|
|
835
|
-
var PublicApi = class extends HttpApi.make("anpord-public").add(PublicEvalsGroup).add(PublicPromptsGroup).prefix("/v1").annotate(OpenApi.Title, "Anpord API").annotate(OpenApi.Version, "1.0.0").annotate(OpenApi.Description, "Run agent evals and manage prompts. Every endpoint takes a JSON body over POST and authenticates with a bearer API key.").annotate(OpenApi.Servers, [{
|
|
836
|
-
description: "Production",
|
|
837
|
-
url: "https://api.anpord.com"
|
|
838
|
-
}]) {};
|
|
839
|
-
//#endregion
|
|
840
|
-
//#region ../schema/src/public/client.ts
|
|
841
|
-
const DEFAULT_BASE_URL = "https://api.anpord.com";
|
|
842
|
-
const make = ({ apiKey, baseUrl = DEFAULT_BASE_URL }) => HttpApiClient.make(PublicApi, {
|
|
843
|
-
baseUrl,
|
|
844
|
-
transformClient: HttpClient.mapRequest(HttpClientRequest.bearerToken(Redacted.value(apiKey)))
|
|
845
|
-
});
|
|
846
|
-
var AnpordApi = class extends Effect.Tag("@anpord/sdk/AnpordApi")() {};
|
|
847
|
-
const layer = (options) => Layer.effect(AnpordApi, make(options)).pipe(Layer.provide(FetchHttpClient.layer));
|
|
848
|
-
//#endregion
|
|
849
|
-
export { ChannelName as a, make as i, DEFAULT_BASE_URL as n, PromptId as o, layer as r, VersionNumber as s, AnpordApi as t };
|