anpord 0.1.11 → 0.1.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/README.md +56 -78
  2. package/dist/bin.cjs +211 -204
  3. package/dist/bin.mjs +211 -204
  4. package/dist/cli-runtime.cjs +2 -2
  5. package/dist/cli-runtime.d.cts +1 -1
  6. package/dist/cli-runtime.d.mts +1 -1
  7. package/dist/cli-runtime.mjs +2 -2
  8. package/dist/client-BfdjFtlV.mjs +305 -0
  9. package/dist/{client-C9m0uLCz.d.cts → client-BgRs8JvW.d.mts} +2150 -174
  10. package/dist/{client-TFCLuMv8.d.mts → client-BirFO3jz.d.cts} +2150 -174
  11. package/dist/client-C-P2I00z.cjs +346 -0
  12. package/dist/{compiler-DXlpU_In.mjs → compiler-D96wI7UP.mjs} +108 -75
  13. package/dist/{compiler-Ap8DM-JU.cjs → compiler-uwb1pkdK.cjs} +116 -83
  14. package/dist/config.cjs +1 -1
  15. package/dist/config.d.cts +1 -1
  16. package/dist/config.d.mts +1 -1
  17. package/dist/config.mjs +1 -1
  18. package/dist/{errors-C9bQUcA3.mjs → errors-B0YknR5V.mjs} +3 -19
  19. package/dist/{errors-Boum34f5.cjs → errors-BX1wry8K.cjs} +3 -19
  20. package/dist/{errors-BK3f7Rdr.d.cts → errors-aGrd21jy.d.cts} +0 -3
  21. package/dist/{errors-BK3f7Rdr.d.mts → errors-aGrd21jy.d.mts} +0 -3
  22. package/dist/eval-judges-BkmiVLLq.cjs +59 -0
  23. package/dist/eval-judges-CwTvTq7N.mjs +42 -0
  24. package/dist/eval-judges-DPPUftbh.d.cts +48 -0
  25. package/dist/eval-judges-DPPUftbh.d.mts +48 -0
  26. package/dist/eval-validations-BRdoZrTQ.mjs +113 -0
  27. package/dist/eval-validations-_MlnQ_uv.cjs +148 -0
  28. package/dist/eval.cjs +1 -1
  29. package/dist/eval.d.cts +60 -6
  30. package/dist/eval.d.mts +60 -6
  31. package/dist/eval.mjs +1 -1
  32. package/dist/evals-B6aT3xgH.d.cts +1364 -0
  33. package/dist/evals-B6aT3xgH.d.mts +1364 -0
  34. package/dist/evals-Dcc_x6jW.mjs +582 -0
  35. package/dist/evals-DouMsrsM.cjs +737 -0
  36. package/dist/index.cjs +23 -99
  37. package/dist/index.d.cts +138 -34
  38. package/dist/index.d.mts +138 -34
  39. package/dist/index.mjs +23 -99
  40. package/dist/source.d.cts +1 -1
  41. package/dist/source.d.mts +1 -1
  42. package/dist/{types-oaMF6-E1.d.mts → types-Codjaq-o.d.mts} +146 -45
  43. package/dist/{types-H9L450Iu.d.cts → types-ISsteq95.d.cts} +146 -45
  44. package/dist/validator-runtime.cjs +238 -0
  45. package/dist/validator-runtime.d.cts +9 -0
  46. package/dist/validator-runtime.d.mts +9 -0
  47. package/dist/validator-runtime.mjs +237 -0
  48. package/dist/validators.cjs +10 -0
  49. package/dist/validators.d.cts +8 -0
  50. package/dist/validators.d.mts +8 -0
  51. package/dist/validators.mjs +9 -0
  52. package/package.json +11 -1
  53. package/dist/client-5K3dMI5Q.mjs +0 -849
  54. package/dist/client-Ci0woWLW.cjs +0 -890
  55. package/dist/evals-BEgEmfiO.d.cts +0 -603
  56. package/dist/evals-BEgEmfiO.d.mts +0 -603
  57. package/dist/harness-profile-BMjsN460.mjs +0 -43
  58. package/dist/harness-profile-CDQpPLPd.cjs +0 -78
@@ -1,849 +0,0 @@
1
- import { n as HarnessProfile, o as profileFitsHarness, r as PROFILE_HARNESS_RULE } from "./harness-profile-BMjsN460.mjs";
2
- import { FetchHttpClient, HttpApi, HttpApiClient, HttpApiEndpoint, HttpApiGroup, HttpApiMiddleware, HttpApiSchema, HttpApiSecurity, HttpClient, HttpClientRequest, OpenApi } from "@effect/platform";
3
- import { Context, Effect, Layer, Redacted, Schema } from "effect";
4
- //#region ../schema/src/domain/prompts.ts
5
- const ChannelName = Schema.String.pipe(Schema.minLength(1), Schema.maxLength(36), Schema.pattern(/^[a-z0-9][a-z0-9_-]*$/, { message: () => "Channel must be lowercase alphanumeric, optionally with - or _" }), Schema.brand("ChannelName")).annotations({ description: "Addresses a version. `latest` is derived from the highest version rather than stored, so it cannot drift from the version table." });
6
- ChannelName.make("production");
7
- ChannelName.make("latest");
8
- const PromptId = Schema.String.pipe(Schema.minLength(1), Schema.maxLength(255), Schema.pattern(/^[a-z0-9][a-z0-9/_-]*$/, { message: () => "Prompt id must be lowercase alphanumeric, optionally with / _ or -" }), Schema.brand("PromptId"));
9
- const PromptName = Schema.String.pipe(Schema.minLength(1), Schema.maxLength(255), Schema.brand("PromptName"));
10
- const VersionNumber = Schema.Int.pipe(Schema.positive(), Schema.brand("VersionNumber"));
11
- Schema.NumberFromString.pipe(Schema.int(), Schema.positive(), Schema.brand("VersionNumber"));
12
- Schema.NumberFromString.pipe(Schema.int(), Schema.positive(), Schema.lessThanOrEqualTo(100));
13
- const Timestamp = Schema.Union(Schema.DateFromSelf, Schema.Date);
14
- const CommitMessage = Schema.String.pipe(Schema.maxLength(500));
15
- const PromptContent = Schema.String.pipe(Schema.minLength(1), Schema.maxLength(262144, { message: () => "Prompt content is longer than 256 KB" }));
16
- const PromptDescription = Schema.String.pipe(Schema.maxLength(2e3));
17
- const PromptConfig = Schema.Record({
18
- key: Schema.String,
19
- value: Schema.Unknown
20
- });
21
- const Author = Schema.Struct({
22
- image: Schema.NullOr(Schema.String),
23
- name: Schema.String
24
- });
25
- Schema.Struct({
26
- author: Schema.NullOr(Author),
27
- channel: Schema.NullOr(ChannelName),
28
- commitMessage: Schema.NullOr(CommitMessage),
29
- config: PromptConfig,
30
- content: Schema.String,
31
- createdAt: Timestamp,
32
- id: PromptId,
33
- name: PromptName,
34
- version: VersionNumber,
35
- versionId: Schema.String
36
- });
37
- Schema.Struct({
38
- channel: ChannelName,
39
- updatedAt: Timestamp,
40
- updatedBy: Schema.NullOr(Author),
41
- version: VersionNumber
42
- });
43
- const PromptSummary = Schema.Struct({
44
- author: Schema.NullOr(Author),
45
- description: Schema.NullOr(Schema.String),
46
- id: PromptId,
47
- latestVersion: Schema.NullOr(VersionNumber),
48
- name: PromptName,
49
- productionVersion: Schema.NullOr(VersionNumber),
50
- updatedAt: Timestamp
51
- });
52
- Schema.Struct({
53
- id: PromptId,
54
- updatedAt: Timestamp
55
- });
56
- Schema.Literal("all", "draft", "live");
57
- Schema.Literal("name", "updated");
58
- Schema.Struct({
59
- items: Schema.Array(PromptSummary),
60
- /** Opaque to callers, and null once the last page has been read. */
61
- nextCursor: Schema.NullOr(Schema.String)
62
- });
63
- Schema.Struct({
64
- commitMessage: Schema.optional(CommitMessage),
65
- config: Schema.optional(PromptConfig),
66
- content: PromptContent,
67
- description: Schema.optional(PromptDescription),
68
- id: PromptId,
69
- name: PromptName,
70
- publish: Schema.optional(Schema.Boolean)
71
- });
72
- Schema.Struct({
73
- commitMessage: Schema.optional(CommitMessage),
74
- config: Schema.optional(PromptConfig),
75
- content: PromptContent,
76
- publish: Schema.optional(Schema.Boolean)
77
- });
78
- Schema.Struct({
79
- commitMessage: Schema.optional(CommitMessage),
80
- config: Schema.optional(PromptConfig),
81
- content: PromptContent
82
- });
83
- Schema.Struct({
84
- description: Schema.optional(PromptDescription),
85
- id: Schema.optional(PromptId),
86
- name: Schema.optional(PromptName)
87
- });
88
- Schema.Struct({
89
- channel: ChannelName,
90
- version: VersionNumber
91
- });
92
- Schema.Struct({
93
- channel: Schema.optional(ChannelName),
94
- version: Schema.optional(VersionNumber)
95
- });
96
- //#endregion
97
- //#region ../schema/src/domain/errors.ts
98
- var NotFound = class extends Schema.TaggedError()("NotFound", { message: Schema.String }, HttpApiSchema.annotations({ status: 404 })) {};
99
- var Conflict = class extends Schema.TaggedError()("Conflict", { message: Schema.String }, HttpApiSchema.annotations({ status: 409 })) {};
100
- var BadRequest = class extends Schema.TaggedError()("BadRequest", { message: Schema.String }, HttpApiSchema.annotations({ status: 400 })) {};
101
- var Unauthorized = class extends Schema.TaggedError()("Unauthorized", { message: Schema.String }, HttpApiSchema.annotations({ status: 401 })) {};
102
- /** Distinct from Unauthorized: the caller is who they say they are, and the
103
- * answer is still no. Signing in again cannot help, so the client must not
104
- * treat this as an expired session. */
105
- var Forbidden = class extends Schema.TaggedError()("Forbidden", { message: Schema.String }, HttpApiSchema.annotations({ status: 403 })) {};
106
- Schema.TaggedError()("InternalError", { message: Schema.String }, HttpApiSchema.annotations({ status: 500 }));
107
- //#endregion
108
- //#region ../schema/src/domain/eval-limits.ts
109
- /**
110
- * How long a string a trial may carry to the sandbox.
111
- *
112
- * Every one of these is `shellQuote`d into a single command line and handed to
113
- * `sh -c` as one argv element, and Linux caps one argument at `MAX_ARG_STRLEN`
114
- * -- 128 KiB. Past that the trial dies as an opaque `E2BIG` inside a VM that
115
- * is already billing, which reads as a broken provider rather than as a
116
- * request nobody should have accepted.
117
- *
118
- * The budget is in characters against a byte ceiling, so it holds for text
119
- * that is not ASCII: 128 KiB is 32768 characters even where every one of them
120
- * costs the four bytes UTF-8 allows. The prompt takes half of that and the
121
- * verifier a quarter, leaving the rest for the harness's own flags, the
122
- * workspace path, the model name, the profile arguments, and the escaping
123
- * `shellQuote` adds -- which turns each apostrophe into four characters.
124
- *
125
- * These are ceilings on what the wire accepts, not targets. The prompts people
126
- * actually send are hundreds of characters and the longest verifier stored is
127
- * under three thousand.
128
- */
129
- const PROMPT_LIMIT = 16384;
130
- const VERIFY_LIMIT = 8192;
131
- const NAME_LIMIT = 200;
132
- const VARIABLE_VALUE_LIMIT = 2048;
133
- const EvalPrompt = Schema.String.pipe(Schema.maxLength(PROMPT_LIMIT), Schema.annotations({
134
- description: "What the agent is asked to do.",
135
- identifier: "EvalPrompt",
136
- message: () => `A prompt must be at most ${PROMPT_LIMIT} characters, because it is quoted into one sandbox command line.`
137
- }));
138
- const EvalVerify = Schema.String.pipe(Schema.maxLength(VERIFY_LIMIT), Schema.annotations({
139
- description: "The shell command that decides whether a trial passed.",
140
- identifier: "EvalVerify",
141
- message: () => `A verifier must be at most ${VERIFY_LIMIT} characters, because it is quoted into one sandbox command line.`
142
- }));
143
- const EvalCaseName = Schema.String.pipe(Schema.maxLength(NAME_LIMIT), Schema.annotations({
144
- description: "What a case is called.",
145
- identifier: "EvalCaseName",
146
- message: () => `A case name must be at most ${NAME_LIMIT} characters.`
147
- }));
148
- const EvalVariableValue = Schema.String.pipe(Schema.maxLength(VARIABLE_VALUE_LIMIT), Schema.annotations({
149
- identifier: "EvalVariableValue",
150
- message: () => `A variable value must be at most ${VARIABLE_VALUE_LIMIT} characters, because it is substituted into the prompt before it is quoted.`
151
- }));
152
- //#endregion
153
- //#region ../schema/src/domain/credentials.ts
154
- const CredentialScope = Schema.Literal("organization", "personal");
155
- const CredentialStatus = Schema.Literal("active", "invalid");
156
- const CredentialField = Schema.Struct({
157
- /** The shape of the value, shown as a placeholder beside the label. Absent
158
- * where the label already says it, as a bare API key does. */
159
- hint: Schema.optional(Schema.String),
160
- label: Schema.String,
161
- name: Schema.String,
162
- required: Schema.Boolean,
163
- secret: Schema.Boolean
164
- });
165
- const CredentialAuthMethod = Schema.Struct({
166
- fields: Schema.Array(CredentialField),
167
- id: Schema.String,
168
- kind: Schema.Literal("secret", "device", "env"),
169
- label: Schema.String
170
- });
171
- Schema.Struct({
172
- authMethods: Schema.Array(CredentialAuthMethod),
173
- category: Schema.Literal("harness", "sandbox"),
174
- id: Schema.String,
175
- label: Schema.String
176
- });
177
- Schema.Struct({
178
- integrationId: Schema.String,
179
- /** Display names of the members holding one, so the reader knows who to
180
- * ask. Never ids: this is read by everyone in the organization. */
181
- owners: Schema.Array(Schema.String)
182
- }).annotations({
183
- description: "Who else in the organization has an integration connected.",
184
- identifier: "IntegrationAwareness"
185
- });
186
- Schema.Struct({
187
- authMethodId: Schema.String,
188
- createdAt: Schema.DateTimeUtc,
189
- id: Schema.String,
190
- integrationId: Schema.String,
191
- isDefault: Schema.Boolean,
192
- lastUsedAt: Schema.NullOr(Schema.DateTimeUtc),
193
- name: Schema.String,
194
- scope: CredentialScope,
195
- status: CredentialStatus
196
- });
197
- const CredentialValues = Schema.Record({
198
- key: Schema.String,
199
- value: Schema.String
200
- });
201
- Schema.Struct({
202
- authMethodId: Schema.String,
203
- connectionId: Schema.String,
204
- integrationId: Schema.String,
205
- revision: Schema.Int,
206
- values: CredentialValues
207
- });
208
- Schema.Struct({
209
- authMethodId: Schema.String,
210
- integrationId: Schema.String,
211
- isDefault: Schema.optionalWith(Schema.Boolean, { default: () => false }),
212
- name: Schema.String.pipe(Schema.minLength(1), Schema.maxLength(80)),
213
- scope: CredentialScope,
214
- values: CredentialValues
215
- });
216
- const CredentialBindings = Schema.Struct({
217
- harnessConnectionId: Schema.optional(Schema.String),
218
- sandboxConnectionId: Schema.optional(Schema.String)
219
- });
220
- const CredentialSelections = Schema.Record({
221
- key: Schema.String,
222
- value: Schema.String
223
- });
224
- Schema.Struct({ values: CredentialValues });
225
- Schema.Struct({
226
- integrationId: Schema.Literal("codex"),
227
- name: Schema.String.pipe(Schema.minLength(1), Schema.maxLength(80)),
228
- scope: CredentialScope
229
- });
230
- Schema.Struct({
231
- attemptId: Schema.String,
232
- code: Schema.String,
233
- expiresAt: Schema.DateTimeUtc,
234
- verificationUrl: Schema.String
235
- });
236
- Schema.Struct({
237
- connectionId: Schema.NullOr(Schema.String),
238
- status: Schema.Literal("pending", "complete", "failed", "expired")
239
- });
240
- //#endregion
241
- //#region ../schema/src/domain/evals.ts
242
- const EvalProvider = Schema.Literal("daytona", "e2b", "upstash", "modal", "cloudflare", "vercel");
243
- EvalProvider.literals;
244
- const EvalHarness = Schema.Literal("codex", "opencode", "pi", "fx", "claude", "gemini", "qwen", "cursor", "command");
245
- const EvalTrialStatus = Schema.Literal("queued", "running", "passed", "failed", "void");
246
- const EvalRunStatus = Schema.Literal("running", "finished", "failed");
247
- const EvalSource = Schema.Union(Schema.Struct({ kind: Schema.Literal("empty") }), Schema.Struct({
248
- kind: Schema.Literal("repo"),
249
- ref: Schema.NullOr(Schema.String),
250
- url: Schema.String.pipe(Schema.minLength(1), Schema.annotations({ message: () => "Give the repository a URL." }))
251
- }), Schema.Struct({
252
- files: Schema.Record({
253
- key: Schema.String,
254
- value: Schema.String
255
- }),
256
- kind: Schema.Literal("files")
257
- })).annotations({
258
- description: "The workspace available to the harness before setup runs.",
259
- identifier: "EvalSource"
260
- });
261
- const EvalValidator = Schema.Struct({
262
- name: Schema.String.pipe(Schema.minLength(1), Schema.maxLength(100)),
263
- source: Schema.String.pipe(Schema.minLength(1), Schema.maxLength(1e6))
264
- }).annotations({
265
- description: "A bundled TypeScript validator and its exported function name.",
266
- identifier: "EvalValidator"
267
- });
268
- const EvalPrepare = Schema.Struct({
269
- name: Schema.String.pipe(Schema.minLength(1), Schema.maxLength(100)),
270
- source: Schema.String.pipe(Schema.minLength(1), Schema.maxLength(1e6))
271
- }).annotations({
272
- description: "A bundled TypeScript workspace setup and its exported function name.",
273
- identifier: "EvalPrepare"
274
- });
275
- const EvalPrepareValue = Schema.Record({
276
- key: Schema.String,
277
- value: Schema.Unknown
278
- }).annotations({
279
- description: "What a workspace setup returned, handed to the validator.",
280
- identifier: "EvalPrepareValue"
281
- });
282
- const EvalVariables = Schema.Record({
283
- key: Schema.String,
284
- value: EvalVariableValue
285
- }).annotations({
286
- description: "Values for the placeholders the run prompt names, such as {{task}}.",
287
- identifier: "EvalVariables"
288
- });
289
- /**
290
- * A directory worth keeping between runs of a case.
291
- *
292
- * Declared on the case rather than reported by its prepare, because a restore
293
- * happens before the prepare runs and so cannot be told where to look by it.
294
- * The shape CI caches use, for the same reason.
295
- */
296
- const CaseCache = Schema.Struct({
297
- key: Schema.String.pipe(Schema.minLength(1)),
298
- path: Schema.String.pipe(Schema.minLength(1), Schema.filter((value) => !(value.startsWith("/") || value.split("/").includes("..")), { message: () => "a cache path must stay inside the workspace" }))
299
- }).annotations({ identifier: "CaseCache" });
300
- const EvalCase = Schema.Struct({
301
- cache: Schema.optional(CaseCache),
302
- name: EvalCaseName,
303
- prepare: Schema.NullOr(EvalPrepare),
304
- source: EvalSource,
305
- variables: Schema.optionalWith(EvalVariables, { default: () => ({}) }),
306
- validator: Schema.optionalWith(Schema.NullOr(EvalValidator), { default: () => null }),
307
- verify: Schema.NullOr(EvalVerify)
308
- });
309
- const EvalTaskProfile = Schema.Struct({
310
- name: Schema.String,
311
- version: Schema.String
312
- }).annotations({
313
- description: "The profile a cell's harness ran under, by name and version.",
314
- identifier: "EvalTaskProfile"
315
- });
316
- const EvalTask = Schema.Struct({
317
- harness: EvalHarness,
318
- harnessVersion: Schema.String,
319
- model: Schema.String,
320
- profile: Schema.optional(Schema.NullOr(EvalTaskProfile)),
321
- provider: EvalProvider
322
- }).annotations({
323
- description: "The harness, installed version, profile, model, and sandbox for a cell.",
324
- identifier: "EvalTask"
325
- });
326
- const EvalTaskRequest = Schema.Struct({
327
- credentials: Schema.optional(CredentialBindings),
328
- harness: EvalHarness,
329
- model: Schema.String,
330
- profile: Schema.optional(HarnessProfile),
331
- provider: EvalProvider
332
- }).pipe(Schema.filter(profileFitsHarness, { message: () => PROFILE_HARNESS_RULE }));
333
- const EvalName = Schema.String.pipe(Schema.minLength(1), Schema.maxLength(100));
334
- Schema.Struct({
335
- cases: Schema.Array(EvalCase).pipe(Schema.minItems(1), Schema.maxItems(100)),
336
- name: Schema.optional(EvalName),
337
- prompt: EvalPrompt,
338
- tasks: Schema.Array(EvalTaskRequest).pipe(Schema.minItems(1), Schema.maxItems(20)),
339
- trials: Schema.Int.pipe(Schema.between(1, 10))
340
- });
341
- const OccurredAtMillis = Schema.NullOr(Schema.Number);
342
- const EvalUsage = Schema.Struct({
343
- cacheReadTokens: Schema.Int,
344
- cacheWriteTokens: Schema.Int,
345
- costUsd: Schema.optional(Schema.NullOr(Schema.Number)),
346
- inputTokens: Schema.Int,
347
- outputTokens: Schema.Int,
348
- totalTokens: Schema.Int
349
- }).annotations({
350
- description: "Token usage reported by the harness.",
351
- identifier: "EvalUsage"
352
- });
353
- const EvalJournalEntry = Schema.Union(Schema.Struct({
354
- _tag: Schema.Literal("command"),
355
- command: Schema.String,
356
- exitCode: Schema.NullOr(Schema.Int),
357
- finishedAtMillis: OccurredAtMillis,
358
- output: Schema.String,
359
- startedAtMillis: OccurredAtMillis
360
- }), Schema.Struct({
361
- _tag: Schema.Literal("message"),
362
- finishedAtMillis: OccurredAtMillis,
363
- text: Schema.String,
364
- usage: Schema.optional(Schema.NullOr(EvalUsage))
365
- }), Schema.Struct({
366
- _tag: Schema.Literal("toolCall"),
367
- finishedAtMillis: OccurredAtMillis,
368
- name: Schema.String,
369
- startedAtMillis: Schema.optional(OccurredAtMillis),
370
- status: Schema.NullOr(Schema.String)
371
- }), Schema.Struct({
372
- _tag: Schema.Literal("fileChange"),
373
- finishedAtMillis: OccurredAtMillis,
374
- paths: Schema.Array(Schema.String)
375
- })).annotations({
376
- description: "A normalized event recorded from the harness trajectory.",
377
- identifier: "EvalJournalEntry"
378
- });
379
- const EvalVerifyStep = Schema.Struct({
380
- command: Schema.String,
381
- exitCode: Schema.Int
382
- }).annotations({
383
- description: "One condition of the verifier, and how it exited. Only the steps that ran are listed: the script stops at the first failure.",
384
- identifier: "EvalVerifyStep"
385
- });
386
- /**
387
- * How much of a cost is known, and on what basis.
388
- *
389
- * The distinction is the point: a public-rate calculation is not an invoice, a
390
- * subscription's marginal price is not zero, and a cost the platform absorbs
391
- * is not one the customer paid. Collapsing any of those into a number produces
392
- * a total that reads as authoritative and is not.
393
- */
394
- const CostClassification = Schema.Literal("actual", "allocated", "estimate", "included", "managed", "unknown");
395
- const CostComponentName = Schema.Literal("harness", "model", "platform", "sandbox");
396
- const EvalCostComponent = Schema.Struct({
397
- classification: CostClassification,
398
- component: CostComponentName,
399
- detail: Schema.Record({
400
- key: Schema.String,
401
- value: Schema.Unknown
402
- }),
403
- explanation: Schema.String,
404
- source: Schema.String,
405
- usd: Schema.NullOr(Schema.Number)
406
- }).annotations({
407
- description: "What one layer of a trial cost, and how far that is known.",
408
- identifier: "EvalCostComponent"
409
- });
410
- /**
411
- * What a run, case, or trial cost, kept apart by how it is known.
412
- *
413
- * No single total, deliberately. Adding an estimate to an actual charge and an
414
- * allocated share produces a number that means none of the three.
415
- */
416
- const EvalCosts = Schema.Struct({
417
- allocatedUsd: Schema.Number,
418
- components: Schema.Array(EvalCostComponent),
419
- estimatedEquivalentUsd: Schema.Number,
420
- incomplete: Schema.Boolean,
421
- knownActualUsd: Schema.Number
422
- }).annotations({
423
- description: "Cost by component, kept apart by classification.",
424
- identifier: "EvalCosts"
425
- });
426
- const EvalTrial = Schema.Struct({
427
- commands: Schema.Int,
428
- costs: Schema.NullOr(EvalCosts),
429
- prepared: Schema.NullOr(EvalPrepareValue),
430
- exitCode: Schema.Int,
431
- failedCommands: Schema.Int,
432
- filesChanged: Schema.Array(Schema.String),
433
- modelMs: Schema.Int,
434
- ordinal: Schema.Int,
435
- passed: Schema.Boolean,
436
- sandboxId: Schema.NullOr(Schema.String),
437
- sandboxMs: Schema.Int,
438
- status: EvalTrialStatus,
439
- timed: Schema.Boolean,
440
- trajectory: Schema.Array(EvalJournalEntry),
441
- usage: Schema.NullOr(EvalUsage),
442
- verifySteps: Schema.Array(EvalVerifyStep),
443
- voidFields: Schema.Array(Schema.String)
444
- }).annotations({
445
- description: "One sandbox attempt for a grid cell.",
446
- identifier: "EvalTrial"
447
- });
448
- const EvalDistribution = Schema.Struct({
449
- commandMax: Schema.Int,
450
- commandMedian: Schema.Number,
451
- commandMin: Schema.Int,
452
- deterministic: Schema.Boolean,
453
- failed: Schema.Int,
454
- passRate: Schema.Number,
455
- passed: Schema.Int,
456
- scored: Schema.Int,
457
- trials: Schema.Int,
458
- voided: Schema.Int
459
- }).annotations({
460
- description: "The scored outcome across all trials in a cell.",
461
- identifier: "EvalDistribution"
462
- });
463
- const EvalVerdict = Schema.Literal("improved", "incomparable", "regressed", "unchanged");
464
- const EvalComparison = Schema.Struct({
465
- baselineHarnessVersion: Schema.String,
466
- baselinePassRate: Schema.Number,
467
- baselineProfileVersion: Schema.NullOr(Schema.String),
468
- candidateHarnessVersion: Schema.String,
469
- candidatePassRate: Schema.Number,
470
- candidateProfileVersion: Schema.NullOr(Schema.String),
471
- delta: Schema.Number,
472
- determinismLost: Schema.Boolean,
473
- reason: Schema.NullOr(Schema.String),
474
- verdict: EvalVerdict
475
- }).annotations({
476
- description: "The cell result compared with its most recent baseline.",
477
- identifier: "EvalComparison"
478
- });
479
- const EvalSetup = Schema.Struct({
480
- prompt: Schema.String,
481
- repoRef: Schema.NullOr(Schema.String),
482
- repoUrl: Schema.NullOr(Schema.String),
483
- prepareName: Schema.NullOr(Schema.String),
484
- validatorName: Schema.NullOr(Schema.String),
485
- verifyCommand: Schema.NullOr(Schema.String),
486
- workspace: Schema.String
487
- }).annotations({
488
- description: "The prompt, workspace, setup, and verifier used by a cell.",
489
- identifier: "EvalSetup"
490
- });
491
- const EvalCell = Schema.Struct({
492
- caseName: Schema.String,
493
- cellKey: Schema.NullOr(Schema.String),
494
- costs: Schema.NullOr(EvalCosts),
495
- comparison: Schema.NullOr(EvalComparison),
496
- distribution: Schema.NullOr(EvalDistribution),
497
- internalId: Schema.NullOr(Schema.String),
498
- setup: Schema.NullOr(EvalSetup),
499
- status: EvalRunStatus,
500
- taskIndex: Schema.Int,
501
- trials: Schema.Array(EvalTrial)
502
- }).annotations({
503
- description: "One case and task combination in an eval grid.",
504
- identifier: "EvalCell"
505
- });
506
- const EvalTimestamp = Schema.DateTimeUtc.annotations({
507
- description: "An ISO-8601 timestamp in UTC.",
508
- identifier: "EvalTimestamp",
509
- jsonSchema: { format: "date-time" }
510
- });
511
- const EvalRun = Schema.Struct({
512
- cases: Schema.Array(Schema.String),
513
- cells: Schema.Array(EvalCell),
514
- costs: Schema.NullOr(EvalCosts),
515
- failure: Schema.NullOr(Schema.String),
516
- finishedAt: Schema.NullOr(EvalTimestamp),
517
- id: Schema.String,
518
- name: Schema.NullOr(EvalName),
519
- startedAt: EvalTimestamp,
520
- status: EvalRunStatus,
521
- tasks: Schema.Array(EvalTask)
522
- }).annotations({
523
- description: "A complete eval run with its cells and trials.",
524
- identifier: "EvalRun"
525
- });
526
- const EvalRunSummary = Schema.Struct({
527
- caseCount: Schema.Int,
528
- columns: Schema.Array(EvalTask),
529
- commandMax: Schema.NullOr(Schema.Int),
530
- commandMin: Schema.NullOr(Schema.Int),
531
- failure: Schema.NullOr(Schema.String),
532
- finishedAt: Schema.NullOr(EvalTimestamp),
533
- firstCaseName: Schema.NullOr(Schema.String),
534
- id: Schema.String,
535
- name: Schema.NullOr(Schema.String),
536
- passed: Schema.Int,
537
- scored: Schema.Int,
538
- startedAt: EvalTimestamp,
539
- status: EvalRunStatus,
540
- taskCount: Schema.Int,
541
- voided: Schema.Int
542
- }).annotations({
543
- description: "A compact eval run returned by the run list.",
544
- identifier: "EvalRunSummary"
545
- });
546
- const EvalPageCursor = Schema.Struct({
547
- id: Schema.String,
548
- startedAtMillis: Schema.Int
549
- });
550
- /** One page of runs. `next` is null at the end rather than an empty cursor, so
551
- * a caller stops because there is nothing more rather than because a fetch
552
- * came back empty. */
553
- const EvalRunPage = Schema.Struct({
554
- next: Schema.NullOr(EvalPageCursor),
555
- runs: Schema.Array(EvalRunSummary),
556
- /** Every run the organization has, so a listing can say how far it goes
557
- * rather than only whether there is more. */
558
- total: Schema.Int
559
- });
560
- const EvalCellHistoryEntry = Schema.Struct({
561
- distribution: EvalDistribution,
562
- finishedAt: Schema.NullOr(EvalTimestamp),
563
- harnessVersion: Schema.String,
564
- internalId: Schema.String,
565
- profileVersion: Schema.NullOr(Schema.String),
566
- runId: Schema.String,
567
- trials: Schema.Array(EvalTrial)
568
- }).annotations({
569
- description: "A previous scored result for the same cell identity.",
570
- identifier: "EvalCellHistoryEntry"
571
- });
572
- const PlaygroundCaseView = Schema.Struct({
573
- name: Schema.String,
574
- setup: Schema.NullOr(Schema.String),
575
- source: EvalSource,
576
- variables: Schema.optionalWith(EvalVariables, { default: () => ({}) }),
577
- verify: Schema.NullOr(Schema.String)
578
- });
579
- const PlaygroundColumnView = Schema.Struct({
580
- harness: EvalHarness,
581
- model: Schema.String,
582
- provider: EvalProvider
583
- });
584
- const PlaygroundConfigView = Schema.Struct({
585
- cases: Schema.Array(PlaygroundCaseView),
586
- columns: Schema.Array(PlaygroundColumnView),
587
- connections: Schema.optionalWith(CredentialSelections, { default: () => ({}) }),
588
- prompt: Schema.String,
589
- trials: Schema.Int.pipe(Schema.between(1, 10))
590
- });
591
- const EvalDraftCase = Schema.Struct({
592
- name: Schema.String.pipe(Schema.minLength(1), Schema.annotations({ message: () => "Name this case." })),
593
- setup: Schema.NullOr(Schema.String),
594
- source: EvalSource,
595
- variables: EvalVariables.pipe(Schema.filter((values) => Object.values(values).some((value) => value.trim() !== ""), { message: () => "Say what the agent should do." })),
596
- verify: Schema.NullOr(Schema.String)
597
- });
598
- const EvalAgent = Schema.Struct({
599
- harness: EvalHarness,
600
- model: Schema.String
601
- });
602
- Schema.Struct({
603
- agents: Schema.mutable(Schema.Array(EvalAgent)).pipe(Schema.minItems(1), Schema.annotations({ message: () => "Choose at least one agent." })),
604
- cases: Schema.mutable(Schema.Array(EvalDraftCase)).pipe(Schema.minItems(1), Schema.annotations({ message: () => "Add at least one case." })),
605
- connections: CredentialSelections,
606
- name: Schema.String,
607
- prompt: Schema.String,
608
- providers: Schema.mutable(Schema.Array(EvalProvider)).pipe(Schema.minItems(1), Schema.annotations({ message: () => "Choose at least one sandbox." })),
609
- trials: Schema.Int.pipe(Schema.between(1, 10), Schema.annotations({ message: () => "Run between 1 and 10 trials." }))
610
- });
611
- Schema.Struct({
612
- config: PlaygroundConfigView,
613
- id: Schema.String,
614
- lastRunId: Schema.NullOr(Schema.String),
615
- name: Schema.String,
616
- problems: Schema.Array(Schema.String),
617
- ungated: Schema.Array(Schema.String),
618
- updatedAt: Schema.DateTimeUtc
619
- });
620
- Schema.Struct({ name: Schema.String.pipe(Schema.minLength(1)) });
621
- Schema.Struct({
622
- config: PlaygroundConfigView,
623
- name: Schema.String.pipe(Schema.minLength(1))
624
- });
625
- const StartedEval = Schema.Struct({ id: Schema.String }).annotations({
626
- description: "The id of an eval run accepted for background execution.",
627
- identifier: "StartedEval"
628
- });
629
- const RerunCellRequest = Schema.Struct({ trials: Schema.Int.pipe(Schema.between(1, 10)) });
630
- const CatalogueModel = Schema.Struct({
631
- displayName: Schema.String,
632
- id: Schema.String,
633
- summary: Schema.NullOr(Schema.String),
634
- vendor: Schema.NullOr(Schema.String)
635
- }).annotations({
636
- description: "A model available to the installed harness.",
637
- identifier: "CatalogueModel"
638
- });
639
- const ModelCatalogue = Schema.Struct({
640
- harness: EvalHarness,
641
- models: Schema.Array(CatalogueModel),
642
- total: Schema.Int
643
- }).annotations({
644
- description: "Models available to the installed harness.",
645
- identifier: "ModelCatalogue"
646
- });
647
- //#endregion
648
- //#region ../schema/src/internal/authentication.ts
649
- var CurrentActor = class extends Context.Tag("@anpord/schema/CurrentActor")() {};
650
- HttpApiMiddleware.Tag()("@anpord/schema/Authentication", {
651
- failure: Unauthorized,
652
- provides: CurrentActor,
653
- security: { session: HttpApiSecurity.apiKey({
654
- in: "cookie",
655
- key: "anpord.session_token"
656
- }) }
657
- });
658
- //#endregion
659
- //#region ../schema/src/public/authentication.ts
660
- var ApiKeyAuthentication = class extends HttpApiMiddleware.Tag()("@anpord/schema/ApiKeyAuthentication", {
661
- failure: Unauthorized,
662
- provides: CurrentActor,
663
- security: { bearer: HttpApiSecurity.bearer }
664
- }) {};
665
- //#endregion
666
- //#region ../schema/src/public/evals-api.ts
667
- const EvalRunRequest = Schema.Struct({ id: Schema.String }).annotations({
668
- description: "Select an eval run by id.",
669
- identifier: "EvalRunRequest"
670
- });
671
- const EvalCellRequest = Schema.Struct({ cellKey: Schema.String }).annotations({
672
- description: "Select an eval cell by its stable key.",
673
- identifier: "EvalCellRequest"
674
- });
675
- const EvalModelsRequest = Schema.Struct({
676
- harness: EvalHarness,
677
- /** Narrows to models whose id or name contains this. */
678
- q: Schema.optional(Schema.String)
679
- }).annotations({
680
- description: "Select a harness whose available models should be listed.",
681
- identifier: "EvalModelsRequest"
682
- });
683
- const PublicEvalProvider = EvalProvider.annotations({
684
- description: "A hosted sandbox provider.",
685
- identifier: "PublicEvalProvider"
686
- });
687
- const ListEvalsRequest = Schema.Struct({
688
- cursor: Schema.optional(Schema.NullOr(EvalPageCursor)),
689
- limit: Schema.optional(Schema.Int)
690
- }).annotations({
691
- description: "Where to read from, and how much.",
692
- identifier: "ListEvalsRequest"
693
- });
694
- const PublicEvalCase = Schema.Struct({
695
- cache: Schema.optional(CaseCache),
696
- name: EvalCaseName,
697
- prepare: Schema.optional(Schema.NullOr(EvalPrepare)),
698
- source: Schema.optional(EvalSource),
699
- validator: Schema.optional(Schema.NullOr(EvalValidator)),
700
- variables: Schema.optional(EvalVariables),
701
- verify: Schema.NullOr(EvalVerify)
702
- }).pipe(Schema.filter(({ validator, verify }) => !(validator !== void 0 && validator !== null && verify !== null), { message: () => "Use either validator or verify, not both." })).annotations({
703
- description: "A task, workspace source, setup command, and verifier.",
704
- identifier: "StartEvalCase"
705
- });
706
- const PublicEvalTask = Schema.Struct({
707
- harness: EvalHarness,
708
- model: Schema.String.pipe(Schema.minLength(1)),
709
- profile: Schema.optional(HarnessProfile),
710
- provider: PublicEvalProvider
711
- }).pipe(Schema.filter(profileFitsHarness, { message: () => PROFILE_HARNESS_RULE })).annotations({
712
- description: `A harness, model, and hosted sandbox combination, with an optional profile layered on the harness. ${PROFILE_HARNESS_RULE}`,
713
- identifier: "StartEvalTask"
714
- });
715
- const PublicStartEvalRequest = Schema.Struct({
716
- cases: Schema.Array(PublicEvalCase).pipe(Schema.minItems(1), Schema.maxItems(100)),
717
- name: Schema.optional(EvalName),
718
- prompt: EvalPrompt,
719
- tasks: Schema.Array(PublicEvalTask).pipe(Schema.minItems(1), Schema.maxItems(20)),
720
- trials: Schema.Int.pipe(Schema.between(1, 10))
721
- }).annotations({
722
- description: `Start a grid with at most 100 total case, task, and trial combinations.`,
723
- identifier: "StartEvalRequest"
724
- });
725
- var PublicEvalsGroup = class extends HttpApiGroup.make("evals").add(HttpApiEndpoint.post("list", "/evals.list").setPayload(ListEvalsRequest).addSuccess(EvalRunPage).annotate(OpenApi.Summary, "List eval runs").annotate(OpenApi.Description, "Newest first. Pass the `next` cursor from a response to read the page after it; a null `next` means there are no more.")).add(HttpApiEndpoint.post("start", "/evals.start").setPayload(PublicStartEvalRequest).addSuccess(StartedEval).annotate(OpenApi.Summary, "Start an eval run").annotate(OpenApi.Description, `Starts the grid and returns its id while trials continue in the background. ${PROFILE_HARNESS_RULE}`)).add(HttpApiEndpoint.post("get", "/evals.get").setPayload(EvalRunRequest).addSuccess(EvalRun).annotate(OpenApi.Summary, "Get an eval run")).add(HttpApiEndpoint.post("cellHistory", "/evals.cellHistory").setPayload(EvalCellRequest).addSuccess(Schema.Array(EvalCellHistoryEntry)).annotate(OpenApi.Summary, "List a cell's history").annotate(OpenApi.Description, "Returns the 20 most recent results.")).add(HttpApiEndpoint.post("rerunCell", "/evals.rerunCell").setPayload(Schema.extend(EvalRunRequest, Schema.extend(EvalCellRequest, RerunCellRequest))).addSuccess(StartedEval).annotate(OpenApi.Summary, "Rerun one cell")).add(HttpApiEndpoint.post("models", "/evals.models").setPayload(EvalModelsRequest).addSuccess(ModelCatalogue).annotate(OpenApi.Summary, "List models available to the harness").annotate(OpenApi.Description, "The command harness has no catalogue of its own, so its list is empty: the model is whatever the profile's run command reads from ANPORD_MODEL.")).addError(BadRequest).addError(Conflict).addError(Forbidden).addError(NotFound).middleware(ApiKeyAuthentication).annotate(OpenApi.Title, "Evals").annotate(OpenApi.Description, "Run cases across harness, model, and sandbox combinations and compare the results with their baselines.") {};
726
- //#endregion
727
- //#region ../schema/src/public/requests.ts
728
- const GetPromptRequest = Schema.Struct({
729
- channel: Schema.optional(ChannelName),
730
- id: PromptId,
731
- includeVersions: Schema.optional(Schema.Boolean),
732
- version: Schema.optional(VersionNumber)
733
- }).annotations({
734
- description: "Resolve a prompt. Give a version to pin one, a channel to follow one, or neither for the organization's default channel.",
735
- identifier: "GetPromptRequest"
736
- });
737
- Schema.Struct({ id: PromptId }).annotations({
738
- description: "Show a prompt's version history.",
739
- identifier: "ListVersionsRequest"
740
- });
741
- const ListPromptsRequest = Schema.Record({
742
- key: Schema.String,
743
- value: Schema.Never
744
- }).annotations({
745
- description: "No parameters; returns up to 100 prompts in the organization.",
746
- identifier: "ListPromptsRequest"
747
- });
748
- const CreatePromptRequest = Schema.Struct({
749
- config: Schema.optional(PromptConfig),
750
- content: PromptContent,
751
- description: Schema.optional(Schema.String),
752
- id: PromptId,
753
- message: Schema.optional(CommitMessage),
754
- name: PromptName
755
- }).annotations({
756
- description: "Create a prompt and publish its first version to the default channel.",
757
- identifier: "CreatePromptRequest"
758
- });
759
- const UpdatePromptRequest = Schema.Struct({
760
- config: Schema.optional(PromptConfig),
761
- content: PromptContent,
762
- id: PromptId,
763
- message: Schema.optional(CommitMessage)
764
- }).annotations({
765
- description: "Add a version to a prompt. The new version becomes latest, and omitted config is stored as an empty object.",
766
- identifier: "UpdatePromptRequest"
767
- });
768
- const PromotePromptRequest = Schema.Struct({
769
- channel: ChannelName,
770
- id: PromptId,
771
- version: VersionNumber
772
- }).annotations({
773
- description: "Point a channel at a version.",
774
- identifier: "PromotePromptRequest"
775
- });
776
- Schema.Struct({ id: PromptId }).annotations({
777
- description: "Archive a prompt. Existing selectors stay readable.",
778
- identifier: "ArchivePromptRequest"
779
- });
780
- const Ok = Schema.Struct({ ok: Schema.Literal(true) }).annotations({
781
- description: "The operation succeeded.",
782
- identifier: "Ok"
783
- });
784
- //#endregion
785
- //#region ../schema/src/public/shapes.ts
786
- const Instant = Schema.DateTimeUtc.annotations({
787
- description: "An ISO-8601 timestamp in UTC.",
788
- identifier: "Instant",
789
- jsonSchema: { format: "date-time" }
790
- });
791
- const PublicPrompt = Schema.Struct({
792
- channel: Schema.NullOr(ChannelName),
793
- config: PromptConfig,
794
- content: Schema.String,
795
- createdAt: Instant,
796
- id: PromptId,
797
- message: Schema.NullOr(CommitMessage),
798
- name: PromptName,
799
- version: VersionNumber
800
- }).annotations({
801
- description: "A prompt resolved at a specific version.",
802
- identifier: "ResolvedPrompt"
803
- });
804
- const PublicVersion = Schema.Struct({
805
- createdAt: Instant,
806
- message: Schema.NullOr(CommitMessage),
807
- version: VersionNumber
808
- }).annotations({
809
- description: "One entry in a prompt's history.",
810
- identifier: "Version"
811
- });
812
- const PublicPromptWithVersions = Schema.extend(PublicPrompt, Schema.Struct({ versions: Schema.optional(Schema.Array(PublicVersion)) })).annotations({
813
- description: "A prompt, with its version history when `includeVersions` is set.",
814
- identifier: "Prompt"
815
- });
816
- const PublicPromptSummary = Schema.Struct({
817
- id: PromptId,
818
- latestVersion: Schema.NullOr(VersionNumber),
819
- name: PromptName,
820
- productionVersion: Schema.NullOr(VersionNumber),
821
- updatedAt: Instant
822
- }).annotations({
823
- description: "A prompt without its content.",
824
- identifier: "PromptSummary"
825
- });
826
- const PromptList = Schema.Struct({ data: Schema.Array(PublicPromptSummary) }).annotations({
827
- description: "Every prompt in the organization.",
828
- identifier: "PromptList"
829
- });
830
- //#endregion
831
- //#region ../schema/src/public/prompts-api.ts
832
- var PublicPromptsGroup = class extends HttpApiGroup.make("prompts").add(HttpApiEndpoint.post("get", "/prompts.get").setPayload(GetPromptRequest).addSuccess(PublicPromptWithVersions).annotate(OpenApi.Summary, "Resolve a prompt").annotate(OpenApi.Description, "Returns the content a caller should send to a model. With no selector this follows the organization's default channel.")).add(HttpApiEndpoint.post("list", "/prompts.list").setPayload(ListPromptsRequest).addSuccess(PromptList).annotate(OpenApi.Summary, "List prompts").annotate(OpenApi.Description, "Up to 100 prompts in the organization, without content.")).add(HttpApiEndpoint.post("create", "/prompts.create").setPayload(CreatePromptRequest).addSuccess(PublicPromptWithVersions).annotate(OpenApi.Summary, "Create a prompt").annotate(OpenApi.Description, "Creates the prompt and its first version in one call.")).add(HttpApiEndpoint.post("update", "/prompts.update").setPayload(UpdatePromptRequest).addSuccess(PublicPromptWithVersions).annotate(OpenApi.Summary, "Add a version").annotate(OpenApi.Description, "Content is versioned, so updating a prompt appends a version rather than overwriting one. Earlier versions stay readable.")).add(HttpApiEndpoint.post("promote", "/prompts.promote").setPayload(PromotePromptRequest).addSuccess(Ok).annotate(OpenApi.Summary, "Promote a version to a channel").annotate(OpenApi.Description, "Points a channel, such as production, at a version. This is how a version goes live without callers changing anything.")).addError(BadRequest).addError(Forbidden).addError(Conflict).addError(NotFound).middleware(ApiKeyAuthentication).annotate(OpenApi.Title, "Prompts").annotate(OpenApi.Description, "Read and write prompts. Content is versioned, so writes append rather than overwrite, and channels decide which version callers receive.") {};
833
- //#endregion
834
- //#region ../schema/src/public/api.ts
835
- var PublicApi = class extends HttpApi.make("anpord-public").add(PublicEvalsGroup).add(PublicPromptsGroup).prefix("/v1").annotate(OpenApi.Title, "Anpord API").annotate(OpenApi.Version, "1.0.0").annotate(OpenApi.Description, "Run agent evals and manage prompts. Every endpoint takes a JSON body over POST and authenticates with a bearer API key.").annotate(OpenApi.Servers, [{
836
- description: "Production",
837
- url: "https://api.anpord.com"
838
- }]) {};
839
- //#endregion
840
- //#region ../schema/src/public/client.ts
841
- const DEFAULT_BASE_URL = "https://api.anpord.com";
842
- const make = ({ apiKey, baseUrl = DEFAULT_BASE_URL }) => HttpApiClient.make(PublicApi, {
843
- baseUrl,
844
- transformClient: HttpClient.mapRequest(HttpClientRequest.bearerToken(Redacted.value(apiKey)))
845
- });
846
- var AnpordApi = class extends Effect.Tag("@anpord/sdk/AnpordApi")() {};
847
- const layer = (options) => Layer.effect(AnpordApi, make(options)).pipe(Layer.provide(FetchHttpClient.layer));
848
- //#endregion
849
- export { ChannelName as a, make as i, DEFAULT_BASE_URL as n, PromptId as o, layer as r, VersionNumber as s, AnpordApi as t };