anpord 0.1.24 → 0.1.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -19
- package/dist/api-context.cjs +1 -1
- package/dist/api-context.d.cts +1 -1
- package/dist/api-context.d.mts +1 -1
- package/dist/api-context.mjs +1 -1
- package/dist/api-mocks-C0uY0UBb.mjs +183 -0
- package/dist/api-mocks-CMB0IJFX.cjs +296 -0
- package/dist/api-mocks-CmytjGmi.d.cts +227 -0
- package/dist/api-mocks-CmytjGmi.d.mts +227 -0
- package/dist/api-runtime.cjs +1 -1
- package/dist/api-runtime.d.cts +1 -1
- package/dist/api-runtime.d.mts +1 -1
- package/dist/api-runtime.mjs +1 -1
- package/dist/api.cjs +1 -1
- package/dist/api.d.cts +1 -1
- package/dist/api.d.mts +1 -1
- package/dist/api.mjs +1 -1
- package/dist/bin.cjs +3629 -2447
- package/dist/bin.mjs +3621 -2439
- package/dist/{calls-DJGZJPrF.d.cts → calls-DYqBi5FA.d.cts} +1 -1
- package/dist/{calls-DJGZJPrF.d.mts → calls-DYqBi5FA.d.mts} +1 -1
- package/dist/{calls-6Z1EoEa-.d.cts → calls-Drj_dZpu.d.cts} +1 -1
- package/dist/{calls-6Z1EoEa-.d.mts → calls-Drj_dZpu.d.mts} +1 -1
- package/dist/cli.d.cts +1 -1
- package/dist/cli.d.mts +1 -1
- package/dist/command-C_8h4IdD.d.cts +9 -0
- package/dist/command-C_8h4IdD.d.mts +9 -0
- package/dist/command.cjs +13 -0
- package/dist/command.d.cts +2 -0
- package/dist/command.d.mts +2 -0
- package/dist/command.mjs +11 -0
- package/dist/{compiler-D9jidpZ0.mjs → compiler-BtVNqh3R.mjs} +510 -125
- package/dist/{compiler-B1UrHfCL.cjs → compiler-s8dJGvE6.cjs} +598 -141
- package/dist/errors-B6JK0hbj.mjs +1018 -0
- package/dist/errors-ZfXkocHp.cjs +1143 -0
- package/dist/eval-definition-Dz3dHY1S.d.cts +494 -0
- package/dist/eval-definition-Dz3dHY1S.d.mts +494 -0
- package/dist/eval-judges-CO0ds-HQ.cjs +134 -0
- package/dist/eval-judges-DVxvZQl5.mjs +99 -0
- package/dist/{eval-turns-DcJ8_1qr.cjs → eval-turns-B9dd8Vfb.cjs} +13 -9
- package/dist/eval-turns-BzdjxU_X.d.cts +144 -0
- package/dist/eval-turns-BzdjxU_X.d.mts +144 -0
- package/dist/{eval-turns-BC-GMHD-.mjs → eval-turns-JXtnOI0D.mjs} +13 -9
- package/dist/eval.cjs +1 -1
- package/dist/eval.d.cts +4 -3
- package/dist/eval.d.mts +4 -3
- package/dist/eval.mjs +1 -1
- package/dist/index.cjs +91 -99
- package/dist/index.d.cts +4446 -235
- package/dist/index.d.mts +4446 -235
- package/dist/index.mjs +92 -91
- package/dist/mcp.d.cts +1 -1
- package/dist/mcp.d.mts +1 -1
- package/dist/{runtime-CYr8RbsW.mjs → runtime-BTq5czA4.mjs} +2 -3
- package/dist/{runtime-Bfs2j0jW.cjs → runtime-LjI7Hqj3.cjs} +3 -4
- package/dist/{source-aW5nSo2h.cjs → source-BEcBfWv6.cjs} +7 -0
- package/dist/{source-BP11pzVV.mjs → source-BWXW4jA0.mjs} +2 -1
- package/dist/source.cjs +2 -1
- package/dist/source.d.cts +3 -2
- package/dist/source.d.mts +3 -2
- package/dist/source.mjs +2 -2
- package/dist/types-BsW7R-jL.d.mts +2004 -0
- package/dist/types-D8kjPlPR.d.cts +2004 -0
- package/dist/validator-runtime.cjs +17 -34
- package/dist/validator-runtime.d.cts +1 -1
- package/dist/validator-runtime.d.mts +1 -1
- package/dist/validator-runtime.mjs +14 -31
- package/dist/validators.cjs +2 -41
- package/dist/validators.d.cts +2 -36
- package/dist/validators.d.mts +2 -36
- package/dist/validators.mjs +3 -41
- package/package.json +2 -8
- package/dist/api-mocks-BYWvJp9v.mjs +0 -33
- package/dist/api-mocks-CgiC7T9U.cjs +0 -74
- package/dist/api-mocks-ofqP55P_.d.cts +0 -101
- package/dist/api-mocks-ofqP55P_.d.mts +0 -101
- package/dist/client-B3LQU8cc.d.cts +0 -9410
- package/dist/client-CWVAt_wk.cjs +0 -281
- package/dist/client-CfPf0hIc.mjs +0 -240
- package/dist/client-Cg1qFD01.d.mts +0 -9410
- package/dist/config.cjs +0 -18
- package/dist/config.d.cts +0 -13
- package/dist/config.d.mts +0 -13
- package/dist/config.mjs +0 -13
- package/dist/errors-B0YknR5V.mjs +0 -58
- package/dist/errors-BX1wry8K.cjs +0 -81
- package/dist/errors-aGrd21jy.d.cts +0 -34
- package/dist/errors-aGrd21jy.d.mts +0 -34
- package/dist/eval-judges-1zF4PVqV.cjs +0 -59
- package/dist/eval-judges-Bee_ABJt.mjs +0 -42
- package/dist/eval-turns-CTCoSuB6.d.cts +0 -77
- package/dist/eval-turns-CTCoSuB6.d.mts +0 -77
- package/dist/eval-validations-CZm7vVtQ.cjs +0 -166
- package/dist/eval-validations-DNO-qwkU.mjs +0 -113
- package/dist/evals-Dw2Ek-mx.d.cts +0 -1364
- package/dist/evals-Dw2Ek-mx.d.mts +0 -1364
- package/dist/evals-api-DUiZWsyC.cjs +0 -976
- package/dist/evals-api-NLUUV9tM.mjs +0 -845
- package/dist/types-BnCBSiXY.d.cts +0 -352
- package/dist/types-DiNkJRDh.d.mts +0 -352
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# anpord
|
|
2
2
|
|
|
3
|
-
TypeScript SDK for [Anpord](https://anpord.com). Define coding-agent evals, run them
|
|
3
|
+
TypeScript SDK and CLI for [Anpord](https://anpord.com). Define coding-agent evals, run them on different harnesses, models and sandboxes, and manage versioned prompts.
|
|
4
4
|
|
|
5
5
|
## Install
|
|
6
6
|
|
|
@@ -8,32 +8,44 @@ TypeScript SDK for [Anpord](https://anpord.com). Define coding-agent evals, run
|
|
|
8
8
|
npm install anpord
|
|
9
9
|
```
|
|
10
10
|
|
|
11
|
-
## Define
|
|
11
|
+
## Define a suite
|
|
12
|
+
|
|
13
|
+
A suite groups cases that share a prompt and setup. Suites and cases need an `id`, a lowercase handle such as `writes-hello`; a `name` is optional and defaults to the id. Each variant is a harness, model and sandbox to run every case on, with an optional profile.
|
|
12
14
|
|
|
13
15
|
```ts
|
|
14
|
-
import { suite, empty } from "anpord";
|
|
16
|
+
import { command, suite, empty } from "anpord";
|
|
15
17
|
|
|
16
18
|
export default suite({
|
|
17
|
-
|
|
19
|
+
id: "greeting",
|
|
18
20
|
source: empty,
|
|
19
21
|
prompt: "{{task}}",
|
|
20
22
|
cases: [
|
|
21
23
|
{
|
|
22
|
-
|
|
24
|
+
id: "writes-hello",
|
|
23
25
|
variables: { task: "Create hello.txt containing exactly hello" },
|
|
24
|
-
|
|
26
|
+
validate: command('test "$(cat hello.txt)" = hello'),
|
|
27
|
+
},
|
|
28
|
+
],
|
|
29
|
+
variants: [
|
|
30
|
+
{ harness: "codex", model: "gpt-5.6-sol" },
|
|
31
|
+
{ harness: "claude", model: "sonnet", sandbox: "daytona" },
|
|
32
|
+
{
|
|
33
|
+
harness: "opencode",
|
|
34
|
+
model: "anthropic/claude-sonnet-4.6",
|
|
35
|
+
profile: { dir: "./profiles/strict", name: "strict" },
|
|
25
36
|
},
|
|
26
37
|
],
|
|
27
|
-
tasks: [{ harness: "codex", model: "model-id", provider: "daytona" }],
|
|
28
38
|
trials: 3,
|
|
29
39
|
});
|
|
30
40
|
```
|
|
31
41
|
|
|
42
|
+
A case has one `validate`: a shell `command(...)` whose exit code decides the trial, a validator function, a model judge, or an array of validators and judges that must all pass. A command decides a case on its own and cannot go in an array.
|
|
43
|
+
|
|
32
44
|
```sh
|
|
33
45
|
npx anpord eval ./greeting.eval.ts
|
|
34
46
|
```
|
|
35
47
|
|
|
36
|
-
With no file, the CLI
|
|
48
|
+
Each file starts a batch: one run per case per variant. Each run makes `trials` attempts, each in its own sandbox. With no file, the CLI finds every `*.eval.ts` file under the current directory.
|
|
37
49
|
|
|
38
50
|
## Mock MCP and CLI
|
|
39
51
|
|
|
@@ -82,7 +94,7 @@ export const usersCli = cli({
|
|
|
82
94
|
});
|
|
83
95
|
```
|
|
84
96
|
|
|
85
|
-
Attach
|
|
97
|
+
Attach them to a suite with `mcp: [usersMcp]` or `cli: [usersCli]`. Schemas type the handlers and validate calls at runtime. The `command` from `anpord/cli` declares a mock CLI command, unlike the `command` validator from `anpord`, so alias one of them in a file that imports both.
|
|
86
98
|
|
|
87
99
|
## Model judges
|
|
88
100
|
|
|
@@ -99,28 +111,45 @@ const correctness = judge({
|
|
|
99
111
|
});
|
|
100
112
|
```
|
|
101
113
|
|
|
102
|
-
Set `validate: correctness
|
|
114
|
+
Set `validate: correctness` on a case, or combine it with code: `validate: [checkToolCalls, correctness]`. Every check must pass. A judge with `harness` uses that harness connection, including a ChatGPT sign-in for Codex. To call OpenAI directly, use `provider: "openai"` and add an OpenAI model connection.
|
|
103
115
|
|
|
104
|
-
See [model judges](https://docs.anpord.com/evals/judges) for isolation, authentication
|
|
116
|
+
See [model judges](https://docs.anpord.com/evals/judges) for isolation, authentication and unscored failures.
|
|
105
117
|
|
|
106
|
-
##
|
|
118
|
+
## Start a batch from code
|
|
107
119
|
|
|
108
120
|
```ts
|
|
109
121
|
import { Anpord } from "anpord";
|
|
110
|
-
import
|
|
122
|
+
import greeting from "./greeting.eval";
|
|
111
123
|
|
|
112
124
|
const anpord = new Anpord();
|
|
113
|
-
const
|
|
114
|
-
await compileEval("./greeting.eval.ts")
|
|
115
|
-
);
|
|
125
|
+
const batch = await anpord.evals.batches.startAndWait(greeting);
|
|
116
126
|
|
|
117
|
-
console.log(
|
|
127
|
+
console.log(batch.status, batch.runs);
|
|
118
128
|
await anpord.dispose();
|
|
119
129
|
```
|
|
120
130
|
|
|
121
|
-
The client reads `ANPORD_API_KEY`.
|
|
131
|
+
The client reads `ANPORD_API_KEY`. `batch.runs` holds one run per case and variant, each with its trials and pass rate. `evals.batches.start` returns as soon as the batch is started, and `evals.batches.wait({ id })` waits on one started earlier.
|
|
132
|
+
|
|
133
|
+
## Read and rerun from code
|
|
134
|
+
|
|
135
|
+
```ts
|
|
136
|
+
const { cases } = await anpord.evals.cases.list({ suite: "greeting" });
|
|
137
|
+
const detail = await anpord.evals.cases.get({ id: "writes-hello" });
|
|
138
|
+
|
|
139
|
+
const rerun = await anpord.evals.cases.run({
|
|
140
|
+
id: "writes-hello",
|
|
141
|
+
trials: 3,
|
|
142
|
+
variants: detail.variants.map(({ variant }) => variant.id),
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
const { runs } = await anpord.evals.runs.list({ caseId: "writes-hello" });
|
|
146
|
+
const run = await anpord.evals.runs.get({ id: rerun.runs[0].id });
|
|
147
|
+
const models = await anpord.evals.models.list({ harness: "codex" });
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
`evals.batches`, `evals.cases`, `evals.runs` and `evals.models` follow the data model: a suite holds cases, a case runs on variants, and each run of a case on a variant holds its trials. A batch is the runs started together.
|
|
122
151
|
|
|
123
|
-
See the [documentation](https://docs.anpord.com) for cases, validators, profiles,
|
|
152
|
+
See the [documentation](https://docs.anpord.com) for cases, validators, profiles, mocks, prompt releases and the API reference.
|
|
124
153
|
|
|
125
154
|
## License
|
|
126
155
|
|
package/dist/api-context.cjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
-
const require_api_mocks = require("./api-mocks-
|
|
2
|
+
const require_api_mocks = require("./api-mocks-CMB0IJFX.cjs");
|
|
3
3
|
let effect = require("effect");
|
|
4
4
|
let node_fs_promises = require("node:fs/promises");
|
|
5
5
|
//#region src/mock-api/context.ts
|
package/dist/api-context.d.cts
CHANGED
package/dist/api-context.d.mts
CHANGED
package/dist/api-context.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { n as API_MANIFEST, o as ApiCall, s as ApiManifest, t as API_JOURNAL } from "./api-mocks-C0uY0UBb.mjs";
|
|
2
2
|
import { Schema } from "effect";
|
|
3
3
|
import { readFile } from "node:fs/promises";
|
|
4
4
|
//#region src/mock-api/context.ts
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
import { n as EvalJudgment } from "./eval-judges-DVxvZQl5.mjs";
|
|
2
|
+
import { Schema } from "effect";
|
|
3
|
+
//#region ../schema/src/domain/secret-text.ts
|
|
4
|
+
const REDACTED = "[redacted]";
|
|
5
|
+
const KNOWN_MINIMUM = 8;
|
|
6
|
+
const EDGE = "(?<![A-Za-z0-9_-])";
|
|
7
|
+
const HAS_DIGIT = "(?=[A-Za-z0-9._~+/-]*[0-9])";
|
|
8
|
+
const CREDENTIAL = new RegExp([
|
|
9
|
+
`${EDGE}sk-${HAS_DIGIT}[A-Za-z0-9_-]{20,}`,
|
|
10
|
+
`${EDGE}[rs]k_(?:live|test)_[A-Za-z0-9]{10,}`,
|
|
11
|
+
`${EDGE}am_sk_[A-Za-z0-9_-]+`,
|
|
12
|
+
`${EDGE}anp_[A-Za-z0-9_-]{16,}`,
|
|
13
|
+
`${EDGE}gh[pousr]_[A-Za-z0-9]{20,}`,
|
|
14
|
+
`${EDGE}github_pat_[A-Za-z0-9_]{20,}`,
|
|
15
|
+
`${EDGE}xox[abeoprs]-[A-Za-z0-9-]{10,}`,
|
|
16
|
+
`${EDGE}(?:AKIA|ASIA)[A-Z0-9]{16}`,
|
|
17
|
+
`${EDGE}AIza[A-Za-z0-9_-]{35}`,
|
|
18
|
+
`${EDGE}eyJ[A-Za-z0-9_-]{8,}\\.[A-Za-z0-9_-]{8,}\\.[A-Za-z0-9_-]{8,}`,
|
|
19
|
+
`(?<=\\b[Bb]earer\\s+)${HAS_DIGIT}[A-Za-z0-9._~+/-]{16,}=*`,
|
|
20
|
+
"(?<=[a-z][a-z0-9+.-]*://[^/\\s:@]+:)[^/\\s@]+(?=@)",
|
|
21
|
+
"-----BEGIN [A-Z ]*PRIVATE KEY-----[\\s\\S]*?-----END [A-Z ]*PRIVATE KEY-----"
|
|
22
|
+
].join("|"), "g");
|
|
23
|
+
const escapedForms = (value) => [value, JSON.stringify(value).slice(1, -1)];
|
|
24
|
+
const redactSecrets = (text, known = []) => known.filter((value) => value.length >= KNOWN_MINIMUM).flatMap(escapedForms).toSorted((left, right) => right.length - left.length).reduce((redacted, value) => redacted.replaceAll(value, REDACTED), text).replace(CREDENTIAL, REDACTED);
|
|
25
|
+
//#endregion
|
|
26
|
+
//#region ../schema/src/domain/eval-validations.ts
|
|
27
|
+
const VALIDATION_TEXT_LIMIT = 16e3;
|
|
28
|
+
const VALIDATION_BUDGET = 64e3;
|
|
29
|
+
const VALIDATION_FRAME = "ANPORD_VALIDATION=";
|
|
30
|
+
const EVIDENCE_LIMITS = {
|
|
31
|
+
budget: VALIDATION_BUDGET,
|
|
32
|
+
text: VALIDATION_TEXT_LIMIT
|
|
33
|
+
};
|
|
34
|
+
const REPORTED_LIMITS = {
|
|
35
|
+
budget: 192e3,
|
|
36
|
+
text: 64e3
|
|
37
|
+
};
|
|
38
|
+
const valueWithin = (limit) => Schema.Struct({
|
|
39
|
+
text: Schema.String.pipe(Schema.maxLength(limit)),
|
|
40
|
+
format: Schema.Literal("text", "json"),
|
|
41
|
+
state: Schema.Literal("captured", "disabled", "unavailable"),
|
|
42
|
+
truncated: Schema.Boolean
|
|
43
|
+
});
|
|
44
|
+
const callOf = (value) => Schema.Struct({
|
|
45
|
+
index: Schema.NonNegativeInt,
|
|
46
|
+
method: Schema.Literal("answer", "transcript", "turns", "readText", "exists", "exec", "cli.calls", "api.calls", "api.url", "mcp.calls"),
|
|
47
|
+
startedAt: Schema.Number,
|
|
48
|
+
durationMs: Schema.NullOr(Schema.NonNegativeInt),
|
|
49
|
+
input: value,
|
|
50
|
+
output: value,
|
|
51
|
+
error: Schema.NullOr(value)
|
|
52
|
+
});
|
|
53
|
+
const validationOf = (value, call) => Schema.Struct({
|
|
54
|
+
id: Schema.String.pipe(Schema.minLength(1), Schema.maxLength(200)),
|
|
55
|
+
index: Schema.NonNegativeInt,
|
|
56
|
+
name: Schema.String.pipe(Schema.maxLength(200)),
|
|
57
|
+
kind: Schema.Literal("code", "judge", "command"),
|
|
58
|
+
status: Schema.Literal("queued", "running", "passed", "failed", "error", "skipped"),
|
|
59
|
+
startedAt: Schema.NullOr(Schema.Number),
|
|
60
|
+
durationMs: Schema.NullOr(Schema.NonNegativeInt),
|
|
61
|
+
message: Schema.String.pipe(Schema.maxLength(2e3)),
|
|
62
|
+
exitCode: Schema.NullOr(Schema.Int),
|
|
63
|
+
truncated: Schema.Boolean,
|
|
64
|
+
judgment: Schema.optional(EvalJudgment),
|
|
65
|
+
input: value,
|
|
66
|
+
output: value,
|
|
67
|
+
error: Schema.NullOr(value),
|
|
68
|
+
metadata: Schema.optional(value),
|
|
69
|
+
calls: Schema.Array(call).pipe(Schema.maxItems(64)),
|
|
70
|
+
logs: Schema.Array(Schema.Struct({
|
|
71
|
+
index: Schema.NonNegativeInt,
|
|
72
|
+
at: Schema.Number,
|
|
73
|
+
level: Schema.Literal("stdout", "stderr"),
|
|
74
|
+
value
|
|
75
|
+
})).pipe(Schema.maxItems(64))
|
|
76
|
+
});
|
|
77
|
+
const ValidationValue = valueWithin(VALIDATION_TEXT_LIMIT).annotations({ identifier: "ValidationValue" });
|
|
78
|
+
const EvalValidation = validationOf(ValidationValue, callOf(ValidationValue).annotations({ identifier: "ValidationCall" })).annotations({ identifier: "EvalValidation" });
|
|
79
|
+
const ReportedValue = valueWithin(REPORTED_LIMITS.text);
|
|
80
|
+
const ReportedValidation = validationOf(ReportedValue, callOf(ReportedValue));
|
|
81
|
+
const EvalValidations = Schema.Array(EvalValidation).pipe(Schema.maxItems(40)).annotations({ identifier: "EvalValidations" });
|
|
82
|
+
const validationSnapshot = (value) => ({
|
|
83
|
+
...value,
|
|
84
|
+
truncated: value.truncated || [
|
|
85
|
+
value.input,
|
|
86
|
+
value.output,
|
|
87
|
+
value.error,
|
|
88
|
+
value.metadata,
|
|
89
|
+
...value.calls.flatMap((call) => [
|
|
90
|
+
call.input,
|
|
91
|
+
call.output,
|
|
92
|
+
call.error
|
|
93
|
+
]),
|
|
94
|
+
...value.logs.map((log) => log.value)
|
|
95
|
+
].some((field) => field?.truncated)
|
|
96
|
+
});
|
|
97
|
+
const unavailableValue = {
|
|
98
|
+
text: "",
|
|
99
|
+
format: "text",
|
|
100
|
+
state: "unavailable",
|
|
101
|
+
truncated: false
|
|
102
|
+
};
|
|
103
|
+
const validationCapture = (enabled = true, secrets = [], limits = EVIDENCE_LIMITS) => {
|
|
104
|
+
let remaining = limits.budget;
|
|
105
|
+
return (value, format = "json") => {
|
|
106
|
+
if (!enabled) return {
|
|
107
|
+
...unavailableValue,
|
|
108
|
+
state: "disabled",
|
|
109
|
+
format
|
|
110
|
+
};
|
|
111
|
+
try {
|
|
112
|
+
const encoded = format === "text" ? String(value) : JSON.stringify(value);
|
|
113
|
+
if (encoded === void 0) return {
|
|
114
|
+
...unavailableValue,
|
|
115
|
+
format
|
|
116
|
+
};
|
|
117
|
+
const raw = redactSecrets(encoded, secrets);
|
|
118
|
+
const text = raw.slice(0, Math.min(remaining, limits.text));
|
|
119
|
+
remaining -= text.length;
|
|
120
|
+
return {
|
|
121
|
+
text,
|
|
122
|
+
format,
|
|
123
|
+
state: "captured",
|
|
124
|
+
truncated: text.length < raw.length
|
|
125
|
+
};
|
|
126
|
+
} catch {
|
|
127
|
+
return {
|
|
128
|
+
...unavailableValue,
|
|
129
|
+
format
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
};
|
|
133
|
+
};
|
|
134
|
+
const validationExecution = (identity, startedAt) => ({
|
|
135
|
+
...identity,
|
|
136
|
+
startedAt,
|
|
137
|
+
status: startedAt === null ? "skipped" : "running",
|
|
138
|
+
durationMs: null,
|
|
139
|
+
exitCode: null,
|
|
140
|
+
message: "",
|
|
141
|
+
truncated: false,
|
|
142
|
+
input: unavailableValue,
|
|
143
|
+
output: unavailableValue,
|
|
144
|
+
error: null,
|
|
145
|
+
calls: [],
|
|
146
|
+
logs: []
|
|
147
|
+
});
|
|
148
|
+
//#endregion
|
|
149
|
+
//#region ../schema/src/domain/api-mocks.ts
|
|
150
|
+
const API_PROGRAM = "workspace/.anpord/api/program.json";
|
|
151
|
+
const API_MANIFEST = ".anpord/api/manifest.json";
|
|
152
|
+
const API_JOURNAL = ".anpord/api/calls.jsonl";
|
|
153
|
+
const API_READY = "ANPORD_API_READY=";
|
|
154
|
+
const API_REPORTED_LIMITS = {
|
|
155
|
+
budget: 96e3,
|
|
156
|
+
text: 64e3
|
|
157
|
+
};
|
|
158
|
+
const ApiProgram = Schema.Struct({ entry: Schema.Literal(".anpord/api/server.mjs") });
|
|
159
|
+
const ApiManifest = Schema.Array(Schema.Struct({
|
|
160
|
+
name: Schema.String,
|
|
161
|
+
url: Schema.String,
|
|
162
|
+
endpoints: Schema.Array(Schema.Struct({
|
|
163
|
+
method: Schema.String,
|
|
164
|
+
path: Schema.String,
|
|
165
|
+
description: Schema.optional(Schema.String)
|
|
166
|
+
}))
|
|
167
|
+
}));
|
|
168
|
+
var ApiCall = class extends Schema.Class("ApiCall")({
|
|
169
|
+
api: Schema.String,
|
|
170
|
+
index: Schema.NonNegativeInt,
|
|
171
|
+
method: Schema.String,
|
|
172
|
+
path: Schema.String,
|
|
173
|
+
matched: Schema.Boolean,
|
|
174
|
+
startedAt: Schema.Number,
|
|
175
|
+
durationMs: Schema.NonNegativeInt,
|
|
176
|
+
input: ReportedValue,
|
|
177
|
+
output: ReportedValue,
|
|
178
|
+
status: Schema.Int,
|
|
179
|
+
error: Schema.NullOr(Schema.String.pipe(Schema.maxLength(2e3))),
|
|
180
|
+
logs: Schema.Array(ReportedValue).pipe(Schema.maxItems(64))
|
|
181
|
+
}) {};
|
|
182
|
+
//#endregion
|
|
183
|
+
export { validationExecution as _, API_REPORTED_LIMITS as a, ApiProgram as c, REPORTED_LIMITS as d, ReportedValidation as f, validationCapture as g, unavailableValue as h, API_READY as i, EVIDENCE_LIMITS as l, VALIDATION_TEXT_LIMIT as m, API_MANIFEST as n, ApiCall as o, VALIDATION_FRAME as p, API_PROGRAM as r, ApiManifest as s, API_JOURNAL as t, EvalValidations as u, validationSnapshot as v, redactSecrets as y };
|
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
const require_eval_judges = require("./eval-judges-CO0ds-HQ.cjs");
|
|
2
|
+
let effect = require("effect");
|
|
3
|
+
//#region ../schema/src/domain/secret-text.ts
|
|
4
|
+
const REDACTED = "[redacted]";
|
|
5
|
+
const KNOWN_MINIMUM = 8;
|
|
6
|
+
const EDGE = "(?<![A-Za-z0-9_-])";
|
|
7
|
+
const HAS_DIGIT = "(?=[A-Za-z0-9._~+/-]*[0-9])";
|
|
8
|
+
const CREDENTIAL = new RegExp([
|
|
9
|
+
`${EDGE}sk-${HAS_DIGIT}[A-Za-z0-9_-]{20,}`,
|
|
10
|
+
`${EDGE}[rs]k_(?:live|test)_[A-Za-z0-9]{10,}`,
|
|
11
|
+
`${EDGE}am_sk_[A-Za-z0-9_-]+`,
|
|
12
|
+
`${EDGE}anp_[A-Za-z0-9_-]{16,}`,
|
|
13
|
+
`${EDGE}gh[pousr]_[A-Za-z0-9]{20,}`,
|
|
14
|
+
`${EDGE}github_pat_[A-Za-z0-9_]{20,}`,
|
|
15
|
+
`${EDGE}xox[abeoprs]-[A-Za-z0-9-]{10,}`,
|
|
16
|
+
`${EDGE}(?:AKIA|ASIA)[A-Z0-9]{16}`,
|
|
17
|
+
`${EDGE}AIza[A-Za-z0-9_-]{35}`,
|
|
18
|
+
`${EDGE}eyJ[A-Za-z0-9_-]{8,}\\.[A-Za-z0-9_-]{8,}\\.[A-Za-z0-9_-]{8,}`,
|
|
19
|
+
`(?<=\\b[Bb]earer\\s+)${HAS_DIGIT}[A-Za-z0-9._~+/-]{16,}=*`,
|
|
20
|
+
"(?<=[a-z][a-z0-9+.-]*://[^/\\s:@]+:)[^/\\s@]+(?=@)",
|
|
21
|
+
"-----BEGIN [A-Z ]*PRIVATE KEY-----[\\s\\S]*?-----END [A-Z ]*PRIVATE KEY-----"
|
|
22
|
+
].join("|"), "g");
|
|
23
|
+
const escapedForms = (value) => [value, JSON.stringify(value).slice(1, -1)];
|
|
24
|
+
const redactSecrets = (text, known = []) => known.filter((value) => value.length >= KNOWN_MINIMUM).flatMap(escapedForms).toSorted((left, right) => right.length - left.length).reduce((redacted, value) => redacted.replaceAll(value, REDACTED), text).replace(CREDENTIAL, REDACTED);
|
|
25
|
+
//#endregion
|
|
26
|
+
//#region ../schema/src/domain/eval-validations.ts
|
|
27
|
+
const VALIDATION_TEXT_LIMIT = 16e3;
|
|
28
|
+
const VALIDATION_BUDGET = 64e3;
|
|
29
|
+
const VALIDATION_FRAME = "ANPORD_VALIDATION=";
|
|
30
|
+
const EVIDENCE_LIMITS = {
|
|
31
|
+
budget: VALIDATION_BUDGET,
|
|
32
|
+
text: VALIDATION_TEXT_LIMIT
|
|
33
|
+
};
|
|
34
|
+
const REPORTED_LIMITS = {
|
|
35
|
+
budget: 192e3,
|
|
36
|
+
text: 64e3
|
|
37
|
+
};
|
|
38
|
+
const valueWithin = (limit) => effect.Schema.Struct({
|
|
39
|
+
text: effect.Schema.String.pipe(effect.Schema.maxLength(limit)),
|
|
40
|
+
format: effect.Schema.Literal("text", "json"),
|
|
41
|
+
state: effect.Schema.Literal("captured", "disabled", "unavailable"),
|
|
42
|
+
truncated: effect.Schema.Boolean
|
|
43
|
+
});
|
|
44
|
+
const callOf = (value) => effect.Schema.Struct({
|
|
45
|
+
index: effect.Schema.NonNegativeInt,
|
|
46
|
+
method: effect.Schema.Literal("answer", "transcript", "turns", "readText", "exists", "exec", "cli.calls", "api.calls", "api.url", "mcp.calls"),
|
|
47
|
+
startedAt: effect.Schema.Number,
|
|
48
|
+
durationMs: effect.Schema.NullOr(effect.Schema.NonNegativeInt),
|
|
49
|
+
input: value,
|
|
50
|
+
output: value,
|
|
51
|
+
error: effect.Schema.NullOr(value)
|
|
52
|
+
});
|
|
53
|
+
const validationOf = (value, call) => effect.Schema.Struct({
|
|
54
|
+
id: effect.Schema.String.pipe(effect.Schema.minLength(1), effect.Schema.maxLength(200)),
|
|
55
|
+
index: effect.Schema.NonNegativeInt,
|
|
56
|
+
name: effect.Schema.String.pipe(effect.Schema.maxLength(200)),
|
|
57
|
+
kind: effect.Schema.Literal("code", "judge", "command"),
|
|
58
|
+
status: effect.Schema.Literal("queued", "running", "passed", "failed", "error", "skipped"),
|
|
59
|
+
startedAt: effect.Schema.NullOr(effect.Schema.Number),
|
|
60
|
+
durationMs: effect.Schema.NullOr(effect.Schema.NonNegativeInt),
|
|
61
|
+
message: effect.Schema.String.pipe(effect.Schema.maxLength(2e3)),
|
|
62
|
+
exitCode: effect.Schema.NullOr(effect.Schema.Int),
|
|
63
|
+
truncated: effect.Schema.Boolean,
|
|
64
|
+
judgment: effect.Schema.optional(require_eval_judges.EvalJudgment),
|
|
65
|
+
input: value,
|
|
66
|
+
output: value,
|
|
67
|
+
error: effect.Schema.NullOr(value),
|
|
68
|
+
metadata: effect.Schema.optional(value),
|
|
69
|
+
calls: effect.Schema.Array(call).pipe(effect.Schema.maxItems(64)),
|
|
70
|
+
logs: effect.Schema.Array(effect.Schema.Struct({
|
|
71
|
+
index: effect.Schema.NonNegativeInt,
|
|
72
|
+
at: effect.Schema.Number,
|
|
73
|
+
level: effect.Schema.Literal("stdout", "stderr"),
|
|
74
|
+
value
|
|
75
|
+
})).pipe(effect.Schema.maxItems(64))
|
|
76
|
+
});
|
|
77
|
+
const ValidationValue = valueWithin(VALIDATION_TEXT_LIMIT).annotations({ identifier: "ValidationValue" });
|
|
78
|
+
const EvalValidation = validationOf(ValidationValue, callOf(ValidationValue).annotations({ identifier: "ValidationCall" })).annotations({ identifier: "EvalValidation" });
|
|
79
|
+
const ReportedValue = valueWithin(REPORTED_LIMITS.text);
|
|
80
|
+
const ReportedValidation = validationOf(ReportedValue, callOf(ReportedValue));
|
|
81
|
+
const EvalValidations = effect.Schema.Array(EvalValidation).pipe(effect.Schema.maxItems(40)).annotations({ identifier: "EvalValidations" });
|
|
82
|
+
const validationSnapshot = (value) => ({
|
|
83
|
+
...value,
|
|
84
|
+
truncated: value.truncated || [
|
|
85
|
+
value.input,
|
|
86
|
+
value.output,
|
|
87
|
+
value.error,
|
|
88
|
+
value.metadata,
|
|
89
|
+
...value.calls.flatMap((call) => [
|
|
90
|
+
call.input,
|
|
91
|
+
call.output,
|
|
92
|
+
call.error
|
|
93
|
+
]),
|
|
94
|
+
...value.logs.map((log) => log.value)
|
|
95
|
+
].some((field) => field?.truncated)
|
|
96
|
+
});
|
|
97
|
+
const unavailableValue = {
|
|
98
|
+
text: "",
|
|
99
|
+
format: "text",
|
|
100
|
+
state: "unavailable",
|
|
101
|
+
truncated: false
|
|
102
|
+
};
|
|
103
|
+
const validationCapture = (enabled = true, secrets = [], limits = EVIDENCE_LIMITS) => {
|
|
104
|
+
let remaining = limits.budget;
|
|
105
|
+
return (value, format = "json") => {
|
|
106
|
+
if (!enabled) return {
|
|
107
|
+
...unavailableValue,
|
|
108
|
+
state: "disabled",
|
|
109
|
+
format
|
|
110
|
+
};
|
|
111
|
+
try {
|
|
112
|
+
const encoded = format === "text" ? String(value) : JSON.stringify(value);
|
|
113
|
+
if (encoded === void 0) return {
|
|
114
|
+
...unavailableValue,
|
|
115
|
+
format
|
|
116
|
+
};
|
|
117
|
+
const raw = redactSecrets(encoded, secrets);
|
|
118
|
+
const text = raw.slice(0, Math.min(remaining, limits.text));
|
|
119
|
+
remaining -= text.length;
|
|
120
|
+
return {
|
|
121
|
+
text,
|
|
122
|
+
format,
|
|
123
|
+
state: "captured",
|
|
124
|
+
truncated: text.length < raw.length
|
|
125
|
+
};
|
|
126
|
+
} catch {
|
|
127
|
+
return {
|
|
128
|
+
...unavailableValue,
|
|
129
|
+
format
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
};
|
|
133
|
+
};
|
|
134
|
+
const validationExecution = (identity, startedAt) => ({
|
|
135
|
+
...identity,
|
|
136
|
+
startedAt,
|
|
137
|
+
status: startedAt === null ? "skipped" : "running",
|
|
138
|
+
durationMs: null,
|
|
139
|
+
exitCode: null,
|
|
140
|
+
message: "",
|
|
141
|
+
truncated: false,
|
|
142
|
+
input: unavailableValue,
|
|
143
|
+
output: unavailableValue,
|
|
144
|
+
error: null,
|
|
145
|
+
calls: [],
|
|
146
|
+
logs: []
|
|
147
|
+
});
|
|
148
|
+
//#endregion
|
|
149
|
+
//#region ../schema/src/domain/api-mocks.ts
|
|
150
|
+
const API_PROGRAM = "workspace/.anpord/api/program.json";
|
|
151
|
+
const API_MANIFEST = ".anpord/api/manifest.json";
|
|
152
|
+
const API_JOURNAL = ".anpord/api/calls.jsonl";
|
|
153
|
+
const API_READY = "ANPORD_API_READY=";
|
|
154
|
+
const API_REPORTED_LIMITS = {
|
|
155
|
+
budget: 96e3,
|
|
156
|
+
text: 64e3
|
|
157
|
+
};
|
|
158
|
+
const ApiProgram = effect.Schema.Struct({ entry: effect.Schema.Literal(".anpord/api/server.mjs") });
|
|
159
|
+
const ApiManifest = effect.Schema.Array(effect.Schema.Struct({
|
|
160
|
+
name: effect.Schema.String,
|
|
161
|
+
url: effect.Schema.String,
|
|
162
|
+
endpoints: effect.Schema.Array(effect.Schema.Struct({
|
|
163
|
+
method: effect.Schema.String,
|
|
164
|
+
path: effect.Schema.String,
|
|
165
|
+
description: effect.Schema.optional(effect.Schema.String)
|
|
166
|
+
}))
|
|
167
|
+
}));
|
|
168
|
+
var ApiCall = class extends effect.Schema.Class("ApiCall")({
|
|
169
|
+
api: effect.Schema.String,
|
|
170
|
+
index: effect.Schema.NonNegativeInt,
|
|
171
|
+
method: effect.Schema.String,
|
|
172
|
+
path: effect.Schema.String,
|
|
173
|
+
matched: effect.Schema.Boolean,
|
|
174
|
+
startedAt: effect.Schema.Number,
|
|
175
|
+
durationMs: effect.Schema.NonNegativeInt,
|
|
176
|
+
input: ReportedValue,
|
|
177
|
+
output: ReportedValue,
|
|
178
|
+
status: effect.Schema.Int,
|
|
179
|
+
error: effect.Schema.NullOr(effect.Schema.String.pipe(effect.Schema.maxLength(2e3))),
|
|
180
|
+
logs: effect.Schema.Array(ReportedValue).pipe(effect.Schema.maxItems(64))
|
|
181
|
+
}) {};
|
|
182
|
+
//#endregion
|
|
183
|
+
Object.defineProperty(exports, "API_JOURNAL", {
|
|
184
|
+
enumerable: true,
|
|
185
|
+
get: function() {
|
|
186
|
+
return API_JOURNAL;
|
|
187
|
+
}
|
|
188
|
+
});
|
|
189
|
+
Object.defineProperty(exports, "API_MANIFEST", {
|
|
190
|
+
enumerable: true,
|
|
191
|
+
get: function() {
|
|
192
|
+
return API_MANIFEST;
|
|
193
|
+
}
|
|
194
|
+
});
|
|
195
|
+
Object.defineProperty(exports, "API_PROGRAM", {
|
|
196
|
+
enumerable: true,
|
|
197
|
+
get: function() {
|
|
198
|
+
return API_PROGRAM;
|
|
199
|
+
}
|
|
200
|
+
});
|
|
201
|
+
Object.defineProperty(exports, "API_READY", {
|
|
202
|
+
enumerable: true,
|
|
203
|
+
get: function() {
|
|
204
|
+
return API_READY;
|
|
205
|
+
}
|
|
206
|
+
});
|
|
207
|
+
Object.defineProperty(exports, "API_REPORTED_LIMITS", {
|
|
208
|
+
enumerable: true,
|
|
209
|
+
get: function() {
|
|
210
|
+
return API_REPORTED_LIMITS;
|
|
211
|
+
}
|
|
212
|
+
});
|
|
213
|
+
Object.defineProperty(exports, "ApiCall", {
|
|
214
|
+
enumerable: true,
|
|
215
|
+
get: function() {
|
|
216
|
+
return ApiCall;
|
|
217
|
+
}
|
|
218
|
+
});
|
|
219
|
+
Object.defineProperty(exports, "ApiManifest", {
|
|
220
|
+
enumerable: true,
|
|
221
|
+
get: function() {
|
|
222
|
+
return ApiManifest;
|
|
223
|
+
}
|
|
224
|
+
});
|
|
225
|
+
Object.defineProperty(exports, "ApiProgram", {
|
|
226
|
+
enumerable: true,
|
|
227
|
+
get: function() {
|
|
228
|
+
return ApiProgram;
|
|
229
|
+
}
|
|
230
|
+
});
|
|
231
|
+
Object.defineProperty(exports, "EVIDENCE_LIMITS", {
|
|
232
|
+
enumerable: true,
|
|
233
|
+
get: function() {
|
|
234
|
+
return EVIDENCE_LIMITS;
|
|
235
|
+
}
|
|
236
|
+
});
|
|
237
|
+
Object.defineProperty(exports, "EvalValidations", {
|
|
238
|
+
enumerable: true,
|
|
239
|
+
get: function() {
|
|
240
|
+
return EvalValidations;
|
|
241
|
+
}
|
|
242
|
+
});
|
|
243
|
+
Object.defineProperty(exports, "REPORTED_LIMITS", {
|
|
244
|
+
enumerable: true,
|
|
245
|
+
get: function() {
|
|
246
|
+
return REPORTED_LIMITS;
|
|
247
|
+
}
|
|
248
|
+
});
|
|
249
|
+
Object.defineProperty(exports, "ReportedValidation", {
|
|
250
|
+
enumerable: true,
|
|
251
|
+
get: function() {
|
|
252
|
+
return ReportedValidation;
|
|
253
|
+
}
|
|
254
|
+
});
|
|
255
|
+
Object.defineProperty(exports, "VALIDATION_FRAME", {
|
|
256
|
+
enumerable: true,
|
|
257
|
+
get: function() {
|
|
258
|
+
return VALIDATION_FRAME;
|
|
259
|
+
}
|
|
260
|
+
});
|
|
261
|
+
Object.defineProperty(exports, "VALIDATION_TEXT_LIMIT", {
|
|
262
|
+
enumerable: true,
|
|
263
|
+
get: function() {
|
|
264
|
+
return VALIDATION_TEXT_LIMIT;
|
|
265
|
+
}
|
|
266
|
+
});
|
|
267
|
+
Object.defineProperty(exports, "redactSecrets", {
|
|
268
|
+
enumerable: true,
|
|
269
|
+
get: function() {
|
|
270
|
+
return redactSecrets;
|
|
271
|
+
}
|
|
272
|
+
});
|
|
273
|
+
Object.defineProperty(exports, "unavailableValue", {
|
|
274
|
+
enumerable: true,
|
|
275
|
+
get: function() {
|
|
276
|
+
return unavailableValue;
|
|
277
|
+
}
|
|
278
|
+
});
|
|
279
|
+
Object.defineProperty(exports, "validationCapture", {
|
|
280
|
+
enumerable: true,
|
|
281
|
+
get: function() {
|
|
282
|
+
return validationCapture;
|
|
283
|
+
}
|
|
284
|
+
});
|
|
285
|
+
Object.defineProperty(exports, "validationExecution", {
|
|
286
|
+
enumerable: true,
|
|
287
|
+
get: function() {
|
|
288
|
+
return validationExecution;
|
|
289
|
+
}
|
|
290
|
+
});
|
|
291
|
+
Object.defineProperty(exports, "validationSnapshot", {
|
|
292
|
+
enumerable: true,
|
|
293
|
+
get: function() {
|
|
294
|
+
return validationSnapshot;
|
|
295
|
+
}
|
|
296
|
+
});
|