anpord 0.1.24 → 0.1.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/README.md +48 -19
  2. package/dist/api-context.cjs +1 -1
  3. package/dist/api-context.d.cts +1 -1
  4. package/dist/api-context.d.mts +1 -1
  5. package/dist/api-context.mjs +1 -1
  6. package/dist/api-mocks-C0uY0UBb.mjs +183 -0
  7. package/dist/api-mocks-CMB0IJFX.cjs +296 -0
  8. package/dist/api-mocks-CmytjGmi.d.cts +227 -0
  9. package/dist/api-mocks-CmytjGmi.d.mts +227 -0
  10. package/dist/api-runtime.cjs +1 -1
  11. package/dist/api-runtime.d.cts +1 -1
  12. package/dist/api-runtime.d.mts +1 -1
  13. package/dist/api-runtime.mjs +1 -1
  14. package/dist/api.cjs +1 -1
  15. package/dist/api.d.cts +1 -1
  16. package/dist/api.d.mts +1 -1
  17. package/dist/api.mjs +1 -1
  18. package/dist/bin.cjs +3629 -2447
  19. package/dist/bin.mjs +3621 -2439
  20. package/dist/{calls-DJGZJPrF.d.cts → calls-DYqBi5FA.d.cts} +1 -1
  21. package/dist/{calls-DJGZJPrF.d.mts → calls-DYqBi5FA.d.mts} +1 -1
  22. package/dist/{calls-6Z1EoEa-.d.cts → calls-Drj_dZpu.d.cts} +1 -1
  23. package/dist/{calls-6Z1EoEa-.d.mts → calls-Drj_dZpu.d.mts} +1 -1
  24. package/dist/cli.d.cts +1 -1
  25. package/dist/cli.d.mts +1 -1
  26. package/dist/command-C_8h4IdD.d.cts +9 -0
  27. package/dist/command-C_8h4IdD.d.mts +9 -0
  28. package/dist/command.cjs +13 -0
  29. package/dist/command.d.cts +2 -0
  30. package/dist/command.d.mts +2 -0
  31. package/dist/command.mjs +11 -0
  32. package/dist/{compiler-D9jidpZ0.mjs → compiler-BtVNqh3R.mjs} +510 -125
  33. package/dist/{compiler-B1UrHfCL.cjs → compiler-s8dJGvE6.cjs} +598 -141
  34. package/dist/errors-B6JK0hbj.mjs +1018 -0
  35. package/dist/errors-ZfXkocHp.cjs +1143 -0
  36. package/dist/eval-definition-Dz3dHY1S.d.cts +494 -0
  37. package/dist/eval-definition-Dz3dHY1S.d.mts +494 -0
  38. package/dist/eval-judges-CO0ds-HQ.cjs +134 -0
  39. package/dist/eval-judges-DVxvZQl5.mjs +99 -0
  40. package/dist/{eval-turns-DcJ8_1qr.cjs → eval-turns-B9dd8Vfb.cjs} +13 -9
  41. package/dist/eval-turns-BzdjxU_X.d.cts +144 -0
  42. package/dist/eval-turns-BzdjxU_X.d.mts +144 -0
  43. package/dist/{eval-turns-BC-GMHD-.mjs → eval-turns-JXtnOI0D.mjs} +13 -9
  44. package/dist/eval.cjs +1 -1
  45. package/dist/eval.d.cts +4 -3
  46. package/dist/eval.d.mts +4 -3
  47. package/dist/eval.mjs +1 -1
  48. package/dist/index.cjs +91 -99
  49. package/dist/index.d.cts +4446 -235
  50. package/dist/index.d.mts +4446 -235
  51. package/dist/index.mjs +92 -91
  52. package/dist/mcp.d.cts +1 -1
  53. package/dist/mcp.d.mts +1 -1
  54. package/dist/{runtime-CYr8RbsW.mjs → runtime-BTq5czA4.mjs} +2 -3
  55. package/dist/{runtime-Bfs2j0jW.cjs → runtime-LjI7Hqj3.cjs} +3 -4
  56. package/dist/{source-aW5nSo2h.cjs → source-BEcBfWv6.cjs} +7 -0
  57. package/dist/{source-BP11pzVV.mjs → source-BWXW4jA0.mjs} +2 -1
  58. package/dist/source.cjs +2 -1
  59. package/dist/source.d.cts +3 -2
  60. package/dist/source.d.mts +3 -2
  61. package/dist/source.mjs +2 -2
  62. package/dist/types-BsW7R-jL.d.mts +2004 -0
  63. package/dist/types-D8kjPlPR.d.cts +2004 -0
  64. package/dist/validator-runtime.cjs +17 -34
  65. package/dist/validator-runtime.d.cts +1 -1
  66. package/dist/validator-runtime.d.mts +1 -1
  67. package/dist/validator-runtime.mjs +14 -31
  68. package/dist/validators.cjs +2 -41
  69. package/dist/validators.d.cts +2 -36
  70. package/dist/validators.d.mts +2 -36
  71. package/dist/validators.mjs +3 -41
  72. package/package.json +2 -8
  73. package/dist/api-mocks-BYWvJp9v.mjs +0 -33
  74. package/dist/api-mocks-CgiC7T9U.cjs +0 -74
  75. package/dist/api-mocks-ofqP55P_.d.cts +0 -101
  76. package/dist/api-mocks-ofqP55P_.d.mts +0 -101
  77. package/dist/client-B3LQU8cc.d.cts +0 -9410
  78. package/dist/client-CWVAt_wk.cjs +0 -281
  79. package/dist/client-CfPf0hIc.mjs +0 -240
  80. package/dist/client-Cg1qFD01.d.mts +0 -9410
  81. package/dist/config.cjs +0 -18
  82. package/dist/config.d.cts +0 -13
  83. package/dist/config.d.mts +0 -13
  84. package/dist/config.mjs +0 -13
  85. package/dist/errors-B0YknR5V.mjs +0 -58
  86. package/dist/errors-BX1wry8K.cjs +0 -81
  87. package/dist/errors-aGrd21jy.d.cts +0 -34
  88. package/dist/errors-aGrd21jy.d.mts +0 -34
  89. package/dist/eval-judges-1zF4PVqV.cjs +0 -59
  90. package/dist/eval-judges-Bee_ABJt.mjs +0 -42
  91. package/dist/eval-turns-CTCoSuB6.d.cts +0 -77
  92. package/dist/eval-turns-CTCoSuB6.d.mts +0 -77
  93. package/dist/eval-validations-CZm7vVtQ.cjs +0 -166
  94. package/dist/eval-validations-DNO-qwkU.mjs +0 -113
  95. package/dist/evals-Dw2Ek-mx.d.cts +0 -1364
  96. package/dist/evals-Dw2Ek-mx.d.mts +0 -1364
  97. package/dist/evals-api-DUiZWsyC.cjs +0 -976
  98. package/dist/evals-api-NLUUV9tM.mjs +0 -845
  99. package/dist/types-BnCBSiXY.d.cts +0 -352
  100. package/dist/types-DiNkJRDh.d.mts +0 -352
package/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # anpord
2
2
 
3
- TypeScript SDK for [Anpord](https://anpord.com). Define coding-agent evals, run them across agents and sandboxes, and manage versioned prompts.
3
+ TypeScript SDK and CLI for [Anpord](https://anpord.com). Define coding-agent evals, run them on different harnesses, models and sandboxes, and manage versioned prompts.
4
4
 
5
5
  ## Install
6
6
 
@@ -8,32 +8,44 @@ TypeScript SDK for [Anpord](https://anpord.com). Define coding-agent evals, run
8
8
  npm install anpord
9
9
  ```
10
10
 
11
- ## Define an eval
11
+ ## Define a suite
12
+
13
+ A suite groups cases that share a prompt and setup. Suites and cases need an `id`, a lowercase handle such as `writes-hello`; a `name` is optional and defaults to the id. Each variant is a harness, model and sandbox to run every case on, with an optional profile.
12
14
 
13
15
  ```ts
14
- import { suite, empty } from "anpord";
16
+ import { command, suite, empty } from "anpord";
15
17
 
16
18
  export default suite({
17
- name: "greeting",
19
+ id: "greeting",
18
20
  source: empty,
19
21
  prompt: "{{task}}",
20
22
  cases: [
21
23
  {
22
- name: "writes hello",
24
+ id: "writes-hello",
23
25
  variables: { task: "Create hello.txt containing exactly hello" },
24
- verify: 'test "$(cat hello.txt)" = hello',
26
+ validate: command('test "$(cat hello.txt)" = hello'),
27
+ },
28
+ ],
29
+ variants: [
30
+ { harness: "codex", model: "gpt-5.6-sol" },
31
+ { harness: "claude", model: "sonnet", sandbox: "daytona" },
32
+ {
33
+ harness: "opencode",
34
+ model: "anthropic/claude-sonnet-4.6",
35
+ profile: { dir: "./profiles/strict", name: "strict" },
25
36
  },
26
37
  ],
27
- tasks: [{ harness: "codex", model: "model-id", provider: "daytona" }],
28
38
  trials: 3,
29
39
  });
30
40
  ```
31
41
 
42
+ A case has one `validate`: a shell `command(...)` whose exit code decides the trial, a validator function, a model judge, or an array of validators and judges that must all pass. A command decides a case on its own and cannot go in an array.
43
+
32
44
  ```sh
33
45
  npx anpord eval ./greeting.eval.ts
34
46
  ```
35
47
 
36
- With no file, the CLI discovers every `**/*.eval.ts` suite.
48
+ Each file starts a batch: one run per case per variant. Each run makes `trials` attempts, each in its own sandbox. With no file, the CLI finds every `*.eval.ts` file under the current directory.
37
49
 
38
50
  ## Mock MCP and CLI
39
51
 
@@ -82,7 +94,7 @@ export const usersCli = cli({
82
94
  });
83
95
  ```
84
96
 
85
- Attach definitions with `mcp: [usersMcp]` or `cli: [usersCli]`. Handlers use plain TypeScript. Schemas infer handler types and validate calls at runtime.
97
+ Attach them to a suite with `mcp: [usersMcp]` or `cli: [usersCli]`. Schemas type the handlers and validate calls at runtime. The `command` from `anpord/cli` declares a mock CLI command, unlike the `command` validator from `anpord`, so alias one of them in a file that imports both.
86
98
 
87
99
  ## Model judges
88
100
 
@@ -99,28 +111,45 @@ const correctness = judge({
99
111
  });
100
112
  ```
101
113
 
102
- Set `validate: correctness`, or combine it with code: `validate: [checkToolCalls, correctness]`. Every check must pass. Judges use the organization's harness connection, including Codex subscriptions. For direct OpenAI calls, use `provider: "openai"` and configure an OpenAI key in the environment connection.
114
+ Set `validate: correctness` on a case, or combine it with code: `validate: [checkToolCalls, correctness]`. Every check must pass. A judge with `harness` uses that harness connection, including a ChatGPT sign-in for Codex. To call OpenAI directly, use `provider: "openai"` and add an OpenAI model connection.
103
115
 
104
- See [model judges](https://docs.anpord.com/evals/judges) for isolation, authentication, and unscored failures.
116
+ See [model judges](https://docs.anpord.com/evals/judges) for isolation, authentication and unscored failures.
105
117
 
106
- ## Use the API
118
+ ## Start a batch from code
107
119
 
108
120
  ```ts
109
121
  import { Anpord } from "anpord";
110
- import { compileEval } from "anpord/eval";
122
+ import greeting from "./greeting.eval";
111
123
 
112
124
  const anpord = new Anpord();
113
- const run = await anpord.evals.startAndWait(
114
- await compileEval("./greeting.eval.ts")
115
- );
125
+ const batch = await anpord.evals.batches.startAndWait(greeting);
116
126
 
117
- console.log(run.status, run.cells);
127
+ console.log(batch.status, batch.runs);
118
128
  await anpord.dispose();
119
129
  ```
120
130
 
121
- The client reads `ANPORD_API_KEY`.
131
+ The client reads `ANPORD_API_KEY`. `batch.runs` holds one run per case and variant, each with its trials and pass rate. `evals.batches.start` returns as soon as the batch is started, and `evals.batches.wait({ id })` waits on one started earlier.
132
+
133
+ ## Read and rerun from code
134
+
135
+ ```ts
136
+ const { cases } = await anpord.evals.cases.list({ suite: "greeting" });
137
+ const detail = await anpord.evals.cases.get({ id: "writes-hello" });
138
+
139
+ const rerun = await anpord.evals.cases.run({
140
+ id: "writes-hello",
141
+ trials: 3,
142
+ variants: detail.variants.map(({ variant }) => variant.id),
143
+ });
144
+
145
+ const { runs } = await anpord.evals.runs.list({ caseId: "writes-hello" });
146
+ const run = await anpord.evals.runs.get({ id: rerun.runs[0].id });
147
+ const models = await anpord.evals.models.list({ harness: "codex" });
148
+ ```
149
+
150
+ `evals.batches`, `evals.cases`, `evals.runs` and `evals.models` follow the data model: a suite holds cases, a case runs on variants, and each run of a case on a variant holds its trials. A batch is the runs started together.
122
151
 
123
- See the [documentation](https://docs.anpord.com) for cases, validators, profiles, mock interfaces, prompt releases, and the API reference.
152
+ See the [documentation](https://docs.anpord.com) for cases, validators, profiles, mocks, prompt releases and the API reference.
124
153
 
125
154
  ## License
126
155
 
@@ -1,5 +1,5 @@
1
1
  Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
2
- const require_api_mocks = require("./api-mocks-CgiC7T9U.cjs");
2
+ const require_api_mocks = require("./api-mocks-CMB0IJFX.cjs");
3
3
  let effect = require("effect");
4
4
  let node_fs_promises = require("node:fs/promises");
5
5
  //#region src/mock-api/context.ts
@@ -1,4 +1,4 @@
1
- import { t as ApiCall } from "./api-mocks-ofqP55P_.cjs";
1
+ import { t as ApiCall } from "./api-mocks-CmytjGmi.cjs";
2
2
  //#region src/mock-api/context.d.ts
3
3
  declare const apiContext: {
4
4
  url: (name: string) => Promise<string>;
@@ -1,4 +1,4 @@
1
- import { t as ApiCall } from "./api-mocks-ofqP55P_.mjs";
1
+ import { t as ApiCall } from "./api-mocks-CmytjGmi.mjs";
2
2
  //#region src/mock-api/context.d.ts
3
3
  declare const apiContext: {
4
4
  url: (name: string) => Promise<string>;
@@ -1,4 +1,4 @@
1
- import { a as ApiCall, n as API_MANIFEST, o as ApiManifest, t as API_JOURNAL } from "./api-mocks-BYWvJp9v.mjs";
1
+ import { n as API_MANIFEST, o as ApiCall, s as ApiManifest, t as API_JOURNAL } from "./api-mocks-C0uY0UBb.mjs";
2
2
  import { Schema } from "effect";
3
3
  import { readFile } from "node:fs/promises";
4
4
  //#region src/mock-api/context.ts
@@ -0,0 +1,183 @@
1
+ import { n as EvalJudgment } from "./eval-judges-DVxvZQl5.mjs";
2
+ import { Schema } from "effect";
3
+ //#region ../schema/src/domain/secret-text.ts
4
+ const REDACTED = "[redacted]";
5
+ const KNOWN_MINIMUM = 8;
6
+ const EDGE = "(?<![A-Za-z0-9_-])";
7
+ const HAS_DIGIT = "(?=[A-Za-z0-9._~+/-]*[0-9])";
8
+ const CREDENTIAL = new RegExp([
9
+ `${EDGE}sk-${HAS_DIGIT}[A-Za-z0-9_-]{20,}`,
10
+ `${EDGE}[rs]k_(?:live|test)_[A-Za-z0-9]{10,}`,
11
+ `${EDGE}am_sk_[A-Za-z0-9_-]+`,
12
+ `${EDGE}anp_[A-Za-z0-9_-]{16,}`,
13
+ `${EDGE}gh[pousr]_[A-Za-z0-9]{20,}`,
14
+ `${EDGE}github_pat_[A-Za-z0-9_]{20,}`,
15
+ `${EDGE}xox[abeoprs]-[A-Za-z0-9-]{10,}`,
16
+ `${EDGE}(?:AKIA|ASIA)[A-Z0-9]{16}`,
17
+ `${EDGE}AIza[A-Za-z0-9_-]{35}`,
18
+ `${EDGE}eyJ[A-Za-z0-9_-]{8,}\\.[A-Za-z0-9_-]{8,}\\.[A-Za-z0-9_-]{8,}`,
19
+ `(?<=\\b[Bb]earer\\s+)${HAS_DIGIT}[A-Za-z0-9._~+/-]{16,}=*`,
20
+ "(?<=[a-z][a-z0-9+.-]*://[^/\\s:@]+:)[^/\\s@]+(?=@)",
21
+ "-----BEGIN [A-Z ]*PRIVATE KEY-----[\\s\\S]*?-----END [A-Z ]*PRIVATE KEY-----"
22
+ ].join("|"), "g");
23
+ const escapedForms = (value) => [value, JSON.stringify(value).slice(1, -1)];
24
+ const redactSecrets = (text, known = []) => known.filter((value) => value.length >= KNOWN_MINIMUM).flatMap(escapedForms).toSorted((left, right) => right.length - left.length).reduce((redacted, value) => redacted.replaceAll(value, REDACTED), text).replace(CREDENTIAL, REDACTED);
25
+ //#endregion
26
+ //#region ../schema/src/domain/eval-validations.ts
27
+ const VALIDATION_TEXT_LIMIT = 16e3;
28
+ const VALIDATION_BUDGET = 64e3;
29
+ const VALIDATION_FRAME = "ANPORD_VALIDATION=";
30
+ const EVIDENCE_LIMITS = {
31
+ budget: VALIDATION_BUDGET,
32
+ text: VALIDATION_TEXT_LIMIT
33
+ };
34
+ const REPORTED_LIMITS = {
35
+ budget: 192e3,
36
+ text: 64e3
37
+ };
38
+ const valueWithin = (limit) => Schema.Struct({
39
+ text: Schema.String.pipe(Schema.maxLength(limit)),
40
+ format: Schema.Literal("text", "json"),
41
+ state: Schema.Literal("captured", "disabled", "unavailable"),
42
+ truncated: Schema.Boolean
43
+ });
44
+ const callOf = (value) => Schema.Struct({
45
+ index: Schema.NonNegativeInt,
46
+ method: Schema.Literal("answer", "transcript", "turns", "readText", "exists", "exec", "cli.calls", "api.calls", "api.url", "mcp.calls"),
47
+ startedAt: Schema.Number,
48
+ durationMs: Schema.NullOr(Schema.NonNegativeInt),
49
+ input: value,
50
+ output: value,
51
+ error: Schema.NullOr(value)
52
+ });
53
+ const validationOf = (value, call) => Schema.Struct({
54
+ id: Schema.String.pipe(Schema.minLength(1), Schema.maxLength(200)),
55
+ index: Schema.NonNegativeInt,
56
+ name: Schema.String.pipe(Schema.maxLength(200)),
57
+ kind: Schema.Literal("code", "judge", "command"),
58
+ status: Schema.Literal("queued", "running", "passed", "failed", "error", "skipped"),
59
+ startedAt: Schema.NullOr(Schema.Number),
60
+ durationMs: Schema.NullOr(Schema.NonNegativeInt),
61
+ message: Schema.String.pipe(Schema.maxLength(2e3)),
62
+ exitCode: Schema.NullOr(Schema.Int),
63
+ truncated: Schema.Boolean,
64
+ judgment: Schema.optional(EvalJudgment),
65
+ input: value,
66
+ output: value,
67
+ error: Schema.NullOr(value),
68
+ metadata: Schema.optional(value),
69
+ calls: Schema.Array(call).pipe(Schema.maxItems(64)),
70
+ logs: Schema.Array(Schema.Struct({
71
+ index: Schema.NonNegativeInt,
72
+ at: Schema.Number,
73
+ level: Schema.Literal("stdout", "stderr"),
74
+ value
75
+ })).pipe(Schema.maxItems(64))
76
+ });
77
+ const ValidationValue = valueWithin(VALIDATION_TEXT_LIMIT).annotations({ identifier: "ValidationValue" });
78
+ const EvalValidation = validationOf(ValidationValue, callOf(ValidationValue).annotations({ identifier: "ValidationCall" })).annotations({ identifier: "EvalValidation" });
79
+ const ReportedValue = valueWithin(REPORTED_LIMITS.text);
80
+ const ReportedValidation = validationOf(ReportedValue, callOf(ReportedValue));
81
+ const EvalValidations = Schema.Array(EvalValidation).pipe(Schema.maxItems(40)).annotations({ identifier: "EvalValidations" });
82
+ const validationSnapshot = (value) => ({
83
+ ...value,
84
+ truncated: value.truncated || [
85
+ value.input,
86
+ value.output,
87
+ value.error,
88
+ value.metadata,
89
+ ...value.calls.flatMap((call) => [
90
+ call.input,
91
+ call.output,
92
+ call.error
93
+ ]),
94
+ ...value.logs.map((log) => log.value)
95
+ ].some((field) => field?.truncated)
96
+ });
97
+ const unavailableValue = {
98
+ text: "",
99
+ format: "text",
100
+ state: "unavailable",
101
+ truncated: false
102
+ };
103
+ const validationCapture = (enabled = true, secrets = [], limits = EVIDENCE_LIMITS) => {
104
+ let remaining = limits.budget;
105
+ return (value, format = "json") => {
106
+ if (!enabled) return {
107
+ ...unavailableValue,
108
+ state: "disabled",
109
+ format
110
+ };
111
+ try {
112
+ const encoded = format === "text" ? String(value) : JSON.stringify(value);
113
+ if (encoded === void 0) return {
114
+ ...unavailableValue,
115
+ format
116
+ };
117
+ const raw = redactSecrets(encoded, secrets);
118
+ const text = raw.slice(0, Math.min(remaining, limits.text));
119
+ remaining -= text.length;
120
+ return {
121
+ text,
122
+ format,
123
+ state: "captured",
124
+ truncated: text.length < raw.length
125
+ };
126
+ } catch {
127
+ return {
128
+ ...unavailableValue,
129
+ format
130
+ };
131
+ }
132
+ };
133
+ };
134
+ const validationExecution = (identity, startedAt) => ({
135
+ ...identity,
136
+ startedAt,
137
+ status: startedAt === null ? "skipped" : "running",
138
+ durationMs: null,
139
+ exitCode: null,
140
+ message: "",
141
+ truncated: false,
142
+ input: unavailableValue,
143
+ output: unavailableValue,
144
+ error: null,
145
+ calls: [],
146
+ logs: []
147
+ });
148
+ //#endregion
149
+ //#region ../schema/src/domain/api-mocks.ts
150
+ const API_PROGRAM = "workspace/.anpord/api/program.json";
151
+ const API_MANIFEST = ".anpord/api/manifest.json";
152
+ const API_JOURNAL = ".anpord/api/calls.jsonl";
153
+ const API_READY = "ANPORD_API_READY=";
154
+ const API_REPORTED_LIMITS = {
155
+ budget: 96e3,
156
+ text: 64e3
157
+ };
158
+ const ApiProgram = Schema.Struct({ entry: Schema.Literal(".anpord/api/server.mjs") });
159
+ const ApiManifest = Schema.Array(Schema.Struct({
160
+ name: Schema.String,
161
+ url: Schema.String,
162
+ endpoints: Schema.Array(Schema.Struct({
163
+ method: Schema.String,
164
+ path: Schema.String,
165
+ description: Schema.optional(Schema.String)
166
+ }))
167
+ }));
168
+ var ApiCall = class extends Schema.Class("ApiCall")({
169
+ api: Schema.String,
170
+ index: Schema.NonNegativeInt,
171
+ method: Schema.String,
172
+ path: Schema.String,
173
+ matched: Schema.Boolean,
174
+ startedAt: Schema.Number,
175
+ durationMs: Schema.NonNegativeInt,
176
+ input: ReportedValue,
177
+ output: ReportedValue,
178
+ status: Schema.Int,
179
+ error: Schema.NullOr(Schema.String.pipe(Schema.maxLength(2e3))),
180
+ logs: Schema.Array(ReportedValue).pipe(Schema.maxItems(64))
181
+ }) {};
182
+ //#endregion
183
+ export { validationExecution as _, API_REPORTED_LIMITS as a, ApiProgram as c, REPORTED_LIMITS as d, ReportedValidation as f, validationCapture as g, unavailableValue as h, API_READY as i, EVIDENCE_LIMITS as l, VALIDATION_TEXT_LIMIT as m, API_MANIFEST as n, ApiCall as o, VALIDATION_FRAME as p, API_PROGRAM as r, ApiManifest as s, API_JOURNAL as t, EvalValidations as u, validationSnapshot as v, redactSecrets as y };
@@ -0,0 +1,296 @@
1
+ const require_eval_judges = require("./eval-judges-CO0ds-HQ.cjs");
2
+ let effect = require("effect");
3
+ //#region ../schema/src/domain/secret-text.ts
4
+ const REDACTED = "[redacted]";
5
+ const KNOWN_MINIMUM = 8;
6
+ const EDGE = "(?<![A-Za-z0-9_-])";
7
+ const HAS_DIGIT = "(?=[A-Za-z0-9._~+/-]*[0-9])";
8
+ const CREDENTIAL = new RegExp([
9
+ `${EDGE}sk-${HAS_DIGIT}[A-Za-z0-9_-]{20,}`,
10
+ `${EDGE}[rs]k_(?:live|test)_[A-Za-z0-9]{10,}`,
11
+ `${EDGE}am_sk_[A-Za-z0-9_-]+`,
12
+ `${EDGE}anp_[A-Za-z0-9_-]{16,}`,
13
+ `${EDGE}gh[pousr]_[A-Za-z0-9]{20,}`,
14
+ `${EDGE}github_pat_[A-Za-z0-9_]{20,}`,
15
+ `${EDGE}xox[abeoprs]-[A-Za-z0-9-]{10,}`,
16
+ `${EDGE}(?:AKIA|ASIA)[A-Z0-9]{16}`,
17
+ `${EDGE}AIza[A-Za-z0-9_-]{35}`,
18
+ `${EDGE}eyJ[A-Za-z0-9_-]{8,}\\.[A-Za-z0-9_-]{8,}\\.[A-Za-z0-9_-]{8,}`,
19
+ `(?<=\\b[Bb]earer\\s+)${HAS_DIGIT}[A-Za-z0-9._~+/-]{16,}=*`,
20
+ "(?<=[a-z][a-z0-9+.-]*://[^/\\s:@]+:)[^/\\s@]+(?=@)",
21
+ "-----BEGIN [A-Z ]*PRIVATE KEY-----[\\s\\S]*?-----END [A-Z ]*PRIVATE KEY-----"
22
+ ].join("|"), "g");
23
+ const escapedForms = (value) => [value, JSON.stringify(value).slice(1, -1)];
24
+ const redactSecrets = (text, known = []) => known.filter((value) => value.length >= KNOWN_MINIMUM).flatMap(escapedForms).toSorted((left, right) => right.length - left.length).reduce((redacted, value) => redacted.replaceAll(value, REDACTED), text).replace(CREDENTIAL, REDACTED);
25
+ //#endregion
26
+ //#region ../schema/src/domain/eval-validations.ts
27
+ const VALIDATION_TEXT_LIMIT = 16e3;
28
+ const VALIDATION_BUDGET = 64e3;
29
+ const VALIDATION_FRAME = "ANPORD_VALIDATION=";
30
+ const EVIDENCE_LIMITS = {
31
+ budget: VALIDATION_BUDGET,
32
+ text: VALIDATION_TEXT_LIMIT
33
+ };
34
+ const REPORTED_LIMITS = {
35
+ budget: 192e3,
36
+ text: 64e3
37
+ };
38
+ const valueWithin = (limit) => effect.Schema.Struct({
39
+ text: effect.Schema.String.pipe(effect.Schema.maxLength(limit)),
40
+ format: effect.Schema.Literal("text", "json"),
41
+ state: effect.Schema.Literal("captured", "disabled", "unavailable"),
42
+ truncated: effect.Schema.Boolean
43
+ });
44
+ const callOf = (value) => effect.Schema.Struct({
45
+ index: effect.Schema.NonNegativeInt,
46
+ method: effect.Schema.Literal("answer", "transcript", "turns", "readText", "exists", "exec", "cli.calls", "api.calls", "api.url", "mcp.calls"),
47
+ startedAt: effect.Schema.Number,
48
+ durationMs: effect.Schema.NullOr(effect.Schema.NonNegativeInt),
49
+ input: value,
50
+ output: value,
51
+ error: effect.Schema.NullOr(value)
52
+ });
53
+ const validationOf = (value, call) => effect.Schema.Struct({
54
+ id: effect.Schema.String.pipe(effect.Schema.minLength(1), effect.Schema.maxLength(200)),
55
+ index: effect.Schema.NonNegativeInt,
56
+ name: effect.Schema.String.pipe(effect.Schema.maxLength(200)),
57
+ kind: effect.Schema.Literal("code", "judge", "command"),
58
+ status: effect.Schema.Literal("queued", "running", "passed", "failed", "error", "skipped"),
59
+ startedAt: effect.Schema.NullOr(effect.Schema.Number),
60
+ durationMs: effect.Schema.NullOr(effect.Schema.NonNegativeInt),
61
+ message: effect.Schema.String.pipe(effect.Schema.maxLength(2e3)),
62
+ exitCode: effect.Schema.NullOr(effect.Schema.Int),
63
+ truncated: effect.Schema.Boolean,
64
+ judgment: effect.Schema.optional(require_eval_judges.EvalJudgment),
65
+ input: value,
66
+ output: value,
67
+ error: effect.Schema.NullOr(value),
68
+ metadata: effect.Schema.optional(value),
69
+ calls: effect.Schema.Array(call).pipe(effect.Schema.maxItems(64)),
70
+ logs: effect.Schema.Array(effect.Schema.Struct({
71
+ index: effect.Schema.NonNegativeInt,
72
+ at: effect.Schema.Number,
73
+ level: effect.Schema.Literal("stdout", "stderr"),
74
+ value
75
+ })).pipe(effect.Schema.maxItems(64))
76
+ });
77
+ const ValidationValue = valueWithin(VALIDATION_TEXT_LIMIT).annotations({ identifier: "ValidationValue" });
78
+ const EvalValidation = validationOf(ValidationValue, callOf(ValidationValue).annotations({ identifier: "ValidationCall" })).annotations({ identifier: "EvalValidation" });
79
+ const ReportedValue = valueWithin(REPORTED_LIMITS.text);
80
+ const ReportedValidation = validationOf(ReportedValue, callOf(ReportedValue));
81
+ const EvalValidations = effect.Schema.Array(EvalValidation).pipe(effect.Schema.maxItems(40)).annotations({ identifier: "EvalValidations" });
82
+ const validationSnapshot = (value) => ({
83
+ ...value,
84
+ truncated: value.truncated || [
85
+ value.input,
86
+ value.output,
87
+ value.error,
88
+ value.metadata,
89
+ ...value.calls.flatMap((call) => [
90
+ call.input,
91
+ call.output,
92
+ call.error
93
+ ]),
94
+ ...value.logs.map((log) => log.value)
95
+ ].some((field) => field?.truncated)
96
+ });
97
+ const unavailableValue = {
98
+ text: "",
99
+ format: "text",
100
+ state: "unavailable",
101
+ truncated: false
102
+ };
103
+ const validationCapture = (enabled = true, secrets = [], limits = EVIDENCE_LIMITS) => {
104
+ let remaining = limits.budget;
105
+ return (value, format = "json") => {
106
+ if (!enabled) return {
107
+ ...unavailableValue,
108
+ state: "disabled",
109
+ format
110
+ };
111
+ try {
112
+ const encoded = format === "text" ? String(value) : JSON.stringify(value);
113
+ if (encoded === void 0) return {
114
+ ...unavailableValue,
115
+ format
116
+ };
117
+ const raw = redactSecrets(encoded, secrets);
118
+ const text = raw.slice(0, Math.min(remaining, limits.text));
119
+ remaining -= text.length;
120
+ return {
121
+ text,
122
+ format,
123
+ state: "captured",
124
+ truncated: text.length < raw.length
125
+ };
126
+ } catch {
127
+ return {
128
+ ...unavailableValue,
129
+ format
130
+ };
131
+ }
132
+ };
133
+ };
134
+ const validationExecution = (identity, startedAt) => ({
135
+ ...identity,
136
+ startedAt,
137
+ status: startedAt === null ? "skipped" : "running",
138
+ durationMs: null,
139
+ exitCode: null,
140
+ message: "",
141
+ truncated: false,
142
+ input: unavailableValue,
143
+ output: unavailableValue,
144
+ error: null,
145
+ calls: [],
146
+ logs: []
147
+ });
148
+ //#endregion
149
+ //#region ../schema/src/domain/api-mocks.ts
150
+ const API_PROGRAM = "workspace/.anpord/api/program.json";
151
+ const API_MANIFEST = ".anpord/api/manifest.json";
152
+ const API_JOURNAL = ".anpord/api/calls.jsonl";
153
+ const API_READY = "ANPORD_API_READY=";
154
+ const API_REPORTED_LIMITS = {
155
+ budget: 96e3,
156
+ text: 64e3
157
+ };
158
+ const ApiProgram = effect.Schema.Struct({ entry: effect.Schema.Literal(".anpord/api/server.mjs") });
159
+ const ApiManifest = effect.Schema.Array(effect.Schema.Struct({
160
+ name: effect.Schema.String,
161
+ url: effect.Schema.String,
162
+ endpoints: effect.Schema.Array(effect.Schema.Struct({
163
+ method: effect.Schema.String,
164
+ path: effect.Schema.String,
165
+ description: effect.Schema.optional(effect.Schema.String)
166
+ }))
167
+ }));
168
+ var ApiCall = class extends effect.Schema.Class("ApiCall")({
169
+ api: effect.Schema.String,
170
+ index: effect.Schema.NonNegativeInt,
171
+ method: effect.Schema.String,
172
+ path: effect.Schema.String,
173
+ matched: effect.Schema.Boolean,
174
+ startedAt: effect.Schema.Number,
175
+ durationMs: effect.Schema.NonNegativeInt,
176
+ input: ReportedValue,
177
+ output: ReportedValue,
178
+ status: effect.Schema.Int,
179
+ error: effect.Schema.NullOr(effect.Schema.String.pipe(effect.Schema.maxLength(2e3))),
180
+ logs: effect.Schema.Array(ReportedValue).pipe(effect.Schema.maxItems(64))
181
+ }) {};
182
+ //#endregion
183
+ Object.defineProperty(exports, "API_JOURNAL", {
184
+ enumerable: true,
185
+ get: function() {
186
+ return API_JOURNAL;
187
+ }
188
+ });
189
+ Object.defineProperty(exports, "API_MANIFEST", {
190
+ enumerable: true,
191
+ get: function() {
192
+ return API_MANIFEST;
193
+ }
194
+ });
195
+ Object.defineProperty(exports, "API_PROGRAM", {
196
+ enumerable: true,
197
+ get: function() {
198
+ return API_PROGRAM;
199
+ }
200
+ });
201
+ Object.defineProperty(exports, "API_READY", {
202
+ enumerable: true,
203
+ get: function() {
204
+ return API_READY;
205
+ }
206
+ });
207
+ Object.defineProperty(exports, "API_REPORTED_LIMITS", {
208
+ enumerable: true,
209
+ get: function() {
210
+ return API_REPORTED_LIMITS;
211
+ }
212
+ });
213
+ Object.defineProperty(exports, "ApiCall", {
214
+ enumerable: true,
215
+ get: function() {
216
+ return ApiCall;
217
+ }
218
+ });
219
+ Object.defineProperty(exports, "ApiManifest", {
220
+ enumerable: true,
221
+ get: function() {
222
+ return ApiManifest;
223
+ }
224
+ });
225
+ Object.defineProperty(exports, "ApiProgram", {
226
+ enumerable: true,
227
+ get: function() {
228
+ return ApiProgram;
229
+ }
230
+ });
231
+ Object.defineProperty(exports, "EVIDENCE_LIMITS", {
232
+ enumerable: true,
233
+ get: function() {
234
+ return EVIDENCE_LIMITS;
235
+ }
236
+ });
237
+ Object.defineProperty(exports, "EvalValidations", {
238
+ enumerable: true,
239
+ get: function() {
240
+ return EvalValidations;
241
+ }
242
+ });
243
+ Object.defineProperty(exports, "REPORTED_LIMITS", {
244
+ enumerable: true,
245
+ get: function() {
246
+ return REPORTED_LIMITS;
247
+ }
248
+ });
249
+ Object.defineProperty(exports, "ReportedValidation", {
250
+ enumerable: true,
251
+ get: function() {
252
+ return ReportedValidation;
253
+ }
254
+ });
255
+ Object.defineProperty(exports, "VALIDATION_FRAME", {
256
+ enumerable: true,
257
+ get: function() {
258
+ return VALIDATION_FRAME;
259
+ }
260
+ });
261
+ Object.defineProperty(exports, "VALIDATION_TEXT_LIMIT", {
262
+ enumerable: true,
263
+ get: function() {
264
+ return VALIDATION_TEXT_LIMIT;
265
+ }
266
+ });
267
+ Object.defineProperty(exports, "redactSecrets", {
268
+ enumerable: true,
269
+ get: function() {
270
+ return redactSecrets;
271
+ }
272
+ });
273
+ Object.defineProperty(exports, "unavailableValue", {
274
+ enumerable: true,
275
+ get: function() {
276
+ return unavailableValue;
277
+ }
278
+ });
279
+ Object.defineProperty(exports, "validationCapture", {
280
+ enumerable: true,
281
+ get: function() {
282
+ return validationCapture;
283
+ }
284
+ });
285
+ Object.defineProperty(exports, "validationExecution", {
286
+ enumerable: true,
287
+ get: function() {
288
+ return validationExecution;
289
+ }
290
+ });
291
+ Object.defineProperty(exports, "validationSnapshot", {
292
+ enumerable: true,
293
+ get: function() {
294
+ return validationSnapshot;
295
+ }
296
+ });