anpord 0.1.25 → 0.1.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api-context.cjs +1 -1
- package/dist/api-context.d.cts +1 -1
- package/dist/api-context.d.mts +1 -1
- package/dist/api-context.mjs +1 -1
- package/dist/{api-mocks-RCkffSgG.mjs → api-mocks-C0uY0UBb.mjs} +66 -26
- package/dist/{api-mocks-BEorMdN5.cjs → api-mocks-CMB0IJFX.cjs} +91 -27
- package/dist/api-mocks-CmytjGmi.d.cts +227 -0
- package/dist/api-mocks-CmytjGmi.d.mts +227 -0
- package/dist/api-runtime.cjs +1 -1
- package/dist/api-runtime.d.cts +1 -1
- package/dist/api-runtime.d.mts +1 -1
- package/dist/api-runtime.mjs +1 -1
- package/dist/api.cjs +1 -1
- package/dist/api.d.cts +1 -1
- package/dist/api.d.mts +1 -1
- package/dist/api.mjs +1 -1
- package/dist/bin.cjs +1449 -11375
- package/dist/bin.mjs +1471 -11399
- package/dist/{compiler-BHRygPwI.mjs → compiler-BtVNqh3R.mjs} +67 -16
- package/dist/{compiler-yPa4imwi.cjs → compiler-s8dJGvE6.cjs} +71 -14
- package/dist/{errors-CvwbNKSb.mjs → errors-B6JK0hbj.mjs} +250 -81
- package/dist/{errors-C3VZHByw.cjs → errors-ZfXkocHp.cjs} +285 -86
- package/dist/{eval-definition-BP1gJPPO.d.cts → eval-definition-Dz3dHY1S.d.cts} +82 -9
- package/dist/{eval-definition-BP1gJPPO.d.mts → eval-definition-Dz3dHY1S.d.mts} +82 -9
- package/dist/eval-judges-CO0ds-HQ.cjs +134 -0
- package/dist/eval-judges-DVxvZQl5.mjs +99 -0
- package/dist/{eval-turns-DcJ8_1qr.cjs → eval-turns-B9dd8Vfb.cjs} +13 -9
- package/dist/eval-turns-BzdjxU_X.d.cts +144 -0
- package/dist/eval-turns-BzdjxU_X.d.mts +144 -0
- package/dist/{eval-turns-BC-GMHD-.mjs → eval-turns-JXtnOI0D.mjs} +13 -9
- package/dist/eval.cjs +1 -1
- package/dist/eval.d.cts +2 -2
- package/dist/eval.d.mts +2 -2
- package/dist/eval.mjs +1 -1
- package/dist/index.cjs +11 -7
- package/dist/index.d.cts +743 -361
- package/dist/index.d.mts +743 -361
- package/dist/index.mjs +11 -7
- package/dist/{runtime-B58_CBI5.mjs → runtime-BTq5czA4.mjs} +2 -2
- package/dist/{runtime-DX_9jN8J.cjs → runtime-LjI7Hqj3.cjs} +2 -2
- package/dist/{source-aW5nSo2h.cjs → source-BEcBfWv6.cjs} +7 -0
- package/dist/{source-BP11pzVV.mjs → source-BWXW4jA0.mjs} +2 -1
- package/dist/source.cjs +2 -1
- package/dist/source.d.cts +3 -2
- package/dist/source.d.mts +3 -2
- package/dist/source.mjs +2 -2
- package/dist/{types-l55NqLxW.d.mts → types-BsW7R-jL.d.mts} +419 -59
- package/dist/{types-C26_zN2f.d.cts → types-D8kjPlPR.d.cts} +419 -59
- package/dist/validator-runtime.cjs +14 -31
- package/dist/validator-runtime.d.cts +1 -1
- package/dist/validator-runtime.d.mts +1 -1
- package/dist/validator-runtime.mjs +14 -31
- package/dist/validators.cjs +2 -2
- package/dist/validators.d.cts +1 -1
- package/dist/validators.d.mts +1 -1
- package/dist/validators.mjs +2 -2
- package/package.json +1 -1
- package/dist/api-mocks-bVa7eYtc.d.cts +0 -101
- package/dist/api-mocks-bVa7eYtc.d.mts +0 -101
- package/dist/eval-judges-1zF4PVqV.cjs +0 -59
- package/dist/eval-judges-Bee_ABJt.mjs +0 -42
- package/dist/eval-turns-Uwe0G7PS.d.cts +0 -77
- package/dist/eval-turns-Uwe0G7PS.d.mts +0 -77
package/dist/api-context.cjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
-
const require_api_mocks = require("./api-mocks-
|
|
2
|
+
const require_api_mocks = require("./api-mocks-CMB0IJFX.cjs");
|
|
3
3
|
let effect = require("effect");
|
|
4
4
|
let node_fs_promises = require("node:fs/promises");
|
|
5
5
|
//#region src/mock-api/context.ts
|
package/dist/api-context.d.cts
CHANGED
package/dist/api-context.d.mts
CHANGED
package/dist/api-context.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { n as API_MANIFEST, o as ApiCall, s as ApiManifest, t as API_JOURNAL } from "./api-mocks-C0uY0UBb.mjs";
|
|
2
2
|
import { Schema } from "effect";
|
|
3
3
|
import { readFile } from "node:fs/promises";
|
|
4
4
|
//#region src/mock-api/context.ts
|
|
@@ -1,25 +1,56 @@
|
|
|
1
|
-
import { n as EvalJudgment } from "./eval-judges-
|
|
1
|
+
import { n as EvalJudgment } from "./eval-judges-DVxvZQl5.mjs";
|
|
2
2
|
import { Schema } from "effect";
|
|
3
|
+
//#region ../schema/src/domain/secret-text.ts
|
|
4
|
+
const REDACTED = "[redacted]";
|
|
5
|
+
const KNOWN_MINIMUM = 8;
|
|
6
|
+
const EDGE = "(?<![A-Za-z0-9_-])";
|
|
7
|
+
const HAS_DIGIT = "(?=[A-Za-z0-9._~+/-]*[0-9])";
|
|
8
|
+
const CREDENTIAL = new RegExp([
|
|
9
|
+
`${EDGE}sk-${HAS_DIGIT}[A-Za-z0-9_-]{20,}`,
|
|
10
|
+
`${EDGE}[rs]k_(?:live|test)_[A-Za-z0-9]{10,}`,
|
|
11
|
+
`${EDGE}am_sk_[A-Za-z0-9_-]+`,
|
|
12
|
+
`${EDGE}anp_[A-Za-z0-9_-]{16,}`,
|
|
13
|
+
`${EDGE}gh[pousr]_[A-Za-z0-9]{20,}`,
|
|
14
|
+
`${EDGE}github_pat_[A-Za-z0-9_]{20,}`,
|
|
15
|
+
`${EDGE}xox[abeoprs]-[A-Za-z0-9-]{10,}`,
|
|
16
|
+
`${EDGE}(?:AKIA|ASIA)[A-Z0-9]{16}`,
|
|
17
|
+
`${EDGE}AIza[A-Za-z0-9_-]{35}`,
|
|
18
|
+
`${EDGE}eyJ[A-Za-z0-9_-]{8,}\\.[A-Za-z0-9_-]{8,}\\.[A-Za-z0-9_-]{8,}`,
|
|
19
|
+
`(?<=\\b[Bb]earer\\s+)${HAS_DIGIT}[A-Za-z0-9._~+/-]{16,}=*`,
|
|
20
|
+
"(?<=[a-z][a-z0-9+.-]*://[^/\\s:@]+:)[^/\\s@]+(?=@)",
|
|
21
|
+
"-----BEGIN [A-Z ]*PRIVATE KEY-----[\\s\\S]*?-----END [A-Z ]*PRIVATE KEY-----"
|
|
22
|
+
].join("|"), "g");
|
|
23
|
+
const escapedForms = (value) => [value, JSON.stringify(value).slice(1, -1)];
|
|
24
|
+
const redactSecrets = (text, known = []) => known.filter((value) => value.length >= KNOWN_MINIMUM).flatMap(escapedForms).toSorted((left, right) => right.length - left.length).reduce((redacted, value) => redacted.replaceAll(value, REDACTED), text).replace(CREDENTIAL, REDACTED);
|
|
25
|
+
//#endregion
|
|
3
26
|
//#region ../schema/src/domain/eval-validations.ts
|
|
4
27
|
const VALIDATION_TEXT_LIMIT = 16e3;
|
|
5
28
|
const VALIDATION_BUDGET = 64e3;
|
|
6
29
|
const VALIDATION_FRAME = "ANPORD_VALIDATION=";
|
|
7
|
-
const
|
|
8
|
-
|
|
30
|
+
const EVIDENCE_LIMITS = {
|
|
31
|
+
budget: VALIDATION_BUDGET,
|
|
32
|
+
text: VALIDATION_TEXT_LIMIT
|
|
33
|
+
};
|
|
34
|
+
const REPORTED_LIMITS = {
|
|
35
|
+
budget: 192e3,
|
|
36
|
+
text: 64e3
|
|
37
|
+
};
|
|
38
|
+
const valueWithin = (limit) => Schema.Struct({
|
|
39
|
+
text: Schema.String.pipe(Schema.maxLength(limit)),
|
|
9
40
|
format: Schema.Literal("text", "json"),
|
|
10
41
|
state: Schema.Literal("captured", "disabled", "unavailable"),
|
|
11
42
|
truncated: Schema.Boolean
|
|
12
|
-
})
|
|
13
|
-
const
|
|
43
|
+
});
|
|
44
|
+
const callOf = (value) => Schema.Struct({
|
|
14
45
|
index: Schema.NonNegativeInt,
|
|
15
46
|
method: Schema.Literal("answer", "transcript", "turns", "readText", "exists", "exec", "cli.calls", "api.calls", "api.url", "mcp.calls"),
|
|
16
47
|
startedAt: Schema.Number,
|
|
17
48
|
durationMs: Schema.NullOr(Schema.NonNegativeInt),
|
|
18
|
-
input:
|
|
19
|
-
output:
|
|
20
|
-
error: Schema.NullOr(
|
|
21
|
-
})
|
|
22
|
-
const
|
|
49
|
+
input: value,
|
|
50
|
+
output: value,
|
|
51
|
+
error: Schema.NullOr(value)
|
|
52
|
+
});
|
|
53
|
+
const validationOf = (value, call) => Schema.Struct({
|
|
23
54
|
id: Schema.String.pipe(Schema.minLength(1), Schema.maxLength(200)),
|
|
24
55
|
index: Schema.NonNegativeInt,
|
|
25
56
|
name: Schema.String.pipe(Schema.maxLength(200)),
|
|
@@ -31,18 +62,22 @@ const EvalValidation = Schema.Struct({
|
|
|
31
62
|
exitCode: Schema.NullOr(Schema.Int),
|
|
32
63
|
truncated: Schema.Boolean,
|
|
33
64
|
judgment: Schema.optional(EvalJudgment),
|
|
34
|
-
input:
|
|
35
|
-
output:
|
|
36
|
-
error: Schema.NullOr(
|
|
37
|
-
metadata: Schema.optional(
|
|
38
|
-
calls: Schema.Array(
|
|
65
|
+
input: value,
|
|
66
|
+
output: value,
|
|
67
|
+
error: Schema.NullOr(value),
|
|
68
|
+
metadata: Schema.optional(value),
|
|
69
|
+
calls: Schema.Array(call).pipe(Schema.maxItems(64)),
|
|
39
70
|
logs: Schema.Array(Schema.Struct({
|
|
40
71
|
index: Schema.NonNegativeInt,
|
|
41
72
|
at: Schema.Number,
|
|
42
73
|
level: Schema.Literal("stdout", "stderr"),
|
|
43
|
-
value
|
|
74
|
+
value
|
|
44
75
|
})).pipe(Schema.maxItems(64))
|
|
45
|
-
})
|
|
76
|
+
});
|
|
77
|
+
const ValidationValue = valueWithin(VALIDATION_TEXT_LIMIT).annotations({ identifier: "ValidationValue" });
|
|
78
|
+
const EvalValidation = validationOf(ValidationValue, callOf(ValidationValue).annotations({ identifier: "ValidationCall" })).annotations({ identifier: "EvalValidation" });
|
|
79
|
+
const ReportedValue = valueWithin(REPORTED_LIMITS.text);
|
|
80
|
+
const ReportedValidation = validationOf(ReportedValue, callOf(ReportedValue));
|
|
46
81
|
const EvalValidations = Schema.Array(EvalValidation).pipe(Schema.maxItems(40)).annotations({ identifier: "EvalValidations" });
|
|
47
82
|
const validationSnapshot = (value) => ({
|
|
48
83
|
...value,
|
|
@@ -65,8 +100,8 @@ const unavailableValue = {
|
|
|
65
100
|
state: "unavailable",
|
|
66
101
|
truncated: false
|
|
67
102
|
};
|
|
68
|
-
const validationCapture = (enabled = true) => {
|
|
69
|
-
let remaining =
|
|
103
|
+
const validationCapture = (enabled = true, secrets = [], limits = EVIDENCE_LIMITS) => {
|
|
104
|
+
let remaining = limits.budget;
|
|
70
105
|
return (value, format = "json") => {
|
|
71
106
|
if (!enabled) return {
|
|
72
107
|
...unavailableValue,
|
|
@@ -74,12 +109,13 @@ const validationCapture = (enabled = true) => {
|
|
|
74
109
|
format
|
|
75
110
|
};
|
|
76
111
|
try {
|
|
77
|
-
const
|
|
78
|
-
if (
|
|
112
|
+
const encoded = format === "text" ? String(value) : JSON.stringify(value);
|
|
113
|
+
if (encoded === void 0) return {
|
|
79
114
|
...unavailableValue,
|
|
80
115
|
format
|
|
81
116
|
};
|
|
82
|
-
const
|
|
117
|
+
const raw = redactSecrets(encoded, secrets);
|
|
118
|
+
const text = raw.slice(0, Math.min(remaining, limits.text));
|
|
83
119
|
remaining -= text.length;
|
|
84
120
|
return {
|
|
85
121
|
text,
|
|
@@ -115,6 +151,10 @@ const API_PROGRAM = "workspace/.anpord/api/program.json";
|
|
|
115
151
|
const API_MANIFEST = ".anpord/api/manifest.json";
|
|
116
152
|
const API_JOURNAL = ".anpord/api/calls.jsonl";
|
|
117
153
|
const API_READY = "ANPORD_API_READY=";
|
|
154
|
+
const API_REPORTED_LIMITS = {
|
|
155
|
+
budget: 96e3,
|
|
156
|
+
text: 64e3
|
|
157
|
+
};
|
|
118
158
|
const ApiProgram = Schema.Struct({ entry: Schema.Literal(".anpord/api/server.mjs") });
|
|
119
159
|
const ApiManifest = Schema.Array(Schema.Struct({
|
|
120
160
|
name: Schema.String,
|
|
@@ -133,11 +173,11 @@ var ApiCall = class extends Schema.Class("ApiCall")({
|
|
|
133
173
|
matched: Schema.Boolean,
|
|
134
174
|
startedAt: Schema.Number,
|
|
135
175
|
durationMs: Schema.NonNegativeInt,
|
|
136
|
-
input:
|
|
137
|
-
output:
|
|
176
|
+
input: ReportedValue,
|
|
177
|
+
output: ReportedValue,
|
|
138
178
|
status: Schema.Int,
|
|
139
179
|
error: Schema.NullOr(Schema.String.pipe(Schema.maxLength(2e3))),
|
|
140
|
-
logs: Schema.Array(
|
|
180
|
+
logs: Schema.Array(ReportedValue).pipe(Schema.maxItems(64))
|
|
141
181
|
}) {};
|
|
142
182
|
//#endregion
|
|
143
|
-
export {
|
|
183
|
+
export { validationExecution as _, API_REPORTED_LIMITS as a, ApiProgram as c, REPORTED_LIMITS as d, ReportedValidation as f, validationCapture as g, unavailableValue as h, API_READY as i, EVIDENCE_LIMITS as l, VALIDATION_TEXT_LIMIT as m, API_MANIFEST as n, ApiCall as o, VALIDATION_FRAME as p, API_PROGRAM as r, ApiManifest as s, API_JOURNAL as t, EvalValidations as u, validationSnapshot as v, redactSecrets as y };
|
|
@@ -1,25 +1,56 @@
|
|
|
1
|
-
const require_eval_judges = require("./eval-judges-
|
|
1
|
+
const require_eval_judges = require("./eval-judges-CO0ds-HQ.cjs");
|
|
2
2
|
let effect = require("effect");
|
|
3
|
+
//#region ../schema/src/domain/secret-text.ts
|
|
4
|
+
const REDACTED = "[redacted]";
|
|
5
|
+
const KNOWN_MINIMUM = 8;
|
|
6
|
+
const EDGE = "(?<![A-Za-z0-9_-])";
|
|
7
|
+
const HAS_DIGIT = "(?=[A-Za-z0-9._~+/-]*[0-9])";
|
|
8
|
+
const CREDENTIAL = new RegExp([
|
|
9
|
+
`${EDGE}sk-${HAS_DIGIT}[A-Za-z0-9_-]{20,}`,
|
|
10
|
+
`${EDGE}[rs]k_(?:live|test)_[A-Za-z0-9]{10,}`,
|
|
11
|
+
`${EDGE}am_sk_[A-Za-z0-9_-]+`,
|
|
12
|
+
`${EDGE}anp_[A-Za-z0-9_-]{16,}`,
|
|
13
|
+
`${EDGE}gh[pousr]_[A-Za-z0-9]{20,}`,
|
|
14
|
+
`${EDGE}github_pat_[A-Za-z0-9_]{20,}`,
|
|
15
|
+
`${EDGE}xox[abeoprs]-[A-Za-z0-9-]{10,}`,
|
|
16
|
+
`${EDGE}(?:AKIA|ASIA)[A-Z0-9]{16}`,
|
|
17
|
+
`${EDGE}AIza[A-Za-z0-9_-]{35}`,
|
|
18
|
+
`${EDGE}eyJ[A-Za-z0-9_-]{8,}\\.[A-Za-z0-9_-]{8,}\\.[A-Za-z0-9_-]{8,}`,
|
|
19
|
+
`(?<=\\b[Bb]earer\\s+)${HAS_DIGIT}[A-Za-z0-9._~+/-]{16,}=*`,
|
|
20
|
+
"(?<=[a-z][a-z0-9+.-]*://[^/\\s:@]+:)[^/\\s@]+(?=@)",
|
|
21
|
+
"-----BEGIN [A-Z ]*PRIVATE KEY-----[\\s\\S]*?-----END [A-Z ]*PRIVATE KEY-----"
|
|
22
|
+
].join("|"), "g");
|
|
23
|
+
const escapedForms = (value) => [value, JSON.stringify(value).slice(1, -1)];
|
|
24
|
+
const redactSecrets = (text, known = []) => known.filter((value) => value.length >= KNOWN_MINIMUM).flatMap(escapedForms).toSorted((left, right) => right.length - left.length).reduce((redacted, value) => redacted.replaceAll(value, REDACTED), text).replace(CREDENTIAL, REDACTED);
|
|
25
|
+
//#endregion
|
|
3
26
|
//#region ../schema/src/domain/eval-validations.ts
|
|
4
27
|
const VALIDATION_TEXT_LIMIT = 16e3;
|
|
5
28
|
const VALIDATION_BUDGET = 64e3;
|
|
6
29
|
const VALIDATION_FRAME = "ANPORD_VALIDATION=";
|
|
7
|
-
const
|
|
8
|
-
|
|
30
|
+
const EVIDENCE_LIMITS = {
|
|
31
|
+
budget: VALIDATION_BUDGET,
|
|
32
|
+
text: VALIDATION_TEXT_LIMIT
|
|
33
|
+
};
|
|
34
|
+
const REPORTED_LIMITS = {
|
|
35
|
+
budget: 192e3,
|
|
36
|
+
text: 64e3
|
|
37
|
+
};
|
|
38
|
+
const valueWithin = (limit) => effect.Schema.Struct({
|
|
39
|
+
text: effect.Schema.String.pipe(effect.Schema.maxLength(limit)),
|
|
9
40
|
format: effect.Schema.Literal("text", "json"),
|
|
10
41
|
state: effect.Schema.Literal("captured", "disabled", "unavailable"),
|
|
11
42
|
truncated: effect.Schema.Boolean
|
|
12
|
-
})
|
|
13
|
-
const
|
|
43
|
+
});
|
|
44
|
+
const callOf = (value) => effect.Schema.Struct({
|
|
14
45
|
index: effect.Schema.NonNegativeInt,
|
|
15
46
|
method: effect.Schema.Literal("answer", "transcript", "turns", "readText", "exists", "exec", "cli.calls", "api.calls", "api.url", "mcp.calls"),
|
|
16
47
|
startedAt: effect.Schema.Number,
|
|
17
48
|
durationMs: effect.Schema.NullOr(effect.Schema.NonNegativeInt),
|
|
18
|
-
input:
|
|
19
|
-
output:
|
|
20
|
-
error: effect.Schema.NullOr(
|
|
21
|
-
})
|
|
22
|
-
const
|
|
49
|
+
input: value,
|
|
50
|
+
output: value,
|
|
51
|
+
error: effect.Schema.NullOr(value)
|
|
52
|
+
});
|
|
53
|
+
const validationOf = (value, call) => effect.Schema.Struct({
|
|
23
54
|
id: effect.Schema.String.pipe(effect.Schema.minLength(1), effect.Schema.maxLength(200)),
|
|
24
55
|
index: effect.Schema.NonNegativeInt,
|
|
25
56
|
name: effect.Schema.String.pipe(effect.Schema.maxLength(200)),
|
|
@@ -31,18 +62,22 @@ const EvalValidation = effect.Schema.Struct({
|
|
|
31
62
|
exitCode: effect.Schema.NullOr(effect.Schema.Int),
|
|
32
63
|
truncated: effect.Schema.Boolean,
|
|
33
64
|
judgment: effect.Schema.optional(require_eval_judges.EvalJudgment),
|
|
34
|
-
input:
|
|
35
|
-
output:
|
|
36
|
-
error: effect.Schema.NullOr(
|
|
37
|
-
metadata: effect.Schema.optional(
|
|
38
|
-
calls: effect.Schema.Array(
|
|
65
|
+
input: value,
|
|
66
|
+
output: value,
|
|
67
|
+
error: effect.Schema.NullOr(value),
|
|
68
|
+
metadata: effect.Schema.optional(value),
|
|
69
|
+
calls: effect.Schema.Array(call).pipe(effect.Schema.maxItems(64)),
|
|
39
70
|
logs: effect.Schema.Array(effect.Schema.Struct({
|
|
40
71
|
index: effect.Schema.NonNegativeInt,
|
|
41
72
|
at: effect.Schema.Number,
|
|
42
73
|
level: effect.Schema.Literal("stdout", "stderr"),
|
|
43
|
-
value
|
|
74
|
+
value
|
|
44
75
|
})).pipe(effect.Schema.maxItems(64))
|
|
45
|
-
})
|
|
76
|
+
});
|
|
77
|
+
const ValidationValue = valueWithin(VALIDATION_TEXT_LIMIT).annotations({ identifier: "ValidationValue" });
|
|
78
|
+
const EvalValidation = validationOf(ValidationValue, callOf(ValidationValue).annotations({ identifier: "ValidationCall" })).annotations({ identifier: "EvalValidation" });
|
|
79
|
+
const ReportedValue = valueWithin(REPORTED_LIMITS.text);
|
|
80
|
+
const ReportedValidation = validationOf(ReportedValue, callOf(ReportedValue));
|
|
46
81
|
const EvalValidations = effect.Schema.Array(EvalValidation).pipe(effect.Schema.maxItems(40)).annotations({ identifier: "EvalValidations" });
|
|
47
82
|
const validationSnapshot = (value) => ({
|
|
48
83
|
...value,
|
|
@@ -65,8 +100,8 @@ const unavailableValue = {
|
|
|
65
100
|
state: "unavailable",
|
|
66
101
|
truncated: false
|
|
67
102
|
};
|
|
68
|
-
const validationCapture = (enabled = true) => {
|
|
69
|
-
let remaining =
|
|
103
|
+
const validationCapture = (enabled = true, secrets = [], limits = EVIDENCE_LIMITS) => {
|
|
104
|
+
let remaining = limits.budget;
|
|
70
105
|
return (value, format = "json") => {
|
|
71
106
|
if (!enabled) return {
|
|
72
107
|
...unavailableValue,
|
|
@@ -74,12 +109,13 @@ const validationCapture = (enabled = true) => {
|
|
|
74
109
|
format
|
|
75
110
|
};
|
|
76
111
|
try {
|
|
77
|
-
const
|
|
78
|
-
if (
|
|
112
|
+
const encoded = format === "text" ? String(value) : JSON.stringify(value);
|
|
113
|
+
if (encoded === void 0) return {
|
|
79
114
|
...unavailableValue,
|
|
80
115
|
format
|
|
81
116
|
};
|
|
82
|
-
const
|
|
117
|
+
const raw = redactSecrets(encoded, secrets);
|
|
118
|
+
const text = raw.slice(0, Math.min(remaining, limits.text));
|
|
83
119
|
remaining -= text.length;
|
|
84
120
|
return {
|
|
85
121
|
text,
|
|
@@ -115,6 +151,10 @@ const API_PROGRAM = "workspace/.anpord/api/program.json";
|
|
|
115
151
|
const API_MANIFEST = ".anpord/api/manifest.json";
|
|
116
152
|
const API_JOURNAL = ".anpord/api/calls.jsonl";
|
|
117
153
|
const API_READY = "ANPORD_API_READY=";
|
|
154
|
+
const API_REPORTED_LIMITS = {
|
|
155
|
+
budget: 96e3,
|
|
156
|
+
text: 64e3
|
|
157
|
+
};
|
|
118
158
|
const ApiProgram = effect.Schema.Struct({ entry: effect.Schema.Literal(".anpord/api/server.mjs") });
|
|
119
159
|
const ApiManifest = effect.Schema.Array(effect.Schema.Struct({
|
|
120
160
|
name: effect.Schema.String,
|
|
@@ -133,11 +173,11 @@ var ApiCall = class extends effect.Schema.Class("ApiCall")({
|
|
|
133
173
|
matched: effect.Schema.Boolean,
|
|
134
174
|
startedAt: effect.Schema.Number,
|
|
135
175
|
durationMs: effect.Schema.NonNegativeInt,
|
|
136
|
-
input:
|
|
137
|
-
output:
|
|
176
|
+
input: ReportedValue,
|
|
177
|
+
output: ReportedValue,
|
|
138
178
|
status: effect.Schema.Int,
|
|
139
179
|
error: effect.Schema.NullOr(effect.Schema.String.pipe(effect.Schema.maxLength(2e3))),
|
|
140
|
-
logs: effect.Schema.Array(
|
|
180
|
+
logs: effect.Schema.Array(ReportedValue).pipe(effect.Schema.maxItems(64))
|
|
141
181
|
}) {};
|
|
142
182
|
//#endregion
|
|
143
183
|
Object.defineProperty(exports, "API_JOURNAL", {
|
|
@@ -164,6 +204,12 @@ Object.defineProperty(exports, "API_READY", {
|
|
|
164
204
|
return API_READY;
|
|
165
205
|
}
|
|
166
206
|
});
|
|
207
|
+
Object.defineProperty(exports, "API_REPORTED_LIMITS", {
|
|
208
|
+
enumerable: true,
|
|
209
|
+
get: function() {
|
|
210
|
+
return API_REPORTED_LIMITS;
|
|
211
|
+
}
|
|
212
|
+
});
|
|
167
213
|
Object.defineProperty(exports, "ApiCall", {
|
|
168
214
|
enumerable: true,
|
|
169
215
|
get: function() {
|
|
@@ -182,10 +228,10 @@ Object.defineProperty(exports, "ApiProgram", {
|
|
|
182
228
|
return ApiProgram;
|
|
183
229
|
}
|
|
184
230
|
});
|
|
185
|
-
Object.defineProperty(exports, "
|
|
231
|
+
Object.defineProperty(exports, "EVIDENCE_LIMITS", {
|
|
186
232
|
enumerable: true,
|
|
187
233
|
get: function() {
|
|
188
|
-
return
|
|
234
|
+
return EVIDENCE_LIMITS;
|
|
189
235
|
}
|
|
190
236
|
});
|
|
191
237
|
Object.defineProperty(exports, "EvalValidations", {
|
|
@@ -194,6 +240,18 @@ Object.defineProperty(exports, "EvalValidations", {
|
|
|
194
240
|
return EvalValidations;
|
|
195
241
|
}
|
|
196
242
|
});
|
|
243
|
+
Object.defineProperty(exports, "REPORTED_LIMITS", {
|
|
244
|
+
enumerable: true,
|
|
245
|
+
get: function() {
|
|
246
|
+
return REPORTED_LIMITS;
|
|
247
|
+
}
|
|
248
|
+
});
|
|
249
|
+
Object.defineProperty(exports, "ReportedValidation", {
|
|
250
|
+
enumerable: true,
|
|
251
|
+
get: function() {
|
|
252
|
+
return ReportedValidation;
|
|
253
|
+
}
|
|
254
|
+
});
|
|
197
255
|
Object.defineProperty(exports, "VALIDATION_FRAME", {
|
|
198
256
|
enumerable: true,
|
|
199
257
|
get: function() {
|
|
@@ -206,6 +264,12 @@ Object.defineProperty(exports, "VALIDATION_TEXT_LIMIT", {
|
|
|
206
264
|
return VALIDATION_TEXT_LIMIT;
|
|
207
265
|
}
|
|
208
266
|
});
|
|
267
|
+
Object.defineProperty(exports, "redactSecrets", {
|
|
268
|
+
enumerable: true,
|
|
269
|
+
get: function() {
|
|
270
|
+
return redactSecrets;
|
|
271
|
+
}
|
|
272
|
+
});
|
|
209
273
|
Object.defineProperty(exports, "unavailableValue", {
|
|
210
274
|
enumerable: true,
|
|
211
275
|
get: function() {
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
import { Schema } from "effect";
|
|
2
|
+
//#region ../schema/src/domain/eval-validations.d.ts
|
|
3
|
+
declare const ValidationValue: Schema.Struct<{
|
|
4
|
+
text: Schema.filter<typeof Schema.String>;
|
|
5
|
+
format: Schema.Literal<["text", "json"]>;
|
|
6
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
7
|
+
truncated: typeof Schema.Boolean;
|
|
8
|
+
}>;
|
|
9
|
+
type ValidationValue = typeof ValidationValue.Type;
|
|
10
|
+
declare const ValidationCall: Schema.Struct<{
|
|
11
|
+
index: Schema.refine<number, typeof Schema.NonNegative>;
|
|
12
|
+
method: Schema.Literal<["answer", "transcript", "turns", "readText", "exists", "exec", "cli.calls", "api.calls", "api.url", "mcp.calls"]>;
|
|
13
|
+
startedAt: typeof Schema.Number;
|
|
14
|
+
durationMs: Schema.NullOr<Schema.refine<number, typeof Schema.NonNegative>>;
|
|
15
|
+
input: Schema.Struct<{
|
|
16
|
+
text: Schema.filter<typeof Schema.String>;
|
|
17
|
+
format: Schema.Literal<["text", "json"]>;
|
|
18
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
19
|
+
truncated: typeof Schema.Boolean;
|
|
20
|
+
}>;
|
|
21
|
+
output: Schema.Struct<{
|
|
22
|
+
text: Schema.filter<typeof Schema.String>;
|
|
23
|
+
format: Schema.Literal<["text", "json"]>;
|
|
24
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
25
|
+
truncated: typeof Schema.Boolean;
|
|
26
|
+
}>;
|
|
27
|
+
error: Schema.NullOr<Schema.Struct<{
|
|
28
|
+
text: Schema.filter<typeof Schema.String>;
|
|
29
|
+
format: Schema.Literal<["text", "json"]>;
|
|
30
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
31
|
+
truncated: typeof Schema.Boolean;
|
|
32
|
+
}>>;
|
|
33
|
+
}>;
|
|
34
|
+
type ValidationCall = typeof ValidationCall.Type;
|
|
35
|
+
declare const EvalValidation: Schema.Struct<{
|
|
36
|
+
id: Schema.filter<Schema.filter<typeof Schema.String>>;
|
|
37
|
+
index: Schema.refine<number, typeof Schema.NonNegative>;
|
|
38
|
+
name: Schema.filter<typeof Schema.String>;
|
|
39
|
+
kind: Schema.Literal<["code", "judge", "command"]>;
|
|
40
|
+
status: Schema.Literal<["queued", "running", "passed", "failed", "error", "skipped"]>;
|
|
41
|
+
startedAt: Schema.NullOr<typeof Schema.Number>;
|
|
42
|
+
durationMs: Schema.NullOr<Schema.refine<number, typeof Schema.NonNegative>>;
|
|
43
|
+
message: Schema.filter<typeof Schema.String>;
|
|
44
|
+
exitCode: Schema.NullOr<typeof Schema.Int>;
|
|
45
|
+
truncated: typeof Schema.Boolean;
|
|
46
|
+
judgment: Schema.optional<Schema.Struct<{
|
|
47
|
+
name: typeof Schema.String;
|
|
48
|
+
model: typeof Schema.String;
|
|
49
|
+
evaluator: typeof Schema.String;
|
|
50
|
+
score: Schema.NullOr<Schema.filter<typeof Schema.Number>>;
|
|
51
|
+
choice: Schema.NullOr<typeof Schema.String>;
|
|
52
|
+
reason: typeof Schema.String;
|
|
53
|
+
threshold: Schema.filter<typeof Schema.Number>;
|
|
54
|
+
durationMs: Schema.refine<number, typeof Schema.NonNegative>;
|
|
55
|
+
error: Schema.NullOr<typeof Schema.String>;
|
|
56
|
+
usage: Schema.optional<Schema.Struct<{
|
|
57
|
+
cacheReadTokens: typeof Schema.Int;
|
|
58
|
+
cacheWriteTokens: typeof Schema.Int;
|
|
59
|
+
cacheWrite1hTokens: Schema.optional<typeof Schema.Int>;
|
|
60
|
+
costUsd: Schema.optional<typeof Schema.Number>;
|
|
61
|
+
inputTokens: typeof Schema.Int;
|
|
62
|
+
outputTokens: typeof Schema.Int;
|
|
63
|
+
totalTokens: typeof Schema.Int;
|
|
64
|
+
}>>;
|
|
65
|
+
}>>;
|
|
66
|
+
input: Schema.Struct<{
|
|
67
|
+
text: Schema.filter<typeof Schema.String>;
|
|
68
|
+
format: Schema.Literal<["text", "json"]>;
|
|
69
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
70
|
+
truncated: typeof Schema.Boolean;
|
|
71
|
+
}>;
|
|
72
|
+
output: Schema.Struct<{
|
|
73
|
+
text: Schema.filter<typeof Schema.String>;
|
|
74
|
+
format: Schema.Literal<["text", "json"]>;
|
|
75
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
76
|
+
truncated: typeof Schema.Boolean;
|
|
77
|
+
}>;
|
|
78
|
+
error: Schema.NullOr<Schema.Struct<{
|
|
79
|
+
text: Schema.filter<typeof Schema.String>;
|
|
80
|
+
format: Schema.Literal<["text", "json"]>;
|
|
81
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
82
|
+
truncated: typeof Schema.Boolean;
|
|
83
|
+
}>>;
|
|
84
|
+
metadata: Schema.optional<Schema.Struct<{
|
|
85
|
+
text: Schema.filter<typeof Schema.String>;
|
|
86
|
+
format: Schema.Literal<["text", "json"]>;
|
|
87
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
88
|
+
truncated: typeof Schema.Boolean;
|
|
89
|
+
}>>;
|
|
90
|
+
calls: Schema.filter<Schema.Array$<Schema.Struct<{
|
|
91
|
+
index: Schema.refine<number, typeof Schema.NonNegative>;
|
|
92
|
+
method: Schema.Literal<["answer", "transcript", "turns", "readText", "exists", "exec", "cli.calls", "api.calls", "api.url", "mcp.calls"]>;
|
|
93
|
+
startedAt: typeof Schema.Number;
|
|
94
|
+
durationMs: Schema.NullOr<Schema.refine<number, typeof Schema.NonNegative>>;
|
|
95
|
+
input: Schema.Struct<{
|
|
96
|
+
text: Schema.filter<typeof Schema.String>;
|
|
97
|
+
format: Schema.Literal<["text", "json"]>;
|
|
98
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
99
|
+
truncated: typeof Schema.Boolean;
|
|
100
|
+
}>;
|
|
101
|
+
output: Schema.Struct<{
|
|
102
|
+
text: Schema.filter<typeof Schema.String>;
|
|
103
|
+
format: Schema.Literal<["text", "json"]>;
|
|
104
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
105
|
+
truncated: typeof Schema.Boolean;
|
|
106
|
+
}>;
|
|
107
|
+
error: Schema.NullOr<Schema.Struct<{
|
|
108
|
+
text: Schema.filter<typeof Schema.String>;
|
|
109
|
+
format: Schema.Literal<["text", "json"]>;
|
|
110
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
111
|
+
truncated: typeof Schema.Boolean;
|
|
112
|
+
}>>;
|
|
113
|
+
}>>>;
|
|
114
|
+
logs: Schema.filter<Schema.Array$<Schema.Struct<{
|
|
115
|
+
index: Schema.refine<number, typeof Schema.NonNegative>;
|
|
116
|
+
at: typeof Schema.Number;
|
|
117
|
+
level: Schema.Literal<["stdout", "stderr"]>;
|
|
118
|
+
value: Schema.Struct<{
|
|
119
|
+
text: Schema.filter<typeof Schema.String>;
|
|
120
|
+
format: Schema.Literal<["text", "json"]>;
|
|
121
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
122
|
+
truncated: typeof Schema.Boolean;
|
|
123
|
+
}>;
|
|
124
|
+
}>>>;
|
|
125
|
+
}>;
|
|
126
|
+
type EvalValidation = typeof EvalValidation.Type;
|
|
127
|
+
//#endregion
|
|
128
|
+
//#region ../schema/src/domain/api-mocks.d.ts
|
|
129
|
+
declare const ApiCall_base: Schema.Class<ApiCall, {
|
|
130
|
+
api: typeof Schema.String;
|
|
131
|
+
index: Schema.refine<number, typeof Schema.NonNegative>;
|
|
132
|
+
method: typeof Schema.String;
|
|
133
|
+
path: typeof Schema.String;
|
|
134
|
+
matched: typeof Schema.Boolean;
|
|
135
|
+
startedAt: typeof Schema.Number;
|
|
136
|
+
durationMs: Schema.refine<number, typeof Schema.NonNegative>;
|
|
137
|
+
input: Schema.Struct<{
|
|
138
|
+
text: Schema.filter<typeof Schema.String>;
|
|
139
|
+
format: Schema.Literal<["text", "json"]>;
|
|
140
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
141
|
+
truncated: typeof Schema.Boolean;
|
|
142
|
+
}>;
|
|
143
|
+
output: Schema.Struct<{
|
|
144
|
+
text: Schema.filter<typeof Schema.String>;
|
|
145
|
+
format: Schema.Literal<["text", "json"]>;
|
|
146
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
147
|
+
truncated: typeof Schema.Boolean;
|
|
148
|
+
}>;
|
|
149
|
+
status: typeof Schema.Int;
|
|
150
|
+
error: Schema.NullOr<Schema.filter<typeof Schema.String>>;
|
|
151
|
+
logs: Schema.filter<Schema.Array$<Schema.Struct<{
|
|
152
|
+
text: Schema.filter<typeof Schema.String>;
|
|
153
|
+
format: Schema.Literal<["text", "json"]>;
|
|
154
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
155
|
+
truncated: typeof Schema.Boolean;
|
|
156
|
+
}>>>;
|
|
157
|
+
}, Schema.Struct.Encoded<{
|
|
158
|
+
api: typeof Schema.String;
|
|
159
|
+
index: Schema.refine<number, typeof Schema.NonNegative>;
|
|
160
|
+
method: typeof Schema.String;
|
|
161
|
+
path: typeof Schema.String;
|
|
162
|
+
matched: typeof Schema.Boolean;
|
|
163
|
+
startedAt: typeof Schema.Number;
|
|
164
|
+
durationMs: Schema.refine<number, typeof Schema.NonNegative>;
|
|
165
|
+
input: Schema.Struct<{
|
|
166
|
+
text: Schema.filter<typeof Schema.String>;
|
|
167
|
+
format: Schema.Literal<["text", "json"]>;
|
|
168
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
169
|
+
truncated: typeof Schema.Boolean;
|
|
170
|
+
}>;
|
|
171
|
+
output: Schema.Struct<{
|
|
172
|
+
text: Schema.filter<typeof Schema.String>;
|
|
173
|
+
format: Schema.Literal<["text", "json"]>;
|
|
174
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
175
|
+
truncated: typeof Schema.Boolean;
|
|
176
|
+
}>;
|
|
177
|
+
status: typeof Schema.Int;
|
|
178
|
+
error: Schema.NullOr<Schema.filter<typeof Schema.String>>;
|
|
179
|
+
logs: Schema.filter<Schema.Array$<Schema.Struct<{
|
|
180
|
+
text: Schema.filter<typeof Schema.String>;
|
|
181
|
+
format: Schema.Literal<["text", "json"]>;
|
|
182
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
183
|
+
truncated: typeof Schema.Boolean;
|
|
184
|
+
}>>>;
|
|
185
|
+
}>, never, {
|
|
186
|
+
readonly api: string;
|
|
187
|
+
} & {
|
|
188
|
+
readonly output: {
|
|
189
|
+
readonly text: string;
|
|
190
|
+
readonly format: "text" | "json";
|
|
191
|
+
readonly state: "captured" | "disabled" | "unavailable";
|
|
192
|
+
readonly truncated: boolean;
|
|
193
|
+
};
|
|
194
|
+
} & {
|
|
195
|
+
readonly input: {
|
|
196
|
+
readonly text: string;
|
|
197
|
+
readonly format: "text" | "json";
|
|
198
|
+
readonly state: "captured" | "disabled" | "unavailable";
|
|
199
|
+
readonly truncated: boolean;
|
|
200
|
+
};
|
|
201
|
+
} & {
|
|
202
|
+
readonly path: string;
|
|
203
|
+
} & {
|
|
204
|
+
readonly index: number;
|
|
205
|
+
} & {
|
|
206
|
+
readonly method: string;
|
|
207
|
+
} & {
|
|
208
|
+
readonly matched: boolean;
|
|
209
|
+
} & {
|
|
210
|
+
readonly startedAt: number;
|
|
211
|
+
} & {
|
|
212
|
+
readonly durationMs: number;
|
|
213
|
+
} & {
|
|
214
|
+
readonly status: number;
|
|
215
|
+
} & {
|
|
216
|
+
readonly error: string | null;
|
|
217
|
+
} & {
|
|
218
|
+
readonly logs: readonly {
|
|
219
|
+
readonly text: string;
|
|
220
|
+
readonly format: "text" | "json";
|
|
221
|
+
readonly state: "captured" | "disabled" | "unavailable";
|
|
222
|
+
readonly truncated: boolean;
|
|
223
|
+
}[];
|
|
224
|
+
}, {}, {}>;
|
|
225
|
+
declare class ApiCall extends ApiCall_base {}
|
|
226
|
+
//#endregion
|
|
227
|
+
export { ValidationValue as i, EvalValidation as n, ValidationCall as r, ApiCall as t };
|