anpord 0.1.27 → 0.1.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api-context.cjs +1 -1
- package/dist/api-context.d.cts +1 -1
- package/dist/api-context.d.mts +1 -1
- package/dist/api-context.mjs +1 -1
- package/dist/{api-mocks-CBa3MWVP.d.cts → api-mocks-Bp5izg-M.d.cts} +3 -3
- package/dist/{api-mocks-CBa3MWVP.d.mts → api-mocks-Bp5izg-M.d.mts} +3 -3
- package/dist/{api-mocks-DH1C7any.mjs → api-mocks-DDs4SgSW.mjs} +33 -3
- package/dist/{api-mocks-BsUGpsim.cjs → api-mocks-p7nRxMS0.cjs} +44 -2
- package/dist/api-runtime.cjs +1 -1
- package/dist/api-runtime.d.cts +1 -1
- package/dist/api-runtime.d.mts +1 -1
- package/dist/api-runtime.mjs +1 -1
- package/dist/api.cjs +1 -1
- package/dist/api.d.cts +1 -1
- package/dist/api.d.mts +1 -1
- package/dist/api.mjs +1 -1
- package/dist/bin.cjs +110 -35
- package/dist/bin.mjs +110 -35
- package/dist/cli-runtime.cjs +1 -1
- package/dist/cli-runtime.mjs +1 -1
- package/dist/{compiler-DKWUVkxB.cjs → compiler-8DrTWGac.cjs} +1 -1
- package/dist/{compiler-DDc7Towh.mjs → compiler-dmc6AOx6.mjs} +1 -1
- package/dist/{errors-D1L76YFH.cjs → errors-C1vD70D6.cjs} +3 -2
- package/dist/{errors-DaYv6_WJ.mjs → errors-H6woEW0d.mjs} +3 -2
- package/dist/{eval-definition-B365FMXW.d.cts → eval-definition-Cs6o8JLJ.d.cts} +10 -10
- package/dist/{eval-definition-B365FMXW.d.mts → eval-definition-Cs6o8JLJ.d.mts} +10 -10
- package/dist/eval.cjs +1 -1
- package/dist/eval.d.cts +2 -2
- package/dist/eval.d.mts +2 -2
- package/dist/eval.mjs +1 -1
- package/dist/index.cjs +2 -2
- package/dist/index.d.cts +58 -51
- package/dist/index.d.mts +58 -51
- package/dist/index.mjs +2 -2
- package/dist/mcp-runtime.cjs +1 -1
- package/dist/mcp-runtime.mjs +1 -1
- package/dist/{runtime-CjDUik7_.mjs → runtime-DxvthKib.mjs} +1 -1
- package/dist/{runtime-BcX6BpI5.cjs → runtime-V4yw0sb-.cjs} +1 -1
- package/dist/source.d.cts +1 -1
- package/dist/source.d.mts +1 -1
- package/dist/{types-DuH3gZVT.d.mts → types-CZqrXnhd.d.mts} +5 -5
- package/dist/{types-QRY2ArEX.d.cts → types-D1BHBwZd.d.cts} +5 -5
- package/dist/validator-runtime.cjs +1 -1
- package/dist/validator-runtime.d.cts +1 -1
- package/dist/validator-runtime.d.mts +1 -1
- package/dist/validator-runtime.mjs +1 -1
- package/package.json +1 -1
package/dist/api-context.cjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
-
const require_api_mocks = require("./api-mocks-
|
|
2
|
+
const require_api_mocks = require("./api-mocks-p7nRxMS0.cjs");
|
|
3
3
|
let effect = require("effect");
|
|
4
4
|
let node_fs_promises = require("node:fs/promises");
|
|
5
5
|
//#region src/mock-api/context.ts
|
package/dist/api-context.d.cts
CHANGED
package/dist/api-context.d.mts
CHANGED
package/dist/api-context.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { n as API_MANIFEST, o as ApiCall, s as ApiManifest, t as API_JOURNAL } from "./api-mocks-
|
|
1
|
+
import { n as API_MANIFEST, o as ApiCall, s as ApiManifest, t as API_JOURNAL } from "./api-mocks-DDs4SgSW.mjs";
|
|
2
2
|
import { Schema } from "effect";
|
|
3
3
|
import { readFile } from "node:fs/promises";
|
|
4
4
|
//#region src/mock-api/context.ts
|
|
@@ -11,7 +11,7 @@ declare const ValidationCall: Schema.Struct<{
|
|
|
11
11
|
index: Schema.refine<number, typeof Schema.NonNegative>;
|
|
12
12
|
method: Schema.Literal<["answer", "transcript", "turns", "readText", "exists", "exec", "cli.calls", "api.calls", "api.url", "mcp.calls"]>;
|
|
13
13
|
startedAt: typeof Schema.Number;
|
|
14
|
-
durationMs: Schema.NullOr<Schema.refine<number, typeof Schema.NonNegative
|
|
14
|
+
durationMs: Schema.NullOr<Schema.transform<typeof Schema.Number, Schema.refine<number, typeof Schema.NonNegative>>>;
|
|
15
15
|
input: Schema.Struct<{
|
|
16
16
|
text: Schema.filter<typeof Schema.String>;
|
|
17
17
|
format: Schema.Literal<["text", "json"]>;
|
|
@@ -39,7 +39,7 @@ declare const EvalValidation: Schema.Struct<{
|
|
|
39
39
|
kind: Schema.Literal<["code", "judge", "command"]>;
|
|
40
40
|
status: Schema.Literal<["queued", "running", "passed", "failed", "error", "skipped"]>;
|
|
41
41
|
startedAt: Schema.NullOr<typeof Schema.Number>;
|
|
42
|
-
durationMs: Schema.NullOr<Schema.refine<number, typeof Schema.NonNegative
|
|
42
|
+
durationMs: Schema.NullOr<Schema.transform<typeof Schema.Number, Schema.refine<number, typeof Schema.NonNegative>>>;
|
|
43
43
|
message: Schema.filter<typeof Schema.String>;
|
|
44
44
|
exitCode: Schema.NullOr<typeof Schema.Int>;
|
|
45
45
|
truncated: typeof Schema.Boolean;
|
|
@@ -91,7 +91,7 @@ declare const EvalValidation: Schema.Struct<{
|
|
|
91
91
|
index: Schema.refine<number, typeof Schema.NonNegative>;
|
|
92
92
|
method: Schema.Literal<["answer", "transcript", "turns", "readText", "exists", "exec", "cli.calls", "api.calls", "api.url", "mcp.calls"]>;
|
|
93
93
|
startedAt: typeof Schema.Number;
|
|
94
|
-
durationMs: Schema.NullOr<Schema.refine<number, typeof Schema.NonNegative
|
|
94
|
+
durationMs: Schema.NullOr<Schema.transform<typeof Schema.Number, Schema.refine<number, typeof Schema.NonNegative>>>;
|
|
95
95
|
input: Schema.Struct<{
|
|
96
96
|
text: Schema.filter<typeof Schema.String>;
|
|
97
97
|
format: Schema.Literal<["text", "json"]>;
|
|
@@ -11,7 +11,7 @@ declare const ValidationCall: Schema.Struct<{
|
|
|
11
11
|
index: Schema.refine<number, typeof Schema.NonNegative>;
|
|
12
12
|
method: Schema.Literal<["answer", "transcript", "turns", "readText", "exists", "exec", "cli.calls", "api.calls", "api.url", "mcp.calls"]>;
|
|
13
13
|
startedAt: typeof Schema.Number;
|
|
14
|
-
durationMs: Schema.NullOr<Schema.refine<number, typeof Schema.NonNegative
|
|
14
|
+
durationMs: Schema.NullOr<Schema.transform<typeof Schema.Number, Schema.refine<number, typeof Schema.NonNegative>>>;
|
|
15
15
|
input: Schema.Struct<{
|
|
16
16
|
text: Schema.filter<typeof Schema.String>;
|
|
17
17
|
format: Schema.Literal<["text", "json"]>;
|
|
@@ -39,7 +39,7 @@ declare const EvalValidation: Schema.Struct<{
|
|
|
39
39
|
kind: Schema.Literal<["code", "judge", "command"]>;
|
|
40
40
|
status: Schema.Literal<["queued", "running", "passed", "failed", "error", "skipped"]>;
|
|
41
41
|
startedAt: Schema.NullOr<typeof Schema.Number>;
|
|
42
|
-
durationMs: Schema.NullOr<Schema.refine<number, typeof Schema.NonNegative
|
|
42
|
+
durationMs: Schema.NullOr<Schema.transform<typeof Schema.Number, Schema.refine<number, typeof Schema.NonNegative>>>;
|
|
43
43
|
message: Schema.filter<typeof Schema.String>;
|
|
44
44
|
exitCode: Schema.NullOr<typeof Schema.Int>;
|
|
45
45
|
truncated: typeof Schema.Boolean;
|
|
@@ -91,7 +91,7 @@ declare const EvalValidation: Schema.Struct<{
|
|
|
91
91
|
index: Schema.refine<number, typeof Schema.NonNegative>;
|
|
92
92
|
method: Schema.Literal<["answer", "transcript", "turns", "readText", "exists", "exec", "cli.calls", "api.calls", "api.url", "mcp.calls"]>;
|
|
93
93
|
startedAt: typeof Schema.Number;
|
|
94
|
-
durationMs: Schema.NullOr<Schema.refine<number, typeof Schema.NonNegative
|
|
94
|
+
durationMs: Schema.NullOr<Schema.transform<typeof Schema.Number, Schema.refine<number, typeof Schema.NonNegative>>>;
|
|
95
95
|
input: Schema.Struct<{
|
|
96
96
|
text: Schema.filter<typeof Schema.String>;
|
|
97
97
|
format: Schema.Literal<["text", "json"]>;
|
|
@@ -35,6 +35,12 @@ const REPORTED_LIMITS = {
|
|
|
35
35
|
budget: 192e3,
|
|
36
36
|
text: 64e3
|
|
37
37
|
};
|
|
38
|
+
const DurationMs = Schema.transform(Schema.Number, Schema.NonNegativeInt, {
|
|
39
|
+
decode: (value) => Math.max(0, Math.round(value)),
|
|
40
|
+
encode: (value) => value,
|
|
41
|
+
strict: true
|
|
42
|
+
});
|
|
43
|
+
const validationDuration = (startedAt, finished) => startedAt === null ? null : Math.max(0, Math.round(finished - startedAt));
|
|
38
44
|
const valueWithin = (limit) => Schema.Struct({
|
|
39
45
|
text: Schema.String.pipe(Schema.maxLength(limit)),
|
|
40
46
|
format: Schema.Literal("text", "json"),
|
|
@@ -45,7 +51,7 @@ const callOf = (value) => Schema.Struct({
|
|
|
45
51
|
index: Schema.NonNegativeInt,
|
|
46
52
|
method: Schema.Literal("answer", "transcript", "turns", "readText", "exists", "exec", "cli.calls", "api.calls", "api.url", "mcp.calls"),
|
|
47
53
|
startedAt: Schema.Number,
|
|
48
|
-
durationMs: Schema.NullOr(
|
|
54
|
+
durationMs: Schema.NullOr(DurationMs),
|
|
49
55
|
input: value,
|
|
50
56
|
output: value,
|
|
51
57
|
error: Schema.NullOr(value)
|
|
@@ -57,7 +63,7 @@ const validationOf = (value, call) => Schema.Struct({
|
|
|
57
63
|
kind: Schema.Literal("code", "judge", "command"),
|
|
58
64
|
status: Schema.Literal("queued", "running", "passed", "failed", "error", "skipped"),
|
|
59
65
|
startedAt: Schema.NullOr(Schema.Number),
|
|
60
|
-
durationMs: Schema.NullOr(
|
|
66
|
+
durationMs: Schema.NullOr(DurationMs),
|
|
61
67
|
message: Schema.String.pipe(Schema.maxLength(2e3)),
|
|
62
68
|
exitCode: Schema.NullOr(Schema.Int),
|
|
63
69
|
truncated: Schema.Boolean,
|
|
@@ -100,6 +106,30 @@ const unavailableValue = {
|
|
|
100
106
|
state: "unavailable",
|
|
101
107
|
truncated: false
|
|
102
108
|
};
|
|
109
|
+
const withoutText = (value) => value.text === "" ? value : {
|
|
110
|
+
...unavailableValue,
|
|
111
|
+
format: value.format,
|
|
112
|
+
truncated: true
|
|
113
|
+
};
|
|
114
|
+
const orNull = (value) => value === null ? null : withoutText(value);
|
|
115
|
+
const validationWithoutEvidence = (value) => ({
|
|
116
|
+
...value,
|
|
117
|
+
calls: value.calls.map((call) => ({
|
|
118
|
+
...call,
|
|
119
|
+
error: orNull(call.error),
|
|
120
|
+
input: withoutText(call.input),
|
|
121
|
+
output: withoutText(call.output)
|
|
122
|
+
})),
|
|
123
|
+
error: orNull(value.error),
|
|
124
|
+
input: withoutText(value.input),
|
|
125
|
+
logs: value.logs.map((log) => ({
|
|
126
|
+
...log,
|
|
127
|
+
value: withoutText(log.value)
|
|
128
|
+
})),
|
|
129
|
+
output: withoutText(value.output),
|
|
130
|
+
truncated: true,
|
|
131
|
+
...value.metadata === void 0 ? {} : { metadata: withoutText(value.metadata) }
|
|
132
|
+
});
|
|
103
133
|
const validationCapture = (enabled = true, secrets = [], limits = EVIDENCE_LIMITS) => {
|
|
104
134
|
let remaining = limits.budget;
|
|
105
135
|
return (value, format = "json") => {
|
|
@@ -182,4 +212,4 @@ var ApiCall = class extends Schema.Class("ApiCall")({
|
|
|
182
212
|
logs: Schema.Array(ReportedValue).pipe(Schema.maxItems(64))
|
|
183
213
|
}) {};
|
|
184
214
|
//#endregion
|
|
185
|
-
export { unavailableValue as _, API_REPORTED_LIMITS as a,
|
|
215
|
+
export { redactSecrets as C, validationWithoutEvidence as S, unavailableValue as _, API_REPORTED_LIMITS as a, validationExecution as b, ApiProgram as c, EVIDENCE_LIMITS as d, EvalValidations as f, VALIDATION_TEXT_LIMIT as g, VALIDATION_FRAME as h, API_READY as i, CLI_JOURNAL as l, ReportedValidation as m, API_MANIFEST as n, ApiCall as o, REPORTED_LIMITS as p, API_PROGRAM as r, ApiManifest as s, API_JOURNAL as t, MCP_JOURNAL as u, validationCapture as v, validationSnapshot as x, validationDuration as y };
|
|
@@ -35,6 +35,12 @@ const REPORTED_LIMITS = {
|
|
|
35
35
|
budget: 192e3,
|
|
36
36
|
text: 64e3
|
|
37
37
|
};
|
|
38
|
+
const DurationMs = effect.Schema.transform(effect.Schema.Number, effect.Schema.NonNegativeInt, {
|
|
39
|
+
decode: (value) => Math.max(0, Math.round(value)),
|
|
40
|
+
encode: (value) => value,
|
|
41
|
+
strict: true
|
|
42
|
+
});
|
|
43
|
+
const validationDuration = (startedAt, finished) => startedAt === null ? null : Math.max(0, Math.round(finished - startedAt));
|
|
38
44
|
const valueWithin = (limit) => effect.Schema.Struct({
|
|
39
45
|
text: effect.Schema.String.pipe(effect.Schema.maxLength(limit)),
|
|
40
46
|
format: effect.Schema.Literal("text", "json"),
|
|
@@ -45,7 +51,7 @@ const callOf = (value) => effect.Schema.Struct({
|
|
|
45
51
|
index: effect.Schema.NonNegativeInt,
|
|
46
52
|
method: effect.Schema.Literal("answer", "transcript", "turns", "readText", "exists", "exec", "cli.calls", "api.calls", "api.url", "mcp.calls"),
|
|
47
53
|
startedAt: effect.Schema.Number,
|
|
48
|
-
durationMs: effect.Schema.NullOr(
|
|
54
|
+
durationMs: effect.Schema.NullOr(DurationMs),
|
|
49
55
|
input: value,
|
|
50
56
|
output: value,
|
|
51
57
|
error: effect.Schema.NullOr(value)
|
|
@@ -57,7 +63,7 @@ const validationOf = (value, call) => effect.Schema.Struct({
|
|
|
57
63
|
kind: effect.Schema.Literal("code", "judge", "command"),
|
|
58
64
|
status: effect.Schema.Literal("queued", "running", "passed", "failed", "error", "skipped"),
|
|
59
65
|
startedAt: effect.Schema.NullOr(effect.Schema.Number),
|
|
60
|
-
durationMs: effect.Schema.NullOr(
|
|
66
|
+
durationMs: effect.Schema.NullOr(DurationMs),
|
|
61
67
|
message: effect.Schema.String.pipe(effect.Schema.maxLength(2e3)),
|
|
62
68
|
exitCode: effect.Schema.NullOr(effect.Schema.Int),
|
|
63
69
|
truncated: effect.Schema.Boolean,
|
|
@@ -100,6 +106,30 @@ const unavailableValue = {
|
|
|
100
106
|
state: "unavailable",
|
|
101
107
|
truncated: false
|
|
102
108
|
};
|
|
109
|
+
const withoutText = (value) => value.text === "" ? value : {
|
|
110
|
+
...unavailableValue,
|
|
111
|
+
format: value.format,
|
|
112
|
+
truncated: true
|
|
113
|
+
};
|
|
114
|
+
const orNull = (value) => value === null ? null : withoutText(value);
|
|
115
|
+
const validationWithoutEvidence = (value) => ({
|
|
116
|
+
...value,
|
|
117
|
+
calls: value.calls.map((call) => ({
|
|
118
|
+
...call,
|
|
119
|
+
error: orNull(call.error),
|
|
120
|
+
input: withoutText(call.input),
|
|
121
|
+
output: withoutText(call.output)
|
|
122
|
+
})),
|
|
123
|
+
error: orNull(value.error),
|
|
124
|
+
input: withoutText(value.input),
|
|
125
|
+
logs: value.logs.map((log) => ({
|
|
126
|
+
...log,
|
|
127
|
+
value: withoutText(log.value)
|
|
128
|
+
})),
|
|
129
|
+
output: withoutText(value.output),
|
|
130
|
+
truncated: true,
|
|
131
|
+
...value.metadata === void 0 ? {} : { metadata: withoutText(value.metadata) }
|
|
132
|
+
});
|
|
103
133
|
const validationCapture = (enabled = true, secrets = [], limits = EVIDENCE_LIMITS) => {
|
|
104
134
|
let remaining = limits.budget;
|
|
105
135
|
return (value, format = "json") => {
|
|
@@ -296,6 +326,12 @@ Object.defineProperty(exports, "validationCapture", {
|
|
|
296
326
|
return validationCapture;
|
|
297
327
|
}
|
|
298
328
|
});
|
|
329
|
+
Object.defineProperty(exports, "validationDuration", {
|
|
330
|
+
enumerable: true,
|
|
331
|
+
get: function() {
|
|
332
|
+
return validationDuration;
|
|
333
|
+
}
|
|
334
|
+
});
|
|
299
335
|
Object.defineProperty(exports, "validationExecution", {
|
|
300
336
|
enumerable: true,
|
|
301
337
|
get: function() {
|
|
@@ -308,3 +344,9 @@ Object.defineProperty(exports, "validationSnapshot", {
|
|
|
308
344
|
return validationSnapshot;
|
|
309
345
|
}
|
|
310
346
|
});
|
|
347
|
+
Object.defineProperty(exports, "validationWithoutEvidence", {
|
|
348
|
+
enumerable: true,
|
|
349
|
+
get: function() {
|
|
350
|
+
return validationWithoutEvidence;
|
|
351
|
+
}
|
|
352
|
+
});
|
package/dist/api-runtime.cjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
-
const require_runtime = require("./runtime-
|
|
2
|
+
const require_runtime = require("./runtime-V4yw0sb-.cjs");
|
|
3
3
|
exports.runApiServers = require_runtime.runApiServers;
|
|
4
4
|
exports.withApi = require_runtime.withApi;
|
package/dist/api-runtime.d.cts
CHANGED
package/dist/api-runtime.d.mts
CHANGED
package/dist/api-runtime.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { n as withApi, t as runApiServers } from "./runtime-
|
|
1
|
+
import { n as withApi, t as runApiServers } from "./runtime-DxvthKib.mjs";
|
|
2
2
|
export { runApiServers, withApi };
|
package/dist/api.cjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
-
const require_runtime = require("./runtime-
|
|
2
|
+
const require_runtime = require("./runtime-V4yw0sb-.cjs");
|
|
3
3
|
exports.api = require_runtime.api;
|
|
4
4
|
exports.endpoint = require_runtime.endpoint;
|
|
5
5
|
exports.withApi = require_runtime.withApi;
|
package/dist/api.d.cts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { t as ApiCall } from "./api-mocks-
|
|
1
|
+
import { t as ApiCall } from "./api-mocks-Bp5izg-M.cjs";
|
|
2
2
|
import { a as endpoint, i as api, n as ApiHandlerContext, r as EndpointDefinition, t as ApiDefinition } from "./define-NZvmQuIv.cjs";
|
|
3
3
|
import { withApi } from "./api-runtime.cjs";
|
|
4
4
|
export { type ApiCall, type ApiDefinition, type ApiHandlerContext, type EndpointDefinition, api, endpoint, withApi };
|
package/dist/api.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { t as ApiCall } from "./api-mocks-
|
|
1
|
+
import { t as ApiCall } from "./api-mocks-Bp5izg-M.mjs";
|
|
2
2
|
import { a as endpoint, i as api, n as ApiHandlerContext, r as EndpointDefinition, t as ApiDefinition } from "./define-NZvmQuIv.mjs";
|
|
3
3
|
import { withApi } from "./api-runtime.mjs";
|
|
4
4
|
export { type ApiCall, type ApiDefinition, type ApiHandlerContext, type EndpointDefinition, api, endpoint, withApi };
|
package/dist/api.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { i as endpoint, n as withApi, r as api } from "./runtime-
|
|
1
|
+
import { i as endpoint, n as withApi, r as api } from "./runtime-DxvthKib.mjs";
|
|
2
2
|
export { api, endpoint, withApi };
|
package/dist/bin.cjs
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
const require_eval_judges = require("./eval-judges-CO0ds-HQ.cjs");
|
|
3
|
-
const require_api_mocks = require("./api-mocks-
|
|
4
|
-
const require_errors = require("./errors-
|
|
5
|
-
const require_compiler = require("./compiler-
|
|
3
|
+
const require_api_mocks = require("./api-mocks-p7nRxMS0.cjs");
|
|
4
|
+
const require_errors = require("./errors-C1vD70D6.cjs");
|
|
5
|
+
const require_compiler = require("./compiler-8DrTWGac.cjs");
|
|
6
6
|
const require_eval_turns = require("./eval-turns-B9dd8Vfb.cjs");
|
|
7
7
|
const require_sandbox_env = require("./sandbox-env-B8cLIcku.cjs");
|
|
8
8
|
let effect = require("effect");
|
|
@@ -17,7 +17,7 @@ let node_os = require("node:os");
|
|
|
17
17
|
let node_child_process = require("node:child_process");
|
|
18
18
|
let node_crypto = require("node:crypto");
|
|
19
19
|
//#region package.json
|
|
20
|
-
var version$1 = "0.1.
|
|
20
|
+
var version$1 = "0.1.29";
|
|
21
21
|
//#endregion
|
|
22
22
|
//#region ../template/src/extract.ts
|
|
23
23
|
/** Reads escapes too, so text the renderer leaves literal is never reported as
|
|
@@ -680,6 +680,14 @@ const SHADES = {
|
|
|
680
680
|
135
|
|
681
681
|
]
|
|
682
682
|
},
|
|
683
|
+
lavender: {
|
|
684
|
+
fallback: 146,
|
|
685
|
+
rgb: [
|
|
686
|
+
170,
|
|
687
|
+
160,
|
|
688
|
+
204
|
|
689
|
+
]
|
|
690
|
+
},
|
|
683
691
|
magenta: {
|
|
684
692
|
fallback: 141,
|
|
685
693
|
rgb: [
|
|
@@ -696,6 +704,38 @@ const SHADES = {
|
|
|
696
704
|
107
|
|
697
705
|
]
|
|
698
706
|
},
|
|
707
|
+
rose: {
|
|
708
|
+
fallback: 181,
|
|
709
|
+
rgb: [
|
|
710
|
+
204,
|
|
711
|
+
158,
|
|
712
|
+
172
|
|
713
|
+
]
|
|
714
|
+
},
|
|
715
|
+
sand: {
|
|
716
|
+
fallback: 180,
|
|
717
|
+
rgb: [
|
|
718
|
+
200,
|
|
719
|
+
176,
|
|
720
|
+
138
|
|
721
|
+
]
|
|
722
|
+
},
|
|
723
|
+
slate: {
|
|
724
|
+
fallback: 110,
|
|
725
|
+
rgb: [
|
|
726
|
+
138,
|
|
727
|
+
164,
|
|
728
|
+
204
|
|
729
|
+
]
|
|
730
|
+
},
|
|
731
|
+
teal: {
|
|
732
|
+
fallback: 109,
|
|
733
|
+
rgb: [
|
|
734
|
+
128,
|
|
735
|
+
184,
|
|
736
|
+
178
|
|
737
|
+
]
|
|
738
|
+
},
|
|
699
739
|
yellow: {
|
|
700
740
|
fallback: 179,
|
|
701
741
|
rgb: [
|
|
@@ -709,26 +749,8 @@ const TRUE_COLOUR = /* @__PURE__ */ new Set(["truecolor", "24bit"]);
|
|
|
709
749
|
const wrap = (code) => (text) => `${ESCAPE}[${code}m${text}${ESCAPE}[0m`;
|
|
710
750
|
const shadeCode = ({ fallback, rgb }, trueColour) => trueColour ? `38;2;${rgb.join(";")}` : `38;5;${fallback}`;
|
|
711
751
|
const unpainted = (text) => text;
|
|
712
|
-
const colouredWith = (trueColour) => (
|
|
713
|
-
|
|
714
|
-
bold: wrap("1"),
|
|
715
|
-
cyan: wrap(shadeCode(SHADES.cyan, trueColour)),
|
|
716
|
-
dim: wrap(shadeCode(SHADES.dim, trueColour)),
|
|
717
|
-
green: wrap(shadeCode(SHADES.green, trueColour)),
|
|
718
|
-
magenta: wrap(shadeCode(SHADES.magenta, trueColour)),
|
|
719
|
-
red: wrap(shadeCode(SHADES.red, trueColour)),
|
|
720
|
-
yellow: wrap(shadeCode(SHADES.yellow, trueColour))
|
|
721
|
-
});
|
|
722
|
-
const MONOCHROME = {
|
|
723
|
-
blue: unpainted,
|
|
724
|
-
bold: unpainted,
|
|
725
|
-
cyan: unpainted,
|
|
726
|
-
dim: unpainted,
|
|
727
|
-
green: unpainted,
|
|
728
|
-
magenta: unpainted,
|
|
729
|
-
red: unpainted,
|
|
730
|
-
yellow: unpainted
|
|
731
|
-
};
|
|
752
|
+
const colouredWith = (trueColour) => Object.fromEntries([["bold", wrap("1")], ...Object.entries(SHADES).map(([name, shade]) => [name, wrap(shadeCode(shade, trueColour))])]);
|
|
753
|
+
const MONOCHROME = Object.fromEntries(["bold", ...Object.keys(SHADES)].map((name) => [name, unpainted]));
|
|
732
754
|
const paletteFor = (colour) => colour ? colouredWith(TRUE_COLOUR.has(process.env.COLORTERM ?? "")) : MONOCHROME;
|
|
733
755
|
//#endregion
|
|
734
756
|
//#region src/cli/text-wrap.ts
|
|
@@ -1238,6 +1260,12 @@ const BRANCH_WIDTH = 2;
|
|
|
1238
1260
|
const VERB_WIDTH = 7;
|
|
1239
1261
|
const NOTABLE_MS = 1e3;
|
|
1240
1262
|
const GAP = " ";
|
|
1263
|
+
const VERB_TONES = {
|
|
1264
|
+
ran: "sand",
|
|
1265
|
+
read: "slate",
|
|
1266
|
+
search: "teal",
|
|
1267
|
+
wrote: "rose"
|
|
1268
|
+
};
|
|
1241
1269
|
const unless = (text, tone) => text === "" ? "" : tone(text);
|
|
1242
1270
|
const tookBetween = (startedAt, finishedAt) => startedAt === null || startedAt === void 0 || finishedAt === null ? null : Math.max(0, finishedAt - startedAt);
|
|
1243
1271
|
const commandParts = (step) => {
|
|
@@ -1270,7 +1298,7 @@ const stepLine = (step, { branch, paint, room }) => {
|
|
|
1270
1298
|
const failed = failure === null ? "" : `${GAP}✗ ${failure}`;
|
|
1271
1299
|
const label = verb.padEnd(VERB_WIDTH);
|
|
1272
1300
|
const width = room - BRANCH_WIDTH - label.length - 1 - took.length - failed.length;
|
|
1273
|
-
return branch(`${(failure === null ? paint
|
|
1301
|
+
return branch(`${(failure === null ? paint[VERB_TONES[verb] ?? "lavender"] : paint.red)(label)} ${clipped$1(detail, Math.max(1, width))}${unless(took, paint.dim)}${unless(failed, paint.red)}`);
|
|
1274
1302
|
};
|
|
1275
1303
|
//#endregion
|
|
1276
1304
|
//#region src/cli/transcript-turn.ts
|
|
@@ -1767,6 +1795,21 @@ const profileOfRequest = (profile) => profile === void 0 ? null : {
|
|
|
1767
1795
|
//#endregion
|
|
1768
1796
|
//#region ../eval/src/domain/journal-entries.ts
|
|
1769
1797
|
const JOURNAL_OUTPUT_LIMIT = 4e3;
|
|
1798
|
+
const cut = (text) => text.length > JOURNAL_OUTPUT_LIMIT ? `${text.slice(0, JOURNAL_OUTPUT_LIMIT)}\n[the rest was cut before this was sent, ${text.length} characters in all]` : text;
|
|
1799
|
+
const cutIfPresent = (text) => text === void 0 ? text : cut(text);
|
|
1800
|
+
const cutToJournal = (event) => {
|
|
1801
|
+
if (event._tag === "Command") return {
|
|
1802
|
+
...event,
|
|
1803
|
+
output: cut(event.output)
|
|
1804
|
+
};
|
|
1805
|
+
if (event._tag === "ToolCall") return {
|
|
1806
|
+
...event,
|
|
1807
|
+
error: cutIfPresent(event.error),
|
|
1808
|
+
input: cut(event.input),
|
|
1809
|
+
output: cutIfPresent(event.output)
|
|
1810
|
+
};
|
|
1811
|
+
return event;
|
|
1812
|
+
};
|
|
1770
1813
|
const millisOrNull = (at) => at ?? null;
|
|
1771
1814
|
const asEntries = (event) => {
|
|
1772
1815
|
if (event._tag === "Command") return [{
|
|
@@ -3940,6 +3983,10 @@ const TRANSCRIPT_PATH = (home) => `${home}/.anpord-transcript.txt`;
|
|
|
3940
3983
|
const TURNS_PATH = (home) => `${home}/.anpord-turns.json`;
|
|
3941
3984
|
//#endregion
|
|
3942
3985
|
//#region ../eval/src/domain/journal.ts
|
|
3986
|
+
const relativeToWorkspace = (workspace, events) => events.map((event) => event._tag === "FileChange" ? {
|
|
3987
|
+
...event,
|
|
3988
|
+
paths: event.paths.map((path) => node_path.posix.isAbsolute(path) ? node_path.posix.relative(workspace, path) : path)
|
|
3989
|
+
} : event);
|
|
3943
3990
|
const commandsIn = (events) => events.filter((event) => event._tag === "Command").length;
|
|
3944
3991
|
const filesIn = (events) => [...new Set(events.flatMap((event) => event._tag === "FileChange" ? event.paths : []))];
|
|
3945
3992
|
const failedCommandsIn = (events) => events.filter((event) => event._tag === "Command" && event.exitCode !== null && event.exitCode !== 0).length;
|
|
@@ -4090,7 +4137,7 @@ const completeValidations = (records, message, finished, error, exitCode) => {
|
|
|
4090
4137
|
exitCode,
|
|
4091
4138
|
message: message ?? "Validator did not return a complete result",
|
|
4092
4139
|
error: record.error ?? error,
|
|
4093
|
-
durationMs: record.startedAt
|
|
4140
|
+
durationMs: require_api_mocks.validationDuration(record.startedAt, finished)
|
|
4094
4141
|
};
|
|
4095
4142
|
if (record.status === "queued" || record.status === "running") return {
|
|
4096
4143
|
...record,
|
|
@@ -4107,7 +4154,7 @@ const legacyValidation = (record, execution, finished, capture) => {
|
|
|
4107
4154
|
status: raw === null ? "error" : resultStatus(raw.passed),
|
|
4108
4155
|
message: (raw?.message ?? (raw === null ? "Validator returned no valid result" : "")).slice(0, 2e3),
|
|
4109
4156
|
output: raw === null ? record.output : capture(raw),
|
|
4110
|
-
durationMs:
|
|
4157
|
+
durationMs: require_api_mocks.validationDuration(record.startedAt ?? finished, finished),
|
|
4111
4158
|
exitCode: execution.exitCode,
|
|
4112
4159
|
logs: [{
|
|
4113
4160
|
index: 0,
|
|
@@ -5468,7 +5515,7 @@ const AgentTrialLive = effect.Layer.effect(AgentTrial, effect.Effect.gen(functio
|
|
|
5468
5515
|
const apiEvents = yield* api.collect();
|
|
5469
5516
|
yield* effect.Stream.fromIterable(apiEvents).pipe(sink.through, effect.Stream.runDrain);
|
|
5470
5517
|
yield* api.check;
|
|
5471
|
-
const events = [...conversation.events, ...apiEvents];
|
|
5518
|
+
const events = relativeToWorkspace(request.workspace, [...conversation.events, ...apiEvents]);
|
|
5472
5519
|
const modelFinished = yield* effect.Clock.currentTimeMillis;
|
|
5473
5520
|
const artifacts = yield* captureArtifacts(sandbox, request.workspace, filesIn(events), secrets);
|
|
5474
5521
|
const scored = yield* scorer.score({
|
|
@@ -5848,7 +5895,7 @@ const brokenBy = (cause, durationMs, events) => ({
|
|
|
5848
5895
|
timedOut: effect.Option.exists(effect.Cause.failureOption(cause), (error) => error instanceof TrialTimedOut)
|
|
5849
5896
|
});
|
|
5850
5897
|
const reportRequest = (result, ordinal, runId) => result.kind === "scored" ? { payload: {
|
|
5851
|
-
events: result.events,
|
|
5898
|
+
events: result.events.map(cutToJournal),
|
|
5852
5899
|
ordinal,
|
|
5853
5900
|
outcome: result.outcome,
|
|
5854
5901
|
runId,
|
|
@@ -5856,7 +5903,7 @@ const reportRequest = (result, ordinal, runId) => result.kind === "scored" ? { p
|
|
|
5856
5903
|
usage: result.usage,
|
|
5857
5904
|
userSpend: result.userSpend
|
|
5858
5905
|
} } : { payload: {
|
|
5859
|
-
events: result.events,
|
|
5906
|
+
events: result.events.map(cutToJournal),
|
|
5860
5907
|
failure: result.reason,
|
|
5861
5908
|
ordinal,
|
|
5862
5909
|
runId,
|
|
@@ -5875,7 +5922,7 @@ const UNSCORED = {
|
|
|
5875
5922
|
voidFields: []
|
|
5876
5923
|
};
|
|
5877
5924
|
const unscoredForOlderServer = (result, ordinal, runId) => ({ payload: {
|
|
5878
|
-
events: result.events,
|
|
5925
|
+
events: result.events.map(cutToJournal),
|
|
5879
5926
|
ordinal,
|
|
5880
5927
|
outcome: UNSCORED,
|
|
5881
5928
|
runId,
|
|
@@ -5883,6 +5930,17 @@ const unscoredForOlderServer = (result, ordinal, runId) => ({ payload: {
|
|
|
5883
5930
|
usage: null,
|
|
5884
5931
|
userSpend: null
|
|
5885
5932
|
} });
|
|
5933
|
+
const withoutJournal = (result) => ({
|
|
5934
|
+
...result,
|
|
5935
|
+
events: []
|
|
5936
|
+
});
|
|
5937
|
+
const withoutEvidence = (result) => result.kind === "scored" ? {
|
|
5938
|
+
...result,
|
|
5939
|
+
outcome: {
|
|
5940
|
+
...result.outcome,
|
|
5941
|
+
validations: result.outcome.validations.map(require_api_mocks.validationWithoutEvidence)
|
|
5942
|
+
}
|
|
5943
|
+
} : result;
|
|
5886
5944
|
const verdictOf = (result) => result.kind === "scored" ? result.outcome : {
|
|
5887
5945
|
failure: result.reason,
|
|
5888
5946
|
status: result.timedOut ? "timed out" : "void",
|
|
@@ -5908,6 +5966,10 @@ const labelOfRequest = (variant) => formatVariant({
|
|
|
5908
5966
|
model: variant.model,
|
|
5909
5967
|
profile: variant.profile?.name ?? null
|
|
5910
5968
|
});
|
|
5969
|
+
const casesAtOnce = effect.Config.string("EVAL_LOCAL_CASES_AT_ONCE").pipe(effect.Config.withDefault("1"), effect.Config.map((value) => value === "unbounded" ? "unbounded" : Number.parseInt(value, 10)), effect.Config.validate({
|
|
5970
|
+
message: "EVAL_LOCAL_CASES_AT_ONCE must be a positive number or unbounded",
|
|
5971
|
+
validation: (value) => value === "unbounded" || value > 0
|
|
5972
|
+
}));
|
|
5911
5973
|
const localConfig = effect.ConfigProvider.fromEnv().pipe(effect.ConfigProvider.orElse(() => effect.ConfigProvider.fromMap(/* @__PURE__ */ new Map([["ANPORD_LOCAL_SANDBOX", "true"]]))));
|
|
5912
5974
|
const printed = (lines) => lines.length === 0 ? effect.Effect.void : note(lines.join("\n"));
|
|
5913
5975
|
const runVariant = (request, variant, transcript, options) => effect.Effect.gen(function* () {
|
|
@@ -5970,7 +6032,7 @@ const runVariant = (request, variant, transcript, options) => effect.Effect.gen(
|
|
|
5970
6032
|
variant: label
|
|
5971
6033
|
});
|
|
5972
6034
|
});
|
|
5973
|
-
return yield* effect.Effect.forEach(request.cases.flatMap((subject) => ordinals.map((ordinal) => [subject, ordinal])), ([subject, ordinal]) => attempt(subject, ordinal), { concurrency:
|
|
6035
|
+
return yield* effect.Effect.forEach(request.cases.flatMap((subject) => ordinals.map((ordinal) => [subject, ordinal])), ([subject, ordinal]) => attempt(subject, ordinal), { concurrency: yield* casesAtOnce });
|
|
5974
6036
|
}).pipe(effect.Effect.provide(options.credentials === void 0 || options.credentials.size === 0 ? EvalLocalLive : evalLocalWith(credentialResolverFrom(options.credentials))), effect.Effect.scoped);
|
|
5975
6037
|
const runLocally = (request, options = {}) => effect.Effect.gen(function* () {
|
|
5976
6038
|
const transcript = yield* makeTranscriber(stderrStyle());
|
|
@@ -6239,9 +6301,19 @@ const untilClosed = (batchId) => require_errors.AnpordApi.pipe(effect.Effect.fla
|
|
|
6239
6301
|
const refused = require_errors.asAnpordError(error);
|
|
6240
6302
|
return refused.status === CLOSED_BATCH ? effect.Effect.fail(new BatchClosed({ message: refused.message })) : effect.Effect.void;
|
|
6241
6303
|
}), effect.Effect.repeat(effect.Schedule.spaced(LOCAL_BEAT_EVERY)), effect.Effect.zipRight(effect.Effect.never));
|
|
6242
|
-
const
|
|
6304
|
+
const TOO_LARGE = 413;
|
|
6305
|
+
const tooLarge = (error) => _effect_platform.HttpClientError.isHttpClientError(error) && error._tag === "ResponseError" && error.response.status === TOO_LARGE;
|
|
6306
|
+
const SHRUNK = [{
|
|
6307
|
+
gaveUp: "its journal",
|
|
6308
|
+
shrink: withoutJournal
|
|
6309
|
+
}, {
|
|
6310
|
+
gaveUp: "its journal or the evidence its checks captured",
|
|
6311
|
+
shrink: (result) => withoutEvidence(withoutJournal(result))
|
|
6312
|
+
}];
|
|
6313
|
+
const reportTrial = (api, result, ordinal, runId, trial) => {
|
|
6243
6314
|
const send = (request) => retryTransient(api.runner.report(request));
|
|
6244
|
-
|
|
6315
|
+
const sendTrial = (sent) => send(reportRequest(sent, ordinal, runId)).pipe(effect.Effect.catchTag("HttpApiDecodeError", (refused) => sent.kind === "broken" ? send(unscoredForOlderServer(sent, ordinal, runId)) : effect.Effect.fail(refused)));
|
|
6316
|
+
return SHRUNK.reduce((attempt, rung) => attempt.pipe(effect.Effect.catchIf(tooLarge, () => sendTrial(rung.shrink(result)).pipe(effect.Effect.zipRight(note(`A trial of ${trial} was too large for Anpord to take, so its verdict was recorded without ${rung.gaveUp}.`))))), sendTrial(result));
|
|
6245
6317
|
};
|
|
6246
6318
|
const recordInto = (label, request, ui, started) => effect.Effect.gen(function* () {
|
|
6247
6319
|
const api = yield* require_errors.AnpordApi;
|
|
@@ -6254,7 +6326,10 @@ const recordInto = (label, request, ui, started) => effect.Effect.gen(function*
|
|
|
6254
6326
|
return {
|
|
6255
6327
|
cases: yield* runLocally(request, {
|
|
6256
6328
|
credentials: leases,
|
|
6257
|
-
onTrial: (slot, result, ordinal) =>
|
|
6329
|
+
onTrial: (slot, result, ordinal) => {
|
|
6330
|
+
const trial = `${slot.caseId} on ${labelOfRequest(slot.variant)}`;
|
|
6331
|
+
return reportTrial(api, result, ordinal, runIdOf(slot), trial).pipe(effect.Effect.catchAll((error) => note(`A trial of ${trial} was not recorded, so it shows as void. ${require_errors.asAnpordError(error).message}`)));
|
|
6332
|
+
}
|
|
6258
6333
|
}),
|
|
6259
6334
|
costs: yield* finishBatch(started.id),
|
|
6260
6335
|
link: effect.Option.some(link)
|