anpord 0.1.11 → 0.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -78
- package/dist/bin.cjs +211 -204
- package/dist/bin.mjs +211 -204
- package/dist/cli-runtime.cjs +2 -2
- package/dist/cli-runtime.d.cts +1 -1
- package/dist/cli-runtime.d.mts +1 -1
- package/dist/cli-runtime.mjs +2 -2
- package/dist/client-BfdjFtlV.mjs +305 -0
- package/dist/{client-C9m0uLCz.d.cts → client-BgRs8JvW.d.mts} +2150 -174
- package/dist/{client-TFCLuMv8.d.mts → client-BirFO3jz.d.cts} +2150 -174
- package/dist/client-C-P2I00z.cjs +346 -0
- package/dist/{compiler-DXlpU_In.mjs → compiler-D96wI7UP.mjs} +108 -75
- package/dist/{compiler-Ap8DM-JU.cjs → compiler-uwb1pkdK.cjs} +116 -83
- package/dist/config.cjs +1 -1
- package/dist/config.d.cts +1 -1
- package/dist/config.d.mts +1 -1
- package/dist/config.mjs +1 -1
- package/dist/{errors-C9bQUcA3.mjs → errors-B0YknR5V.mjs} +3 -19
- package/dist/{errors-Boum34f5.cjs → errors-BX1wry8K.cjs} +3 -19
- package/dist/{errors-BK3f7Rdr.d.cts → errors-aGrd21jy.d.cts} +0 -3
- package/dist/{errors-BK3f7Rdr.d.mts → errors-aGrd21jy.d.mts} +0 -3
- package/dist/eval-judges-BkmiVLLq.cjs +59 -0
- package/dist/eval-judges-CwTvTq7N.mjs +42 -0
- package/dist/eval-judges-DPPUftbh.d.cts +48 -0
- package/dist/eval-judges-DPPUftbh.d.mts +48 -0
- package/dist/eval-validations-BRdoZrTQ.mjs +113 -0
- package/dist/eval-validations-_MlnQ_uv.cjs +148 -0
- package/dist/eval.cjs +1 -1
- package/dist/eval.d.cts +60 -6
- package/dist/eval.d.mts +60 -6
- package/dist/eval.mjs +1 -1
- package/dist/evals-B6aT3xgH.d.cts +1364 -0
- package/dist/evals-B6aT3xgH.d.mts +1364 -0
- package/dist/evals-Dcc_x6jW.mjs +582 -0
- package/dist/evals-DouMsrsM.cjs +737 -0
- package/dist/index.cjs +23 -99
- package/dist/index.d.cts +138 -34
- package/dist/index.d.mts +138 -34
- package/dist/index.mjs +23 -99
- package/dist/source.d.cts +1 -1
- package/dist/source.d.mts +1 -1
- package/dist/{types-oaMF6-E1.d.mts → types-Codjaq-o.d.mts} +146 -45
- package/dist/{types-H9L450Iu.d.cts → types-ISsteq95.d.cts} +146 -45
- package/dist/validator-runtime.cjs +238 -0
- package/dist/validator-runtime.d.cts +9 -0
- package/dist/validator-runtime.d.mts +9 -0
- package/dist/validator-runtime.mjs +237 -0
- package/dist/validators.cjs +10 -0
- package/dist/validators.d.cts +8 -0
- package/dist/validators.d.mts +8 -0
- package/dist/validators.mjs +9 -0
- package/package.json +11 -1
- package/dist/client-5K3dMI5Q.mjs +0 -849
- package/dist/client-Ci0woWLW.cjs +0 -890
- package/dist/evals-BEgEmfiO.d.cts +0 -603
- package/dist/evals-BEgEmfiO.d.mts +0 -603
- package/dist/harness-profile-BMjsN460.mjs +0 -43
- package/dist/harness-profile-CDQpPLPd.cjs +0 -78
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
const require_eval_validations = require("./eval-validations-_MlnQ_uv.cjs");
|
|
3
|
+
let effect = require("effect");
|
|
4
|
+
let node_fs_promises = require("node:fs/promises");
|
|
5
|
+
let node_child_process = require("node:child_process");
|
|
6
|
+
let node_util = require("node:util");
|
|
7
|
+
let node_async_hooks = require("node:async_hooks");
|
|
8
|
+
//#region src/mcp/calls.ts
|
|
9
|
+
const McpCallSchema = effect.Schema.Struct({
|
|
10
|
+
error: effect.Schema.optional(effect.Schema.String),
|
|
11
|
+
input: effect.Schema.Unknown,
|
|
12
|
+
kind: effect.Schema.Literal("tool", "resource"),
|
|
13
|
+
name: effect.Schema.String,
|
|
14
|
+
output: effect.Schema.optional(effect.Schema.Unknown),
|
|
15
|
+
server: effect.Schema.String
|
|
16
|
+
});
|
|
17
|
+
//#endregion
|
|
18
|
+
//#region src/mock-cli/calls.ts
|
|
19
|
+
const CliCallSchema = effect.Schema.Struct({
|
|
20
|
+
cli: effect.Schema.String,
|
|
21
|
+
command: effect.Schema.String,
|
|
22
|
+
error: effect.Schema.optional(effect.Schema.String),
|
|
23
|
+
input: effect.Schema.Unknown,
|
|
24
|
+
output: effect.Schema.optional(effect.Schema.Unknown)
|
|
25
|
+
});
|
|
26
|
+
//#endregion
|
|
27
|
+
//#region src/evals/validator-runtime.ts
|
|
28
|
+
const Result = effect.Schema.Union(effect.Schema.Boolean, effect.Schema.Struct({
|
|
29
|
+
passed: effect.Schema.Boolean,
|
|
30
|
+
message: effect.Schema.optional(effect.Schema.String)
|
|
31
|
+
}));
|
|
32
|
+
const decodeResult = effect.Schema.decodeUnknownSync(Result, { onExcessProperty: "error" });
|
|
33
|
+
const local = new node_async_hooks.AsyncLocalStorage();
|
|
34
|
+
const write = process.stdout.write.bind(process.stdout);
|
|
35
|
+
const emit = (record) => write(`${require_eval_validations.VALIDATION_FRAME}${JSON.stringify(require_eval_validations.validationSnapshot(record))}\n`);
|
|
36
|
+
const now = () => performance.timeOrigin + performance.now();
|
|
37
|
+
const observe = async (method, input, run) => {
|
|
38
|
+
const state = local.getStore();
|
|
39
|
+
if (!state) return run();
|
|
40
|
+
const { record, capture } = state;
|
|
41
|
+
const call = {
|
|
42
|
+
index: record.calls.length,
|
|
43
|
+
method,
|
|
44
|
+
input: capture(input),
|
|
45
|
+
output: require_eval_validations.unavailableValue,
|
|
46
|
+
error: null,
|
|
47
|
+
startedAt: now(),
|
|
48
|
+
durationMs: null
|
|
49
|
+
};
|
|
50
|
+
if (record.calls.length === 64) {
|
|
51
|
+
state.record = {
|
|
52
|
+
...record,
|
|
53
|
+
truncated: true
|
|
54
|
+
};
|
|
55
|
+
return run();
|
|
56
|
+
}
|
|
57
|
+
state.record = {
|
|
58
|
+
...record,
|
|
59
|
+
calls: [...record.calls, call]
|
|
60
|
+
};
|
|
61
|
+
emit(state.record);
|
|
62
|
+
let completed = call;
|
|
63
|
+
try {
|
|
64
|
+
const value = await run();
|
|
65
|
+
completed = {
|
|
66
|
+
...call,
|
|
67
|
+
output: capture(value)
|
|
68
|
+
};
|
|
69
|
+
return value;
|
|
70
|
+
} catch (error) {
|
|
71
|
+
completed = {
|
|
72
|
+
...call,
|
|
73
|
+
error: capture(error instanceof Error ? error.stack : String(error), "text")
|
|
74
|
+
};
|
|
75
|
+
throw error;
|
|
76
|
+
} finally {
|
|
77
|
+
completed = {
|
|
78
|
+
...completed,
|
|
79
|
+
durationMs: Math.max(0, Math.round(now() - call.startedAt))
|
|
80
|
+
};
|
|
81
|
+
state.record = {
|
|
82
|
+
...state.record,
|
|
83
|
+
calls: state.record.calls.map((entry) => entry.index === call.index ? completed : entry)
|
|
84
|
+
};
|
|
85
|
+
emit(state.record);
|
|
86
|
+
}
|
|
87
|
+
};
|
|
88
|
+
const readOptional = async (path) => {
|
|
89
|
+
if (path === void 0) return "";
|
|
90
|
+
try {
|
|
91
|
+
return await (0, node_fs_promises.readFile)(path, "utf8");
|
|
92
|
+
} catch (error) {
|
|
93
|
+
if (error instanceof Error && "code" in error && error.code === "ENOENT") return "";
|
|
94
|
+
throw error;
|
|
95
|
+
}
|
|
96
|
+
};
|
|
97
|
+
const exec = (command) => new Promise((resolve, reject) => {
|
|
98
|
+
const child = (0, node_child_process.spawn)("/bin/sh", ["-lc", command], { cwd: process.cwd() });
|
|
99
|
+
let stderr = "";
|
|
100
|
+
let stdout = "";
|
|
101
|
+
child.stderr.on("data", (chunk) => {
|
|
102
|
+
stderr += chunk;
|
|
103
|
+
});
|
|
104
|
+
child.stdout.on("data", (chunk) => {
|
|
105
|
+
stdout += chunk;
|
|
106
|
+
});
|
|
107
|
+
child.on("error", reject);
|
|
108
|
+
child.on("close", (code) => resolve({
|
|
109
|
+
exitCode: code ?? 1,
|
|
110
|
+
stderr,
|
|
111
|
+
stdout
|
|
112
|
+
}));
|
|
113
|
+
});
|
|
114
|
+
const calls = async (path, schema) => {
|
|
115
|
+
const text = await readOptional(path);
|
|
116
|
+
const decode = effect.Schema.decodeUnknownSync(effect.Schema.parseJson(schema));
|
|
117
|
+
return text.trim() === "" ? [] : text.trim().split("\n").map((line) => decode(line));
|
|
118
|
+
};
|
|
119
|
+
const context = () => ({
|
|
120
|
+
answer: () => observe("answer", [], () => readOptional(process.env.ANPORD_ANSWER_FILE)),
|
|
121
|
+
transcript: () => observe("transcript", [], () => readOptional(process.env.ANPORD_TRANSCRIPT_FILE)),
|
|
122
|
+
prepared: effect.Schema.decodeUnknownSync(effect.Schema.parseJson(effect.Schema.Record({
|
|
123
|
+
key: effect.Schema.String,
|
|
124
|
+
value: effect.Schema.Unknown
|
|
125
|
+
})))(process.env.ANPORD_PREPARE_VALUE ?? "{}"),
|
|
126
|
+
readText: (path) => observe("readText", [path], () => (0, node_fs_promises.readFile)(path, "utf8")),
|
|
127
|
+
exists: (path) => observe("exists", [path], () => (0, node_fs_promises.access)(path).then(() => true, (error) => {
|
|
128
|
+
if (error instanceof Error && "code" in error && error.code === "ENOENT") return false;
|
|
129
|
+
throw error;
|
|
130
|
+
})),
|
|
131
|
+
exec: (command) => observe("exec", [command], () => exec(command)),
|
|
132
|
+
cli: { calls: (name) => observe("cli.calls", name === void 0 ? [] : [name], async () => (await calls(".anpord/cli-calls.jsonl", CliCallSchema)).filter((call) => name === void 0 || call.cli === name)) },
|
|
133
|
+
mcp: { calls: (name) => observe("mcp.calls", name === void 0 ? [] : [name], async () => (await calls(".anpord/mcp-calls.jsonl", McpCallSchema)).filter((call) => name === void 0 || call.server === name)) }
|
|
134
|
+
});
|
|
135
|
+
const runValidators = async (checks, capture = true) => {
|
|
136
|
+
const original = {
|
|
137
|
+
log: console.log,
|
|
138
|
+
info: console.info,
|
|
139
|
+
warn: console.warn,
|
|
140
|
+
error: console.error,
|
|
141
|
+
debug: console.debug
|
|
142
|
+
};
|
|
143
|
+
const log = (level, values) => {
|
|
144
|
+
const state = local.getStore();
|
|
145
|
+
if (!state) return original[level === "stdout" ? "log" : "error"](...values);
|
|
146
|
+
const record = state.record;
|
|
147
|
+
state.record = record.logs.length === 64 ? {
|
|
148
|
+
...record,
|
|
149
|
+
truncated: true
|
|
150
|
+
} : {
|
|
151
|
+
...record,
|
|
152
|
+
logs: [...record.logs, {
|
|
153
|
+
index: record.logs.length,
|
|
154
|
+
at: now(),
|
|
155
|
+
level,
|
|
156
|
+
value: state.capture((0, node_util.format)(...values), "text")
|
|
157
|
+
}]
|
|
158
|
+
};
|
|
159
|
+
emit(state.record);
|
|
160
|
+
};
|
|
161
|
+
console.log = console.info = console.debug = (...values) => log("stdout", values);
|
|
162
|
+
console.warn = console.error = (...values) => log("stderr", values);
|
|
163
|
+
let stopped = false;
|
|
164
|
+
let passed = true;
|
|
165
|
+
let message;
|
|
166
|
+
try {
|
|
167
|
+
for (const check of checks) {
|
|
168
|
+
const started = stopped ? null : now();
|
|
169
|
+
const state = {
|
|
170
|
+
record: require_eval_validations.validationExecution({
|
|
171
|
+
id: `code:${check.index}`,
|
|
172
|
+
index: check.index,
|
|
173
|
+
name: check.name,
|
|
174
|
+
kind: "code"
|
|
175
|
+
}, started),
|
|
176
|
+
capture: require_eval_validations.validationCapture(capture)
|
|
177
|
+
};
|
|
178
|
+
if (stopped) {
|
|
179
|
+
emit({
|
|
180
|
+
...state.record,
|
|
181
|
+
message: "An earlier code validator did not pass"
|
|
182
|
+
});
|
|
183
|
+
continue;
|
|
184
|
+
}
|
|
185
|
+
emit(state.record);
|
|
186
|
+
await local.run(state, async () => {
|
|
187
|
+
try {
|
|
188
|
+
const input = context();
|
|
189
|
+
state.record = {
|
|
190
|
+
...state.record,
|
|
191
|
+
input: state.capture({ prepared: input.prepared })
|
|
192
|
+
};
|
|
193
|
+
const raw = await check.validate(input);
|
|
194
|
+
state.record = {
|
|
195
|
+
...state.record,
|
|
196
|
+
output: state.capture(raw)
|
|
197
|
+
};
|
|
198
|
+
const result = decodeResult(raw);
|
|
199
|
+
const verdict = typeof result === "boolean" ? { passed: result } : result;
|
|
200
|
+
state.record = {
|
|
201
|
+
...state.record,
|
|
202
|
+
status: verdict.passed ? "passed" : "failed",
|
|
203
|
+
message: (verdict.message ?? "").slice(0, 2e3),
|
|
204
|
+
exitCode: 0
|
|
205
|
+
};
|
|
206
|
+
stopped = !verdict.passed;
|
|
207
|
+
passed &&= verdict.passed;
|
|
208
|
+
message = verdict.message;
|
|
209
|
+
} catch (error) {
|
|
210
|
+
stopped = true;
|
|
211
|
+
passed = false;
|
|
212
|
+
process.exitCode = 1;
|
|
213
|
+
state.record = {
|
|
214
|
+
...state.record,
|
|
215
|
+
status: "error",
|
|
216
|
+
message: "Validator threw or returned an invalid result",
|
|
217
|
+
error: state.capture(error instanceof Error ? error.stack : String(error), "text"),
|
|
218
|
+
exitCode: 1
|
|
219
|
+
};
|
|
220
|
+
} finally {
|
|
221
|
+
state.record = {
|
|
222
|
+
...state.record,
|
|
223
|
+
durationMs: Math.max(0, Math.round(now() - (started ?? now())))
|
|
224
|
+
};
|
|
225
|
+
emit(state.record);
|
|
226
|
+
}
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
write(`ANPORD_VALIDATOR_RESULT=${JSON.stringify({
|
|
230
|
+
passed,
|
|
231
|
+
message
|
|
232
|
+
})}\n`);
|
|
233
|
+
} finally {
|
|
234
|
+
Object.assign(console, original);
|
|
235
|
+
}
|
|
236
|
+
};
|
|
237
|
+
//#endregion
|
|
238
|
+
exports.runValidators = runValidators;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { f as Validator } from "./types-ISsteq95.cjs";
|
|
2
|
+
//#region src/evals/validator-runtime.d.ts
|
|
3
|
+
declare const runValidators: (checks: readonly {
|
|
4
|
+
index: number;
|
|
5
|
+
name: string;
|
|
6
|
+
validate: Validator;
|
|
7
|
+
}[], capture?: boolean) => Promise<void>;
|
|
8
|
+
//#endregion
|
|
9
|
+
export { runValidators };
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { f as Validator } from "./types-Codjaq-o.mjs";
|
|
2
|
+
//#region src/evals/validator-runtime.d.ts
|
|
3
|
+
declare const runValidators: (checks: readonly {
|
|
4
|
+
index: number;
|
|
5
|
+
name: string;
|
|
6
|
+
validate: Validator;
|
|
7
|
+
}[], capture?: boolean) => Promise<void>;
|
|
8
|
+
//#endregion
|
|
9
|
+
export { runValidators };
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
import { a as validationExecution, i as validationCapture, n as VALIDATION_FRAME, o as validationSnapshot, r as unavailableValue } from "./eval-validations-BRdoZrTQ.mjs";
|
|
2
|
+
import { Schema } from "effect";
|
|
3
|
+
import { access, readFile } from "node:fs/promises";
|
|
4
|
+
import { spawn } from "node:child_process";
|
|
5
|
+
import { format } from "node:util";
|
|
6
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
7
|
+
//#region src/mcp/calls.ts
|
|
8
|
+
const McpCallSchema = Schema.Struct({
|
|
9
|
+
error: Schema.optional(Schema.String),
|
|
10
|
+
input: Schema.Unknown,
|
|
11
|
+
kind: Schema.Literal("tool", "resource"),
|
|
12
|
+
name: Schema.String,
|
|
13
|
+
output: Schema.optional(Schema.Unknown),
|
|
14
|
+
server: Schema.String
|
|
15
|
+
});
|
|
16
|
+
//#endregion
|
|
17
|
+
//#region src/mock-cli/calls.ts
|
|
18
|
+
const CliCallSchema = Schema.Struct({
|
|
19
|
+
cli: Schema.String,
|
|
20
|
+
command: Schema.String,
|
|
21
|
+
error: Schema.optional(Schema.String),
|
|
22
|
+
input: Schema.Unknown,
|
|
23
|
+
output: Schema.optional(Schema.Unknown)
|
|
24
|
+
});
|
|
25
|
+
//#endregion
|
|
26
|
+
//#region src/evals/validator-runtime.ts
|
|
27
|
+
const Result = Schema.Union(Schema.Boolean, Schema.Struct({
|
|
28
|
+
passed: Schema.Boolean,
|
|
29
|
+
message: Schema.optional(Schema.String)
|
|
30
|
+
}));
|
|
31
|
+
const decodeResult = Schema.decodeUnknownSync(Result, { onExcessProperty: "error" });
|
|
32
|
+
const local = new AsyncLocalStorage();
|
|
33
|
+
const write = process.stdout.write.bind(process.stdout);
|
|
34
|
+
const emit = (record) => write(`${VALIDATION_FRAME}${JSON.stringify(validationSnapshot(record))}\n`);
|
|
35
|
+
const now = () => performance.timeOrigin + performance.now();
|
|
36
|
+
const observe = async (method, input, run) => {
|
|
37
|
+
const state = local.getStore();
|
|
38
|
+
if (!state) return run();
|
|
39
|
+
const { record, capture } = state;
|
|
40
|
+
const call = {
|
|
41
|
+
index: record.calls.length,
|
|
42
|
+
method,
|
|
43
|
+
input: capture(input),
|
|
44
|
+
output: unavailableValue,
|
|
45
|
+
error: null,
|
|
46
|
+
startedAt: now(),
|
|
47
|
+
durationMs: null
|
|
48
|
+
};
|
|
49
|
+
if (record.calls.length === 64) {
|
|
50
|
+
state.record = {
|
|
51
|
+
...record,
|
|
52
|
+
truncated: true
|
|
53
|
+
};
|
|
54
|
+
return run();
|
|
55
|
+
}
|
|
56
|
+
state.record = {
|
|
57
|
+
...record,
|
|
58
|
+
calls: [...record.calls, call]
|
|
59
|
+
};
|
|
60
|
+
emit(state.record);
|
|
61
|
+
let completed = call;
|
|
62
|
+
try {
|
|
63
|
+
const value = await run();
|
|
64
|
+
completed = {
|
|
65
|
+
...call,
|
|
66
|
+
output: capture(value)
|
|
67
|
+
};
|
|
68
|
+
return value;
|
|
69
|
+
} catch (error) {
|
|
70
|
+
completed = {
|
|
71
|
+
...call,
|
|
72
|
+
error: capture(error instanceof Error ? error.stack : String(error), "text")
|
|
73
|
+
};
|
|
74
|
+
throw error;
|
|
75
|
+
} finally {
|
|
76
|
+
completed = {
|
|
77
|
+
...completed,
|
|
78
|
+
durationMs: Math.max(0, Math.round(now() - call.startedAt))
|
|
79
|
+
};
|
|
80
|
+
state.record = {
|
|
81
|
+
...state.record,
|
|
82
|
+
calls: state.record.calls.map((entry) => entry.index === call.index ? completed : entry)
|
|
83
|
+
};
|
|
84
|
+
emit(state.record);
|
|
85
|
+
}
|
|
86
|
+
};
|
|
87
|
+
const readOptional = async (path) => {
|
|
88
|
+
if (path === void 0) return "";
|
|
89
|
+
try {
|
|
90
|
+
return await readFile(path, "utf8");
|
|
91
|
+
} catch (error) {
|
|
92
|
+
if (error instanceof Error && "code" in error && error.code === "ENOENT") return "";
|
|
93
|
+
throw error;
|
|
94
|
+
}
|
|
95
|
+
};
|
|
96
|
+
const exec = (command) => new Promise((resolve, reject) => {
|
|
97
|
+
const child = spawn("/bin/sh", ["-lc", command], { cwd: process.cwd() });
|
|
98
|
+
let stderr = "";
|
|
99
|
+
let stdout = "";
|
|
100
|
+
child.stderr.on("data", (chunk) => {
|
|
101
|
+
stderr += chunk;
|
|
102
|
+
});
|
|
103
|
+
child.stdout.on("data", (chunk) => {
|
|
104
|
+
stdout += chunk;
|
|
105
|
+
});
|
|
106
|
+
child.on("error", reject);
|
|
107
|
+
child.on("close", (code) => resolve({
|
|
108
|
+
exitCode: code ?? 1,
|
|
109
|
+
stderr,
|
|
110
|
+
stdout
|
|
111
|
+
}));
|
|
112
|
+
});
|
|
113
|
+
const calls = async (path, schema) => {
|
|
114
|
+
const text = await readOptional(path);
|
|
115
|
+
const decode = Schema.decodeUnknownSync(Schema.parseJson(schema));
|
|
116
|
+
return text.trim() === "" ? [] : text.trim().split("\n").map((line) => decode(line));
|
|
117
|
+
};
|
|
118
|
+
const context = () => ({
|
|
119
|
+
answer: () => observe("answer", [], () => readOptional(process.env.ANPORD_ANSWER_FILE)),
|
|
120
|
+
transcript: () => observe("transcript", [], () => readOptional(process.env.ANPORD_TRANSCRIPT_FILE)),
|
|
121
|
+
prepared: Schema.decodeUnknownSync(Schema.parseJson(Schema.Record({
|
|
122
|
+
key: Schema.String,
|
|
123
|
+
value: Schema.Unknown
|
|
124
|
+
})))(process.env.ANPORD_PREPARE_VALUE ?? "{}"),
|
|
125
|
+
readText: (path) => observe("readText", [path], () => readFile(path, "utf8")),
|
|
126
|
+
exists: (path) => observe("exists", [path], () => access(path).then(() => true, (error) => {
|
|
127
|
+
if (error instanceof Error && "code" in error && error.code === "ENOENT") return false;
|
|
128
|
+
throw error;
|
|
129
|
+
})),
|
|
130
|
+
exec: (command) => observe("exec", [command], () => exec(command)),
|
|
131
|
+
cli: { calls: (name) => observe("cli.calls", name === void 0 ? [] : [name], async () => (await calls(".anpord/cli-calls.jsonl", CliCallSchema)).filter((call) => name === void 0 || call.cli === name)) },
|
|
132
|
+
mcp: { calls: (name) => observe("mcp.calls", name === void 0 ? [] : [name], async () => (await calls(".anpord/mcp-calls.jsonl", McpCallSchema)).filter((call) => name === void 0 || call.server === name)) }
|
|
133
|
+
});
|
|
134
|
+
const runValidators = async (checks, capture = true) => {
|
|
135
|
+
const original = {
|
|
136
|
+
log: console.log,
|
|
137
|
+
info: console.info,
|
|
138
|
+
warn: console.warn,
|
|
139
|
+
error: console.error,
|
|
140
|
+
debug: console.debug
|
|
141
|
+
};
|
|
142
|
+
const log = (level, values) => {
|
|
143
|
+
const state = local.getStore();
|
|
144
|
+
if (!state) return original[level === "stdout" ? "log" : "error"](...values);
|
|
145
|
+
const record = state.record;
|
|
146
|
+
state.record = record.logs.length === 64 ? {
|
|
147
|
+
...record,
|
|
148
|
+
truncated: true
|
|
149
|
+
} : {
|
|
150
|
+
...record,
|
|
151
|
+
logs: [...record.logs, {
|
|
152
|
+
index: record.logs.length,
|
|
153
|
+
at: now(),
|
|
154
|
+
level,
|
|
155
|
+
value: state.capture(format(...values), "text")
|
|
156
|
+
}]
|
|
157
|
+
};
|
|
158
|
+
emit(state.record);
|
|
159
|
+
};
|
|
160
|
+
console.log = console.info = console.debug = (...values) => log("stdout", values);
|
|
161
|
+
console.warn = console.error = (...values) => log("stderr", values);
|
|
162
|
+
let stopped = false;
|
|
163
|
+
let passed = true;
|
|
164
|
+
let message;
|
|
165
|
+
try {
|
|
166
|
+
for (const check of checks) {
|
|
167
|
+
const started = stopped ? null : now();
|
|
168
|
+
const state = {
|
|
169
|
+
record: validationExecution({
|
|
170
|
+
id: `code:${check.index}`,
|
|
171
|
+
index: check.index,
|
|
172
|
+
name: check.name,
|
|
173
|
+
kind: "code"
|
|
174
|
+
}, started),
|
|
175
|
+
capture: validationCapture(capture)
|
|
176
|
+
};
|
|
177
|
+
if (stopped) {
|
|
178
|
+
emit({
|
|
179
|
+
...state.record,
|
|
180
|
+
message: "An earlier code validator did not pass"
|
|
181
|
+
});
|
|
182
|
+
continue;
|
|
183
|
+
}
|
|
184
|
+
emit(state.record);
|
|
185
|
+
await local.run(state, async () => {
|
|
186
|
+
try {
|
|
187
|
+
const input = context();
|
|
188
|
+
state.record = {
|
|
189
|
+
...state.record,
|
|
190
|
+
input: state.capture({ prepared: input.prepared })
|
|
191
|
+
};
|
|
192
|
+
const raw = await check.validate(input);
|
|
193
|
+
state.record = {
|
|
194
|
+
...state.record,
|
|
195
|
+
output: state.capture(raw)
|
|
196
|
+
};
|
|
197
|
+
const result = decodeResult(raw);
|
|
198
|
+
const verdict = typeof result === "boolean" ? { passed: result } : result;
|
|
199
|
+
state.record = {
|
|
200
|
+
...state.record,
|
|
201
|
+
status: verdict.passed ? "passed" : "failed",
|
|
202
|
+
message: (verdict.message ?? "").slice(0, 2e3),
|
|
203
|
+
exitCode: 0
|
|
204
|
+
};
|
|
205
|
+
stopped = !verdict.passed;
|
|
206
|
+
passed &&= verdict.passed;
|
|
207
|
+
message = verdict.message;
|
|
208
|
+
} catch (error) {
|
|
209
|
+
stopped = true;
|
|
210
|
+
passed = false;
|
|
211
|
+
process.exitCode = 1;
|
|
212
|
+
state.record = {
|
|
213
|
+
...state.record,
|
|
214
|
+
status: "error",
|
|
215
|
+
message: "Validator threw or returned an invalid result",
|
|
216
|
+
error: state.capture(error instanceof Error ? error.stack : String(error), "text"),
|
|
217
|
+
exitCode: 1
|
|
218
|
+
};
|
|
219
|
+
} finally {
|
|
220
|
+
state.record = {
|
|
221
|
+
...state.record,
|
|
222
|
+
durationMs: Math.max(0, Math.round(now() - (started ?? now())))
|
|
223
|
+
};
|
|
224
|
+
emit(state.record);
|
|
225
|
+
}
|
|
226
|
+
});
|
|
227
|
+
}
|
|
228
|
+
write(`ANPORD_VALIDATOR_RESULT=${JSON.stringify({
|
|
229
|
+
passed,
|
|
230
|
+
message
|
|
231
|
+
})}\n`);
|
|
232
|
+
} finally {
|
|
233
|
+
Object.assign(console, original);
|
|
234
|
+
}
|
|
235
|
+
};
|
|
236
|
+
//#endregion
|
|
237
|
+
export { runValidators };
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
const require_eval_judges = require("./eval-judges-BkmiVLLq.cjs");
|
|
3
|
+
let effect = require("effect");
|
|
4
|
+
//#region src/validators.ts
|
|
5
|
+
const judge = (options) => effect.Schema.decodeUnknownSync(require_eval_judges.EvalJudge)({
|
|
6
|
+
...options,
|
|
7
|
+
kind: "judge"
|
|
8
|
+
}, { onExcessProperty: "error" });
|
|
9
|
+
//#endregion
|
|
10
|
+
exports.judge = judge;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { n as EvalJudgment, t as EvalJudge } from "./eval-judges-DPPUftbh.cjs";
|
|
2
|
+
//#region src/validators.d.ts
|
|
3
|
+
type JudgeOptions = typeof EvalJudge.Encoded extends (infer Options) ? Options extends {
|
|
4
|
+
readonly kind: "judge";
|
|
5
|
+
} ? Omit<Options, "kind"> : never : never;
|
|
6
|
+
declare const judge: (options: JudgeOptions) => EvalJudge;
|
|
7
|
+
//#endregion
|
|
8
|
+
export { type EvalJudge as Judge, JudgeOptions, type EvalJudgment as Judgment, judge };
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { n as EvalJudgment, t as EvalJudge } from "./eval-judges-DPPUftbh.mjs";
|
|
2
|
+
//#region src/validators.d.ts
|
|
3
|
+
type JudgeOptions = typeof EvalJudge.Encoded extends (infer Options) ? Options extends {
|
|
4
|
+
readonly kind: "judge";
|
|
5
|
+
} ? Omit<Options, "kind"> : never : never;
|
|
6
|
+
declare const judge: (options: JudgeOptions) => EvalJudge;
|
|
7
|
+
//#endregion
|
|
8
|
+
export { type EvalJudge as Judge, JudgeOptions, type EvalJudgment as Judgment, judge };
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { t as EvalJudge } from "./eval-judges-CwTvTq7N.mjs";
|
|
2
|
+
import { Schema } from "effect";
|
|
3
|
+
//#region src/validators.ts
|
|
4
|
+
const judge = (options) => Schema.decodeUnknownSync(EvalJudge)({
|
|
5
|
+
...options,
|
|
6
|
+
kind: "judge"
|
|
7
|
+
}, { onExcessProperty: "error" });
|
|
8
|
+
//#endregion
|
|
9
|
+
export { judge };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "anpord",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.13",
|
|
4
4
|
"description": "TypeScript SDK and CLI for coding agent evals and prompt management.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -25,6 +25,16 @@
|
|
|
25
25
|
}
|
|
26
26
|
},
|
|
27
27
|
"./package.json": "./package.json",
|
|
28
|
+
"./validators": {
|
|
29
|
+
"import": {
|
|
30
|
+
"types": "./dist/validators.d.mts",
|
|
31
|
+
"default": "./dist/validators.mjs"
|
|
32
|
+
},
|
|
33
|
+
"require": {
|
|
34
|
+
"types": "./dist/validators.d.cts",
|
|
35
|
+
"default": "./dist/validators.cjs"
|
|
36
|
+
}
|
|
37
|
+
},
|
|
28
38
|
"./cli": {
|
|
29
39
|
"import": {
|
|
30
40
|
"types": "./dist/cli.d.mts",
|