@providerkit/agents 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE-APACHE-2.0 +202 -0
- package/README.md +2108 -52
- package/THIRD_PARTY_NOTICES.md +521 -0
- package/dist/a2a/auth.d.ts +10 -0
- package/dist/a2a/auth.d.ts.map +1 -0
- package/dist/a2a/auth.js +52 -0
- package/dist/a2a/auth.js.map +1 -0
- package/dist/a2a/client.d.ts +74 -0
- package/dist/a2a/client.d.ts.map +1 -0
- package/dist/a2a/client.js +601 -0
- package/dist/a2a/client.js.map +1 -0
- package/dist/a2a/index.d.ts +22 -0
- package/dist/a2a/index.d.ts.map +1 -0
- package/dist/a2a/index.js +11 -0
- package/dist/a2a/index.js.map +1 -0
- package/dist/a2a/memory-runs.d.ts +62 -0
- package/dist/a2a/memory-runs.d.ts.map +1 -0
- package/dist/a2a/memory-runs.js +349 -0
- package/dist/a2a/memory-runs.js.map +1 -0
- package/dist/a2a/policy.d.ts +15 -0
- package/dist/a2a/policy.d.ts.map +1 -0
- package/dist/a2a/policy.js +114 -0
- package/dist/a2a/policy.js.map +1 -0
- package/dist/a2a/push.d.ts +34 -0
- package/dist/a2a/push.d.ts.map +1 -0
- package/dist/a2a/push.js +180 -0
- package/dist/a2a/push.js.map +1 -0
- package/dist/a2a/remote.d.ts +155 -0
- package/dist/a2a/remote.d.ts.map +1 -0
- package/dist/a2a/remote.js +1196 -0
- package/dist/a2a/remote.js.map +1 -0
- package/dist/a2a/server.d.ts +110 -0
- package/dist/a2a/server.d.ts.map +1 -0
- package/dist/a2a/server.js +1139 -0
- package/dist/a2a/server.js.map +1 -0
- package/dist/a2a/signing.d.ts +21 -0
- package/dist/a2a/signing.d.ts.map +1 -0
- package/dist/a2a/signing.js +305 -0
- package/dist/a2a/signing.js.map +1 -0
- package/dist/a2a/tasks.d.ts +88 -0
- package/dist/a2a/tasks.d.ts.map +1 -0
- package/dist/a2a/tasks.js +170 -0
- package/dist/a2a/tasks.js.map +1 -0
- package/dist/a2a/wire.d.ts +267 -0
- package/dist/a2a/wire.d.ts.map +1 -0
- package/dist/a2a/wire.js +1202 -0
- package/dist/a2a/wire.js.map +1 -0
- package/dist/background-types.d.ts +76 -0
- package/dist/background-types.d.ts.map +1 -0
- package/dist/background-types.js +2 -0
- package/dist/background-types.js.map +1 -0
- package/dist/child.d.ts +25 -0
- package/dist/child.d.ts.map +1 -0
- package/dist/child.js +129 -0
- package/dist/child.js.map +1 -0
- package/dist/code-driver.d.ts +40 -0
- package/dist/code-driver.d.ts.map +1 -0
- package/dist/code-driver.js +2 -0
- package/dist/code-driver.js.map +1 -0
- package/dist/code-mode.d.ts +3 -0
- package/dist/code-mode.d.ts.map +1 -0
- package/dist/code-mode.js +2 -0
- package/dist/code-mode.js.map +1 -0
- package/dist/code-script.d.ts +3 -0
- package/dist/code-script.d.ts.map +1 -0
- package/dist/code-script.js +149 -0
- package/dist/code-script.js.map +1 -0
- package/dist/code.d.ts +75 -0
- package/dist/code.d.ts.map +1 -0
- package/dist/code.js +326 -0
- package/dist/code.js.map +1 -0
- package/dist/compaction.d.ts +15 -0
- package/dist/compaction.d.ts.map +1 -0
- package/dist/compaction.js +168 -0
- package/dist/compaction.js.map +1 -0
- package/dist/delegation.d.ts +85 -16
- package/dist/delegation.d.ts.map +1 -1
- package/dist/delegation.js +240 -106
- package/dist/delegation.js.map +1 -1
- package/dist/evals/driver.d.ts +10 -0
- package/dist/evals/driver.d.ts.map +1 -0
- package/dist/evals/driver.js +192 -0
- package/dist/evals/driver.js.map +1 -0
- package/dist/evals/journal.d.ts +26 -0
- package/dist/evals/journal.d.ts.map +1 -0
- package/dist/evals/journal.js +483 -0
- package/dist/evals/journal.js.map +1 -0
- package/dist/evals/judge.d.ts +31 -0
- package/dist/evals/judge.d.ts.map +1 -0
- package/dist/evals/judge.js +142 -0
- package/dist/evals/judge.js.map +1 -0
- package/dist/evals/report.d.ts +38 -0
- package/dist/evals/report.d.ts.map +1 -0
- package/dist/evals/report.js +241 -0
- package/dist/evals/report.js.map +1 -0
- package/dist/evals/runner.d.ts +24 -0
- package/dist/evals/runner.d.ts.map +1 -0
- package/dist/evals/runner.js +655 -0
- package/dist/evals/runner.js.map +1 -0
- package/dist/evals/scorers.d.ts +36 -0
- package/dist/evals/scorers.d.ts.map +1 -0
- package/dist/evals/scorers.js +259 -0
- package/dist/evals/scorers.js.map +1 -0
- package/dist/evals/simulation.d.ts +7 -0
- package/dist/evals/simulation.d.ts.map +1 -0
- package/dist/evals/simulation.js +38 -0
- package/dist/evals/simulation.js.map +1 -0
- package/dist/evals/testing.d.ts +36 -0
- package/dist/evals/testing.d.ts.map +1 -0
- package/dist/evals/testing.js +87 -0
- package/dist/evals/testing.js.map +1 -0
- package/dist/evals/trajectory.d.ts +11 -0
- package/dist/evals/trajectory.d.ts.map +1 -0
- package/dist/evals/trajectory.js +180 -0
- package/dist/evals/trajectory.js.map +1 -0
- package/dist/evals/types.d.ts +431 -0
- package/dist/evals/types.d.ts.map +1 -0
- package/dist/evals/types.js +4 -0
- package/dist/evals/types.js.map +1 -0
- package/dist/evals.d.ts +14 -0
- package/dist/evals.d.ts.map +1 -0
- package/dist/evals.js +9 -0
- package/dist/evals.js.map +1 -0
- package/dist/extensions.d.ts +33 -0
- package/dist/extensions.d.ts.map +1 -0
- package/dist/extensions.js +188 -0
- package/dist/extensions.js.map +1 -0
- package/dist/handoff.d.ts +58 -0
- package/dist/handoff.d.ts.map +1 -0
- package/dist/handoff.js +160 -0
- package/dist/handoff.js.map +1 -0
- package/dist/index.d.ts +14 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -1
- package/dist/input.d.ts +42 -0
- package/dist/input.d.ts.map +1 -0
- package/dist/input.js +266 -0
- package/dist/input.js.map +1 -0
- package/dist/mcp.d.ts +77 -0
- package/dist/mcp.d.ts.map +1 -0
- package/dist/mcp.js +505 -0
- package/dist/mcp.js.map +1 -0
- package/dist/memory-store.d.ts +11 -0
- package/dist/memory-store.d.ts.map +1 -0
- package/dist/memory-store.js +272 -0
- package/dist/memory-store.js.map +1 -0
- package/dist/memory.d.ts +212 -0
- package/dist/memory.d.ts.map +1 -0
- package/dist/memory.js +613 -0
- package/dist/memory.js.map +1 -0
- package/dist/otel.d.ts +74 -0
- package/dist/otel.d.ts.map +1 -0
- package/dist/otel.js +427 -0
- package/dist/otel.js.map +1 -0
- package/dist/progress.d.ts +17 -0
- package/dist/progress.d.ts.map +1 -0
- package/dist/progress.js +193 -0
- package/dist/progress.js.map +1 -0
- package/dist/quickjs.d.ts +19 -0
- package/dist/quickjs.d.ts.map +1 -0
- package/dist/quickjs.js +519 -0
- package/dist/quickjs.js.map +1 -0
- package/dist/recovery.d.ts +10 -0
- package/dist/recovery.d.ts.map +1 -0
- package/dist/recovery.js +62 -0
- package/dist/recovery.js.map +1 -0
- package/dist/request.d.ts +88 -0
- package/dist/request.d.ts.map +1 -0
- package/dist/request.js +422 -0
- package/dist/request.js.map +1 -0
- package/dist/run.d.ts +16 -4
- package/dist/run.d.ts.map +1 -1
- package/dist/run.js +2016 -245
- package/dist/run.js.map +1 -1
- package/dist/runs.d.ts +77 -34
- package/dist/runs.d.ts.map +1 -1
- package/dist/runs.js +267 -85
- package/dist/runs.js.map +1 -1
- package/dist/schema-to-ts.d.ts +4 -0
- package/dist/schema-to-ts.d.ts.map +1 -0
- package/dist/schema-to-ts.js +191 -0
- package/dist/schema-to-ts.js.map +1 -0
- package/dist/session.d.ts +23 -0
- package/dist/session.d.ts.map +1 -0
- package/dist/session.js +137 -0
- package/dist/session.js.map +1 -0
- package/dist/snapshot.d.ts +57 -8
- package/dist/snapshot.d.ts.map +1 -1
- package/dist/snapshot.js +1206 -109
- package/dist/snapshot.js.map +1 -1
- package/dist/steering.d.ts +75 -0
- package/dist/steering.d.ts.map +1 -0
- package/dist/steering.js +214 -0
- package/dist/steering.js.map +1 -0
- package/dist/subagents.d.ts +141 -0
- package/dist/subagents.d.ts.map +1 -0
- package/dist/subagents.js +746 -0
- package/dist/subagents.js.map +1 -0
- package/dist/tool-retry.d.ts +12 -0
- package/dist/tool-retry.d.ts.map +1 -0
- package/dist/tool-retry.js +53 -0
- package/dist/tool-retry.js.map +1 -0
- package/dist/tree.d.ts +62 -6
- package/dist/tree.d.ts.map +1 -1
- package/dist/tree.js +192 -23
- package/dist/tree.js.map +1 -1
- package/dist/types.d.ts +767 -52
- package/dist/types.d.ts.map +1 -1
- package/dist/workflow-snapshot.d.ts +8 -0
- package/dist/workflow-snapshot.d.ts.map +1 -0
- package/dist/workflow-snapshot.js +276 -0
- package/dist/workflow-snapshot.js.map +1 -0
- package/dist/workflow-types.d.ts +171 -0
- package/dist/workflow-types.d.ts.map +1 -0
- package/dist/workflow-types.js +2 -0
- package/dist/workflow-types.js.map +1 -0
- package/dist/workflow.d.ts +19 -0
- package/dist/workflow.d.ts.map +1 -0
- package/dist/workflow.js +1053 -0
- package/dist/workflow.js.map +1 -0
- package/examples/a2a-remote.ts +85 -0
- package/examples/a2a-server.ts +100 -0
- package/examples/background.ts +18 -4
- package/examples/code-mode.ts +76 -0
- package/examples/conversation.ts +57 -0
- package/examples/evals.ts +79 -0
- package/examples/guardrails.ts +56 -0
- package/examples/handoff.ts +79 -0
- package/examples/memory.ts +44 -0
- package/examples/otel.ts +99 -0
- package/examples/steering.ts +69 -0
- package/examples/structured-handoff.ts +92 -0
- package/examples/subagents.ts +127 -0
- package/examples/workflow.ts +74 -0
- package/package.json +58 -6
- package/src/a2a/auth.ts +62 -0
- package/src/a2a/client.ts +733 -0
- package/src/a2a/index.ts +74 -0
- package/src/a2a/memory-runs.ts +448 -0
- package/src/a2a/policy.ts +148 -0
- package/src/a2a/push.ts +251 -0
- package/src/a2a/remote.ts +1652 -0
- package/src/a2a/server.ts +1390 -0
- package/src/a2a/signing.ts +348 -0
- package/src/a2a/tasks.ts +260 -0
- package/src/a2a/wire.ts +1443 -0
- package/src/background-types.ts +86 -0
- package/src/child.ts +181 -0
- package/src/code-driver.ts +48 -0
- package/src/code-mode.ts +12 -0
- package/src/code-script.ts +149 -0
- package/src/code.ts +424 -0
- package/src/compaction.ts +212 -0
- package/src/delegation.ts +444 -155
- package/src/evals/driver.ts +222 -0
- package/src/evals/journal.ts +605 -0
- package/src/evals/judge.ts +188 -0
- package/src/evals/report.ts +295 -0
- package/src/evals/runner.ts +826 -0
- package/src/evals/scorers.ts +339 -0
- package/src/evals/simulation.ts +46 -0
- package/src/evals/testing.ts +126 -0
- package/src/evals/trajectory.ts +223 -0
- package/src/evals/types.ts +463 -0
- package/src/evals.ts +72 -0
- package/src/extensions.ts +247 -0
- package/src/handoff.ts +214 -0
- package/src/index.ts +51 -0
- package/src/input.ts +319 -0
- package/src/mcp.ts +690 -0
- package/src/memory-store.ts +366 -0
- package/src/memory.ts +893 -0
- package/src/otel.ts +568 -0
- package/src/progress.ts +214 -0
- package/src/quickjs.ts +524 -0
- package/src/recovery.ts +90 -0
- package/src/request.ts +513 -0
- package/src/run.ts +2678 -313
- package/src/runs.ts +464 -133
- package/src/schema-to-ts.ts +194 -0
- package/src/session.ts +178 -0
- package/src/snapshot.ts +1595 -124
- package/src/steering.ts +329 -0
- package/src/subagents.ts +1084 -0
- package/src/tool-retry.ts +86 -0
- package/src/tree.ts +223 -27
- package/src/types.ts +711 -52
- package/src/workflow-snapshot.ts +366 -0
- package/src/workflow-types.ts +213 -0
- package/src/workflow.ts +1316 -0
|
@@ -0,0 +1,655 @@
|
|
|
1
|
+
import { messageOf } from "@providerkit/core";
|
|
2
|
+
import { createMemorySession, } from "../index.js";
|
|
3
|
+
import { agentSnapshot, copyData, immutableCopy, isId, isRecord } from "../snapshot.js";
|
|
4
|
+
import { caseData, combineUsage, emptyUsage, evalRun, readCases, readJournal, runResult, usageOf, } from "./journal.js";
|
|
5
|
+
import { checkDriver, createDriver } from "./driver.js";
|
|
6
|
+
import { simulationAgent, simulationPrompt } from "./simulation.js";
|
|
7
|
+
import { checkScorers, normalizeScores } from "./scorers.js";
|
|
8
|
+
import { reportGroups, summarize } from "./report.js";
|
|
9
|
+
import { nonNegative } from "./trajectory.js";
|
|
10
|
+
function whole(value, name) {
|
|
11
|
+
if (!Number.isSafeInteger(value) || value < 1)
|
|
12
|
+
throw new TypeError(`${name} must be a positive whole number.`);
|
|
13
|
+
}
|
|
14
|
+
function checkScoring(options) {
|
|
15
|
+
whole(options.maxJudgeRuns ?? 3, "maxJudgeRuns");
|
|
16
|
+
const timeout = options.timeoutMs ?? 60_000;
|
|
17
|
+
whole(timeout, "timeoutMs");
|
|
18
|
+
if (timeout > 2_147_483_647)
|
|
19
|
+
throw new TypeError("timeoutMs must fit a platform timer.");
|
|
20
|
+
for (const value of [options.allowJudge, options.includePrivateData])
|
|
21
|
+
if (value !== undefined && typeof value !== "boolean")
|
|
22
|
+
throw new TypeError("Judge and privacy options must be booleans.");
|
|
23
|
+
if (options.price !== undefined && typeof options.price !== "function")
|
|
24
|
+
throw new TypeError("price must be a function.");
|
|
25
|
+
if (options.signal !== undefined && !(options.signal instanceof AbortSignal))
|
|
26
|
+
throw new TypeError("signal must be an AbortSignal.");
|
|
27
|
+
}
|
|
28
|
+
function checkCase(item) {
|
|
29
|
+
readCases([caseData(item)]);
|
|
30
|
+
}
|
|
31
|
+
function deadline(options, parent) {
|
|
32
|
+
const controller = new AbortController();
|
|
33
|
+
const timer = setTimeout(() => controller.abort(new Error("The case reached its time limit.")), options.timeoutMs ?? 60_000);
|
|
34
|
+
const signal = AbortSignal.any([
|
|
35
|
+
controller.signal,
|
|
36
|
+
...(parent ? [parent] : []),
|
|
37
|
+
...(options.signal ? [options.signal] : []),
|
|
38
|
+
]);
|
|
39
|
+
return { signal, close: () => clearTimeout(timer) };
|
|
40
|
+
}
|
|
41
|
+
/** Host callbacks and custom scorers are cooperative. This stops waiting, not their external
|
|
42
|
+
* effects. No new SDK call is admitted from a retained context after scoring has ended. */
|
|
43
|
+
async function waitFor(work, signal) {
|
|
44
|
+
let abort = () => { };
|
|
45
|
+
try {
|
|
46
|
+
return await Promise.race([
|
|
47
|
+
work,
|
|
48
|
+
new Promise((_resolve, reject) => {
|
|
49
|
+
abort = () => reject(signal.reason);
|
|
50
|
+
if (signal.aborted)
|
|
51
|
+
abort();
|
|
52
|
+
else
|
|
53
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
54
|
+
}),
|
|
55
|
+
]);
|
|
56
|
+
}
|
|
57
|
+
finally {
|
|
58
|
+
signal.removeEventListener("abort", abort);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
function newAccount() {
|
|
62
|
+
return {
|
|
63
|
+
agent: emptyUsage(),
|
|
64
|
+
users: new Map(),
|
|
65
|
+
judges: new Map(),
|
|
66
|
+
judgeCount: 0,
|
|
67
|
+
records: new Map(),
|
|
68
|
+
settledUnknown: false,
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
async function score(run, item, scorers, options, signal, accounting, changed, runs = [run], turn) {
|
|
72
|
+
const results = [];
|
|
73
|
+
const helpers = new Set();
|
|
74
|
+
const helperStop = new AbortController();
|
|
75
|
+
const helperSignal = AbortSignal.any([signal, helperStop.signal]);
|
|
76
|
+
let helperBusy = false, active = true;
|
|
77
|
+
const frozenCase = immutableCopy(caseData(item));
|
|
78
|
+
const context = {
|
|
79
|
+
case: frozenCase,
|
|
80
|
+
turn,
|
|
81
|
+
runs: immutableCopy(runs),
|
|
82
|
+
expected: frozenCase.expected,
|
|
83
|
+
run: immutableCopy(run),
|
|
84
|
+
signal,
|
|
85
|
+
runAgent(agent, input) {
|
|
86
|
+
if (!active)
|
|
87
|
+
throw new Error("Scoring has ended. Start another explicit evaluation to run a judge.");
|
|
88
|
+
helperSignal.throwIfAborted();
|
|
89
|
+
if (options.allowJudge !== true)
|
|
90
|
+
throw new Error("Model judging is off. Set allowJudge to true to send data to the supplied provider.");
|
|
91
|
+
if (helperBusy || accounting.judgeCount >= (options.maxJudgeRuns ?? 3))
|
|
92
|
+
throw new Error("The helper-run limit was reached. Helpers must run one at a time.");
|
|
93
|
+
helperBusy = true;
|
|
94
|
+
const index = accounting.judgeCount++;
|
|
95
|
+
const began = Date.now();
|
|
96
|
+
let latest;
|
|
97
|
+
const capture = (node) => {
|
|
98
|
+
if (!active)
|
|
99
|
+
return;
|
|
100
|
+
const snapshot = agentSnapshot(node);
|
|
101
|
+
latest = snapshot;
|
|
102
|
+
let usage;
|
|
103
|
+
try {
|
|
104
|
+
usage = usageOf(snapshot, options.price);
|
|
105
|
+
}
|
|
106
|
+
catch (error) {
|
|
107
|
+
accounting.judges.set(index, usageOf(snapshot));
|
|
108
|
+
throw error;
|
|
109
|
+
}
|
|
110
|
+
accounting.judges.set(index, usage);
|
|
111
|
+
accounting.records.set(snapshot.runId, runResult(snapshot, usage, "judge", options.includePrivateData, turn, Date.now() - began));
|
|
112
|
+
if (snapshot.state.status !== "running" && usage.unknownCharges)
|
|
113
|
+
accounting.settledUnknown = true;
|
|
114
|
+
changed();
|
|
115
|
+
};
|
|
116
|
+
const work = (async () => {
|
|
117
|
+
try {
|
|
118
|
+
const outcome = await agent.run(input, {
|
|
119
|
+
signal: helperSignal,
|
|
120
|
+
host: { checkpoint: capture },
|
|
121
|
+
});
|
|
122
|
+
capture(readJournal(outcome));
|
|
123
|
+
return outcome;
|
|
124
|
+
}
|
|
125
|
+
finally {
|
|
126
|
+
// If a helper failed after dispatch, the last journal still holds the unknown charge.
|
|
127
|
+
if (latest && active) {
|
|
128
|
+
let usage;
|
|
129
|
+
try {
|
|
130
|
+
usage = usageOf(latest, options.price);
|
|
131
|
+
}
|
|
132
|
+
catch {
|
|
133
|
+
usage = usageOf(latest);
|
|
134
|
+
}
|
|
135
|
+
accounting.judges.set(index, usage);
|
|
136
|
+
accounting.records.set(latest.runId, runResult(latest, usage, "judge", options.includePrivateData, turn, Date.now() - began));
|
|
137
|
+
if (usage.unknownCharges)
|
|
138
|
+
accounting.settledUnknown = true;
|
|
139
|
+
changed();
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
})();
|
|
143
|
+
helpers.add(work);
|
|
144
|
+
void work.then(() => {
|
|
145
|
+
helperBusy = false;
|
|
146
|
+
helpers.delete(work);
|
|
147
|
+
}, () => {
|
|
148
|
+
helperBusy = false;
|
|
149
|
+
helpers.delete(work);
|
|
150
|
+
});
|
|
151
|
+
return work;
|
|
152
|
+
},
|
|
153
|
+
};
|
|
154
|
+
try {
|
|
155
|
+
for (const scorer of scorers) {
|
|
156
|
+
const failure = (code, error) => ({
|
|
157
|
+
scorer: scorer.name,
|
|
158
|
+
name: scorer.name,
|
|
159
|
+
gate: scorer.gate !== false,
|
|
160
|
+
kind: "error",
|
|
161
|
+
error: code,
|
|
162
|
+
...(options.includePrivateData && error !== undefined ? { reason: messageOf(error) } : {}),
|
|
163
|
+
});
|
|
164
|
+
if (signal.aborted) {
|
|
165
|
+
results.push(failure("cancelled"));
|
|
166
|
+
continue;
|
|
167
|
+
}
|
|
168
|
+
let raw;
|
|
169
|
+
try {
|
|
170
|
+
try {
|
|
171
|
+
raw = await waitFor(Promise.resolve().then(() => scorer.score(context)), signal);
|
|
172
|
+
}
|
|
173
|
+
finally {
|
|
174
|
+
// A failed scorer still owns its helpers and their receipts until they settle.
|
|
175
|
+
await waitFor(Promise.all([...helpers]), signal);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
catch (error) {
|
|
179
|
+
results.push(failure(signal.aborted ? "cancelled" : "scorer_failed", error));
|
|
180
|
+
continue;
|
|
181
|
+
}
|
|
182
|
+
try {
|
|
183
|
+
results.push(...normalizeScores(scorer, raw, options.includePrivateData === true));
|
|
184
|
+
}
|
|
185
|
+
catch (error) {
|
|
186
|
+
results.push(failure("invalid_score", error));
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
finally {
|
|
191
|
+
// Stop admission first, then let SDK cancellation save its last available receipts.
|
|
192
|
+
helperStop.abort();
|
|
193
|
+
await Promise.allSettled(helpers);
|
|
194
|
+
active = false;
|
|
195
|
+
}
|
|
196
|
+
return results.map((result) => ({ ...result, ...(turn === undefined ? {} : { turn }) }));
|
|
197
|
+
}
|
|
198
|
+
function verdict(row, item) {
|
|
199
|
+
if (!row.run) {
|
|
200
|
+
row.status = "error";
|
|
201
|
+
row.problems.push("The agent did not return a valid run journal.");
|
|
202
|
+
return;
|
|
203
|
+
}
|
|
204
|
+
const expected = item.expectStatus ?? "completed";
|
|
205
|
+
if (row.run.status !== expected) {
|
|
206
|
+
row.status = "failed";
|
|
207
|
+
row.problems.push(row.run.status === "suspended"
|
|
208
|
+
? "The run is waiting for an answer or a workflow wake time. Supply a matching scripted answer if needed, or expect suspended status. Evals do not resume waiting runs."
|
|
209
|
+
: `Expected run status ${expected}; received ${row.run.status}.`);
|
|
210
|
+
}
|
|
211
|
+
const gates = row.scores.filter((result) => result.gate);
|
|
212
|
+
if (gates.some((result) => result.kind === "error")) {
|
|
213
|
+
row.status = "error";
|
|
214
|
+
row.problems.push("A required scorer failed. Inspect its configuration before running the case again.");
|
|
215
|
+
}
|
|
216
|
+
else if (gates.some((result) => result.passed === false)) {
|
|
217
|
+
row.status = "failed";
|
|
218
|
+
row.problems.push("A required score did not pass.");
|
|
219
|
+
}
|
|
220
|
+
else if (row.status !== "failed") {
|
|
221
|
+
if (gates.some((result) => result.kind === "skipped") ||
|
|
222
|
+
(!gates.some((result) => result.passed !== undefined) && item.expectStatus === undefined)) {
|
|
223
|
+
row.status = "unscored";
|
|
224
|
+
row.problems.push("No complete set of required scores is available. Supply expectations or the missing measurements.");
|
|
225
|
+
}
|
|
226
|
+
else
|
|
227
|
+
row.status = "passed";
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
/** Executes explicitly supplied agents through the SDK. Sandbox real tools. Only the separate
|
|
231
|
+
* trusted driver answers new requests; no stored effect is resumed and no case is retried. */
|
|
232
|
+
export async function runEvals(options) {
|
|
233
|
+
checkScoring(options);
|
|
234
|
+
if (!isId(options.name) || !Array.isArray(options.cases) || !options.cases.length)
|
|
235
|
+
throw new TypeError("An eval needs a name and at least one case.");
|
|
236
|
+
if (!options.agent ||
|
|
237
|
+
(typeof options.agent !== "function" && typeof options.agent.run !== "function"))
|
|
238
|
+
throw new TypeError("Supply an agent or an agent factory.");
|
|
239
|
+
const repeat = options.repeat ?? 1, concurrency = options.concurrency ?? 1;
|
|
240
|
+
whole(repeat, "repeat");
|
|
241
|
+
whole(concurrency, "concurrency");
|
|
242
|
+
const ks = [...(options.k ?? [repeat])];
|
|
243
|
+
if (!ks.length || new Set(ks).size !== ks.length)
|
|
244
|
+
throw new TypeError("k must contain unique positive whole numbers.");
|
|
245
|
+
for (const k of ks)
|
|
246
|
+
whole(k, "k");
|
|
247
|
+
const budget = copyData(options.budget);
|
|
248
|
+
if (budget &&
|
|
249
|
+
(Object.keys(budget).some((key) => key !== "usd" && key !== "tokens") ||
|
|
250
|
+
!Object.keys(budget).length ||
|
|
251
|
+
[budget.usd, budget.tokens].some((v) => v !== undefined && !nonNegative(v))))
|
|
252
|
+
throw new TypeError("Spend thresholds must be finite and non-negative.");
|
|
253
|
+
if (options.run &&
|
|
254
|
+
Object.keys(options.run).some((key) => !["context", "modelSettings", "enabledTools"].includes(key)))
|
|
255
|
+
throw new TypeError("Eval run options accept only context, modelSettings and enabledTools.");
|
|
256
|
+
if (options.host &&
|
|
257
|
+
Object.keys(options.host).some((key) => ![
|
|
258
|
+
"checkpoint",
|
|
259
|
+
"loadMedia",
|
|
260
|
+
"prepareRequest",
|
|
261
|
+
"needsApproval",
|
|
262
|
+
"beforeTool",
|
|
263
|
+
"beforeStep",
|
|
264
|
+
"canRunConcurrently",
|
|
265
|
+
].includes(key)))
|
|
266
|
+
throw new TypeError("An eval host cannot answer requests, resume effects or replace child execution.");
|
|
267
|
+
if (options.allowSimulation !== undefined && typeof options.allowSimulation !== "boolean")
|
|
268
|
+
throw new TypeError("allowSimulation must be a boolean.");
|
|
269
|
+
if (options.resolveAgent !== undefined && typeof options.resolveAgent !== "function")
|
|
270
|
+
throw new TypeError("resolveAgent must be a function.");
|
|
271
|
+
const needsSimulation = options.cases.some((item) => item.user !== undefined);
|
|
272
|
+
if (needsSimulation && (!options.user || options.allowSimulation !== true))
|
|
273
|
+
throw new TypeError("Simulated-user cases need an explicit user provider and allowSimulation: true. Simulation sends conversation text to that provider and can cost money.");
|
|
274
|
+
const simulator = needsSimulation && options.user ? simulationAgent(options.user) : undefined;
|
|
275
|
+
const ids = new Set();
|
|
276
|
+
for (const item of options.cases) {
|
|
277
|
+
checkCase(item);
|
|
278
|
+
if (ids.has(item.id))
|
|
279
|
+
throw new TypeError("Eval case ids must be unique.");
|
|
280
|
+
ids.add(item.id);
|
|
281
|
+
const scorers = [...(options.scorers ?? []), ...(item.scorers ?? [])];
|
|
282
|
+
checkScorers(scorers);
|
|
283
|
+
for (const turn of item.turns ?? [])
|
|
284
|
+
checkScorers([...scorers, ...(turn.scorers ?? [])]);
|
|
285
|
+
}
|
|
286
|
+
if (options.drivers !== undefined && !isRecord(options.drivers))
|
|
287
|
+
throw new TypeError("Drivers must be keyed by case id.");
|
|
288
|
+
for (const [id, driver] of Object.entries(options.drivers ?? {})) {
|
|
289
|
+
if (!ids.has(id))
|
|
290
|
+
throw new TypeError("A driver refers to an unknown case id.");
|
|
291
|
+
checkDriver(driver);
|
|
292
|
+
}
|
|
293
|
+
const startedAt = Date.now();
|
|
294
|
+
const stop = new AbortController();
|
|
295
|
+
const signal = AbortSignal.any([stop.signal, ...(options.signal ? [options.signal] : [])]);
|
|
296
|
+
let stopped;
|
|
297
|
+
const accounts = [];
|
|
298
|
+
function inspect() {
|
|
299
|
+
if (stopped || !budget)
|
|
300
|
+
return;
|
|
301
|
+
const usage = combineUsage(accounts.flatMap((account) => [
|
|
302
|
+
account.agent,
|
|
303
|
+
...account.judges.values(),
|
|
304
|
+
...account.users.values(),
|
|
305
|
+
]));
|
|
306
|
+
if (budget.usd !== undefined && accounts.some((account) => account.settledUnknown))
|
|
307
|
+
stopped = {
|
|
308
|
+
reason: "unknown_cost",
|
|
309
|
+
message: "A call's final charge is unknown. Reconcile it with the provider before starting more paid work.",
|
|
310
|
+
};
|
|
311
|
+
else if (budget.usd !== undefined && usage.unpricedCalls > 0)
|
|
312
|
+
stopped = {
|
|
313
|
+
reason: "unknown_cost",
|
|
314
|
+
message: "Stopped admitting work because a model call has no complete price. Supply caller rates, including hosted-tool rates, or use a token threshold.",
|
|
315
|
+
};
|
|
316
|
+
else if ((budget.usd !== undefined && usage.knownCostUsd >= budget.usd) ||
|
|
317
|
+
(budget.tokens !== undefined &&
|
|
318
|
+
usage.usage.inputTokens + usage.usage.outputTokens >= budget.tokens))
|
|
319
|
+
stopped = {
|
|
320
|
+
reason: "budget",
|
|
321
|
+
message: "The spend threshold was reached. No new work was admitted; calls already sent may still finish and charge.",
|
|
322
|
+
};
|
|
323
|
+
if (stopped)
|
|
324
|
+
stop.abort(new Error(stopped.message));
|
|
325
|
+
}
|
|
326
|
+
// Finite workers avoid one pending promise per case while retaining dataset/repeat order.
|
|
327
|
+
const jobs = options.cases.flatMap((item) => Array.from({ length: repeat }, (_, index) => ({ item, repeat: index + 1 })));
|
|
328
|
+
const rows = jobs.map(({ item, repeat }) => ({
|
|
329
|
+
id: item.id,
|
|
330
|
+
repeat,
|
|
331
|
+
status: "skipped",
|
|
332
|
+
problems: [],
|
|
333
|
+
scores: [],
|
|
334
|
+
runs: [],
|
|
335
|
+
turns: (item.turns ?? [item]).map((_turn, index) => ({
|
|
336
|
+
index: index + 1,
|
|
337
|
+
status: "skipped",
|
|
338
|
+
scores: [],
|
|
339
|
+
problems: [],
|
|
340
|
+
})),
|
|
341
|
+
...(item.user ? { simulation: "incomplete" } : {}),
|
|
342
|
+
agentUsage: emptyUsage(),
|
|
343
|
+
judgeUsage: emptyUsage(),
|
|
344
|
+
userUsage: emptyUsage(),
|
|
345
|
+
durationMs: 0,
|
|
346
|
+
}));
|
|
347
|
+
let next = 0;
|
|
348
|
+
inspect();
|
|
349
|
+
const configuredAgent = options.agent;
|
|
350
|
+
async function worker() {
|
|
351
|
+
while (!signal.aborted) {
|
|
352
|
+
const index = next++, job = jobs[index], pendingRow = rows[index];
|
|
353
|
+
if (!job || !pendingRow)
|
|
354
|
+
return;
|
|
355
|
+
const row = pendingRow;
|
|
356
|
+
const began = Date.now(), timer = deadline(options, signal);
|
|
357
|
+
const account = newAccount();
|
|
358
|
+
accounts.push(account);
|
|
359
|
+
const driver = createDriver(job.item.id, options.drivers?.[job.item.id], options.host);
|
|
360
|
+
const session = job.item.turns || job.item.user ? createMemorySession() : undefined;
|
|
361
|
+
const runs = [];
|
|
362
|
+
let currentTurn;
|
|
363
|
+
async function execute(target, input, role, turn) {
|
|
364
|
+
timer.signal.throwIfAborted();
|
|
365
|
+
const start = Date.now();
|
|
366
|
+
let latest;
|
|
367
|
+
let closed = false;
|
|
368
|
+
const capture = (node, fallback = false) => {
|
|
369
|
+
if (closed)
|
|
370
|
+
return;
|
|
371
|
+
const snapshot = agentSnapshot(node);
|
|
372
|
+
latest = snapshot;
|
|
373
|
+
let usage;
|
|
374
|
+
try {
|
|
375
|
+
usage = usageOf(snapshot, options.price);
|
|
376
|
+
}
|
|
377
|
+
catch (error) {
|
|
378
|
+
usage = usageOf(snapshot);
|
|
379
|
+
if (!fallback)
|
|
380
|
+
throw error;
|
|
381
|
+
}
|
|
382
|
+
const record = runResult(snapshot, usage, role, options.includePrivateData, turn, Date.now() - start);
|
|
383
|
+
account.records.set(snapshot.runId, record);
|
|
384
|
+
if (role === "user")
|
|
385
|
+
account.users.set(snapshot.runId, usage);
|
|
386
|
+
else {
|
|
387
|
+
account.agent = combineUsage([...account.records.values()].filter((entry) => entry.role === "agent"));
|
|
388
|
+
row.run = record;
|
|
389
|
+
if (currentTurn)
|
|
390
|
+
currentTurn.runId = record.runId;
|
|
391
|
+
}
|
|
392
|
+
if (snapshot.state.status !== "running" && usage.unknownCharges)
|
|
393
|
+
account.settledUnknown = true;
|
|
394
|
+
inspect();
|
|
395
|
+
};
|
|
396
|
+
try {
|
|
397
|
+
const outcome = await waitFor(target.run(input, {
|
|
398
|
+
...(role === "agent" ? { ...options.run, session } : {}),
|
|
399
|
+
signal: timer.signal,
|
|
400
|
+
host: {
|
|
401
|
+
...(role === "agent" ? { ...options.host, ...driver.host } : {}),
|
|
402
|
+
checkpoint: async (snapshot) => {
|
|
403
|
+
capture(snapshot);
|
|
404
|
+
if (role === "agent")
|
|
405
|
+
await options.host?.checkpoint?.(snapshot);
|
|
406
|
+
},
|
|
407
|
+
},
|
|
408
|
+
}), timer.signal);
|
|
409
|
+
capture(readJournal(outcome));
|
|
410
|
+
return { outcome, run: evalRun(outcome.snapshot, options.price, Date.now() - start) };
|
|
411
|
+
}
|
|
412
|
+
finally {
|
|
413
|
+
// Preserve partial receipts on timeout or a host/pricing error, never overwrite earlier turns.
|
|
414
|
+
if (latest)
|
|
415
|
+
capture(latest, true);
|
|
416
|
+
closed = true;
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
try {
|
|
420
|
+
let agent = typeof configuredAgent === "function"
|
|
421
|
+
? await waitFor(Promise.resolve().then(() => configuredAgent({
|
|
422
|
+
case: job.item,
|
|
423
|
+
repeat: job.repeat,
|
|
424
|
+
signal: timer.signal,
|
|
425
|
+
stub: driver.stub,
|
|
426
|
+
})), timer.signal)
|
|
427
|
+
: configuredAgent;
|
|
428
|
+
driver.assertApplied();
|
|
429
|
+
const user = job.item.user;
|
|
430
|
+
let input = user?.opening ?? job.item.turns?.[0]?.input ?? job.item.input;
|
|
431
|
+
if (input === undefined)
|
|
432
|
+
throw new TypeError("The case has no input.");
|
|
433
|
+
const maxTurns = user?.maxTurns ?? (user ? 10 : (job.item.turns?.length ?? 1));
|
|
434
|
+
const transcript = [];
|
|
435
|
+
for (let turn = 1; turn <= maxTurns; turn++) {
|
|
436
|
+
timer.signal.throwIfAborted();
|
|
437
|
+
currentTurn = row.turns[turn - 1];
|
|
438
|
+
if (!currentTurn) {
|
|
439
|
+
currentTurn = { index: turn, status: "skipped", scores: [], problems: [] };
|
|
440
|
+
row.turns.push(currentTurn);
|
|
441
|
+
}
|
|
442
|
+
currentTurn.status = "unscored";
|
|
443
|
+
const spec = job.item.turns?.[turn - 1];
|
|
444
|
+
input = spec?.input ?? input;
|
|
445
|
+
const { outcome, run } = await execute(agent, input, "agent", turn);
|
|
446
|
+
runs.push(run);
|
|
447
|
+
const item = { ...job.item, ...(spec ?? {}) };
|
|
448
|
+
const scores = await score(run, item, [...(options.scorers ?? []), ...(job.item.scorers ?? []), ...(spec?.scorers ?? [])], options, timer.signal, account, inspect, runs, turn);
|
|
449
|
+
row.scores.push(...scores);
|
|
450
|
+
currentTurn.scores = scores;
|
|
451
|
+
const result = {
|
|
452
|
+
run: row.run,
|
|
453
|
+
scores,
|
|
454
|
+
status: "unscored",
|
|
455
|
+
problems: currentTurn.problems,
|
|
456
|
+
};
|
|
457
|
+
verdict(result, item);
|
|
458
|
+
currentTurn.status = result.status;
|
|
459
|
+
row.status = result.status;
|
|
460
|
+
row.problems.push(...result.problems);
|
|
461
|
+
// A completed setup turn can lack scores without blocking the next turn.
|
|
462
|
+
if ((result.status !== "passed" && result.status !== "unscored") ||
|
|
463
|
+
outcome.status !== "completed")
|
|
464
|
+
break;
|
|
465
|
+
if (session && outcome.session?.status !== "saved") {
|
|
466
|
+
row.problems.push("The session did not save this turn. Fix the session error before continuing.");
|
|
467
|
+
throw new Error("Session append failed.");
|
|
468
|
+
}
|
|
469
|
+
if (user && simulator) {
|
|
470
|
+
const content = run.snapshot.input;
|
|
471
|
+
transcript.push({
|
|
472
|
+
role: "user",
|
|
473
|
+
text: typeof content === "string"
|
|
474
|
+
? content
|
|
475
|
+
: content
|
|
476
|
+
.filter((part) => part.type === "text")
|
|
477
|
+
.map((part) => part.text)
|
|
478
|
+
.join("\n"),
|
|
479
|
+
});
|
|
480
|
+
// submit_result can finish without assistant text. Send the saved answer only.
|
|
481
|
+
transcript.push({
|
|
482
|
+
role: "assistant",
|
|
483
|
+
text: typeof run.value === "string" ? run.value : JSON.stringify(run.value),
|
|
484
|
+
});
|
|
485
|
+
const simulated = await execute(simulator, simulationPrompt(user, transcript), "user", turn);
|
|
486
|
+
if (simulated.outcome.status !== "completed" ||
|
|
487
|
+
typeof simulated.run.value !== "string" ||
|
|
488
|
+
!simulated.run.value.trim()) {
|
|
489
|
+
row.problems.push("The simulated user did not return a non-empty message. Check its provider and settings.");
|
|
490
|
+
throw new Error("Invalid simulated-user reply.");
|
|
491
|
+
}
|
|
492
|
+
const nextInput = simulated.run.value.trim();
|
|
493
|
+
if (nextInput === "</finished>") {
|
|
494
|
+
row.simulation = "finished";
|
|
495
|
+
break;
|
|
496
|
+
}
|
|
497
|
+
if (turn === maxTurns) {
|
|
498
|
+
row.simulation = "turn_limit";
|
|
499
|
+
row.status = "unscored";
|
|
500
|
+
row.problems.push("The simulated conversation reached its turn limit before the user finished. Review the transcript or raise maxTurns.");
|
|
501
|
+
break;
|
|
502
|
+
}
|
|
503
|
+
input = nextInput;
|
|
504
|
+
}
|
|
505
|
+
if (turn < maxTurns &&
|
|
506
|
+
(run.activeAgent.id !== agent.id || run.activeAgent.version !== agent.version)) {
|
|
507
|
+
const nextAgent = await waitFor(Promise.resolve(options.resolveAgent?.(run.activeAgent)), timer.signal);
|
|
508
|
+
if (!nextAgent ||
|
|
509
|
+
nextAgent.id !== run.activeAgent.id ||
|
|
510
|
+
nextAgent.version !== run.activeAgent.version) {
|
|
511
|
+
row.problems.push("A handoff changed the conversation owner. Supply resolveAgent for that exact agent id and version.");
|
|
512
|
+
throw new Error("The active agent is not registered for the next turn.");
|
|
513
|
+
}
|
|
514
|
+
agent = nextAgent;
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
if (row.status === "passed" &&
|
|
518
|
+
(row.turns.some((turn) => turn.status === "skipped") || row.simulation === "incomplete")) {
|
|
519
|
+
row.status = "unscored";
|
|
520
|
+
row.problems.push("The conversation stopped before all turns finished. Review its pending requests and expected statuses.");
|
|
521
|
+
}
|
|
522
|
+
// A later passing turn cannot erase an earlier missing score.
|
|
523
|
+
if (row.status === "passed" && row.turns.some((turn) => turn.status === "unscored"))
|
|
524
|
+
row.status = "unscored";
|
|
525
|
+
}
|
|
526
|
+
catch (error) {
|
|
527
|
+
row.status = timer.signal.aborted ? "cancelled" : "error";
|
|
528
|
+
if (currentTurn?.status === "unscored")
|
|
529
|
+
currentTurn.status = row.status;
|
|
530
|
+
row.problems.push(timer.signal.aborted
|
|
531
|
+
? "The case was cancelled or reached its time limit. Work already sent may still finish."
|
|
532
|
+
: "The agent, simulator or host failed. Check the supplied agents, tools and host callbacks.");
|
|
533
|
+
if (options.includePrivateData)
|
|
534
|
+
row.problems.push(messageOf(error));
|
|
535
|
+
}
|
|
536
|
+
finally {
|
|
537
|
+
timer.close();
|
|
538
|
+
row.durationMs = Date.now() - began;
|
|
539
|
+
row.runs = [...account.records.values()];
|
|
540
|
+
row.agentUsage = account.agent;
|
|
541
|
+
row.judgeUsage = combineUsage(account.judges.values());
|
|
542
|
+
row.userUsage = combineUsage(account.users.values());
|
|
543
|
+
}
|
|
544
|
+
if (timer.signal.aborted) {
|
|
545
|
+
row.status = "cancelled";
|
|
546
|
+
if (currentTurn && currentTurn.status !== "passed")
|
|
547
|
+
currentTurn.status = "cancelled";
|
|
548
|
+
if (!row.problems.length)
|
|
549
|
+
row.problems.push("The case was cancelled before scoring finished. Work already sent may still finish.");
|
|
550
|
+
}
|
|
551
|
+
for (const turn of row.turns)
|
|
552
|
+
if (turn.status === "skipped")
|
|
553
|
+
turn.problems.push("This turn was not run because an earlier turn or the evaluation stopped.");
|
|
554
|
+
// A settled unknown bill is not a zero. Under a money threshold stop before the next case.
|
|
555
|
+
if (budget?.usd !== undefined &&
|
|
556
|
+
(account.agent.unknownCharges ||
|
|
557
|
+
row.judgeUsage.unknownCharges ||
|
|
558
|
+
row.userUsage.unknownCharges) &&
|
|
559
|
+
!stopped) {
|
|
560
|
+
stopped = {
|
|
561
|
+
reason: "unknown_cost",
|
|
562
|
+
message: "A call's final charge is unknown. Reconcile it with the provider before starting more paid work.",
|
|
563
|
+
};
|
|
564
|
+
stop.abort(new Error(stopped.message));
|
|
565
|
+
}
|
|
566
|
+
inspect();
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, jobs.length) }, worker));
|
|
570
|
+
if (!stopped && signal.aborted)
|
|
571
|
+
stopped = {
|
|
572
|
+
reason: "cancelled",
|
|
573
|
+
message: "Evaluation cancelled. Cases not started remain counted as skipped.",
|
|
574
|
+
};
|
|
575
|
+
for (const row of rows)
|
|
576
|
+
if (row.status === "skipped") {
|
|
577
|
+
row.problems.push("The case was not started because evaluation stopped.");
|
|
578
|
+
for (const turn of row.turns)
|
|
579
|
+
turn.problems.push("The case was not started because evaluation stopped.");
|
|
580
|
+
}
|
|
581
|
+
return {
|
|
582
|
+
format: 1,
|
|
583
|
+
name: options.name,
|
|
584
|
+
startedAt,
|
|
585
|
+
durationMs: Date.now() - startedAt,
|
|
586
|
+
options: { repeat, k: ks, concurrency, ...(budget ? { budget } : {}) },
|
|
587
|
+
cases: rows,
|
|
588
|
+
groups: reportGroups(rows, ks),
|
|
589
|
+
summary: summarize(rows),
|
|
590
|
+
...(stopped ? { stopped } : {}),
|
|
591
|
+
};
|
|
592
|
+
}
|
|
593
|
+
/** Scores a validated journal without re-executing its agent or tools. Only explicitly enabled
|
|
594
|
+
* helper judges can make provider calls. Duration is unknown for a stored journal. */
|
|
595
|
+
export async function scoreRun(raw, scorers, options = {}) {
|
|
596
|
+
checkScoring(options);
|
|
597
|
+
const snapshot = readJournal(raw);
|
|
598
|
+
const item = options.case ?? { id: snapshot.runId, input: snapshot.input };
|
|
599
|
+
checkCase(item);
|
|
600
|
+
checkScorers(scorers);
|
|
601
|
+
const run = evalRun(snapshot, options.price);
|
|
602
|
+
const accounting = newAccount();
|
|
603
|
+
accounting.agent = usageOf(snapshot, options.price);
|
|
604
|
+
const timer = deadline(options);
|
|
605
|
+
try {
|
|
606
|
+
const scores = await score(run, item, scorers, options, timer.signal, accounting, () => { });
|
|
607
|
+
return {
|
|
608
|
+
runId: snapshot.runId,
|
|
609
|
+
status: snapshot.state.status,
|
|
610
|
+
scores,
|
|
611
|
+
agentUsage: accounting.agent,
|
|
612
|
+
judgeUsage: combineUsage(accounting.judges.values()),
|
|
613
|
+
};
|
|
614
|
+
}
|
|
615
|
+
finally {
|
|
616
|
+
timer.close();
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
/** Explicit host call, not a lifecycle hook. No queue, storage, retries or telemetry. A host
|
|
620
|
+
* must deduplicate run IDs if it wants each completed run scored only once. */
|
|
621
|
+
export function createLiveScorer(options) {
|
|
622
|
+
checkScoring(options);
|
|
623
|
+
checkScorers(options.scorers);
|
|
624
|
+
if (!nonNegative(options.rate) || options.rate > 1)
|
|
625
|
+
throw new TypeError("The sampling rate must be between 0 and 1.");
|
|
626
|
+
const concurrency = options.concurrency ?? 1;
|
|
627
|
+
whole(concurrency, "concurrency");
|
|
628
|
+
let active = 0;
|
|
629
|
+
return {
|
|
630
|
+
async score(raw) {
|
|
631
|
+
if (active >= concurrency)
|
|
632
|
+
return { status: "busy" };
|
|
633
|
+
active++;
|
|
634
|
+
try {
|
|
635
|
+
const snapshot = readJournal(raw);
|
|
636
|
+
if (snapshot.state.status !== "completed")
|
|
637
|
+
return { status: "not_completed" };
|
|
638
|
+
// Adapted from Mastra, packages/core/src/evals/hooks.ts (Apache-2.0).
|
|
639
|
+
// Copyright (c) 2025 Kepler Software, Inc. Changed: Web Crypto, no tracing or hook wiring.
|
|
640
|
+
const hash = new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(snapshot.runId)));
|
|
641
|
+
const unit = hash.slice(0, 6).reduce((value, byte) => value * 256 + byte, 0) / 2 ** 48;
|
|
642
|
+
if (unit >= options.rate)
|
|
643
|
+
return { status: "not_sampled" };
|
|
644
|
+
return { status: "scored", result: await scoreRun(snapshot, options.scorers, options) };
|
|
645
|
+
}
|
|
646
|
+
catch {
|
|
647
|
+
return { status: "error" };
|
|
648
|
+
}
|
|
649
|
+
finally {
|
|
650
|
+
active--;
|
|
651
|
+
}
|
|
652
|
+
},
|
|
653
|
+
};
|
|
654
|
+
}
|
|
655
|
+
//# sourceMappingURL=runner.js.map
|