@statelyai/agent 2.0.0-alpha.36 → 2.0.0-alpha.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/dist/{effects-CuOUNYtZ.mjs → effects-BT_cSY0j.mjs} +95 -11
- package/dist/{effects-B23bFZEs.cjs → effects-DDseYtJk.cjs} +95 -11
- package/dist/index.cjs +1 -1
- package/dist/index.d.cts +3 -3
- package/dist/index.d.mts +3 -3
- package/dist/index.mjs +1 -1
- package/dist/otel.d.cts +1 -1
- package/dist/otel.d.mts +1 -1
- package/dist/{run-agent-BcQ-9NRT.d.cts → run-agent-30mH2moE.d.cts} +2 -2
- package/dist/{run-agent-C86drZtN.d.mts → run-agent-DlPzBWj-.d.mts} +2 -2
- package/dist/{setup-agent-jlo4vtaZ.d.cts → setup-agent-BaNyz6Q3.d.cts} +1 -1
- package/dist/{setup-agent-BIKcbVXV.d.mts → setup-agent-C1CG_nfX.d.mts} +1 -1
- package/dist/testing.cjs +2411 -1799
- package/dist/testing.d.cts +167 -4
- package/dist/testing.d.mts +167 -4
- package/dist/testing.mjs +2412 -1801
- package/dist/{usage-HtnrUsl9.d.cts → usage-Bm5nvpow.d.cts} +5 -0
- package/dist/{usage-DoP0Oi84.d.mts → usage-BnikdOca.d.mts} +5 -0
- package/dist/validate.cjs +1 -1
- package/dist/validate.d.cts +2 -2
- package/dist/validate.d.mts +2 -2
- package/dist/validate.mjs +1 -1
- package/dist/{verify-Dpz1u5Pe.mjs → verify-ChyX6jq0.mjs} +75 -20
- package/dist/{verify-Bink-Drw.cjs → verify-CwlombbC.cjs} +75 -20
- package/dist/{verify-CciG8UP-.d.cts → verify-DJHfEfm6.d.cts} +9 -4
- package/dist/{verify-Buh8XSUr.d.mts → verify-i9Hiwc5g.d.mts} +9 -4
- package/docs/event-log.md +2 -1
- package/docs/persistence.md +71 -0
- package/docs/steps.md +2 -0
- package/docs/usage-and-budgets.md +2 -0
- package/docs/verify.md +7 -2
- package/package.json +1 -1
- package/skills/build-machine/SKILL.md +4 -1
package/dist/testing.mjs
CHANGED
|
@@ -1,1958 +1,2569 @@
|
|
|
1
|
-
import { L as getCallUsage, d as canonicalEventJson, ft as isError, ht as AgentError, mt as isRecord, pt as isPlainObject, rt as getStateMeta, z as normalizeGeneratorResult } from "./event-log-DkVlk3Rh.mjs";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { L as getCallUsage, R as isTextLogic, S as replay, c as agentCallSite, d as canonicalEventJson, ft as isError, ht as AgentError, mt as isRecord, pt as isPlainObject, rt as getStateMeta, z as normalizeGeneratorResult } from "./event-log-DkVlk3Rh.mjs";
|
|
2
|
+
import { s as isDecisionLogic } from "./decision-m5hGmHpF.mjs";
|
|
3
|
+
import { i as isActLogic } from "./act-B9hLQls6.mjs";
|
|
4
|
+
import { _ as runToQuiescence, d as createAgentRuntime, h as isStateMachineLogic, n as mapEffects, t as isAsyncIterable, y as toJsonValue } from "./effects-BT_cSY0j.mjs";
|
|
5
|
+
import { n as createInMemoryEventLogStore } from "./event-log-store-Brx9tOHt.mjs";
|
|
6
|
+
import { a as explorePaths, c as simulateAgent, i as canReach, l as textOutput, n as AgentUnknownStateError, o as isExplorationComplete, r as assertAgentMachine, s as lintAgentMachine, t as AgentLintError } from "./verify-ChyX6jq0.mjs";
|
|
4
7
|
import { getInitialMicrosteps, getMicrosteps } from "xstate";
|
|
5
|
-
//#region src/
|
|
8
|
+
//#region src/equivalence.ts
|
|
6
9
|
/**
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
+
* Verified equivalence — prove a port behaves like the code it replaces.
|
|
11
|
+
*
|
|
12
|
+
* A port is equivalent when, given the same model answers and the same tool
|
|
13
|
+
* results, it produces the same OBSERVABLE EFFECTS: the same number of model
|
|
14
|
+
* calls, the same tool calls with the same arguments, and the same output.
|
|
15
|
+
* Prompts, internal steps, and states are free to differ.
|
|
16
|
+
*
|
|
17
|
+
* 1. Record the original: wrap its model (`recorder.model`, an AI SDK
|
|
18
|
+
* `LanguageModelV3`, or `recorder.openai`, an `openai` client) and its tools
|
|
19
|
+
* (`recorder.tool` / `recorder.tools`), run a scenario, call
|
|
20
|
+
* `recorder.output(result)`, and save `recorder.recording`.
|
|
21
|
+
* 2. Replay it on the port: `createRecorder({ replay: recording })` answers
|
|
22
|
+
* every model call and tool call from the recording, with no API key and no
|
|
23
|
+
* side effects, and records what the port did.
|
|
24
|
+
* 3. `compareRecordings(original, port)` reports the first effect that differs.
|
|
25
|
+
*
|
|
26
|
+
* A port that changes how it calls the model on purpose (a model-picked action
|
|
27
|
+
* that became `agent.decide`, a stream that became `doGenerate`) still replays:
|
|
28
|
+
* `translate` turns the recorded answer into one that fits, `fallback` answers
|
|
29
|
+
* the calls the original never made, and `compareRecordings(…, { kinds })`
|
|
30
|
+
* compares only tool calls and output.
|
|
31
|
+
*
|
|
32
|
+
* Any other client (`@anthropic-ai/sdk`, `@google/genai`, a framework's own
|
|
33
|
+
* model client) records through `recorder.client` or `recorder.fn`: each call
|
|
34
|
+
* is a `call` effect, answered in replay by name and order.
|
|
35
|
+
*
|
|
36
|
+
* Provider-free: every seam is wrapped structurally, so this module imports
|
|
37
|
+
* neither `ai` nor `openai`.
|
|
38
|
+
* @module
|
|
10
39
|
*/
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
40
|
+
/**
|
|
41
|
+
* Thrown in replay when the port asks for something the recording cannot
|
|
42
|
+
* answer: a model call past the last recorded one, or a tool call the original
|
|
43
|
+
* never made. The effect is still recorded, so {@link compareRecordings}
|
|
44
|
+
* reports it.
|
|
45
|
+
*/
|
|
46
|
+
var RecordingDivergenceError = class extends Error {
|
|
47
|
+
name = "RecordingDivergenceError";
|
|
48
|
+
};
|
|
49
|
+
function toRecordedError(error) {
|
|
50
|
+
if (!(error instanceof Error)) return {
|
|
51
|
+
name: "Error",
|
|
52
|
+
message: String(error)
|
|
53
|
+
};
|
|
54
|
+
const recorded = {
|
|
55
|
+
name: error.name,
|
|
56
|
+
message: error.message
|
|
57
|
+
};
|
|
58
|
+
if (Object.prototype.propertyIsEnumerable.call(error, "name")) recorded.ownName = true;
|
|
59
|
+
const code = error.code;
|
|
60
|
+
if (code !== void 0) recorded.code = toJson(code);
|
|
61
|
+
const data = Object.fromEntries(Object.entries(error).filter(([key, value]) => key !== "code" && (typeof value === "number" || typeof value === "boolean")));
|
|
62
|
+
if (Object.keys(data).length) recorded.data = data;
|
|
63
|
+
const markers = /* @__PURE__ */ new Set();
|
|
64
|
+
for (let holder = error; holder; holder = Object.getPrototypeOf(holder)) for (const symbol of Object.getOwnPropertySymbols(holder)) {
|
|
65
|
+
const key = Symbol.keyFor(symbol);
|
|
66
|
+
if (key !== void 0 && error[symbol] === true) markers.add(key);
|
|
26
67
|
}
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
68
|
+
if (markers.size) recorded.markers = [...markers];
|
|
69
|
+
return recorded;
|
|
70
|
+
}
|
|
71
|
+
function fromRecordedError({ data, markers, name, ownName, ...fields }) {
|
|
72
|
+
const error = Object.assign(new Error(fields.message), fields, data);
|
|
73
|
+
if (ownName || name !== "Error") Object.defineProperty(error, "name", {
|
|
74
|
+
value: name,
|
|
75
|
+
writable: true,
|
|
76
|
+
configurable: true,
|
|
77
|
+
enumerable: ownName ?? false
|
|
78
|
+
});
|
|
79
|
+
for (const marker of markers ?? []) error[Symbol.for(marker)] = true;
|
|
80
|
+
return error;
|
|
81
|
+
}
|
|
82
|
+
/** What two runs compare of an error: its data and markers may differ. */
|
|
83
|
+
function errorIdentity$1(error) {
|
|
84
|
+
return error && {
|
|
85
|
+
name: error.name,
|
|
86
|
+
message: error.message,
|
|
87
|
+
code: error.code
|
|
30
88
|
};
|
|
31
89
|
}
|
|
32
|
-
/**
|
|
33
|
-
function
|
|
34
|
-
if (
|
|
35
|
-
|
|
36
|
-
|
|
90
|
+
/** A JSON-safe copy: Dates become ISO strings, Errors become {@link RecordedError}. */
|
|
91
|
+
function toJson(value) {
|
|
92
|
+
if (value === void 0) return void 0;
|
|
93
|
+
return JSON.parse(JSON.stringify(value, function(key, item) {
|
|
94
|
+
const raw = this[key];
|
|
95
|
+
if (raw instanceof Date) return raw.toISOString();
|
|
96
|
+
if (item instanceof Error) return toRecordedError(item);
|
|
97
|
+
if (typeof item === "bigint") return item.toString();
|
|
98
|
+
if (item instanceof Uint8Array) return Array.from(item);
|
|
99
|
+
return item;
|
|
100
|
+
}));
|
|
37
101
|
}
|
|
38
|
-
/**
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
return
|
|
102
|
+
/** Restores the `Date` timestamps the AI SDK puts on responses. */
|
|
103
|
+
function reviveTimestamps(value) {
|
|
104
|
+
if (value && typeof value === "object") {
|
|
105
|
+
for (const holder of [value, value.response]) if (holder && typeof holder["timestamp"] === "string") holder["timestamp"] = new Date(holder["timestamp"]);
|
|
106
|
+
}
|
|
107
|
+
return value;
|
|
44
108
|
}
|
|
45
|
-
function
|
|
46
|
-
if (
|
|
47
|
-
|
|
48
|
-
if (!isRecord(a) || !isRecord(b)) return false;
|
|
49
|
-
const keys = Object.keys(a);
|
|
50
|
-
return keys.length === Object.keys(b).length && keys.every((key) => deepEqual(a[key], b[key]));
|
|
109
|
+
function omit(value, keys) {
|
|
110
|
+
if (!value || typeof value !== "object") return value;
|
|
111
|
+
return Object.fromEntries(Object.entries(value).filter(([key]) => !keys.includes(key)));
|
|
51
112
|
}
|
|
52
113
|
/**
|
|
53
|
-
*
|
|
54
|
-
* expectation is classified once instead of once per candidate:
|
|
55
|
-
*
|
|
56
|
-
* - A string expectation matches an event's `type`, or a state value it names.
|
|
57
|
-
* - An event expectation matches an event with the same `type` whose other
|
|
58
|
-
* declared keys deep-equal, so payload details are opt-in.
|
|
59
|
-
* - A state-value expectation matches structurally.
|
|
114
|
+
* Key-order-independent JSON, for equality.
|
|
60
115
|
* @internal
|
|
61
116
|
*/
|
|
62
|
-
function
|
|
63
|
-
if (
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
117
|
+
function stableStringify(value) {
|
|
118
|
+
if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
|
|
119
|
+
if (value && typeof value === "object") return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`).join(",")}}`;
|
|
120
|
+
return JSON.stringify(value) ?? "undefined";
|
|
121
|
+
}
|
|
122
|
+
function jsonEqual(a, b) {
|
|
123
|
+
return stableStringify(toJson(a)) === stableStringify(toJson(b));
|
|
124
|
+
}
|
|
125
|
+
function streamFrom(parts) {
|
|
126
|
+
return new ReadableStream({ start(controller) {
|
|
127
|
+
for (const part of parts) controller.enqueue(reviveTimestamps(structuredClone(part)));
|
|
128
|
+
controller.close();
|
|
129
|
+
} });
|
|
130
|
+
}
|
|
131
|
+
async function* iterate(chunks) {
|
|
132
|
+
for (const chunk of chunks) yield structuredClone(chunk);
|
|
78
133
|
}
|
|
79
134
|
/**
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
* Both trajectories may be state values or events collected from
|
|
83
|
-
* `onTransition`;
|
|
84
|
-
* items are compared by shape, so `['prompting', 'drafting']` scores a state
|
|
85
|
-
* path and `['PROMPT_SUBMITTED', 'SEND']` scores an event log with the same
|
|
86
|
-
* call.
|
|
87
|
-
*
|
|
88
|
-
* Semantics are an ordered subsequence: every expected item must appear in
|
|
89
|
-
* `actual`, in order, with gaps allowed. Pass `{ exact: true }` to require
|
|
90
|
-
* equality instead.
|
|
91
|
-
*
|
|
92
|
-
* @example Trajectory assertions in a test
|
|
93
|
-
* ```ts
|
|
94
|
-
* const statePath: unknown[] = [];
|
|
95
|
-
* const result = await runToQuiescence(
|
|
96
|
-
* createAgentRuntime(machine, {
|
|
97
|
-
* executors,
|
|
98
|
-
* onTransition: (snapshot) => statePath.push(snapshot.value),
|
|
99
|
-
* }),
|
|
100
|
-
* { input },
|
|
101
|
-
* );
|
|
102
|
-
*
|
|
103
|
-
* const path = matchesTrajectory(statePath, ['prompting', 'drafting', 'sent']);
|
|
104
|
-
* expect(path.matched, JSON.stringify(path.firstMiss)).toBe(true);
|
|
105
|
-
* expect(matchesTrajectory(events, ['PROMPT_SUBMITTED', 'SEND']).matched).toBe(true);
|
|
106
|
-
* ```
|
|
107
|
-
*
|
|
108
|
-
* @example Partial credit in an eval scorer
|
|
109
|
-
* ```ts
|
|
110
|
-
* const match = matchesTrajectory(output.statePath, expected.statePath);
|
|
111
|
-
* return { name: 'state_path', score: match.score, metadata: { ...match.firstMiss } };
|
|
112
|
-
* ```
|
|
135
|
+
* Creates a {@link Recorder}. With no options it records: every wrapped call
|
|
136
|
+
* runs for real. With `replay` it answers from a recording instead.
|
|
113
137
|
*/
|
|
114
|
-
function
|
|
115
|
-
const
|
|
116
|
-
|
|
117
|
-
const
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
138
|
+
function createRecorder(options = {}) {
|
|
139
|
+
const effects = [];
|
|
140
|
+
const replay = options.replay?.effects;
|
|
141
|
+
const replayedModels = replay?.filter((effect) => effect.kind === "model");
|
|
142
|
+
const usedModels = /* @__PURE__ */ new Set();
|
|
143
|
+
const usedTools = /* @__PURE__ */ new Set();
|
|
144
|
+
const usedCalls = /* @__PURE__ */ new Set();
|
|
145
|
+
const callCounts = /* @__PURE__ */ new Map();
|
|
146
|
+
const normalizeEffect = (effect, list) => options.normalize?.(effect, list.indexOf(effect), list) ?? effect;
|
|
147
|
+
let modelCalls = 0;
|
|
148
|
+
let toolsInFlight = 0;
|
|
149
|
+
/**
|
|
150
|
+
* The earliest unused recorded call that fits this one: one `translate`
|
|
151
|
+
* adapts, or one with the same method and shape. Not by position, so a call
|
|
152
|
+
* the port added or dropped does not shift every answer after it.
|
|
153
|
+
*/
|
|
154
|
+
function nextReplayedModel(effect) {
|
|
155
|
+
for (const recorded of replayedModels) {
|
|
156
|
+
if (usedModels.has(recorded)) continue;
|
|
157
|
+
const translated = options.translate?.(structuredClone(recorded), effect);
|
|
158
|
+
if (translated) {
|
|
159
|
+
effect.translated = true;
|
|
160
|
+
return {
|
|
161
|
+
recorded,
|
|
162
|
+
answer: translated
|
|
163
|
+
};
|
|
130
164
|
}
|
|
131
|
-
if (
|
|
165
|
+
if (recorded.method === effect.method && modelCallsMatch(options.modelCalls, recorded, effect)) return {
|
|
166
|
+
recorded,
|
|
167
|
+
answer: recorded
|
|
168
|
+
};
|
|
132
169
|
}
|
|
133
|
-
if (found === -1) {
|
|
134
|
-
firstMiss = {
|
|
135
|
-
index,
|
|
136
|
-
expected: want,
|
|
137
|
-
searchedFrom
|
|
138
|
-
};
|
|
139
|
-
break;
|
|
140
|
-
}
|
|
141
|
-
matchedCount++;
|
|
142
|
-
cursor = found + 1;
|
|
143
170
|
}
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
matchedCount,
|
|
149
|
-
expectedCount,
|
|
150
|
-
score: matched ? 1 : denominator === 0 ? 1 : matchedCount / denominator,
|
|
151
|
-
...firstMiss ? { firstMiss } : {}
|
|
152
|
-
};
|
|
153
|
-
}
|
|
154
|
-
//#endregion
|
|
155
|
-
//#region src/seam.ts
|
|
156
|
-
/** The only own keys an executor result carries. @internal */
|
|
157
|
-
const TEXT_RESULT_KEYS = /* @__PURE__ */ new Set([
|
|
158
|
-
"result",
|
|
159
|
-
"messages",
|
|
160
|
-
"usage",
|
|
161
|
-
"raw"
|
|
162
|
-
]);
|
|
163
|
-
/**
|
|
164
|
-
* True when a scripted entry is the executor result rather than the output
|
|
165
|
-
* value: it owns a `result` key and owns NOTHING outside the result's own
|
|
166
|
-
* vocabulary. Bare `'result' in value` would swallow an output object's
|
|
167
|
-
* siblings (`{ result: 'draft', confidence: 0.9 }` would lose `confidence`)
|
|
168
|
-
* and would also match an inherited `result`. @internal
|
|
169
|
-
*/
|
|
170
|
-
function isTextResult(value) {
|
|
171
|
-
return Object.hasOwn(value, "result") && Object.keys(value).every((key) => TEXT_RESULT_KEYS.has(key));
|
|
172
|
-
}
|
|
173
|
-
/**
|
|
174
|
-
* Resolves ONE scripted entry to an executor result: a function entry is
|
|
175
|
-
* called with the request, and the value is taken as the executor result only
|
|
176
|
-
* when it is one (see {@link isTextResult}). @internal
|
|
177
|
-
*/
|
|
178
|
-
async function resolveSeamScriptEntry(entry, request, info) {
|
|
179
|
-
const value = typeof entry === "function" ? await entry(request, info) : entry;
|
|
180
|
-
return isRecord(value) && isTextResult(value) ? value : { result: value };
|
|
181
|
-
}
|
|
182
|
-
/** Names a pending text request in an error message. @internal */
|
|
183
|
-
function describeText(request) {
|
|
184
|
-
return request.name ? `'${request.name}' (model '${request.model}')` : `(model '${request.model}')`;
|
|
185
|
-
}
|
|
186
|
-
/**
|
|
187
|
-
* Stream semantics with no model: the whole text lands as one chunk. @internal
|
|
188
|
-
*/
|
|
189
|
-
function emitScriptedChunk(result, info) {
|
|
190
|
-
const text = isRecord(result) ? result["result"] : void 0;
|
|
191
|
-
if (typeof text === "string") info?.onChunk?.(text);
|
|
192
|
-
}
|
|
193
|
-
/**
|
|
194
|
-
* The seam's own answer, for scoring: the executor result's `result`.
|
|
195
|
-
* @internal
|
|
196
|
-
*/
|
|
197
|
-
async function seamOutputOf(result, request) {
|
|
198
|
-
if (!isRecord(result)) return;
|
|
199
|
-
return await normalizeGeneratorResult(result, `seam '${request.name ?? request.model}'`);
|
|
200
|
-
}
|
|
201
|
-
/**
|
|
202
|
-
* Runs a machine end to end with one model call under test, and slices the run
|
|
203
|
-
* at that call.
|
|
204
|
-
*
|
|
205
|
-
* The `before`/`after` slices are the point: `after.statePath` is the branch
|
|
206
|
-
* the seam chose and `after.events` is the same question against the durable
|
|
207
|
-
* log, both ready for `matchesTrajectory`. The state slice splits where the
|
|
208
|
-
* state path stood when the seam answered; the event slice splits at the
|
|
209
|
-
* seam's own effect completion (the first `xstate.done.*`/`xstate.error.*`
|
|
210
|
-
* entry appended after the call was made).
|
|
211
|
-
*
|
|
212
|
-
* @example No API key: the seam is scripted too, so the whole thing runs offline.
|
|
213
|
-
* ```ts
|
|
214
|
-
* const run = await runSeam(emailDrafter, {
|
|
215
|
-
* scripts: { writeFollowUps: [followUps], emailDrafter: [draft] },
|
|
216
|
-
* seam: { request: 'writeFollowUps' },
|
|
217
|
-
* respond: ({ state }) => (state === 'prompting' ? { type: 'PROMPT_SUBMITTED', prompt } : null),
|
|
218
|
-
* });
|
|
219
|
-
*
|
|
220
|
-
* matchesTrajectory(run.after.statePath, ['needsMoreInfo', 'drafting']);
|
|
221
|
-
* ```
|
|
222
|
-
*
|
|
223
|
-
* @example A candidate prompt at the seam, scored against the same rows.
|
|
224
|
-
* ```ts
|
|
225
|
-
* const { generateText } = createAiSdkExecutors({ models });
|
|
226
|
-
* const run = await runSeam(emailDrafter, { ...row, candidate: generateText });
|
|
227
|
-
* ```
|
|
228
|
-
*/
|
|
229
|
-
async function runSeam(machine, options) {
|
|
230
|
-
const { seam, candidate } = options;
|
|
231
|
-
const queues = new Map(Object.entries(options.scripts ?? {}).map(([key, answers]) => [key, [...answers]]));
|
|
232
|
-
const statePath = [];
|
|
233
|
-
let calls = 0;
|
|
234
|
-
let seamMatches = 0;
|
|
235
|
-
let seamOutput;
|
|
236
|
-
let seamUsage;
|
|
237
|
-
const callLog = [];
|
|
238
|
-
let seamReached = false;
|
|
239
|
-
let callsBeforeSeam = -1;
|
|
240
|
-
let seamStateAt = 0;
|
|
241
|
-
let seamEventAt = 0;
|
|
242
|
-
let liveEvents = 0;
|
|
243
|
-
const queueKeyOf = (request) => request.name !== void 0 && queues.has(request.name) ? request.name : request.model;
|
|
171
|
+
function noAnswerMessage(effect) {
|
|
172
|
+
const unused = replayedModels.filter((recorded) => !usedModels.has(recorded));
|
|
173
|
+
return unused.length ? `Model call #${effect.call} (${effect.method}, ${JSON.stringify(modelCallShape(effect))}) fits no unused recorded call: ${unused.map((recorded) => `#${recorded.call} (${recorded.method}, ${JSON.stringify(modelCallShape(recorded))})`).join(", ")}. A recorded answer only fits a call of the same method and shape; pass \`translate\` to adapt one.` : `Model call #${effect.call} was never made by the recorded run (it made ${replayedModels.length}).`;
|
|
174
|
+
}
|
|
244
175
|
/**
|
|
245
|
-
*
|
|
246
|
-
*
|
|
247
|
-
* running dry.
|
|
176
|
+
* Records a model call around `run`, or answers it from the recording.
|
|
177
|
+
* `fallback` runs it on the stand-in when the recording has no answer.
|
|
248
178
|
*/
|
|
249
|
-
|
|
250
|
-
const
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
const callIndex = calls++;
|
|
261
|
-
const isSeam = request.name === seam.request && seamMatches++ === (seam.occurrence ?? 0);
|
|
262
|
-
const ledger = (source) => {
|
|
263
|
-
callLog.push({
|
|
264
|
-
...request.name !== void 0 ? { name: request.name } : {},
|
|
265
|
-
model: request.model,
|
|
266
|
-
key: queueKeyOf(request),
|
|
267
|
-
source,
|
|
268
|
-
seam: isSeam
|
|
269
|
-
});
|
|
179
|
+
async function modelCall(seam, method, request, streamed, run, fallback) {
|
|
180
|
+
const effect = {
|
|
181
|
+
kind: "model",
|
|
182
|
+
seam,
|
|
183
|
+
method,
|
|
184
|
+
call: modelCalls++,
|
|
185
|
+
request: toJson(omit(request, [
|
|
186
|
+
"abortSignal",
|
|
187
|
+
"headers",
|
|
188
|
+
"signal"
|
|
189
|
+
]))
|
|
270
190
|
};
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
seamEventAt = liveEvents;
|
|
290
|
-
seamOutput = scripted.result;
|
|
291
|
-
seamUsage = scripted.usage;
|
|
292
|
-
return scripted;
|
|
293
|
-
};
|
|
294
|
-
const executors = {
|
|
295
|
-
...options.executors,
|
|
296
|
-
generateText: route,
|
|
297
|
-
streamText: async (request, info) => {
|
|
298
|
-
const result = await route(request, info);
|
|
299
|
-
emitScriptedChunk(result, info);
|
|
300
|
-
return result;
|
|
301
|
-
}
|
|
302
|
-
};
|
|
303
|
-
let snapshot;
|
|
304
|
-
let event;
|
|
305
|
-
const events = [];
|
|
306
|
-
let result;
|
|
307
|
-
const maxTurns = options.maxTurns ?? 12;
|
|
308
|
-
for (let turn = 0; turn <= maxTurns; turn++) {
|
|
309
|
-
result = await runToQuiescence(createAgentRuntime(machine, {
|
|
310
|
-
...options.actors ? { actors: options.actors } : {},
|
|
311
|
-
executors,
|
|
312
|
-
onTransition: (next, causedBy) => {
|
|
313
|
-
if (turn > 0 && causedBy.type === "@xstate.init") return;
|
|
314
|
-
events.push(causedBy);
|
|
315
|
-
liveEvents = events.length;
|
|
316
|
-
statePath.push(next.value);
|
|
191
|
+
effects.push(effect);
|
|
192
|
+
if (replay) {
|
|
193
|
+
const replayed = nextReplayedModel(effect);
|
|
194
|
+
if (replayed) {
|
|
195
|
+
const { recorded, answer } = replayed;
|
|
196
|
+
usedModels.add(recorded);
|
|
197
|
+
effect.replayed = recorded.call;
|
|
198
|
+
if (answer.error) {
|
|
199
|
+
effect.error = answer.error;
|
|
200
|
+
throw fromRecordedError(answer.error);
|
|
201
|
+
}
|
|
202
|
+
effect.response = answer.response;
|
|
203
|
+
effect.stream = answer.stream;
|
|
204
|
+
if (seam === "openai") return streamed ? iterate(answer.stream ?? []) : structuredClone(answer.response);
|
|
205
|
+
return streamed ? {
|
|
206
|
+
...reviveTimestamps(structuredClone(answer.response) ?? {}),
|
|
207
|
+
stream: streamFrom(answer.stream ?? [])
|
|
208
|
+
} : reviveTimestamps(structuredClone(answer.response));
|
|
317
209
|
}
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
if (result.status !== "idle" || turn === maxTurns) break;
|
|
323
|
-
const next = await options.respond?.({
|
|
324
|
-
snapshot: result.snapshot,
|
|
325
|
-
state: result.snapshot.value,
|
|
326
|
-
meta: getStateMeta(result.snapshot),
|
|
327
|
-
turn,
|
|
328
|
-
result
|
|
329
|
-
});
|
|
330
|
-
if (!next) break;
|
|
331
|
-
event = next;
|
|
332
|
-
snapshot = result.persist();
|
|
333
|
-
}
|
|
334
|
-
let splitAt = events.length;
|
|
335
|
-
if (seamReached) {
|
|
336
|
-
splitAt = seamEventAt;
|
|
337
|
-
for (let index = seamEventAt; index < events.length; index++) {
|
|
338
|
-
const type = events[index].type;
|
|
339
|
-
if (type.startsWith("xstate.done") || type.startsWith("xstate.error")) {
|
|
340
|
-
splitAt = index;
|
|
341
|
-
break;
|
|
210
|
+
if (!fallback) {
|
|
211
|
+
const error = new RecordingDivergenceError(noAnswerMessage(effect));
|
|
212
|
+
effect.error = toRecordedError(error);
|
|
213
|
+
throw error;
|
|
342
214
|
}
|
|
215
|
+
effect.unrecorded = true;
|
|
216
|
+
run = fallback;
|
|
343
217
|
}
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
callsBeforeSeam,
|
|
351
|
-
calls: callLog,
|
|
352
|
-
before: {
|
|
353
|
-
statePath: statePath.slice(0, splitStateAt),
|
|
354
|
-
events: events.slice(0, splitAt)
|
|
355
|
-
},
|
|
356
|
-
after: {
|
|
357
|
-
statePath: statePath.slice(splitStateAt),
|
|
358
|
-
events: events.slice(splitAt)
|
|
218
|
+
let result;
|
|
219
|
+
try {
|
|
220
|
+
result = await run();
|
|
221
|
+
} catch (error) {
|
|
222
|
+
effect.error = toRecordedError(error);
|
|
223
|
+
throw error;
|
|
359
224
|
}
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
//#region src/internal/trace-states.ts
|
|
364
|
-
/** The `value` of a trace event's `from` or `snapshot`. @internal */
|
|
365
|
-
function valueOf(holder) {
|
|
366
|
-
return isRecord(holder) ? holder.value : void 0;
|
|
367
|
-
}
|
|
368
|
-
/** A decision request on its first attempt (no failed attempts yet). @internal */
|
|
369
|
-
function isFirstDecision(request) {
|
|
370
|
-
return isRecord(request) && request.kind === "decision" && (!Array.isArray(request.attempts) || request.attempts.length === 0);
|
|
371
|
-
}
|
|
372
|
-
/**
|
|
373
|
-
* Every active leaf of a state value as a dot path:
|
|
374
|
-
* `'a'` → `['a']`; `{ a: 'b' }` → `['a.b']`; `{ p: { x: 'one', y: {} } }` → `['p.x.one', 'p.y']`.
|
|
375
|
-
* @internal
|
|
376
|
-
*/
|
|
377
|
-
function statePaths(value, prefix = "") {
|
|
378
|
-
if (typeof value === "string") return [prefix + value];
|
|
379
|
-
if (!isRecord(value)) return [];
|
|
380
|
-
const entries = Object.entries(value);
|
|
381
|
-
if (entries.length === 0) return prefix ? [prefix.slice(0, -1)] : [];
|
|
382
|
-
return entries.flatMap(([key, child]) => statePaths(child, `${prefix}${key}.`));
|
|
383
|
-
}
|
|
384
|
-
/**
|
|
385
|
-
* A trace's `machine.transition` events as steps, skipping `@xstate.init`.
|
|
386
|
-
* A first-attempt decision's `request.start` names the decision on the step
|
|
387
|
-
* that follows it, skipping `@agent.usage` steps. Reads live and serialized traces alike.
|
|
388
|
-
* @internal
|
|
389
|
-
*/
|
|
390
|
-
function traceSteps(trace) {
|
|
391
|
-
const steps = [];
|
|
392
|
-
let decision;
|
|
393
|
-
for (const event of trace) {
|
|
394
|
-
if (event.type === "request.start" && isFirstDecision(event.request)) {
|
|
395
|
-
const name = event.request.name;
|
|
396
|
-
decision = typeof name === "string" ? name : void 0;
|
|
397
|
-
continue;
|
|
225
|
+
if (!streamed) {
|
|
226
|
+
effect.response = toJson(result);
|
|
227
|
+
return result;
|
|
398
228
|
}
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
if (
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
to: statePaths(toValue),
|
|
410
|
-
toValue,
|
|
411
|
-
ignored: event.ignored === true,
|
|
412
|
-
...decision === void 0 || type === "@agent.usage" ? {} : { decision }
|
|
413
|
-
});
|
|
414
|
-
if (type !== "@agent.usage") decision = void 0;
|
|
415
|
-
}
|
|
416
|
-
return steps;
|
|
417
|
-
}
|
|
418
|
-
//#endregion
|
|
419
|
-
//#region src/fold-traces.ts
|
|
420
|
-
/**
|
|
421
|
-
* Folds many runs' traces into per-state stats: how many runs visited each
|
|
422
|
-
* state, how many of those failed, how many options the guards withheld
|
|
423
|
-
* at decisions made there, how many events it ignored, and how many failures
|
|
424
|
-
* `blame` traced back to it. Pure; reads only:
|
|
425
|
-
*
|
|
426
|
-
* - `machine.transition`: the run visited `from.value` and `snapshot.value`;
|
|
427
|
-
* with `ignored`, each of those states ignored one event.
|
|
428
|
-
* - `request.start` of a decision, first attempt only: its `request.withheld`
|
|
429
|
-
* counts toward the state the run was in, the `from.value` of the
|
|
430
|
-
* transition its choice causes.
|
|
431
|
-
*
|
|
432
|
-
* A trace is one run's events, live (`AgentTraceEvent`) or serialized
|
|
433
|
-
* (`serializeTraceEvent`). A host-driven `decideNext` traces nothing; push
|
|
434
|
-
* `{ type: 'request.start', request: result.request }` onto the trace before
|
|
435
|
-
* sending its event. Parallel states count once per active leaf.
|
|
436
|
-
*
|
|
437
|
-
* `failures` counts a failed run against every state it visited, so states
|
|
438
|
-
* after a bad decision rank as high as the decision. Sort by `blamed` to see
|
|
439
|
-
* where failures start.
|
|
440
|
-
*
|
|
441
|
-
* @example
|
|
442
|
-
* ```ts no-check
|
|
443
|
-
* const stats = foldTraces(traces, {
|
|
444
|
-
* failed: (trace) => trace.at(-1)?.type === 'run.end' && trace.at(-1).status === 'error',
|
|
445
|
-
* });
|
|
446
|
-
* ```
|
|
447
|
-
*/
|
|
448
|
-
function foldTraces(traces, options) {
|
|
449
|
-
const stats = /* @__PURE__ */ new Map();
|
|
450
|
-
const statsFor = (state) => {
|
|
451
|
-
let entry = stats.get(state);
|
|
452
|
-
if (!entry) {
|
|
453
|
-
entry = {
|
|
454
|
-
state,
|
|
455
|
-
visits: 0,
|
|
456
|
-
failures: 0,
|
|
457
|
-
withheld: 0,
|
|
458
|
-
ignored: 0,
|
|
459
|
-
blamed: 0
|
|
460
|
-
};
|
|
461
|
-
stats.set(state, entry);
|
|
229
|
+
const parts = [];
|
|
230
|
+
effect.stream = parts;
|
|
231
|
+
if (seam === "openai") {
|
|
232
|
+
const source = result;
|
|
233
|
+
return (async function* () {
|
|
234
|
+
for await (const chunk of source) {
|
|
235
|
+
parts.push(toJson(chunk));
|
|
236
|
+
yield chunk;
|
|
237
|
+
}
|
|
238
|
+
})();
|
|
462
239
|
}
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
240
|
+
const { stream, ...rest } = result;
|
|
241
|
+
effect.response = toJson(rest);
|
|
242
|
+
return {
|
|
243
|
+
...rest,
|
|
244
|
+
stream: stream.pipeThrough(new TransformStream({ transform(part, controller) {
|
|
245
|
+
parts.push(toJson(part));
|
|
246
|
+
controller.enqueue(part);
|
|
247
|
+
} }))
|
|
471
248
|
};
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
249
|
+
}
|
|
250
|
+
/** `inputOf` picks the recorded input from the call's arguments. */
|
|
251
|
+
function tool(name, fn, inputOf = (args) => args.length === 1 ? args[0] : args) {
|
|
252
|
+
return ((...args) => {
|
|
253
|
+
const input = toJson(inputOf(args));
|
|
254
|
+
const effect = {
|
|
255
|
+
kind: "tool",
|
|
256
|
+
name,
|
|
257
|
+
step: modelCalls,
|
|
258
|
+
input,
|
|
259
|
+
...toolsInFlight > 0 ? { concurrent: true } : {}
|
|
260
|
+
};
|
|
261
|
+
effects.push(effect);
|
|
262
|
+
return track(call(effect, args));
|
|
263
|
+
});
|
|
264
|
+
function call(effect, args) {
|
|
265
|
+
const { input } = effect;
|
|
266
|
+
if (replay && options.tools !== "live") {
|
|
267
|
+
const wanted = normalizeEffect(effect, effects).input;
|
|
268
|
+
const matches = (candidate) => candidate.kind === "tool" && !usedTools.has(candidate) && candidate.name === name && jsonEqual(normalizeEffect(candidate, replay).input, wanted);
|
|
269
|
+
const recorded = replay.find(matches);
|
|
270
|
+
if (!recorded && options.fallback?.tools) {
|
|
271
|
+
effect.unrecorded = true;
|
|
272
|
+
return runLive(effect, args);
|
|
273
|
+
}
|
|
274
|
+
const settle = (value) => {
|
|
275
|
+
if (recorded?.sync) return value();
|
|
276
|
+
try {
|
|
277
|
+
return Promise.resolve(value());
|
|
278
|
+
} catch (error) {
|
|
279
|
+
return Promise.reject(error);
|
|
280
|
+
}
|
|
281
|
+
};
|
|
282
|
+
return settle(() => {
|
|
283
|
+
if (!recorded) {
|
|
284
|
+
const error = new RecordingDivergenceError(`Tool '${name}' was called with ${JSON.stringify(input)}; the recorded run made no such call.`);
|
|
285
|
+
effect.error = toRecordedError(error);
|
|
286
|
+
throw error;
|
|
287
|
+
}
|
|
288
|
+
usedTools.add(recorded);
|
|
289
|
+
if (recorded.sync) effect.sync = true;
|
|
290
|
+
if (recorded.error) {
|
|
291
|
+
effect.error = recorded.error;
|
|
292
|
+
throw fromRecordedError(recorded.error);
|
|
293
|
+
}
|
|
294
|
+
effect.output = recorded.output;
|
|
295
|
+
return structuredClone(recorded.output);
|
|
296
|
+
});
|
|
297
|
+
}
|
|
298
|
+
return runLive(effect, args);
|
|
481
299
|
}
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
300
|
+
function runLive(effect, args) {
|
|
301
|
+
let output;
|
|
302
|
+
try {
|
|
303
|
+
output = fn(...args);
|
|
304
|
+
} catch (error) {
|
|
305
|
+
effect.sync = true;
|
|
306
|
+
effect.error = toRecordedError(error);
|
|
307
|
+
throw error;
|
|
308
|
+
}
|
|
309
|
+
if (!(output instanceof Promise)) {
|
|
310
|
+
effect.sync = true;
|
|
311
|
+
effect.output = toJson(output);
|
|
312
|
+
return output;
|
|
313
|
+
}
|
|
314
|
+
return output.then((value) => {
|
|
315
|
+
effect.output = toJson(value);
|
|
316
|
+
return value;
|
|
317
|
+
}, (error) => {
|
|
318
|
+
effect.error = toRecordedError(error);
|
|
319
|
+
throw error;
|
|
320
|
+
});
|
|
487
321
|
}
|
|
488
|
-
const blamed = failed ? options.blame?.(trace) : void 0;
|
|
489
|
-
if (blamed !== void 0) statsFor(blamed).blamed++;
|
|
490
322
|
}
|
|
491
|
-
|
|
492
|
-
}
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
if (tokens[index] !== eventTokens[index]) return false;
|
|
504
|
-
}
|
|
505
|
-
return true;
|
|
506
|
-
}
|
|
507
|
-
/** Whether a definition's `matches` pattern (e.g. an invoke's `actorId`) accepts the traced event. */
|
|
508
|
-
function matchesPattern(definition, event) {
|
|
509
|
-
const pattern = definition.matches;
|
|
510
|
-
if (!isRecord(pattern) || !isRecord(event)) return true;
|
|
511
|
-
return Object.entries(pattern).every(([key, value]) => Object.is(event[key], value));
|
|
512
|
-
}
|
|
513
|
-
/**
|
|
514
|
-
* The descriptor XState would try first at `node` for `event`, guards aside:
|
|
515
|
-
* the exact key, then partial and wildcard keys longest first, as in
|
|
516
|
-
* xstate@6.0.0-alpha.69 `getCandidates` (src/stateUtils.ts).
|
|
517
|
-
*/
|
|
518
|
-
function candidateAt(node, type, event) {
|
|
519
|
-
const keys = [...node.transitions.keys()].filter((key) => key !== type && matchesDescriptor(type, key)).sort((a, b) => b.length - a.length);
|
|
520
|
-
if (node.transitions.has(type)) keys.unshift(type);
|
|
521
|
-
return keys.find((key) => node.transitions.get(key).some((d) => matchesPattern(d, event)));
|
|
522
|
-
}
|
|
523
|
-
const pathOf = (node) => node.path.join(".");
|
|
524
|
-
const sameLeaves = (a, b) => statePaths(a).sort().join("|") === statePaths(b).sort().join("|");
|
|
525
|
-
/**
|
|
526
|
-
* The leaves a trace passed through inside its macrosteps: states an `always`
|
|
527
|
-
* transition left before the step ended, which no traced snapshot holds.
|
|
528
|
-
* Re-derived with XState's pure `getMicrosteps` from the snapshot before each
|
|
529
|
-
* step (the previous step's, or the fresh start's microsteps from its input).
|
|
530
|
-
* A step that cannot be recomputed, or whose recomputation ends elsewhere
|
|
531
|
-
* than the traced snapshot, adds none. Only `value` and `context` are rebuilt,
|
|
532
|
-
* so a guard that reads live children can take another route to the same
|
|
533
|
-
* leaves, and that route is credited.
|
|
534
|
-
*/
|
|
535
|
-
function passedThrough(machine, trace) {
|
|
536
|
-
const leaves = /* @__PURE__ */ new Set();
|
|
537
|
-
let resumed = false;
|
|
538
|
-
let previous;
|
|
539
|
-
for (const event of trace) {
|
|
540
|
-
if (event.type === "run.start") resumed = event.snapshot !== void 0;
|
|
541
|
-
if (event.type !== "machine.transition" || !isRecord(event.event)) continue;
|
|
542
|
-
const snapshot = isRecord(event.snapshot) ? event.snapshot : void 0;
|
|
543
|
-
try {
|
|
544
|
-
const microsteps = event.event.type === "@xstate.init" ? resumed ? [] : getInitialMicrosteps(machine, event.event.input) : previous && sameLeaves(previous.value, valueOf(event.from)) ? getMicrosteps(machine, machine.resolveState({
|
|
545
|
-
value: previous.value,
|
|
546
|
-
context: previous.context
|
|
547
|
-
}), event.event) : [];
|
|
548
|
-
const last = microsteps.at(-1)?.[0];
|
|
549
|
-
if (last && snapshot && sameLeaves(last.value, snapshot.value)) for (const [step] of microsteps) for (const leaf of statePaths(step.value)) leaves.add(leaf);
|
|
550
|
-
} catch {}
|
|
551
|
-
previous = snapshot;
|
|
552
|
-
}
|
|
553
|
-
return leaves;
|
|
554
|
-
}
|
|
555
|
-
const ratioOf = (visited, total) => total === 0 ? 1 : visited / total;
|
|
556
|
-
/**
|
|
557
|
-
* Measures how much of a machine a set of runs exercised: which atomic and
|
|
558
|
-
* final states they visited, and which handlers they took. A handler is a
|
|
559
|
-
* `(declaring state, event descriptor)` pair from `on`, invoke
|
|
560
|
-
* `onDone`/`onError`, and `after`, with the targets it declares and the
|
|
561
|
-
* leaves the runs landed in after it. Pure; reads only `machine.transition`
|
|
562
|
-
* events, live (`AgentTraceEvent`) or serialized (`serializeTraceEvent`).
|
|
563
|
-
*
|
|
564
|
-
* A step credits one handler per active leaf: the nearest node, from the leaf
|
|
565
|
-
* up, whose exact, partial (`a.*`), or wildcard (`*`) descriptor matches the
|
|
566
|
-
* event, preferred in XState's order. Guards are not replayed, so a step
|
|
567
|
-
* whose nearest handler was rejected and fell through to an ancestor is
|
|
568
|
-
* credited to the nearest one. XState keys invoke and `after` handlers by
|
|
569
|
-
* bare type (`xstate.done.actor`), one per state however many invokes; a
|
|
570
|
-
* definition's `matches` (an invoke's `actorId`) must accept the traced event
|
|
571
|
-
* for its state to be credited. Ignored steps credit no handler. `always`
|
|
572
|
-
* handlers are listed as `unobservable`, never as unvisited; a state one
|
|
573
|
-
* passes through is visited, re-derived from each step's microsteps.
|
|
574
|
-
*
|
|
575
|
-
* @example
|
|
576
|
-
* ```ts no-check
|
|
577
|
-
* const report = traceCoverage(machine, traces, { reachable: await explorePaths(machine) });
|
|
578
|
-
* expect(report.unvisitedHandlers).toEqual([]);
|
|
579
|
-
* ```
|
|
580
|
-
*/
|
|
581
|
-
function traceCoverage(machine, traces, options = {}) {
|
|
582
|
-
const nodes = /* @__PURE__ */ new Map();
|
|
583
|
-
const states = [];
|
|
584
|
-
const handlers = [];
|
|
585
|
-
const handlerAt = /* @__PURE__ */ new Map();
|
|
586
|
-
const unobservable = [];
|
|
587
|
-
const walk = (node) => {
|
|
588
|
-
const state = pathOf(node);
|
|
589
|
-
nodes.set(state, node);
|
|
590
|
-
if (node.type === "atomic" || node.type === "final") states.push({
|
|
591
|
-
state,
|
|
592
|
-
runs: 0
|
|
593
|
-
});
|
|
594
|
-
const byEvent = /* @__PURE__ */ new Map();
|
|
595
|
-
for (const [event, definitions] of node.transitions) {
|
|
596
|
-
const dynamic = definitions.some((d) => typeof d.to === "function");
|
|
597
|
-
const declared = [...new Set(definitions.flatMap((d) => (d.target ?? []).map(pathOf)))];
|
|
598
|
-
const handler = {
|
|
599
|
-
state,
|
|
600
|
-
event,
|
|
601
|
-
runs: 0,
|
|
602
|
-
targets: {
|
|
603
|
-
declared: dynamic ? "dynamic" : declared,
|
|
604
|
-
seen: [],
|
|
605
|
-
unreached: []
|
|
606
|
-
}
|
|
323
|
+
function fn(name, run, callOptions = {}) {
|
|
324
|
+
const { redact, serialize = toJson, revive } = callOptions;
|
|
325
|
+
return (async (...args) => {
|
|
326
|
+
const passed = [...args];
|
|
327
|
+
while (passed.length && passed.at(-1) === void 0) passed.pop();
|
|
328
|
+
const call = callCounts.get(name) ?? 0;
|
|
329
|
+
callCounts.set(name, call + 1);
|
|
330
|
+
const effect = {
|
|
331
|
+
kind: "call",
|
|
332
|
+
name,
|
|
333
|
+
call,
|
|
334
|
+
args: toJson(redact ? redact(passed) : passed)
|
|
607
335
|
};
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
if (unobservable.length > 0) for (const leaf of passedThrough(machine, trace)) visited.add(leaf);
|
|
630
|
-
const credited = /* @__PURE__ */ new Set();
|
|
631
|
-
for (const step of steps) {
|
|
632
|
-
for (const leaf of step.to) visited.add(leaf);
|
|
633
|
-
if (step.ignored) continue;
|
|
634
|
-
const hits = /* @__PURE__ */ new Set();
|
|
635
|
-
for (const leaf of step.from) for (let node = nodes.get(leaf); node; node = node.parent) {
|
|
636
|
-
if (node.type === "final") continue;
|
|
637
|
-
const event = candidateAt(node, step.event, step.payload);
|
|
638
|
-
if (event === void 0) continue;
|
|
639
|
-
hits.add(handlerAt.get(node).get(event));
|
|
640
|
-
break;
|
|
336
|
+
effects.push(effect);
|
|
337
|
+
if (replay) {
|
|
338
|
+
const recorded = replay.find((candidate) => candidate.kind === "call" && candidate.name === name && !usedCalls.has(candidate));
|
|
339
|
+
if (!recorded) {
|
|
340
|
+
const error = new RecordingDivergenceError(`Call #${call} to '${name}' was never made by the recorded run.`);
|
|
341
|
+
effect.error = toRecordedError(error);
|
|
342
|
+
throw error;
|
|
343
|
+
}
|
|
344
|
+
usedCalls.add(recorded);
|
|
345
|
+
if (recorded.error) {
|
|
346
|
+
effect.error = recorded.error;
|
|
347
|
+
throw fromRecordedError(recorded.error);
|
|
348
|
+
}
|
|
349
|
+
if (recorded.stream) {
|
|
350
|
+
effect.stream = recorded.stream;
|
|
351
|
+
const chunks = structuredClone(recorded.stream);
|
|
352
|
+
return revive ? revive(chunks) : iterate(chunks);
|
|
353
|
+
}
|
|
354
|
+
effect.result = recorded.result;
|
|
355
|
+
const result = structuredClone(recorded.result);
|
|
356
|
+
return revive ? revive(result) : result;
|
|
641
357
|
}
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
358
|
+
let result;
|
|
359
|
+
try {
|
|
360
|
+
result = await run(...args);
|
|
361
|
+
} catch (error) {
|
|
362
|
+
effect.error = toRecordedError(error);
|
|
363
|
+
throw error;
|
|
647
364
|
}
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
365
|
+
if (!isAsyncIterable(result)) {
|
|
366
|
+
effect.result = toJson(serialize(result));
|
|
367
|
+
return result;
|
|
368
|
+
}
|
|
369
|
+
const chunks = [];
|
|
370
|
+
effect.stream = chunks;
|
|
371
|
+
const source = result;
|
|
372
|
+
return (async function* () {
|
|
373
|
+
try {
|
|
374
|
+
for await (const chunk of source) {
|
|
375
|
+
chunks.push(toJson(serialize(chunk)));
|
|
376
|
+
yield chunk;
|
|
377
|
+
}
|
|
378
|
+
} catch (error) {
|
|
379
|
+
effect.error = toRecordedError(error);
|
|
380
|
+
throw error;
|
|
381
|
+
}
|
|
382
|
+
})();
|
|
383
|
+
});
|
|
667
384
|
}
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
return {
|
|
675
|
-
states,
|
|
676
|
-
handlers,
|
|
677
|
-
unvisitedStates,
|
|
678
|
-
unvisitedHandlers,
|
|
679
|
-
unobservable,
|
|
680
|
-
...unreachable ? { unreachable } : {},
|
|
681
|
-
...reachableComplete === void 0 ? {} : { reachableComplete },
|
|
682
|
-
ratio: {
|
|
683
|
-
states: ratioOf(counted.length - unvisitedStates.length, counted.length),
|
|
684
|
-
handlers: ratioOf(handlers.length - unvisitedHandlers.length, handlers.length)
|
|
685
|
-
}
|
|
686
|
-
};
|
|
687
|
-
}
|
|
688
|
-
//#endregion
|
|
689
|
-
//#region src/first-divergence.ts
|
|
690
|
-
const defaultKey = (step) => `${step.from.join(",")}|${step.event}|${step.to.join(",")}`;
|
|
691
|
-
const isErrorLike = (value) => isError(value) || isRecord(value) && typeof value.name === "string" && typeof value.message === "string";
|
|
692
|
-
const normalizeErrors = (value, seen = /* @__PURE__ */ new WeakSet()) => {
|
|
693
|
-
if (value === null || typeof value !== "object") return value;
|
|
694
|
-
if (seen.has(value)) return "[circular]";
|
|
695
|
-
seen.add(value);
|
|
696
|
-
try {
|
|
697
|
-
if (isErrorLike(value) && (isError(value) || "stack" in value || "code" in value)) {
|
|
698
|
-
const { name, message, code, cause } = value;
|
|
699
|
-
return {
|
|
700
|
-
name,
|
|
701
|
-
message,
|
|
702
|
-
...typeof code === "string" ? { code } : {},
|
|
703
|
-
...cause !== void 0 ? { cause: normalizeErrors(cause, seen) } : {}
|
|
385
|
+
/** Counts an async tool call as running until it settles. */
|
|
386
|
+
function track(result) {
|
|
387
|
+
if (result instanceof Promise) {
|
|
388
|
+
toolsInFlight++;
|
|
389
|
+
const settled = () => {
|
|
390
|
+
toolsInFlight--;
|
|
704
391
|
};
|
|
392
|
+
result.then(settled, settled);
|
|
705
393
|
}
|
|
706
|
-
|
|
707
|
-
if (!isPlainObject(value)) return value;
|
|
708
|
-
const copy = Object.create(null);
|
|
709
|
-
for (const key of Object.keys(value)) copy[key] = normalizeErrors(value[key], seen);
|
|
710
|
-
return copy;
|
|
711
|
-
} finally {
|
|
712
|
-
seen.delete(value);
|
|
394
|
+
return result;
|
|
713
395
|
}
|
|
714
|
-
};
|
|
715
|
-
const payloadKey = (step) => {
|
|
716
|
-
if (!isRecord(step.payload)) return `${defaultKey(step)}|${canonicalEventJson(step.payload)}`;
|
|
717
|
-
const { sessionId: _sessionId, ...payload } = step.payload;
|
|
718
|
-
return `${defaultKey(step)}|${canonicalEventJson(normalizeErrors(payload))}`;
|
|
719
|
-
};
|
|
720
|
-
const isTrace = (items) => items.length === 0 || items.some((item) => isRecord(item) && (item.type === "machine.transition" || item.type === "run.start" || item.type === "run.end"));
|
|
721
|
-
/**
|
|
722
|
-
* Finds the first step where a run departs from a reference, so a failure is
|
|
723
|
-
* blamed on the state it started in rather than on every state the run went
|
|
724
|
-
* through afterwards.
|
|
725
|
-
*
|
|
726
|
-
* - Against a reference **trace**, such as a passing run of the same
|
|
727
|
-
* scenario, steps are compared in order by `key`. The first step that
|
|
728
|
-
* differs, or the point where either trace ends early, is the divergence.
|
|
729
|
-
* - Against an expected **trajectory** (`TrajectoryItem[]`, as for
|
|
730
|
-
* {@link matchesTrajectory}), the run is laid out as its initial state, then
|
|
731
|
-
* each step's event and new state. The first expected item it never reaches
|
|
732
|
-
* is the divergence, blamed on the step after the last item it did reach. A
|
|
733
|
-
* run that goes on past the trajectory has not diverged.
|
|
734
|
-
*
|
|
735
|
-
* An empty reference is a trace with no steps, so any step diverges from it.
|
|
736
|
-
* A trace is one run's events, live or serialized. Steps the machine ignored
|
|
737
|
-
* changed nothing and are left out on both sides. Returns `undefined` when the
|
|
738
|
-
* run does not diverge.
|
|
739
|
-
*
|
|
740
|
-
* @example
|
|
741
|
-
* ```ts no-check
|
|
742
|
-
* const stats = foldTraces(traces, {
|
|
743
|
-
* failed,
|
|
744
|
-
* blame: (trace) => firstDivergence(trace, passingRunFor(trace))?.state,
|
|
745
|
-
* });
|
|
746
|
-
* ```
|
|
747
|
-
*/
|
|
748
|
-
function firstDivergence(actual, reference, options = {}) {
|
|
749
|
-
const steps = traceSteps(actual).filter((step) => !step.ignored);
|
|
750
|
-
return isTrace(reference) ? againstTrace(steps, traceSteps(reference).filter((step) => !step.ignored), options.key === "payload" ? payloadKey : options.key ?? defaultKey) : againstTrajectory(actual, steps, reference);
|
|
751
|
-
}
|
|
752
|
-
function diverged(steps, index, expected, initial = steps[0]?.fromValue) {
|
|
753
|
-
const step = steps[index];
|
|
754
396
|
return {
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
397
|
+
model(model) {
|
|
398
|
+
const base = model;
|
|
399
|
+
const stand = options.fallback?.model;
|
|
400
|
+
const wrapped = Object.create(base);
|
|
401
|
+
for (const method of [
|
|
402
|
+
"doGenerate",
|
|
403
|
+
"doStream",
|
|
404
|
+
"doDecide",
|
|
405
|
+
"doEvaluate"
|
|
406
|
+
]) {
|
|
407
|
+
if (typeof base[method] !== "function") continue;
|
|
408
|
+
const invoke = (target, callOptions) => target[method].call(target, callOptions);
|
|
409
|
+
wrapped[method] = (callOptions) => modelCall("ai-sdk", method, callOptions, method === "doStream", () => invoke(base, callOptions), stand && typeof stand[method] === "function" ? () => invoke(stand, callOptions) : void 0);
|
|
410
|
+
}
|
|
411
|
+
return wrapped;
|
|
412
|
+
},
|
|
413
|
+
openai(client) {
|
|
414
|
+
const wrapCreate = (resource, method) => new Proxy(resource, { get(target, key, receiver) {
|
|
415
|
+
if (key !== "create") return Reflect.get(target, key, receiver);
|
|
416
|
+
return (params, requestOptions) => {
|
|
417
|
+
let api;
|
|
418
|
+
const create = (resource) => () => {
|
|
419
|
+
api = Reflect.get(resource, "create").call(resource, params, requestOptions);
|
|
420
|
+
return Promise.resolve(api);
|
|
421
|
+
};
|
|
422
|
+
const stand = fallbackResource(method);
|
|
423
|
+
const promise = modelCall("openai", method, params, params?.stream === true, create(target), stand && create(stand));
|
|
424
|
+
return Object.assign(promise, { withResponse: async () => {
|
|
425
|
+
const data = await promise;
|
|
426
|
+
const meta = await api?.withResponse?.();
|
|
427
|
+
return {
|
|
428
|
+
data,
|
|
429
|
+
response: meta?.response ?? new Response(null),
|
|
430
|
+
request_id: meta?.request_id ?? null
|
|
431
|
+
};
|
|
432
|
+
} });
|
|
433
|
+
};
|
|
434
|
+
} });
|
|
435
|
+
const fallbackResource = (method) => {
|
|
436
|
+
let value = options.fallback?.openai;
|
|
437
|
+
for (const key of method.split(".").slice(0, -1)) value = value?.[key];
|
|
438
|
+
return value && typeof value === "object" ? value : void 0;
|
|
439
|
+
};
|
|
440
|
+
const wrapped = /* @__PURE__ */ new Map();
|
|
441
|
+
const wrap = (key, make) => {
|
|
442
|
+
if (!wrapped.has(key)) wrapped.set(key, make());
|
|
443
|
+
return wrapped.get(key);
|
|
444
|
+
};
|
|
445
|
+
return new Proxy(client, { get(target, key, receiver) {
|
|
446
|
+
const value = Reflect.get(target, key, receiver);
|
|
447
|
+
if (key === "responses" && value) return wrap(key, () => wrapCreate(value, "responses.create"));
|
|
448
|
+
if (key === "chat" && value) return wrap(key, () => new Proxy(value, { get(chat, chatKey, chatReceiver) {
|
|
449
|
+
const inner = Reflect.get(chat, chatKey, chatReceiver);
|
|
450
|
+
return chatKey === "completions" && inner ? wrap("chat.completions", () => wrapCreate(inner, "chat.completions.create")) : inner;
|
|
451
|
+
} }));
|
|
452
|
+
return value;
|
|
453
|
+
} });
|
|
454
|
+
},
|
|
455
|
+
tool: (name, fn) => tool(name, fn),
|
|
456
|
+
fn,
|
|
457
|
+
client(name, client, { methods, ...callOptions }) {
|
|
458
|
+
const paths = methods.map((method) => method.split("."));
|
|
459
|
+
const startsWith = (path, prefix) => prefix.every((key, index) => path[index] === key);
|
|
460
|
+
const wrapAt = (target, prefix) => {
|
|
461
|
+
const wrapped = /* @__PURE__ */ new Map();
|
|
462
|
+
return new Proxy(target, { get(object, key, receiver) {
|
|
463
|
+
if (typeof key !== "string") return Reflect.get(object, key, receiver);
|
|
464
|
+
const path = [...prefix, key];
|
|
465
|
+
const method = paths.some((p) => p.length === path.length && startsWith(p, path));
|
|
466
|
+
if (!method && !paths.some((p) => startsWith(p, path))) return Reflect.get(object, key, receiver);
|
|
467
|
+
if (!wrapped.has(key)) {
|
|
468
|
+
const value = Reflect.get(object, key, receiver);
|
|
469
|
+
wrapped.set(key, method ? fn(`${name}.${path.join(".")}`, (...args) => value.apply(object, args), callOptions) : wrapAt(value && typeof value === "object" ? value : {}, path));
|
|
470
|
+
}
|
|
471
|
+
return wrapped.get(key);
|
|
472
|
+
} });
|
|
473
|
+
};
|
|
474
|
+
return wrapAt(client, []);
|
|
475
|
+
},
|
|
476
|
+
tools(tools) {
|
|
477
|
+
return Object.fromEntries(Object.entries(tools).map(([name, definition]) => [name, definition.execute ? {
|
|
478
|
+
...definition,
|
|
479
|
+
execute: tool(name, definition.execute, (args) => args[0])
|
|
480
|
+
} : definition]));
|
|
481
|
+
},
|
|
482
|
+
output(value) {
|
|
483
|
+
effects.push({
|
|
484
|
+
kind: "output",
|
|
485
|
+
value: toJson(value)
|
|
486
|
+
});
|
|
487
|
+
},
|
|
488
|
+
get recording() {
|
|
489
|
+
return {
|
|
490
|
+
version: 1,
|
|
491
|
+
effects: toJson(effects)
|
|
492
|
+
};
|
|
493
|
+
}
|
|
760
494
|
};
|
|
761
495
|
}
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
const step = steps[index];
|
|
766
|
-
const want = reference[index];
|
|
767
|
-
if (step && want && key(step) === key(want)) continue;
|
|
768
|
-
return diverged(steps, index, want, steps[0]?.fromValue ?? reference[0]?.fromValue);
|
|
769
|
-
}
|
|
496
|
+
/** True when two model calls offer the same tool names and response format. */
|
|
497
|
+
function sameModelCallShape(expected, actual) {
|
|
498
|
+
return jsonEqual(modelCallShape(expected), modelCallShape(actual));
|
|
770
499
|
}
|
|
771
|
-
function
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
steps.forEach((step, index) => {
|
|
776
|
-
timeline.push(step.payload);
|
|
777
|
-
stepAt.push(index);
|
|
778
|
-
if (JSON.stringify(step.toValue) !== JSON.stringify(step.fromValue)) {
|
|
779
|
-
timeline.push({ value: step.toValue });
|
|
780
|
-
stepAt.push(index);
|
|
781
|
-
}
|
|
782
|
-
});
|
|
783
|
-
const match = matchesTrajectory(timeline, expected);
|
|
784
|
-
if (!match.firstMiss) return void 0;
|
|
785
|
-
const at = match.firstMiss.searchedFrom;
|
|
786
|
-
return diverged(steps, at < timeline.length ? stepAt[at] ?? steps.length : steps.length, match.firstMiss.expected, initial);
|
|
500
|
+
function modelCallsMatch(match, expected, actual) {
|
|
501
|
+
if (match === "none") return true;
|
|
502
|
+
if (typeof match === "function") return match(expected, actual);
|
|
503
|
+
return sameModelCallShape(expected, actual);
|
|
787
504
|
}
|
|
788
|
-
function
|
|
789
|
-
|
|
790
|
-
return steps[0]?.fromValue;
|
|
505
|
+
function shapeMessage(expected, actual) {
|
|
506
|
+
return `Model call #${expected.call} changed shape: ${JSON.stringify(modelCallShape(expected))} → ${JSON.stringify(modelCallShape(actual))}. A recorded answer only fits a call of the same shape.`;
|
|
791
507
|
}
|
|
792
|
-
//#endregion
|
|
793
|
-
//#region src/propose-from-traces.ts
|
|
794
508
|
/**
|
|
795
|
-
*
|
|
796
|
-
*
|
|
797
|
-
* a person to review, and nothing here changes a machine.
|
|
798
|
-
*
|
|
799
|
-
* Reads `machine.transition` events: a verdict is a transition, not ignored,
|
|
800
|
-
* out of `approvalState` on an approve or reject event. Rows come in the
|
|
801
|
-
* order keys were first seen.
|
|
802
|
-
*
|
|
803
|
-
* @example
|
|
804
|
-
* ```ts no-check
|
|
805
|
-
* const rows = proposeApprovalRules(traces, {
|
|
806
|
-
* approvalState: "awaitingApproval",
|
|
807
|
-
* approveEvents: ["APPROVE"],
|
|
808
|
-
* rejectEvents: ["REJECT"],
|
|
809
|
-
* key: ({ snapshot }) => `${snapshot.context.pending.tool}:${snapshot.context.pending.path}`,
|
|
810
|
-
* });
|
|
811
|
-
* ```
|
|
509
|
+
* Offered tool names and the response format, or a decision's questions and
|
|
510
|
+
* their options: what a recorded answer depends on.
|
|
812
511
|
*/
|
|
813
|
-
function
|
|
814
|
-
const
|
|
815
|
-
const
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
512
|
+
function modelCallShape(effect) {
|
|
513
|
+
const request = effect.request ?? {};
|
|
514
|
+
const questions = request["questions"];
|
|
515
|
+
if (questions && typeof questions === "object") return { questions: Object.entries(questions).map(([name, question]) => {
|
|
516
|
+
const criteria = question["criteria"];
|
|
517
|
+
if (question["type"] === "choice" && criteria && typeof criteria === "object") return [
|
|
518
|
+
name,
|
|
519
|
+
question["type"],
|
|
520
|
+
Object.keys(criteria).sort()
|
|
521
|
+
];
|
|
522
|
+
if (question["type"] === "score" && Array.isArray(criteria)) return [
|
|
523
|
+
name,
|
|
524
|
+
question["type"],
|
|
525
|
+
criteria.length
|
|
526
|
+
];
|
|
527
|
+
return [name, question["type"]];
|
|
528
|
+
}).sort((x, y) => String(x[0]).localeCompare(String(y[0]))) };
|
|
529
|
+
return {
|
|
530
|
+
tools: (Array.isArray(request["tools"]) ? request["tools"] : []).map((tool) => tool["name"] ?? tool["function"]?.["name"]).sort(),
|
|
531
|
+
format: request["responseFormat"]?.["type"] ?? request["response_format"]?.["type"] ?? (request["text"]?.["format"])?.["type"] ?? "text"
|
|
830
532
|
};
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
}
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
533
|
+
}
|
|
534
|
+
/**
|
|
535
|
+
* Puts each group of concurrent tool calls in a stable order, so parallel tool
|
|
536
|
+
* calls that finished in a different order still compare equal. A tool call
|
|
537
|
+
* that started after the previous one settled begins a new group, so
|
|
538
|
+
* sequential calls keep their order.
|
|
539
|
+
*/
|
|
540
|
+
function normalize$1(effects) {
|
|
541
|
+
const result = [];
|
|
542
|
+
let batch = [];
|
|
543
|
+
const flush = () => {
|
|
544
|
+
result.push(...batch.sort((a, b) => stableStringify([a.name, a.input]).localeCompare(stableStringify([b.name, b.input]))));
|
|
545
|
+
batch = [];
|
|
546
|
+
};
|
|
547
|
+
for (const effect of effects) {
|
|
548
|
+
if (effect.kind !== "tool" || !effect.concurrent) flush();
|
|
549
|
+
if (effect.kind === "tool") batch.push(effect);
|
|
550
|
+
else result.push(effect);
|
|
551
|
+
}
|
|
552
|
+
flush();
|
|
553
|
+
return result;
|
|
554
|
+
}
|
|
555
|
+
function describe(effect) {
|
|
556
|
+
switch (effect.kind) {
|
|
557
|
+
case "model": return `model call #${effect.call} (${effect.method})`;
|
|
558
|
+
case "tool": return `tool '${effect.name}' with ${JSON.stringify(effect.input)}`;
|
|
559
|
+
case "call": return `call #${effect.call} to '${effect.name}' with ${JSON.stringify(effect.args)}`;
|
|
560
|
+
case "output": return `output ${JSON.stringify(effect.value)}`;
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
/** Why two effects differ, or `undefined` when they are the same. */
|
|
564
|
+
function differ(left, right, options) {
|
|
565
|
+
if (left.kind !== right.kind) return {
|
|
566
|
+
reason: "kind",
|
|
567
|
+
message: `Expected ${describe(left)}; the port made ${describe(right)}.`
|
|
568
|
+
};
|
|
569
|
+
if (left.kind === "model" && right.kind === "model") {
|
|
570
|
+
if (right.translated && right.replayed === left.call) return void 0;
|
|
571
|
+
if (left.method !== right.method) return {
|
|
572
|
+
reason: "model-method",
|
|
573
|
+
message: `Model call #${left.call} used ${left.method}; the port used ${right.method}.`
|
|
574
|
+
};
|
|
575
|
+
if (!modelCallsMatch(options.modelCalls, left, right)) return {
|
|
576
|
+
reason: "model-call-shape",
|
|
577
|
+
message: shapeMessage(left, right)
|
|
578
|
+
};
|
|
579
|
+
if (!jsonEqual(errorIdentity$1(left.error), errorIdentity$1(right.error))) {
|
|
580
|
+
const outcome = (error) => error ? `failed with ${error.name}: ${error.message}` : "succeeded";
|
|
581
|
+
return {
|
|
582
|
+
reason: "model-error",
|
|
583
|
+
message: `Model call #${left.call} ${outcome(left.error)}; in the port it ${outcome(right.error)}.`
|
|
584
|
+
};
|
|
862
585
|
}
|
|
586
|
+
return;
|
|
863
587
|
}
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
588
|
+
if (left.kind === "tool" && right.kind === "tool") {
|
|
589
|
+
if (left.name !== right.name) return {
|
|
590
|
+
reason: "tool-name",
|
|
591
|
+
message: `Expected ${describe(left)}; the port made ${describe(right)}.`
|
|
592
|
+
};
|
|
593
|
+
if (!jsonEqual(left.input, right.input)) return {
|
|
594
|
+
reason: "tool-input",
|
|
595
|
+
message: `Expected ${describe(left)}; the port passed ${JSON.stringify(right.input)}.`
|
|
596
|
+
};
|
|
597
|
+
if (!jsonEqual(left.output, right.output) || !jsonEqual(errorIdentity$1(left.error), errorIdentity$1(right.error))) return {
|
|
598
|
+
reason: "tool-result",
|
|
599
|
+
message: `Tool '${left.name}' returned a different result.`
|
|
600
|
+
};
|
|
601
|
+
return;
|
|
867
602
|
}
|
|
868
|
-
|
|
603
|
+
if (left.kind === "call" && right.kind === "call") {
|
|
604
|
+
if (left.name !== right.name) return {
|
|
605
|
+
reason: "call-name",
|
|
606
|
+
message: `Expected ${describe(left)}; the port made ${describe(right)}.`
|
|
607
|
+
};
|
|
608
|
+
if (!jsonEqual(left.args, right.args)) return {
|
|
609
|
+
reason: "call-args",
|
|
610
|
+
message: `Expected ${describe(left)}; the port passed ${JSON.stringify(right.args)}.`
|
|
611
|
+
};
|
|
612
|
+
if (!jsonEqual(left.result, right.result) || !jsonEqual(left.stream, right.stream) || !jsonEqual(errorIdentity$1(left.error), errorIdentity$1(right.error))) return {
|
|
613
|
+
reason: "call-result",
|
|
614
|
+
message: `Call #${left.call} to '${left.name}' returned a different result.`
|
|
615
|
+
};
|
|
616
|
+
return;
|
|
617
|
+
}
|
|
618
|
+
if (left.kind === "output" && right.kind === "output" && !jsonEqual(left.value, right.value)) return {
|
|
619
|
+
reason: "output",
|
|
620
|
+
message: `Expected ${describe(left)}; the port returned ${JSON.stringify(right.value)}.`
|
|
621
|
+
};
|
|
869
622
|
}
|
|
870
623
|
/**
|
|
871
|
-
*
|
|
872
|
-
*
|
|
873
|
-
*
|
|
874
|
-
*
|
|
875
|
-
*
|
|
876
|
-
*
|
|
877
|
-
* @example
|
|
878
|
-
* ```ts no-check
|
|
879
|
-
* const [retries] = proposeCaps(traces, {
|
|
880
|
-
* caps: [{ name: "retries", current: 3, failureState: "gaveUp", valueFromRun: countRetries }],
|
|
881
|
-
* });
|
|
882
|
-
* ```
|
|
624
|
+
* Compares two recordings and reports every effect that differs. `expected`
|
|
625
|
+
* is the original's recording, `actual` the port's. Effects that are the same
|
|
626
|
+
* on both sides are lined up first (longest common subsequence), so one extra
|
|
627
|
+
* call does not shift every effect after it; between them, effects of the same
|
|
628
|
+
* kind pair up and report what changed, and the rest are missing or extra.
|
|
883
629
|
*/
|
|
884
|
-
function
|
|
885
|
-
const
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
630
|
+
function compareRecordings(expected, actual, options = {}) {
|
|
631
|
+
const select = (effects) => normalize$1(effects.flatMap((effect, index) => {
|
|
632
|
+
if (options.kinds && !options.kinds.includes(effect.kind)) return [];
|
|
633
|
+
const mapped = options.normalize ? options.normalize(effect, index, effects) : effect;
|
|
634
|
+
return mapped ? [mapped] : [];
|
|
635
|
+
}));
|
|
636
|
+
const a = select(expected.effects);
|
|
637
|
+
const b = select(actual.effects);
|
|
638
|
+
const same = a.map((left) => b.map((right) => !differ(left, right, options)));
|
|
639
|
+
const lcs = Array.from({ length: a.length + 1 }, () => Array.from({ length: b.length + 1 }, () => 0));
|
|
640
|
+
for (let i = a.length - 1; i >= 0; i--) for (let j = b.length - 1; j >= 0; j--) lcs[i][j] = same[i][j] ? lcs[i + 1][j + 1] + 1 : Math.max(lcs[i + 1][j], lcs[i][j + 1]);
|
|
641
|
+
const divergences = [];
|
|
642
|
+
let lefts = [];
|
|
643
|
+
let rights = [];
|
|
644
|
+
const flushGap = (at) => {
|
|
645
|
+
const unpairedRights = [...rights];
|
|
646
|
+
const changed = [];
|
|
647
|
+
for (const index of lefts) {
|
|
648
|
+
const left = a[index];
|
|
649
|
+
const k = unpairedRights.findIndex((j) => b[j].kind === left.kind);
|
|
650
|
+
if (k === -1) {
|
|
651
|
+
changed.push({
|
|
652
|
+
index,
|
|
653
|
+
reason: left.kind === "output" ? "output" : "missing-effect",
|
|
654
|
+
expected: left,
|
|
655
|
+
message: left.kind === "output" ? `The port never returned; the original returned ${JSON.stringify(left.value)}.` : `The port never made ${describe(left)}.`
|
|
656
|
+
});
|
|
657
|
+
continue;
|
|
658
|
+
}
|
|
659
|
+
const right = b[unpairedRights.splice(k, 1)[0]];
|
|
660
|
+
const difference = differ(left, right, options);
|
|
661
|
+
if (difference) changed.push({
|
|
662
|
+
index,
|
|
663
|
+
expected: left,
|
|
664
|
+
actual: right,
|
|
665
|
+
...difference
|
|
666
|
+
});
|
|
897
667
|
}
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
p95,
|
|
907
|
-
proposal
|
|
908
|
-
};
|
|
909
|
-
});
|
|
910
|
-
}
|
|
911
|
-
function hitCap(trace, cap) {
|
|
912
|
-
for (const event of trace) {
|
|
913
|
-
if (cap.failureState !== void 0 && event.type === "machine.transition" && statePaths(valueOf(event.snapshot)).some((path) => path === cap.failureState || path.startsWith(`${cap.failureState}.`))) return true;
|
|
914
|
-
if (cap.failureOutput && event.type === "run.end" && event.status === "done") {
|
|
915
|
-
if (cap.failureOutput(event.output)) return true;
|
|
668
|
+
for (const j of unpairedRights) {
|
|
669
|
+
const extra = b[j];
|
|
670
|
+
divergences.push({
|
|
671
|
+
index: lefts[0] ?? at,
|
|
672
|
+
reason: extra.kind === "output" ? "output" : "extra-effect",
|
|
673
|
+
actual: extra,
|
|
674
|
+
message: extra.kind === "output" ? `The original never returned; the port returned ${JSON.stringify(extra.value)}.` : `The port made ${describe(extra)}, which the original never made.`
|
|
675
|
+
});
|
|
916
676
|
}
|
|
917
|
-
|
|
918
|
-
|
|
677
|
+
divergences.push(...changed);
|
|
678
|
+
lefts = [];
|
|
679
|
+
rights = [];
|
|
680
|
+
};
|
|
681
|
+
let i = 0;
|
|
682
|
+
let j = 0;
|
|
683
|
+
while (i < a.length || j < b.length) if (i < a.length && j < b.length && same[i][j]) {
|
|
684
|
+
flushGap(i);
|
|
685
|
+
i++;
|
|
686
|
+
j++;
|
|
687
|
+
} else if (j < b.length && (i === a.length || lcs[i][j + 1] >= lcs[i + 1][j])) rights.push(j++);
|
|
688
|
+
else lefts.push(i++);
|
|
689
|
+
flushGap(i);
|
|
690
|
+
divergences.sort((x, y) => x.index - y.index);
|
|
691
|
+
return divergences.length ? {
|
|
692
|
+
equivalent: false,
|
|
693
|
+
divergence: divergences[0],
|
|
694
|
+
divergences
|
|
695
|
+
} : {
|
|
696
|
+
equivalent: true,
|
|
697
|
+
divergences
|
|
698
|
+
};
|
|
919
699
|
}
|
|
920
700
|
//#endregion
|
|
921
|
-
//#region src/
|
|
701
|
+
//#region src/crash-recovery.ts
|
|
702
|
+
var CrashError = class extends Error {
|
|
703
|
+
constructor() {
|
|
704
|
+
super("checkCrashRecovery: simulated crash");
|
|
705
|
+
this.name = "CrashError";
|
|
706
|
+
}
|
|
707
|
+
};
|
|
708
|
+
const DONE_ACTOR = "xstate.done.actor";
|
|
709
|
+
const ERROR_ACTOR = "xstate.error.actor";
|
|
922
710
|
/**
|
|
923
|
-
*
|
|
924
|
-
*
|
|
925
|
-
*
|
|
926
|
-
*
|
|
711
|
+
* Runs `machine` once uninterrupted, then once per crash point: killed after
|
|
712
|
+
* each log append and while each effect is in flight, then resumed from the
|
|
713
|
+
* store's log by a fresh runtime. Each resumed run is checked against the
|
|
714
|
+
* uninterrupted one:
|
|
927
715
|
*
|
|
928
|
-
*
|
|
929
|
-
*
|
|
930
|
-
*
|
|
931
|
-
*
|
|
716
|
+
* - An effect whose completion was durable before the crash never runs again.
|
|
717
|
+
* Only an effect in flight at the crash may (execution is at-least-once).
|
|
718
|
+
* - Every effect of the uninterrupted run runs, and no other: a re-executed
|
|
719
|
+
* model call keeps its `callKey` and its request.
|
|
720
|
+
* - The run ends with the same status, output, state value, and context.
|
|
721
|
+
* - The resumed log replays to that state.
|
|
932
722
|
*
|
|
933
|
-
*
|
|
934
|
-
*
|
|
935
|
-
*
|
|
936
|
-
* for (const { request } of rows) console.log((await decide(request)).event);
|
|
937
|
-
* ```
|
|
723
|
+
* Model calls the uninterrupted run made are answered from its recording.
|
|
724
|
+
* Plain actors run live, so they must give the same result however many times
|
|
725
|
+
* they run (pass `reset` for fakes that keep state).
|
|
938
726
|
*/
|
|
939
|
-
function
|
|
940
|
-
const
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
727
|
+
async function checkCrashRecovery(machine, options = {}) {
|
|
728
|
+
const timeout = options.timeout ?? 5e3;
|
|
729
|
+
options.reset?.();
|
|
730
|
+
const baselineWorld = newWorld(() => false);
|
|
731
|
+
const baselineStore = observedStore((options.store ?? createInMemoryEventLogStore)(), baselineWorld, () => false);
|
|
732
|
+
const baseline = await withTimeout(runTurns(machine, options, baselineWorld, baselineStore.store), timeout);
|
|
733
|
+
if (baseline === TIMED_OUT) throw new Error(`checkCrashRecovery: the uninterrupted run did not settle within ${timeout}ms`);
|
|
734
|
+
if (baselineWorld.unsent.length > 0) {
|
|
735
|
+
const keys = baselineWorld.unsent.map((index) => {
|
|
736
|
+
const { during, occurrence = 0 } = options.sendDuring[index];
|
|
737
|
+
return occurrence === 0 ? `'${during}'` : `'${during}' (start ${occurrence})`;
|
|
948
738
|
});
|
|
739
|
+
throw new Error(`checkCrashRecovery: sendDuring names effects the uninterrupted run never started: ${keys.join(", ")}`);
|
|
949
740
|
}
|
|
950
|
-
|
|
741
|
+
if (baselineWorld.undelivered > 0) throw new Error(`checkCrashRecovery: the uninterrupted run settled '${baseline.status}' with ${baselineWorld.undelivered} of \`events\` undelivered`);
|
|
742
|
+
const baselineEntries = await baselineStore.store.read(THREAD);
|
|
743
|
+
const baselineCounts = countBy(baselineWorld.effects);
|
|
744
|
+
const expected = outcomeOf(baseline);
|
|
745
|
+
const points = [];
|
|
746
|
+
for (const entries of /* @__PURE__ */ new Set([0, ...baselineStore.appends])) points.push({
|
|
747
|
+
kind: "log",
|
|
748
|
+
entries
|
|
749
|
+
});
|
|
750
|
+
const seen = /* @__PURE__ */ new Map();
|
|
751
|
+
baselineWorld.effects.forEach((effect, index) => {
|
|
752
|
+
const occurrence = seen.get(effect.key) ?? 0;
|
|
753
|
+
seen.set(effect.key, occurrence + 1);
|
|
754
|
+
points.push({
|
|
755
|
+
kind: "effect",
|
|
756
|
+
effect: {
|
|
757
|
+
kind: effect.kind,
|
|
758
|
+
key: effect.key
|
|
759
|
+
},
|
|
760
|
+
index,
|
|
761
|
+
occurrence
|
|
762
|
+
});
|
|
763
|
+
});
|
|
764
|
+
const failures = [];
|
|
765
|
+
const unreached = [];
|
|
766
|
+
const ctx = {
|
|
767
|
+
machine,
|
|
768
|
+
options,
|
|
769
|
+
timeout,
|
|
770
|
+
baselineCounts,
|
|
771
|
+
recorded: baselineWorld.answers,
|
|
772
|
+
expected,
|
|
773
|
+
sentAt: new Map([...baselineWorld.sentEntries].map(([index, entry]) => [index, entry.index]))
|
|
774
|
+
};
|
|
775
|
+
for (const point of points) {
|
|
776
|
+
let found = await crashAt(point, ctx);
|
|
777
|
+
for (let retry = 1; !Array.isArray(found) && retry < UNREACHED_RETRIES; retry++) found = await crashAt(point, ctx);
|
|
778
|
+
if (Array.isArray(found)) failures.push(...found);
|
|
779
|
+
else unreached.push({
|
|
780
|
+
point,
|
|
781
|
+
reason: found.unreached
|
|
782
|
+
});
|
|
783
|
+
}
|
|
784
|
+
return {
|
|
785
|
+
ok: failures.length === 0 && unreached.length === 0,
|
|
786
|
+
baseline: {
|
|
787
|
+
status: baseline.status,
|
|
788
|
+
...baseline.status === "done" ? { output: baseline.output } : {},
|
|
789
|
+
entries: baselineEntries.length,
|
|
790
|
+
effects: baselineWorld.effects.map(({ kind, key }) => ({
|
|
791
|
+
kind,
|
|
792
|
+
key
|
|
793
|
+
}))
|
|
794
|
+
},
|
|
795
|
+
points,
|
|
796
|
+
unreached,
|
|
797
|
+
failures
|
|
798
|
+
};
|
|
951
799
|
}
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
const
|
|
972
|
-
|
|
973
|
-
const
|
|
974
|
-
const
|
|
975
|
-
if (
|
|
976
|
-
|
|
977
|
-
|
|
800
|
+
const THREAD = "crash-recovery";
|
|
801
|
+
const UNREACHED_RETRIES = 3;
|
|
802
|
+
const SKIPPED_TURN = "(no timer pending)";
|
|
803
|
+
const TIMED_OUT = Symbol("timed out");
|
|
804
|
+
async function crashAt(point, ctx) {
|
|
805
|
+
const { machine, options, timeout } = ctx;
|
|
806
|
+
const fail = (problem, message, key) => ({
|
|
807
|
+
point,
|
|
808
|
+
problem,
|
|
809
|
+
message,
|
|
810
|
+
...key !== void 0 ? { key } : {}
|
|
811
|
+
});
|
|
812
|
+
options.reset?.();
|
|
813
|
+
let backing;
|
|
814
|
+
try {
|
|
815
|
+
backing = (options.store ?? createInMemoryEventLogStore)();
|
|
816
|
+
} catch (error) {
|
|
817
|
+
return [fail("threw", `the store factory threw: ${String(error)}`)];
|
|
818
|
+
}
|
|
819
|
+
const controller = new AbortController();
|
|
820
|
+
let runs = 0;
|
|
821
|
+
const crashed = newWorld((effect) => point.kind === "effect" && effect.key === point.effect.key && runs++ === point.occurrence, ctx.recorded);
|
|
822
|
+
const crash = () => {
|
|
823
|
+
if (crashed.crashed) return;
|
|
824
|
+
crashed.crashed = true;
|
|
825
|
+
controller.abort(new CrashError());
|
|
978
826
|
};
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
827
|
+
let missed = point.kind === "log" && point.entries === 0 ? void 0 : point.kind === "log" ? "the log never reached that length" : "the run never started that effect";
|
|
828
|
+
const onEffect = crashed.onEffect;
|
|
829
|
+
crashed.onEffect = (effect) => {
|
|
830
|
+
const dies = onEffect(effect);
|
|
831
|
+
if (dies) {
|
|
832
|
+
missed = void 0;
|
|
833
|
+
queueMicrotask(crash);
|
|
834
|
+
}
|
|
835
|
+
return dies;
|
|
983
836
|
};
|
|
837
|
+
const dying = observedStore(backing, crashed, (accepted) => {
|
|
838
|
+
if (point.kind !== "log" || accepted < point.entries) return false;
|
|
839
|
+
missed = accepted === point.entries ? void 0 : `one write took the log to ${accepted} entries`;
|
|
840
|
+
crash();
|
|
841
|
+
return true;
|
|
842
|
+
});
|
|
843
|
+
if (point.kind === "log" && point.entries === 0) dying.freeze();
|
|
844
|
+
let died;
|
|
845
|
+
try {
|
|
846
|
+
died = await withTimeout(runTurns(machine, options, crashed, dying.store, void 0, controller.signal), timeout);
|
|
847
|
+
} catch (error) {
|
|
848
|
+
if (!(error instanceof CrashError)) {
|
|
849
|
+
crash();
|
|
850
|
+
return [fail("threw", `the crashed run threw: ${String(error)}`)];
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
crash();
|
|
854
|
+
if (died === TIMED_OUT) return [fail("stuck", `the run did not stop within ${timeout}ms after the crash`)];
|
|
855
|
+
if (missed !== void 0) return { unreached: missed };
|
|
856
|
+
const resumed = newWorld(() => false, ctx.recorded);
|
|
857
|
+
const resumeController = new AbortController();
|
|
858
|
+
let result;
|
|
859
|
+
try {
|
|
860
|
+
result = await withTimeout(runTurns(machine, options, resumed, backing, {
|
|
861
|
+
crashed,
|
|
862
|
+
sentAt: ctx.sentAt
|
|
863
|
+
}, resumeController.signal), timeout);
|
|
864
|
+
} catch (error) {
|
|
865
|
+
resumed.crashed = true;
|
|
866
|
+
resumeController.abort(new CrashError());
|
|
867
|
+
return [fail("threw", `the resume threw: ${String(error)}`)];
|
|
868
|
+
}
|
|
869
|
+
if (result === TIMED_OUT) {
|
|
870
|
+
resumed.crashed = true;
|
|
871
|
+
resumeController.abort(new CrashError());
|
|
872
|
+
return [fail("stuck", `the resume did not settle within ${timeout}ms`)];
|
|
873
|
+
}
|
|
874
|
+
const failures = [];
|
|
875
|
+
for (const index of resumed.unsentAfterCrash) {
|
|
876
|
+
const { during, occurrence = 0 } = options.sendDuring[index];
|
|
877
|
+
failures.push(fail("undelivered", `the sendDuring event for '${during}' (start ${occurrence}) was lost in the crash and never sent again`));
|
|
878
|
+
}
|
|
879
|
+
const total = countBy([...crashed.effects, ...resumed.effects]);
|
|
880
|
+
const inFlight = countBy(crashed.effects.filter((effect) => !effect.completed));
|
|
881
|
+
for (const [key, count] of total) {
|
|
882
|
+
const expected = ctx.baselineCounts.get(key) ?? 0;
|
|
883
|
+
const allowed = expected + (inFlight.get(key) ?? 0);
|
|
884
|
+
if (expected === 0) failures.push(fail("unexpected", `'${key}' ran ${count}x; the uninterrupted run never made it`, key));
|
|
885
|
+
else if (count > allowed) failures.push(fail("reran", `'${key}' ran ${count}x, at most ${allowed} expected: its completion was durable before the crash`, key));
|
|
886
|
+
}
|
|
887
|
+
for (const [key, expected] of ctx.baselineCounts) {
|
|
888
|
+
const count = total.get(key) ?? 0;
|
|
889
|
+
if (count < expected) failures.push(fail("skipped", `'${key}' ran ${count}x, ${expected} expected`, key));
|
|
890
|
+
}
|
|
891
|
+
const crashedRequests = new Map(crashed.effects.filter((effect) => effect.kind === "model").map((effect) => [effect.key, effect.request]));
|
|
892
|
+
for (const effect of resumed.effects) {
|
|
893
|
+
if (effect.kind !== "model" || !crashedRequests.has(effect.key)) continue;
|
|
894
|
+
const before = crashedRequests.get(effect.key);
|
|
895
|
+
if (before === void 0 || effect.request === void 0) continue;
|
|
896
|
+
if (before !== effect.request) failures.push(fail("request-changed", `'${effect.key}' re-executed with a different request`, effect.key));
|
|
897
|
+
}
|
|
898
|
+
const actual = outcomeOf(result);
|
|
899
|
+
if (actual !== ctx.expected) failures.push(fail("outcome", `resumed run ended ${actual}, expected ${ctx.expected}`));
|
|
900
|
+
const log = await backing.read(THREAD);
|
|
901
|
+
try {
|
|
902
|
+
const replayed = replay(options.actors !== void 0 ? machine.provide({ actors: options.actors }) : machine, log);
|
|
903
|
+
const fromLog = stableStringify(toJsonValue({
|
|
904
|
+
value: replayed.snapshot.value,
|
|
905
|
+
context: replayed.snapshot.context
|
|
906
|
+
}));
|
|
907
|
+
const live = stableStringify(toJsonValue({
|
|
908
|
+
value: result.snapshot.value,
|
|
909
|
+
context: result.snapshot.context
|
|
910
|
+
}));
|
|
911
|
+
if (fromLog !== live) failures.push(fail("replay", `the log replays to ${fromLog}, the resumed run ended at ${live}`));
|
|
912
|
+
} catch (error) {
|
|
913
|
+
failures.push(fail("replay", `the log does not replay: ${String(error)}`));
|
|
914
|
+
}
|
|
915
|
+
return failures;
|
|
984
916
|
}
|
|
985
917
|
/**
|
|
986
|
-
* Runs
|
|
987
|
-
*
|
|
988
|
-
*
|
|
989
|
-
*
|
|
990
|
-
* With a live model, one run per scenario says little: τ-bench found pass^k
|
|
991
|
-
* falls well below the single-run rate. Report pass^k next to it.
|
|
992
|
-
*
|
|
993
|
-
* @example
|
|
994
|
-
* ```ts no-check
|
|
995
|
-
* const report = await repeatRuns({
|
|
996
|
-
* scenarios,
|
|
997
|
-
* k: 5,
|
|
998
|
-
* run: async (scenario) => {
|
|
999
|
-
* const trace: AgentTraceEvent[] = [];
|
|
1000
|
-
* const result = await runToQuiescence(
|
|
1001
|
-
* createAgentRuntime(machine, { executors, onTrace: (event) => trace.push(event) }),
|
|
1002
|
-
* { input: scenario.input },
|
|
1003
|
-
* );
|
|
1004
|
-
* return { trace, outcome: String(result.snapshot.value) };
|
|
1005
|
-
* },
|
|
1006
|
-
* passed: (scenario, { outcome }) => outcome === scenario.expected,
|
|
1007
|
-
* });
|
|
1008
|
-
* report.passHat(5);
|
|
1009
|
-
* ```
|
|
918
|
+
* Runs the turns: start from `input`, then one host event per idle settle,
|
|
919
|
+
* with `sendDuring` events sent as their effects start and host timers fired
|
|
920
|
+
* when the run is idle. Resuming, what the crashed run's log holds is skipped.
|
|
1010
921
|
*/
|
|
1011
|
-
async function
|
|
1012
|
-
|
|
1013
|
-
const
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
922
|
+
async function runTurns(machine, options, world, store, resuming, signal) {
|
|
923
|
+
const tracked = trackActors(machine, world);
|
|
924
|
+
const executors = trackExecutors(options.executors ?? {}, world);
|
|
925
|
+
const actors = options.actors !== void 0 ? Object.fromEntries(Object.entries(options.actors).map(([key, logic]) => [key, isStateMachineLogic(logic) ? trackActors(logic, world) : trackLogic(logic, world)])) : void 0;
|
|
926
|
+
const before = resuming?.crashed;
|
|
927
|
+
const log = before !== void 0 ? await store.read(THREAD) : [];
|
|
928
|
+
const durable = new Set(log.map((entry) => entry.id));
|
|
929
|
+
const events = options.events ?? [];
|
|
930
|
+
let next = 0;
|
|
931
|
+
if (before !== void 0) while (next < before.hostEntries.length && (before.hostEntries[next] === SKIPPED_TURN || durable.has(before.hostEntries[next]))) next++;
|
|
932
|
+
const sends = options.sendDuring ?? [];
|
|
933
|
+
const unsent = /* @__PURE__ */ new Set();
|
|
934
|
+
const redeliver = [];
|
|
935
|
+
const sendAt = /* @__PURE__ */ new Map();
|
|
936
|
+
const waitsFor = /* @__PURE__ */ new Map();
|
|
937
|
+
sends.forEach(({ during, occurrence = 0 }, index) => {
|
|
938
|
+
if (resuming === void 0) {
|
|
939
|
+
waitsFor.set(index, occurrence);
|
|
940
|
+
unsent.add(index);
|
|
941
|
+
return;
|
|
1028
942
|
}
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
943
|
+
const entry = before.sentEntries.get(index);
|
|
944
|
+
if (entry !== void 0 && durable.has(entry.id)) return;
|
|
945
|
+
const starts = before.effects.filter((effect) => effect.key === during);
|
|
946
|
+
if (starts[occurrence]?.completed !== true) {
|
|
947
|
+
waitsFor.set(index, occurrence - starts.filter((effect) => effect.completed).length);
|
|
948
|
+
unsent.add(index);
|
|
949
|
+
return;
|
|
950
|
+
}
|
|
951
|
+
const at = resuming.sentAt.get(index);
|
|
952
|
+
if (at === void 0) return;
|
|
953
|
+
if (at <= log.length) redeliver.push(index);
|
|
954
|
+
else sendAt.set(index, at);
|
|
955
|
+
});
|
|
956
|
+
const awaitingSends = [];
|
|
957
|
+
let live;
|
|
958
|
+
const send = (index) => {
|
|
959
|
+
const { event } = sends[index];
|
|
960
|
+
awaitingSends.push({
|
|
961
|
+
index,
|
|
962
|
+
json: canonicalEventJson(event)
|
|
963
|
+
});
|
|
964
|
+
live?.send(event);
|
|
965
|
+
};
|
|
966
|
+
const startsSeen = /* @__PURE__ */ new Map();
|
|
967
|
+
const onEffect = world.onEffect;
|
|
968
|
+
world.onEffect = (effect) => {
|
|
969
|
+
const nth = startsSeen.get(effect.key) ?? 0;
|
|
970
|
+
startsSeen.set(effect.key, nth + 1);
|
|
971
|
+
for (const index of unsent) {
|
|
972
|
+
if (sends[index].during !== effect.key || waitsFor.get(index) !== nth) continue;
|
|
973
|
+
unsent.delete(index);
|
|
974
|
+
send(index);
|
|
975
|
+
}
|
|
976
|
+
return onEffect(effect);
|
|
977
|
+
};
|
|
978
|
+
const pendingTimers = /* @__PURE__ */ new Map();
|
|
979
|
+
world.clock = {
|
|
980
|
+
now: before?.clock.now ?? 0,
|
|
981
|
+
due: pendingTimers
|
|
982
|
+
};
|
|
983
|
+
const dueBefore = new Map(before?.clock.due ?? []);
|
|
984
|
+
const timers = options.timers === "host" ? {
|
|
985
|
+
schedule: ({ id, delay }) => {
|
|
986
|
+
if (!pendingTimers.has(id)) {
|
|
987
|
+
const restored = dueBefore.get(id);
|
|
988
|
+
dueBefore.delete(id);
|
|
989
|
+
pendingTimers.set(id, restored ?? world.clock.now + delay);
|
|
990
|
+
}
|
|
991
|
+
},
|
|
992
|
+
cancel: (id) => {
|
|
993
|
+
pendingTimers.delete(id);
|
|
994
|
+
}
|
|
995
|
+
} : void 0;
|
|
996
|
+
/** `track`: the turn delivers one of `events` (a host event, or a timer's turn), whose entry is recorded. */
|
|
997
|
+
const turn = (event, track) => {
|
|
998
|
+
let awaiting = event !== void 0 && track === "host" ? canonicalEventJson(event) : void 0;
|
|
999
|
+
let awaitingTimer = track === "timer";
|
|
1000
|
+
live = createAgentRuntime(tracked, {
|
|
1001
|
+
...options.runtime,
|
|
1002
|
+
onEvent: (entry) => {
|
|
1003
|
+
options.runtime?.onEvent?.(entry);
|
|
1004
|
+
for (const [index, at] of sendAt) {
|
|
1005
|
+
if (entry.index + 1 < at) continue;
|
|
1006
|
+
sendAt.delete(index);
|
|
1007
|
+
send(index);
|
|
1008
|
+
}
|
|
1009
|
+
if (entry.actorPath !== void 0) return;
|
|
1010
|
+
const json = canonicalEventJson(entry.event);
|
|
1011
|
+
if (awaiting !== void 0 && json === awaiting) {
|
|
1012
|
+
world.hostEntries.push(entry.id);
|
|
1013
|
+
awaiting = void 0;
|
|
1014
|
+
return;
|
|
1015
|
+
}
|
|
1016
|
+
if (awaitingTimer && entry.event.type === "xstate.timer") {
|
|
1017
|
+
world.hostEntries.push(entry.id);
|
|
1018
|
+
awaitingTimer = false;
|
|
1019
|
+
return;
|
|
1020
|
+
}
|
|
1021
|
+
const sent = awaitingSends.findIndex((item) => item.json === json);
|
|
1022
|
+
if (sent !== -1) {
|
|
1023
|
+
world.sentEntries.set(awaitingSends[sent].index, {
|
|
1024
|
+
id: entry.id,
|
|
1025
|
+
index: entry.index
|
|
1026
|
+
});
|
|
1027
|
+
awaitingSends.splice(sent, 1);
|
|
1028
|
+
}
|
|
1029
|
+
},
|
|
1030
|
+
executors,
|
|
1031
|
+
...actors !== void 0 ? { actors } : {},
|
|
1032
|
+
store,
|
|
1033
|
+
threadId: THREAD,
|
|
1034
|
+
...timers !== void 0 ? { timers } : {},
|
|
1035
|
+
...signal !== void 0 ? { signal } : {}
|
|
1033
1036
|
});
|
|
1037
|
+
for (const index of redeliver.splice(0)) send(index);
|
|
1038
|
+
return runToQuiescence(live, {
|
|
1039
|
+
...options.input !== void 0 ? { input: options.input } : {},
|
|
1040
|
+
...event !== void 0 ? { event } : {}
|
|
1041
|
+
});
|
|
1042
|
+
};
|
|
1043
|
+
const soonestTimer = () => {
|
|
1044
|
+
let soonest;
|
|
1045
|
+
for (const timer of pendingTimers) if (soonest === void 0 || timer[1] < soonest[1]) soonest = timer;
|
|
1046
|
+
if (soonest === void 0) return void 0;
|
|
1047
|
+
pendingTimers.delete(soonest[0]);
|
|
1048
|
+
world.clock.now = soonest[1];
|
|
1049
|
+
return {
|
|
1050
|
+
type: "xstate.timer",
|
|
1051
|
+
id: soonest[0]
|
|
1052
|
+
};
|
|
1053
|
+
};
|
|
1054
|
+
let result = await turn();
|
|
1055
|
+
while (result.status === "idle" && !world.crashed) {
|
|
1056
|
+
let event;
|
|
1057
|
+
let host = true;
|
|
1058
|
+
let marked = false;
|
|
1059
|
+
if (next < events.length) {
|
|
1060
|
+
event = events[next++];
|
|
1061
|
+
if (event.type === "xstate.timer" && !("id" in event)) {
|
|
1062
|
+
event = soonestTimer();
|
|
1063
|
+
host = false;
|
|
1064
|
+
marked = true;
|
|
1065
|
+
if (event === void 0) {
|
|
1066
|
+
world.hostEntries.push(SKIPPED_TURN);
|
|
1067
|
+
continue;
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
1070
|
+
} else {
|
|
1071
|
+
event = soonestTimer();
|
|
1072
|
+
host = false;
|
|
1073
|
+
}
|
|
1074
|
+
if (event === void 0) break;
|
|
1075
|
+
result = await turn(event, host ? "host" : marked ? "timer" : void 0);
|
|
1076
|
+
await result.drain?.();
|
|
1034
1077
|
}
|
|
1035
|
-
|
|
1078
|
+
await result.drain?.();
|
|
1079
|
+
world.unsent = [...unsent];
|
|
1080
|
+
world.unsentAfterCrash = resuming !== void 0 ? [...unsent, ...sendAt.keys()] : [];
|
|
1081
|
+
world.undelivered = events.length - next;
|
|
1082
|
+
return result;
|
|
1036
1083
|
}
|
|
1037
|
-
function
|
|
1038
|
-
const passes = group.runs.filter((run) => run.passed).length;
|
|
1039
|
-
const traces = group.runs.flatMap((run) => run.trace ? [run.trace] : []);
|
|
1040
|
-
const paths = new Set(traces.map((trace) => traceSteps(trace).filter((step) => !step.ignored).map((step) => `${step.from.join(",")}|${step.event}|${step.to.join(",")}`).join(" ")));
|
|
1041
|
-
const reference = traces.find((trace) => trace.some((event) => event.type === "machine.transition"));
|
|
1042
|
-
let earliest;
|
|
1043
|
-
for (const trace of reference ? traces : []) {
|
|
1044
|
-
if (trace === reference) continue;
|
|
1045
|
-
const divergence = firstDivergence(trace, reference);
|
|
1046
|
-
if (divergence && (!earliest || divergence.index < earliest.index)) earliest = divergence;
|
|
1047
|
-
}
|
|
1084
|
+
function newWorld(crashesAt, recorded) {
|
|
1048
1085
|
return {
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1086
|
+
effects: [],
|
|
1087
|
+
hostEntries: [],
|
|
1088
|
+
sentEntries: /* @__PURE__ */ new Map(),
|
|
1089
|
+
unsent: [],
|
|
1090
|
+
unsentAfterCrash: [],
|
|
1091
|
+
undelivered: 0,
|
|
1092
|
+
clock: {
|
|
1093
|
+
now: 0,
|
|
1094
|
+
due: /* @__PURE__ */ new Map()
|
|
1095
|
+
},
|
|
1096
|
+
answers: /* @__PURE__ */ new Map(),
|
|
1097
|
+
...recorded !== void 0 ? { recorded } : {},
|
|
1098
|
+
crashed: false,
|
|
1099
|
+
onEffect: (effect) => crashesAt(effect)
|
|
1055
1100
|
};
|
|
1056
1101
|
}
|
|
1057
|
-
/** C(passes, j) / C(runs, j), as a product so large counts don't overflow. */
|
|
1058
|
-
function allPass(passes, runs, j) {
|
|
1059
|
-
let chance = 1;
|
|
1060
|
-
for (let i = 0; i < j; i++) chance *= Math.max(0, passes - i) / (runs - i);
|
|
1061
|
-
return chance;
|
|
1062
|
-
}
|
|
1063
|
-
//#endregion
|
|
1064
|
-
//#region src/judge-agreement.ts
|
|
1065
1102
|
/**
|
|
1066
|
-
*
|
|
1067
|
-
*
|
|
1068
|
-
*
|
|
1069
|
-
* (Husain/Shankar), and a correction for the judge's bias on unlabeled rows.
|
|
1070
|
-
* Pure.
|
|
1071
|
-
*
|
|
1072
|
-
* Build rows from confirmed labels only; draft labels gate nothing. Group by
|
|
1073
|
-
* decision state or request name so each narrow judge is aligned on its own.
|
|
1074
|
-
* A row with no group key counts in the totals and in no group. Run it on a
|
|
1075
|
-
* dev split and report the held-out split separately.
|
|
1076
|
-
*
|
|
1077
|
-
* Throws `AgentError` with code `'empty-agreement-rows'` when `rows` is empty.
|
|
1078
|
-
*
|
|
1079
|
-
* @example
|
|
1080
|
-
* ```ts no-check
|
|
1081
|
-
* const report = judgeAgreement(
|
|
1082
|
-
* labeled.map((s) => ({ id: s.id, human: s.label, judge: verdicts[s.id], group: s.request })),
|
|
1083
|
-
* { positive: 'fail' },
|
|
1084
|
-
* );
|
|
1085
|
-
* report.binary; // { tpr: 0.9, tnr: 0.8 }
|
|
1086
|
-
* report.correct?.(0.3); // estimated true failure rate in production
|
|
1087
|
-
* ```
|
|
1103
|
+
* `store`, recording each append's resulting length and marking the effects
|
|
1104
|
+
* whose completion an accepted entry carries. `afterAppend` returns true to
|
|
1105
|
+
* freeze it: every later append rejects, as if the process died.
|
|
1088
1106
|
*/
|
|
1089
|
-
function
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1107
|
+
function observedStore(store, world, afterAppend) {
|
|
1108
|
+
const appends = [];
|
|
1109
|
+
let frozen = false;
|
|
1110
|
+
return {
|
|
1111
|
+
store: {
|
|
1112
|
+
...store,
|
|
1113
|
+
read: (threadId, options) => store.read(threadId, options),
|
|
1114
|
+
length: (threadId) => store.length(threadId),
|
|
1115
|
+
fork: (input) => store.fork(input),
|
|
1116
|
+
async append(input) {
|
|
1117
|
+
if (frozen || world.crashed) throw new CrashError();
|
|
1118
|
+
await store.append(input);
|
|
1119
|
+
for (const entry of input.entries) markCompleted(entry, world.effects);
|
|
1120
|
+
const accepted = input.expectedIndex + input.entries.length;
|
|
1121
|
+
appends.push(accepted);
|
|
1122
|
+
if (afterAppend(accepted)) frozen = true;
|
|
1123
|
+
}
|
|
1124
|
+
},
|
|
1125
|
+
appends,
|
|
1126
|
+
freeze: () => {
|
|
1127
|
+
frozen = true;
|
|
1101
1128
|
}
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1129
|
+
};
|
|
1130
|
+
}
|
|
1131
|
+
/**
|
|
1132
|
+
* Marks the call an accepted entry completes: a decision's chosen event names
|
|
1133
|
+
* its call (`site#n`); a completion completes the oldest open effect at its
|
|
1134
|
+
* invoke site, with the other attempts of the same model call.
|
|
1135
|
+
*/
|
|
1136
|
+
function markCompleted(entry, effects) {
|
|
1137
|
+
if (entry.decidedBy !== void 0) {
|
|
1138
|
+
const call = entry.decidedBy;
|
|
1139
|
+
for (const effect of effects) if (effect.key === call || effect.key.startsWith(`${call}.`)) effect.completed = true;
|
|
1140
|
+
return;
|
|
1105
1141
|
}
|
|
1106
|
-
return
|
|
1142
|
+
if (entry.event.type !== DONE_ACTOR && entry.event.type !== ERROR_ACTOR) return;
|
|
1143
|
+
const actorId = entry.event.actorId;
|
|
1144
|
+
if (typeof actorId !== "string") return;
|
|
1145
|
+
const site = agentCallSite(entry.actorPath ?? [], actorId);
|
|
1146
|
+
const oldest = effects.find((effect) => effect.site === site && !effect.completed);
|
|
1147
|
+
if (oldest === void 0) return;
|
|
1148
|
+
const call = oldest.kind === "model" ? callOfModelKey(oldest.key) : void 0;
|
|
1149
|
+
for (const effect of effects) if (effect === oldest || call !== void 0 && effect.site === site && callOfModelKey(effect.key) === call) effect.completed = true;
|
|
1150
|
+
}
|
|
1151
|
+
function countBy(effects) {
|
|
1152
|
+
const counts = /* @__PURE__ */ new Map();
|
|
1153
|
+
for (const { key } of effects) counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
1154
|
+
return counts;
|
|
1155
|
+
}
|
|
1156
|
+
function outcomeOf(result) {
|
|
1157
|
+
const snapshot = result.snapshot;
|
|
1158
|
+
return stableStringify(toJsonValue({
|
|
1159
|
+
status: result.status,
|
|
1160
|
+
...result.status === "done" ? { output: result.output } : {},
|
|
1161
|
+
...result.status === "error" ? { cause: result.cause } : {},
|
|
1162
|
+
value: snapshot.value,
|
|
1163
|
+
context: snapshot.context
|
|
1164
|
+
}));
|
|
1107
1165
|
}
|
|
1108
|
-
function
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1166
|
+
async function withTimeout(promise, ms) {
|
|
1167
|
+
let timer;
|
|
1168
|
+
try {
|
|
1169
|
+
return await Promise.race([promise, new Promise((resolve) => {
|
|
1170
|
+
timer = setTimeout(() => resolve(TIMED_OUT), ms);
|
|
1171
|
+
})]);
|
|
1172
|
+
} finally {
|
|
1173
|
+
clearTimeout(timer);
|
|
1174
|
+
}
|
|
1175
|
+
}
|
|
1176
|
+
/** `callKey` without its execution id: the same call has the same key in every run. */
|
|
1177
|
+
function modelKey(callKey) {
|
|
1178
|
+
return callKey.slice(callKey.indexOf(":") + 1);
|
|
1179
|
+
}
|
|
1180
|
+
function siteOfModelKey(key) {
|
|
1181
|
+
const hash = key.lastIndexOf("#");
|
|
1182
|
+
return hash === -1 ? key : key.slice(0, hash);
|
|
1183
|
+
}
|
|
1184
|
+
/** `site#n` of a model key: the call, without its attempt or strategy suffix. */
|
|
1185
|
+
function callOfModelKey(key) {
|
|
1186
|
+
const hash = key.lastIndexOf("#");
|
|
1187
|
+
if (hash === -1) return key;
|
|
1188
|
+
const dot = key.indexOf(".", hash);
|
|
1189
|
+
return dot === -1 ? key : key.slice(0, dot);
|
|
1190
|
+
}
|
|
1191
|
+
function trackExecutors(executors, world) {
|
|
1192
|
+
const tracked = {};
|
|
1193
|
+
for (const [slot, executor] of Object.entries(executors)) {
|
|
1194
|
+
if (typeof executor !== "function") continue;
|
|
1195
|
+
tracked[slot] = async (request, info) => {
|
|
1196
|
+
if (world.crashed) return never(info?.signal);
|
|
1197
|
+
const key = info?.callKey !== void 0 ? modelKey(info.callKey) : `${slot}:?`;
|
|
1198
|
+
const effect = {
|
|
1199
|
+
kind: "model",
|
|
1200
|
+
key,
|
|
1201
|
+
site: siteOfModelKey(key),
|
|
1202
|
+
completed: false,
|
|
1203
|
+
request: requestFingerprint(request)
|
|
1204
|
+
};
|
|
1205
|
+
world.effects.push(effect);
|
|
1206
|
+
const dies = world.onEffect(effect);
|
|
1207
|
+
const recorded = world.recorded?.get(key);
|
|
1208
|
+
let result;
|
|
1209
|
+
if (recorded !== void 0) {
|
|
1210
|
+
await new Promise((resolve) => setTimeout(resolve, recorded.ms));
|
|
1211
|
+
result = recorded.result;
|
|
1212
|
+
} else {
|
|
1213
|
+
const started = Date.now();
|
|
1214
|
+
result = await executor(request, info);
|
|
1215
|
+
if (info?.callKey !== void 0) world.answers.set(key, {
|
|
1216
|
+
result,
|
|
1217
|
+
ms: Date.now() - started
|
|
1218
|
+
});
|
|
1219
|
+
}
|
|
1220
|
+
if (dies || world.crashed) return never(info?.signal);
|
|
1221
|
+
return result;
|
|
1117
1222
|
};
|
|
1118
|
-
const row = confusion[human] ??= Object.create(null);
|
|
1119
|
-
label.human++;
|
|
1120
|
-
row[judge] = (row[judge] ?? 0) + 1;
|
|
1121
|
-
if (human === judge) {
|
|
1122
|
-
label.agreed++;
|
|
1123
|
-
agreed++;
|
|
1124
|
-
} else disagreements.push({
|
|
1125
|
-
id,
|
|
1126
|
-
human,
|
|
1127
|
-
judge
|
|
1128
|
-
});
|
|
1129
1223
|
}
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
let negatives = 0;
|
|
1141
|
-
let trueNegatives = 0;
|
|
1142
|
-
for (const { human, judge } of rows) if (human === positive) {
|
|
1143
|
-
positives++;
|
|
1144
|
-
if (judge === positive) truePositives++;
|
|
1145
|
-
} else {
|
|
1146
|
-
negatives++;
|
|
1147
|
-
if (judge !== positive) trueNegatives++;
|
|
1224
|
+
return tracked;
|
|
1225
|
+
}
|
|
1226
|
+
const VOLATILE_REQUEST_KEYS = /* @__PURE__ */ new Set(["runId"]);
|
|
1227
|
+
function requestFingerprint(request) {
|
|
1228
|
+
try {
|
|
1229
|
+
const json = toJsonValue(request);
|
|
1230
|
+
if (json && typeof json === "object" && !Array.isArray(json)) for (const key of VOLATILE_REQUEST_KEYS) delete json[key];
|
|
1231
|
+
return stableStringify(json);
|
|
1232
|
+
} catch {
|
|
1233
|
+
return;
|
|
1148
1234
|
}
|
|
1149
|
-
const tpr = positives === 0 ? void 0 : truePositives / positives;
|
|
1150
|
-
const tnr = negatives === 0 ? void 0 : trueNegatives / negatives;
|
|
1151
|
-
summary.binary = {
|
|
1152
|
-
tpr,
|
|
1153
|
-
tnr
|
|
1154
|
-
};
|
|
1155
|
-
if (tpr !== void 0 && tnr !== void 0 && tpr + tnr > 1) summary.correct = (p) => Math.min(1, Math.max(0, (p + tnr - 1) / (tpr + tnr - 1)));
|
|
1156
|
-
return summary;
|
|
1157
1235
|
}
|
|
1158
|
-
|
|
1159
|
-
|
|
1236
|
+
function never(signal) {
|
|
1237
|
+
return new Promise((_, reject) => {
|
|
1238
|
+
if (signal?.aborted) reject(signal.reason);
|
|
1239
|
+
signal?.addEventListener("abort", () => reject(signal.reason), { once: true });
|
|
1240
|
+
});
|
|
1241
|
+
}
|
|
1242
|
+
/** `machine` with every plain actor source, at any depth, tracked. */
|
|
1243
|
+
function trackActors(machine, world, ancestors = /* @__PURE__ */ new Set()) {
|
|
1244
|
+
if (ancestors.has(machine)) return machine;
|
|
1245
|
+
const visited = /* @__PURE__ */ new Set([...ancestors, machine]);
|
|
1246
|
+
const sources = machine.sources?.actors ?? {};
|
|
1247
|
+
const wrapped = {};
|
|
1248
|
+
for (const [key, logic] of Object.entries(sources)) {
|
|
1249
|
+
if (isTextLogic(logic) || isDecisionLogic(logic) || isActLogic(logic)) continue;
|
|
1250
|
+
if (isStateMachineLogic(logic)) {
|
|
1251
|
+
const inner = trackActors(logic, world, visited);
|
|
1252
|
+
if (inner !== logic) wrapped[key] = inner;
|
|
1253
|
+
continue;
|
|
1254
|
+
}
|
|
1255
|
+
if (typeof logic?.transition === "function") wrapped[key] = trackLogic(logic, world);
|
|
1256
|
+
}
|
|
1257
|
+
return Object.keys(wrapped).length > 0 ? machine.provide({ actors: wrapped }) : machine;
|
|
1258
|
+
}
|
|
1259
|
+
/** The invoke site of an actor: its path's last id, under its parents' path. */
|
|
1260
|
+
function actorSite(self) {
|
|
1261
|
+
const path = [];
|
|
1262
|
+
for (let at = self; at?._parent !== void 0; at = at._parent) path.unshift(String(at.id));
|
|
1263
|
+
return agentCallSite(path.slice(0, -1), path.at(-1) ?? "");
|
|
1264
|
+
}
|
|
1160
1265
|
/**
|
|
1161
|
-
*
|
|
1162
|
-
*
|
|
1163
|
-
* replaces and where it differs on purpose.
|
|
1164
|
-
*
|
|
1165
|
-
* Parity is outcome-level: the terminal outcome and the effects in the world,
|
|
1166
|
-
* not the sequence of tool calls, since a machine usually reorders steps. A
|
|
1167
|
-
* candidate that broke no rule where the baseline broke one is intended, even
|
|
1168
|
-
* when the outcome matches. A difference also counts as intended when only the
|
|
1169
|
-
* candidate reached the scenario's `expected` outcome. Every other difference,
|
|
1170
|
-
* and every candidate rule break, is a regression for a person to review.
|
|
1171
|
-
*
|
|
1172
|
-
* Runs are sequential, baseline first. A run that throws is recorded with
|
|
1173
|
-
* outcome `'threw'` and the error message as its `detail`. Parity holds for
|
|
1174
|
-
* these scenarios and these runs only; with a live model, run it more than once.
|
|
1175
|
-
*
|
|
1176
|
-
* @example
|
|
1177
|
-
* ```ts no-check
|
|
1178
|
-
* const report = await compareRuns({
|
|
1179
|
-
* scenarios,
|
|
1180
|
-
* baseline: async (s) => toParityRun(await runRawLoop(s)),
|
|
1181
|
-
* candidate: async (s) => toParityRun(await runDesk(s)),
|
|
1182
|
-
* });
|
|
1183
|
-
* expect(report.ok).toBe(true);
|
|
1184
|
-
* ```
|
|
1266
|
+
* `logic`, recording each effect it starts (`@xstate.effect`: an async
|
|
1267
|
+
* actor's run, a callback's body). After the crash, effects do not start.
|
|
1185
1268
|
*/
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
const
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1269
|
+
function trackLogic(logic, world) {
|
|
1270
|
+
if (isTextLogic(logic) || isDecisionLogic(logic) || isActLogic(logic)) return logic;
|
|
1271
|
+
const track = (result, self) => {
|
|
1272
|
+
const [snapshot, effects] = result;
|
|
1273
|
+
if (!Array.isArray(effects) || !effects.some((e) => e?.type === "@xstate.effect")) return result;
|
|
1274
|
+
return [snapshot, effects.map((effect) => {
|
|
1275
|
+
if (effect?.type !== "@xstate.effect") return effect;
|
|
1276
|
+
const exec = effect.exec;
|
|
1277
|
+
const tracked = (...args) => {
|
|
1278
|
+
if (world.crashed) return;
|
|
1279
|
+
const site = actorSite(self);
|
|
1280
|
+
const started = {
|
|
1281
|
+
kind: "actor",
|
|
1282
|
+
key: `actor:${site}`,
|
|
1283
|
+
site,
|
|
1284
|
+
completed: false
|
|
1285
|
+
};
|
|
1286
|
+
world.effects.push(started);
|
|
1287
|
+
world.onEffect(started);
|
|
1288
|
+
return exec(...args);
|
|
1289
|
+
};
|
|
1290
|
+
return {
|
|
1291
|
+
...effect,
|
|
1292
|
+
exec: tracked,
|
|
1293
|
+
action: tracked
|
|
1294
|
+
};
|
|
1295
|
+
})];
|
|
1211
1296
|
};
|
|
1297
|
+
const overrides = { transition: { value: (...args) => track(logic.transition(...args), args[2].self) } };
|
|
1298
|
+
if (typeof logic.initialTransition === "function") overrides["initialTransition"] = { value: (...args) => track(logic.initialTransition(...args), args[1].self) };
|
|
1299
|
+
if (typeof logic.start === "function") overrides["start"] = { value: (snapshot, scope, startOptions) => logic.start(snapshot, new Proxy(scope, { get(target, property) {
|
|
1300
|
+
if (property === "actionExecutor") return (effect) => {
|
|
1301
|
+
const [, [tracked]] = track([void 0, [effect]], target.self);
|
|
1302
|
+
target.actionExecutor(tracked);
|
|
1303
|
+
};
|
|
1304
|
+
return Reflect.get(target, property, target);
|
|
1305
|
+
} }), startOptions) };
|
|
1306
|
+
return Object.create(logic, overrides);
|
|
1212
1307
|
}
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1308
|
+
//#endregion
|
|
1309
|
+
//#region src/trajectory.ts
|
|
1310
|
+
/**
|
|
1311
|
+
* Classifies one item. Event-shaped things (log entries, events) become events;
|
|
1312
|
+
* everything else is a state value. A snapshot is unwrapped to its `value`.
|
|
1313
|
+
* @internal
|
|
1314
|
+
*/
|
|
1315
|
+
function normalize(item) {
|
|
1316
|
+
if (isRecord(item)) {
|
|
1317
|
+
const nested = item["event"];
|
|
1318
|
+
if (isRecord(nested) && typeof nested["type"] === "string") return {
|
|
1319
|
+
kind: "event",
|
|
1320
|
+
event: nested
|
|
1321
|
+
};
|
|
1322
|
+
if (typeof item["type"] === "string") return {
|
|
1323
|
+
kind: "event",
|
|
1324
|
+
event: item
|
|
1325
|
+
};
|
|
1326
|
+
if ("value" in item) return {
|
|
1327
|
+
kind: "state",
|
|
1328
|
+
value: item["value"]
|
|
1220
1329
|
};
|
|
1221
1330
|
}
|
|
1222
|
-
}
|
|
1223
|
-
function labelRun(expected, baseline, candidate) {
|
|
1224
|
-
if (candidate.violations?.length) return {
|
|
1225
|
-
label: "regression",
|
|
1226
|
-
reason: `candidate broke a rule: ${list(candidate.violations)}`
|
|
1227
|
-
};
|
|
1228
|
-
if (candidate.outcome === "threw") return {
|
|
1229
|
-
label: "regression",
|
|
1230
|
-
reason: `candidate threw: ${String(candidate.detail)}`
|
|
1231
|
-
};
|
|
1232
|
-
if (baseline.violations?.length) return {
|
|
1233
|
-
label: "intended",
|
|
1234
|
-
reason: `baseline broke a rule: ${list(baseline.violations)}`
|
|
1235
|
-
};
|
|
1236
|
-
const sameOutcome = baseline.outcome === candidate.outcome;
|
|
1237
|
-
if (sameOutcome && structurallyEqual(baseline.effects, candidate.effects)) return {
|
|
1238
|
-
label: "same",
|
|
1239
|
-
reason: ""
|
|
1240
|
-
};
|
|
1241
|
-
if (expected !== void 0 && candidate.outcome === expected && baseline.outcome !== expected) return {
|
|
1242
|
-
label: "intended",
|
|
1243
|
-
reason: `only the candidate reached the expected '${expected}'`
|
|
1244
|
-
};
|
|
1245
1331
|
return {
|
|
1246
|
-
|
|
1247
|
-
|
|
1332
|
+
kind: "state",
|
|
1333
|
+
value: item
|
|
1248
1334
|
};
|
|
1249
1335
|
}
|
|
1250
|
-
|
|
1251
|
-
|
|
1336
|
+
/** Every leaf of a state value as a dot path (`{ review: 'editing' }` → `review.editing`). @internal */
|
|
1337
|
+
function statePaths$1(value) {
|
|
1338
|
+
if (typeof value === "string") return [value];
|
|
1339
|
+
if (!isRecord(value)) return [];
|
|
1340
|
+
return Object.entries(value).flatMap(([key, child]) => statePaths$1(child).map((path) => `${key}.${path}`));
|
|
1252
1341
|
}
|
|
1253
|
-
|
|
1342
|
+
/**
|
|
1343
|
+
* A string expectation matches a state value when it names one of its leaves or
|
|
1344
|
+
* an ancestor of one, so `'review'` matches `{ review: 'editing' }`. @internal
|
|
1345
|
+
*/
|
|
1346
|
+
function stateMatchesPath(path, value) {
|
|
1347
|
+
return statePaths$1(value).some((leaf) => leaf === path || leaf.startsWith(`${path}.`));
|
|
1348
|
+
}
|
|
1349
|
+
function deepEqual(a, b) {
|
|
1254
1350
|
if (Object.is(a, b)) return true;
|
|
1255
|
-
if (a
|
|
1256
|
-
if (a instanceof Map || b instanceof Map) return a instanceof Map && b instanceof Map && structurallyEqual([...a], [...b]);
|
|
1257
|
-
if (a instanceof Set || b instanceof Set) return a instanceof Set && b instanceof Set && structurallyEqual([...a], [...b]);
|
|
1258
|
-
if (Array.isArray(a) || Array.isArray(b)) return Array.isArray(a) && Array.isArray(b) && a.length === b.length && a.every((item, index) => structurallyEqual(item, b[index]));
|
|
1351
|
+
if (Array.isArray(a) || Array.isArray(b)) return Array.isArray(a) && Array.isArray(b) && a.length === b.length && a.every((item, index) => deepEqual(item, b[index]));
|
|
1259
1352
|
if (!isRecord(a) || !isRecord(b)) return false;
|
|
1260
1353
|
const keys = Object.keys(a);
|
|
1261
|
-
return keys.length === Object.keys(b).length && keys.every((key) =>
|
|
1354
|
+
return keys.length === Object.keys(b).length && keys.every((key) => deepEqual(a[key], b[key]));
|
|
1262
1355
|
}
|
|
1263
|
-
//#endregion
|
|
1264
|
-
//#region src/equivalence.ts
|
|
1265
1356
|
/**
|
|
1266
|
-
*
|
|
1357
|
+
* Normalizes ONE expected item into a predicate over actual items, so the
|
|
1358
|
+
* expectation is classified once instead of once per candidate:
|
|
1267
1359
|
*
|
|
1268
|
-
* A
|
|
1269
|
-
*
|
|
1270
|
-
*
|
|
1271
|
-
*
|
|
1360
|
+
* - A string expectation matches an event's `type`, or a state value it names.
|
|
1361
|
+
* - An event expectation matches an event with the same `type` whose other
|
|
1362
|
+
* declared keys deep-equal, so payload details are opt-in.
|
|
1363
|
+
* - A state-value expectation matches structurally.
|
|
1364
|
+
* @internal
|
|
1365
|
+
*/
|
|
1366
|
+
function matcherFor(expected) {
|
|
1367
|
+
if (typeof expected === "string") return (actual) => {
|
|
1368
|
+
const actualItem = normalize(actual);
|
|
1369
|
+
return actualItem.kind === "event" ? actualItem.event.type === expected : stateMatchesPath(expected, actualItem.value);
|
|
1370
|
+
};
|
|
1371
|
+
const expectedItem = normalize(expected);
|
|
1372
|
+
switch (expectedItem.kind) {
|
|
1373
|
+
case "event": return (actual) => {
|
|
1374
|
+
const actualItem = normalize(actual);
|
|
1375
|
+
return actualItem.kind === "event" && Object.keys(expectedItem.event).every((key) => deepEqual(expectedItem.event[key], actualItem.event[key]));
|
|
1376
|
+
};
|
|
1377
|
+
case "state": return (actual) => {
|
|
1378
|
+
const actualItem = normalize(actual);
|
|
1379
|
+
return actualItem.kind === "state" && deepEqual(expectedItem.value, actualItem.value);
|
|
1380
|
+
};
|
|
1381
|
+
}
|
|
1382
|
+
}
|
|
1383
|
+
/**
|
|
1384
|
+
* Matches a run's trajectory against an expected one.
|
|
1272
1385
|
*
|
|
1273
|
-
*
|
|
1274
|
-
*
|
|
1275
|
-
*
|
|
1276
|
-
*
|
|
1277
|
-
*
|
|
1278
|
-
* every model call and tool call from the recording, with no API key and no
|
|
1279
|
-
* side effects, and records what the port did.
|
|
1280
|
-
* 3. `compareRecordings(original, port)` reports the first effect that differs.
|
|
1386
|
+
* Both trajectories may be state values or events collected from
|
|
1387
|
+
* `onTransition`;
|
|
1388
|
+
* items are compared by shape, so `['prompting', 'drafting']` scores a state
|
|
1389
|
+
* path and `['PROMPT_SUBMITTED', 'SEND']` scores an event log with the same
|
|
1390
|
+
* call.
|
|
1281
1391
|
*
|
|
1282
|
-
*
|
|
1283
|
-
*
|
|
1284
|
-
*
|
|
1285
|
-
* the calls the original never made, and `compareRecordings(…, { kinds })`
|
|
1286
|
-
* compares only tool calls and output.
|
|
1392
|
+
* Semantics are an ordered subsequence: every expected item must appear in
|
|
1393
|
+
* `actual`, in order, with gaps allowed. Pass `{ exact: true }` to require
|
|
1394
|
+
* equality instead.
|
|
1287
1395
|
*
|
|
1288
|
-
*
|
|
1289
|
-
*
|
|
1290
|
-
*
|
|
1396
|
+
* @example Trajectory assertions in a test
|
|
1397
|
+
* ```ts
|
|
1398
|
+
* const statePath: unknown[] = [];
|
|
1399
|
+
* const result = await runToQuiescence(
|
|
1400
|
+
* createAgentRuntime(machine, {
|
|
1401
|
+
* executors,
|
|
1402
|
+
* onTransition: (snapshot) => statePath.push(snapshot.value),
|
|
1403
|
+
* }),
|
|
1404
|
+
* { input },
|
|
1405
|
+
* );
|
|
1291
1406
|
*
|
|
1292
|
-
*
|
|
1293
|
-
*
|
|
1294
|
-
*
|
|
1407
|
+
* const path = matchesTrajectory(statePath, ['prompting', 'drafting', 'sent']);
|
|
1408
|
+
* expect(path.matched, JSON.stringify(path.firstMiss)).toBe(true);
|
|
1409
|
+
* expect(matchesTrajectory(events, ['PROMPT_SUBMITTED', 'SEND']).matched).toBe(true);
|
|
1410
|
+
* ```
|
|
1411
|
+
*
|
|
1412
|
+
* @example Partial credit in an eval scorer
|
|
1413
|
+
* ```ts
|
|
1414
|
+
* const match = matchesTrajectory(output.statePath, expected.statePath);
|
|
1415
|
+
* return { name: 'state_path', score: match.score, metadata: { ...match.firstMiss } };
|
|
1416
|
+
* ```
|
|
1295
1417
|
*/
|
|
1418
|
+
function matchesTrajectory(actual, expected, options = {}) {
|
|
1419
|
+
const expectedCount = expected.length;
|
|
1420
|
+
if (expectedCount === 0) throw new AgentError("empty-trajectory", "matchesTrajectory: expected trajectory must contain at least one item.");
|
|
1421
|
+
const exact = options.exact === true;
|
|
1422
|
+
let cursor = 0;
|
|
1423
|
+
let matchedCount = 0;
|
|
1424
|
+
let firstMiss;
|
|
1425
|
+
for (let index = 0; index < expectedCount; index++) {
|
|
1426
|
+
const want = expected[index];
|
|
1427
|
+
const matches = matcherFor(want);
|
|
1428
|
+
const searchedFrom = exact ? index : cursor;
|
|
1429
|
+
let found = -1;
|
|
1430
|
+
for (let at = searchedFrom; at < actual.length; at++) {
|
|
1431
|
+
if (matches(actual[at])) {
|
|
1432
|
+
found = at;
|
|
1433
|
+
break;
|
|
1434
|
+
}
|
|
1435
|
+
if (exact) break;
|
|
1436
|
+
}
|
|
1437
|
+
if (found === -1) {
|
|
1438
|
+
firstMiss = {
|
|
1439
|
+
index,
|
|
1440
|
+
expected: want,
|
|
1441
|
+
searchedFrom
|
|
1442
|
+
};
|
|
1443
|
+
break;
|
|
1444
|
+
}
|
|
1445
|
+
matchedCount++;
|
|
1446
|
+
cursor = found + 1;
|
|
1447
|
+
}
|
|
1448
|
+
const matched = !firstMiss && (!exact || actual.length === expectedCount);
|
|
1449
|
+
const denominator = exact ? Math.max(expectedCount, actual.length) : expectedCount;
|
|
1450
|
+
return {
|
|
1451
|
+
matched,
|
|
1452
|
+
matchedCount,
|
|
1453
|
+
expectedCount,
|
|
1454
|
+
score: matched ? 1 : denominator === 0 ? 1 : matchedCount / denominator,
|
|
1455
|
+
...firstMiss ? { firstMiss } : {}
|
|
1456
|
+
};
|
|
1457
|
+
}
|
|
1458
|
+
//#endregion
|
|
1459
|
+
//#region src/seam.ts
|
|
1460
|
+
/** The only own keys an executor result carries. @internal */
|
|
1461
|
+
const TEXT_RESULT_KEYS = /* @__PURE__ */ new Set([
|
|
1462
|
+
"result",
|
|
1463
|
+
"messages",
|
|
1464
|
+
"usage",
|
|
1465
|
+
"raw"
|
|
1466
|
+
]);
|
|
1296
1467
|
/**
|
|
1297
|
-
*
|
|
1298
|
-
*
|
|
1299
|
-
*
|
|
1300
|
-
*
|
|
1468
|
+
* True when a scripted entry is the executor result rather than the output
|
|
1469
|
+
* value: it owns a `result` key and owns NOTHING outside the result's own
|
|
1470
|
+
* vocabulary. Bare `'result' in value` would swallow an output object's
|
|
1471
|
+
* siblings (`{ result: 'draft', confidence: 0.9 }` would lose `confidence`)
|
|
1472
|
+
* and would also match an inherited `result`. @internal
|
|
1301
1473
|
*/
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
};
|
|
1305
|
-
function toRecordedError(error) {
|
|
1306
|
-
if (!(error instanceof Error)) return {
|
|
1307
|
-
name: "Error",
|
|
1308
|
-
message: String(error)
|
|
1309
|
-
};
|
|
1310
|
-
const recorded = {
|
|
1311
|
-
name: error.name,
|
|
1312
|
-
message: error.message
|
|
1313
|
-
};
|
|
1314
|
-
if (Object.prototype.propertyIsEnumerable.call(error, "name")) recorded.ownName = true;
|
|
1315
|
-
const code = error.code;
|
|
1316
|
-
if (code !== void 0) recorded.code = toJson(code);
|
|
1317
|
-
const data = Object.fromEntries(Object.entries(error).filter(([key, value]) => key !== "code" && (typeof value === "number" || typeof value === "boolean")));
|
|
1318
|
-
if (Object.keys(data).length) recorded.data = data;
|
|
1319
|
-
const markers = /* @__PURE__ */ new Set();
|
|
1320
|
-
for (let holder = error; holder; holder = Object.getPrototypeOf(holder)) for (const symbol of Object.getOwnPropertySymbols(holder)) {
|
|
1321
|
-
const key = Symbol.keyFor(symbol);
|
|
1322
|
-
if (key !== void 0 && error[symbol] === true) markers.add(key);
|
|
1323
|
-
}
|
|
1324
|
-
if (markers.size) recorded.markers = [...markers];
|
|
1325
|
-
return recorded;
|
|
1326
|
-
}
|
|
1327
|
-
function fromRecordedError({ data, markers, name, ownName, ...fields }) {
|
|
1328
|
-
const error = Object.assign(new Error(fields.message), fields, data);
|
|
1329
|
-
if (ownName || name !== "Error") Object.defineProperty(error, "name", {
|
|
1330
|
-
value: name,
|
|
1331
|
-
writable: true,
|
|
1332
|
-
configurable: true,
|
|
1333
|
-
enumerable: ownName ?? false
|
|
1334
|
-
});
|
|
1335
|
-
for (const marker of markers ?? []) error[Symbol.for(marker)] = true;
|
|
1336
|
-
return error;
|
|
1337
|
-
}
|
|
1338
|
-
/** What two runs compare of an error: its data and markers may differ. */
|
|
1339
|
-
function errorIdentity$1(error) {
|
|
1340
|
-
return error && {
|
|
1341
|
-
name: error.name,
|
|
1342
|
-
message: error.message,
|
|
1343
|
-
code: error.code
|
|
1344
|
-
};
|
|
1474
|
+
function isTextResult(value) {
|
|
1475
|
+
return Object.hasOwn(value, "result") && Object.keys(value).every((key) => TEXT_RESULT_KEYS.has(key));
|
|
1345
1476
|
}
|
|
1346
|
-
/**
|
|
1347
|
-
function
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
if (item instanceof Uint8Array) return Array.from(item);
|
|
1355
|
-
return item;
|
|
1356
|
-
}));
|
|
1477
|
+
/**
|
|
1478
|
+
* Resolves ONE scripted entry to an executor result: a function entry is
|
|
1479
|
+
* called with the request, and the value is taken as the executor result only
|
|
1480
|
+
* when it is one (see {@link isTextResult}). @internal
|
|
1481
|
+
*/
|
|
1482
|
+
async function resolveSeamScriptEntry(entry, request, info) {
|
|
1483
|
+
const value = typeof entry === "function" ? await entry(request, info) : entry;
|
|
1484
|
+
return isRecord(value) && isTextResult(value) ? value : { result: value };
|
|
1357
1485
|
}
|
|
1358
|
-
/**
|
|
1359
|
-
function
|
|
1360
|
-
|
|
1361
|
-
for (const holder of [value, value.response]) if (holder && typeof holder["timestamp"] === "string") holder["timestamp"] = new Date(holder["timestamp"]);
|
|
1362
|
-
}
|
|
1363
|
-
return value;
|
|
1486
|
+
/** Names a pending text request in an error message. @internal */
|
|
1487
|
+
function describeText(request) {
|
|
1488
|
+
return request.name ? `'${request.name}' (model '${request.model}')` : `(model '${request.model}')`;
|
|
1364
1489
|
}
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1490
|
+
/**
|
|
1491
|
+
* Stream semantics with no model: the whole text lands as one chunk. @internal
|
|
1492
|
+
*/
|
|
1493
|
+
function emitScriptedChunk(result, info) {
|
|
1494
|
+
const text = isRecord(result) ? result["result"] : void 0;
|
|
1495
|
+
if (typeof text === "string") info?.onChunk?.(text);
|
|
1368
1496
|
}
|
|
1369
1497
|
/**
|
|
1370
|
-
*
|
|
1498
|
+
* The seam's own answer, for scoring: the executor result's `result`.
|
|
1371
1499
|
* @internal
|
|
1372
1500
|
*/
|
|
1373
|
-
function
|
|
1374
|
-
if (
|
|
1375
|
-
|
|
1376
|
-
return JSON.stringify(value) ?? "undefined";
|
|
1377
|
-
}
|
|
1378
|
-
function jsonEqual(a, b) {
|
|
1379
|
-
return stableStringify(toJson(a)) === stableStringify(toJson(b));
|
|
1380
|
-
}
|
|
1381
|
-
function streamFrom(parts) {
|
|
1382
|
-
return new ReadableStream({ start(controller) {
|
|
1383
|
-
for (const part of parts) controller.enqueue(reviveTimestamps(structuredClone(part)));
|
|
1384
|
-
controller.close();
|
|
1385
|
-
} });
|
|
1386
|
-
}
|
|
1387
|
-
async function* iterate(chunks) {
|
|
1388
|
-
for (const chunk of chunks) yield structuredClone(chunk);
|
|
1501
|
+
async function seamOutputOf(result, request) {
|
|
1502
|
+
if (!isRecord(result)) return;
|
|
1503
|
+
return await normalizeGeneratorResult(result, `seam '${request.name ?? request.model}'`);
|
|
1389
1504
|
}
|
|
1390
1505
|
/**
|
|
1391
|
-
*
|
|
1392
|
-
*
|
|
1506
|
+
* Runs a machine end to end with one model call under test, and slices the run
|
|
1507
|
+
* at that call.
|
|
1508
|
+
*
|
|
1509
|
+
* The `before`/`after` slices are the point: `after.statePath` is the branch
|
|
1510
|
+
* the seam chose and `after.events` is the same question against the durable
|
|
1511
|
+
* log, both ready for `matchesTrajectory`. The state slice splits where the
|
|
1512
|
+
* state path stood when the seam answered; the event slice splits at the
|
|
1513
|
+
* seam's own effect completion (the first `xstate.done.*`/`xstate.error.*`
|
|
1514
|
+
* entry appended after the call was made).
|
|
1515
|
+
*
|
|
1516
|
+
* @example No API key: the seam is scripted too, so the whole thing runs offline.
|
|
1517
|
+
* ```ts
|
|
1518
|
+
* const run = await runSeam(emailDrafter, {
|
|
1519
|
+
* scripts: { writeFollowUps: [followUps], emailDrafter: [draft] },
|
|
1520
|
+
* seam: { request: 'writeFollowUps' },
|
|
1521
|
+
* respond: ({ state }) => (state === 'prompting' ? { type: 'PROMPT_SUBMITTED', prompt } : null),
|
|
1522
|
+
* });
|
|
1523
|
+
*
|
|
1524
|
+
* matchesTrajectory(run.after.statePath, ['needsMoreInfo', 'drafting']);
|
|
1525
|
+
* ```
|
|
1526
|
+
*
|
|
1527
|
+
* @example A candidate prompt at the seam, scored against the same rows.
|
|
1528
|
+
* ```ts
|
|
1529
|
+
* const { generateText } = createAiSdkExecutors({ models });
|
|
1530
|
+
* const run = await runSeam(emailDrafter, { ...row, candidate: generateText });
|
|
1531
|
+
* ```
|
|
1393
1532
|
*/
|
|
1394
|
-
function
|
|
1395
|
-
const
|
|
1396
|
-
const
|
|
1397
|
-
const
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
const
|
|
1403
|
-
let
|
|
1404
|
-
let
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
*/
|
|
1410
|
-
function nextReplayedModel(effect) {
|
|
1411
|
-
for (const recorded of replayedModels) {
|
|
1412
|
-
if (usedModels.has(recorded)) continue;
|
|
1413
|
-
const translated = options.translate?.(structuredClone(recorded), effect);
|
|
1414
|
-
if (translated) {
|
|
1415
|
-
effect.translated = true;
|
|
1416
|
-
return {
|
|
1417
|
-
recorded,
|
|
1418
|
-
answer: translated
|
|
1419
|
-
};
|
|
1420
|
-
}
|
|
1421
|
-
if (recorded.method === effect.method && modelCallsMatch(options.modelCalls, recorded, effect)) return {
|
|
1422
|
-
recorded,
|
|
1423
|
-
answer: recorded
|
|
1424
|
-
};
|
|
1425
|
-
}
|
|
1426
|
-
}
|
|
1427
|
-
function noAnswerMessage(effect) {
|
|
1428
|
-
const unused = replayedModels.filter((recorded) => !usedModels.has(recorded));
|
|
1429
|
-
return unused.length ? `Model call #${effect.call} (${effect.method}, ${JSON.stringify(modelCallShape(effect))}) fits no unused recorded call: ${unused.map((recorded) => `#${recorded.call} (${recorded.method}, ${JSON.stringify(modelCallShape(recorded))})`).join(", ")}. A recorded answer only fits a call of the same method and shape; pass \`translate\` to adapt one.` : `Model call #${effect.call} was never made by the recorded run (it made ${replayedModels.length}).`;
|
|
1430
|
-
}
|
|
1533
|
+
async function runSeam(machine, options) {
|
|
1534
|
+
const { seam, candidate } = options;
|
|
1535
|
+
const queues = new Map(Object.entries(options.scripts ?? {}).map(([key, answers]) => [key, [...answers]]));
|
|
1536
|
+
const statePath = [];
|
|
1537
|
+
let calls = 0;
|
|
1538
|
+
let seamMatches = 0;
|
|
1539
|
+
let seamOutput;
|
|
1540
|
+
let seamUsage;
|
|
1541
|
+
const callLog = [];
|
|
1542
|
+
let seamReached = false;
|
|
1543
|
+
let callsBeforeSeam = -1;
|
|
1544
|
+
let seamStateAt = 0;
|
|
1545
|
+
let seamEventAt = 0;
|
|
1546
|
+
let liveEvents = 0;
|
|
1547
|
+
const queueKeyOf = (request) => request.name !== void 0 && queues.has(request.name) ? request.name : request.model;
|
|
1431
1548
|
/**
|
|
1432
|
-
*
|
|
1433
|
-
*
|
|
1549
|
+
* Consumes this request's slot in the call plan, or resolves `undefined` when
|
|
1550
|
+
* its queue is dry. With `repeatLast`, the last entry is replayed instead of
|
|
1551
|
+
* running dry.
|
|
1434
1552
|
*/
|
|
1435
|
-
|
|
1436
|
-
const
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1553
|
+
const takeScriptedSlot = async (request, info) => {
|
|
1554
|
+
const queue = queues.get(queueKeyOf(request));
|
|
1555
|
+
if (!queue?.length) return;
|
|
1556
|
+
return resolveSeamScriptEntry(options.repeatLast && queue.length === 1 ? queue[0] : queue.shift(), request, info);
|
|
1557
|
+
};
|
|
1558
|
+
const scriptedAnswer = async (request, info) => {
|
|
1559
|
+
const scripted = await takeScriptedSlot(request, info);
|
|
1560
|
+
if (!scripted) throw new AgentError("seam-script-exhausted", `runSeam: no scripted answer left for request ${describeText(request)}. Add an entry to \`scripts.${queueKeyOf(request)}\`, or pass \`repeatLast: true\` to replay its last entry down a longer branch.`);
|
|
1561
|
+
return scripted;
|
|
1562
|
+
};
|
|
1563
|
+
const route = async (request, info) => {
|
|
1564
|
+
const callIndex = calls++;
|
|
1565
|
+
const isSeam = request.name === seam.request && seamMatches++ === (seam.occurrence ?? 0);
|
|
1566
|
+
const ledger = (source) => {
|
|
1567
|
+
callLog.push({
|
|
1568
|
+
...request.name !== void 0 ? { name: request.name } : {},
|
|
1569
|
+
model: request.model,
|
|
1570
|
+
key: queueKeyOf(request),
|
|
1571
|
+
source,
|
|
1572
|
+
seam: isSeam
|
|
1573
|
+
});
|
|
1446
1574
|
};
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
}
|
|
1458
|
-
effect.response = answer.response;
|
|
1459
|
-
effect.stream = answer.stream;
|
|
1460
|
-
if (seam === "openai") return streamed ? iterate(answer.stream ?? []) : structuredClone(answer.response);
|
|
1461
|
-
return streamed ? {
|
|
1462
|
-
...reviveTimestamps(structuredClone(answer.response) ?? {}),
|
|
1463
|
-
stream: streamFrom(answer.stream ?? [])
|
|
1464
|
-
} : reviveTimestamps(structuredClone(answer.response));
|
|
1465
|
-
}
|
|
1466
|
-
if (!fallback) {
|
|
1467
|
-
const error = new RecordingDivergenceError(noAnswerMessage(effect));
|
|
1468
|
-
effect.error = toRecordedError(error);
|
|
1469
|
-
throw error;
|
|
1470
|
-
}
|
|
1471
|
-
effect.unrecorded = true;
|
|
1472
|
-
run = fallback;
|
|
1473
|
-
}
|
|
1474
|
-
let result;
|
|
1475
|
-
try {
|
|
1476
|
-
result = await run();
|
|
1477
|
-
} catch (error) {
|
|
1478
|
-
effect.error = toRecordedError(error);
|
|
1479
|
-
throw error;
|
|
1480
|
-
}
|
|
1481
|
-
if (!streamed) {
|
|
1482
|
-
effect.response = toJson(result);
|
|
1575
|
+
if (isSeam && candidate) {
|
|
1576
|
+
await takeScriptedSlot(request, info);
|
|
1577
|
+
seamReached = true;
|
|
1578
|
+
callsBeforeSeam = callIndex;
|
|
1579
|
+
seamStateAt = statePath.length;
|
|
1580
|
+
seamEventAt = liveEvents;
|
|
1581
|
+
const result = await candidate(request, info);
|
|
1582
|
+
seamOutput = await seamOutputOf(result, request);
|
|
1583
|
+
seamUsage = getCallUsage(result);
|
|
1584
|
+
ledger("candidate");
|
|
1483
1585
|
return result;
|
|
1484
1586
|
}
|
|
1485
|
-
const
|
|
1486
|
-
|
|
1487
|
-
if (
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1587
|
+
const scripted = await scriptedAnswer(request, info);
|
|
1588
|
+
ledger("script");
|
|
1589
|
+
if (!isSeam) return scripted;
|
|
1590
|
+
seamReached = true;
|
|
1591
|
+
callsBeforeSeam = callIndex;
|
|
1592
|
+
seamStateAt = statePath.length;
|
|
1593
|
+
seamEventAt = liveEvents;
|
|
1594
|
+
seamOutput = scripted.result;
|
|
1595
|
+
seamUsage = scripted.usage;
|
|
1596
|
+
return scripted;
|
|
1597
|
+
};
|
|
1598
|
+
const executors = {
|
|
1599
|
+
...options.executors,
|
|
1600
|
+
generateText: route,
|
|
1601
|
+
streamText: async (request, info) => {
|
|
1602
|
+
const result = await route(request, info);
|
|
1603
|
+
emitScriptedChunk(result, info);
|
|
1604
|
+
return result;
|
|
1495
1605
|
}
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
...toolsInFlight > 0 ? { concurrent: true } : {}
|
|
1516
|
-
};
|
|
1517
|
-
effects.push(effect);
|
|
1518
|
-
return track(call(effect, args));
|
|
1606
|
+
};
|
|
1607
|
+
let snapshot;
|
|
1608
|
+
let event;
|
|
1609
|
+
const events = [];
|
|
1610
|
+
let result;
|
|
1611
|
+
const maxTurns = options.maxTurns ?? 12;
|
|
1612
|
+
for (let turn = 0; turn <= maxTurns; turn++) {
|
|
1613
|
+
result = await runToQuiescence(createAgentRuntime(machine, {
|
|
1614
|
+
...options.actors ? { actors: options.actors } : {},
|
|
1615
|
+
executors,
|
|
1616
|
+
onTransition: (next, causedBy) => {
|
|
1617
|
+
if (turn > 0 && causedBy.type === "@xstate.init") return;
|
|
1618
|
+
events.push(causedBy);
|
|
1619
|
+
liveEvents = events.length;
|
|
1620
|
+
statePath.push(next.value);
|
|
1621
|
+
}
|
|
1622
|
+
}), {
|
|
1623
|
+
...snapshot ? { snapshot } : { input: options.input },
|
|
1624
|
+
...event ? { event } : {}
|
|
1519
1625
|
});
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
const error = new RecordingDivergenceError(`Tool '${name}' was called with ${JSON.stringify(input)}; the recorded run made no such call.`);
|
|
1541
|
-
effect.error = toRecordedError(error);
|
|
1542
|
-
throw error;
|
|
1543
|
-
}
|
|
1544
|
-
usedTools.add(recorded);
|
|
1545
|
-
if (recorded.sync) effect.sync = true;
|
|
1546
|
-
if (recorded.error) {
|
|
1547
|
-
effect.error = recorded.error;
|
|
1548
|
-
throw fromRecordedError(recorded.error);
|
|
1549
|
-
}
|
|
1550
|
-
effect.output = recorded.output;
|
|
1551
|
-
return structuredClone(recorded.output);
|
|
1552
|
-
});
|
|
1626
|
+
if (result.status !== "idle" || turn === maxTurns) break;
|
|
1627
|
+
const next = await options.respond?.({
|
|
1628
|
+
snapshot: result.snapshot,
|
|
1629
|
+
state: result.snapshot.value,
|
|
1630
|
+
meta: getStateMeta(result.snapshot),
|
|
1631
|
+
turn,
|
|
1632
|
+
result
|
|
1633
|
+
});
|
|
1634
|
+
if (!next) break;
|
|
1635
|
+
event = next;
|
|
1636
|
+
snapshot = result.persist();
|
|
1637
|
+
}
|
|
1638
|
+
let splitAt = events.length;
|
|
1639
|
+
if (seamReached) {
|
|
1640
|
+
splitAt = seamEventAt;
|
|
1641
|
+
for (let index = seamEventAt; index < events.length; index++) {
|
|
1642
|
+
const type = events[index].type;
|
|
1643
|
+
if (type.startsWith("xstate.done") || type.startsWith("xstate.error")) {
|
|
1644
|
+
splitAt = index;
|
|
1645
|
+
break;
|
|
1553
1646
|
}
|
|
1554
|
-
return runLive(effect, args);
|
|
1555
1647
|
}
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1648
|
+
}
|
|
1649
|
+
const splitStateAt = seamReached ? seamStateAt : statePath.length;
|
|
1650
|
+
return {
|
|
1651
|
+
result,
|
|
1652
|
+
seamOutput,
|
|
1653
|
+
...seamUsage !== void 0 ? { seamUsage } : {},
|
|
1654
|
+
callsBeforeSeam,
|
|
1655
|
+
calls: callLog,
|
|
1656
|
+
before: {
|
|
1657
|
+
statePath: statePath.slice(0, splitStateAt),
|
|
1658
|
+
events: events.slice(0, splitAt)
|
|
1659
|
+
},
|
|
1660
|
+
after: {
|
|
1661
|
+
statePath: statePath.slice(splitStateAt),
|
|
1662
|
+
events: events.slice(splitAt)
|
|
1663
|
+
}
|
|
1664
|
+
};
|
|
1665
|
+
}
|
|
1666
|
+
//#endregion
|
|
1667
|
+
//#region src/internal/trace-states.ts
|
|
1668
|
+
/** The `value` of a trace event's `from` or `snapshot`. @internal */
|
|
1669
|
+
function valueOf(holder) {
|
|
1670
|
+
return isRecord(holder) ? holder.value : void 0;
|
|
1671
|
+
}
|
|
1672
|
+
/** A decision request on its first attempt (no failed attempts yet). @internal */
|
|
1673
|
+
function isFirstDecision(request) {
|
|
1674
|
+
return isRecord(request) && request.kind === "decision" && (!Array.isArray(request.attempts) || request.attempts.length === 0);
|
|
1675
|
+
}
|
|
1676
|
+
/**
|
|
1677
|
+
* Every active leaf of a state value as a dot path:
|
|
1678
|
+
* `'a'` → `['a']`; `{ a: 'b' }` → `['a.b']`; `{ p: { x: 'one', y: {} } }` → `['p.x.one', 'p.y']`.
|
|
1679
|
+
* @internal
|
|
1680
|
+
*/
|
|
1681
|
+
function statePaths(value, prefix = "") {
|
|
1682
|
+
if (typeof value === "string") return [prefix + value];
|
|
1683
|
+
if (!isRecord(value)) return [];
|
|
1684
|
+
const entries = Object.entries(value);
|
|
1685
|
+
if (entries.length === 0) return prefix ? [prefix.slice(0, -1)] : [];
|
|
1686
|
+
return entries.flatMap(([key, child]) => statePaths(child, `${prefix}${key}.`));
|
|
1687
|
+
}
|
|
1688
|
+
/**
|
|
1689
|
+
* A trace's `machine.transition` events as steps, skipping `@xstate.init`.
|
|
1690
|
+
* A first-attempt decision's `request.start` names the decision on the step
|
|
1691
|
+
* that follows it, skipping `@agent.usage` steps. Reads live and serialized traces alike.
|
|
1692
|
+
* @internal
|
|
1693
|
+
*/
|
|
1694
|
+
function traceSteps(trace) {
|
|
1695
|
+
const steps = [];
|
|
1696
|
+
let decision;
|
|
1697
|
+
for (const event of trace) {
|
|
1698
|
+
if (event.type === "request.start" && isFirstDecision(event.request)) {
|
|
1699
|
+
const name = event.request.name;
|
|
1700
|
+
decision = typeof name === "string" ? name : void 0;
|
|
1701
|
+
continue;
|
|
1577
1702
|
}
|
|
1703
|
+
if (event.type !== "machine.transition") continue;
|
|
1704
|
+
const type = isRecord(event.event) ? event.event.type : void 0;
|
|
1705
|
+
if (type === "@xstate.init") continue;
|
|
1706
|
+
const fromValue = valueOf(event.from);
|
|
1707
|
+
const toValue = valueOf(event.snapshot);
|
|
1708
|
+
steps.push({
|
|
1709
|
+
from: statePaths(fromValue),
|
|
1710
|
+
fromValue,
|
|
1711
|
+
event: typeof type === "string" ? type : "",
|
|
1712
|
+
payload: event.event,
|
|
1713
|
+
to: statePaths(toValue),
|
|
1714
|
+
toValue,
|
|
1715
|
+
ignored: event.ignored === true,
|
|
1716
|
+
...decision === void 0 || type === "@agent.usage" ? {} : { decision }
|
|
1717
|
+
});
|
|
1718
|
+
if (type !== "@agent.usage") decision = void 0;
|
|
1578
1719
|
}
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1720
|
+
return steps;
|
|
1721
|
+
}
|
|
1722
|
+
//#endregion
|
|
1723
|
+
//#region src/fold-traces.ts
|
|
1724
|
+
/**
|
|
1725
|
+
* Folds many runs' traces into per-state stats: how many runs visited each
|
|
1726
|
+
* state, how many of those failed, how many options the guards withheld
|
|
1727
|
+
* at decisions made there, how many events it ignored, and how many failures
|
|
1728
|
+
* `blame` traced back to it. Pure; reads only:
|
|
1729
|
+
*
|
|
1730
|
+
* - `machine.transition`: the run visited `from.value` and `snapshot.value`;
|
|
1731
|
+
* with `ignored`, each of those states ignored one event.
|
|
1732
|
+
* - `request.start` of a decision, first attempt only: its `request.withheld`
|
|
1733
|
+
* counts toward the state the run was in, the `from.value` of the
|
|
1734
|
+
* transition its choice causes.
|
|
1735
|
+
*
|
|
1736
|
+
* A trace is one run's events, live (`AgentTraceEvent`) or serialized
|
|
1737
|
+
* (`serializeTraceEvent`). A host-driven `decideNext` traces nothing; push
|
|
1738
|
+
* `{ type: 'request.start', request: result.request }` onto the trace before
|
|
1739
|
+
* sending its event. Parallel states count once per active leaf.
|
|
1740
|
+
*
|
|
1741
|
+
* `failures` counts a failed run against every state it visited, so states
|
|
1742
|
+
* after a bad decision rank as high as the decision. Sort by `blamed` to see
|
|
1743
|
+
* where failures start.
|
|
1744
|
+
*
|
|
1745
|
+
* @example
|
|
1746
|
+
* ```ts no-check
|
|
1747
|
+
* const stats = foldTraces(traces, {
|
|
1748
|
+
* failed: (trace) => trace.at(-1)?.type === 'run.end' && trace.at(-1).status === 'error',
|
|
1749
|
+
* });
|
|
1750
|
+
* ```
|
|
1751
|
+
*/
|
|
1752
|
+
function foldTraces(traces, options) {
|
|
1753
|
+
const stats = /* @__PURE__ */ new Map();
|
|
1754
|
+
const statsFor = (state) => {
|
|
1755
|
+
let entry = stats.get(state);
|
|
1756
|
+
if (!entry) {
|
|
1757
|
+
entry = {
|
|
1758
|
+
state,
|
|
1759
|
+
visits: 0,
|
|
1760
|
+
failures: 0,
|
|
1761
|
+
withheld: 0,
|
|
1762
|
+
ignored: 0,
|
|
1763
|
+
blamed: 0
|
|
1764
|
+
};
|
|
1765
|
+
stats.set(state, entry);
|
|
1766
|
+
}
|
|
1767
|
+
return entry;
|
|
1768
|
+
};
|
|
1769
|
+
for (const trace of traces) {
|
|
1770
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1771
|
+
let current = [];
|
|
1772
|
+
const visit = (state) => {
|
|
1773
|
+
statsFor(state);
|
|
1774
|
+
visited.add(state);
|
|
1775
|
+
};
|
|
1776
|
+
for (const event of trace) if (event.type === "machine.transition") {
|
|
1777
|
+
for (const state of statePaths(valueOf(event.from))) visit(state);
|
|
1778
|
+
current = statePaths(valueOf(event.snapshot));
|
|
1779
|
+
for (const state of current) visit(state);
|
|
1780
|
+
if (event.ignored === true) for (const state of current) statsFor(state).ignored++;
|
|
1781
|
+
} else if (event.type === "request.start" && isFirstDecision(event.request)) {
|
|
1782
|
+
const withheld = event.request.withheld;
|
|
1783
|
+
if (!Array.isArray(withheld)) continue;
|
|
1784
|
+
for (const state of current) statsFor(state).withheld += withheld.length;
|
|
1785
|
+
}
|
|
1786
|
+
const failed = options.failed(trace);
|
|
1787
|
+
for (const state of visited) {
|
|
1788
|
+
const entry = statsFor(state);
|
|
1789
|
+
entry.visits++;
|
|
1790
|
+
if (failed) entry.failures++;
|
|
1791
|
+
}
|
|
1792
|
+
const blamed = failed ? options.blame?.(trace) : void 0;
|
|
1793
|
+
if (blamed !== void 0) statsFor(blamed).blamed++;
|
|
1794
|
+
}
|
|
1795
|
+
return [...stats.values()];
|
|
1796
|
+
}
|
|
1797
|
+
//#endregion
|
|
1798
|
+
//#region src/trace-coverage.ts
|
|
1799
|
+
/** Mirrors xstate@6.0.0-alpha.69 `matchesEventDescriptor` (src/utils.ts), which the package does not export. */
|
|
1800
|
+
function matchesDescriptor(eventType, descriptor) {
|
|
1801
|
+
if (descriptor === eventType || descriptor === "*") return true;
|
|
1802
|
+
if (!descriptor.endsWith(".*")) return false;
|
|
1803
|
+
const tokens = descriptor.split(".");
|
|
1804
|
+
const eventTokens = eventType.split(".");
|
|
1805
|
+
for (let index = 0; index < tokens.length; index++) {
|
|
1806
|
+
if (tokens[index] === "*") return index === tokens.length - 1;
|
|
1807
|
+
if (tokens[index] !== eventTokens[index]) return false;
|
|
1808
|
+
}
|
|
1809
|
+
return true;
|
|
1810
|
+
}
|
|
1811
|
+
/** Whether a definition's `matches` pattern (e.g. an invoke's `actorId`) accepts the traced event. */
|
|
1812
|
+
function matchesPattern(definition, event) {
|
|
1813
|
+
const pattern = definition.matches;
|
|
1814
|
+
if (!isRecord(pattern) || !isRecord(event)) return true;
|
|
1815
|
+
return Object.entries(pattern).every(([key, value]) => Object.is(event[key], value));
|
|
1816
|
+
}
|
|
1817
|
+
/**
|
|
1818
|
+
* The descriptor XState would try first at `node` for `event`, guards aside:
|
|
1819
|
+
* the exact key, then partial and wildcard keys longest first, as in
|
|
1820
|
+
* xstate@6.0.0-alpha.69 `getCandidates` (src/stateUtils.ts).
|
|
1821
|
+
*/
|
|
1822
|
+
function candidateAt(node, type, event) {
|
|
1823
|
+
const keys = [...node.transitions.keys()].filter((key) => key !== type && matchesDescriptor(type, key)).sort((a, b) => b.length - a.length);
|
|
1824
|
+
if (node.transitions.has(type)) keys.unshift(type);
|
|
1825
|
+
return keys.find((key) => node.transitions.get(key).some((d) => matchesPattern(d, event)));
|
|
1826
|
+
}
|
|
1827
|
+
const pathOf = (node) => node.path.join(".");
|
|
1828
|
+
const sameLeaves = (a, b) => statePaths(a).sort().join("|") === statePaths(b).sort().join("|");
|
|
1829
|
+
/**
|
|
1830
|
+
* The leaves a trace passed through inside its macrosteps: states an `always`
|
|
1831
|
+
* transition left before the step ended, which no traced snapshot holds.
|
|
1832
|
+
* Re-derived with XState's pure `getMicrosteps` from the snapshot before each
|
|
1833
|
+
* step (the previous step's, or the fresh start's microsteps from its input).
|
|
1834
|
+
* A step that cannot be recomputed, or whose recomputation ends elsewhere
|
|
1835
|
+
* than the traced snapshot, adds none. Only `value` and `context` are rebuilt,
|
|
1836
|
+
* so a guard that reads live children can take another route to the same
|
|
1837
|
+
* leaves, and that route is credited.
|
|
1838
|
+
*/
|
|
1839
|
+
function passedThrough(machine, trace) {
|
|
1840
|
+
const leaves = /* @__PURE__ */ new Set();
|
|
1841
|
+
let resumed = false;
|
|
1842
|
+
let previous;
|
|
1843
|
+
for (const event of trace) {
|
|
1844
|
+
if (event.type === "run.start") resumed = event.snapshot !== void 0;
|
|
1845
|
+
if (event.type !== "machine.transition" || !isRecord(event.event)) continue;
|
|
1846
|
+
const snapshot = isRecord(event.snapshot) ? event.snapshot : void 0;
|
|
1847
|
+
try {
|
|
1848
|
+
const microsteps = event.event.type === "@xstate.init" ? resumed ? [] : getInitialMicrosteps(machine, event.event.input) : previous && sameLeaves(previous.value, valueOf(event.from)) ? getMicrosteps(machine, machine.resolveState({
|
|
1849
|
+
value: previous.value,
|
|
1850
|
+
context: previous.context
|
|
1851
|
+
}), event.event) : [];
|
|
1852
|
+
const last = microsteps.at(-1)?.[0];
|
|
1853
|
+
if (last && snapshot && sameLeaves(last.value, snapshot.value)) for (const [step] of microsteps) for (const leaf of statePaths(step.value)) leaves.add(leaf);
|
|
1854
|
+
} catch {}
|
|
1855
|
+
previous = snapshot;
|
|
1856
|
+
}
|
|
1857
|
+
return leaves;
|
|
1858
|
+
}
|
|
1859
|
+
const ratioOf = (visited, total) => total === 0 ? 1 : visited / total;
|
|
1860
|
+
/**
|
|
1861
|
+
* Measures how much of a machine a set of runs exercised: which atomic and
|
|
1862
|
+
* final states they visited, and which handlers they took. A handler is a
|
|
1863
|
+
* `(declaring state, event descriptor)` pair from `on`, invoke
|
|
1864
|
+
* `onDone`/`onError`, and `after`, with the targets it declares and the
|
|
1865
|
+
* leaves the runs landed in after it. Pure; reads only `machine.transition`
|
|
1866
|
+
* events, live (`AgentTraceEvent`) or serialized (`serializeTraceEvent`).
|
|
1867
|
+
*
|
|
1868
|
+
* A step credits one handler per active leaf: the nearest node, from the leaf
|
|
1869
|
+
* up, whose exact, partial (`a.*`), or wildcard (`*`) descriptor matches the
|
|
1870
|
+
* event, preferred in XState's order. Guards are not replayed, so a step
|
|
1871
|
+
* whose nearest handler was rejected and fell through to an ancestor is
|
|
1872
|
+
* credited to the nearest one. XState keys invoke and `after` handlers by
|
|
1873
|
+
* bare type (`xstate.done.actor`), one per state however many invokes; a
|
|
1874
|
+
* definition's `matches` (an invoke's `actorId`) must accept the traced event
|
|
1875
|
+
* for its state to be credited. Ignored steps credit no handler. `always`
|
|
1876
|
+
* handlers are listed as `unobservable`, never as unvisited; a state one
|
|
1877
|
+
* passes through is visited, re-derived from each step's microsteps.
|
|
1878
|
+
*
|
|
1879
|
+
* @example
|
|
1880
|
+
* ```ts no-check
|
|
1881
|
+
* const report = traceCoverage(machine, traces, { reachable: await explorePaths(machine) });
|
|
1882
|
+
* expect(report.unvisitedHandlers).toEqual([]);
|
|
1883
|
+
* ```
|
|
1884
|
+
*/
|
|
1885
|
+
function traceCoverage(machine, traces, options = {}) {
|
|
1886
|
+
const nodes = /* @__PURE__ */ new Map();
|
|
1887
|
+
const states = [];
|
|
1888
|
+
const handlers = [];
|
|
1889
|
+
const handlerAt = /* @__PURE__ */ new Map();
|
|
1890
|
+
const unobservable = [];
|
|
1891
|
+
const walk = (node) => {
|
|
1892
|
+
const state = pathOf(node);
|
|
1893
|
+
nodes.set(state, node);
|
|
1894
|
+
if (node.type === "atomic" || node.type === "final") states.push({
|
|
1895
|
+
state,
|
|
1896
|
+
runs: 0
|
|
1897
|
+
});
|
|
1898
|
+
const byEvent = /* @__PURE__ */ new Map();
|
|
1899
|
+
for (const [event, definitions] of node.transitions) {
|
|
1900
|
+
const dynamic = definitions.some((d) => typeof d.to === "function");
|
|
1901
|
+
const declared = [...new Set(definitions.flatMap((d) => (d.target ?? []).map(pathOf)))];
|
|
1902
|
+
const handler = {
|
|
1903
|
+
state,
|
|
1904
|
+
event,
|
|
1905
|
+
runs: 0,
|
|
1906
|
+
targets: {
|
|
1907
|
+
declared: dynamic ? "dynamic" : declared,
|
|
1908
|
+
seen: [],
|
|
1909
|
+
unreached: []
|
|
1609
1910
|
}
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1911
|
+
};
|
|
1912
|
+
handlers.push(handler);
|
|
1913
|
+
byEvent.set(event, handler);
|
|
1914
|
+
}
|
|
1915
|
+
handlerAt.set(node, byEvent);
|
|
1916
|
+
if (node.always?.length) unobservable.push({
|
|
1917
|
+
state,
|
|
1918
|
+
event: "always"
|
|
1919
|
+
});
|
|
1920
|
+
for (const child of Object.values(node.states)) walk(child);
|
|
1921
|
+
};
|
|
1922
|
+
walk(machine.root);
|
|
1923
|
+
const stateRuns = new Map(states.map((row) => [row.state, row]));
|
|
1924
|
+
const seen = /* @__PURE__ */ new Map();
|
|
1925
|
+
for (const trace of traces) {
|
|
1926
|
+
const steps = traceSteps(trace);
|
|
1927
|
+
const visited = new Set(steps[0]?.from);
|
|
1928
|
+
for (const event of trace) {
|
|
1929
|
+
if (event.type !== "machine.transition" || !isRecord(event.event)) continue;
|
|
1930
|
+
if (event.event.type !== "@xstate.init" || !isRecord(event.snapshot)) continue;
|
|
1931
|
+
for (const leaf of statePaths(event.snapshot.value)) visited.add(leaf);
|
|
1932
|
+
}
|
|
1933
|
+
if (unobservable.length > 0) for (const leaf of passedThrough(machine, trace)) visited.add(leaf);
|
|
1934
|
+
const credited = /* @__PURE__ */ new Set();
|
|
1935
|
+
for (const step of steps) {
|
|
1936
|
+
for (const leaf of step.to) visited.add(leaf);
|
|
1937
|
+
if (step.ignored) continue;
|
|
1938
|
+
const hits = /* @__PURE__ */ new Set();
|
|
1939
|
+
for (const leaf of step.from) for (let node = nodes.get(leaf); node; node = node.parent) {
|
|
1940
|
+
if (node.type === "final") continue;
|
|
1941
|
+
const event = candidateAt(node, step.event, step.payload);
|
|
1942
|
+
if (event === void 0) continue;
|
|
1943
|
+
hits.add(handlerAt.get(node).get(event));
|
|
1944
|
+
break;
|
|
1620
1945
|
}
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1946
|
+
for (const handler of hits) {
|
|
1947
|
+
credited.add(handler);
|
|
1948
|
+
let leaves = seen.get(handler);
|
|
1949
|
+
if (!leaves) seen.set(handler, leaves = /* @__PURE__ */ new Set());
|
|
1950
|
+
for (const leaf of step.to) leaves.add(leaf);
|
|
1624
1951
|
}
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
const
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1952
|
+
}
|
|
1953
|
+
for (const state of visited) {
|
|
1954
|
+
const row = stateRuns.get(state);
|
|
1955
|
+
if (row) row.runs++;
|
|
1956
|
+
}
|
|
1957
|
+
for (const handler of credited) handler.runs++;
|
|
1958
|
+
}
|
|
1959
|
+
for (const handler of handlers) {
|
|
1960
|
+
const leaves = [...seen.get(handler) ?? []];
|
|
1961
|
+
handler.targets.seen = leaves;
|
|
1962
|
+
const { declared } = handler.targets;
|
|
1963
|
+
if (declared === "dynamic") continue;
|
|
1964
|
+
handler.targets.unreached = declared.filter((target) => !leaves.some((leaf) => leaf === target || leaf.startsWith(`${target}.`)));
|
|
1965
|
+
}
|
|
1966
|
+
const reachableComplete = options.reachable ? isExplorationComplete(options.reachable) : void 0;
|
|
1967
|
+
let unreachable;
|
|
1968
|
+
if (options.reachable && reachableComplete) {
|
|
1969
|
+
const reached = new Set(options.reachable.reachedStates.flatMap((value) => statePaths(value)));
|
|
1970
|
+
unreachable = states.filter((row) => !reached.has(row.state) && row.runs === 0).map((row) => row.state);
|
|
1971
|
+
}
|
|
1972
|
+
const counted = unreachable ? states.filter((row) => !unreachable.includes(row.state)) : states;
|
|
1973
|
+
const unvisitedStates = counted.filter((row) => row.runs === 0).map((row) => row.state);
|
|
1974
|
+
const unvisitedHandlers = handlers.filter((handler) => handler.runs === 0).map(({ state, event }) => ({
|
|
1975
|
+
state,
|
|
1976
|
+
event
|
|
1977
|
+
}));
|
|
1978
|
+
return {
|
|
1979
|
+
states,
|
|
1980
|
+
handlers,
|
|
1981
|
+
unvisitedStates,
|
|
1982
|
+
unvisitedHandlers,
|
|
1983
|
+
unobservable,
|
|
1984
|
+
...unreachable ? { unreachable } : {},
|
|
1985
|
+
...reachableComplete === void 0 ? {} : { reachableComplete },
|
|
1986
|
+
ratio: {
|
|
1987
|
+
states: ratioOf(counted.length - unvisitedStates.length, counted.length),
|
|
1988
|
+
handlers: ratioOf(handlers.length - unvisitedHandlers.length, handlers.length)
|
|
1989
|
+
}
|
|
1990
|
+
};
|
|
1991
|
+
}
|
|
1992
|
+
//#endregion
|
|
1993
|
+
//#region src/first-divergence.ts
|
|
1994
|
+
const defaultKey = (step) => `${step.from.join(",")}|${step.event}|${step.to.join(",")}`;
|
|
1995
|
+
const isErrorLike = (value) => isError(value) || isRecord(value) && typeof value.name === "string" && typeof value.message === "string";
|
|
1996
|
+
const normalizeErrors = (value, seen = /* @__PURE__ */ new WeakSet()) => {
|
|
1997
|
+
if (value === null || typeof value !== "object") return value;
|
|
1998
|
+
if (seen.has(value)) return "[circular]";
|
|
1999
|
+
seen.add(value);
|
|
2000
|
+
try {
|
|
2001
|
+
if (isErrorLike(value) && (isError(value) || "stack" in value || "code" in value)) {
|
|
2002
|
+
const { name, message, code, cause } = value;
|
|
2003
|
+
return {
|
|
2004
|
+
name,
|
|
2005
|
+
message,
|
|
2006
|
+
...typeof code === "string" ? { code } : {},
|
|
2007
|
+
...cause !== void 0 ? { cause: normalizeErrors(cause, seen) } : {}
|
|
2008
|
+
};
|
|
2009
|
+
}
|
|
2010
|
+
if (Array.isArray(value)) return value.map((item) => normalizeErrors(item, seen));
|
|
2011
|
+
if (!isPlainObject(value)) return value;
|
|
2012
|
+
const copy = Object.create(null);
|
|
2013
|
+
for (const key of Object.keys(value)) copy[key] = normalizeErrors(value[key], seen);
|
|
2014
|
+
return copy;
|
|
2015
|
+
} finally {
|
|
2016
|
+
seen.delete(value);
|
|
2017
|
+
}
|
|
2018
|
+
};
|
|
2019
|
+
const payloadKey = (step) => {
|
|
2020
|
+
if (!isRecord(step.payload)) return `${defaultKey(step)}|${canonicalEventJson(step.payload)}`;
|
|
2021
|
+
const { sessionId: _sessionId, ...payload } = step.payload;
|
|
2022
|
+
return `${defaultKey(step)}|${canonicalEventJson(normalizeErrors(payload))}`;
|
|
2023
|
+
};
|
|
2024
|
+
const isTrace = (items) => items.length === 0 || items.some((item) => isRecord(item) && (item.type === "machine.transition" || item.type === "run.start" || item.type === "run.end"));
|
|
2025
|
+
/**
|
|
2026
|
+
* Finds the first step where a run departs from a reference, so a failure is
|
|
2027
|
+
* blamed on the state it started in rather than on every state the run went
|
|
2028
|
+
* through afterwards.
|
|
2029
|
+
*
|
|
2030
|
+
* - Against a reference **trace**, such as a passing run of the same
|
|
2031
|
+
* scenario, steps are compared in order by `key`. The first step that
|
|
2032
|
+
* differs, or the point where either trace ends early, is the divergence.
|
|
2033
|
+
* - Against an expected **trajectory** (`TrajectoryItem[]`, as for
|
|
2034
|
+
* {@link matchesTrajectory}), the run is laid out as its initial state, then
|
|
2035
|
+
* each step's event and new state. The first expected item it never reaches
|
|
2036
|
+
* is the divergence, blamed on the step after the last item it did reach. A
|
|
2037
|
+
* run that goes on past the trajectory has not diverged.
|
|
2038
|
+
*
|
|
2039
|
+
* An empty reference is a trace with no steps, so any step diverges from it.
|
|
2040
|
+
* A trace is one run's events, live or serialized. Steps the machine ignored
|
|
2041
|
+
* changed nothing and are left out on both sides. Returns `undefined` when the
|
|
2042
|
+
* run does not diverge.
|
|
2043
|
+
*
|
|
2044
|
+
* @example
|
|
2045
|
+
* ```ts no-check
|
|
2046
|
+
* const stats = foldTraces(traces, {
|
|
2047
|
+
* failed,
|
|
2048
|
+
* blame: (trace) => firstDivergence(trace, passingRunFor(trace))?.state,
|
|
2049
|
+
* });
|
|
2050
|
+
* ```
|
|
2051
|
+
*/
|
|
2052
|
+
function firstDivergence(actual, reference, options = {}) {
|
|
2053
|
+
const steps = traceSteps(actual).filter((step) => !step.ignored);
|
|
2054
|
+
return isTrace(reference) ? againstTrace(steps, traceSteps(reference).filter((step) => !step.ignored), options.key === "payload" ? payloadKey : options.key ?? defaultKey) : againstTrajectory(actual, steps, reference);
|
|
2055
|
+
}
|
|
2056
|
+
function diverged(steps, index, expected, initial = steps[0]?.fromValue) {
|
|
2057
|
+
const step = steps[index];
|
|
2058
|
+
return {
|
|
2059
|
+
index,
|
|
2060
|
+
state: (step ? step.from[0] : steps.at(-1)?.to[0] ?? statePaths(initial)[0]) ?? "",
|
|
2061
|
+
...step?.decision === void 0 ? {} : { decision: step.decision },
|
|
2062
|
+
...step ? { actual: step } : {},
|
|
2063
|
+
...expected === void 0 ? {} : { expected }
|
|
2064
|
+
};
|
|
2065
|
+
}
|
|
2066
|
+
function againstTrace(steps, reference, key) {
|
|
2067
|
+
const length = Math.max(steps.length, reference.length);
|
|
2068
|
+
for (let index = 0; index < length; index++) {
|
|
2069
|
+
const step = steps[index];
|
|
2070
|
+
const want = reference[index];
|
|
2071
|
+
if (step && want && key(step) === key(want)) continue;
|
|
2072
|
+
return diverged(steps, index, want, steps[0]?.fromValue ?? reference[0]?.fromValue);
|
|
2073
|
+
}
|
|
2074
|
+
}
|
|
2075
|
+
function againstTrajectory(trace, steps, expected) {
|
|
2076
|
+
const initial = initialValue(trace, steps);
|
|
2077
|
+
const timeline = initial === void 0 ? [] : [{ value: initial }];
|
|
2078
|
+
const stepAt = initial === void 0 ? [] : [0];
|
|
2079
|
+
steps.forEach((step, index) => {
|
|
2080
|
+
timeline.push(step.payload);
|
|
2081
|
+
stepAt.push(index);
|
|
2082
|
+
if (JSON.stringify(step.toValue) !== JSON.stringify(step.fromValue)) {
|
|
2083
|
+
timeline.push({ value: step.toValue });
|
|
2084
|
+
stepAt.push(index);
|
|
2085
|
+
}
|
|
2086
|
+
});
|
|
2087
|
+
const match = matchesTrajectory(timeline, expected);
|
|
2088
|
+
if (!match.firstMiss) return void 0;
|
|
2089
|
+
const at = match.firstMiss.searchedFrom;
|
|
2090
|
+
return diverged(steps, at < timeline.length ? stepAt[at] ?? steps.length : steps.length, match.firstMiss.expected, initial);
|
|
2091
|
+
}
|
|
2092
|
+
function initialValue(trace, steps) {
|
|
2093
|
+
for (const event of trace) if (event.type === "machine.transition" && isRecord(event.snapshot)) return event.snapshot.value;
|
|
2094
|
+
return steps[0]?.fromValue;
|
|
2095
|
+
}
|
|
2096
|
+
//#endregion
|
|
2097
|
+
//#region src/propose-from-traces.ts
|
|
2098
|
+
/**
|
|
2099
|
+
* Groups the approval verdicts in many runs' traces by `key` and proposes an
|
|
2100
|
+
* `allow` or `deny` rule for each. Pure and deterministic; a proposal is for
|
|
2101
|
+
* a person to review, and nothing here changes a machine.
|
|
2102
|
+
*
|
|
2103
|
+
* Reads `machine.transition` events: a verdict is a transition, not ignored,
|
|
2104
|
+
* out of `approvalState` on an approve or reject event. Rows come in the
|
|
2105
|
+
* order keys were first seen.
|
|
2106
|
+
*
|
|
2107
|
+
* @example
|
|
2108
|
+
* ```ts no-check
|
|
2109
|
+
* const rows = proposeApprovalRules(traces, {
|
|
2110
|
+
* approvalState: "awaitingApproval",
|
|
2111
|
+
* approveEvents: ["APPROVE"],
|
|
2112
|
+
* rejectEvents: ["REJECT"],
|
|
2113
|
+
* key: ({ snapshot }) => `${snapshot.context.pending.tool}:${snapshot.context.pending.path}`,
|
|
2114
|
+
* });
|
|
2115
|
+
* ```
|
|
2116
|
+
*/
|
|
2117
|
+
function proposeApprovalRules(traces, options) {
|
|
2118
|
+
const rows = /* @__PURE__ */ new Map();
|
|
2119
|
+
const rowFor = (key) => {
|
|
2120
|
+
let row = rows.get(key);
|
|
2121
|
+
if (!row) {
|
|
2122
|
+
row = {
|
|
2123
|
+
key,
|
|
2124
|
+
approved: 0,
|
|
2125
|
+
rejected: 0,
|
|
2126
|
+
violations: 0,
|
|
2127
|
+
runs: [],
|
|
2128
|
+
proposal: null,
|
|
2129
|
+
support: 0
|
|
2130
|
+
};
|
|
2131
|
+
rows.set(key, row);
|
|
2132
|
+
}
|
|
2133
|
+
return row;
|
|
2134
|
+
};
|
|
2135
|
+
const inState = (value) => statePaths(value).some((path) => path === options.approvalState || path.startsWith(`${options.approvalState}.`));
|
|
2136
|
+
let index = 0;
|
|
2137
|
+
for (const trace of traces) {
|
|
2138
|
+
const run = index++;
|
|
2139
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2140
|
+
let snapshot;
|
|
2141
|
+
for (const event of trace) {
|
|
2142
|
+
if (event.type !== "machine.transition") continue;
|
|
2143
|
+
const before = snapshot;
|
|
2144
|
+
snapshot = event.snapshot;
|
|
2145
|
+
const type = isRecord(event.event) ? event.event.type : void 0;
|
|
2146
|
+
if (event.ignored === true || typeof type !== "string" || !inState(valueOf(event.from))) continue;
|
|
2147
|
+
const approve = options.approveEvents.includes(type);
|
|
2148
|
+
if (!approve && !options.rejectEvents.includes(type)) continue;
|
|
2149
|
+
const key = options.key({
|
|
2150
|
+
event: event.event,
|
|
2151
|
+
snapshot: before,
|
|
2152
|
+
trace
|
|
2153
|
+
});
|
|
2154
|
+
if (key === void 0) continue;
|
|
2155
|
+
const row = rowFor(key);
|
|
2156
|
+
if (approve) row.approved++;
|
|
2157
|
+
else row.rejected++;
|
|
2158
|
+
if (!seen.has(key)) {
|
|
2159
|
+
seen.add(key);
|
|
2160
|
+
row.runs.push(run);
|
|
2161
|
+
}
|
|
2162
|
+
}
|
|
2163
|
+
for (const key of options.violations?.(trace) ?? []) {
|
|
2164
|
+
const row = rows.get(key);
|
|
2165
|
+
if (row) row.violations++;
|
|
2166
|
+
}
|
|
2167
|
+
}
|
|
2168
|
+
for (const row of rows.values()) {
|
|
2169
|
+
row.support = row.runs.length;
|
|
2170
|
+
row.proposal = row.approved > 0 && row.rejected === 0 && row.violations === 0 ? "allow" : row.rejected > 0 && row.approved === 0 ? "deny" : null;
|
|
2171
|
+
}
|
|
2172
|
+
return [...rows.values()];
|
|
2173
|
+
}
|
|
2174
|
+
/**
|
|
2175
|
+
* Proposes a value for each cap (retry count, steps, batch size) from many
|
|
2176
|
+
* runs' traces: the 95th percentile of what passing runs used, plus one, and
|
|
2177
|
+
* never below the current cap while runs still hit it. Pure and
|
|
2178
|
+
* deterministic; a proposal is for a person to review, and nothing here
|
|
2179
|
+
* changes a machine.
|
|
2180
|
+
*
|
|
2181
|
+
* @example
|
|
2182
|
+
* ```ts no-check
|
|
2183
|
+
* const [retries] = proposeCaps(traces, {
|
|
2184
|
+
* caps: [{ name: "retries", current: 3, failureState: "gaveUp", valueFromRun: countRetries }],
|
|
2185
|
+
* });
|
|
2186
|
+
* ```
|
|
2187
|
+
*/
|
|
2188
|
+
function proposeCaps(traces, options) {
|
|
2189
|
+
const runs = [...traces];
|
|
2190
|
+
return options.caps.map((cap) => {
|
|
2191
|
+
let hits = 0;
|
|
2192
|
+
let max = null;
|
|
2193
|
+
const passing = [];
|
|
2194
|
+
for (const trace of runs) {
|
|
2195
|
+
const hit = hitCap(trace, cap);
|
|
2196
|
+
if (hit) hits++;
|
|
2197
|
+
const value = cap.valueFromRun(trace);
|
|
2198
|
+
if (value === void 0) continue;
|
|
2199
|
+
max = max === null ? value : Math.max(max, value);
|
|
2200
|
+
if (!hit) passing.push(value);
|
|
2201
|
+
}
|
|
2202
|
+
passing.sort((a, b) => a - b);
|
|
2203
|
+
const p95 = passing.length ? passing[Math.ceil(.95 * passing.length) - 1] : null;
|
|
2204
|
+
const proposal = p95 === null ? null : hits > 0 && cap.current !== void 0 ? Math.max(p95 + 1, cap.current) : p95 + 1;
|
|
2205
|
+
return {
|
|
2206
|
+
name: cap.name,
|
|
2207
|
+
hits,
|
|
2208
|
+
passing: passing.length,
|
|
2209
|
+
max,
|
|
2210
|
+
p95,
|
|
2211
|
+
proposal
|
|
2212
|
+
};
|
|
2213
|
+
});
|
|
2214
|
+
}
|
|
2215
|
+
function hitCap(trace, cap) {
|
|
2216
|
+
for (const event of trace) {
|
|
2217
|
+
if (cap.failureState !== void 0 && event.type === "machine.transition" && statePaths(valueOf(event.snapshot)).some((path) => path === cap.failureState || path.startsWith(`${cap.failureState}.`))) return true;
|
|
2218
|
+
if (cap.failureOutput && event.type === "run.end" && event.status === "done") {
|
|
2219
|
+
if (cap.failureOutput(event.output)) return true;
|
|
2220
|
+
}
|
|
2221
|
+
}
|
|
2222
|
+
return false;
|
|
2223
|
+
}
|
|
2224
|
+
//#endregion
|
|
2225
|
+
//#region src/decision-requests.ts
|
|
2226
|
+
/**
|
|
2227
|
+
* The decisions a run asked for, read from its trace: each decision's first
|
|
2228
|
+
* `request.start`, with the states the run was in. A trace is one run's events,
|
|
2229
|
+
* live or serialized with `serializeTraceEvent`. Retries of the same decision
|
|
2230
|
+
* are left out, so each decision appears once.
|
|
2231
|
+
*
|
|
2232
|
+
* Use the requests as eval rows (run a decider on each and compare its event
|
|
2233
|
+
* with a label) or as probe inputs for a policy. A serialized request keeps its
|
|
2234
|
+
* `options`, `metadata`, `system`, and `prompt`; event schemas do not survive
|
|
2235
|
+
* serialization, so replay closed choices (`options`) rather than open ones.
|
|
2236
|
+
*
|
|
2237
|
+
* @example
|
|
2238
|
+
* ```ts no-check
|
|
2239
|
+
* const rows = traces.flatMap((trace) => getDecisionRequests(trace, { name: "chooseResolution" }));
|
|
2240
|
+
* for (const { request } of rows) console.log((await decide(request)).event);
|
|
2241
|
+
* ```
|
|
2242
|
+
*/
|
|
2243
|
+
function getDecisionRequests(trace, options = {}) {
|
|
2244
|
+
const requests = [];
|
|
2245
|
+
let current = [];
|
|
2246
|
+
for (const event of trace) if (event.type === "machine.transition") current = statePaths(valueOf(event.snapshot));
|
|
2247
|
+
else if (event.type === "request.start" && isFirstDecision(event.request)) {
|
|
2248
|
+
if (options.name !== void 0 && event.request.name !== options.name) continue;
|
|
2249
|
+
requests.push({
|
|
2250
|
+
states: [...current],
|
|
2251
|
+
request: event.request
|
|
2252
|
+
});
|
|
2253
|
+
}
|
|
2254
|
+
return requests;
|
|
2255
|
+
}
|
|
2256
|
+
//#endregion
|
|
2257
|
+
//#region src/repeat-runs.ts
|
|
2258
|
+
/**
|
|
2259
|
+
* Summarizes repeated runs of each scenario: how often each passed, whether
|
|
2260
|
+
* all its runs passed (pass^k), how many distinct state paths they took, and
|
|
2261
|
+
* where they first went different ways. Pure. Use it for runs collected
|
|
2262
|
+
* elsewhere, such as CI shards or replays; {@link repeatRuns} collects them and
|
|
2263
|
+
* calls it.
|
|
2264
|
+
*
|
|
2265
|
+
* A scenario that passes 3 of 5 runs and `divergedAt: 'deciding'` points at one
|
|
2266
|
+
* decision, not at a flaky agent.
|
|
2267
|
+
*
|
|
2268
|
+
* @example
|
|
2269
|
+
* ```ts no-check
|
|
2270
|
+
* const report = summarizeRepeats(shards.map((s) => ({ id: s.id, scenario: s, runs: s.runs })));
|
|
2271
|
+
* report.passHat(3); // the chance 3 runs of a scenario all pass
|
|
2272
|
+
* ```
|
|
2273
|
+
*/
|
|
2274
|
+
function summarizeRepeats(groups) {
|
|
2275
|
+
const scenarios = groups.map((group) => summarize$1(group));
|
|
2276
|
+
const traces = groups.flatMap((group) => group.runs.flatMap((run) => run.trace ? [run.trace] : []));
|
|
2277
|
+
const most = Math.max(0, ...groups.map((group) => group.runs.length));
|
|
2278
|
+
const passHat = (j) => {
|
|
2279
|
+
if (!Number.isInteger(j) || j < 1 || j > most) throw new AgentError("pass-hat-out-of-range", `passHat: j must be a whole number from 1 to ${most}, got ${j}.`);
|
|
2280
|
+
const eligible = scenarios.filter((scenario) => scenario.runs.length >= j);
|
|
2281
|
+
return eligible.reduce((total, scenario) => total + allPass(scenario.passes, scenario.runs.length, j), 0) / eligible.length;
|
|
2282
|
+
};
|
|
2283
|
+
return {
|
|
2284
|
+
scenarios,
|
|
2285
|
+
passHat,
|
|
2286
|
+
traces
|
|
2287
|
+
};
|
|
2288
|
+
}
|
|
2289
|
+
/**
|
|
2290
|
+
* Runs every scenario `k` times and summarizes the runs with
|
|
2291
|
+
* {@link summarizeRepeats}. Runs are sequential, scenario by scenario. A run
|
|
2292
|
+
* that throws fails, with its message as `error` and no trace.
|
|
2293
|
+
*
|
|
2294
|
+
* With a live model, one run per scenario says little: τ-bench found pass^k
|
|
2295
|
+
* falls well below the single-run rate. Report pass^k next to it.
|
|
2296
|
+
*
|
|
2297
|
+
* @example
|
|
2298
|
+
* ```ts no-check
|
|
2299
|
+
* const report = await repeatRuns({
|
|
2300
|
+
* scenarios,
|
|
2301
|
+
* k: 5,
|
|
2302
|
+
* run: async (scenario) => {
|
|
2303
|
+
* const trace: AgentTraceEvent[] = [];
|
|
2304
|
+
* const result = await runToQuiescence(
|
|
2305
|
+
* createAgentRuntime(machine, { executors, onTrace: (event) => trace.push(event) }),
|
|
2306
|
+
* { input: scenario.input },
|
|
2307
|
+
* );
|
|
2308
|
+
* return { trace, outcome: String(result.snapshot.value) };
|
|
2309
|
+
* },
|
|
2310
|
+
* passed: (scenario, { outcome }) => outcome === scenario.expected,
|
|
2311
|
+
* });
|
|
2312
|
+
* report.passHat(5);
|
|
2313
|
+
* ```
|
|
2314
|
+
*/
|
|
2315
|
+
async function repeatRuns(options) {
|
|
2316
|
+
if (!Number.isInteger(options.k) || options.k < 1) throw new AgentError("invalid-repeat-count", `repeatRuns: k must be a whole number of 1 or more, got ${options.k}.`);
|
|
2317
|
+
const groups = [];
|
|
2318
|
+
for (const scenario of options.scenarios) {
|
|
2319
|
+
const runs = [];
|
|
2320
|
+
for (let attempt = 0; attempt < options.k; attempt++) try {
|
|
2321
|
+
const result = await options.run(scenario, attempt);
|
|
2322
|
+
runs.push({
|
|
2323
|
+
passed: options.passed(scenario, result),
|
|
2324
|
+
trace: result.trace,
|
|
2325
|
+
...result.outcome === void 0 ? {} : { outcome: result.outcome }
|
|
2326
|
+
});
|
|
2327
|
+
} catch (error) {
|
|
2328
|
+
runs.push({
|
|
2329
|
+
passed: false,
|
|
2330
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2331
|
+
});
|
|
2332
|
+
}
|
|
2333
|
+
groups.push({
|
|
2334
|
+
id: scenario.id,
|
|
2335
|
+
scenario,
|
|
2336
|
+
runs
|
|
1639
2337
|
});
|
|
1640
2338
|
}
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
2339
|
+
return summarizeRepeats(groups);
|
|
2340
|
+
}
|
|
2341
|
+
function summarize$1(group) {
|
|
2342
|
+
const passes = group.runs.filter((run) => run.passed).length;
|
|
2343
|
+
const traces = group.runs.flatMap((run) => run.trace ? [run.trace] : []);
|
|
2344
|
+
const paths = new Set(traces.map((trace) => traceSteps(trace).filter((step) => !step.ignored).map((step) => `${step.from.join(",")}|${step.event}|${step.to.join(",")}`).join(" ")));
|
|
2345
|
+
const reference = traces.find((trace) => trace.some((event) => event.type === "machine.transition"));
|
|
2346
|
+
let earliest;
|
|
2347
|
+
for (const trace of reference ? traces : []) {
|
|
2348
|
+
if (trace === reference) continue;
|
|
2349
|
+
const divergence = firstDivergence(trace, reference);
|
|
2350
|
+
if (divergence && (!earliest || divergence.index < earliest.index)) earliest = divergence;
|
|
1651
2351
|
}
|
|
1652
2352
|
return {
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
"doStream",
|
|
1660
|
-
"doDecide",
|
|
1661
|
-
"doEvaluate"
|
|
1662
|
-
]) {
|
|
1663
|
-
if (typeof base[method] !== "function") continue;
|
|
1664
|
-
const invoke = (target, callOptions) => target[method].call(target, callOptions);
|
|
1665
|
-
wrapped[method] = (callOptions) => modelCall("ai-sdk", method, callOptions, method === "doStream", () => invoke(base, callOptions), stand && typeof stand[method] === "function" ? () => invoke(stand, callOptions) : void 0);
|
|
1666
|
-
}
|
|
1667
|
-
return wrapped;
|
|
1668
|
-
},
|
|
1669
|
-
openai(client) {
|
|
1670
|
-
const wrapCreate = (resource, method) => new Proxy(resource, { get(target, key, receiver) {
|
|
1671
|
-
if (key !== "create") return Reflect.get(target, key, receiver);
|
|
1672
|
-
return (params, requestOptions) => {
|
|
1673
|
-
let api;
|
|
1674
|
-
const create = (resource) => () => {
|
|
1675
|
-
api = Reflect.get(resource, "create").call(resource, params, requestOptions);
|
|
1676
|
-
return Promise.resolve(api);
|
|
1677
|
-
};
|
|
1678
|
-
const stand = fallbackResource(method);
|
|
1679
|
-
const promise = modelCall("openai", method, params, params?.stream === true, create(target), stand && create(stand));
|
|
1680
|
-
return Object.assign(promise, { withResponse: async () => {
|
|
1681
|
-
const data = await promise;
|
|
1682
|
-
const meta = await api?.withResponse?.();
|
|
1683
|
-
return {
|
|
1684
|
-
data,
|
|
1685
|
-
response: meta?.response ?? new Response(null),
|
|
1686
|
-
request_id: meta?.request_id ?? null
|
|
1687
|
-
};
|
|
1688
|
-
} });
|
|
1689
|
-
};
|
|
1690
|
-
} });
|
|
1691
|
-
const fallbackResource = (method) => {
|
|
1692
|
-
let value = options.fallback?.openai;
|
|
1693
|
-
for (const key of method.split(".").slice(0, -1)) value = value?.[key];
|
|
1694
|
-
return value && typeof value === "object" ? value : void 0;
|
|
1695
|
-
};
|
|
1696
|
-
const wrapped = /* @__PURE__ */ new Map();
|
|
1697
|
-
const wrap = (key, make) => {
|
|
1698
|
-
if (!wrapped.has(key)) wrapped.set(key, make());
|
|
1699
|
-
return wrapped.get(key);
|
|
1700
|
-
};
|
|
1701
|
-
return new Proxy(client, { get(target, key, receiver) {
|
|
1702
|
-
const value = Reflect.get(target, key, receiver);
|
|
1703
|
-
if (key === "responses" && value) return wrap(key, () => wrapCreate(value, "responses.create"));
|
|
1704
|
-
if (key === "chat" && value) return wrap(key, () => new Proxy(value, { get(chat, chatKey, chatReceiver) {
|
|
1705
|
-
const inner = Reflect.get(chat, chatKey, chatReceiver);
|
|
1706
|
-
return chatKey === "completions" && inner ? wrap("chat.completions", () => wrapCreate(inner, "chat.completions.create")) : inner;
|
|
1707
|
-
} }));
|
|
1708
|
-
return value;
|
|
1709
|
-
} });
|
|
1710
|
-
},
|
|
1711
|
-
tool: (name, fn) => tool(name, fn),
|
|
1712
|
-
fn,
|
|
1713
|
-
client(name, client, { methods, ...callOptions }) {
|
|
1714
|
-
const paths = methods.map((method) => method.split("."));
|
|
1715
|
-
const startsWith = (path, prefix) => prefix.every((key, index) => path[index] === key);
|
|
1716
|
-
const wrapAt = (target, prefix) => {
|
|
1717
|
-
const wrapped = /* @__PURE__ */ new Map();
|
|
1718
|
-
return new Proxy(target, { get(object, key, receiver) {
|
|
1719
|
-
if (typeof key !== "string") return Reflect.get(object, key, receiver);
|
|
1720
|
-
const path = [...prefix, key];
|
|
1721
|
-
const method = paths.some((p) => p.length === path.length && startsWith(p, path));
|
|
1722
|
-
if (!method && !paths.some((p) => startsWith(p, path))) return Reflect.get(object, key, receiver);
|
|
1723
|
-
if (!wrapped.has(key)) {
|
|
1724
|
-
const value = Reflect.get(object, key, receiver);
|
|
1725
|
-
wrapped.set(key, method ? fn(`${name}.${path.join(".")}`, (...args) => value.apply(object, args), callOptions) : wrapAt(value && typeof value === "object" ? value : {}, path));
|
|
1726
|
-
}
|
|
1727
|
-
return wrapped.get(key);
|
|
1728
|
-
} });
|
|
1729
|
-
};
|
|
1730
|
-
return wrapAt(client, []);
|
|
1731
|
-
},
|
|
1732
|
-
tools(tools) {
|
|
1733
|
-
return Object.fromEntries(Object.entries(tools).map(([name, definition]) => [name, definition.execute ? {
|
|
1734
|
-
...definition,
|
|
1735
|
-
execute: tool(name, definition.execute, (args) => args[0])
|
|
1736
|
-
} : definition]));
|
|
1737
|
-
},
|
|
1738
|
-
output(value) {
|
|
1739
|
-
effects.push({
|
|
1740
|
-
kind: "output",
|
|
1741
|
-
value: toJson(value)
|
|
1742
|
-
});
|
|
1743
|
-
},
|
|
1744
|
-
get recording() {
|
|
1745
|
-
return {
|
|
1746
|
-
version: 1,
|
|
1747
|
-
effects: toJson(effects)
|
|
1748
|
-
};
|
|
1749
|
-
}
|
|
2353
|
+
...group,
|
|
2354
|
+
passes,
|
|
2355
|
+
passAny: passes > 0,
|
|
2356
|
+
passAll: group.runs.length > 0 && passes === group.runs.length,
|
|
2357
|
+
distinctPaths: paths.size,
|
|
2358
|
+
...earliest ? { divergedAt: earliest.state } : {}
|
|
1750
2359
|
};
|
|
1751
2360
|
}
|
|
1752
|
-
/**
|
|
1753
|
-
function
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
if (match === "none") return true;
|
|
1758
|
-
if (typeof match === "function") return match(expected, actual);
|
|
1759
|
-
return sameModelCallShape(expected, actual);
|
|
1760
|
-
}
|
|
1761
|
-
function shapeMessage(expected, actual) {
|
|
1762
|
-
return `Model call #${expected.call} changed shape: ${JSON.stringify(modelCallShape(expected))} → ${JSON.stringify(modelCallShape(actual))}. A recorded answer only fits a call of the same shape.`;
|
|
2361
|
+
/** C(passes, j) / C(runs, j), as a product so large counts don't overflow. */
|
|
2362
|
+
function allPass(passes, runs, j) {
|
|
2363
|
+
let chance = 1;
|
|
2364
|
+
for (let i = 0; i < j; i++) chance *= Math.max(0, passes - i) / (runs - i);
|
|
2365
|
+
return chance;
|
|
1763
2366
|
}
|
|
2367
|
+
//#endregion
|
|
2368
|
+
//#region src/judge-agreement.ts
|
|
1764
2369
|
/**
|
|
1765
|
-
*
|
|
1766
|
-
*
|
|
2370
|
+
* Measures how well a judge agrees with people on the same cases: accuracy,
|
|
2371
|
+
* per-label recall, the confusion matrix, and the rows that disagree. With
|
|
2372
|
+
* `positive`, also the TPR and TNR, the recommended view for binary judges
|
|
2373
|
+
* (Husain/Shankar), and a correction for the judge's bias on unlabeled rows.
|
|
2374
|
+
* Pure.
|
|
2375
|
+
*
|
|
2376
|
+
* Build rows from confirmed labels only; draft labels gate nothing. Group by
|
|
2377
|
+
* decision state or request name so each narrow judge is aligned on its own.
|
|
2378
|
+
* A row with no group key counts in the totals and in no group. Run it on a
|
|
2379
|
+
* dev split and report the held-out split separately.
|
|
2380
|
+
*
|
|
2381
|
+
* Throws `AgentError` with code `'empty-agreement-rows'` when `rows` is empty.
|
|
2382
|
+
*
|
|
2383
|
+
* @example
|
|
2384
|
+
* ```ts no-check
|
|
2385
|
+
* const report = judgeAgreement(
|
|
2386
|
+
* labeled.map((s) => ({ id: s.id, human: s.label, judge: verdicts[s.id], group: s.request })),
|
|
2387
|
+
* { positive: 'fail' },
|
|
2388
|
+
* );
|
|
2389
|
+
* report.binary; // { tpr: 0.9, tnr: 0.8 }
|
|
2390
|
+
* report.correct?.(0.3); // estimated true failure rate in production
|
|
2391
|
+
* ```
|
|
1767
2392
|
*/
|
|
1768
|
-
function
|
|
1769
|
-
|
|
1770
|
-
const
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
];
|
|
1783
|
-
|
|
1784
|
-
}
|
|
1785
|
-
return
|
|
1786
|
-
|
|
1787
|
-
|
|
2393
|
+
function judgeAgreement(rows, options = {}) {
|
|
2394
|
+
if (rows.length === 0) throw new AgentError("empty-agreement-rows", "judgeAgreement: rows must contain at least one row.");
|
|
2395
|
+
const report = summarize(rows, options.positive);
|
|
2396
|
+
const groupOf = options.groupBy ?? ((row) => row.group);
|
|
2397
|
+
if (options.groupBy || rows.some((row) => row.group !== void 0)) {
|
|
2398
|
+
const grouped = /* @__PURE__ */ new Map();
|
|
2399
|
+
for (const row of rows) {
|
|
2400
|
+
const key = groupOf(row);
|
|
2401
|
+
if (key === void 0) continue;
|
|
2402
|
+
const members = grouped.get(key);
|
|
2403
|
+
if (members) members.push(row);
|
|
2404
|
+
else grouped.set(key, [row]);
|
|
2405
|
+
}
|
|
2406
|
+
const groups = Object.create(null);
|
|
2407
|
+
for (const [key, members] of grouped) groups[key] = summarize(members, options.positive);
|
|
2408
|
+
report.groups = groups;
|
|
2409
|
+
}
|
|
2410
|
+
return report;
|
|
2411
|
+
}
|
|
2412
|
+
function summarize(rows, positive) {
|
|
2413
|
+
const perLabel = Object.create(null);
|
|
2414
|
+
const confusion = Object.create(null);
|
|
2415
|
+
const disagreements = [];
|
|
2416
|
+
let agreed = 0;
|
|
2417
|
+
for (const { id, human, judge } of rows) {
|
|
2418
|
+
const label = perLabel[human] ??= {
|
|
2419
|
+
human: 0,
|
|
2420
|
+
agreed: 0
|
|
2421
|
+
};
|
|
2422
|
+
const row = confusion[human] ??= Object.create(null);
|
|
2423
|
+
label.human++;
|
|
2424
|
+
row[judge] = (row[judge] ?? 0) + 1;
|
|
2425
|
+
if (human === judge) {
|
|
2426
|
+
label.agreed++;
|
|
2427
|
+
agreed++;
|
|
2428
|
+
} else disagreements.push({
|
|
2429
|
+
id,
|
|
2430
|
+
human,
|
|
2431
|
+
judge
|
|
2432
|
+
});
|
|
2433
|
+
}
|
|
2434
|
+
const summary = {
|
|
2435
|
+
total: rows.length,
|
|
2436
|
+
accuracy: agreed / rows.length,
|
|
2437
|
+
perLabel,
|
|
2438
|
+
confusion,
|
|
2439
|
+
disagreements
|
|
2440
|
+
};
|
|
2441
|
+
if (positive === void 0) return summary;
|
|
2442
|
+
let positives = 0;
|
|
2443
|
+
let truePositives = 0;
|
|
2444
|
+
let negatives = 0;
|
|
2445
|
+
let trueNegatives = 0;
|
|
2446
|
+
for (const { human, judge } of rows) if (human === positive) {
|
|
2447
|
+
positives++;
|
|
2448
|
+
if (judge === positive) truePositives++;
|
|
2449
|
+
} else {
|
|
2450
|
+
negatives++;
|
|
2451
|
+
if (judge !== positive) trueNegatives++;
|
|
2452
|
+
}
|
|
2453
|
+
const tpr = positives === 0 ? void 0 : truePositives / positives;
|
|
2454
|
+
const tnr = negatives === 0 ? void 0 : trueNegatives / negatives;
|
|
2455
|
+
summary.binary = {
|
|
2456
|
+
tpr,
|
|
2457
|
+
tnr
|
|
1788
2458
|
};
|
|
2459
|
+
if (tpr !== void 0 && tnr !== void 0 && tpr + tnr > 1) summary.correct = (p) => Math.min(1, Math.max(0, (p + tnr - 1) / (tpr + tnr - 1)));
|
|
2460
|
+
return summary;
|
|
1789
2461
|
}
|
|
2462
|
+
//#endregion
|
|
2463
|
+
//#region src/compare-runs.ts
|
|
1790
2464
|
/**
|
|
1791
|
-
*
|
|
1792
|
-
*
|
|
1793
|
-
*
|
|
1794
|
-
*
|
|
2465
|
+
* Runs every scenario through a baseline and a candidate implementation and
|
|
2466
|
+
* labels each difference, so a port can show where it behaves like the code it
|
|
2467
|
+
* replaces and where it differs on purpose.
|
|
2468
|
+
*
|
|
2469
|
+
* Parity is outcome-level: the terminal outcome and the effects in the world,
|
|
2470
|
+
* not the sequence of tool calls, since a machine usually reorders steps. A
|
|
2471
|
+
* candidate that broke no rule where the baseline broke one is intended, even
|
|
2472
|
+
* when the outcome matches. A difference also counts as intended when only the
|
|
2473
|
+
* candidate reached the scenario's `expected` outcome. Every other difference,
|
|
2474
|
+
* and every candidate rule break, is a regression for a person to review.
|
|
2475
|
+
*
|
|
2476
|
+
* Runs are sequential, baseline first. A run that throws is recorded with
|
|
2477
|
+
* outcome `'threw'` and the error message as its `detail`. Parity holds for
|
|
2478
|
+
* these scenarios and these runs only; with a live model, run it more than once.
|
|
2479
|
+
*
|
|
2480
|
+
* @example
|
|
2481
|
+
* ```ts no-check
|
|
2482
|
+
* const report = await compareRuns({
|
|
2483
|
+
* scenarios,
|
|
2484
|
+
* baseline: async (s) => toParityRun(await runRawLoop(s)),
|
|
2485
|
+
* candidate: async (s) => toParityRun(await runDesk(s)),
|
|
2486
|
+
* });
|
|
2487
|
+
* expect(report.ok).toBe(true);
|
|
2488
|
+
* ```
|
|
1795
2489
|
*/
|
|
1796
|
-
function
|
|
1797
|
-
const
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
2490
|
+
async function compareRuns(options) {
|
|
2491
|
+
const scenarios = [];
|
|
2492
|
+
const counts = {
|
|
2493
|
+
same: 0,
|
|
2494
|
+
intended: 0,
|
|
2495
|
+
regression: 0
|
|
1802
2496
|
};
|
|
1803
|
-
for (const
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
2497
|
+
for (const scenario of options.scenarios) {
|
|
2498
|
+
const baseline = await settle$1(() => options.baseline(scenario));
|
|
2499
|
+
const candidate = await settle$1(() => options.candidate(scenario));
|
|
2500
|
+
const { label, reason } = labelRun(scenario.expected, baseline, candidate);
|
|
2501
|
+
counts[label]++;
|
|
2502
|
+
scenarios.push({
|
|
2503
|
+
id: scenario.id,
|
|
2504
|
+
scenario,
|
|
2505
|
+
label,
|
|
2506
|
+
reason,
|
|
2507
|
+
baseline,
|
|
2508
|
+
candidate
|
|
2509
|
+
});
|
|
1807
2510
|
}
|
|
1808
|
-
|
|
1809
|
-
|
|
2511
|
+
return {
|
|
2512
|
+
scenarios,
|
|
2513
|
+
counts,
|
|
2514
|
+
ok: counts.regression === 0
|
|
2515
|
+
};
|
|
1810
2516
|
}
|
|
1811
|
-
function
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
2517
|
+
async function settle$1(run) {
|
|
2518
|
+
try {
|
|
2519
|
+
return await run();
|
|
2520
|
+
} catch (error) {
|
|
2521
|
+
return {
|
|
2522
|
+
outcome: "threw",
|
|
2523
|
+
detail: error instanceof Error ? error.message : String(error)
|
|
2524
|
+
};
|
|
1817
2525
|
}
|
|
1818
2526
|
}
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
reason:
|
|
1823
|
-
message: `Expected ${describe(left)}; the port made ${describe(right)}.`
|
|
2527
|
+
function labelRun(expected, baseline, candidate) {
|
|
2528
|
+
if (candidate.violations?.length) return {
|
|
2529
|
+
label: "regression",
|
|
2530
|
+
reason: `candidate broke a rule: ${list(candidate.violations)}`
|
|
1824
2531
|
};
|
|
1825
|
-
if (
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
reason: "model-method",
|
|
1829
|
-
message: `Model call #${left.call} used ${left.method}; the port used ${right.method}.`
|
|
1830
|
-
};
|
|
1831
|
-
if (!modelCallsMatch(options.modelCalls, left, right)) return {
|
|
1832
|
-
reason: "model-call-shape",
|
|
1833
|
-
message: shapeMessage(left, right)
|
|
1834
|
-
};
|
|
1835
|
-
if (!jsonEqual(errorIdentity$1(left.error), errorIdentity$1(right.error))) {
|
|
1836
|
-
const outcome = (error) => error ? `failed with ${error.name}: ${error.message}` : "succeeded";
|
|
1837
|
-
return {
|
|
1838
|
-
reason: "model-error",
|
|
1839
|
-
message: `Model call #${left.call} ${outcome(left.error)}; in the port it ${outcome(right.error)}.`
|
|
1840
|
-
};
|
|
1841
|
-
}
|
|
1842
|
-
return;
|
|
1843
|
-
}
|
|
1844
|
-
if (left.kind === "tool" && right.kind === "tool") {
|
|
1845
|
-
if (left.name !== right.name) return {
|
|
1846
|
-
reason: "tool-name",
|
|
1847
|
-
message: `Expected ${describe(left)}; the port made ${describe(right)}.`
|
|
1848
|
-
};
|
|
1849
|
-
if (!jsonEqual(left.input, right.input)) return {
|
|
1850
|
-
reason: "tool-input",
|
|
1851
|
-
message: `Expected ${describe(left)}; the port passed ${JSON.stringify(right.input)}.`
|
|
1852
|
-
};
|
|
1853
|
-
if (!jsonEqual(left.output, right.output) || !jsonEqual(errorIdentity$1(left.error), errorIdentity$1(right.error))) return {
|
|
1854
|
-
reason: "tool-result",
|
|
1855
|
-
message: `Tool '${left.name}' returned a different result.`
|
|
1856
|
-
};
|
|
1857
|
-
return;
|
|
1858
|
-
}
|
|
1859
|
-
if (left.kind === "call" && right.kind === "call") {
|
|
1860
|
-
if (left.name !== right.name) return {
|
|
1861
|
-
reason: "call-name",
|
|
1862
|
-
message: `Expected ${describe(left)}; the port made ${describe(right)}.`
|
|
1863
|
-
};
|
|
1864
|
-
if (!jsonEqual(left.args, right.args)) return {
|
|
1865
|
-
reason: "call-args",
|
|
1866
|
-
message: `Expected ${describe(left)}; the port passed ${JSON.stringify(right.args)}.`
|
|
1867
|
-
};
|
|
1868
|
-
if (!jsonEqual(left.result, right.result) || !jsonEqual(left.stream, right.stream) || !jsonEqual(errorIdentity$1(left.error), errorIdentity$1(right.error))) return {
|
|
1869
|
-
reason: "call-result",
|
|
1870
|
-
message: `Call #${left.call} to '${left.name}' returned a different result.`
|
|
1871
|
-
};
|
|
1872
|
-
return;
|
|
1873
|
-
}
|
|
1874
|
-
if (left.kind === "output" && right.kind === "output" && !jsonEqual(left.value, right.value)) return {
|
|
1875
|
-
reason: "output",
|
|
1876
|
-
message: `Expected ${describe(left)}; the port returned ${JSON.stringify(right.value)}.`
|
|
2532
|
+
if (candidate.outcome === "threw") return {
|
|
2533
|
+
label: "regression",
|
|
2534
|
+
reason: `candidate threw: ${String(candidate.detail)}`
|
|
1877
2535
|
};
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
* is the original's recording, `actual` the port's. Effects that are the same
|
|
1882
|
-
* on both sides are lined up first (longest common subsequence), so one extra
|
|
1883
|
-
* call does not shift every effect after it; between them, effects of the same
|
|
1884
|
-
* kind pair up and report what changed, and the rest are missing or extra.
|
|
1885
|
-
*/
|
|
1886
|
-
function compareRecordings(expected, actual, options = {}) {
|
|
1887
|
-
const select = (effects) => normalize(effects.flatMap((effect, index) => {
|
|
1888
|
-
if (options.kinds && !options.kinds.includes(effect.kind)) return [];
|
|
1889
|
-
const mapped = options.normalize ? options.normalize(effect, index, effects) : effect;
|
|
1890
|
-
return mapped ? [mapped] : [];
|
|
1891
|
-
}));
|
|
1892
|
-
const a = select(expected.effects);
|
|
1893
|
-
const b = select(actual.effects);
|
|
1894
|
-
const same = a.map((left) => b.map((right) => !differ(left, right, options)));
|
|
1895
|
-
const lcs = Array.from({ length: a.length + 1 }, () => Array.from({ length: b.length + 1 }, () => 0));
|
|
1896
|
-
for (let i = a.length - 1; i >= 0; i--) for (let j = b.length - 1; j >= 0; j--) lcs[i][j] = same[i][j] ? lcs[i + 1][j + 1] + 1 : Math.max(lcs[i + 1][j], lcs[i][j + 1]);
|
|
1897
|
-
const divergences = [];
|
|
1898
|
-
let lefts = [];
|
|
1899
|
-
let rights = [];
|
|
1900
|
-
const flushGap = (at) => {
|
|
1901
|
-
const unpairedRights = [...rights];
|
|
1902
|
-
const changed = [];
|
|
1903
|
-
for (const index of lefts) {
|
|
1904
|
-
const left = a[index];
|
|
1905
|
-
const k = unpairedRights.findIndex((j) => b[j].kind === left.kind);
|
|
1906
|
-
if (k === -1) {
|
|
1907
|
-
changed.push({
|
|
1908
|
-
index,
|
|
1909
|
-
reason: left.kind === "output" ? "output" : "missing-effect",
|
|
1910
|
-
expected: left,
|
|
1911
|
-
message: left.kind === "output" ? `The port never returned; the original returned ${JSON.stringify(left.value)}.` : `The port never made ${describe(left)}.`
|
|
1912
|
-
});
|
|
1913
|
-
continue;
|
|
1914
|
-
}
|
|
1915
|
-
const right = b[unpairedRights.splice(k, 1)[0]];
|
|
1916
|
-
const difference = differ(left, right, options);
|
|
1917
|
-
if (difference) changed.push({
|
|
1918
|
-
index,
|
|
1919
|
-
expected: left,
|
|
1920
|
-
actual: right,
|
|
1921
|
-
...difference
|
|
1922
|
-
});
|
|
1923
|
-
}
|
|
1924
|
-
for (const j of unpairedRights) {
|
|
1925
|
-
const extra = b[j];
|
|
1926
|
-
divergences.push({
|
|
1927
|
-
index: lefts[0] ?? at,
|
|
1928
|
-
reason: extra.kind === "output" ? "output" : "extra-effect",
|
|
1929
|
-
actual: extra,
|
|
1930
|
-
message: extra.kind === "output" ? `The original never returned; the port returned ${JSON.stringify(extra.value)}.` : `The port made ${describe(extra)}, which the original never made.`
|
|
1931
|
-
});
|
|
1932
|
-
}
|
|
1933
|
-
divergences.push(...changed);
|
|
1934
|
-
lefts = [];
|
|
1935
|
-
rights = [];
|
|
2536
|
+
if (baseline.violations?.length) return {
|
|
2537
|
+
label: "intended",
|
|
2538
|
+
reason: `baseline broke a rule: ${list(baseline.violations)}`
|
|
1936
2539
|
};
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
divergence: divergences[0],
|
|
1950
|
-
divergences
|
|
1951
|
-
} : {
|
|
1952
|
-
equivalent: true,
|
|
1953
|
-
divergences
|
|
2540
|
+
const sameOutcome = baseline.outcome === candidate.outcome;
|
|
2541
|
+
if (sameOutcome && structurallyEqual(baseline.effects, candidate.effects)) return {
|
|
2542
|
+
label: "same",
|
|
2543
|
+
reason: ""
|
|
2544
|
+
};
|
|
2545
|
+
if (expected !== void 0 && candidate.outcome === expected && baseline.outcome !== expected) return {
|
|
2546
|
+
label: "intended",
|
|
2547
|
+
reason: `only the candidate reached the expected '${expected}'`
|
|
2548
|
+
};
|
|
2549
|
+
return {
|
|
2550
|
+
label: "regression",
|
|
2551
|
+
reason: sameOutcome ? "effects differ" : `outcome differs: ${baseline.outcome} → ${candidate.outcome}`
|
|
1954
2552
|
};
|
|
1955
2553
|
}
|
|
2554
|
+
function list(items) {
|
|
2555
|
+
return items.map((item) => typeof item === "string" ? item : JSON.stringify(item)).join(", ");
|
|
2556
|
+
}
|
|
2557
|
+
function structurallyEqual(a, b) {
|
|
2558
|
+
if (Object.is(a, b)) return true;
|
|
2559
|
+
if (a instanceof Date || b instanceof Date) return a instanceof Date && b instanceof Date && a.getTime() === b.getTime();
|
|
2560
|
+
if (a instanceof Map || b instanceof Map) return a instanceof Map && b instanceof Map && structurallyEqual([...a], [...b]);
|
|
2561
|
+
if (a instanceof Set || b instanceof Set) return a instanceof Set && b instanceof Set && structurallyEqual([...a], [...b]);
|
|
2562
|
+
if (Array.isArray(a) || Array.isArray(b)) return Array.isArray(a) && Array.isArray(b) && a.length === b.length && a.every((item, index) => structurallyEqual(item, b[index]));
|
|
2563
|
+
if (!isRecord(a) || !isRecord(b)) return false;
|
|
2564
|
+
const keys = Object.keys(a);
|
|
2565
|
+
return keys.length === Object.keys(b).length && keys.every((key) => Object.hasOwn(b, key) && structurallyEqual(a[key], b[key]));
|
|
2566
|
+
}
|
|
1956
2567
|
//#endregion
|
|
1957
2568
|
//#region src/check-parity.ts
|
|
1958
2569
|
/**
|
|
@@ -2057,4 +2668,4 @@ async function checkParity(options) {
|
|
|
2057
2668
|
};
|
|
2058
2669
|
}
|
|
2059
2670
|
//#endregion
|
|
2060
|
-
export { AgentLintError, AgentUnknownStateError, RecordingDivergenceError, assertAgentMachine, canReach, checkParity, compareRecordings, compareRuns, createRecorder, explorePaths, firstDivergence, foldTraces, getDecisionRequests, judgeAgreement, lintAgentMachine, matchesTrajectory, proposeApprovalRules, proposeCaps, repeatRuns, runSeam, sameModelCallShape, simulateAgent, summarizeRepeats, textOutput, traceCoverage };
|
|
2671
|
+
export { AgentLintError, AgentUnknownStateError, RecordingDivergenceError, assertAgentMachine, canReach, checkCrashRecovery, checkParity, compareRecordings, compareRuns, createRecorder, explorePaths, firstDivergence, foldTraces, getDecisionRequests, judgeAgreement, lintAgentMachine, matchesTrajectory, proposeApprovalRules, proposeCaps, repeatRuns, runSeam, sameModelCallShape, simulateAgent, summarizeRepeats, textOutput, traceCoverage };
|