@statelyai/agent 2.0.0-alpha.38 → 2.0.0-alpha.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +45 -0
- package/dist/{act-CrX0JwWY.cjs → act-B0CVbmo4.cjs} +1 -1
- package/dist/{act-B9hLQls6.mjs → act-Bsz-hUfD.mjs} +1 -1
- package/dist/ai-sdk.cjs +4 -4
- package/dist/ai-sdk.d.cts +2 -3
- package/dist/ai-sdk.d.mts +2 -3
- package/dist/ai-sdk.mjs +4 -4
- package/dist/{decision-m5hGmHpF.mjs → decision-B41YBx5U.mjs} +1 -1
- package/dist/{decision-CMeuNZ1g.cjs → decision-D8bcIGXQ.cjs} +1 -1
- package/dist/{decision-object-DBcLWVvv.d.cts → decision-object-BB8XV5Ec.d.cts} +1 -1
- package/dist/{decision-object-B5zGSPFp.d.mts → decision-object-CH_sE694.d.mts} +1 -1
- package/dist/effects-B9IvUPVh.mjs +23 -0
- package/dist/effects-DrWcUGgc.cjs +34 -0
- package/dist/evals.cjs +810 -0
- package/dist/evals.d.cts +595 -0
- package/dist/evals.d.mts +595 -0
- package/dist/evals.mjs +801 -0
- package/dist/{event-log-DkVlk3Rh.mjs → event-log-CEq_bVXQ.mjs} +8 -4
- package/dist/{event-log-Dkc-yIYu.cjs → event-log-dSOnXHuG.cjs} +7 -15
- package/dist/{event-log-store-Brx9tOHt.mjs → event-log-store-DLAfNM34.mjs} +1 -1
- package/dist/{event-log-store-BBS8so8a.d.mts → event-log-store-DYun526m.d.mts} +15 -14
- package/dist/{event-log-store-Cl-YWDnq.cjs → event-log-store-Dey9ufA4.cjs} +1 -1
- package/dist/{event-log-store-bc4ICGcS.d.cts → event-log-store-aFXK6yZX.d.cts} +15 -14
- package/dist/index.cjs +131 -93
- package/dist/index.d.cts +73 -134
- package/dist/index.d.mts +73 -134
- package/dist/index.mjs +112 -70
- package/dist/internal-BzlpzSux.d.cts +14 -0
- package/dist/internal-BzlpzSux.d.mts +14 -0
- package/dist/log.cjs +2 -3
- package/dist/log.d.cts +2 -2
- package/dist/log.d.mts +2 -2
- package/dist/log.mjs +3 -3
- package/dist/machines.cjs +1 -719
- package/dist/machines.d.cts +2 -424
- package/dist/machines.d.mts +2 -424
- package/dist/machines.mjs +2 -709
- package/dist/{model-source-kjGwqAV8.mjs → model-source-BrsLDvHL.mjs} +1 -1
- package/dist/{model-source-B4WQ8Idp.cjs → model-source-YpxjmWuX.cjs} +1 -1
- package/dist/openai.cjs +3 -3
- package/dist/openai.d.cts +2 -3
- package/dist/openai.d.mts +2 -3
- package/dist/openai.mjs +3 -3
- package/dist/otel.d.cts +1 -1
- package/dist/otel.d.mts +1 -1
- package/dist/policy.cjs +77 -0
- package/dist/policy.d.cts +143 -0
- package/dist/policy.d.mts +143 -0
- package/dist/policy.mjs +75 -0
- package/dist/{effects-BT_cSY0j.mjs → run-agent-BE7DOWD6.mjs} +150 -184
- package/dist/{run-agent-DlPzBWj-.d.mts → run-agent-DHlq87G7.d.mts} +18 -34
- package/dist/{effects-DDseYtJk.cjs → run-agent-De3xVFGw.cjs} +149 -201
- package/dist/{run-agent-30mH2moE.d.cts → run-agent-Dq0oSFRm.d.cts} +18 -34
- package/dist/{setup-agent-BDmNr6PC.mjs → setup-agent-CLm6QVGy.mjs} +3 -3
- package/dist/{setup-agent-BaNyz6Q3.d.cts → setup-agent-Cv8Hk3mX.d.cts} +12 -11
- package/dist/{setup-agent-C1CG_nfX.d.mts → setup-agent-DgFAhaPJ.d.mts} +12 -11
- package/dist/{setup-agent-CmEToB8y.cjs → setup-agent-Dv5VpuHe.cjs} +3 -3
- package/dist/tanstack-ai.cjs +4 -4
- package/dist/tanstack-ai.d.cts +2 -3
- package/dist/tanstack-ai.d.mts +2 -3
- package/dist/tanstack-ai.mjs +4 -4
- package/dist/testing.cjs +122 -996
- package/dist/testing.d.cts +36 -695
- package/dist/testing.d.mts +36 -695
- package/dist/testing.mjs +112 -977
- package/dist/{text-logic-BqxalgMr.d.mts → text-logic-x17uLBwA.d.cts} +238 -2
- package/dist/{text-logic-CSf9bqpP.d.cts → text-logic-x17uLBwA.d.mts} +238 -2
- package/dist/trajectory-BFjXaooB.d.cts +96 -0
- package/dist/trajectory-BFjXaooB.d.mts +96 -0
- package/dist/trajectory-BUXj_1Ue.mjs +152 -0
- package/dist/trajectory-CoZSv4wF.cjs +157 -0
- package/dist/{usage-Bm5nvpow.d.cts → usage-B4B-TQSV.d.mts} +1 -1
- package/dist/{usage-BnikdOca.d.mts → usage-DvGcoK6x.d.cts} +1 -1
- package/dist/{utils-7lSa4H-i.d.cts → utils-B8k-YPYI.d.mts} +23 -17
- package/dist/{utils-CBbKw2fM.d.mts → utils-DAclF-kw.d.cts} +23 -17
- package/dist/validate.cjs +4 -4
- package/dist/validate.d.cts +2 -2
- package/dist/validate.d.mts +2 -2
- package/dist/validate.mjs +4 -4
- package/dist/{verify-DJHfEfm6.d.cts → verify-BM0sGmsf.d.cts} +42 -6
- package/dist/{verify-ChyX6jq0.mjs → verify-CWlg3j7B.mjs} +153 -45
- package/dist/{verify-CwlombbC.cjs → verify-Dek6gHCS.cjs} +153 -45
- package/dist/{verify-i9Hiwc5g.d.mts → verify-H3Pr2gNw.d.mts} +42 -6
- package/dist/{with-model-calls-B_WLmOSl.cjs → with-model-calls-D50Ca6qW.cjs} +2 -2
- package/dist/{with-model-calls-BkOwdroS.mjs → with-model-calls-uY-KKliA.mjs} +2 -2
- package/docs/advanced.md +5 -5
- package/docs/choosing-a-run-mode.md +7 -7
- package/docs/debugging.md +6 -8
- package/docs/decisions.md +5 -4
- package/docs/environment-and-policy.md +1 -1
- package/docs/evals.md +57 -43
- package/docs/event-log.md +13 -15
- package/docs/from-a-loop.md +3 -3
- package/docs/generate-machines.md +4 -4
- package/docs/hosts.md +20 -27
- package/docs/human-in-the-loop.md +5 -8
- package/docs/index.md +4 -4
- package/docs/judgments.md +5 -7
- package/docs/langgraph-comparison.md +6 -6
- package/docs/machines-as-data.md +7 -12
- package/docs/machines-presets.md +17 -120
- package/docs/machines.md +13 -13
- package/docs/models-and-providers.md +19 -25
- package/docs/multi-agent.md +1 -1
- package/docs/observability.md +6 -8
- package/docs/persistence.md +10 -20
- package/docs/quickstart.md +3 -3
- package/docs/statechart-policy-examples.md +1 -1
- package/docs/steps.md +5 -6
- package/docs/strangler-fig.md +72 -7
- package/docs/text-requests.md +10 -12
- package/docs/thinking-in-state-machines.md +8 -8
- package/docs/tools.md +10 -12
- package/docs/upgrading.md +1 -1
- package/docs/usage-and-budgets.md +51 -56
- package/docs/validate-and-repair.md +9 -14
- package/docs/verify.md +53 -7
- package/docs/xstate-4-and-5-apps.md +19 -28
- package/llms.txt +1 -1
- package/package.json +31 -6
- package/readme.md +32 -24
- package/dist/internal-C3Lqu538.d.cts +0 -43
- package/dist/internal-DUcdhk-i.d.mts +0 -43
- package/dist/types-DQUOOaZO.d.cts +0 -238
- package/dist/types-DQUOOaZO.d.mts +0 -238
- package/skills/add-evals/SKILL.md +0 -152
- package/skills/add-traces/SKILL.md +0 -152
- package/skills/build-machine/SKILL.md +0 -84
- package/skills/generate-machine/SKILL.md +0 -335
- package/skills/graph-engineering/SKILL.md +0 -43
- package/skills/improve-machine/SKILL.md +0 -261
- package/skills/improve-machine/reference.md +0 -101
- package/skills/loop-to-machine/SKILL.md +0 -205
- package/skills/loop-to-machine/policy.md +0 -154
- package/skills/loop-to-machine/reference.md +0 -260
package/dist/testing.cjs
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
-
const require_event_log = require("./event-log-
|
|
3
|
-
const require_decision = require("./decision-
|
|
4
|
-
const require_act = require("./act-
|
|
5
|
-
const
|
|
6
|
-
const
|
|
7
|
-
const
|
|
8
|
-
|
|
2
|
+
const require_event_log = require("./event-log-dSOnXHuG.cjs");
|
|
3
|
+
const require_decision = require("./decision-D8bcIGXQ.cjs");
|
|
4
|
+
const require_act = require("./act-B0CVbmo4.cjs");
|
|
5
|
+
const require_event_log_store = require("./event-log-store-Dey9ufA4.cjs");
|
|
6
|
+
const require_run_agent = require("./run-agent-De3xVFGw.cjs");
|
|
7
|
+
const require_effects = require("./effects-DrWcUGgc.cjs");
|
|
8
|
+
const require_verify = require("./verify-Dek6gHCS.cjs");
|
|
9
|
+
const require_trajectory = require("./trajectory-CoZSv4wF.cjs");
|
|
9
10
|
//#region src/equivalence.ts
|
|
10
11
|
/**
|
|
11
12
|
* Verified equivalence — prove a port behaves like the code it replaces.
|
|
@@ -47,11 +48,12 @@ let xstate = require("xstate");
|
|
|
47
48
|
var RecordingDivergenceError = class extends Error {
|
|
48
49
|
name = "RecordingDivergenceError";
|
|
49
50
|
};
|
|
50
|
-
function toRecordedError(error) {
|
|
51
|
+
function toRecordedError(error, path = /* @__PURE__ */ new Set()) {
|
|
51
52
|
if (!(error instanceof Error)) return {
|
|
52
53
|
name: "Error",
|
|
53
54
|
message: String(error)
|
|
54
55
|
};
|
|
56
|
+
path.add(error);
|
|
55
57
|
const recorded = {
|
|
56
58
|
name: error.name,
|
|
57
59
|
message: error.message
|
|
@@ -67,10 +69,39 @@ function toRecordedError(error) {
|
|
|
67
69
|
if (key !== void 0 && error[symbol] === true) markers.add(key);
|
|
68
70
|
}
|
|
69
71
|
if (markers.size) recorded.markers = [...markers];
|
|
72
|
+
if (error.cause instanceof Error && !path.has(error.cause)) recorded.cause = toRecordedError(error.cause, path);
|
|
73
|
+
if (error instanceof AggregateError && Array.isArray(error.errors)) recorded.errors = error.errors.flatMap((item) => {
|
|
74
|
+
if (!(item instanceof Error)) try {
|
|
75
|
+
return [{ value: toJson(item) }];
|
|
76
|
+
} catch {
|
|
77
|
+
try {
|
|
78
|
+
return [{ value: String(item) }];
|
|
79
|
+
} catch {
|
|
80
|
+
return [{ value: "[unserializable]" }];
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
return path.has(item) ? [] : [toRecordedError(item, path)];
|
|
84
|
+
});
|
|
85
|
+
path.delete(error);
|
|
70
86
|
return recorded;
|
|
71
87
|
}
|
|
72
|
-
function fromRecordedError(
|
|
73
|
-
const
|
|
88
|
+
function fromRecordedError(recorded, reviveError) {
|
|
89
|
+
const { data, markers, name, ownName, cause, errors, ...fields } = recorded;
|
|
90
|
+
const revivedErrors = errors?.map((item) => "name" in item ? fromRecordedError(item, reviveError) : structuredClone(item.value));
|
|
91
|
+
const define = (target, key, value) => Object.defineProperty(target, key, {
|
|
92
|
+
value,
|
|
93
|
+
writable: true,
|
|
94
|
+
configurable: true
|
|
95
|
+
});
|
|
96
|
+
const custom = reviveError?.(structuredClone(recorded));
|
|
97
|
+
if (custom) {
|
|
98
|
+
if (cause && custom.cause === void 0) define(custom, "cause", fromRecordedError(cause, reviveError));
|
|
99
|
+
if (revivedErrors && custom instanceof AggregateError) define(custom, "errors", revivedErrors);
|
|
100
|
+
return custom;
|
|
101
|
+
}
|
|
102
|
+
const base = revivedErrors ? new AggregateError(revivedErrors, fields.message) : new Error(fields.message);
|
|
103
|
+
const error = Object.assign(base, fields, data);
|
|
104
|
+
if (cause) define(error, "cause", fromRecordedError(cause, reviveError));
|
|
74
105
|
if (ownName || name !== "Error") Object.defineProperty(error, "name", {
|
|
75
106
|
value: name,
|
|
76
107
|
writable: true,
|
|
@@ -133,6 +164,25 @@ async function* iterate(chunks) {
|
|
|
133
164
|
for (const chunk of chunks) yield structuredClone(chunk);
|
|
134
165
|
}
|
|
135
166
|
/**
|
|
167
|
+
* Yields what `revived` yields (awaited), if it is an async iterable, then
|
|
168
|
+
* throws the recorded `failure`, if any. Like the recording wrapper, it calls
|
|
169
|
+
* `onError` only for a failure the port reads to, or one its consumer throws
|
|
170
|
+
* in (`.throw()`); `.return()` ends it with no error.
|
|
171
|
+
*/
|
|
172
|
+
async function* replayStream(revived, onError, failure) {
|
|
173
|
+
try {
|
|
174
|
+
const value = await revived;
|
|
175
|
+
if (require_effects.isAsyncIterable(value)) yield* value;
|
|
176
|
+
} catch (error) {
|
|
177
|
+
onError(toRecordedError(error));
|
|
178
|
+
throw error;
|
|
179
|
+
}
|
|
180
|
+
if (failure) {
|
|
181
|
+
onError(failure.recorded);
|
|
182
|
+
throw failure.error;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
136
186
|
* Creates a {@link Recorder}. With no options it records: every wrapped call
|
|
137
187
|
* runs for real. With `replay` it answers from a recording instead.
|
|
138
188
|
*/
|
|
@@ -147,6 +197,7 @@ function createRecorder(options = {}) {
|
|
|
147
197
|
const normalizeEffect = (effect, list) => options.normalize?.(effect, list.indexOf(effect), list) ?? effect;
|
|
148
198
|
let modelCalls = 0;
|
|
149
199
|
let toolsInFlight = 0;
|
|
200
|
+
const reviveRecordedError = (error) => fromRecordedError(error, options.reviveError);
|
|
150
201
|
/**
|
|
151
202
|
* The earliest unused recorded call that fits this one: one `translate`
|
|
152
203
|
* adapts, or one with the same method and shape. Not by position, so a call
|
|
@@ -196,13 +247,23 @@ function createRecorder(options = {}) {
|
|
|
196
247
|
const { recorded, answer } = replayed;
|
|
197
248
|
usedModels.add(recorded);
|
|
198
249
|
effect.replayed = recorded.call;
|
|
250
|
+
const recordError = (error) => {
|
|
251
|
+
effect.error = error;
|
|
252
|
+
};
|
|
253
|
+
if (answer.error && answer.stream && streamed && seam === "openai") {
|
|
254
|
+
effect.stream = answer.stream;
|
|
255
|
+
return replayStream(iterate(answer.stream), recordError, {
|
|
256
|
+
recorded: answer.error,
|
|
257
|
+
error: reviveRecordedError(answer.error)
|
|
258
|
+
});
|
|
259
|
+
}
|
|
199
260
|
if (answer.error) {
|
|
200
261
|
effect.error = answer.error;
|
|
201
|
-
throw
|
|
262
|
+
throw reviveRecordedError(answer.error);
|
|
202
263
|
}
|
|
203
264
|
effect.response = answer.response;
|
|
204
265
|
effect.stream = answer.stream;
|
|
205
|
-
if (seam === "openai") return streamed ? iterate(answer.stream ?? []) : structuredClone(answer.response);
|
|
266
|
+
if (seam === "openai") return streamed ? replayStream(iterate(answer.stream ?? []), recordError) : structuredClone(answer.response);
|
|
206
267
|
return streamed ? {
|
|
207
268
|
...reviveTimestamps(structuredClone(answer.response) ?? {}),
|
|
208
269
|
stream: streamFrom(answer.stream ?? [])
|
|
@@ -232,9 +293,14 @@ function createRecorder(options = {}) {
|
|
|
232
293
|
if (seam === "openai") {
|
|
233
294
|
const source = result;
|
|
234
295
|
return (async function* () {
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
296
|
+
try {
|
|
297
|
+
for await (const chunk of source) {
|
|
298
|
+
parts.push(toJson(chunk));
|
|
299
|
+
yield chunk;
|
|
300
|
+
}
|
|
301
|
+
} catch (error) {
|
|
302
|
+
effect.error = toRecordedError(error);
|
|
303
|
+
throw error;
|
|
238
304
|
}
|
|
239
305
|
})();
|
|
240
306
|
}
|
|
@@ -290,7 +356,7 @@ function createRecorder(options = {}) {
|
|
|
290
356
|
if (recorded.sync) effect.sync = true;
|
|
291
357
|
if (recorded.error) {
|
|
292
358
|
effect.error = recorded.error;
|
|
293
|
-
throw
|
|
359
|
+
throw reviveRecordedError(recorded.error);
|
|
294
360
|
}
|
|
295
361
|
effect.output = recorded.output;
|
|
296
362
|
return structuredClone(recorded.output);
|
|
@@ -343,14 +409,23 @@ function createRecorder(options = {}) {
|
|
|
343
409
|
throw error;
|
|
344
410
|
}
|
|
345
411
|
usedCalls.add(recorded);
|
|
346
|
-
if (recorded.error) {
|
|
347
|
-
effect.error = recorded.error;
|
|
348
|
-
throw fromRecordedError(recorded.error);
|
|
349
|
-
}
|
|
350
412
|
if (recorded.stream) {
|
|
351
413
|
effect.stream = recorded.stream;
|
|
352
414
|
const chunks = structuredClone(recorded.stream);
|
|
353
|
-
|
|
415
|
+
const revived = revive ? revive(chunks) : iterate(chunks);
|
|
416
|
+
const { error } = recorded;
|
|
417
|
+
if (!error && revive) return revived;
|
|
418
|
+
const recordError = (thrown) => {
|
|
419
|
+
effect.error = thrown;
|
|
420
|
+
};
|
|
421
|
+
return replayStream(revived, recordError, error && {
|
|
422
|
+
recorded: error,
|
|
423
|
+
error: reviveRecordedError(error)
|
|
424
|
+
});
|
|
425
|
+
}
|
|
426
|
+
if (recorded.error) {
|
|
427
|
+
effect.error = recorded.error;
|
|
428
|
+
throw reviveRecordedError(recorded.error);
|
|
354
429
|
}
|
|
355
430
|
effect.result = recorded.result;
|
|
356
431
|
const result = structuredClone(recorded.result);
|
|
@@ -538,7 +613,7 @@ function modelCallShape(effect) {
|
|
|
538
613
|
* that started after the previous one settled begins a new group, so
|
|
539
614
|
* sequential calls keep their order.
|
|
540
615
|
*/
|
|
541
|
-
function normalize
|
|
616
|
+
function normalize(effects) {
|
|
542
617
|
const result = [];
|
|
543
618
|
let batch = [];
|
|
544
619
|
const flush = () => {
|
|
@@ -629,7 +704,7 @@ function differ(left, right, options) {
|
|
|
629
704
|
* kind pair up and report what changed, and the rest are missing or extra.
|
|
630
705
|
*/
|
|
631
706
|
function compareRecordings(expected, actual, options = {}) {
|
|
632
|
-
const select = (effects) => normalize
|
|
707
|
+
const select = (effects) => normalize(effects.flatMap((effect, index) => {
|
|
633
708
|
if (options.kinds && !options.kinds.includes(effect.kind)) return [];
|
|
634
709
|
const mapped = options.normalize ? options.normalize(effect, index, effects) : effect;
|
|
635
710
|
return mapped ? [mapped] : [];
|
|
@@ -901,11 +976,11 @@ async function crashAt(point, ctx) {
|
|
|
901
976
|
const log = await backing.read(THREAD);
|
|
902
977
|
try {
|
|
903
978
|
const replayed = require_event_log.replay(options.actors !== void 0 ? machine.provide({ actors: options.actors }) : machine, log);
|
|
904
|
-
const fromLog = stableStringify(
|
|
979
|
+
const fromLog = stableStringify(require_run_agent.toJsonValue({
|
|
905
980
|
value: replayed.snapshot.value,
|
|
906
981
|
context: replayed.snapshot.context
|
|
907
982
|
}));
|
|
908
|
-
const live = stableStringify(
|
|
983
|
+
const live = stableStringify(require_run_agent.toJsonValue({
|
|
909
984
|
value: result.snapshot.value,
|
|
910
985
|
context: result.snapshot.context
|
|
911
986
|
}));
|
|
@@ -923,7 +998,7 @@ async function crashAt(point, ctx) {
|
|
|
923
998
|
async function runTurns(machine, options, world, store, resuming, signal) {
|
|
924
999
|
const tracked = trackActors(machine, world);
|
|
925
1000
|
const executors = trackExecutors(options.executors ?? {}, world);
|
|
926
|
-
const actors = options.actors !== void 0 ? Object.fromEntries(Object.entries(options.actors).map(([key, logic]) => [key,
|
|
1001
|
+
const actors = options.actors !== void 0 ? Object.fromEntries(Object.entries(options.actors).map(([key, logic]) => [key, require_run_agent.isStateMachineLogic(logic) ? trackActors(logic, world) : trackLogic(logic, world)])) : void 0;
|
|
927
1002
|
const before = resuming?.crashed;
|
|
928
1003
|
const log = before !== void 0 ? await store.read(THREAD) : [];
|
|
929
1004
|
const durable = new Set(log.map((entry) => entry.id));
|
|
@@ -998,7 +1073,7 @@ async function runTurns(machine, options, world, store, resuming, signal) {
|
|
|
998
1073
|
const turn = (event, track) => {
|
|
999
1074
|
let awaiting = event !== void 0 && track === "host" ? require_event_log.canonicalEventJson(event) : void 0;
|
|
1000
1075
|
let awaitingTimer = track === "timer";
|
|
1001
|
-
live =
|
|
1076
|
+
live = require_run_agent.createAgentRuntime(tracked, {
|
|
1002
1077
|
...options.runtime,
|
|
1003
1078
|
onEvent: (entry) => {
|
|
1004
1079
|
options.runtime?.onEvent?.(entry);
|
|
@@ -1036,7 +1111,7 @@ async function runTurns(machine, options, world, store, resuming, signal) {
|
|
|
1036
1111
|
...signal !== void 0 ? { signal } : {}
|
|
1037
1112
|
});
|
|
1038
1113
|
for (const index of redeliver.splice(0)) send(index);
|
|
1039
|
-
return
|
|
1114
|
+
return require_run_agent.runToQuiescence(live, {
|
|
1040
1115
|
...options.input !== void 0 ? { input: options.input } : {},
|
|
1041
1116
|
...event !== void 0 ? { event } : {}
|
|
1042
1117
|
});
|
|
@@ -1156,7 +1231,7 @@ function countBy(effects) {
|
|
|
1156
1231
|
}
|
|
1157
1232
|
function outcomeOf(result) {
|
|
1158
1233
|
const snapshot = result.snapshot;
|
|
1159
|
-
return stableStringify(
|
|
1234
|
+
return stableStringify(require_run_agent.toJsonValue({
|
|
1160
1235
|
status: result.status,
|
|
1161
1236
|
...result.status === "done" ? { output: result.output } : {},
|
|
1162
1237
|
...result.status === "error" ? { cause: result.cause } : {},
|
|
@@ -1227,7 +1302,7 @@ function trackExecutors(executors, world) {
|
|
|
1227
1302
|
const VOLATILE_REQUEST_KEYS = /* @__PURE__ */ new Set(["runId"]);
|
|
1228
1303
|
function requestFingerprint(request) {
|
|
1229
1304
|
try {
|
|
1230
|
-
const json =
|
|
1305
|
+
const json = require_run_agent.toJsonValue(request);
|
|
1231
1306
|
if (json && typeof json === "object" && !Array.isArray(json)) for (const key of VOLATILE_REQUEST_KEYS) delete json[key];
|
|
1232
1307
|
return stableStringify(json);
|
|
1233
1308
|
} catch {
|
|
@@ -1248,7 +1323,7 @@ function trackActors(machine, world, ancestors = /* @__PURE__ */ new Set()) {
|
|
|
1248
1323
|
const wrapped = {};
|
|
1249
1324
|
for (const [key, logic] of Object.entries(sources)) {
|
|
1250
1325
|
if (require_event_log.isTextLogic(logic) || require_decision.isDecisionLogic(logic) || require_act.isActLogic(logic)) continue;
|
|
1251
|
-
if (
|
|
1326
|
+
if (require_run_agent.isStateMachineLogic(logic)) {
|
|
1252
1327
|
const inner = trackActors(logic, world, visited);
|
|
1253
1328
|
if (inner !== logic) wrapped[key] = inner;
|
|
1254
1329
|
continue;
|
|
@@ -1307,156 +1382,6 @@ function trackLogic(logic, world) {
|
|
|
1307
1382
|
return Object.create(logic, overrides);
|
|
1308
1383
|
}
|
|
1309
1384
|
//#endregion
|
|
1310
|
-
//#region src/trajectory.ts
|
|
1311
|
-
/**
|
|
1312
|
-
* Classifies one item. Event-shaped things (log entries, events) become events;
|
|
1313
|
-
* everything else is a state value. A snapshot is unwrapped to its `value`.
|
|
1314
|
-
* @internal
|
|
1315
|
-
*/
|
|
1316
|
-
function normalize(item) {
|
|
1317
|
-
if (require_event_log.isRecord(item)) {
|
|
1318
|
-
const nested = item["event"];
|
|
1319
|
-
if (require_event_log.isRecord(nested) && typeof nested["type"] === "string") return {
|
|
1320
|
-
kind: "event",
|
|
1321
|
-
event: nested
|
|
1322
|
-
};
|
|
1323
|
-
if (typeof item["type"] === "string") return {
|
|
1324
|
-
kind: "event",
|
|
1325
|
-
event: item
|
|
1326
|
-
};
|
|
1327
|
-
if ("value" in item) return {
|
|
1328
|
-
kind: "state",
|
|
1329
|
-
value: item["value"]
|
|
1330
|
-
};
|
|
1331
|
-
}
|
|
1332
|
-
return {
|
|
1333
|
-
kind: "state",
|
|
1334
|
-
value: item
|
|
1335
|
-
};
|
|
1336
|
-
}
|
|
1337
|
-
/** Every leaf of a state value as a dot path (`{ review: 'editing' }` → `review.editing`). @internal */
|
|
1338
|
-
function statePaths$1(value) {
|
|
1339
|
-
if (typeof value === "string") return [value];
|
|
1340
|
-
if (!require_event_log.isRecord(value)) return [];
|
|
1341
|
-
return Object.entries(value).flatMap(([key, child]) => statePaths$1(child).map((path) => `${key}.${path}`));
|
|
1342
|
-
}
|
|
1343
|
-
/**
|
|
1344
|
-
* A string expectation matches a state value when it names one of its leaves or
|
|
1345
|
-
* an ancestor of one, so `'review'` matches `{ review: 'editing' }`. @internal
|
|
1346
|
-
*/
|
|
1347
|
-
function stateMatchesPath(path, value) {
|
|
1348
|
-
return statePaths$1(value).some((leaf) => leaf === path || leaf.startsWith(`${path}.`));
|
|
1349
|
-
}
|
|
1350
|
-
function deepEqual(a, b) {
|
|
1351
|
-
if (Object.is(a, b)) return true;
|
|
1352
|
-
if (Array.isArray(a) || Array.isArray(b)) return Array.isArray(a) && Array.isArray(b) && a.length === b.length && a.every((item, index) => deepEqual(item, b[index]));
|
|
1353
|
-
if (!require_event_log.isRecord(a) || !require_event_log.isRecord(b)) return false;
|
|
1354
|
-
const keys = Object.keys(a);
|
|
1355
|
-
return keys.length === Object.keys(b).length && keys.every((key) => deepEqual(a[key], b[key]));
|
|
1356
|
-
}
|
|
1357
|
-
/**
|
|
1358
|
-
* Normalizes ONE expected item into a predicate over actual items, so the
|
|
1359
|
-
* expectation is classified once instead of once per candidate:
|
|
1360
|
-
*
|
|
1361
|
-
* - A string expectation matches an event's `type`, or a state value it names.
|
|
1362
|
-
* - An event expectation matches an event with the same `type` whose other
|
|
1363
|
-
* declared keys deep-equal, so payload details are opt-in.
|
|
1364
|
-
* - A state-value expectation matches structurally.
|
|
1365
|
-
* @internal
|
|
1366
|
-
*/
|
|
1367
|
-
function matcherFor(expected) {
|
|
1368
|
-
if (typeof expected === "string") return (actual) => {
|
|
1369
|
-
const actualItem = normalize(actual);
|
|
1370
|
-
return actualItem.kind === "event" ? actualItem.event.type === expected : stateMatchesPath(expected, actualItem.value);
|
|
1371
|
-
};
|
|
1372
|
-
const expectedItem = normalize(expected);
|
|
1373
|
-
switch (expectedItem.kind) {
|
|
1374
|
-
case "event": return (actual) => {
|
|
1375
|
-
const actualItem = normalize(actual);
|
|
1376
|
-
return actualItem.kind === "event" && Object.keys(expectedItem.event).every((key) => deepEqual(expectedItem.event[key], actualItem.event[key]));
|
|
1377
|
-
};
|
|
1378
|
-
case "state": return (actual) => {
|
|
1379
|
-
const actualItem = normalize(actual);
|
|
1380
|
-
return actualItem.kind === "state" && deepEqual(expectedItem.value, actualItem.value);
|
|
1381
|
-
};
|
|
1382
|
-
}
|
|
1383
|
-
}
|
|
1384
|
-
/**
|
|
1385
|
-
* Matches a run's trajectory against an expected one.
|
|
1386
|
-
*
|
|
1387
|
-
* Both trajectories may be state values or events collected from
|
|
1388
|
-
* `onTransition`;
|
|
1389
|
-
* items are compared by shape, so `['prompting', 'drafting']` scores a state
|
|
1390
|
-
* path and `['PROMPT_SUBMITTED', 'SEND']` scores an event log with the same
|
|
1391
|
-
* call.
|
|
1392
|
-
*
|
|
1393
|
-
* Semantics are an ordered subsequence: every expected item must appear in
|
|
1394
|
-
* `actual`, in order, with gaps allowed. Pass `{ exact: true }` to require
|
|
1395
|
-
* equality instead.
|
|
1396
|
-
*
|
|
1397
|
-
* @example Trajectory assertions in a test
|
|
1398
|
-
* ```ts
|
|
1399
|
-
* const statePath: unknown[] = [];
|
|
1400
|
-
* const result = await runToQuiescence(
|
|
1401
|
-
* createAgentRuntime(machine, {
|
|
1402
|
-
* executors,
|
|
1403
|
-
* onTransition: (snapshot) => statePath.push(snapshot.value),
|
|
1404
|
-
* }),
|
|
1405
|
-
* { input },
|
|
1406
|
-
* );
|
|
1407
|
-
*
|
|
1408
|
-
* const path = matchesTrajectory(statePath, ['prompting', 'drafting', 'sent']);
|
|
1409
|
-
* expect(path.matched, JSON.stringify(path.firstMiss)).toBe(true);
|
|
1410
|
-
* expect(matchesTrajectory(events, ['PROMPT_SUBMITTED', 'SEND']).matched).toBe(true);
|
|
1411
|
-
* ```
|
|
1412
|
-
*
|
|
1413
|
-
* @example Partial credit in an eval scorer
|
|
1414
|
-
* ```ts
|
|
1415
|
-
* const match = matchesTrajectory(output.statePath, expected.statePath);
|
|
1416
|
-
* return { name: 'state_path', score: match.score, metadata: { ...match.firstMiss } };
|
|
1417
|
-
* ```
|
|
1418
|
-
*/
|
|
1419
|
-
function matchesTrajectory(actual, expected, options = {}) {
|
|
1420
|
-
const expectedCount = expected.length;
|
|
1421
|
-
if (expectedCount === 0) throw new require_event_log.AgentError("empty-trajectory", "matchesTrajectory: expected trajectory must contain at least one item.");
|
|
1422
|
-
const exact = options.exact === true;
|
|
1423
|
-
let cursor = 0;
|
|
1424
|
-
let matchedCount = 0;
|
|
1425
|
-
let firstMiss;
|
|
1426
|
-
for (let index = 0; index < expectedCount; index++) {
|
|
1427
|
-
const want = expected[index];
|
|
1428
|
-
const matches = matcherFor(want);
|
|
1429
|
-
const searchedFrom = exact ? index : cursor;
|
|
1430
|
-
let found = -1;
|
|
1431
|
-
for (let at = searchedFrom; at < actual.length; at++) {
|
|
1432
|
-
if (matches(actual[at])) {
|
|
1433
|
-
found = at;
|
|
1434
|
-
break;
|
|
1435
|
-
}
|
|
1436
|
-
if (exact) break;
|
|
1437
|
-
}
|
|
1438
|
-
if (found === -1) {
|
|
1439
|
-
firstMiss = {
|
|
1440
|
-
index,
|
|
1441
|
-
expected: want,
|
|
1442
|
-
searchedFrom
|
|
1443
|
-
};
|
|
1444
|
-
break;
|
|
1445
|
-
}
|
|
1446
|
-
matchedCount++;
|
|
1447
|
-
cursor = found + 1;
|
|
1448
|
-
}
|
|
1449
|
-
const matched = !firstMiss && (!exact || actual.length === expectedCount);
|
|
1450
|
-
const denominator = exact ? Math.max(expectedCount, actual.length) : expectedCount;
|
|
1451
|
-
return {
|
|
1452
|
-
matched,
|
|
1453
|
-
matchedCount,
|
|
1454
|
-
expectedCount,
|
|
1455
|
-
score: matched ? 1 : denominator === 0 ? 1 : matchedCount / denominator,
|
|
1456
|
-
...firstMiss ? { firstMiss } : {}
|
|
1457
|
-
};
|
|
1458
|
-
}
|
|
1459
|
-
//#endregion
|
|
1460
1385
|
//#region src/seam.ts
|
|
1461
1386
|
/** The only own keys an executor result carries. @internal */
|
|
1462
1387
|
const TEXT_RESULT_KEYS = /* @__PURE__ */ new Set([
|
|
@@ -1611,7 +1536,7 @@ async function runSeam(machine, options) {
|
|
|
1611
1536
|
let result;
|
|
1612
1537
|
const maxTurns = options.maxTurns ?? 12;
|
|
1613
1538
|
for (let turn = 0; turn <= maxTurns; turn++) {
|
|
1614
|
-
result = await
|
|
1539
|
+
result = await require_run_agent.runToQuiescence(require_run_agent.createAgentRuntime(machine, {
|
|
1615
1540
|
...options.actors ? { actors: options.actors } : {},
|
|
1616
1541
|
executors,
|
|
1617
1542
|
onTransition: (next, causedBy) => {
|
|
@@ -1665,802 +1590,6 @@ async function runSeam(machine, options) {
|
|
|
1665
1590
|
};
|
|
1666
1591
|
}
|
|
1667
1592
|
//#endregion
|
|
1668
|
-
//#region src/internal/trace-states.ts
|
|
1669
|
-
/** The `value` of a trace event's `from` or `snapshot`. @internal */
|
|
1670
|
-
function valueOf(holder) {
|
|
1671
|
-
return require_event_log.isRecord(holder) ? holder.value : void 0;
|
|
1672
|
-
}
|
|
1673
|
-
/** A decision request on its first attempt (no failed attempts yet). @internal */
|
|
1674
|
-
function isFirstDecision(request) {
|
|
1675
|
-
return require_event_log.isRecord(request) && request.kind === "decision" && (!Array.isArray(request.attempts) || request.attempts.length === 0);
|
|
1676
|
-
}
|
|
1677
|
-
/**
|
|
1678
|
-
* Every active leaf of a state value as a dot path:
|
|
1679
|
-
* `'a'` → `['a']`; `{ a: 'b' }` → `['a.b']`; `{ p: { x: 'one', y: {} } }` → `['p.x.one', 'p.y']`.
|
|
1680
|
-
* @internal
|
|
1681
|
-
*/
|
|
1682
|
-
function statePaths(value, prefix = "") {
|
|
1683
|
-
if (typeof value === "string") return [prefix + value];
|
|
1684
|
-
if (!require_event_log.isRecord(value)) return [];
|
|
1685
|
-
const entries = Object.entries(value);
|
|
1686
|
-
if (entries.length === 0) return prefix ? [prefix.slice(0, -1)] : [];
|
|
1687
|
-
return entries.flatMap(([key, child]) => statePaths(child, `${prefix}${key}.`));
|
|
1688
|
-
}
|
|
1689
|
-
/**
|
|
1690
|
-
* A trace's `machine.transition` events as steps, skipping `@xstate.init`.
|
|
1691
|
-
* A first-attempt decision's `request.start` names the decision on the step
|
|
1692
|
-
* that follows it, skipping `@agent.usage` steps. Reads live and serialized traces alike.
|
|
1693
|
-
* @internal
|
|
1694
|
-
*/
|
|
1695
|
-
function traceSteps(trace) {
|
|
1696
|
-
const steps = [];
|
|
1697
|
-
let decision;
|
|
1698
|
-
for (const event of trace) {
|
|
1699
|
-
if (event.type === "request.start" && isFirstDecision(event.request)) {
|
|
1700
|
-
const name = event.request.name;
|
|
1701
|
-
decision = typeof name === "string" ? name : void 0;
|
|
1702
|
-
continue;
|
|
1703
|
-
}
|
|
1704
|
-
if (event.type !== "machine.transition") continue;
|
|
1705
|
-
const type = require_event_log.isRecord(event.event) ? event.event.type : void 0;
|
|
1706
|
-
if (type === "@xstate.init") continue;
|
|
1707
|
-
const fromValue = valueOf(event.from);
|
|
1708
|
-
const toValue = valueOf(event.snapshot);
|
|
1709
|
-
steps.push({
|
|
1710
|
-
from: statePaths(fromValue),
|
|
1711
|
-
fromValue,
|
|
1712
|
-
event: typeof type === "string" ? type : "",
|
|
1713
|
-
payload: event.event,
|
|
1714
|
-
to: statePaths(toValue),
|
|
1715
|
-
toValue,
|
|
1716
|
-
ignored: event.ignored === true,
|
|
1717
|
-
...decision === void 0 || type === "@agent.usage" ? {} : { decision }
|
|
1718
|
-
});
|
|
1719
|
-
if (type !== "@agent.usage") decision = void 0;
|
|
1720
|
-
}
|
|
1721
|
-
return steps;
|
|
1722
|
-
}
|
|
1723
|
-
//#endregion
|
|
1724
|
-
//#region src/fold-traces.ts
|
|
1725
|
-
/**
|
|
1726
|
-
* Folds many runs' traces into per-state stats: how many runs visited each
|
|
1727
|
-
* state, how many of those failed, how many options the guards withheld
|
|
1728
|
-
* at decisions made there, how many events it ignored, and how many failures
|
|
1729
|
-
* `blame` traced back to it. Pure; reads only:
|
|
1730
|
-
*
|
|
1731
|
-
* - `machine.transition`: the run visited `from.value` and `snapshot.value`;
|
|
1732
|
-
* with `ignored`, each of those states ignored one event.
|
|
1733
|
-
* - `request.start` of a decision, first attempt only: its `request.withheld`
|
|
1734
|
-
* counts toward the state the run was in, the `from.value` of the
|
|
1735
|
-
* transition its choice causes.
|
|
1736
|
-
*
|
|
1737
|
-
* A trace is one run's events, live (`AgentTraceEvent`) or serialized
|
|
1738
|
-
* (`serializeTraceEvent`). A host-driven `decideNext` traces nothing; push
|
|
1739
|
-
* `{ type: 'request.start', request: result.request }` onto the trace before
|
|
1740
|
-
* sending its event. Parallel states count once per active leaf.
|
|
1741
|
-
*
|
|
1742
|
-
* `failures` counts a failed run against every state it visited, so states
|
|
1743
|
-
* after a bad decision rank as high as the decision. Sort by `blamed` to see
|
|
1744
|
-
* where failures start.
|
|
1745
|
-
*
|
|
1746
|
-
* @example
|
|
1747
|
-
* ```ts no-check
|
|
1748
|
-
* const stats = foldTraces(traces, {
|
|
1749
|
-
* failed: (trace) => trace.at(-1)?.type === 'run.end' && trace.at(-1).status === 'error',
|
|
1750
|
-
* });
|
|
1751
|
-
* ```
|
|
1752
|
-
*/
|
|
1753
|
-
function foldTraces(traces, options) {
|
|
1754
|
-
const stats = /* @__PURE__ */ new Map();
|
|
1755
|
-
const statsFor = (state) => {
|
|
1756
|
-
let entry = stats.get(state);
|
|
1757
|
-
if (!entry) {
|
|
1758
|
-
entry = {
|
|
1759
|
-
state,
|
|
1760
|
-
visits: 0,
|
|
1761
|
-
failures: 0,
|
|
1762
|
-
withheld: 0,
|
|
1763
|
-
ignored: 0,
|
|
1764
|
-
blamed: 0
|
|
1765
|
-
};
|
|
1766
|
-
stats.set(state, entry);
|
|
1767
|
-
}
|
|
1768
|
-
return entry;
|
|
1769
|
-
};
|
|
1770
|
-
for (const trace of traces) {
|
|
1771
|
-
const visited = /* @__PURE__ */ new Set();
|
|
1772
|
-
let current = [];
|
|
1773
|
-
const visit = (state) => {
|
|
1774
|
-
statsFor(state);
|
|
1775
|
-
visited.add(state);
|
|
1776
|
-
};
|
|
1777
|
-
for (const event of trace) if (event.type === "machine.transition") {
|
|
1778
|
-
for (const state of statePaths(valueOf(event.from))) visit(state);
|
|
1779
|
-
current = statePaths(valueOf(event.snapshot));
|
|
1780
|
-
for (const state of current) visit(state);
|
|
1781
|
-
if (event.ignored === true) for (const state of current) statsFor(state).ignored++;
|
|
1782
|
-
} else if (event.type === "request.start" && isFirstDecision(event.request)) {
|
|
1783
|
-
const withheld = event.request.withheld;
|
|
1784
|
-
if (!Array.isArray(withheld)) continue;
|
|
1785
|
-
for (const state of current) statsFor(state).withheld += withheld.length;
|
|
1786
|
-
}
|
|
1787
|
-
const failed = options.failed(trace);
|
|
1788
|
-
for (const state of visited) {
|
|
1789
|
-
const entry = statsFor(state);
|
|
1790
|
-
entry.visits++;
|
|
1791
|
-
if (failed) entry.failures++;
|
|
1792
|
-
}
|
|
1793
|
-
const blamed = failed ? options.blame?.(trace) : void 0;
|
|
1794
|
-
if (blamed !== void 0) statsFor(blamed).blamed++;
|
|
1795
|
-
}
|
|
1796
|
-
return [...stats.values()];
|
|
1797
|
-
}
|
|
1798
|
-
//#endregion
|
|
1799
|
-
//#region src/trace-coverage.ts
|
|
1800
|
-
/** Mirrors xstate@6.0.0-alpha.69 `matchesEventDescriptor` (src/utils.ts), which the package does not export. */
|
|
1801
|
-
function matchesDescriptor(eventType, descriptor) {
|
|
1802
|
-
if (descriptor === eventType || descriptor === "*") return true;
|
|
1803
|
-
if (!descriptor.endsWith(".*")) return false;
|
|
1804
|
-
const tokens = descriptor.split(".");
|
|
1805
|
-
const eventTokens = eventType.split(".");
|
|
1806
|
-
for (let index = 0; index < tokens.length; index++) {
|
|
1807
|
-
if (tokens[index] === "*") return index === tokens.length - 1;
|
|
1808
|
-
if (tokens[index] !== eventTokens[index]) return false;
|
|
1809
|
-
}
|
|
1810
|
-
return true;
|
|
1811
|
-
}
|
|
1812
|
-
/** Whether a definition's `matches` pattern (e.g. an invoke's `actorId`) accepts the traced event. */
|
|
1813
|
-
function matchesPattern(definition, event) {
|
|
1814
|
-
const pattern = definition.matches;
|
|
1815
|
-
if (!require_event_log.isRecord(pattern) || !require_event_log.isRecord(event)) return true;
|
|
1816
|
-
return Object.entries(pattern).every(([key, value]) => Object.is(event[key], value));
|
|
1817
|
-
}
|
|
1818
|
-
/**
|
|
1819
|
-
* The descriptor XState would try first at `node` for `event`, guards aside:
|
|
1820
|
-
* the exact key, then partial and wildcard keys longest first, as in
|
|
1821
|
-
* xstate@6.0.0-alpha.69 `getCandidates` (src/stateUtils.ts).
|
|
1822
|
-
*/
|
|
1823
|
-
function candidateAt(node, type, event) {
|
|
1824
|
-
const keys = [...node.transitions.keys()].filter((key) => key !== type && matchesDescriptor(type, key)).sort((a, b) => b.length - a.length);
|
|
1825
|
-
if (node.transitions.has(type)) keys.unshift(type);
|
|
1826
|
-
return keys.find((key) => node.transitions.get(key).some((d) => matchesPattern(d, event)));
|
|
1827
|
-
}
|
|
1828
|
-
const pathOf = (node) => node.path.join(".");
|
|
1829
|
-
const sameLeaves = (a, b) => statePaths(a).sort().join("|") === statePaths(b).sort().join("|");
|
|
1830
|
-
/**
|
|
1831
|
-
* The leaves a trace passed through inside its macrosteps: states an `always`
|
|
1832
|
-
* transition left before the step ended, which no traced snapshot holds.
|
|
1833
|
-
* Re-derived with XState's pure `getMicrosteps` from the snapshot before each
|
|
1834
|
-
* step (the previous step's, or the fresh start's microsteps from its input).
|
|
1835
|
-
* A step that cannot be recomputed, or whose recomputation ends elsewhere
|
|
1836
|
-
* than the traced snapshot, adds none. Only `value` and `context` are rebuilt,
|
|
1837
|
-
* so a guard that reads live children can take another route to the same
|
|
1838
|
-
* leaves, and that route is credited.
|
|
1839
|
-
*/
|
|
1840
|
-
function passedThrough(machine, trace) {
|
|
1841
|
-
const leaves = /* @__PURE__ */ new Set();
|
|
1842
|
-
let resumed = false;
|
|
1843
|
-
let previous;
|
|
1844
|
-
for (const event of trace) {
|
|
1845
|
-
if (event.type === "run.start") resumed = event.snapshot !== void 0;
|
|
1846
|
-
if (event.type !== "machine.transition" || !require_event_log.isRecord(event.event)) continue;
|
|
1847
|
-
const snapshot = require_event_log.isRecord(event.snapshot) ? event.snapshot : void 0;
|
|
1848
|
-
try {
|
|
1849
|
-
const microsteps = event.event.type === "@xstate.init" ? resumed ? [] : (0, xstate.getInitialMicrosteps)(machine, event.event.input) : previous && sameLeaves(previous.value, valueOf(event.from)) ? (0, xstate.getMicrosteps)(machine, machine.resolveState({
|
|
1850
|
-
value: previous.value,
|
|
1851
|
-
context: previous.context
|
|
1852
|
-
}), event.event) : [];
|
|
1853
|
-
const last = microsteps.at(-1)?.[0];
|
|
1854
|
-
if (last && snapshot && sameLeaves(last.value, snapshot.value)) for (const [step] of microsteps) for (const leaf of statePaths(step.value)) leaves.add(leaf);
|
|
1855
|
-
} catch {}
|
|
1856
|
-
previous = snapshot;
|
|
1857
|
-
}
|
|
1858
|
-
return leaves;
|
|
1859
|
-
}
|
|
1860
|
-
const ratioOf = (visited, total) => total === 0 ? 1 : visited / total;
|
|
1861
|
-
/**
|
|
1862
|
-
* Measures how much of a machine a set of runs exercised: which atomic and
|
|
1863
|
-
* final states they visited, and which handlers they took. A handler is a
|
|
1864
|
-
* `(declaring state, event descriptor)` pair from `on`, invoke
|
|
1865
|
-
* `onDone`/`onError`, and `after`, with the targets it declares and the
|
|
1866
|
-
* leaves the runs landed in after it. Pure; reads only `machine.transition`
|
|
1867
|
-
* events, live (`AgentTraceEvent`) or serialized (`serializeTraceEvent`).
|
|
1868
|
-
*
|
|
1869
|
-
* A step credits one handler per active leaf: the nearest node, from the leaf
|
|
1870
|
-
* up, whose exact, partial (`a.*`), or wildcard (`*`) descriptor matches the
|
|
1871
|
-
* event, preferred in XState's order. Guards are not replayed, so a step
|
|
1872
|
-
* whose nearest handler was rejected and fell through to an ancestor is
|
|
1873
|
-
* credited to the nearest one. XState keys invoke and `after` handlers by
|
|
1874
|
-
* bare type (`xstate.done.actor`), one per state however many invokes; a
|
|
1875
|
-
* definition's `matches` (an invoke's `actorId`) must accept the traced event
|
|
1876
|
-
* for its state to be credited. Ignored steps credit no handler. `always`
|
|
1877
|
-
* handlers are listed as `unobservable`, never as unvisited; a state one
|
|
1878
|
-
* passes through is visited, re-derived from each step's microsteps.
|
|
1879
|
-
*
|
|
1880
|
-
* @example
|
|
1881
|
-
* ```ts no-check
|
|
1882
|
-
* const report = traceCoverage(machine, traces, { reachable: await explorePaths(machine) });
|
|
1883
|
-
* expect(report.unvisitedHandlers).toEqual([]);
|
|
1884
|
-
* ```
|
|
1885
|
-
*/
|
|
1886
|
-
function traceCoverage(machine, traces, options = {}) {
|
|
1887
|
-
const nodes = /* @__PURE__ */ new Map();
|
|
1888
|
-
const states = [];
|
|
1889
|
-
const handlers = [];
|
|
1890
|
-
const handlerAt = /* @__PURE__ */ new Map();
|
|
1891
|
-
const unobservable = [];
|
|
1892
|
-
const walk = (node) => {
|
|
1893
|
-
const state = pathOf(node);
|
|
1894
|
-
nodes.set(state, node);
|
|
1895
|
-
if (node.type === "atomic" || node.type === "final") states.push({
|
|
1896
|
-
state,
|
|
1897
|
-
runs: 0
|
|
1898
|
-
});
|
|
1899
|
-
const byEvent = /* @__PURE__ */ new Map();
|
|
1900
|
-
for (const [event, definitions] of node.transitions) {
|
|
1901
|
-
const dynamic = definitions.some((d) => typeof d.to === "function");
|
|
1902
|
-
const declared = [...new Set(definitions.flatMap((d) => (d.target ?? []).map(pathOf)))];
|
|
1903
|
-
const handler = {
|
|
1904
|
-
state,
|
|
1905
|
-
event,
|
|
1906
|
-
runs: 0,
|
|
1907
|
-
targets: {
|
|
1908
|
-
declared: dynamic ? "dynamic" : declared,
|
|
1909
|
-
seen: [],
|
|
1910
|
-
unreached: []
|
|
1911
|
-
}
|
|
1912
|
-
};
|
|
1913
|
-
handlers.push(handler);
|
|
1914
|
-
byEvent.set(event, handler);
|
|
1915
|
-
}
|
|
1916
|
-
handlerAt.set(node, byEvent);
|
|
1917
|
-
if (node.always?.length) unobservable.push({
|
|
1918
|
-
state,
|
|
1919
|
-
event: "always"
|
|
1920
|
-
});
|
|
1921
|
-
for (const child of Object.values(node.states)) walk(child);
|
|
1922
|
-
};
|
|
1923
|
-
walk(machine.root);
|
|
1924
|
-
const stateRuns = new Map(states.map((row) => [row.state, row]));
|
|
1925
|
-
const seen = /* @__PURE__ */ new Map();
|
|
1926
|
-
for (const trace of traces) {
|
|
1927
|
-
const steps = traceSteps(trace);
|
|
1928
|
-
const visited = new Set(steps[0]?.from);
|
|
1929
|
-
for (const event of trace) {
|
|
1930
|
-
if (event.type !== "machine.transition" || !require_event_log.isRecord(event.event)) continue;
|
|
1931
|
-
if (event.event.type !== "@xstate.init" || !require_event_log.isRecord(event.snapshot)) continue;
|
|
1932
|
-
for (const leaf of statePaths(event.snapshot.value)) visited.add(leaf);
|
|
1933
|
-
}
|
|
1934
|
-
if (unobservable.length > 0) for (const leaf of passedThrough(machine, trace)) visited.add(leaf);
|
|
1935
|
-
const credited = /* @__PURE__ */ new Set();
|
|
1936
|
-
for (const step of steps) {
|
|
1937
|
-
for (const leaf of step.to) visited.add(leaf);
|
|
1938
|
-
if (step.ignored) continue;
|
|
1939
|
-
const hits = /* @__PURE__ */ new Set();
|
|
1940
|
-
for (const leaf of step.from) for (let node = nodes.get(leaf); node; node = node.parent) {
|
|
1941
|
-
if (node.type === "final") continue;
|
|
1942
|
-
const event = candidateAt(node, step.event, step.payload);
|
|
1943
|
-
if (event === void 0) continue;
|
|
1944
|
-
hits.add(handlerAt.get(node).get(event));
|
|
1945
|
-
break;
|
|
1946
|
-
}
|
|
1947
|
-
for (const handler of hits) {
|
|
1948
|
-
credited.add(handler);
|
|
1949
|
-
let leaves = seen.get(handler);
|
|
1950
|
-
if (!leaves) seen.set(handler, leaves = /* @__PURE__ */ new Set());
|
|
1951
|
-
for (const leaf of step.to) leaves.add(leaf);
|
|
1952
|
-
}
|
|
1953
|
-
}
|
|
1954
|
-
for (const state of visited) {
|
|
1955
|
-
const row = stateRuns.get(state);
|
|
1956
|
-
if (row) row.runs++;
|
|
1957
|
-
}
|
|
1958
|
-
for (const handler of credited) handler.runs++;
|
|
1959
|
-
}
|
|
1960
|
-
for (const handler of handlers) {
|
|
1961
|
-
const leaves = [...seen.get(handler) ?? []];
|
|
1962
|
-
handler.targets.seen = leaves;
|
|
1963
|
-
const { declared } = handler.targets;
|
|
1964
|
-
if (declared === "dynamic") continue;
|
|
1965
|
-
handler.targets.unreached = declared.filter((target) => !leaves.some((leaf) => leaf === target || leaf.startsWith(`${target}.`)));
|
|
1966
|
-
}
|
|
1967
|
-
const reachableComplete = options.reachable ? require_verify.isExplorationComplete(options.reachable) : void 0;
|
|
1968
|
-
let unreachable;
|
|
1969
|
-
if (options.reachable && reachableComplete) {
|
|
1970
|
-
const reached = new Set(options.reachable.reachedStates.flatMap((value) => statePaths(value)));
|
|
1971
|
-
unreachable = states.filter((row) => !reached.has(row.state) && row.runs === 0).map((row) => row.state);
|
|
1972
|
-
}
|
|
1973
|
-
const counted = unreachable ? states.filter((row) => !unreachable.includes(row.state)) : states;
|
|
1974
|
-
const unvisitedStates = counted.filter((row) => row.runs === 0).map((row) => row.state);
|
|
1975
|
-
const unvisitedHandlers = handlers.filter((handler) => handler.runs === 0).map(({ state, event }) => ({
|
|
1976
|
-
state,
|
|
1977
|
-
event
|
|
1978
|
-
}));
|
|
1979
|
-
return {
|
|
1980
|
-
states,
|
|
1981
|
-
handlers,
|
|
1982
|
-
unvisitedStates,
|
|
1983
|
-
unvisitedHandlers,
|
|
1984
|
-
unobservable,
|
|
1985
|
-
...unreachable ? { unreachable } : {},
|
|
1986
|
-
...reachableComplete === void 0 ? {} : { reachableComplete },
|
|
1987
|
-
ratio: {
|
|
1988
|
-
states: ratioOf(counted.length - unvisitedStates.length, counted.length),
|
|
1989
|
-
handlers: ratioOf(handlers.length - unvisitedHandlers.length, handlers.length)
|
|
1990
|
-
}
|
|
1991
|
-
};
|
|
1992
|
-
}
|
|
1993
|
-
//#endregion
|
|
1994
|
-
//#region src/first-divergence.ts
|
|
1995
|
-
const defaultKey = (step) => `${step.from.join(",")}|${step.event}|${step.to.join(",")}`;
|
|
1996
|
-
const isErrorLike = (value) => require_event_log.isError(value) || require_event_log.isRecord(value) && typeof value.name === "string" && typeof value.message === "string";
|
|
1997
|
-
const normalizeErrors = (value, seen = /* @__PURE__ */ new WeakSet()) => {
|
|
1998
|
-
if (value === null || typeof value !== "object") return value;
|
|
1999
|
-
if (seen.has(value)) return "[circular]";
|
|
2000
|
-
seen.add(value);
|
|
2001
|
-
try {
|
|
2002
|
-
if (isErrorLike(value) && (require_event_log.isError(value) || "stack" in value || "code" in value)) {
|
|
2003
|
-
const { name, message, code, cause } = value;
|
|
2004
|
-
return {
|
|
2005
|
-
name,
|
|
2006
|
-
message,
|
|
2007
|
-
...typeof code === "string" ? { code } : {},
|
|
2008
|
-
...cause !== void 0 ? { cause: normalizeErrors(cause, seen) } : {}
|
|
2009
|
-
};
|
|
2010
|
-
}
|
|
2011
|
-
if (Array.isArray(value)) return value.map((item) => normalizeErrors(item, seen));
|
|
2012
|
-
if (!require_event_log.isPlainObject(value)) return value;
|
|
2013
|
-
const copy = Object.create(null);
|
|
2014
|
-
for (const key of Object.keys(value)) copy[key] = normalizeErrors(value[key], seen);
|
|
2015
|
-
return copy;
|
|
2016
|
-
} finally {
|
|
2017
|
-
seen.delete(value);
|
|
2018
|
-
}
|
|
2019
|
-
};
|
|
2020
|
-
const payloadKey = (step) => {
|
|
2021
|
-
if (!require_event_log.isRecord(step.payload)) return `${defaultKey(step)}|${require_event_log.canonicalEventJson(step.payload)}`;
|
|
2022
|
-
const { sessionId: _sessionId, ...payload } = step.payload;
|
|
2023
|
-
return `${defaultKey(step)}|${require_event_log.canonicalEventJson(normalizeErrors(payload))}`;
|
|
2024
|
-
};
|
|
2025
|
-
const isTrace = (items) => items.length === 0 || items.some((item) => require_event_log.isRecord(item) && (item.type === "machine.transition" || item.type === "run.start" || item.type === "run.end"));
|
|
2026
|
-
/**
|
|
2027
|
-
* Finds the first step where a run departs from a reference, so a failure is
|
|
2028
|
-
* blamed on the state it started in rather than on every state the run went
|
|
2029
|
-
* through afterwards.
|
|
2030
|
-
*
|
|
2031
|
-
* - Against a reference **trace**, such as a passing run of the same
|
|
2032
|
-
* scenario, steps are compared in order by `key`. The first step that
|
|
2033
|
-
* differs, or the point where either trace ends early, is the divergence.
|
|
2034
|
-
* - Against an expected **trajectory** (`TrajectoryItem[]`, as for
|
|
2035
|
-
* {@link matchesTrajectory}), the run is laid out as its initial state, then
|
|
2036
|
-
* each step's event and new state. The first expected item it never reaches
|
|
2037
|
-
* is the divergence, blamed on the step after the last item it did reach. A
|
|
2038
|
-
* run that goes on past the trajectory has not diverged.
|
|
2039
|
-
*
|
|
2040
|
-
* An empty reference is a trace with no steps, so any step diverges from it.
|
|
2041
|
-
* A trace is one run's events, live or serialized. Steps the machine ignored
|
|
2042
|
-
* changed nothing and are left out on both sides. Returns `undefined` when the
|
|
2043
|
-
* run does not diverge.
|
|
2044
|
-
*
|
|
2045
|
-
* @example
|
|
2046
|
-
* ```ts no-check
|
|
2047
|
-
* const stats = foldTraces(traces, {
|
|
2048
|
-
* failed,
|
|
2049
|
-
* blame: (trace) => firstDivergence(trace, passingRunFor(trace))?.state,
|
|
2050
|
-
* });
|
|
2051
|
-
* ```
|
|
2052
|
-
*/
|
|
2053
|
-
function firstDivergence(actual, reference, options = {}) {
|
|
2054
|
-
const steps = traceSteps(actual).filter((step) => !step.ignored);
|
|
2055
|
-
return isTrace(reference) ? againstTrace(steps, traceSteps(reference).filter((step) => !step.ignored), options.key === "payload" ? payloadKey : options.key ?? defaultKey) : againstTrajectory(actual, steps, reference);
|
|
2056
|
-
}
|
|
2057
|
-
function diverged(steps, index, expected, initial = steps[0]?.fromValue) {
|
|
2058
|
-
const step = steps[index];
|
|
2059
|
-
return {
|
|
2060
|
-
index,
|
|
2061
|
-
state: (step ? step.from[0] : steps.at(-1)?.to[0] ?? statePaths(initial)[0]) ?? "",
|
|
2062
|
-
...step?.decision === void 0 ? {} : { decision: step.decision },
|
|
2063
|
-
...step ? { actual: step } : {},
|
|
2064
|
-
...expected === void 0 ? {} : { expected }
|
|
2065
|
-
};
|
|
2066
|
-
}
|
|
2067
|
-
function againstTrace(steps, reference, key) {
|
|
2068
|
-
const length = Math.max(steps.length, reference.length);
|
|
2069
|
-
for (let index = 0; index < length; index++) {
|
|
2070
|
-
const step = steps[index];
|
|
2071
|
-
const want = reference[index];
|
|
2072
|
-
if (step && want && key(step) === key(want)) continue;
|
|
2073
|
-
return diverged(steps, index, want, steps[0]?.fromValue ?? reference[0]?.fromValue);
|
|
2074
|
-
}
|
|
2075
|
-
}
|
|
2076
|
-
function againstTrajectory(trace, steps, expected) {
|
|
2077
|
-
const initial = initialValue(trace, steps);
|
|
2078
|
-
const timeline = initial === void 0 ? [] : [{ value: initial }];
|
|
2079
|
-
const stepAt = initial === void 0 ? [] : [0];
|
|
2080
|
-
steps.forEach((step, index) => {
|
|
2081
|
-
timeline.push(step.payload);
|
|
2082
|
-
stepAt.push(index);
|
|
2083
|
-
if (JSON.stringify(step.toValue) !== JSON.stringify(step.fromValue)) {
|
|
2084
|
-
timeline.push({ value: step.toValue });
|
|
2085
|
-
stepAt.push(index);
|
|
2086
|
-
}
|
|
2087
|
-
});
|
|
2088
|
-
const match = matchesTrajectory(timeline, expected);
|
|
2089
|
-
if (!match.firstMiss) return void 0;
|
|
2090
|
-
const at = match.firstMiss.searchedFrom;
|
|
2091
|
-
return diverged(steps, at < timeline.length ? stepAt[at] ?? steps.length : steps.length, match.firstMiss.expected, initial);
|
|
2092
|
-
}
|
|
2093
|
-
function initialValue(trace, steps) {
|
|
2094
|
-
for (const event of trace) if (event.type === "machine.transition" && require_event_log.isRecord(event.snapshot)) return event.snapshot.value;
|
|
2095
|
-
return steps[0]?.fromValue;
|
|
2096
|
-
}
|
|
2097
|
-
//#endregion
|
|
2098
|
-
//#region src/propose-from-traces.ts
|
|
2099
|
-
/**
|
|
2100
|
-
* Groups the approval verdicts in many runs' traces by `key` and proposes an
|
|
2101
|
-
* `allow` or `deny` rule for each. Pure and deterministic; a proposal is for
|
|
2102
|
-
* a person to review, and nothing here changes a machine.
|
|
2103
|
-
*
|
|
2104
|
-
* Reads `machine.transition` events: a verdict is a transition, not ignored,
|
|
2105
|
-
* out of `approvalState` on an approve or reject event. Rows come in the
|
|
2106
|
-
* order keys were first seen.
|
|
2107
|
-
*
|
|
2108
|
-
* @example
|
|
2109
|
-
* ```ts no-check
|
|
2110
|
-
* const rows = proposeApprovalRules(traces, {
|
|
2111
|
-
* approvalState: "awaitingApproval",
|
|
2112
|
-
* approveEvents: ["APPROVE"],
|
|
2113
|
-
* rejectEvents: ["REJECT"],
|
|
2114
|
-
* key: ({ snapshot }) => `${snapshot.context.pending.tool}:${snapshot.context.pending.path}`,
|
|
2115
|
-
* });
|
|
2116
|
-
* ```
|
|
2117
|
-
*/
|
|
2118
|
-
function proposeApprovalRules(traces, options) {
|
|
2119
|
-
const rows = /* @__PURE__ */ new Map();
|
|
2120
|
-
const rowFor = (key) => {
|
|
2121
|
-
let row = rows.get(key);
|
|
2122
|
-
if (!row) {
|
|
2123
|
-
row = {
|
|
2124
|
-
key,
|
|
2125
|
-
approved: 0,
|
|
2126
|
-
rejected: 0,
|
|
2127
|
-
violations: 0,
|
|
2128
|
-
runs: [],
|
|
2129
|
-
proposal: null,
|
|
2130
|
-
support: 0
|
|
2131
|
-
};
|
|
2132
|
-
rows.set(key, row);
|
|
2133
|
-
}
|
|
2134
|
-
return row;
|
|
2135
|
-
};
|
|
2136
|
-
const inState = (value) => statePaths(value).some((path) => path === options.approvalState || path.startsWith(`${options.approvalState}.`));
|
|
2137
|
-
let index = 0;
|
|
2138
|
-
for (const trace of traces) {
|
|
2139
|
-
const run = index++;
|
|
2140
|
-
const seen = /* @__PURE__ */ new Set();
|
|
2141
|
-
let snapshot;
|
|
2142
|
-
for (const event of trace) {
|
|
2143
|
-
if (event.type !== "machine.transition") continue;
|
|
2144
|
-
const before = snapshot;
|
|
2145
|
-
snapshot = event.snapshot;
|
|
2146
|
-
const type = require_event_log.isRecord(event.event) ? event.event.type : void 0;
|
|
2147
|
-
if (event.ignored === true || typeof type !== "string" || !inState(valueOf(event.from))) continue;
|
|
2148
|
-
const approve = options.approveEvents.includes(type);
|
|
2149
|
-
if (!approve && !options.rejectEvents.includes(type)) continue;
|
|
2150
|
-
const key = options.key({
|
|
2151
|
-
event: event.event,
|
|
2152
|
-
snapshot: before,
|
|
2153
|
-
trace
|
|
2154
|
-
});
|
|
2155
|
-
if (key === void 0) continue;
|
|
2156
|
-
const row = rowFor(key);
|
|
2157
|
-
if (approve) row.approved++;
|
|
2158
|
-
else row.rejected++;
|
|
2159
|
-
if (!seen.has(key)) {
|
|
2160
|
-
seen.add(key);
|
|
2161
|
-
row.runs.push(run);
|
|
2162
|
-
}
|
|
2163
|
-
}
|
|
2164
|
-
for (const key of options.violations?.(trace) ?? []) {
|
|
2165
|
-
const row = rows.get(key);
|
|
2166
|
-
if (row) row.violations++;
|
|
2167
|
-
}
|
|
2168
|
-
}
|
|
2169
|
-
for (const row of rows.values()) {
|
|
2170
|
-
row.support = row.runs.length;
|
|
2171
|
-
row.proposal = row.approved > 0 && row.rejected === 0 && row.violations === 0 ? "allow" : row.rejected > 0 && row.approved === 0 ? "deny" : null;
|
|
2172
|
-
}
|
|
2173
|
-
return [...rows.values()];
|
|
2174
|
-
}
|
|
2175
|
-
/**
|
|
2176
|
-
* Proposes a value for each cap (retry count, steps, batch size) from many
|
|
2177
|
-
* runs' traces: the 95th percentile of what passing runs used, plus one, and
|
|
2178
|
-
* never below the current cap while runs still hit it. Pure and
|
|
2179
|
-
* deterministic; a proposal is for a person to review, and nothing here
|
|
2180
|
-
* changes a machine.
|
|
2181
|
-
*
|
|
2182
|
-
* @example
|
|
2183
|
-
* ```ts no-check
|
|
2184
|
-
* const [retries] = proposeCaps(traces, {
|
|
2185
|
-
* caps: [{ name: "retries", current: 3, failureState: "gaveUp", valueFromRun: countRetries }],
|
|
2186
|
-
* });
|
|
2187
|
-
* ```
|
|
2188
|
-
*/
|
|
2189
|
-
function proposeCaps(traces, options) {
|
|
2190
|
-
const runs = [...traces];
|
|
2191
|
-
return options.caps.map((cap) => {
|
|
2192
|
-
let hits = 0;
|
|
2193
|
-
let max = null;
|
|
2194
|
-
const passing = [];
|
|
2195
|
-
for (const trace of runs) {
|
|
2196
|
-
const hit = hitCap(trace, cap);
|
|
2197
|
-
if (hit) hits++;
|
|
2198
|
-
const value = cap.valueFromRun(trace);
|
|
2199
|
-
if (value === void 0) continue;
|
|
2200
|
-
max = max === null ? value : Math.max(max, value);
|
|
2201
|
-
if (!hit) passing.push(value);
|
|
2202
|
-
}
|
|
2203
|
-
passing.sort((a, b) => a - b);
|
|
2204
|
-
const p95 = passing.length ? passing[Math.ceil(.95 * passing.length) - 1] : null;
|
|
2205
|
-
const proposal = p95 === null ? null : hits > 0 && cap.current !== void 0 ? Math.max(p95 + 1, cap.current) : p95 + 1;
|
|
2206
|
-
return {
|
|
2207
|
-
name: cap.name,
|
|
2208
|
-
hits,
|
|
2209
|
-
passing: passing.length,
|
|
2210
|
-
max,
|
|
2211
|
-
p95,
|
|
2212
|
-
proposal
|
|
2213
|
-
};
|
|
2214
|
-
});
|
|
2215
|
-
}
|
|
2216
|
-
function hitCap(trace, cap) {
|
|
2217
|
-
for (const event of trace) {
|
|
2218
|
-
if (cap.failureState !== void 0 && event.type === "machine.transition" && statePaths(valueOf(event.snapshot)).some((path) => path === cap.failureState || path.startsWith(`${cap.failureState}.`))) return true;
|
|
2219
|
-
if (cap.failureOutput && event.type === "run.end" && event.status === "done") {
|
|
2220
|
-
if (cap.failureOutput(event.output)) return true;
|
|
2221
|
-
}
|
|
2222
|
-
}
|
|
2223
|
-
return false;
|
|
2224
|
-
}
|
|
2225
|
-
//#endregion
|
|
2226
|
-
//#region src/decision-requests.ts
|
|
2227
|
-
/**
|
|
2228
|
-
* The decisions a run asked for, read from its trace: each decision's first
|
|
2229
|
-
* `request.start`, with the states the run was in. A trace is one run's events,
|
|
2230
|
-
* live or serialized with `serializeTraceEvent`. Retries of the same decision
|
|
2231
|
-
* are left out, so each decision appears once.
|
|
2232
|
-
*
|
|
2233
|
-
* Use the requests as eval rows (run a decider on each and compare its event
|
|
2234
|
-
* with a label) or as probe inputs for a policy. A serialized request keeps its
|
|
2235
|
-
* `options`, `metadata`, `system`, and `prompt`; event schemas do not survive
|
|
2236
|
-
* serialization, so replay closed choices (`options`) rather than open ones.
|
|
2237
|
-
*
|
|
2238
|
-
* @example
|
|
2239
|
-
* ```ts no-check
|
|
2240
|
-
* const rows = traces.flatMap((trace) => getDecisionRequests(trace, { name: "chooseResolution" }));
|
|
2241
|
-
* for (const { request } of rows) console.log((await decide(request)).event);
|
|
2242
|
-
* ```
|
|
2243
|
-
*/
|
|
2244
|
-
function getDecisionRequests(trace, options = {}) {
|
|
2245
|
-
const requests = [];
|
|
2246
|
-
let current = [];
|
|
2247
|
-
for (const event of trace) if (event.type === "machine.transition") current = statePaths(valueOf(event.snapshot));
|
|
2248
|
-
else if (event.type === "request.start" && isFirstDecision(event.request)) {
|
|
2249
|
-
if (options.name !== void 0 && event.request.name !== options.name) continue;
|
|
2250
|
-
requests.push({
|
|
2251
|
-
states: [...current],
|
|
2252
|
-
request: event.request
|
|
2253
|
-
});
|
|
2254
|
-
}
|
|
2255
|
-
return requests;
|
|
2256
|
-
}
|
|
2257
|
-
//#endregion
|
|
2258
|
-
//#region src/repeat-runs.ts
|
|
2259
|
-
/**
|
|
2260
|
-
* Summarizes repeated runs of each scenario: how often each passed, whether
|
|
2261
|
-
* all its runs passed (pass^k), how many distinct state paths they took, and
|
|
2262
|
-
* where they first went different ways. Pure. Use it for runs collected
|
|
2263
|
-
* elsewhere, such as CI shards or replays; {@link repeatRuns} collects them and
|
|
2264
|
-
* calls it.
|
|
2265
|
-
*
|
|
2266
|
-
* A scenario that passes 3 of 5 runs and `divergedAt: 'deciding'` points at one
|
|
2267
|
-
* decision, not at a flaky agent.
|
|
2268
|
-
*
|
|
2269
|
-
* @example
|
|
2270
|
-
* ```ts no-check
|
|
2271
|
-
* const report = summarizeRepeats(shards.map((s) => ({ id: s.id, scenario: s, runs: s.runs })));
|
|
2272
|
-
* report.passHat(3); // the chance 3 runs of a scenario all pass
|
|
2273
|
-
* ```
|
|
2274
|
-
*/
|
|
2275
|
-
function summarizeRepeats(groups) {
|
|
2276
|
-
const scenarios = groups.map((group) => summarize$1(group));
|
|
2277
|
-
const traces = groups.flatMap((group) => group.runs.flatMap((run) => run.trace ? [run.trace] : []));
|
|
2278
|
-
const most = Math.max(0, ...groups.map((group) => group.runs.length));
|
|
2279
|
-
const passHat = (j) => {
|
|
2280
|
-
if (!Number.isInteger(j) || j < 1 || j > most) throw new require_event_log.AgentError("pass-hat-out-of-range", `passHat: j must be a whole number from 1 to ${most}, got ${j}.`);
|
|
2281
|
-
const eligible = scenarios.filter((scenario) => scenario.runs.length >= j);
|
|
2282
|
-
return eligible.reduce((total, scenario) => total + allPass(scenario.passes, scenario.runs.length, j), 0) / eligible.length;
|
|
2283
|
-
};
|
|
2284
|
-
return {
|
|
2285
|
-
scenarios,
|
|
2286
|
-
passHat,
|
|
2287
|
-
traces
|
|
2288
|
-
};
|
|
2289
|
-
}
|
|
2290
|
-
/**
|
|
2291
|
-
* Runs every scenario `k` times and summarizes the runs with
|
|
2292
|
-
* {@link summarizeRepeats}. Runs are sequential, scenario by scenario. A run
|
|
2293
|
-
* that throws fails, with its message as `error` and no trace.
|
|
2294
|
-
*
|
|
2295
|
-
* With a live model, one run per scenario says little: τ-bench found pass^k
|
|
2296
|
-
* falls well below the single-run rate. Report pass^k next to it.
|
|
2297
|
-
*
|
|
2298
|
-
* @example
|
|
2299
|
-
* ```ts no-check
|
|
2300
|
-
* const report = await repeatRuns({
|
|
2301
|
-
* scenarios,
|
|
2302
|
-
* k: 5,
|
|
2303
|
-
* run: async (scenario) => {
|
|
2304
|
-
* const trace: AgentTraceEvent[] = [];
|
|
2305
|
-
* const result = await runToQuiescence(
|
|
2306
|
-
* createAgentRuntime(machine, { executors, onTrace: (event) => trace.push(event) }),
|
|
2307
|
-
* { input: scenario.input },
|
|
2308
|
-
* );
|
|
2309
|
-
* return { trace, outcome: String(result.snapshot.value) };
|
|
2310
|
-
* },
|
|
2311
|
-
* passed: (scenario, { outcome }) => outcome === scenario.expected,
|
|
2312
|
-
* });
|
|
2313
|
-
* report.passHat(5);
|
|
2314
|
-
* ```
|
|
2315
|
-
*/
|
|
2316
|
-
async function repeatRuns(options) {
|
|
2317
|
-
if (!Number.isInteger(options.k) || options.k < 1) throw new require_event_log.AgentError("invalid-repeat-count", `repeatRuns: k must be a whole number of 1 or more, got ${options.k}.`);
|
|
2318
|
-
const groups = [];
|
|
2319
|
-
for (const scenario of options.scenarios) {
|
|
2320
|
-
const runs = [];
|
|
2321
|
-
for (let attempt = 0; attempt < options.k; attempt++) try {
|
|
2322
|
-
const result = await options.run(scenario, attempt);
|
|
2323
|
-
runs.push({
|
|
2324
|
-
passed: options.passed(scenario, result),
|
|
2325
|
-
trace: result.trace,
|
|
2326
|
-
...result.outcome === void 0 ? {} : { outcome: result.outcome }
|
|
2327
|
-
});
|
|
2328
|
-
} catch (error) {
|
|
2329
|
-
runs.push({
|
|
2330
|
-
passed: false,
|
|
2331
|
-
error: error instanceof Error ? error.message : String(error)
|
|
2332
|
-
});
|
|
2333
|
-
}
|
|
2334
|
-
groups.push({
|
|
2335
|
-
id: scenario.id,
|
|
2336
|
-
scenario,
|
|
2337
|
-
runs
|
|
2338
|
-
});
|
|
2339
|
-
}
|
|
2340
|
-
return summarizeRepeats(groups);
|
|
2341
|
-
}
|
|
2342
|
-
function summarize$1(group) {
|
|
2343
|
-
const passes = group.runs.filter((run) => run.passed).length;
|
|
2344
|
-
const traces = group.runs.flatMap((run) => run.trace ? [run.trace] : []);
|
|
2345
|
-
const paths = new Set(traces.map((trace) => traceSteps(trace).filter((step) => !step.ignored).map((step) => `${step.from.join(",")}|${step.event}|${step.to.join(",")}`).join(" ")));
|
|
2346
|
-
const reference = traces.find((trace) => trace.some((event) => event.type === "machine.transition"));
|
|
2347
|
-
let earliest;
|
|
2348
|
-
for (const trace of reference ? traces : []) {
|
|
2349
|
-
if (trace === reference) continue;
|
|
2350
|
-
const divergence = firstDivergence(trace, reference);
|
|
2351
|
-
if (divergence && (!earliest || divergence.index < earliest.index)) earliest = divergence;
|
|
2352
|
-
}
|
|
2353
|
-
return {
|
|
2354
|
-
...group,
|
|
2355
|
-
passes,
|
|
2356
|
-
passAny: passes > 0,
|
|
2357
|
-
passAll: group.runs.length > 0 && passes === group.runs.length,
|
|
2358
|
-
distinctPaths: paths.size,
|
|
2359
|
-
...earliest ? { divergedAt: earliest.state } : {}
|
|
2360
|
-
};
|
|
2361
|
-
}
|
|
2362
|
-
/** C(passes, j) / C(runs, j), as a product so large counts don't overflow. */
|
|
2363
|
-
function allPass(passes, runs, j) {
|
|
2364
|
-
let chance = 1;
|
|
2365
|
-
for (let i = 0; i < j; i++) chance *= Math.max(0, passes - i) / (runs - i);
|
|
2366
|
-
return chance;
|
|
2367
|
-
}
|
|
2368
|
-
//#endregion
|
|
2369
|
-
//#region src/judge-agreement.ts
|
|
2370
|
-
/**
|
|
2371
|
-
* Measures how well a judge agrees with people on the same cases: accuracy,
|
|
2372
|
-
* per-label recall, the confusion matrix, and the rows that disagree. With
|
|
2373
|
-
* `positive`, also the TPR and TNR, the recommended view for binary judges
|
|
2374
|
-
* (Husain/Shankar), and a correction for the judge's bias on unlabeled rows.
|
|
2375
|
-
* Pure.
|
|
2376
|
-
*
|
|
2377
|
-
* Build rows from confirmed labels only; draft labels gate nothing. Group by
|
|
2378
|
-
* decision state or request name so each narrow judge is aligned on its own.
|
|
2379
|
-
* A row with no group key counts in the totals and in no group. Run it on a
|
|
2380
|
-
* dev split and report the held-out split separately.
|
|
2381
|
-
*
|
|
2382
|
-
* Throws `AgentError` with code `'empty-agreement-rows'` when `rows` is empty.
|
|
2383
|
-
*
|
|
2384
|
-
* @example
|
|
2385
|
-
* ```ts no-check
|
|
2386
|
-
* const report = judgeAgreement(
|
|
2387
|
-
* labeled.map((s) => ({ id: s.id, human: s.label, judge: verdicts[s.id], group: s.request })),
|
|
2388
|
-
* { positive: 'fail' },
|
|
2389
|
-
* );
|
|
2390
|
-
* report.binary; // { tpr: 0.9, tnr: 0.8 }
|
|
2391
|
-
* report.correct?.(0.3); // estimated true failure rate in production
|
|
2392
|
-
* ```
|
|
2393
|
-
*/
|
|
2394
|
-
function judgeAgreement(rows, options = {}) {
|
|
2395
|
-
if (rows.length === 0) throw new require_event_log.AgentError("empty-agreement-rows", "judgeAgreement: rows must contain at least one row.");
|
|
2396
|
-
const report = summarize(rows, options.positive);
|
|
2397
|
-
const groupOf = options.groupBy ?? ((row) => row.group);
|
|
2398
|
-
if (options.groupBy || rows.some((row) => row.group !== void 0)) {
|
|
2399
|
-
const grouped = /* @__PURE__ */ new Map();
|
|
2400
|
-
for (const row of rows) {
|
|
2401
|
-
const key = groupOf(row);
|
|
2402
|
-
if (key === void 0) continue;
|
|
2403
|
-
const members = grouped.get(key);
|
|
2404
|
-
if (members) members.push(row);
|
|
2405
|
-
else grouped.set(key, [row]);
|
|
2406
|
-
}
|
|
2407
|
-
const groups = Object.create(null);
|
|
2408
|
-
for (const [key, members] of grouped) groups[key] = summarize(members, options.positive);
|
|
2409
|
-
report.groups = groups;
|
|
2410
|
-
}
|
|
2411
|
-
return report;
|
|
2412
|
-
}
|
|
2413
|
-
function summarize(rows, positive) {
|
|
2414
|
-
const perLabel = Object.create(null);
|
|
2415
|
-
const confusion = Object.create(null);
|
|
2416
|
-
const disagreements = [];
|
|
2417
|
-
let agreed = 0;
|
|
2418
|
-
for (const { id, human, judge } of rows) {
|
|
2419
|
-
const label = perLabel[human] ??= {
|
|
2420
|
-
human: 0,
|
|
2421
|
-
agreed: 0
|
|
2422
|
-
};
|
|
2423
|
-
const row = confusion[human] ??= Object.create(null);
|
|
2424
|
-
label.human++;
|
|
2425
|
-
row[judge] = (row[judge] ?? 0) + 1;
|
|
2426
|
-
if (human === judge) {
|
|
2427
|
-
label.agreed++;
|
|
2428
|
-
agreed++;
|
|
2429
|
-
} else disagreements.push({
|
|
2430
|
-
id,
|
|
2431
|
-
human,
|
|
2432
|
-
judge
|
|
2433
|
-
});
|
|
2434
|
-
}
|
|
2435
|
-
const summary = {
|
|
2436
|
-
total: rows.length,
|
|
2437
|
-
accuracy: agreed / rows.length,
|
|
2438
|
-
perLabel,
|
|
2439
|
-
confusion,
|
|
2440
|
-
disagreements
|
|
2441
|
-
};
|
|
2442
|
-
if (positive === void 0) return summary;
|
|
2443
|
-
let positives = 0;
|
|
2444
|
-
let truePositives = 0;
|
|
2445
|
-
let negatives = 0;
|
|
2446
|
-
let trueNegatives = 0;
|
|
2447
|
-
for (const { human, judge } of rows) if (human === positive) {
|
|
2448
|
-
positives++;
|
|
2449
|
-
if (judge === positive) truePositives++;
|
|
2450
|
-
} else {
|
|
2451
|
-
negatives++;
|
|
2452
|
-
if (judge !== positive) trueNegatives++;
|
|
2453
|
-
}
|
|
2454
|
-
const tpr = positives === 0 ? void 0 : truePositives / positives;
|
|
2455
|
-
const tnr = negatives === 0 ? void 0 : trueNegatives / negatives;
|
|
2456
|
-
summary.binary = {
|
|
2457
|
-
tpr,
|
|
2458
|
-
tnr
|
|
2459
|
-
};
|
|
2460
|
-
if (tpr !== void 0 && tnr !== void 0 && tpr + tnr > 1) summary.correct = (p) => Math.min(1, Math.max(0, (p + tnr - 1) / (tpr + tnr - 1)));
|
|
2461
|
-
return summary;
|
|
2462
|
-
}
|
|
2463
|
-
//#endregion
|
|
2464
1593
|
//#region src/compare-runs.ts
|
|
2465
1594
|
/**
|
|
2466
1595
|
* Runs every scenario through a baseline and a candidate implementation and
|
|
@@ -2573,8 +1702,8 @@ function structurallyEqual(a, b) {
|
|
|
2573
1702
|
* @module
|
|
2574
1703
|
*/
|
|
2575
1704
|
/** Each function effect wrapped with `recorder.fn`, by its key. */
|
|
2576
|
-
function recordEffects(recorder, effects) {
|
|
2577
|
-
return require_effects.mapEffects(effects, (name, effect) => recorder.fn(name, effect));
|
|
1705
|
+
function recordEffects(recorder, effects, recordOptions) {
|
|
1706
|
+
return require_effects.mapEffects(effects, (name, effect) => recorder.fn(name, effect, recordOptions?.[name]));
|
|
2578
1707
|
}
|
|
2579
1708
|
async function settle(run, recorder, normalize) {
|
|
2580
1709
|
try {
|
|
@@ -2591,7 +1720,7 @@ function errorIdentity(outcome, normalize) {
|
|
|
2591
1720
|
if (error instanceof Error) return `${error.name}: ${error.message}`;
|
|
2592
1721
|
if (!normalize) return String(error);
|
|
2593
1722
|
try {
|
|
2594
|
-
return stableStringify(
|
|
1723
|
+
return stableStringify(require_run_agent.toJsonValue(error) ?? null);
|
|
2595
1724
|
} catch {
|
|
2596
1725
|
return String(error);
|
|
2597
1726
|
}
|
|
@@ -2622,14 +1751,20 @@ function errorIdentity(outcome, normalize) {
|
|
|
2622
1751
|
async function checkParity(options) {
|
|
2623
1752
|
const rows = [];
|
|
2624
1753
|
for (const scenario of options.scenarios) {
|
|
1754
|
+
const setup = (side) => options.setup ? options.setup(scenario, side) : void 0;
|
|
2625
1755
|
const recorder = createRecorder();
|
|
2626
|
-
const
|
|
1756
|
+
const live = await setup("before");
|
|
1757
|
+
const effects = recordEffects(recorder, options.effects(scenario, live), options.recordOptions);
|
|
2627
1758
|
const normalize = (output) => options.normalizeOutput ? options.normalizeOutput(output, scenario) : output;
|
|
2628
|
-
const before = await settle(() => options.before(scenario.input, effects), recorder, normalize);
|
|
1759
|
+
const before = await settle(() => options.before(scenario.input, effects, live), recorder, normalize);
|
|
2629
1760
|
const recording = recorder.recording;
|
|
2630
|
-
const replayer = createRecorder({
|
|
2631
|
-
|
|
2632
|
-
|
|
1761
|
+
const replayer = createRecorder({
|
|
1762
|
+
replay: recording,
|
|
1763
|
+
reviveError: options.reviveError
|
|
1764
|
+
});
|
|
1765
|
+
const liveAfter = await setup("after");
|
|
1766
|
+
const stand = recordEffects(replayer, options.effects(scenario, liveAfter), options.recordOptions);
|
|
1767
|
+
const after = await settle(() => options.after(scenario.input, stand, liveAfter), replayer, normalize);
|
|
2633
1768
|
const replay = replayer.recording;
|
|
2634
1769
|
const { divergences } = compareRecordings(recording, replay, options.compare);
|
|
2635
1770
|
const normalizeError = options.normalizeError && ((error) => options.normalizeError(error, scenario));
|
|
@@ -2680,18 +1815,9 @@ exports.compareRecordings = compareRecordings;
|
|
|
2680
1815
|
exports.compareRuns = compareRuns;
|
|
2681
1816
|
exports.createRecorder = createRecorder;
|
|
2682
1817
|
exports.explorePaths = require_verify.explorePaths;
|
|
2683
|
-
exports.firstDivergence = firstDivergence;
|
|
2684
|
-
exports.foldTraces = foldTraces;
|
|
2685
|
-
exports.getDecisionRequests = getDecisionRequests;
|
|
2686
|
-
exports.judgeAgreement = judgeAgreement;
|
|
2687
1818
|
exports.lintAgentMachine = require_verify.lintAgentMachine;
|
|
2688
|
-
exports.matchesTrajectory = matchesTrajectory;
|
|
2689
|
-
exports.proposeApprovalRules = proposeApprovalRules;
|
|
2690
|
-
exports.proposeCaps = proposeCaps;
|
|
2691
|
-
exports.repeatRuns = repeatRuns;
|
|
1819
|
+
exports.matchesTrajectory = require_trajectory.matchesTrajectory;
|
|
2692
1820
|
exports.runSeam = runSeam;
|
|
2693
1821
|
exports.sameModelCallShape = sameModelCallShape;
|
|
2694
1822
|
exports.simulateAgent = require_verify.simulateAgent;
|
|
2695
|
-
exports.summarizeRepeats = summarizeRepeats;
|
|
2696
1823
|
exports.textOutput = require_verify.textOutput;
|
|
2697
|
-
exports.traceCoverage = traceCoverage;
|