@statelyai/agent 2.0.0-alpha.32 → 2.0.0-alpha.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/dist/ai-sdk.cjs +114 -63
- package/dist/ai-sdk.d.cts +11 -2
- package/dist/ai-sdk.d.mts +11 -2
- package/dist/ai-sdk.mjs +115 -64
- package/dist/{decision-BPdEaxPf.cjs → decision-BEsATcr1.cjs} +18 -2
- package/dist/{decision-CV7WiV5C.mjs → decision-COkJNjOW.mjs} +18 -2
- package/dist/{event-log-CKWeC7H-.cjs → event-log-BWSSLUff.cjs} +139 -10
- package/dist/{event-log-DAqsalT1.mjs → event-log-C7jeH0y8.mjs} +123 -12
- package/dist/{event-log-store-C2NjVlWU.d.mts → event-log-store-BzXehP7R.d.cts} +28 -3
- package/dist/{event-log-store-s73r1Hxf.d.cts → event-log-store-CWjN6lmM.d.mts} +28 -3
- package/dist/{event-log-store-DKyE7JKr.cjs → event-log-store-CleWpsp5.cjs} +1 -1
- package/dist/{event-log-store-BGehvVTb.mjs → event-log-store-W49TCJXi.mjs} +1 -1
- package/dist/index.cjs +5 -5
- package/dist/index.d.cts +7 -7
- package/dist/index.d.mts +7 -7
- package/dist/index.mjs +5 -5
- package/dist/log.cjs +2 -2
- package/dist/log.d.cts +1 -1
- package/dist/log.d.mts +1 -1
- package/dist/log.mjs +2 -2
- package/dist/machines.cjs +1 -1
- package/dist/machines.mjs +1 -1
- package/dist/{model-source-BilDEyqk.mjs → model-source-CK3F4YuU.mjs} +1 -1
- package/dist/{model-source-BQ97_l1-.cjs → model-source-DmhTMK39.cjs} +1 -1
- package/dist/openai.cjs +377 -95
- package/dist/openai.d.cts +55 -20
- package/dist/openai.d.mts +55 -20
- package/dist/openai.mjs +375 -96
- package/dist/otel.d.cts +1 -1
- package/dist/otel.d.mts +1 -1
- package/dist/{run-agent-B5QhabdY.d.cts → run-agent-B1-Eqviq.d.cts} +13 -5
- package/dist/{run-agent-Atu-wEk7.d.mts → run-agent-BGdx5913.d.mts} +13 -5
- package/dist/{run-agent-2UnSUhDb.mjs → run-agent-Ct_GZw_f.mjs} +124 -38
- package/dist/{run-agent-Cg0xd3K-.cjs → run-agent-s5D32w99.cjs} +123 -37
- package/dist/{setup-agent-CmhB-U5P.mjs → setup-agent-05uw4dPZ.mjs} +2 -2
- package/dist/{setup-agent-BiF1N3td.d.cts → setup-agent-BtfeFfF6.d.cts} +2 -2
- package/dist/{setup-agent-J9f_Wu5T.d.mts → setup-agent-C6nqzv_X.d.mts} +2 -2
- package/dist/{setup-agent-CwUJyqKB.cjs → setup-agent-CXEBURdo.cjs} +2 -2
- package/dist/tanstack-ai.cjs +91 -59
- package/dist/tanstack-ai.d.cts +13 -2
- package/dist/tanstack-ai.d.mts +13 -2
- package/dist/tanstack-ai.mjs +91 -59
- package/dist/testing.cjs +1026 -10
- package/dist/testing.d.cts +585 -5
- package/dist/testing.d.mts +585 -5
- package/dist/testing.mjs +1018 -11
- package/dist/{text-logic-CYP502VT.d.cts → text-logic-CV_4KGr6.d.cts} +28 -3
- package/dist/{text-logic-DuK9HIFM.d.mts → text-logic-Ckh-rJFV.d.mts} +28 -3
- package/dist/{usage-D7JKAxLc.d.mts → usage-BDiJKyv9.d.cts} +11 -1
- package/dist/{usage-DVw-646x.d.cts → usage-DOoVUGVq.d.mts} +11 -1
- package/dist/validate.cjs +3 -3
- package/dist/validate.d.cts +2 -2
- package/dist/validate.d.mts +2 -2
- package/dist/validate.mjs +3 -3
- package/dist/{verify-Dk3Vwnha.d.cts → verify-BQpE8hUz.d.cts} +1 -1
- package/dist/{verify-CoCvR1eM.mjs → verify-Bpahmr34.mjs} +14 -4
- package/dist/{verify-BVz4cURF.d.mts → verify-CqeuEmIk.d.mts} +1 -1
- package/dist/{verify-D_llRCTr.cjs → verify-DCLZa9R0.cjs} +19 -3
- package/dist/with-model-calls-BXYdBdcu.cjs +45 -0
- package/dist/with-model-calls-Bv4V4EHC.mjs +34 -0
- package/package.json +5 -5
- package/readme.md +14 -13
- package/skills/add-evals/SKILL.md +8 -2
- package/skills/improve-machine/SKILL.md +1 -1
- package/skills/loop-to-machine/SKILL.md +40 -2
package/dist/testing.mjs
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { p as runToQuiescence, s as createAgentRuntime } from "./run-agent-
|
|
3
|
-
import { a as explorePaths, i as canReach, n as AgentUnknownStateError, o as
|
|
1
|
+
import { F as getCallUsage, L as normalizeGeneratorResult, d as canonicalEventJson, dt as isPlainObject, ft as isRecord, pt as AgentError, tt as getStateMeta, ut as isError } from "./event-log-C7jeH0y8.mjs";
|
|
2
|
+
import { p as runToQuiescence, s as createAgentRuntime } from "./run-agent-Ct_GZw_f.mjs";
|
|
3
|
+
import { a as explorePaths, c as simulateAgent, i as canReach, n as AgentUnknownStateError, o as isExplorationComplete, r as assertAgentMachine, s as lintAgentMachine, t as AgentLintError } from "./verify-Bpahmr34.mjs";
|
|
4
|
+
import { getInitialMicrosteps, getMicrosteps } from "xstate";
|
|
4
5
|
//#region src/trajectory.ts
|
|
5
6
|
/**
|
|
6
7
|
* Classifies one item. Event-shaped things (log entries, events) become events;
|
|
7
8
|
* everything else is a state value. A snapshot is unwrapped to its `value`.
|
|
8
9
|
* @internal
|
|
9
10
|
*/
|
|
10
|
-
function normalize(item) {
|
|
11
|
+
function normalize$1(item) {
|
|
11
12
|
if (isRecord(item)) {
|
|
12
13
|
const nested = item["event"];
|
|
13
14
|
if (isRecord(nested) && typeof nested["type"] === "string") return {
|
|
@@ -60,17 +61,17 @@ function deepEqual(a, b) {
|
|
|
60
61
|
*/
|
|
61
62
|
function matcherFor(expected) {
|
|
62
63
|
if (typeof expected === "string") return (actual) => {
|
|
63
|
-
const actualItem = normalize(actual);
|
|
64
|
+
const actualItem = normalize$1(actual);
|
|
64
65
|
return actualItem.kind === "event" ? actualItem.event.type === expected : stateMatchesPath(expected, actualItem.value);
|
|
65
66
|
};
|
|
66
|
-
const expectedItem = normalize(expected);
|
|
67
|
+
const expectedItem = normalize$1(expected);
|
|
67
68
|
switch (expectedItem.kind) {
|
|
68
69
|
case "event": return (actual) => {
|
|
69
|
-
const actualItem = normalize(actual);
|
|
70
|
+
const actualItem = normalize$1(actual);
|
|
70
71
|
return actualItem.kind === "event" && Object.keys(expectedItem.event).every((key) => deepEqual(expectedItem.event[key], actualItem.event[key]));
|
|
71
72
|
};
|
|
72
73
|
case "state": return (actual) => {
|
|
73
|
-
const actualItem = normalize(actual);
|
|
74
|
+
const actualItem = normalize$1(actual);
|
|
74
75
|
return actualItem.kind === "state" && deepEqual(expectedItem.value, actualItem.value);
|
|
75
76
|
};
|
|
76
77
|
}
|
|
@@ -380,12 +381,47 @@ function statePaths(value, prefix = "") {
|
|
|
380
381
|
if (entries.length === 0) return prefix ? [prefix.slice(0, -1)] : [];
|
|
381
382
|
return entries.flatMap(([key, child]) => statePaths(child, `${prefix}${key}.`));
|
|
382
383
|
}
|
|
384
|
+
/**
|
|
385
|
+
* A trace's `machine.transition` events as steps, skipping `@xstate.init`.
|
|
386
|
+
* A first-attempt decision's `request.start` names the decision on the step
|
|
387
|
+
* that follows it, skipping `@agent.usage` steps. Reads live and serialized traces alike.
|
|
388
|
+
* @internal
|
|
389
|
+
*/
|
|
390
|
+
function traceSteps(trace) {
|
|
391
|
+
const steps = [];
|
|
392
|
+
let decision;
|
|
393
|
+
for (const event of trace) {
|
|
394
|
+
if (event.type === "request.start" && isFirstDecision(event.request)) {
|
|
395
|
+
const name = event.request.name;
|
|
396
|
+
decision = typeof name === "string" ? name : void 0;
|
|
397
|
+
continue;
|
|
398
|
+
}
|
|
399
|
+
if (event.type !== "machine.transition") continue;
|
|
400
|
+
const type = isRecord(event.event) ? event.event.type : void 0;
|
|
401
|
+
if (type === "@xstate.init") continue;
|
|
402
|
+
const fromValue = valueOf(event.from);
|
|
403
|
+
const toValue = valueOf(event.snapshot);
|
|
404
|
+
steps.push({
|
|
405
|
+
from: statePaths(fromValue),
|
|
406
|
+
fromValue,
|
|
407
|
+
event: typeof type === "string" ? type : "",
|
|
408
|
+
payload: event.event,
|
|
409
|
+
to: statePaths(toValue),
|
|
410
|
+
toValue,
|
|
411
|
+
ignored: event.ignored === true,
|
|
412
|
+
...decision === void 0 || type === "@agent.usage" ? {} : { decision }
|
|
413
|
+
});
|
|
414
|
+
if (type !== "@agent.usage") decision = void 0;
|
|
415
|
+
}
|
|
416
|
+
return steps;
|
|
417
|
+
}
|
|
383
418
|
//#endregion
|
|
384
419
|
//#region src/fold-traces.ts
|
|
385
420
|
/**
|
|
386
421
|
* Folds many runs' traces into per-state stats: how many runs visited each
|
|
387
422
|
* state, how many of those failed, how many options the guards withheld
|
|
388
|
-
* at decisions made there,
|
|
423
|
+
* at decisions made there, how many events it ignored, and how many failures
|
|
424
|
+
* `blame` traced back to it. Pure; reads only:
|
|
389
425
|
*
|
|
390
426
|
* - `machine.transition`: the run visited `from.value` and `snapshot.value`;
|
|
391
427
|
* with `ignored`, each of those states ignored one event.
|
|
@@ -398,6 +434,10 @@ function statePaths(value, prefix = "") {
|
|
|
398
434
|
* `{ type: 'request.start', request: result.request }` onto the trace before
|
|
399
435
|
* sending its event. Parallel states count once per active leaf.
|
|
400
436
|
*
|
|
437
|
+
* `failures` counts a failed run against every state it visited, so states
|
|
438
|
+
* after a bad decision rank as high as the decision. Sort by `blamed` to see
|
|
439
|
+
* where failures start.
|
|
440
|
+
*
|
|
401
441
|
* @example
|
|
402
442
|
* ```ts no-check
|
|
403
443
|
* const stats = foldTraces(traces, {
|
|
@@ -415,7 +455,8 @@ function foldTraces(traces, options) {
|
|
|
415
455
|
visits: 0,
|
|
416
456
|
failures: 0,
|
|
417
457
|
withheld: 0,
|
|
418
|
-
ignored: 0
|
|
458
|
+
ignored: 0,
|
|
459
|
+
blamed: 0
|
|
419
460
|
};
|
|
420
461
|
stats.set(state, entry);
|
|
421
462
|
}
|
|
@@ -444,10 +485,311 @@ function foldTraces(traces, options) {
|
|
|
444
485
|
entry.visits++;
|
|
445
486
|
if (failed) entry.failures++;
|
|
446
487
|
}
|
|
488
|
+
const blamed = failed ? options.blame?.(trace) : void 0;
|
|
489
|
+
if (blamed !== void 0) statsFor(blamed).blamed++;
|
|
447
490
|
}
|
|
448
491
|
return [...stats.values()];
|
|
449
492
|
}
|
|
450
493
|
//#endregion
|
|
494
|
+
//#region src/trace-coverage.ts
|
|
495
|
+
/** Mirrors xstate@6.0.0-alpha.69 `matchesEventDescriptor` (src/utils.ts), which the package does not export. */
|
|
496
|
+
function matchesDescriptor(eventType, descriptor) {
|
|
497
|
+
if (descriptor === eventType || descriptor === "*") return true;
|
|
498
|
+
if (!descriptor.endsWith(".*")) return false;
|
|
499
|
+
const tokens = descriptor.split(".");
|
|
500
|
+
const eventTokens = eventType.split(".");
|
|
501
|
+
for (let index = 0; index < tokens.length; index++) {
|
|
502
|
+
if (tokens[index] === "*") return index === tokens.length - 1;
|
|
503
|
+
if (tokens[index] !== eventTokens[index]) return false;
|
|
504
|
+
}
|
|
505
|
+
return true;
|
|
506
|
+
}
|
|
507
|
+
/** Whether a definition's `matches` pattern (e.g. an invoke's `actorId`) accepts the traced event. */
|
|
508
|
+
function matchesPattern(definition, event) {
|
|
509
|
+
const pattern = definition.matches;
|
|
510
|
+
if (!isRecord(pattern) || !isRecord(event)) return true;
|
|
511
|
+
return Object.entries(pattern).every(([key, value]) => Object.is(event[key], value));
|
|
512
|
+
}
|
|
513
|
+
/**
|
|
514
|
+
* The descriptor XState would try first at `node` for `event`, guards aside:
|
|
515
|
+
* the exact key, then partial and wildcard keys longest first, as in
|
|
516
|
+
* xstate@6.0.0-alpha.69 `getCandidates` (src/stateUtils.ts).
|
|
517
|
+
*/
|
|
518
|
+
function candidateAt(node, type, event) {
|
|
519
|
+
const keys = [...node.transitions.keys()].filter((key) => key !== type && matchesDescriptor(type, key)).sort((a, b) => b.length - a.length);
|
|
520
|
+
if (node.transitions.has(type)) keys.unshift(type);
|
|
521
|
+
return keys.find((key) => node.transitions.get(key).some((d) => matchesPattern(d, event)));
|
|
522
|
+
}
|
|
523
|
+
const pathOf = (node) => node.path.join(".");
|
|
524
|
+
const sameLeaves = (a, b) => statePaths(a).sort().join("|") === statePaths(b).sort().join("|");
|
|
525
|
+
/**
|
|
526
|
+
* The leaves a trace passed through inside its macrosteps: states an `always`
|
|
527
|
+
* transition left before the step ended, which no traced snapshot holds.
|
|
528
|
+
* Re-derived with XState's pure `getMicrosteps` from the snapshot before each
|
|
529
|
+
* step (the previous step's, or the fresh start's microsteps from its input).
|
|
530
|
+
* A step that cannot be recomputed, or whose recomputation ends elsewhere
|
|
531
|
+
* than the traced snapshot, adds none. Only `value` and `context` are rebuilt,
|
|
532
|
+
* so a guard that reads live children can take another route to the same
|
|
533
|
+
* leaves, and that route is credited.
|
|
534
|
+
*/
|
|
535
|
+
function passedThrough(machine, trace) {
|
|
536
|
+
const leaves = /* @__PURE__ */ new Set();
|
|
537
|
+
let resumed = false;
|
|
538
|
+
let previous;
|
|
539
|
+
for (const event of trace) {
|
|
540
|
+
if (event.type === "run.start") resumed = event.snapshot !== void 0;
|
|
541
|
+
if (event.type !== "machine.transition" || !isRecord(event.event)) continue;
|
|
542
|
+
const snapshot = isRecord(event.snapshot) ? event.snapshot : void 0;
|
|
543
|
+
try {
|
|
544
|
+
const microsteps = event.event.type === "@xstate.init" ? resumed ? [] : getInitialMicrosteps(machine, event.event.input) : previous && sameLeaves(previous.value, valueOf(event.from)) ? getMicrosteps(machine, machine.resolveState({
|
|
545
|
+
value: previous.value,
|
|
546
|
+
context: previous.context
|
|
547
|
+
}), event.event) : [];
|
|
548
|
+
const last = microsteps.at(-1)?.[0];
|
|
549
|
+
if (last && snapshot && sameLeaves(last.value, snapshot.value)) for (const [step] of microsteps) for (const leaf of statePaths(step.value)) leaves.add(leaf);
|
|
550
|
+
} catch {}
|
|
551
|
+
previous = snapshot;
|
|
552
|
+
}
|
|
553
|
+
return leaves;
|
|
554
|
+
}
|
|
555
|
+
const ratioOf = (visited, total) => total === 0 ? 1 : visited / total;
|
|
556
|
+
/**
|
|
557
|
+
* Measures how much of a machine a set of runs exercised: which atomic and
|
|
558
|
+
* final states they visited, and which handlers they took. A handler is a
|
|
559
|
+
* `(declaring state, event descriptor)` pair from `on`, invoke
|
|
560
|
+
* `onDone`/`onError`, and `after`, with the targets it declares and the
|
|
561
|
+
* leaves the runs landed in after it. Pure; reads only `machine.transition`
|
|
562
|
+
* events, live (`AgentTraceEvent`) or serialized (`serializeTraceEvent`).
|
|
563
|
+
*
|
|
564
|
+
* A step credits one handler per active leaf: the nearest node, from the leaf
|
|
565
|
+
* up, whose exact, partial (`a.*`), or wildcard (`*`) descriptor matches the
|
|
566
|
+
* event, preferred in XState's order. Guards are not replayed, so a step
|
|
567
|
+
* whose nearest handler was rejected and fell through to an ancestor is
|
|
568
|
+
* credited to the nearest one. XState keys invoke and `after` handlers by
|
|
569
|
+
* bare type (`xstate.done.actor`), one per state however many invokes; a
|
|
570
|
+
* definition's `matches` (an invoke's `actorId`) must accept the traced event
|
|
571
|
+
* for its state to be credited. Ignored steps credit no handler. `always`
|
|
572
|
+
* handlers are listed as `unobservable`, never as unvisited; a state one
|
|
573
|
+
* passes through is visited, re-derived from each step's microsteps.
|
|
574
|
+
*
|
|
575
|
+
* @example
|
|
576
|
+
* ```ts no-check
|
|
577
|
+
* const report = traceCoverage(machine, traces, { reachable: await explorePaths(machine) });
|
|
578
|
+
* expect(report.unvisitedHandlers).toEqual([]);
|
|
579
|
+
* ```
|
|
580
|
+
*/
|
|
581
|
+
function traceCoverage(machine, traces, options = {}) {
|
|
582
|
+
const nodes = /* @__PURE__ */ new Map();
|
|
583
|
+
const states = [];
|
|
584
|
+
const handlers = [];
|
|
585
|
+
const handlerAt = /* @__PURE__ */ new Map();
|
|
586
|
+
const unobservable = [];
|
|
587
|
+
const walk = (node) => {
|
|
588
|
+
const state = pathOf(node);
|
|
589
|
+
nodes.set(state, node);
|
|
590
|
+
if (node.type === "atomic" || node.type === "final") states.push({
|
|
591
|
+
state,
|
|
592
|
+
runs: 0
|
|
593
|
+
});
|
|
594
|
+
const byEvent = /* @__PURE__ */ new Map();
|
|
595
|
+
for (const [event, definitions] of node.transitions) {
|
|
596
|
+
const dynamic = definitions.some((d) => typeof d.to === "function");
|
|
597
|
+
const declared = [...new Set(definitions.flatMap((d) => (d.target ?? []).map(pathOf)))];
|
|
598
|
+
const handler = {
|
|
599
|
+
state,
|
|
600
|
+
event,
|
|
601
|
+
runs: 0,
|
|
602
|
+
targets: {
|
|
603
|
+
declared: dynamic ? "dynamic" : declared,
|
|
604
|
+
seen: [],
|
|
605
|
+
unreached: []
|
|
606
|
+
}
|
|
607
|
+
};
|
|
608
|
+
handlers.push(handler);
|
|
609
|
+
byEvent.set(event, handler);
|
|
610
|
+
}
|
|
611
|
+
handlerAt.set(node, byEvent);
|
|
612
|
+
if (node.always?.length) unobservable.push({
|
|
613
|
+
state,
|
|
614
|
+
event: "always"
|
|
615
|
+
});
|
|
616
|
+
for (const child of Object.values(node.states)) walk(child);
|
|
617
|
+
};
|
|
618
|
+
walk(machine.root);
|
|
619
|
+
const stateRuns = new Map(states.map((row) => [row.state, row]));
|
|
620
|
+
const seen = /* @__PURE__ */ new Map();
|
|
621
|
+
for (const trace of traces) {
|
|
622
|
+
const steps = traceSteps(trace);
|
|
623
|
+
const visited = new Set(steps[0]?.from);
|
|
624
|
+
for (const event of trace) {
|
|
625
|
+
if (event.type !== "machine.transition" || !isRecord(event.event)) continue;
|
|
626
|
+
if (event.event.type !== "@xstate.init" || !isRecord(event.snapshot)) continue;
|
|
627
|
+
for (const leaf of statePaths(event.snapshot.value)) visited.add(leaf);
|
|
628
|
+
}
|
|
629
|
+
if (unobservable.length > 0) for (const leaf of passedThrough(machine, trace)) visited.add(leaf);
|
|
630
|
+
const credited = /* @__PURE__ */ new Set();
|
|
631
|
+
for (const step of steps) {
|
|
632
|
+
for (const leaf of step.to) visited.add(leaf);
|
|
633
|
+
if (step.ignored) continue;
|
|
634
|
+
const hits = /* @__PURE__ */ new Set();
|
|
635
|
+
for (const leaf of step.from) for (let node = nodes.get(leaf); node; node = node.parent) {
|
|
636
|
+
if (node.type === "final") continue;
|
|
637
|
+
const event = candidateAt(node, step.event, step.payload);
|
|
638
|
+
if (event === void 0) continue;
|
|
639
|
+
hits.add(handlerAt.get(node).get(event));
|
|
640
|
+
break;
|
|
641
|
+
}
|
|
642
|
+
for (const handler of hits) {
|
|
643
|
+
credited.add(handler);
|
|
644
|
+
let leaves = seen.get(handler);
|
|
645
|
+
if (!leaves) seen.set(handler, leaves = /* @__PURE__ */ new Set());
|
|
646
|
+
for (const leaf of step.to) leaves.add(leaf);
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
for (const state of visited) {
|
|
650
|
+
const row = stateRuns.get(state);
|
|
651
|
+
if (row) row.runs++;
|
|
652
|
+
}
|
|
653
|
+
for (const handler of credited) handler.runs++;
|
|
654
|
+
}
|
|
655
|
+
for (const handler of handlers) {
|
|
656
|
+
const leaves = [...seen.get(handler) ?? []];
|
|
657
|
+
handler.targets.seen = leaves;
|
|
658
|
+
const { declared } = handler.targets;
|
|
659
|
+
if (declared === "dynamic") continue;
|
|
660
|
+
handler.targets.unreached = declared.filter((target) => !leaves.some((leaf) => leaf === target || leaf.startsWith(`${target}.`)));
|
|
661
|
+
}
|
|
662
|
+
const reachableComplete = options.reachable ? isExplorationComplete(options.reachable) : void 0;
|
|
663
|
+
let unreachable;
|
|
664
|
+
if (options.reachable && reachableComplete) {
|
|
665
|
+
const reached = new Set(options.reachable.reachedStates.flatMap((value) => statePaths(value)));
|
|
666
|
+
unreachable = states.filter((row) => !reached.has(row.state) && row.runs === 0).map((row) => row.state);
|
|
667
|
+
}
|
|
668
|
+
const counted = unreachable ? states.filter((row) => !unreachable.includes(row.state)) : states;
|
|
669
|
+
const unvisitedStates = counted.filter((row) => row.runs === 0).map((row) => row.state);
|
|
670
|
+
const unvisitedHandlers = handlers.filter((handler) => handler.runs === 0).map(({ state, event }) => ({
|
|
671
|
+
state,
|
|
672
|
+
event
|
|
673
|
+
}));
|
|
674
|
+
return {
|
|
675
|
+
states,
|
|
676
|
+
handlers,
|
|
677
|
+
unvisitedStates,
|
|
678
|
+
unvisitedHandlers,
|
|
679
|
+
unobservable,
|
|
680
|
+
...unreachable ? { unreachable } : {},
|
|
681
|
+
...reachableComplete === void 0 ? {} : { reachableComplete },
|
|
682
|
+
ratio: {
|
|
683
|
+
states: ratioOf(counted.length - unvisitedStates.length, counted.length),
|
|
684
|
+
handlers: ratioOf(handlers.length - unvisitedHandlers.length, handlers.length)
|
|
685
|
+
}
|
|
686
|
+
};
|
|
687
|
+
}
|
|
688
|
+
//#endregion
|
|
689
|
+
//#region src/first-divergence.ts
|
|
690
|
+
const defaultKey = (step) => `${step.from.join(",")}|${step.event}|${step.to.join(",")}`;
|
|
691
|
+
const isErrorLike = (value) => isError(value) || isRecord(value) && typeof value.name === "string" && typeof value.message === "string";
|
|
692
|
+
const normalizeErrors = (value, seen = /* @__PURE__ */ new WeakSet()) => {
|
|
693
|
+
if (value === null || typeof value !== "object") return value;
|
|
694
|
+
if (seen.has(value)) return "[circular]";
|
|
695
|
+
seen.add(value);
|
|
696
|
+
try {
|
|
697
|
+
if (isErrorLike(value) && (isError(value) || "stack" in value || "code" in value)) {
|
|
698
|
+
const { name, message, code, cause } = value;
|
|
699
|
+
return {
|
|
700
|
+
name,
|
|
701
|
+
message,
|
|
702
|
+
...typeof code === "string" ? { code } : {},
|
|
703
|
+
...cause !== void 0 ? { cause: normalizeErrors(cause, seen) } : {}
|
|
704
|
+
};
|
|
705
|
+
}
|
|
706
|
+
if (Array.isArray(value)) return value.map((item) => normalizeErrors(item, seen));
|
|
707
|
+
if (!isPlainObject(value)) return value;
|
|
708
|
+
const copy = Object.create(null);
|
|
709
|
+
for (const key of Object.keys(value)) copy[key] = normalizeErrors(value[key], seen);
|
|
710
|
+
return copy;
|
|
711
|
+
} finally {
|
|
712
|
+
seen.delete(value);
|
|
713
|
+
}
|
|
714
|
+
};
|
|
715
|
+
const payloadKey = (step) => {
|
|
716
|
+
if (!isRecord(step.payload)) return `${defaultKey(step)}|${canonicalEventJson(step.payload)}`;
|
|
717
|
+
const { sessionId: _sessionId, ...payload } = step.payload;
|
|
718
|
+
return `${defaultKey(step)}|${canonicalEventJson(normalizeErrors(payload))}`;
|
|
719
|
+
};
|
|
720
|
+
const isTrace = (items) => items.length === 0 || items.some((item) => isRecord(item) && (item.type === "machine.transition" || item.type === "run.start" || item.type === "run.end"));
|
|
721
|
+
/**
|
|
722
|
+
* Finds the first step where a run departs from a reference, so a failure is
|
|
723
|
+
* blamed on the state it started in rather than on every state the run went
|
|
724
|
+
* through afterwards.
|
|
725
|
+
*
|
|
726
|
+
* - Against a reference **trace**, such as a passing run of the same
|
|
727
|
+
* scenario, steps are compared in order by `key`. The first step that
|
|
728
|
+
* differs, or the point where either trace ends early, is the divergence.
|
|
729
|
+
* - Against an expected **trajectory** (`TrajectoryItem[]`, as for
|
|
730
|
+
* {@link matchesTrajectory}), the run is laid out as its initial state, then
|
|
731
|
+
* each step's event and new state. The first expected item it never reaches
|
|
732
|
+
* is the divergence, blamed on the step after the last item it did reach. A
|
|
733
|
+
* run that goes on past the trajectory has not diverged.
|
|
734
|
+
*
|
|
735
|
+
* An empty reference is a trace with no steps, so any step diverges from it.
|
|
736
|
+
* A trace is one run's events, live or serialized. Steps the machine ignored
|
|
737
|
+
* changed nothing and are left out on both sides. Returns `undefined` when the
|
|
738
|
+
* run does not diverge.
|
|
739
|
+
*
|
|
740
|
+
* @example
|
|
741
|
+
* ```ts no-check
|
|
742
|
+
* const stats = foldTraces(traces, {
|
|
743
|
+
* failed,
|
|
744
|
+
* blame: (trace) => firstDivergence(trace, passingRunFor(trace))?.state,
|
|
745
|
+
* });
|
|
746
|
+
* ```
|
|
747
|
+
*/
|
|
748
|
+
function firstDivergence(actual, reference, options = {}) {
|
|
749
|
+
const steps = traceSteps(actual).filter((step) => !step.ignored);
|
|
750
|
+
return isTrace(reference) ? againstTrace(steps, traceSteps(reference).filter((step) => !step.ignored), options.key === "payload" ? payloadKey : options.key ?? defaultKey) : againstTrajectory(actual, steps, reference);
|
|
751
|
+
}
|
|
752
|
+
function diverged(steps, index, expected, initial = steps[0]?.fromValue) {
|
|
753
|
+
const step = steps[index];
|
|
754
|
+
return {
|
|
755
|
+
index,
|
|
756
|
+
state: (step ? step.from[0] : steps.at(-1)?.to[0] ?? statePaths(initial)[0]) ?? "",
|
|
757
|
+
...step?.decision === void 0 ? {} : { decision: step.decision },
|
|
758
|
+
...step ? { actual: step } : {},
|
|
759
|
+
...expected === void 0 ? {} : { expected }
|
|
760
|
+
};
|
|
761
|
+
}
|
|
762
|
+
function againstTrace(steps, reference, key) {
|
|
763
|
+
const length = Math.max(steps.length, reference.length);
|
|
764
|
+
for (let index = 0; index < length; index++) {
|
|
765
|
+
const step = steps[index];
|
|
766
|
+
const want = reference[index];
|
|
767
|
+
if (step && want && key(step) === key(want)) continue;
|
|
768
|
+
return diverged(steps, index, want, steps[0]?.fromValue ?? reference[0]?.fromValue);
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
function againstTrajectory(trace, steps, expected) {
|
|
772
|
+
const initial = initialValue(trace, steps);
|
|
773
|
+
const timeline = initial === void 0 ? [] : [{ value: initial }];
|
|
774
|
+
const stepAt = initial === void 0 ? [] : [0];
|
|
775
|
+
steps.forEach((step, index) => {
|
|
776
|
+
timeline.push(step.payload);
|
|
777
|
+
stepAt.push(index);
|
|
778
|
+
if (JSON.stringify(step.toValue) !== JSON.stringify(step.fromValue)) {
|
|
779
|
+
timeline.push({ value: step.toValue });
|
|
780
|
+
stepAt.push(index);
|
|
781
|
+
}
|
|
782
|
+
});
|
|
783
|
+
const match = matchesTrajectory(timeline, expected);
|
|
784
|
+
if (!match.firstMiss) return void 0;
|
|
785
|
+
const at = match.firstMiss.searchedFrom;
|
|
786
|
+
return diverged(steps, at < timeline.length ? stepAt[at] ?? steps.length : steps.length, match.firstMiss.expected, initial);
|
|
787
|
+
}
|
|
788
|
+
function initialValue(trace, steps) {
|
|
789
|
+
for (const event of trace) if (event.type === "machine.transition" && isRecord(event.snapshot)) return event.snapshot.value;
|
|
790
|
+
return steps[0]?.fromValue;
|
|
791
|
+
}
|
|
792
|
+
//#endregion
|
|
451
793
|
//#region src/decision-requests.ts
|
|
452
794
|
/**
|
|
453
795
|
* The decisions a run asked for, read from its trace: each decision's first
|
|
@@ -480,6 +822,212 @@ function getDecisionRequests(trace, options = {}) {
|
|
|
480
822
|
return requests;
|
|
481
823
|
}
|
|
482
824
|
//#endregion
|
|
825
|
+
//#region src/repeat-runs.ts
|
|
826
|
+
/**
|
|
827
|
+
* Summarizes repeated runs of each scenario: how often each passed, whether
|
|
828
|
+
* all its runs passed (pass^k), how many distinct state paths they took, and
|
|
829
|
+
* where they first went different ways. Pure. Use it for runs collected
|
|
830
|
+
* elsewhere, such as CI shards or replays; {@link repeatRuns} collects them and
|
|
831
|
+
* calls it.
|
|
832
|
+
*
|
|
833
|
+
* A scenario that passes 3 of 5 runs and `divergedAt: 'deciding'` points at one
|
|
834
|
+
* decision, not at a flaky agent.
|
|
835
|
+
*
|
|
836
|
+
* @example
|
|
837
|
+
* ```ts no-check
|
|
838
|
+
* const report = summarizeRepeats(shards.map((s) => ({ id: s.id, scenario: s, runs: s.runs })));
|
|
839
|
+
* report.passHat(3); // the chance 3 runs of a scenario all pass
|
|
840
|
+
* ```
|
|
841
|
+
*/
|
|
842
|
+
function summarizeRepeats(groups) {
|
|
843
|
+
const scenarios = groups.map((group) => summarize$1(group));
|
|
844
|
+
const traces = groups.flatMap((group) => group.runs.flatMap((run) => run.trace ? [run.trace] : []));
|
|
845
|
+
const most = Math.max(0, ...groups.map((group) => group.runs.length));
|
|
846
|
+
const passHat = (j) => {
|
|
847
|
+
if (!Number.isInteger(j) || j < 1 || j > most) throw new AgentError("pass-hat-out-of-range", `passHat: j must be a whole number from 1 to ${most}, got ${j}.`);
|
|
848
|
+
const eligible = scenarios.filter((scenario) => scenario.runs.length >= j);
|
|
849
|
+
return eligible.reduce((total, scenario) => total + allPass(scenario.passes, scenario.runs.length, j), 0) / eligible.length;
|
|
850
|
+
};
|
|
851
|
+
return {
|
|
852
|
+
scenarios,
|
|
853
|
+
passHat,
|
|
854
|
+
traces
|
|
855
|
+
};
|
|
856
|
+
}
|
|
857
|
+
/**
|
|
858
|
+
* Runs every scenario `k` times and summarizes the runs with
|
|
859
|
+
* {@link summarizeRepeats}. Runs are sequential, scenario by scenario. A run
|
|
860
|
+
* that throws fails, with its message as `error` and no trace.
|
|
861
|
+
*
|
|
862
|
+
* With a live model, one run per scenario says little: τ-bench found pass^k
|
|
863
|
+
* falls well below the single-run rate. Report pass^k next to it.
|
|
864
|
+
*
|
|
865
|
+
* @example
|
|
866
|
+
* ```ts no-check
|
|
867
|
+
* const report = await repeatRuns({
|
|
868
|
+
* scenarios,
|
|
869
|
+
* k: 5,
|
|
870
|
+
* run: async (scenario) => {
|
|
871
|
+
* const trace: AgentTraceEvent[] = [];
|
|
872
|
+
* const result = await runToQuiescence(
|
|
873
|
+
* createAgentRuntime(machine, { executors, onTrace: (event) => trace.push(event) }),
|
|
874
|
+
* { input: scenario.input },
|
|
875
|
+
* );
|
|
876
|
+
* return { trace, outcome: String(result.snapshot.value) };
|
|
877
|
+
* },
|
|
878
|
+
* passed: (scenario, { outcome }) => outcome === scenario.expected,
|
|
879
|
+
* });
|
|
880
|
+
* report.passHat(5);
|
|
881
|
+
* ```
|
|
882
|
+
*/
|
|
883
|
+
async function repeatRuns(options) {
|
|
884
|
+
if (!Number.isInteger(options.k) || options.k < 1) throw new AgentError("invalid-repeat-count", `repeatRuns: k must be a whole number of 1 or more, got ${options.k}.`);
|
|
885
|
+
const groups = [];
|
|
886
|
+
for (const scenario of options.scenarios) {
|
|
887
|
+
const runs = [];
|
|
888
|
+
for (let attempt = 0; attempt < options.k; attempt++) try {
|
|
889
|
+
const result = await options.run(scenario, attempt);
|
|
890
|
+
runs.push({
|
|
891
|
+
passed: options.passed(scenario, result),
|
|
892
|
+
trace: result.trace,
|
|
893
|
+
...result.outcome === void 0 ? {} : { outcome: result.outcome }
|
|
894
|
+
});
|
|
895
|
+
} catch (error) {
|
|
896
|
+
runs.push({
|
|
897
|
+
passed: false,
|
|
898
|
+
error: error instanceof Error ? error.message : String(error)
|
|
899
|
+
});
|
|
900
|
+
}
|
|
901
|
+
groups.push({
|
|
902
|
+
id: scenario.id,
|
|
903
|
+
scenario,
|
|
904
|
+
runs
|
|
905
|
+
});
|
|
906
|
+
}
|
|
907
|
+
return summarizeRepeats(groups);
|
|
908
|
+
}
|
|
909
|
+
function summarize$1(group) {
|
|
910
|
+
const passes = group.runs.filter((run) => run.passed).length;
|
|
911
|
+
const traces = group.runs.flatMap((run) => run.trace ? [run.trace] : []);
|
|
912
|
+
const paths = new Set(traces.map((trace) => traceSteps(trace).filter((step) => !step.ignored).map((step) => `${step.from.join(",")}|${step.event}|${step.to.join(",")}`).join(" ")));
|
|
913
|
+
const reference = traces.find((trace) => trace.some((event) => event.type === "machine.transition"));
|
|
914
|
+
let earliest;
|
|
915
|
+
for (const trace of reference ? traces : []) {
|
|
916
|
+
if (trace === reference) continue;
|
|
917
|
+
const divergence = firstDivergence(trace, reference);
|
|
918
|
+
if (divergence && (!earliest || divergence.index < earliest.index)) earliest = divergence;
|
|
919
|
+
}
|
|
920
|
+
return {
|
|
921
|
+
...group,
|
|
922
|
+
passes,
|
|
923
|
+
passAny: passes > 0,
|
|
924
|
+
passAll: group.runs.length > 0 && passes === group.runs.length,
|
|
925
|
+
distinctPaths: paths.size,
|
|
926
|
+
...earliest ? { divergedAt: earliest.state } : {}
|
|
927
|
+
};
|
|
928
|
+
}
|
|
929
|
+
/** C(passes, j) / C(runs, j), as a product so large counts don't overflow. */
|
|
930
|
+
function allPass(passes, runs, j) {
|
|
931
|
+
let chance = 1;
|
|
932
|
+
for (let i = 0; i < j; i++) chance *= Math.max(0, passes - i) / (runs - i);
|
|
933
|
+
return chance;
|
|
934
|
+
}
|
|
935
|
+
//#endregion
|
|
936
|
+
//#region src/judge-agreement.ts
|
|
937
|
+
/**
|
|
938
|
+
* Measures how well a judge agrees with people on the same cases: accuracy,
|
|
939
|
+
* per-label recall, the confusion matrix, and the rows that disagree. With
|
|
940
|
+
* `positive`, also the TPR and TNR, the recommended view for binary judges
|
|
941
|
+
* (Husain/Shankar), and a correction for the judge's bias on unlabeled rows.
|
|
942
|
+
* Pure.
|
|
943
|
+
*
|
|
944
|
+
* Build rows from confirmed labels only; draft labels gate nothing. Group by
|
|
945
|
+
* decision state or request name so each narrow judge is aligned on its own.
|
|
946
|
+
* A row with no group key counts in the totals and in no group. Run it on a
|
|
947
|
+
* dev split and report the held-out split separately.
|
|
948
|
+
*
|
|
949
|
+
* Throws `AgentError` with code `'empty-agreement-rows'` when `rows` is empty.
|
|
950
|
+
*
|
|
951
|
+
* @example
|
|
952
|
+
* ```ts no-check
|
|
953
|
+
* const report = judgeAgreement(
|
|
954
|
+
* labeled.map((s) => ({ id: s.id, human: s.label, judge: verdicts[s.id], group: s.request })),
|
|
955
|
+
* { positive: 'fail' },
|
|
956
|
+
* );
|
|
957
|
+
* report.binary; // { tpr: 0.9, tnr: 0.8 }
|
|
958
|
+
* report.correct?.(0.3); // estimated true failure rate in production
|
|
959
|
+
* ```
|
|
960
|
+
*/
|
|
961
|
+
function judgeAgreement(rows, options = {}) {
|
|
962
|
+
if (rows.length === 0) throw new AgentError("empty-agreement-rows", "judgeAgreement: rows must contain at least one row.");
|
|
963
|
+
const report = summarize(rows, options.positive);
|
|
964
|
+
const groupOf = options.groupBy ?? ((row) => row.group);
|
|
965
|
+
if (options.groupBy || rows.some((row) => row.group !== void 0)) {
|
|
966
|
+
const grouped = /* @__PURE__ */ new Map();
|
|
967
|
+
for (const row of rows) {
|
|
968
|
+
const key = groupOf(row);
|
|
969
|
+
if (key === void 0) continue;
|
|
970
|
+
const members = grouped.get(key);
|
|
971
|
+
if (members) members.push(row);
|
|
972
|
+
else grouped.set(key, [row]);
|
|
973
|
+
}
|
|
974
|
+
const groups = Object.create(null);
|
|
975
|
+
for (const [key, members] of grouped) groups[key] = summarize(members, options.positive);
|
|
976
|
+
report.groups = groups;
|
|
977
|
+
}
|
|
978
|
+
return report;
|
|
979
|
+
}
|
|
980
|
+
function summarize(rows, positive) {
|
|
981
|
+
const perLabel = Object.create(null);
|
|
982
|
+
const confusion = Object.create(null);
|
|
983
|
+
const disagreements = [];
|
|
984
|
+
let agreed = 0;
|
|
985
|
+
for (const { id, human, judge } of rows) {
|
|
986
|
+
const label = perLabel[human] ??= {
|
|
987
|
+
human: 0,
|
|
988
|
+
agreed: 0
|
|
989
|
+
};
|
|
990
|
+
const row = confusion[human] ??= Object.create(null);
|
|
991
|
+
label.human++;
|
|
992
|
+
row[judge] = (row[judge] ?? 0) + 1;
|
|
993
|
+
if (human === judge) {
|
|
994
|
+
label.agreed++;
|
|
995
|
+
agreed++;
|
|
996
|
+
} else disagreements.push({
|
|
997
|
+
id,
|
|
998
|
+
human,
|
|
999
|
+
judge
|
|
1000
|
+
});
|
|
1001
|
+
}
|
|
1002
|
+
const summary = {
|
|
1003
|
+
total: rows.length,
|
|
1004
|
+
accuracy: agreed / rows.length,
|
|
1005
|
+
perLabel,
|
|
1006
|
+
confusion,
|
|
1007
|
+
disagreements
|
|
1008
|
+
};
|
|
1009
|
+
if (positive === void 0) return summary;
|
|
1010
|
+
let positives = 0;
|
|
1011
|
+
let truePositives = 0;
|
|
1012
|
+
let negatives = 0;
|
|
1013
|
+
let trueNegatives = 0;
|
|
1014
|
+
for (const { human, judge } of rows) if (human === positive) {
|
|
1015
|
+
positives++;
|
|
1016
|
+
if (judge === positive) truePositives++;
|
|
1017
|
+
} else {
|
|
1018
|
+
negatives++;
|
|
1019
|
+
if (judge !== positive) trueNegatives++;
|
|
1020
|
+
}
|
|
1021
|
+
const tpr = positives === 0 ? void 0 : truePositives / positives;
|
|
1022
|
+
const tnr = negatives === 0 ? void 0 : trueNegatives / negatives;
|
|
1023
|
+
summary.binary = {
|
|
1024
|
+
tpr,
|
|
1025
|
+
tnr
|
|
1026
|
+
};
|
|
1027
|
+
if (tpr !== void 0 && tnr !== void 0 && tpr + tnr > 1) summary.correct = (p) => Math.min(1, Math.max(0, (p + tnr - 1) / (tpr + tnr - 1)));
|
|
1028
|
+
return summary;
|
|
1029
|
+
}
|
|
1030
|
+
//#endregion
|
|
483
1031
|
//#region src/compare-runs.ts
|
|
484
1032
|
/**
|
|
485
1033
|
* Runs every scenario through a baseline and a candidate implementation and
|
|
@@ -585,4 +1133,463 @@ function structurallyEqual(a, b) {
|
|
|
585
1133
|
return keys.length === Object.keys(b).length && keys.every((key) => Object.hasOwn(b, key) && structurallyEqual(a[key], b[key]));
|
|
586
1134
|
}
|
|
587
1135
|
//#endregion
|
|
588
|
-
|
|
1136
|
+
//#region src/equivalence.ts
|
|
1137
|
+
/**
|
|
1138
|
+
* Thrown in replay when the port asks for something the recording cannot
|
|
1139
|
+
* answer: a model call past the last recorded one, or a tool call the original
|
|
1140
|
+
* never made. The effect is still recorded, so {@link compareRecordings}
|
|
1141
|
+
* reports it.
|
|
1142
|
+
*/
|
|
1143
|
+
var RecordingDivergenceError = class extends Error {
|
|
1144
|
+
name = "RecordingDivergenceError";
|
|
1145
|
+
};
|
|
1146
|
+
function toRecordedError(error) {
|
|
1147
|
+
if (error instanceof Error) {
|
|
1148
|
+
const code = error.code;
|
|
1149
|
+
return code === void 0 ? {
|
|
1150
|
+
name: error.name,
|
|
1151
|
+
message: error.message
|
|
1152
|
+
} : {
|
|
1153
|
+
name: error.name,
|
|
1154
|
+
message: error.message,
|
|
1155
|
+
code: toJson(code)
|
|
1156
|
+
};
|
|
1157
|
+
}
|
|
1158
|
+
return {
|
|
1159
|
+
name: "Error",
|
|
1160
|
+
message: String(error)
|
|
1161
|
+
};
|
|
1162
|
+
}
|
|
1163
|
+
function fromRecordedError(error) {
|
|
1164
|
+
return Object.assign(new Error(error.message), error);
|
|
1165
|
+
}
|
|
1166
|
+
/** A JSON-safe copy: Dates become ISO strings, Errors become {@link RecordedError}. */
|
|
1167
|
+
function toJson(value) {
|
|
1168
|
+
if (value === void 0) return void 0;
|
|
1169
|
+
return JSON.parse(JSON.stringify(value, function(key, item) {
|
|
1170
|
+
const raw = this[key];
|
|
1171
|
+
if (raw instanceof Date) return raw.toISOString();
|
|
1172
|
+
if (item instanceof Error) return toRecordedError(item);
|
|
1173
|
+
if (typeof item === "bigint") return item.toString();
|
|
1174
|
+
if (item instanceof Uint8Array) return Array.from(item);
|
|
1175
|
+
return item;
|
|
1176
|
+
}));
|
|
1177
|
+
}
|
|
1178
|
+
/** Restores the `Date` timestamps the AI SDK puts on responses. */
|
|
1179
|
+
function reviveTimestamps(value) {
|
|
1180
|
+
if (value && typeof value === "object") {
|
|
1181
|
+
for (const holder of [value, value.response]) if (holder && typeof holder["timestamp"] === "string") holder["timestamp"] = new Date(holder["timestamp"]);
|
|
1182
|
+
}
|
|
1183
|
+
return value;
|
|
1184
|
+
}
|
|
1185
|
+
function omit(value, keys) {
|
|
1186
|
+
if (!value || typeof value !== "object") return value;
|
|
1187
|
+
return Object.fromEntries(Object.entries(value).filter(([key]) => !keys.includes(key)));
|
|
1188
|
+
}
|
|
1189
|
+
/** Key-order-independent JSON, for equality. */
|
|
1190
|
+
function stableStringify(value) {
|
|
1191
|
+
if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
|
|
1192
|
+
if (value && typeof value === "object") return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`).join(",")}}`;
|
|
1193
|
+
return JSON.stringify(value) ?? "undefined";
|
|
1194
|
+
}
|
|
1195
|
+
function jsonEqual(a, b) {
|
|
1196
|
+
return stableStringify(toJson(a)) === stableStringify(toJson(b));
|
|
1197
|
+
}
|
|
1198
|
+
function streamFrom(parts) {
|
|
1199
|
+
return new ReadableStream({ start(controller) {
|
|
1200
|
+
for (const part of parts) controller.enqueue(reviveTimestamps(structuredClone(part)));
|
|
1201
|
+
controller.close();
|
|
1202
|
+
} });
|
|
1203
|
+
}
|
|
1204
|
+
async function* iterate(chunks) {
|
|
1205
|
+
for (const chunk of chunks) yield structuredClone(chunk);
|
|
1206
|
+
}
|
|
1207
|
+
/**
|
|
1208
|
+
* Creates a {@link Recorder}. With no options it records: every wrapped call
|
|
1209
|
+
* runs for real. With `replay` it answers from a recording instead.
|
|
1210
|
+
*/
|
|
1211
|
+
function createRecorder(options = {}) {
|
|
1212
|
+
const effects = [];
|
|
1213
|
+
const replay = options.replay?.effects;
|
|
1214
|
+
const replayedModels = replay?.filter((effect) => effect.kind === "model");
|
|
1215
|
+
const usedTools = /* @__PURE__ */ new Set();
|
|
1216
|
+
const normalizeEffect = options.normalize ?? ((effect) => effect);
|
|
1217
|
+
let modelCalls = 0;
|
|
1218
|
+
let toolsInFlight = 0;
|
|
1219
|
+
function nextReplayedModel(effect) {
|
|
1220
|
+
const recorded = replayedModels[effect.call];
|
|
1221
|
+
if (!recorded) throw new RecordingDivergenceError(`Model call #${effect.call} was never made by the recorded run (it made ${replayedModels.length}).`);
|
|
1222
|
+
if (recorded.method !== effect.method) throw new RecordingDivergenceError(`Model call #${effect.call} uses ${effect.method}; the recorded run used ${recorded.method}.`);
|
|
1223
|
+
if (!modelCallsMatch(options.modelCalls, recorded, effect)) throw new RecordingDivergenceError(shapeMessage(recorded, effect));
|
|
1224
|
+
return recorded;
|
|
1225
|
+
}
|
|
1226
|
+
/** Records a model call around `run`, or answers it from the recording. */
|
|
1227
|
+
async function modelCall(seam, method, request, streamed, run) {
|
|
1228
|
+
const effect = {
|
|
1229
|
+
kind: "model",
|
|
1230
|
+
seam,
|
|
1231
|
+
method,
|
|
1232
|
+
call: modelCalls++,
|
|
1233
|
+
request: toJson(omit(request, [
|
|
1234
|
+
"abortSignal",
|
|
1235
|
+
"headers",
|
|
1236
|
+
"signal"
|
|
1237
|
+
]))
|
|
1238
|
+
};
|
|
1239
|
+
effects.push(effect);
|
|
1240
|
+
if (replay) {
|
|
1241
|
+
let recorded;
|
|
1242
|
+
try {
|
|
1243
|
+
recorded = nextReplayedModel(effect);
|
|
1244
|
+
} catch (error) {
|
|
1245
|
+
effect.error = toRecordedError(error);
|
|
1246
|
+
throw error;
|
|
1247
|
+
}
|
|
1248
|
+
if (recorded.error) {
|
|
1249
|
+
effect.error = recorded.error;
|
|
1250
|
+
throw fromRecordedError(recorded.error);
|
|
1251
|
+
}
|
|
1252
|
+
effect.response = recorded.response;
|
|
1253
|
+
effect.stream = recorded.stream;
|
|
1254
|
+
if (seam === "openai") return streamed ? iterate(recorded.stream ?? []) : structuredClone(recorded.response);
|
|
1255
|
+
return streamed ? {
|
|
1256
|
+
...reviveTimestamps(structuredClone(recorded.response) ?? {}),
|
|
1257
|
+
stream: streamFrom(recorded.stream ?? [])
|
|
1258
|
+
} : reviveTimestamps(structuredClone(recorded.response));
|
|
1259
|
+
}
|
|
1260
|
+
let result;
|
|
1261
|
+
try {
|
|
1262
|
+
result = await run();
|
|
1263
|
+
} catch (error) {
|
|
1264
|
+
effect.error = toRecordedError(error);
|
|
1265
|
+
throw error;
|
|
1266
|
+
}
|
|
1267
|
+
if (!streamed) {
|
|
1268
|
+
effect.response = toJson(result);
|
|
1269
|
+
return result;
|
|
1270
|
+
}
|
|
1271
|
+
const parts = [];
|
|
1272
|
+
effect.stream = parts;
|
|
1273
|
+
if (seam === "openai") {
|
|
1274
|
+
const source = result;
|
|
1275
|
+
return (async function* () {
|
|
1276
|
+
for await (const chunk of source) {
|
|
1277
|
+
parts.push(toJson(chunk));
|
|
1278
|
+
yield chunk;
|
|
1279
|
+
}
|
|
1280
|
+
})();
|
|
1281
|
+
}
|
|
1282
|
+
const { stream, ...rest } = result;
|
|
1283
|
+
effect.response = toJson(rest);
|
|
1284
|
+
return {
|
|
1285
|
+
...rest,
|
|
1286
|
+
stream: stream.pipeThrough(new TransformStream({ transform(part, controller) {
|
|
1287
|
+
parts.push(toJson(part));
|
|
1288
|
+
controller.enqueue(part);
|
|
1289
|
+
} }))
|
|
1290
|
+
};
|
|
1291
|
+
}
|
|
1292
|
+
/** `inputOf` picks the recorded input from the call's arguments. */
|
|
1293
|
+
function tool(name, fn, inputOf = (args) => args.length === 1 ? args[0] : args) {
|
|
1294
|
+
return ((...args) => {
|
|
1295
|
+
const input = toJson(inputOf(args));
|
|
1296
|
+
const effect = {
|
|
1297
|
+
kind: "tool",
|
|
1298
|
+
name,
|
|
1299
|
+
step: modelCalls,
|
|
1300
|
+
input,
|
|
1301
|
+
...toolsInFlight > 0 ? { concurrent: true } : {}
|
|
1302
|
+
};
|
|
1303
|
+
effects.push(effect);
|
|
1304
|
+
return track(call(effect, args));
|
|
1305
|
+
});
|
|
1306
|
+
function call(effect, args) {
|
|
1307
|
+
const { input } = effect;
|
|
1308
|
+
if (replay && options.tools !== "live") {
|
|
1309
|
+
const wanted = normalizeEffect(effect).input;
|
|
1310
|
+
const recorded = replay.find((candidate) => candidate.kind === "tool" && !usedTools.has(candidate) && candidate.step === effect.step && candidate.name === name && jsonEqual(normalizeEffect(candidate).input, wanted));
|
|
1311
|
+
const settle = (value) => {
|
|
1312
|
+
if (recorded?.sync) return value();
|
|
1313
|
+
try {
|
|
1314
|
+
return Promise.resolve(value());
|
|
1315
|
+
} catch (error) {
|
|
1316
|
+
return Promise.reject(error);
|
|
1317
|
+
}
|
|
1318
|
+
};
|
|
1319
|
+
return settle(() => {
|
|
1320
|
+
if (!recorded) {
|
|
1321
|
+
const error = new RecordingDivergenceError(`Tool '${name}' was called with ${JSON.stringify(input)} ${effect.step === 0 ? "before any model call" : `after model call #${effect.step - 1}`}; the recorded run made no such call.`);
|
|
1322
|
+
effect.error = toRecordedError(error);
|
|
1323
|
+
throw error;
|
|
1324
|
+
}
|
|
1325
|
+
usedTools.add(recorded);
|
|
1326
|
+
if (recorded.sync) effect.sync = true;
|
|
1327
|
+
if (recorded.error) {
|
|
1328
|
+
effect.error = recorded.error;
|
|
1329
|
+
throw fromRecordedError(recorded.error);
|
|
1330
|
+
}
|
|
1331
|
+
effect.output = recorded.output;
|
|
1332
|
+
return structuredClone(recorded.output);
|
|
1333
|
+
});
|
|
1334
|
+
}
|
|
1335
|
+
let output;
|
|
1336
|
+
try {
|
|
1337
|
+
output = fn(...args);
|
|
1338
|
+
} catch (error) {
|
|
1339
|
+
effect.sync = true;
|
|
1340
|
+
effect.error = toRecordedError(error);
|
|
1341
|
+
throw error;
|
|
1342
|
+
}
|
|
1343
|
+
if (!(output instanceof Promise)) {
|
|
1344
|
+
effect.sync = true;
|
|
1345
|
+
effect.output = toJson(output);
|
|
1346
|
+
return output;
|
|
1347
|
+
}
|
|
1348
|
+
return output.then((value) => {
|
|
1349
|
+
effect.output = toJson(value);
|
|
1350
|
+
return value;
|
|
1351
|
+
}, (error) => {
|
|
1352
|
+
effect.error = toRecordedError(error);
|
|
1353
|
+
throw error;
|
|
1354
|
+
});
|
|
1355
|
+
}
|
|
1356
|
+
}
|
|
1357
|
+
/** Counts an async tool call as running until it settles. */
|
|
1358
|
+
function track(result) {
|
|
1359
|
+
if (result instanceof Promise) {
|
|
1360
|
+
toolsInFlight++;
|
|
1361
|
+
const settled = () => {
|
|
1362
|
+
toolsInFlight--;
|
|
1363
|
+
};
|
|
1364
|
+
result.then(settled, settled);
|
|
1365
|
+
}
|
|
1366
|
+
return result;
|
|
1367
|
+
}
|
|
1368
|
+
return {
|
|
1369
|
+
model(model) {
|
|
1370
|
+
const base = model;
|
|
1371
|
+
const wrapped = Object.create(base);
|
|
1372
|
+
wrapped["doGenerate"] = (callOptions) => modelCall("ai-sdk", "doGenerate", callOptions, false, () => base["doGenerate"].call(base, callOptions));
|
|
1373
|
+
wrapped["doStream"] = (callOptions) => modelCall("ai-sdk", "doStream", callOptions, true, () => base["doStream"].call(base, callOptions));
|
|
1374
|
+
return wrapped;
|
|
1375
|
+
},
|
|
1376
|
+
openai(client) {
|
|
1377
|
+
const wrapCreate = (resource, method) => new Proxy(resource, { get(target, key, receiver) {
|
|
1378
|
+
if (key !== "create") return Reflect.get(target, key, receiver);
|
|
1379
|
+
return (params, requestOptions) => {
|
|
1380
|
+
let api;
|
|
1381
|
+
const promise = modelCall("openai", method, params, params?.stream === true, () => {
|
|
1382
|
+
api = Reflect.get(target, "create").call(target, params, requestOptions);
|
|
1383
|
+
return Promise.resolve(api);
|
|
1384
|
+
});
|
|
1385
|
+
return Object.assign(promise, { withResponse: async () => {
|
|
1386
|
+
const data = await promise;
|
|
1387
|
+
const meta = await api?.withResponse?.();
|
|
1388
|
+
return {
|
|
1389
|
+
data,
|
|
1390
|
+
response: meta?.response ?? new Response(null),
|
|
1391
|
+
request_id: meta?.request_id ?? null
|
|
1392
|
+
};
|
|
1393
|
+
} });
|
|
1394
|
+
};
|
|
1395
|
+
} });
|
|
1396
|
+
const wrapped = /* @__PURE__ */ new Map();
|
|
1397
|
+
const wrap = (key, make) => {
|
|
1398
|
+
if (!wrapped.has(key)) wrapped.set(key, make());
|
|
1399
|
+
return wrapped.get(key);
|
|
1400
|
+
};
|
|
1401
|
+
return new Proxy(client, { get(target, key, receiver) {
|
|
1402
|
+
const value = Reflect.get(target, key, receiver);
|
|
1403
|
+
if (key === "responses" && value) return wrap(key, () => wrapCreate(value, "responses.create"));
|
|
1404
|
+
if (key === "chat" && value) return wrap(key, () => new Proxy(value, { get(chat, chatKey, chatReceiver) {
|
|
1405
|
+
const inner = Reflect.get(chat, chatKey, chatReceiver);
|
|
1406
|
+
return chatKey === "completions" && inner ? wrap("chat.completions", () => wrapCreate(inner, "chat.completions.create")) : inner;
|
|
1407
|
+
} }));
|
|
1408
|
+
return value;
|
|
1409
|
+
} });
|
|
1410
|
+
},
|
|
1411
|
+
tool: (name, fn) => tool(name, fn),
|
|
1412
|
+
tools(tools) {
|
|
1413
|
+
return Object.fromEntries(Object.entries(tools).map(([name, definition]) => [name, definition.execute ? {
|
|
1414
|
+
...definition,
|
|
1415
|
+
execute: tool(name, definition.execute, (args) => args[0])
|
|
1416
|
+
} : definition]));
|
|
1417
|
+
},
|
|
1418
|
+
output(value) {
|
|
1419
|
+
effects.push({
|
|
1420
|
+
kind: "output",
|
|
1421
|
+
value: toJson(value)
|
|
1422
|
+
});
|
|
1423
|
+
},
|
|
1424
|
+
get recording() {
|
|
1425
|
+
return {
|
|
1426
|
+
version: 1,
|
|
1427
|
+
effects: toJson(effects)
|
|
1428
|
+
};
|
|
1429
|
+
}
|
|
1430
|
+
};
|
|
1431
|
+
}
|
|
1432
|
+
/** True when two model calls offer the same tool names and response format. */
|
|
1433
|
+
function sameModelCallShape(expected, actual) {
|
|
1434
|
+
return jsonEqual(modelCallShape(expected), modelCallShape(actual));
|
|
1435
|
+
}
|
|
1436
|
+
function modelCallsMatch(match, expected, actual) {
|
|
1437
|
+
if (match === "none") return true;
|
|
1438
|
+
if (typeof match === "function") return match(expected, actual);
|
|
1439
|
+
return sameModelCallShape(expected, actual);
|
|
1440
|
+
}
|
|
1441
|
+
function shapeMessage(expected, actual) {
|
|
1442
|
+
return `Model call #${expected.call} changed shape: ${JSON.stringify(modelCallShape(expected))} → ${JSON.stringify(modelCallShape(actual))}. A recorded answer only fits a call of the same shape.`;
|
|
1443
|
+
}
|
|
1444
|
+
/** Offered tool names and the response format: what a recorded answer depends on. */
|
|
1445
|
+
function modelCallShape(effect) {
|
|
1446
|
+
const request = effect.request ?? {};
|
|
1447
|
+
return {
|
|
1448
|
+
tools: (Array.isArray(request["tools"]) ? request["tools"] : []).map((tool) => tool["name"] ?? tool["function"]?.["name"]).sort(),
|
|
1449
|
+
format: request["responseFormat"]?.["type"] ?? request["response_format"]?.["type"] ?? (request["text"]?.["format"])?.["type"] ?? "text"
|
|
1450
|
+
};
|
|
1451
|
+
}
|
|
1452
|
+
/**
|
|
1453
|
+
* Puts each group of concurrent tool calls in a stable order, so parallel tool
|
|
1454
|
+
* calls that finished in a different order still compare equal. A tool call
|
|
1455
|
+
* that started after the previous one settled begins a new group, so
|
|
1456
|
+
* sequential calls keep their order.
|
|
1457
|
+
*/
|
|
1458
|
+
function normalize(effects) {
|
|
1459
|
+
const result = [];
|
|
1460
|
+
let batch = [];
|
|
1461
|
+
const flush = () => {
|
|
1462
|
+
result.push(...batch.sort((a, b) => stableStringify([a.name, a.input]).localeCompare(stableStringify([b.name, b.input]))));
|
|
1463
|
+
batch = [];
|
|
1464
|
+
};
|
|
1465
|
+
for (const effect of effects) {
|
|
1466
|
+
if (effect.kind !== "tool" || !effect.concurrent) flush();
|
|
1467
|
+
if (effect.kind === "tool") batch.push(effect);
|
|
1468
|
+
else result.push(effect);
|
|
1469
|
+
}
|
|
1470
|
+
flush();
|
|
1471
|
+
return result;
|
|
1472
|
+
}
|
|
1473
|
+
function describe(effect) {
|
|
1474
|
+
switch (effect.kind) {
|
|
1475
|
+
case "model": return `model call #${effect.call} (${effect.method})`;
|
|
1476
|
+
case "tool": return `tool '${effect.name}' with ${JSON.stringify(effect.input)}`;
|
|
1477
|
+
case "output": return `output ${JSON.stringify(effect.value)}`;
|
|
1478
|
+
}
|
|
1479
|
+
}
|
|
1480
|
+
/** Why two effects differ, or `undefined` when they are the same. */
|
|
1481
|
+
function differ(left, right, options) {
|
|
1482
|
+
if (left.kind !== right.kind) return {
|
|
1483
|
+
reason: "kind",
|
|
1484
|
+
message: `Expected ${describe(left)}; the port made ${describe(right)}.`
|
|
1485
|
+
};
|
|
1486
|
+
if (left.kind === "model" && right.kind === "model") {
|
|
1487
|
+
if (left.method !== right.method) return {
|
|
1488
|
+
reason: "model-method",
|
|
1489
|
+
message: `Model call #${left.call} used ${left.method}; the port used ${right.method}.`
|
|
1490
|
+
};
|
|
1491
|
+
if (!modelCallsMatch(options.modelCalls, left, right)) return {
|
|
1492
|
+
reason: "model-call-shape",
|
|
1493
|
+
message: shapeMessage(left, right)
|
|
1494
|
+
};
|
|
1495
|
+
if (!jsonEqual(left.error, right.error)) {
|
|
1496
|
+
const outcome = (error) => error ? `failed with ${error.name}: ${error.message}` : "succeeded";
|
|
1497
|
+
return {
|
|
1498
|
+
reason: "model-error",
|
|
1499
|
+
message: `Model call #${left.call} ${outcome(left.error)}; in the port it ${outcome(right.error)}.`
|
|
1500
|
+
};
|
|
1501
|
+
}
|
|
1502
|
+
return;
|
|
1503
|
+
}
|
|
1504
|
+
if (left.kind === "tool" && right.kind === "tool") {
|
|
1505
|
+
if (left.name !== right.name) return {
|
|
1506
|
+
reason: "tool-name",
|
|
1507
|
+
message: `Expected ${describe(left)}; the port made ${describe(right)}.`
|
|
1508
|
+
};
|
|
1509
|
+
if (!jsonEqual(left.input, right.input)) return {
|
|
1510
|
+
reason: "tool-input",
|
|
1511
|
+
message: `Expected ${describe(left)}; the port passed ${JSON.stringify(right.input)}.`
|
|
1512
|
+
};
|
|
1513
|
+
if (!jsonEqual(left.output, right.output) || !jsonEqual(left.error, right.error)) return {
|
|
1514
|
+
reason: "tool-result",
|
|
1515
|
+
message: `Tool '${left.name}' returned a different result.`
|
|
1516
|
+
};
|
|
1517
|
+
return;
|
|
1518
|
+
}
|
|
1519
|
+
if (left.kind === "output" && right.kind === "output" && !jsonEqual(left.value, right.value)) return {
|
|
1520
|
+
reason: "output",
|
|
1521
|
+
message: `Expected ${describe(left)}; the port returned ${JSON.stringify(right.value)}.`
|
|
1522
|
+
};
|
|
1523
|
+
}
|
|
1524
|
+
/**
|
|
1525
|
+
* Compares two recordings and reports every effect that differs. `expected`
|
|
1526
|
+
* is the original's recording, `actual` the port's. Effects that are the same
|
|
1527
|
+
* on both sides are lined up first (longest common subsequence), so one extra
|
|
1528
|
+
* call does not shift every effect after it; between them, effects of the same
|
|
1529
|
+
* kind pair up and report what changed, and the rest are missing or extra.
|
|
1530
|
+
*/
|
|
1531
|
+
function compareRecordings(expected, actual, options = {}) {
|
|
1532
|
+
const map = options.normalize ?? ((effect) => effect);
|
|
1533
|
+
const a = normalize(expected.effects.map(map));
|
|
1534
|
+
const b = normalize(actual.effects.map(map));
|
|
1535
|
+
const same = a.map((left) => b.map((right) => !differ(left, right, options)));
|
|
1536
|
+
const lcs = Array.from({ length: a.length + 1 }, () => Array.from({ length: b.length + 1 }, () => 0));
|
|
1537
|
+
for (let i = a.length - 1; i >= 0; i--) for (let j = b.length - 1; j >= 0; j--) lcs[i][j] = same[i][j] ? lcs[i + 1][j + 1] + 1 : Math.max(lcs[i + 1][j], lcs[i][j + 1]);
|
|
1538
|
+
const divergences = [];
|
|
1539
|
+
let lefts = [];
|
|
1540
|
+
let rights = [];
|
|
1541
|
+
const flushGap = (at) => {
|
|
1542
|
+
const unpairedRights = [...rights];
|
|
1543
|
+
const changed = [];
|
|
1544
|
+
for (const index of lefts) {
|
|
1545
|
+
const left = a[index];
|
|
1546
|
+
const k = unpairedRights.findIndex((j) => b[j].kind === left.kind);
|
|
1547
|
+
if (k === -1) {
|
|
1548
|
+
changed.push({
|
|
1549
|
+
index,
|
|
1550
|
+
reason: "missing-effect",
|
|
1551
|
+
expected: left,
|
|
1552
|
+
message: `The port never made ${describe(left)}.`
|
|
1553
|
+
});
|
|
1554
|
+
continue;
|
|
1555
|
+
}
|
|
1556
|
+
const right = b[unpairedRights.splice(k, 1)[0]];
|
|
1557
|
+
const difference = differ(left, right, options);
|
|
1558
|
+
if (difference) changed.push({
|
|
1559
|
+
index,
|
|
1560
|
+
expected: left,
|
|
1561
|
+
actual: right,
|
|
1562
|
+
...difference
|
|
1563
|
+
});
|
|
1564
|
+
}
|
|
1565
|
+
for (const j of unpairedRights) divergences.push({
|
|
1566
|
+
index: lefts[0] ?? at,
|
|
1567
|
+
reason: "extra-effect",
|
|
1568
|
+
actual: b[j],
|
|
1569
|
+
message: `The port made ${describe(b[j])}, which the original never made.`
|
|
1570
|
+
});
|
|
1571
|
+
divergences.push(...changed);
|
|
1572
|
+
lefts = [];
|
|
1573
|
+
rights = [];
|
|
1574
|
+
};
|
|
1575
|
+
let i = 0;
|
|
1576
|
+
let j = 0;
|
|
1577
|
+
while (i < a.length || j < b.length) if (i < a.length && j < b.length && same[i][j]) {
|
|
1578
|
+
flushGap(i);
|
|
1579
|
+
i++;
|
|
1580
|
+
j++;
|
|
1581
|
+
} else if (j < b.length && (i === a.length || lcs[i][j + 1] >= lcs[i + 1][j])) rights.push(j++);
|
|
1582
|
+
else lefts.push(i++);
|
|
1583
|
+
flushGap(i);
|
|
1584
|
+
divergences.sort((x, y) => x.index - y.index);
|
|
1585
|
+
return divergences.length ? {
|
|
1586
|
+
equivalent: false,
|
|
1587
|
+
divergence: divergences[0],
|
|
1588
|
+
divergences
|
|
1589
|
+
} : {
|
|
1590
|
+
equivalent: true,
|
|
1591
|
+
divergences
|
|
1592
|
+
};
|
|
1593
|
+
}
|
|
1594
|
+
//#endregion
|
|
1595
|
+
export { AgentLintError, AgentUnknownStateError, RecordingDivergenceError, assertAgentMachine, canReach, compareRecordings, compareRuns, createRecorder, explorePaths, firstDivergence, foldTraces, getDecisionRequests, judgeAgreement, lintAgentMachine, matchesTrajectory, repeatRuns, runSeam, sameModelCallShape, simulateAgent, summarizeRepeats, traceCoverage };
|