@statelyai/agent 2.0.0-alpha.37 → 2.0.0-alpha.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +51 -0
- package/dist/{act-CrX0JwWY.cjs → act-B0CVbmo4.cjs} +1 -1
- package/dist/{act-B9hLQls6.mjs → act-Bsz-hUfD.mjs} +1 -1
- package/dist/ai-sdk.cjs +4 -4
- package/dist/ai-sdk.d.cts +2 -3
- package/dist/ai-sdk.d.mts +2 -3
- package/dist/ai-sdk.mjs +4 -4
- package/dist/{decision-m5hGmHpF.mjs → decision-B41YBx5U.mjs} +1 -1
- package/dist/{decision-CMeuNZ1g.cjs → decision-D8bcIGXQ.cjs} +1 -1
- package/dist/{decision-object-DBcLWVvv.d.cts → decision-object-BB8XV5Ec.d.cts} +1 -1
- package/dist/{decision-object-B5zGSPFp.d.mts → decision-object-CH_sE694.d.mts} +1 -1
- package/dist/effects-B9IvUPVh.mjs +23 -0
- package/dist/effects-DrWcUGgc.cjs +34 -0
- package/dist/evals.cjs +810 -0
- package/dist/evals.d.cts +595 -0
- package/dist/evals.d.mts +595 -0
- package/dist/evals.mjs +801 -0
- package/dist/{event-log-DkVlk3Rh.mjs → event-log-CEq_bVXQ.mjs} +8 -4
- package/dist/{event-log-Dkc-yIYu.cjs → event-log-dSOnXHuG.cjs} +7 -15
- package/dist/{event-log-store-Brx9tOHt.mjs → event-log-store-DLAfNM34.mjs} +1 -1
- package/dist/{event-log-store-BBS8so8a.d.mts → event-log-store-DYun526m.d.mts} +15 -14
- package/dist/{event-log-store-Cl-YWDnq.cjs → event-log-store-Dey9ufA4.cjs} +1 -1
- package/dist/{event-log-store-bc4ICGcS.d.cts → event-log-store-aFXK6yZX.d.cts} +15 -14
- package/dist/index.cjs +136 -98
- package/dist/index.d.cts +73 -134
- package/dist/index.d.mts +73 -134
- package/dist/index.mjs +117 -75
- package/dist/internal-BzlpzSux.d.cts +14 -0
- package/dist/internal-BzlpzSux.d.mts +14 -0
- package/dist/log.cjs +2 -3
- package/dist/log.d.cts +2 -2
- package/dist/log.d.mts +2 -2
- package/dist/log.mjs +3 -3
- package/dist/machines.cjs +1 -719
- package/dist/machines.d.cts +2 -424
- package/dist/machines.d.mts +2 -424
- package/dist/machines.mjs +2 -709
- package/dist/{model-source-kjGwqAV8.mjs → model-source-BrsLDvHL.mjs} +1 -1
- package/dist/{model-source-B4WQ8Idp.cjs → model-source-YpxjmWuX.cjs} +1 -1
- package/dist/openai.cjs +3 -3
- package/dist/openai.d.cts +2 -3
- package/dist/openai.d.mts +2 -3
- package/dist/openai.mjs +3 -3
- package/dist/otel.d.cts +1 -1
- package/dist/otel.d.mts +1 -1
- package/dist/policy.cjs +77 -0
- package/dist/policy.d.cts +143 -0
- package/dist/policy.d.mts +143 -0
- package/dist/policy.mjs +75 -0
- package/dist/{effects-BT_cSY0j.mjs → run-agent-BE7DOWD6.mjs} +150 -184
- package/dist/{run-agent-DlPzBWj-.d.mts → run-agent-DHlq87G7.d.mts} +18 -34
- package/dist/{effects-DDseYtJk.cjs → run-agent-De3xVFGw.cjs} +149 -201
- package/dist/{run-agent-30mH2moE.d.cts → run-agent-Dq0oSFRm.d.cts} +18 -34
- package/dist/{setup-agent-BDmNr6PC.mjs → setup-agent-CLm6QVGy.mjs} +3 -3
- package/dist/{setup-agent-BaNyz6Q3.d.cts → setup-agent-Cv8Hk3mX.d.cts} +12 -11
- package/dist/{setup-agent-C1CG_nfX.d.mts → setup-agent-DgFAhaPJ.d.mts} +12 -11
- package/dist/{setup-agent-CmEToB8y.cjs → setup-agent-Dv5VpuHe.cjs} +3 -3
- package/dist/tanstack-ai.cjs +4 -4
- package/dist/tanstack-ai.d.cts +2 -3
- package/dist/tanstack-ai.d.mts +2 -3
- package/dist/tanstack-ai.mjs +4 -4
- package/dist/testing.cjs +122 -996
- package/dist/testing.d.cts +36 -695
- package/dist/testing.d.mts +36 -695
- package/dist/testing.mjs +112 -977
- package/dist/{text-logic-BqxalgMr.d.mts → text-logic-x17uLBwA.d.cts} +238 -2
- package/dist/{text-logic-CSf9bqpP.d.cts → text-logic-x17uLBwA.d.mts} +238 -2
- package/dist/trajectory-BFjXaooB.d.cts +96 -0
- package/dist/trajectory-BFjXaooB.d.mts +96 -0
- package/dist/trajectory-BUXj_1Ue.mjs +152 -0
- package/dist/trajectory-CoZSv4wF.cjs +157 -0
- package/dist/{usage-Bm5nvpow.d.cts → usage-B4B-TQSV.d.mts} +1 -1
- package/dist/{usage-BnikdOca.d.mts → usage-DvGcoK6x.d.cts} +1 -1
- package/dist/{utils-7lSa4H-i.d.cts → utils-B8k-YPYI.d.mts} +23 -17
- package/dist/{utils-CBbKw2fM.d.mts → utils-DAclF-kw.d.cts} +23 -17
- package/dist/validate.cjs +4 -4
- package/dist/validate.d.cts +2 -2
- package/dist/validate.d.mts +2 -2
- package/dist/validate.mjs +4 -4
- package/dist/{verify-DJHfEfm6.d.cts → verify-BM0sGmsf.d.cts} +42 -6
- package/dist/{verify-ChyX6jq0.mjs → verify-CWlg3j7B.mjs} +153 -45
- package/dist/{verify-CwlombbC.cjs → verify-Dek6gHCS.cjs} +153 -45
- package/dist/{verify-i9Hiwc5g.d.mts → verify-H3Pr2gNw.d.mts} +42 -6
- package/dist/{with-model-calls-B_WLmOSl.cjs → with-model-calls-D50Ca6qW.cjs} +2 -2
- package/dist/{with-model-calls-BkOwdroS.mjs → with-model-calls-uY-KKliA.mjs} +2 -2
- package/docs/advanced.md +5 -5
- package/docs/choosing-a-run-mode.md +7 -7
- package/docs/debugging.md +6 -8
- package/docs/decisions.md +6 -5
- package/docs/environment-and-policy.md +1 -1
- package/docs/evals.md +57 -43
- package/docs/event-log.md +14 -16
- package/docs/from-a-loop.md +3 -3
- package/docs/generate-machines.md +4 -4
- package/docs/hosts.md +22 -29
- package/docs/human-in-the-loop.md +5 -8
- package/docs/index.md +4 -4
- package/docs/judgments.md +5 -7
- package/docs/langgraph-comparison.md +6 -6
- package/docs/machines-as-data.md +12 -17
- package/docs/machines-presets.md +17 -120
- package/docs/machines.md +13 -13
- package/docs/models-and-providers.md +19 -25
- package/docs/multi-agent.md +1 -1
- package/docs/observability.md +6 -8
- package/docs/persistence.md +15 -25
- package/docs/quickstart.md +4 -4
- package/docs/statechart-policy-examples.md +1 -1
- package/docs/steps.md +5 -6
- package/docs/strangler-fig.md +72 -7
- package/docs/text-requests.md +10 -12
- package/docs/thinking-in-state-machines.md +8 -8
- package/docs/tools.md +10 -12
- package/docs/upgrading.md +1 -1
- package/docs/usage-and-budgets.md +51 -56
- package/docs/validate-and-repair.md +9 -14
- package/docs/verify.md +53 -7
- package/docs/xstate-4-and-5-apps.md +19 -28
- package/llms.txt +1 -1
- package/package.json +31 -6
- package/readme.md +32 -24
- package/dist/internal-C3Lqu538.d.cts +0 -43
- package/dist/internal-DUcdhk-i.d.mts +0 -43
- package/dist/types-DQUOOaZO.d.cts +0 -238
- package/dist/types-DQUOOaZO.d.mts +0 -238
- package/skills/add-evals/SKILL.md +0 -152
- package/skills/add-traces/SKILL.md +0 -152
- package/skills/build-machine/SKILL.md +0 -84
- package/skills/generate-machine/SKILL.md +0 -335
- package/skills/graph-engineering/SKILL.md +0 -43
- package/skills/improve-machine/SKILL.md +0 -261
- package/skills/improve-machine/reference.md +0 -101
- package/skills/loop-to-machine/SKILL.md +0 -205
- package/skills/loop-to-machine/policy.md +0 -154
- package/skills/loop-to-machine/reference.md +0 -260
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { StateValue } from "xstate";
|
|
2
|
+
//#region src/trajectory.d.ts
|
|
3
|
+
/** An event-shaped trajectory item: anything with a string `type`. */
|
|
4
|
+
interface TrajectoryEvent {
|
|
5
|
+
type: string;
|
|
6
|
+
[key: string]: unknown;
|
|
7
|
+
}
|
|
8
|
+
/** Any host-owned envelope around an event. */
|
|
9
|
+
interface TrajectoryEventEnvelope {
|
|
10
|
+
event: TrajectoryEvent;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* One item in a trajectory, on either side of the comparison:
|
|
14
|
+
*
|
|
15
|
+
* - a state value — `'drafting'`, a dot path `'review.editing'`, or the nested
|
|
16
|
+
* object XState reports (`{ review: 'editing' }`),
|
|
17
|
+
* - an event object, or just the event type as a string,
|
|
18
|
+
* - a machine snapshot, whose `value` is used.
|
|
19
|
+
*/
|
|
20
|
+
type TrajectoryItem = StateValue | TrajectoryEvent | TrajectoryEventEnvelope | {
|
|
21
|
+
value: StateValue;
|
|
22
|
+
};
|
|
23
|
+
/** Options for {@link matchesTrajectory}. */
|
|
24
|
+
interface MatchTrajectoryOptions {
|
|
25
|
+
/**
|
|
26
|
+
* Require the trajectories to be equal: same length, item for item. Default
|
|
27
|
+
* `false` — expected items must appear in `actual` in order, gaps allowed.
|
|
28
|
+
*/
|
|
29
|
+
exact?: boolean;
|
|
30
|
+
}
|
|
31
|
+
/** The first expected item that could not be found. */
|
|
32
|
+
interface TrajectoryMiss {
|
|
33
|
+
/** Its position in `expected`. */
|
|
34
|
+
index: number;
|
|
35
|
+
/** The expected item itself. */
|
|
36
|
+
expected: TrajectoryItem;
|
|
37
|
+
/** Where the search for it started in `actual` (everything before was already consumed). */
|
|
38
|
+
searchedFrom: number;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* The result of {@link matchesTrajectory}: a boolean for tests, plus enough
|
|
42
|
+
* detail for a scorer's partial credit and a failure message's diagnosis.
|
|
43
|
+
*/
|
|
44
|
+
interface TrajectoryMatch {
|
|
45
|
+
/** Every expected item was found, in order (and, in `exact` mode, nothing else). */
|
|
46
|
+
matched: boolean;
|
|
47
|
+
/** How many expected items were matched before the first miss. */
|
|
48
|
+
matchedCount: number;
|
|
49
|
+
/** `expected.length`, so a scorer can compute its own ratio. */
|
|
50
|
+
expectedCount: number;
|
|
51
|
+
/**
|
|
52
|
+
* Partial credit, 0..1. `1` for an empty expectation. Denominator is
|
|
53
|
+
* `max(expectedCount, actual.length)` in `exact` mode, `expectedCount` otherwise.
|
|
54
|
+
*/
|
|
55
|
+
score: number;
|
|
56
|
+
/** Absent when `matched`. */
|
|
57
|
+
firstMiss?: TrajectoryMiss;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Matches a run's trajectory against an expected one.
|
|
61
|
+
*
|
|
62
|
+
* Both trajectories may be state values or events collected from
|
|
63
|
+
* `onTransition`;
|
|
64
|
+
* items are compared by shape, so `['prompting', 'drafting']` scores a state
|
|
65
|
+
* path and `['PROMPT_SUBMITTED', 'SEND']` scores an event log with the same
|
|
66
|
+
* call.
|
|
67
|
+
*
|
|
68
|
+
* Semantics are an ordered subsequence: every expected item must appear in
|
|
69
|
+
* `actual`, in order, with gaps allowed. Pass `{ exact: true }` to require
|
|
70
|
+
* equality instead.
|
|
71
|
+
*
|
|
72
|
+
* @example Trajectory assertions in a test
|
|
73
|
+
* ```ts
|
|
74
|
+
* const statePath: unknown[] = [];
|
|
75
|
+
* const result = await runToQuiescence(
|
|
76
|
+
* createAgentRuntime(machine, {
|
|
77
|
+
* executors,
|
|
78
|
+
* onTransition: (snapshot) => statePath.push(snapshot.value),
|
|
79
|
+
* }),
|
|
80
|
+
* { input },
|
|
81
|
+
* );
|
|
82
|
+
*
|
|
83
|
+
* const path = matchesTrajectory(statePath, ['prompting', 'drafting', 'sent']);
|
|
84
|
+
* expect(path.matched, JSON.stringify(path.firstMiss)).toBe(true);
|
|
85
|
+
* expect(matchesTrajectory(events, ['PROMPT_SUBMITTED', 'SEND']).matched).toBe(true);
|
|
86
|
+
* ```
|
|
87
|
+
*
|
|
88
|
+
* @example Partial credit in an eval scorer
|
|
89
|
+
* ```ts
|
|
90
|
+
* const match = matchesTrajectory(output.statePath, expected.statePath);
|
|
91
|
+
* return { name: 'state_path', score: match.score, metadata: { ...match.firstMiss } };
|
|
92
|
+
* ```
|
|
93
|
+
*/
|
|
94
|
+
declare function matchesTrajectory(actual: readonly unknown[], expected: readonly TrajectoryItem[], options?: MatchTrajectoryOptions): TrajectoryMatch;
|
|
95
|
+
//#endregion
|
|
96
|
+
export { TrajectoryMatch as a, TrajectoryItem as i, TrajectoryEvent as n, TrajectoryMiss as o, TrajectoryEventEnvelope as r, matchesTrajectory as s, MatchTrajectoryOptions as t };
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import { ft as isRecord, pt as AgentError } from "./event-log-CEq_bVXQ.mjs";
|
|
2
|
+
//#region src/trajectory.ts
|
|
3
|
+
/**
|
|
4
|
+
* Classifies one item. Event-shaped things (log entries, events) become events;
|
|
5
|
+
* everything else is a state value. A snapshot is unwrapped to its `value`.
|
|
6
|
+
* @internal
|
|
7
|
+
*/
|
|
8
|
+
function normalize(item) {
|
|
9
|
+
if (isRecord(item)) {
|
|
10
|
+
const nested = item["event"];
|
|
11
|
+
if (isRecord(nested) && typeof nested["type"] === "string") return {
|
|
12
|
+
kind: "event",
|
|
13
|
+
event: nested
|
|
14
|
+
};
|
|
15
|
+
if (typeof item["type"] === "string") return {
|
|
16
|
+
kind: "event",
|
|
17
|
+
event: item
|
|
18
|
+
};
|
|
19
|
+
if ("value" in item) return {
|
|
20
|
+
kind: "state",
|
|
21
|
+
value: item["value"]
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
return {
|
|
25
|
+
kind: "state",
|
|
26
|
+
value: item
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
/** Every leaf of a state value as a dot path (`{ review: 'editing' }` → `review.editing`). @internal */
|
|
30
|
+
function statePaths(value) {
|
|
31
|
+
if (typeof value === "string") return [value];
|
|
32
|
+
if (!isRecord(value)) return [];
|
|
33
|
+
return Object.entries(value).flatMap(([key, child]) => statePaths(child).map((path) => `${key}.${path}`));
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* A string expectation matches a state value when it names one of its leaves or
|
|
37
|
+
* an ancestor of one, so `'review'` matches `{ review: 'editing' }`. @internal
|
|
38
|
+
*/
|
|
39
|
+
function stateMatchesPath(path, value) {
|
|
40
|
+
return statePaths(value).some((leaf) => leaf === path || leaf.startsWith(`${path}.`));
|
|
41
|
+
}
|
|
42
|
+
function deepEqual(a, b) {
|
|
43
|
+
if (Object.is(a, b)) return true;
|
|
44
|
+
if (Array.isArray(a) || Array.isArray(b)) return Array.isArray(a) && Array.isArray(b) && a.length === b.length && a.every((item, index) => deepEqual(item, b[index]));
|
|
45
|
+
if (!isRecord(a) || !isRecord(b)) return false;
|
|
46
|
+
const keys = Object.keys(a);
|
|
47
|
+
return keys.length === Object.keys(b).length && keys.every((key) => deepEqual(a[key], b[key]));
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Normalizes ONE expected item into a predicate over actual items, so the
|
|
51
|
+
* expectation is classified once instead of once per candidate:
|
|
52
|
+
*
|
|
53
|
+
* - A string expectation matches an event's `type`, or a state value it names.
|
|
54
|
+
* - An event expectation matches an event with the same `type` whose other
|
|
55
|
+
* declared keys deep-equal, so payload details are opt-in.
|
|
56
|
+
* - A state-value expectation matches structurally.
|
|
57
|
+
* @internal
|
|
58
|
+
*/
|
|
59
|
+
function matcherFor(expected) {
|
|
60
|
+
if (typeof expected === "string") return (actual) => {
|
|
61
|
+
const actualItem = normalize(actual);
|
|
62
|
+
return actualItem.kind === "event" ? actualItem.event.type === expected : stateMatchesPath(expected, actualItem.value);
|
|
63
|
+
};
|
|
64
|
+
const expectedItem = normalize(expected);
|
|
65
|
+
switch (expectedItem.kind) {
|
|
66
|
+
case "event": return (actual) => {
|
|
67
|
+
const actualItem = normalize(actual);
|
|
68
|
+
return actualItem.kind === "event" && Object.keys(expectedItem.event).every((key) => deepEqual(expectedItem.event[key], actualItem.event[key]));
|
|
69
|
+
};
|
|
70
|
+
case "state": return (actual) => {
|
|
71
|
+
const actualItem = normalize(actual);
|
|
72
|
+
return actualItem.kind === "state" && deepEqual(expectedItem.value, actualItem.value);
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Matches a run's trajectory against an expected one.
|
|
78
|
+
*
|
|
79
|
+
* Both trajectories may be state values or events collected from
|
|
80
|
+
* `onTransition`;
|
|
81
|
+
* items are compared by shape, so `['prompting', 'drafting']` scores a state
|
|
82
|
+
* path and `['PROMPT_SUBMITTED', 'SEND']` scores an event log with the same
|
|
83
|
+
* call.
|
|
84
|
+
*
|
|
85
|
+
* Semantics are an ordered subsequence: every expected item must appear in
|
|
86
|
+
* `actual`, in order, with gaps allowed. Pass `{ exact: true }` to require
|
|
87
|
+
* equality instead.
|
|
88
|
+
*
|
|
89
|
+
* @example Trajectory assertions in a test
|
|
90
|
+
* ```ts
|
|
91
|
+
* const statePath: unknown[] = [];
|
|
92
|
+
* const result = await runToQuiescence(
|
|
93
|
+
* createAgentRuntime(machine, {
|
|
94
|
+
* executors,
|
|
95
|
+
* onTransition: (snapshot) => statePath.push(snapshot.value),
|
|
96
|
+
* }),
|
|
97
|
+
* { input },
|
|
98
|
+
* );
|
|
99
|
+
*
|
|
100
|
+
* const path = matchesTrajectory(statePath, ['prompting', 'drafting', 'sent']);
|
|
101
|
+
* expect(path.matched, JSON.stringify(path.firstMiss)).toBe(true);
|
|
102
|
+
* expect(matchesTrajectory(events, ['PROMPT_SUBMITTED', 'SEND']).matched).toBe(true);
|
|
103
|
+
* ```
|
|
104
|
+
*
|
|
105
|
+
* @example Partial credit in an eval scorer
|
|
106
|
+
* ```ts
|
|
107
|
+
* const match = matchesTrajectory(output.statePath, expected.statePath);
|
|
108
|
+
* return { name: 'state_path', score: match.score, metadata: { ...match.firstMiss } };
|
|
109
|
+
* ```
|
|
110
|
+
*/
|
|
111
|
+
function matchesTrajectory(actual, expected, options = {}) {
|
|
112
|
+
const expectedCount = expected.length;
|
|
113
|
+
if (expectedCount === 0) throw new AgentError("empty-trajectory", "matchesTrajectory: expected trajectory must contain at least one item.");
|
|
114
|
+
const exact = options.exact === true;
|
|
115
|
+
let cursor = 0;
|
|
116
|
+
let matchedCount = 0;
|
|
117
|
+
let firstMiss;
|
|
118
|
+
for (let index = 0; index < expectedCount; index++) {
|
|
119
|
+
const want = expected[index];
|
|
120
|
+
const matches = matcherFor(want);
|
|
121
|
+
const searchedFrom = exact ? index : cursor;
|
|
122
|
+
let found = -1;
|
|
123
|
+
for (let at = searchedFrom; at < actual.length; at++) {
|
|
124
|
+
if (matches(actual[at])) {
|
|
125
|
+
found = at;
|
|
126
|
+
break;
|
|
127
|
+
}
|
|
128
|
+
if (exact) break;
|
|
129
|
+
}
|
|
130
|
+
if (found === -1) {
|
|
131
|
+
firstMiss = {
|
|
132
|
+
index,
|
|
133
|
+
expected: want,
|
|
134
|
+
searchedFrom
|
|
135
|
+
};
|
|
136
|
+
break;
|
|
137
|
+
}
|
|
138
|
+
matchedCount++;
|
|
139
|
+
cursor = found + 1;
|
|
140
|
+
}
|
|
141
|
+
const matched = !firstMiss && (!exact || actual.length === expectedCount);
|
|
142
|
+
const denominator = exact ? Math.max(expectedCount, actual.length) : expectedCount;
|
|
143
|
+
return {
|
|
144
|
+
matched,
|
|
145
|
+
matchedCount,
|
|
146
|
+
expectedCount,
|
|
147
|
+
score: matched ? 1 : denominator === 0 ? 1 : matchedCount / denominator,
|
|
148
|
+
...firstMiss ? { firstMiss } : {}
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
//#endregion
|
|
152
|
+
export { matchesTrajectory as t };
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
const require_event_log = require("./event-log-dSOnXHuG.cjs");
|
|
2
|
+
//#region src/trajectory.ts
|
|
3
|
+
/**
|
|
4
|
+
* Classifies one item. Event-shaped things (log entries, events) become events;
|
|
5
|
+
* everything else is a state value. A snapshot is unwrapped to its `value`.
|
|
6
|
+
* @internal
|
|
7
|
+
*/
|
|
8
|
+
function normalize(item) {
|
|
9
|
+
if (require_event_log.isRecord(item)) {
|
|
10
|
+
const nested = item["event"];
|
|
11
|
+
if (require_event_log.isRecord(nested) && typeof nested["type"] === "string") return {
|
|
12
|
+
kind: "event",
|
|
13
|
+
event: nested
|
|
14
|
+
};
|
|
15
|
+
if (typeof item["type"] === "string") return {
|
|
16
|
+
kind: "event",
|
|
17
|
+
event: item
|
|
18
|
+
};
|
|
19
|
+
if ("value" in item) return {
|
|
20
|
+
kind: "state",
|
|
21
|
+
value: item["value"]
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
return {
|
|
25
|
+
kind: "state",
|
|
26
|
+
value: item
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
/** Every leaf of a state value as a dot path (`{ review: 'editing' }` → `review.editing`). @internal */
|
|
30
|
+
function statePaths(value) {
|
|
31
|
+
if (typeof value === "string") return [value];
|
|
32
|
+
if (!require_event_log.isRecord(value)) return [];
|
|
33
|
+
return Object.entries(value).flatMap(([key, child]) => statePaths(child).map((path) => `${key}.${path}`));
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* A string expectation matches a state value when it names one of its leaves or
|
|
37
|
+
* an ancestor of one, so `'review'` matches `{ review: 'editing' }`. @internal
|
|
38
|
+
*/
|
|
39
|
+
function stateMatchesPath(path, value) {
|
|
40
|
+
return statePaths(value).some((leaf) => leaf === path || leaf.startsWith(`${path}.`));
|
|
41
|
+
}
|
|
42
|
+
function deepEqual(a, b) {
|
|
43
|
+
if (Object.is(a, b)) return true;
|
|
44
|
+
if (Array.isArray(a) || Array.isArray(b)) return Array.isArray(a) && Array.isArray(b) && a.length === b.length && a.every((item, index) => deepEqual(item, b[index]));
|
|
45
|
+
if (!require_event_log.isRecord(a) || !require_event_log.isRecord(b)) return false;
|
|
46
|
+
const keys = Object.keys(a);
|
|
47
|
+
return keys.length === Object.keys(b).length && keys.every((key) => deepEqual(a[key], b[key]));
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Normalizes ONE expected item into a predicate over actual items, so the
|
|
51
|
+
* expectation is classified once instead of once per candidate:
|
|
52
|
+
*
|
|
53
|
+
* - A string expectation matches an event's `type`, or a state value it names.
|
|
54
|
+
* - An event expectation matches an event with the same `type` whose other
|
|
55
|
+
* declared keys deep-equal, so payload details are opt-in.
|
|
56
|
+
* - A state-value expectation matches structurally.
|
|
57
|
+
* @internal
|
|
58
|
+
*/
|
|
59
|
+
function matcherFor(expected) {
|
|
60
|
+
if (typeof expected === "string") return (actual) => {
|
|
61
|
+
const actualItem = normalize(actual);
|
|
62
|
+
return actualItem.kind === "event" ? actualItem.event.type === expected : stateMatchesPath(expected, actualItem.value);
|
|
63
|
+
};
|
|
64
|
+
const expectedItem = normalize(expected);
|
|
65
|
+
switch (expectedItem.kind) {
|
|
66
|
+
case "event": return (actual) => {
|
|
67
|
+
const actualItem = normalize(actual);
|
|
68
|
+
return actualItem.kind === "event" && Object.keys(expectedItem.event).every((key) => deepEqual(expectedItem.event[key], actualItem.event[key]));
|
|
69
|
+
};
|
|
70
|
+
case "state": return (actual) => {
|
|
71
|
+
const actualItem = normalize(actual);
|
|
72
|
+
return actualItem.kind === "state" && deepEqual(expectedItem.value, actualItem.value);
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Matches a run's trajectory against an expected one.
|
|
78
|
+
*
|
|
79
|
+
* Both trajectories may be state values or events collected from
|
|
80
|
+
* `onTransition`;
|
|
81
|
+
* items are compared by shape, so `['prompting', 'drafting']` scores a state
|
|
82
|
+
* path and `['PROMPT_SUBMITTED', 'SEND']` scores an event log with the same
|
|
83
|
+
* call.
|
|
84
|
+
*
|
|
85
|
+
* Semantics are an ordered subsequence: every expected item must appear in
|
|
86
|
+
* `actual`, in order, with gaps allowed. Pass `{ exact: true }` to require
|
|
87
|
+
* equality instead.
|
|
88
|
+
*
|
|
89
|
+
* @example Trajectory assertions in a test
|
|
90
|
+
* ```ts
|
|
91
|
+
* const statePath: unknown[] = [];
|
|
92
|
+
* const result = await runToQuiescence(
|
|
93
|
+
* createAgentRuntime(machine, {
|
|
94
|
+
* executors,
|
|
95
|
+
* onTransition: (snapshot) => statePath.push(snapshot.value),
|
|
96
|
+
* }),
|
|
97
|
+
* { input },
|
|
98
|
+
* );
|
|
99
|
+
*
|
|
100
|
+
* const path = matchesTrajectory(statePath, ['prompting', 'drafting', 'sent']);
|
|
101
|
+
* expect(path.matched, JSON.stringify(path.firstMiss)).toBe(true);
|
|
102
|
+
* expect(matchesTrajectory(events, ['PROMPT_SUBMITTED', 'SEND']).matched).toBe(true);
|
|
103
|
+
* ```
|
|
104
|
+
*
|
|
105
|
+
* @example Partial credit in an eval scorer
|
|
106
|
+
* ```ts
|
|
107
|
+
* const match = matchesTrajectory(output.statePath, expected.statePath);
|
|
108
|
+
* return { name: 'state_path', score: match.score, metadata: { ...match.firstMiss } };
|
|
109
|
+
* ```
|
|
110
|
+
*/
|
|
111
|
+
function matchesTrajectory(actual, expected, options = {}) {
|
|
112
|
+
const expectedCount = expected.length;
|
|
113
|
+
if (expectedCount === 0) throw new require_event_log.AgentError("empty-trajectory", "matchesTrajectory: expected trajectory must contain at least one item.");
|
|
114
|
+
const exact = options.exact === true;
|
|
115
|
+
let cursor = 0;
|
|
116
|
+
let matchedCount = 0;
|
|
117
|
+
let firstMiss;
|
|
118
|
+
for (let index = 0; index < expectedCount; index++) {
|
|
119
|
+
const want = expected[index];
|
|
120
|
+
const matches = matcherFor(want);
|
|
121
|
+
const searchedFrom = exact ? index : cursor;
|
|
122
|
+
let found = -1;
|
|
123
|
+
for (let at = searchedFrom; at < actual.length; at++) {
|
|
124
|
+
if (matches(actual[at])) {
|
|
125
|
+
found = at;
|
|
126
|
+
break;
|
|
127
|
+
}
|
|
128
|
+
if (exact) break;
|
|
129
|
+
}
|
|
130
|
+
if (found === -1) {
|
|
131
|
+
firstMiss = {
|
|
132
|
+
index,
|
|
133
|
+
expected: want,
|
|
134
|
+
searchedFrom
|
|
135
|
+
};
|
|
136
|
+
break;
|
|
137
|
+
}
|
|
138
|
+
matchedCount++;
|
|
139
|
+
cursor = found + 1;
|
|
140
|
+
}
|
|
141
|
+
const matched = !firstMiss && (!exact || actual.length === expectedCount);
|
|
142
|
+
const denominator = exact ? Math.max(expectedCount, actual.length) : expectedCount;
|
|
143
|
+
return {
|
|
144
|
+
matched,
|
|
145
|
+
matchedCount,
|
|
146
|
+
expectedCount,
|
|
147
|
+
score: matched ? 1 : denominator === 0 ? 1 : matchedCount / denominator,
|
|
148
|
+
...firstMiss ? { firstMiss } : {}
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
//#endregion
|
|
152
|
+
Object.defineProperty(exports, "matchesTrajectory", {
|
|
153
|
+
enumerable: true,
|
|
154
|
+
get: function() {
|
|
155
|
+
return matchesTrajectory;
|
|
156
|
+
}
|
|
157
|
+
});
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { t as AgentCallUsage } from "./text-logic-
|
|
1
|
+
import { t as AgentCallUsage } from "./text-logic-x17uLBwA.mjs";
|
|
2
2
|
import { EventObject } from "xstate";
|
|
3
3
|
//#region src/usage.d.ts
|
|
4
4
|
/** Reserved event delivered after a model call reports token usage. */
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { t as AgentCallUsage } from "./text-logic-
|
|
1
|
+
import { t as AgentCallUsage } from "./text-logic-x17uLBwA.cjs";
|
|
2
2
|
import { EventObject } from "xstate";
|
|
3
3
|
//#region src/usage.d.ts
|
|
4
4
|
/** Reserved event delivered after a model call reports token usage. */
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { At as SystemMessage, Ct as FilePart, Ft as ToolResultPart, It as UserMessage, Mt as ToolCallPart, Nt as ToolMessage, jt as TextPart, kt as StandardSchemaV1, lt as AgentMessage, vt as AssistantMessage, wt as ImagePart } from "./text-logic-x17uLBwA.mjs";
|
|
2
2
|
import { AnyMachineSnapshot, AnyStateMachine, StateValue } from "xstate";
|
|
3
3
|
//#region src/equivalence.d.ts
|
|
4
4
|
/** An error as recorded: JSON-safe. */
|
|
@@ -15,6 +15,14 @@ interface RecordedError {
|
|
|
15
15
|
markers?: string[];
|
|
16
16
|
/** `name` was an own enumerable field, as the AI SDK's errors set it; replay keeps it so. */
|
|
17
17
|
ownName?: true;
|
|
18
|
+
/** The error's `cause`, when it was an Error; replay restores it. */
|
|
19
|
+
cause?: RecordedError;
|
|
20
|
+
/** An `AggregateError`'s `errors`, each a recorded error or `{ value }`
|
|
21
|
+
* for a reason that was not an Error (`{}` for `undefined`); replay revives
|
|
22
|
+
* an `AggregateError`. */
|
|
23
|
+
errors?: (RecordedError | {
|
|
24
|
+
value?: unknown;
|
|
25
|
+
})[];
|
|
18
26
|
}
|
|
19
27
|
/** One model call. `call` is its 0-based position among model calls. */
|
|
20
28
|
interface RecordedModelCall {
|
|
@@ -110,6 +118,19 @@ interface Recording {
|
|
|
110
118
|
interface CreateRecorderOptions {
|
|
111
119
|
/** Answer model and tool calls from this recording instead of running them. */
|
|
112
120
|
replay?: Recording;
|
|
121
|
+
/**
|
|
122
|
+
* In replay, builds the error a recorded failure throws, so code that checks
|
|
123
|
+
* `instanceof` sees its own class. Called for the error and each `cause`
|
|
124
|
+
* and `errors` entry; return `undefined` for the default, an `Error` (or
|
|
125
|
+
* `AggregateError`) with the recorded name, message, and fields.
|
|
126
|
+
*
|
|
127
|
+
* @example
|
|
128
|
+
* ```ts
|
|
129
|
+
* reviveError: (recorded) =>
|
|
130
|
+
* recorded.name === "AbortError" ? new AbortError(recorded.message) : undefined,
|
|
131
|
+
* ```
|
|
132
|
+
*/
|
|
133
|
+
reviveError?: (recorded: RecordedError) => Error | undefined;
|
|
113
134
|
/**
|
|
114
135
|
* In replay, `"replay"` (default) returns each tool's recorded result
|
|
115
136
|
* without running it; `"live"` runs the wrapped tool (your stub) and records
|
|
@@ -298,21 +319,6 @@ declare function compareRecordings(expected: Recording, actual: Recording, optio
|
|
|
298
319
|
//#region src/utils.d.ts
|
|
299
320
|
/** Returns the human-readable text carried by a message, ignoring non-text parts. */
|
|
300
321
|
declare function getMessageText(message: AgentMessage): string;
|
|
301
|
-
/**
|
|
302
|
-
* A stable, dependency-free structural fingerprint of a machine — a short hex
|
|
303
|
-
* `djb2` hash over its **structural** config only: state ids/nesting, transition
|
|
304
|
-
* event types and targets, invoke `src`s, `initial`, and any other serializable
|
|
305
|
-
* config fields. Function values (context/output builders, prompts, inline
|
|
306
|
-
* guards/actions) are excluded entirely, so two machines that differ only in
|
|
307
|
-
* their prompts or executors hash identically; adding/removing/retargeting a
|
|
308
|
-
* state or transition changes the hash.
|
|
309
|
-
*
|
|
310
|
-
* Used by {@link createAgentRuntime} to stamp settled snapshots with a `version` and to
|
|
311
|
-
* detect a structurally-edited machine on resume. It is a change detector, not
|
|
312
|
-
* a cryptographic digest — collisions are possible but unlikely for real
|
|
313
|
-
* configs. Declare `createMachine({ version })` to pin an explicit version instead.
|
|
314
|
-
*/
|
|
315
|
-
declare function getMachineStructuralHash(machine: AnyStateMachine): string;
|
|
316
322
|
/** Builds a {@link UserMessage} from a string or multimodal content parts. */
|
|
317
323
|
declare function userMessage(content: string | Array<TextPart | ImagePart | FilePart>): UserMessage;
|
|
318
324
|
/** Builds an {@link AssistantMessage} from a string or content parts (text, files, tool calls/results). */
|
|
@@ -400,4 +406,4 @@ declare function getJsonSchema(schema?: StandardSchemaV1): Promise<Record<string
|
|
|
400
406
|
*/
|
|
401
407
|
declare function getJsonSchemaSync(schema?: StandardSchemaV1): Record<string, unknown> | undefined;
|
|
402
408
|
//#endregion
|
|
403
|
-
export {
|
|
409
|
+
export { RecordedToolCall as A, RecordableLanguageModel as C, RecordedError as D, RecordedEffect as E, createRecorder as F, sameModelCallShape as I, Recording as M, RecordingDivergenceError as N, RecordedModelCall as O, compareRecordings as P, RecordableDecisionModel as S, RecordedCall as T, ModelAnswer as _, getMessageText as a, RecordCallOptions as b, isStandardSchema as c, userMessage as d, CompareRecordingsOptions as f, EquivalenceReport as g, EffectDivergenceReason as h, getJsonSchemaSync as i, Recorder as j, RecordedOutput as k, systemMessage as l, EffectDivergence as m, assistantMessage as n, getStateMeta as o, CreateRecorderOptions as p, getJsonSchema as r, getStatePath as s, MetaOfSnapshot as t, toolMessage as u, ModelCallMatch as v, RecordableModel as w, RecordClientOptions as x, NormalizeEffect as y };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { At as SystemMessage, Ct as FilePart, Ft as ToolResultPart, It as UserMessage, Mt as ToolCallPart, Nt as ToolMessage, jt as TextPart, kt as StandardSchemaV1, lt as AgentMessage, vt as AssistantMessage, wt as ImagePart } from "./text-logic-x17uLBwA.cjs";
|
|
2
2
|
import { AnyMachineSnapshot, AnyStateMachine, StateValue } from "xstate";
|
|
3
3
|
//#region src/equivalence.d.ts
|
|
4
4
|
/** An error as recorded: JSON-safe. */
|
|
@@ -15,6 +15,14 @@ interface RecordedError {
|
|
|
15
15
|
markers?: string[];
|
|
16
16
|
/** `name` was an own enumerable field, as the AI SDK's errors set it; replay keeps it so. */
|
|
17
17
|
ownName?: true;
|
|
18
|
+
/** The error's `cause`, when it was an Error; replay restores it. */
|
|
19
|
+
cause?: RecordedError;
|
|
20
|
+
/** An `AggregateError`'s `errors`, each a recorded error or `{ value }`
|
|
21
|
+
* for a reason that was not an Error (`{}` for `undefined`); replay revives
|
|
22
|
+
* an `AggregateError`. */
|
|
23
|
+
errors?: (RecordedError | {
|
|
24
|
+
value?: unknown;
|
|
25
|
+
})[];
|
|
18
26
|
}
|
|
19
27
|
/** One model call. `call` is its 0-based position among model calls. */
|
|
20
28
|
interface RecordedModelCall {
|
|
@@ -110,6 +118,19 @@ interface Recording {
|
|
|
110
118
|
interface CreateRecorderOptions {
|
|
111
119
|
/** Answer model and tool calls from this recording instead of running them. */
|
|
112
120
|
replay?: Recording;
|
|
121
|
+
/**
|
|
122
|
+
* In replay, builds the error a recorded failure throws, so code that checks
|
|
123
|
+
* `instanceof` sees its own class. Called for the error and each `cause`
|
|
124
|
+
* and `errors` entry; return `undefined` for the default, an `Error` (or
|
|
125
|
+
* `AggregateError`) with the recorded name, message, and fields.
|
|
126
|
+
*
|
|
127
|
+
* @example
|
|
128
|
+
* ```ts
|
|
129
|
+
* reviveError: (recorded) =>
|
|
130
|
+
* recorded.name === "AbortError" ? new AbortError(recorded.message) : undefined,
|
|
131
|
+
* ```
|
|
132
|
+
*/
|
|
133
|
+
reviveError?: (recorded: RecordedError) => Error | undefined;
|
|
113
134
|
/**
|
|
114
135
|
* In replay, `"replay"` (default) returns each tool's recorded result
|
|
115
136
|
* without running it; `"live"` runs the wrapped tool (your stub) and records
|
|
@@ -298,21 +319,6 @@ declare function compareRecordings(expected: Recording, actual: Recording, optio
|
|
|
298
319
|
//#region src/utils.d.ts
|
|
299
320
|
/** Returns the human-readable text carried by a message, ignoring non-text parts. */
|
|
300
321
|
declare function getMessageText(message: AgentMessage): string;
|
|
301
|
-
/**
|
|
302
|
-
* A stable, dependency-free structural fingerprint of a machine — a short hex
|
|
303
|
-
* `djb2` hash over its **structural** config only: state ids/nesting, transition
|
|
304
|
-
* event types and targets, invoke `src`s, `initial`, and any other serializable
|
|
305
|
-
* config fields. Function values (context/output builders, prompts, inline
|
|
306
|
-
* guards/actions) are excluded entirely, so two machines that differ only in
|
|
307
|
-
* their prompts or executors hash identically; adding/removing/retargeting a
|
|
308
|
-
* state or transition changes the hash.
|
|
309
|
-
*
|
|
310
|
-
* Used by {@link createAgentRuntime} to stamp settled snapshots with a `version` and to
|
|
311
|
-
* detect a structurally-edited machine on resume. It is a change detector, not
|
|
312
|
-
* a cryptographic digest — collisions are possible but unlikely for real
|
|
313
|
-
* configs. Declare `createMachine({ version })` to pin an explicit version instead.
|
|
314
|
-
*/
|
|
315
|
-
declare function getMachineStructuralHash(machine: AnyStateMachine): string;
|
|
316
322
|
/** Builds a {@link UserMessage} from a string or multimodal content parts. */
|
|
317
323
|
declare function userMessage(content: string | Array<TextPart | ImagePart | FilePart>): UserMessage;
|
|
318
324
|
/** Builds an {@link AssistantMessage} from a string or content parts (text, files, tool calls/results). */
|
|
@@ -400,4 +406,4 @@ declare function getJsonSchema(schema?: StandardSchemaV1): Promise<Record<string
|
|
|
400
406
|
*/
|
|
401
407
|
declare function getJsonSchemaSync(schema?: StandardSchemaV1): Record<string, unknown> | undefined;
|
|
402
408
|
//#endregion
|
|
403
|
-
export {
|
|
409
|
+
export { RecordedToolCall as A, RecordableLanguageModel as C, RecordedError as D, RecordedEffect as E, createRecorder as F, sameModelCallShape as I, Recording as M, RecordingDivergenceError as N, RecordedModelCall as O, compareRecordings as P, RecordableDecisionModel as S, RecordedCall as T, ModelAnswer as _, getMessageText as a, RecordCallOptions as b, isStandardSchema as c, userMessage as d, CompareRecordingsOptions as f, EquivalenceReport as g, EffectDivergenceReason as h, getJsonSchemaSync as i, Recorder as j, RecordedOutput as k, systemMessage as l, EffectDivergence as m, assistantMessage as n, getStateMeta as o, CreateRecorderOptions as p, getJsonSchema as r, getStatePath as s, MetaOfSnapshot as t, toolMessage as u, ModelCallMatch as v, RecordableModel as w, RecordClientOptions as x, NormalizeEffect as y };
|
package/dist/validate.cjs
CHANGED
|
@@ -21,10 +21,10 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
21
21
|
enumerable: true
|
|
22
22
|
}) : target, mod));
|
|
23
23
|
//#endregion
|
|
24
|
-
const require_event_log = require("./event-log-
|
|
25
|
-
const require_decision = require("./decision-
|
|
26
|
-
const require_setup_agent = require("./setup-agent-
|
|
27
|
-
const require_verify = require("./verify-
|
|
24
|
+
const require_event_log = require("./event-log-dSOnXHuG.cjs");
|
|
25
|
+
const require_decision = require("./decision-D8bcIGXQ.cjs");
|
|
26
|
+
const require_setup_agent = require("./setup-agent-Dv5VpuHe.cjs");
|
|
27
|
+
const require_verify = require("./verify-Dek6gHCS.cjs");
|
|
28
28
|
let ajv_dist_2020_js = require("ajv/dist/2020.js");
|
|
29
29
|
ajv_dist_2020_js = __toESM(ajv_dist_2020_js, 1);
|
|
30
30
|
//#region schemas/agent-workflow.json
|
package/dist/validate.d.cts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { d as ExplorePathsOptions } from "./verify-BM0sGmsf.cjs";
|
|
2
|
+
import { _ as FromConfigOptions, f as AgentWorkflowConfig, y as SchemaCompiler } from "./setup-agent-Cv8Hk3mX.cjs";
|
|
3
3
|
import { AnyActorLogic, AnyStateMachine } from "xstate";
|
|
4
4
|
//#region src/validate/validate-config.d.ts
|
|
5
5
|
/**
|
package/dist/validate.d.mts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { d as ExplorePathsOptions } from "./verify-H3Pr2gNw.mjs";
|
|
2
|
+
import { _ as FromConfigOptions, f as AgentWorkflowConfig, y as SchemaCompiler } from "./setup-agent-DgFAhaPJ.mjs";
|
|
3
3
|
import { AnyActorLogic, AnyStateMachine } from "xstate";
|
|
4
4
|
//#region src/validate/validate-config.d.ts
|
|
5
5
|
/**
|
package/dist/validate.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { p as payloadNeedsFillModel } from "./decision-
|
|
3
|
-
import { i as setupAgentFromConfig } from "./setup-agent-
|
|
4
|
-
import { a as explorePaths, s as lintAgentMachine } from "./verify-
|
|
1
|
+
import { D as DECIDE_ACTOR, O as GENERATE_TEXT_ACTOR, T as ACT_ACTOR, k as STREAM_TEXT_ACTOR } from "./event-log-CEq_bVXQ.mjs";
|
|
2
|
+
import { p as payloadNeedsFillModel } from "./decision-B41YBx5U.mjs";
|
|
3
|
+
import { i as setupAgentFromConfig } from "./setup-agent-CLm6QVGy.mjs";
|
|
4
|
+
import { a as explorePaths, s as lintAgentMachine } from "./verify-CWlg3j7B.mjs";
|
|
5
5
|
import Ajv2020 from "ajv/dist/2020.js";
|
|
6
6
|
//#region schemas/agent-workflow.json
|
|
7
7
|
var agent_workflow_default = {
|