@statelyai/agent 2.0.0-alpha.40 → 2.0.0-alpha.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/dist/evals.cjs +1 -1
- package/dist/evals.d.cts +1 -1
- package/dist/evals.d.mts +1 -1
- package/dist/evals.mjs +1 -1
- package/dist/index.d.cts +1 -1
- package/dist/index.d.mts +1 -1
- package/dist/testing.cjs +376 -17
- package/dist/testing.d.cts +181 -5
- package/dist/testing.d.mts +181 -5
- package/dist/testing.mjs +377 -20
- package/dist/{utils-DAclF-kw.d.cts → utils-BMxUcjay.d.cts} +57 -1
- package/dist/{utils-B8k-YPYI.d.mts → utils-BV7pxhD-.d.mts} +57 -1
- package/dist/validate.cjs +1 -1
- package/dist/validate.d.cts +1 -1
- package/dist/validate.d.mts +1 -1
- package/dist/validate.mjs +1 -1
- package/dist/{verify-CIu4mqIZ.d.mts → verify-C1UU84eq.d.mts} +7 -1
- package/dist/{verify-CTHlmBbh.d.cts → verify-DlN6tK8_.d.cts} +7 -1
- package/dist/{verify-D8UDqU4l.cjs → verify-IVYsI026.cjs} +65 -14
- package/dist/{verify-Cacz6XSk.mjs → verify-lKyu3buS.mjs} +65 -14
- package/docs/evals.md +1 -0
- package/docs/index.md +2 -2
- package/docs/meta.json +1 -0
- package/docs/verify-in-place.md +326 -0
- package/docs/verify.md +67 -2
- package/llms.txt +1 -0
- package/package.json +1 -1
- package/readme.md +1 -0
package/dist/testing.mjs
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
import { I as getCallUsage, L as isTextLogic, R as normalizeGeneratorResult, c as agentCallSite, d as canonicalEventJson, ft as isRecord, pt as AgentError, tt as getStateMeta, x as replay } from "./event-log-CEq_bVXQ.mjs";
|
|
1
|
+
import { I as getCallUsage, K as getRegisteredAgentExecutionOptions, L as isTextLogic, R as normalizeGeneratorResult, c as agentCallSite, d as canonicalEventJson, ft as isRecord, pt as AgentError, tt as getStateMeta, x as replay } from "./event-log-CEq_bVXQ.mjs";
|
|
2
2
|
import { s as isDecisionLogic } from "./decision-B41YBx5U.mjs";
|
|
3
3
|
import { i as isActLogic } from "./act-Bsz-hUfD.mjs";
|
|
4
4
|
import { n as createInMemoryEventLogStore } from "./event-log-store-DLAfNM34.mjs";
|
|
5
|
-
import { f as isStateMachineLogic, g as toJsonValue, l as createAgentRuntime, m as runToQuiescence } from "./run-agent-D3gW9wfd.mjs";
|
|
5
|
+
import { f as isStateMachineLogic, g as toJsonValue, l as createAgentRuntime, m as runToQuiescence, p as runAgent } from "./run-agent-D3gW9wfd.mjs";
|
|
6
6
|
import { n as mapEffects, t as isAsyncIterable } from "./effects-B9IvUPVh.mjs";
|
|
7
|
-
import { a as explorePaths, c as simulateAgent, i as canReach, l as textOutput, n as AgentUnknownStateError, r as assertAgentMachine, s as lintAgentMachine, t as AgentLintError } from "./verify-
|
|
7
|
+
import { a as explorePaths, c as simulateAgent, i as canReach, l as textOutput, n as AgentUnknownStateError, r as assertAgentMachine, s as lintAgentMachine, t as AgentLintError } from "./verify-lKyu3buS.mjs";
|
|
8
8
|
import { t as matchesTrajectory } from "./trajectory-BUXj_1Ue.mjs";
|
|
9
|
+
import { createAsyncLogic } from "xstate";
|
|
9
10
|
//#region src/equivalence.ts
|
|
10
11
|
/**
|
|
11
12
|
* Verified equivalence — prove a port behaves like the code it replaces.
|
|
@@ -38,6 +39,86 @@ import { t as matchesTrajectory } from "./trajectory-BUXj_1Ue.mjs";
|
|
|
38
39
|
* neither `ai` nor `openai`.
|
|
39
40
|
* @module
|
|
40
41
|
*/
|
|
42
|
+
const MODEL_METADATA = [
|
|
43
|
+
"specificationVersion",
|
|
44
|
+
"provider",
|
|
45
|
+
"modelId",
|
|
46
|
+
"supportedUrls"
|
|
47
|
+
];
|
|
48
|
+
const MODEL_METHODS = [
|
|
49
|
+
"doGenerate",
|
|
50
|
+
"doStream",
|
|
51
|
+
"doDecide",
|
|
52
|
+
"doEvaluate"
|
|
53
|
+
];
|
|
54
|
+
const MODEL_KEYS = [...MODEL_METADATA, ...MODEL_METHODS];
|
|
55
|
+
const aiSdkInternals = /* @__PURE__ */ new WeakMap();
|
|
56
|
+
/**
|
|
57
|
+
* Records AI SDK code that builds its model inline or passes module-level
|
|
58
|
+
* tools to `generateText`: there is nothing to inject, so the test mocks `"ai"`
|
|
59
|
+
* and routes each call through the recorder. Every `generateText` and
|
|
60
|
+
* `streamText` call runs on the real AI SDK with `seam.model` in place of the
|
|
61
|
+
* code's model (also in place of a model `prepareStep` returns) and its
|
|
62
|
+
* `tools` wrapped with `recorder.tools`. Everything else in `"ai"` stays real,
|
|
63
|
+
* so the code's own tool loop runs.
|
|
64
|
+
*
|
|
65
|
+
* Each model attempt made during the call carries the call's index as
|
|
66
|
+
* {@link RecordedModelCall.aiSdkCall} when its model is a `recorder.model`
|
|
67
|
+
* wrapper passed as is and the recorder came from `createRecorder`. Other
|
|
68
|
+
* attempts carry no index, and who resent them is unknown.
|
|
69
|
+
*
|
|
70
|
+
* `current` returns the seam of the run being recorded. A call while it returns
|
|
71
|
+
* `undefined` throws, so no test reaches a real provider.
|
|
72
|
+
*
|
|
73
|
+
* @example
|
|
74
|
+
* ```ts
|
|
75
|
+
* vi.mock("ai", async (importOriginal) => {
|
|
76
|
+
* const ai = await importOriginal<typeof import("ai")>();
|
|
77
|
+
* const { currentSeam } = await import("./seam"); // { recorder, model: recorder.model(standIn) }
|
|
78
|
+
* return { ...ai, ...recordAiSdk(ai, currentSeam) };
|
|
79
|
+
* });
|
|
80
|
+
* ```
|
|
81
|
+
*/
|
|
82
|
+
function recordAiSdk(ai, current) {
|
|
83
|
+
const wrap = (name, call) => (options) => {
|
|
84
|
+
const seam = current();
|
|
85
|
+
if (!seam) throw new Error(`recordAiSdk: ${name} was called while no run is recording. Return { recorder } from current() while the code runs.`);
|
|
86
|
+
const internals = aiSdkInternals.get(seam.recorder);
|
|
87
|
+
const index = internals ? internals.calls++ : 0;
|
|
88
|
+
const tag = (model) => {
|
|
89
|
+
const base = internals?.unwrap(model);
|
|
90
|
+
return base ? internals.wrap(base, index) : model;
|
|
91
|
+
};
|
|
92
|
+
const model = seam.model ?? options.model;
|
|
93
|
+
const prepareStep = options.prepareStep;
|
|
94
|
+
return call({
|
|
95
|
+
...options,
|
|
96
|
+
...model !== void 0 ? { model: tag(model) } : {},
|
|
97
|
+
...prepareStep ? { prepareStep: async (...args) => {
|
|
98
|
+
const step = await prepareStep(...args);
|
|
99
|
+
if (step?.model === void 0) return step;
|
|
100
|
+
const model = tag(seam.model ?? step.model);
|
|
101
|
+
const own = (key) => key === "model" ? model : Reflect.get(step, key, step);
|
|
102
|
+
return new Proxy({}, {
|
|
103
|
+
get: (_target, key) => own(key),
|
|
104
|
+
has: (_target, key) => key === "model" || key in step,
|
|
105
|
+
ownKeys: () => [.../* @__PURE__ */ new Set([...Reflect.ownKeys(step), "model"])],
|
|
106
|
+
getOwnPropertyDescriptor: (_target, key) => key === "model" || Object.prototype.hasOwnProperty.call(step, key) ? {
|
|
107
|
+
value: own(key),
|
|
108
|
+
enumerable: true,
|
|
109
|
+
configurable: true,
|
|
110
|
+
writable: true
|
|
111
|
+
} : void 0
|
|
112
|
+
});
|
|
113
|
+
} } : {},
|
|
114
|
+
...options.tools ? { tools: seam.recorder.tools(options.tools) } : {}
|
|
115
|
+
});
|
|
116
|
+
};
|
|
117
|
+
return {
|
|
118
|
+
generateText: wrap("generateText", ai.generateText),
|
|
119
|
+
...ai.streamText ? { streamText: wrap("streamText", ai.streamText) } : {}
|
|
120
|
+
};
|
|
121
|
+
}
|
|
41
122
|
/**
|
|
42
123
|
* Thrown in replay when the port asks for something the recording cannot
|
|
43
124
|
* answer: a model call past the last recorded one, or a tool call the original
|
|
@@ -227,7 +308,28 @@ function createRecorder(options = {}) {
|
|
|
227
308
|
* Records a model call around `run`, or answers it from the recording.
|
|
228
309
|
* `fallback` runs it on the stand-in when the recording has no answer.
|
|
229
310
|
*/
|
|
230
|
-
|
|
311
|
+
/**
|
|
312
|
+
* A retry resends the request of a model call that failed. Within one AI SDK
|
|
313
|
+
* call, the failed attempt is its own last attempt, even when a concurrent
|
|
314
|
+
* call came in between or claimed it first. Otherwise it is the last model
|
|
315
|
+
* call, which the code resent.
|
|
316
|
+
*/
|
|
317
|
+
function linkRetry(effect) {
|
|
318
|
+
const models = effects.filter((earlier) => earlier.kind === "model").reverse();
|
|
319
|
+
const failedWith = (earlier) => earlier?.error !== void 0 && earlier.method === effect.method && stableStringify(earlier.request) === stableStringify(effect.request);
|
|
320
|
+
const { aiSdkCall } = effect;
|
|
321
|
+
if (aiSdkCall !== void 0) {
|
|
322
|
+
const own = models.find((earlier) => earlier.aiSdkCall === aiSdkCall);
|
|
323
|
+
const claimedBy = own?.retriedBy === void 0 ? void 0 : models.find((earlier) => earlier.call === own.retriedBy);
|
|
324
|
+
if (own && failedWith(own) && claimedBy?.aiSdkCall !== aiSdkCall) {
|
|
325
|
+
own.retriedBy = effect.call;
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
const last = models[0];
|
|
330
|
+
if (last && failedWith(last) && last.retriedBy === void 0) last.retriedBy = effect.call;
|
|
331
|
+
}
|
|
332
|
+
async function modelCall(seam, method, request, streamed, run, fallback, aiSdkCall) {
|
|
231
333
|
const effect = {
|
|
232
334
|
kind: "model",
|
|
233
335
|
seam,
|
|
@@ -239,6 +341,8 @@ function createRecorder(options = {}) {
|
|
|
239
341
|
"signal"
|
|
240
342
|
]))
|
|
241
343
|
};
|
|
344
|
+
if (aiSdkCall !== void 0) effect.aiSdkCall = aiSdkCall;
|
|
345
|
+
linkRetry(effect);
|
|
242
346
|
effects.push(effect);
|
|
243
347
|
if (replay) {
|
|
244
348
|
const replayed = nextReplayedModel(effect);
|
|
@@ -468,22 +572,47 @@ function createRecorder(options = {}) {
|
|
|
468
572
|
}
|
|
469
573
|
return result;
|
|
470
574
|
}
|
|
471
|
-
|
|
575
|
+
const wrappedModels = /* @__PURE__ */ new WeakMap();
|
|
576
|
+
function wrapModel(model, aiSdkCall) {
|
|
577
|
+
const base = model;
|
|
578
|
+
const stand = options.fallback?.model;
|
|
579
|
+
const wrapped = Object.create(base);
|
|
580
|
+
for (const key of MODEL_METADATA) {
|
|
581
|
+
if (!(key in base)) continue;
|
|
582
|
+
Object.defineProperty(wrapped, key, {
|
|
583
|
+
get: () => Reflect.get(base, key, base),
|
|
584
|
+
enumerable: true,
|
|
585
|
+
configurable: true
|
|
586
|
+
});
|
|
587
|
+
}
|
|
588
|
+
for (const method of MODEL_METHODS) {
|
|
589
|
+
if (typeof base[method] !== "function") continue;
|
|
590
|
+
const invoke = (target, callOptions) => target[method].call(target, callOptions);
|
|
591
|
+
Object.defineProperty(wrapped, method, {
|
|
592
|
+
value: (callOptions) => modelCall("ai-sdk", method, callOptions, method === "doStream", () => invoke(base, callOptions), stand && typeof stand[method] === "function" ? () => invoke(stand, callOptions) : void 0, aiSdkCall),
|
|
593
|
+
writable: true,
|
|
594
|
+
enumerable: true,
|
|
595
|
+
configurable: true
|
|
596
|
+
});
|
|
597
|
+
}
|
|
598
|
+
wrappedModels.set(wrapped, {
|
|
599
|
+
base,
|
|
600
|
+
made: new Map([...MODEL_KEYS].map((key) => [key, Object.getOwnPropertyDescriptor(wrapped, key)]))
|
|
601
|
+
});
|
|
602
|
+
return wrapped;
|
|
603
|
+
}
|
|
604
|
+
function unwrapModel(model) {
|
|
605
|
+
const entry = wrappedModels.get(model);
|
|
606
|
+
if (!entry) return void 0;
|
|
607
|
+
for (const [key, made] of entry.made) {
|
|
608
|
+
const now = Object.getOwnPropertyDescriptor(model, key);
|
|
609
|
+
if (now?.value !== made?.value || now?.get !== made?.get) return void 0;
|
|
610
|
+
}
|
|
611
|
+
return entry.base;
|
|
612
|
+
}
|
|
613
|
+
const recorder = {
|
|
472
614
|
model(model) {
|
|
473
|
-
|
|
474
|
-
const stand = options.fallback?.model;
|
|
475
|
-
const wrapped = Object.create(base);
|
|
476
|
-
for (const method of [
|
|
477
|
-
"doGenerate",
|
|
478
|
-
"doStream",
|
|
479
|
-
"doDecide",
|
|
480
|
-
"doEvaluate"
|
|
481
|
-
]) {
|
|
482
|
-
if (typeof base[method] !== "function") continue;
|
|
483
|
-
const invoke = (target, callOptions) => target[method].call(target, callOptions);
|
|
484
|
-
wrapped[method] = (callOptions) => modelCall("ai-sdk", method, callOptions, method === "doStream", () => invoke(base, callOptions), stand && typeof stand[method] === "function" ? () => invoke(stand, callOptions) : void 0);
|
|
485
|
-
}
|
|
486
|
-
return wrapped;
|
|
615
|
+
return wrapModel(model);
|
|
487
616
|
},
|
|
488
617
|
openai(client) {
|
|
489
618
|
const wrapCreate = (resource, method) => new Proxy(resource, { get(target, key, receiver) {
|
|
@@ -567,6 +696,12 @@ function createRecorder(options = {}) {
|
|
|
567
696
|
};
|
|
568
697
|
}
|
|
569
698
|
};
|
|
699
|
+
aiSdkInternals.set(recorder, {
|
|
700
|
+
unwrap: (model) => model !== null && (typeof model === "object" || typeof model === "function") ? unwrapModel(model) : void 0,
|
|
701
|
+
wrap: wrapModel,
|
|
702
|
+
calls: 0
|
|
703
|
+
});
|
|
704
|
+
return recorder;
|
|
570
705
|
}
|
|
571
706
|
/** True when two model calls offer the same tool names and response format. */
|
|
572
707
|
function sameModelCallShape(expected, actual) {
|
|
@@ -1694,6 +1829,228 @@ function structurallyEqual(a, b) {
|
|
|
1694
1829
|
return keys.length === Object.keys(b).length && keys.every((key) => Object.hasOwn(b, key) && structurallyEqual(a[key], b[key]));
|
|
1695
1830
|
}
|
|
1696
1831
|
//#endregion
|
|
1832
|
+
//#region src/conformance.ts
|
|
1833
|
+
/**
|
|
1834
|
+
* Conformance: check a recorded run of existing code against a model of it.
|
|
1835
|
+
*
|
|
1836
|
+
* The model is an ordinary machine whose actors stand for the code's work.
|
|
1837
|
+
* `checkConformance` runs the model with every actor answered from the
|
|
1838
|
+
* recording: each one consumes the next observation of its kind. The first
|
|
1839
|
+
* time the model does something the recording does not show next, the run
|
|
1840
|
+
* stops and the step is reported. The code itself is not changed or run.
|
|
1841
|
+
*/
|
|
1842
|
+
const NEVER = new Promise(() => {});
|
|
1843
|
+
function inlineInvokeStates(machine) {
|
|
1844
|
+
const found = [];
|
|
1845
|
+
const stack = [machine.root];
|
|
1846
|
+
while (stack.length > 0) {
|
|
1847
|
+
const node = stack.pop();
|
|
1848
|
+
const invoke = node.config.invoke;
|
|
1849
|
+
if ((invoke === void 0 ? [] : Array.isArray(invoke) ? invoke : [invoke]).some((entry) => typeof entry.src !== "string")) found.push(node.path.join(".") || "(root)");
|
|
1850
|
+
stack.push(...Object.values(node.states));
|
|
1851
|
+
}
|
|
1852
|
+
return found.sort();
|
|
1853
|
+
}
|
|
1854
|
+
function actorSources(machine) {
|
|
1855
|
+
const actors = getRegisteredAgentExecutionOptions(machine).actors ?? machine.sources?.actors ?? {};
|
|
1856
|
+
return Object.keys(actors);
|
|
1857
|
+
}
|
|
1858
|
+
/**
|
|
1859
|
+
* Checks a recorded run of existing code against a model of that code.
|
|
1860
|
+
*
|
|
1861
|
+
* The model runs with `runAgent`, and every actor in `actors` is answered from
|
|
1862
|
+
* `observations` instead of running: an actor with `observes` consumes the next
|
|
1863
|
+
* observation, which must be of that kind. The report names the first
|
|
1864
|
+
* deviation: where the model started something the recording does not show
|
|
1865
|
+
* next, or ended while the recording continued.
|
|
1866
|
+
*
|
|
1867
|
+
* Use it to verify existing code in place: prove the model's rules with
|
|
1868
|
+
* `canReach`, then check that recorded runs of the code conform to the model.
|
|
1869
|
+
* A conforming run shows the code behaved as the model allows on that run; it
|
|
1870
|
+
* does not prove the code can never break a rule.
|
|
1871
|
+
*
|
|
1872
|
+
* Every actor the model invokes must be declared by name: a model whose invoke
|
|
1873
|
+
* names logic inline throws `inline-actor-source`, because that logic would
|
|
1874
|
+
* run. A callback in `actors` that throws ends the check with
|
|
1875
|
+
* `conformance-callback-error` instead of counting as an actor failure.
|
|
1876
|
+
*
|
|
1877
|
+
* @example
|
|
1878
|
+
* ```ts
|
|
1879
|
+
* const report = await checkConformance(loopModel, {
|
|
1880
|
+
* input: { maxIter: 3 },
|
|
1881
|
+
* observations: recording.effects.flatMap(toObservation),
|
|
1882
|
+
* actors: {
|
|
1883
|
+
* route: { observes: "route", output: ({ observation }) => observation.names },
|
|
1884
|
+
* runTurn: { observes: "turn", matches: ({ input, observation }) => input.name === observation.agent },
|
|
1885
|
+
* save: {}, // not visible in this recording
|
|
1886
|
+
* },
|
|
1887
|
+
* });
|
|
1888
|
+
* report.conforms; // false when the run did something the model does not allow
|
|
1889
|
+
* ```
|
|
1890
|
+
*/
|
|
1891
|
+
async function checkConformance(machine, options) {
|
|
1892
|
+
const { observations, actors: specs, events = {} } = options;
|
|
1893
|
+
const inline = inlineInvokeStates(machine);
|
|
1894
|
+
if (inline.length > 0) throw new AgentError("inline-actor-source", `checkConformance: ${inline.join(", ")} invoke${inline.length === 1 ? "s" : ""} inline actor logic, which would run for real. Declare each actor by name in setup ({ actors }) and invoke it by src, so the recording can answer it.`);
|
|
1895
|
+
for (const src of Object.keys(specs)) if (!actorSources(machine).includes(src)) throw new AgentError("unknown-actor-source", `checkConformance: actors.${src} names no actor source the machine declares (${actorSources(machine).join(", ") || "none"}).`);
|
|
1896
|
+
let index = 0;
|
|
1897
|
+
let deviation;
|
|
1898
|
+
let callbackError;
|
|
1899
|
+
let state = machine.config.initial;
|
|
1900
|
+
const unobserved = /* @__PURE__ */ new Set();
|
|
1901
|
+
const trace = [];
|
|
1902
|
+
const controller = new AbortController();
|
|
1903
|
+
const callbackFailure = (name, cause) => new AgentError("conformance-callback-error", `checkConformance: ${name} threw: ${cause instanceof Error ? cause.message : String(cause)}`, { cause });
|
|
1904
|
+
const callback = (name, fn) => {
|
|
1905
|
+
try {
|
|
1906
|
+
return fn();
|
|
1907
|
+
} catch (cause) {
|
|
1908
|
+
throw callbackFailure(name, cause);
|
|
1909
|
+
}
|
|
1910
|
+
};
|
|
1911
|
+
const deviate = (reason, expected, actorInput) => {
|
|
1912
|
+
deviation ??= {
|
|
1913
|
+
reason,
|
|
1914
|
+
index,
|
|
1915
|
+
expected,
|
|
1916
|
+
observed: observations[index],
|
|
1917
|
+
state,
|
|
1918
|
+
...actorInput !== void 0 ? { actorInput } : {}
|
|
1919
|
+
};
|
|
1920
|
+
controller.abort();
|
|
1921
|
+
return NEVER;
|
|
1922
|
+
};
|
|
1923
|
+
const answer = (src, spec) => createAsyncLogic({ run: async ({ input }) => {
|
|
1924
|
+
if (deviation || callbackError) return NEVER;
|
|
1925
|
+
if (!spec) return deviate("unmapped-actor", src, input);
|
|
1926
|
+
let observation;
|
|
1927
|
+
if (spec.observes === void 0) unobserved.add(src);
|
|
1928
|
+
else {
|
|
1929
|
+
const next = observations[index];
|
|
1930
|
+
if (next === void 0) return deviate("missing-observation", src, input);
|
|
1931
|
+
if (next.kind !== spec.observes) return deviate("unexpected-observation", src, input);
|
|
1932
|
+
observation = next;
|
|
1933
|
+
index += 1;
|
|
1934
|
+
}
|
|
1935
|
+
const args = {
|
|
1936
|
+
input,
|
|
1937
|
+
observation,
|
|
1938
|
+
observations,
|
|
1939
|
+
index
|
|
1940
|
+
};
|
|
1941
|
+
const call = (name, fn) => {
|
|
1942
|
+
try {
|
|
1943
|
+
return fn();
|
|
1944
|
+
} catch (cause) {
|
|
1945
|
+
callbackError ??= callbackFailure(`actors.${src}.${name}`, cause);
|
|
1946
|
+
controller.abort();
|
|
1947
|
+
return NEVER;
|
|
1948
|
+
}
|
|
1949
|
+
};
|
|
1950
|
+
if (spec.matches) {
|
|
1951
|
+
const fits = call("matches", () => spec.matches(args));
|
|
1952
|
+
if (fits === NEVER) return NEVER;
|
|
1953
|
+
if (!fits) {
|
|
1954
|
+
index -= observation ? 1 : 0;
|
|
1955
|
+
return deviate("input-mismatch", src, input);
|
|
1956
|
+
}
|
|
1957
|
+
}
|
|
1958
|
+
const error = call("error", () => spec.error?.(args));
|
|
1959
|
+
if (error === NEVER) return NEVER;
|
|
1960
|
+
if (error !== void 0) throw error;
|
|
1961
|
+
return call("output", () => spec.output?.(args));
|
|
1962
|
+
} });
|
|
1963
|
+
const shared = {
|
|
1964
|
+
actors: Object.fromEntries(actorSources(machine).map((src) => [src, answer(src, specs[src])])),
|
|
1965
|
+
signal: controller.signal,
|
|
1966
|
+
onTrace: (event) => trace.push(event),
|
|
1967
|
+
onTransition: (snapshot) => {
|
|
1968
|
+
state = snapshot.value;
|
|
1969
|
+
}
|
|
1970
|
+
};
|
|
1971
|
+
let result = await runAgent(machine, {
|
|
1972
|
+
...shared,
|
|
1973
|
+
...options.input !== void 0 ? { input: options.input } : {}
|
|
1974
|
+
});
|
|
1975
|
+
while (!deviation && !callbackError && result.status === "idle") {
|
|
1976
|
+
const next = observations[index];
|
|
1977
|
+
const entry = next === void 0 ? void 0 : events[next.kind];
|
|
1978
|
+
if (!entry || next === void 0) break;
|
|
1979
|
+
const { event: toEvent, matches } = typeof entry === "function" ? { event: entry } : entry;
|
|
1980
|
+
if (!callback(`events.${next.kind}.matches`, () => matches ? matches({
|
|
1981
|
+
observation: next,
|
|
1982
|
+
context: result.snapshot.context,
|
|
1983
|
+
state
|
|
1984
|
+
}) : true)) {
|
|
1985
|
+
deviation = {
|
|
1986
|
+
reason: "input-mismatch",
|
|
1987
|
+
index,
|
|
1988
|
+
expected: `event ${next.kind}`,
|
|
1989
|
+
observed: next,
|
|
1990
|
+
state
|
|
1991
|
+
};
|
|
1992
|
+
break;
|
|
1993
|
+
}
|
|
1994
|
+
const event = callback(`events.${next.kind}`, () => toEvent(next));
|
|
1995
|
+
index += 1;
|
|
1996
|
+
result = await runAgent(machine, {
|
|
1997
|
+
...shared,
|
|
1998
|
+
snapshot: result.persist(),
|
|
1999
|
+
event
|
|
2000
|
+
});
|
|
2001
|
+
if (result.ignored !== void 0) {
|
|
2002
|
+
index -= 1;
|
|
2003
|
+
deviation = {
|
|
2004
|
+
reason: "ignored-event",
|
|
2005
|
+
index,
|
|
2006
|
+
expected: `an event ${JSON.stringify(state)} handles`,
|
|
2007
|
+
observed: next,
|
|
2008
|
+
state
|
|
2009
|
+
};
|
|
2010
|
+
break;
|
|
2011
|
+
}
|
|
2012
|
+
}
|
|
2013
|
+
if (callbackError) throw callbackError;
|
|
2014
|
+
const outputSpec = options.output;
|
|
2015
|
+
const last = observations[index];
|
|
2016
|
+
if (!deviation && outputSpec && result.status === "done" && last === void 0) deviation = {
|
|
2017
|
+
reason: "missing-observation",
|
|
2018
|
+
index,
|
|
2019
|
+
expected: `output (${outputSpec.observes})`,
|
|
2020
|
+
observed: void 0,
|
|
2021
|
+
state
|
|
2022
|
+
};
|
|
2023
|
+
else if (!deviation && outputSpec && result.status === "done" && last?.kind === outputSpec.observes) {
|
|
2024
|
+
const output = result.output;
|
|
2025
|
+
if (callback("output.matches", () => outputSpec.matches ? outputSpec.matches({
|
|
2026
|
+
output,
|
|
2027
|
+
observation: last
|
|
2028
|
+
}) : true)) index += 1;
|
|
2029
|
+
else deviation = {
|
|
2030
|
+
reason: "output-mismatch",
|
|
2031
|
+
index,
|
|
2032
|
+
expected: `output ${JSON.stringify(output)}`,
|
|
2033
|
+
observed: last,
|
|
2034
|
+
state
|
|
2035
|
+
};
|
|
2036
|
+
}
|
|
2037
|
+
if (!deviation && index < observations.length) deviation = {
|
|
2038
|
+
reason: "extra-observations",
|
|
2039
|
+
index,
|
|
2040
|
+
expected: result.status === "idle" ? "idle, waiting for an event" : "end of run",
|
|
2041
|
+
observed: observations[index],
|
|
2042
|
+
state
|
|
2043
|
+
};
|
|
2044
|
+
return {
|
|
2045
|
+
conforms: deviation === void 0,
|
|
2046
|
+
...deviation ? { deviation } : {},
|
|
2047
|
+
consumed: index,
|
|
2048
|
+
unobserved: [...unobserved],
|
|
2049
|
+
status: result.status,
|
|
2050
|
+
trace
|
|
2051
|
+
};
|
|
2052
|
+
}
|
|
2053
|
+
//#endregion
|
|
1697
2054
|
//#region src/check-parity.ts
|
|
1698
2055
|
/**
|
|
1699
2056
|
* `checkParity` — one call that records a workflow, replays its port, and
|
|
@@ -1803,4 +2160,4 @@ async function checkParity(options) {
|
|
|
1803
2160
|
};
|
|
1804
2161
|
}
|
|
1805
2162
|
//#endregion
|
|
1806
|
-
export { AgentLintError, AgentUnknownStateError, RecordingDivergenceError, assertAgentMachine, canReach, checkCrashRecovery, checkParity, compareRecordings, compareRuns, createRecorder, explorePaths, lintAgentMachine, matchesTrajectory, runSeam, sameModelCallShape, simulateAgent, textOutput };
|
|
2163
|
+
export { AgentLintError, AgentUnknownStateError, RecordingDivergenceError, assertAgentMachine, canReach, checkConformance, checkCrashRecovery, checkParity, compareRecordings, compareRuns, createRecorder, explorePaths, lintAgentMachine, matchesTrajectory, recordAiSdk, runSeam, sameModelCallShape, simulateAgent, textOutput };
|
|
@@ -39,6 +39,23 @@ interface RecordedModelCall {
|
|
|
39
39
|
/** The stream's parts or chunks, in order. */
|
|
40
40
|
stream?: unknown[];
|
|
41
41
|
error?: RecordedError;
|
|
42
|
+
/**
|
|
43
|
+
* This call failed, and model call `retriedBy` sent the same request after
|
|
44
|
+
* it: a retry, by the AI SDK (`maxRetries`) or by the code. Compare
|
|
45
|
+
* {@link RecordedModelCall.aiSdkCall} to tell which.
|
|
46
|
+
*/
|
|
47
|
+
retriedBy?: number;
|
|
48
|
+
/**
|
|
49
|
+
* The 0-based `generateText` or `streamText` call this attempt belongs to,
|
|
50
|
+
* when {@link recordAiSdk} routed that call. Attempts with the same value
|
|
51
|
+
* are one call the code made: its tool-loop steps and the SDK's retries. A
|
|
52
|
+
* failed attempt whose `retriedBy` attempt has the same `aiSdkCall` was
|
|
53
|
+
* retried by the SDK; one with a different `aiSdkCall` was resent by the code.
|
|
54
|
+
* Absent when the attempt was not routed through `recordAiSdk`, or its model
|
|
55
|
+
* was not a `recorder.model` wrapper passed as is (a proxy around one, one
|
|
56
|
+
* with a replaced method): who resent it is then unknown.
|
|
57
|
+
*/
|
|
58
|
+
aiSdkCall?: number;
|
|
42
59
|
/** In replay, the recorded call (its `call`) whose answer served this one. */
|
|
43
60
|
replayed?: number;
|
|
44
61
|
/** In replay, `translate` adapted the recorded answer to this call. */
|
|
@@ -254,6 +271,45 @@ interface Recorder {
|
|
|
254
271
|
/** What this run did so far. */
|
|
255
272
|
readonly recording: Recording;
|
|
256
273
|
}
|
|
274
|
+
/** What {@link recordAiSdk} routes an AI SDK call through. */
|
|
275
|
+
interface AiSdkSeam {
|
|
276
|
+
recorder: Recorder;
|
|
277
|
+
/**
|
|
278
|
+
* Replaces the model the code passes, such as `openai("gpt-…")` built inline.
|
|
279
|
+
* Pass `recorder.model(standIn)`, so the stand-in's calls are recorded.
|
|
280
|
+
*/
|
|
281
|
+
model?: unknown;
|
|
282
|
+
}
|
|
283
|
+
/**
|
|
284
|
+
* Records AI SDK code that builds its model inline or passes module-level
|
|
285
|
+
* tools to `generateText`: there is nothing to inject, so the test mocks `"ai"`
|
|
286
|
+
* and routes each call through the recorder. Every `generateText` and
|
|
287
|
+
* `streamText` call runs on the real AI SDK with `seam.model` in place of the
|
|
288
|
+
* code's model (also in place of a model `prepareStep` returns) and its
|
|
289
|
+
* `tools` wrapped with `recorder.tools`. Everything else in `"ai"` stays real,
|
|
290
|
+
* so the code's own tool loop runs.
|
|
291
|
+
*
|
|
292
|
+
* Each model attempt made during the call carries the call's index as
|
|
293
|
+
* {@link RecordedModelCall.aiSdkCall} when its model is a `recorder.model`
|
|
294
|
+
* wrapper passed as is and the recorder came from `createRecorder`. Other
|
|
295
|
+
* attempts carry no index, and who resent them is unknown.
|
|
296
|
+
*
|
|
297
|
+
* `current` returns the seam of the run being recorded. A call while it returns
|
|
298
|
+
* `undefined` throws, so no test reaches a real provider.
|
|
299
|
+
*
|
|
300
|
+
* @example
|
|
301
|
+
* ```ts
|
|
302
|
+
* vi.mock("ai", async (importOriginal) => {
|
|
303
|
+
* const ai = await importOriginal<typeof import("ai")>();
|
|
304
|
+
* const { currentSeam } = await import("./seam"); // { recorder, model: recorder.model(standIn) }
|
|
305
|
+
* return { ...ai, ...recordAiSdk(ai, currentSeam) };
|
|
306
|
+
* });
|
|
307
|
+
* ```
|
|
308
|
+
*/
|
|
309
|
+
declare function recordAiSdk<TAi extends {
|
|
310
|
+
generateText: AnyFunction;
|
|
311
|
+
streamText?: AnyFunction;
|
|
312
|
+
}>(ai: TAi, current: () => AiSdkSeam | undefined): Pick<TAi, "generateText" | "streamText">;
|
|
257
313
|
/**
|
|
258
314
|
* Thrown in replay when the port asks for something the recording cannot
|
|
259
315
|
* answer: a model call past the last recorded one, or a tool call the original
|
|
@@ -406,4 +462,4 @@ declare function getJsonSchema(schema?: StandardSchemaV1): Promise<Record<string
|
|
|
406
462
|
*/
|
|
407
463
|
declare function getJsonSchemaSync(schema?: StandardSchemaV1): Record<string, unknown> | undefined;
|
|
408
464
|
//#endregion
|
|
409
|
-
export {
|
|
465
|
+
export { RecordedOutput as A, RecordableDecisionModel as C, RecordedEffect as D, RecordedCall as E, compareRecordings as F, createRecorder as I, recordAiSdk as L, Recorder as M, Recording as N, RecordedError as O, RecordingDivergenceError as P, sameModelCallShape as R, RecordClientOptions as S, RecordableModel as T, EquivalenceReport as _, getMessageText as a, NormalizeEffect as b, isStandardSchema as c, userMessage as d, AiSdkSeam as f, EffectDivergenceReason as g, EffectDivergence as h, getJsonSchemaSync as i, RecordedToolCall as j, RecordedModelCall as k, systemMessage as l, CreateRecorderOptions as m, assistantMessage as n, getStateMeta as o, CompareRecordingsOptions as p, getJsonSchema as r, getStatePath as s, MetaOfSnapshot as t, toolMessage as u, ModelAnswer as v, RecordableLanguageModel as w, RecordCallOptions as x, ModelCallMatch as y };
|
|
@@ -39,6 +39,23 @@ interface RecordedModelCall {
|
|
|
39
39
|
/** The stream's parts or chunks, in order. */
|
|
40
40
|
stream?: unknown[];
|
|
41
41
|
error?: RecordedError;
|
|
42
|
+
/**
|
|
43
|
+
* This call failed, and model call `retriedBy` sent the same request after
|
|
44
|
+
* it: a retry, by the AI SDK (`maxRetries`) or by the code. Compare
|
|
45
|
+
* {@link RecordedModelCall.aiSdkCall} to tell which.
|
|
46
|
+
*/
|
|
47
|
+
retriedBy?: number;
|
|
48
|
+
/**
|
|
49
|
+
* The 0-based `generateText` or `streamText` call this attempt belongs to,
|
|
50
|
+
* when {@link recordAiSdk} routed that call. Attempts with the same value
|
|
51
|
+
* are one call the code made: its tool-loop steps and the SDK's retries. A
|
|
52
|
+
* failed attempt whose `retriedBy` attempt has the same `aiSdkCall` was
|
|
53
|
+
* retried by the SDK; one with a different `aiSdkCall` was resent by the code.
|
|
54
|
+
* Absent when the attempt was not routed through `recordAiSdk`, or its model
|
|
55
|
+
* was not a `recorder.model` wrapper passed as is (a proxy around one, one
|
|
56
|
+
* with a replaced method): who resent it is then unknown.
|
|
57
|
+
*/
|
|
58
|
+
aiSdkCall?: number;
|
|
42
59
|
/** In replay, the recorded call (its `call`) whose answer served this one. */
|
|
43
60
|
replayed?: number;
|
|
44
61
|
/** In replay, `translate` adapted the recorded answer to this call. */
|
|
@@ -254,6 +271,45 @@ interface Recorder {
|
|
|
254
271
|
/** What this run did so far. */
|
|
255
272
|
readonly recording: Recording;
|
|
256
273
|
}
|
|
274
|
+
/** What {@link recordAiSdk} routes an AI SDK call through. */
|
|
275
|
+
interface AiSdkSeam {
|
|
276
|
+
recorder: Recorder;
|
|
277
|
+
/**
|
|
278
|
+
* Replaces the model the code passes, such as `openai("gpt-…")` built inline.
|
|
279
|
+
* Pass `recorder.model(standIn)`, so the stand-in's calls are recorded.
|
|
280
|
+
*/
|
|
281
|
+
model?: unknown;
|
|
282
|
+
}
|
|
283
|
+
/**
|
|
284
|
+
* Records AI SDK code that builds its model inline or passes module-level
|
|
285
|
+
* tools to `generateText`: there is nothing to inject, so the test mocks `"ai"`
|
|
286
|
+
* and routes each call through the recorder. Every `generateText` and
|
|
287
|
+
* `streamText` call runs on the real AI SDK with `seam.model` in place of the
|
|
288
|
+
* code's model (also in place of a model `prepareStep` returns) and its
|
|
289
|
+
* `tools` wrapped with `recorder.tools`. Everything else in `"ai"` stays real,
|
|
290
|
+
* so the code's own tool loop runs.
|
|
291
|
+
*
|
|
292
|
+
* Each model attempt made during the call carries the call's index as
|
|
293
|
+
* {@link RecordedModelCall.aiSdkCall} when its model is a `recorder.model`
|
|
294
|
+
* wrapper passed as is and the recorder came from `createRecorder`. Other
|
|
295
|
+
* attempts carry no index, and who resent them is unknown.
|
|
296
|
+
*
|
|
297
|
+
* `current` returns the seam of the run being recorded. A call while it returns
|
|
298
|
+
* `undefined` throws, so no test reaches a real provider.
|
|
299
|
+
*
|
|
300
|
+
* @example
|
|
301
|
+
* ```ts
|
|
302
|
+
* vi.mock("ai", async (importOriginal) => {
|
|
303
|
+
* const ai = await importOriginal<typeof import("ai")>();
|
|
304
|
+
* const { currentSeam } = await import("./seam"); // { recorder, model: recorder.model(standIn) }
|
|
305
|
+
* return { ...ai, ...recordAiSdk(ai, currentSeam) };
|
|
306
|
+
* });
|
|
307
|
+
* ```
|
|
308
|
+
*/
|
|
309
|
+
declare function recordAiSdk<TAi extends {
|
|
310
|
+
generateText: AnyFunction;
|
|
311
|
+
streamText?: AnyFunction;
|
|
312
|
+
}>(ai: TAi, current: () => AiSdkSeam | undefined): Pick<TAi, "generateText" | "streamText">;
|
|
257
313
|
/**
|
|
258
314
|
* Thrown in replay when the port asks for something the recording cannot
|
|
259
315
|
* answer: a model call past the last recorded one, or a tool call the original
|
|
@@ -406,4 +462,4 @@ declare function getJsonSchema(schema?: StandardSchemaV1): Promise<Record<string
|
|
|
406
462
|
*/
|
|
407
463
|
declare function getJsonSchemaSync(schema?: StandardSchemaV1): Record<string, unknown> | undefined;
|
|
408
464
|
//#endregion
|
|
409
|
-
export {
|
|
465
|
+
export { RecordedOutput as A, RecordableDecisionModel as C, RecordedEffect as D, RecordedCall as E, compareRecordings as F, createRecorder as I, recordAiSdk as L, Recorder as M, Recording as N, RecordedError as O, RecordingDivergenceError as P, sameModelCallShape as R, RecordClientOptions as S, RecordableModel as T, EquivalenceReport as _, getMessageText as a, NormalizeEffect as b, isStandardSchema as c, userMessage as d, AiSdkSeam as f, EffectDivergenceReason as g, EffectDivergence as h, getJsonSchemaSync as i, RecordedToolCall as j, RecordedModelCall as k, systemMessage as l, CreateRecorderOptions as m, assistantMessage as n, getStateMeta as o, CompareRecordingsOptions as p, getJsonSchema as r, getStatePath as s, MetaOfSnapshot as t, toolMessage as u, ModelAnswer as v, RecordableLanguageModel as w, RecordCallOptions as x, ModelCallMatch as y };
|
package/dist/validate.cjs
CHANGED
|
@@ -24,7 +24,7 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
24
24
|
const require_event_log = require("./event-log-dSOnXHuG.cjs");
|
|
25
25
|
const require_decision = require("./decision-D8bcIGXQ.cjs");
|
|
26
26
|
const require_setup_agent = require("./setup-agent-Dv5VpuHe.cjs");
|
|
27
|
-
const require_verify = require("./verify-
|
|
27
|
+
const require_verify = require("./verify-IVYsI026.cjs");
|
|
28
28
|
let ajv_dist_2020_js = require("ajv/dist/2020.js");
|
|
29
29
|
ajv_dist_2020_js = __toESM(ajv_dist_2020_js, 1);
|
|
30
30
|
//#region schemas/agent-workflow.json
|
package/dist/validate.d.cts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { d as ExplorePathsOptions } from "./verify-
|
|
1
|
+
import { d as ExplorePathsOptions } from "./verify-DlN6tK8_.cjs";
|
|
2
2
|
import { _ as FromConfigOptions, f as AgentWorkflowConfig, y as SchemaCompiler } from "./setup-agent-Cv8Hk3mX.cjs";
|
|
3
3
|
import { AnyActorLogic, AnyStateMachine } from "xstate";
|
|
4
4
|
//#region src/validate/validate-config.d.ts
|
package/dist/validate.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { d as ExplorePathsOptions } from "./verify-
|
|
1
|
+
import { d as ExplorePathsOptions } from "./verify-C1UU84eq.mjs";
|
|
2
2
|
import { _ as FromConfigOptions, f as AgentWorkflowConfig, y as SchemaCompiler } from "./setup-agent-DgFAhaPJ.mjs";
|
|
3
3
|
import { AnyActorLogic, AnyStateMachine } from "xstate";
|
|
4
4
|
//#region src/validate/validate-config.d.ts
|
package/dist/validate.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { D as DECIDE_ACTOR, O as GENERATE_TEXT_ACTOR, T as ACT_ACTOR, k as STREAM_TEXT_ACTOR } from "./event-log-CEq_bVXQ.mjs";
|
|
2
2
|
import { p as payloadNeedsFillModel } from "./decision-B41YBx5U.mjs";
|
|
3
3
|
import { i as setupAgentFromConfig } from "./setup-agent-CLm6QVGy.mjs";
|
|
4
|
-
import { a as explorePaths, s as lintAgentMachine } from "./verify-
|
|
4
|
+
import { a as explorePaths, s as lintAgentMachine } from "./verify-lKyu3buS.mjs";
|
|
5
5
|
import Ajv2020 from "ajv/dist/2020.js";
|
|
6
6
|
//#region schemas/agent-workflow.json
|
|
7
7
|
var agent_workflow_default = {
|
|
@@ -467,7 +467,13 @@ interface ExplorePathsOptions {
|
|
|
467
467
|
* at a shallower depth), so payload and output branching that converges
|
|
468
468
|
* stays tractable. Return everything a guard, choice, or output reads:
|
|
469
469
|
* a key that drops a field the machine reads makes the search unsound.
|
|
470
|
-
* Without it, every path
|
|
470
|
+
* Without it, `explorePaths` explores every path in full, and `canReach`
|
|
471
|
+
* uses a default key: the state value, context, status, history, pending
|
|
472
|
+
* requests and their invoke inputs, children and their inputs, armed
|
|
473
|
+
* timers, and coupled children's states. A step whose context or inputs
|
|
474
|
+
* are not plain data (a `Map`, a class instance) is not deduplicated. The
|
|
475
|
+
* key holds structure, not identity: a guard that compares context with an
|
|
476
|
+
* object held outside the snapshot needs its own `dedupeBy`.
|
|
471
477
|
*/
|
|
472
478
|
dedupeBy?: (snapshot: ExploreSnapshot) => string;
|
|
473
479
|
/**
|
|
@@ -467,7 +467,13 @@ interface ExplorePathsOptions {
|
|
|
467
467
|
* at a shallower depth), so payload and output branching that converges
|
|
468
468
|
* stays tractable. Return everything a guard, choice, or output reads:
|
|
469
469
|
* a key that drops a field the machine reads makes the search unsound.
|
|
470
|
-
* Without it, every path
|
|
470
|
+
* Without it, `explorePaths` explores every path in full, and `canReach`
|
|
471
|
+
* uses a default key: the state value, context, status, history, pending
|
|
472
|
+
* requests and their invoke inputs, children and their inputs, armed
|
|
473
|
+
* timers, and coupled children's states. A step whose context or inputs
|
|
474
|
+
* are not plain data (a `Map`, a class instance) is not deduplicated. The
|
|
475
|
+
* key holds structure, not identity: a guard that compares context with an
|
|
476
|
+
* object held outside the snapshot needs its own `dedupeBy`.
|
|
471
477
|
*/
|
|
472
478
|
dedupeBy?: (snapshot: ExploreSnapshot) => string;
|
|
473
479
|
/**
|