@statelyai/agent 2.0.0-alpha.40 → 2.0.0-alpha.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/dist/evals.cjs +1 -1
- package/dist/evals.d.cts +1 -1
- package/dist/evals.d.mts +1 -1
- package/dist/evals.mjs +1 -1
- package/dist/index.d.cts +1 -1
- package/dist/index.d.mts +1 -1
- package/dist/testing.cjs +376 -17
- package/dist/testing.d.cts +181 -5
- package/dist/testing.d.mts +181 -5
- package/dist/testing.mjs +377 -20
- package/dist/{utils-DAclF-kw.d.cts → utils-BMxUcjay.d.cts} +57 -1
- package/dist/{utils-B8k-YPYI.d.mts → utils-BV7pxhD-.d.mts} +57 -1
- package/dist/validate.cjs +1 -1
- package/dist/validate.d.cts +1 -1
- package/dist/validate.d.mts +1 -1
- package/dist/validate.mjs +1 -1
- package/dist/{verify-CIu4mqIZ.d.mts → verify-C1UU84eq.d.mts} +7 -1
- package/dist/{verify-CTHlmBbh.d.cts → verify-DlN6tK8_.d.cts} +7 -1
- package/dist/{verify-D8UDqU4l.cjs → verify-IVYsI026.cjs} +65 -14
- package/dist/{verify-Cacz6XSk.mjs → verify-lKyu3buS.mjs} +65 -14
- package/docs/evals.md +1 -0
- package/docs/index.md +2 -2
- package/docs/meta.json +1 -0
- package/docs/verify-in-place.md +326 -0
- package/docs/verify.md +67 -2
- package/llms.txt +1 -0
- package/package.json +1 -1
- package/readme.md +1 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,26 @@
|
|
|
1
1
|
# @statelyai/agent
|
|
2
2
|
|
|
3
|
+
## 2.0.0-alpha.41
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- [#252](https://github.com/statelyai/agent/pull/252) [`4a3a10b`](https://github.com/statelyai/agent/commit/4a3a10b192a895b569c4cc1f2346dd14b6279082) Thanks [@davidkpiano](https://github.com/davidkpiano)! - Add `checkConformance` to `@statelyai/agent/testing`. It checks a recorded run of existing code against a model of that code. The model runs with each actor answered from the recording's observations, and the report names the first deviation (`unexpected-observation`, `missing-observation`, `input-mismatch`, `extra-observations`, `ignored-event`, or `unmapped-actor`) with the model's state, plus the actors the recording could not observe. Pass the reports' `trace` to `traceCoverage` for the states no recorded run reached.
|
|
8
|
+
|
|
9
|
+
- `output: { observes, matches }` checks the code's recorded return value against the model's output (`output-mismatch`).
|
|
10
|
+
- An `events` entry can be `{ event, matches }`, which compares the observation with the model's context before sending it, such as the draft a person approved.
|
|
11
|
+
- `recordAiSdk(ai, current)` records AI SDK code that builds its model inline or passes module-level tools to `generateText`: mock `"ai"` with it, and each `generateText` and `streamText` call uses the seam's model and records its tools.
|
|
12
|
+
- A model effect from `recorder.model` that failed and was retried with the same request has `retriedBy`, so a mapping can read one observation per call the code made.
|
|
13
|
+
|
|
14
|
+
The new [Verify in place](https://github.com/statelyai/agent/blob/next/docs/verify-in-place.md) guide and `examples/verify-in-place` walk through the track: a model of an unchanged refund loop, proofs, recorded runs checked against the model, coverage, and a seeded regression caught.
|
|
15
|
+
|
|
16
|
+
- [#255](https://github.com/statelyai/agent/pull/255) [`7ef9389`](https://github.com/statelyai/agent/commit/7ef9389f726920462d9f769666ee561a092dbf16) Thanks [@davidkpiano](https://github.com/davidkpiano)! - `recordAiSdk` stamps each recorded model attempt with `aiSdkCall`, the index of the `generateText` or `streamText` call it belongs to. Attempts with the same `aiSdkCall` are one call the code made (its tool-loop steps and the SDK's retries), so a mapping can tell a retry by the SDK (the `retriedBy` attempt has the same `aiSdkCall`) from a resend by the code (a different one), including with `maxRetries: 0` and after the SDK's last attempt. Steps that use a model from `prepareStep` run on the seam's model and carry the same `aiSdkCall`. `recorder.model` now wraps frozen models, and reads metadata getters such as `provider` on the model itself.
|
|
17
|
+
|
|
18
|
+
The Verify in place guide now checks a run where the code threw with `output`, by recording the throw as the outcome, instead of leaving its outcome unchecked.
|
|
19
|
+
|
|
20
|
+
### Patch Changes
|
|
21
|
+
|
|
22
|
+
- [#253](https://github.com/statelyai/agent/pull/253) [`d9101a0`](https://github.com/statelyai/agent/commit/d9101a08e5d9b019561fbbeeadbde2bfab634655) Thanks [@davidkpiano](https://github.com/davidkpiano)! - `canReach` without `dedupeBy` now deduplicates on a default key (state value, context, status, history, pending requests and invoke inputs, children and their inputs, armed timers, coupled children's states). An `interleave: "pending"` search that explored every interleaving now finishes, and a loop over finitely many states is a complete search. A step whose context is not plain data is not deduplicated. `explorePaths` is unchanged.
|
|
23
|
+
|
|
3
24
|
## 2.0.0-alpha.40
|
|
4
25
|
|
|
5
26
|
### Patch Changes
|
package/dist/evals.cjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
2
|
const require_event_log = require("./event-log-dSOnXHuG.cjs");
|
|
3
|
-
const require_verify = require("./verify-
|
|
3
|
+
const require_verify = require("./verify-IVYsI026.cjs");
|
|
4
4
|
const require_trajectory = require("./trajectory-CoZSv4wF.cjs");
|
|
5
5
|
let xstate = require("xstate");
|
|
6
6
|
//#region src/internal/trace-states.ts
|
package/dist/evals.d.cts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { R as AgentExecutorDecisionRequest } from "./text-logic-x17uLBwA.cjs";
|
|
2
|
-
import { i as AgentPathReport } from "./verify-
|
|
2
|
+
import { i as AgentPathReport } from "./verify-DlN6tK8_.cjs";
|
|
3
3
|
import { i as TrajectoryItem } from "./trajectory-BFjXaooB.cjs";
|
|
4
4
|
import { AnyStateMachine } from "xstate";
|
|
5
5
|
//#region src/fold-traces.d.ts
|
package/dist/evals.d.mts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { R as AgentExecutorDecisionRequest } from "./text-logic-x17uLBwA.mjs";
|
|
2
|
-
import { i as AgentPathReport } from "./verify-
|
|
2
|
+
import { i as AgentPathReport } from "./verify-C1UU84eq.mjs";
|
|
3
3
|
import { i as TrajectoryItem } from "./trajectory-BFjXaooB.mjs";
|
|
4
4
|
import { AnyStateMachine } from "xstate";
|
|
5
5
|
//#region src/fold-traces.d.ts
|
package/dist/evals.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { d as canonicalEventJson, dt as isPlainObject, ft as isRecord, pt as AgentError, ut as isError } from "./event-log-CEq_bVXQ.mjs";
|
|
2
|
-
import { o as isExplorationComplete } from "./verify-
|
|
2
|
+
import { o as isExplorationComplete } from "./verify-lKyu3buS.mjs";
|
|
3
3
|
import { t as matchesTrajectory } from "./trajectory-BUXj_1Ue.mjs";
|
|
4
4
|
import { getInitialMicrosteps, getMicrosteps } from "xstate";
|
|
5
5
|
//#region src/internal/trace-states.ts
|
package/dist/index.d.cts
CHANGED
|
@@ -4,7 +4,7 @@ import { C as AgentInteractionEvent, D as eventFromInteraction, E as GetInteract
|
|
|
4
4
|
import { C as runAgent, D as AgentStepRequest, E as AgentRequest, S as isAgentIdle, T as traceTransitions, _ as AgentTraceEvent, a as AgentMaxModelCallsExceededError, b as createAgentInbox, c as AgentRunMeta, d as AgentRunStart, f as AgentRuntime, g as AgentTimerScheduler, h as AgentTimerEvent, i as AgentInputFrom, l as AgentRunOptions, m as AgentSnapshotDivergedError, n as AgentEffects, o as AgentRunErrorCause, p as AgentRuntimeOptions, r as AgentInbox, s as AgentRunInit, t as AGENT_TRACE_SCHEMA_VERSION, u as AgentRunResult, v as AgentTransitionHandler, w as serializeTraceEvent, x as createAgentRuntime, y as JsonSerializableTraceEvent } from "./run-agent-BO9RSb7i.cjs";
|
|
5
5
|
import { c as AgentLogEntry, d as AgentMachineVersionMismatchError, h as JsonValue, n as AgentEventLogStore, t as AgentEventLogConflictError } from "./event-log-store-aFXK6yZX.cjs";
|
|
6
6
|
import { t as PresetMachine } from "./internal-BzlpzSux.cjs";
|
|
7
|
-
import {
|
|
7
|
+
import { M as Recorder, a as getMessageText, c as isStandardSchema, d as userMessage, i as getJsonSchemaSync, l as systemMessage, n as assistantMessage, o as getStateMeta, r as getJsonSchema, s as getStatePath, t as MetaOfSnapshot, u as toolMessage, x as RecordCallOptions } from "./utils-BMxUcjay.cjs";
|
|
8
8
|
import { AnyActorLogic, AnyMachineSnapshot, AnyStateMachine, ContextFrom, DoneActorEvent, EventFromLogic, InputFrom, OutputFrom, SnapshotFrom, StateValueFrom } from "xstate";
|
|
9
9
|
//#region src/type-helpers.d.ts
|
|
10
10
|
type SchemaAt<T, TKey extends PropertyKey> = T extends {
|
package/dist/index.d.mts
CHANGED
|
@@ -4,7 +4,7 @@ import { C as AgentInteractionEvent, D as eventFromInteraction, E as GetInteract
|
|
|
4
4
|
import { C as runAgent, D as AgentStepRequest, E as AgentRequest, S as isAgentIdle, T as traceTransitions, _ as AgentTraceEvent, a as AgentMaxModelCallsExceededError, b as createAgentInbox, c as AgentRunMeta, d as AgentRunStart, f as AgentRuntime, g as AgentTimerScheduler, h as AgentTimerEvent, i as AgentInputFrom, l as AgentRunOptions, m as AgentSnapshotDivergedError, n as AgentEffects, o as AgentRunErrorCause, p as AgentRuntimeOptions, r as AgentInbox, s as AgentRunInit, t as AGENT_TRACE_SCHEMA_VERSION, u as AgentRunResult, v as AgentTransitionHandler, w as serializeTraceEvent, x as createAgentRuntime, y as JsonSerializableTraceEvent } from "./run-agent-BxkeuAYR.mjs";
|
|
5
5
|
import { c as AgentLogEntry, d as AgentMachineVersionMismatchError, h as JsonValue, n as AgentEventLogStore, t as AgentEventLogConflictError } from "./event-log-store-DYun526m.mjs";
|
|
6
6
|
import { t as PresetMachine } from "./internal-BzlpzSux.mjs";
|
|
7
|
-
import {
|
|
7
|
+
import { M as Recorder, a as getMessageText, c as isStandardSchema, d as userMessage, i as getJsonSchemaSync, l as systemMessage, n as assistantMessage, o as getStateMeta, r as getJsonSchema, s as getStatePath, t as MetaOfSnapshot, u as toolMessage, x as RecordCallOptions } from "./utils-BV7pxhD-.mjs";
|
|
8
8
|
import { AnyActorLogic, AnyMachineSnapshot, AnyStateMachine, ContextFrom, DoneActorEvent, EventFromLogic, InputFrom, OutputFrom, SnapshotFrom, StateValueFrom } from "xstate";
|
|
9
9
|
//#region src/type-helpers.d.ts
|
|
10
10
|
type SchemaAt<T, TKey extends PropertyKey> = T extends {
|
package/dist/testing.cjs
CHANGED
|
@@ -5,8 +5,9 @@ const require_act = require("./act-B0CVbmo4.cjs");
|
|
|
5
5
|
const require_event_log_store = require("./event-log-store-Dey9ufA4.cjs");
|
|
6
6
|
const require_run_agent = require("./run-agent-COdR9FpM.cjs");
|
|
7
7
|
const require_effects = require("./effects-DrWcUGgc.cjs");
|
|
8
|
-
const require_verify = require("./verify-
|
|
8
|
+
const require_verify = require("./verify-IVYsI026.cjs");
|
|
9
9
|
const require_trajectory = require("./trajectory-CoZSv4wF.cjs");
|
|
10
|
+
let xstate = require("xstate");
|
|
10
11
|
//#region src/equivalence.ts
|
|
11
12
|
/**
|
|
12
13
|
* Verified equivalence — prove a port behaves like the code it replaces.
|
|
@@ -39,6 +40,86 @@ const require_trajectory = require("./trajectory-CoZSv4wF.cjs");
|
|
|
39
40
|
* neither `ai` nor `openai`.
|
|
40
41
|
* @module
|
|
41
42
|
*/
|
|
43
|
+
const MODEL_METADATA = [
|
|
44
|
+
"specificationVersion",
|
|
45
|
+
"provider",
|
|
46
|
+
"modelId",
|
|
47
|
+
"supportedUrls"
|
|
48
|
+
];
|
|
49
|
+
const MODEL_METHODS = [
|
|
50
|
+
"doGenerate",
|
|
51
|
+
"doStream",
|
|
52
|
+
"doDecide",
|
|
53
|
+
"doEvaluate"
|
|
54
|
+
];
|
|
55
|
+
const MODEL_KEYS = [...MODEL_METADATA, ...MODEL_METHODS];
|
|
56
|
+
const aiSdkInternals = /* @__PURE__ */ new WeakMap();
|
|
57
|
+
/**
|
|
58
|
+
* Records AI SDK code that builds its model inline or passes module-level
|
|
59
|
+
* tools to `generateText`: there is nothing to inject, so the test mocks `"ai"`
|
|
60
|
+
* and routes each call through the recorder. Every `generateText` and
|
|
61
|
+
* `streamText` call runs on the real AI SDK with `seam.model` in place of the
|
|
62
|
+
* code's model (also in place of a model `prepareStep` returns) and its
|
|
63
|
+
* `tools` wrapped with `recorder.tools`. Everything else in `"ai"` stays real,
|
|
64
|
+
* so the code's own tool loop runs.
|
|
65
|
+
*
|
|
66
|
+
* Each model attempt made during the call carries the call's index as
|
|
67
|
+
* {@link RecordedModelCall.aiSdkCall} when its model is a `recorder.model`
|
|
68
|
+
* wrapper passed as is and the recorder came from `createRecorder`. Other
|
|
69
|
+
* attempts carry no index, and who resent them is unknown.
|
|
70
|
+
*
|
|
71
|
+
* `current` returns the seam of the run being recorded. A call while it returns
|
|
72
|
+
* `undefined` throws, so no test reaches a real provider.
|
|
73
|
+
*
|
|
74
|
+
* @example
|
|
75
|
+
* ```ts
|
|
76
|
+
* vi.mock("ai", async (importOriginal) => {
|
|
77
|
+
* const ai = await importOriginal<typeof import("ai")>();
|
|
78
|
+
* const { currentSeam } = await import("./seam"); // { recorder, model: recorder.model(standIn) }
|
|
79
|
+
* return { ...ai, ...recordAiSdk(ai, currentSeam) };
|
|
80
|
+
* });
|
|
81
|
+
* ```
|
|
82
|
+
*/
|
|
83
|
+
function recordAiSdk(ai, current) {
|
|
84
|
+
const wrap = (name, call) => (options) => {
|
|
85
|
+
const seam = current();
|
|
86
|
+
if (!seam) throw new Error(`recordAiSdk: ${name} was called while no run is recording. Return { recorder } from current() while the code runs.`);
|
|
87
|
+
const internals = aiSdkInternals.get(seam.recorder);
|
|
88
|
+
const index = internals ? internals.calls++ : 0;
|
|
89
|
+
const tag = (model) => {
|
|
90
|
+
const base = internals?.unwrap(model);
|
|
91
|
+
return base ? internals.wrap(base, index) : model;
|
|
92
|
+
};
|
|
93
|
+
const model = seam.model ?? options.model;
|
|
94
|
+
const prepareStep = options.prepareStep;
|
|
95
|
+
return call({
|
|
96
|
+
...options,
|
|
97
|
+
...model !== void 0 ? { model: tag(model) } : {},
|
|
98
|
+
...prepareStep ? { prepareStep: async (...args) => {
|
|
99
|
+
const step = await prepareStep(...args);
|
|
100
|
+
if (step?.model === void 0) return step;
|
|
101
|
+
const model = tag(seam.model ?? step.model);
|
|
102
|
+
const own = (key) => key === "model" ? model : Reflect.get(step, key, step);
|
|
103
|
+
return new Proxy({}, {
|
|
104
|
+
get: (_target, key) => own(key),
|
|
105
|
+
has: (_target, key) => key === "model" || key in step,
|
|
106
|
+
ownKeys: () => [.../* @__PURE__ */ new Set([...Reflect.ownKeys(step), "model"])],
|
|
107
|
+
getOwnPropertyDescriptor: (_target, key) => key === "model" || Object.prototype.hasOwnProperty.call(step, key) ? {
|
|
108
|
+
value: own(key),
|
|
109
|
+
enumerable: true,
|
|
110
|
+
configurable: true,
|
|
111
|
+
writable: true
|
|
112
|
+
} : void 0
|
|
113
|
+
});
|
|
114
|
+
} } : {},
|
|
115
|
+
...options.tools ? { tools: seam.recorder.tools(options.tools) } : {}
|
|
116
|
+
});
|
|
117
|
+
};
|
|
118
|
+
return {
|
|
119
|
+
generateText: wrap("generateText", ai.generateText),
|
|
120
|
+
...ai.streamText ? { streamText: wrap("streamText", ai.streamText) } : {}
|
|
121
|
+
};
|
|
122
|
+
}
|
|
42
123
|
/**
|
|
43
124
|
* Thrown in replay when the port asks for something the recording cannot
|
|
44
125
|
* answer: a model call past the last recorded one, or a tool call the original
|
|
@@ -228,7 +309,28 @@ function createRecorder(options = {}) {
|
|
|
228
309
|
* Records a model call around `run`, or answers it from the recording.
|
|
229
310
|
* `fallback` runs it on the stand-in when the recording has no answer.
|
|
230
311
|
*/
|
|
231
|
-
|
|
312
|
+
/**
|
|
313
|
+
* A retry resends the request of a model call that failed. Within one AI SDK
|
|
314
|
+
* call, the failed attempt is its own last attempt, even when a concurrent
|
|
315
|
+
* call came in between or claimed it first. Otherwise it is the last model
|
|
316
|
+
* call, which the code resent.
|
|
317
|
+
*/
|
|
318
|
+
function linkRetry(effect) {
|
|
319
|
+
const models = effects.filter((earlier) => earlier.kind === "model").reverse();
|
|
320
|
+
const failedWith = (earlier) => earlier?.error !== void 0 && earlier.method === effect.method && stableStringify(earlier.request) === stableStringify(effect.request);
|
|
321
|
+
const { aiSdkCall } = effect;
|
|
322
|
+
if (aiSdkCall !== void 0) {
|
|
323
|
+
const own = models.find((earlier) => earlier.aiSdkCall === aiSdkCall);
|
|
324
|
+
const claimedBy = own?.retriedBy === void 0 ? void 0 : models.find((earlier) => earlier.call === own.retriedBy);
|
|
325
|
+
if (own && failedWith(own) && claimedBy?.aiSdkCall !== aiSdkCall) {
|
|
326
|
+
own.retriedBy = effect.call;
|
|
327
|
+
return;
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
const last = models[0];
|
|
331
|
+
if (last && failedWith(last) && last.retriedBy === void 0) last.retriedBy = effect.call;
|
|
332
|
+
}
|
|
333
|
+
async function modelCall(seam, method, request, streamed, run, fallback, aiSdkCall) {
|
|
232
334
|
const effect = {
|
|
233
335
|
kind: "model",
|
|
234
336
|
seam,
|
|
@@ -240,6 +342,8 @@ function createRecorder(options = {}) {
|
|
|
240
342
|
"signal"
|
|
241
343
|
]))
|
|
242
344
|
};
|
|
345
|
+
if (aiSdkCall !== void 0) effect.aiSdkCall = aiSdkCall;
|
|
346
|
+
linkRetry(effect);
|
|
243
347
|
effects.push(effect);
|
|
244
348
|
if (replay) {
|
|
245
349
|
const replayed = nextReplayedModel(effect);
|
|
@@ -469,22 +573,47 @@ function createRecorder(options = {}) {
|
|
|
469
573
|
}
|
|
470
574
|
return result;
|
|
471
575
|
}
|
|
472
|
-
|
|
576
|
+
const wrappedModels = /* @__PURE__ */ new WeakMap();
|
|
577
|
+
function wrapModel(model, aiSdkCall) {
|
|
578
|
+
const base = model;
|
|
579
|
+
const stand = options.fallback?.model;
|
|
580
|
+
const wrapped = Object.create(base);
|
|
581
|
+
for (const key of MODEL_METADATA) {
|
|
582
|
+
if (!(key in base)) continue;
|
|
583
|
+
Object.defineProperty(wrapped, key, {
|
|
584
|
+
get: () => Reflect.get(base, key, base),
|
|
585
|
+
enumerable: true,
|
|
586
|
+
configurable: true
|
|
587
|
+
});
|
|
588
|
+
}
|
|
589
|
+
for (const method of MODEL_METHODS) {
|
|
590
|
+
if (typeof base[method] !== "function") continue;
|
|
591
|
+
const invoke = (target, callOptions) => target[method].call(target, callOptions);
|
|
592
|
+
Object.defineProperty(wrapped, method, {
|
|
593
|
+
value: (callOptions) => modelCall("ai-sdk", method, callOptions, method === "doStream", () => invoke(base, callOptions), stand && typeof stand[method] === "function" ? () => invoke(stand, callOptions) : void 0, aiSdkCall),
|
|
594
|
+
writable: true,
|
|
595
|
+
enumerable: true,
|
|
596
|
+
configurable: true
|
|
597
|
+
});
|
|
598
|
+
}
|
|
599
|
+
wrappedModels.set(wrapped, {
|
|
600
|
+
base,
|
|
601
|
+
made: new Map([...MODEL_KEYS].map((key) => [key, Object.getOwnPropertyDescriptor(wrapped, key)]))
|
|
602
|
+
});
|
|
603
|
+
return wrapped;
|
|
604
|
+
}
|
|
605
|
+
function unwrapModel(model) {
|
|
606
|
+
const entry = wrappedModels.get(model);
|
|
607
|
+
if (!entry) return void 0;
|
|
608
|
+
for (const [key, made] of entry.made) {
|
|
609
|
+
const now = Object.getOwnPropertyDescriptor(model, key);
|
|
610
|
+
if (now?.value !== made?.value || now?.get !== made?.get) return void 0;
|
|
611
|
+
}
|
|
612
|
+
return entry.base;
|
|
613
|
+
}
|
|
614
|
+
const recorder = {
|
|
473
615
|
model(model) {
|
|
474
|
-
|
|
475
|
-
const stand = options.fallback?.model;
|
|
476
|
-
const wrapped = Object.create(base);
|
|
477
|
-
for (const method of [
|
|
478
|
-
"doGenerate",
|
|
479
|
-
"doStream",
|
|
480
|
-
"doDecide",
|
|
481
|
-
"doEvaluate"
|
|
482
|
-
]) {
|
|
483
|
-
if (typeof base[method] !== "function") continue;
|
|
484
|
-
const invoke = (target, callOptions) => target[method].call(target, callOptions);
|
|
485
|
-
wrapped[method] = (callOptions) => modelCall("ai-sdk", method, callOptions, method === "doStream", () => invoke(base, callOptions), stand && typeof stand[method] === "function" ? () => invoke(stand, callOptions) : void 0);
|
|
486
|
-
}
|
|
487
|
-
return wrapped;
|
|
616
|
+
return wrapModel(model);
|
|
488
617
|
},
|
|
489
618
|
openai(client) {
|
|
490
619
|
const wrapCreate = (resource, method) => new Proxy(resource, { get(target, key, receiver) {
|
|
@@ -568,6 +697,12 @@ function createRecorder(options = {}) {
|
|
|
568
697
|
};
|
|
569
698
|
}
|
|
570
699
|
};
|
|
700
|
+
aiSdkInternals.set(recorder, {
|
|
701
|
+
unwrap: (model) => model !== null && (typeof model === "object" || typeof model === "function") ? unwrapModel(model) : void 0,
|
|
702
|
+
wrap: wrapModel,
|
|
703
|
+
calls: 0
|
|
704
|
+
});
|
|
705
|
+
return recorder;
|
|
571
706
|
}
|
|
572
707
|
/** True when two model calls offer the same tool names and response format. */
|
|
573
708
|
function sameModelCallShape(expected, actual) {
|
|
@@ -1695,6 +1830,228 @@ function structurallyEqual(a, b) {
|
|
|
1695
1830
|
return keys.length === Object.keys(b).length && keys.every((key) => Object.hasOwn(b, key) && structurallyEqual(a[key], b[key]));
|
|
1696
1831
|
}
|
|
1697
1832
|
//#endregion
|
|
1833
|
+
//#region src/conformance.ts
|
|
1834
|
+
/**
|
|
1835
|
+
* Conformance: check a recorded run of existing code against a model of it.
|
|
1836
|
+
*
|
|
1837
|
+
* The model is an ordinary machine whose actors stand for the code's work.
|
|
1838
|
+
* `checkConformance` runs the model with every actor answered from the
|
|
1839
|
+
* recording: each one consumes the next observation of its kind. The first
|
|
1840
|
+
* time the model does something the recording does not show next, the run
|
|
1841
|
+
* stops and the step is reported. The code itself is not changed or run.
|
|
1842
|
+
*/
|
|
1843
|
+
const NEVER = new Promise(() => {});
|
|
1844
|
+
function inlineInvokeStates(machine) {
|
|
1845
|
+
const found = [];
|
|
1846
|
+
const stack = [machine.root];
|
|
1847
|
+
while (stack.length > 0) {
|
|
1848
|
+
const node = stack.pop();
|
|
1849
|
+
const invoke = node.config.invoke;
|
|
1850
|
+
if ((invoke === void 0 ? [] : Array.isArray(invoke) ? invoke : [invoke]).some((entry) => typeof entry.src !== "string")) found.push(node.path.join(".") || "(root)");
|
|
1851
|
+
stack.push(...Object.values(node.states));
|
|
1852
|
+
}
|
|
1853
|
+
return found.sort();
|
|
1854
|
+
}
|
|
1855
|
+
function actorSources(machine) {
|
|
1856
|
+
const actors = require_event_log.getRegisteredAgentExecutionOptions(machine).actors ?? machine.sources?.actors ?? {};
|
|
1857
|
+
return Object.keys(actors);
|
|
1858
|
+
}
|
|
1859
|
+
/**
|
|
1860
|
+
* Checks a recorded run of existing code against a model of that code.
|
|
1861
|
+
*
|
|
1862
|
+
* The model runs with `runAgent`, and every actor in `actors` is answered from
|
|
1863
|
+
* `observations` instead of running: an actor with `observes` consumes the next
|
|
1864
|
+
* observation, which must be of that kind. The report names the first
|
|
1865
|
+
* deviation: where the model started something the recording does not show
|
|
1866
|
+
* next, or ended while the recording continued.
|
|
1867
|
+
*
|
|
1868
|
+
* Use it to verify existing code in place: prove the model's rules with
|
|
1869
|
+
* `canReach`, then check that recorded runs of the code conform to the model.
|
|
1870
|
+
* A conforming run shows the code behaved as the model allows on that run; it
|
|
1871
|
+
* does not prove the code can never break a rule.
|
|
1872
|
+
*
|
|
1873
|
+
* Every actor the model invokes must be declared by name: a model whose invoke
|
|
1874
|
+
* names logic inline throws `inline-actor-source`, because that logic would
|
|
1875
|
+
* run. A callback in `actors` that throws ends the check with
|
|
1876
|
+
* `conformance-callback-error` instead of counting as an actor failure.
|
|
1877
|
+
*
|
|
1878
|
+
* @example
|
|
1879
|
+
* ```ts
|
|
1880
|
+
* const report = await checkConformance(loopModel, {
|
|
1881
|
+
* input: { maxIter: 3 },
|
|
1882
|
+
* observations: recording.effects.flatMap(toObservation),
|
|
1883
|
+
* actors: {
|
|
1884
|
+
* route: { observes: "route", output: ({ observation }) => observation.names },
|
|
1885
|
+
* runTurn: { observes: "turn", matches: ({ input, observation }) => input.name === observation.agent },
|
|
1886
|
+
* save: {}, // not visible in this recording
|
|
1887
|
+
* },
|
|
1888
|
+
* });
|
|
1889
|
+
* report.conforms; // false when the run did something the model does not allow
|
|
1890
|
+
* ```
|
|
1891
|
+
*/
|
|
1892
|
+
async function checkConformance(machine, options) {
|
|
1893
|
+
const { observations, actors: specs, events = {} } = options;
|
|
1894
|
+
const inline = inlineInvokeStates(machine);
|
|
1895
|
+
if (inline.length > 0) throw new require_event_log.AgentError("inline-actor-source", `checkConformance: ${inline.join(", ")} invoke${inline.length === 1 ? "s" : ""} inline actor logic, which would run for real. Declare each actor by name in setup ({ actors }) and invoke it by src, so the recording can answer it.`);
|
|
1896
|
+
for (const src of Object.keys(specs)) if (!actorSources(machine).includes(src)) throw new require_event_log.AgentError("unknown-actor-source", `checkConformance: actors.${src} names no actor source the machine declares (${actorSources(machine).join(", ") || "none"}).`);
|
|
1897
|
+
let index = 0;
|
|
1898
|
+
let deviation;
|
|
1899
|
+
let callbackError;
|
|
1900
|
+
let state = machine.config.initial;
|
|
1901
|
+
const unobserved = /* @__PURE__ */ new Set();
|
|
1902
|
+
const trace = [];
|
|
1903
|
+
const controller = new AbortController();
|
|
1904
|
+
const callbackFailure = (name, cause) => new require_event_log.AgentError("conformance-callback-error", `checkConformance: ${name} threw: ${cause instanceof Error ? cause.message : String(cause)}`, { cause });
|
|
1905
|
+
const callback = (name, fn) => {
|
|
1906
|
+
try {
|
|
1907
|
+
return fn();
|
|
1908
|
+
} catch (cause) {
|
|
1909
|
+
throw callbackFailure(name, cause);
|
|
1910
|
+
}
|
|
1911
|
+
};
|
|
1912
|
+
const deviate = (reason, expected, actorInput) => {
|
|
1913
|
+
deviation ??= {
|
|
1914
|
+
reason,
|
|
1915
|
+
index,
|
|
1916
|
+
expected,
|
|
1917
|
+
observed: observations[index],
|
|
1918
|
+
state,
|
|
1919
|
+
...actorInput !== void 0 ? { actorInput } : {}
|
|
1920
|
+
};
|
|
1921
|
+
controller.abort();
|
|
1922
|
+
return NEVER;
|
|
1923
|
+
};
|
|
1924
|
+
const answer = (src, spec) => (0, xstate.createAsyncLogic)({ run: async ({ input }) => {
|
|
1925
|
+
if (deviation || callbackError) return NEVER;
|
|
1926
|
+
if (!spec) return deviate("unmapped-actor", src, input);
|
|
1927
|
+
let observation;
|
|
1928
|
+
if (spec.observes === void 0) unobserved.add(src);
|
|
1929
|
+
else {
|
|
1930
|
+
const next = observations[index];
|
|
1931
|
+
if (next === void 0) return deviate("missing-observation", src, input);
|
|
1932
|
+
if (next.kind !== spec.observes) return deviate("unexpected-observation", src, input);
|
|
1933
|
+
observation = next;
|
|
1934
|
+
index += 1;
|
|
1935
|
+
}
|
|
1936
|
+
const args = {
|
|
1937
|
+
input,
|
|
1938
|
+
observation,
|
|
1939
|
+
observations,
|
|
1940
|
+
index
|
|
1941
|
+
};
|
|
1942
|
+
const call = (name, fn) => {
|
|
1943
|
+
try {
|
|
1944
|
+
return fn();
|
|
1945
|
+
} catch (cause) {
|
|
1946
|
+
callbackError ??= callbackFailure(`actors.${src}.${name}`, cause);
|
|
1947
|
+
controller.abort();
|
|
1948
|
+
return NEVER;
|
|
1949
|
+
}
|
|
1950
|
+
};
|
|
1951
|
+
if (spec.matches) {
|
|
1952
|
+
const fits = call("matches", () => spec.matches(args));
|
|
1953
|
+
if (fits === NEVER) return NEVER;
|
|
1954
|
+
if (!fits) {
|
|
1955
|
+
index -= observation ? 1 : 0;
|
|
1956
|
+
return deviate("input-mismatch", src, input);
|
|
1957
|
+
}
|
|
1958
|
+
}
|
|
1959
|
+
const error = call("error", () => spec.error?.(args));
|
|
1960
|
+
if (error === NEVER) return NEVER;
|
|
1961
|
+
if (error !== void 0) throw error;
|
|
1962
|
+
return call("output", () => spec.output?.(args));
|
|
1963
|
+
} });
|
|
1964
|
+
const shared = {
|
|
1965
|
+
actors: Object.fromEntries(actorSources(machine).map((src) => [src, answer(src, specs[src])])),
|
|
1966
|
+
signal: controller.signal,
|
|
1967
|
+
onTrace: (event) => trace.push(event),
|
|
1968
|
+
onTransition: (snapshot) => {
|
|
1969
|
+
state = snapshot.value;
|
|
1970
|
+
}
|
|
1971
|
+
};
|
|
1972
|
+
let result = await require_run_agent.runAgent(machine, {
|
|
1973
|
+
...shared,
|
|
1974
|
+
...options.input !== void 0 ? { input: options.input } : {}
|
|
1975
|
+
});
|
|
1976
|
+
while (!deviation && !callbackError && result.status === "idle") {
|
|
1977
|
+
const next = observations[index];
|
|
1978
|
+
const entry = next === void 0 ? void 0 : events[next.kind];
|
|
1979
|
+
if (!entry || next === void 0) break;
|
|
1980
|
+
const { event: toEvent, matches } = typeof entry === "function" ? { event: entry } : entry;
|
|
1981
|
+
if (!callback(`events.${next.kind}.matches`, () => matches ? matches({
|
|
1982
|
+
observation: next,
|
|
1983
|
+
context: result.snapshot.context,
|
|
1984
|
+
state
|
|
1985
|
+
}) : true)) {
|
|
1986
|
+
deviation = {
|
|
1987
|
+
reason: "input-mismatch",
|
|
1988
|
+
index,
|
|
1989
|
+
expected: `event ${next.kind}`,
|
|
1990
|
+
observed: next,
|
|
1991
|
+
state
|
|
1992
|
+
};
|
|
1993
|
+
break;
|
|
1994
|
+
}
|
|
1995
|
+
const event = callback(`events.${next.kind}`, () => toEvent(next));
|
|
1996
|
+
index += 1;
|
|
1997
|
+
result = await require_run_agent.runAgent(machine, {
|
|
1998
|
+
...shared,
|
|
1999
|
+
snapshot: result.persist(),
|
|
2000
|
+
event
|
|
2001
|
+
});
|
|
2002
|
+
if (result.ignored !== void 0) {
|
|
2003
|
+
index -= 1;
|
|
2004
|
+
deviation = {
|
|
2005
|
+
reason: "ignored-event",
|
|
2006
|
+
index,
|
|
2007
|
+
expected: `an event ${JSON.stringify(state)} handles`,
|
|
2008
|
+
observed: next,
|
|
2009
|
+
state
|
|
2010
|
+
};
|
|
2011
|
+
break;
|
|
2012
|
+
}
|
|
2013
|
+
}
|
|
2014
|
+
if (callbackError) throw callbackError;
|
|
2015
|
+
const outputSpec = options.output;
|
|
2016
|
+
const last = observations[index];
|
|
2017
|
+
if (!deviation && outputSpec && result.status === "done" && last === void 0) deviation = {
|
|
2018
|
+
reason: "missing-observation",
|
|
2019
|
+
index,
|
|
2020
|
+
expected: `output (${outputSpec.observes})`,
|
|
2021
|
+
observed: void 0,
|
|
2022
|
+
state
|
|
2023
|
+
};
|
|
2024
|
+
else if (!deviation && outputSpec && result.status === "done" && last?.kind === outputSpec.observes) {
|
|
2025
|
+
const output = result.output;
|
|
2026
|
+
if (callback("output.matches", () => outputSpec.matches ? outputSpec.matches({
|
|
2027
|
+
output,
|
|
2028
|
+
observation: last
|
|
2029
|
+
}) : true)) index += 1;
|
|
2030
|
+
else deviation = {
|
|
2031
|
+
reason: "output-mismatch",
|
|
2032
|
+
index,
|
|
2033
|
+
expected: `output ${JSON.stringify(output)}`,
|
|
2034
|
+
observed: last,
|
|
2035
|
+
state
|
|
2036
|
+
};
|
|
2037
|
+
}
|
|
2038
|
+
if (!deviation && index < observations.length) deviation = {
|
|
2039
|
+
reason: "extra-observations",
|
|
2040
|
+
index,
|
|
2041
|
+
expected: result.status === "idle" ? "idle, waiting for an event" : "end of run",
|
|
2042
|
+
observed: observations[index],
|
|
2043
|
+
state
|
|
2044
|
+
};
|
|
2045
|
+
return {
|
|
2046
|
+
conforms: deviation === void 0,
|
|
2047
|
+
...deviation ? { deviation } : {},
|
|
2048
|
+
consumed: index,
|
|
2049
|
+
unobserved: [...unobserved],
|
|
2050
|
+
status: result.status,
|
|
2051
|
+
trace
|
|
2052
|
+
};
|
|
2053
|
+
}
|
|
2054
|
+
//#endregion
|
|
1698
2055
|
//#region src/check-parity.ts
|
|
1699
2056
|
/**
|
|
1700
2057
|
* `checkParity` — one call that records a workflow, replays its port, and
|
|
@@ -1809,6 +2166,7 @@ exports.AgentUnknownStateError = require_verify.AgentUnknownStateError;
|
|
|
1809
2166
|
exports.RecordingDivergenceError = RecordingDivergenceError;
|
|
1810
2167
|
exports.assertAgentMachine = require_verify.assertAgentMachine;
|
|
1811
2168
|
exports.canReach = require_verify.canReach;
|
|
2169
|
+
exports.checkConformance = checkConformance;
|
|
1812
2170
|
exports.checkCrashRecovery = checkCrashRecovery;
|
|
1813
2171
|
exports.checkParity = checkParity;
|
|
1814
2172
|
exports.compareRecordings = compareRecordings;
|
|
@@ -1817,6 +2175,7 @@ exports.createRecorder = createRecorder;
|
|
|
1817
2175
|
exports.explorePaths = require_verify.explorePaths;
|
|
1818
2176
|
exports.lintAgentMachine = require_verify.lintAgentMachine;
|
|
1819
2177
|
exports.matchesTrajectory = require_trajectory.matchesTrajectory;
|
|
2178
|
+
exports.recordAiSdk = recordAiSdk;
|
|
1820
2179
|
exports.runSeam = runSeam;
|
|
1821
2180
|
exports.sameModelCallShape = sameModelCallShape;
|
|
1822
2181
|
exports.simulateAgent = require_verify.simulateAgent;
|