@statelyai/agent 2.0.0-alpha.40 → 2.0.0-alpha.41

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,26 @@
1
1
  # @statelyai/agent
2
2
 
3
+ ## 2.0.0-alpha.41
4
+
5
+ ### Minor Changes
6
+
7
+ - [#252](https://github.com/statelyai/agent/pull/252) [`4a3a10b`](https://github.com/statelyai/agent/commit/4a3a10b192a895b569c4cc1f2346dd14b6279082) Thanks [@davidkpiano](https://github.com/davidkpiano)! - Add `checkConformance` to `@statelyai/agent/testing`. It checks a recorded run of existing code against a model of that code. The model runs with each actor answered from the recording's observations, and the report names the first deviation (`unexpected-observation`, `missing-observation`, `input-mismatch`, `extra-observations`, `ignored-event`, or `unmapped-actor`) with the model's state, plus the actors the recording could not observe. Pass the reports' `trace` to `traceCoverage` for the states no recorded run reached.
8
+
9
+ - `output: { observes, matches }` checks the code's recorded return value against the model's output (`output-mismatch`).
10
+ - An `events` entry can be `{ event, matches }`, which compares the observation with the model's context before sending it, such as the draft a person approved.
11
+ - `recordAiSdk(ai, current)` records AI SDK code that builds its model inline or passes module-level tools to `generateText`: mock `"ai"` with it, and each `generateText` and `streamText` call uses the seam's model and records its tools.
12
+ - A model effect from `recorder.model` that failed and was retried with the same request has `retriedBy`, so a mapping can read one observation per call the code made.
13
+
14
+ The new [Verify in place](https://github.com/statelyai/agent/blob/next/docs/verify-in-place.md) guide and `examples/verify-in-place` walk through the track: a model of an unchanged refund loop, proofs, recorded runs checked against the model, coverage, and a seeded regression caught.
15
+
16
+ - [#255](https://github.com/statelyai/agent/pull/255) [`7ef9389`](https://github.com/statelyai/agent/commit/7ef9389f726920462d9f769666ee561a092dbf16) Thanks [@davidkpiano](https://github.com/davidkpiano)! - `recordAiSdk` stamps each recorded model attempt with `aiSdkCall`, the index of the `generateText` or `streamText` call it belongs to. Attempts with the same `aiSdkCall` are one call the code made (its tool-loop steps and the SDK's retries), so a mapping can tell a retry by the SDK (the `retriedBy` attempt has the same `aiSdkCall`) from a resend by the code (a different one), including with `maxRetries: 0` and after the SDK's last attempt. Steps that use a model from `prepareStep` run on the seam's model and carry the same `aiSdkCall`. `recorder.model` now wraps frozen models, and reads metadata getters such as `provider` on the model itself.
17
+
18
+ The Verify in place guide now checks a run where the code threw with `output`, by recording the throw as the outcome, instead of leaving its outcome unchecked.
19
+
20
+ ### Patch Changes
21
+
22
+ - [#253](https://github.com/statelyai/agent/pull/253) [`d9101a0`](https://github.com/statelyai/agent/commit/d9101a08e5d9b019561fbbeeadbde2bfab634655) Thanks [@davidkpiano](https://github.com/davidkpiano)! - `canReach` without `dedupeBy` now deduplicates on a default key (state value, context, status, history, pending requests and invoke inputs, children and their inputs, armed timers, coupled children's states). An `interleave: "pending"` search that explored every interleaving now finishes, and a loop over finitely many states is a complete search. A step whose context is not plain data is not deduplicated. `explorePaths` is unchanged.
23
+
3
24
  ## 2.0.0-alpha.40
4
25
 
5
26
  ### Patch Changes
package/dist/evals.cjs CHANGED
@@ -1,6 +1,6 @@
1
1
  Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
2
2
  const require_event_log = require("./event-log-dSOnXHuG.cjs");
3
- const require_verify = require("./verify-D8UDqU4l.cjs");
3
+ const require_verify = require("./verify-IVYsI026.cjs");
4
4
  const require_trajectory = require("./trajectory-CoZSv4wF.cjs");
5
5
  let xstate = require("xstate");
6
6
  //#region src/internal/trace-states.ts
package/dist/evals.d.cts CHANGED
@@ -1,5 +1,5 @@
1
1
  import { R as AgentExecutorDecisionRequest } from "./text-logic-x17uLBwA.cjs";
2
- import { i as AgentPathReport } from "./verify-CTHlmBbh.cjs";
2
+ import { i as AgentPathReport } from "./verify-DlN6tK8_.cjs";
3
3
  import { i as TrajectoryItem } from "./trajectory-BFjXaooB.cjs";
4
4
  import { AnyStateMachine } from "xstate";
5
5
  //#region src/fold-traces.d.ts
package/dist/evals.d.mts CHANGED
@@ -1,5 +1,5 @@
1
1
  import { R as AgentExecutorDecisionRequest } from "./text-logic-x17uLBwA.mjs";
2
- import { i as AgentPathReport } from "./verify-CIu4mqIZ.mjs";
2
+ import { i as AgentPathReport } from "./verify-C1UU84eq.mjs";
3
3
  import { i as TrajectoryItem } from "./trajectory-BFjXaooB.mjs";
4
4
  import { AnyStateMachine } from "xstate";
5
5
  //#region src/fold-traces.d.ts
package/dist/evals.mjs CHANGED
@@ -1,5 +1,5 @@
1
1
  import { d as canonicalEventJson, dt as isPlainObject, ft as isRecord, pt as AgentError, ut as isError } from "./event-log-CEq_bVXQ.mjs";
2
- import { o as isExplorationComplete } from "./verify-Cacz6XSk.mjs";
2
+ import { o as isExplorationComplete } from "./verify-lKyu3buS.mjs";
3
3
  import { t as matchesTrajectory } from "./trajectory-BUXj_1Ue.mjs";
4
4
  import { getInitialMicrosteps, getMicrosteps } from "xstate";
5
5
  //#region src/internal/trace-states.ts
package/dist/index.d.cts CHANGED
@@ -4,7 +4,7 @@ import { C as AgentInteractionEvent, D as eventFromInteraction, E as GetInteract
4
4
  import { C as runAgent, D as AgentStepRequest, E as AgentRequest, S as isAgentIdle, T as traceTransitions, _ as AgentTraceEvent, a as AgentMaxModelCallsExceededError, b as createAgentInbox, c as AgentRunMeta, d as AgentRunStart, f as AgentRuntime, g as AgentTimerScheduler, h as AgentTimerEvent, i as AgentInputFrom, l as AgentRunOptions, m as AgentSnapshotDivergedError, n as AgentEffects, o as AgentRunErrorCause, p as AgentRuntimeOptions, r as AgentInbox, s as AgentRunInit, t as AGENT_TRACE_SCHEMA_VERSION, u as AgentRunResult, v as AgentTransitionHandler, w as serializeTraceEvent, x as createAgentRuntime, y as JsonSerializableTraceEvent } from "./run-agent-BO9RSb7i.cjs";
5
5
  import { c as AgentLogEntry, d as AgentMachineVersionMismatchError, h as JsonValue, n as AgentEventLogStore, t as AgentEventLogConflictError } from "./event-log-store-aFXK6yZX.cjs";
6
6
  import { t as PresetMachine } from "./internal-BzlpzSux.cjs";
7
- import { a as getMessageText, b as RecordCallOptions, c as isStandardSchema, d as userMessage, i as getJsonSchemaSync, j as Recorder, l as systemMessage, n as assistantMessage, o as getStateMeta, r as getJsonSchema, s as getStatePath, t as MetaOfSnapshot, u as toolMessage } from "./utils-DAclF-kw.cjs";
7
+ import { M as Recorder, a as getMessageText, c as isStandardSchema, d as userMessage, i as getJsonSchemaSync, l as systemMessage, n as assistantMessage, o as getStateMeta, r as getJsonSchema, s as getStatePath, t as MetaOfSnapshot, u as toolMessage, x as RecordCallOptions } from "./utils-BMxUcjay.cjs";
8
8
  import { AnyActorLogic, AnyMachineSnapshot, AnyStateMachine, ContextFrom, DoneActorEvent, EventFromLogic, InputFrom, OutputFrom, SnapshotFrom, StateValueFrom } from "xstate";
9
9
  //#region src/type-helpers.d.ts
10
10
  type SchemaAt<T, TKey extends PropertyKey> = T extends {
package/dist/index.d.mts CHANGED
@@ -4,7 +4,7 @@ import { C as AgentInteractionEvent, D as eventFromInteraction, E as GetInteract
4
4
  import { C as runAgent, D as AgentStepRequest, E as AgentRequest, S as isAgentIdle, T as traceTransitions, _ as AgentTraceEvent, a as AgentMaxModelCallsExceededError, b as createAgentInbox, c as AgentRunMeta, d as AgentRunStart, f as AgentRuntime, g as AgentTimerScheduler, h as AgentTimerEvent, i as AgentInputFrom, l as AgentRunOptions, m as AgentSnapshotDivergedError, n as AgentEffects, o as AgentRunErrorCause, p as AgentRuntimeOptions, r as AgentInbox, s as AgentRunInit, t as AGENT_TRACE_SCHEMA_VERSION, u as AgentRunResult, v as AgentTransitionHandler, w as serializeTraceEvent, x as createAgentRuntime, y as JsonSerializableTraceEvent } from "./run-agent-BxkeuAYR.mjs";
5
5
  import { c as AgentLogEntry, d as AgentMachineVersionMismatchError, h as JsonValue, n as AgentEventLogStore, t as AgentEventLogConflictError } from "./event-log-store-DYun526m.mjs";
6
6
  import { t as PresetMachine } from "./internal-BzlpzSux.mjs";
7
- import { a as getMessageText, b as RecordCallOptions, c as isStandardSchema, d as userMessage, i as getJsonSchemaSync, j as Recorder, l as systemMessage, n as assistantMessage, o as getStateMeta, r as getJsonSchema, s as getStatePath, t as MetaOfSnapshot, u as toolMessage } from "./utils-B8k-YPYI.mjs";
7
+ import { M as Recorder, a as getMessageText, c as isStandardSchema, d as userMessage, i as getJsonSchemaSync, l as systemMessage, n as assistantMessage, o as getStateMeta, r as getJsonSchema, s as getStatePath, t as MetaOfSnapshot, u as toolMessage, x as RecordCallOptions } from "./utils-BV7pxhD-.mjs";
8
8
  import { AnyActorLogic, AnyMachineSnapshot, AnyStateMachine, ContextFrom, DoneActorEvent, EventFromLogic, InputFrom, OutputFrom, SnapshotFrom, StateValueFrom } from "xstate";
9
9
  //#region src/type-helpers.d.ts
10
10
  type SchemaAt<T, TKey extends PropertyKey> = T extends {
package/dist/testing.cjs CHANGED
@@ -5,8 +5,9 @@ const require_act = require("./act-B0CVbmo4.cjs");
5
5
  const require_event_log_store = require("./event-log-store-Dey9ufA4.cjs");
6
6
  const require_run_agent = require("./run-agent-COdR9FpM.cjs");
7
7
  const require_effects = require("./effects-DrWcUGgc.cjs");
8
- const require_verify = require("./verify-D8UDqU4l.cjs");
8
+ const require_verify = require("./verify-IVYsI026.cjs");
9
9
  const require_trajectory = require("./trajectory-CoZSv4wF.cjs");
10
+ let xstate = require("xstate");
10
11
  //#region src/equivalence.ts
11
12
  /**
12
13
  * Verified equivalence — prove a port behaves like the code it replaces.
@@ -39,6 +40,86 @@ const require_trajectory = require("./trajectory-CoZSv4wF.cjs");
39
40
  * neither `ai` nor `openai`.
40
41
  * @module
41
42
  */
43
+ const MODEL_METADATA = [
44
+ "specificationVersion",
45
+ "provider",
46
+ "modelId",
47
+ "supportedUrls"
48
+ ];
49
+ const MODEL_METHODS = [
50
+ "doGenerate",
51
+ "doStream",
52
+ "doDecide",
53
+ "doEvaluate"
54
+ ];
55
+ const MODEL_KEYS = [...MODEL_METADATA, ...MODEL_METHODS];
56
+ const aiSdkInternals = /* @__PURE__ */ new WeakMap();
57
+ /**
58
+ * Records AI SDK code that builds its model inline or passes module-level
59
+ * tools to `generateText`: there is nothing to inject, so the test mocks `"ai"`
60
+ * and routes each call through the recorder. Every `generateText` and
61
+ * `streamText` call runs on the real AI SDK with `seam.model` in place of the
62
+ * code's model (also in place of a model `prepareStep` returns) and its
63
+ * `tools` wrapped with `recorder.tools`. Everything else in `"ai"` stays real,
64
+ * so the code's own tool loop runs.
65
+ *
66
+ * Each model attempt made during the call carries the call's index as
67
+ * {@link RecordedModelCall.aiSdkCall} when its model is a `recorder.model`
68
+ * wrapper passed as is and the recorder came from `createRecorder`. Other
69
+ * attempts carry no index, and who resent them is unknown.
70
+ *
71
+ * `current` returns the seam of the run being recorded. A call while it returns
72
+ * `undefined` throws, so no test reaches a real provider.
73
+ *
74
+ * @example
75
+ * ```ts
76
+ * vi.mock("ai", async (importOriginal) => {
77
+ * const ai = await importOriginal<typeof import("ai")>();
78
+ * const { currentSeam } = await import("./seam"); // { recorder, model: recorder.model(standIn) }
79
+ * return { ...ai, ...recordAiSdk(ai, currentSeam) };
80
+ * });
81
+ * ```
82
+ */
83
+ function recordAiSdk(ai, current) {
84
+ const wrap = (name, call) => (options) => {
85
+ const seam = current();
86
+ if (!seam) throw new Error(`recordAiSdk: ${name} was called while no run is recording. Return { recorder } from current() while the code runs.`);
87
+ const internals = aiSdkInternals.get(seam.recorder);
88
+ const index = internals ? internals.calls++ : 0;
89
+ const tag = (model) => {
90
+ const base = internals?.unwrap(model);
91
+ return base ? internals.wrap(base, index) : model;
92
+ };
93
+ const model = seam.model ?? options.model;
94
+ const prepareStep = options.prepareStep;
95
+ return call({
96
+ ...options,
97
+ ...model !== void 0 ? { model: tag(model) } : {},
98
+ ...prepareStep ? { prepareStep: async (...args) => {
99
+ const step = await prepareStep(...args);
100
+ if (step?.model === void 0) return step;
101
+ const model = tag(seam.model ?? step.model);
102
+ const own = (key) => key === "model" ? model : Reflect.get(step, key, step);
103
+ return new Proxy({}, {
104
+ get: (_target, key) => own(key),
105
+ has: (_target, key) => key === "model" || key in step,
106
+ ownKeys: () => [.../* @__PURE__ */ new Set([...Reflect.ownKeys(step), "model"])],
107
+ getOwnPropertyDescriptor: (_target, key) => key === "model" || Object.prototype.hasOwnProperty.call(step, key) ? {
108
+ value: own(key),
109
+ enumerable: true,
110
+ configurable: true,
111
+ writable: true
112
+ } : void 0
113
+ });
114
+ } } : {},
115
+ ...options.tools ? { tools: seam.recorder.tools(options.tools) } : {}
116
+ });
117
+ };
118
+ return {
119
+ generateText: wrap("generateText", ai.generateText),
120
+ ...ai.streamText ? { streamText: wrap("streamText", ai.streamText) } : {}
121
+ };
122
+ }
42
123
  /**
43
124
  * Thrown in replay when the port asks for something the recording cannot
44
125
  * answer: a model call past the last recorded one, or a tool call the original
@@ -228,7 +309,28 @@ function createRecorder(options = {}) {
228
309
  * Records a model call around `run`, or answers it from the recording.
229
310
  * `fallback` runs it on the stand-in when the recording has no answer.
230
311
  */
231
- async function modelCall(seam, method, request, streamed, run, fallback) {
312
+ /**
313
+ * A retry resends the request of a model call that failed. Within one AI SDK
314
+ * call, the failed attempt is its own last attempt, even when a concurrent
315
+ * call came in between or claimed it first. Otherwise it is the last model
316
+ * call, which the code resent.
317
+ */
318
+ function linkRetry(effect) {
319
+ const models = effects.filter((earlier) => earlier.kind === "model").reverse();
320
+ const failedWith = (earlier) => earlier?.error !== void 0 && earlier.method === effect.method && stableStringify(earlier.request) === stableStringify(effect.request);
321
+ const { aiSdkCall } = effect;
322
+ if (aiSdkCall !== void 0) {
323
+ const own = models.find((earlier) => earlier.aiSdkCall === aiSdkCall);
324
+ const claimedBy = own?.retriedBy === void 0 ? void 0 : models.find((earlier) => earlier.call === own.retriedBy);
325
+ if (own && failedWith(own) && claimedBy?.aiSdkCall !== aiSdkCall) {
326
+ own.retriedBy = effect.call;
327
+ return;
328
+ }
329
+ }
330
+ const last = models[0];
331
+ if (last && failedWith(last) && last.retriedBy === void 0) last.retriedBy = effect.call;
332
+ }
333
+ async function modelCall(seam, method, request, streamed, run, fallback, aiSdkCall) {
232
334
  const effect = {
233
335
  kind: "model",
234
336
  seam,
@@ -240,6 +342,8 @@ function createRecorder(options = {}) {
240
342
  "signal"
241
343
  ]))
242
344
  };
345
+ if (aiSdkCall !== void 0) effect.aiSdkCall = aiSdkCall;
346
+ linkRetry(effect);
243
347
  effects.push(effect);
244
348
  if (replay) {
245
349
  const replayed = nextReplayedModel(effect);
@@ -469,22 +573,47 @@ function createRecorder(options = {}) {
469
573
  }
470
574
  return result;
471
575
  }
472
- return {
576
+ const wrappedModels = /* @__PURE__ */ new WeakMap();
577
+ function wrapModel(model, aiSdkCall) {
578
+ const base = model;
579
+ const stand = options.fallback?.model;
580
+ const wrapped = Object.create(base);
581
+ for (const key of MODEL_METADATA) {
582
+ if (!(key in base)) continue;
583
+ Object.defineProperty(wrapped, key, {
584
+ get: () => Reflect.get(base, key, base),
585
+ enumerable: true,
586
+ configurable: true
587
+ });
588
+ }
589
+ for (const method of MODEL_METHODS) {
590
+ if (typeof base[method] !== "function") continue;
591
+ const invoke = (target, callOptions) => target[method].call(target, callOptions);
592
+ Object.defineProperty(wrapped, method, {
593
+ value: (callOptions) => modelCall("ai-sdk", method, callOptions, method === "doStream", () => invoke(base, callOptions), stand && typeof stand[method] === "function" ? () => invoke(stand, callOptions) : void 0, aiSdkCall),
594
+ writable: true,
595
+ enumerable: true,
596
+ configurable: true
597
+ });
598
+ }
599
+ wrappedModels.set(wrapped, {
600
+ base,
601
+ made: new Map([...MODEL_KEYS].map((key) => [key, Object.getOwnPropertyDescriptor(wrapped, key)]))
602
+ });
603
+ return wrapped;
604
+ }
605
+ function unwrapModel(model) {
606
+ const entry = wrappedModels.get(model);
607
+ if (!entry) return void 0;
608
+ for (const [key, made] of entry.made) {
609
+ const now = Object.getOwnPropertyDescriptor(model, key);
610
+ if (now?.value !== made?.value || now?.get !== made?.get) return void 0;
611
+ }
612
+ return entry.base;
613
+ }
614
+ const recorder = {
473
615
  model(model) {
474
- const base = model;
475
- const stand = options.fallback?.model;
476
- const wrapped = Object.create(base);
477
- for (const method of [
478
- "doGenerate",
479
- "doStream",
480
- "doDecide",
481
- "doEvaluate"
482
- ]) {
483
- if (typeof base[method] !== "function") continue;
484
- const invoke = (target, callOptions) => target[method].call(target, callOptions);
485
- wrapped[method] = (callOptions) => modelCall("ai-sdk", method, callOptions, method === "doStream", () => invoke(base, callOptions), stand && typeof stand[method] === "function" ? () => invoke(stand, callOptions) : void 0);
486
- }
487
- return wrapped;
616
+ return wrapModel(model);
488
617
  },
489
618
  openai(client) {
490
619
  const wrapCreate = (resource, method) => new Proxy(resource, { get(target, key, receiver) {
@@ -568,6 +697,12 @@ function createRecorder(options = {}) {
568
697
  };
569
698
  }
570
699
  };
700
+ aiSdkInternals.set(recorder, {
701
+ unwrap: (model) => model !== null && (typeof model === "object" || typeof model === "function") ? unwrapModel(model) : void 0,
702
+ wrap: wrapModel,
703
+ calls: 0
704
+ });
705
+ return recorder;
571
706
  }
572
707
  /** True when two model calls offer the same tool names and response format. */
573
708
  function sameModelCallShape(expected, actual) {
@@ -1695,6 +1830,228 @@ function structurallyEqual(a, b) {
1695
1830
  return keys.length === Object.keys(b).length && keys.every((key) => Object.hasOwn(b, key) && structurallyEqual(a[key], b[key]));
1696
1831
  }
1697
1832
  //#endregion
1833
+ //#region src/conformance.ts
1834
+ /**
1835
+ * Conformance: check a recorded run of existing code against a model of it.
1836
+ *
1837
+ * The model is an ordinary machine whose actors stand for the code's work.
1838
+ * `checkConformance` runs the model with every actor answered from the
1839
+ * recording: each one consumes the next observation of its kind. The first
1840
+ * time the model does something the recording does not show next, the run
1841
+ * stops and the step is reported. The code itself is not changed or run.
1842
+ */
1843
+ const NEVER = new Promise(() => {});
1844
+ function inlineInvokeStates(machine) {
1845
+ const found = [];
1846
+ const stack = [machine.root];
1847
+ while (stack.length > 0) {
1848
+ const node = stack.pop();
1849
+ const invoke = node.config.invoke;
1850
+ if ((invoke === void 0 ? [] : Array.isArray(invoke) ? invoke : [invoke]).some((entry) => typeof entry.src !== "string")) found.push(node.path.join(".") || "(root)");
1851
+ stack.push(...Object.values(node.states));
1852
+ }
1853
+ return found.sort();
1854
+ }
1855
+ function actorSources(machine) {
1856
+ const actors = require_event_log.getRegisteredAgentExecutionOptions(machine).actors ?? machine.sources?.actors ?? {};
1857
+ return Object.keys(actors);
1858
+ }
1859
+ /**
1860
+ * Checks a recorded run of existing code against a model of that code.
1861
+ *
1862
+ * The model runs with `runAgent`, and every actor in `actors` is answered from
1863
+ * `observations` instead of running: an actor with `observes` consumes the next
1864
+ * observation, which must be of that kind. The report names the first
1865
+ * deviation: where the model started something the recording does not show
1866
+ * next, or ended while the recording continued.
1867
+ *
1868
+ * Use it to verify existing code in place: prove the model's rules with
1869
+ * `canReach`, then check that recorded runs of the code conform to the model.
1870
+ * A conforming run shows the code behaved as the model allows on that run; it
1871
+ * does not prove the code can never break a rule.
1872
+ *
1873
+ * Every actor the model invokes must be declared by name: a model whose invoke
1874
+ * names logic inline throws `inline-actor-source`, because that logic would
1875
+ * run. A callback in `actors` that throws ends the check with
1876
+ * `conformance-callback-error` instead of counting as an actor failure.
1877
+ *
1878
+ * @example
1879
+ * ```ts
1880
+ * const report = await checkConformance(loopModel, {
1881
+ * input: { maxIter: 3 },
1882
+ * observations: recording.effects.flatMap(toObservation),
1883
+ * actors: {
1884
+ * route: { observes: "route", output: ({ observation }) => observation.names },
1885
+ * runTurn: { observes: "turn", matches: ({ input, observation }) => input.name === observation.agent },
1886
+ * save: {}, // not visible in this recording
1887
+ * },
1888
+ * });
1889
+ * report.conforms; // false when the run did something the model does not allow
1890
+ * ```
1891
+ */
1892
+ async function checkConformance(machine, options) {
1893
+ const { observations, actors: specs, events = {} } = options;
1894
+ const inline = inlineInvokeStates(machine);
1895
+ if (inline.length > 0) throw new require_event_log.AgentError("inline-actor-source", `checkConformance: ${inline.join(", ")} invoke${inline.length === 1 ? "s" : ""} inline actor logic, which would run for real. Declare each actor by name in setup ({ actors }) and invoke it by src, so the recording can answer it.`);
1896
+ for (const src of Object.keys(specs)) if (!actorSources(machine).includes(src)) throw new require_event_log.AgentError("unknown-actor-source", `checkConformance: actors.${src} names no actor source the machine declares (${actorSources(machine).join(", ") || "none"}).`);
1897
+ let index = 0;
1898
+ let deviation;
1899
+ let callbackError;
1900
+ let state = machine.config.initial;
1901
+ const unobserved = /* @__PURE__ */ new Set();
1902
+ const trace = [];
1903
+ const controller = new AbortController();
1904
+ const callbackFailure = (name, cause) => new require_event_log.AgentError("conformance-callback-error", `checkConformance: ${name} threw: ${cause instanceof Error ? cause.message : String(cause)}`, { cause });
1905
+ const callback = (name, fn) => {
1906
+ try {
1907
+ return fn();
1908
+ } catch (cause) {
1909
+ throw callbackFailure(name, cause);
1910
+ }
1911
+ };
1912
+ const deviate = (reason, expected, actorInput) => {
1913
+ deviation ??= {
1914
+ reason,
1915
+ index,
1916
+ expected,
1917
+ observed: observations[index],
1918
+ state,
1919
+ ...actorInput !== void 0 ? { actorInput } : {}
1920
+ };
1921
+ controller.abort();
1922
+ return NEVER;
1923
+ };
1924
+ const answer = (src, spec) => (0, xstate.createAsyncLogic)({ run: async ({ input }) => {
1925
+ if (deviation || callbackError) return NEVER;
1926
+ if (!spec) return deviate("unmapped-actor", src, input);
1927
+ let observation;
1928
+ if (spec.observes === void 0) unobserved.add(src);
1929
+ else {
1930
+ const next = observations[index];
1931
+ if (next === void 0) return deviate("missing-observation", src, input);
1932
+ if (next.kind !== spec.observes) return deviate("unexpected-observation", src, input);
1933
+ observation = next;
1934
+ index += 1;
1935
+ }
1936
+ const args = {
1937
+ input,
1938
+ observation,
1939
+ observations,
1940
+ index
1941
+ };
1942
+ const call = (name, fn) => {
1943
+ try {
1944
+ return fn();
1945
+ } catch (cause) {
1946
+ callbackError ??= callbackFailure(`actors.${src}.${name}`, cause);
1947
+ controller.abort();
1948
+ return NEVER;
1949
+ }
1950
+ };
1951
+ if (spec.matches) {
1952
+ const fits = call("matches", () => spec.matches(args));
1953
+ if (fits === NEVER) return NEVER;
1954
+ if (!fits) {
1955
+ index -= observation ? 1 : 0;
1956
+ return deviate("input-mismatch", src, input);
1957
+ }
1958
+ }
1959
+ const error = call("error", () => spec.error?.(args));
1960
+ if (error === NEVER) return NEVER;
1961
+ if (error !== void 0) throw error;
1962
+ return call("output", () => spec.output?.(args));
1963
+ } });
1964
+ const shared = {
1965
+ actors: Object.fromEntries(actorSources(machine).map((src) => [src, answer(src, specs[src])])),
1966
+ signal: controller.signal,
1967
+ onTrace: (event) => trace.push(event),
1968
+ onTransition: (snapshot) => {
1969
+ state = snapshot.value;
1970
+ }
1971
+ };
1972
+ let result = await require_run_agent.runAgent(machine, {
1973
+ ...shared,
1974
+ ...options.input !== void 0 ? { input: options.input } : {}
1975
+ });
1976
+ while (!deviation && !callbackError && result.status === "idle") {
1977
+ const next = observations[index];
1978
+ const entry = next === void 0 ? void 0 : events[next.kind];
1979
+ if (!entry || next === void 0) break;
1980
+ const { event: toEvent, matches } = typeof entry === "function" ? { event: entry } : entry;
1981
+ if (!callback(`events.${next.kind}.matches`, () => matches ? matches({
1982
+ observation: next,
1983
+ context: result.snapshot.context,
1984
+ state
1985
+ }) : true)) {
1986
+ deviation = {
1987
+ reason: "input-mismatch",
1988
+ index,
1989
+ expected: `event ${next.kind}`,
1990
+ observed: next,
1991
+ state
1992
+ };
1993
+ break;
1994
+ }
1995
+ const event = callback(`events.${next.kind}`, () => toEvent(next));
1996
+ index += 1;
1997
+ result = await require_run_agent.runAgent(machine, {
1998
+ ...shared,
1999
+ snapshot: result.persist(),
2000
+ event
2001
+ });
2002
+ if (result.ignored !== void 0) {
2003
+ index -= 1;
2004
+ deviation = {
2005
+ reason: "ignored-event",
2006
+ index,
2007
+ expected: `an event ${JSON.stringify(state)} handles`,
2008
+ observed: next,
2009
+ state
2010
+ };
2011
+ break;
2012
+ }
2013
+ }
2014
+ if (callbackError) throw callbackError;
2015
+ const outputSpec = options.output;
2016
+ const last = observations[index];
2017
+ if (!deviation && outputSpec && result.status === "done" && last === void 0) deviation = {
2018
+ reason: "missing-observation",
2019
+ index,
2020
+ expected: `output (${outputSpec.observes})`,
2021
+ observed: void 0,
2022
+ state
2023
+ };
2024
+ else if (!deviation && outputSpec && result.status === "done" && last?.kind === outputSpec.observes) {
2025
+ const output = result.output;
2026
+ if (callback("output.matches", () => outputSpec.matches ? outputSpec.matches({
2027
+ output,
2028
+ observation: last
2029
+ }) : true)) index += 1;
2030
+ else deviation = {
2031
+ reason: "output-mismatch",
2032
+ index,
2033
+ expected: `output ${JSON.stringify(output)}`,
2034
+ observed: last,
2035
+ state
2036
+ };
2037
+ }
2038
+ if (!deviation && index < observations.length) deviation = {
2039
+ reason: "extra-observations",
2040
+ index,
2041
+ expected: result.status === "idle" ? "idle, waiting for an event" : "end of run",
2042
+ observed: observations[index],
2043
+ state
2044
+ };
2045
+ return {
2046
+ conforms: deviation === void 0,
2047
+ ...deviation ? { deviation } : {},
2048
+ consumed: index,
2049
+ unobserved: [...unobserved],
2050
+ status: result.status,
2051
+ trace
2052
+ };
2053
+ }
2054
+ //#endregion
1698
2055
  //#region src/check-parity.ts
1699
2056
  /**
1700
2057
  * `checkParity` — one call that records a workflow, replays its port, and
@@ -1809,6 +2166,7 @@ exports.AgentUnknownStateError = require_verify.AgentUnknownStateError;
1809
2166
  exports.RecordingDivergenceError = RecordingDivergenceError;
1810
2167
  exports.assertAgentMachine = require_verify.assertAgentMachine;
1811
2168
  exports.canReach = require_verify.canReach;
2169
+ exports.checkConformance = checkConformance;
1812
2170
  exports.checkCrashRecovery = checkCrashRecovery;
1813
2171
  exports.checkParity = checkParity;
1814
2172
  exports.compareRecordings = compareRecordings;
@@ -1817,6 +2175,7 @@ exports.createRecorder = createRecorder;
1817
2175
  exports.explorePaths = require_verify.explorePaths;
1818
2176
  exports.lintAgentMachine = require_verify.lintAgentMachine;
1819
2177
  exports.matchesTrajectory = require_trajectory.matchesTrajectory;
2178
+ exports.recordAiSdk = recordAiSdk;
1820
2179
  exports.runSeam = runSeam;
1821
2180
  exports.sameModelCallShape = sameModelCallShape;
1822
2181
  exports.simulateAgent = require_verify.simulateAgent;