@statelyai/agent 2.0.0-alpha.34 → 2.0.0-alpha.35

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,18 @@
1
1
  # @statelyai/agent
2
2
 
3
+ ## 2.0.0-alpha.35
4
+
5
+ ### Patch Changes
6
+
7
+ - [#191](https://github.com/statelyai/agent/pull/191) [`8dcb0c8`](https://github.com/statelyai/agent/commit/8dcb0c8643b1bd38e53538999c3d27c58c4b096f) Thanks [@davidkpiano](https://github.com/davidkpiano)! - **Replay ports that change on purpose.** `createRecorder({ replay })` now answers each model call with the earliest unused recorded call of the same method and shape, and each tool call with the earliest unused recorded call of the same name and input, instead of by position. A call the port added or dropped no longer shifts the answers after it, unless another call has the same shape (or tool name and input): those are answered in recorded order.
8
+
9
+ - `translate(recorded, actual)` adapts a recorded answer to a call you changed on purpose (a stream that became `doGenerate`, text that became a decision). `compareRecordings` counts the pair as the same call.
10
+ - `fallback: { model, openai, tools }` answers what the recording cannot: a stand-in model or client answers the call, and the wrapped tool runs. Those effects are marked `unrecorded` and still count as extra.
11
+ - `compareRecordings(a, b, { kinds: ["tool", "output"] })` compares only what a port did, for a port that calls the model another way.
12
+ - `normalize(effect, index, effects)` can return `undefined` to leave an effect out of the comparison, such as a repeated idempotent write. Replay still answers from it.
13
+ - `recorder.model` wraps a decision model's `doDecide` (and `doEvaluate`). Its shape is the questions, their choice options, and their score levels.
14
+ - A recorded error keeps its number and boolean fields (`statusCode`, `isRetryable`) and its `Symbol.for` markers. In replay, `APICallError.isInstance` and retry behavior hold. Comparisons still check only an error's name, message, and code.
15
+
3
16
  ## 2.0.0-alpha.34
4
17
 
5
18
  ### Patch Changes
package/dist/testing.cjs CHANGED
@@ -1145,24 +1145,38 @@ var RecordingDivergenceError = class extends Error {
1145
1145
  name = "RecordingDivergenceError";
1146
1146
  };
1147
1147
  function toRecordedError(error) {
1148
- if (error instanceof Error) {
1149
- const code = error.code;
1150
- return code === void 0 ? {
1151
- name: error.name,
1152
- message: error.message
1153
- } : {
1154
- name: error.name,
1155
- message: error.message,
1156
- code: toJson(code)
1157
- };
1158
- }
1159
- return {
1148
+ if (!(error instanceof Error)) return {
1160
1149
  name: "Error",
1161
1150
  message: String(error)
1162
1151
  };
1152
+ const recorded = {
1153
+ name: error.name,
1154
+ message: error.message
1155
+ };
1156
+ const code = error.code;
1157
+ if (code !== void 0) recorded.code = toJson(code);
1158
+ const data = Object.fromEntries(Object.entries(error).filter(([key, value]) => key !== "code" && (typeof value === "number" || typeof value === "boolean")));
1159
+ if (Object.keys(data).length) recorded.data = data;
1160
+ const markers = /* @__PURE__ */ new Set();
1161
+ for (let holder = error; holder; holder = Object.getPrototypeOf(holder)) for (const symbol of Object.getOwnPropertySymbols(holder)) {
1162
+ const key = Symbol.keyFor(symbol);
1163
+ if (key !== void 0 && error[symbol] === true) markers.add(key);
1164
+ }
1165
+ if (markers.size) recorded.markers = [...markers];
1166
+ return recorded;
1167
+ }
1168
+ function fromRecordedError({ data, markers, ...fields }) {
1169
+ const error = Object.assign(new Error(fields.message), fields, data);
1170
+ for (const marker of markers ?? []) error[Symbol.for(marker)] = true;
1171
+ return error;
1163
1172
  }
1164
- function fromRecordedError(error) {
1165
- return Object.assign(new Error(error.message), error);
1173
+ /** What two runs compare of an error: its data and markers may differ. */
1174
+ function errorIdentity(error) {
1175
+ return error && {
1176
+ name: error.name,
1177
+ message: error.message,
1178
+ code: error.code
1179
+ };
1166
1180
  }
1167
1181
  /** A JSON-safe copy: Dates become ISO strings, Errors become {@link RecordedError}. */
1168
1182
  function toJson(value) {
@@ -1213,19 +1227,42 @@ function createRecorder(options = {}) {
1213
1227
  const effects = [];
1214
1228
  const replay = options.replay?.effects;
1215
1229
  const replayedModels = replay?.filter((effect) => effect.kind === "model");
1230
+ const usedModels = /* @__PURE__ */ new Set();
1216
1231
  const usedTools = /* @__PURE__ */ new Set();
1217
- const normalizeEffect = options.normalize ?? ((effect) => effect);
1232
+ const normalizeEffect = (effect, list) => options.normalize?.(effect, list.indexOf(effect), list) ?? effect;
1218
1233
  let modelCalls = 0;
1219
1234
  let toolsInFlight = 0;
1235
+ /**
1236
+ * The earliest unused recorded call that fits this one: one `translate`
1237
+ * adapts, or one with the same method and shape. Not by position, so a call
1238
+ * the port added or dropped does not shift every answer after it.
1239
+ */
1220
1240
  function nextReplayedModel(effect) {
1221
- const recorded = replayedModels[effect.call];
1222
- if (!recorded) throw new RecordingDivergenceError(`Model call #${effect.call} was never made by the recorded run (it made ${replayedModels.length}).`);
1223
- if (recorded.method !== effect.method) throw new RecordingDivergenceError(`Model call #${effect.call} uses ${effect.method}; the recorded run used ${recorded.method}.`);
1224
- if (!modelCallsMatch(options.modelCalls, recorded, effect)) throw new RecordingDivergenceError(shapeMessage(recorded, effect));
1225
- return recorded;
1241
+ for (const recorded of replayedModels) {
1242
+ if (usedModels.has(recorded)) continue;
1243
+ const translated = options.translate?.(structuredClone(recorded), effect);
1244
+ if (translated) {
1245
+ effect.translated = true;
1246
+ return {
1247
+ recorded,
1248
+ answer: translated
1249
+ };
1250
+ }
1251
+ if (recorded.method === effect.method && modelCallsMatch(options.modelCalls, recorded, effect)) return {
1252
+ recorded,
1253
+ answer: recorded
1254
+ };
1255
+ }
1256
+ }
1257
+ function noAnswerMessage(effect) {
1258
+ const unused = replayedModels.filter((recorded) => !usedModels.has(recorded));
1259
+ return unused.length ? `Model call #${effect.call} (${effect.method}, ${JSON.stringify(modelCallShape(effect))}) fits no unused recorded call: ${unused.map((recorded) => `#${recorded.call} (${recorded.method}, ${JSON.stringify(modelCallShape(recorded))})`).join(", ")}. A recorded answer only fits a call of the same method and shape; pass \`translate\` to adapt one.` : `Model call #${effect.call} was never made by the recorded run (it made ${replayedModels.length}).`;
1226
1260
  }
1227
- /** Records a model call around `run`, or answers it from the recording. */
1228
- async function modelCall(seam, method, request, streamed, run) {
1261
+ /**
1262
+ * Records a model call around `run`, or answers it from the recording.
1263
+ * `fallback` runs it on the stand-in when the recording has no answer.
1264
+ */
1265
+ async function modelCall(seam, method, request, streamed, run, fallback) {
1229
1266
  const effect = {
1230
1267
  kind: "model",
1231
1268
  seam,
@@ -1239,24 +1276,30 @@ function createRecorder(options = {}) {
1239
1276
  };
1240
1277
  effects.push(effect);
1241
1278
  if (replay) {
1242
- let recorded;
1243
- try {
1244
- recorded = nextReplayedModel(effect);
1245
- } catch (error) {
1279
+ const replayed = nextReplayedModel(effect);
1280
+ if (replayed) {
1281
+ const { recorded, answer } = replayed;
1282
+ usedModels.add(recorded);
1283
+ effect.replayed = recorded.call;
1284
+ if (answer.error) {
1285
+ effect.error = answer.error;
1286
+ throw fromRecordedError(answer.error);
1287
+ }
1288
+ effect.response = answer.response;
1289
+ effect.stream = answer.stream;
1290
+ if (seam === "openai") return streamed ? iterate(answer.stream ?? []) : structuredClone(answer.response);
1291
+ return streamed ? {
1292
+ ...reviveTimestamps(structuredClone(answer.response) ?? {}),
1293
+ stream: streamFrom(answer.stream ?? [])
1294
+ } : reviveTimestamps(structuredClone(answer.response));
1295
+ }
1296
+ if (!fallback) {
1297
+ const error = new RecordingDivergenceError(noAnswerMessage(effect));
1246
1298
  effect.error = toRecordedError(error);
1247
1299
  throw error;
1248
1300
  }
1249
- if (recorded.error) {
1250
- effect.error = recorded.error;
1251
- throw fromRecordedError(recorded.error);
1252
- }
1253
- effect.response = recorded.response;
1254
- effect.stream = recorded.stream;
1255
- if (seam === "openai") return streamed ? iterate(recorded.stream ?? []) : structuredClone(recorded.response);
1256
- return streamed ? {
1257
- ...reviveTimestamps(structuredClone(recorded.response) ?? {}),
1258
- stream: streamFrom(recorded.stream ?? [])
1259
- } : reviveTimestamps(structuredClone(recorded.response));
1301
+ effect.unrecorded = true;
1302
+ run = fallback;
1260
1303
  }
1261
1304
  let result;
1262
1305
  try {
@@ -1307,8 +1350,13 @@ function createRecorder(options = {}) {
1307
1350
  function call(effect, args) {
1308
1351
  const { input } = effect;
1309
1352
  if (replay && options.tools !== "live") {
1310
- const wanted = normalizeEffect(effect).input;
1311
- const recorded = replay.find((candidate) => candidate.kind === "tool" && !usedTools.has(candidate) && candidate.step === effect.step && candidate.name === name && jsonEqual(normalizeEffect(candidate).input, wanted));
1353
+ const wanted = normalizeEffect(effect, effects).input;
1354
+ const matches = (candidate) => candidate.kind === "tool" && !usedTools.has(candidate) && candidate.name === name && jsonEqual(normalizeEffect(candidate, replay).input, wanted);
1355
+ const recorded = replay.find(matches);
1356
+ if (!recorded && options.fallback?.tools) {
1357
+ effect.unrecorded = true;
1358
+ return runLive(effect, args);
1359
+ }
1312
1360
  const settle = (value) => {
1313
1361
  if (recorded?.sync) return value();
1314
1362
  try {
@@ -1319,7 +1367,7 @@ function createRecorder(options = {}) {
1319
1367
  };
1320
1368
  return settle(() => {
1321
1369
  if (!recorded) {
1322
- const error = new RecordingDivergenceError(`Tool '${name}' was called with ${JSON.stringify(input)} ${effect.step === 0 ? "before any model call" : `after model call #${effect.step - 1}`}; the recorded run made no such call.`);
1370
+ const error = new RecordingDivergenceError(`Tool '${name}' was called with ${JSON.stringify(input)}; the recorded run made no such call.`);
1323
1371
  effect.error = toRecordedError(error);
1324
1372
  throw error;
1325
1373
  }
@@ -1333,6 +1381,9 @@ function createRecorder(options = {}) {
1333
1381
  return structuredClone(recorded.output);
1334
1382
  });
1335
1383
  }
1384
+ return runLive(effect, args);
1385
+ }
1386
+ function runLive(effect, args) {
1336
1387
  let output;
1337
1388
  try {
1338
1389
  output = fn(...args);
@@ -1369,9 +1420,18 @@ function createRecorder(options = {}) {
1369
1420
  return {
1370
1421
  model(model) {
1371
1422
  const base = model;
1423
+ const stand = options.fallback?.model;
1372
1424
  const wrapped = Object.create(base);
1373
- wrapped["doGenerate"] = (callOptions) => modelCall("ai-sdk", "doGenerate", callOptions, false, () => base["doGenerate"].call(base, callOptions));
1374
- wrapped["doStream"] = (callOptions) => modelCall("ai-sdk", "doStream", callOptions, true, () => base["doStream"].call(base, callOptions));
1425
+ for (const method of [
1426
+ "doGenerate",
1427
+ "doStream",
1428
+ "doDecide",
1429
+ "doEvaluate"
1430
+ ]) {
1431
+ if (typeof base[method] !== "function") continue;
1432
+ const invoke = (target, callOptions) => target[method].call(target, callOptions);
1433
+ wrapped[method] = (callOptions) => modelCall("ai-sdk", method, callOptions, method === "doStream", () => invoke(base, callOptions), stand && typeof stand[method] === "function" ? () => invoke(stand, callOptions) : void 0);
1434
+ }
1375
1435
  return wrapped;
1376
1436
  },
1377
1437
  openai(client) {
@@ -1379,10 +1439,12 @@ function createRecorder(options = {}) {
1379
1439
  if (key !== "create") return Reflect.get(target, key, receiver);
1380
1440
  return (params, requestOptions) => {
1381
1441
  let api;
1382
- const promise = modelCall("openai", method, params, params?.stream === true, () => {
1383
- api = Reflect.get(target, "create").call(target, params, requestOptions);
1442
+ const create = (resource) => () => {
1443
+ api = Reflect.get(resource, "create").call(resource, params, requestOptions);
1384
1444
  return Promise.resolve(api);
1385
- });
1445
+ };
1446
+ const stand = fallbackResource(method);
1447
+ const promise = modelCall("openai", method, params, params?.stream === true, create(target), stand && create(stand));
1386
1448
  return Object.assign(promise, { withResponse: async () => {
1387
1449
  const data = await promise;
1388
1450
  const meta = await api?.withResponse?.();
@@ -1394,6 +1456,11 @@ function createRecorder(options = {}) {
1394
1456
  } });
1395
1457
  };
1396
1458
  } });
1459
+ const fallbackResource = (method) => {
1460
+ let value = options.fallback?.openai;
1461
+ for (const key of method.split(".").slice(0, -1)) value = value?.[key];
1462
+ return value && typeof value === "object" ? value : void 0;
1463
+ };
1397
1464
  const wrapped = /* @__PURE__ */ new Map();
1398
1465
  const wrap = (key, make) => {
1399
1466
  if (!wrapped.has(key)) wrapped.set(key, make());
@@ -1442,9 +1509,27 @@ function modelCallsMatch(match, expected, actual) {
1442
1509
  function shapeMessage(expected, actual) {
1443
1510
  return `Model call #${expected.call} changed shape: ${JSON.stringify(modelCallShape(expected))} → ${JSON.stringify(modelCallShape(actual))}. A recorded answer only fits a call of the same shape.`;
1444
1511
  }
1445
- /** Offered tool names and the response format: what a recorded answer depends on. */
1512
+ /**
1513
+ * Offered tool names and the response format, or a decision's questions and
1514
+ * their options: what a recorded answer depends on.
1515
+ */
1446
1516
  function modelCallShape(effect) {
1447
1517
  const request = effect.request ?? {};
1518
+ const questions = request["questions"];
1519
+ if (questions && typeof questions === "object") return { questions: Object.entries(questions).map(([name, question]) => {
1520
+ const criteria = question["criteria"];
1521
+ if (question["type"] === "choice" && criteria && typeof criteria === "object") return [
1522
+ name,
1523
+ question["type"],
1524
+ Object.keys(criteria).sort()
1525
+ ];
1526
+ if (question["type"] === "score" && Array.isArray(criteria)) return [
1527
+ name,
1528
+ question["type"],
1529
+ criteria.length
1530
+ ];
1531
+ return [name, question["type"]];
1532
+ }).sort((x, y) => String(x[0]).localeCompare(String(y[0]))) };
1448
1533
  return {
1449
1534
  tools: (Array.isArray(request["tools"]) ? request["tools"] : []).map((tool) => tool["name"] ?? tool["function"]?.["name"]).sort(),
1450
1535
  format: request["responseFormat"]?.["type"] ?? request["response_format"]?.["type"] ?? (request["text"]?.["format"])?.["type"] ?? "text"
@@ -1485,6 +1570,7 @@ function differ(left, right, options) {
1485
1570
  message: `Expected ${describe(left)}; the port made ${describe(right)}.`
1486
1571
  };
1487
1572
  if (left.kind === "model" && right.kind === "model") {
1573
+ if (right.translated && right.replayed === left.call) return void 0;
1488
1574
  if (left.method !== right.method) return {
1489
1575
  reason: "model-method",
1490
1576
  message: `Model call #${left.call} used ${left.method}; the port used ${right.method}.`
@@ -1493,7 +1579,7 @@ function differ(left, right, options) {
1493
1579
  reason: "model-call-shape",
1494
1580
  message: shapeMessage(left, right)
1495
1581
  };
1496
- if (!jsonEqual(left.error, right.error)) {
1582
+ if (!jsonEqual(errorIdentity(left.error), errorIdentity(right.error))) {
1497
1583
  const outcome = (error) => error ? `failed with ${error.name}: ${error.message}` : "succeeded";
1498
1584
  return {
1499
1585
  reason: "model-error",
@@ -1511,7 +1597,7 @@ function differ(left, right, options) {
1511
1597
  reason: "tool-input",
1512
1598
  message: `Expected ${describe(left)}; the port passed ${JSON.stringify(right.input)}.`
1513
1599
  };
1514
- if (!jsonEqual(left.output, right.output) || !jsonEqual(left.error, right.error)) return {
1600
+ if (!jsonEqual(left.output, right.output) || !jsonEqual(errorIdentity(left.error), errorIdentity(right.error))) return {
1515
1601
  reason: "tool-result",
1516
1602
  message: `Tool '${left.name}' returned a different result.`
1517
1603
  };
@@ -1530,9 +1616,13 @@ function differ(left, right, options) {
1530
1616
  * kind pair up and report what changed, and the rest are missing or extra.
1531
1617
  */
1532
1618
  function compareRecordings(expected, actual, options = {}) {
1533
- const map = options.normalize ?? ((effect) => effect);
1534
- const a = normalize(expected.effects.map(map));
1535
- const b = normalize(actual.effects.map(map));
1619
+ const select = (effects) => normalize(effects.flatMap((effect, index) => {
1620
+ if (options.kinds && !options.kinds.includes(effect.kind)) return [];
1621
+ const mapped = options.normalize ? options.normalize(effect, index, effects) : effect;
1622
+ return mapped ? [mapped] : [];
1623
+ }));
1624
+ const a = select(expected.effects);
1625
+ const b = select(actual.effects);
1536
1626
  const same = a.map((left) => b.map((right) => !differ(left, right, options)));
1537
1627
  const lcs = Array.from({ length: a.length + 1 }, () => Array.from({ length: b.length + 1 }, () => 0));
1538
1628
  for (let i = a.length - 1; i >= 0; i--) for (let j = b.length - 1; j >= 0; j--) lcs[i][j] = same[i][j] ? lcs[i + 1][j + 1] + 1 : Math.max(lcs[i + 1][j], lcs[i][j + 1]);
@@ -847,6 +847,12 @@ export declare function compareRuns<TScenario extends {
847
847
  * side effects, and records what the port did.
848
848
  * 3. `compareRecordings(original, port)` reports the first effect that differs.
849
849
  *
850
+ * A port that changes how it calls the model on purpose (a model-picked action
851
+ * that became `agent.decide`, a stream that became `doGenerate`) still replays:
852
+ * `translate` turns the recorded answer into one that fits, `fallback` answers
853
+ * the calls the original never made, and `compareRecordings(…, { kinds })`
854
+ * compares only tool calls and output.
855
+ *
850
856
  * Provider-free: both seams are wrapped structurally, so this module imports
851
857
  * neither `ai` nor `openai`.
852
858
  * @module
@@ -856,13 +862,20 @@ interface RecordedError {
856
862
  name: string;
857
863
  message: string;
858
864
  code?: unknown;
865
+ /** The error's own number and boolean fields, such as `statusCode` and
866
+ * `isRetryable`; replay restores them. */
867
+ data?: Record<string, number | boolean>;
868
+ /** `Symbol.for` keys set to `true` on the error, such as the AI SDK's
869
+ * `vercel.ai.error.AI_APICallError`; replay restores them, so checks like
870
+ * `APICallError.isInstance` still pass. */
871
+ markers?: string[];
859
872
  }
860
873
  /** One model call. `call` is its 0-based position among model calls. */
861
874
  interface RecordedModelCall {
862
875
  kind: "model";
863
876
  /** Which wrapper saw it. */
864
877
  seam: "ai-sdk" | "openai";
865
- /** `doGenerate`/`doStream`, or `chat.completions.create`/`responses.create`. */
878
+ /** `doGenerate`/`doStream`/`doDecide`, or `chat.completions.create`/`responses.create`. */
866
879
  method: string;
867
880
  call: number;
868
881
  /** The call options or params, JSON-safe (abort signals and headers dropped). */
@@ -872,6 +885,12 @@ interface RecordedModelCall {
872
885
  /** The stream's parts or chunks, in order. */
873
886
  stream?: unknown[];
874
887
  error?: RecordedError;
888
+ /** In replay, the recorded call (its `call`) whose answer served this one. */
889
+ replayed?: number;
890
+ /** In replay, `translate` adapted the recorded answer to this call. */
891
+ translated?: true;
892
+ /** In replay, no recorded call fit, so `fallback` answered it. */
893
+ unrecorded?: true;
875
894
  }
876
895
  /** One tool call. `step` is the number of model calls made before it. */
877
896
  interface RecordedToolCall {
@@ -886,6 +905,8 @@ interface RecordedToolCall {
886
905
  /** Started while an earlier tool call was still running. Concurrent calls
887
906
  * may finish in either order; sequential ones must keep theirs. */
888
907
  concurrent?: true;
908
+ /** In replay, no recorded call matched, so `fallback.tools` ran the tool. */
909
+ unrecorded?: true;
889
910
  }
890
911
  /** The run's final result. */
891
912
  interface RecordedOutput {
@@ -917,10 +938,53 @@ interface CreateRecorderOptions {
917
938
  /**
918
939
  * In replay, maps the recorded and the new tool call before matching them,
919
940
  * like {@link CompareRecordingsOptions.normalize}: redact a generated id in a
920
- * tool's input so the recorded result still answers it.
941
+ * tool's input so the recorded result still answers it. A dropped
942
+ * (`undefined`) effect matches as recorded.
943
+ */
944
+ normalize?: NormalizeEffect;
945
+ /**
946
+ * In replay, adapts a recorded answer to a model call you changed on
947
+ * purpose. Called with each unused recorded call, in order, until it
948
+ * returns an answer; that answer serves the call regardless of method and
949
+ * shape, and {@link compareRecordings} counts the pair as the same call.
950
+ * Return `undefined` to leave a recorded call to the usual matching.
951
+ *
952
+ * @example
953
+ * ```ts
954
+ * // The original streamed; the port generates.
955
+ * translate: (recorded, actual) =>
956
+ * actual.method === "doGenerate" && recorded.method === "doStream"
957
+ * ? { response: toGenerateResult(recorded.stream) }
958
+ * : undefined,
959
+ * ```
921
960
  */
922
- normalize?: (effect: RecordedEffect) => RecordedEffect;
961
+ translate?: (recorded: RecordedModelCall, actual: RecordedModelCall) => ModelAnswer | undefined;
962
+ /**
963
+ * In replay, answers what the recording cannot, instead of throwing
964
+ * {@link RecordingDivergenceError}: a model call no recorded call fits (the
965
+ * port's retry or an added call) goes to `model` (AI SDK seam) or `openai`
966
+ * (client seam), and with `tools: true` a tool call no recorded call matches
967
+ * runs the wrapped tool. Such effects are marked `unrecorded` and still
968
+ * count as extra in {@link compareRecordings}.
969
+ */
970
+ fallback?: {
971
+ model?: RecordableModel;
972
+ openai?: object;
973
+ tools?: boolean;
974
+ };
975
+ }
976
+ /** A model call's answer: the result, the stream's parts, or an error. */
977
+ interface ModelAnswer {
978
+ response?: unknown;
979
+ stream?: unknown[];
980
+ error?: RecordedError;
923
981
  }
982
+ /**
983
+ * Maps an effect before matching or comparing; `undefined` drops it.
984
+ * `index` and `effects` are its position in the whole recording, so a repeat
985
+ * of an earlier effect (an idempotent write made again) can be dropped.
986
+ */
987
+ type NormalizeEffect = (effect: RecordedEffect, index: number, effects: readonly RecordedEffect[]) => RecordedEffect | undefined;
924
988
  /**
925
989
  * When two model calls count as the same: `"shape"` compares offered tool
926
990
  * names and response format, `"none"` only that a call happened, and a
@@ -935,15 +999,21 @@ interface RecordableLanguageModel {
935
999
  stream: ReadableStream<unknown>;
936
1000
  }>;
937
1001
  }
1002
+ /** The structural slice of an AI SDK decision model (`experimental_decide`). */
1003
+ interface RecordableDecisionModel {
1004
+ doDecide(options: never): PromiseLike<unknown>;
1005
+ }
1006
+ /** An AI SDK model the recorder can wrap. */
1007
+ type RecordableModel = RecordableLanguageModel | RecordableDecisionModel;
938
1008
  type AnyFunction = (...args: never[]) => unknown;
939
1009
  /** Records (or replays) the effects of one run. */
940
1010
  interface Recorder {
941
1011
  /**
942
- * Wraps an AI SDK `LanguageModelV3` (`openai("gpt-…")`, `anthropic(…)`),
943
- * keeping its type. In replay the model is never called, so a provider
944
- * model built with no API key works.
1012
+ * Wraps an AI SDK `LanguageModelV3` (`openai("gpt-…")`, `anthropic(…)`) or
1013
+ * decision model (`doDecide`), keeping its type. In replay the model is
1014
+ * never called, so a provider model built with no API key works.
945
1015
  */
946
- model<TModel extends RecordableLanguageModel>(model: TModel): TModel;
1016
+ model<TModel extends RecordableModel>(model: TModel): TModel;
947
1017
  /**
948
1018
  * Wraps an `openai` client's `chat.completions.create` and
949
1019
  * `responses.create`, keeping its type. A streamed result is an async
@@ -1002,10 +1072,16 @@ interface CompareRecordingsOptions {
1002
1072
  modelCalls?: ModelCallMatch;
1003
1073
  /**
1004
1074
  * Maps each effect before comparing, on both sides: redact values that
1005
- * differ on every run (generated ids, timestamps) or project a tool's input
1006
- * to the fields that matter.
1075
+ * differ on every run (generated ids, timestamps), project a tool's input
1076
+ * to the fields that matter, or return `undefined` to drop the effect.
1077
+ */
1078
+ normalize?: NormalizeEffect;
1079
+ /**
1080
+ * Which kinds of effect to compare. Default all. `["tool", "output"]`
1081
+ * compares a port that calls the model differently (another protocol, a
1082
+ * structured draft instead of a tool loop) by what it did, not how it asked.
1007
1083
  */
1008
- normalize?: (effect: RecordedEffect) => RecordedEffect;
1084
+ kinds?: readonly RecordedEffect["kind"][];
1009
1085
  }
1010
1086
  /** True when two model calls offer the same tool names and response format. */
1011
1087
  export declare function sameModelCallShape(expected: RecordedModelCall, actual: RecordedModelCall): boolean;
@@ -1018,4 +1094,4 @@ export declare function sameModelCallShape(expected: RecordedModelCall, actual:
1018
1094
  */
1019
1095
  export declare function compareRecordings(expected: Recording, actual: Recording, options?: CompareRecordingsOptions): EquivalenceReport;
1020
1096
  //#endregion
1021
- export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AgreementReport, type AgreementRow, type AgreementSummary, type AssertAgentMachineOptions, type CanReachResult, type CanReachTarget, type CompareRecordingsOptions, type CompareRunsOptions, type CoverageReport, type CreateRecorderOptions, type Divergence, type EffectDivergence, type EffectDivergenceReason, type EquivalenceReport, type ExplorePathsOptions, type ExploreSnapshot, type FirstDivergenceOptions, type FoldTracesOptions, type GetDecisionRequestsOptions, type HandlerCoverage, type HandlerRef, type JudgeAgreementOptions, type LintAgentMachineOptions, type MatchTrajectoryOptions, type ModelCallMatch, type ParityLabel, type ParityReport, type ParityResult, type ParityRun, type RecordableLanguageModel, type RecordedEffect, type RecordedError, type RecordedModelCall, type RecordedOutput, type RecordedToolCall, type Recorder, type Recording, type RepeatGroup, type RepeatReport, type RepeatResult, type RepeatRun, type RepeatRunsOptions, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type StateCoverage, type StateStats, type TraceCoverageOptions, type TraceStep, type TracedDecisionRequest, type TrajectoryEvent, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, explorePaths, lintAgentMachine, simulateAgent };
1097
+ export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AgreementReport, type AgreementRow, type AgreementSummary, type AssertAgentMachineOptions, type CanReachResult, type CanReachTarget, type CompareRecordingsOptions, type CompareRunsOptions, type CoverageReport, type CreateRecorderOptions, type Divergence, type EffectDivergence, type EffectDivergenceReason, type EquivalenceReport, type ExplorePathsOptions, type ExploreSnapshot, type FirstDivergenceOptions, type FoldTracesOptions, type GetDecisionRequestsOptions, type HandlerCoverage, type HandlerRef, type JudgeAgreementOptions, type LintAgentMachineOptions, type MatchTrajectoryOptions, type ModelAnswer, type ModelCallMatch, type NormalizeEffect, type ParityLabel, type ParityReport, type ParityResult, type ParityRun, type RecordableDecisionModel, type RecordableLanguageModel, type RecordableModel, type RecordedEffect, type RecordedError, type RecordedModelCall, type RecordedOutput, type RecordedToolCall, type Recorder, type Recording, type RepeatGroup, type RepeatReport, type RepeatResult, type RepeatRun, type RepeatRunsOptions, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type StateCoverage, type StateStats, type TraceCoverageOptions, type TraceStep, type TracedDecisionRequest, type TrajectoryEvent, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, explorePaths, lintAgentMachine, simulateAgent };
@@ -847,6 +847,12 @@ export declare function compareRuns<TScenario extends {
847
847
  * side effects, and records what the port did.
848
848
  * 3. `compareRecordings(original, port)` reports the first effect that differs.
849
849
  *
850
+ * A port that changes how it calls the model on purpose (a model-picked action
851
+ * that became `agent.decide`, a stream that became `doGenerate`) still replays:
852
+ * `translate` turns the recorded answer into one that fits, `fallback` answers
853
+ * the calls the original never made, and `compareRecordings(…, { kinds })`
854
+ * compares only tool calls and output.
855
+ *
850
856
  * Provider-free: both seams are wrapped structurally, so this module imports
851
857
  * neither `ai` nor `openai`.
852
858
  * @module
@@ -856,13 +862,20 @@ interface RecordedError {
856
862
  name: string;
857
863
  message: string;
858
864
  code?: unknown;
865
+ /** The error's own number and boolean fields, such as `statusCode` and
866
+ * `isRetryable`; replay restores them. */
867
+ data?: Record<string, number | boolean>;
868
+ /** `Symbol.for` keys set to `true` on the error, such as the AI SDK's
869
+ * `vercel.ai.error.AI_APICallError`; replay restores them, so checks like
870
+ * `APICallError.isInstance` still pass. */
871
+ markers?: string[];
859
872
  }
860
873
  /** One model call. `call` is its 0-based position among model calls. */
861
874
  interface RecordedModelCall {
862
875
  kind: "model";
863
876
  /** Which wrapper saw it. */
864
877
  seam: "ai-sdk" | "openai";
865
- /** `doGenerate`/`doStream`, or `chat.completions.create`/`responses.create`. */
878
+ /** `doGenerate`/`doStream`/`doDecide`, or `chat.completions.create`/`responses.create`. */
866
879
  method: string;
867
880
  call: number;
868
881
  /** The call options or params, JSON-safe (abort signals and headers dropped). */
@@ -872,6 +885,12 @@ interface RecordedModelCall {
872
885
  /** The stream's parts or chunks, in order. */
873
886
  stream?: unknown[];
874
887
  error?: RecordedError;
888
+ /** In replay, the recorded call (its `call`) whose answer served this one. */
889
+ replayed?: number;
890
+ /** In replay, `translate` adapted the recorded answer to this call. */
891
+ translated?: true;
892
+ /** In replay, no recorded call fit, so `fallback` answered it. */
893
+ unrecorded?: true;
875
894
  }
876
895
  /** One tool call. `step` is the number of model calls made before it. */
877
896
  interface RecordedToolCall {
@@ -886,6 +905,8 @@ interface RecordedToolCall {
886
905
  /** Started while an earlier tool call was still running. Concurrent calls
887
906
  * may finish in either order; sequential ones must keep theirs. */
888
907
  concurrent?: true;
908
+ /** In replay, no recorded call matched, so `fallback.tools` ran the tool. */
909
+ unrecorded?: true;
889
910
  }
890
911
  /** The run's final result. */
891
912
  interface RecordedOutput {
@@ -917,10 +938,53 @@ interface CreateRecorderOptions {
917
938
  /**
918
939
  * In replay, maps the recorded and the new tool call before matching them,
919
940
  * like {@link CompareRecordingsOptions.normalize}: redact a generated id in a
920
- * tool's input so the recorded result still answers it.
941
+ * tool's input so the recorded result still answers it. A dropped
942
+ * (`undefined`) effect matches as recorded.
943
+ */
944
+ normalize?: NormalizeEffect;
945
+ /**
946
+ * In replay, adapts a recorded answer to a model call you changed on
947
+ * purpose. Called with each unused recorded call, in order, until it
948
+ * returns an answer; that answer serves the call regardless of method and
949
+ * shape, and {@link compareRecordings} counts the pair as the same call.
950
+ * Return `undefined` to leave a recorded call to the usual matching.
951
+ *
952
+ * @example
953
+ * ```ts
954
+ * // The original streamed; the port generates.
955
+ * translate: (recorded, actual) =>
956
+ * actual.method === "doGenerate" && recorded.method === "doStream"
957
+ * ? { response: toGenerateResult(recorded.stream) }
958
+ * : undefined,
959
+ * ```
921
960
  */
922
- normalize?: (effect: RecordedEffect) => RecordedEffect;
961
+ translate?: (recorded: RecordedModelCall, actual: RecordedModelCall) => ModelAnswer | undefined;
962
+ /**
963
+ * In replay, answers what the recording cannot, instead of throwing
964
+ * {@link RecordingDivergenceError}: a model call no recorded call fits (the
965
+ * port's retry or an added call) goes to `model` (AI SDK seam) or `openai`
966
+ * (client seam), and with `tools: true` a tool call no recorded call matches
967
+ * runs the wrapped tool. Such effects are marked `unrecorded` and still
968
+ * count as extra in {@link compareRecordings}.
969
+ */
970
+ fallback?: {
971
+ model?: RecordableModel;
972
+ openai?: object;
973
+ tools?: boolean;
974
+ };
975
+ }
976
+ /** A model call's answer: the result, the stream's parts, or an error. */
977
+ interface ModelAnswer {
978
+ response?: unknown;
979
+ stream?: unknown[];
980
+ error?: RecordedError;
923
981
  }
982
+ /**
983
+ * Maps an effect before matching or comparing; `undefined` drops it.
984
+ * `index` and `effects` are its position in the whole recording, so a repeat
985
+ * of an earlier effect (an idempotent write made again) can be dropped.
986
+ */
987
+ type NormalizeEffect = (effect: RecordedEffect, index: number, effects: readonly RecordedEffect[]) => RecordedEffect | undefined;
924
988
  /**
925
989
  * When two model calls count as the same: `"shape"` compares offered tool
926
990
  * names and response format, `"none"` only that a call happened, and a
@@ -935,15 +999,21 @@ interface RecordableLanguageModel {
935
999
  stream: ReadableStream<unknown>;
936
1000
  }>;
937
1001
  }
1002
+ /** The structural slice of an AI SDK decision model (`experimental_decide`). */
1003
+ interface RecordableDecisionModel {
1004
+ doDecide(options: never): PromiseLike<unknown>;
1005
+ }
1006
+ /** An AI SDK model the recorder can wrap. */
1007
+ type RecordableModel = RecordableLanguageModel | RecordableDecisionModel;
938
1008
  type AnyFunction = (...args: never[]) => unknown;
939
1009
  /** Records (or replays) the effects of one run. */
940
1010
  interface Recorder {
941
1011
  /**
942
- * Wraps an AI SDK `LanguageModelV3` (`openai("gpt-…")`, `anthropic(…)`),
943
- * keeping its type. In replay the model is never called, so a provider
944
- * model built with no API key works.
1012
+ * Wraps an AI SDK `LanguageModelV3` (`openai("gpt-…")`, `anthropic(…)`) or
1013
+ * decision model (`doDecide`), keeping its type. In replay the model is
1014
+ * never called, so a provider model built with no API key works.
945
1015
  */
946
- model<TModel extends RecordableLanguageModel>(model: TModel): TModel;
1016
+ model<TModel extends RecordableModel>(model: TModel): TModel;
947
1017
  /**
948
1018
  * Wraps an `openai` client's `chat.completions.create` and
949
1019
  * `responses.create`, keeping its type. A streamed result is an async
@@ -1002,10 +1072,16 @@ interface CompareRecordingsOptions {
1002
1072
  modelCalls?: ModelCallMatch;
1003
1073
  /**
1004
1074
  * Maps each effect before comparing, on both sides: redact values that
1005
- * differ on every run (generated ids, timestamps) or project a tool's input
1006
- * to the fields that matter.
1075
+ * differ on every run (generated ids, timestamps), project a tool's input
1076
+ * to the fields that matter, or return `undefined` to drop the effect.
1077
+ */
1078
+ normalize?: NormalizeEffect;
1079
+ /**
1080
+ * Which kinds of effect to compare. Default all. `["tool", "output"]`
1081
+ * compares a port that calls the model differently (another protocol, a
1082
+ * structured draft instead of a tool loop) by what it did, not how it asked.
1007
1083
  */
1008
- normalize?: (effect: RecordedEffect) => RecordedEffect;
1084
+ kinds?: readonly RecordedEffect["kind"][];
1009
1085
  }
1010
1086
  /** True when two model calls offer the same tool names and response format. */
1011
1087
  export declare function sameModelCallShape(expected: RecordedModelCall, actual: RecordedModelCall): boolean;
@@ -1018,4 +1094,4 @@ export declare function sameModelCallShape(expected: RecordedModelCall, actual:
1018
1094
  */
1019
1095
  export declare function compareRecordings(expected: Recording, actual: Recording, options?: CompareRecordingsOptions): EquivalenceReport;
1020
1096
  //#endregion
1021
- export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AgreementReport, type AgreementRow, type AgreementSummary, type AssertAgentMachineOptions, type CanReachResult, type CanReachTarget, type CompareRecordingsOptions, type CompareRunsOptions, type CoverageReport, type CreateRecorderOptions, type Divergence, type EffectDivergence, type EffectDivergenceReason, type EquivalenceReport, type ExplorePathsOptions, type ExploreSnapshot, type FirstDivergenceOptions, type FoldTracesOptions, type GetDecisionRequestsOptions, type HandlerCoverage, type HandlerRef, type JudgeAgreementOptions, type LintAgentMachineOptions, type MatchTrajectoryOptions, type ModelCallMatch, type ParityLabel, type ParityReport, type ParityResult, type ParityRun, type RecordableLanguageModel, type RecordedEffect, type RecordedError, type RecordedModelCall, type RecordedOutput, type RecordedToolCall, type Recorder, type Recording, type RepeatGroup, type RepeatReport, type RepeatResult, type RepeatRun, type RepeatRunsOptions, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type StateCoverage, type StateStats, type TraceCoverageOptions, type TraceStep, type TracedDecisionRequest, type TrajectoryEvent, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, explorePaths, lintAgentMachine, simulateAgent };
1097
+ export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AgreementReport, type AgreementRow, type AgreementSummary, type AssertAgentMachineOptions, type CanReachResult, type CanReachTarget, type CompareRecordingsOptions, type CompareRunsOptions, type CoverageReport, type CreateRecorderOptions, type Divergence, type EffectDivergence, type EffectDivergenceReason, type EquivalenceReport, type ExplorePathsOptions, type ExploreSnapshot, type FirstDivergenceOptions, type FoldTracesOptions, type GetDecisionRequestsOptions, type HandlerCoverage, type HandlerRef, type JudgeAgreementOptions, type LintAgentMachineOptions, type MatchTrajectoryOptions, type ModelAnswer, type ModelCallMatch, type NormalizeEffect, type ParityLabel, type ParityReport, type ParityResult, type ParityRun, type RecordableDecisionModel, type RecordableLanguageModel, type RecordableModel, type RecordedEffect, type RecordedError, type RecordedModelCall, type RecordedOutput, type RecordedToolCall, type Recorder, type Recording, type RepeatGroup, type RepeatReport, type RepeatResult, type RepeatRun, type RepeatRunsOptions, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type StateCoverage, type StateStats, type TraceCoverageOptions, type TraceStep, type TracedDecisionRequest, type TrajectoryEvent, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, explorePaths, lintAgentMachine, simulateAgent };
package/dist/testing.mjs CHANGED
@@ -1144,24 +1144,38 @@ var RecordingDivergenceError = class extends Error {
1144
1144
  name = "RecordingDivergenceError";
1145
1145
  };
1146
1146
  function toRecordedError(error) {
1147
- if (error instanceof Error) {
1148
- const code = error.code;
1149
- return code === void 0 ? {
1150
- name: error.name,
1151
- message: error.message
1152
- } : {
1153
- name: error.name,
1154
- message: error.message,
1155
- code: toJson(code)
1156
- };
1157
- }
1158
- return {
1147
+ if (!(error instanceof Error)) return {
1159
1148
  name: "Error",
1160
1149
  message: String(error)
1161
1150
  };
1151
+ const recorded = {
1152
+ name: error.name,
1153
+ message: error.message
1154
+ };
1155
+ const code = error.code;
1156
+ if (code !== void 0) recorded.code = toJson(code);
1157
+ const data = Object.fromEntries(Object.entries(error).filter(([key, value]) => key !== "code" && (typeof value === "number" || typeof value === "boolean")));
1158
+ if (Object.keys(data).length) recorded.data = data;
1159
+ const markers = /* @__PURE__ */ new Set();
1160
+ for (let holder = error; holder; holder = Object.getPrototypeOf(holder)) for (const symbol of Object.getOwnPropertySymbols(holder)) {
1161
+ const key = Symbol.keyFor(symbol);
1162
+ if (key !== void 0 && error[symbol] === true) markers.add(key);
1163
+ }
1164
+ if (markers.size) recorded.markers = [...markers];
1165
+ return recorded;
1166
+ }
1167
+ function fromRecordedError({ data, markers, ...fields }) {
1168
+ const error = Object.assign(new Error(fields.message), fields, data);
1169
+ for (const marker of markers ?? []) error[Symbol.for(marker)] = true;
1170
+ return error;
1162
1171
  }
1163
- function fromRecordedError(error) {
1164
- return Object.assign(new Error(error.message), error);
1172
+ /** What two runs compare of an error: its data and markers may differ. */
1173
+ function errorIdentity(error) {
1174
+ return error && {
1175
+ name: error.name,
1176
+ message: error.message,
1177
+ code: error.code
1178
+ };
1165
1179
  }
1166
1180
  /** A JSON-safe copy: Dates become ISO strings, Errors become {@link RecordedError}. */
1167
1181
  function toJson(value) {
@@ -1212,19 +1226,42 @@ function createRecorder(options = {}) {
1212
1226
  const effects = [];
1213
1227
  const replay = options.replay?.effects;
1214
1228
  const replayedModels = replay?.filter((effect) => effect.kind === "model");
1229
+ const usedModels = /* @__PURE__ */ new Set();
1215
1230
  const usedTools = /* @__PURE__ */ new Set();
1216
- const normalizeEffect = options.normalize ?? ((effect) => effect);
1231
+ const normalizeEffect = (effect, list) => options.normalize?.(effect, list.indexOf(effect), list) ?? effect;
1217
1232
  let modelCalls = 0;
1218
1233
  let toolsInFlight = 0;
1234
+ /**
1235
+ * The earliest unused recorded call that fits this one: one `translate`
1236
+ * adapts, or one with the same method and shape. Not by position, so a call
1237
+ * the port added or dropped does not shift every answer after it.
1238
+ */
1219
1239
  function nextReplayedModel(effect) {
1220
- const recorded = replayedModels[effect.call];
1221
- if (!recorded) throw new RecordingDivergenceError(`Model call #${effect.call} was never made by the recorded run (it made ${replayedModels.length}).`);
1222
- if (recorded.method !== effect.method) throw new RecordingDivergenceError(`Model call #${effect.call} uses ${effect.method}; the recorded run used ${recorded.method}.`);
1223
- if (!modelCallsMatch(options.modelCalls, recorded, effect)) throw new RecordingDivergenceError(shapeMessage(recorded, effect));
1224
- return recorded;
1240
+ for (const recorded of replayedModels) {
1241
+ if (usedModels.has(recorded)) continue;
1242
+ const translated = options.translate?.(structuredClone(recorded), effect);
1243
+ if (translated) {
1244
+ effect.translated = true;
1245
+ return {
1246
+ recorded,
1247
+ answer: translated
1248
+ };
1249
+ }
1250
+ if (recorded.method === effect.method && modelCallsMatch(options.modelCalls, recorded, effect)) return {
1251
+ recorded,
1252
+ answer: recorded
1253
+ };
1254
+ }
1255
+ }
1256
+ function noAnswerMessage(effect) {
1257
+ const unused = replayedModels.filter((recorded) => !usedModels.has(recorded));
1258
+ return unused.length ? `Model call #${effect.call} (${effect.method}, ${JSON.stringify(modelCallShape(effect))}) fits no unused recorded call: ${unused.map((recorded) => `#${recorded.call} (${recorded.method}, ${JSON.stringify(modelCallShape(recorded))})`).join(", ")}. A recorded answer only fits a call of the same method and shape; pass \`translate\` to adapt one.` : `Model call #${effect.call} was never made by the recorded run (it made ${replayedModels.length}).`;
1225
1259
  }
1226
- /** Records a model call around `run`, or answers it from the recording. */
1227
- async function modelCall(seam, method, request, streamed, run) {
1260
+ /**
1261
+ * Records a model call around `run`, or answers it from the recording.
1262
+ * `fallback` runs it on the stand-in when the recording has no answer.
1263
+ */
1264
+ async function modelCall(seam, method, request, streamed, run, fallback) {
1228
1265
  const effect = {
1229
1266
  kind: "model",
1230
1267
  seam,
@@ -1238,24 +1275,30 @@ function createRecorder(options = {}) {
1238
1275
  };
1239
1276
  effects.push(effect);
1240
1277
  if (replay) {
1241
- let recorded;
1242
- try {
1243
- recorded = nextReplayedModel(effect);
1244
- } catch (error) {
1278
+ const replayed = nextReplayedModel(effect);
1279
+ if (replayed) {
1280
+ const { recorded, answer } = replayed;
1281
+ usedModels.add(recorded);
1282
+ effect.replayed = recorded.call;
1283
+ if (answer.error) {
1284
+ effect.error = answer.error;
1285
+ throw fromRecordedError(answer.error);
1286
+ }
1287
+ effect.response = answer.response;
1288
+ effect.stream = answer.stream;
1289
+ if (seam === "openai") return streamed ? iterate(answer.stream ?? []) : structuredClone(answer.response);
1290
+ return streamed ? {
1291
+ ...reviveTimestamps(structuredClone(answer.response) ?? {}),
1292
+ stream: streamFrom(answer.stream ?? [])
1293
+ } : reviveTimestamps(structuredClone(answer.response));
1294
+ }
1295
+ if (!fallback) {
1296
+ const error = new RecordingDivergenceError(noAnswerMessage(effect));
1245
1297
  effect.error = toRecordedError(error);
1246
1298
  throw error;
1247
1299
  }
1248
- if (recorded.error) {
1249
- effect.error = recorded.error;
1250
- throw fromRecordedError(recorded.error);
1251
- }
1252
- effect.response = recorded.response;
1253
- effect.stream = recorded.stream;
1254
- if (seam === "openai") return streamed ? iterate(recorded.stream ?? []) : structuredClone(recorded.response);
1255
- return streamed ? {
1256
- ...reviveTimestamps(structuredClone(recorded.response) ?? {}),
1257
- stream: streamFrom(recorded.stream ?? [])
1258
- } : reviveTimestamps(structuredClone(recorded.response));
1300
+ effect.unrecorded = true;
1301
+ run = fallback;
1259
1302
  }
1260
1303
  let result;
1261
1304
  try {
@@ -1306,8 +1349,13 @@ function createRecorder(options = {}) {
1306
1349
  function call(effect, args) {
1307
1350
  const { input } = effect;
1308
1351
  if (replay && options.tools !== "live") {
1309
- const wanted = normalizeEffect(effect).input;
1310
- const recorded = replay.find((candidate) => candidate.kind === "tool" && !usedTools.has(candidate) && candidate.step === effect.step && candidate.name === name && jsonEqual(normalizeEffect(candidate).input, wanted));
1352
+ const wanted = normalizeEffect(effect, effects).input;
1353
+ const matches = (candidate) => candidate.kind === "tool" && !usedTools.has(candidate) && candidate.name === name && jsonEqual(normalizeEffect(candidate, replay).input, wanted);
1354
+ const recorded = replay.find(matches);
1355
+ if (!recorded && options.fallback?.tools) {
1356
+ effect.unrecorded = true;
1357
+ return runLive(effect, args);
1358
+ }
1311
1359
  const settle = (value) => {
1312
1360
  if (recorded?.sync) return value();
1313
1361
  try {
@@ -1318,7 +1366,7 @@ function createRecorder(options = {}) {
1318
1366
  };
1319
1367
  return settle(() => {
1320
1368
  if (!recorded) {
1321
- const error = new RecordingDivergenceError(`Tool '${name}' was called with ${JSON.stringify(input)} ${effect.step === 0 ? "before any model call" : `after model call #${effect.step - 1}`}; the recorded run made no such call.`);
1369
+ const error = new RecordingDivergenceError(`Tool '${name}' was called with ${JSON.stringify(input)}; the recorded run made no such call.`);
1322
1370
  effect.error = toRecordedError(error);
1323
1371
  throw error;
1324
1372
  }
@@ -1332,6 +1380,9 @@ function createRecorder(options = {}) {
1332
1380
  return structuredClone(recorded.output);
1333
1381
  });
1334
1382
  }
1383
+ return runLive(effect, args);
1384
+ }
1385
+ function runLive(effect, args) {
1335
1386
  let output;
1336
1387
  try {
1337
1388
  output = fn(...args);
@@ -1368,9 +1419,18 @@ function createRecorder(options = {}) {
1368
1419
  return {
1369
1420
  model(model) {
1370
1421
  const base = model;
1422
+ const stand = options.fallback?.model;
1371
1423
  const wrapped = Object.create(base);
1372
- wrapped["doGenerate"] = (callOptions) => modelCall("ai-sdk", "doGenerate", callOptions, false, () => base["doGenerate"].call(base, callOptions));
1373
- wrapped["doStream"] = (callOptions) => modelCall("ai-sdk", "doStream", callOptions, true, () => base["doStream"].call(base, callOptions));
1424
+ for (const method of [
1425
+ "doGenerate",
1426
+ "doStream",
1427
+ "doDecide",
1428
+ "doEvaluate"
1429
+ ]) {
1430
+ if (typeof base[method] !== "function") continue;
1431
+ const invoke = (target, callOptions) => target[method].call(target, callOptions);
1432
+ wrapped[method] = (callOptions) => modelCall("ai-sdk", method, callOptions, method === "doStream", () => invoke(base, callOptions), stand && typeof stand[method] === "function" ? () => invoke(stand, callOptions) : void 0);
1433
+ }
1374
1434
  return wrapped;
1375
1435
  },
1376
1436
  openai(client) {
@@ -1378,10 +1438,12 @@ function createRecorder(options = {}) {
1378
1438
  if (key !== "create") return Reflect.get(target, key, receiver);
1379
1439
  return (params, requestOptions) => {
1380
1440
  let api;
1381
- const promise = modelCall("openai", method, params, params?.stream === true, () => {
1382
- api = Reflect.get(target, "create").call(target, params, requestOptions);
1441
+ const create = (resource) => () => {
1442
+ api = Reflect.get(resource, "create").call(resource, params, requestOptions);
1383
1443
  return Promise.resolve(api);
1384
- });
1444
+ };
1445
+ const stand = fallbackResource(method);
1446
+ const promise = modelCall("openai", method, params, params?.stream === true, create(target), stand && create(stand));
1385
1447
  return Object.assign(promise, { withResponse: async () => {
1386
1448
  const data = await promise;
1387
1449
  const meta = await api?.withResponse?.();
@@ -1393,6 +1455,11 @@ function createRecorder(options = {}) {
1393
1455
  } });
1394
1456
  };
1395
1457
  } });
1458
+ const fallbackResource = (method) => {
1459
+ let value = options.fallback?.openai;
1460
+ for (const key of method.split(".").slice(0, -1)) value = value?.[key];
1461
+ return value && typeof value === "object" ? value : void 0;
1462
+ };
1396
1463
  const wrapped = /* @__PURE__ */ new Map();
1397
1464
  const wrap = (key, make) => {
1398
1465
  if (!wrapped.has(key)) wrapped.set(key, make());
@@ -1441,9 +1508,27 @@ function modelCallsMatch(match, expected, actual) {
1441
1508
  function shapeMessage(expected, actual) {
1442
1509
  return `Model call #${expected.call} changed shape: ${JSON.stringify(modelCallShape(expected))} → ${JSON.stringify(modelCallShape(actual))}. A recorded answer only fits a call of the same shape.`;
1443
1510
  }
1444
- /** Offered tool names and the response format: what a recorded answer depends on. */
1511
+ /**
1512
+ * Offered tool names and the response format, or a decision's questions and
1513
+ * their options: what a recorded answer depends on.
1514
+ */
1445
1515
  function modelCallShape(effect) {
1446
1516
  const request = effect.request ?? {};
1517
+ const questions = request["questions"];
1518
+ if (questions && typeof questions === "object") return { questions: Object.entries(questions).map(([name, question]) => {
1519
+ const criteria = question["criteria"];
1520
+ if (question["type"] === "choice" && criteria && typeof criteria === "object") return [
1521
+ name,
1522
+ question["type"],
1523
+ Object.keys(criteria).sort()
1524
+ ];
1525
+ if (question["type"] === "score" && Array.isArray(criteria)) return [
1526
+ name,
1527
+ question["type"],
1528
+ criteria.length
1529
+ ];
1530
+ return [name, question["type"]];
1531
+ }).sort((x, y) => String(x[0]).localeCompare(String(y[0]))) };
1447
1532
  return {
1448
1533
  tools: (Array.isArray(request["tools"]) ? request["tools"] : []).map((tool) => tool["name"] ?? tool["function"]?.["name"]).sort(),
1449
1534
  format: request["responseFormat"]?.["type"] ?? request["response_format"]?.["type"] ?? (request["text"]?.["format"])?.["type"] ?? "text"
@@ -1484,6 +1569,7 @@ function differ(left, right, options) {
1484
1569
  message: `Expected ${describe(left)}; the port made ${describe(right)}.`
1485
1570
  };
1486
1571
  if (left.kind === "model" && right.kind === "model") {
1572
+ if (right.translated && right.replayed === left.call) return void 0;
1487
1573
  if (left.method !== right.method) return {
1488
1574
  reason: "model-method",
1489
1575
  message: `Model call #${left.call} used ${left.method}; the port used ${right.method}.`
@@ -1492,7 +1578,7 @@ function differ(left, right, options) {
1492
1578
  reason: "model-call-shape",
1493
1579
  message: shapeMessage(left, right)
1494
1580
  };
1495
- if (!jsonEqual(left.error, right.error)) {
1581
+ if (!jsonEqual(errorIdentity(left.error), errorIdentity(right.error))) {
1496
1582
  const outcome = (error) => error ? `failed with ${error.name}: ${error.message}` : "succeeded";
1497
1583
  return {
1498
1584
  reason: "model-error",
@@ -1510,7 +1596,7 @@ function differ(left, right, options) {
1510
1596
  reason: "tool-input",
1511
1597
  message: `Expected ${describe(left)}; the port passed ${JSON.stringify(right.input)}.`
1512
1598
  };
1513
- if (!jsonEqual(left.output, right.output) || !jsonEqual(left.error, right.error)) return {
1599
+ if (!jsonEqual(left.output, right.output) || !jsonEqual(errorIdentity(left.error), errorIdentity(right.error))) return {
1514
1600
  reason: "tool-result",
1515
1601
  message: `Tool '${left.name}' returned a different result.`
1516
1602
  };
@@ -1529,9 +1615,13 @@ function differ(left, right, options) {
1529
1615
  * kind pair up and report what changed, and the rest are missing or extra.
1530
1616
  */
1531
1617
  function compareRecordings(expected, actual, options = {}) {
1532
- const map = options.normalize ?? ((effect) => effect);
1533
- const a = normalize(expected.effects.map(map));
1534
- const b = normalize(actual.effects.map(map));
1618
+ const select = (effects) => normalize(effects.flatMap((effect, index) => {
1619
+ if (options.kinds && !options.kinds.includes(effect.kind)) return [];
1620
+ const mapped = options.normalize ? options.normalize(effect, index, effects) : effect;
1621
+ return mapped ? [mapped] : [];
1622
+ }));
1623
+ const a = select(expected.effects);
1624
+ const b = select(actual.effects);
1535
1625
  const same = a.map((left) => b.map((right) => !differ(left, right, options)));
1536
1626
  const lcs = Array.from({ length: a.length + 1 }, () => Array.from({ length: b.length + 1 }, () => 0));
1537
1627
  for (let i = a.length - 1; i >= 0; i--) for (let j = b.length - 1; j >= 0; j--) lcs[i][j] = same[i][j] ? lcs[i + 1][j + 1] + 1 : Math.max(lcs[i + 1][j], lcs[i][j + 1]);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@statelyai/agent",
3
- "version": "2.0.0-alpha.34",
3
+ "version": "2.0.0-alpha.35",
4
4
  "description": "Make invalid agent actions impossible. Agent logic as state machines: deterministic, inspectable, resumable, runs anywhere.",
5
5
  "keywords": [
6
6
  "agent",
@@ -225,11 +225,13 @@ test.each(SCENARIOS)("equivalent to the recorded loop: $id", async (scenario) =>
225
225
  const INTENDED: Record<string, Array<Pick<EffectDivergence, "reason" | "message">>> = {};
226
226
  ```
227
227
 
228
- - Equivalent means the same model calls (count and shape: offered tool names, response format), the same tool calls with the same arguments, and the same output. Prompts and states may differ. Tool calls between two model calls compare in any order.
229
- - Values that differ on every run (generated ids, branch names, timestamps) fail `tool-input`. Pass the same `normalize` to `createRecorder({ replay, normalize })` and `compareRecordings(a, b, { normalize })` to redact them on both sides; never edit the fixtures.
228
+ - Equivalent means the same model calls (count and shape: offered tool names, response format), the same tool calls with the same arguments, and the same output. Prompts and states may differ. Tool calls that ran at the same time compare in any order; sequential ones keep theirs.
229
+ - Values that differ on every run (generated ids, branch names, timestamps) fail `tool-input`. Pass the same `normalize` to `createRecorder({ replay, normalize })` and `compareRecordings(a, b, { normalize })` to redact them on both sides; never edit the fixtures. `normalize(effect, index, effects)` returns `undefined` to leave an effect out of `compareRecordings`, such as a repeat of an idempotent write; replay still answers from it. Drop an effect only when repeating it is harmless: a dropped `send` or `charge` hides a duplicate.
230
230
  - `divergences` lists every difference, lined up so one extra call does not shift the rest; `divergence` is the first.
231
- - In replay, a model call past the recorded ones or a tool call the old code never made throws `RecordingDivergenceError`, and `compareRecordings` names it.
232
- - `model-call-shape`: the machine changed a call's tools or output format, for example a model-picked action that became `agent.decide`. The recorded answer does not fit the new call, so equivalence ends there. Port the call unchanged first, prove equivalence, then change it in a separate step whose divergences are `INTENDED`. To keep replaying past a call you changed on purpose, pass the same `modelCalls` function to `createRecorder({ replay, modelCalls })` and `compareRecordings`: `(expected, actual) => expected.call === 2 || sameModelCallShape(expected, actual)`. The recorded answer must still fit the new call.
231
+ - Replay answers each model call with the earliest unused recorded call of the same method and shape, and each tool call with the earliest unused recorded call of the same name and input, not by position. A call the machine added or dropped does not shift the answers after it, as long as no other call has the same method and shape (or tool name and input). Calls that do are answered in recorded order, so an added or dropped one takes or leaves the next one's answer.
232
+ - A call no recorded answer fits throws `RecordingDivergenceError`, and `compareRecordings` names it. To run the machine's own path instead (its retry, its added check), pass `fallback: { model: standIn, tools: true }`: a stand-in model answers the call and the wrapped tool runs. Those effects are marked `unrecorded` and still count as extra.
233
+ - `model-call-shape`: the machine changed a call's tools or output format, for example a model-picked action that became `agent.decide`. Port the call unchanged first, prove equivalence, then change it in a separate step whose divergences are `INTENDED`. To keep replaying past a call you changed on purpose, pass `translate(recorded, actual)` to `createRecorder`: it returns the answer that fits the new call (`{ response }`, `{ stream }`, or `{ error }`), or `undefined` to leave that recorded call alone, and `compareRecordings` counts the pair as the same call. When the recorded answer already fits, a `modelCalls` function passed to both is enough: `(expected, actual) => expected.call === 2 || sameModelCallShape(expected, actual)`.
234
+ - When the machine asks the model a different way altogether (another protocol, one structured draft instead of a tool loop), compare what it did, not how it asked: `compareRecordings(a, b, { kinds: ["tool", "output"] })`, with the machine recorded on a stand-in model.
233
235
  - A guard that blocks a rule break diverges on purpose. List it in `INTENDED` with the exact divergence.
234
236
 
235
237
  - Every `regression` row: fix the machine, or, when the old behavior was wrong but broke no rule, ask the user. If the outcomes differ, the answer may become the scenario's `expected`. If only effects differ (the loop asked three times, the machine once), list the row in a reviewed-differences list in the test, with the exact difference asserted and the person who accepted it. Never edit a scenario to make a row pass.