@statelyai/agent 2.0.0-alpha.34 → 2.0.0-alpha.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/dist/testing.cjs +141 -51
- package/dist/testing.d.cts +87 -11
- package/dist/testing.d.mts +87 -11
- package/dist/testing.mjs +141 -51
- package/package.json +1 -1
- package/skills/loop-to-machine/SKILL.md +6 -4
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,18 @@
|
|
|
1
1
|
# @statelyai/agent
|
|
2
2
|
|
|
3
|
+
## 2.0.0-alpha.35
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- [#191](https://github.com/statelyai/agent/pull/191) [`8dcb0c8`](https://github.com/statelyai/agent/commit/8dcb0c8643b1bd38e53538999c3d27c58c4b096f) Thanks [@davidkpiano](https://github.com/davidkpiano)! - **Replay ports that change on purpose.** `createRecorder({ replay })` now answers each model call with the earliest unused recorded call of the same method and shape, and each tool call with the earliest unused recorded call of the same name and input, instead of by position. A call the port added or dropped no longer shifts the answers after it, unless another call has the same shape (or tool name and input): those are answered in recorded order.
|
|
8
|
+
|
|
9
|
+
- `translate(recorded, actual)` adapts a recorded answer to a call you changed on purpose (a stream that became `doGenerate`, text that became a decision). `compareRecordings` counts the pair as the same call.
|
|
10
|
+
- `fallback: { model, openai, tools }` answers what the recording cannot: a stand-in model or client answers the call, and the wrapped tool runs. Those effects are marked `unrecorded` and still count as extra.
|
|
11
|
+
- `compareRecordings(a, b, { kinds: ["tool", "output"] })` compares only what a port did, for a port that calls the model another way.
|
|
12
|
+
- `normalize(effect, index, effects)` can return `undefined` to leave an effect out of the comparison, such as a repeated idempotent write. Replay still answers from it.
|
|
13
|
+
- `recorder.model` wraps a decision model's `doDecide` (and `doEvaluate`). Its shape is the questions, their choice options, and their score levels.
|
|
14
|
+
- A recorded error keeps its number and boolean fields (`statusCode`, `isRetryable`) and its `Symbol.for` markers. In replay, `APICallError.isInstance` and retry behavior hold. Comparisons still check only an error's name, message, and code.
|
|
15
|
+
|
|
3
16
|
## 2.0.0-alpha.34
|
|
4
17
|
|
|
5
18
|
### Patch Changes
|
package/dist/testing.cjs
CHANGED
|
@@ -1145,24 +1145,38 @@ var RecordingDivergenceError = class extends Error {
|
|
|
1145
1145
|
name = "RecordingDivergenceError";
|
|
1146
1146
|
};
|
|
1147
1147
|
function toRecordedError(error) {
|
|
1148
|
-
if (error instanceof Error) {
|
|
1149
|
-
const code = error.code;
|
|
1150
|
-
return code === void 0 ? {
|
|
1151
|
-
name: error.name,
|
|
1152
|
-
message: error.message
|
|
1153
|
-
} : {
|
|
1154
|
-
name: error.name,
|
|
1155
|
-
message: error.message,
|
|
1156
|
-
code: toJson(code)
|
|
1157
|
-
};
|
|
1158
|
-
}
|
|
1159
|
-
return {
|
|
1148
|
+
if (!(error instanceof Error)) return {
|
|
1160
1149
|
name: "Error",
|
|
1161
1150
|
message: String(error)
|
|
1162
1151
|
};
|
|
1152
|
+
const recorded = {
|
|
1153
|
+
name: error.name,
|
|
1154
|
+
message: error.message
|
|
1155
|
+
};
|
|
1156
|
+
const code = error.code;
|
|
1157
|
+
if (code !== void 0) recorded.code = toJson(code);
|
|
1158
|
+
const data = Object.fromEntries(Object.entries(error).filter(([key, value]) => key !== "code" && (typeof value === "number" || typeof value === "boolean")));
|
|
1159
|
+
if (Object.keys(data).length) recorded.data = data;
|
|
1160
|
+
const markers = /* @__PURE__ */ new Set();
|
|
1161
|
+
for (let holder = error; holder; holder = Object.getPrototypeOf(holder)) for (const symbol of Object.getOwnPropertySymbols(holder)) {
|
|
1162
|
+
const key = Symbol.keyFor(symbol);
|
|
1163
|
+
if (key !== void 0 && error[symbol] === true) markers.add(key);
|
|
1164
|
+
}
|
|
1165
|
+
if (markers.size) recorded.markers = [...markers];
|
|
1166
|
+
return recorded;
|
|
1167
|
+
}
|
|
1168
|
+
function fromRecordedError({ data, markers, ...fields }) {
|
|
1169
|
+
const error = Object.assign(new Error(fields.message), fields, data);
|
|
1170
|
+
for (const marker of markers ?? []) error[Symbol.for(marker)] = true;
|
|
1171
|
+
return error;
|
|
1163
1172
|
}
|
|
1164
|
-
|
|
1165
|
-
|
|
1173
|
+
/** What two runs compare of an error: its data and markers may differ. */
|
|
1174
|
+
function errorIdentity(error) {
|
|
1175
|
+
return error && {
|
|
1176
|
+
name: error.name,
|
|
1177
|
+
message: error.message,
|
|
1178
|
+
code: error.code
|
|
1179
|
+
};
|
|
1166
1180
|
}
|
|
1167
1181
|
/** A JSON-safe copy: Dates become ISO strings, Errors become {@link RecordedError}. */
|
|
1168
1182
|
function toJson(value) {
|
|
@@ -1213,19 +1227,42 @@ function createRecorder(options = {}) {
|
|
|
1213
1227
|
const effects = [];
|
|
1214
1228
|
const replay = options.replay?.effects;
|
|
1215
1229
|
const replayedModels = replay?.filter((effect) => effect.kind === "model");
|
|
1230
|
+
const usedModels = /* @__PURE__ */ new Set();
|
|
1216
1231
|
const usedTools = /* @__PURE__ */ new Set();
|
|
1217
|
-
const normalizeEffect = options.normalize
|
|
1232
|
+
const normalizeEffect = (effect, list) => options.normalize?.(effect, list.indexOf(effect), list) ?? effect;
|
|
1218
1233
|
let modelCalls = 0;
|
|
1219
1234
|
let toolsInFlight = 0;
|
|
1235
|
+
/**
|
|
1236
|
+
* The earliest unused recorded call that fits this one: one `translate`
|
|
1237
|
+
* adapts, or one with the same method and shape. Not by position, so a call
|
|
1238
|
+
* the port added or dropped does not shift every answer after it.
|
|
1239
|
+
*/
|
|
1220
1240
|
function nextReplayedModel(effect) {
|
|
1221
|
-
const recorded
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1241
|
+
for (const recorded of replayedModels) {
|
|
1242
|
+
if (usedModels.has(recorded)) continue;
|
|
1243
|
+
const translated = options.translate?.(structuredClone(recorded), effect);
|
|
1244
|
+
if (translated) {
|
|
1245
|
+
effect.translated = true;
|
|
1246
|
+
return {
|
|
1247
|
+
recorded,
|
|
1248
|
+
answer: translated
|
|
1249
|
+
};
|
|
1250
|
+
}
|
|
1251
|
+
if (recorded.method === effect.method && modelCallsMatch(options.modelCalls, recorded, effect)) return {
|
|
1252
|
+
recorded,
|
|
1253
|
+
answer: recorded
|
|
1254
|
+
};
|
|
1255
|
+
}
|
|
1256
|
+
}
|
|
1257
|
+
function noAnswerMessage(effect) {
|
|
1258
|
+
const unused = replayedModels.filter((recorded) => !usedModels.has(recorded));
|
|
1259
|
+
return unused.length ? `Model call #${effect.call} (${effect.method}, ${JSON.stringify(modelCallShape(effect))}) fits no unused recorded call: ${unused.map((recorded) => `#${recorded.call} (${recorded.method}, ${JSON.stringify(modelCallShape(recorded))})`).join(", ")}. A recorded answer only fits a call of the same method and shape; pass \`translate\` to adapt one.` : `Model call #${effect.call} was never made by the recorded run (it made ${replayedModels.length}).`;
|
|
1226
1260
|
}
|
|
1227
|
-
/**
|
|
1228
|
-
|
|
1261
|
+
/**
|
|
1262
|
+
* Records a model call around `run`, or answers it from the recording.
|
|
1263
|
+
* `fallback` runs it on the stand-in when the recording has no answer.
|
|
1264
|
+
*/
|
|
1265
|
+
async function modelCall(seam, method, request, streamed, run, fallback) {
|
|
1229
1266
|
const effect = {
|
|
1230
1267
|
kind: "model",
|
|
1231
1268
|
seam,
|
|
@@ -1239,24 +1276,30 @@ function createRecorder(options = {}) {
|
|
|
1239
1276
|
};
|
|
1240
1277
|
effects.push(effect);
|
|
1241
1278
|
if (replay) {
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
recorded =
|
|
1245
|
-
|
|
1279
|
+
const replayed = nextReplayedModel(effect);
|
|
1280
|
+
if (replayed) {
|
|
1281
|
+
const { recorded, answer } = replayed;
|
|
1282
|
+
usedModels.add(recorded);
|
|
1283
|
+
effect.replayed = recorded.call;
|
|
1284
|
+
if (answer.error) {
|
|
1285
|
+
effect.error = answer.error;
|
|
1286
|
+
throw fromRecordedError(answer.error);
|
|
1287
|
+
}
|
|
1288
|
+
effect.response = answer.response;
|
|
1289
|
+
effect.stream = answer.stream;
|
|
1290
|
+
if (seam === "openai") return streamed ? iterate(answer.stream ?? []) : structuredClone(answer.response);
|
|
1291
|
+
return streamed ? {
|
|
1292
|
+
...reviveTimestamps(structuredClone(answer.response) ?? {}),
|
|
1293
|
+
stream: streamFrom(answer.stream ?? [])
|
|
1294
|
+
} : reviveTimestamps(structuredClone(answer.response));
|
|
1295
|
+
}
|
|
1296
|
+
if (!fallback) {
|
|
1297
|
+
const error = new RecordingDivergenceError(noAnswerMessage(effect));
|
|
1246
1298
|
effect.error = toRecordedError(error);
|
|
1247
1299
|
throw error;
|
|
1248
1300
|
}
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
throw fromRecordedError(recorded.error);
|
|
1252
|
-
}
|
|
1253
|
-
effect.response = recorded.response;
|
|
1254
|
-
effect.stream = recorded.stream;
|
|
1255
|
-
if (seam === "openai") return streamed ? iterate(recorded.stream ?? []) : structuredClone(recorded.response);
|
|
1256
|
-
return streamed ? {
|
|
1257
|
-
...reviveTimestamps(structuredClone(recorded.response) ?? {}),
|
|
1258
|
-
stream: streamFrom(recorded.stream ?? [])
|
|
1259
|
-
} : reviveTimestamps(structuredClone(recorded.response));
|
|
1301
|
+
effect.unrecorded = true;
|
|
1302
|
+
run = fallback;
|
|
1260
1303
|
}
|
|
1261
1304
|
let result;
|
|
1262
1305
|
try {
|
|
@@ -1307,8 +1350,13 @@ function createRecorder(options = {}) {
|
|
|
1307
1350
|
function call(effect, args) {
|
|
1308
1351
|
const { input } = effect;
|
|
1309
1352
|
if (replay && options.tools !== "live") {
|
|
1310
|
-
const wanted = normalizeEffect(effect).input;
|
|
1311
|
-
const
|
|
1353
|
+
const wanted = normalizeEffect(effect, effects).input;
|
|
1354
|
+
const matches = (candidate) => candidate.kind === "tool" && !usedTools.has(candidate) && candidate.name === name && jsonEqual(normalizeEffect(candidate, replay).input, wanted);
|
|
1355
|
+
const recorded = replay.find(matches);
|
|
1356
|
+
if (!recorded && options.fallback?.tools) {
|
|
1357
|
+
effect.unrecorded = true;
|
|
1358
|
+
return runLive(effect, args);
|
|
1359
|
+
}
|
|
1312
1360
|
const settle = (value) => {
|
|
1313
1361
|
if (recorded?.sync) return value();
|
|
1314
1362
|
try {
|
|
@@ -1319,7 +1367,7 @@ function createRecorder(options = {}) {
|
|
|
1319
1367
|
};
|
|
1320
1368
|
return settle(() => {
|
|
1321
1369
|
if (!recorded) {
|
|
1322
|
-
const error = new RecordingDivergenceError(`Tool '${name}' was called with ${JSON.stringify(input)}
|
|
1370
|
+
const error = new RecordingDivergenceError(`Tool '${name}' was called with ${JSON.stringify(input)}; the recorded run made no such call.`);
|
|
1323
1371
|
effect.error = toRecordedError(error);
|
|
1324
1372
|
throw error;
|
|
1325
1373
|
}
|
|
@@ -1333,6 +1381,9 @@ function createRecorder(options = {}) {
|
|
|
1333
1381
|
return structuredClone(recorded.output);
|
|
1334
1382
|
});
|
|
1335
1383
|
}
|
|
1384
|
+
return runLive(effect, args);
|
|
1385
|
+
}
|
|
1386
|
+
function runLive(effect, args) {
|
|
1336
1387
|
let output;
|
|
1337
1388
|
try {
|
|
1338
1389
|
output = fn(...args);
|
|
@@ -1369,9 +1420,18 @@ function createRecorder(options = {}) {
|
|
|
1369
1420
|
return {
|
|
1370
1421
|
model(model) {
|
|
1371
1422
|
const base = model;
|
|
1423
|
+
const stand = options.fallback?.model;
|
|
1372
1424
|
const wrapped = Object.create(base);
|
|
1373
|
-
|
|
1374
|
-
|
|
1425
|
+
for (const method of [
|
|
1426
|
+
"doGenerate",
|
|
1427
|
+
"doStream",
|
|
1428
|
+
"doDecide",
|
|
1429
|
+
"doEvaluate"
|
|
1430
|
+
]) {
|
|
1431
|
+
if (typeof base[method] !== "function") continue;
|
|
1432
|
+
const invoke = (target, callOptions) => target[method].call(target, callOptions);
|
|
1433
|
+
wrapped[method] = (callOptions) => modelCall("ai-sdk", method, callOptions, method === "doStream", () => invoke(base, callOptions), stand && typeof stand[method] === "function" ? () => invoke(stand, callOptions) : void 0);
|
|
1434
|
+
}
|
|
1375
1435
|
return wrapped;
|
|
1376
1436
|
},
|
|
1377
1437
|
openai(client) {
|
|
@@ -1379,10 +1439,12 @@ function createRecorder(options = {}) {
|
|
|
1379
1439
|
if (key !== "create") return Reflect.get(target, key, receiver);
|
|
1380
1440
|
return (params, requestOptions) => {
|
|
1381
1441
|
let api;
|
|
1382
|
-
const
|
|
1383
|
-
api = Reflect.get(
|
|
1442
|
+
const create = (resource) => () => {
|
|
1443
|
+
api = Reflect.get(resource, "create").call(resource, params, requestOptions);
|
|
1384
1444
|
return Promise.resolve(api);
|
|
1385
|
-
}
|
|
1445
|
+
};
|
|
1446
|
+
const stand = fallbackResource(method);
|
|
1447
|
+
const promise = modelCall("openai", method, params, params?.stream === true, create(target), stand && create(stand));
|
|
1386
1448
|
return Object.assign(promise, { withResponse: async () => {
|
|
1387
1449
|
const data = await promise;
|
|
1388
1450
|
const meta = await api?.withResponse?.();
|
|
@@ -1394,6 +1456,11 @@ function createRecorder(options = {}) {
|
|
|
1394
1456
|
} });
|
|
1395
1457
|
};
|
|
1396
1458
|
} });
|
|
1459
|
+
const fallbackResource = (method) => {
|
|
1460
|
+
let value = options.fallback?.openai;
|
|
1461
|
+
for (const key of method.split(".").slice(0, -1)) value = value?.[key];
|
|
1462
|
+
return value && typeof value === "object" ? value : void 0;
|
|
1463
|
+
};
|
|
1397
1464
|
const wrapped = /* @__PURE__ */ new Map();
|
|
1398
1465
|
const wrap = (key, make) => {
|
|
1399
1466
|
if (!wrapped.has(key)) wrapped.set(key, make());
|
|
@@ -1442,9 +1509,27 @@ function modelCallsMatch(match, expected, actual) {
|
|
|
1442
1509
|
function shapeMessage(expected, actual) {
|
|
1443
1510
|
return `Model call #${expected.call} changed shape: ${JSON.stringify(modelCallShape(expected))} → ${JSON.stringify(modelCallShape(actual))}. A recorded answer only fits a call of the same shape.`;
|
|
1444
1511
|
}
|
|
1445
|
-
/**
|
|
1512
|
+
/**
|
|
1513
|
+
* Offered tool names and the response format, or a decision's questions and
|
|
1514
|
+
* their options: what a recorded answer depends on.
|
|
1515
|
+
*/
|
|
1446
1516
|
function modelCallShape(effect) {
|
|
1447
1517
|
const request = effect.request ?? {};
|
|
1518
|
+
const questions = request["questions"];
|
|
1519
|
+
if (questions && typeof questions === "object") return { questions: Object.entries(questions).map(([name, question]) => {
|
|
1520
|
+
const criteria = question["criteria"];
|
|
1521
|
+
if (question["type"] === "choice" && criteria && typeof criteria === "object") return [
|
|
1522
|
+
name,
|
|
1523
|
+
question["type"],
|
|
1524
|
+
Object.keys(criteria).sort()
|
|
1525
|
+
];
|
|
1526
|
+
if (question["type"] === "score" && Array.isArray(criteria)) return [
|
|
1527
|
+
name,
|
|
1528
|
+
question["type"],
|
|
1529
|
+
criteria.length
|
|
1530
|
+
];
|
|
1531
|
+
return [name, question["type"]];
|
|
1532
|
+
}).sort((x, y) => String(x[0]).localeCompare(String(y[0]))) };
|
|
1448
1533
|
return {
|
|
1449
1534
|
tools: (Array.isArray(request["tools"]) ? request["tools"] : []).map((tool) => tool["name"] ?? tool["function"]?.["name"]).sort(),
|
|
1450
1535
|
format: request["responseFormat"]?.["type"] ?? request["response_format"]?.["type"] ?? (request["text"]?.["format"])?.["type"] ?? "text"
|
|
@@ -1485,6 +1570,7 @@ function differ(left, right, options) {
|
|
|
1485
1570
|
message: `Expected ${describe(left)}; the port made ${describe(right)}.`
|
|
1486
1571
|
};
|
|
1487
1572
|
if (left.kind === "model" && right.kind === "model") {
|
|
1573
|
+
if (right.translated && right.replayed === left.call) return void 0;
|
|
1488
1574
|
if (left.method !== right.method) return {
|
|
1489
1575
|
reason: "model-method",
|
|
1490
1576
|
message: `Model call #${left.call} used ${left.method}; the port used ${right.method}.`
|
|
@@ -1493,7 +1579,7 @@ function differ(left, right, options) {
|
|
|
1493
1579
|
reason: "model-call-shape",
|
|
1494
1580
|
message: shapeMessage(left, right)
|
|
1495
1581
|
};
|
|
1496
|
-
if (!jsonEqual(left.error, right.error)) {
|
|
1582
|
+
if (!jsonEqual(errorIdentity(left.error), errorIdentity(right.error))) {
|
|
1497
1583
|
const outcome = (error) => error ? `failed with ${error.name}: ${error.message}` : "succeeded";
|
|
1498
1584
|
return {
|
|
1499
1585
|
reason: "model-error",
|
|
@@ -1511,7 +1597,7 @@ function differ(left, right, options) {
|
|
|
1511
1597
|
reason: "tool-input",
|
|
1512
1598
|
message: `Expected ${describe(left)}; the port passed ${JSON.stringify(right.input)}.`
|
|
1513
1599
|
};
|
|
1514
|
-
if (!jsonEqual(left.output, right.output) || !jsonEqual(left.error, right.error)) return {
|
|
1600
|
+
if (!jsonEqual(left.output, right.output) || !jsonEqual(errorIdentity(left.error), errorIdentity(right.error))) return {
|
|
1515
1601
|
reason: "tool-result",
|
|
1516
1602
|
message: `Tool '${left.name}' returned a different result.`
|
|
1517
1603
|
};
|
|
@@ -1530,9 +1616,13 @@ function differ(left, right, options) {
|
|
|
1530
1616
|
* kind pair up and report what changed, and the rest are missing or extra.
|
|
1531
1617
|
*/
|
|
1532
1618
|
function compareRecordings(expected, actual, options = {}) {
|
|
1533
|
-
const
|
|
1534
|
-
|
|
1535
|
-
|
|
1619
|
+
const select = (effects) => normalize(effects.flatMap((effect, index) => {
|
|
1620
|
+
if (options.kinds && !options.kinds.includes(effect.kind)) return [];
|
|
1621
|
+
const mapped = options.normalize ? options.normalize(effect, index, effects) : effect;
|
|
1622
|
+
return mapped ? [mapped] : [];
|
|
1623
|
+
}));
|
|
1624
|
+
const a = select(expected.effects);
|
|
1625
|
+
const b = select(actual.effects);
|
|
1536
1626
|
const same = a.map((left) => b.map((right) => !differ(left, right, options)));
|
|
1537
1627
|
const lcs = Array.from({ length: a.length + 1 }, () => Array.from({ length: b.length + 1 }, () => 0));
|
|
1538
1628
|
for (let i = a.length - 1; i >= 0; i--) for (let j = b.length - 1; j >= 0; j--) lcs[i][j] = same[i][j] ? lcs[i + 1][j + 1] + 1 : Math.max(lcs[i + 1][j], lcs[i][j + 1]);
|
package/dist/testing.d.cts
CHANGED
|
@@ -847,6 +847,12 @@ export declare function compareRuns<TScenario extends {
|
|
|
847
847
|
* side effects, and records what the port did.
|
|
848
848
|
* 3. `compareRecordings(original, port)` reports the first effect that differs.
|
|
849
849
|
*
|
|
850
|
+
* A port that changes how it calls the model on purpose (a model-picked action
|
|
851
|
+
* that became `agent.decide`, a stream that became `doGenerate`) still replays:
|
|
852
|
+
* `translate` turns the recorded answer into one that fits, `fallback` answers
|
|
853
|
+
* the calls the original never made, and `compareRecordings(…, { kinds })`
|
|
854
|
+
* compares only tool calls and output.
|
|
855
|
+
*
|
|
850
856
|
* Provider-free: both seams are wrapped structurally, so this module imports
|
|
851
857
|
* neither `ai` nor `openai`.
|
|
852
858
|
* @module
|
|
@@ -856,13 +862,20 @@ interface RecordedError {
|
|
|
856
862
|
name: string;
|
|
857
863
|
message: string;
|
|
858
864
|
code?: unknown;
|
|
865
|
+
/** The error's own number and boolean fields, such as `statusCode` and
|
|
866
|
+
* `isRetryable`; replay restores them. */
|
|
867
|
+
data?: Record<string, number | boolean>;
|
|
868
|
+
/** `Symbol.for` keys set to `true` on the error, such as the AI SDK's
|
|
869
|
+
* `vercel.ai.error.AI_APICallError`; replay restores them, so checks like
|
|
870
|
+
* `APICallError.isInstance` still pass. */
|
|
871
|
+
markers?: string[];
|
|
859
872
|
}
|
|
860
873
|
/** One model call. `call` is its 0-based position among model calls. */
|
|
861
874
|
interface RecordedModelCall {
|
|
862
875
|
kind: "model";
|
|
863
876
|
/** Which wrapper saw it. */
|
|
864
877
|
seam: "ai-sdk" | "openai";
|
|
865
|
-
/** `doGenerate`/`doStream`, or `chat.completions.create`/`responses.create`. */
|
|
878
|
+
/** `doGenerate`/`doStream`/`doDecide`, or `chat.completions.create`/`responses.create`. */
|
|
866
879
|
method: string;
|
|
867
880
|
call: number;
|
|
868
881
|
/** The call options or params, JSON-safe (abort signals and headers dropped). */
|
|
@@ -872,6 +885,12 @@ interface RecordedModelCall {
|
|
|
872
885
|
/** The stream's parts or chunks, in order. */
|
|
873
886
|
stream?: unknown[];
|
|
874
887
|
error?: RecordedError;
|
|
888
|
+
/** In replay, the recorded call (its `call`) whose answer served this one. */
|
|
889
|
+
replayed?: number;
|
|
890
|
+
/** In replay, `translate` adapted the recorded answer to this call. */
|
|
891
|
+
translated?: true;
|
|
892
|
+
/** In replay, no recorded call fit, so `fallback` answered it. */
|
|
893
|
+
unrecorded?: true;
|
|
875
894
|
}
|
|
876
895
|
/** One tool call. `step` is the number of model calls made before it. */
|
|
877
896
|
interface RecordedToolCall {
|
|
@@ -886,6 +905,8 @@ interface RecordedToolCall {
|
|
|
886
905
|
/** Started while an earlier tool call was still running. Concurrent calls
|
|
887
906
|
* may finish in either order; sequential ones must keep theirs. */
|
|
888
907
|
concurrent?: true;
|
|
908
|
+
/** In replay, no recorded call matched, so `fallback.tools` ran the tool. */
|
|
909
|
+
unrecorded?: true;
|
|
889
910
|
}
|
|
890
911
|
/** The run's final result. */
|
|
891
912
|
interface RecordedOutput {
|
|
@@ -917,10 +938,53 @@ interface CreateRecorderOptions {
|
|
|
917
938
|
/**
|
|
918
939
|
* In replay, maps the recorded and the new tool call before matching them,
|
|
919
940
|
* like {@link CompareRecordingsOptions.normalize}: redact a generated id in a
|
|
920
|
-
* tool's input so the recorded result still answers it.
|
|
941
|
+
* tool's input so the recorded result still answers it. A dropped
|
|
942
|
+
* (`undefined`) effect matches as recorded.
|
|
943
|
+
*/
|
|
944
|
+
normalize?: NormalizeEffect;
|
|
945
|
+
/**
|
|
946
|
+
* In replay, adapts a recorded answer to a model call you changed on
|
|
947
|
+
* purpose. Called with each unused recorded call, in order, until it
|
|
948
|
+
* returns an answer; that answer serves the call regardless of method and
|
|
949
|
+
* shape, and {@link compareRecordings} counts the pair as the same call.
|
|
950
|
+
* Return `undefined` to leave a recorded call to the usual matching.
|
|
951
|
+
*
|
|
952
|
+
* @example
|
|
953
|
+
* ```ts
|
|
954
|
+
* // The original streamed; the port generates.
|
|
955
|
+
* translate: (recorded, actual) =>
|
|
956
|
+
* actual.method === "doGenerate" && recorded.method === "doStream"
|
|
957
|
+
* ? { response: toGenerateResult(recorded.stream) }
|
|
958
|
+
* : undefined,
|
|
959
|
+
* ```
|
|
921
960
|
*/
|
|
922
|
-
|
|
961
|
+
translate?: (recorded: RecordedModelCall, actual: RecordedModelCall) => ModelAnswer | undefined;
|
|
962
|
+
/**
|
|
963
|
+
* In replay, answers what the recording cannot, instead of throwing
|
|
964
|
+
* {@link RecordingDivergenceError}: a model call no recorded call fits (the
|
|
965
|
+
* port's retry or an added call) goes to `model` (AI SDK seam) or `openai`
|
|
966
|
+
* (client seam), and with `tools: true` a tool call no recorded call matches
|
|
967
|
+
* runs the wrapped tool. Such effects are marked `unrecorded` and still
|
|
968
|
+
* count as extra in {@link compareRecordings}.
|
|
969
|
+
*/
|
|
970
|
+
fallback?: {
|
|
971
|
+
model?: RecordableModel;
|
|
972
|
+
openai?: object;
|
|
973
|
+
tools?: boolean;
|
|
974
|
+
};
|
|
975
|
+
}
|
|
976
|
+
/** A model call's answer: the result, the stream's parts, or an error. */
|
|
977
|
+
interface ModelAnswer {
|
|
978
|
+
response?: unknown;
|
|
979
|
+
stream?: unknown[];
|
|
980
|
+
error?: RecordedError;
|
|
923
981
|
}
|
|
982
|
+
/**
|
|
983
|
+
* Maps an effect before matching or comparing; `undefined` drops it.
|
|
984
|
+
* `index` and `effects` are its position in the whole recording, so a repeat
|
|
985
|
+
* of an earlier effect (an idempotent write made again) can be dropped.
|
|
986
|
+
*/
|
|
987
|
+
type NormalizeEffect = (effect: RecordedEffect, index: number, effects: readonly RecordedEffect[]) => RecordedEffect | undefined;
|
|
924
988
|
/**
|
|
925
989
|
* When two model calls count as the same: `"shape"` compares offered tool
|
|
926
990
|
* names and response format, `"none"` only that a call happened, and a
|
|
@@ -935,15 +999,21 @@ interface RecordableLanguageModel {
|
|
|
935
999
|
stream: ReadableStream<unknown>;
|
|
936
1000
|
}>;
|
|
937
1001
|
}
|
|
1002
|
+
/** The structural slice of an AI SDK decision model (`experimental_decide`). */
|
|
1003
|
+
interface RecordableDecisionModel {
|
|
1004
|
+
doDecide(options: never): PromiseLike<unknown>;
|
|
1005
|
+
}
|
|
1006
|
+
/** An AI SDK model the recorder can wrap. */
|
|
1007
|
+
type RecordableModel = RecordableLanguageModel | RecordableDecisionModel;
|
|
938
1008
|
type AnyFunction = (...args: never[]) => unknown;
|
|
939
1009
|
/** Records (or replays) the effects of one run. */
|
|
940
1010
|
interface Recorder {
|
|
941
1011
|
/**
|
|
942
|
-
* Wraps an AI SDK `LanguageModelV3` (`openai("gpt-…")`, `anthropic(…)`)
|
|
943
|
-
* keeping its type. In replay the model is
|
|
944
|
-
* model built with no API key works.
|
|
1012
|
+
* Wraps an AI SDK `LanguageModelV3` (`openai("gpt-…")`, `anthropic(…)`) or
|
|
1013
|
+
* decision model (`doDecide`), keeping its type. In replay the model is
|
|
1014
|
+
* never called, so a provider model built with no API key works.
|
|
945
1015
|
*/
|
|
946
|
-
model<TModel extends
|
|
1016
|
+
model<TModel extends RecordableModel>(model: TModel): TModel;
|
|
947
1017
|
/**
|
|
948
1018
|
* Wraps an `openai` client's `chat.completions.create` and
|
|
949
1019
|
* `responses.create`, keeping its type. A streamed result is an async
|
|
@@ -1002,10 +1072,16 @@ interface CompareRecordingsOptions {
|
|
|
1002
1072
|
modelCalls?: ModelCallMatch;
|
|
1003
1073
|
/**
|
|
1004
1074
|
* Maps each effect before comparing, on both sides: redact values that
|
|
1005
|
-
* differ on every run (generated ids, timestamps)
|
|
1006
|
-
* to the fields that matter.
|
|
1075
|
+
* differ on every run (generated ids, timestamps), project a tool's input
|
|
1076
|
+
* to the fields that matter, or return `undefined` to drop the effect.
|
|
1077
|
+
*/
|
|
1078
|
+
normalize?: NormalizeEffect;
|
|
1079
|
+
/**
|
|
1080
|
+
* Which kinds of effect to compare. Default all. `["tool", "output"]`
|
|
1081
|
+
* compares a port that calls the model differently (another protocol, a
|
|
1082
|
+
* structured draft instead of a tool loop) by what it did, not how it asked.
|
|
1007
1083
|
*/
|
|
1008
|
-
|
|
1084
|
+
kinds?: readonly RecordedEffect["kind"][];
|
|
1009
1085
|
}
|
|
1010
1086
|
/** True when two model calls offer the same tool names and response format. */
|
|
1011
1087
|
export declare function sameModelCallShape(expected: RecordedModelCall, actual: RecordedModelCall): boolean;
|
|
@@ -1018,4 +1094,4 @@ export declare function sameModelCallShape(expected: RecordedModelCall, actual:
|
|
|
1018
1094
|
*/
|
|
1019
1095
|
export declare function compareRecordings(expected: Recording, actual: Recording, options?: CompareRecordingsOptions): EquivalenceReport;
|
|
1020
1096
|
//#endregion
|
|
1021
|
-
export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AgreementReport, type AgreementRow, type AgreementSummary, type AssertAgentMachineOptions, type CanReachResult, type CanReachTarget, type CompareRecordingsOptions, type CompareRunsOptions, type CoverageReport, type CreateRecorderOptions, type Divergence, type EffectDivergence, type EffectDivergenceReason, type EquivalenceReport, type ExplorePathsOptions, type ExploreSnapshot, type FirstDivergenceOptions, type FoldTracesOptions, type GetDecisionRequestsOptions, type HandlerCoverage, type HandlerRef, type JudgeAgreementOptions, type LintAgentMachineOptions, type MatchTrajectoryOptions, type ModelCallMatch, type ParityLabel, type ParityReport, type ParityResult, type ParityRun, type RecordableLanguageModel, type RecordedEffect, type RecordedError, type RecordedModelCall, type RecordedOutput, type RecordedToolCall, type Recorder, type Recording, type RepeatGroup, type RepeatReport, type RepeatResult, type RepeatRun, type RepeatRunsOptions, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type StateCoverage, type StateStats, type TraceCoverageOptions, type TraceStep, type TracedDecisionRequest, type TrajectoryEvent, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, explorePaths, lintAgentMachine, simulateAgent };
|
|
1097
|
+
export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AgreementReport, type AgreementRow, type AgreementSummary, type AssertAgentMachineOptions, type CanReachResult, type CanReachTarget, type CompareRecordingsOptions, type CompareRunsOptions, type CoverageReport, type CreateRecorderOptions, type Divergence, type EffectDivergence, type EffectDivergenceReason, type EquivalenceReport, type ExplorePathsOptions, type ExploreSnapshot, type FirstDivergenceOptions, type FoldTracesOptions, type GetDecisionRequestsOptions, type HandlerCoverage, type HandlerRef, type JudgeAgreementOptions, type LintAgentMachineOptions, type MatchTrajectoryOptions, type ModelAnswer, type ModelCallMatch, type NormalizeEffect, type ParityLabel, type ParityReport, type ParityResult, type ParityRun, type RecordableDecisionModel, type RecordableLanguageModel, type RecordableModel, type RecordedEffect, type RecordedError, type RecordedModelCall, type RecordedOutput, type RecordedToolCall, type Recorder, type Recording, type RepeatGroup, type RepeatReport, type RepeatResult, type RepeatRun, type RepeatRunsOptions, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type StateCoverage, type StateStats, type TraceCoverageOptions, type TraceStep, type TracedDecisionRequest, type TrajectoryEvent, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, explorePaths, lintAgentMachine, simulateAgent };
|
package/dist/testing.d.mts
CHANGED
|
@@ -847,6 +847,12 @@ export declare function compareRuns<TScenario extends {
|
|
|
847
847
|
* side effects, and records what the port did.
|
|
848
848
|
* 3. `compareRecordings(original, port)` reports the first effect that differs.
|
|
849
849
|
*
|
|
850
|
+
* A port that changes how it calls the model on purpose (a model-picked action
|
|
851
|
+
* that became `agent.decide`, a stream that became `doGenerate`) still replays:
|
|
852
|
+
* `translate` turns the recorded answer into one that fits, `fallback` answers
|
|
853
|
+
* the calls the original never made, and `compareRecordings(…, { kinds })`
|
|
854
|
+
* compares only tool calls and output.
|
|
855
|
+
*
|
|
850
856
|
* Provider-free: both seams are wrapped structurally, so this module imports
|
|
851
857
|
* neither `ai` nor `openai`.
|
|
852
858
|
* @module
|
|
@@ -856,13 +862,20 @@ interface RecordedError {
|
|
|
856
862
|
name: string;
|
|
857
863
|
message: string;
|
|
858
864
|
code?: unknown;
|
|
865
|
+
/** The error's own number and boolean fields, such as `statusCode` and
|
|
866
|
+
* `isRetryable`; replay restores them. */
|
|
867
|
+
data?: Record<string, number | boolean>;
|
|
868
|
+
/** `Symbol.for` keys set to `true` on the error, such as the AI SDK's
|
|
869
|
+
* `vercel.ai.error.AI_APICallError`; replay restores them, so checks like
|
|
870
|
+
* `APICallError.isInstance` still pass. */
|
|
871
|
+
markers?: string[];
|
|
859
872
|
}
|
|
860
873
|
/** One model call. `call` is its 0-based position among model calls. */
|
|
861
874
|
interface RecordedModelCall {
|
|
862
875
|
kind: "model";
|
|
863
876
|
/** Which wrapper saw it. */
|
|
864
877
|
seam: "ai-sdk" | "openai";
|
|
865
|
-
/** `doGenerate`/`doStream`, or `chat.completions.create`/`responses.create`. */
|
|
878
|
+
/** `doGenerate`/`doStream`/`doDecide`, or `chat.completions.create`/`responses.create`. */
|
|
866
879
|
method: string;
|
|
867
880
|
call: number;
|
|
868
881
|
/** The call options or params, JSON-safe (abort signals and headers dropped). */
|
|
@@ -872,6 +885,12 @@ interface RecordedModelCall {
|
|
|
872
885
|
/** The stream's parts or chunks, in order. */
|
|
873
886
|
stream?: unknown[];
|
|
874
887
|
error?: RecordedError;
|
|
888
|
+
/** In replay, the recorded call (its `call`) whose answer served this one. */
|
|
889
|
+
replayed?: number;
|
|
890
|
+
/** In replay, `translate` adapted the recorded answer to this call. */
|
|
891
|
+
translated?: true;
|
|
892
|
+
/** In replay, no recorded call fit, so `fallback` answered it. */
|
|
893
|
+
unrecorded?: true;
|
|
875
894
|
}
|
|
876
895
|
/** One tool call. `step` is the number of model calls made before it. */
|
|
877
896
|
interface RecordedToolCall {
|
|
@@ -886,6 +905,8 @@ interface RecordedToolCall {
|
|
|
886
905
|
/** Started while an earlier tool call was still running. Concurrent calls
|
|
887
906
|
* may finish in either order; sequential ones must keep theirs. */
|
|
888
907
|
concurrent?: true;
|
|
908
|
+
/** In replay, no recorded call matched, so `fallback.tools` ran the tool. */
|
|
909
|
+
unrecorded?: true;
|
|
889
910
|
}
|
|
890
911
|
/** The run's final result. */
|
|
891
912
|
interface RecordedOutput {
|
|
@@ -917,10 +938,53 @@ interface CreateRecorderOptions {
|
|
|
917
938
|
/**
|
|
918
939
|
* In replay, maps the recorded and the new tool call before matching them,
|
|
919
940
|
* like {@link CompareRecordingsOptions.normalize}: redact a generated id in a
|
|
920
|
-
* tool's input so the recorded result still answers it.
|
|
941
|
+
* tool's input so the recorded result still answers it. A dropped
|
|
942
|
+
* (`undefined`) effect matches as recorded.
|
|
943
|
+
*/
|
|
944
|
+
normalize?: NormalizeEffect;
|
|
945
|
+
/**
|
|
946
|
+
* In replay, adapts a recorded answer to a model call you changed on
|
|
947
|
+
* purpose. Called with each unused recorded call, in order, until it
|
|
948
|
+
* returns an answer; that answer serves the call regardless of method and
|
|
949
|
+
* shape, and {@link compareRecordings} counts the pair as the same call.
|
|
950
|
+
* Return `undefined` to leave a recorded call to the usual matching.
|
|
951
|
+
*
|
|
952
|
+
* @example
|
|
953
|
+
* ```ts
|
|
954
|
+
* // The original streamed; the port generates.
|
|
955
|
+
* translate: (recorded, actual) =>
|
|
956
|
+
* actual.method === "doGenerate" && recorded.method === "doStream"
|
|
957
|
+
* ? { response: toGenerateResult(recorded.stream) }
|
|
958
|
+
* : undefined,
|
|
959
|
+
* ```
|
|
921
960
|
*/
|
|
922
|
-
|
|
961
|
+
translate?: (recorded: RecordedModelCall, actual: RecordedModelCall) => ModelAnswer | undefined;
|
|
962
|
+
/**
|
|
963
|
+
* In replay, answers what the recording cannot, instead of throwing
|
|
964
|
+
* {@link RecordingDivergenceError}: a model call no recorded call fits (the
|
|
965
|
+
* port's retry or an added call) goes to `model` (AI SDK seam) or `openai`
|
|
966
|
+
* (client seam), and with `tools: true` a tool call no recorded call matches
|
|
967
|
+
* runs the wrapped tool. Such effects are marked `unrecorded` and still
|
|
968
|
+
* count as extra in {@link compareRecordings}.
|
|
969
|
+
*/
|
|
970
|
+
fallback?: {
|
|
971
|
+
model?: RecordableModel;
|
|
972
|
+
openai?: object;
|
|
973
|
+
tools?: boolean;
|
|
974
|
+
};
|
|
975
|
+
}
|
|
976
|
+
/** A model call's answer: the result, the stream's parts, or an error. */
|
|
977
|
+
interface ModelAnswer {
|
|
978
|
+
response?: unknown;
|
|
979
|
+
stream?: unknown[];
|
|
980
|
+
error?: RecordedError;
|
|
923
981
|
}
|
|
982
|
+
/**
|
|
983
|
+
* Maps an effect before matching or comparing; `undefined` drops it.
|
|
984
|
+
* `index` and `effects` are its position in the whole recording, so a repeat
|
|
985
|
+
* of an earlier effect (an idempotent write made again) can be dropped.
|
|
986
|
+
*/
|
|
987
|
+
type NormalizeEffect = (effect: RecordedEffect, index: number, effects: readonly RecordedEffect[]) => RecordedEffect | undefined;
|
|
924
988
|
/**
|
|
925
989
|
* When two model calls count as the same: `"shape"` compares offered tool
|
|
926
990
|
* names and response format, `"none"` only that a call happened, and a
|
|
@@ -935,15 +999,21 @@ interface RecordableLanguageModel {
|
|
|
935
999
|
stream: ReadableStream<unknown>;
|
|
936
1000
|
}>;
|
|
937
1001
|
}
|
|
1002
|
+
/** The structural slice of an AI SDK decision model (`experimental_decide`). */
|
|
1003
|
+
interface RecordableDecisionModel {
|
|
1004
|
+
doDecide(options: never): PromiseLike<unknown>;
|
|
1005
|
+
}
|
|
1006
|
+
/** An AI SDK model the recorder can wrap. */
|
|
1007
|
+
type RecordableModel = RecordableLanguageModel | RecordableDecisionModel;
|
|
938
1008
|
type AnyFunction = (...args: never[]) => unknown;
|
|
939
1009
|
/** Records (or replays) the effects of one run. */
|
|
940
1010
|
interface Recorder {
|
|
941
1011
|
/**
|
|
942
|
-
* Wraps an AI SDK `LanguageModelV3` (`openai("gpt-…")`, `anthropic(…)`)
|
|
943
|
-
* keeping its type. In replay the model is
|
|
944
|
-
* model built with no API key works.
|
|
1012
|
+
* Wraps an AI SDK `LanguageModelV3` (`openai("gpt-…")`, `anthropic(…)`) or
|
|
1013
|
+
* decision model (`doDecide`), keeping its type. In replay the model is
|
|
1014
|
+
* never called, so a provider model built with no API key works.
|
|
945
1015
|
*/
|
|
946
|
-
model<TModel extends
|
|
1016
|
+
model<TModel extends RecordableModel>(model: TModel): TModel;
|
|
947
1017
|
/**
|
|
948
1018
|
* Wraps an `openai` client's `chat.completions.create` and
|
|
949
1019
|
* `responses.create`, keeping its type. A streamed result is an async
|
|
@@ -1002,10 +1072,16 @@ interface CompareRecordingsOptions {
|
|
|
1002
1072
|
modelCalls?: ModelCallMatch;
|
|
1003
1073
|
/**
|
|
1004
1074
|
* Maps each effect before comparing, on both sides: redact values that
|
|
1005
|
-
* differ on every run (generated ids, timestamps)
|
|
1006
|
-
* to the fields that matter.
|
|
1075
|
+
* differ on every run (generated ids, timestamps), project a tool's input
|
|
1076
|
+
* to the fields that matter, or return `undefined` to drop the effect.
|
|
1077
|
+
*/
|
|
1078
|
+
normalize?: NormalizeEffect;
|
|
1079
|
+
/**
|
|
1080
|
+
* Which kinds of effect to compare. Default all. `["tool", "output"]`
|
|
1081
|
+
* compares a port that calls the model differently (another protocol, a
|
|
1082
|
+
* structured draft instead of a tool loop) by what it did, not how it asked.
|
|
1007
1083
|
*/
|
|
1008
|
-
|
|
1084
|
+
kinds?: readonly RecordedEffect["kind"][];
|
|
1009
1085
|
}
|
|
1010
1086
|
/** True when two model calls offer the same tool names and response format. */
|
|
1011
1087
|
export declare function sameModelCallShape(expected: RecordedModelCall, actual: RecordedModelCall): boolean;
|
|
@@ -1018,4 +1094,4 @@ export declare function sameModelCallShape(expected: RecordedModelCall, actual:
|
|
|
1018
1094
|
*/
|
|
1019
1095
|
export declare function compareRecordings(expected: Recording, actual: Recording, options?: CompareRecordingsOptions): EquivalenceReport;
|
|
1020
1096
|
//#endregion
|
|
1021
|
-
export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AgreementReport, type AgreementRow, type AgreementSummary, type AssertAgentMachineOptions, type CanReachResult, type CanReachTarget, type CompareRecordingsOptions, type CompareRunsOptions, type CoverageReport, type CreateRecorderOptions, type Divergence, type EffectDivergence, type EffectDivergenceReason, type EquivalenceReport, type ExplorePathsOptions, type ExploreSnapshot, type FirstDivergenceOptions, type FoldTracesOptions, type GetDecisionRequestsOptions, type HandlerCoverage, type HandlerRef, type JudgeAgreementOptions, type LintAgentMachineOptions, type MatchTrajectoryOptions, type ModelCallMatch, type ParityLabel, type ParityReport, type ParityResult, type ParityRun, type RecordableLanguageModel, type RecordedEffect, type RecordedError, type RecordedModelCall, type RecordedOutput, type RecordedToolCall, type Recorder, type Recording, type RepeatGroup, type RepeatReport, type RepeatResult, type RepeatRun, type RepeatRunsOptions, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type StateCoverage, type StateStats, type TraceCoverageOptions, type TraceStep, type TracedDecisionRequest, type TrajectoryEvent, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, explorePaths, lintAgentMachine, simulateAgent };
|
|
1097
|
+
export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AgreementReport, type AgreementRow, type AgreementSummary, type AssertAgentMachineOptions, type CanReachResult, type CanReachTarget, type CompareRecordingsOptions, type CompareRunsOptions, type CoverageReport, type CreateRecorderOptions, type Divergence, type EffectDivergence, type EffectDivergenceReason, type EquivalenceReport, type ExplorePathsOptions, type ExploreSnapshot, type FirstDivergenceOptions, type FoldTracesOptions, type GetDecisionRequestsOptions, type HandlerCoverage, type HandlerRef, type JudgeAgreementOptions, type LintAgentMachineOptions, type MatchTrajectoryOptions, type ModelAnswer, type ModelCallMatch, type NormalizeEffect, type ParityLabel, type ParityReport, type ParityResult, type ParityRun, type RecordableDecisionModel, type RecordableLanguageModel, type RecordableModel, type RecordedEffect, type RecordedError, type RecordedModelCall, type RecordedOutput, type RecordedToolCall, type Recorder, type Recording, type RepeatGroup, type RepeatReport, type RepeatResult, type RepeatRun, type RepeatRunsOptions, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type StateCoverage, type StateStats, type TraceCoverageOptions, type TraceStep, type TracedDecisionRequest, type TrajectoryEvent, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, explorePaths, lintAgentMachine, simulateAgent };
|
package/dist/testing.mjs
CHANGED
|
@@ -1144,24 +1144,38 @@ var RecordingDivergenceError = class extends Error {
|
|
|
1144
1144
|
name = "RecordingDivergenceError";
|
|
1145
1145
|
};
|
|
1146
1146
|
function toRecordedError(error) {
|
|
1147
|
-
if (error instanceof Error) {
|
|
1148
|
-
const code = error.code;
|
|
1149
|
-
return code === void 0 ? {
|
|
1150
|
-
name: error.name,
|
|
1151
|
-
message: error.message
|
|
1152
|
-
} : {
|
|
1153
|
-
name: error.name,
|
|
1154
|
-
message: error.message,
|
|
1155
|
-
code: toJson(code)
|
|
1156
|
-
};
|
|
1157
|
-
}
|
|
1158
|
-
return {
|
|
1147
|
+
if (!(error instanceof Error)) return {
|
|
1159
1148
|
name: "Error",
|
|
1160
1149
|
message: String(error)
|
|
1161
1150
|
};
|
|
1151
|
+
const recorded = {
|
|
1152
|
+
name: error.name,
|
|
1153
|
+
message: error.message
|
|
1154
|
+
};
|
|
1155
|
+
const code = error.code;
|
|
1156
|
+
if (code !== void 0) recorded.code = toJson(code);
|
|
1157
|
+
const data = Object.fromEntries(Object.entries(error).filter(([key, value]) => key !== "code" && (typeof value === "number" || typeof value === "boolean")));
|
|
1158
|
+
if (Object.keys(data).length) recorded.data = data;
|
|
1159
|
+
const markers = /* @__PURE__ */ new Set();
|
|
1160
|
+
for (let holder = error; holder; holder = Object.getPrototypeOf(holder)) for (const symbol of Object.getOwnPropertySymbols(holder)) {
|
|
1161
|
+
const key = Symbol.keyFor(symbol);
|
|
1162
|
+
if (key !== void 0 && error[symbol] === true) markers.add(key);
|
|
1163
|
+
}
|
|
1164
|
+
if (markers.size) recorded.markers = [...markers];
|
|
1165
|
+
return recorded;
|
|
1166
|
+
}
|
|
1167
|
+
function fromRecordedError({ data, markers, ...fields }) {
|
|
1168
|
+
const error = Object.assign(new Error(fields.message), fields, data);
|
|
1169
|
+
for (const marker of markers ?? []) error[Symbol.for(marker)] = true;
|
|
1170
|
+
return error;
|
|
1162
1171
|
}
|
|
1163
|
-
|
|
1164
|
-
|
|
1172
|
+
/** What two runs compare of an error: its data and markers may differ. */
|
|
1173
|
+
function errorIdentity(error) {
|
|
1174
|
+
return error && {
|
|
1175
|
+
name: error.name,
|
|
1176
|
+
message: error.message,
|
|
1177
|
+
code: error.code
|
|
1178
|
+
};
|
|
1165
1179
|
}
|
|
1166
1180
|
/** A JSON-safe copy: Dates become ISO strings, Errors become {@link RecordedError}. */
|
|
1167
1181
|
function toJson(value) {
|
|
@@ -1212,19 +1226,42 @@ function createRecorder(options = {}) {
|
|
|
1212
1226
|
const effects = [];
|
|
1213
1227
|
const replay = options.replay?.effects;
|
|
1214
1228
|
const replayedModels = replay?.filter((effect) => effect.kind === "model");
|
|
1229
|
+
const usedModels = /* @__PURE__ */ new Set();
|
|
1215
1230
|
const usedTools = /* @__PURE__ */ new Set();
|
|
1216
|
-
const normalizeEffect = options.normalize
|
|
1231
|
+
const normalizeEffect = (effect, list) => options.normalize?.(effect, list.indexOf(effect), list) ?? effect;
|
|
1217
1232
|
let modelCalls = 0;
|
|
1218
1233
|
let toolsInFlight = 0;
|
|
1234
|
+
/**
|
|
1235
|
+
* The earliest unused recorded call that fits this one: one `translate`
|
|
1236
|
+
* adapts, or one with the same method and shape. Not by position, so a call
|
|
1237
|
+
* the port added or dropped does not shift every answer after it.
|
|
1238
|
+
*/
|
|
1219
1239
|
function nextReplayedModel(effect) {
|
|
1220
|
-
const recorded
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1240
|
+
for (const recorded of replayedModels) {
|
|
1241
|
+
if (usedModels.has(recorded)) continue;
|
|
1242
|
+
const translated = options.translate?.(structuredClone(recorded), effect);
|
|
1243
|
+
if (translated) {
|
|
1244
|
+
effect.translated = true;
|
|
1245
|
+
return {
|
|
1246
|
+
recorded,
|
|
1247
|
+
answer: translated
|
|
1248
|
+
};
|
|
1249
|
+
}
|
|
1250
|
+
if (recorded.method === effect.method && modelCallsMatch(options.modelCalls, recorded, effect)) return {
|
|
1251
|
+
recorded,
|
|
1252
|
+
answer: recorded
|
|
1253
|
+
};
|
|
1254
|
+
}
|
|
1255
|
+
}
|
|
1256
|
+
function noAnswerMessage(effect) {
|
|
1257
|
+
const unused = replayedModels.filter((recorded) => !usedModels.has(recorded));
|
|
1258
|
+
return unused.length ? `Model call #${effect.call} (${effect.method}, ${JSON.stringify(modelCallShape(effect))}) fits no unused recorded call: ${unused.map((recorded) => `#${recorded.call} (${recorded.method}, ${JSON.stringify(modelCallShape(recorded))})`).join(", ")}. A recorded answer only fits a call of the same method and shape; pass \`translate\` to adapt one.` : `Model call #${effect.call} was never made by the recorded run (it made ${replayedModels.length}).`;
|
|
1225
1259
|
}
|
|
1226
|
-
/**
|
|
1227
|
-
|
|
1260
|
+
/**
|
|
1261
|
+
* Records a model call around `run`, or answers it from the recording.
|
|
1262
|
+
* `fallback` runs it on the stand-in when the recording has no answer.
|
|
1263
|
+
*/
|
|
1264
|
+
async function modelCall(seam, method, request, streamed, run, fallback) {
|
|
1228
1265
|
const effect = {
|
|
1229
1266
|
kind: "model",
|
|
1230
1267
|
seam,
|
|
@@ -1238,24 +1275,30 @@ function createRecorder(options = {}) {
|
|
|
1238
1275
|
};
|
|
1239
1276
|
effects.push(effect);
|
|
1240
1277
|
if (replay) {
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
recorded =
|
|
1244
|
-
|
|
1278
|
+
const replayed = nextReplayedModel(effect);
|
|
1279
|
+
if (replayed) {
|
|
1280
|
+
const { recorded, answer } = replayed;
|
|
1281
|
+
usedModels.add(recorded);
|
|
1282
|
+
effect.replayed = recorded.call;
|
|
1283
|
+
if (answer.error) {
|
|
1284
|
+
effect.error = answer.error;
|
|
1285
|
+
throw fromRecordedError(answer.error);
|
|
1286
|
+
}
|
|
1287
|
+
effect.response = answer.response;
|
|
1288
|
+
effect.stream = answer.stream;
|
|
1289
|
+
if (seam === "openai") return streamed ? iterate(answer.stream ?? []) : structuredClone(answer.response);
|
|
1290
|
+
return streamed ? {
|
|
1291
|
+
...reviveTimestamps(structuredClone(answer.response) ?? {}),
|
|
1292
|
+
stream: streamFrom(answer.stream ?? [])
|
|
1293
|
+
} : reviveTimestamps(structuredClone(answer.response));
|
|
1294
|
+
}
|
|
1295
|
+
if (!fallback) {
|
|
1296
|
+
const error = new RecordingDivergenceError(noAnswerMessage(effect));
|
|
1245
1297
|
effect.error = toRecordedError(error);
|
|
1246
1298
|
throw error;
|
|
1247
1299
|
}
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
throw fromRecordedError(recorded.error);
|
|
1251
|
-
}
|
|
1252
|
-
effect.response = recorded.response;
|
|
1253
|
-
effect.stream = recorded.stream;
|
|
1254
|
-
if (seam === "openai") return streamed ? iterate(recorded.stream ?? []) : structuredClone(recorded.response);
|
|
1255
|
-
return streamed ? {
|
|
1256
|
-
...reviveTimestamps(structuredClone(recorded.response) ?? {}),
|
|
1257
|
-
stream: streamFrom(recorded.stream ?? [])
|
|
1258
|
-
} : reviveTimestamps(structuredClone(recorded.response));
|
|
1300
|
+
effect.unrecorded = true;
|
|
1301
|
+
run = fallback;
|
|
1259
1302
|
}
|
|
1260
1303
|
let result;
|
|
1261
1304
|
try {
|
|
@@ -1306,8 +1349,13 @@ function createRecorder(options = {}) {
|
|
|
1306
1349
|
function call(effect, args) {
|
|
1307
1350
|
const { input } = effect;
|
|
1308
1351
|
if (replay && options.tools !== "live") {
|
|
1309
|
-
const wanted = normalizeEffect(effect).input;
|
|
1310
|
-
const
|
|
1352
|
+
const wanted = normalizeEffect(effect, effects).input;
|
|
1353
|
+
const matches = (candidate) => candidate.kind === "tool" && !usedTools.has(candidate) && candidate.name === name && jsonEqual(normalizeEffect(candidate, replay).input, wanted);
|
|
1354
|
+
const recorded = replay.find(matches);
|
|
1355
|
+
if (!recorded && options.fallback?.tools) {
|
|
1356
|
+
effect.unrecorded = true;
|
|
1357
|
+
return runLive(effect, args);
|
|
1358
|
+
}
|
|
1311
1359
|
const settle = (value) => {
|
|
1312
1360
|
if (recorded?.sync) return value();
|
|
1313
1361
|
try {
|
|
@@ -1318,7 +1366,7 @@ function createRecorder(options = {}) {
|
|
|
1318
1366
|
};
|
|
1319
1367
|
return settle(() => {
|
|
1320
1368
|
if (!recorded) {
|
|
1321
|
-
const error = new RecordingDivergenceError(`Tool '${name}' was called with ${JSON.stringify(input)}
|
|
1369
|
+
const error = new RecordingDivergenceError(`Tool '${name}' was called with ${JSON.stringify(input)}; the recorded run made no such call.`);
|
|
1322
1370
|
effect.error = toRecordedError(error);
|
|
1323
1371
|
throw error;
|
|
1324
1372
|
}
|
|
@@ -1332,6 +1380,9 @@ function createRecorder(options = {}) {
|
|
|
1332
1380
|
return structuredClone(recorded.output);
|
|
1333
1381
|
});
|
|
1334
1382
|
}
|
|
1383
|
+
return runLive(effect, args);
|
|
1384
|
+
}
|
|
1385
|
+
function runLive(effect, args) {
|
|
1335
1386
|
let output;
|
|
1336
1387
|
try {
|
|
1337
1388
|
output = fn(...args);
|
|
@@ -1368,9 +1419,18 @@ function createRecorder(options = {}) {
|
|
|
1368
1419
|
return {
|
|
1369
1420
|
model(model) {
|
|
1370
1421
|
const base = model;
|
|
1422
|
+
const stand = options.fallback?.model;
|
|
1371
1423
|
const wrapped = Object.create(base);
|
|
1372
|
-
|
|
1373
|
-
|
|
1424
|
+
for (const method of [
|
|
1425
|
+
"doGenerate",
|
|
1426
|
+
"doStream",
|
|
1427
|
+
"doDecide",
|
|
1428
|
+
"doEvaluate"
|
|
1429
|
+
]) {
|
|
1430
|
+
if (typeof base[method] !== "function") continue;
|
|
1431
|
+
const invoke = (target, callOptions) => target[method].call(target, callOptions);
|
|
1432
|
+
wrapped[method] = (callOptions) => modelCall("ai-sdk", method, callOptions, method === "doStream", () => invoke(base, callOptions), stand && typeof stand[method] === "function" ? () => invoke(stand, callOptions) : void 0);
|
|
1433
|
+
}
|
|
1374
1434
|
return wrapped;
|
|
1375
1435
|
},
|
|
1376
1436
|
openai(client) {
|
|
@@ -1378,10 +1438,12 @@ function createRecorder(options = {}) {
|
|
|
1378
1438
|
if (key !== "create") return Reflect.get(target, key, receiver);
|
|
1379
1439
|
return (params, requestOptions) => {
|
|
1380
1440
|
let api;
|
|
1381
|
-
const
|
|
1382
|
-
api = Reflect.get(
|
|
1441
|
+
const create = (resource) => () => {
|
|
1442
|
+
api = Reflect.get(resource, "create").call(resource, params, requestOptions);
|
|
1383
1443
|
return Promise.resolve(api);
|
|
1384
|
-
}
|
|
1444
|
+
};
|
|
1445
|
+
const stand = fallbackResource(method);
|
|
1446
|
+
const promise = modelCall("openai", method, params, params?.stream === true, create(target), stand && create(stand));
|
|
1385
1447
|
return Object.assign(promise, { withResponse: async () => {
|
|
1386
1448
|
const data = await promise;
|
|
1387
1449
|
const meta = await api?.withResponse?.();
|
|
@@ -1393,6 +1455,11 @@ function createRecorder(options = {}) {
|
|
|
1393
1455
|
} });
|
|
1394
1456
|
};
|
|
1395
1457
|
} });
|
|
1458
|
+
const fallbackResource = (method) => {
|
|
1459
|
+
let value = options.fallback?.openai;
|
|
1460
|
+
for (const key of method.split(".").slice(0, -1)) value = value?.[key];
|
|
1461
|
+
return value && typeof value === "object" ? value : void 0;
|
|
1462
|
+
};
|
|
1396
1463
|
const wrapped = /* @__PURE__ */ new Map();
|
|
1397
1464
|
const wrap = (key, make) => {
|
|
1398
1465
|
if (!wrapped.has(key)) wrapped.set(key, make());
|
|
@@ -1441,9 +1508,27 @@ function modelCallsMatch(match, expected, actual) {
|
|
|
1441
1508
|
function shapeMessage(expected, actual) {
|
|
1442
1509
|
return `Model call #${expected.call} changed shape: ${JSON.stringify(modelCallShape(expected))} → ${JSON.stringify(modelCallShape(actual))}. A recorded answer only fits a call of the same shape.`;
|
|
1443
1510
|
}
|
|
1444
|
-
/**
|
|
1511
|
+
/**
|
|
1512
|
+
* Offered tool names and the response format, or a decision's questions and
|
|
1513
|
+
* their options: what a recorded answer depends on.
|
|
1514
|
+
*/
|
|
1445
1515
|
function modelCallShape(effect) {
|
|
1446
1516
|
const request = effect.request ?? {};
|
|
1517
|
+
const questions = request["questions"];
|
|
1518
|
+
if (questions && typeof questions === "object") return { questions: Object.entries(questions).map(([name, question]) => {
|
|
1519
|
+
const criteria = question["criteria"];
|
|
1520
|
+
if (question["type"] === "choice" && criteria && typeof criteria === "object") return [
|
|
1521
|
+
name,
|
|
1522
|
+
question["type"],
|
|
1523
|
+
Object.keys(criteria).sort()
|
|
1524
|
+
];
|
|
1525
|
+
if (question["type"] === "score" && Array.isArray(criteria)) return [
|
|
1526
|
+
name,
|
|
1527
|
+
question["type"],
|
|
1528
|
+
criteria.length
|
|
1529
|
+
];
|
|
1530
|
+
return [name, question["type"]];
|
|
1531
|
+
}).sort((x, y) => String(x[0]).localeCompare(String(y[0]))) };
|
|
1447
1532
|
return {
|
|
1448
1533
|
tools: (Array.isArray(request["tools"]) ? request["tools"] : []).map((tool) => tool["name"] ?? tool["function"]?.["name"]).sort(),
|
|
1449
1534
|
format: request["responseFormat"]?.["type"] ?? request["response_format"]?.["type"] ?? (request["text"]?.["format"])?.["type"] ?? "text"
|
|
@@ -1484,6 +1569,7 @@ function differ(left, right, options) {
|
|
|
1484
1569
|
message: `Expected ${describe(left)}; the port made ${describe(right)}.`
|
|
1485
1570
|
};
|
|
1486
1571
|
if (left.kind === "model" && right.kind === "model") {
|
|
1572
|
+
if (right.translated && right.replayed === left.call) return void 0;
|
|
1487
1573
|
if (left.method !== right.method) return {
|
|
1488
1574
|
reason: "model-method",
|
|
1489
1575
|
message: `Model call #${left.call} used ${left.method}; the port used ${right.method}.`
|
|
@@ -1492,7 +1578,7 @@ function differ(left, right, options) {
|
|
|
1492
1578
|
reason: "model-call-shape",
|
|
1493
1579
|
message: shapeMessage(left, right)
|
|
1494
1580
|
};
|
|
1495
|
-
if (!jsonEqual(left.error, right.error)) {
|
|
1581
|
+
if (!jsonEqual(errorIdentity(left.error), errorIdentity(right.error))) {
|
|
1496
1582
|
const outcome = (error) => error ? `failed with ${error.name}: ${error.message}` : "succeeded";
|
|
1497
1583
|
return {
|
|
1498
1584
|
reason: "model-error",
|
|
@@ -1510,7 +1596,7 @@ function differ(left, right, options) {
|
|
|
1510
1596
|
reason: "tool-input",
|
|
1511
1597
|
message: `Expected ${describe(left)}; the port passed ${JSON.stringify(right.input)}.`
|
|
1512
1598
|
};
|
|
1513
|
-
if (!jsonEqual(left.output, right.output) || !jsonEqual(left.error, right.error)) return {
|
|
1599
|
+
if (!jsonEqual(left.output, right.output) || !jsonEqual(errorIdentity(left.error), errorIdentity(right.error))) return {
|
|
1514
1600
|
reason: "tool-result",
|
|
1515
1601
|
message: `Tool '${left.name}' returned a different result.`
|
|
1516
1602
|
};
|
|
@@ -1529,9 +1615,13 @@ function differ(left, right, options) {
|
|
|
1529
1615
|
* kind pair up and report what changed, and the rest are missing or extra.
|
|
1530
1616
|
*/
|
|
1531
1617
|
function compareRecordings(expected, actual, options = {}) {
|
|
1532
|
-
const
|
|
1533
|
-
|
|
1534
|
-
|
|
1618
|
+
const select = (effects) => normalize(effects.flatMap((effect, index) => {
|
|
1619
|
+
if (options.kinds && !options.kinds.includes(effect.kind)) return [];
|
|
1620
|
+
const mapped = options.normalize ? options.normalize(effect, index, effects) : effect;
|
|
1621
|
+
return mapped ? [mapped] : [];
|
|
1622
|
+
}));
|
|
1623
|
+
const a = select(expected.effects);
|
|
1624
|
+
const b = select(actual.effects);
|
|
1535
1625
|
const same = a.map((left) => b.map((right) => !differ(left, right, options)));
|
|
1536
1626
|
const lcs = Array.from({ length: a.length + 1 }, () => Array.from({ length: b.length + 1 }, () => 0));
|
|
1537
1627
|
for (let i = a.length - 1; i >= 0; i--) for (let j = b.length - 1; j >= 0; j--) lcs[i][j] = same[i][j] ? lcs[i + 1][j + 1] + 1 : Math.max(lcs[i + 1][j], lcs[i][j + 1]);
|
package/package.json
CHANGED
|
@@ -225,11 +225,13 @@ test.each(SCENARIOS)("equivalent to the recorded loop: $id", async (scenario) =>
|
|
|
225
225
|
const INTENDED: Record<string, Array<Pick<EffectDivergence, "reason" | "message">>> = {};
|
|
226
226
|
```
|
|
227
227
|
|
|
228
|
-
- Equivalent means the same model calls (count and shape: offered tool names, response format), the same tool calls with the same arguments, and the same output. Prompts and states may differ. Tool calls
|
|
229
|
-
- Values that differ on every run (generated ids, branch names, timestamps) fail `tool-input`. Pass the same `normalize` to `createRecorder({ replay, normalize })` and `compareRecordings(a, b, { normalize })` to redact them on both sides; never edit the fixtures.
|
|
228
|
+
- Equivalent means the same model calls (count and shape: offered tool names, response format), the same tool calls with the same arguments, and the same output. Prompts and states may differ. Tool calls that ran at the same time compare in any order; sequential ones keep theirs.
|
|
229
|
+
- Values that differ on every run (generated ids, branch names, timestamps) fail `tool-input`. Pass the same `normalize` to `createRecorder({ replay, normalize })` and `compareRecordings(a, b, { normalize })` to redact them on both sides; never edit the fixtures. `normalize(effect, index, effects)` returns `undefined` to leave an effect out of `compareRecordings`, such as a repeat of an idempotent write; replay still answers from it. Drop an effect only when repeating it is harmless: a dropped `send` or `charge` hides a duplicate.
|
|
230
230
|
- `divergences` lists every difference, lined up so one extra call does not shift the rest; `divergence` is the first.
|
|
231
|
-
-
|
|
232
|
-
-
|
|
231
|
+
- Replay answers each model call with the earliest unused recorded call of the same method and shape, and each tool call with the earliest unused recorded call of the same name and input, not by position. A call the machine added or dropped does not shift the answers after it, as long as no other call has the same method and shape (or tool name and input). Calls that do are answered in recorded order, so an added or dropped one takes or leaves the next one's answer.
|
|
232
|
+
- A call no recorded answer fits throws `RecordingDivergenceError`, and `compareRecordings` names it. To run the machine's own path instead (its retry, its added check), pass `fallback: { model: standIn, tools: true }`: a stand-in model answers the call and the wrapped tool runs. Those effects are marked `unrecorded` and still count as extra.
|
|
233
|
+
- `model-call-shape`: the machine changed a call's tools or output format, for example a model-picked action that became `agent.decide`. Port the call unchanged first, prove equivalence, then change it in a separate step whose divergences are `INTENDED`. To keep replaying past a call you changed on purpose, pass `translate(recorded, actual)` to `createRecorder`: it returns the answer that fits the new call (`{ response }`, `{ stream }`, or `{ error }`), or `undefined` to leave that recorded call alone, and `compareRecordings` counts the pair as the same call. When the recorded answer already fits, a `modelCalls` function passed to both is enough: `(expected, actual) => expected.call === 2 || sameModelCallShape(expected, actual)`.
|
|
234
|
+
- When the machine asks the model a different way altogether (another protocol, one structured draft instead of a tool loop), compare what it did, not how it asked: `compareRecordings(a, b, { kinds: ["tool", "output"] })`, with the machine recorded on a stand-in model.
|
|
233
235
|
- A guard that blocks a rule break diverges on purpose. List it in `INTENDED` with the exact divergence.
|
|
234
236
|
|
|
235
237
|
- Every `regression` row: fix the machine, or, when the old behavior was wrong but broke no rule, ask the user. If the outcomes differ, the answer may become the scenario's `expected`. If only effects differ (the loop asked three times, the machine once), list the row in a reviewed-differences list in the test, with the exact difference asserted and the person who accepted it. Never edit a scenario to make a row pass.
|