@statelyai/agent 2.0.0-alpha.34 → 2.0.0-alpha.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +124 -0
- package/dist/act-B9hLQls6.mjs +19 -0
- package/dist/act-CrX0JwWY.cjs +42 -0
- package/dist/ai-sdk.cjs +63 -7
- package/dist/ai-sdk.d.cts +13 -3
- package/dist/ai-sdk.d.mts +13 -3
- package/dist/ai-sdk.mjs +63 -7
- package/dist/{decision-BEsATcr1.cjs → decision-CMeuNZ1g.cjs} +262 -4
- package/dist/{decision-COkJNjOW.mjs → decision-m5hGmHpF.mjs} +239 -5
- package/dist/decision-object-B5zGSPFp.d.mts +15 -0
- package/dist/decision-object-DBcLWVvv.d.cts +15 -0
- package/dist/{run-agent-s5D32w99.cjs → effects-B23bFZEs.cjs} +948 -102
- package/dist/{run-agent-Ct_GZw_f.mjs → effects-CuOUNYtZ.mjs} +914 -104
- package/dist/{event-log-C7jeH0y8.mjs → event-log-DkVlk3Rh.mjs} +30 -15
- package/dist/{event-log-BWSSLUff.cjs → event-log-Dkc-yIYu.cjs} +41 -14
- package/dist/{event-log-store-BzXehP7R.d.cts → event-log-store-BBS8so8a.d.mts} +2 -2
- package/dist/{event-log-store-W49TCJXi.mjs → event-log-store-Brx9tOHt.mjs} +1 -1
- package/dist/{event-log-store-CleWpsp5.cjs → event-log-store-Cl-YWDnq.cjs} +1 -1
- package/dist/{event-log-store-CWjN6lmM.d.mts → event-log-store-bc4ICGcS.d.cts} +2 -2
- package/dist/index.cjs +660 -286
- package/dist/index.d.cts +168 -9
- package/dist/index.d.mts +168 -9
- package/dist/index.mjs +639 -271
- package/dist/internal-C3Lqu538.d.cts +43 -0
- package/dist/internal-DUcdhk-i.d.mts +43 -0
- package/dist/log.cjs +2 -2
- package/dist/log.d.cts +1 -1
- package/dist/log.d.mts +1 -1
- package/dist/log.mjs +2 -2
- package/dist/machines.cjs +457 -41
- package/dist/machines.d.cts +114 -43
- package/dist/machines.d.mts +114 -43
- package/dist/machines.mjs +457 -42
- package/dist/{model-source-DmhTMK39.cjs → model-source-B4WQ8Idp.cjs} +1 -1
- package/dist/{model-source-CK3F4YuU.mjs → model-source-kjGwqAV8.mjs} +1 -1
- package/dist/openai.cjs +149 -36
- package/dist/openai.d.cts +28 -10
- package/dist/openai.d.mts +28 -10
- package/dist/openai.mjs +149 -36
- package/dist/otel.d.cts +1 -1
- package/dist/otel.d.mts +1 -1
- package/dist/{run-agent-BGdx5913.d.mts → run-agent-BcQ-9NRT.d.cts} +91 -9
- package/dist/{run-agent-B1-Eqviq.d.cts → run-agent-C86drZtN.d.mts} +91 -9
- package/dist/{setup-agent-05uw4dPZ.mjs → setup-agent-BDmNr6PC.mjs} +9 -3
- package/dist/{setup-agent-BtfeFfF6.d.cts → setup-agent-BIKcbVXV.d.mts} +4 -4
- package/dist/{setup-agent-CXEBURdo.cjs → setup-agent-CmEToB8y.cjs} +9 -3
- package/dist/{setup-agent-C6nqzv_X.d.mts → setup-agent-jlo4vtaZ.d.cts} +4 -4
- package/dist/tanstack-ai.cjs +62 -4
- package/dist/tanstack-ai.d.cts +18 -6
- package/dist/tanstack-ai.d.mts +18 -6
- package/dist/tanstack-ai.mjs +62 -4
- package/dist/testing.cjs +536 -67
- package/dist/testing.d.cts +197 -180
- package/dist/testing.d.mts +197 -180
- package/dist/testing.mjs +532 -67
- package/dist/{text-logic-CV_4KGr6.d.cts → text-logic-BqxalgMr.d.mts} +87 -4
- package/dist/{text-logic-Ckh-rJFV.d.mts → text-logic-CSf9bqpP.d.cts} +87 -4
- package/dist/{types-Dp5_Ru4R.d.cts → types-DQUOOaZO.d.cts} +1 -1
- package/dist/{types-Dp5_Ru4R.d.mts → types-DQUOOaZO.d.mts} +1 -1
- package/dist/{usage-BDiJKyv9.d.cts → usage-DoP0Oi84.d.mts} +1 -1
- package/dist/{usage-DOoVUGVq.d.mts → usage-HtnrUsl9.d.cts} +1 -1
- package/dist/utils-7lSa4H-i.d.cts +403 -0
- package/dist/utils-CBbKw2fM.d.mts +403 -0
- package/dist/validate.cjs +44 -12
- package/dist/validate.d.cts +6 -6
- package/dist/validate.d.mts +6 -6
- package/dist/validate.mjs +44 -12
- package/dist/verify-Bink-Drw.cjs +2304 -0
- package/dist/{verify-BQpE8hUz.d.cts → verify-Buh8XSUr.d.mts} +208 -14
- package/dist/{verify-CqeuEmIk.d.mts → verify-CciG8UP-.d.cts} +208 -14
- package/dist/verify-Dpz1u5Pe.mjs +2251 -0
- package/dist/with-model-calls-B_WLmOSl.cjs +248 -0
- package/dist/with-model-calls-BkOwdroS.mjs +213 -0
- package/docs/act.md +102 -0
- package/docs/advanced.md +105 -0
- package/docs/any-stack.md +62 -0
- package/docs/choosing-a-run-mode.md +55 -0
- package/docs/debugging.md +29 -0
- package/docs/decisions.md +473 -0
- package/docs/environment-and-policy.md +188 -0
- package/docs/evals.md +509 -0
- package/docs/event-log.md +311 -0
- package/docs/from-a-loop.md +528 -0
- package/docs/generate-machines.md +110 -0
- package/docs/hosts.md +414 -0
- package/docs/human-in-the-loop.md +177 -0
- package/docs/index.md +48 -0
- package/docs/judgments.md +281 -0
- package/docs/langgraph-comparison.md +198 -0
- package/docs/machines-as-data.md +454 -0
- package/docs/machines-presets.md +198 -0
- package/docs/machines.md +497 -0
- package/docs/messages.md +67 -0
- package/docs/meta.json +52 -0
- package/docs/models-and-providers.md +268 -0
- package/docs/multi-agent.md +68 -0
- package/docs/observability.md +109 -0
- package/docs/patterns.md +31 -0
- package/docs/persistence.md +136 -0
- package/docs/quickstart.md +121 -0
- package/docs/roadmap.md +22 -0
- package/docs/scope.md +75 -0
- package/docs/statechart-policy-examples.md +69 -0
- package/docs/steps.md +54 -0
- package/docs/strangler-fig.md +323 -0
- package/docs/text-requests.md +250 -0
- package/docs/thinking-in-state-machines.md +324 -0
- package/docs/tools.md +249 -0
- package/docs/upgrading.md +37 -0
- package/docs/usage-and-budgets.md +516 -0
- package/docs/validate-and-repair.md +253 -0
- package/docs/verify.md +487 -0
- package/docs/xstate-4-and-5-apps.md +161 -0
- package/llms.txt +53 -0
- package/package.json +10 -3
- package/readme.md +30 -15
- package/skills/add-evals/SKILL.md +3 -1
- package/skills/add-traces/SKILL.md +6 -1
- package/skills/build-machine/SKILL.md +81 -0
- package/skills/generate-machine/SKILL.md +1 -1
- package/skills/graph-engineering/SKILL.md +13 -3
- package/skills/improve-machine/SKILL.md +21 -2
- package/skills/loop-to-machine/SKILL.md +147 -190
- package/skills/loop-to-machine/reference.md +260 -0
- package/dist/utils-BnXdWih3.d.mts +0 -108
- package/dist/utils-ebVpCYoF.d.cts +0 -108
- package/dist/verify-Bpahmr34.mjs +0 -1241
- package/dist/verify-DCLZa9R0.cjs +0 -1288
- package/dist/with-model-calls-BXYdBdcu.cjs +0 -45
- package/dist/with-model-calls-Bv4V4EHC.mjs +0 -34
package/dist/testing.cjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
-
const require_event_log = require("./event-log-
|
|
3
|
-
const
|
|
4
|
-
const require_verify = require("./verify-
|
|
2
|
+
const require_event_log = require("./event-log-Dkc-yIYu.cjs");
|
|
3
|
+
const require_effects = require("./effects-B23bFZEs.cjs");
|
|
4
|
+
const require_verify = require("./verify-Bink-Drw.cjs");
|
|
5
5
|
let xstate = require("xstate");
|
|
6
6
|
//#region src/trajectory.ts
|
|
7
7
|
/**
|
|
@@ -307,7 +307,7 @@ async function runSeam(machine, options) {
|
|
|
307
307
|
let result;
|
|
308
308
|
const maxTurns = options.maxTurns ?? 12;
|
|
309
309
|
for (let turn = 0; turn <= maxTurns; turn++) {
|
|
310
|
-
result = await
|
|
310
|
+
result = await require_effects.runToQuiescence(require_effects.createAgentRuntime(machine, {
|
|
311
311
|
...options.actors ? { actors: options.actors } : {},
|
|
312
312
|
executors,
|
|
313
313
|
onTransition: (next, causedBy) => {
|
|
@@ -791,6 +791,134 @@ function initialValue(trace, steps) {
|
|
|
791
791
|
return steps[0]?.fromValue;
|
|
792
792
|
}
|
|
793
793
|
//#endregion
|
|
794
|
+
//#region src/propose-from-traces.ts
|
|
795
|
+
/**
|
|
796
|
+
* Groups the approval verdicts in many runs' traces by `key` and proposes an
|
|
797
|
+
* `allow` or `deny` rule for each. Pure and deterministic; a proposal is for
|
|
798
|
+
* a person to review, and nothing here changes a machine.
|
|
799
|
+
*
|
|
800
|
+
* Reads `machine.transition` events: a verdict is a transition, not ignored,
|
|
801
|
+
* out of `approvalState` on an approve or reject event. Rows come in the
|
|
802
|
+
* order keys were first seen.
|
|
803
|
+
*
|
|
804
|
+
* @example
|
|
805
|
+
* ```ts no-check
|
|
806
|
+
* const rows = proposeApprovalRules(traces, {
|
|
807
|
+
* approvalState: "awaitingApproval",
|
|
808
|
+
* approveEvents: ["APPROVE"],
|
|
809
|
+
* rejectEvents: ["REJECT"],
|
|
810
|
+
* key: ({ snapshot }) => `${snapshot.context.pending.tool}:${snapshot.context.pending.path}`,
|
|
811
|
+
* });
|
|
812
|
+
* ```
|
|
813
|
+
*/
|
|
814
|
+
function proposeApprovalRules(traces, options) {
|
|
815
|
+
const rows = /* @__PURE__ */ new Map();
|
|
816
|
+
const rowFor = (key) => {
|
|
817
|
+
let row = rows.get(key);
|
|
818
|
+
if (!row) {
|
|
819
|
+
row = {
|
|
820
|
+
key,
|
|
821
|
+
approved: 0,
|
|
822
|
+
rejected: 0,
|
|
823
|
+
violations: 0,
|
|
824
|
+
runs: [],
|
|
825
|
+
proposal: null,
|
|
826
|
+
support: 0
|
|
827
|
+
};
|
|
828
|
+
rows.set(key, row);
|
|
829
|
+
}
|
|
830
|
+
return row;
|
|
831
|
+
};
|
|
832
|
+
const inState = (value) => statePaths(value).some((path) => path === options.approvalState || path.startsWith(`${options.approvalState}.`));
|
|
833
|
+
let index = 0;
|
|
834
|
+
for (const trace of traces) {
|
|
835
|
+
const run = index++;
|
|
836
|
+
const seen = /* @__PURE__ */ new Set();
|
|
837
|
+
let snapshot;
|
|
838
|
+
for (const event of trace) {
|
|
839
|
+
if (event.type !== "machine.transition") continue;
|
|
840
|
+
const before = snapshot;
|
|
841
|
+
snapshot = event.snapshot;
|
|
842
|
+
const type = require_event_log.isRecord(event.event) ? event.event.type : void 0;
|
|
843
|
+
if (event.ignored === true || typeof type !== "string" || !inState(valueOf(event.from))) continue;
|
|
844
|
+
const approve = options.approveEvents.includes(type);
|
|
845
|
+
if (!approve && !options.rejectEvents.includes(type)) continue;
|
|
846
|
+
const key = options.key({
|
|
847
|
+
event: event.event,
|
|
848
|
+
snapshot: before,
|
|
849
|
+
trace
|
|
850
|
+
});
|
|
851
|
+
if (key === void 0) continue;
|
|
852
|
+
const row = rowFor(key);
|
|
853
|
+
if (approve) row.approved++;
|
|
854
|
+
else row.rejected++;
|
|
855
|
+
if (!seen.has(key)) {
|
|
856
|
+
seen.add(key);
|
|
857
|
+
row.runs.push(run);
|
|
858
|
+
}
|
|
859
|
+
}
|
|
860
|
+
for (const key of options.violations?.(trace) ?? []) {
|
|
861
|
+
const row = rows.get(key);
|
|
862
|
+
if (row) row.violations++;
|
|
863
|
+
}
|
|
864
|
+
}
|
|
865
|
+
for (const row of rows.values()) {
|
|
866
|
+
row.support = row.runs.length;
|
|
867
|
+
row.proposal = row.approved > 0 && row.rejected === 0 && row.violations === 0 ? "allow" : row.rejected > 0 && row.approved === 0 ? "deny" : null;
|
|
868
|
+
}
|
|
869
|
+
return [...rows.values()];
|
|
870
|
+
}
|
|
871
|
+
/**
|
|
872
|
+
* Proposes a value for each cap (retry count, steps, batch size) from many
|
|
873
|
+
* runs' traces: the 95th percentile of what passing runs used, plus one, and
|
|
874
|
+
* never below the current cap while runs still hit it. Pure and
|
|
875
|
+
* deterministic; a proposal is for a person to review, and nothing here
|
|
876
|
+
* changes a machine.
|
|
877
|
+
*
|
|
878
|
+
* @example
|
|
879
|
+
* ```ts no-check
|
|
880
|
+
* const [retries] = proposeCaps(traces, {
|
|
881
|
+
* caps: [{ name: "retries", current: 3, failureState: "gaveUp", valueFromRun: countRetries }],
|
|
882
|
+
* });
|
|
883
|
+
* ```
|
|
884
|
+
*/
|
|
885
|
+
function proposeCaps(traces, options) {
|
|
886
|
+
const runs = [...traces];
|
|
887
|
+
return options.caps.map((cap) => {
|
|
888
|
+
let hits = 0;
|
|
889
|
+
let max = null;
|
|
890
|
+
const passing = [];
|
|
891
|
+
for (const trace of runs) {
|
|
892
|
+
const hit = hitCap(trace, cap);
|
|
893
|
+
if (hit) hits++;
|
|
894
|
+
const value = cap.valueFromRun(trace);
|
|
895
|
+
if (value === void 0) continue;
|
|
896
|
+
max = max === null ? value : Math.max(max, value);
|
|
897
|
+
if (!hit) passing.push(value);
|
|
898
|
+
}
|
|
899
|
+
passing.sort((a, b) => a - b);
|
|
900
|
+
const p95 = passing.length ? passing[Math.ceil(.95 * passing.length) - 1] : null;
|
|
901
|
+
const proposal = p95 === null ? null : hits > 0 && cap.current !== void 0 ? Math.max(p95 + 1, cap.current) : p95 + 1;
|
|
902
|
+
return {
|
|
903
|
+
name: cap.name,
|
|
904
|
+
hits,
|
|
905
|
+
passing: passing.length,
|
|
906
|
+
max,
|
|
907
|
+
p95,
|
|
908
|
+
proposal
|
|
909
|
+
};
|
|
910
|
+
});
|
|
911
|
+
}
|
|
912
|
+
function hitCap(trace, cap) {
|
|
913
|
+
for (const event of trace) {
|
|
914
|
+
if (cap.failureState !== void 0 && event.type === "machine.transition" && statePaths(valueOf(event.snapshot)).some((path) => path === cap.failureState || path.startsWith(`${cap.failureState}.`))) return true;
|
|
915
|
+
if (cap.failureOutput && event.type === "run.end" && event.status === "done") {
|
|
916
|
+
if (cap.failureOutput(event.output)) return true;
|
|
917
|
+
}
|
|
918
|
+
}
|
|
919
|
+
return false;
|
|
920
|
+
}
|
|
921
|
+
//#endregion
|
|
794
922
|
//#region src/decision-requests.ts
|
|
795
923
|
/**
|
|
796
924
|
* The decisions a run asked for, read from its trace: each decision's first
|
|
@@ -1064,8 +1192,8 @@ async function compareRuns(options) {
|
|
|
1064
1192
|
regression: 0
|
|
1065
1193
|
};
|
|
1066
1194
|
for (const scenario of options.scenarios) {
|
|
1067
|
-
const baseline = await settle(() => options.baseline(scenario));
|
|
1068
|
-
const candidate = await settle(() => options.candidate(scenario));
|
|
1195
|
+
const baseline = await settle$1(() => options.baseline(scenario));
|
|
1196
|
+
const candidate = await settle$1(() => options.candidate(scenario));
|
|
1069
1197
|
const { label, reason } = labelRun(scenario.expected, baseline, candidate);
|
|
1070
1198
|
counts[label]++;
|
|
1071
1199
|
scenarios.push({
|
|
@@ -1083,7 +1211,7 @@ async function compareRuns(options) {
|
|
|
1083
1211
|
ok: counts.regression === 0
|
|
1084
1212
|
};
|
|
1085
1213
|
}
|
|
1086
|
-
async function settle(run) {
|
|
1214
|
+
async function settle$1(run) {
|
|
1087
1215
|
try {
|
|
1088
1216
|
return await run();
|
|
1089
1217
|
} catch (error) {
|
|
@@ -1136,6 +1264,37 @@ function structurallyEqual(a, b) {
|
|
|
1136
1264
|
//#endregion
|
|
1137
1265
|
//#region src/equivalence.ts
|
|
1138
1266
|
/**
|
|
1267
|
+
* Verified equivalence — prove a port behaves like the code it replaces.
|
|
1268
|
+
*
|
|
1269
|
+
* A port is equivalent when, given the same model answers and the same tool
|
|
1270
|
+
* results, it produces the same OBSERVABLE EFFECTS: the same number of model
|
|
1271
|
+
* calls, the same tool calls with the same arguments, and the same output.
|
|
1272
|
+
* Prompts, internal steps, and states are free to differ.
|
|
1273
|
+
*
|
|
1274
|
+
* 1. Record the original: wrap its model (`recorder.model`, an AI SDK
|
|
1275
|
+
* `LanguageModelV3`, or `recorder.openai`, an `openai` client) and its tools
|
|
1276
|
+
* (`recorder.tool` / `recorder.tools`), run a scenario, call
|
|
1277
|
+
* `recorder.output(result)`, and save `recorder.recording`.
|
|
1278
|
+
* 2. Replay it on the port: `createRecorder({ replay: recording })` answers
|
|
1279
|
+
* every model call and tool call from the recording, with no API key and no
|
|
1280
|
+
* side effects, and records what the port did.
|
|
1281
|
+
* 3. `compareRecordings(original, port)` reports the first effect that differs.
|
|
1282
|
+
*
|
|
1283
|
+
* A port that changes how it calls the model on purpose (a model-picked action
|
|
1284
|
+
* that became `agent.decide`, a stream that became `doGenerate`) still replays:
|
|
1285
|
+
* `translate` turns the recorded answer into one that fits, `fallback` answers
|
|
1286
|
+
* the calls the original never made, and `compareRecordings(…, { kinds })`
|
|
1287
|
+
* compares only tool calls and output.
|
|
1288
|
+
*
|
|
1289
|
+
* Any other client (`@anthropic-ai/sdk`, `@google/genai`, a framework's own
|
|
1290
|
+
* model client) records through `recorder.client` or `recorder.fn`: each call
|
|
1291
|
+
* is a `call` effect, answered in replay by name and order.
|
|
1292
|
+
*
|
|
1293
|
+
* Provider-free: every seam is wrapped structurally, so this module imports
|
|
1294
|
+
* neither `ai` nor `openai`.
|
|
1295
|
+
* @module
|
|
1296
|
+
*/
|
|
1297
|
+
/**
|
|
1139
1298
|
* Thrown in replay when the port asks for something the recording cannot
|
|
1140
1299
|
* answer: a model call past the last recorded one, or a tool call the original
|
|
1141
1300
|
* never made. The effect is still recorded, so {@link compareRecordings}
|
|
@@ -1145,24 +1304,45 @@ var RecordingDivergenceError = class extends Error {
|
|
|
1145
1304
|
name = "RecordingDivergenceError";
|
|
1146
1305
|
};
|
|
1147
1306
|
function toRecordedError(error) {
|
|
1148
|
-
if (error instanceof Error) {
|
|
1149
|
-
const code = error.code;
|
|
1150
|
-
return code === void 0 ? {
|
|
1151
|
-
name: error.name,
|
|
1152
|
-
message: error.message
|
|
1153
|
-
} : {
|
|
1154
|
-
name: error.name,
|
|
1155
|
-
message: error.message,
|
|
1156
|
-
code: toJson(code)
|
|
1157
|
-
};
|
|
1158
|
-
}
|
|
1159
|
-
return {
|
|
1307
|
+
if (!(error instanceof Error)) return {
|
|
1160
1308
|
name: "Error",
|
|
1161
1309
|
message: String(error)
|
|
1162
1310
|
};
|
|
1311
|
+
const recorded = {
|
|
1312
|
+
name: error.name,
|
|
1313
|
+
message: error.message
|
|
1314
|
+
};
|
|
1315
|
+
if (Object.prototype.propertyIsEnumerable.call(error, "name")) recorded.ownName = true;
|
|
1316
|
+
const code = error.code;
|
|
1317
|
+
if (code !== void 0) recorded.code = toJson(code);
|
|
1318
|
+
const data = Object.fromEntries(Object.entries(error).filter(([key, value]) => key !== "code" && (typeof value === "number" || typeof value === "boolean")));
|
|
1319
|
+
if (Object.keys(data).length) recorded.data = data;
|
|
1320
|
+
const markers = /* @__PURE__ */ new Set();
|
|
1321
|
+
for (let holder = error; holder; holder = Object.getPrototypeOf(holder)) for (const symbol of Object.getOwnPropertySymbols(holder)) {
|
|
1322
|
+
const key = Symbol.keyFor(symbol);
|
|
1323
|
+
if (key !== void 0 && error[symbol] === true) markers.add(key);
|
|
1324
|
+
}
|
|
1325
|
+
if (markers.size) recorded.markers = [...markers];
|
|
1326
|
+
return recorded;
|
|
1163
1327
|
}
|
|
1164
|
-
function fromRecordedError(
|
|
1165
|
-
|
|
1328
|
+
function fromRecordedError({ data, markers, name, ownName, ...fields }) {
|
|
1329
|
+
const error = Object.assign(new Error(fields.message), fields, data);
|
|
1330
|
+
if (ownName || name !== "Error") Object.defineProperty(error, "name", {
|
|
1331
|
+
value: name,
|
|
1332
|
+
writable: true,
|
|
1333
|
+
configurable: true,
|
|
1334
|
+
enumerable: ownName ?? false
|
|
1335
|
+
});
|
|
1336
|
+
for (const marker of markers ?? []) error[Symbol.for(marker)] = true;
|
|
1337
|
+
return error;
|
|
1338
|
+
}
|
|
1339
|
+
/** What two runs compare of an error: its data and markers may differ. */
|
|
1340
|
+
function errorIdentity$1(error) {
|
|
1341
|
+
return error && {
|
|
1342
|
+
name: error.name,
|
|
1343
|
+
message: error.message,
|
|
1344
|
+
code: error.code
|
|
1345
|
+
};
|
|
1166
1346
|
}
|
|
1167
1347
|
/** A JSON-safe copy: Dates become ISO strings, Errors become {@link RecordedError}. */
|
|
1168
1348
|
function toJson(value) {
|
|
@@ -1187,7 +1367,10 @@ function omit(value, keys) {
|
|
|
1187
1367
|
if (!value || typeof value !== "object") return value;
|
|
1188
1368
|
return Object.fromEntries(Object.entries(value).filter(([key]) => !keys.includes(key)));
|
|
1189
1369
|
}
|
|
1190
|
-
/**
|
|
1370
|
+
/**
|
|
1371
|
+
* Key-order-independent JSON, for equality.
|
|
1372
|
+
* @internal
|
|
1373
|
+
*/
|
|
1191
1374
|
function stableStringify(value) {
|
|
1192
1375
|
if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
|
|
1193
1376
|
if (value && typeof value === "object") return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`).join(",")}}`;
|
|
@@ -1213,19 +1396,44 @@ function createRecorder(options = {}) {
|
|
|
1213
1396
|
const effects = [];
|
|
1214
1397
|
const replay = options.replay?.effects;
|
|
1215
1398
|
const replayedModels = replay?.filter((effect) => effect.kind === "model");
|
|
1399
|
+
const usedModels = /* @__PURE__ */ new Set();
|
|
1216
1400
|
const usedTools = /* @__PURE__ */ new Set();
|
|
1217
|
-
const
|
|
1401
|
+
const usedCalls = /* @__PURE__ */ new Set();
|
|
1402
|
+
const callCounts = /* @__PURE__ */ new Map();
|
|
1403
|
+
const normalizeEffect = (effect, list) => options.normalize?.(effect, list.indexOf(effect), list) ?? effect;
|
|
1218
1404
|
let modelCalls = 0;
|
|
1219
1405
|
let toolsInFlight = 0;
|
|
1406
|
+
/**
|
|
1407
|
+
* The earliest unused recorded call that fits this one: one `translate`
|
|
1408
|
+
* adapts, or one with the same method and shape. Not by position, so a call
|
|
1409
|
+
* the port added or dropped does not shift every answer after it.
|
|
1410
|
+
*/
|
|
1220
1411
|
function nextReplayedModel(effect) {
|
|
1221
|
-
const recorded
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1412
|
+
for (const recorded of replayedModels) {
|
|
1413
|
+
if (usedModels.has(recorded)) continue;
|
|
1414
|
+
const translated = options.translate?.(structuredClone(recorded), effect);
|
|
1415
|
+
if (translated) {
|
|
1416
|
+
effect.translated = true;
|
|
1417
|
+
return {
|
|
1418
|
+
recorded,
|
|
1419
|
+
answer: translated
|
|
1420
|
+
};
|
|
1421
|
+
}
|
|
1422
|
+
if (recorded.method === effect.method && modelCallsMatch(options.modelCalls, recorded, effect)) return {
|
|
1423
|
+
recorded,
|
|
1424
|
+
answer: recorded
|
|
1425
|
+
};
|
|
1426
|
+
}
|
|
1427
|
+
}
|
|
1428
|
+
function noAnswerMessage(effect) {
|
|
1429
|
+
const unused = replayedModels.filter((recorded) => !usedModels.has(recorded));
|
|
1430
|
+
return unused.length ? `Model call #${effect.call} (${effect.method}, ${JSON.stringify(modelCallShape(effect))}) fits no unused recorded call: ${unused.map((recorded) => `#${recorded.call} (${recorded.method}, ${JSON.stringify(modelCallShape(recorded))})`).join(", ")}. A recorded answer only fits a call of the same method and shape; pass \`translate\` to adapt one.` : `Model call #${effect.call} was never made by the recorded run (it made ${replayedModels.length}).`;
|
|
1226
1431
|
}
|
|
1227
|
-
/**
|
|
1228
|
-
|
|
1432
|
+
/**
|
|
1433
|
+
* Records a model call around `run`, or answers it from the recording.
|
|
1434
|
+
* `fallback` runs it on the stand-in when the recording has no answer.
|
|
1435
|
+
*/
|
|
1436
|
+
async function modelCall(seam, method, request, streamed, run, fallback) {
|
|
1229
1437
|
const effect = {
|
|
1230
1438
|
kind: "model",
|
|
1231
1439
|
seam,
|
|
@@ -1239,24 +1447,30 @@ function createRecorder(options = {}) {
|
|
|
1239
1447
|
};
|
|
1240
1448
|
effects.push(effect);
|
|
1241
1449
|
if (replay) {
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
recorded =
|
|
1245
|
-
|
|
1450
|
+
const replayed = nextReplayedModel(effect);
|
|
1451
|
+
if (replayed) {
|
|
1452
|
+
const { recorded, answer } = replayed;
|
|
1453
|
+
usedModels.add(recorded);
|
|
1454
|
+
effect.replayed = recorded.call;
|
|
1455
|
+
if (answer.error) {
|
|
1456
|
+
effect.error = answer.error;
|
|
1457
|
+
throw fromRecordedError(answer.error);
|
|
1458
|
+
}
|
|
1459
|
+
effect.response = answer.response;
|
|
1460
|
+
effect.stream = answer.stream;
|
|
1461
|
+
if (seam === "openai") return streamed ? iterate(answer.stream ?? []) : structuredClone(answer.response);
|
|
1462
|
+
return streamed ? {
|
|
1463
|
+
...reviveTimestamps(structuredClone(answer.response) ?? {}),
|
|
1464
|
+
stream: streamFrom(answer.stream ?? [])
|
|
1465
|
+
} : reviveTimestamps(structuredClone(answer.response));
|
|
1466
|
+
}
|
|
1467
|
+
if (!fallback) {
|
|
1468
|
+
const error = new RecordingDivergenceError(noAnswerMessage(effect));
|
|
1246
1469
|
effect.error = toRecordedError(error);
|
|
1247
1470
|
throw error;
|
|
1248
1471
|
}
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
throw fromRecordedError(recorded.error);
|
|
1252
|
-
}
|
|
1253
|
-
effect.response = recorded.response;
|
|
1254
|
-
effect.stream = recorded.stream;
|
|
1255
|
-
if (seam === "openai") return streamed ? iterate(recorded.stream ?? []) : structuredClone(recorded.response);
|
|
1256
|
-
return streamed ? {
|
|
1257
|
-
...reviveTimestamps(structuredClone(recorded.response) ?? {}),
|
|
1258
|
-
stream: streamFrom(recorded.stream ?? [])
|
|
1259
|
-
} : reviveTimestamps(structuredClone(recorded.response));
|
|
1472
|
+
effect.unrecorded = true;
|
|
1473
|
+
run = fallback;
|
|
1260
1474
|
}
|
|
1261
1475
|
let result;
|
|
1262
1476
|
try {
|
|
@@ -1307,8 +1521,13 @@ function createRecorder(options = {}) {
|
|
|
1307
1521
|
function call(effect, args) {
|
|
1308
1522
|
const { input } = effect;
|
|
1309
1523
|
if (replay && options.tools !== "live") {
|
|
1310
|
-
const wanted = normalizeEffect(effect).input;
|
|
1311
|
-
const
|
|
1524
|
+
const wanted = normalizeEffect(effect, effects).input;
|
|
1525
|
+
const matches = (candidate) => candidate.kind === "tool" && !usedTools.has(candidate) && candidate.name === name && jsonEqual(normalizeEffect(candidate, replay).input, wanted);
|
|
1526
|
+
const recorded = replay.find(matches);
|
|
1527
|
+
if (!recorded && options.fallback?.tools) {
|
|
1528
|
+
effect.unrecorded = true;
|
|
1529
|
+
return runLive(effect, args);
|
|
1530
|
+
}
|
|
1312
1531
|
const settle = (value) => {
|
|
1313
1532
|
if (recorded?.sync) return value();
|
|
1314
1533
|
try {
|
|
@@ -1319,7 +1538,7 @@ function createRecorder(options = {}) {
|
|
|
1319
1538
|
};
|
|
1320
1539
|
return settle(() => {
|
|
1321
1540
|
if (!recorded) {
|
|
1322
|
-
const error = new RecordingDivergenceError(`Tool '${name}' was called with ${JSON.stringify(input)}
|
|
1541
|
+
const error = new RecordingDivergenceError(`Tool '${name}' was called with ${JSON.stringify(input)}; the recorded run made no such call.`);
|
|
1323
1542
|
effect.error = toRecordedError(error);
|
|
1324
1543
|
throw error;
|
|
1325
1544
|
}
|
|
@@ -1333,6 +1552,9 @@ function createRecorder(options = {}) {
|
|
|
1333
1552
|
return structuredClone(recorded.output);
|
|
1334
1553
|
});
|
|
1335
1554
|
}
|
|
1555
|
+
return runLive(effect, args);
|
|
1556
|
+
}
|
|
1557
|
+
function runLive(effect, args) {
|
|
1336
1558
|
let output;
|
|
1337
1559
|
try {
|
|
1338
1560
|
output = fn(...args);
|
|
@@ -1355,6 +1577,68 @@ function createRecorder(options = {}) {
|
|
|
1355
1577
|
});
|
|
1356
1578
|
}
|
|
1357
1579
|
}
|
|
1580
|
+
function fn(name, run, callOptions = {}) {
|
|
1581
|
+
const { redact, serialize = toJson, revive } = callOptions;
|
|
1582
|
+
return (async (...args) => {
|
|
1583
|
+
const passed = [...args];
|
|
1584
|
+
while (passed.length && passed.at(-1) === void 0) passed.pop();
|
|
1585
|
+
const call = callCounts.get(name) ?? 0;
|
|
1586
|
+
callCounts.set(name, call + 1);
|
|
1587
|
+
const effect = {
|
|
1588
|
+
kind: "call",
|
|
1589
|
+
name,
|
|
1590
|
+
call,
|
|
1591
|
+
args: toJson(redact ? redact(passed) : passed)
|
|
1592
|
+
};
|
|
1593
|
+
effects.push(effect);
|
|
1594
|
+
if (replay) {
|
|
1595
|
+
const recorded = replay.find((candidate) => candidate.kind === "call" && candidate.name === name && !usedCalls.has(candidate));
|
|
1596
|
+
if (!recorded) {
|
|
1597
|
+
const error = new RecordingDivergenceError(`Call #${call} to '${name}' was never made by the recorded run.`);
|
|
1598
|
+
effect.error = toRecordedError(error);
|
|
1599
|
+
throw error;
|
|
1600
|
+
}
|
|
1601
|
+
usedCalls.add(recorded);
|
|
1602
|
+
if (recorded.error) {
|
|
1603
|
+
effect.error = recorded.error;
|
|
1604
|
+
throw fromRecordedError(recorded.error);
|
|
1605
|
+
}
|
|
1606
|
+
if (recorded.stream) {
|
|
1607
|
+
effect.stream = recorded.stream;
|
|
1608
|
+
const chunks = structuredClone(recorded.stream);
|
|
1609
|
+
return revive ? revive(chunks) : iterate(chunks);
|
|
1610
|
+
}
|
|
1611
|
+
effect.result = recorded.result;
|
|
1612
|
+
const result = structuredClone(recorded.result);
|
|
1613
|
+
return revive ? revive(result) : result;
|
|
1614
|
+
}
|
|
1615
|
+
let result;
|
|
1616
|
+
try {
|
|
1617
|
+
result = await run(...args);
|
|
1618
|
+
} catch (error) {
|
|
1619
|
+
effect.error = toRecordedError(error);
|
|
1620
|
+
throw error;
|
|
1621
|
+
}
|
|
1622
|
+
if (!require_effects.isAsyncIterable(result)) {
|
|
1623
|
+
effect.result = toJson(serialize(result));
|
|
1624
|
+
return result;
|
|
1625
|
+
}
|
|
1626
|
+
const chunks = [];
|
|
1627
|
+
effect.stream = chunks;
|
|
1628
|
+
const source = result;
|
|
1629
|
+
return (async function* () {
|
|
1630
|
+
try {
|
|
1631
|
+
for await (const chunk of source) {
|
|
1632
|
+
chunks.push(toJson(serialize(chunk)));
|
|
1633
|
+
yield chunk;
|
|
1634
|
+
}
|
|
1635
|
+
} catch (error) {
|
|
1636
|
+
effect.error = toRecordedError(error);
|
|
1637
|
+
throw error;
|
|
1638
|
+
}
|
|
1639
|
+
})();
|
|
1640
|
+
});
|
|
1641
|
+
}
|
|
1358
1642
|
/** Counts an async tool call as running until it settles. */
|
|
1359
1643
|
function track(result) {
|
|
1360
1644
|
if (result instanceof Promise) {
|
|
@@ -1369,9 +1653,18 @@ function createRecorder(options = {}) {
|
|
|
1369
1653
|
return {
|
|
1370
1654
|
model(model) {
|
|
1371
1655
|
const base = model;
|
|
1656
|
+
const stand = options.fallback?.model;
|
|
1372
1657
|
const wrapped = Object.create(base);
|
|
1373
|
-
|
|
1374
|
-
|
|
1658
|
+
for (const method of [
|
|
1659
|
+
"doGenerate",
|
|
1660
|
+
"doStream",
|
|
1661
|
+
"doDecide",
|
|
1662
|
+
"doEvaluate"
|
|
1663
|
+
]) {
|
|
1664
|
+
if (typeof base[method] !== "function") continue;
|
|
1665
|
+
const invoke = (target, callOptions) => target[method].call(target, callOptions);
|
|
1666
|
+
wrapped[method] = (callOptions) => modelCall("ai-sdk", method, callOptions, method === "doStream", () => invoke(base, callOptions), stand && typeof stand[method] === "function" ? () => invoke(stand, callOptions) : void 0);
|
|
1667
|
+
}
|
|
1375
1668
|
return wrapped;
|
|
1376
1669
|
},
|
|
1377
1670
|
openai(client) {
|
|
@@ -1379,10 +1672,12 @@ function createRecorder(options = {}) {
|
|
|
1379
1672
|
if (key !== "create") return Reflect.get(target, key, receiver);
|
|
1380
1673
|
return (params, requestOptions) => {
|
|
1381
1674
|
let api;
|
|
1382
|
-
const
|
|
1383
|
-
api = Reflect.get(
|
|
1675
|
+
const create = (resource) => () => {
|
|
1676
|
+
api = Reflect.get(resource, "create").call(resource, params, requestOptions);
|
|
1384
1677
|
return Promise.resolve(api);
|
|
1385
|
-
}
|
|
1678
|
+
};
|
|
1679
|
+
const stand = fallbackResource(method);
|
|
1680
|
+
const promise = modelCall("openai", method, params, params?.stream === true, create(target), stand && create(stand));
|
|
1386
1681
|
return Object.assign(promise, { withResponse: async () => {
|
|
1387
1682
|
const data = await promise;
|
|
1388
1683
|
const meta = await api?.withResponse?.();
|
|
@@ -1394,6 +1689,11 @@ function createRecorder(options = {}) {
|
|
|
1394
1689
|
} });
|
|
1395
1690
|
};
|
|
1396
1691
|
} });
|
|
1692
|
+
const fallbackResource = (method) => {
|
|
1693
|
+
let value = options.fallback?.openai;
|
|
1694
|
+
for (const key of method.split(".").slice(0, -1)) value = value?.[key];
|
|
1695
|
+
return value && typeof value === "object" ? value : void 0;
|
|
1696
|
+
};
|
|
1397
1697
|
const wrapped = /* @__PURE__ */ new Map();
|
|
1398
1698
|
const wrap = (key, make) => {
|
|
1399
1699
|
if (!wrapped.has(key)) wrapped.set(key, make());
|
|
@@ -1410,6 +1710,26 @@ function createRecorder(options = {}) {
|
|
|
1410
1710
|
} });
|
|
1411
1711
|
},
|
|
1412
1712
|
tool: (name, fn) => tool(name, fn),
|
|
1713
|
+
fn,
|
|
1714
|
+
client(name, client, { methods, ...callOptions }) {
|
|
1715
|
+
const paths = methods.map((method) => method.split("."));
|
|
1716
|
+
const startsWith = (path, prefix) => prefix.every((key, index) => path[index] === key);
|
|
1717
|
+
const wrapAt = (target, prefix) => {
|
|
1718
|
+
const wrapped = /* @__PURE__ */ new Map();
|
|
1719
|
+
return new Proxy(target, { get(object, key, receiver) {
|
|
1720
|
+
if (typeof key !== "string") return Reflect.get(object, key, receiver);
|
|
1721
|
+
const path = [...prefix, key];
|
|
1722
|
+
const method = paths.some((p) => p.length === path.length && startsWith(p, path));
|
|
1723
|
+
if (!method && !paths.some((p) => startsWith(p, path))) return Reflect.get(object, key, receiver);
|
|
1724
|
+
if (!wrapped.has(key)) {
|
|
1725
|
+
const value = Reflect.get(object, key, receiver);
|
|
1726
|
+
wrapped.set(key, method ? fn(`${name}.${path.join(".")}`, (...args) => value.apply(object, args), callOptions) : wrapAt(value && typeof value === "object" ? value : {}, path));
|
|
1727
|
+
}
|
|
1728
|
+
return wrapped.get(key);
|
|
1729
|
+
} });
|
|
1730
|
+
};
|
|
1731
|
+
return wrapAt(client, []);
|
|
1732
|
+
},
|
|
1413
1733
|
tools(tools) {
|
|
1414
1734
|
return Object.fromEntries(Object.entries(tools).map(([name, definition]) => [name, definition.execute ? {
|
|
1415
1735
|
...definition,
|
|
@@ -1442,9 +1762,27 @@ function modelCallsMatch(match, expected, actual) {
|
|
|
1442
1762
|
function shapeMessage(expected, actual) {
|
|
1443
1763
|
return `Model call #${expected.call} changed shape: ${JSON.stringify(modelCallShape(expected))} → ${JSON.stringify(modelCallShape(actual))}. A recorded answer only fits a call of the same shape.`;
|
|
1444
1764
|
}
|
|
1445
|
-
/**
|
|
1765
|
+
/**
|
|
1766
|
+
* Offered tool names and the response format, or a decision's questions and
|
|
1767
|
+
* their options: what a recorded answer depends on.
|
|
1768
|
+
*/
|
|
1446
1769
|
function modelCallShape(effect) {
|
|
1447
1770
|
const request = effect.request ?? {};
|
|
1771
|
+
const questions = request["questions"];
|
|
1772
|
+
if (questions && typeof questions === "object") return { questions: Object.entries(questions).map(([name, question]) => {
|
|
1773
|
+
const criteria = question["criteria"];
|
|
1774
|
+
if (question["type"] === "choice" && criteria && typeof criteria === "object") return [
|
|
1775
|
+
name,
|
|
1776
|
+
question["type"],
|
|
1777
|
+
Object.keys(criteria).sort()
|
|
1778
|
+
];
|
|
1779
|
+
if (question["type"] === "score" && Array.isArray(criteria)) return [
|
|
1780
|
+
name,
|
|
1781
|
+
question["type"],
|
|
1782
|
+
criteria.length
|
|
1783
|
+
];
|
|
1784
|
+
return [name, question["type"]];
|
|
1785
|
+
}).sort((x, y) => String(x[0]).localeCompare(String(y[0]))) };
|
|
1448
1786
|
return {
|
|
1449
1787
|
tools: (Array.isArray(request["tools"]) ? request["tools"] : []).map((tool) => tool["name"] ?? tool["function"]?.["name"]).sort(),
|
|
1450
1788
|
format: request["responseFormat"]?.["type"] ?? request["response_format"]?.["type"] ?? (request["text"]?.["format"])?.["type"] ?? "text"
|
|
@@ -1475,6 +1813,7 @@ function describe(effect) {
|
|
|
1475
1813
|
switch (effect.kind) {
|
|
1476
1814
|
case "model": return `model call #${effect.call} (${effect.method})`;
|
|
1477
1815
|
case "tool": return `tool '${effect.name}' with ${JSON.stringify(effect.input)}`;
|
|
1816
|
+
case "call": return `call #${effect.call} to '${effect.name}' with ${JSON.stringify(effect.args)}`;
|
|
1478
1817
|
case "output": return `output ${JSON.stringify(effect.value)}`;
|
|
1479
1818
|
}
|
|
1480
1819
|
}
|
|
@@ -1485,6 +1824,7 @@ function differ(left, right, options) {
|
|
|
1485
1824
|
message: `Expected ${describe(left)}; the port made ${describe(right)}.`
|
|
1486
1825
|
};
|
|
1487
1826
|
if (left.kind === "model" && right.kind === "model") {
|
|
1827
|
+
if (right.translated && right.replayed === left.call) return void 0;
|
|
1488
1828
|
if (left.method !== right.method) return {
|
|
1489
1829
|
reason: "model-method",
|
|
1490
1830
|
message: `Model call #${left.call} used ${left.method}; the port used ${right.method}.`
|
|
@@ -1493,7 +1833,7 @@ function differ(left, right, options) {
|
|
|
1493
1833
|
reason: "model-call-shape",
|
|
1494
1834
|
message: shapeMessage(left, right)
|
|
1495
1835
|
};
|
|
1496
|
-
if (!jsonEqual(left.error, right.error)) {
|
|
1836
|
+
if (!jsonEqual(errorIdentity$1(left.error), errorIdentity$1(right.error))) {
|
|
1497
1837
|
const outcome = (error) => error ? `failed with ${error.name}: ${error.message}` : "succeeded";
|
|
1498
1838
|
return {
|
|
1499
1839
|
reason: "model-error",
|
|
@@ -1511,12 +1851,27 @@ function differ(left, right, options) {
|
|
|
1511
1851
|
reason: "tool-input",
|
|
1512
1852
|
message: `Expected ${describe(left)}; the port passed ${JSON.stringify(right.input)}.`
|
|
1513
1853
|
};
|
|
1514
|
-
if (!jsonEqual(left.output, right.output) || !jsonEqual(left.error, right.error)) return {
|
|
1854
|
+
if (!jsonEqual(left.output, right.output) || !jsonEqual(errorIdentity$1(left.error), errorIdentity$1(right.error))) return {
|
|
1515
1855
|
reason: "tool-result",
|
|
1516
1856
|
message: `Tool '${left.name}' returned a different result.`
|
|
1517
1857
|
};
|
|
1518
1858
|
return;
|
|
1519
1859
|
}
|
|
1860
|
+
if (left.kind === "call" && right.kind === "call") {
|
|
1861
|
+
if (left.name !== right.name) return {
|
|
1862
|
+
reason: "call-name",
|
|
1863
|
+
message: `Expected ${describe(left)}; the port made ${describe(right)}.`
|
|
1864
|
+
};
|
|
1865
|
+
if (!jsonEqual(left.args, right.args)) return {
|
|
1866
|
+
reason: "call-args",
|
|
1867
|
+
message: `Expected ${describe(left)}; the port passed ${JSON.stringify(right.args)}.`
|
|
1868
|
+
};
|
|
1869
|
+
if (!jsonEqual(left.result, right.result) || !jsonEqual(left.stream, right.stream) || !jsonEqual(errorIdentity$1(left.error), errorIdentity$1(right.error))) return {
|
|
1870
|
+
reason: "call-result",
|
|
1871
|
+
message: `Call #${left.call} to '${left.name}' returned a different result.`
|
|
1872
|
+
};
|
|
1873
|
+
return;
|
|
1874
|
+
}
|
|
1520
1875
|
if (left.kind === "output" && right.kind === "output" && !jsonEqual(left.value, right.value)) return {
|
|
1521
1876
|
reason: "output",
|
|
1522
1877
|
message: `Expected ${describe(left)}; the port returned ${JSON.stringify(right.value)}.`
|
|
@@ -1530,9 +1885,13 @@ function differ(left, right, options) {
|
|
|
1530
1885
|
* kind pair up and report what changed, and the rest are missing or extra.
|
|
1531
1886
|
*/
|
|
1532
1887
|
function compareRecordings(expected, actual, options = {}) {
|
|
1533
|
-
const
|
|
1534
|
-
|
|
1535
|
-
|
|
1888
|
+
const select = (effects) => normalize(effects.flatMap((effect, index) => {
|
|
1889
|
+
if (options.kinds && !options.kinds.includes(effect.kind)) return [];
|
|
1890
|
+
const mapped = options.normalize ? options.normalize(effect, index, effects) : effect;
|
|
1891
|
+
return mapped ? [mapped] : [];
|
|
1892
|
+
}));
|
|
1893
|
+
const a = select(expected.effects);
|
|
1894
|
+
const b = select(actual.effects);
|
|
1536
1895
|
const same = a.map((left) => b.map((right) => !differ(left, right, options)));
|
|
1537
1896
|
const lcs = Array.from({ length: a.length + 1 }, () => Array.from({ length: b.length + 1 }, () => 0));
|
|
1538
1897
|
for (let i = a.length - 1; i >= 0; i--) for (let j = b.length - 1; j >= 0; j--) lcs[i][j] = same[i][j] ? lcs[i + 1][j + 1] + 1 : Math.max(lcs[i + 1][j], lcs[i][j + 1]);
|
|
@@ -1548,9 +1907,9 @@ function compareRecordings(expected, actual, options = {}) {
|
|
|
1548
1907
|
if (k === -1) {
|
|
1549
1908
|
changed.push({
|
|
1550
1909
|
index,
|
|
1551
|
-
reason: "missing-effect",
|
|
1910
|
+
reason: left.kind === "output" ? "output" : "missing-effect",
|
|
1552
1911
|
expected: left,
|
|
1553
|
-
message: `The port never made ${describe(left)}.`
|
|
1912
|
+
message: left.kind === "output" ? `The port never returned; the original returned ${JSON.stringify(left.value)}.` : `The port never made ${describe(left)}.`
|
|
1554
1913
|
});
|
|
1555
1914
|
continue;
|
|
1556
1915
|
}
|
|
@@ -1563,12 +1922,15 @@ function compareRecordings(expected, actual, options = {}) {
|
|
|
1563
1922
|
...difference
|
|
1564
1923
|
});
|
|
1565
1924
|
}
|
|
1566
|
-
for (const j of unpairedRights)
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1925
|
+
for (const j of unpairedRights) {
|
|
1926
|
+
const extra = b[j];
|
|
1927
|
+
divergences.push({
|
|
1928
|
+
index: lefts[0] ?? at,
|
|
1929
|
+
reason: extra.kind === "output" ? "output" : "extra-effect",
|
|
1930
|
+
actual: extra,
|
|
1931
|
+
message: extra.kind === "output" ? `The original never returned; the port returned ${JSON.stringify(extra.value)}.` : `The port made ${describe(extra)}, which the original never made.`
|
|
1932
|
+
});
|
|
1933
|
+
}
|
|
1572
1934
|
divergences.push(...changed);
|
|
1573
1935
|
lefts = [];
|
|
1574
1936
|
rights = [];
|
|
@@ -1593,11 +1955,115 @@ function compareRecordings(expected, actual, options = {}) {
|
|
|
1593
1955
|
};
|
|
1594
1956
|
}
|
|
1595
1957
|
//#endregion
|
|
1958
|
+
//#region src/check-parity.ts
|
|
1959
|
+
/**
|
|
1960
|
+
* `checkParity` — one call that records a workflow, replays its port, and
|
|
1961
|
+
* compares their effects, per scenario.
|
|
1962
|
+
* @module
|
|
1963
|
+
*/
|
|
1964
|
+
/** Each function effect wrapped with `recorder.fn`, by its key. */
|
|
1965
|
+
function recordEffects(recorder, effects) {
|
|
1966
|
+
return require_effects.mapEffects(effects, (name, effect) => recorder.fn(name, effect));
|
|
1967
|
+
}
|
|
1968
|
+
async function settle(run, recorder, normalize) {
|
|
1969
|
+
try {
|
|
1970
|
+
const output = await run();
|
|
1971
|
+
recorder.output(normalize(output));
|
|
1972
|
+
return { output };
|
|
1973
|
+
} catch (error) {
|
|
1974
|
+
return { error };
|
|
1975
|
+
}
|
|
1976
|
+
}
|
|
1977
|
+
function errorIdentity(outcome, normalize) {
|
|
1978
|
+
if (!("error" in outcome)) return void 0;
|
|
1979
|
+
const error = normalize ? normalize(outcome.error) : outcome.error;
|
|
1980
|
+
if (error instanceof Error) return `${error.name}: ${error.message}`;
|
|
1981
|
+
if (!normalize) return String(error);
|
|
1982
|
+
try {
|
|
1983
|
+
return stableStringify(require_effects.toJsonValue(error) ?? null);
|
|
1984
|
+
} catch {
|
|
1985
|
+
return String(error);
|
|
1986
|
+
}
|
|
1987
|
+
}
|
|
1988
|
+
/**
|
|
1989
|
+
* Checks a port against the workflow it replaces, scenario by scenario: runs
|
|
1990
|
+
* `before` on fresh `effects` under `createRecorder`, replays `after` against
|
|
1991
|
+
* that recording (no effect really runs), and compares the two with
|
|
1992
|
+
* `compareRecordings`. Effects are compared by name, call order, arguments
|
|
1993
|
+
* and results; the output by value.
|
|
1994
|
+
*
|
|
1995
|
+
* Never throws for a port that misbehaves: an `after` that throws (a
|
|
1996
|
+
* `RecordingDivergenceError` for a call the original never made, or its own
|
|
1997
|
+
* error) is a row with `after: { error }`. When both sides throw, they match
|
|
1998
|
+
* on the error's name and message, after `normalizeError` when given.
|
|
1999
|
+
*
|
|
2000
|
+
* @example
|
|
2001
|
+
* ```ts
|
|
2002
|
+
* const report = await checkParity({
|
|
2003
|
+
* scenarios: [{ id: "refund", input: ticket }],
|
|
2004
|
+
* effects: () => createFakeEffects(),
|
|
2005
|
+
* before: (input, effects) => legacyRunTicket(input, effects),
|
|
2006
|
+
* after: (input, effects) => step1.run(input, { effects }),
|
|
2007
|
+
* });
|
|
2008
|
+
* expect(report.equivalent).toBe(true);
|
|
2009
|
+
* ```
|
|
2010
|
+
*/
|
|
2011
|
+
async function checkParity(options) {
|
|
2012
|
+
const rows = [];
|
|
2013
|
+
for (const scenario of options.scenarios) {
|
|
2014
|
+
const recorder = createRecorder();
|
|
2015
|
+
const effects = recordEffects(recorder, options.effects(scenario));
|
|
2016
|
+
const normalize = (output) => options.normalizeOutput ? options.normalizeOutput(output, scenario) : output;
|
|
2017
|
+
const before = await settle(() => options.before(scenario.input, effects), recorder, normalize);
|
|
2018
|
+
const recording = recorder.recording;
|
|
2019
|
+
const replayer = createRecorder({ replay: recording });
|
|
2020
|
+
const stand = recordEffects(replayer, options.effects(scenario));
|
|
2021
|
+
const after = await settle(() => options.after(scenario.input, stand), replayer, normalize);
|
|
2022
|
+
const replay = replayer.recording;
|
|
2023
|
+
const { divergences } = compareRecordings(recording, replay, options.compare);
|
|
2024
|
+
const normalizeError = options.normalizeError && ((error) => options.normalizeError(error, scenario));
|
|
2025
|
+
const thrownBefore = errorIdentity(before, normalizeError);
|
|
2026
|
+
const thrownAfter = errorIdentity(after, normalizeError);
|
|
2027
|
+
if (thrownBefore !== void 0 && thrownAfter !== void 0 && thrownBefore !== thrownAfter) divergences.push({
|
|
2028
|
+
index: recording.effects.length,
|
|
2029
|
+
reason: "output",
|
|
2030
|
+
message: `The original threw ${thrownBefore}; the port threw ${thrownAfter}.`
|
|
2031
|
+
});
|
|
2032
|
+
else if (thrownBefore === void 0 !== (thrownAfter === void 0)) {
|
|
2033
|
+
const returned = (thrownBefore === void 0 ? recording : replay).effects.filter((effect) => effect.kind === "output").at(-1);
|
|
2034
|
+
const value = JSON.stringify(returned?.kind === "output" ? returned.value : void 0);
|
|
2035
|
+
const message = thrownAfter !== void 0 ? `The original returned ${value}; the port threw ${thrownAfter}.` : `The original threw ${thrownBefore}; the port returned ${value}.`;
|
|
2036
|
+
const existing = divergences.find((d) => d.reason === "output" && d.expected?.kind === "output" !== (d.actual?.kind === "output"));
|
|
2037
|
+
if (existing) existing.message = message;
|
|
2038
|
+
else divergences.push({
|
|
2039
|
+
index: recording.effects.length,
|
|
2040
|
+
reason: "output",
|
|
2041
|
+
message
|
|
2042
|
+
});
|
|
2043
|
+
}
|
|
2044
|
+
rows.push({
|
|
2045
|
+
id: scenario.id,
|
|
2046
|
+
scenario,
|
|
2047
|
+
equivalent: divergences.length === 0,
|
|
2048
|
+
divergences,
|
|
2049
|
+
before,
|
|
2050
|
+
after,
|
|
2051
|
+
recording,
|
|
2052
|
+
replay
|
|
2053
|
+
});
|
|
2054
|
+
}
|
|
2055
|
+
return {
|
|
2056
|
+
equivalent: rows.every((row) => row.equivalent),
|
|
2057
|
+
scenarios: rows
|
|
2058
|
+
};
|
|
2059
|
+
}
|
|
2060
|
+
//#endregion
|
|
1596
2061
|
exports.AgentLintError = require_verify.AgentLintError;
|
|
1597
2062
|
exports.AgentUnknownStateError = require_verify.AgentUnknownStateError;
|
|
1598
2063
|
exports.RecordingDivergenceError = RecordingDivergenceError;
|
|
1599
2064
|
exports.assertAgentMachine = require_verify.assertAgentMachine;
|
|
1600
2065
|
exports.canReach = require_verify.canReach;
|
|
2066
|
+
exports.checkParity = checkParity;
|
|
1601
2067
|
exports.compareRecordings = compareRecordings;
|
|
1602
2068
|
exports.compareRuns = compareRuns;
|
|
1603
2069
|
exports.createRecorder = createRecorder;
|
|
@@ -1608,9 +2074,12 @@ exports.getDecisionRequests = getDecisionRequests;
|
|
|
1608
2074
|
exports.judgeAgreement = judgeAgreement;
|
|
1609
2075
|
exports.lintAgentMachine = require_verify.lintAgentMachine;
|
|
1610
2076
|
exports.matchesTrajectory = matchesTrajectory;
|
|
2077
|
+
exports.proposeApprovalRules = proposeApprovalRules;
|
|
2078
|
+
exports.proposeCaps = proposeCaps;
|
|
1611
2079
|
exports.repeatRuns = repeatRuns;
|
|
1612
2080
|
exports.runSeam = runSeam;
|
|
1613
2081
|
exports.sameModelCallShape = sameModelCallShape;
|
|
1614
2082
|
exports.simulateAgent = require_verify.simulateAgent;
|
|
1615
2083
|
exports.summarizeRepeats = summarizeRepeats;
|
|
2084
|
+
exports.textOutput = require_verify.textOutput;
|
|
1616
2085
|
exports.traceCoverage = traceCoverage;
|