@statelyai/agent 2.0.0-alpha.34 → 2.0.0-alpha.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +124 -0
- package/dist/act-B9hLQls6.mjs +19 -0
- package/dist/act-CrX0JwWY.cjs +42 -0
- package/dist/ai-sdk.cjs +63 -7
- package/dist/ai-sdk.d.cts +13 -3
- package/dist/ai-sdk.d.mts +13 -3
- package/dist/ai-sdk.mjs +63 -7
- package/dist/{decision-BEsATcr1.cjs → decision-CMeuNZ1g.cjs} +262 -4
- package/dist/{decision-COkJNjOW.mjs → decision-m5hGmHpF.mjs} +239 -5
- package/dist/decision-object-B5zGSPFp.d.mts +15 -0
- package/dist/decision-object-DBcLWVvv.d.cts +15 -0
- package/dist/{run-agent-s5D32w99.cjs → effects-B23bFZEs.cjs} +948 -102
- package/dist/{run-agent-Ct_GZw_f.mjs → effects-CuOUNYtZ.mjs} +914 -104
- package/dist/{event-log-C7jeH0y8.mjs → event-log-DkVlk3Rh.mjs} +30 -15
- package/dist/{event-log-BWSSLUff.cjs → event-log-Dkc-yIYu.cjs} +41 -14
- package/dist/{event-log-store-BzXehP7R.d.cts → event-log-store-BBS8so8a.d.mts} +2 -2
- package/dist/{event-log-store-W49TCJXi.mjs → event-log-store-Brx9tOHt.mjs} +1 -1
- package/dist/{event-log-store-CleWpsp5.cjs → event-log-store-Cl-YWDnq.cjs} +1 -1
- package/dist/{event-log-store-CWjN6lmM.d.mts → event-log-store-bc4ICGcS.d.cts} +2 -2
- package/dist/index.cjs +660 -286
- package/dist/index.d.cts +168 -9
- package/dist/index.d.mts +168 -9
- package/dist/index.mjs +639 -271
- package/dist/internal-C3Lqu538.d.cts +43 -0
- package/dist/internal-DUcdhk-i.d.mts +43 -0
- package/dist/log.cjs +2 -2
- package/dist/log.d.cts +1 -1
- package/dist/log.d.mts +1 -1
- package/dist/log.mjs +2 -2
- package/dist/machines.cjs +457 -41
- package/dist/machines.d.cts +114 -43
- package/dist/machines.d.mts +114 -43
- package/dist/machines.mjs +457 -42
- package/dist/{model-source-DmhTMK39.cjs → model-source-B4WQ8Idp.cjs} +1 -1
- package/dist/{model-source-CK3F4YuU.mjs → model-source-kjGwqAV8.mjs} +1 -1
- package/dist/openai.cjs +149 -36
- package/dist/openai.d.cts +28 -10
- package/dist/openai.d.mts +28 -10
- package/dist/openai.mjs +149 -36
- package/dist/otel.d.cts +1 -1
- package/dist/otel.d.mts +1 -1
- package/dist/{run-agent-BGdx5913.d.mts → run-agent-BcQ-9NRT.d.cts} +91 -9
- package/dist/{run-agent-B1-Eqviq.d.cts → run-agent-C86drZtN.d.mts} +91 -9
- package/dist/{setup-agent-05uw4dPZ.mjs → setup-agent-BDmNr6PC.mjs} +9 -3
- package/dist/{setup-agent-BtfeFfF6.d.cts → setup-agent-BIKcbVXV.d.mts} +4 -4
- package/dist/{setup-agent-CXEBURdo.cjs → setup-agent-CmEToB8y.cjs} +9 -3
- package/dist/{setup-agent-C6nqzv_X.d.mts → setup-agent-jlo4vtaZ.d.cts} +4 -4
- package/dist/tanstack-ai.cjs +62 -4
- package/dist/tanstack-ai.d.cts +18 -6
- package/dist/tanstack-ai.d.mts +18 -6
- package/dist/tanstack-ai.mjs +62 -4
- package/dist/testing.cjs +536 -67
- package/dist/testing.d.cts +197 -180
- package/dist/testing.d.mts +197 -180
- package/dist/testing.mjs +532 -67
- package/dist/{text-logic-CV_4KGr6.d.cts → text-logic-BqxalgMr.d.mts} +87 -4
- package/dist/{text-logic-Ckh-rJFV.d.mts → text-logic-CSf9bqpP.d.cts} +87 -4
- package/dist/{types-Dp5_Ru4R.d.cts → types-DQUOOaZO.d.cts} +1 -1
- package/dist/{types-Dp5_Ru4R.d.mts → types-DQUOOaZO.d.mts} +1 -1
- package/dist/{usage-BDiJKyv9.d.cts → usage-DoP0Oi84.d.mts} +1 -1
- package/dist/{usage-DOoVUGVq.d.mts → usage-HtnrUsl9.d.cts} +1 -1
- package/dist/utils-7lSa4H-i.d.cts +403 -0
- package/dist/utils-CBbKw2fM.d.mts +403 -0
- package/dist/validate.cjs +44 -12
- package/dist/validate.d.cts +6 -6
- package/dist/validate.d.mts +6 -6
- package/dist/validate.mjs +44 -12
- package/dist/verify-Bink-Drw.cjs +2304 -0
- package/dist/{verify-BQpE8hUz.d.cts → verify-Buh8XSUr.d.mts} +208 -14
- package/dist/{verify-CqeuEmIk.d.mts → verify-CciG8UP-.d.cts} +208 -14
- package/dist/verify-Dpz1u5Pe.mjs +2251 -0
- package/dist/with-model-calls-B_WLmOSl.cjs +248 -0
- package/dist/with-model-calls-BkOwdroS.mjs +213 -0
- package/docs/act.md +102 -0
- package/docs/advanced.md +105 -0
- package/docs/any-stack.md +62 -0
- package/docs/choosing-a-run-mode.md +55 -0
- package/docs/debugging.md +29 -0
- package/docs/decisions.md +473 -0
- package/docs/environment-and-policy.md +188 -0
- package/docs/evals.md +509 -0
- package/docs/event-log.md +311 -0
- package/docs/from-a-loop.md +528 -0
- package/docs/generate-machines.md +110 -0
- package/docs/hosts.md +414 -0
- package/docs/human-in-the-loop.md +177 -0
- package/docs/index.md +48 -0
- package/docs/judgments.md +281 -0
- package/docs/langgraph-comparison.md +198 -0
- package/docs/machines-as-data.md +454 -0
- package/docs/machines-presets.md +198 -0
- package/docs/machines.md +497 -0
- package/docs/messages.md +67 -0
- package/docs/meta.json +52 -0
- package/docs/models-and-providers.md +268 -0
- package/docs/multi-agent.md +68 -0
- package/docs/observability.md +109 -0
- package/docs/patterns.md +31 -0
- package/docs/persistence.md +136 -0
- package/docs/quickstart.md +121 -0
- package/docs/roadmap.md +22 -0
- package/docs/scope.md +75 -0
- package/docs/statechart-policy-examples.md +69 -0
- package/docs/steps.md +54 -0
- package/docs/strangler-fig.md +323 -0
- package/docs/text-requests.md +250 -0
- package/docs/thinking-in-state-machines.md +324 -0
- package/docs/tools.md +249 -0
- package/docs/upgrading.md +37 -0
- package/docs/usage-and-budgets.md +516 -0
- package/docs/validate-and-repair.md +253 -0
- package/docs/verify.md +487 -0
- package/docs/xstate-4-and-5-apps.md +161 -0
- package/llms.txt +53 -0
- package/package.json +10 -3
- package/readme.md +30 -15
- package/skills/add-evals/SKILL.md +3 -1
- package/skills/add-traces/SKILL.md +6 -1
- package/skills/build-machine/SKILL.md +81 -0
- package/skills/generate-machine/SKILL.md +1 -1
- package/skills/graph-engineering/SKILL.md +13 -3
- package/skills/improve-machine/SKILL.md +21 -2
- package/skills/loop-to-machine/SKILL.md +147 -190
- package/skills/loop-to-machine/reference.md +260 -0
- package/dist/utils-BnXdWih3.d.mts +0 -108
- package/dist/utils-ebVpCYoF.d.cts +0 -108
- package/dist/verify-Bpahmr34.mjs +0 -1241
- package/dist/verify-DCLZa9R0.cjs +0 -1288
- package/dist/with-model-calls-BXYdBdcu.cjs +0 -45
- package/dist/with-model-calls-Bv4V4EHC.mjs +0 -34
package/dist/testing.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { a as explorePaths, c as simulateAgent, i as canReach, n as AgentUnknownStateError, o as isExplorationComplete, r as assertAgentMachine, s as lintAgentMachine, t as AgentLintError } from "./verify-
|
|
1
|
+
import { L as getCallUsage, d as canonicalEventJson, ft as isError, ht as AgentError, mt as isRecord, pt as isPlainObject, rt as getStateMeta, z as normalizeGeneratorResult } from "./event-log-DkVlk3Rh.mjs";
|
|
2
|
+
import { _ as runToQuiescence, d as createAgentRuntime, n as mapEffects, t as isAsyncIterable, y as toJsonValue } from "./effects-CuOUNYtZ.mjs";
|
|
3
|
+
import { a as explorePaths, c as simulateAgent, i as canReach, l as textOutput, n as AgentUnknownStateError, o as isExplorationComplete, r as assertAgentMachine, s as lintAgentMachine, t as AgentLintError } from "./verify-Dpz1u5Pe.mjs";
|
|
4
4
|
import { getInitialMicrosteps, getMicrosteps } from "xstate";
|
|
5
5
|
//#region src/trajectory.ts
|
|
6
6
|
/**
|
|
@@ -790,6 +790,134 @@ function initialValue(trace, steps) {
|
|
|
790
790
|
return steps[0]?.fromValue;
|
|
791
791
|
}
|
|
792
792
|
//#endregion
|
|
793
|
+
//#region src/propose-from-traces.ts
|
|
794
|
+
/**
|
|
795
|
+
* Groups the approval verdicts in many runs' traces by `key` and proposes an
|
|
796
|
+
* `allow` or `deny` rule for each. Pure and deterministic; a proposal is for
|
|
797
|
+
* a person to review, and nothing here changes a machine.
|
|
798
|
+
*
|
|
799
|
+
* Reads `machine.transition` events: a verdict is a transition, not ignored,
|
|
800
|
+
* out of `approvalState` on an approve or reject event. Rows come in the
|
|
801
|
+
* order keys were first seen.
|
|
802
|
+
*
|
|
803
|
+
* @example
|
|
804
|
+
* ```ts no-check
|
|
805
|
+
* const rows = proposeApprovalRules(traces, {
|
|
806
|
+
* approvalState: "awaitingApproval",
|
|
807
|
+
* approveEvents: ["APPROVE"],
|
|
808
|
+
* rejectEvents: ["REJECT"],
|
|
809
|
+
* key: ({ snapshot }) => `${snapshot.context.pending.tool}:${snapshot.context.pending.path}`,
|
|
810
|
+
* });
|
|
811
|
+
* ```
|
|
812
|
+
*/
|
|
813
|
+
function proposeApprovalRules(traces, options) {
|
|
814
|
+
const rows = /* @__PURE__ */ new Map();
|
|
815
|
+
const rowFor = (key) => {
|
|
816
|
+
let row = rows.get(key);
|
|
817
|
+
if (!row) {
|
|
818
|
+
row = {
|
|
819
|
+
key,
|
|
820
|
+
approved: 0,
|
|
821
|
+
rejected: 0,
|
|
822
|
+
violations: 0,
|
|
823
|
+
runs: [],
|
|
824
|
+
proposal: null,
|
|
825
|
+
support: 0
|
|
826
|
+
};
|
|
827
|
+
rows.set(key, row);
|
|
828
|
+
}
|
|
829
|
+
return row;
|
|
830
|
+
};
|
|
831
|
+
const inState = (value) => statePaths(value).some((path) => path === options.approvalState || path.startsWith(`${options.approvalState}.`));
|
|
832
|
+
let index = 0;
|
|
833
|
+
for (const trace of traces) {
|
|
834
|
+
const run = index++;
|
|
835
|
+
const seen = /* @__PURE__ */ new Set();
|
|
836
|
+
let snapshot;
|
|
837
|
+
for (const event of trace) {
|
|
838
|
+
if (event.type !== "machine.transition") continue;
|
|
839
|
+
const before = snapshot;
|
|
840
|
+
snapshot = event.snapshot;
|
|
841
|
+
const type = isRecord(event.event) ? event.event.type : void 0;
|
|
842
|
+
if (event.ignored === true || typeof type !== "string" || !inState(valueOf(event.from))) continue;
|
|
843
|
+
const approve = options.approveEvents.includes(type);
|
|
844
|
+
if (!approve && !options.rejectEvents.includes(type)) continue;
|
|
845
|
+
const key = options.key({
|
|
846
|
+
event: event.event,
|
|
847
|
+
snapshot: before,
|
|
848
|
+
trace
|
|
849
|
+
});
|
|
850
|
+
if (key === void 0) continue;
|
|
851
|
+
const row = rowFor(key);
|
|
852
|
+
if (approve) row.approved++;
|
|
853
|
+
else row.rejected++;
|
|
854
|
+
if (!seen.has(key)) {
|
|
855
|
+
seen.add(key);
|
|
856
|
+
row.runs.push(run);
|
|
857
|
+
}
|
|
858
|
+
}
|
|
859
|
+
for (const key of options.violations?.(trace) ?? []) {
|
|
860
|
+
const row = rows.get(key);
|
|
861
|
+
if (row) row.violations++;
|
|
862
|
+
}
|
|
863
|
+
}
|
|
864
|
+
for (const row of rows.values()) {
|
|
865
|
+
row.support = row.runs.length;
|
|
866
|
+
row.proposal = row.approved > 0 && row.rejected === 0 && row.violations === 0 ? "allow" : row.rejected > 0 && row.approved === 0 ? "deny" : null;
|
|
867
|
+
}
|
|
868
|
+
return [...rows.values()];
|
|
869
|
+
}
|
|
870
|
+
/**
|
|
871
|
+
* Proposes a value for each cap (retry count, steps, batch size) from many
|
|
872
|
+
* runs' traces: the 95th percentile of what passing runs used, plus one, and
|
|
873
|
+
* never below the current cap while runs still hit it. Pure and
|
|
874
|
+
* deterministic; a proposal is for a person to review, and nothing here
|
|
875
|
+
* changes a machine.
|
|
876
|
+
*
|
|
877
|
+
* @example
|
|
878
|
+
* ```ts no-check
|
|
879
|
+
* const [retries] = proposeCaps(traces, {
|
|
880
|
+
* caps: [{ name: "retries", current: 3, failureState: "gaveUp", valueFromRun: countRetries }],
|
|
881
|
+
* });
|
|
882
|
+
* ```
|
|
883
|
+
*/
|
|
884
|
+
function proposeCaps(traces, options) {
|
|
885
|
+
const runs = [...traces];
|
|
886
|
+
return options.caps.map((cap) => {
|
|
887
|
+
let hits = 0;
|
|
888
|
+
let max = null;
|
|
889
|
+
const passing = [];
|
|
890
|
+
for (const trace of runs) {
|
|
891
|
+
const hit = hitCap(trace, cap);
|
|
892
|
+
if (hit) hits++;
|
|
893
|
+
const value = cap.valueFromRun(trace);
|
|
894
|
+
if (value === void 0) continue;
|
|
895
|
+
max = max === null ? value : Math.max(max, value);
|
|
896
|
+
if (!hit) passing.push(value);
|
|
897
|
+
}
|
|
898
|
+
passing.sort((a, b) => a - b);
|
|
899
|
+
const p95 = passing.length ? passing[Math.ceil(.95 * passing.length) - 1] : null;
|
|
900
|
+
const proposal = p95 === null ? null : hits > 0 && cap.current !== void 0 ? Math.max(p95 + 1, cap.current) : p95 + 1;
|
|
901
|
+
return {
|
|
902
|
+
name: cap.name,
|
|
903
|
+
hits,
|
|
904
|
+
passing: passing.length,
|
|
905
|
+
max,
|
|
906
|
+
p95,
|
|
907
|
+
proposal
|
|
908
|
+
};
|
|
909
|
+
});
|
|
910
|
+
}
|
|
911
|
+
function hitCap(trace, cap) {
|
|
912
|
+
for (const event of trace) {
|
|
913
|
+
if (cap.failureState !== void 0 && event.type === "machine.transition" && statePaths(valueOf(event.snapshot)).some((path) => path === cap.failureState || path.startsWith(`${cap.failureState}.`))) return true;
|
|
914
|
+
if (cap.failureOutput && event.type === "run.end" && event.status === "done") {
|
|
915
|
+
if (cap.failureOutput(event.output)) return true;
|
|
916
|
+
}
|
|
917
|
+
}
|
|
918
|
+
return false;
|
|
919
|
+
}
|
|
920
|
+
//#endregion
|
|
793
921
|
//#region src/decision-requests.ts
|
|
794
922
|
/**
|
|
795
923
|
* The decisions a run asked for, read from its trace: each decision's first
|
|
@@ -1063,8 +1191,8 @@ async function compareRuns(options) {
|
|
|
1063
1191
|
regression: 0
|
|
1064
1192
|
};
|
|
1065
1193
|
for (const scenario of options.scenarios) {
|
|
1066
|
-
const baseline = await settle(() => options.baseline(scenario));
|
|
1067
|
-
const candidate = await settle(() => options.candidate(scenario));
|
|
1194
|
+
const baseline = await settle$1(() => options.baseline(scenario));
|
|
1195
|
+
const candidate = await settle$1(() => options.candidate(scenario));
|
|
1068
1196
|
const { label, reason } = labelRun(scenario.expected, baseline, candidate);
|
|
1069
1197
|
counts[label]++;
|
|
1070
1198
|
scenarios.push({
|
|
@@ -1082,7 +1210,7 @@ async function compareRuns(options) {
|
|
|
1082
1210
|
ok: counts.regression === 0
|
|
1083
1211
|
};
|
|
1084
1212
|
}
|
|
1085
|
-
async function settle(run) {
|
|
1213
|
+
async function settle$1(run) {
|
|
1086
1214
|
try {
|
|
1087
1215
|
return await run();
|
|
1088
1216
|
} catch (error) {
|
|
@@ -1135,6 +1263,37 @@ function structurallyEqual(a, b) {
|
|
|
1135
1263
|
//#endregion
|
|
1136
1264
|
//#region src/equivalence.ts
|
|
1137
1265
|
/**
|
|
1266
|
+
* Verified equivalence — prove a port behaves like the code it replaces.
|
|
1267
|
+
*
|
|
1268
|
+
* A port is equivalent when, given the same model answers and the same tool
|
|
1269
|
+
* results, it produces the same OBSERVABLE EFFECTS: the same number of model
|
|
1270
|
+
* calls, the same tool calls with the same arguments, and the same output.
|
|
1271
|
+
* Prompts, internal steps, and states are free to differ.
|
|
1272
|
+
*
|
|
1273
|
+
* 1. Record the original: wrap its model (`recorder.model`, an AI SDK
|
|
1274
|
+
* `LanguageModelV3`, or `recorder.openai`, an `openai` client) and its tools
|
|
1275
|
+
* (`recorder.tool` / `recorder.tools`), run a scenario, call
|
|
1276
|
+
* `recorder.output(result)`, and save `recorder.recording`.
|
|
1277
|
+
* 2. Replay it on the port: `createRecorder({ replay: recording })` answers
|
|
1278
|
+
* every model call and tool call from the recording, with no API key and no
|
|
1279
|
+
* side effects, and records what the port did.
|
|
1280
|
+
* 3. `compareRecordings(original, port)` reports the first effect that differs.
|
|
1281
|
+
*
|
|
1282
|
+
* A port that changes how it calls the model on purpose (a model-picked action
|
|
1283
|
+
* that became `agent.decide`, a stream that became `doGenerate`) still replays:
|
|
1284
|
+
* `translate` turns the recorded answer into one that fits, `fallback` answers
|
|
1285
|
+
* the calls the original never made, and `compareRecordings(…, { kinds })`
|
|
1286
|
+
* compares only tool calls and output.
|
|
1287
|
+
*
|
|
1288
|
+
* Any other client (`@anthropic-ai/sdk`, `@google/genai`, a framework's own
|
|
1289
|
+
* model client) records through `recorder.client` or `recorder.fn`: each call
|
|
1290
|
+
* is a `call` effect, answered in replay by name and order.
|
|
1291
|
+
*
|
|
1292
|
+
* Provider-free: every seam is wrapped structurally, so this module imports
|
|
1293
|
+
* neither `ai` nor `openai`.
|
|
1294
|
+
* @module
|
|
1295
|
+
*/
|
|
1296
|
+
/**
|
|
1138
1297
|
* Thrown in replay when the port asks for something the recording cannot
|
|
1139
1298
|
* answer: a model call past the last recorded one, or a tool call the original
|
|
1140
1299
|
* never made. The effect is still recorded, so {@link compareRecordings}
|
|
@@ -1144,24 +1303,45 @@ var RecordingDivergenceError = class extends Error {
|
|
|
1144
1303
|
name = "RecordingDivergenceError";
|
|
1145
1304
|
};
|
|
1146
1305
|
function toRecordedError(error) {
|
|
1147
|
-
if (error instanceof Error) {
|
|
1148
|
-
const code = error.code;
|
|
1149
|
-
return code === void 0 ? {
|
|
1150
|
-
name: error.name,
|
|
1151
|
-
message: error.message
|
|
1152
|
-
} : {
|
|
1153
|
-
name: error.name,
|
|
1154
|
-
message: error.message,
|
|
1155
|
-
code: toJson(code)
|
|
1156
|
-
};
|
|
1157
|
-
}
|
|
1158
|
-
return {
|
|
1306
|
+
if (!(error instanceof Error)) return {
|
|
1159
1307
|
name: "Error",
|
|
1160
1308
|
message: String(error)
|
|
1161
1309
|
};
|
|
1310
|
+
const recorded = {
|
|
1311
|
+
name: error.name,
|
|
1312
|
+
message: error.message
|
|
1313
|
+
};
|
|
1314
|
+
if (Object.prototype.propertyIsEnumerable.call(error, "name")) recorded.ownName = true;
|
|
1315
|
+
const code = error.code;
|
|
1316
|
+
if (code !== void 0) recorded.code = toJson(code);
|
|
1317
|
+
const data = Object.fromEntries(Object.entries(error).filter(([key, value]) => key !== "code" && (typeof value === "number" || typeof value === "boolean")));
|
|
1318
|
+
if (Object.keys(data).length) recorded.data = data;
|
|
1319
|
+
const markers = /* @__PURE__ */ new Set();
|
|
1320
|
+
for (let holder = error; holder; holder = Object.getPrototypeOf(holder)) for (const symbol of Object.getOwnPropertySymbols(holder)) {
|
|
1321
|
+
const key = Symbol.keyFor(symbol);
|
|
1322
|
+
if (key !== void 0 && error[symbol] === true) markers.add(key);
|
|
1323
|
+
}
|
|
1324
|
+
if (markers.size) recorded.markers = [...markers];
|
|
1325
|
+
return recorded;
|
|
1162
1326
|
}
|
|
1163
|
-
function fromRecordedError(
|
|
1164
|
-
|
|
1327
|
+
function fromRecordedError({ data, markers, name, ownName, ...fields }) {
|
|
1328
|
+
const error = Object.assign(new Error(fields.message), fields, data);
|
|
1329
|
+
if (ownName || name !== "Error") Object.defineProperty(error, "name", {
|
|
1330
|
+
value: name,
|
|
1331
|
+
writable: true,
|
|
1332
|
+
configurable: true,
|
|
1333
|
+
enumerable: ownName ?? false
|
|
1334
|
+
});
|
|
1335
|
+
for (const marker of markers ?? []) error[Symbol.for(marker)] = true;
|
|
1336
|
+
return error;
|
|
1337
|
+
}
|
|
1338
|
+
/** What two runs compare of an error: its data and markers may differ. */
|
|
1339
|
+
function errorIdentity$1(error) {
|
|
1340
|
+
return error && {
|
|
1341
|
+
name: error.name,
|
|
1342
|
+
message: error.message,
|
|
1343
|
+
code: error.code
|
|
1344
|
+
};
|
|
1165
1345
|
}
|
|
1166
1346
|
/** A JSON-safe copy: Dates become ISO strings, Errors become {@link RecordedError}. */
|
|
1167
1347
|
function toJson(value) {
|
|
@@ -1186,7 +1366,10 @@ function omit(value, keys) {
|
|
|
1186
1366
|
if (!value || typeof value !== "object") return value;
|
|
1187
1367
|
return Object.fromEntries(Object.entries(value).filter(([key]) => !keys.includes(key)));
|
|
1188
1368
|
}
|
|
1189
|
-
/**
|
|
1369
|
+
/**
|
|
1370
|
+
* Key-order-independent JSON, for equality.
|
|
1371
|
+
* @internal
|
|
1372
|
+
*/
|
|
1190
1373
|
function stableStringify(value) {
|
|
1191
1374
|
if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
|
|
1192
1375
|
if (value && typeof value === "object") return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`).join(",")}}`;
|
|
@@ -1212,19 +1395,44 @@ function createRecorder(options = {}) {
|
|
|
1212
1395
|
const effects = [];
|
|
1213
1396
|
const replay = options.replay?.effects;
|
|
1214
1397
|
const replayedModels = replay?.filter((effect) => effect.kind === "model");
|
|
1398
|
+
const usedModels = /* @__PURE__ */ new Set();
|
|
1215
1399
|
const usedTools = /* @__PURE__ */ new Set();
|
|
1216
|
-
const
|
|
1400
|
+
const usedCalls = /* @__PURE__ */ new Set();
|
|
1401
|
+
const callCounts = /* @__PURE__ */ new Map();
|
|
1402
|
+
const normalizeEffect = (effect, list) => options.normalize?.(effect, list.indexOf(effect), list) ?? effect;
|
|
1217
1403
|
let modelCalls = 0;
|
|
1218
1404
|
let toolsInFlight = 0;
|
|
1405
|
+
/**
|
|
1406
|
+
* The earliest unused recorded call that fits this one: one `translate`
|
|
1407
|
+
* adapts, or one with the same method and shape. Not by position, so a call
|
|
1408
|
+
* the port added or dropped does not shift every answer after it.
|
|
1409
|
+
*/
|
|
1219
1410
|
function nextReplayedModel(effect) {
|
|
1220
|
-
const recorded
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1411
|
+
for (const recorded of replayedModels) {
|
|
1412
|
+
if (usedModels.has(recorded)) continue;
|
|
1413
|
+
const translated = options.translate?.(structuredClone(recorded), effect);
|
|
1414
|
+
if (translated) {
|
|
1415
|
+
effect.translated = true;
|
|
1416
|
+
return {
|
|
1417
|
+
recorded,
|
|
1418
|
+
answer: translated
|
|
1419
|
+
};
|
|
1420
|
+
}
|
|
1421
|
+
if (recorded.method === effect.method && modelCallsMatch(options.modelCalls, recorded, effect)) return {
|
|
1422
|
+
recorded,
|
|
1423
|
+
answer: recorded
|
|
1424
|
+
};
|
|
1425
|
+
}
|
|
1426
|
+
}
|
|
1427
|
+
function noAnswerMessage(effect) {
|
|
1428
|
+
const unused = replayedModels.filter((recorded) => !usedModels.has(recorded));
|
|
1429
|
+
return unused.length ? `Model call #${effect.call} (${effect.method}, ${JSON.stringify(modelCallShape(effect))}) fits no unused recorded call: ${unused.map((recorded) => `#${recorded.call} (${recorded.method}, ${JSON.stringify(modelCallShape(recorded))})`).join(", ")}. A recorded answer only fits a call of the same method and shape; pass \`translate\` to adapt one.` : `Model call #${effect.call} was never made by the recorded run (it made ${replayedModels.length}).`;
|
|
1225
1430
|
}
|
|
1226
|
-
/**
|
|
1227
|
-
|
|
1431
|
+
/**
|
|
1432
|
+
* Records a model call around `run`, or answers it from the recording.
|
|
1433
|
+
* `fallback` runs it on the stand-in when the recording has no answer.
|
|
1434
|
+
*/
|
|
1435
|
+
async function modelCall(seam, method, request, streamed, run, fallback) {
|
|
1228
1436
|
const effect = {
|
|
1229
1437
|
kind: "model",
|
|
1230
1438
|
seam,
|
|
@@ -1238,24 +1446,30 @@ function createRecorder(options = {}) {
|
|
|
1238
1446
|
};
|
|
1239
1447
|
effects.push(effect);
|
|
1240
1448
|
if (replay) {
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
recorded =
|
|
1244
|
-
|
|
1449
|
+
const replayed = nextReplayedModel(effect);
|
|
1450
|
+
if (replayed) {
|
|
1451
|
+
const { recorded, answer } = replayed;
|
|
1452
|
+
usedModels.add(recorded);
|
|
1453
|
+
effect.replayed = recorded.call;
|
|
1454
|
+
if (answer.error) {
|
|
1455
|
+
effect.error = answer.error;
|
|
1456
|
+
throw fromRecordedError(answer.error);
|
|
1457
|
+
}
|
|
1458
|
+
effect.response = answer.response;
|
|
1459
|
+
effect.stream = answer.stream;
|
|
1460
|
+
if (seam === "openai") return streamed ? iterate(answer.stream ?? []) : structuredClone(answer.response);
|
|
1461
|
+
return streamed ? {
|
|
1462
|
+
...reviveTimestamps(structuredClone(answer.response) ?? {}),
|
|
1463
|
+
stream: streamFrom(answer.stream ?? [])
|
|
1464
|
+
} : reviveTimestamps(structuredClone(answer.response));
|
|
1465
|
+
}
|
|
1466
|
+
if (!fallback) {
|
|
1467
|
+
const error = new RecordingDivergenceError(noAnswerMessage(effect));
|
|
1245
1468
|
effect.error = toRecordedError(error);
|
|
1246
1469
|
throw error;
|
|
1247
1470
|
}
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
throw fromRecordedError(recorded.error);
|
|
1251
|
-
}
|
|
1252
|
-
effect.response = recorded.response;
|
|
1253
|
-
effect.stream = recorded.stream;
|
|
1254
|
-
if (seam === "openai") return streamed ? iterate(recorded.stream ?? []) : structuredClone(recorded.response);
|
|
1255
|
-
return streamed ? {
|
|
1256
|
-
...reviveTimestamps(structuredClone(recorded.response) ?? {}),
|
|
1257
|
-
stream: streamFrom(recorded.stream ?? [])
|
|
1258
|
-
} : reviveTimestamps(structuredClone(recorded.response));
|
|
1471
|
+
effect.unrecorded = true;
|
|
1472
|
+
run = fallback;
|
|
1259
1473
|
}
|
|
1260
1474
|
let result;
|
|
1261
1475
|
try {
|
|
@@ -1306,8 +1520,13 @@ function createRecorder(options = {}) {
|
|
|
1306
1520
|
function call(effect, args) {
|
|
1307
1521
|
const { input } = effect;
|
|
1308
1522
|
if (replay && options.tools !== "live") {
|
|
1309
|
-
const wanted = normalizeEffect(effect).input;
|
|
1310
|
-
const
|
|
1523
|
+
const wanted = normalizeEffect(effect, effects).input;
|
|
1524
|
+
const matches = (candidate) => candidate.kind === "tool" && !usedTools.has(candidate) && candidate.name === name && jsonEqual(normalizeEffect(candidate, replay).input, wanted);
|
|
1525
|
+
const recorded = replay.find(matches);
|
|
1526
|
+
if (!recorded && options.fallback?.tools) {
|
|
1527
|
+
effect.unrecorded = true;
|
|
1528
|
+
return runLive(effect, args);
|
|
1529
|
+
}
|
|
1311
1530
|
const settle = (value) => {
|
|
1312
1531
|
if (recorded?.sync) return value();
|
|
1313
1532
|
try {
|
|
@@ -1318,7 +1537,7 @@ function createRecorder(options = {}) {
|
|
|
1318
1537
|
};
|
|
1319
1538
|
return settle(() => {
|
|
1320
1539
|
if (!recorded) {
|
|
1321
|
-
const error = new RecordingDivergenceError(`Tool '${name}' was called with ${JSON.stringify(input)}
|
|
1540
|
+
const error = new RecordingDivergenceError(`Tool '${name}' was called with ${JSON.stringify(input)}; the recorded run made no such call.`);
|
|
1322
1541
|
effect.error = toRecordedError(error);
|
|
1323
1542
|
throw error;
|
|
1324
1543
|
}
|
|
@@ -1332,6 +1551,9 @@ function createRecorder(options = {}) {
|
|
|
1332
1551
|
return structuredClone(recorded.output);
|
|
1333
1552
|
});
|
|
1334
1553
|
}
|
|
1554
|
+
return runLive(effect, args);
|
|
1555
|
+
}
|
|
1556
|
+
function runLive(effect, args) {
|
|
1335
1557
|
let output;
|
|
1336
1558
|
try {
|
|
1337
1559
|
output = fn(...args);
|
|
@@ -1354,6 +1576,68 @@ function createRecorder(options = {}) {
|
|
|
1354
1576
|
});
|
|
1355
1577
|
}
|
|
1356
1578
|
}
|
|
1579
|
+
function fn(name, run, callOptions = {}) {
|
|
1580
|
+
const { redact, serialize = toJson, revive } = callOptions;
|
|
1581
|
+
return (async (...args) => {
|
|
1582
|
+
const passed = [...args];
|
|
1583
|
+
while (passed.length && passed.at(-1) === void 0) passed.pop();
|
|
1584
|
+
const call = callCounts.get(name) ?? 0;
|
|
1585
|
+
callCounts.set(name, call + 1);
|
|
1586
|
+
const effect = {
|
|
1587
|
+
kind: "call",
|
|
1588
|
+
name,
|
|
1589
|
+
call,
|
|
1590
|
+
args: toJson(redact ? redact(passed) : passed)
|
|
1591
|
+
};
|
|
1592
|
+
effects.push(effect);
|
|
1593
|
+
if (replay) {
|
|
1594
|
+
const recorded = replay.find((candidate) => candidate.kind === "call" && candidate.name === name && !usedCalls.has(candidate));
|
|
1595
|
+
if (!recorded) {
|
|
1596
|
+
const error = new RecordingDivergenceError(`Call #${call} to '${name}' was never made by the recorded run.`);
|
|
1597
|
+
effect.error = toRecordedError(error);
|
|
1598
|
+
throw error;
|
|
1599
|
+
}
|
|
1600
|
+
usedCalls.add(recorded);
|
|
1601
|
+
if (recorded.error) {
|
|
1602
|
+
effect.error = recorded.error;
|
|
1603
|
+
throw fromRecordedError(recorded.error);
|
|
1604
|
+
}
|
|
1605
|
+
if (recorded.stream) {
|
|
1606
|
+
effect.stream = recorded.stream;
|
|
1607
|
+
const chunks = structuredClone(recorded.stream);
|
|
1608
|
+
return revive ? revive(chunks) : iterate(chunks);
|
|
1609
|
+
}
|
|
1610
|
+
effect.result = recorded.result;
|
|
1611
|
+
const result = structuredClone(recorded.result);
|
|
1612
|
+
return revive ? revive(result) : result;
|
|
1613
|
+
}
|
|
1614
|
+
let result;
|
|
1615
|
+
try {
|
|
1616
|
+
result = await run(...args);
|
|
1617
|
+
} catch (error) {
|
|
1618
|
+
effect.error = toRecordedError(error);
|
|
1619
|
+
throw error;
|
|
1620
|
+
}
|
|
1621
|
+
if (!isAsyncIterable(result)) {
|
|
1622
|
+
effect.result = toJson(serialize(result));
|
|
1623
|
+
return result;
|
|
1624
|
+
}
|
|
1625
|
+
const chunks = [];
|
|
1626
|
+
effect.stream = chunks;
|
|
1627
|
+
const source = result;
|
|
1628
|
+
return (async function* () {
|
|
1629
|
+
try {
|
|
1630
|
+
for await (const chunk of source) {
|
|
1631
|
+
chunks.push(toJson(serialize(chunk)));
|
|
1632
|
+
yield chunk;
|
|
1633
|
+
}
|
|
1634
|
+
} catch (error) {
|
|
1635
|
+
effect.error = toRecordedError(error);
|
|
1636
|
+
throw error;
|
|
1637
|
+
}
|
|
1638
|
+
})();
|
|
1639
|
+
});
|
|
1640
|
+
}
|
|
1357
1641
|
/** Counts an async tool call as running until it settles. */
|
|
1358
1642
|
function track(result) {
|
|
1359
1643
|
if (result instanceof Promise) {
|
|
@@ -1368,9 +1652,18 @@ function createRecorder(options = {}) {
|
|
|
1368
1652
|
return {
|
|
1369
1653
|
model(model) {
|
|
1370
1654
|
const base = model;
|
|
1655
|
+
const stand = options.fallback?.model;
|
|
1371
1656
|
const wrapped = Object.create(base);
|
|
1372
|
-
|
|
1373
|
-
|
|
1657
|
+
for (const method of [
|
|
1658
|
+
"doGenerate",
|
|
1659
|
+
"doStream",
|
|
1660
|
+
"doDecide",
|
|
1661
|
+
"doEvaluate"
|
|
1662
|
+
]) {
|
|
1663
|
+
if (typeof base[method] !== "function") continue;
|
|
1664
|
+
const invoke = (target, callOptions) => target[method].call(target, callOptions);
|
|
1665
|
+
wrapped[method] = (callOptions) => modelCall("ai-sdk", method, callOptions, method === "doStream", () => invoke(base, callOptions), stand && typeof stand[method] === "function" ? () => invoke(stand, callOptions) : void 0);
|
|
1666
|
+
}
|
|
1374
1667
|
return wrapped;
|
|
1375
1668
|
},
|
|
1376
1669
|
openai(client) {
|
|
@@ -1378,10 +1671,12 @@ function createRecorder(options = {}) {
|
|
|
1378
1671
|
if (key !== "create") return Reflect.get(target, key, receiver);
|
|
1379
1672
|
return (params, requestOptions) => {
|
|
1380
1673
|
let api;
|
|
1381
|
-
const
|
|
1382
|
-
api = Reflect.get(
|
|
1674
|
+
const create = (resource) => () => {
|
|
1675
|
+
api = Reflect.get(resource, "create").call(resource, params, requestOptions);
|
|
1383
1676
|
return Promise.resolve(api);
|
|
1384
|
-
}
|
|
1677
|
+
};
|
|
1678
|
+
const stand = fallbackResource(method);
|
|
1679
|
+
const promise = modelCall("openai", method, params, params?.stream === true, create(target), stand && create(stand));
|
|
1385
1680
|
return Object.assign(promise, { withResponse: async () => {
|
|
1386
1681
|
const data = await promise;
|
|
1387
1682
|
const meta = await api?.withResponse?.();
|
|
@@ -1393,6 +1688,11 @@ function createRecorder(options = {}) {
|
|
|
1393
1688
|
} });
|
|
1394
1689
|
};
|
|
1395
1690
|
} });
|
|
1691
|
+
const fallbackResource = (method) => {
|
|
1692
|
+
let value = options.fallback?.openai;
|
|
1693
|
+
for (const key of method.split(".").slice(0, -1)) value = value?.[key];
|
|
1694
|
+
return value && typeof value === "object" ? value : void 0;
|
|
1695
|
+
};
|
|
1396
1696
|
const wrapped = /* @__PURE__ */ new Map();
|
|
1397
1697
|
const wrap = (key, make) => {
|
|
1398
1698
|
if (!wrapped.has(key)) wrapped.set(key, make());
|
|
@@ -1409,6 +1709,26 @@ function createRecorder(options = {}) {
|
|
|
1409
1709
|
} });
|
|
1410
1710
|
},
|
|
1411
1711
|
tool: (name, fn) => tool(name, fn),
|
|
1712
|
+
fn,
|
|
1713
|
+
client(name, client, { methods, ...callOptions }) {
|
|
1714
|
+
const paths = methods.map((method) => method.split("."));
|
|
1715
|
+
const startsWith = (path, prefix) => prefix.every((key, index) => path[index] === key);
|
|
1716
|
+
const wrapAt = (target, prefix) => {
|
|
1717
|
+
const wrapped = /* @__PURE__ */ new Map();
|
|
1718
|
+
return new Proxy(target, { get(object, key, receiver) {
|
|
1719
|
+
if (typeof key !== "string") return Reflect.get(object, key, receiver);
|
|
1720
|
+
const path = [...prefix, key];
|
|
1721
|
+
const method = paths.some((p) => p.length === path.length && startsWith(p, path));
|
|
1722
|
+
if (!method && !paths.some((p) => startsWith(p, path))) return Reflect.get(object, key, receiver);
|
|
1723
|
+
if (!wrapped.has(key)) {
|
|
1724
|
+
const value = Reflect.get(object, key, receiver);
|
|
1725
|
+
wrapped.set(key, method ? fn(`${name}.${path.join(".")}`, (...args) => value.apply(object, args), callOptions) : wrapAt(value && typeof value === "object" ? value : {}, path));
|
|
1726
|
+
}
|
|
1727
|
+
return wrapped.get(key);
|
|
1728
|
+
} });
|
|
1729
|
+
};
|
|
1730
|
+
return wrapAt(client, []);
|
|
1731
|
+
},
|
|
1412
1732
|
tools(tools) {
|
|
1413
1733
|
return Object.fromEntries(Object.entries(tools).map(([name, definition]) => [name, definition.execute ? {
|
|
1414
1734
|
...definition,
|
|
@@ -1441,9 +1761,27 @@ function modelCallsMatch(match, expected, actual) {
|
|
|
1441
1761
|
function shapeMessage(expected, actual) {
|
|
1442
1762
|
return `Model call #${expected.call} changed shape: ${JSON.stringify(modelCallShape(expected))} → ${JSON.stringify(modelCallShape(actual))}. A recorded answer only fits a call of the same shape.`;
|
|
1443
1763
|
}
|
|
1444
|
-
/**
|
|
1764
|
+
/**
|
|
1765
|
+
* Offered tool names and the response format, or a decision's questions and
|
|
1766
|
+
* their options: what a recorded answer depends on.
|
|
1767
|
+
*/
|
|
1445
1768
|
function modelCallShape(effect) {
|
|
1446
1769
|
const request = effect.request ?? {};
|
|
1770
|
+
const questions = request["questions"];
|
|
1771
|
+
if (questions && typeof questions === "object") return { questions: Object.entries(questions).map(([name, question]) => {
|
|
1772
|
+
const criteria = question["criteria"];
|
|
1773
|
+
if (question["type"] === "choice" && criteria && typeof criteria === "object") return [
|
|
1774
|
+
name,
|
|
1775
|
+
question["type"],
|
|
1776
|
+
Object.keys(criteria).sort()
|
|
1777
|
+
];
|
|
1778
|
+
if (question["type"] === "score" && Array.isArray(criteria)) return [
|
|
1779
|
+
name,
|
|
1780
|
+
question["type"],
|
|
1781
|
+
criteria.length
|
|
1782
|
+
];
|
|
1783
|
+
return [name, question["type"]];
|
|
1784
|
+
}).sort((x, y) => String(x[0]).localeCompare(String(y[0]))) };
|
|
1447
1785
|
return {
|
|
1448
1786
|
tools: (Array.isArray(request["tools"]) ? request["tools"] : []).map((tool) => tool["name"] ?? tool["function"]?.["name"]).sort(),
|
|
1449
1787
|
format: request["responseFormat"]?.["type"] ?? request["response_format"]?.["type"] ?? (request["text"]?.["format"])?.["type"] ?? "text"
|
|
@@ -1474,6 +1812,7 @@ function describe(effect) {
|
|
|
1474
1812
|
switch (effect.kind) {
|
|
1475
1813
|
case "model": return `model call #${effect.call} (${effect.method})`;
|
|
1476
1814
|
case "tool": return `tool '${effect.name}' with ${JSON.stringify(effect.input)}`;
|
|
1815
|
+
case "call": return `call #${effect.call} to '${effect.name}' with ${JSON.stringify(effect.args)}`;
|
|
1477
1816
|
case "output": return `output ${JSON.stringify(effect.value)}`;
|
|
1478
1817
|
}
|
|
1479
1818
|
}
|
|
@@ -1484,6 +1823,7 @@ function differ(left, right, options) {
|
|
|
1484
1823
|
message: `Expected ${describe(left)}; the port made ${describe(right)}.`
|
|
1485
1824
|
};
|
|
1486
1825
|
if (left.kind === "model" && right.kind === "model") {
|
|
1826
|
+
if (right.translated && right.replayed === left.call) return void 0;
|
|
1487
1827
|
if (left.method !== right.method) return {
|
|
1488
1828
|
reason: "model-method",
|
|
1489
1829
|
message: `Model call #${left.call} used ${left.method}; the port used ${right.method}.`
|
|
@@ -1492,7 +1832,7 @@ function differ(left, right, options) {
|
|
|
1492
1832
|
reason: "model-call-shape",
|
|
1493
1833
|
message: shapeMessage(left, right)
|
|
1494
1834
|
};
|
|
1495
|
-
if (!jsonEqual(left.error, right.error)) {
|
|
1835
|
+
if (!jsonEqual(errorIdentity$1(left.error), errorIdentity$1(right.error))) {
|
|
1496
1836
|
const outcome = (error) => error ? `failed with ${error.name}: ${error.message}` : "succeeded";
|
|
1497
1837
|
return {
|
|
1498
1838
|
reason: "model-error",
|
|
@@ -1510,12 +1850,27 @@ function differ(left, right, options) {
|
|
|
1510
1850
|
reason: "tool-input",
|
|
1511
1851
|
message: `Expected ${describe(left)}; the port passed ${JSON.stringify(right.input)}.`
|
|
1512
1852
|
};
|
|
1513
|
-
if (!jsonEqual(left.output, right.output) || !jsonEqual(left.error, right.error)) return {
|
|
1853
|
+
if (!jsonEqual(left.output, right.output) || !jsonEqual(errorIdentity$1(left.error), errorIdentity$1(right.error))) return {
|
|
1514
1854
|
reason: "tool-result",
|
|
1515
1855
|
message: `Tool '${left.name}' returned a different result.`
|
|
1516
1856
|
};
|
|
1517
1857
|
return;
|
|
1518
1858
|
}
|
|
1859
|
+
if (left.kind === "call" && right.kind === "call") {
|
|
1860
|
+
if (left.name !== right.name) return {
|
|
1861
|
+
reason: "call-name",
|
|
1862
|
+
message: `Expected ${describe(left)}; the port made ${describe(right)}.`
|
|
1863
|
+
};
|
|
1864
|
+
if (!jsonEqual(left.args, right.args)) return {
|
|
1865
|
+
reason: "call-args",
|
|
1866
|
+
message: `Expected ${describe(left)}; the port passed ${JSON.stringify(right.args)}.`
|
|
1867
|
+
};
|
|
1868
|
+
if (!jsonEqual(left.result, right.result) || !jsonEqual(left.stream, right.stream) || !jsonEqual(errorIdentity$1(left.error), errorIdentity$1(right.error))) return {
|
|
1869
|
+
reason: "call-result",
|
|
1870
|
+
message: `Call #${left.call} to '${left.name}' returned a different result.`
|
|
1871
|
+
};
|
|
1872
|
+
return;
|
|
1873
|
+
}
|
|
1519
1874
|
if (left.kind === "output" && right.kind === "output" && !jsonEqual(left.value, right.value)) return {
|
|
1520
1875
|
reason: "output",
|
|
1521
1876
|
message: `Expected ${describe(left)}; the port returned ${JSON.stringify(right.value)}.`
|
|
@@ -1529,9 +1884,13 @@ function differ(left, right, options) {
|
|
|
1529
1884
|
* kind pair up and report what changed, and the rest are missing or extra.
|
|
1530
1885
|
*/
|
|
1531
1886
|
function compareRecordings(expected, actual, options = {}) {
|
|
1532
|
-
const
|
|
1533
|
-
|
|
1534
|
-
|
|
1887
|
+
const select = (effects) => normalize(effects.flatMap((effect, index) => {
|
|
1888
|
+
if (options.kinds && !options.kinds.includes(effect.kind)) return [];
|
|
1889
|
+
const mapped = options.normalize ? options.normalize(effect, index, effects) : effect;
|
|
1890
|
+
return mapped ? [mapped] : [];
|
|
1891
|
+
}));
|
|
1892
|
+
const a = select(expected.effects);
|
|
1893
|
+
const b = select(actual.effects);
|
|
1535
1894
|
const same = a.map((left) => b.map((right) => !differ(left, right, options)));
|
|
1536
1895
|
const lcs = Array.from({ length: a.length + 1 }, () => Array.from({ length: b.length + 1 }, () => 0));
|
|
1537
1896
|
for (let i = a.length - 1; i >= 0; i--) for (let j = b.length - 1; j >= 0; j--) lcs[i][j] = same[i][j] ? lcs[i + 1][j + 1] + 1 : Math.max(lcs[i + 1][j], lcs[i][j + 1]);
|
|
@@ -1547,9 +1906,9 @@ function compareRecordings(expected, actual, options = {}) {
|
|
|
1547
1906
|
if (k === -1) {
|
|
1548
1907
|
changed.push({
|
|
1549
1908
|
index,
|
|
1550
|
-
reason: "missing-effect",
|
|
1909
|
+
reason: left.kind === "output" ? "output" : "missing-effect",
|
|
1551
1910
|
expected: left,
|
|
1552
|
-
message: `The port never made ${describe(left)}.`
|
|
1911
|
+
message: left.kind === "output" ? `The port never returned; the original returned ${JSON.stringify(left.value)}.` : `The port never made ${describe(left)}.`
|
|
1553
1912
|
});
|
|
1554
1913
|
continue;
|
|
1555
1914
|
}
|
|
@@ -1562,12 +1921,15 @@ function compareRecordings(expected, actual, options = {}) {
|
|
|
1562
1921
|
...difference
|
|
1563
1922
|
});
|
|
1564
1923
|
}
|
|
1565
|
-
for (const j of unpairedRights)
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1924
|
+
for (const j of unpairedRights) {
|
|
1925
|
+
const extra = b[j];
|
|
1926
|
+
divergences.push({
|
|
1927
|
+
index: lefts[0] ?? at,
|
|
1928
|
+
reason: extra.kind === "output" ? "output" : "extra-effect",
|
|
1929
|
+
actual: extra,
|
|
1930
|
+
message: extra.kind === "output" ? `The original never returned; the port returned ${JSON.stringify(extra.value)}.` : `The port made ${describe(extra)}, which the original never made.`
|
|
1931
|
+
});
|
|
1932
|
+
}
|
|
1571
1933
|
divergences.push(...changed);
|
|
1572
1934
|
lefts = [];
|
|
1573
1935
|
rights = [];
|
|
@@ -1592,4 +1954,107 @@ function compareRecordings(expected, actual, options = {}) {
|
|
|
1592
1954
|
};
|
|
1593
1955
|
}
|
|
1594
1956
|
//#endregion
|
|
1595
|
-
|
|
1957
|
+
//#region src/check-parity.ts
|
|
1958
|
+
/**
|
|
1959
|
+
* `checkParity` — one call that records a workflow, replays its port, and
|
|
1960
|
+
* compares their effects, per scenario.
|
|
1961
|
+
* @module
|
|
1962
|
+
*/
|
|
1963
|
+
/** Each function effect wrapped with `recorder.fn`, by its key. */
|
|
1964
|
+
function recordEffects(recorder, effects) {
|
|
1965
|
+
return mapEffects(effects, (name, effect) => recorder.fn(name, effect));
|
|
1966
|
+
}
|
|
1967
|
+
async function settle(run, recorder, normalize) {
|
|
1968
|
+
try {
|
|
1969
|
+
const output = await run();
|
|
1970
|
+
recorder.output(normalize(output));
|
|
1971
|
+
return { output };
|
|
1972
|
+
} catch (error) {
|
|
1973
|
+
return { error };
|
|
1974
|
+
}
|
|
1975
|
+
}
|
|
1976
|
+
function errorIdentity(outcome, normalize) {
|
|
1977
|
+
if (!("error" in outcome)) return void 0;
|
|
1978
|
+
const error = normalize ? normalize(outcome.error) : outcome.error;
|
|
1979
|
+
if (error instanceof Error) return `${error.name}: ${error.message}`;
|
|
1980
|
+
if (!normalize) return String(error);
|
|
1981
|
+
try {
|
|
1982
|
+
return stableStringify(toJsonValue(error) ?? null);
|
|
1983
|
+
} catch {
|
|
1984
|
+
return String(error);
|
|
1985
|
+
}
|
|
1986
|
+
}
|
|
1987
|
+
/**
|
|
1988
|
+
* Checks a port against the workflow it replaces, scenario by scenario: runs
|
|
1989
|
+
* `before` on fresh `effects` under `createRecorder`, replays `after` against
|
|
1990
|
+
* that recording (no effect really runs), and compares the two with
|
|
1991
|
+
* `compareRecordings`. Effects are compared by name, call order, arguments
|
|
1992
|
+
* and results; the output by value.
|
|
1993
|
+
*
|
|
1994
|
+
* Never throws for a port that misbehaves: an `after` that throws (a
|
|
1995
|
+
* `RecordingDivergenceError` for a call the original never made, or its own
|
|
1996
|
+
* error) is a row with `after: { error }`. When both sides throw, they match
|
|
1997
|
+
* on the error's name and message, after `normalizeError` when given.
|
|
1998
|
+
*
|
|
1999
|
+
* @example
|
|
2000
|
+
* ```ts
|
|
2001
|
+
* const report = await checkParity({
|
|
2002
|
+
* scenarios: [{ id: "refund", input: ticket }],
|
|
2003
|
+
* effects: () => createFakeEffects(),
|
|
2004
|
+
* before: (input, effects) => legacyRunTicket(input, effects),
|
|
2005
|
+
* after: (input, effects) => step1.run(input, { effects }),
|
|
2006
|
+
* });
|
|
2007
|
+
* expect(report.equivalent).toBe(true);
|
|
2008
|
+
* ```
|
|
2009
|
+
*/
|
|
2010
|
+
async function checkParity(options) {
|
|
2011
|
+
const rows = [];
|
|
2012
|
+
for (const scenario of options.scenarios) {
|
|
2013
|
+
const recorder = createRecorder();
|
|
2014
|
+
const effects = recordEffects(recorder, options.effects(scenario));
|
|
2015
|
+
const normalize = (output) => options.normalizeOutput ? options.normalizeOutput(output, scenario) : output;
|
|
2016
|
+
const before = await settle(() => options.before(scenario.input, effects), recorder, normalize);
|
|
2017
|
+
const recording = recorder.recording;
|
|
2018
|
+
const replayer = createRecorder({ replay: recording });
|
|
2019
|
+
const stand = recordEffects(replayer, options.effects(scenario));
|
|
2020
|
+
const after = await settle(() => options.after(scenario.input, stand), replayer, normalize);
|
|
2021
|
+
const replay = replayer.recording;
|
|
2022
|
+
const { divergences } = compareRecordings(recording, replay, options.compare);
|
|
2023
|
+
const normalizeError = options.normalizeError && ((error) => options.normalizeError(error, scenario));
|
|
2024
|
+
const thrownBefore = errorIdentity(before, normalizeError);
|
|
2025
|
+
const thrownAfter = errorIdentity(after, normalizeError);
|
|
2026
|
+
if (thrownBefore !== void 0 && thrownAfter !== void 0 && thrownBefore !== thrownAfter) divergences.push({
|
|
2027
|
+
index: recording.effects.length,
|
|
2028
|
+
reason: "output",
|
|
2029
|
+
message: `The original threw ${thrownBefore}; the port threw ${thrownAfter}.`
|
|
2030
|
+
});
|
|
2031
|
+
else if (thrownBefore === void 0 !== (thrownAfter === void 0)) {
|
|
2032
|
+
const returned = (thrownBefore === void 0 ? recording : replay).effects.filter((effect) => effect.kind === "output").at(-1);
|
|
2033
|
+
const value = JSON.stringify(returned?.kind === "output" ? returned.value : void 0);
|
|
2034
|
+
const message = thrownAfter !== void 0 ? `The original returned ${value}; the port threw ${thrownAfter}.` : `The original threw ${thrownBefore}; the port returned ${value}.`;
|
|
2035
|
+
const existing = divergences.find((d) => d.reason === "output" && d.expected?.kind === "output" !== (d.actual?.kind === "output"));
|
|
2036
|
+
if (existing) existing.message = message;
|
|
2037
|
+
else divergences.push({
|
|
2038
|
+
index: recording.effects.length,
|
|
2039
|
+
reason: "output",
|
|
2040
|
+
message
|
|
2041
|
+
});
|
|
2042
|
+
}
|
|
2043
|
+
rows.push({
|
|
2044
|
+
id: scenario.id,
|
|
2045
|
+
scenario,
|
|
2046
|
+
equivalent: divergences.length === 0,
|
|
2047
|
+
divergences,
|
|
2048
|
+
before,
|
|
2049
|
+
after,
|
|
2050
|
+
recording,
|
|
2051
|
+
replay
|
|
2052
|
+
});
|
|
2053
|
+
}
|
|
2054
|
+
return {
|
|
2055
|
+
equivalent: rows.every((row) => row.equivalent),
|
|
2056
|
+
scenarios: rows
|
|
2057
|
+
};
|
|
2058
|
+
}
|
|
2059
|
+
//#endregion
|
|
2060
|
+
export { AgentLintError, AgentUnknownStateError, RecordingDivergenceError, assertAgentMachine, canReach, checkParity, compareRecordings, compareRuns, createRecorder, explorePaths, firstDivergence, foldTraces, getDecisionRequests, judgeAgreement, lintAgentMachine, matchesTrajectory, proposeApprovalRules, proposeCaps, repeatRuns, runSeam, sameModelCallShape, simulateAgent, summarizeRepeats, textOutput, traceCoverage };
|