@statelyai/agent 2.0.0-alpha.35 → 2.0.0-alpha.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +111 -0
- package/dist/act-B9hLQls6.mjs +19 -0
- package/dist/act-CrX0JwWY.cjs +42 -0
- package/dist/ai-sdk.cjs +63 -7
- package/dist/ai-sdk.d.cts +13 -3
- package/dist/ai-sdk.d.mts +13 -3
- package/dist/ai-sdk.mjs +63 -7
- package/dist/{decision-BEsATcr1.cjs → decision-CMeuNZ1g.cjs} +262 -4
- package/dist/{decision-COkJNjOW.mjs → decision-m5hGmHpF.mjs} +239 -5
- package/dist/decision-object-B5zGSPFp.d.mts +15 -0
- package/dist/decision-object-DBcLWVvv.d.cts +15 -0
- package/dist/{run-agent-s5D32w99.cjs → effects-B23bFZEs.cjs} +948 -102
- package/dist/{run-agent-Ct_GZw_f.mjs → effects-CuOUNYtZ.mjs} +914 -104
- package/dist/{event-log-C7jeH0y8.mjs → event-log-DkVlk3Rh.mjs} +30 -15
- package/dist/{event-log-BWSSLUff.cjs → event-log-Dkc-yIYu.cjs} +41 -14
- package/dist/{event-log-store-BzXehP7R.d.cts → event-log-store-BBS8so8a.d.mts} +2 -2
- package/dist/{event-log-store-W49TCJXi.mjs → event-log-store-Brx9tOHt.mjs} +1 -1
- package/dist/{event-log-store-CleWpsp5.cjs → event-log-store-Cl-YWDnq.cjs} +1 -1
- package/dist/{event-log-store-CWjN6lmM.d.mts → event-log-store-bc4ICGcS.d.cts} +2 -2
- package/dist/index.cjs +660 -286
- package/dist/index.d.cts +168 -9
- package/dist/index.d.mts +168 -9
- package/dist/index.mjs +639 -271
- package/dist/internal-C3Lqu538.d.cts +43 -0
- package/dist/internal-DUcdhk-i.d.mts +43 -0
- package/dist/log.cjs +2 -2
- package/dist/log.d.cts +1 -1
- package/dist/log.d.mts +1 -1
- package/dist/log.mjs +2 -2
- package/dist/machines.cjs +457 -41
- package/dist/machines.d.cts +114 -43
- package/dist/machines.d.mts +114 -43
- package/dist/machines.mjs +457 -42
- package/dist/{model-source-DmhTMK39.cjs → model-source-B4WQ8Idp.cjs} +1 -1
- package/dist/{model-source-CK3F4YuU.mjs → model-source-kjGwqAV8.mjs} +1 -1
- package/dist/openai.cjs +149 -36
- package/dist/openai.d.cts +28 -10
- package/dist/openai.d.mts +28 -10
- package/dist/openai.mjs +149 -36
- package/dist/otel.d.cts +1 -1
- package/dist/otel.d.mts +1 -1
- package/dist/{run-agent-BGdx5913.d.mts → run-agent-BcQ-9NRT.d.cts} +91 -9
- package/dist/{run-agent-B1-Eqviq.d.cts → run-agent-C86drZtN.d.mts} +91 -9
- package/dist/{setup-agent-05uw4dPZ.mjs → setup-agent-BDmNr6PC.mjs} +9 -3
- package/dist/{setup-agent-BtfeFfF6.d.cts → setup-agent-BIKcbVXV.d.mts} +4 -4
- package/dist/{setup-agent-CXEBURdo.cjs → setup-agent-CmEToB8y.cjs} +9 -3
- package/dist/{setup-agent-C6nqzv_X.d.mts → setup-agent-jlo4vtaZ.d.cts} +4 -4
- package/dist/tanstack-ai.cjs +62 -4
- package/dist/tanstack-ai.d.cts +18 -6
- package/dist/tanstack-ai.d.mts +18 -6
- package/dist/tanstack-ai.mjs +62 -4
- package/dist/testing.cjs +399 -20
- package/dist/testing.d.cts +196 -255
- package/dist/testing.d.mts +196 -255
- package/dist/testing.mjs +395 -20
- package/dist/{text-logic-CV_4KGr6.d.cts → text-logic-BqxalgMr.d.mts} +87 -4
- package/dist/{text-logic-Ckh-rJFV.d.mts → text-logic-CSf9bqpP.d.cts} +87 -4
- package/dist/{types-Dp5_Ru4R.d.cts → types-DQUOOaZO.d.cts} +1 -1
- package/dist/{types-Dp5_Ru4R.d.mts → types-DQUOOaZO.d.mts} +1 -1
- package/dist/{usage-BDiJKyv9.d.cts → usage-DoP0Oi84.d.mts} +1 -1
- package/dist/{usage-DOoVUGVq.d.mts → usage-HtnrUsl9.d.cts} +1 -1
- package/dist/utils-7lSa4H-i.d.cts +403 -0
- package/dist/utils-CBbKw2fM.d.mts +403 -0
- package/dist/validate.cjs +44 -12
- package/dist/validate.d.cts +6 -6
- package/dist/validate.d.mts +6 -6
- package/dist/validate.mjs +44 -12
- package/dist/verify-Bink-Drw.cjs +2304 -0
- package/dist/{verify-BQpE8hUz.d.cts → verify-Buh8XSUr.d.mts} +208 -14
- package/dist/{verify-CqeuEmIk.d.mts → verify-CciG8UP-.d.cts} +208 -14
- package/dist/verify-Dpz1u5Pe.mjs +2251 -0
- package/dist/with-model-calls-B_WLmOSl.cjs +248 -0
- package/dist/with-model-calls-BkOwdroS.mjs +213 -0
- package/docs/act.md +102 -0
- package/docs/advanced.md +105 -0
- package/docs/any-stack.md +62 -0
- package/docs/choosing-a-run-mode.md +55 -0
- package/docs/debugging.md +29 -0
- package/docs/decisions.md +473 -0
- package/docs/environment-and-policy.md +188 -0
- package/docs/evals.md +509 -0
- package/docs/event-log.md +311 -0
- package/docs/from-a-loop.md +528 -0
- package/docs/generate-machines.md +110 -0
- package/docs/hosts.md +414 -0
- package/docs/human-in-the-loop.md +177 -0
- package/docs/index.md +48 -0
- package/docs/judgments.md +281 -0
- package/docs/langgraph-comparison.md +198 -0
- package/docs/machines-as-data.md +454 -0
- package/docs/machines-presets.md +198 -0
- package/docs/machines.md +497 -0
- package/docs/messages.md +67 -0
- package/docs/meta.json +52 -0
- package/docs/models-and-providers.md +268 -0
- package/docs/multi-agent.md +68 -0
- package/docs/observability.md +109 -0
- package/docs/patterns.md +31 -0
- package/docs/persistence.md +136 -0
- package/docs/quickstart.md +121 -0
- package/docs/roadmap.md +22 -0
- package/docs/scope.md +75 -0
- package/docs/statechart-policy-examples.md +69 -0
- package/docs/steps.md +54 -0
- package/docs/strangler-fig.md +323 -0
- package/docs/text-requests.md +250 -0
- package/docs/thinking-in-state-machines.md +324 -0
- package/docs/tools.md +249 -0
- package/docs/upgrading.md +37 -0
- package/docs/usage-and-budgets.md +516 -0
- package/docs/validate-and-repair.md +253 -0
- package/docs/verify.md +487 -0
- package/docs/xstate-4-and-5-apps.md +161 -0
- package/llms.txt +53 -0
- package/package.json +10 -3
- package/readme.md +30 -15
- package/skills/add-evals/SKILL.md +3 -1
- package/skills/add-traces/SKILL.md +6 -1
- package/skills/build-machine/SKILL.md +81 -0
- package/skills/generate-machine/SKILL.md +1 -1
- package/skills/graph-engineering/SKILL.md +13 -3
- package/skills/improve-machine/SKILL.md +21 -2
- package/skills/loop-to-machine/SKILL.md +147 -192
- package/skills/loop-to-machine/reference.md +260 -0
- package/dist/utils-BnXdWih3.d.mts +0 -108
- package/dist/utils-ebVpCYoF.d.cts +0 -108
- package/dist/verify-Bpahmr34.mjs +0 -1241
- package/dist/verify-DCLZa9R0.cjs +0 -1288
- package/dist/with-model-calls-BXYdBdcu.cjs +0 -45
- package/dist/with-model-calls-Bv4V4EHC.mjs +0 -34
package/dist/testing.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { a as explorePaths, c as simulateAgent, i as canReach, n as AgentUnknownStateError, o as isExplorationComplete, r as assertAgentMachine, s as lintAgentMachine, t as AgentLintError } from "./verify-
|
|
1
|
+
import { L as getCallUsage, d as canonicalEventJson, ft as isError, ht as AgentError, mt as isRecord, pt as isPlainObject, rt as getStateMeta, z as normalizeGeneratorResult } from "./event-log-DkVlk3Rh.mjs";
|
|
2
|
+
import { _ as runToQuiescence, d as createAgentRuntime, n as mapEffects, t as isAsyncIterable, y as toJsonValue } from "./effects-CuOUNYtZ.mjs";
|
|
3
|
+
import { a as explorePaths, c as simulateAgent, i as canReach, l as textOutput, n as AgentUnknownStateError, o as isExplorationComplete, r as assertAgentMachine, s as lintAgentMachine, t as AgentLintError } from "./verify-Dpz1u5Pe.mjs";
|
|
4
4
|
import { getInitialMicrosteps, getMicrosteps } from "xstate";
|
|
5
5
|
//#region src/trajectory.ts
|
|
6
6
|
/**
|
|
@@ -790,6 +790,134 @@ function initialValue(trace, steps) {
|
|
|
790
790
|
return steps[0]?.fromValue;
|
|
791
791
|
}
|
|
792
792
|
//#endregion
|
|
793
|
+
//#region src/propose-from-traces.ts
|
|
794
|
+
/**
|
|
795
|
+
* Groups the approval verdicts in many runs' traces by `key` and proposes an
|
|
796
|
+
* `allow` or `deny` rule for each. Pure and deterministic; a proposal is for
|
|
797
|
+
* a person to review, and nothing here changes a machine.
|
|
798
|
+
*
|
|
799
|
+
* Reads `machine.transition` events: a verdict is a transition, not ignored,
|
|
800
|
+
* out of `approvalState` on an approve or reject event. Rows come in the
|
|
801
|
+
* order keys were first seen.
|
|
802
|
+
*
|
|
803
|
+
* @example
|
|
804
|
+
* ```ts no-check
|
|
805
|
+
* const rows = proposeApprovalRules(traces, {
|
|
806
|
+
* approvalState: "awaitingApproval",
|
|
807
|
+
* approveEvents: ["APPROVE"],
|
|
808
|
+
* rejectEvents: ["REJECT"],
|
|
809
|
+
* key: ({ snapshot }) => `${snapshot.context.pending.tool}:${snapshot.context.pending.path}`,
|
|
810
|
+
* });
|
|
811
|
+
* ```
|
|
812
|
+
*/
|
|
813
|
+
function proposeApprovalRules(traces, options) {
|
|
814
|
+
const rows = /* @__PURE__ */ new Map();
|
|
815
|
+
const rowFor = (key) => {
|
|
816
|
+
let row = rows.get(key);
|
|
817
|
+
if (!row) {
|
|
818
|
+
row = {
|
|
819
|
+
key,
|
|
820
|
+
approved: 0,
|
|
821
|
+
rejected: 0,
|
|
822
|
+
violations: 0,
|
|
823
|
+
runs: [],
|
|
824
|
+
proposal: null,
|
|
825
|
+
support: 0
|
|
826
|
+
};
|
|
827
|
+
rows.set(key, row);
|
|
828
|
+
}
|
|
829
|
+
return row;
|
|
830
|
+
};
|
|
831
|
+
const inState = (value) => statePaths(value).some((path) => path === options.approvalState || path.startsWith(`${options.approvalState}.`));
|
|
832
|
+
let index = 0;
|
|
833
|
+
for (const trace of traces) {
|
|
834
|
+
const run = index++;
|
|
835
|
+
const seen = /* @__PURE__ */ new Set();
|
|
836
|
+
let snapshot;
|
|
837
|
+
for (const event of trace) {
|
|
838
|
+
if (event.type !== "machine.transition") continue;
|
|
839
|
+
const before = snapshot;
|
|
840
|
+
snapshot = event.snapshot;
|
|
841
|
+
const type = isRecord(event.event) ? event.event.type : void 0;
|
|
842
|
+
if (event.ignored === true || typeof type !== "string" || !inState(valueOf(event.from))) continue;
|
|
843
|
+
const approve = options.approveEvents.includes(type);
|
|
844
|
+
if (!approve && !options.rejectEvents.includes(type)) continue;
|
|
845
|
+
const key = options.key({
|
|
846
|
+
event: event.event,
|
|
847
|
+
snapshot: before,
|
|
848
|
+
trace
|
|
849
|
+
});
|
|
850
|
+
if (key === void 0) continue;
|
|
851
|
+
const row = rowFor(key);
|
|
852
|
+
if (approve) row.approved++;
|
|
853
|
+
else row.rejected++;
|
|
854
|
+
if (!seen.has(key)) {
|
|
855
|
+
seen.add(key);
|
|
856
|
+
row.runs.push(run);
|
|
857
|
+
}
|
|
858
|
+
}
|
|
859
|
+
for (const key of options.violations?.(trace) ?? []) {
|
|
860
|
+
const row = rows.get(key);
|
|
861
|
+
if (row) row.violations++;
|
|
862
|
+
}
|
|
863
|
+
}
|
|
864
|
+
for (const row of rows.values()) {
|
|
865
|
+
row.support = row.runs.length;
|
|
866
|
+
row.proposal = row.approved > 0 && row.rejected === 0 && row.violations === 0 ? "allow" : row.rejected > 0 && row.approved === 0 ? "deny" : null;
|
|
867
|
+
}
|
|
868
|
+
return [...rows.values()];
|
|
869
|
+
}
|
|
870
|
+
/**
|
|
871
|
+
* Proposes a value for each cap (retry count, steps, batch size) from many
|
|
872
|
+
* runs' traces: the 95th percentile of what passing runs used, plus one, and
|
|
873
|
+
* never below the current cap while runs still hit it. Pure and
|
|
874
|
+
* deterministic; a proposal is for a person to review, and nothing here
|
|
875
|
+
* changes a machine.
|
|
876
|
+
*
|
|
877
|
+
* @example
|
|
878
|
+
* ```ts no-check
|
|
879
|
+
* const [retries] = proposeCaps(traces, {
|
|
880
|
+
* caps: [{ name: "retries", current: 3, failureState: "gaveUp", valueFromRun: countRetries }],
|
|
881
|
+
* });
|
|
882
|
+
* ```
|
|
883
|
+
*/
|
|
884
|
+
function proposeCaps(traces, options) {
|
|
885
|
+
const runs = [...traces];
|
|
886
|
+
return options.caps.map((cap) => {
|
|
887
|
+
let hits = 0;
|
|
888
|
+
let max = null;
|
|
889
|
+
const passing = [];
|
|
890
|
+
for (const trace of runs) {
|
|
891
|
+
const hit = hitCap(trace, cap);
|
|
892
|
+
if (hit) hits++;
|
|
893
|
+
const value = cap.valueFromRun(trace);
|
|
894
|
+
if (value === void 0) continue;
|
|
895
|
+
max = max === null ? value : Math.max(max, value);
|
|
896
|
+
if (!hit) passing.push(value);
|
|
897
|
+
}
|
|
898
|
+
passing.sort((a, b) => a - b);
|
|
899
|
+
const p95 = passing.length ? passing[Math.ceil(.95 * passing.length) - 1] : null;
|
|
900
|
+
const proposal = p95 === null ? null : hits > 0 && cap.current !== void 0 ? Math.max(p95 + 1, cap.current) : p95 + 1;
|
|
901
|
+
return {
|
|
902
|
+
name: cap.name,
|
|
903
|
+
hits,
|
|
904
|
+
passing: passing.length,
|
|
905
|
+
max,
|
|
906
|
+
p95,
|
|
907
|
+
proposal
|
|
908
|
+
};
|
|
909
|
+
});
|
|
910
|
+
}
|
|
911
|
+
function hitCap(trace, cap) {
|
|
912
|
+
for (const event of trace) {
|
|
913
|
+
if (cap.failureState !== void 0 && event.type === "machine.transition" && statePaths(valueOf(event.snapshot)).some((path) => path === cap.failureState || path.startsWith(`${cap.failureState}.`))) return true;
|
|
914
|
+
if (cap.failureOutput && event.type === "run.end" && event.status === "done") {
|
|
915
|
+
if (cap.failureOutput(event.output)) return true;
|
|
916
|
+
}
|
|
917
|
+
}
|
|
918
|
+
return false;
|
|
919
|
+
}
|
|
920
|
+
//#endregion
|
|
793
921
|
//#region src/decision-requests.ts
|
|
794
922
|
/**
|
|
795
923
|
* The decisions a run asked for, read from its trace: each decision's first
|
|
@@ -1063,8 +1191,8 @@ async function compareRuns(options) {
|
|
|
1063
1191
|
regression: 0
|
|
1064
1192
|
};
|
|
1065
1193
|
for (const scenario of options.scenarios) {
|
|
1066
|
-
const baseline = await settle(() => options.baseline(scenario));
|
|
1067
|
-
const candidate = await settle(() => options.candidate(scenario));
|
|
1194
|
+
const baseline = await settle$1(() => options.baseline(scenario));
|
|
1195
|
+
const candidate = await settle$1(() => options.candidate(scenario));
|
|
1068
1196
|
const { label, reason } = labelRun(scenario.expected, baseline, candidate);
|
|
1069
1197
|
counts[label]++;
|
|
1070
1198
|
scenarios.push({
|
|
@@ -1082,7 +1210,7 @@ async function compareRuns(options) {
|
|
|
1082
1210
|
ok: counts.regression === 0
|
|
1083
1211
|
};
|
|
1084
1212
|
}
|
|
1085
|
-
async function settle(run) {
|
|
1213
|
+
async function settle$1(run) {
|
|
1086
1214
|
try {
|
|
1087
1215
|
return await run();
|
|
1088
1216
|
} catch (error) {
|
|
@@ -1135,6 +1263,37 @@ function structurallyEqual(a, b) {
|
|
|
1135
1263
|
//#endregion
|
|
1136
1264
|
//#region src/equivalence.ts
|
|
1137
1265
|
/**
|
|
1266
|
+
* Verified equivalence — prove a port behaves like the code it replaces.
|
|
1267
|
+
*
|
|
1268
|
+
* A port is equivalent when, given the same model answers and the same tool
|
|
1269
|
+
* results, it produces the same OBSERVABLE EFFECTS: the same number of model
|
|
1270
|
+
* calls, the same tool calls with the same arguments, and the same output.
|
|
1271
|
+
* Prompts, internal steps, and states are free to differ.
|
|
1272
|
+
*
|
|
1273
|
+
* 1. Record the original: wrap its model (`recorder.model`, an AI SDK
|
|
1274
|
+
* `LanguageModelV3`, or `recorder.openai`, an `openai` client) and its tools
|
|
1275
|
+
* (`recorder.tool` / `recorder.tools`), run a scenario, call
|
|
1276
|
+
* `recorder.output(result)`, and save `recorder.recording`.
|
|
1277
|
+
* 2. Replay it on the port: `createRecorder({ replay: recording })` answers
|
|
1278
|
+
* every model call and tool call from the recording, with no API key and no
|
|
1279
|
+
* side effects, and records what the port did.
|
|
1280
|
+
* 3. `compareRecordings(original, port)` reports the first effect that differs.
|
|
1281
|
+
*
|
|
1282
|
+
* A port that changes how it calls the model on purpose (a model-picked action
|
|
1283
|
+
* that became `agent.decide`, a stream that became `doGenerate`) still replays:
|
|
1284
|
+
* `translate` turns the recorded answer into one that fits, `fallback` answers
|
|
1285
|
+
* the calls the original never made, and `compareRecordings(…, { kinds })`
|
|
1286
|
+
* compares only tool calls and output.
|
|
1287
|
+
*
|
|
1288
|
+
* Any other client (`@anthropic-ai/sdk`, `@google/genai`, a framework's own
|
|
1289
|
+
* model client) records through `recorder.client` or `recorder.fn`: each call
|
|
1290
|
+
* is a `call` effect, answered in replay by name and order.
|
|
1291
|
+
*
|
|
1292
|
+
* Provider-free: every seam is wrapped structurally, so this module imports
|
|
1293
|
+
* neither `ai` nor `openai`.
|
|
1294
|
+
* @module
|
|
1295
|
+
*/
|
|
1296
|
+
/**
|
|
1138
1297
|
* Thrown in replay when the port asks for something the recording cannot
|
|
1139
1298
|
* answer: a model call past the last recorded one, or a tool call the original
|
|
1140
1299
|
* never made. The effect is still recorded, so {@link compareRecordings}
|
|
@@ -1152,6 +1311,7 @@ function toRecordedError(error) {
|
|
|
1152
1311
|
name: error.name,
|
|
1153
1312
|
message: error.message
|
|
1154
1313
|
};
|
|
1314
|
+
if (Object.prototype.propertyIsEnumerable.call(error, "name")) recorded.ownName = true;
|
|
1155
1315
|
const code = error.code;
|
|
1156
1316
|
if (code !== void 0) recorded.code = toJson(code);
|
|
1157
1317
|
const data = Object.fromEntries(Object.entries(error).filter(([key, value]) => key !== "code" && (typeof value === "number" || typeof value === "boolean")));
|
|
@@ -1164,13 +1324,19 @@ function toRecordedError(error) {
|
|
|
1164
1324
|
if (markers.size) recorded.markers = [...markers];
|
|
1165
1325
|
return recorded;
|
|
1166
1326
|
}
|
|
1167
|
-
function fromRecordedError({ data, markers, ...fields }) {
|
|
1327
|
+
function fromRecordedError({ data, markers, name, ownName, ...fields }) {
|
|
1168
1328
|
const error = Object.assign(new Error(fields.message), fields, data);
|
|
1329
|
+
if (ownName || name !== "Error") Object.defineProperty(error, "name", {
|
|
1330
|
+
value: name,
|
|
1331
|
+
writable: true,
|
|
1332
|
+
configurable: true,
|
|
1333
|
+
enumerable: ownName ?? false
|
|
1334
|
+
});
|
|
1169
1335
|
for (const marker of markers ?? []) error[Symbol.for(marker)] = true;
|
|
1170
1336
|
return error;
|
|
1171
1337
|
}
|
|
1172
1338
|
/** What two runs compare of an error: its data and markers may differ. */
|
|
1173
|
-
function errorIdentity(error) {
|
|
1339
|
+
function errorIdentity$1(error) {
|
|
1174
1340
|
return error && {
|
|
1175
1341
|
name: error.name,
|
|
1176
1342
|
message: error.message,
|
|
@@ -1200,7 +1366,10 @@ function omit(value, keys) {
|
|
|
1200
1366
|
if (!value || typeof value !== "object") return value;
|
|
1201
1367
|
return Object.fromEntries(Object.entries(value).filter(([key]) => !keys.includes(key)));
|
|
1202
1368
|
}
|
|
1203
|
-
/**
|
|
1369
|
+
/**
|
|
1370
|
+
* Key-order-independent JSON, for equality.
|
|
1371
|
+
* @internal
|
|
1372
|
+
*/
|
|
1204
1373
|
function stableStringify(value) {
|
|
1205
1374
|
if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
|
|
1206
1375
|
if (value && typeof value === "object") return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`).join(",")}}`;
|
|
@@ -1228,6 +1397,8 @@ function createRecorder(options = {}) {
|
|
|
1228
1397
|
const replayedModels = replay?.filter((effect) => effect.kind === "model");
|
|
1229
1398
|
const usedModels = /* @__PURE__ */ new Set();
|
|
1230
1399
|
const usedTools = /* @__PURE__ */ new Set();
|
|
1400
|
+
const usedCalls = /* @__PURE__ */ new Set();
|
|
1401
|
+
const callCounts = /* @__PURE__ */ new Map();
|
|
1231
1402
|
const normalizeEffect = (effect, list) => options.normalize?.(effect, list.indexOf(effect), list) ?? effect;
|
|
1232
1403
|
let modelCalls = 0;
|
|
1233
1404
|
let toolsInFlight = 0;
|
|
@@ -1405,6 +1576,68 @@ function createRecorder(options = {}) {
|
|
|
1405
1576
|
});
|
|
1406
1577
|
}
|
|
1407
1578
|
}
|
|
1579
|
+
function fn(name, run, callOptions = {}) {
|
|
1580
|
+
const { redact, serialize = toJson, revive } = callOptions;
|
|
1581
|
+
return (async (...args) => {
|
|
1582
|
+
const passed = [...args];
|
|
1583
|
+
while (passed.length && passed.at(-1) === void 0) passed.pop();
|
|
1584
|
+
const call = callCounts.get(name) ?? 0;
|
|
1585
|
+
callCounts.set(name, call + 1);
|
|
1586
|
+
const effect = {
|
|
1587
|
+
kind: "call",
|
|
1588
|
+
name,
|
|
1589
|
+
call,
|
|
1590
|
+
args: toJson(redact ? redact(passed) : passed)
|
|
1591
|
+
};
|
|
1592
|
+
effects.push(effect);
|
|
1593
|
+
if (replay) {
|
|
1594
|
+
const recorded = replay.find((candidate) => candidate.kind === "call" && candidate.name === name && !usedCalls.has(candidate));
|
|
1595
|
+
if (!recorded) {
|
|
1596
|
+
const error = new RecordingDivergenceError(`Call #${call} to '${name}' was never made by the recorded run.`);
|
|
1597
|
+
effect.error = toRecordedError(error);
|
|
1598
|
+
throw error;
|
|
1599
|
+
}
|
|
1600
|
+
usedCalls.add(recorded);
|
|
1601
|
+
if (recorded.error) {
|
|
1602
|
+
effect.error = recorded.error;
|
|
1603
|
+
throw fromRecordedError(recorded.error);
|
|
1604
|
+
}
|
|
1605
|
+
if (recorded.stream) {
|
|
1606
|
+
effect.stream = recorded.stream;
|
|
1607
|
+
const chunks = structuredClone(recorded.stream);
|
|
1608
|
+
return revive ? revive(chunks) : iterate(chunks);
|
|
1609
|
+
}
|
|
1610
|
+
effect.result = recorded.result;
|
|
1611
|
+
const result = structuredClone(recorded.result);
|
|
1612
|
+
return revive ? revive(result) : result;
|
|
1613
|
+
}
|
|
1614
|
+
let result;
|
|
1615
|
+
try {
|
|
1616
|
+
result = await run(...args);
|
|
1617
|
+
} catch (error) {
|
|
1618
|
+
effect.error = toRecordedError(error);
|
|
1619
|
+
throw error;
|
|
1620
|
+
}
|
|
1621
|
+
if (!isAsyncIterable(result)) {
|
|
1622
|
+
effect.result = toJson(serialize(result));
|
|
1623
|
+
return result;
|
|
1624
|
+
}
|
|
1625
|
+
const chunks = [];
|
|
1626
|
+
effect.stream = chunks;
|
|
1627
|
+
const source = result;
|
|
1628
|
+
return (async function* () {
|
|
1629
|
+
try {
|
|
1630
|
+
for await (const chunk of source) {
|
|
1631
|
+
chunks.push(toJson(serialize(chunk)));
|
|
1632
|
+
yield chunk;
|
|
1633
|
+
}
|
|
1634
|
+
} catch (error) {
|
|
1635
|
+
effect.error = toRecordedError(error);
|
|
1636
|
+
throw error;
|
|
1637
|
+
}
|
|
1638
|
+
})();
|
|
1639
|
+
});
|
|
1640
|
+
}
|
|
1408
1641
|
/** Counts an async tool call as running until it settles. */
|
|
1409
1642
|
function track(result) {
|
|
1410
1643
|
if (result instanceof Promise) {
|
|
@@ -1476,6 +1709,26 @@ function createRecorder(options = {}) {
|
|
|
1476
1709
|
} });
|
|
1477
1710
|
},
|
|
1478
1711
|
tool: (name, fn) => tool(name, fn),
|
|
1712
|
+
fn,
|
|
1713
|
+
client(name, client, { methods, ...callOptions }) {
|
|
1714
|
+
const paths = methods.map((method) => method.split("."));
|
|
1715
|
+
const startsWith = (path, prefix) => prefix.every((key, index) => path[index] === key);
|
|
1716
|
+
const wrapAt = (target, prefix) => {
|
|
1717
|
+
const wrapped = /* @__PURE__ */ new Map();
|
|
1718
|
+
return new Proxy(target, { get(object, key, receiver) {
|
|
1719
|
+
if (typeof key !== "string") return Reflect.get(object, key, receiver);
|
|
1720
|
+
const path = [...prefix, key];
|
|
1721
|
+
const method = paths.some((p) => p.length === path.length && startsWith(p, path));
|
|
1722
|
+
if (!method && !paths.some((p) => startsWith(p, path))) return Reflect.get(object, key, receiver);
|
|
1723
|
+
if (!wrapped.has(key)) {
|
|
1724
|
+
const value = Reflect.get(object, key, receiver);
|
|
1725
|
+
wrapped.set(key, method ? fn(`${name}.${path.join(".")}`, (...args) => value.apply(object, args), callOptions) : wrapAt(value && typeof value === "object" ? value : {}, path));
|
|
1726
|
+
}
|
|
1727
|
+
return wrapped.get(key);
|
|
1728
|
+
} });
|
|
1729
|
+
};
|
|
1730
|
+
return wrapAt(client, []);
|
|
1731
|
+
},
|
|
1479
1732
|
tools(tools) {
|
|
1480
1733
|
return Object.fromEntries(Object.entries(tools).map(([name, definition]) => [name, definition.execute ? {
|
|
1481
1734
|
...definition,
|
|
@@ -1559,6 +1812,7 @@ function describe(effect) {
|
|
|
1559
1812
|
switch (effect.kind) {
|
|
1560
1813
|
case "model": return `model call #${effect.call} (${effect.method})`;
|
|
1561
1814
|
case "tool": return `tool '${effect.name}' with ${JSON.stringify(effect.input)}`;
|
|
1815
|
+
case "call": return `call #${effect.call} to '${effect.name}' with ${JSON.stringify(effect.args)}`;
|
|
1562
1816
|
case "output": return `output ${JSON.stringify(effect.value)}`;
|
|
1563
1817
|
}
|
|
1564
1818
|
}
|
|
@@ -1578,7 +1832,7 @@ function differ(left, right, options) {
|
|
|
1578
1832
|
reason: "model-call-shape",
|
|
1579
1833
|
message: shapeMessage(left, right)
|
|
1580
1834
|
};
|
|
1581
|
-
if (!jsonEqual(errorIdentity(left.error), errorIdentity(right.error))) {
|
|
1835
|
+
if (!jsonEqual(errorIdentity$1(left.error), errorIdentity$1(right.error))) {
|
|
1582
1836
|
const outcome = (error) => error ? `failed with ${error.name}: ${error.message}` : "succeeded";
|
|
1583
1837
|
return {
|
|
1584
1838
|
reason: "model-error",
|
|
@@ -1596,12 +1850,27 @@ function differ(left, right, options) {
|
|
|
1596
1850
|
reason: "tool-input",
|
|
1597
1851
|
message: `Expected ${describe(left)}; the port passed ${JSON.stringify(right.input)}.`
|
|
1598
1852
|
};
|
|
1599
|
-
if (!jsonEqual(left.output, right.output) || !jsonEqual(errorIdentity(left.error), errorIdentity(right.error))) return {
|
|
1853
|
+
if (!jsonEqual(left.output, right.output) || !jsonEqual(errorIdentity$1(left.error), errorIdentity$1(right.error))) return {
|
|
1600
1854
|
reason: "tool-result",
|
|
1601
1855
|
message: `Tool '${left.name}' returned a different result.`
|
|
1602
1856
|
};
|
|
1603
1857
|
return;
|
|
1604
1858
|
}
|
|
1859
|
+
if (left.kind === "call" && right.kind === "call") {
|
|
1860
|
+
if (left.name !== right.name) return {
|
|
1861
|
+
reason: "call-name",
|
|
1862
|
+
message: `Expected ${describe(left)}; the port made ${describe(right)}.`
|
|
1863
|
+
};
|
|
1864
|
+
if (!jsonEqual(left.args, right.args)) return {
|
|
1865
|
+
reason: "call-args",
|
|
1866
|
+
message: `Expected ${describe(left)}; the port passed ${JSON.stringify(right.args)}.`
|
|
1867
|
+
};
|
|
1868
|
+
if (!jsonEqual(left.result, right.result) || !jsonEqual(left.stream, right.stream) || !jsonEqual(errorIdentity$1(left.error), errorIdentity$1(right.error))) return {
|
|
1869
|
+
reason: "call-result",
|
|
1870
|
+
message: `Call #${left.call} to '${left.name}' returned a different result.`
|
|
1871
|
+
};
|
|
1872
|
+
return;
|
|
1873
|
+
}
|
|
1605
1874
|
if (left.kind === "output" && right.kind === "output" && !jsonEqual(left.value, right.value)) return {
|
|
1606
1875
|
reason: "output",
|
|
1607
1876
|
message: `Expected ${describe(left)}; the port returned ${JSON.stringify(right.value)}.`
|
|
@@ -1637,9 +1906,9 @@ function compareRecordings(expected, actual, options = {}) {
|
|
|
1637
1906
|
if (k === -1) {
|
|
1638
1907
|
changed.push({
|
|
1639
1908
|
index,
|
|
1640
|
-
reason: "missing-effect",
|
|
1909
|
+
reason: left.kind === "output" ? "output" : "missing-effect",
|
|
1641
1910
|
expected: left,
|
|
1642
|
-
message: `The port never made ${describe(left)}.`
|
|
1911
|
+
message: left.kind === "output" ? `The port never returned; the original returned ${JSON.stringify(left.value)}.` : `The port never made ${describe(left)}.`
|
|
1643
1912
|
});
|
|
1644
1913
|
continue;
|
|
1645
1914
|
}
|
|
@@ -1652,12 +1921,15 @@ function compareRecordings(expected, actual, options = {}) {
|
|
|
1652
1921
|
...difference
|
|
1653
1922
|
});
|
|
1654
1923
|
}
|
|
1655
|
-
for (const j of unpairedRights)
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1924
|
+
for (const j of unpairedRights) {
|
|
1925
|
+
const extra = b[j];
|
|
1926
|
+
divergences.push({
|
|
1927
|
+
index: lefts[0] ?? at,
|
|
1928
|
+
reason: extra.kind === "output" ? "output" : "extra-effect",
|
|
1929
|
+
actual: extra,
|
|
1930
|
+
message: extra.kind === "output" ? `The original never returned; the port returned ${JSON.stringify(extra.value)}.` : `The port made ${describe(extra)}, which the original never made.`
|
|
1931
|
+
});
|
|
1932
|
+
}
|
|
1661
1933
|
divergences.push(...changed);
|
|
1662
1934
|
lefts = [];
|
|
1663
1935
|
rights = [];
|
|
@@ -1682,4 +1954,107 @@ function compareRecordings(expected, actual, options = {}) {
|
|
|
1682
1954
|
};
|
|
1683
1955
|
}
|
|
1684
1956
|
//#endregion
|
|
1685
|
-
|
|
1957
|
+
//#region src/check-parity.ts
|
|
1958
|
+
/**
|
|
1959
|
+
* `checkParity` — one call that records a workflow, replays its port, and
|
|
1960
|
+
* compares their effects, per scenario.
|
|
1961
|
+
* @module
|
|
1962
|
+
*/
|
|
1963
|
+
/** Each function effect wrapped with `recorder.fn`, by its key. */
|
|
1964
|
+
function recordEffects(recorder, effects) {
|
|
1965
|
+
return mapEffects(effects, (name, effect) => recorder.fn(name, effect));
|
|
1966
|
+
}
|
|
1967
|
+
async function settle(run, recorder, normalize) {
|
|
1968
|
+
try {
|
|
1969
|
+
const output = await run();
|
|
1970
|
+
recorder.output(normalize(output));
|
|
1971
|
+
return { output };
|
|
1972
|
+
} catch (error) {
|
|
1973
|
+
return { error };
|
|
1974
|
+
}
|
|
1975
|
+
}
|
|
1976
|
+
function errorIdentity(outcome, normalize) {
|
|
1977
|
+
if (!("error" in outcome)) return void 0;
|
|
1978
|
+
const error = normalize ? normalize(outcome.error) : outcome.error;
|
|
1979
|
+
if (error instanceof Error) return `${error.name}: ${error.message}`;
|
|
1980
|
+
if (!normalize) return String(error);
|
|
1981
|
+
try {
|
|
1982
|
+
return stableStringify(toJsonValue(error) ?? null);
|
|
1983
|
+
} catch {
|
|
1984
|
+
return String(error);
|
|
1985
|
+
}
|
|
1986
|
+
}
|
|
1987
|
+
/**
|
|
1988
|
+
* Checks a port against the workflow it replaces, scenario by scenario: runs
|
|
1989
|
+
* `before` on fresh `effects` under `createRecorder`, replays `after` against
|
|
1990
|
+
* that recording (no effect really runs), and compares the two with
|
|
1991
|
+
* `compareRecordings`. Effects are compared by name, call order, arguments
|
|
1992
|
+
* and results; the output by value.
|
|
1993
|
+
*
|
|
1994
|
+
* Never throws for a port that misbehaves: an `after` that throws (a
|
|
1995
|
+
* `RecordingDivergenceError` for a call the original never made, or its own
|
|
1996
|
+
* error) is a row with `after: { error }`. When both sides throw, they match
|
|
1997
|
+
* on the error's name and message, after `normalizeError` when given.
|
|
1998
|
+
*
|
|
1999
|
+
* @example
|
|
2000
|
+
* ```ts
|
|
2001
|
+
* const report = await checkParity({
|
|
2002
|
+
* scenarios: [{ id: "refund", input: ticket }],
|
|
2003
|
+
* effects: () => createFakeEffects(),
|
|
2004
|
+
* before: (input, effects) => legacyRunTicket(input, effects),
|
|
2005
|
+
* after: (input, effects) => step1.run(input, { effects }),
|
|
2006
|
+
* });
|
|
2007
|
+
* expect(report.equivalent).toBe(true);
|
|
2008
|
+
* ```
|
|
2009
|
+
*/
|
|
2010
|
+
async function checkParity(options) {
|
|
2011
|
+
const rows = [];
|
|
2012
|
+
for (const scenario of options.scenarios) {
|
|
2013
|
+
const recorder = createRecorder();
|
|
2014
|
+
const effects = recordEffects(recorder, options.effects(scenario));
|
|
2015
|
+
const normalize = (output) => options.normalizeOutput ? options.normalizeOutput(output, scenario) : output;
|
|
2016
|
+
const before = await settle(() => options.before(scenario.input, effects), recorder, normalize);
|
|
2017
|
+
const recording = recorder.recording;
|
|
2018
|
+
const replayer = createRecorder({ replay: recording });
|
|
2019
|
+
const stand = recordEffects(replayer, options.effects(scenario));
|
|
2020
|
+
const after = await settle(() => options.after(scenario.input, stand), replayer, normalize);
|
|
2021
|
+
const replay = replayer.recording;
|
|
2022
|
+
const { divergences } = compareRecordings(recording, replay, options.compare);
|
|
2023
|
+
const normalizeError = options.normalizeError && ((error) => options.normalizeError(error, scenario));
|
|
2024
|
+
const thrownBefore = errorIdentity(before, normalizeError);
|
|
2025
|
+
const thrownAfter = errorIdentity(after, normalizeError);
|
|
2026
|
+
if (thrownBefore !== void 0 && thrownAfter !== void 0 && thrownBefore !== thrownAfter) divergences.push({
|
|
2027
|
+
index: recording.effects.length,
|
|
2028
|
+
reason: "output",
|
|
2029
|
+
message: `The original threw ${thrownBefore}; the port threw ${thrownAfter}.`
|
|
2030
|
+
});
|
|
2031
|
+
else if (thrownBefore === void 0 !== (thrownAfter === void 0)) {
|
|
2032
|
+
const returned = (thrownBefore === void 0 ? recording : replay).effects.filter((effect) => effect.kind === "output").at(-1);
|
|
2033
|
+
const value = JSON.stringify(returned?.kind === "output" ? returned.value : void 0);
|
|
2034
|
+
const message = thrownAfter !== void 0 ? `The original returned ${value}; the port threw ${thrownAfter}.` : `The original threw ${thrownBefore}; the port returned ${value}.`;
|
|
2035
|
+
const existing = divergences.find((d) => d.reason === "output" && d.expected?.kind === "output" !== (d.actual?.kind === "output"));
|
|
2036
|
+
if (existing) existing.message = message;
|
|
2037
|
+
else divergences.push({
|
|
2038
|
+
index: recording.effects.length,
|
|
2039
|
+
reason: "output",
|
|
2040
|
+
message
|
|
2041
|
+
});
|
|
2042
|
+
}
|
|
2043
|
+
rows.push({
|
|
2044
|
+
id: scenario.id,
|
|
2045
|
+
scenario,
|
|
2046
|
+
equivalent: divergences.length === 0,
|
|
2047
|
+
divergences,
|
|
2048
|
+
before,
|
|
2049
|
+
after,
|
|
2050
|
+
recording,
|
|
2051
|
+
replay
|
|
2052
|
+
});
|
|
2053
|
+
}
|
|
2054
|
+
return {
|
|
2055
|
+
equivalent: rows.every((row) => row.equivalent),
|
|
2056
|
+
scenarios: rows
|
|
2057
|
+
};
|
|
2058
|
+
}
|
|
2059
|
+
//#endregion
|
|
2060
|
+
export { AgentLintError, AgentUnknownStateError, RecordingDivergenceError, assertAgentMachine, canReach, checkParity, compareRecordings, compareRuns, createRecorder, explorePaths, firstDivergence, foldTraces, getDecisionRequests, judgeAgreement, lintAgentMachine, matchesTrajectory, proposeApprovalRules, proposeCaps, repeatRuns, runSeam, sameModelCallShape, simulateAgent, summarizeRepeats, textOutput, traceCoverage };
|