@statelyai/agent 2.0.0-alpha.33 → 2.0.0-alpha.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/dist/ai-sdk.cjs +114 -63
- package/dist/ai-sdk.d.cts +11 -2
- package/dist/ai-sdk.d.mts +11 -2
- package/dist/ai-sdk.mjs +115 -64
- package/dist/{decision-BPdEaxPf.cjs → decision-BEsATcr1.cjs} +18 -2
- package/dist/{decision-CV7WiV5C.mjs → decision-COkJNjOW.mjs} +18 -2
- package/dist/{event-log-CKWeC7H-.cjs → event-log-BWSSLUff.cjs} +139 -10
- package/dist/{event-log-DAqsalT1.mjs → event-log-C7jeH0y8.mjs} +123 -12
- package/dist/{event-log-store-C2NjVlWU.d.mts → event-log-store-BzXehP7R.d.cts} +28 -3
- package/dist/{event-log-store-s73r1Hxf.d.cts → event-log-store-CWjN6lmM.d.mts} +28 -3
- package/dist/{event-log-store-DKyE7JKr.cjs → event-log-store-CleWpsp5.cjs} +1 -1
- package/dist/{event-log-store-BGehvVTb.mjs → event-log-store-W49TCJXi.mjs} +1 -1
- package/dist/index.cjs +5 -5
- package/dist/index.d.cts +7 -7
- package/dist/index.d.mts +7 -7
- package/dist/index.mjs +5 -5
- package/dist/log.cjs +2 -2
- package/dist/log.d.cts +1 -1
- package/dist/log.d.mts +1 -1
- package/dist/log.mjs +2 -2
- package/dist/machines.cjs +1 -1
- package/dist/machines.mjs +1 -1
- package/dist/{model-source-BilDEyqk.mjs → model-source-CK3F4YuU.mjs} +1 -1
- package/dist/{model-source-BQ97_l1-.cjs → model-source-DmhTMK39.cjs} +1 -1
- package/dist/openai.cjs +377 -95
- package/dist/openai.d.cts +55 -20
- package/dist/openai.d.mts +55 -20
- package/dist/openai.mjs +375 -96
- package/dist/otel.d.cts +1 -1
- package/dist/otel.d.mts +1 -1
- package/dist/{run-agent-B5QhabdY.d.cts → run-agent-B1-Eqviq.d.cts} +13 -5
- package/dist/{run-agent-Atu-wEk7.d.mts → run-agent-BGdx5913.d.mts} +13 -5
- package/dist/{run-agent-2UnSUhDb.mjs → run-agent-Ct_GZw_f.mjs} +124 -38
- package/dist/{run-agent-Cg0xd3K-.cjs → run-agent-s5D32w99.cjs} +123 -37
- package/dist/{setup-agent-CmhB-U5P.mjs → setup-agent-05uw4dPZ.mjs} +2 -2
- package/dist/{setup-agent-BiF1N3td.d.cts → setup-agent-BtfeFfF6.d.cts} +2 -2
- package/dist/{setup-agent-J9f_Wu5T.d.mts → setup-agent-C6nqzv_X.d.mts} +2 -2
- package/dist/{setup-agent-CwUJyqKB.cjs → setup-agent-CXEBURdo.cjs} +2 -2
- package/dist/tanstack-ai.cjs +91 -59
- package/dist/tanstack-ai.d.cts +13 -2
- package/dist/tanstack-ai.d.mts +13 -2
- package/dist/tanstack-ai.mjs +91 -59
- package/dist/testing.cjs +539 -13
- package/dist/testing.d.cts +202 -6
- package/dist/testing.d.mts +202 -6
- package/dist/testing.mjs +536 -14
- package/dist/{text-logic-CYP502VT.d.cts → text-logic-CV_4KGr6.d.cts} +28 -3
- package/dist/{text-logic-DuK9HIFM.d.mts → text-logic-Ckh-rJFV.d.mts} +28 -3
- package/dist/{usage-D7JKAxLc.d.mts → usage-BDiJKyv9.d.cts} +11 -1
- package/dist/{usage-DVw-646x.d.cts → usage-DOoVUGVq.d.mts} +11 -1
- package/dist/validate.cjs +3 -3
- package/dist/validate.d.cts +2 -2
- package/dist/validate.d.mts +2 -2
- package/dist/validate.mjs +3 -3
- package/dist/{verify-Dk3Vwnha.d.cts → verify-BQpE8hUz.d.cts} +1 -1
- package/dist/{verify-DgB8P-Wx.mjs → verify-Bpahmr34.mjs} +2 -2
- package/dist/{verify-BVz4cURF.d.mts → verify-CqeuEmIk.d.mts} +1 -1
- package/dist/{verify-CS_WLRXn.cjs → verify-DCLZa9R0.cjs} +2 -2
- package/dist/with-model-calls-BXYdBdcu.cjs +45 -0
- package/dist/with-model-calls-Bv4V4EHC.mjs +34 -0
- package/package.json +5 -5
- package/readme.md +14 -13
- package/skills/loop-to-machine/SKILL.md +40 -2
package/dist/testing.d.cts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { f as AgentRuntimeOptions, l as AgentRunResult, o as AgentRunInit } from "./run-agent-
|
|
1
|
+
import { F as AgentExecutorDecisionRequest, c as AgentRequestExecutorInfo, f as AgentTextRequest, s as AgentRequestExecutor, t as AgentCallUsage, u as AgentRequestExecutors } from "./text-logic-CV_4KGr6.cjs";
|
|
2
|
+
import { f as AgentRuntimeOptions, l as AgentRunResult, o as AgentRunInit } from "./run-agent-B1-Eqviq.cjs";
|
|
3
3
|
import { t as MetaOfSnapshot } from "./utils-ebVpCYoF.cjs";
|
|
4
|
-
import { _ as assertAgentMachine, a as AgentPathTerminal, b as lintAgentMachine, c as CanReachResult, d as ExploreSnapshot, f as LintAgentMachineOptions, g as SimulationTrailEntry, h as SimulationScript, i as AgentPathReport, l as CanReachTarget, m as SimulateAgentResult, n as AgentLintError, o as AgentUnknownStateError, p as SimulateAgentOptions, r as AgentLintSeverity, s as AssertAgentMachineOptions, t as AgentLintDiagnostic, u as ExplorePathsOptions, v as canReach, x as simulateAgent, y as explorePaths } from "./verify-
|
|
4
|
+
import { _ as assertAgentMachine, a as AgentPathTerminal, b as lintAgentMachine, c as CanReachResult, d as ExploreSnapshot, f as LintAgentMachineOptions, g as SimulationTrailEntry, h as SimulationScript, i as AgentPathReport, l as CanReachTarget, m as SimulateAgentResult, n as AgentLintError, o as AgentUnknownStateError, p as SimulateAgentOptions, r as AgentLintSeverity, s as AssertAgentMachineOptions, t as AgentLintDiagnostic, u as ExplorePathsOptions, v as canReach, x as simulateAgent, y as explorePaths } from "./verify-BQpE8hUz.cjs";
|
|
5
5
|
import { AnyStateMachine, EventFromLogic, InputFrom, SnapshotFrom, StateValue } from "xstate";
|
|
6
6
|
//#region src/trajectory.d.ts
|
|
7
7
|
/** An event-shaped trajectory item: anything with a string `type`. */
|
|
@@ -418,7 +418,8 @@ interface TraceCoverageOptions {
|
|
|
418
418
|
* bare type (`xstate.done.actor`), one per state however many invokes; a
|
|
419
419
|
* definition's `matches` (an invoke's `actorId`) must accept the traced event
|
|
420
420
|
* for its state to be credited. Ignored steps credit no handler. `always`
|
|
421
|
-
* handlers are listed as `unobservable`, never as unvisited
|
|
421
|
+
* handlers are listed as `unobservable`, never as unvisited; a state one
|
|
422
|
+
* passes through is visited, re-derived from each step's microsteps.
|
|
422
423
|
*
|
|
423
424
|
* @example
|
|
424
425
|
* ```ts no-check
|
|
@@ -459,8 +460,14 @@ interface FirstDivergenceOptions {
|
|
|
459
460
|
* `from|event|to`. Override to leave out what may differ between equivalent
|
|
460
461
|
* runs, such as a parallel region that only logs. Steps are still compared
|
|
461
462
|
* in order. Trace references only.
|
|
463
|
+
*
|
|
464
|
+
* `"payload"` also compares the event's payload, such as an actor's output,
|
|
465
|
+
* as key-order-independent JSON without its `sessionId`, and with errors as
|
|
466
|
+
* `{ name, message, code?, cause? }` (an `Error`, or a serialized one with `stack` or
|
|
467
|
+
* `code`). Use it for deterministic or mocked runs; live model outputs usually differ
|
|
468
|
+
* between runs.
|
|
462
469
|
*/
|
|
463
|
-
key?: (step: TraceStep) => string;
|
|
470
|
+
key?: ((step: TraceStep) => string) | "payload";
|
|
464
471
|
}
|
|
465
472
|
/** Where a run first departs from its reference. See {@link firstDivergence}. */
|
|
466
473
|
interface Divergence {
|
|
@@ -822,4 +829,193 @@ export declare function compareRuns<TScenario extends {
|
|
|
822
829
|
expected?: string;
|
|
823
830
|
}>(options: CompareRunsOptions<TScenario>): Promise<ParityReport<TScenario>>;
|
|
824
831
|
//#endregion
|
|
825
|
-
|
|
832
|
+
//#region src/equivalence.d.ts
|
|
833
|
+
/**
|
|
834
|
+
* Verified equivalence — prove a port behaves like the code it replaces.
|
|
835
|
+
*
|
|
836
|
+
* A port is equivalent when, given the same model answers and the same tool
|
|
837
|
+
* results, it produces the same OBSERVABLE EFFECTS: the same number of model
|
|
838
|
+
* calls, the same tool calls with the same arguments, and the same output.
|
|
839
|
+
* Prompts, internal steps, and states are free to differ.
|
|
840
|
+
*
|
|
841
|
+
* 1. Record the original: wrap its model (`recorder.model`, an AI SDK
|
|
842
|
+
* `LanguageModelV3`, or `recorder.openai`, an `openai` client) and its tools
|
|
843
|
+
* (`recorder.tool` / `recorder.tools`), run a scenario, call
|
|
844
|
+
* `recorder.output(result)`, and save `recorder.recording`.
|
|
845
|
+
* 2. Replay it on the port: `createRecorder({ replay: recording })` answers
|
|
846
|
+
* every model call and tool call from the recording, with no API key and no
|
|
847
|
+
* side effects, and records what the port did.
|
|
848
|
+
* 3. `compareRecordings(original, port)` reports the first effect that differs.
|
|
849
|
+
*
|
|
850
|
+
* Provider-free: both seams are wrapped structurally, so this module imports
|
|
851
|
+
* neither `ai` nor `openai`.
|
|
852
|
+
* @module
|
|
853
|
+
*/
|
|
854
|
+
/** An error as recorded: JSON-safe. */
|
|
855
|
+
interface RecordedError {
|
|
856
|
+
name: string;
|
|
857
|
+
message: string;
|
|
858
|
+
code?: unknown;
|
|
859
|
+
}
|
|
860
|
+
/** One model call. `call` is its 0-based position among model calls. */
|
|
861
|
+
interface RecordedModelCall {
|
|
862
|
+
kind: "model";
|
|
863
|
+
/** Which wrapper saw it. */
|
|
864
|
+
seam: "ai-sdk" | "openai";
|
|
865
|
+
/** `doGenerate`/`doStream`, or `chat.completions.create`/`responses.create`. */
|
|
866
|
+
method: string;
|
|
867
|
+
call: number;
|
|
868
|
+
/** The call options or params, JSON-safe (abort signals and headers dropped). */
|
|
869
|
+
request: unknown;
|
|
870
|
+
/** The non-streaming result. */
|
|
871
|
+
response?: unknown;
|
|
872
|
+
/** The stream's parts or chunks, in order. */
|
|
873
|
+
stream?: unknown[];
|
|
874
|
+
error?: RecordedError;
|
|
875
|
+
}
|
|
876
|
+
/** One tool call. `step` is the number of model calls made before it. */
|
|
877
|
+
interface RecordedToolCall {
|
|
878
|
+
kind: "tool";
|
|
879
|
+
name: string;
|
|
880
|
+
step: number;
|
|
881
|
+
input: unknown;
|
|
882
|
+
output?: unknown;
|
|
883
|
+
error?: RecordedError;
|
|
884
|
+
/** The tool returned without a Promise; replay answers synchronously too. */
|
|
885
|
+
sync?: true;
|
|
886
|
+
/** Started while an earlier tool call was still running. Concurrent calls
|
|
887
|
+
* may finish in either order; sequential ones must keep theirs. */
|
|
888
|
+
concurrent?: true;
|
|
889
|
+
}
|
|
890
|
+
/** The run's final result. */
|
|
891
|
+
interface RecordedOutput {
|
|
892
|
+
kind: "output";
|
|
893
|
+
value: unknown;
|
|
894
|
+
}
|
|
895
|
+
type RecordedEffect = RecordedModelCall | RecordedToolCall | RecordedOutput;
|
|
896
|
+
/** A run's effects, in order. JSON-serializable: save it as a golden file. */
|
|
897
|
+
interface Recording {
|
|
898
|
+
version: 1;
|
|
899
|
+
effects: RecordedEffect[];
|
|
900
|
+
}
|
|
901
|
+
/** Options for {@link createRecorder}. Omit for record mode. */
|
|
902
|
+
interface CreateRecorderOptions {
|
|
903
|
+
/** Answer model and tool calls from this recording instead of running them. */
|
|
904
|
+
replay?: Recording;
|
|
905
|
+
/**
|
|
906
|
+
* In replay, `"replay"` (default) returns each tool's recorded result
|
|
907
|
+
* without running it; `"live"` runs the wrapped tool (your stub) and records
|
|
908
|
+
* what it returned.
|
|
909
|
+
*/
|
|
910
|
+
tools?: "replay" | "live";
|
|
911
|
+
/**
|
|
912
|
+
* In replay, which model calls a recorded answer may serve. `"shape"`
|
|
913
|
+
* (default) refuses a call whose offered tools or response format changed,
|
|
914
|
+
* since the recorded answer may not fit it; see {@link ModelCallMatch}.
|
|
915
|
+
*/
|
|
916
|
+
modelCalls?: ModelCallMatch;
|
|
917
|
+
/**
|
|
918
|
+
* In replay, maps the recorded and the new tool call before matching them,
|
|
919
|
+
* like {@link CompareRecordingsOptions.normalize}: redact a generated id in a
|
|
920
|
+
* tool's input so the recorded result still answers it.
|
|
921
|
+
*/
|
|
922
|
+
normalize?: (effect: RecordedEffect) => RecordedEffect;
|
|
923
|
+
}
|
|
924
|
+
/**
|
|
925
|
+
* When two model calls count as the same: `"shape"` compares offered tool
|
|
926
|
+
* names and response format, `"none"` only that a call happened, and a
|
|
927
|
+
* function decides per call (allow the changes you made on purpose and fall
|
|
928
|
+
* back to {@link sameModelCallShape}).
|
|
929
|
+
*/
|
|
930
|
+
type ModelCallMatch = "shape" | "none" | ((expected: RecordedModelCall, actual: RecordedModelCall) => boolean);
|
|
931
|
+
/** The structural slice of an AI SDK `LanguageModelV3` the recorder wraps. */
|
|
932
|
+
interface RecordableLanguageModel {
|
|
933
|
+
doGenerate(options: never): PromiseLike<unknown>;
|
|
934
|
+
doStream(options: never): PromiseLike<{
|
|
935
|
+
stream: ReadableStream<unknown>;
|
|
936
|
+
}>;
|
|
937
|
+
}
|
|
938
|
+
type AnyFunction = (...args: never[]) => unknown;
|
|
939
|
+
/** Records (or replays) the effects of one run. */
|
|
940
|
+
interface Recorder {
|
|
941
|
+
/**
|
|
942
|
+
* Wraps an AI SDK `LanguageModelV3` (`openai("gpt-…")`, `anthropic(…)`),
|
|
943
|
+
* keeping its type. In replay the model is never called, so a provider
|
|
944
|
+
* model built with no API key works.
|
|
945
|
+
*/
|
|
946
|
+
model<TModel extends RecordableLanguageModel>(model: TModel): TModel;
|
|
947
|
+
/**
|
|
948
|
+
* Wraps an `openai` client's `chat.completions.create` and
|
|
949
|
+
* `responses.create`, keeping its type. A streamed result is an async
|
|
950
|
+
* iterable only. In replay the client is never called, so
|
|
951
|
+
* `new OpenAI({ apiKey: "replay" })` works.
|
|
952
|
+
*/
|
|
953
|
+
openai<TClient extends object>(client: TClient): TClient;
|
|
954
|
+
/** Wraps one tool function. Its arguments are the recorded input. */
|
|
955
|
+
tool<TFn extends AnyFunction>(name: string, fn: TFn): TFn;
|
|
956
|
+
/** Wraps every `execute` in an AI SDK tool set, named by key. */
|
|
957
|
+
tools<TTools extends Record<string, {
|
|
958
|
+
execute?: AnyFunction;
|
|
959
|
+
}>>(tools: TTools): TTools;
|
|
960
|
+
/** Records the run's final result. */
|
|
961
|
+
output(value: unknown): void;
|
|
962
|
+
/** What this run did so far. */
|
|
963
|
+
readonly recording: Recording;
|
|
964
|
+
}
|
|
965
|
+
/**
|
|
966
|
+
* Thrown in replay when the port asks for something the recording cannot
|
|
967
|
+
* answer: a model call past the last recorded one, or a tool call the original
|
|
968
|
+
* never made. The effect is still recorded, so {@link compareRecordings}
|
|
969
|
+
* reports it.
|
|
970
|
+
*/
|
|
971
|
+
export declare class RecordingDivergenceError extends Error {
|
|
972
|
+
name: string;
|
|
973
|
+
}
|
|
974
|
+
/**
|
|
975
|
+
* Creates a {@link Recorder}. With no options it records: every wrapped call
|
|
976
|
+
* runs for real. With `replay` it answers from a recording instead.
|
|
977
|
+
*/
|
|
978
|
+
export declare function createRecorder(options?: CreateRecorderOptions): Recorder;
|
|
979
|
+
/** Why two effects differ. */
|
|
980
|
+
type EffectDivergenceReason = "missing-effect" | "extra-effect" | "kind" | "model-method" | "model-call-shape" | "model-error" | "tool-name" | "tool-input" | "tool-result" | "output";
|
|
981
|
+
/** The first effect where two recordings differ. */
|
|
982
|
+
interface EffectDivergence {
|
|
983
|
+
/** Position in the compared (tool-order-normalized) effect lists. */
|
|
984
|
+
index: number;
|
|
985
|
+
reason: EffectDivergenceReason;
|
|
986
|
+
expected?: RecordedEffect;
|
|
987
|
+
actual?: RecordedEffect;
|
|
988
|
+
/** One sentence, for test failures. */
|
|
989
|
+
message: string;
|
|
990
|
+
}
|
|
991
|
+
/** The result of {@link compareRecordings}. */
|
|
992
|
+
interface EquivalenceReport {
|
|
993
|
+
equivalent: boolean;
|
|
994
|
+
/** The first divergence, by position in the original's effects. */
|
|
995
|
+
divergence?: EffectDivergence;
|
|
996
|
+
/** Every divergence, in order. Effects both runs made line up between them. */
|
|
997
|
+
divergences: EffectDivergence[];
|
|
998
|
+
}
|
|
999
|
+
/** Options for {@link compareRecordings}. */
|
|
1000
|
+
interface CompareRecordingsOptions {
|
|
1001
|
+
/** When two model calls count as the same. Default `"shape"`. */
|
|
1002
|
+
modelCalls?: ModelCallMatch;
|
|
1003
|
+
/**
|
|
1004
|
+
* Maps each effect before comparing, on both sides: redact values that
|
|
1005
|
+
* differ on every run (generated ids, timestamps) or project a tool's input
|
|
1006
|
+
* to the fields that matter.
|
|
1007
|
+
*/
|
|
1008
|
+
normalize?: (effect: RecordedEffect) => RecordedEffect;
|
|
1009
|
+
}
|
|
1010
|
+
/** True when two model calls offer the same tool names and response format. */
|
|
1011
|
+
export declare function sameModelCallShape(expected: RecordedModelCall, actual: RecordedModelCall): boolean;
|
|
1012
|
+
/**
|
|
1013
|
+
* Compares two recordings and reports every effect that differs. `expected`
|
|
1014
|
+
* is the original's recording, `actual` the port's. Effects that are the same
|
|
1015
|
+
* on both sides are lined up first (longest common subsequence), so one extra
|
|
1016
|
+
* call does not shift every effect after it; between them, effects of the same
|
|
1017
|
+
* kind pair up and report what changed, and the rest are missing or extra.
|
|
1018
|
+
*/
|
|
1019
|
+
export declare function compareRecordings(expected: Recording, actual: Recording, options?: CompareRecordingsOptions): EquivalenceReport;
|
|
1020
|
+
//#endregion
|
|
1021
|
+
export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AgreementReport, type AgreementRow, type AgreementSummary, type AssertAgentMachineOptions, type CanReachResult, type CanReachTarget, type CompareRecordingsOptions, type CompareRunsOptions, type CoverageReport, type CreateRecorderOptions, type Divergence, type EffectDivergence, type EffectDivergenceReason, type EquivalenceReport, type ExplorePathsOptions, type ExploreSnapshot, type FirstDivergenceOptions, type FoldTracesOptions, type GetDecisionRequestsOptions, type HandlerCoverage, type HandlerRef, type JudgeAgreementOptions, type LintAgentMachineOptions, type MatchTrajectoryOptions, type ModelCallMatch, type ParityLabel, type ParityReport, type ParityResult, type ParityRun, type RecordableLanguageModel, type RecordedEffect, type RecordedError, type RecordedModelCall, type RecordedOutput, type RecordedToolCall, type Recorder, type Recording, type RepeatGroup, type RepeatReport, type RepeatResult, type RepeatRun, type RepeatRunsOptions, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type StateCoverage, type StateStats, type TraceCoverageOptions, type TraceStep, type TracedDecisionRequest, type TrajectoryEvent, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, explorePaths, lintAgentMachine, simulateAgent };
|
package/dist/testing.d.mts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { f as AgentRuntimeOptions, l as AgentRunResult, o as AgentRunInit } from "./run-agent-
|
|
1
|
+
import { F as AgentExecutorDecisionRequest, c as AgentRequestExecutorInfo, f as AgentTextRequest, s as AgentRequestExecutor, t as AgentCallUsage, u as AgentRequestExecutors } from "./text-logic-Ckh-rJFV.mjs";
|
|
2
|
+
import { f as AgentRuntimeOptions, l as AgentRunResult, o as AgentRunInit } from "./run-agent-BGdx5913.mjs";
|
|
3
3
|
import { t as MetaOfSnapshot } from "./utils-BnXdWih3.mjs";
|
|
4
|
-
import { _ as assertAgentMachine, a as AgentPathTerminal, b as lintAgentMachine, c as CanReachResult, d as ExploreSnapshot, f as LintAgentMachineOptions, g as SimulationTrailEntry, h as SimulationScript, i as AgentPathReport, l as CanReachTarget, m as SimulateAgentResult, n as AgentLintError, o as AgentUnknownStateError, p as SimulateAgentOptions, r as AgentLintSeverity, s as AssertAgentMachineOptions, t as AgentLintDiagnostic, u as ExplorePathsOptions, v as canReach, x as simulateAgent, y as explorePaths } from "./verify-
|
|
4
|
+
import { _ as assertAgentMachine, a as AgentPathTerminal, b as lintAgentMachine, c as CanReachResult, d as ExploreSnapshot, f as LintAgentMachineOptions, g as SimulationTrailEntry, h as SimulationScript, i as AgentPathReport, l as CanReachTarget, m as SimulateAgentResult, n as AgentLintError, o as AgentUnknownStateError, p as SimulateAgentOptions, r as AgentLintSeverity, s as AssertAgentMachineOptions, t as AgentLintDiagnostic, u as ExplorePathsOptions, v as canReach, x as simulateAgent, y as explorePaths } from "./verify-CqeuEmIk.mjs";
|
|
5
5
|
import { AnyStateMachine, EventFromLogic, InputFrom, SnapshotFrom, StateValue } from "xstate";
|
|
6
6
|
//#region src/trajectory.d.ts
|
|
7
7
|
/** An event-shaped trajectory item: anything with a string `type`. */
|
|
@@ -418,7 +418,8 @@ interface TraceCoverageOptions {
|
|
|
418
418
|
* bare type (`xstate.done.actor`), one per state however many invokes; a
|
|
419
419
|
* definition's `matches` (an invoke's `actorId`) must accept the traced event
|
|
420
420
|
* for its state to be credited. Ignored steps credit no handler. `always`
|
|
421
|
-
* handlers are listed as `unobservable`, never as unvisited
|
|
421
|
+
* handlers are listed as `unobservable`, never as unvisited; a state one
|
|
422
|
+
* passes through is visited, re-derived from each step's microsteps.
|
|
422
423
|
*
|
|
423
424
|
* @example
|
|
424
425
|
* ```ts no-check
|
|
@@ -459,8 +460,14 @@ interface FirstDivergenceOptions {
|
|
|
459
460
|
* `from|event|to`. Override to leave out what may differ between equivalent
|
|
460
461
|
* runs, such as a parallel region that only logs. Steps are still compared
|
|
461
462
|
* in order. Trace references only.
|
|
463
|
+
*
|
|
464
|
+
* `"payload"` also compares the event's payload, such as an actor's output,
|
|
465
|
+
* as key-order-independent JSON without its `sessionId`, and with errors as
|
|
466
|
+
* `{ name, message, code?, cause? }` (an `Error`, or a serialized one with `stack` or
|
|
467
|
+
* `code`). Use it for deterministic or mocked runs; live model outputs usually differ
|
|
468
|
+
* between runs.
|
|
462
469
|
*/
|
|
463
|
-
key?: (step: TraceStep) => string;
|
|
470
|
+
key?: ((step: TraceStep) => string) | "payload";
|
|
464
471
|
}
|
|
465
472
|
/** Where a run first departs from its reference. See {@link firstDivergence}. */
|
|
466
473
|
interface Divergence {
|
|
@@ -822,4 +829,193 @@ export declare function compareRuns<TScenario extends {
|
|
|
822
829
|
expected?: string;
|
|
823
830
|
}>(options: CompareRunsOptions<TScenario>): Promise<ParityReport<TScenario>>;
|
|
824
831
|
//#endregion
|
|
825
|
-
|
|
832
|
+
//#region src/equivalence.d.ts
|
|
833
|
+
/**
|
|
834
|
+
* Verified equivalence — prove a port behaves like the code it replaces.
|
|
835
|
+
*
|
|
836
|
+
* A port is equivalent when, given the same model answers and the same tool
|
|
837
|
+
* results, it produces the same OBSERVABLE EFFECTS: the same number of model
|
|
838
|
+
* calls, the same tool calls with the same arguments, and the same output.
|
|
839
|
+
* Prompts, internal steps, and states are free to differ.
|
|
840
|
+
*
|
|
841
|
+
* 1. Record the original: wrap its model (`recorder.model`, an AI SDK
|
|
842
|
+
* `LanguageModelV3`, or `recorder.openai`, an `openai` client) and its tools
|
|
843
|
+
* (`recorder.tool` / `recorder.tools`), run a scenario, call
|
|
844
|
+
* `recorder.output(result)`, and save `recorder.recording`.
|
|
845
|
+
* 2. Replay it on the port: `createRecorder({ replay: recording })` answers
|
|
846
|
+
* every model call and tool call from the recording, with no API key and no
|
|
847
|
+
* side effects, and records what the port did.
|
|
848
|
+
* 3. `compareRecordings(original, port)` reports the first effect that differs.
|
|
849
|
+
*
|
|
850
|
+
* Provider-free: both seams are wrapped structurally, so this module imports
|
|
851
|
+
* neither `ai` nor `openai`.
|
|
852
|
+
* @module
|
|
853
|
+
*/
|
|
854
|
+
/** An error as recorded: JSON-safe. */
|
|
855
|
+
interface RecordedError {
|
|
856
|
+
name: string;
|
|
857
|
+
message: string;
|
|
858
|
+
code?: unknown;
|
|
859
|
+
}
|
|
860
|
+
/** One model call. `call` is its 0-based position among model calls. */
|
|
861
|
+
interface RecordedModelCall {
|
|
862
|
+
kind: "model";
|
|
863
|
+
/** Which wrapper saw it. */
|
|
864
|
+
seam: "ai-sdk" | "openai";
|
|
865
|
+
/** `doGenerate`/`doStream`, or `chat.completions.create`/`responses.create`. */
|
|
866
|
+
method: string;
|
|
867
|
+
call: number;
|
|
868
|
+
/** The call options or params, JSON-safe (abort signals and headers dropped). */
|
|
869
|
+
request: unknown;
|
|
870
|
+
/** The non-streaming result. */
|
|
871
|
+
response?: unknown;
|
|
872
|
+
/** The stream's parts or chunks, in order. */
|
|
873
|
+
stream?: unknown[];
|
|
874
|
+
error?: RecordedError;
|
|
875
|
+
}
|
|
876
|
+
/** One tool call. `step` is the number of model calls made before it. */
|
|
877
|
+
interface RecordedToolCall {
|
|
878
|
+
kind: "tool";
|
|
879
|
+
name: string;
|
|
880
|
+
step: number;
|
|
881
|
+
input: unknown;
|
|
882
|
+
output?: unknown;
|
|
883
|
+
error?: RecordedError;
|
|
884
|
+
/** The tool returned without a Promise; replay answers synchronously too. */
|
|
885
|
+
sync?: true;
|
|
886
|
+
/** Started while an earlier tool call was still running. Concurrent calls
|
|
887
|
+
* may finish in either order; sequential ones must keep theirs. */
|
|
888
|
+
concurrent?: true;
|
|
889
|
+
}
|
|
890
|
+
/** The run's final result. */
|
|
891
|
+
interface RecordedOutput {
|
|
892
|
+
kind: "output";
|
|
893
|
+
value: unknown;
|
|
894
|
+
}
|
|
895
|
+
type RecordedEffect = RecordedModelCall | RecordedToolCall | RecordedOutput;
|
|
896
|
+
/** A run's effects, in order. JSON-serializable: save it as a golden file. */
|
|
897
|
+
interface Recording {
|
|
898
|
+
version: 1;
|
|
899
|
+
effects: RecordedEffect[];
|
|
900
|
+
}
|
|
901
|
+
/** Options for {@link createRecorder}. Omit for record mode. */
|
|
902
|
+
interface CreateRecorderOptions {
|
|
903
|
+
/** Answer model and tool calls from this recording instead of running them. */
|
|
904
|
+
replay?: Recording;
|
|
905
|
+
/**
|
|
906
|
+
* In replay, `"replay"` (default) returns each tool's recorded result
|
|
907
|
+
* without running it; `"live"` runs the wrapped tool (your stub) and records
|
|
908
|
+
* what it returned.
|
|
909
|
+
*/
|
|
910
|
+
tools?: "replay" | "live";
|
|
911
|
+
/**
|
|
912
|
+
* In replay, which model calls a recorded answer may serve. `"shape"`
|
|
913
|
+
* (default) refuses a call whose offered tools or response format changed,
|
|
914
|
+
* since the recorded answer may not fit it; see {@link ModelCallMatch}.
|
|
915
|
+
*/
|
|
916
|
+
modelCalls?: ModelCallMatch;
|
|
917
|
+
/**
|
|
918
|
+
* In replay, maps the recorded and the new tool call before matching them,
|
|
919
|
+
* like {@link CompareRecordingsOptions.normalize}: redact a generated id in a
|
|
920
|
+
* tool's input so the recorded result still answers it.
|
|
921
|
+
*/
|
|
922
|
+
normalize?: (effect: RecordedEffect) => RecordedEffect;
|
|
923
|
+
}
|
|
924
|
+
/**
|
|
925
|
+
* When two model calls count as the same: `"shape"` compares offered tool
|
|
926
|
+
* names and response format, `"none"` only that a call happened, and a
|
|
927
|
+
* function decides per call (allow the changes you made on purpose and fall
|
|
928
|
+
* back to {@link sameModelCallShape}).
|
|
929
|
+
*/
|
|
930
|
+
type ModelCallMatch = "shape" | "none" | ((expected: RecordedModelCall, actual: RecordedModelCall) => boolean);
|
|
931
|
+
/** The structural slice of an AI SDK `LanguageModelV3` the recorder wraps. */
|
|
932
|
+
interface RecordableLanguageModel {
|
|
933
|
+
doGenerate(options: never): PromiseLike<unknown>;
|
|
934
|
+
doStream(options: never): PromiseLike<{
|
|
935
|
+
stream: ReadableStream<unknown>;
|
|
936
|
+
}>;
|
|
937
|
+
}
|
|
938
|
+
type AnyFunction = (...args: never[]) => unknown;
|
|
939
|
+
/** Records (or replays) the effects of one run. */
|
|
940
|
+
interface Recorder {
|
|
941
|
+
/**
|
|
942
|
+
* Wraps an AI SDK `LanguageModelV3` (`openai("gpt-…")`, `anthropic(…)`),
|
|
943
|
+
* keeping its type. In replay the model is never called, so a provider
|
|
944
|
+
* model built with no API key works.
|
|
945
|
+
*/
|
|
946
|
+
model<TModel extends RecordableLanguageModel>(model: TModel): TModel;
|
|
947
|
+
/**
|
|
948
|
+
* Wraps an `openai` client's `chat.completions.create` and
|
|
949
|
+
* `responses.create`, keeping its type. A streamed result is an async
|
|
950
|
+
* iterable only. In replay the client is never called, so
|
|
951
|
+
* `new OpenAI({ apiKey: "replay" })` works.
|
|
952
|
+
*/
|
|
953
|
+
openai<TClient extends object>(client: TClient): TClient;
|
|
954
|
+
/** Wraps one tool function. Its arguments are the recorded input. */
|
|
955
|
+
tool<TFn extends AnyFunction>(name: string, fn: TFn): TFn;
|
|
956
|
+
/** Wraps every `execute` in an AI SDK tool set, named by key. */
|
|
957
|
+
tools<TTools extends Record<string, {
|
|
958
|
+
execute?: AnyFunction;
|
|
959
|
+
}>>(tools: TTools): TTools;
|
|
960
|
+
/** Records the run's final result. */
|
|
961
|
+
output(value: unknown): void;
|
|
962
|
+
/** What this run did so far. */
|
|
963
|
+
readonly recording: Recording;
|
|
964
|
+
}
|
|
965
|
+
/**
|
|
966
|
+
* Thrown in replay when the port asks for something the recording cannot
|
|
967
|
+
* answer: a model call past the last recorded one, or a tool call the original
|
|
968
|
+
* never made. The effect is still recorded, so {@link compareRecordings}
|
|
969
|
+
* reports it.
|
|
970
|
+
*/
|
|
971
|
+
export declare class RecordingDivergenceError extends Error {
|
|
972
|
+
name: string;
|
|
973
|
+
}
|
|
974
|
+
/**
|
|
975
|
+
* Creates a {@link Recorder}. With no options it records: every wrapped call
|
|
976
|
+
* runs for real. With `replay` it answers from a recording instead.
|
|
977
|
+
*/
|
|
978
|
+
export declare function createRecorder(options?: CreateRecorderOptions): Recorder;
|
|
979
|
+
/** Why two effects differ. */
|
|
980
|
+
type EffectDivergenceReason = "missing-effect" | "extra-effect" | "kind" | "model-method" | "model-call-shape" | "model-error" | "tool-name" | "tool-input" | "tool-result" | "output";
|
|
981
|
+
/** The first effect where two recordings differ. */
|
|
982
|
+
interface EffectDivergence {
|
|
983
|
+
/** Position in the compared (tool-order-normalized) effect lists. */
|
|
984
|
+
index: number;
|
|
985
|
+
reason: EffectDivergenceReason;
|
|
986
|
+
expected?: RecordedEffect;
|
|
987
|
+
actual?: RecordedEffect;
|
|
988
|
+
/** One sentence, for test failures. */
|
|
989
|
+
message: string;
|
|
990
|
+
}
|
|
991
|
+
/** The result of {@link compareRecordings}. */
|
|
992
|
+
interface EquivalenceReport {
|
|
993
|
+
equivalent: boolean;
|
|
994
|
+
/** The first divergence, by position in the original's effects. */
|
|
995
|
+
divergence?: EffectDivergence;
|
|
996
|
+
/** Every divergence, in order. Effects both runs made line up between them. */
|
|
997
|
+
divergences: EffectDivergence[];
|
|
998
|
+
}
|
|
999
|
+
/** Options for {@link compareRecordings}. */
|
|
1000
|
+
interface CompareRecordingsOptions {
|
|
1001
|
+
/** When two model calls count as the same. Default `"shape"`. */
|
|
1002
|
+
modelCalls?: ModelCallMatch;
|
|
1003
|
+
/**
|
|
1004
|
+
* Maps each effect before comparing, on both sides: redact values that
|
|
1005
|
+
* differ on every run (generated ids, timestamps) or project a tool's input
|
|
1006
|
+
* to the fields that matter.
|
|
1007
|
+
*/
|
|
1008
|
+
normalize?: (effect: RecordedEffect) => RecordedEffect;
|
|
1009
|
+
}
|
|
1010
|
+
/** True when two model calls offer the same tool names and response format. */
|
|
1011
|
+
export declare function sameModelCallShape(expected: RecordedModelCall, actual: RecordedModelCall): boolean;
|
|
1012
|
+
/**
|
|
1013
|
+
* Compares two recordings and reports every effect that differs. `expected`
|
|
1014
|
+
* is the original's recording, `actual` the port's. Effects that are the same
|
|
1015
|
+
* on both sides are lined up first (longest common subsequence), so one extra
|
|
1016
|
+
* call does not shift every effect after it; between them, effects of the same
|
|
1017
|
+
* kind pair up and report what changed, and the rest are missing or extra.
|
|
1018
|
+
*/
|
|
1019
|
+
export declare function compareRecordings(expected: Recording, actual: Recording, options?: CompareRecordingsOptions): EquivalenceReport;
|
|
1020
|
+
//#endregion
|
|
1021
|
+
export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AgreementReport, type AgreementRow, type AgreementSummary, type AssertAgentMachineOptions, type CanReachResult, type CanReachTarget, type CompareRecordingsOptions, type CompareRunsOptions, type CoverageReport, type CreateRecorderOptions, type Divergence, type EffectDivergence, type EffectDivergenceReason, type EquivalenceReport, type ExplorePathsOptions, type ExploreSnapshot, type FirstDivergenceOptions, type FoldTracesOptions, type GetDecisionRequestsOptions, type HandlerCoverage, type HandlerRef, type JudgeAgreementOptions, type LintAgentMachineOptions, type MatchTrajectoryOptions, type ModelCallMatch, type ParityLabel, type ParityReport, type ParityResult, type ParityRun, type RecordableLanguageModel, type RecordedEffect, type RecordedError, type RecordedModelCall, type RecordedOutput, type RecordedToolCall, type Recorder, type Recording, type RepeatGroup, type RepeatReport, type RepeatResult, type RepeatRun, type RepeatRunsOptions, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type StateCoverage, type StateStats, type TraceCoverageOptions, type TraceStep, type TracedDecisionRequest, type TrajectoryEvent, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, explorePaths, lintAgentMachine, simulateAgent };
|