@statelyai/agent 2.0.0-alpha.34 → 2.0.0-alpha.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +124 -0
- package/dist/act-B9hLQls6.mjs +19 -0
- package/dist/act-CrX0JwWY.cjs +42 -0
- package/dist/ai-sdk.cjs +63 -7
- package/dist/ai-sdk.d.cts +13 -3
- package/dist/ai-sdk.d.mts +13 -3
- package/dist/ai-sdk.mjs +63 -7
- package/dist/{decision-BEsATcr1.cjs → decision-CMeuNZ1g.cjs} +262 -4
- package/dist/{decision-COkJNjOW.mjs → decision-m5hGmHpF.mjs} +239 -5
- package/dist/decision-object-B5zGSPFp.d.mts +15 -0
- package/dist/decision-object-DBcLWVvv.d.cts +15 -0
- package/dist/{run-agent-s5D32w99.cjs → effects-B23bFZEs.cjs} +948 -102
- package/dist/{run-agent-Ct_GZw_f.mjs → effects-CuOUNYtZ.mjs} +914 -104
- package/dist/{event-log-C7jeH0y8.mjs → event-log-DkVlk3Rh.mjs} +30 -15
- package/dist/{event-log-BWSSLUff.cjs → event-log-Dkc-yIYu.cjs} +41 -14
- package/dist/{event-log-store-BzXehP7R.d.cts → event-log-store-BBS8so8a.d.mts} +2 -2
- package/dist/{event-log-store-W49TCJXi.mjs → event-log-store-Brx9tOHt.mjs} +1 -1
- package/dist/{event-log-store-CleWpsp5.cjs → event-log-store-Cl-YWDnq.cjs} +1 -1
- package/dist/{event-log-store-CWjN6lmM.d.mts → event-log-store-bc4ICGcS.d.cts} +2 -2
- package/dist/index.cjs +660 -286
- package/dist/index.d.cts +168 -9
- package/dist/index.d.mts +168 -9
- package/dist/index.mjs +639 -271
- package/dist/internal-C3Lqu538.d.cts +43 -0
- package/dist/internal-DUcdhk-i.d.mts +43 -0
- package/dist/log.cjs +2 -2
- package/dist/log.d.cts +1 -1
- package/dist/log.d.mts +1 -1
- package/dist/log.mjs +2 -2
- package/dist/machines.cjs +457 -41
- package/dist/machines.d.cts +114 -43
- package/dist/machines.d.mts +114 -43
- package/dist/machines.mjs +457 -42
- package/dist/{model-source-DmhTMK39.cjs → model-source-B4WQ8Idp.cjs} +1 -1
- package/dist/{model-source-CK3F4YuU.mjs → model-source-kjGwqAV8.mjs} +1 -1
- package/dist/openai.cjs +149 -36
- package/dist/openai.d.cts +28 -10
- package/dist/openai.d.mts +28 -10
- package/dist/openai.mjs +149 -36
- package/dist/otel.d.cts +1 -1
- package/dist/otel.d.mts +1 -1
- package/dist/{run-agent-BGdx5913.d.mts → run-agent-BcQ-9NRT.d.cts} +91 -9
- package/dist/{run-agent-B1-Eqviq.d.cts → run-agent-C86drZtN.d.mts} +91 -9
- package/dist/{setup-agent-05uw4dPZ.mjs → setup-agent-BDmNr6PC.mjs} +9 -3
- package/dist/{setup-agent-BtfeFfF6.d.cts → setup-agent-BIKcbVXV.d.mts} +4 -4
- package/dist/{setup-agent-CXEBURdo.cjs → setup-agent-CmEToB8y.cjs} +9 -3
- package/dist/{setup-agent-C6nqzv_X.d.mts → setup-agent-jlo4vtaZ.d.cts} +4 -4
- package/dist/tanstack-ai.cjs +62 -4
- package/dist/tanstack-ai.d.cts +18 -6
- package/dist/tanstack-ai.d.mts +18 -6
- package/dist/tanstack-ai.mjs +62 -4
- package/dist/testing.cjs +536 -67
- package/dist/testing.d.cts +197 -180
- package/dist/testing.d.mts +197 -180
- package/dist/testing.mjs +532 -67
- package/dist/{text-logic-CV_4KGr6.d.cts → text-logic-BqxalgMr.d.mts} +87 -4
- package/dist/{text-logic-Ckh-rJFV.d.mts → text-logic-CSf9bqpP.d.cts} +87 -4
- package/dist/{types-Dp5_Ru4R.d.cts → types-DQUOOaZO.d.cts} +1 -1
- package/dist/{types-Dp5_Ru4R.d.mts → types-DQUOOaZO.d.mts} +1 -1
- package/dist/{usage-BDiJKyv9.d.cts → usage-DoP0Oi84.d.mts} +1 -1
- package/dist/{usage-DOoVUGVq.d.mts → usage-HtnrUsl9.d.cts} +1 -1
- package/dist/utils-7lSa4H-i.d.cts +403 -0
- package/dist/utils-CBbKw2fM.d.mts +403 -0
- package/dist/validate.cjs +44 -12
- package/dist/validate.d.cts +6 -6
- package/dist/validate.d.mts +6 -6
- package/dist/validate.mjs +44 -12
- package/dist/verify-Bink-Drw.cjs +2304 -0
- package/dist/{verify-BQpE8hUz.d.cts → verify-Buh8XSUr.d.mts} +208 -14
- package/dist/{verify-CqeuEmIk.d.mts → verify-CciG8UP-.d.cts} +208 -14
- package/dist/verify-Dpz1u5Pe.mjs +2251 -0
- package/dist/with-model-calls-B_WLmOSl.cjs +248 -0
- package/dist/with-model-calls-BkOwdroS.mjs +213 -0
- package/docs/act.md +102 -0
- package/docs/advanced.md +105 -0
- package/docs/any-stack.md +62 -0
- package/docs/choosing-a-run-mode.md +55 -0
- package/docs/debugging.md +29 -0
- package/docs/decisions.md +473 -0
- package/docs/environment-and-policy.md +188 -0
- package/docs/evals.md +509 -0
- package/docs/event-log.md +311 -0
- package/docs/from-a-loop.md +528 -0
- package/docs/generate-machines.md +110 -0
- package/docs/hosts.md +414 -0
- package/docs/human-in-the-loop.md +177 -0
- package/docs/index.md +48 -0
- package/docs/judgments.md +281 -0
- package/docs/langgraph-comparison.md +198 -0
- package/docs/machines-as-data.md +454 -0
- package/docs/machines-presets.md +198 -0
- package/docs/machines.md +497 -0
- package/docs/messages.md +67 -0
- package/docs/meta.json +52 -0
- package/docs/models-and-providers.md +268 -0
- package/docs/multi-agent.md +68 -0
- package/docs/observability.md +109 -0
- package/docs/patterns.md +31 -0
- package/docs/persistence.md +136 -0
- package/docs/quickstart.md +121 -0
- package/docs/roadmap.md +22 -0
- package/docs/scope.md +75 -0
- package/docs/statechart-policy-examples.md +69 -0
- package/docs/steps.md +54 -0
- package/docs/strangler-fig.md +323 -0
- package/docs/text-requests.md +250 -0
- package/docs/thinking-in-state-machines.md +324 -0
- package/docs/tools.md +249 -0
- package/docs/upgrading.md +37 -0
- package/docs/usage-and-budgets.md +516 -0
- package/docs/validate-and-repair.md +253 -0
- package/docs/verify.md +487 -0
- package/docs/xstate-4-and-5-apps.md +161 -0
- package/llms.txt +53 -0
- package/package.json +10 -3
- package/readme.md +30 -15
- package/skills/add-evals/SKILL.md +3 -1
- package/skills/add-traces/SKILL.md +6 -1
- package/skills/build-machine/SKILL.md +81 -0
- package/skills/generate-machine/SKILL.md +1 -1
- package/skills/graph-engineering/SKILL.md +13 -3
- package/skills/improve-machine/SKILL.md +21 -2
- package/skills/loop-to-machine/SKILL.md +147 -190
- package/skills/loop-to-machine/reference.md +260 -0
- package/dist/utils-BnXdWih3.d.mts +0 -108
- package/dist/utils-ebVpCYoF.d.cts +0 -108
- package/dist/verify-Bpahmr34.mjs +0 -1241
- package/dist/verify-DCLZa9R0.cjs +0 -1288
- package/dist/with-model-calls-BXYdBdcu.cjs +0 -45
- package/dist/with-model-calls-Bv4V4EHC.mjs +0 -34
package/dist/testing.d.mts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { t as MetaOfSnapshot } from "./utils-
|
|
4
|
-
import { _ as assertAgentMachine, a as AgentPathTerminal, b as lintAgentMachine, c as CanReachResult, d as ExploreSnapshot, f as LintAgentMachineOptions, g as SimulationTrailEntry, h as SimulationScript, i as AgentPathReport, l as CanReachTarget, m as SimulateAgentResult, n as AgentLintError, o as AgentUnknownStateError, p as SimulateAgentOptions, r as AgentLintSeverity, s as AssertAgentMachineOptions, t as AgentLintDiagnostic, u as ExplorePathsOptions, v as canReach, x as simulateAgent, y as explorePaths } from "./verify-
|
|
1
|
+
import { R as AgentExecutorDecisionRequest, c as AgentRequestExecutorInfo, f as AgentTextRequest, s as AgentRequestExecutor, t as AgentCallUsage, u as AgentRequestExecutors } from "./text-logic-BqxalgMr.mjs";
|
|
2
|
+
import { p as AgentRuntimeOptions, s as AgentRunInit, u as AgentRunResult } from "./run-agent-C86drZtN.mjs";
|
|
3
|
+
import { A as RecordedOutput, C as RecordableDecisionModel, D as RecordedEffect, E as RecordedCall, F as compareRecordings, I as createRecorder, L as sameModelCallShape, M as Recorder, N as Recording, O as RecordedError, P as RecordingDivergenceError, S as RecordClientOptions, T as RecordableModel, _ as EquivalenceReport, b as NormalizeEffect, g as EffectDivergenceReason, h as EffectDivergence, j as RecordedToolCall, k as RecordedModelCall, m as CreateRecorderOptions, p as CompareRecordingsOptions, t as MetaOfSnapshot, v as ModelAnswer, w as RecordableLanguageModel, x as RecordCallOptions, y as ModelCallMatch } from "./utils-CBbKw2fM.mjs";
|
|
4
|
+
import { S as textOutput, _ as assertAgentMachine, a as AgentPathTerminal, b as lintAgentMachine, c as CanReachResult, d as ExploreSnapshot, f as LintAgentMachineOptions, g as SimulationTrailEntry, h as SimulationScript, i as AgentPathReport, l as CanReachTarget, m as SimulateAgentResult, n as AgentLintError, o as AgentUnknownStateError, p as SimulateAgentOptions, r as AgentLintSeverity, s as AssertAgentMachineOptions, t as AgentLintDiagnostic, u as ExplorePathsOptions, v as canReach, x as simulateAgent, y as explorePaths } from "./verify-Buh8XSUr.mjs";
|
|
5
5
|
import { AnyStateMachine, EventFromLogic, InputFrom, SnapshotFrom, StateValue } from "xstate";
|
|
6
6
|
//#region src/trajectory.d.ts
|
|
7
7
|
/** An event-shaped trajectory item: anything with a string `type`. */
|
|
@@ -515,6 +515,125 @@ export declare function firstDivergence(actual: readonly {
|
|
|
515
515
|
readonly type: string;
|
|
516
516
|
}[] | readonly TrajectoryItem[], options?: FirstDivergenceOptions): Divergence | undefined;
|
|
517
517
|
//#endregion
|
|
518
|
+
//#region src/propose-from-traces.d.ts
|
|
519
|
+
/** A person's verdict as {@link proposeApprovalRules} hands it to `key`. */
|
|
520
|
+
interface ApprovalVerdict<TTrace> {
|
|
521
|
+
/** The approve or reject event, as traced. */
|
|
522
|
+
event: Record<string, unknown>;
|
|
523
|
+
/** The traced snapshot the verdict arrived in: the approval state, with the context that holds the pending call. */
|
|
524
|
+
snapshot: unknown;
|
|
525
|
+
/** The whole run. */
|
|
526
|
+
trace: TTrace;
|
|
527
|
+
}
|
|
528
|
+
/** Options for {@link proposeApprovalRules}. */
|
|
529
|
+
interface ProposeApprovalRulesOptions<TTrace> {
|
|
530
|
+
/** The state's dot path, such as `'awaitingApproval'`. A verdict counts when the run was in it or in a state inside it. */
|
|
531
|
+
approvalState: string;
|
|
532
|
+
/** Event types that approve, such as `['APPROVE']`. */
|
|
533
|
+
approveEvents: readonly string[];
|
|
534
|
+
/** Event types that reject, such as `['REJECT']`. */
|
|
535
|
+
rejectEvents: readonly string[];
|
|
536
|
+
/**
|
|
537
|
+
* The rule a verdict is evidence for: usually the tool name and a pattern
|
|
538
|
+
* of its arguments, read from `snapshot.context`. `undefined` skips the verdict.
|
|
539
|
+
*/
|
|
540
|
+
key: (verdict: ApprovalVerdict<TTrace>) => string | undefined;
|
|
541
|
+
/**
|
|
542
|
+
* The keys a run broke a rule for, from ground truth (a ledger checked
|
|
543
|
+
* after the fact). Any violation blocks `allow` for that key.
|
|
544
|
+
*/
|
|
545
|
+
violations?: (trace: TTrace) => Iterable<string>;
|
|
546
|
+
}
|
|
547
|
+
/** One key's row from {@link proposeApprovalRules}. */
|
|
548
|
+
interface ApprovalRuleProposal {
|
|
549
|
+
key: string;
|
|
550
|
+
/** Approve verdicts, summed over runs. */
|
|
551
|
+
approved: number;
|
|
552
|
+
/** Reject verdicts, summed over runs. */
|
|
553
|
+
rejected: number;
|
|
554
|
+
/** Violations `violations` reported, summed over runs. 0 without `violations`. */
|
|
555
|
+
violations: number;
|
|
556
|
+
/** Indexes, in input order, of the runs with a verdict for this key. */
|
|
557
|
+
runs: number[];
|
|
558
|
+
/**
|
|
559
|
+
* `'allow'`: approved, never rejected, no violation. `'deny'`: rejected,
|
|
560
|
+
* never approved. `null`: mixed evidence.
|
|
561
|
+
*/
|
|
562
|
+
proposal: "allow" | "deny" | null;
|
|
563
|
+
/** Runs with a verdict for this key: `runs.length`. */
|
|
564
|
+
support: number;
|
|
565
|
+
}
|
|
566
|
+
/**
|
|
567
|
+
* Groups the approval verdicts in many runs' traces by `key` and proposes an
|
|
568
|
+
* `allow` or `deny` rule for each. Pure and deterministic; a proposal is for
|
|
569
|
+
* a person to review, and nothing here changes a machine.
|
|
570
|
+
*
|
|
571
|
+
* Reads `machine.transition` events: a verdict is a transition, not ignored,
|
|
572
|
+
* out of `approvalState` on an approve or reject event. Rows come in the
|
|
573
|
+
* order keys were first seen.
|
|
574
|
+
*
|
|
575
|
+
* @example
|
|
576
|
+
* ```ts no-check
|
|
577
|
+
* const rows = proposeApprovalRules(traces, {
|
|
578
|
+
* approvalState: "awaitingApproval",
|
|
579
|
+
* approveEvents: ["APPROVE"],
|
|
580
|
+
* rejectEvents: ["REJECT"],
|
|
581
|
+
* key: ({ snapshot }) => `${snapshot.context.pending.tool}:${snapshot.context.pending.path}`,
|
|
582
|
+
* });
|
|
583
|
+
* ```
|
|
584
|
+
*/
|
|
585
|
+
export declare function proposeApprovalRules<TTrace extends readonly {
|
|
586
|
+
readonly type: string;
|
|
587
|
+
}[]>(traces: Iterable<TTrace>, options: ProposeApprovalRulesOptions<TTrace>): ApprovalRuleProposal[];
|
|
588
|
+
/** One cap for {@link proposeCaps}. Give `failureState` or `failureOutput`. */
|
|
589
|
+
interface CapSpec<TTrace> {
|
|
590
|
+
/** Names the row, such as `'retries'`. */
|
|
591
|
+
name: string;
|
|
592
|
+
/** The cap's value now. A cap runs hit is never proposed below it. */
|
|
593
|
+
current?: number;
|
|
594
|
+
/** The run hit the cap when it entered this state (dot path), such as `'gaveUp'`. */
|
|
595
|
+
failureState?: string;
|
|
596
|
+
/** The run hit the cap when its `run.end` output satisfies this. */
|
|
597
|
+
failureOutput?: (output: unknown) => boolean;
|
|
598
|
+
/** How much of the cap the run used: retries made, steps taken, batch size. `undefined` leaves the run out. */
|
|
599
|
+
valueFromRun: (trace: TTrace) => number | undefined;
|
|
600
|
+
}
|
|
601
|
+
/** One cap's row from {@link proposeCaps}. */
|
|
602
|
+
interface CapProposal {
|
|
603
|
+
name: string;
|
|
604
|
+
/** Runs that hit the cap. */
|
|
605
|
+
hits: number;
|
|
606
|
+
/** Runs that did not, with a value. */
|
|
607
|
+
passing: number;
|
|
608
|
+
/** The largest value over all runs; `null` with none. */
|
|
609
|
+
max: number | null;
|
|
610
|
+
/** The 95th percentile (nearest rank) over passing runs; `null` with none. */
|
|
611
|
+
p95: number | null;
|
|
612
|
+
/** `p95 + 1`, and never below `current` when any run hit the cap; `null` with no passing run. */
|
|
613
|
+
proposal: number | null;
|
|
614
|
+
}
|
|
615
|
+
/** Options for {@link proposeCaps}. */
|
|
616
|
+
interface ProposeCapsOptions<TTrace> {
|
|
617
|
+
caps: readonly CapSpec<TTrace>[];
|
|
618
|
+
}
|
|
619
|
+
/**
|
|
620
|
+
* Proposes a value for each cap (retry count, steps, batch size) from many
|
|
621
|
+
* runs' traces: the 95th percentile of what passing runs used, plus one, and
|
|
622
|
+
* never below the current cap while runs still hit it. Pure and
|
|
623
|
+
* deterministic; a proposal is for a person to review, and nothing here
|
|
624
|
+
* changes a machine.
|
|
625
|
+
*
|
|
626
|
+
* @example
|
|
627
|
+
* ```ts no-check
|
|
628
|
+
* const [retries] = proposeCaps(traces, {
|
|
629
|
+
* caps: [{ name: "retries", current: 3, failureState: "gaveUp", valueFromRun: countRetries }],
|
|
630
|
+
* });
|
|
631
|
+
* ```
|
|
632
|
+
*/
|
|
633
|
+
export declare function proposeCaps<TTrace extends readonly {
|
|
634
|
+
readonly type: string;
|
|
635
|
+
}[]>(traces: Iterable<TTrace>, options: ProposeCapsOptions<TTrace>): CapProposal[];
|
|
636
|
+
//#endregion
|
|
518
637
|
//#region src/decision-requests.d.ts
|
|
519
638
|
/** One decision a run asked for. See {@link getDecisionRequests}. */
|
|
520
639
|
interface TracedDecisionRequest {
|
|
@@ -829,193 +948,91 @@ export declare function compareRuns<TScenario extends {
|
|
|
829
948
|
expected?: string;
|
|
830
949
|
}>(options: CompareRunsOptions<TScenario>): Promise<ParityReport<TScenario>>;
|
|
831
950
|
//#endregion
|
|
832
|
-
//#region src/
|
|
833
|
-
/**
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
* side effects, and records what the port did.
|
|
848
|
-
* 3. `compareRecordings(original, port)` reports the first effect that differs.
|
|
849
|
-
*
|
|
850
|
-
* Provider-free: both seams are wrapped structurally, so this module imports
|
|
851
|
-
* neither `ai` nor `openai`.
|
|
852
|
-
* @module
|
|
853
|
-
*/
|
|
854
|
-
/** An error as recorded: JSON-safe. */
|
|
855
|
-
interface RecordedError {
|
|
856
|
-
name: string;
|
|
857
|
-
message: string;
|
|
858
|
-
code?: unknown;
|
|
859
|
-
}
|
|
860
|
-
/** One model call. `call` is its 0-based position among model calls. */
|
|
861
|
-
interface RecordedModelCall {
|
|
862
|
-
kind: "model";
|
|
863
|
-
/** Which wrapper saw it. */
|
|
864
|
-
seam: "ai-sdk" | "openai";
|
|
865
|
-
/** `doGenerate`/`doStream`, or `chat.completions.create`/`responses.create`. */
|
|
866
|
-
method: string;
|
|
867
|
-
call: number;
|
|
868
|
-
/** The call options or params, JSON-safe (abort signals and headers dropped). */
|
|
869
|
-
request: unknown;
|
|
870
|
-
/** The non-streaming result. */
|
|
871
|
-
response?: unknown;
|
|
872
|
-
/** The stream's parts or chunks, in order. */
|
|
873
|
-
stream?: unknown[];
|
|
874
|
-
error?: RecordedError;
|
|
875
|
-
}
|
|
876
|
-
/** One tool call. `step` is the number of model calls made before it. */
|
|
877
|
-
interface RecordedToolCall {
|
|
878
|
-
kind: "tool";
|
|
879
|
-
name: string;
|
|
880
|
-
step: number;
|
|
881
|
-
input: unknown;
|
|
882
|
-
output?: unknown;
|
|
883
|
-
error?: RecordedError;
|
|
884
|
-
/** The tool returned without a Promise; replay answers synchronously too. */
|
|
885
|
-
sync?: true;
|
|
886
|
-
/** Started while an earlier tool call was still running. Concurrent calls
|
|
887
|
-
* may finish in either order; sequential ones must keep theirs. */
|
|
888
|
-
concurrent?: true;
|
|
889
|
-
}
|
|
890
|
-
/** The run's final result. */
|
|
891
|
-
interface RecordedOutput {
|
|
892
|
-
kind: "output";
|
|
893
|
-
value: unknown;
|
|
894
|
-
}
|
|
895
|
-
type RecordedEffect = RecordedModelCall | RecordedToolCall | RecordedOutput;
|
|
896
|
-
/** A run's effects, in order. JSON-serializable: save it as a golden file. */
|
|
897
|
-
interface Recording {
|
|
898
|
-
version: 1;
|
|
899
|
-
effects: RecordedEffect[];
|
|
900
|
-
}
|
|
901
|
-
/** Options for {@link createRecorder}. Omit for record mode. */
|
|
902
|
-
interface CreateRecorderOptions {
|
|
903
|
-
/** Answer model and tool calls from this recording instead of running them. */
|
|
904
|
-
replay?: Recording;
|
|
905
|
-
/**
|
|
906
|
-
* In replay, `"replay"` (default) returns each tool's recorded result
|
|
907
|
-
* without running it; `"live"` runs the wrapped tool (your stub) and records
|
|
908
|
-
* what it returned.
|
|
909
|
-
*/
|
|
910
|
-
tools?: "replay" | "live";
|
|
911
|
-
/**
|
|
912
|
-
* In replay, which model calls a recorded answer may serve. `"shape"`
|
|
913
|
-
* (default) refuses a call whose offered tools or response format changed,
|
|
914
|
-
* since the recorded answer may not fit it; see {@link ModelCallMatch}.
|
|
915
|
-
*/
|
|
916
|
-
modelCalls?: ModelCallMatch;
|
|
951
|
+
//#region src/check-parity.d.ts
|
|
952
|
+
/** One scenario: an id and the workflow's input. Extra fields are yours. */
|
|
953
|
+
interface CheckParityScenario<TInput = unknown> {
|
|
954
|
+
id: string;
|
|
955
|
+
input: TInput;
|
|
956
|
+
}
|
|
957
|
+
/** How one side of a scenario ended: what it returned, or what it threw. */
|
|
958
|
+
type CheckParityOutcome<TOutput = unknown> = {
|
|
959
|
+
output: TOutput;
|
|
960
|
+
} | {
|
|
961
|
+
error: unknown;
|
|
962
|
+
};
|
|
963
|
+
/** Options for {@link checkParity}. */
|
|
964
|
+
interface CheckParityOptions<TScenario extends CheckParityScenario, TEffects extends object, TOutput> {
|
|
965
|
+
scenarios: readonly TScenario[];
|
|
917
966
|
/**
|
|
918
|
-
*
|
|
919
|
-
*
|
|
920
|
-
*
|
|
967
|
+
* Fresh effects for one scenario: fakes, stubs, or real clients. `before`
|
|
968
|
+
* runs on them for real, recorded, so a write effect writes once per
|
|
969
|
+
* scenario: point it at an isolated fixture, not production. `after` gets
|
|
970
|
+
* replay stand-ins and never runs them. A class instance's prototype
|
|
971
|
+
* methods count, bound to the instance.
|
|
921
972
|
*/
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
*/
|
|
930
|
-
type ModelCallMatch = "shape" | "none" | ((expected: RecordedModelCall, actual: RecordedModelCall) => boolean);
|
|
931
|
-
/** The structural slice of an AI SDK `LanguageModelV3` the recorder wraps. */
|
|
932
|
-
interface RecordableLanguageModel {
|
|
933
|
-
doGenerate(options: never): PromiseLike<unknown>;
|
|
934
|
-
doStream(options: never): PromiseLike<{
|
|
935
|
-
stream: ReadableStream<unknown>;
|
|
936
|
-
}>;
|
|
937
|
-
}
|
|
938
|
-
type AnyFunction = (...args: never[]) => unknown;
|
|
939
|
-
/** Records (or replays) the effects of one run. */
|
|
940
|
-
interface Recorder {
|
|
973
|
+
effects: (scenario: TScenario) => TEffects;
|
|
974
|
+
/** The original workflow. */
|
|
975
|
+
before: (input: TScenario["input"], effects: TEffects) => PromiseLike<TOutput>;
|
|
976
|
+
/** The port: a `wrapWorkflow` run (`(input, effects) => step.run(input, { effects })`) or any runner with the same signature. */
|
|
977
|
+
after: (input: TScenario["input"], effects: TEffects) => PromiseLike<unknown>;
|
|
978
|
+
/** Passed to `compareRecordings`. */
|
|
979
|
+
compare?: CompareRecordingsOptions;
|
|
941
980
|
/**
|
|
942
|
-
*
|
|
943
|
-
*
|
|
944
|
-
* model built with no API key works.
|
|
981
|
+
* Maps each side's output before it is compared: drop or pin what differs
|
|
982
|
+
* run to run (a duration, a generated id). Rows keep the raw outputs.
|
|
945
983
|
*/
|
|
946
|
-
|
|
984
|
+
normalizeOutput?: (output: unknown, scenario: TScenario) => unknown;
|
|
947
985
|
/**
|
|
948
|
-
*
|
|
949
|
-
*
|
|
950
|
-
*
|
|
951
|
-
*
|
|
986
|
+
* Maps each side's thrown error before it is compared: drop what differs
|
|
987
|
+
* run to run (an embedded stack line, a generated id). An `Error` result
|
|
988
|
+
* matches on name and message, anything else on its JSON. Rows keep the
|
|
989
|
+
* raw errors.
|
|
952
990
|
*/
|
|
953
|
-
|
|
954
|
-
/** Wraps one tool function. Its arguments are the recorded input. */
|
|
955
|
-
tool<TFn extends AnyFunction>(name: string, fn: TFn): TFn;
|
|
956
|
-
/** Wraps every `execute` in an AI SDK tool set, named by key. */
|
|
957
|
-
tools<TTools extends Record<string, {
|
|
958
|
-
execute?: AnyFunction;
|
|
959
|
-
}>>(tools: TTools): TTools;
|
|
960
|
-
/** Records the run's final result. */
|
|
961
|
-
output(value: unknown): void;
|
|
962
|
-
/** What this run did so far. */
|
|
963
|
-
readonly recording: Recording;
|
|
964
|
-
}
|
|
965
|
-
/**
|
|
966
|
-
* Thrown in replay when the port asks for something the recording cannot
|
|
967
|
-
* answer: a model call past the last recorded one, or a tool call the original
|
|
968
|
-
* never made. The effect is still recorded, so {@link compareRecordings}
|
|
969
|
-
* reports it.
|
|
970
|
-
*/
|
|
971
|
-
export declare class RecordingDivergenceError extends Error {
|
|
972
|
-
name: string;
|
|
991
|
+
normalizeError?: (error: unknown, scenario: TScenario) => unknown;
|
|
973
992
|
}
|
|
974
|
-
/**
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
/** Why two effects differ. */
|
|
980
|
-
type EffectDivergenceReason = "missing-effect" | "extra-effect" | "kind" | "model-method" | "model-call-shape" | "model-error" | "tool-name" | "tool-input" | "tool-result" | "output";
|
|
981
|
-
/** The first effect where two recordings differ. */
|
|
982
|
-
interface EffectDivergence {
|
|
983
|
-
/** Position in the compared (tool-order-normalized) effect lists. */
|
|
984
|
-
index: number;
|
|
985
|
-
reason: EffectDivergenceReason;
|
|
986
|
-
expected?: RecordedEffect;
|
|
987
|
-
actual?: RecordedEffect;
|
|
988
|
-
/** One sentence, for test failures. */
|
|
989
|
-
message: string;
|
|
990
|
-
}
|
|
991
|
-
/** The result of {@link compareRecordings}. */
|
|
992
|
-
interface EquivalenceReport {
|
|
993
|
+
/** One scenario's row in a {@link CheckParityReport}. */
|
|
994
|
+
interface CheckParityRow<TScenario extends CheckParityScenario, TOutput> {
|
|
995
|
+
id: string;
|
|
996
|
+
scenario: TScenario;
|
|
997
|
+
/** No divergence, and both sides returned the same value or threw the same error. */
|
|
993
998
|
equivalent: boolean;
|
|
994
|
-
/** The first divergence, by position in the original's effects. */
|
|
995
|
-
divergence?: EffectDivergence;
|
|
996
|
-
/** Every divergence, in order. Effects both runs made line up between them. */
|
|
997
999
|
divergences: EffectDivergence[];
|
|
1000
|
+
before: CheckParityOutcome<TOutput>;
|
|
1001
|
+
after: CheckParityOutcome;
|
|
1002
|
+
/** The golden recording of `before`: save it as a fixture. */
|
|
1003
|
+
recording: Recording;
|
|
1004
|
+
/** What `after` did in replay. */
|
|
1005
|
+
replay: Recording;
|
|
998
1006
|
}
|
|
999
|
-
/**
|
|
1000
|
-
interface
|
|
1001
|
-
/**
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
* Maps each effect before comparing, on both sides: redact values that
|
|
1005
|
-
* differ on every run (generated ids, timestamps) or project a tool's input
|
|
1006
|
-
* to the fields that matter.
|
|
1007
|
-
*/
|
|
1008
|
-
normalize?: (effect: RecordedEffect) => RecordedEffect;
|
|
1007
|
+
/** The result of {@link checkParity}. */
|
|
1008
|
+
interface CheckParityReport<TScenario extends CheckParityScenario, TOutput> {
|
|
1009
|
+
/** Every scenario is equivalent. */
|
|
1010
|
+
equivalent: boolean;
|
|
1011
|
+
scenarios: CheckParityRow<TScenario, TOutput>[];
|
|
1009
1012
|
}
|
|
1010
|
-
/** True when two model calls offer the same tool names and response format. */
|
|
1011
|
-
export declare function sameModelCallShape(expected: RecordedModelCall, actual: RecordedModelCall): boolean;
|
|
1012
1013
|
/**
|
|
1013
|
-
*
|
|
1014
|
-
*
|
|
1015
|
-
*
|
|
1016
|
-
*
|
|
1017
|
-
*
|
|
1014
|
+
* Checks a port against the workflow it replaces, scenario by scenario: runs
|
|
1015
|
+
* `before` on fresh `effects` under `createRecorder`, replays `after` against
|
|
1016
|
+
* that recording (no effect really runs), and compares the two with
|
|
1017
|
+
* `compareRecordings`. Effects are compared by name, call order, arguments
|
|
1018
|
+
* and results; the output by value.
|
|
1019
|
+
*
|
|
1020
|
+
* Never throws for a port that misbehaves: an `after` that throws (a
|
|
1021
|
+
* `RecordingDivergenceError` for a call the original never made, or its own
|
|
1022
|
+
* error) is a row with `after: { error }`. When both sides throw, they match
|
|
1023
|
+
* on the error's name and message, after `normalizeError` when given.
|
|
1024
|
+
*
|
|
1025
|
+
* @example
|
|
1026
|
+
* ```ts
|
|
1027
|
+
* const report = await checkParity({
|
|
1028
|
+
* scenarios: [{ id: "refund", input: ticket }],
|
|
1029
|
+
* effects: () => createFakeEffects(),
|
|
1030
|
+
* before: (input, effects) => legacyRunTicket(input, effects),
|
|
1031
|
+
* after: (input, effects) => step1.run(input, { effects }),
|
|
1032
|
+
* });
|
|
1033
|
+
* expect(report.equivalent).toBe(true);
|
|
1034
|
+
* ```
|
|
1018
1035
|
*/
|
|
1019
|
-
export declare function
|
|
1036
|
+
export declare function checkParity<TScenario extends CheckParityScenario, TEffects extends object, TOutput>(options: CheckParityOptions<TScenario, TEffects, TOutput>): Promise<CheckParityReport<TScenario, TOutput>>;
|
|
1020
1037
|
//#endregion
|
|
1021
|
-
export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AgreementReport, type AgreementRow, type AgreementSummary, type AssertAgentMachineOptions, type CanReachResult, type CanReachTarget, type CompareRecordingsOptions, type CompareRunsOptions, type CoverageReport, type CreateRecorderOptions, type Divergence, type EffectDivergence, type EffectDivergenceReason, type EquivalenceReport, type ExplorePathsOptions, type ExploreSnapshot, type FirstDivergenceOptions, type FoldTracesOptions, type GetDecisionRequestsOptions, type HandlerCoverage, type HandlerRef, type JudgeAgreementOptions, type LintAgentMachineOptions, type MatchTrajectoryOptions, type ModelCallMatch, type ParityLabel, type ParityReport, type ParityResult, type ParityRun, type RecordableLanguageModel, type RecordedEffect, type RecordedError, type RecordedModelCall, type RecordedOutput, type RecordedToolCall, type Recorder, type Recording, type RepeatGroup, type RepeatReport, type RepeatResult, type RepeatRun, type RepeatRunsOptions, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type StateCoverage, type StateStats, type TraceCoverageOptions, type TraceStep, type TracedDecisionRequest, type TrajectoryEvent, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, explorePaths, lintAgentMachine, simulateAgent };
|
|
1038
|
+
export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AgreementReport, type AgreementRow, type AgreementSummary, type ApprovalRuleProposal, type ApprovalVerdict, type AssertAgentMachineOptions, type CanReachResult, type CanReachTarget, type CapProposal, type CapSpec, type CheckParityOptions, type CheckParityOutcome, type CheckParityReport, type CheckParityRow, type CheckParityScenario, type CompareRecordingsOptions, type CompareRunsOptions, type CoverageReport, type CreateRecorderOptions, type Divergence, type EffectDivergence, type EffectDivergenceReason, type EquivalenceReport, type ExplorePathsOptions, type ExploreSnapshot, type FirstDivergenceOptions, type FoldTracesOptions, type GetDecisionRequestsOptions, type HandlerCoverage, type HandlerRef, type JudgeAgreementOptions, type LintAgentMachineOptions, type MatchTrajectoryOptions, type ModelAnswer, type ModelCallMatch, type NormalizeEffect, type ParityLabel, type ParityReport, type ParityResult, type ParityRun, type ProposeApprovalRulesOptions, type ProposeCapsOptions, type RecordCallOptions, type RecordClientOptions, type RecordableDecisionModel, type RecordableLanguageModel, type RecordableModel, type RecordedCall, type RecordedEffect, type RecordedError, type RecordedModelCall, type RecordedOutput, type RecordedToolCall, type Recorder, type Recording, RecordingDivergenceError, type RepeatGroup, type RepeatReport, type RepeatResult, type RepeatRun, type RepeatRunsOptions, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type StateCoverage, type StateStats, type TraceCoverageOptions, type TraceStep, type TracedDecisionRequest, type TrajectoryEvent, type TrajectoryEventEnvelope, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, compareRecordings, createRecorder, explorePaths, lintAgentMachine, sameModelCallShape, simulateAgent, textOutput };
|