@statelyai/agent 2.0.0-alpha.39 → 2.0.0-alpha.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/dist/evals.cjs +1 -1
- package/dist/evals.d.cts +1 -1
- package/dist/evals.d.mts +1 -1
- package/dist/evals.mjs +1 -1
- package/dist/index.cjs +17 -10
- package/dist/index.d.cts +29 -12
- package/dist/index.d.mts +29 -12
- package/dist/index.mjs +17 -10
- package/dist/machine-input-0YTPcnCm.mjs +27 -0
- package/dist/machine-input-DU3owLUr.cjs +32 -0
- package/dist/otel.d.cts +1 -1
- package/dist/otel.d.mts +1 -1
- package/dist/policy.cjs +1 -1
- package/dist/policy.d.cts +1 -1
- package/dist/policy.d.mts +1 -1
- package/dist/policy.mjs +1 -1
- package/dist/{run-agent-Dq0oSFRm.d.cts → run-agent-BO9RSb7i.d.cts} +16 -7
- package/dist/{run-agent-DHlq87G7.d.mts → run-agent-BxkeuAYR.d.mts} +16 -7
- package/dist/{run-agent-De3xVFGw.cjs → run-agent-COdR9FpM.cjs} +26 -27
- package/dist/{run-agent-BE7DOWD6.mjs → run-agent-D3gW9wfd.mjs} +27 -28
- package/dist/testing.cjs +377 -18
- package/dist/testing.d.cts +181 -5
- package/dist/testing.d.mts +181 -5
- package/dist/testing.mjs +377 -20
- package/dist/{utils-DAclF-kw.d.cts → utils-BMxUcjay.d.cts} +57 -1
- package/dist/{utils-B8k-YPYI.d.mts → utils-BV7pxhD-.d.mts} +57 -1
- package/dist/validate.cjs +1 -1
- package/dist/validate.d.cts +1 -1
- package/dist/validate.d.mts +1 -1
- package/dist/validate.mjs +1 -1
- package/dist/{verify-H3Pr2gNw.d.mts → verify-C1UU84eq.d.mts} +22 -2
- package/dist/{verify-BM0sGmsf.d.cts → verify-DlN6tK8_.d.cts} +22 -2
- package/dist/{verify-Dek6gHCS.cjs → verify-IVYsI026.cjs} +84 -17
- package/dist/{verify-CWlg3j7B.mjs → verify-lKyu3buS.mjs} +84 -17
- package/docs/evals.md +1 -0
- package/docs/hosts.md +5 -3
- package/docs/index.md +2 -2
- package/docs/meta.json +1 -0
- package/docs/strangler-fig.md +36 -29
- package/docs/verify-in-place.md +326 -0
- package/docs/verify.md +70 -4
- package/llms.txt +1 -0
- package/package.json +1 -1
- package/readme.md +1 -0
package/dist/testing.d.mts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { c as AgentRequestExecutorInfo, f as AgentTextRequest, s as AgentRequestExecutor, t as AgentCallUsage, u as AgentRequestExecutors } from "./text-logic-x17uLBwA.mjs";
|
|
2
|
-
import { C as simulateAgent, S as lintAgentMachine, _ as SimulationScript, a as AgentPathTerminal, b as canReach, c as CanReachOrdering, d as ExplorePathsOptions, f as ExploreSnapshot, g as SimulateAgentResult, h as SimulateAgentOptions, i as AgentPathReport, l as CanReachResult, m as LintAgentMachineOptions, n as AgentLintError, o as AgentUnknownStateError, p as ExploreStep, r as AgentLintSeverity, s as AssertAgentMachineOptions, t as AgentLintDiagnostic, u as CanReachTarget, v as SimulationTrailEntry, w as textOutput, x as explorePaths, y as assertAgentMachine } from "./verify-
|
|
2
|
+
import { C as simulateAgent, S as lintAgentMachine, _ as SimulationScript, a as AgentPathTerminal, b as canReach, c as CanReachOrdering, d as ExplorePathsOptions, f as ExploreSnapshot, g as SimulateAgentResult, h as SimulateAgentOptions, i as AgentPathReport, l as CanReachResult, m as LintAgentMachineOptions, n as AgentLintError, o as AgentUnknownStateError, p as ExploreStep, r as AgentLintSeverity, s as AssertAgentMachineOptions, t as AgentLintDiagnostic, u as CanReachTarget, v as SimulationTrailEntry, w as textOutput, x as explorePaths, y as assertAgentMachine } from "./verify-C1UU84eq.mjs";
|
|
3
3
|
import { a as TrajectoryMatch, i as TrajectoryItem, n as TrajectoryEvent, o as TrajectoryMiss, r as TrajectoryEventEnvelope, s as matchesTrajectory, t as MatchTrajectoryOptions } from "./trajectory-BFjXaooB.mjs";
|
|
4
|
-
import { p as AgentRuntimeOptions, s as AgentRunInit, u as AgentRunResult } from "./run-agent-
|
|
4
|
+
import { _ as AgentTraceEvent, l as AgentRunOptions, p as AgentRuntimeOptions, s as AgentRunInit, u as AgentRunResult } from "./run-agent-BxkeuAYR.mjs";
|
|
5
5
|
import { n as AgentEventLogStore } from "./event-log-store-DYun526m.mjs";
|
|
6
|
-
import { A as
|
|
7
|
-
import { AnyActorLogic, AnyStateMachine, EventFromLogic, InputFrom, SnapshotFrom, StateValue } from "xstate";
|
|
6
|
+
import { A as RecordedOutput, C as RecordableDecisionModel, D as RecordedEffect, E as RecordedCall, F as compareRecordings, I as createRecorder, L as recordAiSdk, M as Recorder, N as Recording, O as RecordedError, P as RecordingDivergenceError, R as sameModelCallShape, S as RecordClientOptions, T as RecordableModel, _ as EquivalenceReport, b as NormalizeEffect, f as AiSdkSeam, g as EffectDivergenceReason, h as EffectDivergence, j as RecordedToolCall, k as RecordedModelCall, m as CreateRecorderOptions, p as CompareRecordingsOptions, t as MetaOfSnapshot, v as ModelAnswer, w as RecordableLanguageModel, x as RecordCallOptions, y as ModelCallMatch } from "./utils-BV7pxhD-.mjs";
|
|
7
|
+
import { AnyActorLogic, AnyStateMachine, EventFromLogic, EventObject, InputFrom, SnapshotFrom, StateValue } from "xstate";
|
|
8
8
|
//#region src/crash-recovery.d.ts
|
|
9
9
|
/** Options for {@link checkCrashRecovery}. */
|
|
10
10
|
interface CheckCrashRecoveryOptions<TMachine extends AnyStateMachine> {
|
|
@@ -428,6 +428,182 @@ export declare function compareRuns<TScenario extends {
|
|
|
428
428
|
expected?: string;
|
|
429
429
|
}>(options: CompareRunsOptions<TScenario>): Promise<ParityReport<TScenario>>;
|
|
430
430
|
//#endregion
|
|
431
|
+
//#region src/conformance.d.ts
|
|
432
|
+
/** One thing a recorded run did, as the model reads it. `kind` names what it is. */
|
|
433
|
+
interface ConformanceObservation {
|
|
434
|
+
kind: string;
|
|
435
|
+
}
|
|
436
|
+
/** What a {@link ConformanceActor} callback receives. */
|
|
437
|
+
interface ConformanceActorArgs<TObservation extends ConformanceObservation> {
|
|
438
|
+
/** The actor's input, as the model computed it. */
|
|
439
|
+
input: unknown;
|
|
440
|
+
/** The observation this actor consumed, or `undefined` for an actor with no `observes`. */
|
|
441
|
+
observation: TObservation | undefined;
|
|
442
|
+
/** Every observation of the run, in order. */
|
|
443
|
+
observations: readonly TObservation[];
|
|
444
|
+
/** The index of the next observation not yet consumed. */
|
|
445
|
+
index: number;
|
|
446
|
+
}
|
|
447
|
+
/** How one of the model's actors is answered from the recording. */
|
|
448
|
+
interface ConformanceActor<TObservation extends ConformanceObservation> {
|
|
449
|
+
/**
|
|
450
|
+
* The observation kind this actor consumes. When the model starts the actor,
|
|
451
|
+
* the next observation must be of this kind, or the run deviates. Omit it for
|
|
452
|
+
* work the recording cannot see: the actor then consumes nothing and is
|
|
453
|
+
* listed in {@link ConformanceReport.unobserved}.
|
|
454
|
+
*/
|
|
455
|
+
observes?: string;
|
|
456
|
+
/** Return `false` when the actor's input does not fit the observation: an `input-mismatch` deviation. */
|
|
457
|
+
matches?: (args: ConformanceActorArgs<TObservation>) => boolean;
|
|
458
|
+
/**
|
|
459
|
+
* The actor's output, which the model's `onDone` reads. Build it from the
|
|
460
|
+
* observation, and from the model's own `input` for state the recording does
|
|
461
|
+
* not show. Default: `undefined`.
|
|
462
|
+
*/
|
|
463
|
+
output?: (args: ConformanceActorArgs<TObservation>) => unknown;
|
|
464
|
+
/** Return a value to reject the actor with it, for work that failed in the recording. */
|
|
465
|
+
error?: (args: ConformanceActorArgs<TObservation>) => unknown;
|
|
466
|
+
}
|
|
467
|
+
/** What a {@link ConformanceEvent} `matches` receives. */
|
|
468
|
+
interface ConformanceEventArgs<TObservation extends ConformanceObservation> {
|
|
469
|
+
/** The observation the event comes from. */
|
|
470
|
+
observation: TObservation;
|
|
471
|
+
/** The model's context while it waited for the event. */
|
|
472
|
+
context: unknown;
|
|
473
|
+
/** The model's state value while it waited for the event. */
|
|
474
|
+
state: StateValue;
|
|
475
|
+
}
|
|
476
|
+
/**
|
|
477
|
+
* How an observation becomes an event the idle model receives. A function is
|
|
478
|
+
* the `event` alone. Add `matches` to compare the observation with what the
|
|
479
|
+
* model holds, such as the draft a person approved against the draft in context.
|
|
480
|
+
*/
|
|
481
|
+
type ConformanceEvent<TObservation extends ConformanceObservation> = ((observation: TObservation) => EventObject) | {
|
|
482
|
+
event: (observation: TObservation) => EventObject;
|
|
483
|
+
/** Return `false` when the observation does not fit the model's state: an `input-mismatch` deviation. */
|
|
484
|
+
matches?: (args: ConformanceEventArgs<TObservation>) => boolean;
|
|
485
|
+
};
|
|
486
|
+
/** How the code's recorded return value is checked against the model's output. */
|
|
487
|
+
interface ConformanceOutput<TObservation extends ConformanceObservation> {
|
|
488
|
+
/**
|
|
489
|
+
* The observation kind of the code's return value. When the model finishes,
|
|
490
|
+
* the next observation must be of this kind, or the run deviates.
|
|
491
|
+
*/
|
|
492
|
+
observes: string;
|
|
493
|
+
/** Return `false` when the model's output does not fit the recorded one: an `output-mismatch` deviation. */
|
|
494
|
+
matches?: (args: {
|
|
495
|
+
output: unknown;
|
|
496
|
+
observation: TObservation;
|
|
497
|
+
}) => boolean;
|
|
498
|
+
}
|
|
499
|
+
/** Options for {@link checkConformance}. */
|
|
500
|
+
interface CheckConformanceOptions<TMachine extends AnyStateMachine, TObservation extends ConformanceObservation> {
|
|
501
|
+
/** The machine input for this run, validated as `runAgent` validates it. */
|
|
502
|
+
input?: AgentRunOptions<TMachine>["input"];
|
|
503
|
+
/** The recorded run, in order. Map a recording with `recording.effects.flatMap(...)`. */
|
|
504
|
+
observations: readonly TObservation[];
|
|
505
|
+
/**
|
|
506
|
+
* One entry per actor source the model invokes, keyed by `src`. An invoked
|
|
507
|
+
* src with no entry is an `unmapped-actor` deviation.
|
|
508
|
+
*/
|
|
509
|
+
actors: Record<string, ConformanceActor<TObservation>>;
|
|
510
|
+
/**
|
|
511
|
+
* Observations the model receives as events, keyed by kind. When the model
|
|
512
|
+
* settles idle and the next observation has one of these kinds, it is
|
|
513
|
+
* consumed and sent, and the run resumes.
|
|
514
|
+
*/
|
|
515
|
+
events?: Record<string, ConformanceEvent<TObservation>>;
|
|
516
|
+
/**
|
|
517
|
+
* Checks the code's return value. Map the recording's `output` effect to an
|
|
518
|
+
* observation of kind `observes`: the model must finish there, and `matches`
|
|
519
|
+
* compares its output with that observation. Without it, a recorded return
|
|
520
|
+
* value is an `extra-observations` deviation like any other.
|
|
521
|
+
*/
|
|
522
|
+
output?: ConformanceOutput<TObservation>;
|
|
523
|
+
}
|
|
524
|
+
/** Why a run deviated from the model. */
|
|
525
|
+
type ConformanceDeviationReason =
|
|
526
|
+
/** The model started an actor, and the next observation is of another kind. */
|
|
527
|
+
"unexpected-observation" |
|
|
528
|
+
/** The model started an actor that observes something, or finished with `output` set, and the recording had ended. */
|
|
529
|
+
"missing-observation" |
|
|
530
|
+
/** An actor's or event's `matches` returned `false` for the observation. */
|
|
531
|
+
"input-mismatch" |
|
|
532
|
+
/** The model finished, and `output.matches` returned `false` for the recorded return value. */
|
|
533
|
+
"output-mismatch" |
|
|
534
|
+
/** The model finished or settled idle, and the recording continues. */
|
|
535
|
+
"extra-observations" |
|
|
536
|
+
/** The model was idle, and it does not handle the event an observation maps to in that state. */
|
|
537
|
+
"ignored-event" |
|
|
538
|
+
/** The model invoked a src with no entry in `actors`. */
|
|
539
|
+
"unmapped-actor";
|
|
540
|
+
/** The first point where the recorded run and the model part ways. */
|
|
541
|
+
interface ConformanceDeviation<TObservation extends ConformanceObservation> {
|
|
542
|
+
reason: ConformanceDeviationReason;
|
|
543
|
+
/** The index of the observation at the deviation; `observations.length` when the recording had ended. */
|
|
544
|
+
index: number;
|
|
545
|
+
/** What the model did next: the actor src it started, the event, `"end of run"`, or `"idle, waiting for an event"`. */
|
|
546
|
+
expected: string;
|
|
547
|
+
/** What the recording shows next, or `undefined` when it had ended. */
|
|
548
|
+
observed: TObservation | undefined;
|
|
549
|
+
/** The model's state value at the deviation. */
|
|
550
|
+
state: StateValue;
|
|
551
|
+
/** The actor's input, for an actor deviation. */
|
|
552
|
+
actorInput?: unknown;
|
|
553
|
+
}
|
|
554
|
+
/** The result of {@link checkConformance}. */
|
|
555
|
+
interface ConformanceReport<TObservation extends ConformanceObservation> {
|
|
556
|
+
/** True when the model produced the recorded run exactly, observation for observation. */
|
|
557
|
+
conforms: boolean;
|
|
558
|
+
/** The first deviation, when the run does not conform. */
|
|
559
|
+
deviation?: ConformanceDeviation<TObservation>;
|
|
560
|
+
/** How many observations the model consumed. */
|
|
561
|
+
consumed: number;
|
|
562
|
+
/**
|
|
563
|
+
* Actor srcs the model ran without an observation (no `observes`): what they
|
|
564
|
+
* did was not checked against the recording.
|
|
565
|
+
*/
|
|
566
|
+
unobserved: string[];
|
|
567
|
+
/** How the model's run settled. A deviation stops the run, which then settles `error`. */
|
|
568
|
+
status: "done" | "idle" | "error";
|
|
569
|
+
/** The model's trace events. Pass the traces of several runs to `traceCoverage`. */
|
|
570
|
+
trace: AgentTraceEvent[];
|
|
571
|
+
}
|
|
572
|
+
/**
|
|
573
|
+
* Checks a recorded run of existing code against a model of that code.
|
|
574
|
+
*
|
|
575
|
+
* The model runs with `runAgent`, and every actor in `actors` is answered from
|
|
576
|
+
* `observations` instead of running: an actor with `observes` consumes the next
|
|
577
|
+
* observation, which must be of that kind. The report names the first
|
|
578
|
+
* deviation: where the model started something the recording does not show
|
|
579
|
+
* next, or ended while the recording continued.
|
|
580
|
+
*
|
|
581
|
+
* Use it to verify existing code in place: prove the model's rules with
|
|
582
|
+
* `canReach`, then check that recorded runs of the code conform to the model.
|
|
583
|
+
* A conforming run shows the code behaved as the model allows on that run; it
|
|
584
|
+
* does not prove the code can never break a rule.
|
|
585
|
+
*
|
|
586
|
+
* Every actor the model invokes must be declared by name: a model whose invoke
|
|
587
|
+
* names logic inline throws `inline-actor-source`, because that logic would
|
|
588
|
+
* run. A callback in `actors` that throws ends the check with
|
|
589
|
+
* `conformance-callback-error` instead of counting as an actor failure.
|
|
590
|
+
*
|
|
591
|
+
* @example
|
|
592
|
+
* ```ts
|
|
593
|
+
* const report = await checkConformance(loopModel, {
|
|
594
|
+
* input: { maxIter: 3 },
|
|
595
|
+
* observations: recording.effects.flatMap(toObservation),
|
|
596
|
+
* actors: {
|
|
597
|
+
* route: { observes: "route", output: ({ observation }) => observation.names },
|
|
598
|
+
* runTurn: { observes: "turn", matches: ({ input, observation }) => input.name === observation.agent },
|
|
599
|
+
* save: {}, // not visible in this recording
|
|
600
|
+
* },
|
|
601
|
+
* });
|
|
602
|
+
* report.conforms; // false when the run did something the model does not allow
|
|
603
|
+
* ```
|
|
604
|
+
*/
|
|
605
|
+
export declare function checkConformance<TMachine extends AnyStateMachine, TObservation extends ConformanceObservation>(machine: TMachine, options: CheckConformanceOptions<TMachine, TObservation>): Promise<ConformanceReport<TObservation>>;
|
|
606
|
+
//#endregion
|
|
431
607
|
//#region src/check-parity.d.ts
|
|
432
608
|
/** One scenario: an id and the workflow's input. Extra fields are yours. */
|
|
433
609
|
interface CheckParityScenario<TInput = unknown> {
|
|
@@ -539,4 +715,4 @@ interface CheckParityReport<TScenario extends CheckParityScenario, TOutput> {
|
|
|
539
715
|
*/
|
|
540
716
|
export declare function checkParity<TScenario extends CheckParityScenario, TEffects extends object, TOutput, TLive = undefined>(options: CheckParityOptions<TScenario, TEffects, TOutput, TLive>): Promise<CheckParityReport<TScenario, TOutput>>;
|
|
541
717
|
//#endregion
|
|
542
|
-
export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AssertAgentMachineOptions, type CanReachOrdering, type CanReachResult, type CanReachTarget, type CheckCrashRecoveryOptions, type CheckParityBaseOptions, type CheckParityOptions, type CheckParityOutcome, type CheckParityReport, type CheckParityRow, type CheckParityScenario, type CheckParitySetup, type CompareRecordingsOptions, type CompareRunsOptions, type CrashEffect, type CrashPoint, type CrashRecoveryFailure, type CrashRecoveryProblem, type CrashRecoveryReport, type CreateRecorderOptions, type EffectDivergence, type EffectDivergenceReason, type EquivalenceReport, type ExplorePathsOptions, type ExploreSnapshot, type ExploreStep, type LintAgentMachineOptions, type MatchTrajectoryOptions, type ModelAnswer, type ModelCallMatch, type NormalizeEffect, type ParityLabel, type ParityReport, type ParityResult, type ParityRun, type RecordCallOptions, type RecordClientOptions, type RecordableDecisionModel, type RecordableLanguageModel, type RecordableModel, type RecordedCall, type RecordedEffect, type RecordedError, type RecordedModelCall, type RecordedOutput, type RecordedToolCall, type Recorder, type Recording, RecordingDivergenceError, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type TrajectoryEvent, type TrajectoryEventEnvelope, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, compareRecordings, createRecorder, explorePaths, lintAgentMachine, matchesTrajectory, sameModelCallShape, simulateAgent, textOutput };
|
|
718
|
+
export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AiSdkSeam, type AssertAgentMachineOptions, type CanReachOrdering, type CanReachResult, type CanReachTarget, type CheckConformanceOptions, type CheckCrashRecoveryOptions, type CheckParityBaseOptions, type CheckParityOptions, type CheckParityOutcome, type CheckParityReport, type CheckParityRow, type CheckParityScenario, type CheckParitySetup, type CompareRecordingsOptions, type CompareRunsOptions, type ConformanceActor, type ConformanceActorArgs, type ConformanceDeviation, type ConformanceDeviationReason, type ConformanceEvent, type ConformanceEventArgs, type ConformanceObservation, type ConformanceOutput, type ConformanceReport, type CrashEffect, type CrashPoint, type CrashRecoveryFailure, type CrashRecoveryProblem, type CrashRecoveryReport, type CreateRecorderOptions, type EffectDivergence, type EffectDivergenceReason, type EquivalenceReport, type ExplorePathsOptions, type ExploreSnapshot, type ExploreStep, type LintAgentMachineOptions, type MatchTrajectoryOptions, type ModelAnswer, type ModelCallMatch, type NormalizeEffect, type ParityLabel, type ParityReport, type ParityResult, type ParityRun, type RecordCallOptions, type RecordClientOptions, type RecordableDecisionModel, type RecordableLanguageModel, type RecordableModel, type RecordedCall, type RecordedEffect, type RecordedError, type RecordedModelCall, type RecordedOutput, type RecordedToolCall, type Recorder, type Recording, RecordingDivergenceError, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type TrajectoryEvent, type TrajectoryEventEnvelope, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, compareRecordings, createRecorder, explorePaths, lintAgentMachine, matchesTrajectory, recordAiSdk, sameModelCallShape, simulateAgent, textOutput };
|
package/dist/testing.mjs
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
import { I as getCallUsage, L as isTextLogic, R as normalizeGeneratorResult, c as agentCallSite, d as canonicalEventJson, ft as isRecord, pt as AgentError, tt as getStateMeta, x as replay } from "./event-log-CEq_bVXQ.mjs";
|
|
1
|
+
import { I as getCallUsage, K as getRegisteredAgentExecutionOptions, L as isTextLogic, R as normalizeGeneratorResult, c as agentCallSite, d as canonicalEventJson, ft as isRecord, pt as AgentError, tt as getStateMeta, x as replay } from "./event-log-CEq_bVXQ.mjs";
|
|
2
2
|
import { s as isDecisionLogic } from "./decision-B41YBx5U.mjs";
|
|
3
3
|
import { i as isActLogic } from "./act-Bsz-hUfD.mjs";
|
|
4
4
|
import { n as createInMemoryEventLogStore } from "./event-log-store-DLAfNM34.mjs";
|
|
5
|
-
import { f as isStateMachineLogic, g as toJsonValue, l as createAgentRuntime, m as runToQuiescence } from "./run-agent-
|
|
5
|
+
import { f as isStateMachineLogic, g as toJsonValue, l as createAgentRuntime, m as runToQuiescence, p as runAgent } from "./run-agent-D3gW9wfd.mjs";
|
|
6
6
|
import { n as mapEffects, t as isAsyncIterable } from "./effects-B9IvUPVh.mjs";
|
|
7
|
-
import { a as explorePaths, c as simulateAgent, i as canReach, l as textOutput, n as AgentUnknownStateError, r as assertAgentMachine, s as lintAgentMachine, t as AgentLintError } from "./verify-
|
|
7
|
+
import { a as explorePaths, c as simulateAgent, i as canReach, l as textOutput, n as AgentUnknownStateError, r as assertAgentMachine, s as lintAgentMachine, t as AgentLintError } from "./verify-lKyu3buS.mjs";
|
|
8
8
|
import { t as matchesTrajectory } from "./trajectory-BUXj_1Ue.mjs";
|
|
9
|
+
import { createAsyncLogic } from "xstate";
|
|
9
10
|
//#region src/equivalence.ts
|
|
10
11
|
/**
|
|
11
12
|
* Verified equivalence — prove a port behaves like the code it replaces.
|
|
@@ -38,6 +39,86 @@ import { t as matchesTrajectory } from "./trajectory-BUXj_1Ue.mjs";
|
|
|
38
39
|
* neither `ai` nor `openai`.
|
|
39
40
|
* @module
|
|
40
41
|
*/
|
|
42
|
+
const MODEL_METADATA = [
|
|
43
|
+
"specificationVersion",
|
|
44
|
+
"provider",
|
|
45
|
+
"modelId",
|
|
46
|
+
"supportedUrls"
|
|
47
|
+
];
|
|
48
|
+
const MODEL_METHODS = [
|
|
49
|
+
"doGenerate",
|
|
50
|
+
"doStream",
|
|
51
|
+
"doDecide",
|
|
52
|
+
"doEvaluate"
|
|
53
|
+
];
|
|
54
|
+
const MODEL_KEYS = [...MODEL_METADATA, ...MODEL_METHODS];
|
|
55
|
+
const aiSdkInternals = /* @__PURE__ */ new WeakMap();
|
|
56
|
+
/**
|
|
57
|
+
* Records AI SDK code that builds its model inline or passes module-level
|
|
58
|
+
* tools to `generateText`: there is nothing to inject, so the test mocks `"ai"`
|
|
59
|
+
* and routes each call through the recorder. Every `generateText` and
|
|
60
|
+
* `streamText` call runs on the real AI SDK with `seam.model` in place of the
|
|
61
|
+
* code's model (also in place of a model `prepareStep` returns) and its
|
|
62
|
+
* `tools` wrapped with `recorder.tools`. Everything else in `"ai"` stays real,
|
|
63
|
+
* so the code's own tool loop runs.
|
|
64
|
+
*
|
|
65
|
+
* Each model attempt made during the call carries the call's index as
|
|
66
|
+
* {@link RecordedModelCall.aiSdkCall} when its model is a `recorder.model`
|
|
67
|
+
* wrapper passed as is and the recorder came from `createRecorder`. Other
|
|
68
|
+
* attempts carry no index, and who resent them is unknown.
|
|
69
|
+
*
|
|
70
|
+
* `current` returns the seam of the run being recorded. A call while it returns
|
|
71
|
+
* `undefined` throws, so no test reaches a real provider.
|
|
72
|
+
*
|
|
73
|
+
* @example
|
|
74
|
+
* ```ts
|
|
75
|
+
* vi.mock("ai", async (importOriginal) => {
|
|
76
|
+
* const ai = await importOriginal<typeof import("ai")>();
|
|
77
|
+
* const { currentSeam } = await import("./seam"); // { recorder, model: recorder.model(standIn) }
|
|
78
|
+
* return { ...ai, ...recordAiSdk(ai, currentSeam) };
|
|
79
|
+
* });
|
|
80
|
+
* ```
|
|
81
|
+
*/
|
|
82
|
+
function recordAiSdk(ai, current) {
|
|
83
|
+
const wrap = (name, call) => (options) => {
|
|
84
|
+
const seam = current();
|
|
85
|
+
if (!seam) throw new Error(`recordAiSdk: ${name} was called while no run is recording. Return { recorder } from current() while the code runs.`);
|
|
86
|
+
const internals = aiSdkInternals.get(seam.recorder);
|
|
87
|
+
const index = internals ? internals.calls++ : 0;
|
|
88
|
+
const tag = (model) => {
|
|
89
|
+
const base = internals?.unwrap(model);
|
|
90
|
+
return base ? internals.wrap(base, index) : model;
|
|
91
|
+
};
|
|
92
|
+
const model = seam.model ?? options.model;
|
|
93
|
+
const prepareStep = options.prepareStep;
|
|
94
|
+
return call({
|
|
95
|
+
...options,
|
|
96
|
+
...model !== void 0 ? { model: tag(model) } : {},
|
|
97
|
+
...prepareStep ? { prepareStep: async (...args) => {
|
|
98
|
+
const step = await prepareStep(...args);
|
|
99
|
+
if (step?.model === void 0) return step;
|
|
100
|
+
const model = tag(seam.model ?? step.model);
|
|
101
|
+
const own = (key) => key === "model" ? model : Reflect.get(step, key, step);
|
|
102
|
+
return new Proxy({}, {
|
|
103
|
+
get: (_target, key) => own(key),
|
|
104
|
+
has: (_target, key) => key === "model" || key in step,
|
|
105
|
+
ownKeys: () => [.../* @__PURE__ */ new Set([...Reflect.ownKeys(step), "model"])],
|
|
106
|
+
getOwnPropertyDescriptor: (_target, key) => key === "model" || Object.prototype.hasOwnProperty.call(step, key) ? {
|
|
107
|
+
value: own(key),
|
|
108
|
+
enumerable: true,
|
|
109
|
+
configurable: true,
|
|
110
|
+
writable: true
|
|
111
|
+
} : void 0
|
|
112
|
+
});
|
|
113
|
+
} } : {},
|
|
114
|
+
...options.tools ? { tools: seam.recorder.tools(options.tools) } : {}
|
|
115
|
+
});
|
|
116
|
+
};
|
|
117
|
+
return {
|
|
118
|
+
generateText: wrap("generateText", ai.generateText),
|
|
119
|
+
...ai.streamText ? { streamText: wrap("streamText", ai.streamText) } : {}
|
|
120
|
+
};
|
|
121
|
+
}
|
|
41
122
|
/**
|
|
42
123
|
* Thrown in replay when the port asks for something the recording cannot
|
|
43
124
|
* answer: a model call past the last recorded one, or a tool call the original
|
|
@@ -227,7 +308,28 @@ function createRecorder(options = {}) {
|
|
|
227
308
|
* Records a model call around `run`, or answers it from the recording.
|
|
228
309
|
* `fallback` runs it on the stand-in when the recording has no answer.
|
|
229
310
|
*/
|
|
230
|
-
|
|
311
|
+
/**
|
|
312
|
+
* A retry resends the request of a model call that failed. Within one AI SDK
|
|
313
|
+
* call, the failed attempt is its own last attempt, even when a concurrent
|
|
314
|
+
* call came in between or claimed it first. Otherwise it is the last model
|
|
315
|
+
* call, which the code resent.
|
|
316
|
+
*/
|
|
317
|
+
function linkRetry(effect) {
|
|
318
|
+
const models = effects.filter((earlier) => earlier.kind === "model").reverse();
|
|
319
|
+
const failedWith = (earlier) => earlier?.error !== void 0 && earlier.method === effect.method && stableStringify(earlier.request) === stableStringify(effect.request);
|
|
320
|
+
const { aiSdkCall } = effect;
|
|
321
|
+
if (aiSdkCall !== void 0) {
|
|
322
|
+
const own = models.find((earlier) => earlier.aiSdkCall === aiSdkCall);
|
|
323
|
+
const claimedBy = own?.retriedBy === void 0 ? void 0 : models.find((earlier) => earlier.call === own.retriedBy);
|
|
324
|
+
if (own && failedWith(own) && claimedBy?.aiSdkCall !== aiSdkCall) {
|
|
325
|
+
own.retriedBy = effect.call;
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
const last = models[0];
|
|
330
|
+
if (last && failedWith(last) && last.retriedBy === void 0) last.retriedBy = effect.call;
|
|
331
|
+
}
|
|
332
|
+
async function modelCall(seam, method, request, streamed, run, fallback, aiSdkCall) {
|
|
231
333
|
const effect = {
|
|
232
334
|
kind: "model",
|
|
233
335
|
seam,
|
|
@@ -239,6 +341,8 @@ function createRecorder(options = {}) {
|
|
|
239
341
|
"signal"
|
|
240
342
|
]))
|
|
241
343
|
};
|
|
344
|
+
if (aiSdkCall !== void 0) effect.aiSdkCall = aiSdkCall;
|
|
345
|
+
linkRetry(effect);
|
|
242
346
|
effects.push(effect);
|
|
243
347
|
if (replay) {
|
|
244
348
|
const replayed = nextReplayedModel(effect);
|
|
@@ -468,22 +572,47 @@ function createRecorder(options = {}) {
|
|
|
468
572
|
}
|
|
469
573
|
return result;
|
|
470
574
|
}
|
|
471
|
-
|
|
575
|
+
const wrappedModels = /* @__PURE__ */ new WeakMap();
|
|
576
|
+
function wrapModel(model, aiSdkCall) {
|
|
577
|
+
const base = model;
|
|
578
|
+
const stand = options.fallback?.model;
|
|
579
|
+
const wrapped = Object.create(base);
|
|
580
|
+
for (const key of MODEL_METADATA) {
|
|
581
|
+
if (!(key in base)) continue;
|
|
582
|
+
Object.defineProperty(wrapped, key, {
|
|
583
|
+
get: () => Reflect.get(base, key, base),
|
|
584
|
+
enumerable: true,
|
|
585
|
+
configurable: true
|
|
586
|
+
});
|
|
587
|
+
}
|
|
588
|
+
for (const method of MODEL_METHODS) {
|
|
589
|
+
if (typeof base[method] !== "function") continue;
|
|
590
|
+
const invoke = (target, callOptions) => target[method].call(target, callOptions);
|
|
591
|
+
Object.defineProperty(wrapped, method, {
|
|
592
|
+
value: (callOptions) => modelCall("ai-sdk", method, callOptions, method === "doStream", () => invoke(base, callOptions), stand && typeof stand[method] === "function" ? () => invoke(stand, callOptions) : void 0, aiSdkCall),
|
|
593
|
+
writable: true,
|
|
594
|
+
enumerable: true,
|
|
595
|
+
configurable: true
|
|
596
|
+
});
|
|
597
|
+
}
|
|
598
|
+
wrappedModels.set(wrapped, {
|
|
599
|
+
base,
|
|
600
|
+
made: new Map([...MODEL_KEYS].map((key) => [key, Object.getOwnPropertyDescriptor(wrapped, key)]))
|
|
601
|
+
});
|
|
602
|
+
return wrapped;
|
|
603
|
+
}
|
|
604
|
+
function unwrapModel(model) {
|
|
605
|
+
const entry = wrappedModels.get(model);
|
|
606
|
+
if (!entry) return void 0;
|
|
607
|
+
for (const [key, made] of entry.made) {
|
|
608
|
+
const now = Object.getOwnPropertyDescriptor(model, key);
|
|
609
|
+
if (now?.value !== made?.value || now?.get !== made?.get) return void 0;
|
|
610
|
+
}
|
|
611
|
+
return entry.base;
|
|
612
|
+
}
|
|
613
|
+
const recorder = {
|
|
472
614
|
model(model) {
|
|
473
|
-
|
|
474
|
-
const stand = options.fallback?.model;
|
|
475
|
-
const wrapped = Object.create(base);
|
|
476
|
-
for (const method of [
|
|
477
|
-
"doGenerate",
|
|
478
|
-
"doStream",
|
|
479
|
-
"doDecide",
|
|
480
|
-
"doEvaluate"
|
|
481
|
-
]) {
|
|
482
|
-
if (typeof base[method] !== "function") continue;
|
|
483
|
-
const invoke = (target, callOptions) => target[method].call(target, callOptions);
|
|
484
|
-
wrapped[method] = (callOptions) => modelCall("ai-sdk", method, callOptions, method === "doStream", () => invoke(base, callOptions), stand && typeof stand[method] === "function" ? () => invoke(stand, callOptions) : void 0);
|
|
485
|
-
}
|
|
486
|
-
return wrapped;
|
|
615
|
+
return wrapModel(model);
|
|
487
616
|
},
|
|
488
617
|
openai(client) {
|
|
489
618
|
const wrapCreate = (resource, method) => new Proxy(resource, { get(target, key, receiver) {
|
|
@@ -567,6 +696,12 @@ function createRecorder(options = {}) {
|
|
|
567
696
|
};
|
|
568
697
|
}
|
|
569
698
|
};
|
|
699
|
+
aiSdkInternals.set(recorder, {
|
|
700
|
+
unwrap: (model) => model !== null && (typeof model === "object" || typeof model === "function") ? unwrapModel(model) : void 0,
|
|
701
|
+
wrap: wrapModel,
|
|
702
|
+
calls: 0
|
|
703
|
+
});
|
|
704
|
+
return recorder;
|
|
570
705
|
}
|
|
571
706
|
/** True when two model calls offer the same tool names and response format. */
|
|
572
707
|
function sameModelCallShape(expected, actual) {
|
|
@@ -1694,6 +1829,228 @@ function structurallyEqual(a, b) {
|
|
|
1694
1829
|
return keys.length === Object.keys(b).length && keys.every((key) => Object.hasOwn(b, key) && structurallyEqual(a[key], b[key]));
|
|
1695
1830
|
}
|
|
1696
1831
|
//#endregion
|
|
1832
|
+
//#region src/conformance.ts
|
|
1833
|
+
/**
|
|
1834
|
+
* Conformance: check a recorded run of existing code against a model of it.
|
|
1835
|
+
*
|
|
1836
|
+
* The model is an ordinary machine whose actors stand for the code's work.
|
|
1837
|
+
* `checkConformance` runs the model with every actor answered from the
|
|
1838
|
+
* recording: each one consumes the next observation of its kind. The first
|
|
1839
|
+
* time the model does something the recording does not show next, the run
|
|
1840
|
+
* stops and the step is reported. The code itself is not changed or run.
|
|
1841
|
+
*/
|
|
1842
|
+
const NEVER = new Promise(() => {});
|
|
1843
|
+
function inlineInvokeStates(machine) {
|
|
1844
|
+
const found = [];
|
|
1845
|
+
const stack = [machine.root];
|
|
1846
|
+
while (stack.length > 0) {
|
|
1847
|
+
const node = stack.pop();
|
|
1848
|
+
const invoke = node.config.invoke;
|
|
1849
|
+
if ((invoke === void 0 ? [] : Array.isArray(invoke) ? invoke : [invoke]).some((entry) => typeof entry.src !== "string")) found.push(node.path.join(".") || "(root)");
|
|
1850
|
+
stack.push(...Object.values(node.states));
|
|
1851
|
+
}
|
|
1852
|
+
return found.sort();
|
|
1853
|
+
}
|
|
1854
|
+
function actorSources(machine) {
|
|
1855
|
+
const actors = getRegisteredAgentExecutionOptions(machine).actors ?? machine.sources?.actors ?? {};
|
|
1856
|
+
return Object.keys(actors);
|
|
1857
|
+
}
|
|
1858
|
+
/**
|
|
1859
|
+
* Checks a recorded run of existing code against a model of that code.
|
|
1860
|
+
*
|
|
1861
|
+
* The model runs with `runAgent`, and every actor in `actors` is answered from
|
|
1862
|
+
* `observations` instead of running: an actor with `observes` consumes the next
|
|
1863
|
+
* observation, which must be of that kind. The report names the first
|
|
1864
|
+
* deviation: where the model started something the recording does not show
|
|
1865
|
+
* next, or ended while the recording continued.
|
|
1866
|
+
*
|
|
1867
|
+
* Use it to verify existing code in place: prove the model's rules with
|
|
1868
|
+
* `canReach`, then check that recorded runs of the code conform to the model.
|
|
1869
|
+
* A conforming run shows the code behaved as the model allows on that run; it
|
|
1870
|
+
* does not prove the code can never break a rule.
|
|
1871
|
+
*
|
|
1872
|
+
* Every actor the model invokes must be declared by name: a model whose invoke
|
|
1873
|
+
* names logic inline throws `inline-actor-source`, because that logic would
|
|
1874
|
+
* run. A callback in `actors` that throws ends the check with
|
|
1875
|
+
* `conformance-callback-error` instead of counting as an actor failure.
|
|
1876
|
+
*
|
|
1877
|
+
* @example
|
|
1878
|
+
* ```ts
|
|
1879
|
+
* const report = await checkConformance(loopModel, {
|
|
1880
|
+
* input: { maxIter: 3 },
|
|
1881
|
+
* observations: recording.effects.flatMap(toObservation),
|
|
1882
|
+
* actors: {
|
|
1883
|
+
* route: { observes: "route", output: ({ observation }) => observation.names },
|
|
1884
|
+
* runTurn: { observes: "turn", matches: ({ input, observation }) => input.name === observation.agent },
|
|
1885
|
+
* save: {}, // not visible in this recording
|
|
1886
|
+
* },
|
|
1887
|
+
* });
|
|
1888
|
+
* report.conforms; // false when the run did something the model does not allow
|
|
1889
|
+
* ```
|
|
1890
|
+
*/
|
|
1891
|
+
async function checkConformance(machine, options) {
|
|
1892
|
+
const { observations, actors: specs, events = {} } = options;
|
|
1893
|
+
const inline = inlineInvokeStates(machine);
|
|
1894
|
+
if (inline.length > 0) throw new AgentError("inline-actor-source", `checkConformance: ${inline.join(", ")} invoke${inline.length === 1 ? "s" : ""} inline actor logic, which would run for real. Declare each actor by name in setup ({ actors }) and invoke it by src, so the recording can answer it.`);
|
|
1895
|
+
for (const src of Object.keys(specs)) if (!actorSources(machine).includes(src)) throw new AgentError("unknown-actor-source", `checkConformance: actors.${src} names no actor source the machine declares (${actorSources(machine).join(", ") || "none"}).`);
|
|
1896
|
+
let index = 0;
|
|
1897
|
+
let deviation;
|
|
1898
|
+
let callbackError;
|
|
1899
|
+
let state = machine.config.initial;
|
|
1900
|
+
const unobserved = /* @__PURE__ */ new Set();
|
|
1901
|
+
const trace = [];
|
|
1902
|
+
const controller = new AbortController();
|
|
1903
|
+
const callbackFailure = (name, cause) => new AgentError("conformance-callback-error", `checkConformance: ${name} threw: ${cause instanceof Error ? cause.message : String(cause)}`, { cause });
|
|
1904
|
+
const callback = (name, fn) => {
|
|
1905
|
+
try {
|
|
1906
|
+
return fn();
|
|
1907
|
+
} catch (cause) {
|
|
1908
|
+
throw callbackFailure(name, cause);
|
|
1909
|
+
}
|
|
1910
|
+
};
|
|
1911
|
+
const deviate = (reason, expected, actorInput) => {
|
|
1912
|
+
deviation ??= {
|
|
1913
|
+
reason,
|
|
1914
|
+
index,
|
|
1915
|
+
expected,
|
|
1916
|
+
observed: observations[index],
|
|
1917
|
+
state,
|
|
1918
|
+
...actorInput !== void 0 ? { actorInput } : {}
|
|
1919
|
+
};
|
|
1920
|
+
controller.abort();
|
|
1921
|
+
return NEVER;
|
|
1922
|
+
};
|
|
1923
|
+
const answer = (src, spec) => createAsyncLogic({ run: async ({ input }) => {
|
|
1924
|
+
if (deviation || callbackError) return NEVER;
|
|
1925
|
+
if (!spec) return deviate("unmapped-actor", src, input);
|
|
1926
|
+
let observation;
|
|
1927
|
+
if (spec.observes === void 0) unobserved.add(src);
|
|
1928
|
+
else {
|
|
1929
|
+
const next = observations[index];
|
|
1930
|
+
if (next === void 0) return deviate("missing-observation", src, input);
|
|
1931
|
+
if (next.kind !== spec.observes) return deviate("unexpected-observation", src, input);
|
|
1932
|
+
observation = next;
|
|
1933
|
+
index += 1;
|
|
1934
|
+
}
|
|
1935
|
+
const args = {
|
|
1936
|
+
input,
|
|
1937
|
+
observation,
|
|
1938
|
+
observations,
|
|
1939
|
+
index
|
|
1940
|
+
};
|
|
1941
|
+
const call = (name, fn) => {
|
|
1942
|
+
try {
|
|
1943
|
+
return fn();
|
|
1944
|
+
} catch (cause) {
|
|
1945
|
+
callbackError ??= callbackFailure(`actors.${src}.${name}`, cause);
|
|
1946
|
+
controller.abort();
|
|
1947
|
+
return NEVER;
|
|
1948
|
+
}
|
|
1949
|
+
};
|
|
1950
|
+
if (spec.matches) {
|
|
1951
|
+
const fits = call("matches", () => spec.matches(args));
|
|
1952
|
+
if (fits === NEVER) return NEVER;
|
|
1953
|
+
if (!fits) {
|
|
1954
|
+
index -= observation ? 1 : 0;
|
|
1955
|
+
return deviate("input-mismatch", src, input);
|
|
1956
|
+
}
|
|
1957
|
+
}
|
|
1958
|
+
const error = call("error", () => spec.error?.(args));
|
|
1959
|
+
if (error === NEVER) return NEVER;
|
|
1960
|
+
if (error !== void 0) throw error;
|
|
1961
|
+
return call("output", () => spec.output?.(args));
|
|
1962
|
+
} });
|
|
1963
|
+
const shared = {
|
|
1964
|
+
actors: Object.fromEntries(actorSources(machine).map((src) => [src, answer(src, specs[src])])),
|
|
1965
|
+
signal: controller.signal,
|
|
1966
|
+
onTrace: (event) => trace.push(event),
|
|
1967
|
+
onTransition: (snapshot) => {
|
|
1968
|
+
state = snapshot.value;
|
|
1969
|
+
}
|
|
1970
|
+
};
|
|
1971
|
+
let result = await runAgent(machine, {
|
|
1972
|
+
...shared,
|
|
1973
|
+
...options.input !== void 0 ? { input: options.input } : {}
|
|
1974
|
+
});
|
|
1975
|
+
while (!deviation && !callbackError && result.status === "idle") {
|
|
1976
|
+
const next = observations[index];
|
|
1977
|
+
const entry = next === void 0 ? void 0 : events[next.kind];
|
|
1978
|
+
if (!entry || next === void 0) break;
|
|
1979
|
+
const { event: toEvent, matches } = typeof entry === "function" ? { event: entry } : entry;
|
|
1980
|
+
if (!callback(`events.${next.kind}.matches`, () => matches ? matches({
|
|
1981
|
+
observation: next,
|
|
1982
|
+
context: result.snapshot.context,
|
|
1983
|
+
state
|
|
1984
|
+
}) : true)) {
|
|
1985
|
+
deviation = {
|
|
1986
|
+
reason: "input-mismatch",
|
|
1987
|
+
index,
|
|
1988
|
+
expected: `event ${next.kind}`,
|
|
1989
|
+
observed: next,
|
|
1990
|
+
state
|
|
1991
|
+
};
|
|
1992
|
+
break;
|
|
1993
|
+
}
|
|
1994
|
+
const event = callback(`events.${next.kind}`, () => toEvent(next));
|
|
1995
|
+
index += 1;
|
|
1996
|
+
result = await runAgent(machine, {
|
|
1997
|
+
...shared,
|
|
1998
|
+
snapshot: result.persist(),
|
|
1999
|
+
event
|
|
2000
|
+
});
|
|
2001
|
+
if (result.ignored !== void 0) {
|
|
2002
|
+
index -= 1;
|
|
2003
|
+
deviation = {
|
|
2004
|
+
reason: "ignored-event",
|
|
2005
|
+
index,
|
|
2006
|
+
expected: `an event ${JSON.stringify(state)} handles`,
|
|
2007
|
+
observed: next,
|
|
2008
|
+
state
|
|
2009
|
+
};
|
|
2010
|
+
break;
|
|
2011
|
+
}
|
|
2012
|
+
}
|
|
2013
|
+
if (callbackError) throw callbackError;
|
|
2014
|
+
const outputSpec = options.output;
|
|
2015
|
+
const last = observations[index];
|
|
2016
|
+
if (!deviation && outputSpec && result.status === "done" && last === void 0) deviation = {
|
|
2017
|
+
reason: "missing-observation",
|
|
2018
|
+
index,
|
|
2019
|
+
expected: `output (${outputSpec.observes})`,
|
|
2020
|
+
observed: void 0,
|
|
2021
|
+
state
|
|
2022
|
+
};
|
|
2023
|
+
else if (!deviation && outputSpec && result.status === "done" && last?.kind === outputSpec.observes) {
|
|
2024
|
+
const output = result.output;
|
|
2025
|
+
if (callback("output.matches", () => outputSpec.matches ? outputSpec.matches({
|
|
2026
|
+
output,
|
|
2027
|
+
observation: last
|
|
2028
|
+
}) : true)) index += 1;
|
|
2029
|
+
else deviation = {
|
|
2030
|
+
reason: "output-mismatch",
|
|
2031
|
+
index,
|
|
2032
|
+
expected: `output ${JSON.stringify(output)}`,
|
|
2033
|
+
observed: last,
|
|
2034
|
+
state
|
|
2035
|
+
};
|
|
2036
|
+
}
|
|
2037
|
+
if (!deviation && index < observations.length) deviation = {
|
|
2038
|
+
reason: "extra-observations",
|
|
2039
|
+
index,
|
|
2040
|
+
expected: result.status === "idle" ? "idle, waiting for an event" : "end of run",
|
|
2041
|
+
observed: observations[index],
|
|
2042
|
+
state
|
|
2043
|
+
};
|
|
2044
|
+
return {
|
|
2045
|
+
conforms: deviation === void 0,
|
|
2046
|
+
...deviation ? { deviation } : {},
|
|
2047
|
+
consumed: index,
|
|
2048
|
+
unobserved: [...unobserved],
|
|
2049
|
+
status: result.status,
|
|
2050
|
+
trace
|
|
2051
|
+
};
|
|
2052
|
+
}
|
|
2053
|
+
//#endregion
|
|
1697
2054
|
//#region src/check-parity.ts
|
|
1698
2055
|
/**
|
|
1699
2056
|
* `checkParity` — one call that records a workflow, replays its port, and
|
|
@@ -1803,4 +2160,4 @@ async function checkParity(options) {
|
|
|
1803
2160
|
};
|
|
1804
2161
|
}
|
|
1805
2162
|
//#endregion
|
|
1806
|
-
export { AgentLintError, AgentUnknownStateError, RecordingDivergenceError, assertAgentMachine, canReach, checkCrashRecovery, checkParity, compareRecordings, compareRuns, createRecorder, explorePaths, lintAgentMachine, matchesTrajectory, runSeam, sameModelCallShape, simulateAgent, textOutput };
|
|
2163
|
+
export { AgentLintError, AgentUnknownStateError, RecordingDivergenceError, assertAgentMachine, canReach, checkConformance, checkCrashRecovery, checkParity, compareRecordings, compareRuns, createRecorder, explorePaths, lintAgentMachine, matchesTrajectory, recordAiSdk, runSeam, sameModelCallShape, simulateAgent, textOutput };
|