@statelyai/agent 2.0.0-alpha.40 → 2.0.0-alpha.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/dist/evals.cjs +1 -1
- package/dist/evals.d.cts +1 -1
- package/dist/evals.d.mts +1 -1
- package/dist/evals.mjs +1 -1
- package/dist/index.d.cts +1 -1
- package/dist/index.d.mts +1 -1
- package/dist/testing.cjs +376 -17
- package/dist/testing.d.cts +181 -5
- package/dist/testing.d.mts +181 -5
- package/dist/testing.mjs +377 -20
- package/dist/{utils-DAclF-kw.d.cts → utils-BMxUcjay.d.cts} +57 -1
- package/dist/{utils-B8k-YPYI.d.mts → utils-BV7pxhD-.d.mts} +57 -1
- package/dist/validate.cjs +1 -1
- package/dist/validate.d.cts +1 -1
- package/dist/validate.d.mts +1 -1
- package/dist/validate.mjs +1 -1
- package/dist/{verify-CIu4mqIZ.d.mts → verify-C1UU84eq.d.mts} +7 -1
- package/dist/{verify-CTHlmBbh.d.cts → verify-DlN6tK8_.d.cts} +7 -1
- package/dist/{verify-D8UDqU4l.cjs → verify-IVYsI026.cjs} +65 -14
- package/dist/{verify-Cacz6XSk.mjs → verify-lKyu3buS.mjs} +65 -14
- package/docs/evals.md +1 -0
- package/docs/index.md +2 -2
- package/docs/meta.json +1 -0
- package/docs/verify-in-place.md +326 -0
- package/docs/verify.md +67 -2
- package/llms.txt +1 -0
- package/package.json +1 -1
- package/readme.md +1 -0
package/dist/testing.d.cts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { c as AgentRequestExecutorInfo, f as AgentTextRequest, s as AgentRequestExecutor, t as AgentCallUsage, u as AgentRequestExecutors } from "./text-logic-x17uLBwA.cjs";
|
|
2
|
-
import { C as simulateAgent, S as lintAgentMachine, _ as SimulationScript, a as AgentPathTerminal, b as canReach, c as CanReachOrdering, d as ExplorePathsOptions, f as ExploreSnapshot, g as SimulateAgentResult, h as SimulateAgentOptions, i as AgentPathReport, l as CanReachResult, m as LintAgentMachineOptions, n as AgentLintError, o as AgentUnknownStateError, p as ExploreStep, r as AgentLintSeverity, s as AssertAgentMachineOptions, t as AgentLintDiagnostic, u as CanReachTarget, v as SimulationTrailEntry, w as textOutput, x as explorePaths, y as assertAgentMachine } from "./verify-
|
|
2
|
+
import { C as simulateAgent, S as lintAgentMachine, _ as SimulationScript, a as AgentPathTerminal, b as canReach, c as CanReachOrdering, d as ExplorePathsOptions, f as ExploreSnapshot, g as SimulateAgentResult, h as SimulateAgentOptions, i as AgentPathReport, l as CanReachResult, m as LintAgentMachineOptions, n as AgentLintError, o as AgentUnknownStateError, p as ExploreStep, r as AgentLintSeverity, s as AssertAgentMachineOptions, t as AgentLintDiagnostic, u as CanReachTarget, v as SimulationTrailEntry, w as textOutput, x as explorePaths, y as assertAgentMachine } from "./verify-DlN6tK8_.cjs";
|
|
3
3
|
import { a as TrajectoryMatch, i as TrajectoryItem, n as TrajectoryEvent, o as TrajectoryMiss, r as TrajectoryEventEnvelope, s as matchesTrajectory, t as MatchTrajectoryOptions } from "./trajectory-BFjXaooB.cjs";
|
|
4
|
-
import { p as AgentRuntimeOptions, s as AgentRunInit, u as AgentRunResult } from "./run-agent-BO9RSb7i.cjs";
|
|
4
|
+
import { _ as AgentTraceEvent, l as AgentRunOptions, p as AgentRuntimeOptions, s as AgentRunInit, u as AgentRunResult } from "./run-agent-BO9RSb7i.cjs";
|
|
5
5
|
import { n as AgentEventLogStore } from "./event-log-store-aFXK6yZX.cjs";
|
|
6
|
-
import { A as
|
|
7
|
-
import { AnyActorLogic, AnyStateMachine, EventFromLogic, InputFrom, SnapshotFrom, StateValue } from "xstate";
|
|
6
|
+
import { A as RecordedOutput, C as RecordableDecisionModel, D as RecordedEffect, E as RecordedCall, F as compareRecordings, I as createRecorder, L as recordAiSdk, M as Recorder, N as Recording, O as RecordedError, P as RecordingDivergenceError, R as sameModelCallShape, S as RecordClientOptions, T as RecordableModel, _ as EquivalenceReport, b as NormalizeEffect, f as AiSdkSeam, g as EffectDivergenceReason, h as EffectDivergence, j as RecordedToolCall, k as RecordedModelCall, m as CreateRecorderOptions, p as CompareRecordingsOptions, t as MetaOfSnapshot, v as ModelAnswer, w as RecordableLanguageModel, x as RecordCallOptions, y as ModelCallMatch } from "./utils-BMxUcjay.cjs";
|
|
7
|
+
import { AnyActorLogic, AnyStateMachine, EventFromLogic, EventObject, InputFrom, SnapshotFrom, StateValue } from "xstate";
|
|
8
8
|
//#region src/crash-recovery.d.ts
|
|
9
9
|
/** Options for {@link checkCrashRecovery}. */
|
|
10
10
|
interface CheckCrashRecoveryOptions<TMachine extends AnyStateMachine> {
|
|
@@ -428,6 +428,182 @@ export declare function compareRuns<TScenario extends {
|
|
|
428
428
|
expected?: string;
|
|
429
429
|
}>(options: CompareRunsOptions<TScenario>): Promise<ParityReport<TScenario>>;
|
|
430
430
|
//#endregion
|
|
431
|
+
//#region src/conformance.d.ts
|
|
432
|
+
/** One thing a recorded run did, as the model reads it. `kind` names what it is. */
|
|
433
|
+
interface ConformanceObservation {
|
|
434
|
+
kind: string;
|
|
435
|
+
}
|
|
436
|
+
/** What a {@link ConformanceActor} callback receives. */
|
|
437
|
+
interface ConformanceActorArgs<TObservation extends ConformanceObservation> {
|
|
438
|
+
/** The actor's input, as the model computed it. */
|
|
439
|
+
input: unknown;
|
|
440
|
+
/** The observation this actor consumed, or `undefined` for an actor with no `observes`. */
|
|
441
|
+
observation: TObservation | undefined;
|
|
442
|
+
/** Every observation of the run, in order. */
|
|
443
|
+
observations: readonly TObservation[];
|
|
444
|
+
/** The index of the next observation not yet consumed. */
|
|
445
|
+
index: number;
|
|
446
|
+
}
|
|
447
|
+
/** How one of the model's actors is answered from the recording. */
|
|
448
|
+
interface ConformanceActor<TObservation extends ConformanceObservation> {
|
|
449
|
+
/**
|
|
450
|
+
* The observation kind this actor consumes. When the model starts the actor,
|
|
451
|
+
* the next observation must be of this kind, or the run deviates. Omit it for
|
|
452
|
+
* work the recording cannot see: the actor then consumes nothing and is
|
|
453
|
+
* listed in {@link ConformanceReport.unobserved}.
|
|
454
|
+
*/
|
|
455
|
+
observes?: string;
|
|
456
|
+
/** Return `false` when the actor's input does not fit the observation: an `input-mismatch` deviation. */
|
|
457
|
+
matches?: (args: ConformanceActorArgs<TObservation>) => boolean;
|
|
458
|
+
/**
|
|
459
|
+
* The actor's output, which the model's `onDone` reads. Build it from the
|
|
460
|
+
* observation, and from the model's own `input` for state the recording does
|
|
461
|
+
* not show. Default: `undefined`.
|
|
462
|
+
*/
|
|
463
|
+
output?: (args: ConformanceActorArgs<TObservation>) => unknown;
|
|
464
|
+
/** Return a value to reject the actor with it, for work that failed in the recording. */
|
|
465
|
+
error?: (args: ConformanceActorArgs<TObservation>) => unknown;
|
|
466
|
+
}
|
|
467
|
+
/** What a {@link ConformanceEvent} `matches` receives. */
|
|
468
|
+
interface ConformanceEventArgs<TObservation extends ConformanceObservation> {
|
|
469
|
+
/** The observation the event comes from. */
|
|
470
|
+
observation: TObservation;
|
|
471
|
+
/** The model's context while it waited for the event. */
|
|
472
|
+
context: unknown;
|
|
473
|
+
/** The model's state value while it waited for the event. */
|
|
474
|
+
state: StateValue;
|
|
475
|
+
}
|
|
476
|
+
/**
|
|
477
|
+
* How an observation becomes an event the idle model receives. A function is
|
|
478
|
+
* the `event` alone. Add `matches` to compare the observation with what the
|
|
479
|
+
* model holds, such as the draft a person approved against the draft in context.
|
|
480
|
+
*/
|
|
481
|
+
type ConformanceEvent<TObservation extends ConformanceObservation> = ((observation: TObservation) => EventObject) | {
|
|
482
|
+
event: (observation: TObservation) => EventObject;
|
|
483
|
+
/** Return `false` when the observation does not fit the model's state: an `input-mismatch` deviation. */
|
|
484
|
+
matches?: (args: ConformanceEventArgs<TObservation>) => boolean;
|
|
485
|
+
};
|
|
486
|
+
/** How the code's recorded return value is checked against the model's output. */
|
|
487
|
+
interface ConformanceOutput<TObservation extends ConformanceObservation> {
|
|
488
|
+
/**
|
|
489
|
+
* The observation kind of the code's return value. When the model finishes,
|
|
490
|
+
* the next observation must be of this kind, or the run deviates.
|
|
491
|
+
*/
|
|
492
|
+
observes: string;
|
|
493
|
+
/** Return `false` when the model's output does not fit the recorded one: an `output-mismatch` deviation. */
|
|
494
|
+
matches?: (args: {
|
|
495
|
+
output: unknown;
|
|
496
|
+
observation: TObservation;
|
|
497
|
+
}) => boolean;
|
|
498
|
+
}
|
|
499
|
+
/** Options for {@link checkConformance}. */
|
|
500
|
+
interface CheckConformanceOptions<TMachine extends AnyStateMachine, TObservation extends ConformanceObservation> {
|
|
501
|
+
/** The machine input for this run, validated as `runAgent` validates it. */
|
|
502
|
+
input?: AgentRunOptions<TMachine>["input"];
|
|
503
|
+
/** The recorded run, in order. Map a recording with `recording.effects.flatMap(...)`. */
|
|
504
|
+
observations: readonly TObservation[];
|
|
505
|
+
/**
|
|
506
|
+
* One entry per actor source the model invokes, keyed by `src`. An invoked
|
|
507
|
+
* src with no entry is an `unmapped-actor` deviation.
|
|
508
|
+
*/
|
|
509
|
+
actors: Record<string, ConformanceActor<TObservation>>;
|
|
510
|
+
/**
|
|
511
|
+
* Observations the model receives as events, keyed by kind. When the model
|
|
512
|
+
* settles idle and the next observation has one of these kinds, it is
|
|
513
|
+
* consumed and sent, and the run resumes.
|
|
514
|
+
*/
|
|
515
|
+
events?: Record<string, ConformanceEvent<TObservation>>;
|
|
516
|
+
/**
|
|
517
|
+
* Checks the code's return value. Map the recording's `output` effect to an
|
|
518
|
+
* observation of kind `observes`: the model must finish there, and `matches`
|
|
519
|
+
* compares its output with that observation. Without it, a recorded return
|
|
520
|
+
* value is an `extra-observations` deviation like any other.
|
|
521
|
+
*/
|
|
522
|
+
output?: ConformanceOutput<TObservation>;
|
|
523
|
+
}
|
|
524
|
+
/** Why a run deviated from the model. */
|
|
525
|
+
type ConformanceDeviationReason =
|
|
526
|
+
/** The model started an actor, and the next observation is of another kind. */
|
|
527
|
+
"unexpected-observation" |
|
|
528
|
+
/** The model started an actor that observes something, or finished with `output` set, and the recording had ended. */
|
|
529
|
+
"missing-observation" |
|
|
530
|
+
/** An actor's or event's `matches` returned `false` for the observation. */
|
|
531
|
+
"input-mismatch" |
|
|
532
|
+
/** The model finished, and `output.matches` returned `false` for the recorded return value. */
|
|
533
|
+
"output-mismatch" |
|
|
534
|
+
/** The model finished or settled idle, and the recording continues. */
|
|
535
|
+
"extra-observations" |
|
|
536
|
+
/** The model was idle, and it does not handle the event an observation maps to in that state. */
|
|
537
|
+
"ignored-event" |
|
|
538
|
+
/** The model invoked a src with no entry in `actors`. */
|
|
539
|
+
"unmapped-actor";
|
|
540
|
+
/** The first point where the recorded run and the model part ways. */
|
|
541
|
+
interface ConformanceDeviation<TObservation extends ConformanceObservation> {
|
|
542
|
+
reason: ConformanceDeviationReason;
|
|
543
|
+
/** The index of the observation at the deviation; `observations.length` when the recording had ended. */
|
|
544
|
+
index: number;
|
|
545
|
+
/** What the model did next: the actor src it started, the event, `"end of run"`, or `"idle, waiting for an event"`. */
|
|
546
|
+
expected: string;
|
|
547
|
+
/** What the recording shows next, or `undefined` when it had ended. */
|
|
548
|
+
observed: TObservation | undefined;
|
|
549
|
+
/** The model's state value at the deviation. */
|
|
550
|
+
state: StateValue;
|
|
551
|
+
/** The actor's input, for an actor deviation. */
|
|
552
|
+
actorInput?: unknown;
|
|
553
|
+
}
|
|
554
|
+
/** The result of {@link checkConformance}. */
|
|
555
|
+
interface ConformanceReport<TObservation extends ConformanceObservation> {
|
|
556
|
+
/** True when the model produced the recorded run exactly, observation for observation. */
|
|
557
|
+
conforms: boolean;
|
|
558
|
+
/** The first deviation, when the run does not conform. */
|
|
559
|
+
deviation?: ConformanceDeviation<TObservation>;
|
|
560
|
+
/** How many observations the model consumed. */
|
|
561
|
+
consumed: number;
|
|
562
|
+
/**
|
|
563
|
+
* Actor srcs the model ran without an observation (no `observes`): what they
|
|
564
|
+
* did was not checked against the recording.
|
|
565
|
+
*/
|
|
566
|
+
unobserved: string[];
|
|
567
|
+
/** How the model's run settled. A deviation stops the run, which then settles `error`. */
|
|
568
|
+
status: "done" | "idle" | "error";
|
|
569
|
+
/** The model's trace events. Pass the traces of several runs to `traceCoverage`. */
|
|
570
|
+
trace: AgentTraceEvent[];
|
|
571
|
+
}
|
|
572
|
+
/**
|
|
573
|
+
* Checks a recorded run of existing code against a model of that code.
|
|
574
|
+
*
|
|
575
|
+
* The model runs with `runAgent`, and every actor in `actors` is answered from
|
|
576
|
+
* `observations` instead of running: an actor with `observes` consumes the next
|
|
577
|
+
* observation, which must be of that kind. The report names the first
|
|
578
|
+
* deviation: where the model started something the recording does not show
|
|
579
|
+
* next, or ended while the recording continued.
|
|
580
|
+
*
|
|
581
|
+
* Use it to verify existing code in place: prove the model's rules with
|
|
582
|
+
* `canReach`, then check that recorded runs of the code conform to the model.
|
|
583
|
+
* A conforming run shows the code behaved as the model allows on that run; it
|
|
584
|
+
* does not prove the code can never break a rule.
|
|
585
|
+
*
|
|
586
|
+
* Every actor the model invokes must be declared by name: a model whose invoke
|
|
587
|
+
* names logic inline throws `inline-actor-source`, because that logic would
|
|
588
|
+
* run. A callback in `actors` that throws ends the check with
|
|
589
|
+
* `conformance-callback-error` instead of counting as an actor failure.
|
|
590
|
+
*
|
|
591
|
+
* @example
|
|
592
|
+
* ```ts
|
|
593
|
+
* const report = await checkConformance(loopModel, {
|
|
594
|
+
* input: { maxIter: 3 },
|
|
595
|
+
* observations: recording.effects.flatMap(toObservation),
|
|
596
|
+
* actors: {
|
|
597
|
+
* route: { observes: "route", output: ({ observation }) => observation.names },
|
|
598
|
+
* runTurn: { observes: "turn", matches: ({ input, observation }) => input.name === observation.agent },
|
|
599
|
+
* save: {}, // not visible in this recording
|
|
600
|
+
* },
|
|
601
|
+
* });
|
|
602
|
+
* report.conforms; // false when the run did something the model does not allow
|
|
603
|
+
* ```
|
|
604
|
+
*/
|
|
605
|
+
export declare function checkConformance<TMachine extends AnyStateMachine, TObservation extends ConformanceObservation>(machine: TMachine, options: CheckConformanceOptions<TMachine, TObservation>): Promise<ConformanceReport<TObservation>>;
|
|
606
|
+
//#endregion
|
|
431
607
|
//#region src/check-parity.d.ts
|
|
432
608
|
/** One scenario: an id and the workflow's input. Extra fields are yours. */
|
|
433
609
|
interface CheckParityScenario<TInput = unknown> {
|
|
@@ -539,4 +715,4 @@ interface CheckParityReport<TScenario extends CheckParityScenario, TOutput> {
|
|
|
539
715
|
*/
|
|
540
716
|
export declare function checkParity<TScenario extends CheckParityScenario, TEffects extends object, TOutput, TLive = undefined>(options: CheckParityOptions<TScenario, TEffects, TOutput, TLive>): Promise<CheckParityReport<TScenario, TOutput>>;
|
|
541
717
|
//#endregion
|
|
542
|
-
export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AssertAgentMachineOptions, type CanReachOrdering, type CanReachResult, type CanReachTarget, type CheckCrashRecoveryOptions, type CheckParityBaseOptions, type CheckParityOptions, type CheckParityOutcome, type CheckParityReport, type CheckParityRow, type CheckParityScenario, type CheckParitySetup, type CompareRecordingsOptions, type CompareRunsOptions, type CrashEffect, type CrashPoint, type CrashRecoveryFailure, type CrashRecoveryProblem, type CrashRecoveryReport, type CreateRecorderOptions, type EffectDivergence, type EffectDivergenceReason, type EquivalenceReport, type ExplorePathsOptions, type ExploreSnapshot, type ExploreStep, type LintAgentMachineOptions, type MatchTrajectoryOptions, type ModelAnswer, type ModelCallMatch, type NormalizeEffect, type ParityLabel, type ParityReport, type ParityResult, type ParityRun, type RecordCallOptions, type RecordClientOptions, type RecordableDecisionModel, type RecordableLanguageModel, type RecordableModel, type RecordedCall, type RecordedEffect, type RecordedError, type RecordedModelCall, type RecordedOutput, type RecordedToolCall, type Recorder, type Recording, RecordingDivergenceError, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type TrajectoryEvent, type TrajectoryEventEnvelope, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, compareRecordings, createRecorder, explorePaths, lintAgentMachine, matchesTrajectory, sameModelCallShape, simulateAgent, textOutput };
|
|
718
|
+
export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AiSdkSeam, type AssertAgentMachineOptions, type CanReachOrdering, type CanReachResult, type CanReachTarget, type CheckConformanceOptions, type CheckCrashRecoveryOptions, type CheckParityBaseOptions, type CheckParityOptions, type CheckParityOutcome, type CheckParityReport, type CheckParityRow, type CheckParityScenario, type CheckParitySetup, type CompareRecordingsOptions, type CompareRunsOptions, type ConformanceActor, type ConformanceActorArgs, type ConformanceDeviation, type ConformanceDeviationReason, type ConformanceEvent, type ConformanceEventArgs, type ConformanceObservation, type ConformanceOutput, type ConformanceReport, type CrashEffect, type CrashPoint, type CrashRecoveryFailure, type CrashRecoveryProblem, type CrashRecoveryReport, type CreateRecorderOptions, type EffectDivergence, type EffectDivergenceReason, type EquivalenceReport, type ExplorePathsOptions, type ExploreSnapshot, type ExploreStep, type LintAgentMachineOptions, type MatchTrajectoryOptions, type ModelAnswer, type ModelCallMatch, type NormalizeEffect, type ParityLabel, type ParityReport, type ParityResult, type ParityRun, type RecordCallOptions, type RecordClientOptions, type RecordableDecisionModel, type RecordableLanguageModel, type RecordableModel, type RecordedCall, type RecordedEffect, type RecordedError, type RecordedModelCall, type RecordedOutput, type RecordedToolCall, type Recorder, type Recording, RecordingDivergenceError, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type TrajectoryEvent, type TrajectoryEventEnvelope, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, compareRecordings, createRecorder, explorePaths, lintAgentMachine, matchesTrajectory, recordAiSdk, sameModelCallShape, simulateAgent, textOutput };
|
package/dist/testing.d.mts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { c as AgentRequestExecutorInfo, f as AgentTextRequest, s as AgentRequestExecutor, t as AgentCallUsage, u as AgentRequestExecutors } from "./text-logic-x17uLBwA.mjs";
|
|
2
|
-
import { C as simulateAgent, S as lintAgentMachine, _ as SimulationScript, a as AgentPathTerminal, b as canReach, c as CanReachOrdering, d as ExplorePathsOptions, f as ExploreSnapshot, g as SimulateAgentResult, h as SimulateAgentOptions, i as AgentPathReport, l as CanReachResult, m as LintAgentMachineOptions, n as AgentLintError, o as AgentUnknownStateError, p as ExploreStep, r as AgentLintSeverity, s as AssertAgentMachineOptions, t as AgentLintDiagnostic, u as CanReachTarget, v as SimulationTrailEntry, w as textOutput, x as explorePaths, y as assertAgentMachine } from "./verify-
|
|
2
|
+
import { C as simulateAgent, S as lintAgentMachine, _ as SimulationScript, a as AgentPathTerminal, b as canReach, c as CanReachOrdering, d as ExplorePathsOptions, f as ExploreSnapshot, g as SimulateAgentResult, h as SimulateAgentOptions, i as AgentPathReport, l as CanReachResult, m as LintAgentMachineOptions, n as AgentLintError, o as AgentUnknownStateError, p as ExploreStep, r as AgentLintSeverity, s as AssertAgentMachineOptions, t as AgentLintDiagnostic, u as CanReachTarget, v as SimulationTrailEntry, w as textOutput, x as explorePaths, y as assertAgentMachine } from "./verify-C1UU84eq.mjs";
|
|
3
3
|
import { a as TrajectoryMatch, i as TrajectoryItem, n as TrajectoryEvent, o as TrajectoryMiss, r as TrajectoryEventEnvelope, s as matchesTrajectory, t as MatchTrajectoryOptions } from "./trajectory-BFjXaooB.mjs";
|
|
4
|
-
import { p as AgentRuntimeOptions, s as AgentRunInit, u as AgentRunResult } from "./run-agent-BxkeuAYR.mjs";
|
|
4
|
+
import { _ as AgentTraceEvent, l as AgentRunOptions, p as AgentRuntimeOptions, s as AgentRunInit, u as AgentRunResult } from "./run-agent-BxkeuAYR.mjs";
|
|
5
5
|
import { n as AgentEventLogStore } from "./event-log-store-DYun526m.mjs";
|
|
6
|
-
import { A as
|
|
7
|
-
import { AnyActorLogic, AnyStateMachine, EventFromLogic, InputFrom, SnapshotFrom, StateValue } from "xstate";
|
|
6
|
+
import { A as RecordedOutput, C as RecordableDecisionModel, D as RecordedEffect, E as RecordedCall, F as compareRecordings, I as createRecorder, L as recordAiSdk, M as Recorder, N as Recording, O as RecordedError, P as RecordingDivergenceError, R as sameModelCallShape, S as RecordClientOptions, T as RecordableModel, _ as EquivalenceReport, b as NormalizeEffect, f as AiSdkSeam, g as EffectDivergenceReason, h as EffectDivergence, j as RecordedToolCall, k as RecordedModelCall, m as CreateRecorderOptions, p as CompareRecordingsOptions, t as MetaOfSnapshot, v as ModelAnswer, w as RecordableLanguageModel, x as RecordCallOptions, y as ModelCallMatch } from "./utils-BV7pxhD-.mjs";
|
|
7
|
+
import { AnyActorLogic, AnyStateMachine, EventFromLogic, EventObject, InputFrom, SnapshotFrom, StateValue } from "xstate";
|
|
8
8
|
//#region src/crash-recovery.d.ts
|
|
9
9
|
/** Options for {@link checkCrashRecovery}. */
|
|
10
10
|
interface CheckCrashRecoveryOptions<TMachine extends AnyStateMachine> {
|
|
@@ -428,6 +428,182 @@ export declare function compareRuns<TScenario extends {
|
|
|
428
428
|
expected?: string;
|
|
429
429
|
}>(options: CompareRunsOptions<TScenario>): Promise<ParityReport<TScenario>>;
|
|
430
430
|
//#endregion
|
|
431
|
+
//#region src/conformance.d.ts
|
|
432
|
+
/** One thing a recorded run did, as the model reads it. `kind` names what it is. */
|
|
433
|
+
interface ConformanceObservation {
|
|
434
|
+
kind: string;
|
|
435
|
+
}
|
|
436
|
+
/** What a {@link ConformanceActor} callback receives. */
|
|
437
|
+
interface ConformanceActorArgs<TObservation extends ConformanceObservation> {
|
|
438
|
+
/** The actor's input, as the model computed it. */
|
|
439
|
+
input: unknown;
|
|
440
|
+
/** The observation this actor consumed, or `undefined` for an actor with no `observes`. */
|
|
441
|
+
observation: TObservation | undefined;
|
|
442
|
+
/** Every observation of the run, in order. */
|
|
443
|
+
observations: readonly TObservation[];
|
|
444
|
+
/** The index of the next observation not yet consumed. */
|
|
445
|
+
index: number;
|
|
446
|
+
}
|
|
447
|
+
/** How one of the model's actors is answered from the recording. */
|
|
448
|
+
interface ConformanceActor<TObservation extends ConformanceObservation> {
|
|
449
|
+
/**
|
|
450
|
+
* The observation kind this actor consumes. When the model starts the actor,
|
|
451
|
+
* the next observation must be of this kind, or the run deviates. Omit it for
|
|
452
|
+
* work the recording cannot see: the actor then consumes nothing and is
|
|
453
|
+
* listed in {@link ConformanceReport.unobserved}.
|
|
454
|
+
*/
|
|
455
|
+
observes?: string;
|
|
456
|
+
/** Return `false` when the actor's input does not fit the observation: an `input-mismatch` deviation. */
|
|
457
|
+
matches?: (args: ConformanceActorArgs<TObservation>) => boolean;
|
|
458
|
+
/**
|
|
459
|
+
* The actor's output, which the model's `onDone` reads. Build it from the
|
|
460
|
+
* observation, and from the model's own `input` for state the recording does
|
|
461
|
+
* not show. Default: `undefined`.
|
|
462
|
+
*/
|
|
463
|
+
output?: (args: ConformanceActorArgs<TObservation>) => unknown;
|
|
464
|
+
/** Return a value to reject the actor with it, for work that failed in the recording. */
|
|
465
|
+
error?: (args: ConformanceActorArgs<TObservation>) => unknown;
|
|
466
|
+
}
|
|
467
|
+
/** What a {@link ConformanceEvent} `matches` receives. */
|
|
468
|
+
interface ConformanceEventArgs<TObservation extends ConformanceObservation> {
|
|
469
|
+
/** The observation the event comes from. */
|
|
470
|
+
observation: TObservation;
|
|
471
|
+
/** The model's context while it waited for the event. */
|
|
472
|
+
context: unknown;
|
|
473
|
+
/** The model's state value while it waited for the event. */
|
|
474
|
+
state: StateValue;
|
|
475
|
+
}
|
|
476
|
+
/**
|
|
477
|
+
* How an observation becomes an event the idle model receives. A function is
|
|
478
|
+
* the `event` alone. Add `matches` to compare the observation with what the
|
|
479
|
+
* model holds, such as the draft a person approved against the draft in context.
|
|
480
|
+
*/
|
|
481
|
+
type ConformanceEvent<TObservation extends ConformanceObservation> = ((observation: TObservation) => EventObject) | {
|
|
482
|
+
event: (observation: TObservation) => EventObject;
|
|
483
|
+
/** Return `false` when the observation does not fit the model's state: an `input-mismatch` deviation. */
|
|
484
|
+
matches?: (args: ConformanceEventArgs<TObservation>) => boolean;
|
|
485
|
+
};
|
|
486
|
+
/** How the code's recorded return value is checked against the model's output. */
|
|
487
|
+
interface ConformanceOutput<TObservation extends ConformanceObservation> {
|
|
488
|
+
/**
|
|
489
|
+
* The observation kind of the code's return value. When the model finishes,
|
|
490
|
+
* the next observation must be of this kind, or the run deviates.
|
|
491
|
+
*/
|
|
492
|
+
observes: string;
|
|
493
|
+
/** Return `false` when the model's output does not fit the recorded one: an `output-mismatch` deviation. */
|
|
494
|
+
matches?: (args: {
|
|
495
|
+
output: unknown;
|
|
496
|
+
observation: TObservation;
|
|
497
|
+
}) => boolean;
|
|
498
|
+
}
|
|
499
|
+
/** Options for {@link checkConformance}. */
|
|
500
|
+
interface CheckConformanceOptions<TMachine extends AnyStateMachine, TObservation extends ConformanceObservation> {
|
|
501
|
+
/** The machine input for this run, validated as `runAgent` validates it. */
|
|
502
|
+
input?: AgentRunOptions<TMachine>["input"];
|
|
503
|
+
/** The recorded run, in order. Map a recording with `recording.effects.flatMap(...)`. */
|
|
504
|
+
observations: readonly TObservation[];
|
|
505
|
+
/**
|
|
506
|
+
* One entry per actor source the model invokes, keyed by `src`. An invoked
|
|
507
|
+
* src with no entry is an `unmapped-actor` deviation.
|
|
508
|
+
*/
|
|
509
|
+
actors: Record<string, ConformanceActor<TObservation>>;
|
|
510
|
+
/**
|
|
511
|
+
* Observations the model receives as events, keyed by kind. When the model
|
|
512
|
+
* settles idle and the next observation has one of these kinds, it is
|
|
513
|
+
* consumed and sent, and the run resumes.
|
|
514
|
+
*/
|
|
515
|
+
events?: Record<string, ConformanceEvent<TObservation>>;
|
|
516
|
+
/**
|
|
517
|
+
* Checks the code's return value. Map the recording's `output` effect to an
|
|
518
|
+
* observation of kind `observes`: the model must finish there, and `matches`
|
|
519
|
+
* compares its output with that observation. Without it, a recorded return
|
|
520
|
+
* value is an `extra-observations` deviation like any other.
|
|
521
|
+
*/
|
|
522
|
+
output?: ConformanceOutput<TObservation>;
|
|
523
|
+
}
|
|
524
|
+
/** Why a run deviated from the model. */
|
|
525
|
+
type ConformanceDeviationReason =
|
|
526
|
+
/** The model started an actor, and the next observation is of another kind. */
|
|
527
|
+
"unexpected-observation" |
|
|
528
|
+
/** The model started an actor that observes something, or finished with `output` set, and the recording had ended. */
|
|
529
|
+
"missing-observation" |
|
|
530
|
+
/** An actor's or event's `matches` returned `false` for the observation. */
|
|
531
|
+
"input-mismatch" |
|
|
532
|
+
/** The model finished, and `output.matches` returned `false` for the recorded return value. */
|
|
533
|
+
"output-mismatch" |
|
|
534
|
+
/** The model finished or settled idle, and the recording continues. */
|
|
535
|
+
"extra-observations" |
|
|
536
|
+
/** The model was idle, and it does not handle the event an observation maps to in that state. */
|
|
537
|
+
"ignored-event" |
|
|
538
|
+
/** The model invoked a src with no entry in `actors`. */
|
|
539
|
+
"unmapped-actor";
|
|
540
|
+
/** The first point where the recorded run and the model part ways. */
|
|
541
|
+
interface ConformanceDeviation<TObservation extends ConformanceObservation> {
|
|
542
|
+
reason: ConformanceDeviationReason;
|
|
543
|
+
/** The index of the observation at the deviation; `observations.length` when the recording had ended. */
|
|
544
|
+
index: number;
|
|
545
|
+
/** What the model did next: the actor src it started, the event, `"end of run"`, or `"idle, waiting for an event"`. */
|
|
546
|
+
expected: string;
|
|
547
|
+
/** What the recording shows next, or `undefined` when it had ended. */
|
|
548
|
+
observed: TObservation | undefined;
|
|
549
|
+
/** The model's state value at the deviation. */
|
|
550
|
+
state: StateValue;
|
|
551
|
+
/** The actor's input, for an actor deviation. */
|
|
552
|
+
actorInput?: unknown;
|
|
553
|
+
}
|
|
554
|
+
/** The result of {@link checkConformance}. */
|
|
555
|
+
interface ConformanceReport<TObservation extends ConformanceObservation> {
|
|
556
|
+
/** True when the model produced the recorded run exactly, observation for observation. */
|
|
557
|
+
conforms: boolean;
|
|
558
|
+
/** The first deviation, when the run does not conform. */
|
|
559
|
+
deviation?: ConformanceDeviation<TObservation>;
|
|
560
|
+
/** How many observations the model consumed. */
|
|
561
|
+
consumed: number;
|
|
562
|
+
/**
|
|
563
|
+
* Actor srcs the model ran without an observation (no `observes`): what they
|
|
564
|
+
* did was not checked against the recording.
|
|
565
|
+
*/
|
|
566
|
+
unobserved: string[];
|
|
567
|
+
/** How the model's run settled. A deviation stops the run, which then settles `error`. */
|
|
568
|
+
status: "done" | "idle" | "error";
|
|
569
|
+
/** The model's trace events. Pass the traces of several runs to `traceCoverage`. */
|
|
570
|
+
trace: AgentTraceEvent[];
|
|
571
|
+
}
|
|
572
|
+
/**
|
|
573
|
+
* Checks a recorded run of existing code against a model of that code.
|
|
574
|
+
*
|
|
575
|
+
* The model runs with `runAgent`, and every actor in `actors` is answered from
|
|
576
|
+
* `observations` instead of running: an actor with `observes` consumes the next
|
|
577
|
+
* observation, which must be of that kind. The report names the first
|
|
578
|
+
* deviation: where the model started something the recording does not show
|
|
579
|
+
* next, or ended while the recording continued.
|
|
580
|
+
*
|
|
581
|
+
* Use it to verify existing code in place: prove the model's rules with
|
|
582
|
+
* `canReach`, then check that recorded runs of the code conform to the model.
|
|
583
|
+
* A conforming run shows the code behaved as the model allows on that run; it
|
|
584
|
+
* does not prove the code can never break a rule.
|
|
585
|
+
*
|
|
586
|
+
* Every actor the model invokes must be declared by name: a model whose invoke
|
|
587
|
+
* names logic inline throws `inline-actor-source`, because that logic would
|
|
588
|
+
* run. A callback in `actors` that throws ends the check with
|
|
589
|
+
* `conformance-callback-error` instead of counting as an actor failure.
|
|
590
|
+
*
|
|
591
|
+
* @example
|
|
592
|
+
* ```ts
|
|
593
|
+
* const report = await checkConformance(loopModel, {
|
|
594
|
+
* input: { maxIter: 3 },
|
|
595
|
+
* observations: recording.effects.flatMap(toObservation),
|
|
596
|
+
* actors: {
|
|
597
|
+
* route: { observes: "route", output: ({ observation }) => observation.names },
|
|
598
|
+
* runTurn: { observes: "turn", matches: ({ input, observation }) => input.name === observation.agent },
|
|
599
|
+
* save: {}, // not visible in this recording
|
|
600
|
+
* },
|
|
601
|
+
* });
|
|
602
|
+
* report.conforms; // false when the run did something the model does not allow
|
|
603
|
+
* ```
|
|
604
|
+
*/
|
|
605
|
+
export declare function checkConformance<TMachine extends AnyStateMachine, TObservation extends ConformanceObservation>(machine: TMachine, options: CheckConformanceOptions<TMachine, TObservation>): Promise<ConformanceReport<TObservation>>;
|
|
606
|
+
//#endregion
|
|
431
607
|
//#region src/check-parity.d.ts
|
|
432
608
|
/** One scenario: an id and the workflow's input. Extra fields are yours. */
|
|
433
609
|
interface CheckParityScenario<TInput = unknown> {
|
|
@@ -539,4 +715,4 @@ interface CheckParityReport<TScenario extends CheckParityScenario, TOutput> {
|
|
|
539
715
|
*/
|
|
540
716
|
export declare function checkParity<TScenario extends CheckParityScenario, TEffects extends object, TOutput, TLive = undefined>(options: CheckParityOptions<TScenario, TEffects, TOutput, TLive>): Promise<CheckParityReport<TScenario, TOutput>>;
|
|
541
717
|
//#endregion
|
|
542
|
-
export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AssertAgentMachineOptions, type CanReachOrdering, type CanReachResult, type CanReachTarget, type CheckCrashRecoveryOptions, type CheckParityBaseOptions, type CheckParityOptions, type CheckParityOutcome, type CheckParityReport, type CheckParityRow, type CheckParityScenario, type CheckParitySetup, type CompareRecordingsOptions, type CompareRunsOptions, type CrashEffect, type CrashPoint, type CrashRecoveryFailure, type CrashRecoveryProblem, type CrashRecoveryReport, type CreateRecorderOptions, type EffectDivergence, type EffectDivergenceReason, type EquivalenceReport, type ExplorePathsOptions, type ExploreSnapshot, type ExploreStep, type LintAgentMachineOptions, type MatchTrajectoryOptions, type ModelAnswer, type ModelCallMatch, type NormalizeEffect, type ParityLabel, type ParityReport, type ParityResult, type ParityRun, type RecordCallOptions, type RecordClientOptions, type RecordableDecisionModel, type RecordableLanguageModel, type RecordableModel, type RecordedCall, type RecordedEffect, type RecordedError, type RecordedModelCall, type RecordedOutput, type RecordedToolCall, type Recorder, type Recording, RecordingDivergenceError, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type TrajectoryEvent, type TrajectoryEventEnvelope, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, compareRecordings, createRecorder, explorePaths, lintAgentMachine, matchesTrajectory, sameModelCallShape, simulateAgent, textOutput };
|
|
718
|
+
export { type AgentLintDiagnostic, AgentLintError, type AgentLintSeverity, type AgentPathReport, type AgentPathTerminal, AgentUnknownStateError, type AiSdkSeam, type AssertAgentMachineOptions, type CanReachOrdering, type CanReachResult, type CanReachTarget, type CheckConformanceOptions, type CheckCrashRecoveryOptions, type CheckParityBaseOptions, type CheckParityOptions, type CheckParityOutcome, type CheckParityReport, type CheckParityRow, type CheckParityScenario, type CheckParitySetup, type CompareRecordingsOptions, type CompareRunsOptions, type ConformanceActor, type ConformanceActorArgs, type ConformanceDeviation, type ConformanceDeviationReason, type ConformanceEvent, type ConformanceEventArgs, type ConformanceObservation, type ConformanceOutput, type ConformanceReport, type CrashEffect, type CrashPoint, type CrashRecoveryFailure, type CrashRecoveryProblem, type CrashRecoveryReport, type CreateRecorderOptions, type EffectDivergence, type EffectDivergenceReason, type EquivalenceReport, type ExplorePathsOptions, type ExploreSnapshot, type ExploreStep, type LintAgentMachineOptions, type MatchTrajectoryOptions, type ModelAnswer, type ModelCallMatch, type NormalizeEffect, type ParityLabel, type ParityReport, type ParityResult, type ParityRun, type RecordCallOptions, type RecordClientOptions, type RecordableDecisionModel, type RecordableLanguageModel, type RecordableModel, type RecordedCall, type RecordedEffect, type RecordedError, type RecordedModelCall, type RecordedOutput, type RecordedToolCall, type Recorder, type Recording, RecordingDivergenceError, type RunSeamOptions, type RunSeamResult, type SeamCall, type SeamRef, type SeamScriptEntry, type SeamSlice, type SeamTurn, type SimulateAgentOptions, type SimulateAgentResult, type SimulationScript, type SimulationTrailEntry, type TrajectoryEvent, type TrajectoryEventEnvelope, type TrajectoryItem, type TrajectoryMatch, type TrajectoryMiss, assertAgentMachine, canReach, compareRecordings, createRecorder, explorePaths, lintAgentMachine, matchesTrajectory, recordAiSdk, sameModelCallShape, simulateAgent, textOutput };
|