@attest-ai/core 1.0.0-rc.2 → 1.0.0-rc.4

package/dist/esm/index.d.ts

@@ -1,12 +1,12 @@
-import { InitializeResult, ProtocolLogger, ProtocolDiagnostic, Trace, Assertion, BatchBudget, EvaluateBatchResult, DriftReport, SimulatePersona, ConversationMessage, SimulateFaultConfig, Step, AssertionResult, Usage, Outcome } from './proto/index.js';
-export { AgentId, AssertionId, AssertionSpec, BiasProbe, ConstraintSpec, ContentSpec, ERR_ASSERTION_ERROR, ERR_ENGINE_ERROR, ERR_INVALID_TRACE, ERR_PROVIDER_ERROR, ERR_SESSION_ERROR, ERR_TIMEOUT, EmbeddingSpec, EngineTimeoutError, ErrorData, EvaluateBatchParams, Exchange, FailureClass, GenerateUserMessageParams, GenerateUserMessageResult, Handoff, InitializeParams, JudgeAgreement, JudgeMetadata, LlmJudgeSpec, OTLPConversionError, OUTCOME_FAIL, OUTCOME_PASS, OUTCOME_SKIP, PluginSpec, ProtocolDesyncError, ProtocolDiagnosticBuffer, ProtocolDiagnosticKind, ProtocolError, QueryDriftParams, QueryDriftResult, REASON_BUDGET_EXHAUSTED, REASON_JUDGE_ABSTAINED, REASON_MISSING_EVIDENCE, REASON_PROVIDER_NOT_CONFIGURED, REASON_PROVIDER_UNAVAILABLE, REASON_SHORT_CIRCUITED, REASON_SIMULATION, RPCError, RPCRequest, RPCResponse, STATUS_HARD_FAIL, STATUS_NOT_EVALUATED, STATUS_PASS, STATUS_SOFT_FAIL, STEP_AGENT_CALL, STEP_LLM_CALL, STEP_RETRIEVAL, STEP_TOOL_CALL, SchemaSpec, ShutdownResult, StepError, SubmitPluginResultParams, TYPE_CONSTRAINT, TYPE_CONTENT, TYPE_EMBEDDING, TYPE_LLM_JUDGE, TYPE_PLUGIN, TYPE_SCHEMA, TYPE_TRACE, TYPE_TRACE_TREE, TraceId, TraceMetadata, TraceSpec, TraceTreeSpec, Turn, agentId, assertionId, decodeResponse, encodeRequest, extractId, extractResult, layerForType, previewLine, traceId } from './proto/index.js';
+import { InitializeResult, ProtocolLogger, ProtocolDiagnostic, Trace, Assertion, BatchBudget, BatchContext, EvaluateBatchResult, DriftReport, SimulatePersona, ConversationMessage, SimulateFaultConfig, Step, AssertionResult, Usage, Outcome } from './proto/index.js';
+export { AgentId, AssertionId, AssertionSpec, BiasProbe, ConstraintSpec, ContentSpec, ERR_ASSERTION_ERROR, ERR_ENGINE_ERROR, ERR_INVALID_TRACE, ERR_PROVIDER_ERROR, ERR_SESSION_ERROR, ERR_TIMEOUT, EffectSpec, EmbeddingSpec, EngineTimeoutError, ErrorData, EvaluateBatchParams, Exchange, FailureClass, GenerateUserMessageParams, GenerateUserMessageResult, Handoff, InitializeParams, JudgeAgreement, JudgeMetadata, LlmJudgeSpec, OTLPConversionError, OUTCOME_FAIL, OUTCOME_PASS, OUTCOME_SKIP, PluginSpec, ProtocolDesyncError, ProtocolDiagnosticBuffer, ProtocolDiagnosticKind, ProtocolError, QueryDriftParams, QueryDriftResult, REASON_BUDGET_EXHAUSTED, REASON_JUDGE_ABSTAINED, REASON_MISSING_EVIDENCE, REASON_PROVIDER_NOT_CONFIGURED, REASON_PROVIDER_UNAVAILABLE, REASON_REPLAY_MISS, REASON_SHORT_CIRCUITED, REASON_SIMULATION, RPCError, RPCRequest, RPCResponse, STATUS_HARD_FAIL, STATUS_NOT_EVALUATED, STATUS_PASS, STATUS_SOFT_FAIL, STEP_AGENT_CALL, STEP_LLM_CALL, STEP_RETRIEVAL, STEP_TOOL_CALL, SchemaSpec, ShutdownResult, StepError, SubmitPluginResultParams, TYPE_CONSTRAINT, TYPE_CONTENT, TYPE_EFFECT, TYPE_EMBEDDING, TYPE_LLM_JUDGE, TYPE_PLUGIN, TYPE_SCHEMA, TYPE_TRACE, TYPE_TRACE_TREE, TraceId, TraceMetadata, TraceSpec, TraceTreeSpec, Turn, agentId, assertionId, decodeResponse, encodeRequest, extractId, extractResult, layerForType, previewLine, traceId } from './proto/index.js';
 import { ChildProcess } from 'node:child_process';
 import { Interface } from 'node:readline';
 import { AsyncLocalStorage } from 'node:async_hooks';
 import { EventEmitter } from 'node:events';
 
-declare const VERSION = "1.0.0-rc.2";
-declare const ENGINE_VERSION = "1.0.0-rc.2";
+declare const VERSION = "1.0.0-rc.4";
+declare const ENGINE_VERSION = "1.0.0-rc.4";
 
 declare class EngineManager {
     private enginePath;
@@ -84,6 +84,7 @@ declare class AttestClient {
     evaluateBatch(trace: Trace, assertions: readonly Assertion[], options?: {
         timeout?: number;
         budget?: BatchBudget;
+        context?: BatchContext;
     }): Promise<EvaluateBatchResult>;
     submitPluginResult(traceId: string, pluginName: string, assertionId: string, status: string, score: number, explanation: string, options?: {
         timeout?: number;
@@ -248,9 +249,52 @@ declare class TraceTree {
 
 type TrajectoryMode = "strict" | "unordered" | "subset" | "superset";
 type ArgOperator = "eq" | "ne" | "lt" | "lte" | "gt" | "gte" | "in" | "not_in";
+/** A resource's records filtered by exact field values. */
+type Where = Record<string, unknown>;
+/**
+ * Effect checks against one world snapshot, from {@link ExpectChain.world}. Each method adds
+ * an `effect` assertion (Layer 3) to the parent chain and returns it.
+ */
+declare class WorldChecks<C> {
+    private readonly _snapshot;
+    private readonly _add;
+    constructor(snapshot: Record<string, unknown> | null, add: (spec: Record<string, unknown>) => C);
+    private addCheck;
+    /**
+     * Assert how many records of `resource` match `where`. Give `equals`, or `atMost` and/or
+     * `atLeast`. A violation reads `effect violated: 2 refunds for O-1042 (expected 1)`.
+     *
+     * @throws Error when no bound is given, or `equals` is combined with another bound.
+     *
+     * @example
+     * ```ts
+     * expect(trace).world(await world.state()).count("refunds", { equals: 1, where: { order_id: "O-1042" } });
+     * ```
+     */
+    count(resource: string, opts: {
+        equals?: number;
+        atMost?: number;
+        atLeast?: number;
+        where?: Where;
+    }): C;
+    /** Assert no two matching records share the same values for field(s) `by`. */
+    unique(resource: string, opts: {
+        by: string | string[];
+        where?: Where;
+    }): C;
+    /**
+     * Assert every matching record carries a distinct value in field `key` (default
+     * `idempotency_key`), the record field that stores the idempotency key.
+     */
+    idempotent(resource: string, opts?: {
+        key?: string;
+        where?: Where;
+    }): C;
+}
 declare class ExpectChain {
     private readonly _result;
     private readonly _assertions;
+    private _worldBytes;
     constructor(result: AgentResult);
     get assertions(): Assertion[];
     get trace(): Trace;
@@ -565,6 +609,12 @@ declare class ExpectChain {
     toolArgSatisfies(tool: string, arg: string, op: ArgOperator, value: unknown, opts?: {
         soft?: boolean;
     }): this;
+    /**
+     * Check effects in a world snapshot:
+     * `expect(trace).world(await world.state()).count(...)`. `null` means the test ran without
+     * a world helper: the checks are `not_evaluated` (`missing_evidence`).
+     */
+    world(snapshot: Record<string, unknown> | null): WorldChecks<this>;
     /**
      * Assert the output message contains the given string.
      *
@@ -955,6 +1005,264 @@ declare class ExpectChain {
  */
 declare function attestExpect(result: AgentResult | Trace): ExpectChain;
 
+/**
+ * Capture: turn a run into a verified fixture, and fixtures into tests.
+ *
+ * `attest capture --name <name> -- <command>` (the engine) records a run's trace and model
+ * exchanges into `tests/attest/fixtures/<name>.attest.json`. This module is the TypeScript
+ * side:
+ *
+ * - `record()` captures the code in a callback in this process.
+ * - `loadFixture()` reads a fixture (`proto/attest/v1/fixture-v1.schema.json`).
+ * - `emitVitest()` writes the fixture's drafted contracts as a Vitest file, using the
+ *   `attestExpect()` DSL. `attest capture` and `record()` call it.
+ */
+
+/** A capture did not produce a fixture, or produced one that is not verified. */
+declare class CaptureError extends Error {
+    /** The fixture written, when the engine wrote one. */
+    readonly fixture: string | null;
+    constructor(message: string, fixture?: string | null);
+}
+/** A drafted contract: an engine assertion spec plus its id and description. */
+interface Contract {
+    id: string;
+    kind: string;
+    description: string;
+    type: string;
+    spec: Record<string, unknown>;
+}
+/** A contract's status on the captured trace. */
+interface ContractResult {
+    contract: string;
+    status: string;
+    explanation: string;
+}
+/** A capture fixture (fixture-v1). */
+interface Fixture {
+    name: string;
+    source: {
+        kind: string;
+        replay_capable: boolean;
+        command?: string[];
+        exit_code?: number;
+    };
+    trace: Trace;
+    exchanges: Record<string, unknown>[];
+    contracts: Contract[];
+    verification: {
+        verified: boolean;
+        expect_fail: boolean;
+        trace_check: {
+            ok: boolean;
+            errors: number;
+            warnings: number;
+        };
+        results: ContractResult[];
+        expected_failures: string[];
+    };
+    world: Record<string, unknown> | null;
+}
+/** Read a capture fixture written by `attest capture` or `record()`. */
+declare function loadFixture(file: string | URL): Fixture;
+/** The `ExpectChain` method call that emits `contract`'s spec; throws when there is none. */
+declare function dslCall(contract: Contract): string;
+/**
+ * Where the drafted test for a fixture goes: `tests/attest/<name>.test.ts` for
+ * `tests/attest/fixtures/<name>.attest.json`.
+ */
+declare function testPathFor(fixturePath: string): string;
+/**
+ * Render a fixture's drafted contracts as a Vitest module.
+ *
+ * A verified fixture becomes a test that enforces every contract with `evaluate`. An
+ * incident reproduction (`expect_fail`) becomes a test that checks, with `inspect`, that
+ * the expected failures still fail and the other contracts still pass; it stays green
+ * while it reproduces the incident.
+ */
+declare function renderVitest(fixture: Fixture, fixturePath: string, testPath: string): string;
+/**
+ * Write the drafted Vitest file for a fixture unless it already exists. Returns the path
+ * written, or null when a test file is already there (re-capturing never overwrites an
+ * edited test).
+ */
+declare function emitVitest(fixturePath: string): string | null;
+/** A running in-process capture. */
+interface Capture {
+    name: string;
+    /** The variables `record()` set in `process.env` for the callback. */
+    env: Record<string, string>;
+}
+interface RecordOptions {
+    /** Fixture directory (default `tests/attest/fixtures`). */
+    outDir?: string;
+    /** Save an incident reproduction: verified when drafted contracts fail. */
+    expectFail?: boolean;
+    /** Trace `agent_id` (default: the first turn's agent name). */
+    agentId?: string;
+    /** Write the drafted Vitest file (default true). */
+    emitTest?: boolean;
+    /** Engine binary (default: the SDK's engine discovery). */
+    enginePath?: string;
+    /**
+     * A `world.yaml` to host for the callback (the engine's `--world`): its base URL variable
+     * (`PAYMENTS_BASE_URL`) is set for the callback and the world's final state is saved in
+     * the fixture's `world` section.
+     */
+    world?: string;
+    /**
+     * Called after the callback returns and before the fixture is built; pass the tracer
+     * provider's `forceFlush` so batched spans reach the receiver.
+     */
+    flush?: () => unknown;
+}
+interface RecordResult<T> {
+    value: T;
+    fixture: string;
+    verified: boolean;
+    testFile: string | null;
+}
+/**
+ * Capture `fn` as fixture `<outDir>/<name>.attest.json`.
+ *
+ * Starts `attest-engine capture --serve`, which hosts the recording proxy and the
+ * OTLP/HTTP receiver on 127.0.0.1, and sets `OPENAI_BASE_URL`, `ANTHROPIC_BASE_URL` and
+ * `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` while `fn` runs. Model clients and the
+ * OpenTelemetry exporter read these when they are constructed, so construct them inside
+ * `fn`, and pass the tracer provider's `forceFlush` as `flush`. Afterwards the variables
+ * are restored and the engine writes the fixture; for a verified fixture with `emitTest`
+ * the drafted Vitest file is written unless one exists. If `fn` throws, the capture is
+ * abandoned and no fixture is written. Captures cannot overlap.
+ *
+ * @throws CaptureError when another capture is running, the engine writes no fixture,
+ *   or the fixture is not verified (its drafted contracts did not hold; the fixture is
+ *   still written and no test is drafted).
+ */
+declare function record<T>(name: string, fn: (capture: Capture) => T | Promise<T>, opts?: RecordOptions): Promise<RecordResult<T>>;
+
+/**
+ * Replay: run the agent's real code with model responses served from a fixture.
+ *
+ * `startReplay(name)` starts `attest-engine replay --serve`, which hosts the replay
+ * proxy on 127.0.0.1, and sets `OPENAI_BASE_URL`, `ANTHROPIC_BASE_URL` and
+ * `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` until `finish()`. The proxy answers every model
+ * request from `<fixturesDir>/<name>.attest.json` and makes no provider call. A request
+ * that matches no recording is a miss; `finish()` returns the engine's summary.
+ */
+type ReplayMode = "strict" | "auto";
+type ReplayModeSetting = ReplayMode | "off";
+/** One request the agent made during the replay. */
+interface ReplayRecord {
+    seq: number;
+    provider: string;
+    api: string;
+    outcome: "replayed" | "live" | "miss";
+    exchange?: number;
+    message?: string;
+}
+/** What the replay session served, as the engine reports it. */
+interface ReplaySummary {
+    fixture: string;
+    mode: ReplayMode;
+    replayed: number;
+    live: number;
+    misses: number;
+    unused: number;
+    live_cost_usd: number;
+    unpriced_live_calls: number;
+    records: ReplayRecord[];
+}
+/** A strict-mode replay request matched no recording. `message` is the engine's miss message. */
+declare class ReplayMissError extends Error {
+    readonly summary: ReplaySummary;
+    constructor(message: string, summary: ReplaySummary);
+}
+/** An auto-mode replay request matched no recording and no provider key was set: `not_evaluated: replay_miss`. */
+declare class ReplayNotEvaluatedError extends Error {
+    readonly summary: ReplaySummary;
+    constructor(message: string, summary: ReplaySummary);
+}
+declare class ReplayError extends Error {
+    constructor(message: string);
+}
+interface ReplayOptions {
+    /** Fixture directory (default `tests/attest/fixtures`). */
+    fixturesDir?: string;
+    /** `strict`, `auto` or `off`; default per `resolveMode`. `off` starts nothing. */
+    mode?: ReplayModeSetting;
+    /**
+     * Pass `--no-live` to the engine: no miss is forwarded live. Default: `ATTEST_EGRESS=block`.
+     * An installed egress guard always implies it.
+     */
+    noLive?: boolean;
+    /** Engine binary (default: the SDK's engine discovery). */
+    enginePath?: string;
+}
+interface ReplaySession {
+    readonly name: string;
+    readonly mode: ReplayModeSetting;
+    /** The variables the session set in `process.env`. */
+    readonly env: Record<string, string>;
+    /** The summary once `finish()` has run, else `null`. */
+    readonly summary: ReplaySummary | null;
+    /** Stop the engine, restore the environment and return the summary (`null` for `off`). Idempotent. */
+    finish(): Promise<ReplaySummary | null>;
+    /** The error a miss raises (valid after `finish()`): strict => miss, auto => not evaluated. */
+    error(): ReplayMissError | ReplayNotEvaluatedError | null;
+    /** `finish()`, then throw `error()` if any. */
+    check(): Promise<void>;
+}
+/**
+ * Replay mode from `explicit`, else `ATTEST_REPLAY` (`strict|auto|off`), else `strict`
+ * when `CI` is set, else `auto`.
+ */
+declare function resolveMode(explicit?: ReplayModeSetting, env?: NodeJS.ProcessEnv): ReplayModeSetting;
+/** Start a replay session for fixture `name`. Sessions cannot overlap. */
+declare function startReplay(name: string, opts?: ReplayOptions): Promise<ReplaySession>;
+/** The first miss message of a summary, or `null`. */
+declare function firstMiss(summary: ReplaySummary): string | null;
+/** `N live model calls · $X.XX`, plus ` (+N unpriced)`. */
+declare function liveLine(live: number, costUsd: number, unpriced?: number): string;
+
+/** A world-v1 snapshot. */
+type WorldSnapshot = Record<string, unknown>;
+declare class WorldError extends Error {
+    constructor(message: string);
+}
+interface WorldOptions {
+    /** Engine binary (default: the SDK's engine discovery). */
+    enginePath?: string;
+}
+interface WorldSession {
+    /** The absolute `world.yaml` path. */
+    readonly path: string;
+    /** The variable the session set in `process.env`: the world's base URL. */
+    readonly env: Record<string, string>;
+    /** The world's base URL. */
+    readonly baseUrl: string;
+    /** The current world-v1 snapshot, read from the running world. */
+    state(): Promise<WorldSnapshot>;
+    /** Stop the world, restore the environment and return the final snapshot. Idempotent. */
+    stop(): Promise<WorldSnapshot>;
+}
+/** Host the world defined by `worldFile` until the session stops. Worlds cannot overlap. */
+declare function startWorld(worldFile: string, opts?: WorldOptions): Promise<WorldSession>;
+
+declare class EgressBlockedError extends Error {
+    readonly host: string;
+    constructor(host: string);
+}
+declare function isProviderHost(host: string): boolean;
+/** True when `ATTEST_EGRESS=block`. */
+declare function egressBlockRequested(env?: NodeJS.ProcessEnv): boolean;
+/**
+ * Install the guard. Installs are counted: the returned function releases only this
+ * install (once), and the guard comes off when every install has been released.
+ */
+declare function installEgressGuard(): () => void;
+/** Remove the guard now, whatever the install count. */
+declare function uninstallEgressGuard(): void;
+
 declare class Agent {
     readonly name: string;
     private readonly fn;
@@ -1286,7 +1594,7 @@ declare class OTelAdapter {
      * @throws RangeError when a parsed export holds a non-finite number.
      * @throws OTLPConversionError when the engine times out, overflows the output buffer or
      *   prints no trace.
-     * @throws Error from `spawnSync` when the engine cannot start.
+     * @throws Error when the engine cannot start.
      */
     static fromOtlp(otlp: OTLPInput, agentId?: string): Promise<Trace>;
 }
@@ -1355,4 +1663,5 @@ interface TraceAdapter {
     traceFromResponse(response: unknown, options?: Record<string, unknown>): Trace;
 }
 
-export { ADVERSARIAL_USER, Agent, AgentResult, type AgreementResult, AlertDispatcher, type AlertPayload, AnthropicAdapter, Assertion, AssertionResult, AttestClient, type AttestClientOptions, type AttestPlugin, BatchBudget, CONFUSED_USER, COOPERATIVE_USER, ContinuousEvalRunner, ConversationMessage, DriftReport, ENGINE_VERSION, EngineManager, EvaluateBatchResult, ExpectChain, FRIENDLY_USER, GeminiAdapter, InitializeResult, type LabelPair, type LabeledRecord, LangChainAdapter, ManualAdapter, OTelAdapter, OllamaAdapter, OpenAIAdapter, Outcome, PluginRegistry, type PluginResult, ProtocolDiagnostic, ProtocolLogger, type RenderOptions, Sampler, SimulateFaultConfig, SimulatePersona, Step, TIER_1, TIER_2, TIER_3, Trace, type TraceAdapter, TraceBuilder, TraceTree, Usage, VERSION, activeBuilder, addUsage, agent, attestExpect, computeAgreement, config, delegate, executePluginAssertion, formatUsage, getSampleRate, isSimulationMode, loadLabelsCSV, loadLabelsJSONL, parseUsage, promptHash, renderAssertionFailure, renderDiagnostics, renderSummary, requireSimulationCapability, resetConfig, simulationEnvEnabled, sumUsage, tier, usageSummaryLine, zeroUsage };
+export { ADVERSARIAL_USER, Agent, AgentResult, AlertDispatcher, AnthropicAdapter, Assertion, AssertionResult, AttestClient, BatchBudget, BatchContext, CONFUSED_USER, COOPERATIVE_USER, CaptureError, ContinuousEvalRunner, ConversationMessage, DriftReport, ENGINE_VERSION, EgressBlockedError, EngineManager, EvaluateBatchResult, ExpectChain, FRIENDLY_USER, GeminiAdapter, InitializeResult, LangChainAdapter, ManualAdapter, OTelAdapter, OllamaAdapter, OpenAIAdapter, Outcome, PluginRegistry, ProtocolDiagnostic, ProtocolLogger, ReplayError, ReplayMissError, ReplayNotEvaluatedError, Sampler, SimulateFaultConfig, SimulatePersona, Step, TIER_1, TIER_2, TIER_3, Trace, TraceBuilder, TraceTree, Usage, VERSION, WorldChecks, WorldError, activeBuilder, addUsage, agent, attestExpect, computeAgreement, config, delegate, dslCall, egressBlockRequested, emitVitest, executePluginAssertion, firstMiss, formatUsage, getSampleRate, installEgressGuard, isProviderHost, isSimulationMode, liveLine, loadFixture, loadLabelsCSV, loadLabelsJSONL, parseUsage, promptHash, record, renderAssertionFailure, renderDiagnostics, renderSummary, renderVitest, requireSimulationCapability, resetConfig, resolveMode, simulationEnvEnabled, startReplay, startWorld, sumUsage, testPathFor, tier, uninstallEgressGuard, usageSummaryLine, zeroUsage };
+export type { AgreementResult, AlertPayload, AttestClientOptions, AttestPlugin, Capture, Contract, ContractResult, Fixture, LabelPair, LabeledRecord, PluginResult, RecordOptions, RecordResult, RenderOptions, ReplayMode, ReplayModeSetting, ReplayOptions, ReplayRecord, ReplaySession, ReplaySummary, TraceAdapter, WorldOptions, WorldSession, WorldSnapshot };