@attest-ai/core 1.0.0-rc.4 → 1.0.0-rc.5

package/dist/cjs/index.d.cts

@@ -5,8 +5,8 @@ import { Interface } from 'node:readline';
 import { AsyncLocalStorage } from 'node:async_hooks';
 import { EventEmitter } from 'node:events';
 
-declare const VERSION = "1.0.0-rc.4";
-declare const ENGINE_VERSION = "1.0.0-rc.4";
+declare const VERSION = "1.0.0-rc.5";
+declare const ENGINE_VERSION = "1.0.0-rc.5";
 
 declare class EngineManager {
     private enginePath;
@@ -1337,6 +1337,20 @@ declare class PluginRegistry {
     listPlugins(pluginType?: string): string[];
 }
 declare function executePluginAssertion(plugin: AttestPlugin, trace: Trace, spec: Record<string, unknown>, client: AttestClient, traceId: string, assertionId: string, timeoutMs?: number): Promise<PluginResult>;
+/** Explicitly register a plugin instance into the registry. */
+declare function registerPlugin(registry: PluginRegistry, name: string, plugin: AttestPlugin): void;
+/**
+ * Discover and load plugins declared by the project's direct dependencies.
+ *
+ * An installed dependency declares plugins in its `package.json` as
+ * `"attest": {"plugins": ["./dist/plugin.js"]}`. Each entry is imported and
+ * its default export (or, without one, every export implementing the
+ * plugin shape) is registered under the plugin's `name`. A failure loading
+ * one entry, or reading the project's `package.json`, is logged and does not
+ * raise, as with Python's `load_entrypoint_plugins`. Dependencies that are declared but not
+ * installed are skipped. Returns the number of plugins loaded.
+ */
+declare function loadPackagePlugins(registry: PluginRegistry, projectRoot?: string): Promise<number>;
 
 declare class Sampler {
     private readonly rate;
@@ -1439,6 +1453,132 @@ declare const ADVERSARIAL_USER: SimulatePersona;
 declare const CONFUSED_USER: SimulatePersona;
 declare const COOPERATIVE_USER: SimulatePersona;
 
+/** Mock tool implementations active for the current async scope. */
+type MockToolFn = (args: Record<string, unknown>) => unknown;
+
+/** Registry of mock tool implementations, activated for the duration of `run`. */
+declare class MockToolRegistry {
+    private readonly tools;
+    register(name: string, fn: MockToolFn): void;
+    /**
+     * Run `fn` with the registered mocks active. `TraceBuilder.addToolCall`
+     * returns the mock's result for a registered tool name. Scopes are
+     * isolated per async call chain (AsyncLocalStorage).
+     */
+    run<T>(fn: () => T): T;
+}
+type MockTool<F extends MockToolFn> = F & {
+    readonly mockToolName: string;
+};
+/** Tag `fn` as the mock for tool `name`; the returned function behaves like `fn`. */
+declare function mockTool<F extends MockToolFn>(name: string, fn: F): MockTool<F>;
+
+interface FaultInjectOptions {
+    /** Probability in [0, 1] that a call throws. Default 0. */
+    errorRate?: number;
+    /** Upper bound of uniform random delay per call, in ms. Default 0. */
+    latencyJitterMs?: number;
+    /** Seed for a deterministic fault sequence. */
+    seed?: number;
+}
+/**
+ * Wrap `fn` so calls throw `Error("Injected fault in <name>")` at
+ * `errorRate` and are delayed by up to `latencyJitterMs`. The RNG is
+ * created once per wrapped function, so a fixed seed gives a fixed
+ * sequence across calls.
+ */
+declare function faultInject<A extends unknown[], R>(options: FaultInjectOptions, fn: (...args: A) => R | Promise<R>): (...args: A) => Promise<R>;
+
+/**
+ * Anything with a test outcome: `AgentResult`, `RepeatResult` and
+ * `ScenarioResult`. Aggregates nest.
+ */
+interface Verdict {
+    readonly notEvaluatedReasons: Readonly<Record<string, number>>;
+    outcome(options?: {
+        strict?: boolean;
+    }): Outcome;
+    skipReason(): string;
+}
+type Renderable = Verdict & {
+    renderDiagnostics?(options?: {
+        color?: boolean;
+    }): string;
+};
+/** Diagnostics of one verdict: an `AgentResult` or an aggregate. */
+declare function renderVerdictDiagnostics(verdict: Renderable, color?: boolean): string;
+
+/** Results of `n` trials, aggregated under the assertion-status rules. */
+declare class RepeatResult implements Verdict {
+    readonly results: readonly Verdict[];
+    constructor(results?: readonly Verdict[]);
+    get count(): number;
+    /** Trials skipped because an assertion was not evaluated. */
+    get notEvaluated(): number;
+    /** Passing / evaluated trials; null when no trial was evaluated. */
+    get passRate(): number | null;
+    /** True when every trial's outcome is pass (soft failures included). */
+    get allPassed(): boolean;
+    outcome(options?: {
+        strict?: boolean;
+    }): Outcome;
+    get notEvaluatedReasons(): Readonly<Record<string, number>>;
+    skipReason(): string;
+    renderDiagnostics(options?: {
+        color?: boolean;
+    }): string;
+}
+/** Run `fn` `n` times sequentially and collect the verdicts. */
+declare function repeat<A extends unknown[]>(n: number, fn: (...args: A) => Verdict | Promise<Verdict>): (...args: A) => Promise<RepeatResult>;
+
+interface ScenarioConfig {
+    persona?: SimulatePersona;
+    mockTools: Readonly<Record<string, MockToolFn>>;
+    faultErrorRate: number;
+    faultLatencyJitterMs: number;
+    maxTurns: number;
+    seed?: number;
+}
+/** Passed to the scenario body on every turn. */
+interface ScenarioContext {
+    readonly persona: SimulatePersona | undefined;
+    /** Zero-based turn index. */
+    readonly turn: number;
+}
+/** One verdict per turn; the aggregate follows the assertion-status rules. */
+declare class ScenarioResult implements Verdict {
+    readonly config: ScenarioConfig;
+    readonly results: readonly AgentResult[];
+    constructor(config: ScenarioConfig, results?: readonly AgentResult[]);
+    /** True when the aggregate `outcome()` is pass. */
+    get passed(): boolean;
+    outcome(options?: {
+        strict?: boolean;
+    }): Outcome;
+    get notEvaluatedReasons(): Readonly<Record<string, number>>;
+    skipReason(): string;
+    renderDiagnostics(options?: {
+        color?: boolean;
+    }): string;
+}
+interface ScenarioOptions {
+    persona?: SimulatePersona;
+    mockTools?: Record<string, MockToolFn>;
+    /** Probability per turn of throwing `Injected fault at turn N`. Default 0. */
+    faultErrorRate?: number;
+    /** Upper bound of uniform random delay per turn, in ms. Default 0. */
+    faultLatencyJitterMs?: number;
+    /** Turns to run. Default 1. */
+    maxTurns?: number;
+    /** Seed for a deterministic fault sequence. */
+    seed?: number;
+}
+/**
+ * Run `fn` for `maxTurns` turns with mock tools active, injecting faults
+ * and latency per turn. `fn` receives the persona and turn index.
+ */
+declare function scenario<A extends unknown[]>(options: ScenarioOptions, fn: (ctx: ScenarioContext, ...args: A) => AgentResult | Promise<AgentResult>): (...args: A) => Promise<ScenarioResult>;
+
 declare class ManualAdapter {
     private readonly agentId;
     constructor(agentId?: string);
@@ -1663,5 +1803,5 @@ interface TraceAdapter {
     traceFromResponse(response: unknown, options?: Record<string, unknown>): Trace;
 }
 
-export { ADVERSARIAL_USER, Agent, AgentResult, AlertDispatcher, AnthropicAdapter, Assertion, AssertionResult, AttestClient, BatchBudget, BatchContext, CONFUSED_USER, COOPERATIVE_USER, CaptureError, ContinuousEvalRunner, ConversationMessage, DriftReport, ENGINE_VERSION, EgressBlockedError, EngineManager, EvaluateBatchResult, ExpectChain, FRIENDLY_USER, GeminiAdapter, InitializeResult, LangChainAdapter, ManualAdapter, OTelAdapter, OllamaAdapter, OpenAIAdapter, Outcome, PluginRegistry, ProtocolDiagnostic, ProtocolLogger, ReplayError, ReplayMissError, ReplayNotEvaluatedError, Sampler, SimulateFaultConfig, SimulatePersona, Step, TIER_1, TIER_2, TIER_3, Trace, TraceBuilder, TraceTree, Usage, VERSION, WorldChecks, WorldError, activeBuilder, addUsage, agent, attestExpect, computeAgreement, config, delegate, dslCall, egressBlockRequested, emitVitest, executePluginAssertion, firstMiss, formatUsage, getSampleRate, installEgressGuard, isProviderHost, isSimulationMode, liveLine, loadFixture, loadLabelsCSV, loadLabelsJSONL, parseUsage, promptHash, record, renderAssertionFailure, renderDiagnostics, renderSummary, renderVitest, requireSimulationCapability, resetConfig, resolveMode, simulationEnvEnabled, startReplay, startWorld, sumUsage, testPathFor, tier, uninstallEgressGuard, usageSummaryLine, zeroUsage };
-export type { AgreementResult, AlertPayload, AttestClientOptions, AttestPlugin, Capture, Contract, ContractResult, Fixture, LabelPair, LabeledRecord, PluginResult, RecordOptions, RecordResult, RenderOptions, ReplayMode, ReplayModeSetting, ReplayOptions, ReplayRecord, ReplaySession, ReplaySummary, TraceAdapter, WorldOptions, WorldSession, WorldSnapshot };
+export { ADVERSARIAL_USER, Agent, AgentResult, AlertDispatcher, AnthropicAdapter, Assertion, AssertionResult, AttestClient, BatchBudget, BatchContext, CONFUSED_USER, COOPERATIVE_USER, CaptureError, ContinuousEvalRunner, ConversationMessage, DriftReport, ENGINE_VERSION, EgressBlockedError, EngineManager, EvaluateBatchResult, ExpectChain, FRIENDLY_USER, GeminiAdapter, InitializeResult, LangChainAdapter, ManualAdapter, MockToolRegistry, OTelAdapter, OllamaAdapter, OpenAIAdapter, Outcome, PluginRegistry, ProtocolDiagnostic, ProtocolLogger, RepeatResult, ReplayError, ReplayMissError, ReplayNotEvaluatedError, Sampler, ScenarioResult, SimulateFaultConfig, SimulatePersona, Step, TIER_1, TIER_2, TIER_3, Trace, TraceBuilder, TraceTree, Usage, VERSION, WorldChecks, WorldError, activeBuilder, addUsage, agent, attestExpect, computeAgreement, config, delegate, dslCall, egressBlockRequested, emitVitest, executePluginAssertion, faultInject, firstMiss, formatUsage, getSampleRate, installEgressGuard, isProviderHost, isSimulationMode, liveLine, loadFixture, loadLabelsCSV, loadLabelsJSONL, loadPackagePlugins, mockTool, parseUsage, promptHash, record, registerPlugin, renderAssertionFailure, renderDiagnostics, renderSummary, renderVerdictDiagnostics, renderVitest, repeat, requireSimulationCapability, resetConfig, resolveMode, scenario, simulationEnvEnabled, startReplay, startWorld, sumUsage, testPathFor, tier, uninstallEgressGuard, usageSummaryLine, zeroUsage };
+export type { AgreementResult, AlertPayload, AttestClientOptions, AttestPlugin, Capture, Contract, ContractResult, FaultInjectOptions, Fixture, LabelPair, LabeledRecord, MockTool, MockToolFn, PluginResult, RecordOptions, RecordResult, RenderOptions, ReplayMode, ReplayModeSetting, ReplayOptions, ReplayRecord, ReplaySession, ReplaySummary, ScenarioConfig, ScenarioContext, ScenarioOptions, TraceAdapter, Verdict, WorldOptions, WorldSession, WorldSnapshot };

package/dist/esm/cli.js

@@ -8,8 +8,8 @@ import * as path4 from "path";
 import { fileURLToPath as fileURLToPath2 } from "url";
 
 // src/version.ts
-var VERSION = "1.0.0-rc.4";
-var ENGINE_VERSION = "1.0.0-rc.4";
+var VERSION = "1.0.0-rc.5";
+var ENGINE_VERSION = "1.0.0-rc.5";
 
 // src/engine-manager.ts
 import { spawn, spawnSync } from "child_process";
@@ -319,10 +319,17 @@ jobs:
       - name: Run tests
         id: tests
         continue-on-error: true
+        # Traces (prompts, outputs, tool args; not redacted) travel in the artifact; on a public
+        # repository any signed-in GitHub user can download it. Enable on private repositories only:
+        # env:
+        #   ATTEST_REPORT_TRACES: "1"
         run: npx vitest run ${REPORT_ARGS}
       - name: Write the JUnit report
         if: always()
         run: npx attest report junit --report attest-report.json --out attest-junit.xml
+      - name: Write the failure viewer
+        if: always()
+        run: npx attest view attest-report.json --no-open --out attest-report.html
       - name: Upload the Attest report
         if: always()
         uses: actions/upload-artifact@v7
@@ -331,6 +338,7 @@ jobs:
           path: |
             attest-report.json
             attest-junit.xml
+            attest-report.html
           if-no-files-found: error
       - name: Evaluate the merge policy
         if: always()