@tangle-network/agent-eval 0.117.1 → 0.118.1

package/dist/default-registry-DaK8b3fv.d.ts

@@ -1,155 +0,0 @@
-import { AxAIService } from '@ax-llm/ax';
-import { T as TraceAnalystKindSpec } from './kind-factory-ClZmO25A.js';
-import { A as Analyst, a as AnalystContext, b as AnalystRunSummary, c as AnalystFinding, d as AnalystRunResult, C as ChatClient, e as AnalystRunInputs, f as AnalystRunEvent } from './policy-edit-wG9uFEFm.js';
-
-/**
- * AnalystRegistry — orchestrate N analysts against one run.
- *
- * Owns three responsibilities and only three:
- *   1. Registration — ids must be unique; bad registrations fail loudly
- *      at register-time, not run-time.
- *   2. Routing — each analyst declares its `inputKind`; the registry
- *      picks the matching field from AnalystRunInputs and skips the
- *      analyst with a logged reason if it's missing.
- *   3. Isolation — one analyst's exception MUST NOT stop other analysts.
- *      Failed analysts produce zero findings + a 'failed' summary row.
- *
- * Cross-cutting concerns (telemetry, error → finding conversion, cost
- * ingestion, storage rotation) live in `AnalystHooks`. Budget shaping
- * (equal split vs weighted vs custom) lives in `BudgetPolicy`. Both
- * have sensible defaults; consumers override only what they need.
- */
-
-interface AnalystHooks {
-    /** Before analyze() — last chance to mutate ctx (e.g. inject tags, override budget). */
-    onBeforeAnalyze?(args: {
-        analyst: Analyst;
-        ctx: AnalystContext;
-        runId: string;
-    }): void | Promise<void>;
-    /** After every analyst (ok | failed | skipped). Use for telemetry, ingestion, rotation. */
-    onAfterAnalyze?(args: {
-        analyst: Analyst;
-        summary: AnalystRunSummary;
-        findings: AnalystFinding[];
-        runId: string;
-    }): void | Promise<void>;
-    /**
-     * On analyst exception. Hook MAY return findings to convert the
-     * error into structured findings; the summary still reports 'failed'.
-     * Return void to keep the default empty-findings behavior.
-     */
-    onError?(args: {
-        analyst: Analyst;
-        error: Error;
-        runId: string;
-    }): AnalystFinding[] | undefined | Promise<AnalystFinding[] | undefined>;
-    /** Once after registry.run() completes. Use for final aggregation, persistence. */
-    onComplete?(args: {
-        result: AnalystRunResult;
-    }): void | Promise<void>;
-}
-interface BudgetPolicy {
-    /** Overall USD cap across the registry.run(). */
-    totalUsd?: number;
-    /** Per-analyst weight for the default allocator. Missing ids get weight 1. */
-    weights?: Record<string, number>;
-    /**
-     * Custom allocator — receives the analyst, remaining/total budget, and
-     * the count of analysts that will run. Returns the per-analyst budget
-     * (or undefined to leave it uncapped). Overrides weights when set.
-     */
-    allocate?: (args: {
-        analyst: Analyst;
-        totalUsd: number | undefined;
-        remainingUsd: number | undefined;
-        runningCount: number;
-    }) => number | undefined;
-}
-interface AnalystRegistryOptions {
-    /** Shared chat client passed to every LLM analyst via AnalystContext. */
-    chat?: ChatClient;
-    /** Logger callback. Defaults to a no-op. */
-    log?: (msg: string, fields?: Record<string, unknown>) => void;
-    /** Hooks invoked around analyze() — observability + customization seam. */
-    hooks?: AnalystHooks;
-    /** Default budget when run() doesn't override. */
-    defaultBudget?: BudgetPolicy;
-}
-interface RegistryRunOpts {
-    /** Restrict to a subset of registered analysts by id. */
-    only?: string[];
-    /** Skip these analysts even if registered. Useful for cheap iteration. */
-    skip?: string[];
-    /** Budget policy — totalUsd + optional weights/allocator. Falls back to options.defaultBudget. */
-    budget?: BudgetPolicy;
-    /** Wall-clock cap. Analysts SHOULD honor `ctx.deadlineMs`. */
-    timeoutMs?: number;
-    /** Abort signal — forwarded into every analyst's context. */
-    signal?: AbortSignal;
-    /** Tags echoed into AnalystContext.tags — useful for tracking environment/version in findings. */
-    tags?: Record<string, string>;
-    /**
-     * Prior-run findings made available as retrieval context to every
-     * analyst via `ctx.priorFindings`. The registry forwards the slice
-     * whose `analyst_id` matches each registered analyst so a kind sees
-     * only its own history. Pass `{ '*': findings }` to broadcast to
-     * every analyst (useful for cross-kind chaining where the improvement
-     * analyst consumes upstream failure findings).
-     */
-    priorFindings?: ReadonlyArray<AnalystFinding> | Record<string, ReadonlyArray<AnalystFinding>>;
-}
-declare class AnalystRegistry {
-    private readonly analysts;
-    private readonly options;
-    constructor(options?: AnalystRegistryOptions);
-    register(analyst: Analyst): void;
-    list(): ReadonlyArray<{
-        id: string;
-        description: string;
-        version: string;
-        cost: Analyst['cost'];
-    }>;
-    run(runId: string, inputs: AnalystRunInputs, runOpts?: RegistryRunOpts): Promise<AnalystRunResult>;
-    /**
-     * Streaming counterpart to `run()`. Emits `AnalystRunEvent` values
-     * in real time — `run-started`, then per-analyst `skipped` /
-     * `started` / `completed`, then a terminal `run-completed` whose
-     * payload is the full `AnalystRunResult`. UIs use this to render
-     * progress; persistence consumers use `run()` and read the result.
-     *
-     * Hooks (`onBeforeAnalyze` / `onAfterAnalyze` / `onError` /
-     * `onComplete`) fire as before — streaming is additive, not a hook
-     * replacement.
-     */
-    runStream(runId: string, inputs: AnalystRunInputs, runOpts?: RegistryRunOpts): AsyncGenerator<AnalystRunEvent, void, void>;
-    private selectAnalysts;
-    private routeInput;
-}
-
-/**
- * `buildDefaultAnalystRegistry` — the canonical analyst suite, so consumers
- * stop hand-wiring `new AnalystRegistry()` + per-kind `createTraceAnalystKind`.
- *
- * The deterministic `behavioralAnalyst` is ALWAYS registered (it needs no
- * model and is model-agnostic by construction). The agentic RLM kinds are
- * registered only when an `ai` service is supplied — so a caller with no LLM
- * still gets the full behavioral/efficiency diagnosis, and the substrate's
- * "any model (including no model)" guarantee holds at the suite level.
- */
-
-interface DefaultAnalystRegistryOptions {
-    /** Ax service for the agentic RLM kinds. Omit → only the deterministic analyst. */
-    ai?: AxAIService;
-    /** Model for the agentic kinds (falls back to the ai service default). */
-    model?: string;
-    /** Which agentic kinds to register when `ai` is present. Default = the shipped suite. */
-    kinds?: readonly TraceAnalystKindSpec[];
-    /** Set false to omit the deterministic behavioral analyst (default: include). */
-    includeBehavioral?: boolean;
-    /** Forwarded to the AnalystRegistry constructor (signal, tags, priorFindings). */
-    registry?: AnalystRegistryOptions;
-}
-declare function buildDefaultAnalystRegistry(opts?: DefaultAnalystRegistryOptions): AnalystRegistry;
-
-export { AnalystRegistry as A, type BudgetPolicy as B, type DefaultAnalystRegistryOptions as D, type RegistryRunOpts as R, type AnalystHooks as a, type AnalystRegistryOptions as b, buildDefaultAnalystRegistry as c };

package/dist/emitter-CjD7vUwv.d.ts

@@ -1,122 +0,0 @@
-import { a as RunOutcome, R as Run, S as Span, b as SpanKind, L as LlmSpan, T as ToolSpan, c as RetrievalSpan, J as JudgeSpan, d as SandboxSpan, E as EventKind, e as TraceEvent, B as BudgetLedgerEntry, A as Artifact, M as Message } from './schema-B3Q3l9Z_.js';
-import { T as TraceStore } from './store-DGqD0Pyo.js';
-
-/**
- * TraceEmitter — hierarchical span builder that auto-parents using an
- * internal stack. One emitter per Run; emitters do NOT share state.
- *
- * Convenience methods (`llm`, `tool`, `retrieval`, `judge`, `sandbox`)
- * return a `SpanHandle` with `.end()` / `.fail()` so callers don't
- * have to thread spanIds manually. For async workflows that can't use
- * the stack (e.g. fan-out parallel calls), pass `parentSpanId`
- * explicitly.
- */
-
-interface SpanHandle<S extends Span = Span> {
-    span: S;
-    end(patch?: Partial<S>): Promise<void>;
-    fail(error: string | Error, patch?: Partial<S>): Promise<void>;
-}
-interface RunCompleteHookContext {
-    runId: string;
-    emitter: TraceEmitter;
-    store: TraceStore;
-    /** Outcome the caller passed to `endRun` (undefined for `abortRun`). */
-    outcome?: RunOutcome;
-    /** Final run status. */
-    status: 'completed' | 'failed' | 'aborted';
-}
-type RunCompleteHook = (ctx: RunCompleteHookContext) => Promise<void> | void;
-interface TraceEmitterOptions {
-    runId?: string;
-    /** Inject a clock for deterministic tests. */
-    now?: () => number;
-    /** Inject an id generator for deterministic tests. */
-    id?: () => string;
-    /**
-     * Hooks fired after `endRun` / `abortRun` writes the final run state.
-     * Designed for trace-analyst auto-execution, integrity assertions, and
-     * outbound notifications. Hooks run sequentially in the order supplied.
-     *
-     * By default a hook that throws is swallowed and logged as a `note` event
-     * on the run — auto-orchestration must not crash the underlying flow.
-     * Set `hookErrors: 'throw'` to propagate.
-     */
-    onRunComplete?: RunCompleteHook[];
-    /** `'swallow'` (default) | `'throw'`. */
-    hookErrors?: 'swallow' | 'throw';
-}
-declare class TraceEmitter {
-    private store;
-    private stack;
-    private _runId;
-    private now;
-    private id;
-    private hooks;
-    private hookErrors;
-    constructor(store: TraceStore, options?: TraceEmitterOptions);
-    get runId(): string;
-    get traceStore(): TraceStore;
-    /** Append a hook after construction (e.g. attach the trace analyst). */
-    addRunCompleteHook(hook: RunCompleteHook): void;
-    /**
-     * Begin a Run.
-     *
-     * `scenarioId` is required on the persisted Run shape — every Run downstream
-     * gets a non-empty scenarioId so filters and aggregations stay simple — but
-     * the INPUT here accepts it as optional. When omitted, startRun substitutes
-     * a sensible default (`run.layer ?? run.tags?.['kind'] ?? 'runtime'`) so
-     * runtime / operator / meta-eval runs that have no curated-scenario corpus
-     * to anchor to don't have to invent placeholder strings at the call site.
-     */
-    startRun(run: Omit<Run, 'runId' | 'scenarioId' | 'startedAt' | 'status'> & {
-        scenarioId?: string;
-    }): Promise<Run>;
-    endRun(outcome?: RunOutcome): Promise<void>;
-    abortRun(reason: string): Promise<void>;
-    private runHooks;
-    span<S extends Span = Span>(init: {
-        kind: SpanKind;
-        name: string;
-        parentSpanId?: string;
-        attributes?: Record<string, unknown>;
-    } & Partial<Omit<S, 'spanId' | 'runId' | 'startedAt' | 'kind' | 'name'>>): Promise<SpanHandle<S>>;
-    private handle;
-    private pop;
-    llm(init: Omit<LlmSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>): Promise<SpanHandle<LlmSpan>>;
-    tool(init: Omit<ToolSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>): Promise<SpanHandle<ToolSpan>>;
-    retrieval(init: Omit<RetrievalSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>): Promise<SpanHandle<RetrievalSpan>>;
-    recordJudge(verdict: Omit<JudgeSpan, 'spanId' | 'runId' | 'kind' | 'startedAt' | 'endedAt'>): Promise<JudgeSpan>;
-    sandbox(init: Omit<SandboxSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>): Promise<SpanHandle<SandboxSpan>>;
-    emit(event: {
-        kind: EventKind;
-        spanId?: string;
-        payload?: Record<string, unknown>;
-    }): Promise<TraceEvent>;
-    recordBudget(entry: Omit<BudgetLedgerEntry, 'runId' | 'timestamp'> & {
-        timestamp?: number;
-    }): Promise<BudgetLedgerEntry>;
-    recordArtifact(artifact: Omit<Artifact, 'artifactId' | 'runId'>): Promise<Artifact>;
-    /**
-     * Runs `fn` inside a span; auto-ends on success, auto-fails on throw.
-     * Returns the fn's return value. Use this for the 95% case.
-     */
-    within<T>(init: Parameters<TraceEmitter['span']>[0], fn: (handle: SpanHandle) => Promise<T>): Promise<T>;
-}
-/** Helper to build an LLM span handle args object from a provider-shaped response. */
-declare function llmSpanFromProvider(args: {
-    name?: string;
-    model: string;
-    messages: Message[];
-    output: string;
-    usage?: {
-        inputTokens?: number;
-        outputTokens?: number;
-        cachedTokens?: number;
-        reasoningTokens?: number;
-    };
-    costUsd?: number;
-    finishReason?: string;
-}): Omit<LlmSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>;
-
-export { type RunCompleteHook as R, type SpanHandle as S, TraceEmitter as T, type RunCompleteHookContext as a, type TraceEmitterOptions as b, llmSpanFromProvider as l };

package/dist/errors-oeQrLqXC.d.ts

@@ -1,74 +0,0 @@
-/**
- * Error taxonomy for `@tangle-network/agent-eval`.
- *
- * Every error this package throws as part of its *public contract* extends
- * `AgentEvalError`. Consumers can pattern-match by `instanceof <Subclass>` or
- * by the stable string `code` carried on the base class.
- *
- * The codes are stable across minor versions; new codes can be added, but
- * existing codes never change meaning. New subclasses are non-breaking.
- *
- * Internal invariant guards (`throw new Error('this should never happen')`)
- * remain plain `Error`s on purpose — they're programmer-mistake assertions,
- * not consumer-catchable contract failures.
- */
-type AgentEvalErrorCode = 'validation' | 'not_found' | 'config' | 'capture_integrity' | 'judge' | 'verification' | 'replay' | 'backend_integrity' | 'profile_matrix';
-/**
- * Base class for every contract error this package throws — carries the stable
- * string `code` taxonomy so consumers can `instanceof`-match or switch on `code`.
- */
-declare class AgentEvalError extends Error {
-    /** Stable string code. Survives minification; safe to switch on. */
-    readonly code: AgentEvalErrorCode;
-    constructor(code: AgentEvalErrorCode, message: string, options?: {
-        cause?: unknown;
-    });
-}
-/** Caller passed invalid arguments (out of range, mutually-exclusive options, bad shape). */
-declare class ValidationError extends AgentEvalError {
-    constructor(message: string, options?: {
-        cause?: unknown;
-    });
-}
-/** A named resource (run, span, rubric, scenario, dataset row, route) does not exist. */
-declare class NotFoundError extends AgentEvalError {
-    constructor(message: string, options?: {
-        cause?: unknown;
-    });
-}
-/** Configuration missing or malformed (`HOME` unset, required image not supplied, env var absent). */
-declare class ConfigError extends AgentEvalError {
-    constructor(message: string, options?: {
-        cause?: unknown;
-    });
-}
-/**
- * A run is missing the artifacts a launch-grade check requires:
- * raw HTTP capture absent, no LLM spans, route assertion failed, run-end
- * assertion tripped. Block ship on this; do not catch and move on.
- */
-declare class CaptureIntegrityError extends AgentEvalError {
-    constructor(message: string, options?: {
-        cause?: unknown;
-    });
-}
-/** A judge call failed in a way that's not retryable: schema parse failure, bad rubric, conflicting dimensions. */
-declare class JudgeError extends AgentEvalError {
-    constructor(message: string, options?: {
-        cause?: unknown;
-    });
-}
-/** A verifier signalled a hard failure (compile, test, schema) — distinct from a low judge score. */
-declare class VerificationError extends AgentEvalError {
-    constructor(message: string, options?: {
-        cause?: unknown;
-    });
-}
-/** Replay cache cannot satisfy a request: miss with no fallback, sink lacks list(), unsupported URL. */
-declare class ReplayError extends AgentEvalError {
-    constructor(message: string, options?: {
-        cause?: unknown;
-    });
-}
-
-export { AgentEvalError as A, CaptureIntegrityError as C, JudgeError as J, NotFoundError as N, ReplayError as R, ValidationError as V, ConfigError as a, type AgentEvalErrorCode as b, VerificationError as c };

package/dist/failure-cluster-DOAcSJ87.d.ts

@@ -1,76 +0,0 @@
-import { R as Run, S as Span, e as TraceEvent, F as FailureClass } from './schema-B3Q3l9Z_.js';
-import { T as TraceStore } from './store-DGqD0Pyo.js';
-
-/**
- * Failure taxonomy — canonical classes + a default classifier.
- *
- * Every failed run should end up in a named class. The classifier here
- * is rule-based (fast, deterministic); an LLM fallback can be added by
- * the consumer for novel cases and trained into the rule base over time.
- *
- * Consumers call `classifyFailure(run, spans, events)` and persist the
- * returned class as `Run.outcome.failureClass`.
- */
-
-interface FailureContext {
-    run: Run;
-    spans: Span[];
-    events: TraceEvent[];
-}
-interface FailureClassification {
-    failureClass: FailureClass;
-    reason: string;
-    triggerSpanId?: string;
-    triggerEventId?: string;
-}
-/** Ordered rules — first match wins. */
-interface FailureRule {
-    id: string;
-    match: (ctx: FailureContext) => {
-        failureClass: FailureClass;
-        reason: string;
-        triggerSpanId?: string;
-        triggerEventId?: string;
-    } | null;
-}
-declare const DEFAULT_RULES: FailureRule[];
-/** Classify the failure mode of a run using an ordered rule list. */
-declare function classifyFailure(ctx: FailureContext, rules?: FailureRule[]): FailureClassification;
-
-/**
- * FailureClusterView — groups failed runs by (failureClass, triggerTool,
- * argHash-prefix) so weekly reviews can prioritize the top-N clusters.
- *
- * Each cluster includes: N runs, scenarios affected, representative
- * error message, a proposed mitigation hint (rule → action table).
- */
-
-interface FailureCluster {
-    failureClass: FailureClass;
-    /** Tool name when the trigger was a tool span, else undefined. */
-    toolName?: string;
-    /** First 16 chars of argHash — clusters similar args. */
-    argPrefix?: string;
-    /**
-     * Source dimension when the trigger was a judge span (e.g. `'format'`,
-     * `'safety'`, `'correctness'`). Lets cross-template aggregators
-     * group failures by the dimension that fired without overloading
-     * `argPrefix`. Optional — clusters without this field deserialize cleanly.
-     */
-    dimension?: string;
-    runCount: number;
-    scenarioIds: string[];
-    exampleError?: string;
-    exampleRunId: string;
-}
-interface FailureClusterReport {
-    clusters: FailureCluster[];
-    totalFailures: number;
-    totalRuns: number;
-}
-declare function failureClusterView(store: TraceStore, options?: {
-    rules?: FailureRule[];
-    minClusterSize?: number;
-}): Promise<FailureClusterReport>;
-
-export { DEFAULT_RULES as D, type FailureClusterReport as F, type FailureCluster as a, type FailureClassification as b, type FailureContext as c, type FailureRule as d, classifyFailure as e, failureClusterView as f };