@tangle-network/agent-eval 0.117.1 → 0.118.0

package/dist/control-6vuGfmDH.d.ts

@@ -1,258 +0,0 @@
-import { b as FeedbackLabel, A as ProposedSideEffect, a as ControlEvalResult, i as ControlRunResult, j as ControlRuntimeConfig } from './feedback-trajectory-BUnM58xL.js';
-import { T as TraceEmitter } from './emitter-CjD7vUwv.js';
-import { F as FailureClass } from './schema-B3Q3l9Z_.js';
-import { T as TraceStore } from './store-DGqD0Pyo.js';
-import { a as RunSplitTag, c as RunTokenUsage, R as RunRecord } from './run-record-BDH49H2E.js';
-
-interface ActionExecutionPolicy {
-    allowedTypes?: string[];
-    blockedTypes?: string[];
-    alwaysRequireApprovalTypes?: string[];
-    autoApproveTypes?: string[];
-    requireApprovalForExternalSideEffects?: boolean;
-    requireApprovalAboveCostUsd?: number;
-    maxActionCostUsd?: number;
-    remainingBudgetUsd?: number;
-    expectedOutcomeRequired?: boolean;
-    killCriteriaRequired?: boolean;
-}
-interface ActionPolicyDecision {
-    allowed: boolean;
-    blocked: boolean;
-    requiresApproval: boolean;
-    reasons: string[];
-    label?: FeedbackLabel;
-}
-declare function evaluateActionPolicy(action: ProposedSideEffect, policy?: ActionExecutionPolicy, options?: {
-    createdAt?: string;
-}): ActionPolicyDecision;
-
-interface RunEvidenceMetadata {
-    experimentId: string;
-    candidateId: string;
-    seed: number;
-    model: string;
-    promptHash: string;
-    configHash: string;
-    commitSha: string;
-    splitTag: RunSplitTag;
-    tokenUsage: RunTokenUsage;
-    queueMs?: number;
-    judgeMetadata?: RunRecord['judgeMetadata'];
-    raw?: Record<string, number>;
-}
-interface ControlRunToRunRecordOptions extends RunEvidenceMetadata {
-    runId?: string;
-    score?: number;
-    failureMode?: string;
-}
-/**
- * Project a completed control-loop run into the strict RunRecord shape used by
- * release gates, optimizer tables, and research reports.
- *
- * The control loop owns live execution evidence. The caller still supplies the
- * experiment-cell metadata because prompt/config hashes, split assignment,
- * model snapshot, and commit SHA are product/harness concerns.
- */
-declare function controlRunToRunRecord<TState, TAction, TActionResult, TEval extends ControlEvalResult = ControlEvalResult>(run: ControlRunResult<TState, TAction, TActionResult, TEval>, options: ControlRunToRunRecordOptions): RunRecord;
-declare function scoreFromEvals(evals: readonly ControlEvalResult[]): number | undefined;
-
-/**
- * Propose / Verify / Review — the core multi-shot primitive.
- *
- *   shot N:  propose(state, priorReview) → new state
- *            verify(state)               → pass/fail, optional layers
- *            review(state, verification, memory) → observations + next-shot
- *                                                   instruction + shouldContinue
- *            memory.append(entry)
- *
- * Roles are strictly separated:
- *
- *   - The WORKER is whatever the caller wraps in `propose`. It is
- *     stateful — caller owns its resume/session mechanism.
- *   - The VERIFIER grades the state. It produces the ground truth.
- *     The reviewer cannot overturn or downgrade a verification layer.
- *   - The REVIEWER is stateless per call. Its continuity is the
- *     `ReviewMemoryStore` — durable JSONL by default, or any store
- *     implementing the interface. It reads memory + trace summary +
- *     verification and directs the NEXT proposer shot.
- *
- * This shape is load-bearing. The reviewer never grades; the verifier
- * never directs. Two processes, two prompts, two concerns — which is
- * what keeps the loop from confirmation-biasing itself into "all
- * passed" when it didn't.
- *
- * Short-circuits and soft-fails are both first-class:
- *   - verify.pass === true  → reviewer LLM call is skipped, memory
- *     records a success entry, loop exits.
- *   - review throws         → the shot still counts; the loop uses the
- *     last-known instruction (or `fallbackInstruction`) for the next
- *     propose call. A transient reviewer failure must NEVER abort a
- *     valid arc.
- *
- * Composable: `propose` itself can be another `runProposeReview` call.
- * That's the dogfooding path — a harness built on this primitive is in
- * turn evaluable by it.
- */
-
-interface Verification {
-    pass: boolean;
-    score?: number;
-    failingLayers?: string[];
-    details?: unknown;
-}
-interface Review {
-    observations: string;
-    diagnosis: string;
-    nextShotInstruction: string;
-    shouldContinue: boolean;
-    confidence: number;
-}
-interface ReviewMemoryEntry extends Review {
-    shot: number;
-    timestamp: number;
-    verification: {
-        pass: boolean;
-        score?: number;
-        failingLayers?: string[];
-    };
-}
-interface ProposeInput<State> {
-    shot: number;
-    goal: string;
-    state: State;
-    priorReview: Review | null;
-    abortSignal: AbortSignal;
-    emitter?: TraceEmitter;
-}
-interface ProposeOutput<State, Summary = unknown> {
-    state: State;
-    traceSummary?: Summary;
-}
-interface ReviewInput<State, Summary = unknown> {
-    shot: number;
-    goal: string;
-    state: State;
-    verification: Verification;
-    traceSummary: Summary | undefined;
-    memory: ReviewMemoryEntry[];
-}
-type ProposeFn<State, Summary = unknown> = (input: ProposeInput<State>) => Promise<ProposeOutput<State, Summary>>;
-type VerifyFn<State> = (state: State) => Promise<Verification>;
-type ReviewFn<State, Summary = unknown> = (input: ReviewInput<State, Summary>) => Promise<Review>;
-interface ReviewMemoryStore {
-    load(): Promise<ReviewMemoryEntry[]>;
-    append(entry: ReviewMemoryEntry): Promise<void>;
-}
-interface ProposeReviewConfig<State, Summary = unknown> {
-    goal: string;
-    initialState: State;
-    propose: ProposeFn<State, Summary>;
-    verify: VerifyFn<State>;
-    review: ReviewFn<State, Summary>;
-    /** Hard shot cap. Default 10. */
-    maxShots?: number;
-    /** Wall-clock cap in ms. Default 10 min. */
-    maxWallMs?: number;
-    /**
-     * If the reviewer returns confidence ≤ floor on `confidenceFloorWindow`
-     * consecutive shots, terminate early. Default floor 0.3, window 2.
-     * Set window to 0 or floor to <0 to disable.
-     */
-    confidenceFloor?: number;
-    confidenceFloorWindow?: number;
-    /** Defaults to an in-memory store if omitted. */
-    memory?: ReviewMemoryStore;
-    /** If provided, emit a Run + per-shot spans. */
-    store?: TraceStore;
-    scenarioId?: string;
-    projectId?: string;
-    variantId?: string;
-    /**
-     * Used when the reviewer soft-fails on shot 1 (no prior instruction to
-     * fall back to). Default is a generic "inspect failures and fix".
-     */
-    fallbackInstruction?: string;
-}
-interface ProposeReviewShot<State, Summary = unknown> {
-    shot: number;
-    state: State;
-    verification: Verification;
-    traceSummary: Summary | undefined;
-    review: Review;
-    reviewAvailable: boolean;
-    reviewError?: string;
-    durationMs: number;
-}
-interface ProposeReviewReport<State, Summary = unknown> {
-    runId: string | null;
-    completed: boolean;
-    shots: ProposeReviewShot<State, Summary>[];
-    finalState: State;
-    finalVerification: Verification;
-    failureClass?: FailureClass;
-    wallMs: number;
-    score: number;
-}
-declare function inMemoryReviewStore(initial?: ReviewMemoryEntry[]): ReviewMemoryStore;
-declare function jsonlReviewStore(path: string): ReviewMemoryStore;
-declare function runProposeReview<State, Summary = unknown>(config: ProposeReviewConfig<State, Summary>): Promise<ProposeReviewReport<State, Summary>>;
-type LlmJsonCall = (req: {
-    system: string;
-    user: string;
-}) => Promise<unknown>;
-interface LlmReviewerConfig<State, Summary = unknown> {
-    callJson: LlmJsonCall;
-    renderState?: (state: State) => string;
-    renderTraceSummary?: (summary: Summary | undefined) => string;
-    /** Appended to the default system prompt. */
-    systemPromptAddendum?: string;
-}
-declare function createLlmReviewer<State, Summary = unknown>(cfg: LlmReviewerConfig<State, Summary>): ReviewFn<State, Summary>;
-
-interface ProposeReviewControlState<State, Summary = unknown> {
-    shot: number;
-    state: State;
-    priorReview: Review | null;
-    verification: Verification;
-    traceSummary?: Summary;
-    memory: ReviewMemoryEntry[];
-    completed: boolean;
-    reviewAvailable: boolean;
-    reviewError?: string;
-}
-interface ProposeReviewControlAction {
-    type: 'propose-review-shot';
-    shot: number;
-}
-interface ProposeReviewControlResult<State, Summary = unknown> {
-    state: State;
-    verification: Verification;
-    traceSummary?: Summary;
-    review: Review | null;
-    reviewAvailable: boolean;
-    reviewError?: string;
-}
-interface ProposeReviewControlConfig<State, Summary = unknown> {
-    goal: string;
-    initialState: State;
-    propose: ProposeFn<State, Summary>;
-    verify: VerifyFn<State>;
-    review: ReviewFn<State, Summary>;
-    maxShots?: number;
-    maxWallMs?: number;
-    memory?: ReviewMemoryStore;
-    store?: TraceStore;
-    scenarioId?: string;
-    projectId?: string;
-    variantId?: string;
-    fallbackInstruction?: string;
-    confidenceFloor?: number;
-    confidenceFloorWindow?: number;
-    failureClassFromVerification?: (verification: Verification) => FailureClass | undefined;
-    actionFailure?: ControlRuntimeConfig<ProposeReviewControlState<State, Summary>, ProposeReviewControlAction, ProposeReviewControlResult<State, Summary>>['actionFailure'];
-}
-declare function runProposeReviewAsControlLoop<State, Summary = unknown>(config: ProposeReviewControlConfig<State, Summary>): Promise<ControlRunResult<ProposeReviewControlState<State, Summary>, ProposeReviewControlAction, ProposeReviewControlResult<State, Summary>>>;
-declare function controlFailureClassFromVerification(verification: Verification): FailureClass | undefined;
-
-export { type ActionExecutionPolicy as A, type ControlRunToRunRecordOptions as C, type LlmJsonCall as L, type ProposeFn as P, type Review as R, type Verification as V, type ActionPolicyDecision as a, type LlmReviewerConfig as b, type ProposeInput as c, type ProposeOutput as d, type ProposeReviewConfig as e, type ProposeReviewControlAction as f, type ProposeReviewControlConfig as g, type ProposeReviewControlResult as h, type ProposeReviewControlState as i, type ProposeReviewReport as j, type ProposeReviewShot as k, type ReviewFn as l, type ReviewInput as m, type ReviewMemoryEntry as n, type ReviewMemoryStore as o, type RunEvidenceMetadata as p, type VerifyFn as q, controlFailureClassFromVerification as r, controlRunToRunRecord as s, createLlmReviewer as t, evaluateActionPolicy as u, inMemoryReviewStore as v, jsonlReviewStore as w, runProposeReview as x, runProposeReviewAsControlLoop as y, scoreFromEvals as z };

package/dist/cost-ledger-DWy3XdJc.d.ts

@@ -1,183 +0,0 @@
-import { V as ValidationError } from './errors-oeQrLqXC.js';
-
-type CostChannel = 'agent' | 'judge' | 'verifier' | 'analyst' | 'driver' | (string & {});
-interface CostUsage {
-    inputTokens: number;
-    outputTokens: number;
-    cachedTokens?: number;
-}
-interface CostCallBase {
-    callId: string;
-    channel: CostChannel;
-    phase: string;
-    actor: string;
-    model: string;
-    maximumCostUsd?: number;
-    tags?: Record<string, string>;
-    timestamp: number;
-}
-interface CostReceipt extends CostCallBase, CostUsage {
-    status: 'settled';
-    costUsd: number;
-    costUnknown: boolean;
-    usageUnknown?: boolean;
-    pricing?: {
-        inputUsdPerThousand: number;
-        outputUsdPerThousand: number;
-    };
-    actualCostUsd?: number;
-    error?: string;
-}
-/** @deprecated Read-only compatibility shape. New paid work uses `runPaidCall`. */
-type CostLedgerEntry = Omit<CostReceipt, 'status' | 'callId' | 'phase' | 'actor' | 'maximumCostUsd' | 'usageUnknown' | 'pricing' | 'error'>;
-interface CostReceiptInput extends CostUsage {
-    model: string;
-    actualCostUsd?: number;
-    costUnknown?: boolean;
-    usageUnknown?: boolean;
-}
-type MaximumCharge = {
-    externallyEnforcedMaximumUsd: number;
-} | ({
-    model: string;
-} & CostUsage);
-interface RunPaidCallInput<T> {
-    callId?: string;
-    channel: CostChannel;
-    phase: string;
-    actor: string;
-    /** Used before a provider receipt exists and on failures without one. */
-    model?: string;
-    tags?: Record<string, string>;
-    signal?: AbortSignal;
-    /** Provider-enforced dollar maximum, or maximum priced token usage. Required when capped. */
-    maximumCharge?: MaximumCharge;
-    /** `callId` can be forwarded as the provider's idempotency key. */
-    execute(signal: AbortSignal, callId: string): Promise<T>;
-    receipt(value: T): CostReceiptInput;
-    receiptFromError?(error: Error): CostReceiptInput | undefined;
-}
-type PaidCallResult<T> = {
-    succeeded: true;
-    callId: string;
-    value: T;
-    receipt: CostReceipt;
-} | {
-    succeeded: false;
-    callId?: string;
-    error: Error;
-    receipt?: CostReceipt;
-};
-interface ChannelRollup {
-    channel: CostChannel;
-    calls: number;
-    inputTokens: number;
-    outputTokens: number;
-    cachedTokens: number;
-    costUsd: number;
-    unpricedCalls: number;
-    unknownUsageCalls: number;
-}
-interface CostLedgerSummary {
-    totalCalls: number;
-    pendingCalls: number;
-    unresolvedCalls: number;
-    reservedCostUsd: number;
-    inputTokens: number;
-    outputTokens: number;
-    cachedTokens: number;
-    totalCostUsd: number;
-    byChannel: ChannelRollup[];
-    unpricedModels: string[];
-    fullyPriced: boolean;
-    usageComplete: boolean;
-    accountingComplete: boolean;
-    incompleteReasons: string[];
-}
-interface CostLedgerFilter {
-    channel?: CostChannel;
-    phase?: string;
-    tags?: Record<string, string>;
-}
-/** Append-only storage. `append` must atomically reject stale revisions. */
-interface CostLedgerPersistence {
-    read(): {
-        revision: string;
-        events: string;
-    };
-    append(expectedRevision: string, event: string): string | undefined;
-}
-interface CostLedgerOptions {
-    costCeilingUsd?: number;
-    persistence?: CostLedgerPersistence;
-    /** Import already-settled receipts without admitting new paid work. */
-    receipts?: readonly CostReceipt[];
-}
-declare class CostCeilingReachedError extends ValidationError {
-    constructor(ceilingUsd: number, committedAndReservedUsd: number, requestedUsd: number, phase: string, actor: string);
-}
-declare class CostAccountingIncompleteError extends ValidationError {
-}
-declare class CostReservationExceededError extends ValidationError {
-    constructor(actor: string, actualUsd: number, maximumUsd: number);
-}
-declare class CostCallConflictError extends ValidationError {
-    readonly callId?: string;
-    readonly receipt?: CostReceipt;
-    constructor(message: string, options?: {
-        callId?: string;
-        receipt?: CostReceipt;
-        cause?: unknown;
-    });
-}
-declare class CostLedgerPersistenceError extends ValidationError {
-    readonly callId?: string;
-    readonly receipt?: CostReceipt;
-    constructor(cause: unknown, callId?: string, receipt?: CostReceipt);
-}
-declare class CostReceiptCaptureError extends ValidationError {
-    readonly callId: string;
-    readonly receipt?: CostReceipt;
-    readonly receiptError: Error;
-    constructor(callId: string, cause: unknown, receiptError: unknown, receipt?: CostReceipt);
-}
-/** Run-wide paid-call admission, durable call state, receipts, and summaries. */
-declare class CostLedger {
-    private readonly records;
-    private readonly activeCallIds;
-    private readonly lateCallIds;
-    private completedTasks;
-    private revision;
-    private costLimitPersisted;
-    readonly costCeilingUsd?: number;
-    private readonly persistence?;
-    constructor(input?: number | CostLedgerOptions);
-    runPaidCall<T>(input: RunPaidCallInput<T>): Promise<PaidCallResult<T>>;
-    /** Settle a call left pending by a crashed process after reconciling with the provider. */
-    reconcile(callId: string, observed: CostReceiptInput, options?: {
-        error?: string;
-    }): CostReceipt;
-    list(filter?: CostLedgerFilter): CostReceipt[];
-    summary(filter?: CostLedgerFilter): CostLedgerSummary;
-    markCompleted(count?: number): void;
-    costPerCompletedTask(): number | null;
-    private execute;
-    private captureLateOutcome;
-    private commitOutcome;
-    private captureFailure;
-    private commitReceipt;
-    private resolveMaximum;
-    private hasIncompleteSettledCall;
-    private appendRecord;
-    private ensureCostLimitPersisted;
-    private appendEvent;
-}
-/** Return the canonical pricing-table key, or null when the model is unpriced. */
-declare function modelPriceKey(model: string): string | null;
-interface CostResult {
-    costUsd: number;
-    costUnknown: boolean;
-}
-declare function costForUsage(model: string, usage: CostUsage): CostResult;
-
-export { CostLedger as C, type MaximumCharge as M, type PaidCallResult as P, type RunPaidCallInput as R, type CostChannel as a, type CostLedgerSummary as b, type CostReceiptInput as c, type CostReceipt as d, type ChannelRollup as e, CostAccountingIncompleteError as f, CostCallConflictError as g, CostCeilingReachedError as h, type CostLedgerEntry as i, type CostLedgerFilter as j, type CostLedgerOptions as k, type CostLedgerPersistence as l, CostLedgerPersistenceError as m, CostReceiptCaptureError as n, CostReservationExceededError as o, type CostResult as p, type CostUsage as q, costForUsage as r, modelPriceKey as s };

package/dist/dataset-NENEzRgk.d.ts

@@ -1,115 +0,0 @@
-import { V as ValidationError } from './errors-oeQrLqXC.js';
-
-/**
- * Dataset — versioned, sliceable, content-hashed scenario collection.
- *
- * Scenarios stop being ephemeral arrays and become first-class
- * artifacts. Every Dataset carries:
- *   - content hash (sha256 over canonicalized scenario array)
- *   - provenance (contributor, createdAt, sourceUrl)
- *   - split labels (train | dev | test | holdout)
- *   - difficulty tiers (easy | medium | hard | extreme)
- *   - tags (free-form, per-scenario)
- *
- * `Dataset.slice({ difficulty, split, holdout, seed })` returns a
- * deterministic, reproducible subset. Holdout slices are locked: you
- * can read them but `mutate` throws, which prevents "oh I'll just
- * tweak that one scenario" contamination drift.
- */
-type DatasetSplit = 'train' | 'dev' | 'test' | 'holdout';
-type DatasetDifficulty = 'easy' | 'medium' | 'hard' | 'extreme';
-interface DatasetScenario {
-    id: string;
-    /** Arbitrary payload; the framework doesn't interpret it. */
-    payload: unknown;
-    split?: DatasetSplit;
-    difficulty?: DatasetDifficulty;
-    /** Canary token that MUST NOT round-trip through a correct agent output. */
-    canary?: string;
-    /**
-     * Behavioral-canary forbidden pattern. A string OR a serialized regex
-     * (`/.../flags`) that the agent under test MUST NOT emit. Used by
-     * {@link import('./canary').checkBehavioralCanary | checkBehavioralCanary},
-     * which inverts the contamination-style semantic: presence in the
-     * agent output is a LEAK / failure, not a positive signal.
-     *
-     * Falls back to {@link canary} when omitted.
-     */
-    forbiddenPattern?: string;
-    tags?: Record<string, string>;
-}
-interface DatasetProvenance {
-    contributor?: string;
-    createdAt: string;
-    sourceUrl?: string;
-    license?: string;
-    description?: string;
-    /** Monotonic human-readable version (e.g. "2026.04.20"). */
-    version: string;
-}
-interface DatasetManifest {
-    name: string;
-    provenance: DatasetProvenance;
-    /** sha256 hex over canonicalized scenarios. */
-    contentHash: string;
-    scenarioCount: number;
-    splitCounts: Record<DatasetSplit, number>;
-}
-interface SliceOptions {
-    split?: DatasetSplit;
-    difficulty?: DatasetDifficulty;
-    /** Number of scenarios (random sample, seeded). Omit to take all that match. */
-    limit?: number;
-    seed?: number;
-    /** Predicate narrowing. Applied after split/difficulty filters. */
-    filter?: (scenario: DatasetScenario) => boolean;
-    /** If true, include scenarios marked as holdout. Default false. */
-    includeHoldout?: boolean;
-}
-
-/** Locked holdouts — throws on mutate. Callers that need a mutable dataset fork it. */
-declare class HoldoutLockedError extends ValidationError {
-    constructor(datasetName: string);
-}
-declare class Dataset {
-    readonly name: string;
-    readonly provenance: DatasetProvenance;
-    private scenarios;
-    private locked;
-    constructor(init: {
-        name: string;
-        provenance: DatasetProvenance;
-        scenarios: DatasetScenario[];
-        locked?: boolean;
-    });
-    /** All scenarios. Readonly — callers must go through `slice` or `clone`. */
-    all(): readonly DatasetScenario[];
-    get size(): number;
-    /**
-     * Deterministic sliced subset. Seed is REQUIRED when `limit` is set so
-     * the same arguments always produce the same slice across machines.
-     */
-    slice(options?: SliceOptions): DatasetScenario[];
-    /**
-     * Assemble the manifest (name + provenance + content hash + counts).
-     * Content hash is deterministic over canonicalized scenarios.
-     */
-    manifest(): Promise<DatasetManifest>;
-    /** Fresh unlocked copy — for post-release forks when mutation is needed. */
-    clone(overrides?: Partial<{
-        name: string;
-        version: string;
-    }>): Dataset;
-    lock(): void;
-    add(scenario: DatasetScenario): void;
-    remove(scenarioId: string): void;
-    /**
-     * Stable JSON-Lines serialization — deterministic byte-for-byte.
-     * Write to disk for contamination-verifiable archives.
-     */
-    toJsonl(): string;
-    static fromJsonl(jsonl: string, manifest: Omit<DatasetManifest, 'contentHash' | 'scenarioCount' | 'splitCounts'>): Dataset;
-}
-declare function hashScenarios(scenarios: DatasetScenario[]): Promise<string>;
-
-export { type DatasetScenario as D, HoldoutLockedError as H, type SliceOptions as S, type DatasetSplit as a, type DatasetManifest as b, Dataset as c, type DatasetDifficulty as d, type DatasetProvenance as e, hashScenarios as h };