@tangle-network/agent-eval 0.115.3 → 0.117.0

package/dist/cost-ledger-DWy3XdJc.d.ts

@@ -0,0 +1,183 @@
+import { V as ValidationError } from './errors-oeQrLqXC.js';
+
+type CostChannel = 'agent' | 'judge' | 'verifier' | 'analyst' | 'driver' | (string & {});
+interface CostUsage {
+    inputTokens: number;
+    outputTokens: number;
+    cachedTokens?: number;
+}
+interface CostCallBase {
+    callId: string;
+    channel: CostChannel;
+    phase: string;
+    actor: string;
+    model: string;
+    maximumCostUsd?: number;
+    tags?: Record<string, string>;
+    timestamp: number;
+}
+interface CostReceipt extends CostCallBase, CostUsage {
+    status: 'settled';
+    costUsd: number;
+    costUnknown: boolean;
+    usageUnknown?: boolean;
+    pricing?: {
+        inputUsdPerThousand: number;
+        outputUsdPerThousand: number;
+    };
+    actualCostUsd?: number;
+    error?: string;
+}
+/** @deprecated Read-only compatibility shape. New paid work uses `runPaidCall`. */
+type CostLedgerEntry = Omit<CostReceipt, 'status' | 'callId' | 'phase' | 'actor' | 'maximumCostUsd' | 'usageUnknown' | 'pricing' | 'error'>;
+interface CostReceiptInput extends CostUsage {
+    model: string;
+    actualCostUsd?: number;
+    costUnknown?: boolean;
+    usageUnknown?: boolean;
+}
+type MaximumCharge = {
+    externallyEnforcedMaximumUsd: number;
+} | ({
+    model: string;
+} & CostUsage);
+interface RunPaidCallInput<T> {
+    callId?: string;
+    channel: CostChannel;
+    phase: string;
+    actor: string;
+    /** Used before a provider receipt exists and on failures without one. */
+    model?: string;
+    tags?: Record<string, string>;
+    signal?: AbortSignal;
+    /** Provider-enforced dollar maximum, or maximum priced token usage. Required when capped. */
+    maximumCharge?: MaximumCharge;
+    /** `callId` can be forwarded as the provider's idempotency key. */
+    execute(signal: AbortSignal, callId: string): Promise<T>;
+    receipt(value: T): CostReceiptInput;
+    receiptFromError?(error: Error): CostReceiptInput | undefined;
+}
+type PaidCallResult<T> = {
+    succeeded: true;
+    callId: string;
+    value: T;
+    receipt: CostReceipt;
+} | {
+    succeeded: false;
+    callId?: string;
+    error: Error;
+    receipt?: CostReceipt;
+};
+interface ChannelRollup {
+    channel: CostChannel;
+    calls: number;
+    inputTokens: number;
+    outputTokens: number;
+    cachedTokens: number;
+    costUsd: number;
+    unpricedCalls: number;
+    unknownUsageCalls: number;
+}
+interface CostLedgerSummary {
+    totalCalls: number;
+    pendingCalls: number;
+    unresolvedCalls: number;
+    reservedCostUsd: number;
+    inputTokens: number;
+    outputTokens: number;
+    cachedTokens: number;
+    totalCostUsd: number;
+    byChannel: ChannelRollup[];
+    unpricedModels: string[];
+    fullyPriced: boolean;
+    usageComplete: boolean;
+    accountingComplete: boolean;
+    incompleteReasons: string[];
+}
+interface CostLedgerFilter {
+    channel?: CostChannel;
+    phase?: string;
+    tags?: Record<string, string>;
+}
+/** Append-only storage. `append` must atomically reject stale revisions. */
+interface CostLedgerPersistence {
+    read(): {
+        revision: string;
+        events: string;
+    };
+    append(expectedRevision: string, event: string): string | undefined;
+}
+interface CostLedgerOptions {
+    costCeilingUsd?: number;
+    persistence?: CostLedgerPersistence;
+    /** Import already-settled receipts without admitting new paid work. */
+    receipts?: readonly CostReceipt[];
+}
+declare class CostCeilingReachedError extends ValidationError {
+    constructor(ceilingUsd: number, committedAndReservedUsd: number, requestedUsd: number, phase: string, actor: string);
+}
+declare class CostAccountingIncompleteError extends ValidationError {
+}
+declare class CostReservationExceededError extends ValidationError {
+    constructor(actor: string, actualUsd: number, maximumUsd: number);
+}
+declare class CostCallConflictError extends ValidationError {
+    readonly callId?: string;
+    readonly receipt?: CostReceipt;
+    constructor(message: string, options?: {
+        callId?: string;
+        receipt?: CostReceipt;
+        cause?: unknown;
+    });
+}
+declare class CostLedgerPersistenceError extends ValidationError {
+    readonly callId?: string;
+    readonly receipt?: CostReceipt;
+    constructor(cause: unknown, callId?: string, receipt?: CostReceipt);
+}
+declare class CostReceiptCaptureError extends ValidationError {
+    readonly callId: string;
+    readonly receipt?: CostReceipt;
+    readonly receiptError: Error;
+    constructor(callId: string, cause: unknown, receiptError: unknown, receipt?: CostReceipt);
+}
+/** Run-wide paid-call admission, durable call state, receipts, and summaries. */
+declare class CostLedger {
+    private readonly records;
+    private readonly activeCallIds;
+    private readonly lateCallIds;
+    private completedTasks;
+    private revision;
+    private costLimitPersisted;
+    readonly costCeilingUsd?: number;
+    private readonly persistence?;
+    constructor(input?: number | CostLedgerOptions);
+    runPaidCall<T>(input: RunPaidCallInput<T>): Promise<PaidCallResult<T>>;
+    /** Settle a call left pending by a crashed process after reconciling with the provider. */
+    reconcile(callId: string, observed: CostReceiptInput, options?: {
+        error?: string;
+    }): CostReceipt;
+    list(filter?: CostLedgerFilter): CostReceipt[];
+    summary(filter?: CostLedgerFilter): CostLedgerSummary;
+    markCompleted(count?: number): void;
+    costPerCompletedTask(): number | null;
+    private execute;
+    private captureLateOutcome;
+    private commitOutcome;
+    private captureFailure;
+    private commitReceipt;
+    private resolveMaximum;
+    private hasIncompleteSettledCall;
+    private appendRecord;
+    private ensureCostLimitPersisted;
+    private appendEvent;
+}
+/** Return the canonical pricing-table key, or null when the model is unpriced. */
+declare function modelPriceKey(model: string): string | null;
+interface CostResult {
+    costUsd: number;
+    costUnknown: boolean;
+}
+declare function costForUsage(model: string, usage: CostUsage): CostResult;
+
+export { CostLedger as C, type MaximumCharge as M, type PaidCallResult as P, type RunPaidCallInput as R, type CostChannel as a, type CostLedgerSummary as b, type CostReceiptInput as c, type CostReceipt as d, type ChannelRollup as e, CostAccountingIncompleteError as f, CostCallConflictError as g, CostCeilingReachedError as h, type CostLedgerEntry as i, type CostLedgerFilter as j, type CostLedgerOptions as k, type CostLedgerPersistence as l, CostLedgerPersistenceError as m, CostReceiptCaptureError as n, CostReservationExceededError as o, type CostResult as p, type CostUsage as q, costForUsage as r, modelPriceKey as s };

package/dist/{default-registry-DltpYR5u.d.ts → default-registry-DaK8b3fv.d.ts}

@@ -1,5 +1,6 @@
 import { AxAIService } from '@ax-llm/ax';
-import { a as Analyst, b as AnalystContext, c as AnalystRunSummary, A as AnalystFinding, d as AnalystRunResult, C as ChatClient, e as AnalystRunInputs, f as AnalystRunEvent, T as TraceAnalystKindSpec } from './kind-factory-DcNg13sZ.js';
+import { T as TraceAnalystKindSpec } from './kind-factory-ClZmO25A.js';
+import { A as Analyst, a as AnalystContext, b as AnalystRunSummary, c as AnalystFinding, d as AnalystRunResult, C as ChatClient, e as AnalystRunInputs, f as AnalystRunEvent } from './policy-edit-wG9uFEFm.js';
 
 /**
  * AnalystRegistry — orchestrate N analysts against one run.

package/dist/{emitter-BRchAAAx.d.ts → emitter-CjD7vUwv.d.ts}

@@ -1,5 +1,5 @@
-import { a as RunOutcome, R as Run, S as Span, b as SpanKind, L as LlmSpan, T as ToolSpan, c as RetrievalSpan, J as JudgeSpan, d as SandboxSpan, E as EventKind, e as TraceEvent, B as BudgetLedgerEntry, A as Artifact, M as Message } from './schema-SGWcK9wa.js';
-import { T as TraceStore } from './store-BsVi7ncX.js';
+import { a as RunOutcome, R as Run, S as Span, b as SpanKind, L as LlmSpan, T as ToolSpan, c as RetrievalSpan, J as JudgeSpan, d as SandboxSpan, E as EventKind, e as TraceEvent, B as BudgetLedgerEntry, A as Artifact, M as Message } from './schema-B3Q3l9Z_.js';
+import { T as TraceStore } from './store-DGqD0Pyo.js';
 
 /**
  * TraceEmitter — hierarchical span builder that auto-parents using an

package/dist/{failure-cluster-C48PiReX.d.ts → failure-cluster-DOAcSJ87.d.ts}

@@ -1,5 +1,5 @@
-import { R as Run, S as Span, e as TraceEvent, F as FailureClass } from './schema-SGWcK9wa.js';
-import { T as TraceStore } from './store-BsVi7ncX.js';
+import { R as Run, S as Span, e as TraceEvent, F as FailureClass } from './schema-B3Q3l9Z_.js';
+import { T as TraceStore } from './store-DGqD0Pyo.js';
 
 /**
  * Failure taxonomy — canonical classes + a default classifier.

package/dist/{feedback-trajectory-pDcz1lQ1.d.ts → feedback-trajectory-BUnM58xL.d.ts}

@@ -1,6 +1,6 @@
-import { T as TraceEmitter } from './emitter-BRchAAAx.js';
-import { F as FailureClass } from './schema-SGWcK9wa.js';
-import { T as TraceStore } from './store-BsVi7ncX.js';
+import { T as TraceEmitter } from './emitter-CjD7vUwv.js';
+import { F as FailureClass } from './schema-B3Q3l9Z_.js';
+import { T as TraceStore } from './store-DGqD0Pyo.js';
 import { a as DatasetSplit, D as DatasetScenario } from './dataset-NENEzRgk.js';
 
 /**

package/dist/fuzz.d.ts

@@ -1,6 +1,7 @@
-import { C as CostLedger, a as CostChannel } from './cost-ledger-DuSqlw5B.js';
+import { M as MaximumCharge, C as CostLedger, a as CostChannel } from './cost-ledger-DWy3XdJc.js';
 import { D as DefaultVerdict } from './verdict-C9MlYujm.js';
 import { A as AdversarialMutation } from './adversarial-B7loGVVX.js';
+import './errors-oeQrLqXC.js';
 
 /**
  * Behavior-space exploration — types.
@@ -289,12 +290,11 @@ interface ExploreOptions<S> {
      * every other cost option (`costBudgetUsd` / `ledger` / `onCost`).
      */
     costOf?: (scenario: S, cell: Cell, ev: Evaluation) => RunCost | null;
+    /** Pre-call hard maximum. Required whenever the explorer uses a capped ledger. */
+    maximumChargeOf?: (scenario: S, cell: Cell) => MaximumCharge;
     /**
-     * Hard dollar ceiling on accumulated KNOWN cost (same semantics as the
-     * control-runtime `budget.maxCostUsd`: nonnegative finite, the session stops
-     * once spent ≥ ceiling; no new evaluation starts after that). Unknown-cost
-     * runs do not consume budget — they are reported separately, so the ceiling
-     * is honest about what it can see.
+     * Hard dollar cap. Each evaluation reserves `maximumChargeOf` before it starts;
+     * calls that do not fit are rejected. Unknown totals stop further paid work.
      */
     costBudgetUsd?: number;
     /**
@@ -423,21 +423,13 @@ declare class BehaviorExplorer<S> {
     private consecutiveEvalErrors;
     private stoppedEarly;
     private rngState;
-    /** Accumulated KNOWN dollars — unknown-cost runs never inflate it. */
-    private spentKnownUsd;
-    private costUnknownRuns;
+    private readonly costLedger?;
+    private readonly costPhase;
     constructor(opts: ExploreOptions<S>);
     private rng;
     private binId;
     private allocate;
     private objectiveContext;
-    /** Mirrors control-runtime: stop once accumulated KNOWN cost ≥ the ceiling. */
-    private costExhausted;
-    /** Fold one run's cost in: null counts as unknown (never $0); a known cost
-     *  accrues toward the budget, lands in the ledger, and fires `onCost`. */
-    /** Returns the run's known cost, `null` when tracked-but-unknown, `undefined`
-     *  when cost tracking is not wired — the log row mirrors this exactly. */
-    private recordRunCost;
     /** Elites whose INPUT cell matches — what the proposer mutates/deepens from. */
     private elitesFor;
     /** One allocate → propose → evaluate → gate → archive round. */

package/dist/fuzz.js

@@ -1,6 +1,11 @@
 import {
   varianceBasedCurriculum
 } from "./chunk-VZSRQ272.js";
+import {
+  CostLedger,
+  CostReceiptCaptureError
+} from "./chunk-VCTY3W6J.js";
+import "./chunk-VI2UW6B6.js";
 import {
   ValidationError
 } from "./chunk-ONWEPEDO.js";
@@ -326,6 +331,17 @@ var BehaviorExplorer = class {
         "BehaviorExplorer: costBudgetUsd/ledger/onCost require costOf \u2014 the explorer cannot know run cost without it; supply costOf or drop the cost options"
       );
     }
+    if ((opts.costBudgetUsd !== void 0 || opts.ledger?.costCeilingUsd !== void 0) && !opts.maximumChargeOf) {
+      throw new ValidationError(
+        "BehaviorExplorer: capped cost tracking requires maximumChargeOf before evaluation"
+      );
+    }
+    if (opts.ledger && opts.costBudgetUsd !== void 0 && opts.ledger.costCeilingUsd !== opts.costBudgetUsd) {
+      throw new ValidationError(
+        "BehaviorExplorer: costBudgetUsd must match the supplied CostLedger ceiling"
+      );
+    }
+    if (opts.costOf) this.costLedger = opts.ledger ?? new CostLedger(opts.costBudgetUsd);
     this.cellById = new Map(this.cells.map((c) => [c.id, c]));
     this.objective = opts.objective ?? adversarialObjective(0.5);
     this.threshold = this.objective.threshold ?? 0.5;
@@ -354,9 +370,8 @@ var BehaviorExplorer = class {
   consecutiveEvalErrors = 0;
   stoppedEarly;
   rngState;
-  /** Accumulated KNOWN dollars — unknown-cost runs never inflate it. */
-  spentKnownUsd = 0;
-  costUnknownRuns = 0;
+  costLedger;
+  costPhase = "fuzz.explore";
   // mulberry32 — deterministic per session.
   rng = () => {
     this.rngState = this.rngState + 1831565813 | 0;
@@ -393,37 +408,6 @@ var BehaviorExplorer = class {
       archiveDescriptors: entries.map((e) => e.evaluation.descriptor)
     };
   }
-  /** Mirrors control-runtime: stop once accumulated KNOWN cost ≥ the ceiling. */
-  costExhausted() {
-    return this.opts.costBudgetUsd !== void 0 && this.spentKnownUsd >= this.opts.costBudgetUsd;
-  }
-  /** Fold one run's cost in: null counts as unknown (never $0); a known cost
-   *  accrues toward the budget, lands in the ledger, and fires `onCost`. */
-  /** Returns the run's known cost, `null` when tracked-but-unknown, `undefined`
-   *  when cost tracking is not wired — the log row mirrors this exactly. */
-  recordRunCost(scenario, cell, ev) {
-    if (!this.opts.costOf) return void 0;
-    const cost = this.opts.costOf(scenario, cell, ev);
-    if (cost === null) {
-      this.costUnknownRuns++;
-      return null;
-    }
-    if (typeof cost.usd !== "number" || !Number.isFinite(cost.usd) || cost.usd < 0) {
-      throw new RangeError(
-        `BehaviorExplorer: costOf returned an invalid usd (${String(cost?.usd)}) \u2014 return null when cost is unknown, never a fabricated number`
-      );
-    }
-    this.spentKnownUsd += cost.usd;
-    this.opts.ledger?.record({
-      model: cost.model ?? "unattributed",
-      channel: "agent",
-      usage: { inputTokens: 0, outputTokens: 0 },
-      actualCostUsd: cost.usd,
-      tags: { target: this.opts.target, cell: cell.id }
-    });
-    this.opts.onCost?.({ usd: cost.usd, channel: "agent" });
-    return cost.usd;
-  }
   /** Elites whose INPUT cell matches — what the proposer mutates/deepens from. */
   elitesFor(cellId2) {
     const out = [];
@@ -433,13 +417,12 @@ var BehaviorExplorer = class {
   /** One allocate → propose → evaluate → gate → archive round. */
   async step() {
     const remaining = this.opts.budget - this.runsUsed;
-    if (remaining <= 0 || this.costExhausted() || this.opts.signal?.aborted)
-      return { runs: 0, findings: [] };
+    if (remaining <= 0 || this.opts.signal?.aborted) return { runs: 0, findings: [] };
     const allocations = this.allocate(Math.min(this.perRoundBudget, remaining));
     const newFindings = [];
     let runsThisStep = 0;
     for (const alloc of allocations) {
-      if (this.runsUsed >= this.opts.budget || this.costExhausted() || this.stoppedEarly !== void 0 || this.opts.signal?.aborted)
+      if (this.runsUsed >= this.opts.budget || this.stoppedEarly !== void 0 || this.opts.signal?.aborted)
         break;
       const cell = this.cellById.get(alloc.cellId);
       if (!cell) continue;
@@ -461,16 +444,56 @@ var BehaviorExplorer = class {
       await pMap(
         toEval,
         async (scenario) => {
-          if (this.runsUsed >= this.opts.budget || this.costExhausted() || this.stoppedEarly !== void 0 || this.opts.signal?.aborted)
+          if (this.runsUsed >= this.opts.budget || this.stoppedEarly !== void 0 || this.opts.signal?.aborted)
             return;
           try {
             const startedAt = performance.now();
-            const ev = await this.opts.evaluate(scenario, cell);
+            const paid = this.costLedger ? await this.costLedger.runPaidCall({
+              channel: "agent",
+              phase: this.costPhase,
+              actor: "fuzz.evaluate",
+              tags: { target: this.opts.target, cell: cell.id },
+              signal: this.opts.signal,
+              maximumCharge: this.opts.maximumChargeOf?.(scenario, cell),
+              execute: () => this.opts.evaluate(scenario, cell),
+              receipt: (evaluation) => {
+                const cost = this.opts.costOf(scenario, cell, evaluation);
+                if (cost === null) {
+                  return {
+                    model: "unattributed",
+                    inputTokens: 0,
+                    outputTokens: 0,
+                    costUnknown: true
+                  };
+                }
+                if (!Number.isFinite(cost.usd) || cost.usd < 0) {
+                  throw new RangeError(
+                    `BehaviorExplorer: costOf returned an invalid usd (${String(cost.usd)}) \u2014 return null when cost is unknown, never a fabricated number`
+                  );
+                }
+                return {
+                  model: cost.model ?? "unattributed",
+                  inputTokens: 0,
+                  outputTokens: 0,
+                  actualCostUsd: cost.usd
+                };
+              }
+            }) : void 0;
+            if (paid && !paid.succeeded) {
+              if (paid.error instanceof CostReceiptCaptureError && paid.error.receiptError instanceof RangeError) {
+                throw paid.error.receiptError;
+              }
+              throw paid.error;
+            }
+            const ev = paid ? paid.value : await this.opts.evaluate(scenario, cell);
             const latencyMs = ev.latencyMs ?? performance.now() - startedAt;
             this.runsUsed++;
             runsThisStep++;
             this.consecutiveEvalErrors = 0;
-            const costUsd = this.recordRunCost(scenario, cell, ev);
+            const costUsd = paid ? paid.receipt.costUnknown ? null : paid.receipt.costUsd : void 0;
+            if (paid?.receipt.actualCostUsd !== void 0) {
+              this.opts.onCost?.({ usd: paid.receipt.actualCostUsd, channel: "agent" });
+            }
             const interest = this.objective.interest(ev, this.objectiveContext());
             this.log.push({
               cell,
@@ -535,7 +558,7 @@ var BehaviorExplorer = class {
   /** Loop `step()` until the run or dollar budget is spent, the signal aborts,
    *  or no progress is made. */
   async run() {
-    while (this.runsUsed < this.opts.budget && !this.costExhausted() && this.stoppedEarly === void 0 && !this.opts.signal?.aborted) {
+    while (this.runsUsed < this.opts.budget && this.stoppedEarly === void 0 && !this.opts.signal?.aborted) {
       const { runs } = await this.step();
       if (runs === 0 && this.stoppedEarly === void 0) break;
     }
@@ -548,6 +571,10 @@ var BehaviorExplorer = class {
     return [...this._findings].sort((a, b) => b.interest - a.interest);
   }
   capsule() {
+    const cost = this.costLedger?.summary({
+      phase: this.costPhase,
+      tags: { target: this.opts.target }
+    });
     return buildCapsule({
       target: this.opts.target,
       objective: this.objective.kind,
@@ -558,7 +585,10 @@ var BehaviorExplorer = class {
       findings: this._findings,
       candidateFindings: this.candidateFindings,
       runsUsed: this.runsUsed,
-      cost: this.opts.costOf ? { costUsd: this.spentKnownUsd, costUnknownRuns: this.costUnknownRuns } : void 0,
+      cost: cost ? {
+        costUsd: cost.totalCostUsd,
+        costUnknownRuns: cost.byChannel.reduce((sum, row) => sum + row.unpricedCalls, 0)
+      } : void 0,
       evalErrors: this.evalErrors,
       stoppedEarly: this.stoppedEarly
     });