@tangle-network/agent-eval 0.125.0 → 0.126.1

package/dist/benchmarks/index.d.ts

@@ -1,31 +1,3 @@
-type AgentProfileCellSchemaVersion = 'agent-profile-cell/v1';
-type AgentProfileJsonObject = {
-    [key: string]: AgentProfileJson;
-};
-type AgentProfileJson = string | number | boolean | null | AgentProfileJson[] | AgentProfileJsonObject;
-type AgentProfileDimensionValue = string | number | boolean | null;
-interface AgentProfileSource {
-    /** Runtime/profile contract being fingerprinted, e.g. `agent-interface-profile`. */
-    kind: string;
-    /** sha256 over the canonical source profile object. */
-    hash: string;
-}
-interface AgentProfileHarness {
-    id: string;
-    version?: string;
-    hash?: string;
-}
-interface AgentProfileCell {
-    schemaVersion: AgentProfileCellSchemaVersion;
-    cellId: string;
-    profileId: string;
-    sourceProfile: AgentProfileSource;
-    harness?: AgentProfileHarness;
-    model?: string;
-    promptHash?: string;
-    dimensions?: Record<string, AgentProfileDimensionValue>;
-}
-
 type CostChannel = 'agent' | 'judge' | 'verifier' | 'analyst' | 'driver' | (string & {});
 interface CostUsage {
     inputTokens: number;
@@ -53,29 +25,40 @@ interface CostReceipt extends CostCallBase, CostUsage {
     costUsd: number;
     costUnknown: boolean;
     usageUnknown?: boolean;
+    /** Rates used to estimate cost locally. Absent when cost is provider-reported or unknown. */
     pricing?: {
         inputUsdPerThousand: number;
+        cachedInputUsdPerThousand?: number;
+        cacheWriteUsdPerThousand?: number;
         outputUsdPerThousand: number;
     };
+    /** Cost reported by the provider, not a local token-price calculation. */
     actualCostUsd?: number;
     error?: string;
 }
 interface CostReceiptInput extends CostUsage {
     model: string;
+    /** Caller-supplied rates for a local estimate when the provider does not report billed cost. */
+    customTokenPricing?: CustomTokenPricing;
     actualCostUsd?: number;
     costUnknown?: boolean;
     usageUnknown?: boolean;
 }
 /** Per-million token rates for a model or endpoint not covered by package pricing. */
 interface CustomTokenPricing {
+    /** Non-cached input tokens. */
     inputUsdPerMillion: number;
+    /** Cache-read tokens. Falls back to the normal input rate when omitted. */
+    cachedInputUsdPerMillion?: number;
+    /** Cache-creation or cache-write tokens. Falls back to the normal input rate when omitted. */
+    cacheWriteUsdPerMillion?: number;
     outputUsdPerMillion: number;
 }
 type MaximumCharge = {
     externallyEnforcedMaximumUsd: number;
 } | ({
     customTokenPricing: CustomTokenPricing;
-} & Pick<CostUsage, 'inputTokens' | 'outputTokens'>) | ({
+} & Pick<CostUsage, 'inputTokens' | 'outputTokens' | 'cachedTokens' | 'cacheWriteTokens'>) | ({
     model: string;
 } & CostUsage);
 interface RunPaidCallInput<T> {
@@ -136,46 +119,6 @@ interface CostLedgerSummary {
     incompleteReasons: string[];
 }
 
-/**
- * Paper-grade RunRecord schema + runtime validator.
- *
- * Every run that participates in a promotion gate, paper table, or
- * researcher loop SHOULD be recorded as a `RunRecord`. The mandatory
- * fields are exactly those the paper "Two Loops, Three Roles" requires
- * for reproducibility: who/what/when/cost/seed/hash, plus the search vs
- * holdout split tag and either a `searchScore` or a `holdoutScore`.
- *
- * This is intentionally NOT a replacement for the rich `Run` /
- * `ProposeReviewReport` / `ScenarioResult` types already in the
- * package. Those are runtime structures with full provenance. A
- * `RunRecord` is the analysis-time projection — the JSON-friendly
- * row you'd put in a parquet file or paste into a notebook.
- *
- * Validate at the boundary:
- *
- *   const rec = validateRunRecord(rawJson)         // throws on missing
- *   const ok  = isRunRecord(rawJson)               // boolean check
- *   const rec = parseRunRecordSafe(rawJson)        // { ok, value | error }
- *
- * The validator runs in pure TS — zod is intentionally NOT a
- * dependency. Round-trip tested in `tests/run-record.test.ts`.
- */
-
-/** Search/dev/holdout split tag. 'search' is the paper-grade alias for the
- *  combined train+test pool that the optimizer is allowed to read. */
-type RunSplitTag = 'search' | 'dev' | 'holdout';
-interface RunTokenUsage {
-    input: number;
-    /** All generated tokens charged as output, including reasoning tokens. */
-    output: number;
-    /** Reasoning-token subset of `output`, when the provider reports it. */
-    reasoning?: number;
-    /** Prompt tokens served from a provider cache. */
-    cached?: number;
-    /** Prompt tokens written into a provider cache. */
-    cacheWrite?: number;
-}
-
 /**
  * LLM client with graceful degrade.
  *
@@ -244,101 +187,43 @@ interface LlmCallResult {
 type LlmCallMetadata = Pick<LlmCallResult, 'usage' | 'costUsd' | 'model' | 'durationMs'>;
 
 /**
- * Analyst contract — the missing orchestration layer over agent-eval's
- * existing analyzers (analyzeTraces, MultiLayerVerifier, RunCritic,
- * SemanticConceptJudge, JudgeFn, ...).
+ * Paper-grade RunRecord schema + runtime validator.
+ *
+ * Every run that participates in a promotion gate, paper table, or
+ * researcher loop SHOULD be recorded as a `RunRecord`. The mandatory
+ * fields are exactly those the paper "Two Loops, Three Roles" requires
+ * for reproducibility: who/what/when/cost/seed/hash, plus the search vs
+ * holdout split tag and either a `searchScore` or a `holdoutScore`.
+ *
+ * This is intentionally NOT a replacement for the rich `Run` /
+ * `ProposeReviewReport` / `ScenarioResult` types already in the
+ * package. Those are runtime structures with full provenance. A
+ * `RunRecord` is the analysis-time projection — the JSON-friendly
+ * row you'd put in a parquet file or paste into a notebook.
  *
- * Each existing primitive returns its own output shape. The Analyst
- * contract is the single envelope every primitive lifts into, so a
- * registry can run N analysts against a run and a single renderer can
- * compose findings without knowing which analyzer produced them.
+ * Validate at the boundary:
+ *
+ *   const rec = validateRunRecord(rawJson)         // throws on missing
+ *   const ok  = isRunRecord(rawJson)               // boolean check
+ *   const rec = parseRunRecordSafe(rawJson)        // { ok, value | error }
  *
- * The contract is intentionally domain-agnostic: nothing here knows
- * about code, voice, RAG, or any particular agent stack. Analysts
- * declare what INPUT KIND they need (a trace store, an artifact dir,
- * a RunRecord, a JudgeInput, or `custom`), and the registry routes
- * the matching input from `AnalystRunInputs`.
+ * The validator runs in pure TS — zod is intentionally NOT a
+ * dependency. Round-trip tested in `tests/run-record.test.ts`.
  */
 
-interface EvidenceRef {
-    /**
-     * Where the evidence lives. `span` and `event` refer to OTLP trace
-     * elements; `artifact` to a file inside the run's artifact tree;
-     * `finding` to another AnalystFinding (cross-analyst chaining);
-     * `metric` to a named scalar reading the renderer knows how to read.
-     */
-    kind: 'span' | 'event' | 'artifact' | 'finding' | 'metric';
-    uri: string;
-    excerpt?: string;
-}
-
-type PolicyEditSchemaVersion = 'policy-edit/v1';
-declare const POLICY_EDIT_AXES: readonly ["carrier", "representation", "budget", "sampling", "output_contract", "tool_contract", "routing", "memory", "agent_profile", "deployment_target"];
-type PolicyEditAxis = (typeof POLICY_EDIT_AXES)[number];
-declare const POLICY_EDIT_TARGET_SURFACES: readonly ["prompt", "tool-contract", "runtime-config", "memory", "agent-profile", "code", "deployment"];
-type PolicyEditTargetSurface = (typeof POLICY_EDIT_TARGET_SURFACES)[number];
-type PolicyEditRisk = 'low' | 'medium' | 'high' | 'unknown';
-type PolicyEditGainDirection = 'increase' | 'decrease';
-type PolicyEditGainUnit = 'absolute' | 'relative' | 'percent' | 'score';
-interface PolicyEditTarget {
-    surface: PolicyEditTargetSurface;
-    /** Stable path inside the target surface, for example `system-prompt:tools`
-     * or `budget.maxTurns`. */
-    path?: string;
-    /** Optional canonical deployment identity. Store the existing cell, not a
-     * local profile shape. */
-    agentProfileCell?: AgentProfileCell;
-    /** Human label when the path is not enough for a readable audit trail. */
-    label?: string;
-}
-type PolicyEditChange = {
-    kind: 'text';
-    mode: 'append' | 'prepend' | 'replace';
-    value: string;
-    /** Required when `mode === 'replace'`; exact match only. */
-    find?: string;
-} | {
-    kind: 'json';
-    mode: 'set' | 'merge' | 'remove';
-    path: string;
-    value?: AgentProfileJson;
-};
-interface PolicyEditExpectedGain {
-    /** Metric this edit is expected to move, e.g. `holdout.composite`. */
-    metric: string;
-    direction: PolicyEditGainDirection;
-    /** Positive magnitude in the metric's native units. */
-    amount: number;
-    unit?: PolicyEditGainUnit;
-    rationale?: string;
-}
-interface PolicyEditSource {
-    findingIds: string[];
-    analystIds: string[];
-    evidenceRefs: EvidenceRef[];
-    /** Mirrors `AnalystFinding.derived_from_judge`; admission rejects it. */
-    derivedFromJudge?: boolean;
-}
-interface PolicyEdit {
-    schemaVersion: PolicyEditSchemaVersion;
-    editId: string;
-    axis: PolicyEditAxis;
-    target: PolicyEditTarget;
-    change: PolicyEditChange;
-    claim: string;
-    expectedGain: PolicyEditExpectedGain;
-    confidence: number;
-    risk: PolicyEditRisk;
-    source: PolicyEditSource;
-    rationale?: string;
-    validationPlan?: string;
-    metadata?: Record<string, unknown>;
-}
-declare const POLICY_EDIT_CANDIDATE_RECORD_SCHEMA: "tangle.policy-edit-candidate.v1";
-/** JSON-safe attribution carried with a measured candidate and its scores. */
-interface PolicyEditCandidateRecord {
-    schema: typeof POLICY_EDIT_CANDIDATE_RECORD_SCHEMA;
-    policyEdit: PolicyEdit;
+/** Search/dev/holdout split tag. 'search' is the paper-grade alias for the
+ *  combined train+test pool that the optimizer is allowed to read. */
+type RunSplitTag = 'search' | 'dev' | 'holdout';
+interface RunTokenUsage {
+    input: number;
+    /** All generated tokens charged as output, including reasoning tokens. */
+    output: number;
+    /** Reasoning-token subset of `output`, when the provider reports it. */
+    reasoning?: number;
+    /** Prompt tokens served from a provider cache. */
+    cached?: number;
+    /** Prompt tokens written into a provider cache. */
+    cacheWrite?: number;
 }
 
 /**
@@ -569,8 +454,6 @@ interface GenerationCandidate {
      *  "because rationale Z" the audit requires to survive to the result.
      *  Present when the proposer returned a `ProposedCandidate`. */
     rationale?: string;
-    /** Exact structured cause threaded from the proposer, when available. */
-    candidateRecord?: PolicyEditCandidateRecord;
 }
 interface CampaignAggregates {
     byJudge: Record<string, JudgeAggregate>;

package/dist/benchmarks/index.js

@@ -16,23 +16,22 @@ import {
   routing_exports,
   runBenchmarkAdapter,
   summarizeBenchmarkCampaign
-} from "../chunk-JKDNAOF5.js";
-import "../chunk-A62YMFWA.js";
-import "../chunk-PMITBABE.js";
-import "../chunk-D5JZ7UDZ.js";
-import "../chunk-DT7OXY3C.js";
+} from "../chunk-W4L6C2XT.js";
+import "../chunk-NGUYT5CI.js";
+import "../chunk-VMUENW6F.js";
+import "../chunk-UCLVDLCH.js";
+import "../chunk-WGXIEX7P.js";
 import "../chunk-ARU2PZFM.js";
-import "../chunk-EQUK3RFS.js";
+import "../chunk-J5SQWP6Y.js";
 import "../chunk-PJQFMIOX.js";
-import "../chunk-GRCDRKII.js";
+import "../chunk-WS3NZZQQ.js";
 import "../chunk-VI2UW6B6.js";
 import "../chunk-NUKSVU3W.js";
 import "../chunk-GGE4NNQT.js";
 import "../chunk-IR3KBHOY.js";
 import "../chunk-PC4UYEBM.js";
-import "../chunk-LOW3U7JZ.js";
+import "../chunk-IILEIWGW.js";
 import "../chunk-MA6HLL3S.js";
-import "../chunk-GC4ATIKK.js";
 import "../chunk-VSMTAMNK.js";
 import "../chunk-ONWEPEDO.js";
 import "../chunk-K4DBDHLK.js";