@tangle-network/agent-eval 0.124.0 → 0.126.0

package/dist/rollout/index.d.ts

@@ -876,8 +876,16 @@ interface ClaudeTranscript {
     endedAt: string | null;
     model: string | null;
 }
+interface ReadClaudeTranscriptOptions {
+    /**
+     * Read the sidechain (subagent) thread instead of skipping it. Subagent
+     * transcripts under `<session>/subagents/agent-<id>.jsonl` are sidechain
+     * lines end to end, so their usage is invisible without this.
+     */
+    readonly includeSidechain?: boolean;
+}
 /** Parse one transcript jsonl into canonical messages + usage totals. */
-declare function readClaudeTranscript(path: string): Promise<ClaudeTranscript>;
+declare function readClaudeTranscript(path: string, options?: ReadClaudeTranscriptOptions): Promise<ClaudeTranscript>;
 
 /**
  * Read-only backfill reader over the opencode sqlite store

package/dist/rollout/index.js

@@ -1,18 +1,18 @@
 import {
-  DEFAULT_CLAUDE_PROJECTS_DIR,
-  claudeProjectSlug,
-  findClaudeTranscripts,
   mintRolloutRows,
-  readClaudeTranscript,
   rolloutReward
-} from "../chunk-MGGFVCJ7.js";
+} from "../chunk-M7AH34KV.js";
 import {
+  DEFAULT_CLAUDE_PROJECTS_DIR,
   DEFAULT_OPENCODE_DB,
+  claudeProjectSlug,
+  findClaudeTranscripts,
   findOpencodeSessionById,
   findOpencodeSessionsByDirectory,
   openOpencodeDb,
+  readClaudeTranscript,
   readOpencodeSessionMessages
-} from "../chunk-R7ZRE2KV.js";
+} from "../chunk-VBQ3CRKH.js";
 import {
   FORMAT_FILES,
   RELEASE_FORMATS,

package/dist/{run-campaign-I3JXKVAK.js → run-campaign-LVFKZCEU.js}

@@ -1,9 +1,9 @@
 import {
   planCampaignRun,
   runCampaign
-} from "./chunk-D5JZ7UDZ.js";
+} from "./chunk-UCLVDLCH.js";
 import "./chunk-PJQFMIOX.js";
-import "./chunk-GRCDRKII.js";
+import "./chunk-WS3NZZQQ.js";
 import "./chunk-VI2UW6B6.js";
 import "./chunk-ONWEPEDO.js";
 import "./chunk-PZ5AY32C.js";
@@ -11,4 +11,4 @@ export {
   planCampaignRun,
   runCampaign
 };
-//# sourceMappingURL=run-campaign-I3JXKVAK.js.map
+//# sourceMappingURL=run-campaign-LVFKZCEU.js.map

package/dist/supervisor-run/index.d.ts

@@ -225,7 +225,37 @@ interface WorkerLogSource {
     readonly inbox: string | null;
     /** Worker patch byte length, or null when absent. */
     readonly patchBytes: number | null;
+    /** Where this worker's transcript lives, for the rollout row. Null = no such artifact. */
+    readonly transcriptRef?: string | null;
+    /** Where this worker's delivered patch lives. Null = the store keeps no patch per worker. */
+    readonly patchPath?: string | null;
+    /** This worker's own inference tokens, when the store records them per worker. */
+    readonly tokensIn?: number | null;
+    readonly tokensOut?: number | null;
+    readonly cacheRead?: number | null;
+    readonly cacheWrite?: number | null;
 }
+/**
+ * Facts a SOURCE structurally cannot express, each with the reason.
+ *
+ * The difference between "the artifact is missing" and "this store never
+ * records that fact" is the difference between a run that spent $0 and a
+ * harness that does not price inference — and the second harness is where a
+ * loops-shaped assumption becomes a fabricated zero. A reader declares its
+ * limits once; the analyzer reports `unavailable` for everything downstream.
+ *
+ * `null` on a field means the source DOES carry that fact.
+ */
+interface SourceLimits {
+    /** Reason inference spend has no price in this store (null = the store prices it). */
+    readonly spendUsd: string | null;
+    /** Reason workers carry no pass/fail verdict (null = verdicts are recorded). */
+    readonly workerVerdicts: string | null;
+    /** Reason no delivered artifact (patch/diff) is retained per worker (null = retained). */
+    readonly deliverables: string | null;
+}
+/** A source that carries every fact the analyzer can use. */
+declare const NO_SOURCE_LIMITS: SourceLimits;
 /**
  * Everything the pure analyzer reads — already-read bytes, never paths. Each
  * field is `null` when its artifact was absent, which is what turns the
@@ -279,8 +309,24 @@ interface SupervisorRunSources {
         sessions: number;
         input: number;
         output: number;
+        /** Cached prompt tokens, when the store counts them separately. */
+        cacheRead?: number;
+        cacheWrite?: number;
     } | null;
     readonly harnessMissingReason: string | null;
+    /** What this store structurally cannot record. See `SourceLimits`. */
+    readonly limits: SourceLimits;
+    /**
+     * Where the ROOT invocation's transcript lives. Undefined lets the rollout
+     * minter fall back to the loops layout (`<supRunDir>/journal.jsonl`); any
+     * other store must say, or the row points at a path that never existed.
+     */
+    readonly rootTranscriptRef?: string | null;
+    /**
+     * The `traces` CLI command that covers this run's harness-session layer.
+     * Null falls back to the analyzer's default (an opencode worker fleet).
+     */
+    readonly traceCommand: string | null;
 }
 /**
  * A source of supervisor-run bytes. Implementations own their storage layout;
@@ -352,15 +398,23 @@ interface DecisionMetrics {
 interface RoleSpend {
     readonly tokensIn: Measured<number>;
     readonly tokensOut: Measured<number>;
+    /**
+     * Cached prompt tokens read/written. On a harness that caches aggressively
+     * these dwarf `tokensIn`, so a report that omits them understates the context
+     * each invocation actually consumed. `unavailable` = the store has no such counter.
+     */
+    readonly cacheRead: Measured<number>;
+    readonly cacheWrite: Measured<number>;
     readonly usd: Measured<number>;
     readonly source: string;
 }
 interface PerWorkerRow {
     readonly worker: string;
     readonly wallMs: number | null;
-    readonly tokensIn: number;
-    readonly tokensOut: number;
-    readonly usd: number;
+    /** `null` = this store does not attribute tokens per worker (NOT "zero tokens"). */
+    readonly tokensIn: number | null;
+    readonly tokensOut: number | null;
+    readonly usd: number | null;
     readonly patchBytes: number | null;
     readonly passed: boolean | null;
 }
@@ -482,6 +536,10 @@ interface SupervisorRunTree {
 interface Tokens {
     input: number;
     output: number;
+    cacheRead: number;
+    cacheWrite: number;
+    /** False when the event carried no cache counters at all — not "zero cached". */
+    hasCache: boolean;
 }
 interface SpendLike {
     tokens: Tokens;
@@ -500,6 +558,8 @@ interface CloseRow {
     verdict: string | null;
     at: number | null;
     spend: SpendLike;
+    /** False when the close event carried no spend object — not "spent nothing". */
+    hasSpend: boolean;
 }
 interface WorkerLogFacts {
     started: number | null;
@@ -527,6 +587,10 @@ interface SupervisorTreeFacts {
     readonly brain: {
         tokensIn: number;
         tokensOut: number;
+        cacheRead: number;
+        cacheWrite: number;
+        /** False when no metered event carried cache counters — not "nothing cached". */
+        hasCache: boolean;
         usd: number;
         meteredCount: number;
     };
@@ -549,6 +613,94 @@ declare function parsePatch(text: string): PatchStats;
  */
 declare function rollupSupervisorRuns(reports: readonly SupervisorRunReport[]): SupervisorRunRollup;
 
+/**
+ * Supervision-tree reader over a THIRD-PARTY harness: Claude Code.
+ *
+ * `loops-reader.ts` reads a supervisor we wrote, whose journal was designed
+ * for this analysis. This reader reads a harness we do not control, whose
+ * transcript was designed for replaying a chat — and recovers the same tree
+ * from it. If both produce a `SupervisorRunSources`, the tree model is a
+ * property of multi-agent runs, not of our journal format.
+ *
+ * ## Where the tree hides in a Claude Code transcript
+ *
+ * | Tree fact | Claude Code evidence |
+ * |---|---|
+ * | spawn      | assistant `tool_use` (`Agent` / `Task`), answered by a `tool_result` whose `toolUseResult.agentId` names the child |
+ * | settle     | a `<task-notification>` block in a later user line: `<task-id>` = agentId, `<status>` |
+ * | steer      | assistant `tool_use` (`SendMessage`) with `input.to` = agentId — mid-task, to a LIVE child |
+ * | delivered  | that steer's `tool_result` carrying `success` / `resumedAgentId` |
+ * | cancel     | assistant `tool_use` (`TaskStop`) targeting an agentId |
+ * | brain spend| `message.usage` on the main thread's assistant lines |
+ * | worker spend| `message.usage` inside `<session>/subagents/agent-<id>.jsonl` |
+ * | depth      | a child transcript that itself contains `Agent` tool_use lines |
+ *
+ * Every one of those is read through `parseClaudeEntries` — the SAME line
+ * parser `src/rollout/readers/claude-jsonl.ts` uses for solo rollouts. There
+ * is no second transcript parser.
+ *
+ * ## What Claude Code cannot say
+ *
+ * It records tokens but never a price, runs no per-worker verify, and keeps no
+ * per-worker patch. Those are declared once in `limits`, so the analyzer
+ * reports `unavailable — <reason>` instead of the $0 / 0-accepted that summing
+ * an empty field would produce. See `SourceLimits`.
+ *
+ * ## Metric coverage vs the loops journal
+ *
+ * Measured on a real 52-agent session (fixture:
+ * `tests/fixtures/supervisor-run/claude-code-session-*`).
+ *
+ * | Metric | loops | Claude Code | Why |
+ * |---|---|---|---|
+ * | workersSpawned / Settled / Cancelled | full | full | spawn tool_use + task-notification + TaskStop |
+ * | steers / steersDelivered / steersByWorker | full | full | `SendMessage`; delivery from its tool_result |
+ * | waves / waveSizes / maxConcurrency | full | full | derived from spawn/settle instants |
+ * | respawns / repeatedLabels | full | full | same derivation |
+ * | delegationDepth | full | full | a child transcript's own spawn calls |
+ * | timeToFirstSpawn / supervisorWall | full | full | transcript instants |
+ * | idleMs / idlePct / workerUtilization | full | PARTIAL | an agent that never notifies is counted live to the end of the transcript |
+ * | observeThenRespawn / respawnWithoutEvidence | full | full | ordering of spawn vs settle instants |
+ * | workerEvidenceBytes | full | PARTIAL | the child's closing message; 0 for pruned transcripts |
+ * | brain tokens in/out + cache | full | full | main-thread `message.usage` |
+ * | worker tokens in/out + cache | via harness join | PARTIAL | only for retained subagent transcripts |
+ * | perWorker wall | full | full | spawn → settle instants |
+ * | accepted / rejected / emptyPass / settledVerdicts | full | NONE | no per-worker verify step exists |
+ * | brain/worker/total usd, costPerAcceptedPatch | full | NONE | transcripts carry no price |
+ * | patch stats, delivered, verifyPass/Rc | full | NONE | no diff is handed back |
+ * | judgeResolved / Score / Passed / Total | full | NONE | no judge in the loop |
+ * | driverSteerCalls, brainTruncations | full | NONE | no outer driver log, no per-call finish_reason tap |
+ */
+
+/** Tool names that spawn a child agent. `Task` is the older name for `Agent`. */
+declare const DEFAULT_SPAWN_TOOLS: readonly ["Agent", "Task"];
+/** Tool names that deliver a message to an ALREADY-RUNNING child agent. */
+declare const DEFAULT_STEER_TOOLS: readonly ["SendMessage"];
+/** Tool names that stop a running child agent. */
+declare const DEFAULT_CANCEL_TOOLS: readonly ["TaskStop", "KillAgent"];
+interface ClaudeCodeReaderOptions {
+    /** The main session transcript: `~/.claude/projects/<slug>/<sessionId>.jsonl`. */
+    readonly transcriptPath: string;
+    /**
+     * Directory of child transcripts. Defaults to `<transcript-dir>/<sessionId>/subagents`.
+     * `null` skips the join, and every per-worker token count becomes unavailable.
+     */
+    readonly subagentsDir?: string | null;
+    readonly runRef?: string;
+    readonly instanceId?: string | null;
+    readonly arm?: string | null;
+    readonly spawnTools?: readonly string[];
+    readonly steerTools?: readonly string[];
+    readonly cancelTools?: readonly string[];
+}
+/**
+ * Read a Claude Code session (plus its subagent transcripts) as supervision-tree
+ * source bytes. Never throws on a missing artifact.
+ */
+declare function readClaudeCodeSupervisorRun(opts: ClaudeCodeReaderOptions): Promise<SupervisorRunSources>;
+/** A `SupervisorRunReader` over a Claude Code session — the same contract loops implements. */
+declare function claudeCodeSupervisorRunReader(opts: ClaudeCodeReaderOptions): SupervisorRunReader;
+
 /**
  * ONE implementation of `SupervisorRunReader`: the on-disk layout the loops
  * supervisor writes — `<runDir>/ws/.loops/supervisor/<id>/{journal.jsonl,
@@ -702,4 +854,4 @@ interface SupervisorRolloutOptions {
  */
 declare function supervisorRunRolloutLines(src: SupervisorRunSources, opts?: SupervisorRolloutOptions): SupervisorRunTree;
 
-export { type CloseRow, type DecisionMetrics, type EconomicsMetrics, type LoopsReaderOptions, type Measured, type OrchestrationMetrics, type OutcomeMetrics, type PatchStats, type PerWorkerRow, type RoleSpend, type RollupCellRow, SUPERVISOR_RUN_ROLLUP_SCHEMA, SUPERVISOR_RUN_SCHEMA, type SpawnRow, type SteerBreakdown, type SupervisorRolloutOptions, type SupervisorRunReader, type SupervisorRunReport, type SupervisorRunRollup, type SupervisorRunSources, type SupervisorRunTree, type SupervisorTreeFacts, type Unavailable, type WallDistribution, type WorkerLogFacts, type WorkerLogSource, type WriteSupervisorRunOptions, analyzeSupervisorRun, analyzeSupervisorRunSources, findSupervisorRunDirIn, findSupervisorRunDirs, isUnavailable, loopsSupervisorRunReader, parsePatch, parseSupervisorTree, readLoopsSupervisorRun, renderSupervisorRollupMarkdown, renderSupervisorRunHeadline, renderSupervisorRunMarkdown, reportSupervisorRound, rollupSupervisorRuns, showMeasured, supervisorReportStem, supervisorRunRolloutLines, unavailable, writeSupervisorRunReport, writeSupervisorRunReportSafe };
+export { type ClaudeCodeReaderOptions, type CloseRow, DEFAULT_CANCEL_TOOLS, DEFAULT_SPAWN_TOOLS, DEFAULT_STEER_TOOLS, type DecisionMetrics, type EconomicsMetrics, type LoopsReaderOptions, type Measured, NO_SOURCE_LIMITS, type OrchestrationMetrics, type OutcomeMetrics, type PatchStats, type PerWorkerRow, type RoleSpend, type RollupCellRow, SUPERVISOR_RUN_ROLLUP_SCHEMA, SUPERVISOR_RUN_SCHEMA, type SourceLimits, type SpawnRow, type SteerBreakdown, type SupervisorRolloutOptions, type SupervisorRunReader, type SupervisorRunReport, type SupervisorRunRollup, type SupervisorRunSources, type SupervisorRunTree, type SupervisorTreeFacts, type Unavailable, type WallDistribution, type WorkerLogFacts, type WorkerLogSource, type WriteSupervisorRunOptions, analyzeSupervisorRun, analyzeSupervisorRunSources, claudeCodeSupervisorRunReader, findSupervisorRunDirIn, findSupervisorRunDirs, isUnavailable, loopsSupervisorRunReader, parsePatch, parseSupervisorTree, readClaudeCodeSupervisorRun, readLoopsSupervisorRun, renderSupervisorRollupMarkdown, renderSupervisorRunHeadline, renderSupervisorRunMarkdown, reportSupervisorRound, rollupSupervisorRuns, showMeasured, supervisorReportStem, supervisorRunRolloutLines, unavailable, writeSupervisorRunReport, writeSupervisorRunReportSafe };

package/dist/supervisor-run/index.js

@@ -1,14 +1,20 @@
 import {
+  DEFAULT_CANCEL_TOOLS,
+  DEFAULT_SPAWN_TOOLS,
+  DEFAULT_STEER_TOOLS,
+  NO_SOURCE_LIMITS,
   SUPERVISOR_RUN_ROLLUP_SCHEMA,
   SUPERVISOR_RUN_SCHEMA,
   analyzeSupervisorRun,
   analyzeSupervisorRunSources,
+  claudeCodeSupervisorRunReader,
   findSupervisorRunDirIn,
   findSupervisorRunDirs,
   isUnavailable,
   loopsSupervisorRunReader,
   parsePatch,
   parseSupervisorTree,
+  readClaudeCodeSupervisorRun,
   readLoopsSupervisorRun,
   renderSupervisorRollupMarkdown,
   renderSupervisorRunHeadline,
@@ -21,21 +27,27 @@ import {
   unavailable,
   writeSupervisorRunReport,
   writeSupervisorRunReportSafe
-} from "../chunk-4Y7AAATF.js";
-import "../chunk-R7ZRE2KV.js";
+} from "../chunk-LKKT3IVV.js";
+import "../chunk-VBQ3CRKH.js";
 import "../chunk-MAX3TN3C.js";
 import "../chunk-PZ5AY32C.js";
 export {
+  DEFAULT_CANCEL_TOOLS,
+  DEFAULT_SPAWN_TOOLS,
+  DEFAULT_STEER_TOOLS,
+  NO_SOURCE_LIMITS,
   SUPERVISOR_RUN_ROLLUP_SCHEMA,
   SUPERVISOR_RUN_SCHEMA,
   analyzeSupervisorRun,
   analyzeSupervisorRunSources,
+  claudeCodeSupervisorRunReader,
   findSupervisorRunDirIn,
   findSupervisorRunDirs,
   isUnavailable,
   loopsSupervisorRunReader,
   parsePatch,
   parseSupervisorTree,
+  readClaudeCodeSupervisorRun,
   readLoopsSupervisorRun,
   renderSupervisorRollupMarkdown,
   renderSupervisorRunHeadline,

package/dist/traces.js

@@ -27,7 +27,7 @@ import {
   scoreTraceInsightReadiness,
   tokenizeDomainWords,
   traceAnalystOnRunComplete
-} from "./chunk-QFQZ3U3X.js";
+} from "./chunk-OCFJACJU.js";
 import "./chunk-H5UD2323.js";
 import {
   extractUsage,
@@ -91,7 +91,7 @@ import {
   TraceEmitter,
   llmSpanFromProvider
 } from "./chunk-VQMK5FMP.js";
-import "./chunk-LOW3U7JZ.js";
+import "./chunk-IILEIWGW.js";
 import {
   FAILURE_CLASSES,
   TRACE_SCHEMA_VERSION,
@@ -101,7 +101,6 @@ import {
   isSandboxSpan,
   isToolSpan
 } from "./chunk-MA6HLL3S.js";
-import "./chunk-GC4ATIKK.js";
 import "./chunk-VSMTAMNK.js";
 import "./chunk-ONWEPEDO.js";
 import {

package/dist/wire/index.d.ts

@@ -37,29 +37,40 @@ interface CostReceipt extends CostCallBase, CostUsage {
     costUsd: number;
     costUnknown: boolean;
     usageUnknown?: boolean;
+    /** Rates used to estimate cost locally. Absent when cost is provider-reported or unknown. */
     pricing?: {
         inputUsdPerThousand: number;
+        cachedInputUsdPerThousand?: number;
+        cacheWriteUsdPerThousand?: number;
         outputUsdPerThousand: number;
     };
+    /** Cost reported by the provider, not a local token-price calculation. */
     actualCostUsd?: number;
     error?: string;
 }
 interface CostReceiptInput extends CostUsage {
     model: string;
+    /** Caller-supplied rates for a local estimate when the provider does not report billed cost. */
+    customTokenPricing?: CustomTokenPricing;
     actualCostUsd?: number;
     costUnknown?: boolean;
     usageUnknown?: boolean;
 }
 /** Per-million token rates for a model or endpoint not covered by package pricing. */
 interface CustomTokenPricing {
+    /** Non-cached input tokens. */
     inputUsdPerMillion: number;
+    /** Cache-read tokens. Falls back to the normal input rate when omitted. */
+    cachedInputUsdPerMillion?: number;
+    /** Cache-creation or cache-write tokens. Falls back to the normal input rate when omitted. */
+    cacheWriteUsdPerMillion?: number;
     outputUsdPerMillion: number;
 }
 type MaximumCharge = {
     externallyEnforcedMaximumUsd: number;
 } | ({
     customTokenPricing: CustomTokenPricing;
-} & Pick<CostUsage, 'inputTokens' | 'outputTokens'>) | ({
+} & Pick<CostUsage, 'inputTokens' | 'outputTokens' | 'cachedTokens' | 'cacheWriteTokens'>) | ({
     model: string;
 } & CostUsage);
 interface RunPaidCallInput<T> {
@@ -127,6 +138,8 @@ interface CostLedgerFilter {
 interface CostLedgerWaitOptions {
     /** Maximum time to wait for active provider calls. Default 5 seconds. */
     timeoutMs?: number;
+    /** Wait only for calls matching this attribution filter. */
+    filter?: CostLedgerFilter;
 }
 /** Append-only storage. `append` must atomically reject stale revisions. */
 interface CostLedgerPersistence {

package/dist/wire/index.js

@@ -34,9 +34,9 @@ import {
   runRpcOnce,
   startServer,
   startServerAsync
-} from "../chunk-HM6V7F3M.js";
-import "../chunk-EQUK3RFS.js";
-import "../chunk-GRCDRKII.js";
+} from "../chunk-FO7HEH76.js";
+import "../chunk-J5SQWP6Y.js";
+import "../chunk-WS3NZZQQ.js";
 import "../chunk-VI2UW6B6.js";
 import "../chunk-PC4UYEBM.js";
 import "../chunk-ONWEPEDO.js";