@tangle-network/agent-eval 0.145.22 → 0.147.0

package/dist/matrix-C-2Qx1Zr.d.ts

@@ -0,0 +1,383 @@
+import { p as CostProvenance } from "./cost-ledger-DbQdN3nO.js";
+import { w as JudgeScore } from "./types-C1Bmeb8X.js";
+import { o as MatrixResult } from "./index-D_P7Ye43.js";
+import { AgentProfile } from "@tangle-network/agent-interface";
+//#region src/multishot/judges.d.ts
+declare const DEFAULT_JUDGE_MODEL = "openai/gpt-4o-mini";
+interface JudgeDimension {
+  /** JSON field name + score key. */
+  key: string;
+  /** Description shown in the judge's user prompt. */
+  description: string;
+}
+interface JudgeConfig<TInput> {
+  /** Display name (for trace + log). */
+  name: string;
+  /** Model used for this judge. */
+  model?: string;
+  /** 0-10 scored dimensions. */
+  dimensions: JudgeDimension[];
+  /** Judge system prompt — sets persona + JSON-only constraint. */
+  systemPrompt: string;
+  /** Build the user prompt from the typed input. Must include "Respond with
+   *  ONLY this JSON: { ... }" listing each dimension key. */
+  buildPrompt: (input: TInput) => string;
+  /** Optional model + api overrides. */
+  apiKey?: string;
+  baseUrl?: string;
+  /** Maximum output tokens for the judge response. Defaults to 1500. */
+  maxTokens?: number;
+}
+interface JudgeRunResult {
+  /** Semantic result; failed scores remain non-throwing so matrix aggregation can exclude them. */
+  score: JudgeScore;
+  /** Cost of the completed call, separate from diagnostic provider metadata. */
+  cost: CostProvenance;
+}
+declare function runJudge<TInput>(judge: JudgeConfig<TInput>, input: TInput): Promise<JudgeRunResult>;
+/** Convenience: stringified dimension list for inclusion in a judge prompt.
+ *  Returns lines like `- audience_fit: Does this match what the audience cares about? (0-10)`. */
+declare function renderDimensions(dims: readonly JudgeDimension[]): string;
+/** Convenience: build the "Respond with ONLY this JSON" footer for a judge prompt. */
+declare function renderJsonFooter(dims: readonly JudgeDimension[]): string;
+//#endregion
+//#region src/multishot/types.d.ts
+interface MultishotMessage {
+  role: 'user' | 'assistant' | 'tool';
+  content: string;
+  toolCallId?: string;
+  toolCalls?: Array<{
+    id: string;
+    name: string;
+    args: Record<string, unknown>;
+  }>;
+}
+interface MultishotArtifact {
+  type: string;
+  turn: number;
+  invocation: {
+    name: string;
+    args: Record<string, unknown>;
+  };
+  content: string;
+}
+interface MultishotResult {
+  transcript: MultishotMessage[];
+  artifacts: MultishotArtifact[];
+  toolCalls: number;
+  durationMs: number;
+  /** Known spend. A subtotal, not a total, when `costProvenance.kind` is
+   *  `uncaptured`. */
+  costUsd: number;
+  /** Origin of `costUsd`. A shot that priced every call reports `estimated`
+   *  or `observed`; a shot with a call the router priced at nothing reports
+   *  `uncaptured`, and the matrix records the cell as under-counted instead of
+   *  presenting the subtotal as a complete estimate.
+   *
+   *  Optional so an engine written before this field keeps working; the matrix
+   *  then judges the cell on judge receipts alone, as it did before. */
+  costProvenance?: CostProvenance;
+}
+interface MultishotToolDefinition {
+  type: 'function';
+  function: {
+    name: string;
+    description: string;
+    parameters: Record<string, unknown>;
+  };
+}
+/** One chat-completion request the multishot loop issues for a single agent
+ *  (or driver) inference step. Mirrors the OpenAI-compat body the loop would
+ *  otherwise POST to the Tangle router. */
+interface MultishotTransportRequest {
+  model: string;
+  messages: Array<Record<string, unknown>>;
+  tools?: MultishotToolDefinition[];
+  temperature?: number;
+  maxTokens?: number;
+  signal?: AbortSignal;
+}
+interface MultishotTransportToolCall {
+  id: string;
+  type: 'function';
+  function: {
+    name: string;
+    arguments: string;
+  };
+}
+interface MultishotTransportResponse {
+  message: {
+    content?: string | null;
+    tool_calls?: MultishotTransportToolCall[];
+  };
+  usage?: {
+    prompt_tokens?: number;
+    completion_tokens?: number;
+  };
+  /** Actual spend for this call. When omitted, the loop meters cost from
+   *  `usage` via the per-model router estimator (estimateRouterCost). */
+  costUsd?: number;
+}
+/** Execution seam for one leg of the multishot loop. When provided, it
+ *  replaces the internal router HTTP call for that leg — the loop still owns
+ *  turn scheduling, tool dispatch, transcript capture, and cost metering.
+ *  agent-eval has no dependency on agent-runtime; adapt agent-runtime's
+ *  resolveAgentBackend (or any sandbox/cli-bridge/router client) into this
+ *  signature product-side. */
+type MultishotTransport = (req: MultishotTransportRequest) => Promise<MultishotTransportResponse>;
+type MultishotToolExecutor = (args: Record<string, unknown>, ctx: {
+  apiKey: string;
+  baseUrl: string;
+  signal?: AbortSignal;
+}) => Promise<{
+  content: string;
+  costUsd: number;
+}>;
+interface MultishotPersona {
+  /** Stable identifier — used for per-cell artifact paths + matrix axis keys. */
+  id: string;
+  /** Per-domain payload (income/profile/voice/etc.) shaped by the consumer. */
+  [k: string]: unknown;
+}
+/**
+ * Persona-shaping callbacks. Both are OPTIONAL: when omitted, the loop derives
+ * them from the `AgentProfile` + persona payload (see `defaultShapeFromProfile`)
+ * so a pure-profile call — `runMultishot({ profile, persona })` — works with no
+ * role-builder functions. Provide callbacks only to override the derived shape.
+ */
+interface MultishotShape<TPersona extends MultishotPersona> {
+  /** Opening user message (turn 0) — the persona's first ask. */
+  buildOpener?: (persona: TPersona) => string;
+  /** System prompt the driver LLM uses to roleplay the persona. Should set
+   *  voice, goals, constraints, time-pressure, and the "never go silent" rule. */
+  buildDriverSystemPrompt?: (persona: TPersona) => string;
+}
+declare class MultishotDriverEmptyError extends Error {
+  readonly turn: number;
+  constructor(turn: number);
+}
+declare class MultishotFatalToolError extends Error {
+  constructor(message: string);
+}
+declare class MultishotShotResultError extends Error {
+  constructor(reason: string);
+}
+/** Contract guard for the value a caller-supplied shot resolves with. The
+ *  matrix writes per-cell artifacts, builds judge inputs, and meters cost from
+ *  this value, so a malformed result must stop the cell instead of scoring a
+ *  degraded one. Two silent degradations this closes: an artifact with no
+ *  `type` matches neither the code nor the content artifact set, so the cell
+ *  scores as though the artifact was never produced; a non-finite `costUsd`
+ *  reaches `summary.totalCostUsd` and makes every cost number NaN.
+ *
+ *  A rejected cell is still billed: the matrix cell reads the shot's own
+ *  `costUsd` when it is a usable amount and declares that spend on the throw,
+ *  so money the shot spent before returning a malformed result stays in the
+ *  cumulative sum the cost ceiling reads. A result whose `costUsd` is itself
+ *  malformed carries no usable amount, and the cell records as `uncaptured`.
+ *
+ *  Every required field of `MultishotMessage` and `MultishotArtifact` is
+ *  checked, including `toolCalls` elements and `invocation.args`. Optional
+ *  fields are checked only when present. */
+declare function assertMultishotShotResult(value: unknown): asserts value is MultishotResult;
+//#endregion
+//#region src/multishot/multishot.d.ts
+interface RunMultishotOptions<TPersona extends MultishotPersona> {
+  profile: AgentProfile;
+  persona: TPersona;
+  /** Persona-shaping callbacks. Optional — omitted callbacks are derived from
+   *  the profile + persona payload, so a pure-profile call works. */
+  shape?: MultishotShape<TPersona>;
+  /** Tool definitions advertised to the agent. Defaults to delegate_research + delegate_code. */
+  tools?: MultishotToolDefinition[];
+  /** Map from tool name → executor invoked inline when the agent emits a tool_call. */
+  toolExecutors?: Record<string, MultishotToolExecutor>;
+  /** Map from tool name → artifact type label written into MultishotArtifact.type.
+   *  Tools without a mapping still execute, but their results aren't surfaced as
+   *  typed artifacts (only as tool messages in the transcript). */
+  artifactTypeFor?: (toolName: string) => string | undefined;
+  maxTurns?: number;
+  agentModel?: string;
+  driverModel?: string;
+  /** Fallback driver models tried when the primary simulated-user model returns empty twice. */
+  driverFallbackModels?: string[];
+  /** Maximum output tokens for the first agent call in each assistant turn. */
+  agentMaxTokens?: number;
+  /** Maximum output tokens for agent follow-up calls after tool results. */
+  toolFollowupMaxTokens?: number;
+  /** Maximum output tokens for each simulated-user driver response. */
+  driverMaxTokens?: number;
+  /** Maximum tool calls the agent may dispatch inside one assistant turn. */
+  maxToolDispatches?: number;
+  /** Execution seam for the agent leg. When provided, every agent inference
+   *  step goes through this function instead of the router HTTP call; the
+   *  string levers (agentModel, apiKey, baseUrl) stop applying to that leg.
+   *  apiKey/baseUrl are still resolved for tool executors and any leg
+   *  without an injected transport. */
+  agentTransport?: MultishotTransport;
+  /** Execution seam for the simulated-user driver leg (symmetric to
+   *  agentTransport). Driver model fallback rotation still applies — the
+   *  transport receives each candidate model in turn. */
+  driverTransport?: MultishotTransport;
+  apiKey?: string;
+  baseUrl?: string;
+  signal?: AbortSignal;
+}
+/** One multishot shot — the conversation engine `runMultishotMatrix` invokes
+ *  once per cell. `runMultishot` is the default implementation.
+ *
+ *  An alternative engine (a graph-backed conversation, a replay of a recorded
+ *  transcript, a sandbox-hosted agent) implements this exact signature and
+ *  reaches the matrix through `RunMultishotMatrixOptions.runShot`. The matrix
+ *  keeps every other cell mechanic — cell fan-out, concurrency, the cost
+ *  ceiling, the judge slots, the cell composite, and the per-cell writers — so
+ *  swapping the engine needs no copy of the cell body. */
+type MultishotShot<TPersona extends MultishotPersona> = (opts: RunMultishotOptions<TPersona>) => Promise<MultishotResult>;
+declare function runMultishot<TPersona extends MultishotPersona>(opts: RunMultishotOptions<TPersona>): Promise<MultishotResult>;
+//#endregion
+//#region src/multishot/matrix.d.ts
+interface ConversationJudgeInput<TPersona extends MultishotPersona> {
+  transcript: MultishotMessage[];
+  persona: TPersona;
+}
+interface ArtifactJudgeInput<TPersona extends MultishotPersona> {
+  artifact: MultishotArtifact;
+  persona: TPersona;
+}
+interface MultishotJudges<TPersona extends MultishotPersona> {
+  /** Scores the full transcript end-to-end (always runs). */
+  conversation: JudgeConfig<ConversationJudgeInput<TPersona>>;
+  /** Scores each code-type artifact. Optional — omit when domain has no code artifacts. */
+  codeReview?: JudgeConfig<ArtifactJudgeInput<TPersona>>;
+  /** Scores each non-code (research/content/template) artifact. Optional. */
+  contentQuality?: JudgeConfig<ArtifactJudgeInput<TPersona>>;
+  /** Which artifact types route to codeReview. Defaults to ['code']. */
+  codeArtifactTypes?: string[];
+  /** Which artifact types route to contentQuality. Defaults to ['research']. */
+  contentArtifactTypes?: string[];
+}
+interface CellCompositeScore {
+  composite: number;
+  conversation: JudgeScore;
+  codeReview?: {
+    perArtifact: Array<JudgeScore & {
+      turn: number;
+      type: string;
+    }>;
+    composite: number;
+  };
+  contentQuality?: {
+    perArtifact: Array<JudgeScore & {
+      turn: number;
+      type: string;
+    }>;
+    composite: number;
+  };
+}
+interface RunMultishotMatrixOptions<TPersona extends MultishotPersona> {
+  /** AgentProfile axis (matrix primary). */
+  profiles: Array<{
+    id: string;
+    value: AgentProfile;
+  }>;
+  /** Persona axis. */
+  personas: TPersona[];
+  /** Persona-shaping callbacks. Optional — omitted callbacks are derived per
+   *  cell from that cell's profile + persona payload (pure-profile path). */
+  shape?: MultishotShape<TPersona>;
+  /** Judge configurations. */
+  judges: MultishotJudges<TPersona>;
+  /** Tool definitions advertised to the agent. Defaults to delegate_research + delegate_code. */
+  tools?: MultishotToolDefinition[];
+  /** Map from tool name → inline executor. Must align with `tools`. */
+  toolExecutors?: Record<string, MultishotToolExecutor>;
+  /** Tool name → artifact type label. Defaults to research/code mapping. */
+  artifactTypeFor?: (toolName: string) => string | undefined;
+  /** Where per-cell artifacts land. Cells write to `<runDir>/<profileId>/<personaId>/rep-N/`. */
+  runDir: string;
+  /** Replicates per (profile, persona) cell. */
+  reps?: number;
+  /** Max conversation turns per cell. */
+  maxTurns?: number;
+  /** Maximum tool calls the agent may dispatch inside one assistant turn. */
+  maxToolDispatches?: number;
+  /** Max concurrent cells. */
+  maxConcurrency?: number;
+  /** Total $ ceiling across the matrix; cells aborted past this. */
+  costCeiling?: number;
+  /** Upper bound on what one cell can spend. A cell whose cost is a subtotal
+   *  is charged this bound against `costCeiling` instead of its known amount,
+   *  so hidden spend cannot walk the run past its budget. */
+  maxCellCostUsd?: number;
+  /** Agent model. */
+  agentModel?: string;
+  /** Driver model. */
+  driverModel?: string;
+  /** Fallback driver models tried when the primary simulated-user model returns empty twice. */
+  driverFallbackModels?: string[];
+  /** Maximum output tokens for the first agent call in each assistant turn. */
+  agentMaxTokens?: number;
+  /** Maximum output tokens for agent follow-up calls after tool results. */
+  toolFollowupMaxTokens?: number;
+  /** Maximum output tokens for each simulated-user driver response. */
+  driverMaxTokens?: number;
+  /** Maximum output tokens for each judge response. */
+  judgeMaxTokens?: number;
+  /** Execution seam for the agent leg of every cell — replaces the router
+   *  HTTP call when provided (see RunMultishotOptions.agentTransport).
+   *  Judges are unaffected; configure those via MultishotJudges. */
+  agentTransport?: MultishotTransport;
+  /** Execution seam for the simulated-user driver leg of every cell. */
+  driverTransport?: MultishotTransport;
+  /** Conversation engine for every cell. Defaults to `runMultishot`.
+   *
+   *  The matrix owns everything around the shot — cell fan-out, concurrency,
+   *  the cost ceiling, the judge slots, the cell composite, the per-cell
+   *  artifact writers and the run summary — and forwards the whole cell input
+   *  to this function, so an alternative engine replaces ONLY the
+   *  conversation. Every option on this interface that `runMultishot` accepts
+   *  reaches the shot unchanged. `RunMultishotOptions.signal` has no
+   *  matrix-level counterpart and is not forwarded; a shot owns its own
+   *  cancellation.
+   *
+   *  A shot that resolves with a value outside `MultishotResult` throws
+   *  `MultishotShotResultError` for that cell. The default engine is never
+   *  used as a fallback. */
+  runShot?: MultishotShot<TPersona>;
+  /** Pass-thru fields. */
+  apiKey?: string;
+  baseUrl?: string;
+}
+/** Per-cell output the multishot matrix records in `MatrixResult.cells`.
+ *  A consumer that supplies its own `runShot` reads the matrix through this
+ *  type instead of declaring a structural copy. */
+interface MultishotCellOutput {
+  turns: number;
+  toolCalls: number;
+  artifactCount: number;
+}
+interface CellCompositeInput {
+  conversation: JudgeScore;
+  /** Present iff the codeReview judge is configured. */
+  codeReviews?: ReadonlyArray<JudgeScore>;
+  /** Present iff the contentQuality judge is configured. */
+  contentReviews?: ReadonlyArray<JudgeScore>;
+}
+/** Cell composite = mean over configured judge slots, excluding failed
+ *  scores: a failed conversation judge or an all-failed artifact slot carries
+ *  no signal and is dropped from the mean. `composite` is 0 only when EVERY
+ *  configured slot failed (`allJudgesFailed` distinguishes that from a real
+ *  zero). Pure — exported for deterministic testing. */
+declare function computeCellComposite(input: CellCompositeInput): {
+  composite: number;
+  codeComposite: number;
+  contentComposite: number;
+  allJudgesFailed: boolean;
+};
+interface RunMultishotMatrixResult {
+  matrix: MatrixResult<MultishotCellOutput>;
+}
+declare function runMultishotMatrix<TPersona extends MultishotPersona>(opts: RunMultishotMatrixOptions<TPersona>): Promise<RunMultishotMatrixResult>;
+//#endregion
+export { JudgeConfig as A, MultishotToolExecutor as C, MultishotTransportToolCall as D, MultishotTransportResponse as E, runJudge as F, JudgeRunResult as M, renderDimensions as N, assertMultishotShotResult as O, renderJsonFooter as P, MultishotToolDefinition as S, MultishotTransportRequest as T, MultishotMessage as _, MultishotCellOutput as a, MultishotShape as b, RunMultishotMatrixResult as c, MultishotShot as d, RunMultishotOptions as f, MultishotFatalToolError as g, MultishotDriverEmptyError as h, ConversationJudgeInput as i, JudgeDimension as j, DEFAULT_JUDGE_MODEL as k, computeCellComposite as l, MultishotArtifact as m, CellCompositeInput as n, MultishotJudges as o, runMultishot as p, CellCompositeScore as r, RunMultishotMatrixOptions as s, ArtifactJudgeInput as t, runMultishotMatrix as u, MultishotPersona as v, MultishotTransport as w, MultishotShotResultError as x, MultishotResult as y };
+//# sourceMappingURL=matrix-C-2Qx1Zr.d.ts.map

package/dist/matrix-C-2Qx1Zr.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"matrix-C-2Qx1Zr.d.ts","names":[],"sources":["../src/multishot/judges.ts","../src/multishot/types.ts","../src/multishot/multishot.ts","../src/multishot/matrix.ts"],"mappings":";;;;;cAyBa;UAEI;;EAEf;;EAEA;;UAGe,YAAY;;EAE3B;;EAEA;;EAEA,YAAY;;EAEZ;;;EAGA,cAAc,OAAO;;EAErB;EACA;;EAEA;;UAGe;;EAEf,OAAO;;EAEP,MAAM;;iBAGc,SAAS,QAC7B,OAAO,YAAY,SACnB,OAAO,SACN,QAAQ;;;iBAkIK,iBAAiB,eAAe;;iBAKhC,iBAAiB,eAAe;;;UClM/B;EACf;EACA;EACA;EACA,YAAY;IAAQ;IAAY;IAAc,MAAM;;;UAGrC;EACf;EACA;EACA;IAAc;IAAc,MAAM;;EAClC;;UAGe;EACf,YAAY;EACZ,WAAW;EACX;EACA;;;EAGA;;;;;;;;EAQA,iBAAiB;;UAGF;EACf;EACA;IACE;IACA;IACA,YAAY;;;;;;UAOC;EACf;EACA,UAAU,MAAM;EAChB,QAAQ;EACR;EACA;EACA,SAAS;;UAGM;EACf;EACA;EACA;IAAY;IAAc;;;UAGX;EACf;IAAW;IAAyB,aAAa;;EACjD;IAAU;IAAwB;;;;EAGlC;;;;;;;;KASU,sBACV,KAAK,8BACF,QAAQ;KAED,yBACV,MAAM,yBACN;EAAO;EAAgB;EAAiB,SAAS;MAC9C;EAAU;EAAiB;;UAEf;;EAEf;;GAEC;;;;;;;;UASc,eAAe,iBAAiB;;EAE/C,eAAe,SAAS;;;EAGxB,2BAA2B,SAAS;;cAGzB,kCAAkC;WACjB;EAA5B,YAA4B;;cAMjB,gCAAgC;EAC3C,YAAY;;cAMD,iCAAiC;EAC5C,YAAY;;;;;;;;;;;;;;;;;;;iBAyBE,0BAA0B,yBAAyB,SAAS;;;UCpH3D,oBAAoB,iBAAiB;EACpD,SAAS;EACT,SAAS;;;EAGT,QAAQ,eAAe;;EAEvB,QAAQ;;EAER,gBAAgB,eAAe;;;;EAI/B,mBAAmB;EACnB;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;EAMA,iBAAiB;;;;EAIjB,kBAAkB;EAClB;EACA;EACA,SAAS;;;;;;;;;;;KAYC,cAAc,iBAAiB,qBACzC,MAAM,oBAAoB,cACvB,QAAQ;iBAWS,aAAa,iBAAiB,kBAClD,MAAM,oBAAoB,YACzB,QAAQ;;;UCtEM,uBAAuB,iBAAiB;EACvD,YAAY;EACZ,SAAS;;UAGM,mBAAmB,iBAAiB;EACnD,UAAU;EACV,SAAS;;UAGM,gBAAgB,iBAAiB;;EAEhD,cAAc,YAAY,uBAAuB;;EAEjD,aAAa,YAAY,mBAAmB;;EAE5C,iBAAiB,YAAY,mBAAmB;;EAEhD;;EAEA;;UAGe;EACf;EACA,cAAc;EACd;IACE,aAAa,MAAM;MAAe;MAAc;;IAChD;;EAEF;IACE,aAAa,MAAM;MAAe;MAAc;;IAChD;;;UAIa,0BAA0B,iBAAiB;;EAE1D,UAAU;IAAQ;IAAY,OAAO;;;EAErC,UAAU;;;EAGV,QAAQ,eAAe;;EAEvB,QAAQ,gBAAgB;;EAExB,QAAQ;;EAER,gBAAgB,eAAe;;EAE/B,mBAAmB;;EAEnB;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;EAIA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;EAIA,iBAAiB;;EAEjB,kBAAkB;;;;;;;;;;;;;;;EAelB,UAAU,cAAc;;EAExB;EACA;;;;;UAMe;EACf;EACA;EACA;;UAoBe;EACf,cAAc;;EAEd,cAAc,cAAc;;EAE5B,iBAAiB,cAAc;;;;;;;iBAQjB,qBAAqB,OAAO;EAC1C;EACA;EACA;EACA;;UAuBe;EACf,QAAQ,aAAa;;iBAGD,mBAAmB,iBAAiB,kBACxD,MAAM,0BAA0B,YAC/B,QAAQ"}

package/dist/meta-eval/index.d.ts

@@ -1,7 +1,7 @@
 import { f as Run } from "../schema-BtVldJ3T.js";
 import { s as TraceStore } from "../store-CT9YIIve.js";
 import { a as ContinuousCalibrationResult, n as CandidateScore, o as GoldenItem, r as ContinuousAgreement, t as CalibrationResult } from "../judge-calibration-C5CbMYce.js";
-import { n as SeriesConvergenceResult, o as CorpusAgreementReport, t as SeriesConvergenceOptions } from "../series-convergence-C9G-GNYK.js";
+import { n as SeriesConvergenceResult, o as CorpusAgreementReport, t as SeriesConvergenceOptions } from "../series-convergence-D2fsoJ2w.js";
 import { a as OutcomeFilter, i as InMemoryOutcomeStore, n as FileSystemOutcomeStore, o as OutcomeStore, r as FileSystemOutcomeStoreOptions, t as DeploymentOutcome } from "../outcome-store-BYHIuO0e.js";
 import { a as rubricPredictiveValidity, i as RubricRanking, n as RubricPredictiveValidityInput, r as RubricPredictiveValidityReport, t as RubricOutcomePair } from "../rubric-predictive-validity-C7LnNvF2.js";
 //#region src/meta-eval/correlation-study.d.ts

package/dist/{metrics-Cl0L1KUy.js → metrics-Qv-cpptD.js}

@@ -24,6 +24,10 @@ const MODEL_PRICING = {
 	"claude-3-haiku-20240307": {
 		input: 25e-5,
 		output: .00125
+	},
+	"glm-5.3": {
+		input: .00168,
+		output: .00528
 	}
 };
 /** Family-level pricing fallbacks (per-1K), matched against a normalized id
@@ -88,19 +92,22 @@ const FAMILY_PRICING = [
 	}]
 ];
 /** Normalize a model id for pricing: drop a `@snapshot` suffix, lowercase,
-*  and keep the final harness/provider-prefixed segment so family regexes
-*  match (`opencode/zai-coding-plan/glm-5.1` → `glm-5.1`). */
-function normalizeModelId(model) {
-	return (model.split("@")[0] ?? model).trim().toLowerCase();
+*  and return both the complete harness/provider-prefixed id and its final
+*  served-model segment (`opencode/zai-coding-plan/glm-5.3` → full id plus
+*  `glm-5.3`). Exact rows are checked against both before family fallback. */
+function normalizedModelIds(model) {
+	const id = (model.split("@")[0] ?? model).trim().toLowerCase();
+	const finalSegment = id.slice(id.lastIndexOf("/") + 1);
+	return finalSegment === id ? [id] : [id, finalSegment];
 }
 /** Resolve pricing for a model id: exact table, then family fallback.
 *  Returns null when the id matches nothing (caller decides — never a
 *  silent-zero masquerading as a real $0 cost). */
 function resolveModelPricing(model) {
 	if (MODEL_PRICING[model]) return MODEL_PRICING[model];
-	const id = normalizeModelId(model);
-	if (MODEL_PRICING[id]) return MODEL_PRICING[id];
-	for (const [pattern, price] of FAMILY_PRICING) if (pattern.test(id)) return price;
+	const ids = normalizedModelIds(model);
+	for (const id of ids) if (MODEL_PRICING[id]) return MODEL_PRICING[id];
+	for (const [pattern, price] of FAMILY_PRICING) if (ids.some((id) => pattern.test(id))) return price;
 	return null;
 }
 /** True when `model` has known pricing (exact or family). Lets cost-aware
@@ -130,4 +137,4 @@ function estimateCost(inputTokens, outputTokens, model) {
 //#endregion
 export { resolveModelPricing as a, isModelPriced as i, estimateCost as n, estimateTokens as r, MODEL_PRICING as t };
 
-//# sourceMappingURL=metrics-Cl0L1KUy.js.map
+//# sourceMappingURL=metrics-Qv-cpptD.js.map

package/dist/metrics-Qv-cpptD.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics-Qv-cpptD.js","names":[],"sources":["../src/metrics.ts"],"sourcesContent":["interface TokenPrice {\n  input: number\n  output: number\n}\n\n/** Per-1K token pricing for exact model ids. */\nexport const MODEL_PRICING: Record<string, TokenPrice> = {\n  'gpt-4o': { input: 0.0025, output: 0.01 },\n  'gpt-4o-mini': { input: 0.00015, output: 0.0006 },\n  'gpt-4-turbo': { input: 0.01, output: 0.03 },\n  'claude-sonnet-4-20250514': { input: 0.003, output: 0.015 },\n  'claude-opus-4-20250514': { input: 0.015, output: 0.075 },\n  'claude-3-haiku-20240307': { input: 0.00025, output: 0.00125 },\n  // Router response-header rates observed for the exact served model on\n  // 2026-08-15. Keep this exact row ahead of the deliberately coarse family\n  // fallback so glm-5.3 campaign costs are not understated by ~2.5x.\n  'glm-5.3': { input: 0.00168, output: 0.00528 },\n}\n\n/** Family-level pricing fallbacks (per-1K), matched against a normalized id\n *  after exact lookup misses. Ordered — first match wins. Covers the model\n *  ids actually used through the Tangle router + cli-bridge harnesses\n *  (`claude-code/sonnet`, `opencode/zai-coding-plan/glm-5.1`,\n *  `kimi-code/kimi-k2.6`, `deepseek-v4-pro`, `anthropic/claude-sonnet-4-6`, …),\n *  none of which appear in the exact table above — without this they priced\n *  to a silent $0, blanking every cost/Pareto axis downstream. */\nconst FAMILY_PRICING: Array<[RegExp, TokenPrice]> = [\n  [/claude.*opus/, { input: 0.015, output: 0.075 }],\n  [/claude.*haiku/, { input: 0.0008, output: 0.004 }],\n  [/claude.*sonnet|claude-code|claude-sonnet/, { input: 0.003, output: 0.015 }],\n  [/gpt-4o-mini/, { input: 0.00015, output: 0.0006 }],\n  [/gpt-5|gpt-4\\.1|o[134]\\b/, { input: 0.00125, output: 0.01 }],\n  [/gpt-4o|gpt-4/, { input: 0.0025, output: 0.01 }],\n  [/deepseek/, { input: 0.0003, output: 0.0011 }],\n  [/glm|zhipu|zai/, { input: 0.0006, output: 0.0022 }],\n  [/kimi|moonshot/, { input: 0.0006, output: 0.0025 }],\n  [/qwen/, { input: 0.0004, output: 0.0012 }],\n  [/gemini.*flash/, { input: 0.0001, output: 0.0004 }],\n  [/gemini/, { input: 0.00125, output: 0.005 }],\n  [/llama/, { input: 0.0002, output: 0.0006 }],\n]\n\n/** Normalize a model id for pricing: drop a `@snapshot` suffix, lowercase,\n *  and return both the complete harness/provider-prefixed id and its final\n *  served-model segment (`opencode/zai-coding-plan/glm-5.3` → full id plus\n *  `glm-5.3`). Exact rows are checked against both before family fallback. */\nfunction normalizedModelIds(model: string): string[] {\n  const id = (model.split('@')[0] ?? model).trim().toLowerCase()\n  const finalSegment = id.slice(id.lastIndexOf('/') + 1)\n  return finalSegment === id ? [id] : [id, finalSegment]\n}\n\n/** Resolve pricing for a model id: exact table, then family fallback.\n *  Returns null when the id matches nothing (caller decides — never a\n *  silent-zero masquerading as a real $0 cost). */\nexport function resolveModelPricing(model: string): TokenPrice | null {\n  if (MODEL_PRICING[model]) return MODEL_PRICING[model]\n  const ids = normalizedModelIds(model)\n  for (const id of ids) {\n    if (MODEL_PRICING[id]) return MODEL_PRICING[id]\n  }\n  for (const [pattern, price] of FAMILY_PRICING) {\n    if (ids.some((id) => pattern.test(id))) return price\n  }\n  return null\n}\n\n/** True when `model` has known pricing (exact or family). Lets cost-aware\n *  callers distinguish a real $0 from an unpriced model. */\nexport function isModelPriced(model: string): boolean {\n  return resolveModelPricing(model) !== null\n}\n\nconst warnedUnpricedModels = new Set<string>()\n\n/** Estimate token count from string length (chars / 4 approximation) */\nexport function estimateTokens(text: string): number {\n  return Math.ceil(text.length / 4)\n}\n\n/** Calculate cost in USD from token counts and model. Unknown models warn\n *  once (not a silent zero) and return 0 so callers that ignore pricing keep\n *  working; cost-sensitive callers should gate on {@link isModelPriced}. */\nexport function estimateCost(inputTokens: number, outputTokens: number, model: string): number {\n  const pricing = resolveModelPricing(model)\n  if (!pricing) {\n    if (!warnedUnpricedModels.has(model)) {\n      warnedUnpricedModels.add(model)\n      console.warn(\n        `estimateCost: no pricing for model \"${model}\" — returning 0; add it to ` +\n          'MODEL_PRICING/FAMILY_PRICING (cost/Pareto axes will be blank until then)',\n      )\n    }\n    return 0\n  }\n  return (inputTokens / 1000) * pricing.input + (outputTokens / 1000) * pricing.output\n}\n\n/**\n * TokenCounter — accumulates token usage and cost across turns.\n */\nexport class TokenCounter {\n  private totalInput = 0\n  private totalOutput = 0\n  private totalCost = 0\n  private model: string\n\n  constructor(model = 'gpt-4o') {\n    this.model = model\n  }\n\n  /** Record tokens for a turn, returns per-turn cost */\n  record(inputTokens: number, outputTokens: number): number {\n    this.totalInput += inputTokens\n    this.totalOutput += outputTokens\n    const cost = estimateCost(inputTokens, outputTokens, this.model)\n    this.totalCost += cost\n    return cost\n  }\n\n  /** Estimate and record from raw text */\n  recordFromText(\n    inputText: string,\n    outputText: string,\n  ): { inputTokens: number; outputTokens: number; cost: number } {\n    const inputTokens = estimateTokens(inputText)\n    const outputTokens = estimateTokens(outputText)\n    const cost = this.record(inputTokens, outputTokens)\n    return { inputTokens, outputTokens, cost }\n  }\n\n  getTotalInput(): number {\n    return this.totalInput\n  }\n  getTotalOutput(): number {\n    return this.totalOutput\n  }\n  getTotalCost(): number {\n    return this.totalCost\n  }\n}\n"],"mappings":";;AAMA,MAAa,gBAA4C;CACvD,UAAU;EAAE,OAAO;EAAQ,QAAQ;CAAK;CACxC,eAAe;EAAE,OAAO;EAAS,QAAQ;CAAO;CAChD,eAAe;EAAE,OAAO;EAAM,QAAQ;CAAK;CAC3C,4BAA4B;EAAE,OAAO;EAAO,QAAQ;CAAM;CAC1D,0BAA0B;EAAE,OAAO;EAAO,QAAQ;CAAM;CACxD,2BAA2B;EAAE,OAAO;EAAS,QAAQ;CAAQ;CAI7D,WAAW;EAAE,OAAO;EAAS,QAAQ;CAAQ;AAC/C;;;;;;;;AASA,MAAM,iBAA8C;CAClD,CAAC,gBAAgB;EAAE,OAAO;EAAO,QAAQ;CAAM,CAAC;CAChD,CAAC,iBAAiB;EAAE,OAAO;EAAQ,QAAQ;CAAM,CAAC;CAClD,CAAC,4CAA4C;EAAE,OAAO;EAAO,QAAQ;CAAM,CAAC;CAC5E,CAAC,eAAe;EAAE,OAAO;EAAS,QAAQ;CAAO,CAAC;CAClD,CAAC,2BAA2B;EAAE,OAAO;EAAS,QAAQ;CAAK,CAAC;CAC5D,CAAC,gBAAgB;EAAE,OAAO;EAAQ,QAAQ;CAAK,CAAC;CAChD,CAAC,YAAY;EAAE,OAAO;EAAQ,QAAQ;CAAO,CAAC;CAC9C,CAAC,iBAAiB;EAAE,OAAO;EAAQ,QAAQ;CAAO,CAAC;CACnD,CAAC,iBAAiB;EAAE,OAAO;EAAQ,QAAQ;CAAO,CAAC;CACnD,CAAC,QAAQ;EAAE,OAAO;EAAQ,QAAQ;CAAO,CAAC;CAC1C,CAAC,iBAAiB;EAAE,OAAO;EAAQ,QAAQ;CAAO,CAAC;CACnD,CAAC,UAAU;EAAE,OAAO;EAAS,QAAQ;CAAM,CAAC;CAC5C,CAAC,SAAS;EAAE,OAAO;EAAQ,QAAQ;CAAO,CAAC;AAC7C;;;;;AAMA,SAAS,mBAAmB,OAAyB;CACnD,MAAM,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,MAAA,CAAO,KAAK,CAAC,CAAC,YAAY;CAC7D,MAAM,eAAe,GAAG,MAAM,GAAG,YAAY,GAAG,IAAI,CAAC;CACrD,OAAO,iBAAiB,KAAK,CAAC,EAAE,IAAI,CAAC,IAAI,YAAY;AACvD;;;;AAKA,SAAgB,oBAAoB,OAAkC;CACpE,IAAI,cAAc,QAAQ,OAAO,cAAc;CAC/C,MAAM,MAAM,mBAAmB,KAAK;CACpC,KAAK,MAAM,MAAM,KACf,IAAI,cAAc,KAAK,OAAO,cAAc;CAE9C,KAAK,MAAM,CAAC,SAAS,UAAU,gBAC7B,IAAI,IAAI,MAAM,OAAO,QAAQ,KAAK,EAAE,CAAC,GAAG,OAAO;CAEjD,OAAO;AACT;;;AAIA,SAAgB,cAAc,OAAwB;CACpD,OAAO,oBAAoB,KAAK,MAAM;AACxC;AAEA,MAAM,uCAAuB,IAAI,IAAY;;AAG7C,SAAgB,eAAe,MAAsB;CACnD,OAAO,KAAK,KAAK,KAAK,SAAS,CAAC;AAClC;;;;AAKA,SAAgB,aAAa,aAAqB,cAAsB,OAAuB;CAC7F,MAAM,UAAU,oBAAoB,KAAK;CACzC,IAAI,CAAC,SAAS;EACZ,IAAI,CAAC,qBAAqB,IAAI,KAAK,GAAG;GACpC,qBAAqB,IAAI,KAAK;GAC9B,QAAQ,KACN,uCAAuC,MAAM,oGAE/C;EACF;EACA,OAAO;CACT;CACA,OAAQ,cAAc,MAAQ,QAAQ,QAAS,eAAe,MAAQ,QAAQ;AAChF"}