@mupt-ai/dari-router 0.1.0 → 0.2.0

package/src/cost.ts

@@ -1,455 +0,0 @@
-// Routing-only cost estimates for comparing candidate models before selection.
-import {
-  ANTHROPIC_LOOKBACK_BLOCKS,
-  CROSS_TOKENIZER_RATIO,
-  isAnthropicFamily,
-  isOpenAiFamily,
-  openAiCachedPrefixTokens,
-  promptCacheProviderForModel,
-  providerCacheableTokens,
-  providerMinCacheTokens,
-  tokenizerFamily,
-  type PromptCacheProvider,
-} from "./cache_behavior.js";
-import { reasoningCacheScope, type ReasoningCacheScopeLookup } from "./cache_scope.js";
-import {
-  FIXED_TURN_CACHE_HIT_PROBABILITY,
-  FIXED_TURN_COST_PROJECTED_TURNS,
-} from "./fixed_turn_cost_config.js";
-import { routingCandidateKey } from "./types.js";
-import type {
-  CandidateCostEstimate,
-  ChainHits,
-  FixedTurnCostEstimate,
-  PrefixHit,
-  ReasoningEffort,
-  RouterPrefixHit,
-  RoutingCandidate,
-  TurnCostProjection,
-} from "./types.js";
-
-export const CHARS_PER_TOKEN = 4;
-const MTOK = 1_000_000;
-
-export type ModelPricing = {
-  input: number;
-  output: number;
-  cacheRead: number;
-  cacheWrite: number;
-};
-
-// A model missing from the resolved catalog must surface as null (pricing
-// unknown), never as $0.
-export type PricingLookup = (model: string) => ModelPricing | null;
-
-export type CandidatePromptEstimate = {
-  chars: number;
-  reusesStoredPromptTokens: boolean;
-};
-
-// perModel/perModelBuckets key on raw model-id strings deliberately:
-// entry.model is a verbatim copy of the router's enabled_models entry that
-// served the turn, and candidate lookups use members of the same list, so
-// both sides are byte-identical by construction. Canonicalizing here could
-// merge two distinct custom model ids differing only in provider-prefix case
-// and cross-contaminate their warm prefixes (an unsafe over-claim), whereas
-// raw keying can at worst under-claim warmth if an id is ever re-spelled
-// mid-conversation.
-// Provider-visible chains can differ when one candidate receives compatible
-// opaque continuation state and another sees readable history only. A hit is
-// accepted solely against the chain for the model that originally wrote it;
-// client-declared continuation provenance cannot make another candidate warm.
-export function deriveModelChainHits(
-  chainsByModel: ReadonlyMap<string, string[]>,
-  hits: RouterPrefixHit[] | undefined
-): ChainHits {
-  const depthsByModel = new Map(
-    [...chainsByModel].map(([model, chain]) => [
-      model,
-      new Map(chain.map((hash, index) => [hash, index + 1])),
-    ])
-  );
-  return collectChainHits(hits, (entry) => depthsByModel.get(entry.model)?.get(entry.hash));
-}
-
-// Conversation identity accepts either hash tier: the strict per-model chain
-// (byte-faithful echoes) or the host's loose chain (clients that drop
-// reasoning payloads or rewrite item ids). Warmth estimation must NOT use
-// this — a loose-only match means the provider-visible bytes diverged, so
-// the cache is cold even though the conversation is the same.
-// Loose hashes are not unique: two transcript twins (identical runs whose
-// bytes differ only in reasoning payloads, item ids, or whitespace) can both
-// match at the same depth. Ties are content-equivalent, but the pick must be
-// deterministic: a strict match (byte-identical, provably this transcript)
-// beats a loose-only one, then the most recently active entry wins.
-export function deepestIdentityHit(
-  chainsByModel: ReadonlyMap<string, string[]>,
-  looseChain: readonly string[] | undefined,
-  hits: RouterPrefixHit[] | undefined
-): PrefixHit | undefined {
-  const depthsByModel = new Map(
-    [...chainsByModel].map(([model, chain]) => [
-      model,
-      new Map(chain.map((hash, index) => [hash, index + 1])),
-    ])
-  );
-  const looseDepths = new Map((looseChain ?? []).map((hash, index) => [hash, index + 1]));
-  let deepest: PrefixHit | undefined;
-  let deepestStrict = false;
-  let deepestUpdatedAtMs = Number.NEGATIVE_INFINITY;
-  for (const entry of hits ?? []) {
-    const strictDepth = depthsByModel.get(entry.model)?.get(entry.hash) ?? 0;
-    const looseDepth = (entry.loose_hash && looseDepths.get(entry.loose_hash)) || 0;
-    const depth = Math.max(strictDepth, looseDepth);
-    if (depth === 0) continue;
-    const strict = strictDepth > 0;
-    const parsedUpdatedAtMs = Date.parse(entry.updated_at);
-    const updatedAtMs = Number.isFinite(parsedUpdatedAtMs)
-      ? parsedUpdatedAtMs
-      : Number.NEGATIVE_INFINITY;
-    const wins =
-      !deepest ||
-      depth > deepest.depth ||
-      (depth === deepest.depth &&
-        (strict !== deepestStrict
-          ? strict
-          : updatedAtMs > deepestUpdatedAtMs));
-    if (wins) {
-      deepest = { entry, depth };
-      deepestStrict = strict;
-      deepestUpdatedAtMs = updatedAtMs;
-    }
-  }
-  return deepest;
-}
-
-function collectChainHits(
-  hits: RouterPrefixHit[] | undefined,
-  depthFor: (entry: RouterPrefixHit) => number | undefined
-): ChainHits {
-  let deepest: PrefixHit | undefined;
-  const perModel = new Map<string, PrefixHit>();
-  const perModelBuckets = new Map<string, Map<string | null, PrefixHit>>();
-  for (const entry of hits ?? []) {
-    const depth = depthFor(entry);
-    if (depth === undefined) continue;
-    const hit: PrefixHit = { entry, depth };
-    if (!deepest || depth > deepest.depth) deepest = hit;
-    const existing = perModel.get(entry.model);
-    if (!existing || depth > existing.depth) perModel.set(entry.model, hit);
-    const bucket = entry.reasoning_bucket ?? null;
-    let byBucket = perModelBuckets.get(entry.model);
-    if (!byBucket) {
-      byBucket = new Map();
-      perModelBuckets.set(entry.model, byBucket);
-    }
-    const existingBucketHit = byBucket.get(bucket);
-    if (!existingBucketHit || depth > existingBucketHit.depth) byBucket.set(bucket, hit);
-  }
-  return { deepest, perModel, perModelBuckets };
-}
-
-// The warm-prefix hit a candidate can actually re-read, given how its
-// provider scopes cache identity across reasoning payloads. Shared-scope
-// providers reuse one partition regardless of effort; effort-keyed providers
-// only re-read the partition warmed under the same effective effort, so the
-// hit must come from the matching bucket. Entries written before buckets
-// existed (null) are accepted for any effort so a rollout never estimates a
-// genuinely warm prefix as cold.
-export function selectPrefixHit(
-  hits: ChainHits,
-  model: string,
-  reasoningBucket: string,
-  cacheScope: ReasoningCacheScopeLookup = reasoningCacheScope
-): PrefixHit | undefined {
-  if (cacheScope(model) === "shared") return hits.perModel.get(model);
-  const byBucket = hits.perModelBuckets.get(model);
-  if (!byBucket) return undefined;
-  const matched = byBucket.get(reasoningBucket);
-  const legacy = byBucket.get(null);
-  if (matched) return matched;
-  return legacy;
-}
-
-export function selectPromptAnchor(
-  hits: ChainHits,
-  model: string,
-  reasoningBucket: string,
-  cacheScope: ReasoningCacheScopeLookup = reasoningCacheScope
-): PrefixHit | undefined {
-  // Prompt-token baselines are provider/model-specific. Cross-model hits can
-  // recover conversation identity elsewhere, but cannot anchor this model's
-  // provider-visible prompt estimate.
-  return selectPrefixHit(hits, model, reasoningBucket, cacheScope);
-}
-
-export function estimateCandidateCosts(args: {
-  candidates: RoutingCandidate[];
-  hits: ChainHits;
-  incomingBlockCount: number;
-  incomingProviderBlockCountFor?: (model: string) => number;
-  promptEstimatesByCandidate: ReadonlyMap<string, CandidatePromptEstimate>;
-  toolChoiceFp: string;
-  responseFormatFp: string;
-  pricing: PricingLookup;
-  averageOutputTokensByModel?: Readonly<
-    Record<string, Partial<Record<ReasoningEffort, number>> | null>
-  >;
-  cacheScope?: ReasoningCacheScopeLookup;
-}): CandidateCostEstimate[] {
-  const cacheScope = args.cacheScope ?? reasoningCacheScope;
-  return args.candidates.map((candidate) => {
-    const { model, reasoningEffort } = candidate;
-    const price = args.pricing(model);
-    const anthropicStyle = isAnthropicFamily(model);
-    const openAiStyle = isOpenAiFamily(model);
-    const promptEstimate = args.promptEstimatesByCandidate.get(
-      routingCandidateKey(candidate),
-    );
-    if (promptEstimate === undefined) {
-      throw new Error(
-        `missing prompt estimate for ${model}/${reasoningEffort}`,
-      );
-    }
-    const anchor = selectPromptAnchor(args.hits, model, reasoningEffort, cacheScope);
-    const reusesStoredPromptTokens = promptEstimate.reusesStoredPromptTokens;
-    const serializedTokens = Math.ceil(
-      promptEstimate.chars / CHARS_PER_TOKEN,
-    );
-    const tokenizerReference = anchor ?? args.hits.deepest;
-    const crossesTokenizerFamilies =
-      tokenizerReference !== undefined
-      && tokenizerFamily(model)
-        !== tokenizerFamily(tokenizerReference.entry.model);
-    // Candidate-specific serialization captures provider wire-shape
-    // differences. Preserve the separate pessimistic margin for tokenizer
-    // differences without borrowing the foreign model's measured token count.
-    const newTokens = crossesTokenizerFamilies
-      ? Math.round(serializedTokens * CROSS_TOKENIZER_RATIO)
-      : serializedTokens;
-
-    let estPrompt = newTokens;
-    if (anchor && reusesStoredPromptTokens) {
-      const knownTokens = anchor.entry.prompt_tokens;
-      estPrompt = knownTokens + newTokens;
-    }
-
-    const hit = selectPrefixHit(args.hits, model, reasoningEffort, cacheScope);
-    let warmTokens = 0;
-    if (hit) {
-      // tool_choice flips bust Anthropic's message cache but not automatic
-      // vendor caches; response_format schemas are prompt content everywhere.
-      const fingerprintsMatch = anthropicStyle
-        ? hit.entry.tool_choice_fp === args.toolChoiceFp &&
-          hit.entry.response_format_fp === args.responseFormatFp
-        : hit.entry.response_format_fp === args.responseFormatFp;
-      const withinLookback =
-        !anthropicStyle
-        || withinAnthropicLookback({
-          hit: hit.entry,
-          incomingLegacyDepth: args.incomingBlockCount,
-          incomingProviderDepth:
-            args.incomingProviderBlockCountFor?.(model),
-        });
-      // Provider minimums apply to the cached prefix being re-read — this
-      // candidate's own hit, counted by its own provider — not to the global
-      // prompt estimate (which may anchor on another provider's deeper entry).
-      const meetsMinimum =
-        hit.entry.prompt_tokens >= providerMinCacheTokens(model);
-      if (fingerprintsMatch && withinLookback && meetsMinimum) {
-        warmTokens = openAiStyle
-          ? openAiCachedPrefixTokens(hit.entry.prompt_tokens)
-          : hit.entry.prompt_tokens;
-      }
-    }
-    // Anchored estimates contain only a fresh suffix; complete-prompt
-    // estimates (used for legacy rows without an input depth) already include
-    // the warm prefix and must not add it a second time.
-    estPrompt = Math.max(
-      estPrompt,
-      reusesStoredPromptTokens ? warmTokens + newTokens : warmTokens,
-    );
-    // An unanchored hit (for example, a legacy row) leaves only a coarse
-    // complete-prompt estimate. If that estimate undercuts the confirmed warm
-    // prefix, its fresh/cached split is unknowable. Conservatively charge the
-    // coarse estimate as fresh instead of making the fresh portion free.
-    if (
-      warmTokens > 0
-      && !reusesStoredPromptTokens
-      && newTokens <= warmTokens
-    ) {
-      estPrompt = warmTokens + newTokens;
-    }
-
-    let estCost: number | null = null;
-    if (price) {
-      const remainder = Math.max(0, estPrompt - warmTokens);
-      estCost =
-        (warmTokens * price.cacheRead + remainder * freshInputRate(model, price, estPrompt)) /
-        1_000_000;
-    }
-
-    return {
-      model,
-      reasoning_effort: reasoningEffort,
-      warm_tokens: warmTokens,
-      est_prompt_tokens: estPrompt,
-      est_input_cost_usd: estCost,
-      output_cost_per_mtok: price?.output ?? null,
-      pricing_known: price !== null,
-      fixed_turn_cost_estimate: price
-        ? fixedTurnCostEstimate({
-            model,
-            price,
-            estPromptTokens: estPrompt,
-            warmTokens,
-            reasoningEffort,
-            outputTokens: args.averageOutputTokensByModel?.[model]?.[reasoningEffort],
-          })
-        : null,
-    } satisfies CandidateCostEstimate;
-  });
-}
-
-function withinAnthropicLookback(args: {
-  hit: RouterPrefixHit;
-  incomingLegacyDepth: number;
-  incomingProviderDepth: number | undefined;
-}): boolean {
-  const storedProviderDepth = args.hit.provider_block_depth;
-  if (
-    storedProviderDepth !== undefined
-    && storedProviderDepth !== null
-  ) {
-    if (args.incomingProviderDepth === undefined) return false;
-    const delta = args.incomingProviderDepth - storedProviderDepth;
-    return delta >= 0 && delta <= ANTHROPIC_LOOKBACK_BLOCKS;
-  }
-  const delta = args.incomingLegacyDepth - args.hit.block_depth;
-  return delta >= 0 && delta <= ANTHROPIC_LOOKBACK_BLOCKS;
-}
-
-function fixedTurnCostEstimate(args: {
-  model: string;
-  price: ModelPricing;
-  estPromptTokens: number;
-  warmTokens: number;
-  reasoningEffort: ReasoningEffort;
-  outputTokens?: number;
-}): FixedTurnCostEstimate | null {
-  const provider = promptCacheProviderForModel(args.model);
-  if (!provider || args.outputTokens === undefined) return null;
-
-  // One pass to the longest horizon, snapshotting the running total at each
-  // shorter one: turn N's cost depends on every turn before it, so the
-  // horizons are prefixes of the same accumulation rather than separate runs.
-  const horizons: number[] = [...FIXED_TURN_COST_PROJECTED_TURNS].sort(
-    (left, right) => left - right,
-  );
-  const longestHorizon = horizons[horizons.length - 1] ?? 0;
-  const projections: TurnCostProjection[] = [];
-  const outputTokens = args.outputTokens;
-  let totalCost = 0;
-
-  for (let turn = 1; turn <= longestHorizon; turn += 1) {
-    const inputTokens = args.estPromptTokens + (turn - 1) * outputTokens;
-    totalCost +=
-      turn === 1
-        ? deterministicTurnCost({
-            model: args.model,
-            price: args.price,
-            inputTokens,
-            cachedTokens: args.warmTokens,
-            outputTokens,
-          })
-        : laterTurnCost({
-            model: args.model,
-            price: args.price,
-            provider,
-            inputTokens,
-            // Later requests can only read prefixes that appeared in prior
-            // request inputs. The immediately previous output is fresh on the
-            // next request.
-            reusablePrefix: args.estPromptTokens + (turn - 2) * outputTokens,
-            outputTokens,
-          });
-    if (horizons.includes(turn)) {
-      projections.push({ projected_turns: turn, total_cost_usd: totalCost });
-    }
-  }
-
-  return {
-    output_tokens_per_turn: outputTokens,
-    assumed_reasoning_effort: args.reasoningEffort,
-    projections,
-  };
-}
-
-function laterTurnCost(args: {
-  model: string;
-  price: ModelPricing;
-  provider: PromptCacheProvider;
-  inputTokens: number;
-  reusablePrefix: number;
-  outputTokens: number;
-}): number {
-  const cacheableTokens = providerCacheableTokens(
-    args.model,
-    args.provider,
-    args.reusablePrefix,
-  );
-  const missCost = deterministicTurnCost({
-    model: args.model,
-    price: args.price,
-    inputTokens: args.inputTokens,
-    cachedTokens: 0,
-    outputTokens: args.outputTokens,
-  });
-  if (cacheableTokens <= 0) return missCost;
-
-  const hitCost = deterministicTurnCost({
-    model: args.model,
-    price: args.price,
-    inputTokens: args.inputTokens,
-    cachedTokens: cacheableTokens,
-    outputTokens: args.outputTokens,
-  });
-  const hitProbability = FIXED_TURN_CACHE_HIT_PROBABILITY[args.provider];
-  return hitCost * hitProbability + missCost * (1 - hitProbability);
-}
-
-// The loop cost at one horizon, for callers that need a single scalar rather
-// than the whole curve.
-export function turnCostUsdAt(
-  estimate: FixedTurnCostEstimate | null | undefined,
-  turns: number,
-): number | null {
-  const projection = estimate?.projections.find(
-    (entry) => entry.projected_turns === turns,
-  );
-  return projection?.total_cost_usd ?? null;
-}
-
-// A caching provider writes whatever the prompt did not read, and several bill
-// that write above their input rate. An effort-keyed cache partitions on effort,
-// so a switch of either model or effort leaves nothing warm and writes it all.
-function freshInputRate(model: string, price: ModelPricing, promptTokens: number): number {
-  return price.cacheWrite > 0 && promptTokens >= providerMinCacheTokens(model)
-    ? price.cacheWrite
-    : price.input;
-}
-
-function deterministicTurnCost(args: {
-  model: string;
-  price: ModelPricing;
-  inputTokens: number;
-  cachedTokens: number;
-  outputTokens: number;
-}): number {
-  const freshTokens = args.inputTokens - args.cachedTokens;
-  return (
-    (args.cachedTokens * args.price.cacheRead) / MTOK +
-    (freshTokens * freshInputRate(args.model, args.price, args.inputTokens)) / MTOK +
-    (args.outputTokens * args.price.output) / MTOK
-  );
-}