@mupt-ai/dari-gateway 0.0.0-stage → 0.2.0

package/src/context/provider_prompt_accounting.ts

@@ -0,0 +1,241 @@
+import {
+  isRecord,
+  stableStringify,
+  type CandidatePromptAccounting,
+} from "@mupt-ai/dari-router/policy-engine";
+
+import { toolsByWireName } from "../interaction/tool_identity";
+import type {
+  InteractionContent,
+  InteractionRequest,
+  ProviderIdentity,
+} from "../interaction/types";
+import { buildContext } from "../providers/pi_context";
+import { isOpenAIResponsesApi } from "../providers/model_capabilities";
+import type { InteractionPrefixContext } from "./interaction_fingerprint";
+
+type PromptMetrics = {
+  blockDepth: number;
+  serializedChars: number;
+};
+
+const ACCOUNTING_IMAGE_URL = "data:image/png;base64,AA==";
+const IMAGE_ESTIMATED_TOKENS = {
+  low: 512,
+  auto: 1024,
+  high: 2048,
+  original: 2048,
+} as const;
+const CHARS_PER_ESTIMATED_TOKEN = 4;
+
+// Cost and cache-lookback estimates are candidate-specific because compatible
+// continuation state and provider serialization can change what each model
+// sees.
+export function providerPromptAccounting(args: {
+  request: InteractionRequest;
+  prefix: InteractionPrefixContext;
+  targetsByModel: ReadonlyMap<string, ProviderIdentity>;
+}): CandidatePromptAccounting {
+  const fullByModel = new Map<string, PromptMetrics>();
+  const prefixCharsByModelAndItem = new Map<string, number>();
+
+  const targetFor = (model: string): ProviderIdentity => {
+    const target = args.targetsByModel.get(model);
+    if (!target) {
+      throw new Error(`missing provider target for ${model}`);
+    }
+    return target;
+  };
+  const fullMetrics = (model: string): PromptMetrics => {
+    const cached = fullByModel.get(model);
+    if (cached) return cached;
+    const target = targetFor(model);
+    const metrics = promptMetrics(args.request, target);
+    fullByModel.set(model, metrics);
+    return metrics;
+  };
+
+  return {
+    anchorSemantics: "stored_input",
+    providerBlockDepthFor: (model) => fullMetrics(model).blockDepth,
+    promptEstimateFor: (model, anchor) => {
+      const full = fullMetrics(model);
+      if (!anchor) {
+        return {
+          chars: full.serializedChars,
+          reusesStoredPromptTokens: false,
+        };
+      }
+
+      const itemIndex = args.prefix.itemIndexByHashByModel
+        .get(model)
+        ?.get(anchor.hash);
+      if (itemIndex === undefined) {
+        return {
+          chars: full.serializedChars,
+          reusesStoredPromptTokens: false,
+        };
+      }
+
+      const key = `${model}\u0000${itemIndex}`;
+      let prefixChars = prefixCharsByModelAndItem.get(key);
+      if (prefixChars === undefined) {
+        prefixChars = promptMetrics(
+          {
+            ...args.request,
+            items: args.request.items.slice(0, itemIndex + 1),
+          },
+          targetFor(model),
+        ).serializedChars;
+        prefixCharsByModelAndItem.set(key, prefixChars);
+      }
+      return {
+        chars: Math.max(0, full.serializedChars - prefixChars),
+        reusesStoredPromptTokens: true,
+      };
+    },
+  };
+}
+
+export function promptMetrics(
+  request: InteractionRequest,
+  target: ProviderIdentity,
+): PromptMetrics {
+  const accountingInput = promptAccountingInput(request);
+  const context = buildContext(accountingInput.request, target);
+  const messages = context.messages.flatMap((message) => {
+    const visible = providerVisibleMessage(message);
+    return visible ? [visible] : [];
+  });
+  const tools = providerVisibleTools(request, context.tools, target);
+  const prompt = {
+    ...(context.systemPrompt !== undefined
+      ? { systemPrompt: context.systemPrompt }
+      : {}),
+    ...(tools.length ? { tools } : {}),
+    messages,
+    ...(request.responseFormat !== undefined
+      ? { responseFormat: request.responseFormat }
+      : {}),
+    ...(request.toolChoice !== undefined
+      ? { toolChoice: request.toolChoice }
+      : {}),
+    ...(request.parallelToolCalls !== undefined
+      ? { parallelToolCalls: request.parallelToolCalls }
+      : {}),
+  };
+  return {
+    blockDepth: messages.reduce(
+      (total, message) => total + providerVisibleBlockCount(message),
+      0,
+    ),
+    serializedChars:
+      stableStringify(prompt).length + accountingInput.imageEstimatedChars,
+  };
+}
+
+function providerVisibleMessage(
+  value: unknown,
+): Record<string, unknown> | null {
+  if (!isRecord(value) || typeof value.role !== "string") return null;
+  if (value.role === "assistant") {
+    return {
+      role: value.role,
+      content: sanitizeProviderContent(
+        Array.isArray(value.content) ? value.content : [],
+      ),
+    };
+  }
+  if (value.role === "toolResult") {
+    return {
+      role: value.role,
+      toolCallId: value.toolCallId ?? null,
+      toolName: value.toolName ?? null,
+      content: sanitizeProviderContent(
+        Array.isArray(value.content) ? value.content : [],
+      ),
+      isError: value.isError === true,
+    };
+  }
+  return {
+    role: value.role,
+    content: sanitizeProviderContent(value.content ?? ""),
+  };
+}
+
+function sanitizeProviderContent(value: unknown): unknown {
+  if (!Array.isArray(value)) return value;
+  return value.map((part) => {
+    if (!isRecord(part)) return part;
+    if (part.type === "providerItem" && isRecord(part.item)) {
+      return part.item;
+    }
+    if (
+      part.type === "image"
+      && typeof part.mimeType === "string"
+      && typeof part.data === "string"
+    ) {
+      return { ...part, data: "<image-data>" };
+    }
+    return part;
+  });
+}
+
+function providerVisibleBlockCount(
+  message: Record<string, unknown>,
+): number {
+  if (message.role === "toolResult") return 1;
+  const content = message.content;
+  return Math.max(1, Array.isArray(content) ? content.length : 1);
+}
+
+function providerVisibleTools(
+  request: InteractionRequest,
+  contextTools: Array<Record<string, unknown>> | undefined,
+  target: ProviderIdentity,
+): unknown[] {
+  const definitions = toolsByWireName(request.tools);
+  const tools = (contextTools ?? []).map((tool) => {
+    if (typeof tool.name !== "string") return tool;
+    const definition = definitions.get(tool.name);
+    return definition?.kind !== "function"
+      || definition.strict === undefined
+      ? tool
+      : { ...tool, strict: definition.strict };
+  });
+  if (!isOpenAIResponsesApi(target.api)) return tools;
+  return [
+    ...tools,
+    ...request.tools.flatMap((tool) =>
+      tool.kind === "hosted" ? [tool.config] : []
+    ),
+  ];
+}
+
+function promptAccountingInput(request: InteractionRequest): {
+  request: InteractionRequest;
+  imageEstimatedChars: number;
+} {
+  let imageEstimatedChars = 0;
+  const sanitizeContent = (parts: InteractionContent[]) =>
+    parts.map((part) => {
+      if (part.type !== "image") return part;
+      const detail = part.detail ?? "auto";
+      imageEstimatedChars +=
+        IMAGE_ESTIMATED_TOKENS[detail] * CHARS_PER_ESTIMATED_TOKEN;
+      return { ...part, url: ACCOUNTING_IMAGE_URL };
+    });
+  const items = request.items.map((item) => {
+    if (item.type === "message") {
+      return { ...item, content: sanitizeContent(item.content) };
+    }
+    if (item.type === "tool_result" && Array.isArray(item.output)) {
+      return { ...item, output: sanitizeContent(item.output) };
+    }
+    return item;
+  });
+  return {
+    request: { ...request, items },
+    imageEstimatedChars,
+  };
+}

package/src/context/routing_context.ts

@@ -0,0 +1,337 @@
+import {
+  prepareCandidates,
+  type ActiveLease,
+  type PendingLease,
+  type CandidateCostEstimate,
+  type CandidateModelMetadata,
+  type ChainHits,
+  type PrefixHit,
+  type ChatMessage,
+  type HardCapability,
+  type PreviousDecision,
+  type PricingLookup,
+  type SelectorLeaseHistoryEntry,
+  type StrategyCandidateResolution,
+} from "@mupt-ai/dari-router/policy-engine";
+import { randomUUID } from "node:crypto";
+
+import { HttpError } from "../http/errors";
+import { effectiveContextWindow } from "../models/context_limits";
+import { keySourceForModel } from "../models/execution_provider";
+import { requiredCapabilitiesFor } from "../interaction/capabilities";
+import { providerIdentityFromModel } from "../interaction/provider_continuation_state";
+import { isBackgroundRequest, type InteractionRequest, type ProviderIdentity } from "../interaction/types";
+import { requireCandidateModels } from "../models/ids";
+import { modelMetadataLookup } from "../providers/pi";
+import {
+  modelSupportsImageInput,
+  modelSupportsHostedWebSearch,
+  modelSupportsReasoningConfigurationUpdate,
+  providerApiCanSerializeResponseFormat,
+} from "../providers/model_capabilities";
+import {
+  selectorMessagesForInteraction,
+  type SelectorImage,
+} from "../strategies/selector_projection";
+import type { ResolvedRouter } from "../types";
+import {
+  interactionPrefixContext,
+  interactionPrefixLookupHashes,
+  type InteractionPrefixContext,
+} from "./interaction_fingerprint";
+import { pricingLookupForRouter } from "./pricing";
+import { providerPromptAccounting } from "./provider_prompt_accounting";
+import { leaseHistoryFromPrefixHits } from "./lease_history";
+import { interactionLeaseFenceDepth } from "./interaction_lease_depth";
+
+export type ImageCapabilityPrunedModel = {
+  model: string;
+  reason: "image_input_unsupported";
+};
+
+export type RoutingContext = {
+  // Target-independent request depth used only to fence asynchronous lease
+  // writes. Provider cache lookback uses providerBlockDepthFor instead.
+  leaseDepth: number;
+  hits: ChainHits;
+  reportingCostEstimates: CandidateCostEstimate[];
+  previousDecision?: PreviousDecision;
+  // Unexpired lease commitment recovered from this conversation's prefix
+  // chain; the runtime serves it without consulting the selector.
+  activeLease?: ActiveLease;
+  // Next lease prefetched before expiry; adopted when no lease is active.
+  pendingLease?: PendingLease;
+  pricing: PricingLookup;
+  conversationId: string;
+  isNewConversation: boolean;
+  // Depth of the identity hit that recovered the conversation id (strict or
+  // loose tier, warm or merely recent); null for a fresh conversation.
+  identityMatchDepth: number | null;
+  identityHit?: PrefixHit;
+  zdr: boolean;
+  candidateResolution: StrategyCandidateResolution;
+  // Models excluded before strategy resolution because the pending request
+  // carries image content they cannot accept; absent when nothing was pruned.
+  imageCapabilityPrunedModels?: ImageCapabilityPrunedModel[];
+  prefix: InteractionPrefixContext;
+  targetsByModel: ReadonlyMap<string, ProviderIdentity>;
+  selectorMessages: ChatMessage[];
+  // Images referenced by `<image N>` placeholders in selectorMessages; empty
+  // unless the router opted into selector image input.
+  selectorImages: SelectorImage[];
+  // Absent when the router has lease context disabled: no reconstruction runs
+  // and downstream selector inputs must stay in the pre-lease-context shape.
+  leaseHistory?: SelectorLeaseHistoryEntry[];
+  providerBlockDepthFor: (model: string) => number;
+};
+
+// Router resolution only needs lookup hashes. Model-specific chains are
+// derived after resolve, when the enabled providers and APIs are known.
+export function derivePrefixLookupHashes(
+  routerId: string,
+  interaction: InteractionRequest,
+): string[] {
+  try {
+    return interactionPrefixLookupHashes(routerId, interaction);
+  } catch (error) {
+    console.error("prefix fingerprinting failed", error);
+    return [];
+  }
+}
+
+export async function buildRoutingContext(args: {
+  routerId: string;
+  interaction: InteractionRequest;
+  credentials: ResolvedRouter;
+  previousDecision?: PreviousDecision;
+  explicitConversationId?: string;
+  // Warmth freshness is measured from request arrival; callers without a
+  // request clock (experiments) default to now.
+  receivedAt?: Date;
+}): Promise<RoutingContext> {
+  const configuredCandidateModels = requireCandidateModels(
+    args.credentials.enabled_models,
+  );
+  // With credits exhausted, route only among models a personal subscription
+  // serves. The credit gate refuses routers with none.
+  const fundedCandidateModels = args.credentials.credit_exhausted
+    ? configuredCandidateModels.filter((model) => keySourceForModel(args.credentials, model) === "subscription")
+    : configuredCandidateModels;
+  const { messages: selectorMessages, images: selectorImages } =
+    selectorMessagesForInteraction(args.interaction, {
+      imageInput: args.credentials.selector_image_input_enabled === true,
+    });
+  const leaseDepth = interactionLeaseFenceDepth(args.interaction);
+  const metadataLookup = modelMetadataLookup(args.credentials.model_configs);
+  const targetsByModel = new Map<string, ProviderIdentity>();
+  for (const model of configuredCandidateModels) {
+    targetsByModel.set(
+      model,
+      providerIdentityFromModel(metadataLookup(model).model),
+    );
+  }
+  const capabilityLookup = (modelId: string): CandidateModelMetadata => {
+    const metadata = metadataLookup(modelId);
+    return {
+      provider: typeof metadata.model.provider === "string" ? metadata.model.provider : "",
+      api: typeof metadata.model.api === "string" ? metadata.model.api : "",
+      supportsImageInput: modelSupportsImageInput(metadata),
+      supportsHostedWebSearch: modelSupportsHostedWebSearch(metadata),
+      supportsReasoningConfigurationUpdate: modelSupportsReasoningConfigurationUpdate(metadata),
+      supportsStructuredOutput: providerApiCanSerializeResponseFormat(metadata),
+      supportedThinkingLevels: metadata.supportedThinkingLevels,
+    };
+  };
+  // Image capability filtering (router setting, absent = ON) narrows the
+  // candidate set before strategy resolution — an image-bearing conversation
+  // must not route (or hold a lease) to a text-only model. Prefix chains and
+  // provider accounting still cover every configured model.
+  const imageFilter = resolveImageCapableCandidates({
+    candidateModels: fundedCandidateModels.length > 0 ? fundedCandidateModels : configuredCandidateModels,
+    requiredCapabilities: requiredCapabilitiesFor(args.interaction),
+    enabled: args.credentials.image_capability_filtering !== false,
+    supportsImageInput: (model) => capabilityLookup(model).supportsImageInput,
+  });
+  const prefix = derivePrefixContext({
+    routerId: args.routerId,
+    interaction: args.interaction,
+    targetsByModel,
+  });
+  const promptAccounting = providerPromptAccounting({
+    request: args.interaction,
+    prefix,
+    targetsByModel,
+  });
+  const pricing = pricingLookupForRouter(args.credentials);
+  // A validated harness identity is authoritative. Matching text from another
+  // thread must not contribute cache warmth or selector continuity.
+  const background = isBackgroundRequest(args.interaction);
+  const prefixHits = (background ? [] : args.credentials.prefix_hits ?? []).filter(
+    (hit) => args.explicitConversationId === undefined
+      || hit.conversation_id === args.explicitConversationId,
+  );
+  const prepared = prepareCandidates({
+    candidateModels: imageFilter.candidateModels,
+    metadataLookup: capabilityLookup,
+    requiredCapabilities: imageFilter.requiredCapabilities,
+    requestedReasoningEffort: args.interaction.reasoning?.effort,
+    thinkingEnabled: args.interaction.reasoning?.enabled === true,
+    toolChoice: args.interaction.toolChoice,
+    modelThinkingLevels: args.credentials.model_thinking_levels,
+    strategy: args.credentials.routing_strategy === "custom" ? "custom" : "dari",
+    customConfig: args.credentials.custom_config,
+    modelPrices: args.credentials.model_prices ?? {},
+    pricing,
+    promptTokenLimitFor: (model) => effectiveContextWindow(
+      args.credentials.model_configs?.[model],
+      args.credentials.allow_long_context === true,
+    ),
+    requiresPromptEstimateFor: (model) => args.credentials.allow_long_context !== true
+      && args.credentials.model_configs?.[model]?.standard_context_limit != null,
+    averageOutputTokensByModel: Object.fromEntries(
+      Object.entries(args.credentials.model_configs ?? {}).map(([model, config]) => [
+        model,
+        config.average_output_tokens ?? null,
+      ]),
+    ),
+    promptAccounting,
+    chainsByModel: prefix.chainByModel,
+    // Resolve returns hits up to the platform's conversation lookback;
+    // core narrows them to the warmth window for cost estimation and keeps the
+    // rest for conversation identity and the previous decision.
+    prefixHits,
+    nowMs: (args.receivedAt ?? new Date()).getTime(),
+    looseChain: prefix.looseChain,
+    conversationId: args.explicitConversationId,
+    toolChoiceFp: prefix.fingerprints.tool_choice_fp,
+    responseFormatFp: prefix.fingerprints.response_format_fp,
+    previousDecision: background ? undefined : args.previousDecision,
+  });
+  for (const warning of prepared.warnings) {
+    console.error("cache cost estimation failed", warning.error);
+  }
+
+  const conversationId = args.explicitConversationId
+    ?? prepared.conversationId
+    ?? randomUUID();
+  // A harness identity always produces an explicit conversation id, so it
+  // cannot distinguish a brand-new session from a recovered one on its own.
+  // Prefix lookup is already scoped to that id; no recovered conversation
+  // therefore means this is its first observed turn.
+  const isNewConversation = prepared.conversationId === null;
+  // Lease context (router setting, absent = ON) feeds the selector the
+  // original task and reconstructed lease history. Disabled routers skip the
+  // reconstruction entirely so their selector inputs stay byte-identical to
+  // the pre-lease-context behavior.
+  const leaseHistory = !background && args.credentials.lease_context_enabled !== false
+    ? leaseHistoryFromPrefixHits({
+        prefixHits,
+        conversationId,
+        candidates: prepared.candidateResolution.candidates,
+        messages: selectorMessages,
+      })
+    : undefined;
+  return {
+    leaseDepth,
+    hits: prepared.hits,
+    reportingCostEstimates: prepared.costEstimates,
+    ...(prepared.previousDecision !== undefined
+      ? { previousDecision: prepared.previousDecision }
+      : {}),
+    ...(prepared.activeLease !== undefined ? { activeLease: prepared.activeLease } : {}),
+    ...(prepared.pendingLease !== undefined ? { pendingLease: prepared.pendingLease } : {}),
+    pricing,
+    conversationId,
+    isNewConversation,
+    identityMatchDepth: prepared.identityMatchDepth,
+    ...(prepared.identityHit !== undefined
+      ? { identityHit: prepared.identityHit }
+      : {}),
+    zdr: args.credentials.zdr === true,
+    candidateResolution: prepared.candidateResolution,
+    ...(imageFilter.prunedModels.length > 0
+      ? { imageCapabilityPrunedModels: imageFilter.prunedModels }
+      : {}),
+    prefix,
+    targetsByModel,
+    selectorMessages,
+    selectorImages,
+    ...(leaseHistory !== undefined ? { leaseHistory } : {}),
+    providerBlockDepthFor: promptAccounting.providerBlockDepthFor,
+  };
+}
+
+// When the filter is disabled, "image_input" is also dropped from the hard
+// capabilities so core preparation keeps today's pass-through behavior instead
+// of re-applying the same exclusion. When enabled, the capability stays as a
+// core-side invariant over the already-narrowed set.
+function resolveImageCapableCandidates(args: {
+  candidateModels: string[];
+  requiredCapabilities: HardCapability[];
+  enabled: boolean;
+  supportsImageInput: (model: string) => boolean;
+}): {
+  candidateModels: string[];
+  requiredCapabilities: HardCapability[];
+  prunedModels: ImageCapabilityPrunedModel[];
+} {
+  if (!args.requiredCapabilities.includes("image_input")) {
+    return {
+      candidateModels: args.candidateModels,
+      requiredCapabilities: args.requiredCapabilities,
+      prunedModels: [],
+    };
+  }
+  if (!args.enabled) {
+    return {
+      candidateModels: args.candidateModels,
+      requiredCapabilities: args.requiredCapabilities.filter(
+        (capability) => capability !== "image_input",
+      ),
+      prunedModels: [],
+    };
+  }
+  const capable = args.candidateModels.filter(args.supportsImageInput);
+  if (capable.length === 0) {
+    throw new HttpError(
+      400,
+      "This conversation contains image input, but no enabled model on this router accepts images.",
+      "no_image_capable_candidates",
+    );
+  }
+  return {
+    candidateModels: capable,
+    requiredCapabilities: args.requiredCapabilities,
+    prunedModels: args.candidateModels
+      .filter((model) => !args.supportsImageInput(model))
+      .map((model) => ({ model, reason: "image_input_unsupported" as const })),
+  };
+}
+
+function derivePrefixContext(args: {
+  routerId: string;
+  interaction: InteractionRequest;
+  targetsByModel: ReadonlyMap<string, ProviderIdentity>;
+}): InteractionPrefixContext {
+  try {
+    return interactionPrefixContext(
+      args.routerId,
+      args.interaction,
+      args.targetsByModel,
+    );
+  } catch (error) {
+    console.error("prefix fingerprinting failed", error);
+    return {
+      chainByModel: new Map(
+        [...args.targetsByModel].map(([model]) => [model, []]),
+      ),
+      itemIndexByHashByModel: new Map(
+        [...args.targetsByModel].map(
+          ([model]) => [model, new Map<string, number>()],
+        ),
+      ),
+      looseChain: [],
+      fingerprints: { tool_choice_fp: "", response_format_fp: "" },
+    };
+  }
+}

package/src/http/auth.ts

@@ -0,0 +1,43 @@
+import { timingSafeEqual } from "node:crypto";
+
+import { HttpError } from "./errors";
+
+export function bearerToken(headers: Headers): string {
+  const value = headers.get("authorization") ?? "";
+  const [scheme, token] = value.split(/\s+/, 2);
+  if (scheme?.toLowerCase() !== "bearer" || !token) {
+    throw new HttpError(401, "Missing or invalid routing API key.", "authentication_error");
+  }
+  return publicRoutingToken(token);
+}
+
+export function anthropicToken(headers: Headers): string {
+  const authorization = headers.get("authorization");
+  if (authorization) return bearerToken(headers);
+  const apiKey = headers.get("x-api-key");
+  if (!apiKey) {
+    throw new HttpError(401, "Missing or invalid routing API key.", "authentication_error");
+  }
+  return publicRoutingToken(apiKey);
+}
+
+function publicRoutingToken(token: string): string {
+  const internalToken = process.env.DARI_ROUTING_INTERNAL_TOKEN;
+  if (internalToken && matchesInternalToken(token, internalToken)) {
+    throw new HttpError(
+      401,
+      "Internal routing credentials cannot authorize public inference.",
+      "authentication_error",
+    );
+  }
+  return token;
+}
+
+function matchesInternalToken(token: string, internalToken: string): boolean {
+  // Compare byte lengths, not character counts: timingSafeEqual throws on
+  // buffers of different sizes, and a multi-byte character can match the
+  // secret's character count without matching its byte length.
+  const candidate = Buffer.from(token);
+  const expected = Buffer.from(internalToken);
+  return candidate.length === expected.length && timingSafeEqual(candidate, expected);
+}