@mupt-ai/dari-gateway 0.1.0 → 0.2.0

package/README.md

@@ -111,7 +111,14 @@ const handler = createRequestHandler({ platform, selectorBackend: createPriority
 Bun.serve({ port: 8080, fetch: handler, idleTimeout: 0 });
 ```
 
-`src/self_host/` is the reference platform.
+The root entry needs no Bun APIs, so any runtime with web-standard `fetch`, `Request`, and `Response` can serve the handler. `startServer` and `startSelfHostedGateway` (Bun.serve and bun:sqlite) are in `@mupt-ai/dari-gateway/bun`. `src/self_host/` is the reference platform.
+
+Platforms can also:
+
+- implement `resolveProviderCredential({ provider, model, routerId, organizationId, apiKey })` to supply a per-request `{ apiKey, baseUrl?, headers? }`, such as a tenant's own key, in place of the router's resolved key;
+- resolve routers with `strict: true` to serve only the requested `dari/model/<id>` at the requested thinking level, with no selector, lease, fallback, or personal subscription;
+- read exact usage from `recordRequest`: the summary carries input, output, cache-read, and cache-write tokens and `total_cost_usd`, preferring the provider's billed cost (`cost_source: "provider"`, from OpenRouter and Vercel AI Gateway). Failed and aborted requests are recorded too;
+- attribute cost with `X-Dari-Tags: run=r1,trial=3`, which is parsed into `RequestRecord.tags` and the `recordUsage` payload's `tags`.
 
 ## Development
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mupt-ai/dari-gateway",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Self-hostable LLM gateway for coding agents and APIs: one endpoint for Anthropic Messages, OpenAI Chat Completions, and OpenAI Responses, with subscription pooling, failover, and prompt-cache-aware routing.",
   "license": "Apache-2.0",
   "repository": {
@@ -10,6 +10,7 @@
   "type": "module",
   "exports": {
     ".": "./src/index.ts",
+    "./bun": "./src/bun.ts",
     "./*": "./src/*.ts"
   },
   "bin": {
@@ -19,6 +20,7 @@
     "src",
     "!src/**/*.test.ts",
     "!src/**/*.fixtures.ts",
+    "!src/**/*.cases.ts",
     "!src/test_support",
     "!src/**/test_support.ts",
     "dari-gateway.example.yaml",

package/src/bun.ts

@@ -0,0 +1,4 @@
+// Bun-only entry points: the Bun.serve HTTP server and the self-hosted
+// gateway, whose state lives in bun:sqlite.
+export { installShutdownHandlers, startServer } from "./http/serve";
+export { startSelfHostedGateway } from "./self_host/main";

package/src/context/routing_context.ts

@@ -122,7 +122,7 @@ export async function buildRoutingContext(args: {
       imageInput: args.credentials.selector_image_input_enabled === true,
     });
   const leaseDepth = interactionLeaseFenceDepth(args.interaction);
-  const metadataLookup = await modelMetadataLookup(args.credentials.model_configs);
+  const metadataLookup = modelMetadataLookup(args.credentials.model_configs);
   const targetsByModel = new Map<string, ProviderIdentity>();
   for (const model of configuredCandidateModels) {
     targetsByModel.set(

package/src/index.ts

@@ -1,21 +1,22 @@
 // Public entry points. Every module is also importable by path
 // (`@mupt-ai/dari-gateway/<dir>/<module>`) for deployments that compose the
-// gateway with their own platform.
+// gateway with their own platform. This entry needs no Bun APIs; the Bun
+// server and the SQLite-backed self-hosted gateway live in `./bun`.
 export { createRequestHandler, type GatewayHandlerOptions } from "./http/handler";
-export { installShutdownHandlers, startServer } from "./http/serve";
 export type {
   GatewayPlatform,
   OperationalErrorContext,
   PendingLeaseWrite,
   PrefixStateWrite,
+  ProviderCredentialRequest,
   RequestErrorContext,
   RequestRecord,
   RequestSummary,
+  RequestTags,
   ResolveRouterRequest,
   SubscriptionHold,
 } from "./platform/platform";
 export { createAutoRoutingSelectorBackend } from "./strategies/auto_routing_selector";
 export { createPrioritySelectorBackend } from "./strategies/priority_selector";
 export { createLlmSelectorBackend, type SelectorBackend } from "./strategies/routing";
-export type { ResolvedRouter } from "./types";
-export { startSelfHostedGateway } from "./self_host/main";
+export type { ProviderCredential, ResolvedRouter } from "./types";

package/src/ingress/models.ts

@@ -33,7 +33,7 @@ export async function handleModels(request: Request, routerId: string): Promise<
         },
       });
     }
-    const metadataLookup = await modelMetadataLookup(router.model_configs);
+    const metadataLookup = modelMetadataLookup(router.model_configs);
     const imageInputFor = (model: string) => modelSupportsImageInput(metadataLookup(model));
     return Response.json(codexModelsResponse(router, imageInputFor), {
       headers: {

package/src/interaction/types.ts

@@ -264,6 +264,9 @@ export type InteractionUsage = {
     cacheWrite: number;
     total: number;
   };
+  // "provider" when cost.total is the provider's billed amount rather than a
+  // price from the model's rates.
+  costSource?: "provider";
 };
 
 export type InteractionOutput = {

package/src/platform/platform.ts

@@ -6,7 +6,12 @@ import type {
 import type { HarnessConversationIdentity } from "../runtime/harness_identity";
 import type { InteractionRequest, SourceProtocol } from "../interaction/types";
 import type { SubscriptionUsageLimit } from "../providers/subscription_limits";
-import type { ClientHarness, ProviderKeySource, ResolvedRouter } from "../types";
+import type {
+  ClientHarness,
+  ProviderCredential,
+  ProviderKeySource,
+  ResolvedRouter,
+} from "../types";
 
 /**
  * Everything the gateway runtime needs from the deployment around it: router
@@ -24,6 +29,15 @@ export type GatewayPlatform = {
    * `prefixHashes`. Throws HttpError(401) for an unknown key.
    */
   resolveRouter(request: ResolveRouterRequest): Promise<ResolvedRouter>;
+  /**
+   * Optional. Supply the credential for one enabled model of this request,
+   * consulted after `resolveRouter` and before its provider keys (for the
+   * self-hosted gateway, its environment keys), so an embedder can serve
+   * per-tenant keys from its own store. Undefined keeps the resolved key.
+   */
+  resolveProviderCredential?(
+    request: ProviderCredentialRequest,
+  ): Promise<ProviderCredential | undefined>;
   /** Persist one completed turn's prefix entry and/or refresh a read entry. */
   writePrefixState(write: PrefixStateWrite): Promise<void>;
   /** Set or clear a conversation's prefetched routing lease. */
@@ -53,6 +67,15 @@ export type ResolveRouterRequest = {
   clientHarness?: ClientHarness;
 };
 
+export type ProviderCredentialRequest = {
+  provider: Provider;
+  model: string;
+  routerId: string;
+  organizationId: string;
+  // The gateway key the request authenticated with.
+  apiKey: string;
+};
+
 export type PrefixStateWrite = {
   organization_id: string;
   router_id: string;
@@ -94,8 +117,14 @@ export type RequestRecord = {
   isNewConversation: boolean;
   zdr: boolean;
   harnessConversationIdentity?: HarnessConversationIdentity;
+  // Parsed from the request's X-Dari-Tags header; empty when absent.
+  tags: RequestTags;
 };
 
+// Caller-supplied attribution labels (run, trial, setting), opaque to the
+// gateway.
+export type RequestTags = Record<string, string>;
+
 export type RequestSummary = {
   request_id: string;
   organization_id: string;
@@ -130,7 +159,10 @@ export type RequestSummary = {
   cache_write_tokens: number | null;
   output_tokens: number | null;
   total_tokens: number | null;
+  // Provider-reported cost when the provider returns one (OpenRouter, Vercel
+  // AI Gateway), otherwise priced from the model's rates when known.
   total_cost_usd: number | null;
+  cost_source: "provider" | "pricing" | null;
   pricing_known: boolean | null;
 };
 

package/src/providers/pi.ts

@@ -1,3 +1,22 @@
+import {
+  getSupportedThinkingLevels,
+  normalizeContext,
+  type Api,
+  type AssistantMessageEventStream,
+  type Context,
+  type Model,
+  type ProviderStreams,
+} from "@mupt-ai/pi-ai";
+import { anthropicMessagesApi } from "@mupt-ai/pi-ai/api/anthropic-messages.lazy";
+import { azureOpenAIResponsesApi } from "@mupt-ai/pi-ai/api/azure-openai-responses.lazy";
+import { bedrockConverseStreamApi } from "@mupt-ai/pi-ai/api/bedrock-converse-stream.lazy";
+import { googleGenerativeAIApi } from "@mupt-ai/pi-ai/api/google-generative-ai.lazy";
+import { googleVertexApi } from "@mupt-ai/pi-ai/api/google-vertex.lazy";
+import { mistralConversationsApi } from "@mupt-ai/pi-ai/api/mistral-conversations.lazy";
+import { openAICodexResponsesApi } from "@mupt-ai/pi-ai/api/openai-codex-responses.lazy";
+import { openAICompletionsApi } from "@mupt-ai/pi-ai/api/openai-completions.lazy";
+import { openAIResponsesApi } from "@mupt-ai/pi-ai/api/openai-responses.lazy";
+import { getBuiltinModel } from "@mupt-ai/pi-ai/providers/all";
 import {
   REASONING_EFFORTS,
   isRecord,
@@ -22,6 +41,12 @@ import {
   messageWithSafeguardResults,
   safeguardResultsTap,
 } from "./anthropic_safeguards";
+import {
+  eventsWithProviderReportedCost,
+  messageWithProviderReportedCost,
+  providerReportedCostTap,
+  reportsProviderCost,
+} from "./provider_reported_cost";
 import {
   eventsWithSubscriptionCacheWrites,
   messageWithSubscriptionCacheWrites,
@@ -43,6 +68,12 @@ export type ModelRuntimeMetadata = {
   supportedThinkingLevels: string[];
 };
 
+// A platform-supplied endpoint for one call (see ProviderCredential).
+export type ProviderEndpoint = {
+  baseUrl?: string;
+  headers?: Record<string, string>;
+};
+
 export type ProviderCompletion = {
   message: Record<string, unknown>;
   source: ProviderIdentity;
@@ -55,14 +86,40 @@ export type ProviderEventStream = {
 
 export type ModelMetadataLookup = (modelId: string, pricing?: ModelPricing | null) => ModelRuntimeMetadata;
 
-let piAiPromise: Promise<PiAi> | null = null;
+// Pi's built-in providers stream through these API implementations unchanged.
+// Dari always passes resolved credentials, so it dispatches on the API rather
+// than through a Models collection with its own auth resolution.
+const PI_APIS: Record<string, ProviderStreams> = {
+  "anthropic-messages": anthropicMessagesApi(),
+  "azure-openai-responses": azureOpenAIResponsesApi(),
+  "bedrock-converse-stream": bedrockConverseStreamApi(),
+  "google-generative-ai": googleGenerativeAIApi(),
+  "google-vertex": googleVertexApi(),
+  "mistral-conversations": mistralConversationsApi(),
+  "openai-codex-responses": openAICodexResponsesApi(),
+  "openai-completions": openAICompletionsApi(),
+  "openai-responses": openAIResponsesApi(),
+};
 
-async function piAi(): Promise<PiAi> {
-  // Pi 0.80 keeps the legacy global dispatch and catalog surface under /compat.
-  piAiPromise ??= import("@mupt-ai/pi-ai/compat") as unknown as Promise<PiAi>;
-  return piAiPromise;
+function piStreamSimple(
+  model: Record<string, unknown>,
+  context: Record<string, unknown>,
+  options: Record<string, unknown>,
+): AssistantMessageEventStream {
+  const api = PI_APIS[String(model.api)];
+  if (!api) throw new Error(`No Pi API implementation for ${String(model.api)}.`);
+  return api.streamSimple(model as unknown as Model<Api>, normalizeContext(context as unknown as Context), options);
 }
 
+export const pi: PiAi = {
+  completeSimple: async (model, context, options) =>
+    await piStreamSimple(model, context, options).result() as unknown as Record<string, unknown>,
+  streamSimple: (model, context, options) =>
+    piStreamSimple(model, context, options) as unknown as AsyncIterable<Record<string, unknown>>,
+  getModel: (provider, id) => getBuiltinModel(provider as never, id as never) as unknown as Record<string, unknown> | undefined,
+  getSupportedThinkingLevels: (model) => getSupportedThinkingLevels(model as unknown as Model<Api>),
+};
+
 export async function completeInteractionModel(args: {
   model: string;
   request: InteractionRequest;
@@ -70,18 +127,19 @@ export async function completeInteractionModel(args: {
   auth?: ResolvedProviderAuth;
   pricing?: ModelPricing | null;
   config?: ResolvedRouterModelConfig;
+  endpoint?: ProviderEndpoint;
   sessionId?: string;
   selectedReasoningEffort?: ReasoningEffort;
   timeoutMs?: number;
   signal?: AbortSignal;
   fetch?: ProviderFetch;
 }): Promise<ProviderCompletion> {
-  const pi = await piAi();
-  const model = buildModel(pi, args.model, args.pricing, args.config);
+  const model = modelAtEndpoint(buildModel(pi, args.model, args.pricing, args.config), args.endpoint);
   const source = providerIdentityFromModel(model);
   const context = buildContext(args.request, source);
+  const costTap = reportsProviderCost(model) ? providerReportedCostTap(args.fetch) : undefined;
   const safeguards = forwardsAnthropicSafeguards(args.request, model)
-    ? safeguardResultsTap(args.fetch)
+    ? safeguardResultsTap(costTap?.fetch ?? args.fetch)
     : undefined;
   const options = buildOptions(
     args.request,
@@ -93,16 +151,19 @@ export async function completeInteractionModel(args: {
     args.selectedReasoningEffort,
     args.timeoutMs,
     args.signal,
-    safeguards?.fetch ?? args.fetch,
+    safeguards?.fetch ?? costTap?.fetch ?? args.fetch,
   );
   const usageContext = subscriptionUsageContext(args.model, model, source);
   const message = options.emitProviderEvents === true
     ? await completeProviderEventStream(pi.streamSimple(model, context, options))
     : await pi.completeSimple(model, context, options);
   return {
-    message: messageWithSubscriptionCacheWrites(
-      messageWithSafeguardResults(message, safeguards?.results()),
-      usageContext
+    message: messageWithProviderReportedCost(
+      messageWithSubscriptionCacheWrites(
+        messageWithSafeguardResults(message, safeguards?.results()),
+        usageContext
+      ),
+      costTap?.cost(),
     ),
     source,
   };
@@ -167,16 +228,17 @@ export async function streamInteractionModel(args: {
   auth?: ResolvedProviderAuth;
   pricing?: ModelPricing | null;
   config?: ResolvedRouterModelConfig;
+  endpoint?: ProviderEndpoint;
   sessionId?: string;
   selectedReasoningEffort?: ReasoningEffort;
   signal?: AbortSignal;
   fetch?: ProviderFetch;
 }): Promise<ProviderEventStream> {
-  const pi = await piAi();
-  const model = buildModel(pi, args.model, args.pricing, args.config);
+  const model = modelAtEndpoint(buildModel(pi, args.model, args.pricing, args.config), args.endpoint);
   const source = providerIdentityFromModel(model);
+  const costTap = reportsProviderCost(model) ? providerReportedCostTap(args.fetch) : undefined;
   const safeguards = forwardsAnthropicSafeguards(args.request, model)
-    ? safeguardResultsTap(args.fetch)
+    ? safeguardResultsTap(costTap?.fetch ?? args.fetch)
     : undefined;
   const events = pi.streamSimple(
     model,
@@ -191,20 +253,34 @@ export async function streamInteractionModel(args: {
       args.selectedReasoningEffort,
       undefined,
       args.signal,
-      safeguards?.fetch ?? args.fetch,
+      safeguards?.fetch ?? costTap?.fetch ?? args.fetch,
     )
   );
+  const withCacheWrites = eventsWithSubscriptionCacheWrites(
+    safeguards ? eventsWithSafeguardResults(events, safeguards) : events,
+    subscriptionUsageContext(args.model, model, source)
+  );
   return {
-    events: eventsWithSubscriptionCacheWrites(
-      safeguards ? eventsWithSafeguardResults(events, safeguards) : events,
-      subscriptionUsageContext(args.model, model, source)
-    ),
+    events: costTap ? eventsWithProviderReportedCost(withCacheWrites, costTap) : withCacheWrites,
     source,
   };
 }
 
-export async function modelContextWindow(modelId: string): Promise<number | null> {
-  const pi = await piAi();
+function modelAtEndpoint(
+  model: Record<string, unknown>,
+  endpoint: ProviderEndpoint | undefined,
+): Record<string, unknown> {
+  if (endpoint === undefined) return model;
+  return {
+    ...model,
+    ...(endpoint.baseUrl ? { baseUrl: endpoint.baseUrl } : {}),
+    ...(endpoint.headers
+      ? { headers: { ...(isRecord(model.headers) ? model.headers : {}), ...endpoint.headers } }
+      : {}),
+  };
+}
+
+export function modelContextWindow(modelId: string): number | null {
   const provider = providerForModel(modelId);
   const nativeId = nativeModelId(modelId);
   const registered = resolveModelMetadata(pi, provider, modelId, nativeId);
@@ -215,11 +291,10 @@ export async function modelContextWindow(modelId: string): Promise<number | null
  * Pi's catalog price for a canonical model served by `provider`, in USD per
  * million tokens. Null when Pi does not list the model.
  */
-export async function registeredModelPrice(
+export function registeredModelPrice(
   modelId: string,
   provider: string = providerForModel(modelId),
-): Promise<RouterModelPrice | null> {
-  const pi = await piAi();
+): RouterModelPrice | null {
   const registered = resolveModelMetadata(pi, provider, modelId, nativeModelId(modelId));
   const cost = registered?.cost;
   if (!isRecord(cost) || typeof cost.input !== "number" || typeof cost.output !== "number") {
@@ -233,10 +308,9 @@ export async function registeredModelPrice(
   };
 }
 
-export async function modelMetadataLookup(
+export function modelMetadataLookup(
   configs?: Record<string, ResolvedRouterModelConfig>
-): Promise<ModelMetadataLookup> {
-  const pi = await piAi();
+): ModelMetadataLookup {
   return (modelId: string, pricing?: ModelPricing | null) => {
     const model = buildModel(pi, modelId, pricing, configs?.[modelId]);
     return {