@llmops/app 0.1.4 → 0.1.5-beta.2

package/dist/index.mjs

@@ -1,5 +1,5 @@
 import { Hono } from "hono";
-import { SupportedProviders, createDataLayer, generateId, validateLLMOpsConfig, variantJsonDataSchema } from "@llmops/core";
+import { SupportedProviders, createDataLayer, generateId, logger, validateLLMOpsConfig, variantJsonDataSchema } from "@llmops/core";
 import reactServer from "react-dom/server";
 import { Fragment, jsx, jsxs } from "react/jsx-runtime";
 import { fileURLToPath } from "node:url";
@@ -14,6 +14,7 @@ import { zValidator } from "@hono/zod-validator";
 import { prettyJSON } from "hono/pretty-json";
 import { HTTPException } from "hono/http-exception";
 import { cors } from "hono/cors";
+import { randomUUID } from "node:crypto";
 import gateway from "@llmops/gateway";
 import { env } from "node:process";
 import { createDatabaseFromConnection } from "@llmops/core/db";
@@ -36,7 +37,7 @@ var __export = (all, symbols) => {
 
 //#endregion
 //#region src/client/index.tsx?url
-var client_default = "/assets/index-nHWjhrFW.js";
+var client_default = "/assets/index-BCY9aD4r.js";
 
 //#endregion
 //#region src/client/styles/styles.css?url
@@ -12682,11 +12683,13 @@ var zod_default = external_exports;
 //#endregion
 //#region src/server/lib/zv.ts
 const zv = (target, schema) => zValidator(target, schema, (result, c) => {
-	if (!result.success)
- /**
-	* @todo handle error properly
-	*/
-	return c.json({ message: "Bad Request" }, 400);
+	if (!result.success) return c.json({
+		message: "Bad Request",
+		errors: result.error.issues.map((issue$1) => ({
+			path: issue$1.path.join("."),
+			message: issue$1.message
+		}))
+	}, 400);
 });
 
 //#endregion
@@ -12713,6 +12716,225 @@ const internalServerError = (message, code) => {
 	};
 };
 
+//#endregion
+//#region src/server/handlers/analytics/index.ts
+/**
+* Convert micro-dollars to formatted dollar string
+*/
+function formatCost(microDollars, decimals = 6) {
+	return `$${(microDollars / 1e6).toFixed(decimals)}`;
+}
+/**
+* Parse ISO date string to Date object
+* Accepts both ISO strings (2026-01-02T10:30:00.000Z) and date-only strings (2026-01-02)
+*/
+function parseDate(dateStr) {
+	const date$4 = new Date(dateStr);
+	if (isNaN(date$4.getTime())) throw new Error(`Invalid date string: ${dateStr}`);
+	return date$4;
+}
+/**
+* Parse date string for start of range
+* - ISO strings are used as-is
+* - Date-only strings (YYYY-MM-DD) are treated as start of day UTC
+*/
+function parseStartDate(dateStr) {
+	return parseDate(dateStr);
+}
+/**
+* Parse date string for end of range
+* - ISO strings are used as-is
+* - Date-only strings (YYYY-MM-DD) are set to end of day (23:59:59.999 UTC)
+*/
+function parseEndDate(dateStr) {
+	const date$4 = parseDate(dateStr);
+	if (!dateStr.includes("T")) date$4.setUTCHours(23, 59, 59, 999);
+	return date$4;
+}
+/**
+* Zod schema for ISO date strings
+* Validates that the string can be parsed as a valid date
+*/
+const isoDateString = zod_default.string().refine((val) => !isNaN(new Date(val).getTime()), { message: "Invalid date format. Expected ISO 8601 string (e.g., 2026-01-02T10:30:00.000Z) or date string (e.g., 2026-01-02)" });
+/**
+* Date range query schema
+* Accepts ISO 8601 date strings or date-only strings (YYYY-MM-DD)
+* - startDate: Used as-is for ISO strings, start of day for date-only
+* - endDate: Used as-is for ISO strings, end of day (23:59:59.999) for date-only
+*/
+const dateRangeSchema = zod_default.object({
+	startDate: isoDateString.transform(parseStartDate),
+	endDate: isoDateString.transform(parseEndDate)
+});
+/**
+* Analytics API routes for cost and usage tracking
+*/
+const app$10 = new Hono().get("/requests", zv("query", zod_default.object({
+	limit: zod_default.string().transform(Number).optional(),
+	offset: zod_default.string().transform(Number).optional(),
+	configId: zod_default.string().uuid().optional(),
+	provider: zod_default.string().optional(),
+	model: zod_default.string().optional(),
+	startDate: isoDateString.optional(),
+	endDate: isoDateString.optional()
+})), async (c) => {
+	const db = c.get("db");
+	const query = c.req.valid("query");
+	try {
+		const requests = await db.listRequests({
+			limit: query.limit,
+			offset: query.offset,
+			configId: query.configId,
+			provider: query.provider,
+			model: query.model,
+			startDate: query.startDate ? parseStartDate(query.startDate) : void 0,
+			endDate: query.endDate ? parseEndDate(query.endDate) : void 0
+		});
+		return c.json(successResponse(requests, 200));
+	} catch (error$45) {
+		console.error("Error fetching requests:", error$45);
+		return c.json(internalServerError("Failed to fetch requests", 500), 500);
+	}
+}).get("/requests/:requestId", zv("param", zod_default.object({ requestId: zod_default.string().uuid() })), async (c) => {
+	const db = c.get("db");
+	const { requestId } = c.req.valid("param");
+	try {
+		const request = await db.getRequestByRequestId(requestId);
+		if (!request) return c.json({ error: "Request not found" }, 404);
+		return c.json(successResponse(request, 200));
+	} catch (error$45) {
+		console.error("Error fetching request:", error$45);
+		return c.json(internalServerError("Failed to fetch request", 500), 500);
+	}
+}).get("/costs/total", zv("query", dateRangeSchema), async (c) => {
+	const db = c.get("db");
+	const { startDate, endDate } = c.req.valid("query");
+	try {
+		const data = await db.getTotalCost({
+			startDate,
+			endDate
+		});
+		if (!data) return c.json(successResponse({
+			totalCost: 0,
+			totalCostFormatted: "$0.000000",
+			totalInputCost: 0,
+			totalOutputCost: 0,
+			totalPromptTokens: 0,
+			totalCompletionTokens: 0,
+			totalTokens: 0,
+			requestCount: 0
+		}, 200));
+		return c.json(successResponse({
+			...data,
+			totalCostFormatted: formatCost(data.totalCost),
+			totalInputCostFormatted: formatCost(data.totalInputCost),
+			totalOutputCostFormatted: formatCost(data.totalOutputCost)
+		}, 200));
+	} catch (error$45) {
+		console.error("Error fetching total costs:", error$45);
+		return c.json(internalServerError("Failed to fetch total costs", 500), 500);
+	}
+}).get("/costs/by-model", zv("query", dateRangeSchema), async (c) => {
+	const db = c.get("db");
+	const { startDate, endDate } = c.req.valid("query");
+	try {
+		const data = await db.getCostByModel({
+			startDate,
+			endDate
+		});
+		return c.json(successResponse(data, 200));
+	} catch (error$45) {
+		console.error("Error fetching costs by model:", error$45);
+		return c.json(internalServerError("Failed to fetch costs by model", 500), 500);
+	}
+}).get("/costs/by-provider", zv("query", dateRangeSchema), async (c) => {
+	const db = c.get("db");
+	const { startDate, endDate } = c.req.valid("query");
+	try {
+		const data = await db.getCostByProvider({
+			startDate,
+			endDate
+		});
+		return c.json(successResponse(data, 200));
+	} catch (error$45) {
+		console.error("Error fetching costs by provider:", error$45);
+		return c.json(internalServerError("Failed to fetch costs by provider", 500), 500);
+	}
+}).get("/costs/by-config", zv("query", dateRangeSchema), async (c) => {
+	const db = c.get("db");
+	const { startDate, endDate } = c.req.valid("query");
+	try {
+		const data = await db.getCostByConfig({
+			startDate,
+			endDate
+		});
+		return c.json(successResponse(data, 200));
+	} catch (error$45) {
+		console.error("Error fetching costs by config:", error$45);
+		return c.json(internalServerError("Failed to fetch costs by config", 500), 500);
+	}
+}).get("/costs/daily", zv("query", dateRangeSchema), async (c) => {
+	const db = c.get("db");
+	const { startDate, endDate } = c.req.valid("query");
+	try {
+		const data = await db.getDailyCosts({
+			startDate,
+			endDate
+		});
+		return c.json(successResponse(data, 200));
+	} catch (error$45) {
+		console.error("Error fetching daily costs:", error$45);
+		return c.json(internalServerError("Failed to fetch daily costs", 500), 500);
+	}
+}).get("/costs/summary", zv("query", dateRangeSchema.extend({ groupBy: zod_default.enum([
+	"day",
+	"hour",
+	"model",
+	"provider",
+	"config"
+]).optional() })), async (c) => {
+	const db = c.get("db");
+	const { startDate, endDate, groupBy } = c.req.valid("query");
+	try {
+		const data = await db.getCostSummary({
+			startDate,
+			endDate,
+			groupBy
+		});
+		return c.json(successResponse(data, 200));
+	} catch (error$45) {
+		console.error("Error fetching cost summary:", error$45);
+		return c.json(internalServerError("Failed to fetch cost summary", 500), 500);
+	}
+}).get("/stats", zv("query", dateRangeSchema), async (c) => {
+	const db = c.get("db");
+	const { startDate, endDate } = c.req.valid("query");
+	try {
+		const data = await db.getRequestStats({
+			startDate,
+			endDate
+		});
+		if (!data) return c.json(successResponse({
+			totalRequests: 0,
+			successfulRequests: 0,
+			failedRequests: 0,
+			streamingRequests: 0,
+			avgLatencyMs: 0,
+			maxLatencyMs: 0,
+			minLatencyMs: 0,
+			successRate: 0
+		}, 200));
+		return c.json(successResponse({
+			...data,
+			successRate: data.totalRequests > 0 ? (data.successfulRequests / data.totalRequests * 100).toFixed(2) : 0
+		}, 200));
+	} catch (error$45) {
+		console.error("Error fetching request stats:", error$45);
+		return c.json(internalServerError("Failed to fetch request stats", 500), 500);
+	}
+});
+var analytics_default = app$10;
+
 //#endregion
 //#region src/server/handlers/configs/index.ts
 const app$9 = new Hono().post("/", zv("json", zod_default.object({ name: zod_default.string().min(1) })), async (c) => {
@@ -13370,7 +13592,7 @@ const app$3 = new Hono().use("*", async (c, next) => {
 		error: "Auth middleware not configured",
 		message: `Auth type "${config$1.auth.type}" requires @llmops/enterprise middleware. Either use basicAuth() from @llmops/sdk or install @llmops/enterprise and add the auth middleware.`
 	}, 501);
-}).route("/configs", configs_default).route("/environments", environments_default).route("/providers", providers_default).route("/targeting", targeting_default).route("/variants", variants_default).route("/workspace-settings", workspace_settings_default);
+}).route("/analytics", analytics_default).route("/configs", configs_default).route("/environments", environments_default).route("/providers", providers_default).route("/targeting", targeting_default).route("/variants", variants_default).route("/workspace-settings", workspace_settings_default);
 var v1_default = app$3;
 
 //#endregion
@@ -13540,7 +13762,7 @@ const createGatewayAdapterMiddleware = () => {
 			if (method === "POST" && contentType === "application/json" && (path.endsWith("/chat/completions") || path.endsWith("/completions"))) {
 				const mergedBody = mergeChatCompletionBody(await c.req.json(), variantConfig, data.modelName);
 				const newHeaders = new Headers(c.req.raw.headers);
-				newHeaders.set("x-portkey-config", JSON.stringify(portkeyConfig));
+				newHeaders.set("x-llmops-config", JSON.stringify(portkeyConfig));
 				const newRequest = new Request(c.req.raw.url, {
 					method: c.req.raw.method,
 					headers: newHeaders,
@@ -13553,9 +13775,11 @@ const createGatewayAdapterMiddleware = () => {
 					configurable: true
 				});
 				c.req.bodyCache = {};
-			} else c.req.raw.headers.set("x-portkey-config", JSON.stringify(portkeyConfig));
+			} else c.req.raw.headers.set("x-llmops-config", JSON.stringify(portkeyConfig));
 			c.set("variantConfig", variantConfig);
 			c.set("variantModel", variantConfig.model || data.modelName);
+			c.set("configId", data.configId);
+			c.set("variantId", data.variantId);
 			await next();
 		} catch (error$45) {
 			console.error("Gateway adapter error:", error$45);
@@ -13567,12 +13791,501 @@ const createGatewayAdapterMiddleware = () => {
 	};
 };
 
+//#endregion
+//#region src/server/lib/streamingCostExtractor.ts
+/**
+* Creates a TransformStream that passes through SSE data while extracting usage info.
+*
+* @param onComplete - Callback invoked when stream completes with extracted usage
+* @returns TransformStream that passes through the original stream
+*
+* @example
+* ```typescript
+* const { stream, usagePromise } = createStreamingCostExtractor();
+*
+* // Pipe the response through the extractor
+* const transformedResponse = originalResponse.body.pipeThrough(stream);
+*
+* // Later, get the usage
+* const usage = await usagePromise;
+* if (usage) {
+*   console.log(`Tokens used: ${usage.totalTokens}`);
+* }
+* ```
+*/
+function createStreamingCostExtractor() {
+	let extractedUsage = null;
+	let buffer = "";
+	let resolveUsage;
+	const usagePromise = new Promise((resolve) => {
+		resolveUsage = resolve;
+	});
+	const decoder = new TextDecoder();
+	return {
+		stream: new TransformStream({
+			transform(chunk, controller) {
+				controller.enqueue(chunk);
+				const text = decoder.decode(chunk, { stream: true });
+				buffer += text;
+				const messages = buffer.split("\n\n");
+				buffer = messages.pop() || "";
+				for (const message of messages) {
+					const trimmed = message.trim();
+					if (!trimmed) continue;
+					if (!trimmed.startsWith("data:")) continue;
+					const jsonPart = trimmed.slice(5).trim();
+					if (jsonPart === "[DONE]") continue;
+					try {
+						const parsed = JSON.parse(jsonPart);
+						if (parsed.usage) extractedUsage = {
+							promptTokens: parsed.usage.prompt_tokens ?? 0,
+							completionTokens: parsed.usage.completion_tokens ?? 0,
+							totalTokens: parsed.usage.total_tokens ?? 0,
+							cachedTokens: parsed.usage.prompt_tokens_details?.cached_tokens
+						};
+					} catch {}
+				}
+			},
+			flush(controller) {
+				if (buffer.trim()) {
+					const trimmed = buffer.trim();
+					if (trimmed.startsWith("data:")) {
+						const jsonPart = trimmed.slice(5).trim();
+						if (jsonPart !== "[DONE]") try {
+							const parsed = JSON.parse(jsonPart);
+							if (parsed.usage) extractedUsage = {
+								promptTokens: parsed.usage.prompt_tokens ?? 0,
+								completionTokens: parsed.usage.completion_tokens ?? 0,
+								totalTokens: parsed.usage.total_tokens ?? 0,
+								cachedTokens: parsed.usage.prompt_tokens_details?.cached_tokens
+							};
+						} catch {}
+					}
+				}
+				resolveUsage(extractedUsage);
+			}
+		}),
+		usagePromise
+	};
+}
+/**
+* Wraps a Response with a streaming body to extract usage information.
+*
+* @param response - Original streaming Response
+* @returns Object with transformed response and promise for usage data
+*
+* @example
+* ```typescript
+* const result = wrapStreamingResponse(originalResponse);
+*
+* // Return the transformed response to the client
+* return result.response;
+*
+* // After response is sent, get usage for cost tracking
+* result.usagePromise.then((usage) => {
+*   if (usage) {
+*     trackCost(usage);
+*   }
+* });
+* ```
+*/
+function wrapStreamingResponse(response) {
+	if (!response.body) return {
+		response,
+		usagePromise: Promise.resolve(null)
+	};
+	const { stream, usagePromise } = createStreamingCostExtractor();
+	const transformedBody = response.body.pipeThrough(stream);
+	return {
+		response: new Response(transformedBody, {
+			status: response.status,
+			statusText: response.statusText,
+			headers: response.headers
+		}),
+		usagePromise
+	};
+}
+/**
+* Ensures stream_options.include_usage is set for cost tracking
+* Modifies the body in place.
+*
+* @param body - Request body (will be modified)
+* @returns Modified body with include_usage enabled
+*/
+function ensureStreamUsageEnabled(body) {
+	if (body.stream === true) body.stream_options = {
+		...body.stream_options || {},
+		include_usage: true
+	};
+	return body;
+}
+
+//#endregion
+//#region src/server/services/batchWriter.ts
+/**
+* Creates a BatchWriter instance
+*
+* @example
+* ```typescript
+* const writer = createBatchWriter(
+*   { batchInsertRequests: db.batchInsertRequests },
+*   { flushIntervalMs: 2000 }
+* );
+*
+* // Enqueue a request
+* writer.enqueue({
+*   requestId: 'req-123',
+*   provider: 'openai',
+*   model: 'gpt-4o',
+*   // ... other fields
+* });
+*
+* // When shutting down
+* await writer.stop();
+* ```
+*/
+function createBatchWriter(deps, config$1 = {}) {
+	const { flushIntervalMs = 2e3, maxBatchSize = 100, debug = false } = config$1;
+	let queue = [];
+	let flushTimer = null;
+	let running = false;
+	let flushing = false;
+	const log = debug ? (msg) => logger.debug(msg) : () => {};
+	/**
+	* Flush all queued requests to the database
+	*/
+	async function flush() {
+		if (flushing || queue.length === 0) return;
+		flushing = true;
+		const batch = queue;
+		queue = [];
+		try {
+			log(`[BatchWriter] Flushing ${batch.length} requests`);
+			log(`[BatchWriter] Flushed ${(await deps.batchInsertRequests(batch)).count} requests successfully`);
+		} catch (error$45) {
+			const errorMsg = error$45 instanceof Error ? error$45.message : String(error$45);
+			logger.error(`[BatchWriter] Flush failed, re-queuing requests: ${errorMsg}`);
+			queue = [...batch, ...queue];
+		} finally {
+			flushing = false;
+		}
+	}
+	/**
+	* Start the periodic flush timer
+	*/
+	function start() {
+		if (running) return;
+		running = true;
+		flushTimer = setInterval(() => {
+			flush().catch((err) => {
+				const errorMsg = err instanceof Error ? err.message : String(err);
+				logger.error(`[BatchWriter] Periodic flush error: ${errorMsg}`);
+			});
+		}, flushIntervalMs);
+		log(`[BatchWriter] Started with ${flushIntervalMs}ms flush interval`);
+	}
+	/**
+	* Stop the batch writer and flush remaining items
+	*/
+	async function stop() {
+		if (!running) return;
+		running = false;
+		if (flushTimer) {
+			clearInterval(flushTimer);
+			flushTimer = null;
+		}
+		await flush();
+		log("[BatchWriter] Stopped");
+	}
+	/**
+	* Add a request to the batch queue
+	*/
+	function enqueue(request) {
+		queue.push(request);
+		log(`[BatchWriter] Enqueued request ${request.requestId}, queue size: ${queue.length}`);
+		if (!running) start();
+		if (queue.length >= maxBatchSize) {
+			log(`[BatchWriter] Max batch size reached, forcing flush`);
+			flush().catch((err) => {
+				const errorMsg = err instanceof Error ? err.message : String(err);
+				logger.error(`[BatchWriter] Forced flush error: ${errorMsg}`);
+			});
+		}
+	}
+	return {
+		enqueue,
+		flush,
+		stop,
+		queueLength: () => queue.length,
+		isRunning: () => running
+	};
+}
+/**
+* Global singleton instance
+* Lazily initialized when first accessed
+*/
+let globalWriter = null;
+/**
+* Get or create the global BatchWriter instance
+*
+* @param deps - Database dependencies (required on first call)
+* @param config - Optional configuration
+* @returns The global BatchWriter instance
+*/
+function getGlobalBatchWriter(deps, config$1) {
+	if (!globalWriter) {
+		if (!deps) throw new Error("BatchWriter dependencies required on first initialization");
+		globalWriter = createBatchWriter(deps, config$1);
+	}
+	return globalWriter;
+}
+
+//#endregion
+//#region src/server/middlewares/costTracking.ts
+/**
+* Calculate cost in micro-dollars
+* 1 dollar = 1,000,000 micro-dollars
+*/
+function calculateCost(usage, pricing) {
+	const inputCost = Math.round(usage.promptTokens * pricing.inputCostPer1M);
+	const outputCost = Math.round(usage.completionTokens * pricing.outputCostPer1M);
+	return {
+		inputCost,
+		outputCost,
+		totalCost: inputCost + outputCost
+	};
+}
+/**
+* Simple pricing provider that fetches from models.dev
+*/
+var PricingProvider = class {
+	cache = /* @__PURE__ */ new Map();
+	lastFetch = 0;
+	cacheTTL = 300 * 1e3;
+	fetchPromise = null;
+	getCacheKey(provider, model) {
+		return `${provider.toLowerCase()}:${model.toLowerCase()}`;
+	}
+	async fetchPricingData() {
+		try {
+			const response = await fetch("https://models.dev/api.json");
+			if (!response.ok) return;
+			const data = await response.json();
+			this.cache.clear();
+			for (const [providerId, provider] of Object.entries(data)) {
+				const p = provider;
+				if (!p.models) continue;
+				for (const [, model] of Object.entries(p.models)) {
+					if (!model.cost) continue;
+					const cacheKey = this.getCacheKey(providerId, model.id);
+					this.cache.set(cacheKey, {
+						inputCostPer1M: model.cost.input ?? 0,
+						outputCostPer1M: model.cost.output ?? 0
+					});
+				}
+			}
+			this.lastFetch = Date.now();
+		} catch {}
+	}
+	async ensureFreshCache() {
+		if (Date.now() - this.lastFetch < this.cacheTTL && this.cache.size > 0) return;
+		if (!this.fetchPromise) this.fetchPromise = this.fetchPricingData().finally(() => {
+			this.fetchPromise = null;
+		});
+		await this.fetchPromise;
+	}
+	async getModelPricing(provider, model) {
+		await this.ensureFreshCache();
+		return this.cache.get(this.getCacheKey(provider, model)) || null;
+	}
+};
+const pricingProvider = new PricingProvider();
+/**
+* Creates cost tracking middleware that logs LLM requests with usage and cost data.
+*
+* Features:
+* - Tracks both streaming and non-streaming requests
+* - Calculates costs using models.dev pricing data
+* - Batches database writes for performance
+* - Adds x-llmops-request-id header for tracing
+*/
+function createCostTrackingMiddleware(config$1 = {}) {
+	const { enabled = true, trackErrors = true, flushIntervalMs = 2e3, debug = false } = config$1;
+	const log = debug ? (msg) => logger.debug(`[CostTracking] ${msg}`) : () => {};
+	return async (c, next) => {
+		if (!enabled) return next();
+		const path = c.req.path;
+		if (!path.endsWith("/chat/completions") && !path.endsWith("/completions")) return next();
+		const requestId = randomUUID();
+		const startTime = Date.now();
+		c.header("x-llmops-request-id", requestId);
+		let body = {};
+		let isStreaming = false;
+		try {
+			body = await c.req.raw.clone().json();
+			isStreaming = body.stream === true;
+			if (isStreaming) {
+				body = ensureStreamUsageEnabled(body);
+				const newHeaders = new Headers(c.req.raw.headers);
+				const newRequest = new Request(c.req.raw.url, {
+					method: c.req.raw.method,
+					headers: newHeaders,
+					body: JSON.stringify(body),
+					duplex: "half"
+				});
+				Object.defineProperty(c.req, "raw", {
+					value: newRequest,
+					writable: true,
+					configurable: true
+				});
+				c.req.bodyCache = {};
+			}
+		} catch {
+			log("Failed to parse request body");
+		}
+		const context = {
+			requestId,
+			startTime,
+			provider: "",
+			model: body.model || "",
+			configId: c.get("configId"),
+			endpoint: path,
+			isStreaming
+		};
+		c.set("__costTrackingContext", context);
+		await next();
+		const response = c.res;
+		const statusCode = response.status;
+		const latencyMs = Date.now() - startTime;
+		const variantModel = c.get("variantModel") || context.model;
+		let provider = "unknown";
+		const llmopsConfigHeader = c.req.header("x-llmops-config");
+		if (llmopsConfigHeader) try {
+			provider = JSON.parse(llmopsConfigHeader).provider || provider;
+		} catch {}
+		if (!variantModel) {
+			log(`Skipping request tracking - no model info`);
+			return;
+		}
+		const db = c.get("db");
+		const batchWriter = getGlobalBatchWriter({ batchInsertRequests: (requests) => db.batchInsertRequests(requests) }, {
+			flushIntervalMs,
+			debug
+		});
+		if (isStreaming && response.body) {
+			const { response: wrappedResponse, usagePromise } = wrapStreamingResponse(response);
+			c.res = wrappedResponse;
+			usagePromise.then(async (usage) => {
+				await processUsageAndLog({
+					requestId,
+					provider,
+					model: variantModel,
+					configId: c.get("configId"),
+					variantId: c.get("variantId"),
+					endpoint: context.endpoint,
+					statusCode,
+					latencyMs,
+					isStreaming: true,
+					usage: usage ? {
+						promptTokens: usage.promptTokens,
+						completionTokens: usage.completionTokens,
+						totalTokens: usage.totalTokens,
+						cachedTokens: usage.cachedTokens
+					} : null,
+					batchWriter,
+					trackErrors,
+					log
+				});
+			}).catch((err) => {
+				logger.error(`[CostTracking] Failed to process streaming usage: ${err}`);
+			});
+		} else {
+			let usage = null;
+			try {
+				const responseBody = await response.clone().json();
+				if (responseBody.usage) usage = {
+					promptTokens: responseBody.usage.prompt_tokens || 0,
+					completionTokens: responseBody.usage.completion_tokens || 0,
+					totalTokens: responseBody.usage.total_tokens || 0,
+					cachedTokens: responseBody.usage.prompt_tokens_details?.cached_tokens
+				};
+			} catch {
+				log("Failed to parse response body for usage");
+			}
+			await processUsageAndLog({
+				requestId,
+				provider,
+				model: variantModel,
+				configId: c.get("configId"),
+				variantId: c.get("variantId"),
+				endpoint: context.endpoint,
+				statusCode,
+				latencyMs,
+				isStreaming: false,
+				usage,
+				batchWriter,
+				trackErrors,
+				log
+			});
+		}
+	};
+}
+/**
+* Process usage data and log to batch writer
+*/
+async function processUsageAndLog(params) {
+	const { requestId, provider, model, configId, variantId, endpoint, statusCode, latencyMs, isStreaming, usage, batchWriter, trackErrors, log } = params;
+	if (!trackErrors && statusCode >= 400) {
+		log(`Skipping error response (${statusCode})`);
+		return;
+	}
+	let cost = 0;
+	let inputCost = 0;
+	let outputCost = 0;
+	if (usage && usage.promptTokens + usage.completionTokens > 0) try {
+		const pricing = await pricingProvider.getModelPricing(provider, model);
+		if (pricing) {
+			const costResult = calculateCost({
+				promptTokens: usage.promptTokens,
+				completionTokens: usage.completionTokens
+			}, pricing);
+			cost = costResult.totalCost;
+			inputCost = costResult.inputCost;
+			outputCost = costResult.outputCost;
+			log(`Calculated cost: ${cost} micro-dollars for ${provider}/${model}`);
+		} else log(`No pricing found for ${provider}/${model}`);
+	} catch (err) {
+		logger.error(`[CostTracking] Failed to calculate cost: ${err}`);
+	}
+	const requestData = {
+		requestId,
+		configId: configId || null,
+		variantId: variantId || null,
+		provider,
+		model,
+		promptTokens: usage?.promptTokens || 0,
+		completionTokens: usage?.completionTokens || 0,
+		totalTokens: usage?.totalTokens || 0,
+		cachedTokens: usage?.cachedTokens || 0,
+		cost,
+		inputCost,
+		outputCost,
+		endpoint,
+		statusCode,
+		latencyMs,
+		isStreaming,
+		tags: {}
+	};
+	batchWriter.enqueue(requestData);
+	log(`Enqueued request ${requestId} for logging`);
+}
+
 //#endregion
 //#region src/server/handlers/genai/index.ts
 const app$2 = new Hono();
 app$2.use("*", prettyJSON()).get("/health", async (c) => {
 	return c.json({ status: "healthy" });
-}).use("*", requestValidator).use("*", createRequestGuardMiddleware()).use("*", createGatewayAdapterMiddleware()).route("/", gateway).notFound((c) => c.json({ error: {
+}).use("*", requestValidator).use("*", createRequestGuardMiddleware()).use("*", createCostTrackingMiddleware()).use("*", createGatewayAdapterMiddleware()).route("/", gateway).notFound((c) => c.json({ error: {
 	message: "Not Found",
 	type: "invalid_request_error"
 } }, 404)).onError((err, c) => {