@llmops/core 0.1.4 → 0.1.5-beta.2

package/dist/index.mjs

@@ -1,5 +1,8 @@
-import { A as record, C as _enum, D as literal, E as boolean, M as union, N as unknown, O as number, S as zod_default, T as array, _ as schemas, a as matchType, b as variantsSchema, c as parsePartialTableData, d as validateTableData, f as SCHEMA_METADATA, g as environmentsSchema, h as environmentSecretsSchema, i as getMigrations, j as string, k as object, l as parseTableData, m as configsSchema, n as createDatabaseFromConnection, o as runAutoMigrations, p as configVariantsSchema, r as detectDatabaseType, s as logger, t as createDatabase, u as validatePartialTableData, v as targetingRulesSchema, w as any, x as workspaceSettingsSchema, y as variantVersionsSchema } from "./db-CGY-vZ3u.mjs";
+import { A as object, C as zod_default, D as boolean, E as array, M as string, N as union, O as literal, P as unknown, S as workspaceSettingsSchema, T as any, _ as llmRequestsSchema, a as matchType, b as variantVersionsSchema, c as parsePartialTableData, d as validateTableData, f as SCHEMA_METADATA, g as environmentsSchema, h as environmentSecretsSchema, i as getMigrations, j as record, k as number, l as parseTableData, m as configsSchema, n as createDatabaseFromConnection, o as runAutoMigrations, p as configVariantsSchema, r as detectDatabaseType, s as logger, t as createDatabase, u as validatePartialTableData, v as schemas, w as _enum, x as variantsSchema, y as targetingRulesSchema } from "./db-DSzwrW4p.mjs";
 import gateway from "@llmops/gateway";
+import { sql } from "kysely";
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
 import { createRandomStringGenerator } from "@better-auth/utils/random";
 import { randomBytes, randomUUID } from "node:crypto";
 
@@ -450,7 +453,7 @@ const authSchema = object({ type: string().min(1, "Auth type is required") }).pa
 const llmopsConfigSchema = object({
 	database: any(),
 	auth: authSchema,
-	basePath: string().min(1, "Base path is required and cannot be empty").refine((path) => path.startsWith("/"), "Base path must start with a forward slash"),
+	basePath: string().min(1, "Base path is required and cannot be empty").refine((path$1) => path$1.startsWith("/"), "Base path must start with a forward slash"),
 	providers: providersSchema,
 	autoMigrate: union([boolean(), literal("development")]).optional().default(false),
 	schema: string().optional().default("llmops")
@@ -464,6 +467,491 @@ function validateLLMOpsConfig(config) {
 	return result.data;
 }
 
+//#endregion
+//#region src/cache/types.ts
+/** Time constants in milliseconds for convenience */
+const MS = {
+	"1_MINUTE": 60 * 1e3,
+	"5_MINUTES": 300 * 1e3,
+	"10_MINUTES": 600 * 1e3,
+	"30_MINUTES": 1800 * 1e3,
+	"1_HOUR": 3600 * 1e3,
+	"6_HOURS": 360 * 60 * 1e3,
+	"12_HOURS": 720 * 60 * 1e3,
+	"1_DAY": 1440 * 60 * 1e3,
+	"7_DAYS": 10080 * 60 * 1e3,
+	"30_DAYS": 720 * 60 * 60 * 1e3
+};
+
+//#endregion
+//#region src/cache/backends/memory.ts
+var MemoryCacheBackend = class {
+	cache = /* @__PURE__ */ new Map();
+	stats = {
+		hits: 0,
+		misses: 0,
+		sets: 0,
+		deletes: 0,
+		size: 0,
+		expired: 0
+	};
+	cleanupInterval;
+	maxSize;
+	constructor(maxSize = 1e4, cleanupIntervalMs = 6e4) {
+		this.maxSize = maxSize;
+		this.startCleanup(cleanupIntervalMs);
+	}
+	startCleanup(intervalMs) {
+		this.cleanupInterval = setInterval(() => {
+			this.cleanup();
+		}, intervalMs);
+	}
+	getFullKey(key, namespace) {
+		return namespace ? `${namespace}:${key}` : key;
+	}
+	isExpired(entry) {
+		return entry.expiresAt !== void 0 && entry.expiresAt <= Date.now();
+	}
+	evictIfNeeded() {
+		if (this.cache.size >= this.maxSize) {
+			const entries = Array.from(this.cache.entries());
+			entries.sort((a, b) => a[1].createdAt - b[1].createdAt);
+			const toRemove = Math.floor(this.maxSize * .1);
+			for (let i = 0; i < toRemove && i < entries.length; i++) this.cache.delete(entries[i][0]);
+		}
+	}
+	async get(key, namespace) {
+		const fullKey = this.getFullKey(key, namespace);
+		const entry = this.cache.get(fullKey);
+		if (!entry) {
+			this.stats.misses++;
+			return null;
+		}
+		if (this.isExpired(entry)) {
+			this.cache.delete(fullKey);
+			this.stats.expired++;
+			this.stats.misses++;
+			return null;
+		}
+		this.stats.hits++;
+		return entry;
+	}
+	async set(key, value, options = {}) {
+		const fullKey = this.getFullKey(key, options.namespace);
+		const now = Date.now();
+		const entry = {
+			value,
+			createdAt: now,
+			expiresAt: options.ttl ? now + options.ttl : void 0,
+			metadata: options.metadata
+		};
+		this.evictIfNeeded();
+		this.cache.set(fullKey, entry);
+		this.stats.sets++;
+		this.stats.size = this.cache.size;
+	}
+	async delete(key, namespace) {
+		const fullKey = this.getFullKey(key, namespace);
+		const deleted = this.cache.delete(fullKey);
+		if (deleted) {
+			this.stats.deletes++;
+			this.stats.size = this.cache.size;
+		}
+		return deleted;
+	}
+	async clear(namespace) {
+		if (namespace) {
+			const prefix = `${namespace}:`;
+			const keysToDelete = Array.from(this.cache.keys()).filter((key) => key.startsWith(prefix));
+			for (const key of keysToDelete) this.cache.delete(key);
+			this.stats.deletes += keysToDelete.length;
+		} else {
+			this.stats.deletes += this.cache.size;
+			this.cache.clear();
+		}
+		this.stats.size = this.cache.size;
+	}
+	async has(key, namespace) {
+		const fullKey = this.getFullKey(key, namespace);
+		const entry = this.cache.get(fullKey);
+		if (!entry) return false;
+		if (this.isExpired(entry)) {
+			this.cache.delete(fullKey);
+			this.stats.expired++;
+			return false;
+		}
+		return true;
+	}
+	async keys(namespace) {
+		const allKeys = Array.from(this.cache.keys());
+		if (namespace) {
+			const prefix = `${namespace}:`;
+			return allKeys.filter((key) => key.startsWith(prefix)).map((key) => key.substring(prefix.length));
+		}
+		return allKeys;
+	}
+	async getStats(namespace) {
+		if (namespace) {
+			const prefix = `${namespace}:`;
+			const namespaceKeys = Array.from(this.cache.keys()).filter((key) => key.startsWith(prefix));
+			let expired = 0;
+			for (const key of namespaceKeys) {
+				const entry = this.cache.get(key);
+				if (entry && this.isExpired(entry)) expired++;
+			}
+			return {
+				...this.stats,
+				size: namespaceKeys.length,
+				expired
+			};
+		}
+		return { ...this.stats };
+	}
+	async cleanup() {
+		let expiredCount = 0;
+		for (const [key, entry] of this.cache.entries()) if (this.isExpired(entry)) {
+			this.cache.delete(key);
+			expiredCount++;
+		}
+		if (expiredCount > 0) {
+			this.stats.expired += expiredCount;
+			this.stats.size = this.cache.size;
+		}
+	}
+	async close() {
+		if (this.cleanupInterval) {
+			clearInterval(this.cleanupInterval);
+			this.cleanupInterval = void 0;
+		}
+		this.cache.clear();
+	}
+};
+
+//#endregion
+//#region src/cache/backends/file.ts
+/**
+* @file src/cache/backends/file.ts
+* File-based cache backend implementation
+*/
+var FileCacheBackend = class {
+	cacheFile;
+	data = {};
+	saveTimer;
+	cleanupInterval;
+	loaded = false;
+	loadPromise;
+	stats = {
+		hits: 0,
+		misses: 0,
+		sets: 0,
+		deletes: 0,
+		size: 0,
+		expired: 0
+	};
+	saveInterval;
+	constructor(dataDir = "data", fileName = "cache.json", saveIntervalMs = 1e3, cleanupIntervalMs = 6e4) {
+		this.cacheFile = path.join(process.cwd(), dataDir, fileName);
+		this.saveInterval = saveIntervalMs;
+		this.loadPromise = this.loadCache();
+		this.loadPromise.then(() => {
+			this.startCleanup(cleanupIntervalMs);
+		});
+	}
+	/** Ensure cache is loaded before any operation */
+	async ensureLoaded() {
+		if (!this.loaded) await this.loadPromise;
+	}
+	async ensureDataDir() {
+		const dir = path.dirname(this.cacheFile);
+		try {
+			await fs.mkdir(dir, { recursive: true });
+		} catch {}
+	}
+	async loadCache() {
+		try {
+			const content = await fs.readFile(this.cacheFile, "utf-8");
+			this.data = JSON.parse(content);
+			this.updateStats();
+			this.loaded = true;
+		} catch {
+			this.data = {};
+			this.loaded = true;
+		}
+	}
+	async saveCache() {
+		try {
+			await this.ensureDataDir();
+			await fs.writeFile(this.cacheFile, JSON.stringify(this.data, null, 2));
+		} catch {}
+	}
+	scheduleSave() {
+		if (this.saveTimer) clearTimeout(this.saveTimer);
+		this.saveTimer = setTimeout(() => {
+			this.saveCache();
+			this.saveTimer = void 0;
+		}, this.saveInterval);
+	}
+	startCleanup(intervalMs) {
+		this.cleanupInterval = setInterval(() => {
+			this.cleanup();
+		}, intervalMs);
+	}
+	isExpired(entry) {
+		return entry.expiresAt !== void 0 && entry.expiresAt <= Date.now();
+	}
+	updateStats() {
+		let totalSize = 0;
+		let totalExpired = 0;
+		for (const namespace of Object.values(this.data)) for (const entry of Object.values(namespace)) {
+			totalSize++;
+			if (this.isExpired(entry)) totalExpired++;
+		}
+		this.stats.size = totalSize;
+		this.stats.expired = totalExpired;
+	}
+	getNamespaceData(namespace = "default") {
+		if (!this.data[namespace]) this.data[namespace] = {};
+		return this.data[namespace];
+	}
+	async get(key, namespace) {
+		await this.ensureLoaded();
+		const namespaceData = this.getNamespaceData(namespace);
+		const entry = namespaceData[key];
+		if (!entry) {
+			this.stats.misses++;
+			return null;
+		}
+		if (this.isExpired(entry)) {
+			delete namespaceData[key];
+			this.stats.expired++;
+			this.stats.misses++;
+			this.scheduleSave();
+			return null;
+		}
+		this.stats.hits++;
+		return entry;
+	}
+	async set(key, value, options = {}) {
+		await this.ensureLoaded();
+		const namespace = options.namespace || "default";
+		const namespaceData = this.getNamespaceData(namespace);
+		const now = Date.now();
+		namespaceData[key] = {
+			value,
+			createdAt: now,
+			expiresAt: options.ttl ? now + options.ttl : void 0,
+			metadata: options.metadata
+		};
+		this.stats.sets++;
+		this.updateStats();
+		this.scheduleSave();
+	}
+	async delete(key, namespace) {
+		const namespaceData = this.getNamespaceData(namespace);
+		const existed = key in namespaceData;
+		if (existed) {
+			delete namespaceData[key];
+			this.stats.deletes++;
+			this.updateStats();
+			this.scheduleSave();
+		}
+		return existed;
+	}
+	async clear(namespace) {
+		if (namespace) {
+			const namespaceData = this.getNamespaceData(namespace);
+			const count = Object.keys(namespaceData).length;
+			this.data[namespace] = {};
+			this.stats.deletes += count;
+		} else {
+			const totalCount = Object.values(this.data).reduce((sum, ns) => sum + Object.keys(ns).length, 0);
+			this.data = {};
+			this.stats.deletes += totalCount;
+		}
+		this.updateStats();
+		this.scheduleSave();
+	}
+	async has(key, namespace) {
+		const namespaceData = this.getNamespaceData(namespace);
+		const entry = namespaceData[key];
+		if (!entry) return false;
+		if (this.isExpired(entry)) {
+			delete namespaceData[key];
+			this.stats.expired++;
+			this.scheduleSave();
+			return false;
+		}
+		return true;
+	}
+	async keys(namespace) {
+		if (namespace) {
+			const namespaceData = this.getNamespaceData(namespace);
+			return Object.keys(namespaceData);
+		}
+		const allKeys = [];
+		for (const namespaceData of Object.values(this.data)) allKeys.push(...Object.keys(namespaceData));
+		return allKeys;
+	}
+	async getStats(namespace) {
+		if (namespace) {
+			const namespaceData = this.getNamespaceData(namespace);
+			const keys = Object.keys(namespaceData);
+			let expired = 0;
+			for (const key of keys) {
+				const entry = namespaceData[key];
+				if (this.isExpired(entry)) expired++;
+			}
+			return {
+				...this.stats,
+				size: keys.length,
+				expired
+			};
+		}
+		this.updateStats();
+		return { ...this.stats };
+	}
+	async cleanup() {
+		let expiredCount = 0;
+		let hasChanges = false;
+		for (const [, namespaceData] of Object.entries(this.data)) for (const [key, entry] of Object.entries(namespaceData)) if (this.isExpired(entry)) {
+			delete namespaceData[key];
+			expiredCount++;
+			hasChanges = true;
+		}
+		if (hasChanges) {
+			this.stats.expired += expiredCount;
+			this.updateStats();
+			this.scheduleSave();
+		}
+	}
+	/** Wait for the cache to be ready (file loaded) */
+	async waitForReady() {
+		await this.loadPromise;
+	}
+	async close() {
+		if (this.saveTimer) {
+			clearTimeout(this.saveTimer);
+			await this.saveCache();
+		}
+		if (this.cleanupInterval) {
+			clearInterval(this.cleanupInterval);
+			this.cleanupInterval = void 0;
+		}
+	}
+};
+
+//#endregion
+//#region src/cache/service.ts
+/**
+* @file src/cache/service.ts
+* Unified cache service with pluggable backends
+*/
+var CacheService = class {
+	backend;
+	defaultTtl;
+	constructor(config) {
+		this.defaultTtl = config.defaultTtl;
+		this.backend = this.createBackend(config);
+	}
+	createBackend(config) {
+		switch (config.backend) {
+			case "memory": return new MemoryCacheBackend(config.maxSize, config.cleanupInterval);
+			case "file": return new FileCacheBackend(config.dataDir, config.fileName, config.saveInterval, config.cleanupInterval);
+			default: throw new Error(`Unsupported cache backend: ${config.backend}`);
+		}
+	}
+	/** Get a value from the cache */
+	async get(key, namespace) {
+		const entry = await this.backend.get(key, namespace);
+		return entry ? entry.value : null;
+	}
+	/** Get the full cache entry (with metadata) */
+	async getEntry(key, namespace) {
+		return this.backend.get(key, namespace);
+	}
+	/** Set a value in the cache */
+	async set(key, value, options = {}) {
+		const finalOptions = {
+			...options,
+			ttl: options.ttl ?? this.defaultTtl
+		};
+		await this.backend.set(key, value, finalOptions);
+	}
+	/** Set a value with TTL in seconds (convenience method) */
+	async setWithTtl(key, value, ttlSeconds, namespace) {
+		await this.set(key, value, {
+			ttl: ttlSeconds * 1e3,
+			namespace
+		});
+	}
+	/** Delete a value from the cache */
+	async delete(key, namespace) {
+		return this.backend.delete(key, namespace);
+	}
+	/** Check if a key exists in the cache */
+	async has(key, namespace) {
+		return this.backend.has(key, namespace);
+	}
+	/** Get all keys in a namespace */
+	async keys(namespace) {
+		return this.backend.keys(namespace);
+	}
+	/** Clear all entries in a namespace (or all entries if no namespace) */
+	async clear(namespace) {
+		await this.backend.clear(namespace);
+	}
+	/** Get cache statistics */
+	async getStats(namespace) {
+		return this.backend.getStats(namespace);
+	}
+	/** Manually trigger cleanup of expired entries */
+	async cleanup() {
+		await this.backend.cleanup();
+	}
+	/** Wait for the backend to be ready */
+	async waitForReady() {
+		if ("waitForReady" in this.backend) await this.backend.waitForReady();
+	}
+	/** Close the cache and cleanup resources */
+	async close() {
+		await this.backend.close();
+	}
+	/** Get or set pattern - get value, or compute and cache it if not found */
+	async getOrSet(key, factory, options = {}) {
+		const existing = await this.get(key, options.namespace);
+		if (existing !== null) return existing;
+		const value = await factory();
+		await this.set(key, value, options);
+		return value;
+	}
+	/** Increment a numeric value (simulated atomic operation) */
+	async increment(key, delta = 1, options = {}) {
+		const newValue = (await this.get(key, options.namespace) || 0) + delta;
+		await this.set(key, newValue, options);
+		return newValue;
+	}
+	/** Set multiple values at once */
+	async setMany(entries, defaultOptions = {}) {
+		const promises = entries.map(({ key, value, options }) => this.set(key, value, {
+			...defaultOptions,
+			...options
+		}));
+		await Promise.all(promises);
+	}
+	/** Get multiple values at once */
+	async getMany(keys, namespace) {
+		const promises = keys.map(async (key) => ({
+			key,
+			value: await this.get(key, namespace)
+		}));
+		return Promise.all(promises);
+	}
+	/** Get the underlying backend (for advanced use cases) */
+	getBackend() {
+		return this.backend;
+	}
+};
+
 //#endregion
 //#region src/utils/id.ts
 const generateId = (size) => {
@@ -792,7 +1280,11 @@ const createConfigVariantDataLayer = (db) => {
 				"version"
 			]).where("variantId", "=", configVariant.variantId).orderBy("version", "desc").limit(1).executeTakeFirst();
 			if (!versionData) throw new LLMOpsError(`No variant version found for variant ${configVariant.variantId}`);
-			return versionData;
+			return {
+				...versionData,
+				configId: resolvedConfigId,
+				variantId: configVariant.variantId
+			};
 		}
 	};
 };
@@ -952,6 +1444,287 @@ const createEnvironmentSecretDataLayer = (db) => {
 	};
 };
 
+//#endregion
+//#region src/datalayer/llmRequests.ts
+/**
+* Schema for inserting a new LLM request log
+*/
+const insertLLMRequestSchema = zod_default.object({
+	requestId: zod_default.string().uuid(),
+	configId: zod_default.string().uuid().nullable().optional(),
+	variantId: zod_default.string().uuid().nullable().optional(),
+	provider: zod_default.string(),
+	model: zod_default.string(),
+	promptTokens: zod_default.number().int().default(0),
+	completionTokens: zod_default.number().int().default(0),
+	totalTokens: zod_default.number().int().default(0),
+	cachedTokens: zod_default.number().int().default(0),
+	cost: zod_default.number().int().default(0),
+	inputCost: zod_default.number().int().default(0),
+	outputCost: zod_default.number().int().default(0),
+	endpoint: zod_default.string(),
+	statusCode: zod_default.number().int(),
+	latencyMs: zod_default.number().int().default(0),
+	isStreaming: zod_default.boolean().default(false),
+	userId: zod_default.string().nullable().optional(),
+	tags: zod_default.record(zod_default.string(), zod_default.string()).default({})
+});
+/**
+* Schema for listing LLM requests
+*/
+const listRequestsSchema = zod_default.object({
+	limit: zod_default.number().int().positive().max(1e3).default(100),
+	offset: zod_default.number().int().nonnegative().default(0),
+	configId: zod_default.string().uuid().optional(),
+	provider: zod_default.string().optional(),
+	model: zod_default.string().optional(),
+	startDate: zod_default.date().optional(),
+	endDate: zod_default.date().optional()
+});
+/**
+* Schema for date range queries
+*/
+const dateRangeSchema = zod_default.object({
+	startDate: zod_default.date(),
+	endDate: zod_default.date()
+});
+/**
+* Schema for cost summary with grouping
+*/
+const costSummarySchema = zod_default.object({
+	startDate: zod_default.date(),
+	endDate: zod_default.date(),
+	groupBy: zod_default.enum([
+		"day",
+		"hour",
+		"model",
+		"provider",
+		"config"
+	]).optional()
+});
+/**
+* Helper to create column reference for SQL
+* Uses sql.ref() to properly quote column names for the database
+*/
+const col = (name) => sql.ref(name);
+const tableCol = (table, name) => sql.ref(`${table}.${name}`);
+const createLLMRequestsDataLayer = (db) => {
+	return {
+		batchInsertRequests: async (requests) => {
+			if (requests.length === 0) return { count: 0 };
+			const validatedRequests = await Promise.all(requests.map(async (req) => {
+				const result = await insertLLMRequestSchema.safeParseAsync(req);
+				if (!result.success) throw new LLMOpsError(`Invalid request data: ${result.error.message}`);
+				return result.data;
+			}));
+			const now = (/* @__PURE__ */ new Date()).toISOString();
+			const values = validatedRequests.map((req) => ({
+				id: randomUUID(),
+				requestId: req.requestId,
+				configId: req.configId ?? null,
+				variantId: req.variantId ?? null,
+				provider: req.provider,
+				model: req.model,
+				promptTokens: req.promptTokens,
+				completionTokens: req.completionTokens,
+				totalTokens: req.totalTokens,
+				cachedTokens: req.cachedTokens,
+				cost: req.cost,
+				inputCost: req.inputCost,
+				outputCost: req.outputCost,
+				endpoint: req.endpoint,
+				statusCode: req.statusCode,
+				latencyMs: req.latencyMs,
+				isStreaming: req.isStreaming,
+				userId: req.userId ?? null,
+				tags: JSON.stringify(req.tags),
+				createdAt: now,
+				updatedAt: now
+			}));
+			await db.insertInto("llm_requests").values(values).execute();
+			return { count: values.length };
+		},
+		insertRequest: async (request) => {
+			const result = await insertLLMRequestSchema.safeParseAsync(request);
+			if (!result.success) throw new LLMOpsError(`Invalid request data: ${result.error.message}`);
+			const req = result.data;
+			const now = (/* @__PURE__ */ new Date()).toISOString();
+			return db.insertInto("llm_requests").values({
+				id: randomUUID(),
+				requestId: req.requestId,
+				configId: req.configId ?? null,
+				variantId: req.variantId ?? null,
+				provider: req.provider,
+				model: req.model,
+				promptTokens: req.promptTokens,
+				completionTokens: req.completionTokens,
+				totalTokens: req.totalTokens,
+				cachedTokens: req.cachedTokens,
+				cost: req.cost,
+				inputCost: req.inputCost,
+				outputCost: req.outputCost,
+				endpoint: req.endpoint,
+				statusCode: req.statusCode,
+				latencyMs: req.latencyMs,
+				isStreaming: req.isStreaming,
+				userId: req.userId ?? null,
+				tags: JSON.stringify(req.tags),
+				createdAt: now,
+				updatedAt: now
+			}).returningAll().executeTakeFirst();
+		},
+		listRequests: async (params) => {
+			const result = await listRequestsSchema.safeParseAsync(params || {});
+			if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
+			const { limit, offset, configId, provider, model, startDate, endDate } = result.data;
+			let baseQuery = db.selectFrom("llm_requests");
+			if (configId) baseQuery = baseQuery.where("configId", "=", configId);
+			if (provider) baseQuery = baseQuery.where("provider", "=", provider);
+			if (model) baseQuery = baseQuery.where("model", "=", model);
+			if (startDate) baseQuery = baseQuery.where(sql`${col("createdAt")} >= ${startDate.toISOString()}`);
+			if (endDate) baseQuery = baseQuery.where(sql`${col("createdAt")} <= ${endDate.toISOString()}`);
+			const countResult = await baseQuery.select(sql`COUNT(*)`.as("total")).executeTakeFirst();
+			const total = Number(countResult?.total ?? 0);
+			return {
+				data: await baseQuery.selectAll().orderBy("createdAt", "desc").limit(limit).offset(offset).execute(),
+				total,
+				limit,
+				offset
+			};
+		},
+		getRequestByRequestId: async (requestId) => {
+			return db.selectFrom("llm_requests").selectAll().where("requestId", "=", requestId).executeTakeFirst();
+		},
+		getTotalCost: async (params) => {
+			const result = await dateRangeSchema.safeParseAsync(params);
+			if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
+			const { startDate, endDate } = result.data;
+			return await db.selectFrom("llm_requests").select([
+				sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
+				sql`COALESCE(SUM(${col("inputCost")}), 0)`.as("totalInputCost"),
+				sql`COALESCE(SUM(${col("outputCost")}), 0)`.as("totalOutputCost"),
+				sql`COALESCE(SUM(${col("promptTokens")}), 0)`.as("totalPromptTokens"),
+				sql`COALESCE(SUM(${col("completionTokens")}), 0)`.as("totalCompletionTokens"),
+				sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
+				sql`COUNT(*)`.as("requestCount")
+			]).where(sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col("createdAt")} <= ${endDate.toISOString()}`).executeTakeFirst();
+		},
+		getCostByModel: async (params) => {
+			const result = await dateRangeSchema.safeParseAsync(params);
+			if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
+			const { startDate, endDate } = result.data;
+			return db.selectFrom("llm_requests").select([
+				"provider",
+				"model",
+				sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
+				sql`COALESCE(SUM(${col("inputCost")}), 0)`.as("totalInputCost"),
+				sql`COALESCE(SUM(${col("outputCost")}), 0)`.as("totalOutputCost"),
+				sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
+				sql`COUNT(*)`.as("requestCount"),
+				sql`AVG(${col("latencyMs")})`.as("avgLatencyMs")
+			]).where(sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col("createdAt")} <= ${endDate.toISOString()}`).groupBy(["provider", "model"]).orderBy(sql`SUM(${col("cost")})`, "desc").execute();
+		},
+		getCostByProvider: async (params) => {
+			const result = await dateRangeSchema.safeParseAsync(params);
+			if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
+			const { startDate, endDate } = result.data;
+			return db.selectFrom("llm_requests").select([
+				"provider",
+				sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
+				sql`COALESCE(SUM(${col("inputCost")}), 0)`.as("totalInputCost"),
+				sql`COALESCE(SUM(${col("outputCost")}), 0)`.as("totalOutputCost"),
+				sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
+				sql`COUNT(*)`.as("requestCount"),
+				sql`AVG(${col("latencyMs")})`.as("avgLatencyMs")
+			]).where(sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col("createdAt")} <= ${endDate.toISOString()}`).groupBy("provider").orderBy(sql`SUM(${col("cost")})`, "desc").execute();
+		},
+		getCostByConfig: async (params) => {
+			const result = await dateRangeSchema.safeParseAsync(params);
+			if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
+			const { startDate, endDate } = result.data;
+			return db.selectFrom("llm_requests").leftJoin("configs", "llm_requests.configId", "configs.id").select([
+				"llm_requests.configId",
+				"configs.name as configName",
+				"configs.slug as configSlug",
+				sql`COALESCE(SUM(${tableCol("llm_requests", "cost")}), 0)`.as("totalCost"),
+				sql`COALESCE(SUM(${tableCol("llm_requests", "inputCost")}), 0)`.as("totalInputCost"),
+				sql`COALESCE(SUM(${tableCol("llm_requests", "outputCost")}), 0)`.as("totalOutputCost"),
+				sql`COALESCE(SUM(${tableCol("llm_requests", "totalTokens")}), 0)`.as("totalTokens"),
+				sql`COUNT(*)`.as("requestCount")
+			]).where(sql`${tableCol("llm_requests", "createdAt")} >= ${startDate.toISOString()}`).where(sql`${tableCol("llm_requests", "createdAt")} <= ${endDate.toISOString()}`).groupBy([
+				"llm_requests.configId",
+				"configs.name",
+				"configs.slug"
+			]).orderBy(sql`SUM(${tableCol("llm_requests", "cost")})`, "desc").execute();
+		},
+		getDailyCosts: async (params) => {
+			const result = await dateRangeSchema.safeParseAsync(params);
+			if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
+			const { startDate, endDate } = result.data;
+			return db.selectFrom("llm_requests").select([
+				sql`DATE(${col("createdAt")})`.as("date"),
+				sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
+				sql`COALESCE(SUM(${col("inputCost")}), 0)`.as("totalInputCost"),
+				sql`COALESCE(SUM(${col("outputCost")}), 0)`.as("totalOutputCost"),
+				sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
+				sql`COUNT(*)`.as("requestCount")
+			]).where(sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col("createdAt")} <= ${endDate.toISOString()}`).groupBy(sql`DATE(${col("createdAt")})`).orderBy(sql`DATE(${col("createdAt")})`, "asc").execute();
+		},
+		getCostSummary: async (params) => {
+			const result = await costSummarySchema.safeParseAsync(params);
+			if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
+			const { startDate, endDate, groupBy } = result.data;
+			const baseQuery = db.selectFrom("llm_requests").where(sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col("createdAt")} <= ${endDate.toISOString()}`);
+			switch (groupBy) {
+				case "day": return baseQuery.select([
+					sql`DATE(${col("createdAt")})`.as("groupKey"),
+					sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
+					sql`COUNT(*)`.as("requestCount")
+				]).groupBy(sql`DATE(${col("createdAt")})`).orderBy(sql`DATE(${col("createdAt")})`, "asc").execute();
+				case "hour": return baseQuery.select([
+					sql`DATE_TRUNC('hour', ${col("createdAt")})`.as("groupKey"),
+					sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
+					sql`COUNT(*)`.as("requestCount")
+				]).groupBy(sql`DATE_TRUNC('hour', ${col("createdAt")})`).orderBy(sql`DATE_TRUNC('hour', ${col("createdAt")})`, "asc").execute();
+				case "model": return baseQuery.select([
+					sql`${col("provider")} || '/' || ${col("model")}`.as("groupKey"),
+					sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
+					sql`COUNT(*)`.as("requestCount")
+				]).groupBy(["provider", "model"]).orderBy(sql`SUM(${col("cost")})`, "desc").execute();
+				case "provider": return baseQuery.select([
+					sql`${col("provider")}`.as("groupKey"),
+					sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
+					sql`COUNT(*)`.as("requestCount")
+				]).groupBy("provider").orderBy(sql`SUM(${col("cost")})`, "desc").execute();
+				case "config": return baseQuery.select([
+					sql`COALESCE(${col("configId")}::text, 'no-config')`.as("groupKey"),
+					sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
+					sql`COUNT(*)`.as("requestCount")
+				]).groupBy("configId").orderBy(sql`SUM(${col("cost")})`, "desc").execute();
+				default: return baseQuery.select([
+					sql`'total'`.as("groupKey"),
+					sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
+					sql`COUNT(*)`.as("requestCount")
+				]).execute();
+			}
+		},
+		getRequestStats: async (params) => {
+			const result = await dateRangeSchema.safeParseAsync(params);
+			if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
+			const { startDate, endDate } = result.data;
+			return await db.selectFrom("llm_requests").select([
+				sql`COUNT(*)`.as("totalRequests"),
+				sql`COUNT(CASE WHEN ${col("statusCode")} >= 200 AND ${col("statusCode")} < 300 THEN 1 END)`.as("successfulRequests"),
+				sql`COUNT(CASE WHEN ${col("statusCode")} >= 400 THEN 1 END)`.as("failedRequests"),
+				sql`COUNT(CASE WHEN ${col("isStreaming")} = true THEN 1 END)`.as("streamingRequests"),
+				sql`AVG(${col("latencyMs")})`.as("avgLatencyMs"),
+				sql`MAX(${col("latencyMs")})`.as("maxLatencyMs"),
+				sql`MIN(${col("latencyMs")})`.as("minLatencyMs")
+			]).where(sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col("createdAt")} <= ${endDate.toISOString()}`).executeTakeFirst();
+		}
+	};
+};
+
 //#endregion
 //#region src/datalayer/targetingRules.ts
 const createTargetingRule = zod_default.object({
@@ -1378,6 +2151,7 @@ const createDataLayer = async (db) => {
 		...createConfigVariantDataLayer(db),
 		...createEnvironmentDataLayer(db),
 		...createEnvironmentSecretDataLayer(db),
+		...createLLMRequestsDataLayer(db),
 		...createTargetingRulesDataLayer(db),
 		...createVariantDataLayer(db),
 		...createVariantVersionsDataLayer(db),
@@ -1386,4 +2160,207 @@ const createDataLayer = async (db) => {
 };
 
 //#endregion
-export { SCHEMA_METADATA, SupportedProviders, chatCompletionCreateParamsBaseSchema, configVariantsSchema, configsSchema, createDataLayer, createDatabase, createDatabaseFromConnection, detectDatabaseType, environmentSecretsSchema, environmentsSchema, gateway, generateId, getMigrations, llmopsConfigSchema, logger, matchType, parsePartialTableData, parseTableData, runAutoMigrations, schemas, targetingRulesSchema, validateLLMOpsConfig, validatePartialTableData, validateTableData, variantJsonDataSchema, variantVersionsSchema, variantsSchema, workspaceSettingsSchema };
+//#region src/pricing/calculator.ts
+/**
+* Calculate the cost of an LLM request in micro-dollars
+*
+* Micro-dollars are used to avoid floating-point precision issues:
+* - 1 dollar = 1,000,000 micro-dollars
+* - $0.001 = 1,000 micro-dollars
+* - $0.000001 = 1 micro-dollar
+*
+* @param usage - Token usage data from the LLM response
+* @param pricing - Model pricing information
+* @returns Cost breakdown in micro-dollars
+*
+* @example
+* ```typescript
+* const usage = { promptTokens: 1000, completionTokens: 500 };
+* const pricing = { inputCostPer1M: 2.5, outputCostPer1M: 10.0 };
+* const cost = calculateCost(usage, pricing);
+* // cost = { inputCost: 2500, outputCost: 5000, totalCost: 7500 }
+* // In dollars: $0.0025 input + $0.005 output = $0.0075 total
+* ```
+*/
+function calculateCost(usage, pricing) {
+	const inputCost = Math.round(usage.promptTokens * pricing.inputCostPer1M);
+	const outputCost = Math.round(usage.completionTokens * pricing.outputCostPer1M);
+	return {
+		inputCost,
+		outputCost,
+		totalCost: inputCost + outputCost
+	};
+}
+/**
+* Convert micro-dollars to dollars
+*
+* @param microDollars - Amount in micro-dollars
+* @returns Amount in dollars
+*
+* @example
+* ```typescript
+* microDollarsToDollars(7500); // 0.0075
+* microDollarsToDollars(1000000); // 1.0
+* ```
+*/
+function microDollarsToDollars(microDollars) {
+	return microDollars / 1e6;
+}
+/**
+* Convert dollars to micro-dollars
+*
+* @param dollars - Amount in dollars
+* @returns Amount in micro-dollars (rounded to nearest integer)
+*
+* @example
+* ```typescript
+* dollarsToMicroDollars(0.0075); // 7500
+* dollarsToMicroDollars(1.0); // 1000000
+* ```
+*/
+function dollarsToMicroDollars(dollars) {
+	return Math.round(dollars * 1e6);
+}
+/**
+* Format micro-dollars as a human-readable dollar string
+*
+* @param microDollars - Amount in micro-dollars
+* @param decimals - Number of decimal places (default: 6)
+* @returns Formatted dollar string
+*
+* @example
+* ```typescript
+* formatCost(7500); // "$0.007500"
+* formatCost(1234567, 2); // "$1.23"
+* ```
+*/
+function formatCost(microDollars, decimals = 6) {
+	return `$${microDollarsToDollars(microDollars).toFixed(decimals)}`;
+}
+
+//#endregion
+//#region src/pricing/provider.ts
+const MODELS_DEV_API = "https://models.dev/api.json";
+/**
+* Pricing provider that fetches data from models.dev API
+*
+* Features:
+* - Caches pricing data with configurable TTL (default 5 minutes)
+* - Supports fallback to local cache on fetch failure
+* - Thread-safe cache refresh
+*/
+var ModelsDevPricingProvider = class {
+	cache = /* @__PURE__ */ new Map();
+	lastFetch = 0;
+	cacheTTL;
+	fetchPromise = null;
+	ready = false;
+	/**
+	* Create a new ModelsDevPricingProvider
+	*
+	* @param cacheTTL - Cache TTL in milliseconds (default: 5 minutes)
+	*/
+	constructor(cacheTTL = 300 * 1e3) {
+		this.cacheTTL = cacheTTL;
+	}
+	/**
+	* Generate a cache key for a provider/model combination
+	*/
+	getCacheKey(provider, model) {
+		return `${provider.toLowerCase()}:${model.toLowerCase()}`;
+	}
+	/**
+	* Fetch pricing data from models.dev API
+	*/
+	async fetchPricingData() {
+		try {
+			logger.debug("[Pricing] Fetching pricing data from models.dev");
+			const response = await fetch(MODELS_DEV_API);
+			if (!response.ok) throw new Error(`Failed to fetch models.dev API: ${response.status}`);
+			const data = await response.json();
+			this.cache.clear();
+			for (const [providerId, provider] of Object.entries(data)) {
+				if (!provider.models) continue;
+				for (const [_modelId, model] of Object.entries(provider.models)) {
+					if (!model.cost) continue;
+					const cacheKey = this.getCacheKey(providerId, model.id);
+					this.cache.set(cacheKey, {
+						inputCostPer1M: model.cost.input ?? 0,
+						outputCostPer1M: model.cost.output ?? 0,
+						cacheReadCostPer1M: model.cost.cache_read,
+						cacheWriteCostPer1M: model.cost.cache_write,
+						reasoningCostPer1M: model.cost.reasoning
+					});
+					const nameKey = this.getCacheKey(providerId, model.name);
+					if (nameKey !== cacheKey) this.cache.set(nameKey, this.cache.get(cacheKey));
+				}
+			}
+			this.lastFetch = Date.now();
+			this.ready = true;
+			logger.debug(`[Pricing] Cached pricing for ${this.cache.size} models from models.dev`);
+		} catch (error) {
+			logger.error(`[Pricing] Failed to fetch pricing data: ${error instanceof Error ? error.message : String(error)}`);
+			if (this.cache.size === 0) throw error;
+		}
+	}
+	/**
+	* Ensure cache is fresh, fetching if necessary
+	*/
+	async ensureFreshCache() {
+		if (!(Date.now() - this.lastFetch > this.cacheTTL) && this.cache.size > 0) return;
+		if (!this.fetchPromise) this.fetchPromise = this.fetchPricingData().finally(() => {
+			this.fetchPromise = null;
+		});
+		await this.fetchPromise;
+	}
+	/**
+	* Get pricing for a specific model
+	*/
+	async getModelPricing(provider, model) {
+		await this.ensureFreshCache();
+		const cacheKey = this.getCacheKey(provider, model);
+		const pricing = this.cache.get(cacheKey);
+		if (!pricing) {
+			logger.debug(`[Pricing] No pricing found for ${provider}/${model}, trying partial match`);
+			for (const [key, value] of this.cache.entries()) if (key.startsWith(`${provider.toLowerCase()}:`)) {
+				const modelPart = key.split(":")[1];
+				if (model.toLowerCase().includes(modelPart)) {
+					logger.debug(`[Pricing] Found partial match: ${key}`);
+					return value;
+				}
+			}
+			return null;
+		}
+		return pricing;
+	}
+	/**
+	* Force refresh the pricing cache
+	*/
+	async refreshCache() {
+		this.lastFetch = 0;
+		await this.ensureFreshCache();
+	}
+	/**
+	* Check if the provider is ready
+	*/
+	isReady() {
+		return this.ready;
+	}
+	/**
+	* Get the number of cached models (for debugging)
+	*/
+	getCacheSize() {
+		return this.cache.size;
+	}
+};
+let defaultProvider = null;
+/**
+* Get the default pricing provider instance
+*/
+function getDefaultPricingProvider() {
+	if (!defaultProvider) defaultProvider = new ModelsDevPricingProvider();
+	return defaultProvider;
+}
+
+//#endregion
+export { CacheService, FileCacheBackend, MS, MemoryCacheBackend, ModelsDevPricingProvider, SCHEMA_METADATA, SupportedProviders, calculateCost, chatCompletionCreateParamsBaseSchema, configVariantsSchema, configsSchema, createDataLayer, createDatabase, createDatabaseFromConnection, createLLMRequestsDataLayer, detectDatabaseType, dollarsToMicroDollars, environmentSecretsSchema, environmentsSchema, formatCost, gateway, generateId, getDefaultPricingProvider, getMigrations, llmRequestsSchema, llmopsConfigSchema, logger, matchType, microDollarsToDollars, parsePartialTableData, parseTableData, runAutoMigrations, schemas, targetingRulesSchema, validateLLMOpsConfig, validatePartialTableData, validateTableData, variantJsonDataSchema, variantVersionsSchema, variantsSchema, workspaceSettingsSchema };