@llmops/core 1.0.0-beta.1 → 1.0.0-beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -1,4 +1,4 @@
1
- import { A as spanEventsSchema, B as array, C as playgroundColumnSchema, D as providerConfigsSchema, E as playgroundsSchema, F as variantsSchema, G as record, H as literal, I as workspaceSettingsSchema, J as unknown, K as string, L as zod_default, M as targetingRulesSchema, N as tracesSchema, O as providerGuardrailOverridesSchema, P as variantVersionsSchema, R as _enum, S as llmRequestsSchema, T as playgroundRunsSchema, U as number, V as boolean, W as object, Y as __require, _ as datasetVersionsSchema, a as matchType, b as environmentsSchema, c as parsePartialTableData, d as validateTableData, f as SCHEMA_METADATA, g as datasetVersionRecordsSchema, h as datasetRecordsSchema, i as getMigrations, j as spansSchema, k as schemas, l as parseTableData, m as configsSchema, n as createDatabaseFromConnection, o as runAutoMigrations, p as configVariantsSchema, q as union, r as detectDatabaseType, s as logger, t as createDatabase, u as validatePartialTableData, v as datasetsSchema, w as playgroundResultsSchema, x as guardrailConfigsSchema, y as environmentSecretsSchema, z as any } from "./db-CjwRQB4N.mjs";
1
+ import { A as spanEventsSchema, B as array, C as playgroundColumnSchema, D as providerConfigsSchema, E as playgroundsSchema, F as variantsSchema, G as record, H as literal, I as workspaceSettingsSchema, J as unknown, K as string, L as zod_default, M as targetingRulesSchema, N as tracesSchema, O as providerGuardrailOverridesSchema, P as variantVersionsSchema, R as _enum, S as llmRequestsSchema, T as playgroundRunsSchema, U as number, V as boolean, W as object, Y as __require, _ as datasetVersionsSchema, a as matchType, b as environmentsSchema, c as parsePartialTableData, d as validateTableData, f as SCHEMA_METADATA, g as datasetVersionRecordsSchema, h as datasetRecordsSchema, i as getMigrations, j as spansSchema, k as schemas, l as parseTableData, m as configsSchema, n as createDatabaseFromConnection, o as runAutoMigrations, p as configVariantsSchema, q as union, r as detectDatabaseType, s as logger, t as createDatabase, u as validatePartialTableData, v as datasetsSchema, w as playgroundResultsSchema, x as guardrailConfigsSchema, y as environmentSecretsSchema, z as any } from "./db-CZ8KtpL-.mjs";
2
2
  import { n as executeWithSchema, t as createNeonDialect } from "./neon-dialect-DySGBYUi.mjs";
3
3
  import gateway from "@llmops/gateway";
4
4
  import { CompiledQuery, Kysely, PostgresDialect, sql } from "kysely";
@@ -1385,754 +1385,247 @@ const createDatasetsDataLayer = (db) => {
1385
1385
  };
1386
1386
 
1387
1387
  //#endregion
1388
- //#region src/datalayer/guardrailConfigs.ts
1389
- const createGuardrailConfig = zod_default.object({
1390
- name: zod_default.string().min(1),
1391
- pluginId: zod_default.string().min(1),
1392
- functionId: zod_default.string().min(1),
1393
- hookType: zod_default.enum(["beforeRequestHook", "afterRequestHook"]),
1394
- parameters: zod_default.record(zod_default.string(), zod_default.unknown()).optional().default({}),
1395
- enabled: zod_default.boolean().optional().default(true),
1396
- priority: zod_default.number().int().optional().default(0),
1397
- onFail: zod_default.enum(["block", "log"]).optional().default("block")
1388
+ //#region src/datalayer/playgrounds.ts
1389
+ const createNewPlayground = zod_default.object({
1390
+ name: zod_default.string(),
1391
+ datasetId: zod_default.string().uuid().nullable().optional(),
1392
+ columns: zod_default.array(playgroundColumnSchema).nullable().optional()
1398
1393
  });
1399
- const updateGuardrailConfig = zod_default.object({
1400
- id: zod_default.string().uuid(),
1401
- name: zod_default.string().min(1).optional(),
1402
- hookType: zod_default.enum(["beforeRequestHook", "afterRequestHook"]).optional(),
1403
- parameters: zod_default.record(zod_default.string(), zod_default.unknown()).optional(),
1404
- enabled: zod_default.boolean().optional(),
1405
- priority: zod_default.number().int().optional(),
1406
- onFail: zod_default.enum(["block", "log"]).optional()
1394
+ const updatePlayground = zod_default.object({
1395
+ playgroundId: zod_default.uuidv4(),
1396
+ name: zod_default.string().optional(),
1397
+ datasetId: zod_default.string().uuid().nullable().optional(),
1398
+ columns: zod_default.array(playgroundColumnSchema).nullable().optional()
1407
1399
  });
1408
- const getGuardrailConfigById = zod_default.object({ id: zod_default.string().uuid() });
1409
- const deleteGuardrailConfig = zod_default.object({ id: zod_default.string().uuid() });
1410
- const listGuardrailConfigs = zod_default.object({
1400
+ const getPlaygroundById = zod_default.object({ playgroundId: zod_default.uuidv4() });
1401
+ const deletePlayground = zod_default.object({ playgroundId: zod_default.uuidv4() });
1402
+ const listPlaygrounds = zod_default.object({
1411
1403
  limit: zod_default.number().int().positive().optional(),
1412
- offset: zod_default.number().int().nonnegative().optional(),
1413
- hookType: zod_default.enum(["beforeRequestHook", "afterRequestHook"]).optional(),
1414
- enabled: zod_default.boolean().optional()
1404
+ offset: zod_default.number().int().nonnegative().optional()
1415
1405
  });
1416
- const createGuardrailConfigsDataLayer = (db) => {
1406
+ const createPlaygroundDataLayer = (db) => {
1417
1407
  return {
1418
- createGuardrailConfig: async (params) => {
1419
- const value = await createGuardrailConfig.safeParseAsync(params);
1408
+ createNewPlayground: async (params) => {
1409
+ const value = await createNewPlayground.safeParseAsync(params);
1420
1410
  if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
1421
- const { name, pluginId, functionId, hookType, parameters, enabled, priority, onFail } = value.data;
1422
- return db.insertInto("guardrail_configs").values({
1411
+ const { name, datasetId, columns } = value.data;
1412
+ return db.insertInto("playgrounds").values({
1423
1413
  id: randomUUID(),
1424
1414
  name,
1425
- pluginId,
1426
- functionId,
1427
- hookType,
1428
- parameters: JSON.stringify(parameters),
1429
- enabled,
1430
- priority,
1431
- onFail,
1415
+ datasetId: datasetId ?? null,
1416
+ columns: columns ? JSON.stringify(columns) : null,
1432
1417
  createdAt: (/* @__PURE__ */ new Date()).toISOString(),
1433
1418
  updatedAt: (/* @__PURE__ */ new Date()).toISOString()
1434
1419
  }).returningAll().executeTakeFirst();
1435
1420
  },
1436
- updateGuardrailConfig: async (params) => {
1437
- const value = await updateGuardrailConfig.safeParseAsync(params);
1421
+ updatePlayground: async (params) => {
1422
+ const value = await updatePlayground.safeParseAsync(params);
1438
1423
  if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
1439
- const { id, name, hookType, parameters, enabled, priority, onFail } = value.data;
1424
+ const { playgroundId, name, datasetId, columns } = value.data;
1440
1425
  const updateData = { updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
1441
1426
  if (name !== void 0) updateData.name = name;
1442
- if (hookType !== void 0) updateData.hookType = hookType;
1443
- if (parameters !== void 0) updateData.parameters = JSON.stringify(parameters);
1444
- if (enabled !== void 0) updateData.enabled = enabled;
1445
- if (priority !== void 0) updateData.priority = priority;
1446
- if (onFail !== void 0) updateData.onFail = onFail;
1447
- return db.updateTable("guardrail_configs").set(updateData).where("id", "=", id).returningAll().executeTakeFirst();
1427
+ if (datasetId !== void 0) updateData.datasetId = datasetId;
1428
+ if (columns !== void 0) updateData.columns = columns ? JSON.stringify(columns) : null;
1429
+ return db.updateTable("playgrounds").set(updateData).where("id", "=", playgroundId).returningAll().executeTakeFirst();
1448
1430
  },
1449
- getGuardrailConfigById: async (params) => {
1450
- const value = await getGuardrailConfigById.safeParseAsync(params);
1431
+ getPlaygroundById: async (params) => {
1432
+ const value = await getPlaygroundById.safeParseAsync(params);
1451
1433
  if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
1452
- const { id } = value.data;
1453
- return db.selectFrom("guardrail_configs").selectAll().where("id", "=", id).executeTakeFirst();
1434
+ const { playgroundId } = value.data;
1435
+ return db.selectFrom("playgrounds").selectAll().where("id", "=", playgroundId).executeTakeFirst();
1454
1436
  },
1455
- deleteGuardrailConfig: async (params) => {
1456
- const value = await deleteGuardrailConfig.safeParseAsync(params);
1437
+ deletePlayground: async (params) => {
1438
+ const value = await deletePlayground.safeParseAsync(params);
1457
1439
  if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
1458
- const { id } = value.data;
1459
- return db.deleteFrom("guardrail_configs").where("id", "=", id).returningAll().executeTakeFirst();
1440
+ const { playgroundId } = value.data;
1441
+ return db.deleteFrom("playgrounds").where("id", "=", playgroundId).returningAll().executeTakeFirst();
1460
1442
  },
1461
- listGuardrailConfigs: async (params) => {
1462
- const value = await listGuardrailConfigs.safeParseAsync(params || {});
1443
+ listPlaygrounds: async (params) => {
1444
+ const value = await listPlaygrounds.safeParseAsync(params || {});
1463
1445
  if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
1464
- const { limit = 100, offset = 0, hookType, enabled } = value.data;
1465
- let query = db.selectFrom("guardrail_configs").selectAll().orderBy("priority", "desc").orderBy("createdAt", "desc").limit(limit).offset(offset);
1466
- if (hookType !== void 0) query = query.where("hookType", "=", hookType);
1467
- if (enabled !== void 0) query = query.where("enabled", "=", enabled);
1468
- return query.execute();
1446
+ const { limit = 100, offset = 0 } = value.data;
1447
+ return db.selectFrom("playgrounds").selectAll().orderBy("createdAt", "desc").limit(limit).offset(offset).execute();
1469
1448
  },
1470
- countGuardrailConfigs: async () => {
1471
- const result = await db.selectFrom("guardrail_configs").select(db.fn.countAll().as("count")).executeTakeFirst();
1449
+ countPlaygrounds: async () => {
1450
+ const result = await db.selectFrom("playgrounds").select(db.fn.countAll().as("count")).executeTakeFirst();
1472
1451
  return Number(result?.count ?? 0);
1473
- },
1474
- getEnabledGuardrailsByHookType: async (hookType) => {
1475
- return db.selectFrom("guardrail_configs").selectAll().where("hookType", "=", hookType).where("enabled", "=", true).orderBy("priority", "desc").execute();
1476
1452
  }
1477
1453
  };
1478
1454
  };
1479
1455
 
1480
1456
  //#endregion
1481
- //#region src/datalayer/llmRequests.ts
1482
- /**
1483
- * Schema for individual guardrail result in telemetry
1484
- */
1485
- const guardrailResultSchema = zod_default.object({
1486
- checkId: zod_default.string(),
1487
- functionId: zod_default.string(),
1488
- hookType: zod_default.enum(["beforeRequestHook", "afterRequestHook"]),
1489
- verdict: zod_default.boolean(),
1490
- latencyMs: zod_default.number()
1491
- });
1492
- /**
1493
- * Schema for guardrail results aggregate
1494
- */
1495
- const guardrailResultsSchema = zod_default.object({
1496
- results: zod_default.array(guardrailResultSchema),
1497
- action: zod_default.enum([
1498
- "allowed",
1499
- "blocked",
1500
- "logged"
1501
- ]),
1502
- totalLatencyMs: zod_default.number()
1503
- });
1504
- /**
1505
- * Schema for inserting a new LLM request log
1506
- */
1507
- const insertLLMRequestSchema = zod_default.object({
1508
- requestId: zod_default.string().uuid(),
1509
- configId: zod_default.string().uuid().nullable().optional(),
1510
- variantId: zod_default.string().uuid().nullable().optional(),
1511
- environmentId: zod_default.string().uuid().nullable().optional(),
1512
- providerConfigId: zod_default.string().uuid().nullable().optional(),
1513
- provider: zod_default.string(),
1514
- model: zod_default.string(),
1515
- promptTokens: zod_default.number().int().default(0),
1516
- completionTokens: zod_default.number().int().default(0),
1517
- totalTokens: zod_default.number().int().default(0),
1518
- cachedTokens: zod_default.number().int().default(0),
1519
- cacheCreationTokens: zod_default.number().int().default(0),
1520
- cost: zod_default.number().int().default(0),
1521
- cacheSavings: zod_default.number().int().default(0),
1522
- inputCost: zod_default.number().int().default(0),
1523
- outputCost: zod_default.number().int().default(0),
1524
- endpoint: zod_default.string(),
1525
- statusCode: zod_default.number().int(),
1526
- latencyMs: zod_default.number().int().default(0),
1527
- isStreaming: zod_default.boolean().default(false),
1528
- userId: zod_default.string().nullable().optional(),
1529
- tags: zod_default.record(zod_default.string(), zod_default.string()).default({}),
1530
- guardrailResults: guardrailResultsSchema.nullable().optional(),
1531
- traceId: zod_default.string().nullable().optional(),
1532
- spanId: zod_default.string().nullable().optional(),
1533
- parentSpanId: zod_default.string().nullable().optional(),
1534
- sessionId: zod_default.string().nullable().optional()
1535
- });
1536
- /**
1537
- * Schema for listing LLM requests
1538
- */
1539
- const listRequestsSchema = zod_default.object({
1540
- limit: zod_default.number().int().positive().max(1e3).default(100),
1541
- offset: zod_default.number().int().nonnegative().default(0),
1542
- configId: zod_default.string().uuid().optional(),
1543
- variantId: zod_default.string().uuid().optional(),
1544
- environmentId: zod_default.string().uuid().optional(),
1545
- providerConfigId: zod_default.string().uuid().optional(),
1546
- provider: zod_default.string().optional(),
1547
- model: zod_default.string().optional(),
1548
- startDate: zod_default.date().optional(),
1549
- endDate: zod_default.date().optional(),
1550
- tags: zod_default.record(zod_default.string(), zod_default.array(zod_default.string())).optional()
1457
+ //#region src/datalayer/playgroundResults.ts
1458
+ const createPlaygroundResult = zod_default.object({
1459
+ runId: zod_default.string().uuid(),
1460
+ columnId: zod_default.string().uuid(),
1461
+ datasetRecordId: zod_default.string().uuid().nullable().optional(),
1462
+ inputVariables: zod_default.record(zod_default.string(), zod_default.unknown()).default({}),
1463
+ status: zod_default.enum([
1464
+ "pending",
1465
+ "running",
1466
+ "completed",
1467
+ "failed"
1468
+ ]).default("pending")
1551
1469
  });
1552
- /**
1553
- * Schema for date range queries with optional filters
1554
- */
1555
- const dateRangeSchema = zod_default.object({
1556
- startDate: zod_default.date(),
1557
- endDate: zod_default.date(),
1558
- configId: zod_default.string().uuid().optional(),
1559
- variantId: zod_default.string().uuid().optional(),
1560
- environmentId: zod_default.string().uuid().optional(),
1561
- tags: zod_default.record(zod_default.string(), zod_default.array(zod_default.string())).optional()
1470
+ const createPlaygroundResultsBatch = zod_default.object({ results: zod_default.array(createPlaygroundResult) });
1471
+ const updatePlaygroundResult = zod_default.object({
1472
+ resultId: zod_default.string().uuid(),
1473
+ outputContent: zod_default.string().nullable().optional(),
1474
+ status: zod_default.enum([
1475
+ "pending",
1476
+ "running",
1477
+ "completed",
1478
+ "failed"
1479
+ ]).optional(),
1480
+ error: zod_default.string().nullable().optional(),
1481
+ latencyMs: zod_default.number().int().nullable().optional(),
1482
+ promptTokens: zod_default.number().int().nullable().optional(),
1483
+ completionTokens: zod_default.number().int().nullable().optional(),
1484
+ totalTokens: zod_default.number().int().nullable().optional(),
1485
+ cost: zod_default.number().int().nullable().optional()
1562
1486
  });
1563
- /**
1564
- * Valid groupBy values for cost summary queries
1565
- */
1566
- const COST_SUMMARY_GROUP_BY = [
1567
- "day",
1568
- "hour",
1569
- "model",
1570
- "provider",
1571
- "endpoint",
1572
- "tags"
1573
- ];
1574
- /**
1575
- * Schema for cost summary with grouping
1576
- */
1577
- const costSummarySchema = zod_default.object({
1578
- startDate: zod_default.date(),
1579
- endDate: zod_default.date(),
1580
- configId: zod_default.string().uuid().optional(),
1581
- variantId: zod_default.string().uuid().optional(),
1582
- environmentId: zod_default.string().uuid().optional(),
1583
- tags: zod_default.record(zod_default.string(), zod_default.array(zod_default.string())).optional(),
1584
- groupBy: zod_default.enum(COST_SUMMARY_GROUP_BY).optional(),
1585
- tagKeys: zod_default.array(zod_default.string()).optional()
1487
+ const getPlaygroundResultById = zod_default.object({ resultId: zod_default.string().uuid() });
1488
+ const listPlaygroundResults = zod_default.object({
1489
+ runId: zod_default.string().uuid(),
1490
+ columnId: zod_default.string().uuid().optional(),
1491
+ limit: zod_default.number().int().positive().optional(),
1492
+ offset: zod_default.number().int().nonnegative().optional()
1586
1493
  });
1587
- /**
1588
- * Helper to create column reference for SQL
1589
- * Uses sql.ref() to properly quote column names for the database
1590
- */
1591
- const col$1 = (name) => sql.ref(name);
1592
- const createLLMRequestsDataLayer = (db) => {
1494
+ const deletePlaygroundResultsByRunId = zod_default.object({ runId: zod_default.string().uuid() });
1495
+ const createPlaygroundResultsDataLayer = (db) => {
1593
1496
  return {
1594
- batchInsertRequests: async (requests) => {
1595
- if (requests.length === 0) return { count: 0 };
1596
- const validatedRequests = await Promise.all(requests.map(async (req) => {
1597
- const result = await insertLLMRequestSchema.safeParseAsync(req);
1598
- if (!result.success) throw new LLMOpsError(`Invalid request data: ${result.error.message}`);
1599
- return result.data;
1600
- }));
1497
+ createPlaygroundResult: async (params) => {
1498
+ const value = await createPlaygroundResult.safeParseAsync(params);
1499
+ if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
1500
+ const { runId, columnId, datasetRecordId, inputVariables, status } = value.data;
1501
+ return db.insertInto("playground_results").values({
1502
+ id: randomUUID(),
1503
+ runId,
1504
+ columnId,
1505
+ datasetRecordId: datasetRecordId ?? null,
1506
+ inputVariables: JSON.stringify(inputVariables),
1507
+ outputContent: null,
1508
+ status,
1509
+ error: null,
1510
+ latencyMs: null,
1511
+ promptTokens: null,
1512
+ completionTokens: null,
1513
+ totalTokens: null,
1514
+ cost: null,
1515
+ createdAt: (/* @__PURE__ */ new Date()).toISOString(),
1516
+ updatedAt: (/* @__PURE__ */ new Date()).toISOString()
1517
+ }).returningAll().executeTakeFirst();
1518
+ },
1519
+ createPlaygroundResultsBatch: async (params) => {
1520
+ const value = await createPlaygroundResultsBatch.safeParseAsync(params);
1521
+ if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
1522
+ const { results } = value.data;
1523
+ if (results.length === 0) return [];
1601
1524
  const now = (/* @__PURE__ */ new Date()).toISOString();
1602
- const values = validatedRequests.map((req) => ({
1525
+ const values = results.map((result) => ({
1603
1526
  id: randomUUID(),
1604
- requestId: req.requestId,
1605
- configId: req.configId ?? null,
1606
- variantId: req.variantId ?? null,
1607
- environmentId: req.environmentId ?? null,
1608
- providerConfigId: req.providerConfigId ?? null,
1609
- provider: req.provider,
1610
- model: req.model,
1611
- promptTokens: req.promptTokens,
1612
- completionTokens: req.completionTokens,
1613
- totalTokens: req.totalTokens,
1614
- cachedTokens: req.cachedTokens,
1615
- cacheCreationTokens: req.cacheCreationTokens,
1616
- cost: req.cost,
1617
- cacheSavings: req.cacheSavings,
1618
- inputCost: req.inputCost,
1619
- outputCost: req.outputCost,
1620
- endpoint: req.endpoint,
1621
- statusCode: req.statusCode,
1622
- latencyMs: req.latencyMs,
1623
- isStreaming: req.isStreaming,
1624
- userId: req.userId ?? null,
1625
- tags: JSON.stringify(req.tags),
1626
- guardrailResults: req.guardrailResults ? JSON.stringify(req.guardrailResults) : null,
1627
- traceId: req.traceId ?? null,
1628
- spanId: req.spanId ?? null,
1629
- parentSpanId: req.parentSpanId ?? null,
1630
- sessionId: req.sessionId ?? null,
1527
+ runId: result.runId,
1528
+ columnId: result.columnId,
1529
+ datasetRecordId: result.datasetRecordId ?? null,
1530
+ inputVariables: JSON.stringify(result.inputVariables),
1531
+ outputContent: null,
1532
+ status: result.status,
1533
+ error: null,
1534
+ latencyMs: null,
1535
+ promptTokens: null,
1536
+ completionTokens: null,
1537
+ totalTokens: null,
1538
+ cost: null,
1631
1539
  createdAt: now,
1632
1540
  updatedAt: now
1633
1541
  }));
1634
- await db.insertInto("llm_requests").values(values).execute();
1635
- return { count: values.length };
1542
+ return db.insertInto("playground_results").values(values).returningAll().execute();
1636
1543
  },
1637
- insertRequest: async (request) => {
1638
- const result = await insertLLMRequestSchema.safeParseAsync(request);
1639
- if (!result.success) throw new LLMOpsError(`Invalid request data: ${result.error.message}`);
1640
- const req = result.data;
1641
- const now = (/* @__PURE__ */ new Date()).toISOString();
1642
- return db.insertInto("llm_requests").values({
1643
- id: randomUUID(),
1644
- requestId: req.requestId,
1645
- configId: req.configId ?? null,
1646
- variantId: req.variantId ?? null,
1647
- environmentId: req.environmentId ?? null,
1648
- providerConfigId: req.providerConfigId ?? null,
1649
- provider: req.provider,
1650
- model: req.model,
1651
- promptTokens: req.promptTokens,
1652
- completionTokens: req.completionTokens,
1653
- totalTokens: req.totalTokens,
1654
- cachedTokens: req.cachedTokens,
1655
- cacheCreationTokens: req.cacheCreationTokens,
1656
- cost: req.cost,
1657
- cacheSavings: req.cacheSavings,
1658
- inputCost: req.inputCost,
1659
- outputCost: req.outputCost,
1660
- endpoint: req.endpoint,
1661
- statusCode: req.statusCode,
1662
- latencyMs: req.latencyMs,
1663
- isStreaming: req.isStreaming,
1664
- userId: req.userId ?? null,
1665
- tags: JSON.stringify(req.tags),
1666
- guardrailResults: req.guardrailResults ? JSON.stringify(req.guardrailResults) : null,
1667
- traceId: req.traceId ?? null,
1668
- spanId: req.spanId ?? null,
1669
- parentSpanId: req.parentSpanId ?? null,
1670
- sessionId: req.sessionId ?? null,
1671
- createdAt: now,
1672
- updatedAt: now
1673
- }).returningAll().executeTakeFirst();
1674
- },
1675
- listRequests: async (params) => {
1676
- const result = await listRequestsSchema.safeParseAsync(params || {});
1677
- if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
1678
- const { limit, offset, configId, variantId, environmentId, providerConfigId, provider, model, startDate, endDate, tags } = result.data;
1679
- let baseQuery = db.selectFrom("llm_requests");
1680
- if (configId) baseQuery = baseQuery.where("configId", "=", configId);
1681
- if (variantId) baseQuery = baseQuery.where("variantId", "=", variantId);
1682
- if (environmentId) baseQuery = baseQuery.where("environmentId", "=", environmentId);
1683
- if (providerConfigId) baseQuery = baseQuery.where("providerConfigId", "=", providerConfigId);
1684
- if (provider) baseQuery = baseQuery.where("provider", "=", provider);
1685
- if (model) baseQuery = baseQuery.where("model", "=", model);
1686
- if (startDate) baseQuery = baseQuery.where(sql`${col$1("createdAt")} >= ${startDate.toISOString()}`);
1687
- if (endDate) baseQuery = baseQuery.where(sql`${col$1("createdAt")} <= ${endDate.toISOString()}`);
1688
- if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
1689
- if (values.length === 0) continue;
1690
- if (values.length === 1) baseQuery = baseQuery.where(sql`${col$1("tags")}->>${key} = ${values[0]}`);
1691
- else {
1692
- const valueList = sql.join(values.map((v) => sql`${v}`));
1693
- baseQuery = baseQuery.where(sql`${col$1("tags")}->>${key} IN (${valueList})`);
1694
- }
1695
- }
1696
- const countResult = await baseQuery.select(sql`COUNT(*)`.as("total")).executeTakeFirst();
1697
- const total = Number(countResult?.total ?? 0);
1698
- return {
1699
- data: await baseQuery.selectAll().orderBy("createdAt", "desc").limit(limit).offset(offset).execute(),
1700
- total,
1701
- limit,
1702
- offset
1703
- };
1704
- },
1705
- getRequestByRequestId: async (requestId) => {
1706
- return db.selectFrom("llm_requests").selectAll().where("requestId", "=", requestId).executeTakeFirst();
1707
- },
1708
- getTotalCost: async (params) => {
1709
- const result = await dateRangeSchema.safeParseAsync(params);
1710
- if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
1711
- const { startDate, endDate, configId, variantId, environmentId, tags } = result.data;
1712
- let query = db.selectFrom("llm_requests").select([
1713
- sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1714
- sql`COALESCE(SUM(${col$1("inputCost")}), 0)`.as("totalInputCost"),
1715
- sql`COALESCE(SUM(${col$1("outputCost")}), 0)`.as("totalOutputCost"),
1716
- sql`COALESCE(SUM(${col$1("promptTokens")}), 0)`.as("totalPromptTokens"),
1717
- sql`COALESCE(SUM(${col$1("completionTokens")}), 0)`.as("totalCompletionTokens"),
1718
- sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens"),
1719
- sql`COALESCE(SUM(${col$1("cachedTokens")}), 0)`.as("totalCachedTokens"),
1720
- sql`COALESCE(SUM(${col$1("cacheSavings")}), 0)`.as("totalCacheSavings"),
1721
- sql`COUNT(*)`.as("requestCount")
1722
- ]).where(sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col$1("createdAt")} <= ${endDate.toISOString()}`);
1723
- if (configId) query = query.where("configId", "=", configId);
1724
- if (variantId) query = query.where("variantId", "=", variantId);
1725
- if (environmentId) query = query.where("environmentId", "=", environmentId);
1726
- if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
1727
- if (values.length === 0) continue;
1728
- if (values.length === 1) query = query.where(sql`${col$1("tags")}->>${key} = ${values[0]}`);
1729
- else {
1730
- const valueList = sql.join(values.map((v) => sql`${v}`));
1731
- query = query.where(sql`${col$1("tags")}->>${key} IN (${valueList})`);
1732
- }
1733
- }
1734
- return await query.executeTakeFirst();
1735
- },
1736
- getCostByModel: async (params) => {
1737
- const result = await dateRangeSchema.safeParseAsync(params);
1738
- if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
1739
- const { startDate, endDate } = result.data;
1740
- return db.selectFrom("llm_requests").select([
1741
- "provider",
1742
- "model",
1743
- sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1744
- sql`COALESCE(SUM(${col$1("inputCost")}), 0)`.as("totalInputCost"),
1745
- sql`COALESCE(SUM(${col$1("outputCost")}), 0)`.as("totalOutputCost"),
1746
- sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens"),
1747
- sql`COUNT(*)`.as("requestCount"),
1748
- sql`AVG(${col$1("latencyMs")})`.as("avgLatencyMs")
1749
- ]).where(sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col$1("createdAt")} <= ${endDate.toISOString()}`).groupBy(["provider", "model"]).orderBy(sql`SUM(${col$1("cost")})`, "desc").execute();
1544
+ updatePlaygroundResult: async (params) => {
1545
+ const value = await updatePlaygroundResult.safeParseAsync(params);
1546
+ if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
1547
+ const { resultId, outputContent, status, error, latencyMs, promptTokens, completionTokens, totalTokens, cost } = value.data;
1548
+ const updateData = { updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
1549
+ if (outputContent !== void 0) updateData.outputContent = outputContent;
1550
+ if (status !== void 0) updateData.status = status;
1551
+ if (error !== void 0) updateData.error = error;
1552
+ if (latencyMs !== void 0) updateData.latencyMs = latencyMs;
1553
+ if (promptTokens !== void 0) updateData.promptTokens = promptTokens;
1554
+ if (completionTokens !== void 0) updateData.completionTokens = completionTokens;
1555
+ if (totalTokens !== void 0) updateData.totalTokens = totalTokens;
1556
+ if (cost !== void 0) updateData.cost = cost;
1557
+ return db.updateTable("playground_results").set(updateData).where("id", "=", resultId).returningAll().executeTakeFirst();
1750
1558
  },
1751
- getCostByProvider: async (params) => {
1752
- const result = await dateRangeSchema.safeParseAsync(params);
1753
- if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
1754
- const { startDate, endDate } = result.data;
1755
- return db.selectFrom("llm_requests").select([
1756
- "provider",
1757
- sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1758
- sql`COALESCE(SUM(${col$1("inputCost")}), 0)`.as("totalInputCost"),
1759
- sql`COALESCE(SUM(${col$1("outputCost")}), 0)`.as("totalOutputCost"),
1760
- sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens"),
1761
- sql`COUNT(*)`.as("requestCount"),
1762
- sql`AVG(${col$1("latencyMs")})`.as("avgLatencyMs")
1763
- ]).where(sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col$1("createdAt")} <= ${endDate.toISOString()}`).groupBy("provider").orderBy(sql`SUM(${col$1("cost")})`, "desc").execute();
1559
+ getPlaygroundResultById: async (params) => {
1560
+ const value = await getPlaygroundResultById.safeParseAsync(params);
1561
+ if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
1562
+ const { resultId } = value.data;
1563
+ return db.selectFrom("playground_results").selectAll().where("id", "=", resultId).executeTakeFirst();
1764
1564
  },
1765
- getDailyCosts: async (params) => {
1766
- const result = await dateRangeSchema.safeParseAsync(params);
1767
- if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
1768
- const { startDate, endDate } = result.data;
1769
- return db.selectFrom("llm_requests").select([
1770
- sql`DATE(${col$1("createdAt")})`.as("date"),
1771
- sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1772
- sql`COALESCE(SUM(${col$1("inputCost")}), 0)`.as("totalInputCost"),
1773
- sql`COALESCE(SUM(${col$1("outputCost")}), 0)`.as("totalOutputCost"),
1774
- sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens"),
1775
- sql`COUNT(*)`.as("requestCount")
1776
- ]).where(sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col$1("createdAt")} <= ${endDate.toISOString()}`).groupBy(sql`DATE(${col$1("createdAt")})`).orderBy(sql`DATE(${col$1("createdAt")})`, "asc").execute();
1565
+ listPlaygroundResults: async (params) => {
1566
+ const value = await listPlaygroundResults.safeParseAsync(params);
1567
+ if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
1568
+ const { runId, columnId, limit = 500, offset = 0 } = value.data;
1569
+ let query = db.selectFrom("playground_results").selectAll().where("runId", "=", runId);
1570
+ if (columnId) query = query.where("columnId", "=", columnId);
1571
+ return query.orderBy("createdAt", "asc").limit(limit).offset(offset).execute();
1777
1572
  },
1778
- getCostSummary: async (params) => {
1779
- const result = await costSummarySchema.safeParseAsync(params);
1780
- if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
1781
- const { startDate, endDate, groupBy, configId, variantId, environmentId, tags, tagKeys } = result.data;
1782
- let baseQuery = db.selectFrom("llm_requests").where(sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col$1("createdAt")} <= ${endDate.toISOString()}`);
1783
- if (configId) baseQuery = baseQuery.where("configId", "=", configId);
1784
- if (variantId) baseQuery = baseQuery.where("variantId", "=", variantId);
1785
- if (environmentId) baseQuery = baseQuery.where("environmentId", "=", environmentId);
1786
- if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
1787
- if (values.length === 0) continue;
1788
- if (values.length === 1) baseQuery = baseQuery.where(sql`${col$1("tags")}->>${key} = ${values[0]}`);
1789
- else {
1790
- const valueList = sql.join(values.map((v) => sql`${v}`));
1791
- baseQuery = baseQuery.where(sql`${col$1("tags")}->>${key} IN (${valueList})`);
1792
- }
1793
- }
1794
- switch (groupBy) {
1795
- case "day": return baseQuery.select([
1796
- sql`DATE(${col$1("createdAt")})`.as("groupKey"),
1797
- sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1798
- sql`COUNT(*)`.as("requestCount"),
1799
- sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens")
1800
- ]).groupBy(sql`DATE(${col$1("createdAt")})`).orderBy(sql`DATE(${col$1("createdAt")})`, "asc").execute();
1801
- case "hour": return baseQuery.select([
1802
- sql`DATE_TRUNC('hour', ${col$1("createdAt")})`.as("groupKey"),
1803
- sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1804
- sql`COUNT(*)`.as("requestCount"),
1805
- sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens")
1806
- ]).groupBy(sql`DATE_TRUNC('hour', ${col$1("createdAt")})`).orderBy(sql`DATE_TRUNC('hour', ${col$1("createdAt")})`, "asc").execute();
1807
- case "model": return baseQuery.select([
1808
- sql`${col$1("provider")} || '/' || ${col$1("model")}`.as("groupKey"),
1809
- sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1810
- sql`COUNT(*)`.as("requestCount")
1811
- ]).groupBy(["provider", "model"]).orderBy(sql`SUM(${col$1("cost")})`, "desc").execute();
1812
- case "provider": return baseQuery.select([
1813
- sql`${col$1("provider")}`.as("groupKey"),
1814
- sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1815
- sql`COUNT(*)`.as("requestCount")
1816
- ]).groupBy("provider").orderBy(sql`SUM(${col$1("cost")})`, "desc").execute();
1817
- case "endpoint": return baseQuery.select([
1818
- sql`COALESCE(${col$1("endpoint")}, 'unknown')`.as("groupKey"),
1819
- sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1820
- sql`COUNT(*)`.as("requestCount")
1821
- ]).groupBy("endpoint").orderBy(sql`SUM(${col$1("cost")})`, "desc").execute();
1822
- case "tags": {
1823
- const conditions = [sql`${col$1("createdAt")} >= ${startDate.toISOString()}`, sql`${col$1("createdAt")} <= ${endDate.toISOString()}`];
1824
- if (configId) conditions.push(sql`${col$1("configId")} = ${configId}`);
1825
- if (variantId) conditions.push(sql`${col$1("variantId")} = ${variantId}`);
1826
- if (environmentId) conditions.push(sql`${col$1("environmentId")} = ${environmentId}`);
1827
- if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
1828
- if (values.length === 0) continue;
1829
- if (values.length === 1) conditions.push(sql`${col$1("tags")}->>${key} = ${values[0]}`);
1830
- else {
1831
- const valueList = sql.join(values.map((v) => sql`${v}`));
1832
- conditions.push(sql`${col$1("tags")}->>${key} IN (${valueList})`);
1833
- }
1834
- }
1835
- if (tagKeys && tagKeys.length > 0) {
1836
- const tagKeyList = sql.join(tagKeys.map((k) => sql`${k}`), sql`, `);
1837
- conditions.push(sql`t.key IN (${tagKeyList})`);
1838
- }
1839
- const whereClause = sql.join(conditions, sql` AND `);
1840
- return (await sql`
1841
- SELECT t.key || ':' || t.value as "groupKey",
1842
- COALESCE(SUM(${col$1("cost")}), 0) as "totalCost",
1843
- COUNT(*) as "requestCount"
1844
- FROM "llm_requests", jsonb_each_text(${col$1("tags")}) t
1845
- WHERE ${whereClause}
1846
- GROUP BY t.key, t.value
1847
- ORDER BY SUM(${col$1("cost")}) DESC
1848
- `.execute(db)).rows;
1849
- }
1850
- default: return baseQuery.select([
1851
- sql`'total'`.as("groupKey"),
1852
- sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1853
- sql`COUNT(*)`.as("requestCount")
1854
- ]).execute();
1855
- }
1573
+ deletePlaygroundResultsByRunId: async (params) => {
1574
+ const value = await deletePlaygroundResultsByRunId.safeParseAsync(params);
1575
+ if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
1576
+ const { runId } = value.data;
1577
+ return db.deleteFrom("playground_results").where("runId", "=", runId).returningAll().execute();
1856
1578
  },
1857
- getRequestStats: async (params) => {
1858
- const result = await dateRangeSchema.safeParseAsync(params);
1859
- if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
1860
- const { startDate, endDate, configId, variantId, environmentId, tags } = result.data;
1861
- let query = db.selectFrom("llm_requests").select([
1862
- sql`COUNT(*)`.as("totalRequests"),
1863
- sql`COUNT(CASE WHEN ${col$1("statusCode")} >= 200 AND ${col$1("statusCode")} < 300 THEN 1 END)`.as("successfulRequests"),
1864
- sql`COUNT(CASE WHEN ${col$1("statusCode")} >= 400 THEN 1 END)`.as("failedRequests"),
1865
- sql`COUNT(CASE WHEN ${col$1("isStreaming")} = true THEN 1 END)`.as("streamingRequests"),
1866
- sql`AVG(${col$1("latencyMs")})`.as("avgLatencyMs"),
1867
- sql`MAX(${col$1("latencyMs")})`.as("maxLatencyMs"),
1868
- sql`MIN(${col$1("latencyMs")})`.as("minLatencyMs")
1869
- ]).where(sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col$1("createdAt")} <= ${endDate.toISOString()}`);
1870
- if (configId) query = query.where("configId", "=", configId);
1871
- if (variantId) query = query.where("variantId", "=", variantId);
1872
- if (environmentId) query = query.where("environmentId", "=", environmentId);
1873
- if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
1874
- if (values.length === 0) continue;
1875
- if (values.length === 1) query = query.where(sql`${col$1("tags")}->>${key} = ${values[0]}`);
1876
- else {
1877
- const valueList = sql.join(values.map((v) => sql`${v}`));
1878
- query = query.where(sql`${col$1("tags")}->>${key} IN (${valueList})`);
1879
- }
1880
- }
1881
- return await query.executeTakeFirst();
1579
+ countPlaygroundResults: async (runId) => {
1580
+ const result = await db.selectFrom("playground_results").select(db.fn.countAll().as("count")).where("runId", "=", runId).executeTakeFirst();
1581
+ return Number(result?.count ?? 0);
1882
1582
  },
1883
- getDistinctTags: async () => {
1884
- return (await sql`
1885
- SELECT DISTINCT key, value
1886
- FROM llm_requests, jsonb_each_text(tags) AS t(key, value)
1887
- WHERE tags != '{}'::jsonb
1888
- ORDER BY key, value
1889
- `.execute(db)).rows;
1583
+ countCompletedPlaygroundResults: async (runId) => {
1584
+ const result = await db.selectFrom("playground_results").select(db.fn.countAll().as("count")).where("runId", "=", runId).where("status", "=", "completed").executeTakeFirst();
1585
+ return Number(result?.count ?? 0);
1890
1586
  }
1891
1587
  };
1892
1588
  };
1893
1589
 
1894
1590
  //#endregion
1895
- //#region src/datalayer/playgrounds.ts
1896
- const createNewPlayground = zod_default.object({
1897
- name: zod_default.string(),
1591
+ //#region src/datalayer/playgroundRuns.ts
1592
+ const createPlaygroundRun = zod_default.object({
1593
+ playgroundId: zod_default.string().uuid(),
1898
1594
  datasetId: zod_default.string().uuid().nullable().optional(),
1899
- columns: zod_default.array(playgroundColumnSchema).nullable().optional()
1595
+ datasetVersionId: zod_default.string().uuid().nullable().optional(),
1596
+ status: zod_default.enum([
1597
+ "pending",
1598
+ "running",
1599
+ "completed",
1600
+ "failed",
1601
+ "cancelled"
1602
+ ]).default("pending"),
1603
+ totalRecords: zod_default.number().int().default(0)
1900
1604
  });
1901
- const updatePlayground = zod_default.object({
1902
- playgroundId: zod_default.uuidv4(),
1903
- name: zod_default.string().optional(),
1904
- datasetId: zod_default.string().uuid().nullable().optional(),
1905
- columns: zod_default.array(playgroundColumnSchema).nullable().optional()
1605
+ const updatePlaygroundRun = zod_default.object({
1606
+ runId: zod_default.string().uuid(),
1607
+ status: zod_default.enum([
1608
+ "pending",
1609
+ "running",
1610
+ "completed",
1611
+ "failed",
1612
+ "cancelled"
1613
+ ]).optional(),
1614
+ startedAt: zod_default.date().nullable().optional(),
1615
+ completedAt: zod_default.date().nullable().optional(),
1616
+ completedRecords: zod_default.number().int().optional()
1906
1617
  });
1907
- const getPlaygroundById = zod_default.object({ playgroundId: zod_default.uuidv4() });
1908
- const deletePlayground = zod_default.object({ playgroundId: zod_default.uuidv4() });
1909
- const listPlaygrounds = zod_default.object({
1618
+ const getPlaygroundRunById = zod_default.object({ runId: zod_default.string().uuid() });
1619
+ const listPlaygroundRuns = zod_default.object({
1620
+ playgroundId: zod_default.string().uuid(),
1910
1621
  limit: zod_default.number().int().positive().optional(),
1911
1622
  offset: zod_default.number().int().nonnegative().optional()
1912
1623
  });
1913
- const createPlaygroundDataLayer = (db) => {
1624
+ const deletePlaygroundRun = zod_default.object({ runId: zod_default.string().uuid() });
1625
+ const createPlaygroundRunsDataLayer = (db) => {
1914
1626
  return {
1915
- createNewPlayground: async (params) => {
1916
- const value = await createNewPlayground.safeParseAsync(params);
1917
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
1918
- const { name, datasetId, columns } = value.data;
1919
- return db.insertInto("playgrounds").values({
1920
- id: randomUUID(),
1921
- name,
1922
- datasetId: datasetId ?? null,
1923
- columns: columns ? JSON.stringify(columns) : null,
1924
- createdAt: (/* @__PURE__ */ new Date()).toISOString(),
1925
- updatedAt: (/* @__PURE__ */ new Date()).toISOString()
1926
- }).returningAll().executeTakeFirst();
1927
- },
1928
- updatePlayground: async (params) => {
1929
- const value = await updatePlayground.safeParseAsync(params);
1930
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
1931
- const { playgroundId, name, datasetId, columns } = value.data;
1932
- const updateData = { updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
1933
- if (name !== void 0) updateData.name = name;
1934
- if (datasetId !== void 0) updateData.datasetId = datasetId;
1935
- if (columns !== void 0) updateData.columns = columns ? JSON.stringify(columns) : null;
1936
- return db.updateTable("playgrounds").set(updateData).where("id", "=", playgroundId).returningAll().executeTakeFirst();
1937
- },
1938
- getPlaygroundById: async (params) => {
1939
- const value = await getPlaygroundById.safeParseAsync(params);
1940
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
1941
- const { playgroundId } = value.data;
1942
- return db.selectFrom("playgrounds").selectAll().where("id", "=", playgroundId).executeTakeFirst();
1943
- },
1944
- deletePlayground: async (params) => {
1945
- const value = await deletePlayground.safeParseAsync(params);
1946
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
1947
- const { playgroundId } = value.data;
1948
- return db.deleteFrom("playgrounds").where("id", "=", playgroundId).returningAll().executeTakeFirst();
1949
- },
1950
- listPlaygrounds: async (params) => {
1951
- const value = await listPlaygrounds.safeParseAsync(params || {});
1952
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
1953
- const { limit = 100, offset = 0 } = value.data;
1954
- return db.selectFrom("playgrounds").selectAll().orderBy("createdAt", "desc").limit(limit).offset(offset).execute();
1955
- },
1956
- countPlaygrounds: async () => {
1957
- const result = await db.selectFrom("playgrounds").select(db.fn.countAll().as("count")).executeTakeFirst();
1958
- return Number(result?.count ?? 0);
1959
- }
1960
- };
1961
- };
1962
-
1963
- //#endregion
1964
- //#region src/datalayer/playgroundResults.ts
1965
- const createPlaygroundResult = zod_default.object({
1966
- runId: zod_default.string().uuid(),
1967
- columnId: zod_default.string().uuid(),
1968
- datasetRecordId: zod_default.string().uuid().nullable().optional(),
1969
- inputVariables: zod_default.record(zod_default.string(), zod_default.unknown()).default({}),
1970
- status: zod_default.enum([
1971
- "pending",
1972
- "running",
1973
- "completed",
1974
- "failed"
1975
- ]).default("pending")
1976
- });
1977
- const createPlaygroundResultsBatch = zod_default.object({ results: zod_default.array(createPlaygroundResult) });
1978
- const updatePlaygroundResult = zod_default.object({
1979
- resultId: zod_default.string().uuid(),
1980
- outputContent: zod_default.string().nullable().optional(),
1981
- status: zod_default.enum([
1982
- "pending",
1983
- "running",
1984
- "completed",
1985
- "failed"
1986
- ]).optional(),
1987
- error: zod_default.string().nullable().optional(),
1988
- latencyMs: zod_default.number().int().nullable().optional(),
1989
- promptTokens: zod_default.number().int().nullable().optional(),
1990
- completionTokens: zod_default.number().int().nullable().optional(),
1991
- totalTokens: zod_default.number().int().nullable().optional(),
1992
- cost: zod_default.number().int().nullable().optional()
1993
- });
1994
- const getPlaygroundResultById = zod_default.object({ resultId: zod_default.string().uuid() });
1995
- const listPlaygroundResults = zod_default.object({
1996
- runId: zod_default.string().uuid(),
1997
- columnId: zod_default.string().uuid().optional(),
1998
- limit: zod_default.number().int().positive().optional(),
1999
- offset: zod_default.number().int().nonnegative().optional()
2000
- });
2001
- const deletePlaygroundResultsByRunId = zod_default.object({ runId: zod_default.string().uuid() });
2002
- const createPlaygroundResultsDataLayer = (db) => {
2003
- return {
2004
- createPlaygroundResult: async (params) => {
2005
- const value = await createPlaygroundResult.safeParseAsync(params);
2006
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
2007
- const { runId, columnId, datasetRecordId, inputVariables, status } = value.data;
2008
- return db.insertInto("playground_results").values({
2009
- id: randomUUID(),
2010
- runId,
2011
- columnId,
2012
- datasetRecordId: datasetRecordId ?? null,
2013
- inputVariables: JSON.stringify(inputVariables),
2014
- outputContent: null,
2015
- status,
2016
- error: null,
2017
- latencyMs: null,
2018
- promptTokens: null,
2019
- completionTokens: null,
2020
- totalTokens: null,
2021
- cost: null,
2022
- createdAt: (/* @__PURE__ */ new Date()).toISOString(),
2023
- updatedAt: (/* @__PURE__ */ new Date()).toISOString()
2024
- }).returningAll().executeTakeFirst();
2025
- },
2026
- createPlaygroundResultsBatch: async (params) => {
2027
- const value = await createPlaygroundResultsBatch.safeParseAsync(params);
2028
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
2029
- const { results } = value.data;
2030
- if (results.length === 0) return [];
2031
- const now = (/* @__PURE__ */ new Date()).toISOString();
2032
- const values = results.map((result) => ({
2033
- id: randomUUID(),
2034
- runId: result.runId,
2035
- columnId: result.columnId,
2036
- datasetRecordId: result.datasetRecordId ?? null,
2037
- inputVariables: JSON.stringify(result.inputVariables),
2038
- outputContent: null,
2039
- status: result.status,
2040
- error: null,
2041
- latencyMs: null,
2042
- promptTokens: null,
2043
- completionTokens: null,
2044
- totalTokens: null,
2045
- cost: null,
2046
- createdAt: now,
2047
- updatedAt: now
2048
- }));
2049
- return db.insertInto("playground_results").values(values).returningAll().execute();
2050
- },
2051
- updatePlaygroundResult: async (params) => {
2052
- const value = await updatePlaygroundResult.safeParseAsync(params);
2053
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
2054
- const { resultId, outputContent, status, error, latencyMs, promptTokens, completionTokens, totalTokens, cost } = value.data;
2055
- const updateData = { updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
2056
- if (outputContent !== void 0) updateData.outputContent = outputContent;
2057
- if (status !== void 0) updateData.status = status;
2058
- if (error !== void 0) updateData.error = error;
2059
- if (latencyMs !== void 0) updateData.latencyMs = latencyMs;
2060
- if (promptTokens !== void 0) updateData.promptTokens = promptTokens;
2061
- if (completionTokens !== void 0) updateData.completionTokens = completionTokens;
2062
- if (totalTokens !== void 0) updateData.totalTokens = totalTokens;
2063
- if (cost !== void 0) updateData.cost = cost;
2064
- return db.updateTable("playground_results").set(updateData).where("id", "=", resultId).returningAll().executeTakeFirst();
2065
- },
2066
- getPlaygroundResultById: async (params) => {
2067
- const value = await getPlaygroundResultById.safeParseAsync(params);
2068
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
2069
- const { resultId } = value.data;
2070
- return db.selectFrom("playground_results").selectAll().where("id", "=", resultId).executeTakeFirst();
2071
- },
2072
- listPlaygroundResults: async (params) => {
2073
- const value = await listPlaygroundResults.safeParseAsync(params);
2074
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
2075
- const { runId, columnId, limit = 500, offset = 0 } = value.data;
2076
- let query = db.selectFrom("playground_results").selectAll().where("runId", "=", runId);
2077
- if (columnId) query = query.where("columnId", "=", columnId);
2078
- return query.orderBy("createdAt", "asc").limit(limit).offset(offset).execute();
2079
- },
2080
- deletePlaygroundResultsByRunId: async (params) => {
2081
- const value = await deletePlaygroundResultsByRunId.safeParseAsync(params);
2082
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
2083
- const { runId } = value.data;
2084
- return db.deleteFrom("playground_results").where("runId", "=", runId).returningAll().execute();
2085
- },
2086
- countPlaygroundResults: async (runId) => {
2087
- const result = await db.selectFrom("playground_results").select(db.fn.countAll().as("count")).where("runId", "=", runId).executeTakeFirst();
2088
- return Number(result?.count ?? 0);
2089
- },
2090
- countCompletedPlaygroundResults: async (runId) => {
2091
- const result = await db.selectFrom("playground_results").select(db.fn.countAll().as("count")).where("runId", "=", runId).where("status", "=", "completed").executeTakeFirst();
2092
- return Number(result?.count ?? 0);
2093
- }
2094
- };
2095
- };
2096
-
2097
- //#endregion
2098
- //#region src/datalayer/playgroundRuns.ts
2099
- const createPlaygroundRun = zod_default.object({
2100
- playgroundId: zod_default.string().uuid(),
2101
- datasetId: zod_default.string().uuid().nullable().optional(),
2102
- datasetVersionId: zod_default.string().uuid().nullable().optional(),
2103
- status: zod_default.enum([
2104
- "pending",
2105
- "running",
2106
- "completed",
2107
- "failed",
2108
- "cancelled"
2109
- ]).default("pending"),
2110
- totalRecords: zod_default.number().int().default(0)
2111
- });
2112
- const updatePlaygroundRun = zod_default.object({
2113
- runId: zod_default.string().uuid(),
2114
- status: zod_default.enum([
2115
- "pending",
2116
- "running",
2117
- "completed",
2118
- "failed",
2119
- "cancelled"
2120
- ]).optional(),
2121
- startedAt: zod_default.date().nullable().optional(),
2122
- completedAt: zod_default.date().nullable().optional(),
2123
- completedRecords: zod_default.number().int().optional()
2124
- });
2125
- const getPlaygroundRunById = zod_default.object({ runId: zod_default.string().uuid() });
2126
- const listPlaygroundRuns = zod_default.object({
2127
- playgroundId: zod_default.string().uuid(),
2128
- limit: zod_default.number().int().positive().optional(),
2129
- offset: zod_default.number().int().nonnegative().optional()
2130
- });
2131
- const deletePlaygroundRun = zod_default.object({ runId: zod_default.string().uuid() });
2132
- const createPlaygroundRunsDataLayer = (db) => {
2133
- return {
2134
- createPlaygroundRun: async (params) => {
2135
- const value = await createPlaygroundRun.safeParseAsync(params);
1627
+ createPlaygroundRun: async (params) => {
1628
+ const value = await createPlaygroundRun.safeParseAsync(params);
2136
1629
  if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
2137
1630
  const { playgroundId, datasetId, datasetVersionId, status, totalRecords } = value.data;
2138
1631
  return db.insertInto("playground_runs").values({
@@ -2186,478 +1679,641 @@ const createPlaygroundRunsDataLayer = (db) => {
2186
1679
  };
2187
1680
 
2188
1681
  //#endregion
2189
- //#region src/datalayer/providerConfigs.ts
2190
- /**
2191
- * Generate a unique slug for a provider config.
2192
- * If the base slug already exists, appends -01, -02, etc.
2193
- */
2194
- async function generateUniqueSlug(db, baseSlug) {
2195
- const existing = await db.selectFrom("provider_configs").select("slug").where("slug", "like", `${baseSlug}%`).execute();
2196
- if (existing.length === 0) return baseSlug;
2197
- const existingSlugs = new Set(existing.map((e) => e.slug));
2198
- if (!existingSlugs.has(baseSlug)) return baseSlug;
2199
- let counter = 1;
2200
- while (counter < 100) {
2201
- const candidateSlug = `${baseSlug}-${counter.toString().padStart(2, "0")}`;
2202
- if (!existingSlugs.has(candidateSlug)) return candidateSlug;
2203
- counter++;
2204
- }
2205
- return `${baseSlug}-${randomUUID().slice(0, 8)}`;
2206
- }
2207
- const createProviderConfig = zod_default.object({
2208
- providerId: zod_default.string().min(1),
2209
- slug: zod_default.string().nullable().optional(),
2210
- name: zod_default.string().nullable().optional(),
2211
- config: zod_default.record(zod_default.string(), zod_default.unknown()),
2212
- enabled: zod_default.boolean().optional().default(true)
2213
- });
2214
- const updateProviderConfig = zod_default.object({
2215
- id: zod_default.uuidv4(),
2216
- slug: zod_default.string().nullable().optional(),
1682
+ //#region src/datalayer/workspaceSettings.ts
1683
+ const updateWorkspaceSettings = zod_default.object({
2217
1684
  name: zod_default.string().nullable().optional(),
2218
- config: zod_default.record(zod_default.string(), zod_default.unknown()).optional(),
2219
- enabled: zod_default.boolean().optional()
2220
- });
2221
- const getProviderConfigById = zod_default.object({ id: zod_default.uuidv4() });
2222
- const getProviderConfigByProviderId = zod_default.object({ providerId: zod_default.string().min(1) });
2223
- const getProviderConfigBySlug = zod_default.object({ slug: zod_default.string().min(1) });
2224
- const deleteProviderConfig = zod_default.object({ id: zod_default.uuidv4() });
2225
- const listProviderConfigs = zod_default.object({
2226
- limit: zod_default.number().int().positive().optional(),
2227
- offset: zod_default.number().int().nonnegative().optional()
1685
+ setupComplete: zod_default.boolean().optional(),
1686
+ superAdminId: zod_default.string().nullable().optional()
2228
1687
  });
2229
- const createProviderConfigsDataLayer = (db) => {
1688
+ const createWorkspaceSettingsDataLayer = (db) => {
2230
1689
  return {
2231
- createProviderConfig: async (params) => {
2232
- const value = await createProviderConfig.safeParseAsync(params);
2233
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
2234
- const { providerId, slug, name, config, enabled } = value.data;
2235
- const finalSlug = slug ?? await generateUniqueSlug(db, providerId);
2236
- return db.insertInto("provider_configs").values({
1690
+ getWorkspaceSettings: async () => {
1691
+ let settings = await db.selectFrom("workspace_settings").selectAll().executeTakeFirst();
1692
+ if (!settings) settings = await db.insertInto("workspace_settings").values({
2237
1693
  id: randomUUID(),
2238
- providerId,
2239
- slug: finalSlug,
2240
- name: name ?? null,
2241
- config: JSON.stringify(config),
2242
- enabled,
1694
+ name: null,
1695
+ setupComplete: false,
2243
1696
  createdAt: (/* @__PURE__ */ new Date()).toISOString(),
2244
1697
  updatedAt: (/* @__PURE__ */ new Date()).toISOString()
2245
1698
  }).returningAll().executeTakeFirst();
1699
+ return settings;
2246
1700
  },
2247
- updateProviderConfig: async (params) => {
2248
- const value = await updateProviderConfig.safeParseAsync(params);
1701
+ updateWorkspaceSettings: async (params) => {
1702
+ const value = await updateWorkspaceSettings.safeParseAsync(params);
2249
1703
  if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
2250
- const { id, slug, name, config, enabled } = value.data;
1704
+ let settings = await db.selectFrom("workspace_settings").selectAll().executeTakeFirst();
1705
+ if (!settings) return db.insertInto("workspace_settings").values({
1706
+ id: randomUUID(),
1707
+ name: value.data.name ?? null,
1708
+ setupComplete: value.data.setupComplete ?? false,
1709
+ createdAt: (/* @__PURE__ */ new Date()).toISOString(),
1710
+ updatedAt: (/* @__PURE__ */ new Date()).toISOString()
1711
+ }).returningAll().executeTakeFirst();
2251
1712
  const updateData = { updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
2252
- if (slug !== void 0) updateData.slug = slug;
2253
- if (name !== void 0) updateData.name = name;
2254
- if (config !== void 0) updateData.config = JSON.stringify(config);
2255
- if (enabled !== void 0) updateData.enabled = enabled;
2256
- return db.updateTable("provider_configs").set(updateData).where("id", "=", id).returningAll().executeTakeFirst();
2257
- },
2258
- getProviderConfigById: async (params) => {
2259
- const value = await getProviderConfigById.safeParseAsync(params);
2260
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
2261
- const { id } = value.data;
2262
- return db.selectFrom("provider_configs").selectAll().where("id", "=", id).executeTakeFirst();
1713
+ if (value.data.name !== void 0) updateData.name = value.data.name ?? null;
1714
+ if (value.data.setupComplete !== void 0) updateData.setupComplete = value.data.setupComplete;
1715
+ if (value.data.superAdminId !== void 0) updateData.superAdminId = value.data.superAdminId ?? null;
1716
+ return db.updateTable("workspace_settings").set(updateData).where("id", "=", settings.id).returningAll().executeTakeFirst();
2263
1717
  },
2264
- getProviderConfigByProviderId: async (params) => {
2265
- const value = await getProviderConfigByProviderId.safeParseAsync(params);
2266
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
2267
- const { providerId } = value.data;
2268
- return db.selectFrom("provider_configs").selectAll().where("providerId", "=", providerId).executeTakeFirst();
2269
- },
2270
- getProviderConfigBySlug: async (params) => {
2271
- const value = await getProviderConfigBySlug.safeParseAsync(params);
2272
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
2273
- const { slug } = value.data;
2274
- return db.selectFrom("provider_configs").selectAll().where("slug", "=", slug).executeTakeFirst();
2275
- },
2276
- deleteProviderConfig: async (params) => {
2277
- const value = await deleteProviderConfig.safeParseAsync(params);
2278
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
2279
- const { id } = value.data;
2280
- return db.deleteFrom("provider_configs").where("id", "=", id).returningAll().executeTakeFirst();
2281
- },
2282
- listProviderConfigs: async (params) => {
2283
- const value = await listProviderConfigs.safeParseAsync(params || {});
2284
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
2285
- const { limit = 100, offset = 0 } = value.data;
2286
- return db.selectFrom("provider_configs").selectAll().orderBy("createdAt", "desc").limit(limit).offset(offset).execute();
2287
- },
2288
- countProviderConfigs: async () => {
2289
- const result = await db.selectFrom("provider_configs").select(db.fn.countAll().as("count")).executeTakeFirst();
2290
- return Number(result?.count ?? 0);
1718
+ getSuperAdminId: async () => {
1719
+ return (await db.selectFrom("workspace_settings").select("superAdminId").executeTakeFirst())?.superAdminId ?? null;
2291
1720
  },
2292
- upsertProviderConfig: async (params) => {
2293
- const value = await createProviderConfig.safeParseAsync(params);
2294
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
2295
- const { providerId, slug, name, config, enabled } = value.data;
2296
- const existing = await db.selectFrom("provider_configs").selectAll().where("providerId", "=", providerId).executeTakeFirst();
2297
- if (existing) {
2298
- const finalSlug$1 = slug ?? existing.slug ?? await generateUniqueSlug(db, providerId);
2299
- return db.updateTable("provider_configs").set({
2300
- slug: finalSlug$1,
2301
- name: name ?? existing.name,
2302
- config: JSON.stringify(config),
2303
- enabled,
1721
+ setSuperAdminId: async (userId) => {
1722
+ let settings = await db.selectFrom("workspace_settings").selectAll().executeTakeFirst();
1723
+ if (settings?.superAdminId) return false;
1724
+ if (!settings) {
1725
+ await db.insertInto("workspace_settings").values({
1726
+ id: randomUUID(),
1727
+ name: null,
1728
+ setupComplete: false,
1729
+ superAdminId: userId,
1730
+ createdAt: (/* @__PURE__ */ new Date()).toISOString(),
2304
1731
  updatedAt: (/* @__PURE__ */ new Date()).toISOString()
2305
- }).where("id", "=", existing.id).returningAll().executeTakeFirst();
1732
+ }).execute();
1733
+ return true;
2306
1734
  }
2307
- const finalSlug = slug ?? await generateUniqueSlug(db, providerId);
2308
- return db.insertInto("provider_configs").values({
2309
- id: randomUUID(),
2310
- providerId,
2311
- slug: finalSlug,
2312
- name: name ?? null,
2313
- config: JSON.stringify(config),
2314
- enabled,
2315
- createdAt: (/* @__PURE__ */ new Date()).toISOString(),
2316
- updatedAt: (/* @__PURE__ */ new Date()).toISOString()
2317
- }).returningAll().executeTakeFirst();
2318
- }
2319
- };
2320
- };
2321
-
2322
- //#endregion
2323
- //#region src/datalayer/providerGuardrailOverrides.ts
2324
- const createProviderGuardrailOverride = zod_default.object({
2325
- providerConfigId: zod_default.string().uuid(),
2326
- guardrailConfigId: zod_default.string().uuid(),
2327
- enabled: zod_default.boolean().optional().default(true),
2328
- parameters: zod_default.record(zod_default.string(), zod_default.unknown()).nullable().optional()
2329
- });
2330
- const updateProviderGuardrailOverride = zod_default.object({
2331
- id: zod_default.string().uuid(),
2332
- enabled: zod_default.boolean().optional(),
2333
- parameters: zod_default.record(zod_default.string(), zod_default.unknown()).nullable().optional()
2334
- });
2335
- const getOverrideById = zod_default.object({ id: zod_default.string().uuid() });
2336
- const deleteOverride = zod_default.object({ id: zod_default.string().uuid() });
2337
- const getOverridesByProviderConfigId = zod_default.object({ providerConfigId: zod_default.string().uuid() });
2338
- const getOverridesByGuardrailConfigId = zod_default.object({ guardrailConfigId: zod_default.string().uuid() });
2339
- const getOverrideByProviderAndGuardrail = zod_default.object({
2340
- providerConfigId: zod_default.string().uuid(),
2341
- guardrailConfigId: zod_default.string().uuid()
2342
- });
2343
- const createProviderGuardrailOverridesDataLayer = (db) => {
2344
- return {
2345
- createProviderGuardrailOverride: async (params) => {
2346
- const value = await createProviderGuardrailOverride.safeParseAsync(params);
2347
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
2348
- const { providerConfigId, guardrailConfigId, enabled, parameters } = value.data;
2349
- return db.insertInto("provider_guardrail_overrides").values({
2350
- id: randomUUID(),
2351
- providerConfigId,
2352
- guardrailConfigId,
2353
- enabled,
2354
- parameters: parameters ? JSON.stringify(parameters) : null,
2355
- createdAt: (/* @__PURE__ */ new Date()).toISOString(),
1735
+ await db.updateTable("workspace_settings").set({
1736
+ superAdminId: userId,
2356
1737
  updatedAt: (/* @__PURE__ */ new Date()).toISOString()
2357
- }).returningAll().executeTakeFirst();
2358
- },
2359
- updateProviderGuardrailOverride: async (params) => {
2360
- const value = await updateProviderGuardrailOverride.safeParseAsync(params);
2361
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
2362
- const { id, enabled, parameters } = value.data;
2363
- const updateData = { updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
2364
- if (enabled !== void 0) updateData.enabled = enabled;
2365
- if (parameters !== void 0) updateData.parameters = parameters ? JSON.stringify(parameters) : null;
2366
- return db.updateTable("provider_guardrail_overrides").set(updateData).where("id", "=", id).returningAll().executeTakeFirst();
2367
- },
2368
- getOverrideById: async (params) => {
2369
- const value = await getOverrideById.safeParseAsync(params);
2370
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
2371
- const { id } = value.data;
2372
- return db.selectFrom("provider_guardrail_overrides").selectAll().where("id", "=", id).executeTakeFirst();
2373
- },
2374
- deleteProviderGuardrailOverride: async (params) => {
2375
- const value = await deleteOverride.safeParseAsync(params);
2376
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
2377
- const { id } = value.data;
2378
- return db.deleteFrom("provider_guardrail_overrides").where("id", "=", id).returningAll().executeTakeFirst();
2379
- },
2380
- getOverridesByProviderConfigId: async (params) => {
2381
- const value = await getOverridesByProviderConfigId.safeParseAsync(params);
2382
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
2383
- const { providerConfigId } = value.data;
2384
- return db.selectFrom("provider_guardrail_overrides").selectAll().where("providerConfigId", "=", providerConfigId).execute();
2385
- },
2386
- getOverridesByGuardrailConfigId: async (params) => {
2387
- const value = await getOverridesByGuardrailConfigId.safeParseAsync(params);
2388
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
2389
- const { guardrailConfigId } = value.data;
2390
- return db.selectFrom("provider_guardrail_overrides").selectAll().where("guardrailConfigId", "=", guardrailConfigId).execute();
1738
+ }).where("id", "=", settings.id).execute();
1739
+ return true;
2391
1740
  },
2392
- getOverrideByProviderAndGuardrail: async (params) => {
2393
- const value = await getOverrideByProviderAndGuardrail.safeParseAsync(params);
2394
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
2395
- const { providerConfigId, guardrailConfigId } = value.data;
2396
- return db.selectFrom("provider_guardrail_overrides").selectAll().where("providerConfigId", "=", providerConfigId).where("guardrailConfigId", "=", guardrailConfigId).executeTakeFirst();
1741
+ isSetupComplete: async () => {
1742
+ try {
1743
+ return (await db.selectFrom("workspace_settings").select("setupComplete").executeTakeFirst())?.setupComplete ?? false;
1744
+ } catch {
1745
+ return false;
1746
+ }
2397
1747
  },
2398
- upsertProviderGuardrailOverride: async (params) => {
2399
- const value = await createProviderGuardrailOverride.safeParseAsync(params);
2400
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
2401
- const { providerConfigId, guardrailConfigId, enabled, parameters } = value.data;
2402
- const existing = await db.selectFrom("provider_guardrail_overrides").selectAll().where("providerConfigId", "=", providerConfigId).where("guardrailConfigId", "=", guardrailConfigId).executeTakeFirst();
2403
- if (existing) return db.updateTable("provider_guardrail_overrides").set({
2404
- enabled,
2405
- parameters: parameters ? JSON.stringify(parameters) : null,
2406
- updatedAt: (/* @__PURE__ */ new Date()).toISOString()
2407
- }).where("id", "=", existing.id).returningAll().executeTakeFirst();
2408
- return db.insertInto("provider_guardrail_overrides").values({
1748
+ markSetupComplete: async () => {
1749
+ let settings = await db.selectFrom("workspace_settings").selectAll().executeTakeFirst();
1750
+ if (!settings) return db.insertInto("workspace_settings").values({
2409
1751
  id: randomUUID(),
2410
- providerConfigId,
2411
- guardrailConfigId,
2412
- enabled,
2413
- parameters: parameters ? JSON.stringify(parameters) : null,
1752
+ name: null,
1753
+ setupComplete: true,
2414
1754
  createdAt: (/* @__PURE__ */ new Date()).toISOString(),
2415
1755
  updatedAt: (/* @__PURE__ */ new Date()).toISOString()
2416
1756
  }).returningAll().executeTakeFirst();
2417
- },
2418
- deleteOverridesByGuardrailConfigId: async (params) => {
2419
- const value = await getOverridesByGuardrailConfigId.safeParseAsync(params);
2420
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
2421
- const { guardrailConfigId } = value.data;
2422
- return db.deleteFrom("provider_guardrail_overrides").where("guardrailConfigId", "=", guardrailConfigId).execute();
1757
+ return db.updateTable("workspace_settings").set({
1758
+ setupComplete: true,
1759
+ updatedAt: (/* @__PURE__ */ new Date()).toISOString()
1760
+ }).where("id", "=", settings.id).returningAll().executeTakeFirst();
2423
1761
  }
2424
1762
  };
2425
1763
  };
2426
1764
 
2427
1765
  //#endregion
2428
- //#region src/datalayer/traces.ts
2429
- const col = (name) => sql.ref(name);
1766
+ //#region src/datalayer/create.ts
2430
1767
  /**
2431
- * Schema for upserting a trace
1768
+ * Create all datalayers from a Kysely database instance.
1769
+ * Returns a flat object with all datalayer methods spread together.
2432
1770
  */
2433
- const upsertTraceSchema = zod_default.object({
2434
- traceId: zod_default.string(),
2435
- name: zod_default.string().nullable().optional(),
2436
- sessionId: zod_default.string().nullable().optional(),
2437
- userId: zod_default.string().nullable().optional(),
2438
- status: zod_default.enum([
2439
- "unset",
2440
- "ok",
2441
- "error"
2442
- ]).default("unset"),
2443
- startTime: zod_default.date(),
2444
- endTime: zod_default.date().nullable().optional(),
2445
- durationMs: zod_default.number().int().nullable().optional(),
2446
- spanCount: zod_default.number().int().default(1),
2447
- totalInputTokens: zod_default.number().int().default(0),
2448
- totalOutputTokens: zod_default.number().int().default(0),
2449
- totalTokens: zod_default.number().int().default(0),
2450
- totalCost: zod_default.number().int().default(0),
2451
- tags: zod_default.record(zod_default.string(), zod_default.string()).default({}),
2452
- metadata: zod_default.record(zod_default.string(), zod_default.unknown()).default({})
2453
- });
1771
+ function createDataLayer(db) {
1772
+ return {
1773
+ ...createDatasetsDataLayer(db),
1774
+ ...createPlaygroundDataLayer(db),
1775
+ ...createPlaygroundResultsDataLayer(db),
1776
+ ...createPlaygroundRunsDataLayer(db),
1777
+ ...createWorkspaceSettingsDataLayer(db)
1778
+ };
1779
+ }
1780
+
1781
+ //#endregion
1782
+ //#region src/pricing/calculator.ts
2454
1783
  /**
2455
- * Schema for inserting spans
1784
+ * Calculate the cost of an LLM request in micro-dollars
1785
+ *
1786
+ * Micro-dollars are used to avoid floating-point precision issues:
1787
+ * - 1 dollar = 1,000,000 micro-dollars
1788
+ * - $0.001 = 1,000 micro-dollars
1789
+ * - $0.000001 = 1 micro-dollar
1790
+ *
1791
+ * @param usage - Token usage data from the LLM response
1792
+ * @param pricing - Model pricing information
1793
+ * @returns Cost breakdown in micro-dollars
1794
+ *
1795
+ * @example
1796
+ * ```typescript
1797
+ * const usage = { promptTokens: 1000, completionTokens: 500 };
1798
+ * const pricing = { inputCostPer1M: 2.5, outputCostPer1M: 10.0 };
1799
+ * const cost = calculateCost(usage, pricing);
1800
+ * // cost = { inputCost: 2500, outputCost: 5000, totalCost: 7500 }
1801
+ * // In dollars: $0.0025 input + $0.005 output = $0.0075 total
1802
+ * ```
2456
1803
  */
2457
- const insertSpanSchema = zod_default.object({
2458
- traceId: zod_default.string(),
2459
- spanId: zod_default.string(),
2460
- parentSpanId: zod_default.string().nullable().optional(),
2461
- name: zod_default.string(),
2462
- kind: zod_default.number().int().default(1),
2463
- status: zod_default.number().int().default(0),
2464
- statusMessage: zod_default.string().nullable().optional(),
2465
- startTime: zod_default.date(),
2466
- endTime: zod_default.date().nullable().optional(),
2467
- durationMs: zod_default.number().int().nullable().optional(),
2468
- provider: zod_default.string().nullable().optional(),
2469
- model: zod_default.string().nullable().optional(),
2470
- promptTokens: zod_default.number().int().default(0),
2471
- completionTokens: zod_default.number().int().default(0),
2472
- totalTokens: zod_default.number().int().default(0),
2473
- cost: zod_default.number().int().default(0),
2474
- configId: zod_default.string().uuid().nullable().optional(),
2475
- variantId: zod_default.string().uuid().nullable().optional(),
2476
- environmentId: zod_default.string().uuid().nullable().optional(),
2477
- providerConfigId: zod_default.string().uuid().nullable().optional(),
2478
- requestId: zod_default.string().uuid().nullable().optional(),
2479
- source: zod_default.enum([
2480
- "gateway",
2481
- "otlp",
2482
- "langsmith"
2483
- ]).default("gateway"),
2484
- input: zod_default.unknown().nullable().optional(),
2485
- output: zod_default.unknown().nullable().optional(),
2486
- attributes: zod_default.record(zod_default.string(), zod_default.unknown()).default({})
2487
- });
1804
+ function calculateCost(usage, pricing) {
1805
+ const inputCost = Math.round(usage.promptTokens * pricing.inputCostPer1M);
1806
+ const outputCost = Math.round(usage.completionTokens * pricing.outputCostPer1M);
1807
+ return {
1808
+ inputCost,
1809
+ outputCost,
1810
+ totalCost: inputCost + outputCost,
1811
+ cacheSavings: 0
1812
+ };
1813
+ }
2488
1814
  /**
2489
- * Schema for inserting span events
1815
+ * Get default cache read rate as a fraction of input cost per provider.
1816
+ * Used when models.dev doesn't provide cache pricing.
2490
1817
  */
2491
- const insertSpanEventSchema = zod_default.object({
2492
- traceId: zod_default.string(),
2493
- spanId: zod_default.string(),
2494
- name: zod_default.string(),
2495
- timestamp: zod_default.date(),
2496
- attributes: zod_default.record(zod_default.string(), zod_default.unknown()).default({})
2497
- });
1818
+ function getDefaultCacheReadRate(provider, inputCostPer1M) {
1819
+ switch (provider?.toLowerCase()) {
1820
+ case "anthropic": return inputCostPer1M * .1;
1821
+ case "openai":
1822
+ case "azure-openai": return inputCostPer1M * .5;
1823
+ case "google":
1824
+ case "gemini":
1825
+ case "vertex_ai": return inputCostPer1M * .25;
1826
+ default: return inputCostPer1M * .5;
1827
+ }
1828
+ }
2498
1829
  /**
2499
- * Schema for listing traces
1830
+ * Get default cache write/creation rate as a fraction of input cost per provider.
1831
+ * Used when models.dev doesn't provide cache pricing.
2500
1832
  */
2501
- const listTracesSchema = zod_default.object({
2502
- limit: zod_default.number().int().positive().max(1e3).default(50),
2503
- offset: zod_default.number().int().nonnegative().default(0),
2504
- sessionId: zod_default.string().optional(),
2505
- userId: zod_default.string().optional(),
2506
- status: zod_default.enum([
2507
- "unset",
2508
- "ok",
2509
- "error"
2510
- ]).optional(),
2511
- name: zod_default.string().optional(),
2512
- startDate: zod_default.date().optional(),
2513
- endDate: zod_default.date().optional(),
2514
- tags: zod_default.record(zod_default.string(), zod_default.array(zod_default.string())).optional()
2515
- });
1833
+ function getDefaultCacheWriteRate(provider, inputCostPer1M) {
1834
+ switch (provider?.toLowerCase()) {
1835
+ case "anthropic": return inputCostPer1M * 1.25;
1836
+ default: return inputCostPer1M;
1837
+ }
1838
+ }
2516
1839
  /**
2517
- * Schema for trace stats query
1840
+ * Calculate cache-aware cost of an LLM request in micro-dollars.
1841
+ *
1842
+ * Splits input tokens into uncached, cache-read, and cache-creation buckets,
1843
+ * each priced at different rates. Falls back to provider-specific multipliers
1844
+ * when models.dev doesn't provide cache pricing.
1845
+ *
1846
+ * @param usage - Token usage data (with cachedTokens and cacheCreationTokens)
1847
+ * @param pricing - Model pricing (may include cacheReadCostPer1M / cacheWriteCostPer1M)
1848
+ * @param provider - Provider name for fallback rate selection
1849
+ * @returns Cost breakdown in micro-dollars
2518
1850
  */
2519
- const traceStatsSchema = zod_default.object({
2520
- startDate: zod_default.date(),
2521
- endDate: zod_default.date(),
2522
- sessionId: zod_default.string().optional(),
2523
- userId: zod_default.string().optional()
2524
- });
2525
- const createTracesDataLayer = (db) => {
1851
+ function calculateCacheAwareCost(usage, pricing, provider) {
1852
+ const cachedTokens = usage.cachedTokens ?? 0;
1853
+ const cacheCreationTokens = usage.cacheCreationTokens ?? 0;
1854
+ if (cachedTokens === 0 && cacheCreationTokens === 0) return calculateCost(usage, pricing);
1855
+ const cacheReadRate = pricing.cacheReadCostPer1M ?? getDefaultCacheReadRate(provider, pricing.inputCostPer1M);
1856
+ const cacheWriteRate = pricing.cacheWriteCostPer1M ?? getDefaultCacheWriteRate(provider, pricing.inputCostPer1M);
1857
+ const uncachedInputTokens = Math.max(0, usage.promptTokens - cachedTokens - cacheCreationTokens);
1858
+ const regularInputCost = Math.round(uncachedInputTokens * pricing.inputCostPer1M);
1859
+ const cacheReadCost = Math.round(cachedTokens * cacheReadRate);
1860
+ const cacheWriteCost = Math.round(cacheCreationTokens * cacheWriteRate);
1861
+ const outputCost = Math.round(usage.completionTokens * pricing.outputCostPer1M);
1862
+ const inputCost = regularInputCost + cacheReadCost + cacheWriteCost;
2526
1863
  return {
2527
- upsertTrace: async (data) => {
2528
- const result = await upsertTraceSchema.safeParseAsync(data);
2529
- if (!result.success) throw new LLMOpsError(`Invalid trace data: ${result.error.message}`);
2530
- const trace = result.data;
2531
- const now = (/* @__PURE__ */ new Date()).toISOString();
2532
- await sql`
2533
- INSERT INTO "traces" (
2534
- "id", "traceId", "name", "sessionId", "userId", "status",
2535
- "startTime", "endTime", "durationMs", "spanCount",
2536
- "totalInputTokens", "totalOutputTokens", "totalTokens", "totalCost",
2537
- "tags", "metadata", "createdAt", "updatedAt"
2538
- ) VALUES (
2539
- ${randomUUID()}, ${trace.traceId}, ${trace.name ?? null}, ${trace.sessionId ?? null},
2540
- ${trace.userId ?? null}, ${trace.status},
2541
- ${trace.startTime.toISOString()}, ${trace.endTime?.toISOString() ?? null},
2542
- ${trace.durationMs ?? null}, ${trace.spanCount},
2543
- ${trace.totalInputTokens}, ${trace.totalOutputTokens},
2544
- ${trace.totalTokens}, ${trace.totalCost},
2545
- ${JSON.stringify(trace.tags)}::jsonb, ${JSON.stringify(trace.metadata)}::jsonb,
2546
- ${now}, ${now}
2547
- )
2548
- ON CONFLICT ("traceId") DO UPDATE SET
2549
- "name" = COALESCE(EXCLUDED."name", "traces"."name"),
2550
- "sessionId" = COALESCE(EXCLUDED."sessionId", "traces"."sessionId"),
2551
- "userId" = COALESCE(EXCLUDED."userId", "traces"."userId"),
2552
- "status" = CASE
2553
- WHEN EXCLUDED."status" = 'error' THEN 'error'
2554
- WHEN EXCLUDED."status" = 'ok' AND "traces"."status" != 'error' THEN 'ok'
2555
- ELSE "traces"."status"
2556
- END,
2557
- "startTime" = LEAST("traces"."startTime", EXCLUDED."startTime"),
2558
- "endTime" = GREATEST(
2559
- COALESCE("traces"."endTime", EXCLUDED."endTime"),
2560
- COALESCE(EXCLUDED."endTime", "traces"."endTime")
2561
- ),
2562
- "durationMs" = EXTRACT(EPOCH FROM (
2563
- GREATEST(
2564
- COALESCE("traces"."endTime", EXCLUDED."endTime"),
2565
- COALESCE(EXCLUDED."endTime", "traces"."endTime")
2566
- ) -
2567
- LEAST("traces"."startTime", EXCLUDED."startTime")
2568
- ))::integer * 1000,
2569
- "spanCount" = "traces"."spanCount" + EXCLUDED."spanCount",
2570
- "totalInputTokens" = "traces"."totalInputTokens" + EXCLUDED."totalInputTokens",
2571
- "totalOutputTokens" = "traces"."totalOutputTokens" + EXCLUDED."totalOutputTokens",
2572
- "totalTokens" = "traces"."totalTokens" + EXCLUDED."totalTokens",
2573
- "totalCost" = "traces"."totalCost" + EXCLUDED."totalCost",
2574
- "tags" = "traces"."tags" || EXCLUDED."tags",
2575
- "metadata" = "traces"."metadata" || EXCLUDED."metadata",
2576
- "updatedAt" = ${now}
2577
- `.execute(db);
2578
- },
2579
- batchInsertSpans: async (spans) => {
2580
- if (spans.length === 0) return { count: 0 };
2581
- const validatedSpans = [];
2582
- for (const span of spans) {
2583
- const result = await insertSpanSchema.safeParseAsync(span);
2584
- if (!result.success) {
2585
- logger.warn(`[batchInsertSpans] Skipping invalid span ${span.spanId}: ${result.error.message}`);
2586
- continue;
2587
- }
2588
- validatedSpans.push(result.data);
1864
+ inputCost,
1865
+ outputCost,
1866
+ totalCost: inputCost + outputCost,
1867
+ cacheSavings: Math.round((cachedTokens + cacheCreationTokens) * pricing.inputCostPer1M) - cacheReadCost - cacheWriteCost
1868
+ };
1869
+ }
1870
+ /**
1871
+ * Convert micro-dollars to dollars
1872
+ *
1873
+ * @param microDollars - Amount in micro-dollars
1874
+ * @returns Amount in dollars
1875
+ *
1876
+ * @example
1877
+ * ```typescript
1878
+ * microDollarsToDollars(7500); // 0.0075
1879
+ * microDollarsToDollars(1000000); // 1.0
1880
+ * ```
1881
+ */
1882
+ function microDollarsToDollars(microDollars) {
1883
+ return microDollars / 1e6;
1884
+ }
1885
+ /**
1886
+ * Convert dollars to micro-dollars
1887
+ *
1888
+ * @param dollars - Amount in dollars
1889
+ * @returns Amount in micro-dollars (rounded to nearest integer)
1890
+ *
1891
+ * @example
1892
+ * ```typescript
1893
+ * dollarsToMicroDollars(0.0075); // 7500
1894
+ * dollarsToMicroDollars(1.0); // 1000000
1895
+ * ```
1896
+ */
1897
+ function dollarsToMicroDollars(dollars) {
1898
+ return Math.round(dollars * 1e6);
1899
+ }
1900
+ /**
1901
+ * Format micro-dollars as a human-readable dollar string
1902
+ *
1903
+ * @param microDollars - Amount in micro-dollars
1904
+ * @param decimals - Number of decimal places (default: 6)
1905
+ * @returns Formatted dollar string
1906
+ *
1907
+ * @example
1908
+ * ```typescript
1909
+ * formatCost(7500); // "$0.007500"
1910
+ * formatCost(1234567, 2); // "$1.23"
1911
+ * ```
1912
+ */
1913
+ function formatCost(microDollars, decimals = 6) {
1914
+ return `$${microDollarsToDollars(microDollars).toFixed(decimals)}`;
1915
+ }
1916
+
1917
+ //#endregion
1918
+ //#region src/pricing/provider.ts
1919
+ const LLMOPS_MODELS_API = "https://models.llmops.build";
1920
+ /**
1921
+ * Convert price from USD cents per token to dollars per 1M tokens.
1922
+ *
1923
+ * API returns cents/token. Our system uses dollars/1M tokens.
1924
+ * Formula: (centsPerToken / 100) * 1_000_000 = centsPerToken * 10_000
1925
+ */
1926
+ function centsPerTokenToCostPer1M(centsPerToken) {
1927
+ return centsPerToken * 1e4;
1928
+ }
1929
+ /**
1930
+ * Pricing provider that fetches per-model data from the LLMOps Models API.
1931
+ *
1932
+ * Features:
1933
+ * - Per-model in-memory cache with configurable TTL (default 5 minutes)
1934
+ * - Deduplicates concurrent fetches for the same model
1935
+ * - Caches null results (404s) to avoid repeated lookups
1936
+ * - Falls back to stale cache on fetch errors
1937
+ */
1938
+ var LLMOpsPricingProvider = class {
1939
+ cache = /* @__PURE__ */ new Map();
1940
+ pendingFetches = /* @__PURE__ */ new Map();
1941
+ cacheTTL;
1942
+ baseUrl;
1943
+ constructor(options) {
1944
+ this.cacheTTL = options?.cacheTTL ?? 300 * 1e3;
1945
+ this.baseUrl = options?.baseUrl ?? LLMOPS_MODELS_API;
1946
+ }
1947
+ getCacheKey(provider, model) {
1948
+ return `${provider.toLowerCase()}:${model.toLowerCase()}`;
1949
+ }
1950
+ /**
1951
+ * Fetch pricing for a single model from the API
1952
+ */
1953
+ async fetchModelPricing(provider, model) {
1954
+ const url = `${this.baseUrl}/model-configs/pricing/${encodeURIComponent(provider)}/${model}`;
1955
+ try {
1956
+ logger.debug(`[Pricing] GET ${url}`);
1957
+ const startTime = Date.now();
1958
+ const response = await fetch(url);
1959
+ const elapsed = Date.now() - startTime;
1960
+ logger.debug(`[Pricing] GET ${url} -> ${response.status} (${elapsed}ms)`);
1961
+ if (response.status === 404) {
1962
+ logger.debug(`[Pricing] No pricing found for ${provider}/${model}`);
1963
+ return null;
2589
1964
  }
2590
- if (validatedSpans.length === 0) return { count: 0 };
1965
+ if (!response.ok) throw new Error(`API returned ${response.status}`);
1966
+ const data = await response.json();
1967
+ if (!data.pay_as_you_go) return null;
1968
+ const payg = data.pay_as_you_go;
1969
+ const pricing = {
1970
+ inputCostPer1M: centsPerTokenToCostPer1M(payg.request_token?.price ?? 0),
1971
+ outputCostPer1M: centsPerTokenToCostPer1M(payg.response_token?.price ?? 0),
1972
+ cacheReadCostPer1M: payg.cache_read_input_token?.price != null ? centsPerTokenToCostPer1M(payg.cache_read_input_token.price) : void 0,
1973
+ cacheWriteCostPer1M: payg.cache_write_input_token?.price != null ? centsPerTokenToCostPer1M(payg.cache_write_input_token.price) : void 0
1974
+ };
1975
+ logger.debug(`[Pricing] Cached pricing for ${provider}/${model}: input=$${pricing.inputCostPer1M}/1M, output=$${pricing.outputCostPer1M}/1M`);
1976
+ return pricing;
1977
+ } catch (error) {
1978
+ logger.error(`[Pricing] Failed to fetch pricing for ${provider}/${model}: ${error instanceof Error ? error.message : String(error)}`);
1979
+ const cacheKey = this.getCacheKey(provider, model);
1980
+ const stale = this.cache.get(cacheKey);
1981
+ if (stale) {
1982
+ logger.debug(`[Pricing] Using stale cache for ${provider}/${model}`);
1983
+ return stale.pricing;
1984
+ }
1985
+ return null;
1986
+ }
1987
+ }
1988
+ /**
1989
+ * Internal: fetch with cache and deduplication for a specific provider+model
1990
+ */
1991
+ async getCachedPricing(provider, model) {
1992
+ const cacheKey = this.getCacheKey(provider, model);
1993
+ const cached = this.cache.get(cacheKey);
1994
+ if (cached && Date.now() - cached.fetchedAt < this.cacheTTL) return cached.pricing;
1995
+ let pending = this.pendingFetches.get(cacheKey);
1996
+ if (!pending) {
1997
+ pending = this.fetchModelPricing(provider, model).then((pricing) => {
1998
+ this.cache.set(cacheKey, {
1999
+ pricing,
2000
+ fetchedAt: Date.now()
2001
+ });
2002
+ return pricing;
2003
+ }).finally(() => {
2004
+ this.pendingFetches.delete(cacheKey);
2005
+ });
2006
+ this.pendingFetches.set(cacheKey, pending);
2007
+ }
2008
+ return pending;
2009
+ }
2010
+ /**
2011
+ * Get pricing for a specific model.
2012
+ *
2013
+ * When the model name contains a slash (e.g. "google/gemini-2.5-flash"),
2014
+ * it's likely an OpenRouter model ID. If the initial provider lookup fails,
2015
+ * we automatically retry with "openrouter" as the provider.
2016
+ */
2017
+ async getModelPricing(provider, model) {
2018
+ const pricing = await this.getCachedPricing(provider, model);
2019
+ if (pricing) return pricing;
2020
+ if (!pricing && model.includes("/") && provider.toLowerCase() !== "openrouter") {
2021
+ logger.debug(`[Pricing] Retrying ${provider}/${model} as openrouter/${model}`);
2022
+ return this.getCachedPricing("openrouter", model);
2023
+ }
2024
+ return pricing;
2025
+ }
2026
+ /**
2027
+ * Force refresh the pricing cache (clears all cached entries)
2028
+ */
2029
+ async refreshCache() {
2030
+ this.cache.clear();
2031
+ }
2032
+ /**
2033
+ * Always ready — no bulk pre-fetch needed
2034
+ */
2035
+ isReady() {
2036
+ return true;
2037
+ }
2038
+ /**
2039
+ * Get the number of cached models (for debugging)
2040
+ */
2041
+ getCacheSize() {
2042
+ return this.cache.size;
2043
+ }
2044
+ };
2045
+ let defaultProvider = null;
2046
+ /**
2047
+ * Get the default pricing provider instance
2048
+ */
2049
+ function getDefaultPricingProvider() {
2050
+ if (!defaultProvider) defaultProvider = new LLMOpsPricingProvider();
2051
+ return defaultProvider;
2052
+ }
2053
+
2054
+ //#endregion
2055
+ //#region src/telemetry/postgres.ts
2056
+ const guardrailResultSchema = zod_default.object({
2057
+ checkId: zod_default.string(),
2058
+ functionId: zod_default.string(),
2059
+ hookType: zod_default.enum(["beforeRequestHook", "afterRequestHook"]),
2060
+ verdict: zod_default.boolean(),
2061
+ latencyMs: zod_default.number()
2062
+ });
2063
+ const guardrailResultsSchema = zod_default.object({
2064
+ results: zod_default.array(guardrailResultSchema),
2065
+ action: zod_default.enum([
2066
+ "allowed",
2067
+ "blocked",
2068
+ "logged"
2069
+ ]),
2070
+ totalLatencyMs: zod_default.number()
2071
+ });
2072
+ const insertLLMRequestSchema = zod_default.object({
2073
+ requestId: zod_default.string().uuid(),
2074
+ configId: zod_default.string().uuid().nullable().optional(),
2075
+ variantId: zod_default.string().uuid().nullable().optional(),
2076
+ environmentId: zod_default.string().uuid().nullable().optional(),
2077
+ providerConfigId: zod_default.string().uuid().nullable().optional(),
2078
+ provider: zod_default.string(),
2079
+ model: zod_default.string(),
2080
+ promptTokens: zod_default.number().int().default(0),
2081
+ completionTokens: zod_default.number().int().default(0),
2082
+ totalTokens: zod_default.number().int().default(0),
2083
+ cachedTokens: zod_default.number().int().default(0),
2084
+ cacheCreationTokens: zod_default.number().int().default(0),
2085
+ cost: zod_default.number().int().default(0),
2086
+ cacheSavings: zod_default.number().int().default(0),
2087
+ inputCost: zod_default.number().int().default(0),
2088
+ outputCost: zod_default.number().int().default(0),
2089
+ endpoint: zod_default.string(),
2090
+ statusCode: zod_default.number().int(),
2091
+ latencyMs: zod_default.number().int().default(0),
2092
+ isStreaming: zod_default.boolean().default(false),
2093
+ userId: zod_default.string().nullable().optional(),
2094
+ tags: zod_default.record(zod_default.string(), zod_default.string()).default({}),
2095
+ guardrailResults: guardrailResultsSchema.nullable().optional(),
2096
+ traceId: zod_default.string().nullable().optional(),
2097
+ spanId: zod_default.string().nullable().optional(),
2098
+ parentSpanId: zod_default.string().nullable().optional(),
2099
+ sessionId: zod_default.string().nullable().optional()
2100
+ });
2101
+ const listRequestsSchema = zod_default.object({
2102
+ limit: zod_default.number().int().positive().max(1e3).default(100),
2103
+ offset: zod_default.number().int().nonnegative().default(0),
2104
+ configId: zod_default.string().uuid().optional(),
2105
+ variantId: zod_default.string().uuid().optional(),
2106
+ environmentId: zod_default.string().uuid().optional(),
2107
+ providerConfigId: zod_default.string().uuid().optional(),
2108
+ provider: zod_default.string().optional(),
2109
+ model: zod_default.string().optional(),
2110
+ startDate: zod_default.date().optional(),
2111
+ endDate: zod_default.date().optional(),
2112
+ tags: zod_default.record(zod_default.string(), zod_default.array(zod_default.string())).optional()
2113
+ });
2114
+ const dateRangeSchema = zod_default.object({
2115
+ startDate: zod_default.date(),
2116
+ endDate: zod_default.date(),
2117
+ configId: zod_default.string().uuid().optional(),
2118
+ variantId: zod_default.string().uuid().optional(),
2119
+ environmentId: zod_default.string().uuid().optional(),
2120
+ tags: zod_default.record(zod_default.string(), zod_default.array(zod_default.string())).optional()
2121
+ });
2122
+ const COST_SUMMARY_GROUP_BY = [
2123
+ "day",
2124
+ "hour",
2125
+ "model",
2126
+ "provider",
2127
+ "endpoint",
2128
+ "tags"
2129
+ ];
2130
+ const costSummarySchema = zod_default.object({
2131
+ startDate: zod_default.date(),
2132
+ endDate: zod_default.date(),
2133
+ configId: zod_default.string().uuid().optional(),
2134
+ variantId: zod_default.string().uuid().optional(),
2135
+ environmentId: zod_default.string().uuid().optional(),
2136
+ tags: zod_default.record(zod_default.string(), zod_default.array(zod_default.string())).optional(),
2137
+ groupBy: zod_default.enum(COST_SUMMARY_GROUP_BY).optional(),
2138
+ tagKeys: zod_default.array(zod_default.string()).optional()
2139
+ });
2140
+ const upsertTraceSchema = zod_default.object({
2141
+ traceId: zod_default.string(),
2142
+ name: zod_default.string().nullable().optional(),
2143
+ sessionId: zod_default.string().nullable().optional(),
2144
+ userId: zod_default.string().nullable().optional(),
2145
+ status: zod_default.enum([
2146
+ "unset",
2147
+ "ok",
2148
+ "error"
2149
+ ]).default("unset"),
2150
+ startTime: zod_default.date(),
2151
+ endTime: zod_default.date().nullable().optional(),
2152
+ durationMs: zod_default.number().int().nullable().optional(),
2153
+ spanCount: zod_default.number().int().default(1),
2154
+ totalInputTokens: zod_default.number().int().default(0),
2155
+ totalOutputTokens: zod_default.number().int().default(0),
2156
+ totalTokens: zod_default.number().int().default(0),
2157
+ totalCost: zod_default.number().int().default(0),
2158
+ tags: zod_default.record(zod_default.string(), zod_default.string()).default({}),
2159
+ metadata: zod_default.record(zod_default.string(), zod_default.unknown()).default({})
2160
+ });
2161
+ const insertSpanSchema = zod_default.object({
2162
+ traceId: zod_default.string(),
2163
+ spanId: zod_default.string(),
2164
+ parentSpanId: zod_default.string().nullable().optional(),
2165
+ name: zod_default.string(),
2166
+ kind: zod_default.number().int().default(1),
2167
+ status: zod_default.number().int().default(0),
2168
+ statusMessage: zod_default.string().nullable().optional(),
2169
+ startTime: zod_default.date(),
2170
+ endTime: zod_default.date().nullable().optional(),
2171
+ durationMs: zod_default.number().int().nullable().optional(),
2172
+ provider: zod_default.string().nullable().optional(),
2173
+ model: zod_default.string().nullable().optional(),
2174
+ promptTokens: zod_default.number().int().default(0),
2175
+ completionTokens: zod_default.number().int().default(0),
2176
+ totalTokens: zod_default.number().int().default(0),
2177
+ cost: zod_default.number().int().default(0),
2178
+ configId: zod_default.string().uuid().nullable().optional(),
2179
+ variantId: zod_default.string().uuid().nullable().optional(),
2180
+ environmentId: zod_default.string().uuid().nullable().optional(),
2181
+ providerConfigId: zod_default.string().uuid().nullable().optional(),
2182
+ requestId: zod_default.string().uuid().nullable().optional(),
2183
+ source: zod_default.enum([
2184
+ "gateway",
2185
+ "otlp",
2186
+ "langsmith"
2187
+ ]).default("gateway"),
2188
+ input: zod_default.unknown().nullable().optional(),
2189
+ output: zod_default.unknown().nullable().optional(),
2190
+ attributes: zod_default.record(zod_default.string(), zod_default.unknown()).default({})
2191
+ });
2192
+ const insertSpanEventSchema = zod_default.object({
2193
+ traceId: zod_default.string(),
2194
+ spanId: zod_default.string(),
2195
+ name: zod_default.string(),
2196
+ timestamp: zod_default.date(),
2197
+ attributes: zod_default.record(zod_default.string(), zod_default.unknown()).default({})
2198
+ });
2199
+ const listTracesSchema = zod_default.object({
2200
+ limit: zod_default.number().int().positive().max(1e3).default(50),
2201
+ offset: zod_default.number().int().nonnegative().default(0),
2202
+ sessionId: zod_default.string().optional(),
2203
+ userId: zod_default.string().optional(),
2204
+ status: zod_default.enum([
2205
+ "unset",
2206
+ "ok",
2207
+ "error"
2208
+ ]).optional(),
2209
+ name: zod_default.string().optional(),
2210
+ startDate: zod_default.date().optional(),
2211
+ endDate: zod_default.date().optional(),
2212
+ tags: zod_default.record(zod_default.string(), zod_default.array(zod_default.string())).optional()
2213
+ });
2214
+ const traceStatsSchema = zod_default.object({
2215
+ startDate: zod_default.date(),
2216
+ endDate: zod_default.date(),
2217
+ sessionId: zod_default.string().optional(),
2218
+ userId: zod_default.string().optional()
2219
+ });
2220
+ const col = (name) => sql.ref(name);
2221
+ function createLLMRequestsStore(db) {
2222
+ return {
2223
+ batchInsertRequests: async (requests) => {
2224
+ if (requests.length === 0) return { count: 0 };
2225
+ const validatedRequests = await Promise.all(requests.map(async (req) => {
2226
+ const result = await insertLLMRequestSchema.safeParseAsync(req);
2227
+ if (!result.success) throw new LLMOpsError(`Invalid request data: ${result.error.message}`);
2228
+ return result.data;
2229
+ }));
2591
2230
  const now = (/* @__PURE__ */ new Date()).toISOString();
2592
- const values = validatedSpans.map((span) => ({
2231
+ const values = validatedRequests.map((req) => ({
2593
2232
  id: randomUUID(),
2594
- traceId: span.traceId,
2595
- spanId: span.spanId,
2596
- parentSpanId: span.parentSpanId ?? null,
2597
- name: span.name,
2598
- kind: span.kind,
2599
- status: span.status,
2600
- statusMessage: span.statusMessage ?? null,
2601
- startTime: span.startTime.toISOString(),
2602
- endTime: span.endTime?.toISOString() ?? null,
2603
- durationMs: span.durationMs ?? null,
2604
- provider: span.provider ?? null,
2605
- model: span.model ?? null,
2606
- promptTokens: span.promptTokens,
2607
- completionTokens: span.completionTokens,
2608
- totalTokens: span.totalTokens,
2609
- cost: span.cost,
2610
- configId: span.configId ?? null,
2611
- variantId: span.variantId ?? null,
2612
- environmentId: span.environmentId ?? null,
2613
- providerConfigId: span.providerConfigId ?? null,
2614
- requestId: span.requestId ?? null,
2615
- source: span.source,
2616
- input: span.input != null ? JSON.stringify(span.input) : null,
2617
- output: span.output != null ? JSON.stringify(span.output) : null,
2618
- attributes: JSON.stringify(span.attributes),
2233
+ requestId: req.requestId,
2234
+ configId: req.configId ?? null,
2235
+ variantId: req.variantId ?? null,
2236
+ environmentId: req.environmentId ?? null,
2237
+ providerConfigId: req.providerConfigId ?? null,
2238
+ provider: req.provider,
2239
+ model: req.model,
2240
+ promptTokens: req.promptTokens,
2241
+ completionTokens: req.completionTokens,
2242
+ totalTokens: req.totalTokens,
2243
+ cachedTokens: req.cachedTokens,
2244
+ cacheCreationTokens: req.cacheCreationTokens,
2245
+ cost: req.cost,
2246
+ cacheSavings: req.cacheSavings,
2247
+ inputCost: req.inputCost,
2248
+ outputCost: req.outputCost,
2249
+ endpoint: req.endpoint,
2250
+ statusCode: req.statusCode,
2251
+ latencyMs: req.latencyMs,
2252
+ isStreaming: req.isStreaming,
2253
+ userId: req.userId ?? null,
2254
+ tags: JSON.stringify(req.tags),
2255
+ guardrailResults: req.guardrailResults ? JSON.stringify(req.guardrailResults) : null,
2256
+ traceId: req.traceId ?? null,
2257
+ spanId: req.spanId ?? null,
2258
+ parentSpanId: req.parentSpanId ?? null,
2259
+ sessionId: req.sessionId ?? null,
2619
2260
  createdAt: now,
2620
2261
  updatedAt: now
2621
2262
  }));
2622
- await db.insertInto("spans").values(values).onConflict((oc) => oc.column("spanId").doNothing()).execute();
2623
- return { count: values.length };
2624
- },
2625
- batchInsertSpanEvents: async (events) => {
2626
- if (events.length === 0) return { count: 0 };
2627
- const validatedEvents = [];
2628
- for (const event of events) {
2629
- const result = await insertSpanEventSchema.safeParseAsync(event);
2630
- if (!result.success) {
2631
- logger.warn(`[batchInsertSpanEvents] Skipping invalid event: ${result.error.message}`);
2632
- continue;
2633
- }
2634
- validatedEvents.push(result.data);
2635
- }
2636
- if (validatedEvents.length === 0) return { count: 0 };
2637
- const now = (/* @__PURE__ */ new Date()).toISOString();
2638
- const values = validatedEvents.map((event) => ({
2639
- id: randomUUID(),
2640
- traceId: event.traceId,
2641
- spanId: event.spanId,
2642
- name: event.name,
2643
- timestamp: event.timestamp.toISOString(),
2644
- attributes: JSON.stringify(event.attributes),
2645
- createdAt: now
2646
- }));
2647
- await db.insertInto("span_events").values(values).execute();
2263
+ await db.insertInto("llm_requests").values(values).execute();
2648
2264
  return { count: values.length };
2649
2265
  },
2650
- listTraces: async (params) => {
2651
- const result = await listTracesSchema.safeParseAsync(params || {});
2266
+ insertRequest: async (request) => {
2267
+ const result = await insertLLMRequestSchema.safeParseAsync(request);
2268
+ if (!result.success) throw new LLMOpsError(`Invalid request data: ${result.error.message}`);
2269
+ const req = result.data;
2270
+ const now = (/* @__PURE__ */ new Date()).toISOString();
2271
+ return db.insertInto("llm_requests").values({
2272
+ id: randomUUID(),
2273
+ requestId: req.requestId,
2274
+ configId: req.configId ?? null,
2275
+ variantId: req.variantId ?? null,
2276
+ environmentId: req.environmentId ?? null,
2277
+ providerConfigId: req.providerConfigId ?? null,
2278
+ provider: req.provider,
2279
+ model: req.model,
2280
+ promptTokens: req.promptTokens,
2281
+ completionTokens: req.completionTokens,
2282
+ totalTokens: req.totalTokens,
2283
+ cachedTokens: req.cachedTokens,
2284
+ cacheCreationTokens: req.cacheCreationTokens,
2285
+ cost: req.cost,
2286
+ cacheSavings: req.cacheSavings,
2287
+ inputCost: req.inputCost,
2288
+ outputCost: req.outputCost,
2289
+ endpoint: req.endpoint,
2290
+ statusCode: req.statusCode,
2291
+ latencyMs: req.latencyMs,
2292
+ isStreaming: req.isStreaming,
2293
+ userId: req.userId ?? null,
2294
+ tags: JSON.stringify(req.tags),
2295
+ guardrailResults: req.guardrailResults ? JSON.stringify(req.guardrailResults) : null,
2296
+ traceId: req.traceId ?? null,
2297
+ spanId: req.spanId ?? null,
2298
+ parentSpanId: req.parentSpanId ?? null,
2299
+ sessionId: req.sessionId ?? null,
2300
+ createdAt: now,
2301
+ updatedAt: now
2302
+ }).returningAll().executeTakeFirst();
2303
+ },
2304
+ listRequests: async (params) => {
2305
+ const result = await listRequestsSchema.safeParseAsync(params || {});
2652
2306
  if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2653
- const { limit, offset, sessionId, userId, status, name, startDate, endDate, tags } = result.data;
2654
- let baseQuery = db.selectFrom("traces");
2655
- if (sessionId) baseQuery = baseQuery.where("sessionId", "=", sessionId);
2656
- if (userId) baseQuery = baseQuery.where("userId", "=", userId);
2657
- if (status) baseQuery = baseQuery.where("status", "=", status);
2658
- if (name) baseQuery = baseQuery.where(sql`${col("name")} ILIKE ${"%" + name + "%"}`);
2659
- if (startDate) baseQuery = baseQuery.where(sql`${col("startTime")} >= ${startDate.toISOString()}`);
2660
- if (endDate) baseQuery = baseQuery.where(sql`${col("startTime")} <= ${endDate.toISOString()}`);
2307
+ const { limit, offset, configId, variantId, environmentId, providerConfigId, provider, model, startDate, endDate, tags } = result.data;
2308
+ let baseQuery = db.selectFrom("llm_requests");
2309
+ if (configId) baseQuery = baseQuery.where("configId", "=", configId);
2310
+ if (variantId) baseQuery = baseQuery.where("variantId", "=", variantId);
2311
+ if (environmentId) baseQuery = baseQuery.where("environmentId", "=", environmentId);
2312
+ if (providerConfigId) baseQuery = baseQuery.where("providerConfigId", "=", providerConfigId);
2313
+ if (provider) baseQuery = baseQuery.where("provider", "=", provider);
2314
+ if (model) baseQuery = baseQuery.where("model", "=", model);
2315
+ if (startDate) baseQuery = baseQuery.where(sql`${col("createdAt")} >= ${startDate.toISOString()}`);
2316
+ if (endDate) baseQuery = baseQuery.where(sql`${col("createdAt")} <= ${endDate.toISOString()}`);
2661
2317
  if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
2662
2318
  if (values.length === 0) continue;
2663
2319
  if (values.length === 1) baseQuery = baseQuery.where(sql`${col("tags")}->>${key} = ${values[0]}`);
@@ -2669,426 +2325,387 @@ const createTracesDataLayer = (db) => {
2669
2325
  const countResult = await baseQuery.select(sql`COUNT(*)`.as("total")).executeTakeFirst();
2670
2326
  const total = Number(countResult?.total ?? 0);
2671
2327
  return {
2672
- data: await baseQuery.selectAll().orderBy("startTime", "desc").limit(limit).offset(offset).execute(),
2328
+ data: await baseQuery.selectAll().orderBy("createdAt", "desc").limit(limit).offset(offset).execute(),
2673
2329
  total,
2674
2330
  limit,
2675
2331
  offset
2676
2332
  };
2677
2333
  },
2678
- getTraceWithSpans: async (traceId) => {
2679
- const trace = await db.selectFrom("traces").selectAll().where("traceId", "=", traceId).executeTakeFirst();
2680
- if (!trace) return void 0;
2681
- return {
2682
- trace,
2683
- spans: await db.selectFrom("spans").selectAll().where("traceId", "=", traceId).orderBy("startTime", "asc").execute(),
2684
- events: await db.selectFrom("span_events").selectAll().where("traceId", "=", traceId).orderBy("timestamp", "asc").execute()
2685
- };
2334
+ getRequestByRequestId: async (requestId) => {
2335
+ return db.selectFrom("llm_requests").selectAll().where("requestId", "=", requestId).executeTakeFirst();
2686
2336
  },
2687
- getTraceStats: async (params) => {
2688
- const result = await traceStatsSchema.safeParseAsync(params);
2337
+ getTotalCost: async (params) => {
2338
+ const result = await dateRangeSchema.safeParseAsync(params);
2689
2339
  if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2690
- const { startDate, endDate, sessionId, userId } = result.data;
2691
- let query = db.selectFrom("traces").select([
2692
- sql`COUNT(*)`.as("totalTraces"),
2693
- sql`COALESCE(AVG(${col("durationMs")}), 0)`.as("avgDurationMs"),
2694
- sql`COUNT(CASE WHEN ${col("status")} = 'error' THEN 1 END)`.as("errorCount"),
2695
- sql`COALESCE(SUM(${col("totalCost")}), 0)`.as("totalCost"),
2340
+ const { startDate, endDate, configId, variantId, environmentId, tags } = result.data;
2341
+ let query = db.selectFrom("llm_requests").select([
2342
+ sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2343
+ sql`COALESCE(SUM(${col("inputCost")}), 0)`.as("totalInputCost"),
2344
+ sql`COALESCE(SUM(${col("outputCost")}), 0)`.as("totalOutputCost"),
2345
+ sql`COALESCE(SUM(${col("promptTokens")}), 0)`.as("totalPromptTokens"),
2346
+ sql`COALESCE(SUM(${col("completionTokens")}), 0)`.as("totalCompletionTokens"),
2696
2347
  sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
2697
- sql`COALESCE(SUM(${col("spanCount")}), 0)`.as("totalSpans")
2698
- ]).where(sql`${col("startTime")} >= ${startDate.toISOString()}`).where(sql`${col("startTime")} <= ${endDate.toISOString()}`);
2699
- if (sessionId) query = query.where("sessionId", "=", sessionId);
2700
- if (userId) query = query.where("userId", "=", userId);
2348
+ sql`COALESCE(SUM(${col("cachedTokens")}), 0)`.as("totalCachedTokens"),
2349
+ sql`COALESCE(SUM(${col("cacheSavings")}), 0)`.as("totalCacheSavings"),
2350
+ sql`COUNT(*)`.as("requestCount")
2351
+ ]).where(sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col("createdAt")} <= ${endDate.toISOString()}`);
2352
+ if (configId) query = query.where("configId", "=", configId);
2353
+ if (variantId) query = query.where("variantId", "=", variantId);
2354
+ if (environmentId) query = query.where("environmentId", "=", environmentId);
2355
+ if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
2356
+ if (values.length === 0) continue;
2357
+ if (values.length === 1) query = query.where(sql`${col("tags")}->>${key} = ${values[0]}`);
2358
+ else {
2359
+ const valueList = sql.join(values.map((v) => sql`${v}`));
2360
+ query = query.where(sql`${col("tags")}->>${key} IN (${valueList})`);
2361
+ }
2362
+ }
2701
2363
  return query.executeTakeFirst();
2702
- }
2703
- };
2704
- };
2705
-
2706
- //#endregion
2707
- //#region src/datalayer/workspaceSettings.ts
2708
- const updateWorkspaceSettings = zod_default.object({
2709
- name: zod_default.string().nullable().optional(),
2710
- setupComplete: zod_default.boolean().optional(),
2711
- superAdminId: zod_default.string().nullable().optional()
2712
- });
2713
- const createWorkspaceSettingsDataLayer = (db) => {
2714
- return {
2715
- getWorkspaceSettings: async () => {
2716
- let settings = await db.selectFrom("workspace_settings").selectAll().executeTakeFirst();
2717
- if (!settings) settings = await db.insertInto("workspace_settings").values({
2718
- id: randomUUID(),
2719
- name: null,
2720
- setupComplete: false,
2721
- createdAt: (/* @__PURE__ */ new Date()).toISOString(),
2722
- updatedAt: (/* @__PURE__ */ new Date()).toISOString()
2723
- }).returningAll().executeTakeFirst();
2724
- return settings;
2725
2364
  },
2726
- updateWorkspaceSettings: async (params) => {
2727
- const value = await updateWorkspaceSettings.safeParseAsync(params);
2728
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
2729
- let settings = await db.selectFrom("workspace_settings").selectAll().executeTakeFirst();
2730
- if (!settings) return db.insertInto("workspace_settings").values({
2731
- id: randomUUID(),
2732
- name: value.data.name ?? null,
2733
- setupComplete: value.data.setupComplete ?? false,
2734
- createdAt: (/* @__PURE__ */ new Date()).toISOString(),
2735
- updatedAt: (/* @__PURE__ */ new Date()).toISOString()
2736
- }).returningAll().executeTakeFirst();
2737
- const updateData = { updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
2738
- if (value.data.name !== void 0) updateData.name = value.data.name ?? null;
2739
- if (value.data.setupComplete !== void 0) updateData.setupComplete = value.data.setupComplete;
2740
- if (value.data.superAdminId !== void 0) updateData.superAdminId = value.data.superAdminId ?? null;
2741
- return db.updateTable("workspace_settings").set(updateData).where("id", "=", settings.id).returningAll().executeTakeFirst();
2365
+ getCostByModel: async (params) => {
2366
+ const result = await dateRangeSchema.safeParseAsync(params);
2367
+ if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2368
+ const { startDate, endDate } = result.data;
2369
+ return db.selectFrom("llm_requests").select([
2370
+ "provider",
2371
+ "model",
2372
+ sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2373
+ sql`COALESCE(SUM(${col("inputCost")}), 0)`.as("totalInputCost"),
2374
+ sql`COALESCE(SUM(${col("outputCost")}), 0)`.as("totalOutputCost"),
2375
+ sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
2376
+ sql`COUNT(*)`.as("requestCount"),
2377
+ sql`AVG(${col("latencyMs")})`.as("avgLatencyMs")
2378
+ ]).where(sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col("createdAt")} <= ${endDate.toISOString()}`).groupBy(["provider", "model"]).orderBy(sql`SUM(${col("cost")})`, "desc").execute();
2742
2379
  },
2743
- getSuperAdminId: async () => {
2744
- return (await db.selectFrom("workspace_settings").select("superAdminId").executeTakeFirst())?.superAdminId ?? null;
2380
+ getCostByProvider: async (params) => {
2381
+ const result = await dateRangeSchema.safeParseAsync(params);
2382
+ if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2383
+ const { startDate, endDate } = result.data;
2384
+ return db.selectFrom("llm_requests").select([
2385
+ "provider",
2386
+ sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2387
+ sql`COALESCE(SUM(${col("inputCost")}), 0)`.as("totalInputCost"),
2388
+ sql`COALESCE(SUM(${col("outputCost")}), 0)`.as("totalOutputCost"),
2389
+ sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
2390
+ sql`COUNT(*)`.as("requestCount"),
2391
+ sql`AVG(${col("latencyMs")})`.as("avgLatencyMs")
2392
+ ]).where(sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col("createdAt")} <= ${endDate.toISOString()}`).groupBy("provider").orderBy(sql`SUM(${col("cost")})`, "desc").execute();
2745
2393
  },
2746
- setSuperAdminId: async (userId) => {
2747
- let settings = await db.selectFrom("workspace_settings").selectAll().executeTakeFirst();
2748
- if (settings?.superAdminId) return false;
2749
- if (!settings) {
2750
- await db.insertInto("workspace_settings").values({
2751
- id: randomUUID(),
2752
- name: null,
2753
- setupComplete: false,
2754
- superAdminId: userId,
2755
- createdAt: (/* @__PURE__ */ new Date()).toISOString(),
2756
- updatedAt: (/* @__PURE__ */ new Date()).toISOString()
2757
- }).execute();
2758
- return true;
2759
- }
2760
- await db.updateTable("workspace_settings").set({
2761
- superAdminId: userId,
2762
- updatedAt: (/* @__PURE__ */ new Date()).toISOString()
2763
- }).where("id", "=", settings.id).execute();
2764
- return true;
2394
+ getDailyCosts: async (params) => {
2395
+ const result = await dateRangeSchema.safeParseAsync(params);
2396
+ if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2397
+ const { startDate, endDate } = result.data;
2398
+ return db.selectFrom("llm_requests").select([
2399
+ sql`DATE(${col("createdAt")})`.as("date"),
2400
+ sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2401
+ sql`COALESCE(SUM(${col("inputCost")}), 0)`.as("totalInputCost"),
2402
+ sql`COALESCE(SUM(${col("outputCost")}), 0)`.as("totalOutputCost"),
2403
+ sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
2404
+ sql`COUNT(*)`.as("requestCount")
2405
+ ]).where(sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col("createdAt")} <= ${endDate.toISOString()}`).groupBy(sql`DATE(${col("createdAt")})`).orderBy(sql`DATE(${col("createdAt")})`, "asc").execute();
2765
2406
  },
2766
- isSetupComplete: async () => {
2767
- try {
2768
- return (await db.selectFrom("workspace_settings").select("setupComplete").executeTakeFirst())?.setupComplete ?? false;
2769
- } catch {
2770
- return false;
2407
+ getCostSummary: async (params) => {
2408
+ const result = await costSummarySchema.safeParseAsync(params);
2409
+ if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2410
+ const { startDate, endDate, groupBy, configId, variantId, environmentId, tags, tagKeys } = result.data;
2411
+ let baseQuery = db.selectFrom("llm_requests").where(sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col("createdAt")} <= ${endDate.toISOString()}`);
2412
+ if (configId) baseQuery = baseQuery.where("configId", "=", configId);
2413
+ if (variantId) baseQuery = baseQuery.where("variantId", "=", variantId);
2414
+ if (environmentId) baseQuery = baseQuery.where("environmentId", "=", environmentId);
2415
+ if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
2416
+ if (values.length === 0) continue;
2417
+ if (values.length === 1) baseQuery = baseQuery.where(sql`${col("tags")}->>${key} = ${values[0]}`);
2418
+ else {
2419
+ const valueList = sql.join(values.map((v) => sql`${v}`));
2420
+ baseQuery = baseQuery.where(sql`${col("tags")}->>${key} IN (${valueList})`);
2421
+ }
2422
+ }
2423
+ switch (groupBy) {
2424
+ case "day": return baseQuery.select([
2425
+ sql`DATE(${col("createdAt")})`.as("groupKey"),
2426
+ sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2427
+ sql`COUNT(*)`.as("requestCount"),
2428
+ sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens")
2429
+ ]).groupBy(sql`DATE(${col("createdAt")})`).orderBy(sql`DATE(${col("createdAt")})`, "asc").execute();
2430
+ case "hour": return baseQuery.select([
2431
+ sql`DATE_TRUNC('hour', ${col("createdAt")})`.as("groupKey"),
2432
+ sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2433
+ sql`COUNT(*)`.as("requestCount"),
2434
+ sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens")
2435
+ ]).groupBy(sql`DATE_TRUNC('hour', ${col("createdAt")})`).orderBy(sql`DATE_TRUNC('hour', ${col("createdAt")})`, "asc").execute();
2436
+ case "model": return baseQuery.select([
2437
+ sql`${col("provider")} || '/' || ${col("model")}`.as("groupKey"),
2438
+ sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2439
+ sql`COUNT(*)`.as("requestCount")
2440
+ ]).groupBy(["provider", "model"]).orderBy(sql`SUM(${col("cost")})`, "desc").execute();
2441
+ case "provider": return baseQuery.select([
2442
+ sql`${col("provider")}`.as("groupKey"),
2443
+ sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2444
+ sql`COUNT(*)`.as("requestCount")
2445
+ ]).groupBy("provider").orderBy(sql`SUM(${col("cost")})`, "desc").execute();
2446
+ case "endpoint": return baseQuery.select([
2447
+ sql`COALESCE(${col("endpoint")}, 'unknown')`.as("groupKey"),
2448
+ sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2449
+ sql`COUNT(*)`.as("requestCount")
2450
+ ]).groupBy("endpoint").orderBy(sql`SUM(${col("cost")})`, "desc").execute();
2451
+ case "tags": {
2452
+ const conditions = [sql`${col("createdAt")} >= ${startDate.toISOString()}`, sql`${col("createdAt")} <= ${endDate.toISOString()}`];
2453
+ if (configId) conditions.push(sql`${col("configId")} = ${configId}`);
2454
+ if (variantId) conditions.push(sql`${col("variantId")} = ${variantId}`);
2455
+ if (environmentId) conditions.push(sql`${col("environmentId")} = ${environmentId}`);
2456
+ if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
2457
+ if (values.length === 0) continue;
2458
+ if (values.length === 1) conditions.push(sql`${col("tags")}->>${key} = ${values[0]}`);
2459
+ else {
2460
+ const valueList = sql.join(values.map((v) => sql`${v}`));
2461
+ conditions.push(sql`${col("tags")}->>${key} IN (${valueList})`);
2462
+ }
2463
+ }
2464
+ if (tagKeys && tagKeys.length > 0) {
2465
+ const tagKeyList = sql.join(tagKeys.map((k) => sql`${k}`), sql`, `);
2466
+ conditions.push(sql`t.key IN (${tagKeyList})`);
2467
+ }
2468
+ const whereClause = sql.join(conditions, sql` AND `);
2469
+ return (await sql`
2470
+ SELECT t.key || ':' || t.value as "groupKey",
2471
+ COALESCE(SUM(${col("cost")}), 0) as "totalCost",
2472
+ COUNT(*) as "requestCount"
2473
+ FROM "llm_requests", jsonb_each_text(${col("tags")}) t
2474
+ WHERE ${whereClause}
2475
+ GROUP BY t.key, t.value
2476
+ ORDER BY SUM(${col("cost")}) DESC
2477
+ `.execute(db)).rows;
2478
+ }
2479
+ default: return baseQuery.select([
2480
+ sql`'total'`.as("groupKey"),
2481
+ sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2482
+ sql`COUNT(*)`.as("requestCount")
2483
+ ]).execute();
2771
2484
  }
2772
2485
  },
2773
- markSetupComplete: async () => {
2774
- let settings = await db.selectFrom("workspace_settings").selectAll().executeTakeFirst();
2775
- if (!settings) return db.insertInto("workspace_settings").values({
2776
- id: randomUUID(),
2777
- name: null,
2778
- setupComplete: true,
2779
- createdAt: (/* @__PURE__ */ new Date()).toISOString(),
2780
- updatedAt: (/* @__PURE__ */ new Date()).toISOString()
2781
- }).returningAll().executeTakeFirst();
2782
- return db.updateTable("workspace_settings").set({
2783
- setupComplete: true,
2784
- updatedAt: (/* @__PURE__ */ new Date()).toISOString()
2785
- }).where("id", "=", settings.id).returningAll().executeTakeFirst();
2486
+ getRequestStats: async (params) => {
2487
+ const result = await dateRangeSchema.safeParseAsync(params);
2488
+ if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2489
+ const { startDate, endDate, configId, variantId, environmentId, tags } = result.data;
2490
+ let query = db.selectFrom("llm_requests").select([
2491
+ sql`COUNT(*)`.as("totalRequests"),
2492
+ sql`COUNT(CASE WHEN ${col("statusCode")} >= 200 AND ${col("statusCode")} < 300 THEN 1 END)`.as("successfulRequests"),
2493
+ sql`COUNT(CASE WHEN ${col("statusCode")} >= 400 THEN 1 END)`.as("failedRequests"),
2494
+ sql`COUNT(CASE WHEN ${col("isStreaming")} = true THEN 1 END)`.as("streamingRequests"),
2495
+ sql`AVG(${col("latencyMs")})`.as("avgLatencyMs"),
2496
+ sql`MAX(${col("latencyMs")})`.as("maxLatencyMs"),
2497
+ sql`MIN(${col("latencyMs")})`.as("minLatencyMs")
2498
+ ]).where(sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col("createdAt")} <= ${endDate.toISOString()}`);
2499
+ if (configId) query = query.where("configId", "=", configId);
2500
+ if (variantId) query = query.where("variantId", "=", variantId);
2501
+ if (environmentId) query = query.where("environmentId", "=", environmentId);
2502
+ if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
2503
+ if (values.length === 0) continue;
2504
+ if (values.length === 1) query = query.where(sql`${col("tags")}->>${key} = ${values[0]}`);
2505
+ else {
2506
+ const valueList = sql.join(values.map((v) => sql`${v}`));
2507
+ query = query.where(sql`${col("tags")}->>${key} IN (${valueList})`);
2508
+ }
2509
+ }
2510
+ return query.executeTakeFirst();
2511
+ },
2512
+ getDistinctTags: async () => {
2513
+ return (await sql`
2514
+ SELECT DISTINCT key, value
2515
+ FROM llm_requests, jsonb_each_text(tags) AS t(key, value)
2516
+ WHERE tags != '{}'::jsonb
2517
+ ORDER BY key, value
2518
+ `.execute(db)).rows;
2786
2519
  }
2787
2520
  };
2788
- };
2789
-
2790
- //#endregion
2791
- //#region src/datalayer/create.ts
2792
- /**
2793
- * Create all datalayers from a Kysely database instance.
2794
- * Returns a flat object with all datalayer methods spread together.
2795
- */
2796
- function createDataLayer(db) {
2797
- return {
2798
- ...createDatasetsDataLayer(db),
2799
- ...createGuardrailConfigsDataLayer(db),
2800
- ...createLLMRequestsDataLayer(db),
2801
- ...createPlaygroundDataLayer(db),
2802
- ...createPlaygroundResultsDataLayer(db),
2803
- ...createPlaygroundRunsDataLayer(db),
2804
- ...createProviderConfigsDataLayer(db),
2805
- ...createProviderGuardrailOverridesDataLayer(db),
2806
- ...createTracesDataLayer(db),
2807
- ...createWorkspaceSettingsDataLayer(db)
2808
- };
2809
- }
2810
-
2811
- //#endregion
2812
- //#region src/pricing/calculator.ts
2813
- /**
2814
- * Calculate the cost of an LLM request in micro-dollars
2815
- *
2816
- * Micro-dollars are used to avoid floating-point precision issues:
2817
- * - 1 dollar = 1,000,000 micro-dollars
2818
- * - $0.001 = 1,000 micro-dollars
2819
- * - $0.000001 = 1 micro-dollar
2820
- *
2821
- * @param usage - Token usage data from the LLM response
2822
- * @param pricing - Model pricing information
2823
- * @returns Cost breakdown in micro-dollars
2824
- *
2825
- * @example
2826
- * ```typescript
2827
- * const usage = { promptTokens: 1000, completionTokens: 500 };
2828
- * const pricing = { inputCostPer1M: 2.5, outputCostPer1M: 10.0 };
2829
- * const cost = calculateCost(usage, pricing);
2830
- * // cost = { inputCost: 2500, outputCost: 5000, totalCost: 7500 }
2831
- * // In dollars: $0.0025 input + $0.005 output = $0.0075 total
2832
- * ```
2833
- */
2834
- function calculateCost(usage, pricing) {
2835
- const inputCost = Math.round(usage.promptTokens * pricing.inputCostPer1M);
2836
- const outputCost = Math.round(usage.completionTokens * pricing.outputCostPer1M);
2837
- return {
2838
- inputCost,
2839
- outputCost,
2840
- totalCost: inputCost + outputCost,
2841
- cacheSavings: 0
2842
- };
2843
- }
2844
- /**
2845
- * Get default cache read rate as a fraction of input cost per provider.
2846
- * Used when models.dev doesn't provide cache pricing.
2847
- */
2848
- function getDefaultCacheReadRate(provider, inputCostPer1M) {
2849
- switch (provider?.toLowerCase()) {
2850
- case "anthropic": return inputCostPer1M * .1;
2851
- case "openai":
2852
- case "azure-openai": return inputCostPer1M * .5;
2853
- case "google":
2854
- case "gemini":
2855
- case "vertex_ai": return inputCostPer1M * .25;
2856
- default: return inputCostPer1M * .5;
2857
- }
2858
- }
2859
- /**
2860
- * Get default cache write/creation rate as a fraction of input cost per provider.
2861
- * Used when models.dev doesn't provide cache pricing.
2862
- */
2863
- function getDefaultCacheWriteRate(provider, inputCostPer1M) {
2864
- switch (provider?.toLowerCase()) {
2865
- case "anthropic": return inputCostPer1M * 1.25;
2866
- default: return inputCostPer1M;
2867
- }
2868
2521
  }
2869
- /**
2870
- * Calculate cache-aware cost of an LLM request in micro-dollars.
2871
- *
2872
- * Splits input tokens into uncached, cache-read, and cache-creation buckets,
2873
- * each priced at different rates. Falls back to provider-specific multipliers
2874
- * when models.dev doesn't provide cache pricing.
2875
- *
2876
- * @param usage - Token usage data (with cachedTokens and cacheCreationTokens)
2877
- * @param pricing - Model pricing (may include cacheReadCostPer1M / cacheWriteCostPer1M)
2878
- * @param provider - Provider name for fallback rate selection
2879
- * @returns Cost breakdown in micro-dollars
2880
- */
2881
- function calculateCacheAwareCost(usage, pricing, provider) {
2882
- const cachedTokens = usage.cachedTokens ?? 0;
2883
- const cacheCreationTokens = usage.cacheCreationTokens ?? 0;
2884
- if (cachedTokens === 0 && cacheCreationTokens === 0) return calculateCost(usage, pricing);
2885
- const cacheReadRate = pricing.cacheReadCostPer1M ?? getDefaultCacheReadRate(provider, pricing.inputCostPer1M);
2886
- const cacheWriteRate = pricing.cacheWriteCostPer1M ?? getDefaultCacheWriteRate(provider, pricing.inputCostPer1M);
2887
- const uncachedInputTokens = Math.max(0, usage.promptTokens - cachedTokens - cacheCreationTokens);
2888
- const regularInputCost = Math.round(uncachedInputTokens * pricing.inputCostPer1M);
2889
- const cacheReadCost = Math.round(cachedTokens * cacheReadRate);
2890
- const cacheWriteCost = Math.round(cacheCreationTokens * cacheWriteRate);
2891
- const outputCost = Math.round(usage.completionTokens * pricing.outputCostPer1M);
2892
- const inputCost = regularInputCost + cacheReadCost + cacheWriteCost;
2522
+ function createTracesStore(db) {
2893
2523
  return {
2894
- inputCost,
2895
- outputCost,
2896
- totalCost: inputCost + outputCost,
2897
- cacheSavings: Math.round((cachedTokens + cacheCreationTokens) * pricing.inputCostPer1M) - cacheReadCost - cacheWriteCost
2898
- };
2899
- }
2900
- /**
2901
- * Convert micro-dollars to dollars
2902
- *
2903
- * @param microDollars - Amount in micro-dollars
2904
- * @returns Amount in dollars
2905
- *
2906
- * @example
2907
- * ```typescript
2908
- * microDollarsToDollars(7500); // 0.0075
2909
- * microDollarsToDollars(1000000); // 1.0
2910
- * ```
2911
- */
2912
- function microDollarsToDollars(microDollars) {
2913
- return microDollars / 1e6;
2914
- }
2915
- /**
2916
- * Convert dollars to micro-dollars
2917
- *
2918
- * @param dollars - Amount in dollars
2919
- * @returns Amount in micro-dollars (rounded to nearest integer)
2920
- *
2921
- * @example
2922
- * ```typescript
2923
- * dollarsToMicroDollars(0.0075); // 7500
2924
- * dollarsToMicroDollars(1.0); // 1000000
2925
- * ```
2926
- */
2927
- function dollarsToMicroDollars(dollars) {
2928
- return Math.round(dollars * 1e6);
2929
- }
2930
- /**
2931
- * Format micro-dollars as a human-readable dollar string
2932
- *
2933
- * @param microDollars - Amount in micro-dollars
2934
- * @param decimals - Number of decimal places (default: 6)
2935
- * @returns Formatted dollar string
2936
- *
2937
- * @example
2938
- * ```typescript
2939
- * formatCost(7500); // "$0.007500"
2940
- * formatCost(1234567, 2); // "$1.23"
2941
- * ```
2942
- */
2943
- function formatCost(microDollars, decimals = 6) {
2944
- return `$${microDollarsToDollars(microDollars).toFixed(decimals)}`;
2945
- }
2946
-
2947
- //#endregion
2948
- //#region src/pricing/provider.ts
2949
- const LLMOPS_MODELS_API = "https://models.llmops.build";
2950
- /**
2951
- * Convert price from USD cents per token to dollars per 1M tokens.
2952
- *
2953
- * API returns cents/token. Our system uses dollars/1M tokens.
2954
- * Formula: (centsPerToken / 100) * 1_000_000 = centsPerToken * 10_000
2955
- */
2956
- function centsPerTokenToCostPer1M(centsPerToken) {
2957
- return centsPerToken * 1e4;
2958
- }
2959
- /**
2960
- * Pricing provider that fetches per-model data from the LLMOps Models API.
2961
- *
2962
- * Features:
2963
- * - Per-model in-memory cache with configurable TTL (default 5 minutes)
2964
- * - Deduplicates concurrent fetches for the same model
2965
- * - Caches null results (404s) to avoid repeated lookups
2966
- * - Falls back to stale cache on fetch errors
2967
- */
2968
- var LLMOpsPricingProvider = class {
2969
- cache = /* @__PURE__ */ new Map();
2970
- pendingFetches = /* @__PURE__ */ new Map();
2971
- cacheTTL;
2972
- baseUrl;
2973
- constructor(options) {
2974
- this.cacheTTL = options?.cacheTTL ?? 300 * 1e3;
2975
- this.baseUrl = options?.baseUrl ?? LLMOPS_MODELS_API;
2976
- }
2977
- getCacheKey(provider, model) {
2978
- return `${provider.toLowerCase()}:${model.toLowerCase()}`;
2979
- }
2980
- /**
2981
- * Fetch pricing for a single model from the API
2982
- */
2983
- async fetchModelPricing(provider, model) {
2984
- const url = `${this.baseUrl}/model-configs/pricing/${encodeURIComponent(provider)}/${model}`;
2985
- try {
2986
- logger.debug(`[Pricing] GET ${url}`);
2987
- const startTime = Date.now();
2988
- const response = await fetch(url);
2989
- const elapsed = Date.now() - startTime;
2990
- logger.debug(`[Pricing] GET ${url} -> ${response.status} (${elapsed}ms)`);
2991
- if (response.status === 404) {
2992
- logger.debug(`[Pricing] No pricing found for ${provider}/${model}`);
2993
- return null;
2524
+ upsertTrace: async (data) => {
2525
+ const result = await upsertTraceSchema.safeParseAsync(data);
2526
+ if (!result.success) throw new LLMOpsError(`Invalid trace data: ${result.error.message}`);
2527
+ const trace = result.data;
2528
+ const now = (/* @__PURE__ */ new Date()).toISOString();
2529
+ await sql`
2530
+ INSERT INTO "traces" (
2531
+ "id", "traceId", "name", "sessionId", "userId", "status",
2532
+ "startTime", "endTime", "durationMs", "spanCount",
2533
+ "totalInputTokens", "totalOutputTokens", "totalTokens", "totalCost",
2534
+ "tags", "metadata", "createdAt", "updatedAt"
2535
+ ) VALUES (
2536
+ ${randomUUID()}, ${trace.traceId}, ${trace.name ?? null}, ${trace.sessionId ?? null},
2537
+ ${trace.userId ?? null}, ${trace.status},
2538
+ ${trace.startTime.toISOString()}, ${trace.endTime?.toISOString() ?? null},
2539
+ ${trace.durationMs ?? null}, ${trace.spanCount},
2540
+ ${trace.totalInputTokens}, ${trace.totalOutputTokens},
2541
+ ${trace.totalTokens}, ${trace.totalCost},
2542
+ ${JSON.stringify(trace.tags)}::jsonb, ${JSON.stringify(trace.metadata)}::jsonb,
2543
+ ${now}, ${now}
2544
+ )
2545
+ ON CONFLICT ("traceId") DO UPDATE SET
2546
+ "name" = COALESCE(EXCLUDED."name", "traces"."name"),
2547
+ "sessionId" = COALESCE(EXCLUDED."sessionId", "traces"."sessionId"),
2548
+ "userId" = COALESCE(EXCLUDED."userId", "traces"."userId"),
2549
+ "status" = CASE
2550
+ WHEN EXCLUDED."status" = 'error' THEN 'error'
2551
+ WHEN EXCLUDED."status" = 'ok' AND "traces"."status" != 'error' THEN 'ok'
2552
+ ELSE "traces"."status"
2553
+ END,
2554
+ "startTime" = LEAST("traces"."startTime", EXCLUDED."startTime"),
2555
+ "endTime" = GREATEST(
2556
+ COALESCE("traces"."endTime", EXCLUDED."endTime"),
2557
+ COALESCE(EXCLUDED."endTime", "traces"."endTime")
2558
+ ),
2559
+ "durationMs" = EXTRACT(EPOCH FROM (
2560
+ GREATEST(
2561
+ COALESCE("traces"."endTime", EXCLUDED."endTime"),
2562
+ COALESCE(EXCLUDED."endTime", "traces"."endTime")
2563
+ ) -
2564
+ LEAST("traces"."startTime", EXCLUDED."startTime")
2565
+ ))::integer * 1000,
2566
+ "spanCount" = "traces"."spanCount" + EXCLUDED."spanCount",
2567
+ "totalInputTokens" = "traces"."totalInputTokens" + EXCLUDED."totalInputTokens",
2568
+ "totalOutputTokens" = "traces"."totalOutputTokens" + EXCLUDED."totalOutputTokens",
2569
+ "totalTokens" = "traces"."totalTokens" + EXCLUDED."totalTokens",
2570
+ "totalCost" = "traces"."totalCost" + EXCLUDED."totalCost",
2571
+ "tags" = "traces"."tags" || EXCLUDED."tags",
2572
+ "metadata" = "traces"."metadata" || EXCLUDED."metadata",
2573
+ "updatedAt" = ${now}
2574
+ `.execute(db);
2575
+ },
2576
+ batchInsertSpans: async (spans) => {
2577
+ if (spans.length === 0) return { count: 0 };
2578
+ const validatedSpans = [];
2579
+ for (const span of spans) {
2580
+ const result = await insertSpanSchema.safeParseAsync(span);
2581
+ if (!result.success) {
2582
+ logger.warn(`[batchInsertSpans] Skipping invalid span ${span.spanId}: ${result.error.message}`);
2583
+ continue;
2584
+ }
2585
+ validatedSpans.push(result.data);
2994
2586
  }
2995
- if (!response.ok) throw new Error(`API returned ${response.status}`);
2996
- const data = await response.json();
2997
- if (!data.pay_as_you_go) return null;
2998
- const payg = data.pay_as_you_go;
2999
- const pricing = {
3000
- inputCostPer1M: centsPerTokenToCostPer1M(payg.request_token?.price ?? 0),
3001
- outputCostPer1M: centsPerTokenToCostPer1M(payg.response_token?.price ?? 0),
3002
- cacheReadCostPer1M: payg.cache_read_input_token?.price != null ? centsPerTokenToCostPer1M(payg.cache_read_input_token.price) : void 0,
3003
- cacheWriteCostPer1M: payg.cache_write_input_token?.price != null ? centsPerTokenToCostPer1M(payg.cache_write_input_token.price) : void 0
3004
- };
3005
- logger.debug(`[Pricing] Cached pricing for ${provider}/${model}: input=$${pricing.inputCostPer1M}/1M, output=$${pricing.outputCostPer1M}/1M`);
3006
- return pricing;
3007
- } catch (error) {
3008
- logger.error(`[Pricing] Failed to fetch pricing for ${provider}/${model}: ${error instanceof Error ? error.message : String(error)}`);
3009
- const cacheKey = this.getCacheKey(provider, model);
3010
- const stale = this.cache.get(cacheKey);
3011
- if (stale) {
3012
- logger.debug(`[Pricing] Using stale cache for ${provider}/${model}`);
3013
- return stale.pricing;
2587
+ if (validatedSpans.length === 0) return { count: 0 };
2588
+ const now = (/* @__PURE__ */ new Date()).toISOString();
2589
+ const values = validatedSpans.map((span) => ({
2590
+ id: randomUUID(),
2591
+ traceId: span.traceId,
2592
+ spanId: span.spanId,
2593
+ parentSpanId: span.parentSpanId ?? null,
2594
+ name: span.name,
2595
+ kind: span.kind,
2596
+ status: span.status,
2597
+ statusMessage: span.statusMessage ?? null,
2598
+ startTime: span.startTime.toISOString(),
2599
+ endTime: span.endTime?.toISOString() ?? null,
2600
+ durationMs: span.durationMs ?? null,
2601
+ provider: span.provider ?? null,
2602
+ model: span.model ?? null,
2603
+ promptTokens: span.promptTokens,
2604
+ completionTokens: span.completionTokens,
2605
+ totalTokens: span.totalTokens,
2606
+ cost: span.cost,
2607
+ configId: span.configId ?? null,
2608
+ variantId: span.variantId ?? null,
2609
+ environmentId: span.environmentId ?? null,
2610
+ providerConfigId: span.providerConfigId ?? null,
2611
+ requestId: span.requestId ?? null,
2612
+ source: span.source,
2613
+ input: span.input != null ? JSON.stringify(span.input) : null,
2614
+ output: span.output != null ? JSON.stringify(span.output) : null,
2615
+ attributes: JSON.stringify(span.attributes),
2616
+ createdAt: now,
2617
+ updatedAt: now
2618
+ }));
2619
+ await db.insertInto("spans").values(values).onConflict((oc) => oc.column("spanId").doNothing()).execute();
2620
+ return { count: values.length };
2621
+ },
2622
+ batchInsertSpanEvents: async (events) => {
2623
+ if (events.length === 0) return { count: 0 };
2624
+ const validatedEvents = [];
2625
+ for (const event of events) {
2626
+ const result = await insertSpanEventSchema.safeParseAsync(event);
2627
+ if (!result.success) {
2628
+ logger.warn(`[batchInsertSpanEvents] Skipping invalid event: ${result.error.message}`);
2629
+ continue;
2630
+ }
2631
+ validatedEvents.push(result.data);
3014
2632
  }
3015
- return null;
3016
- }
3017
- }
3018
- /**
3019
- * Internal: fetch with cache and deduplication for a specific provider+model
3020
- */
3021
- async getCachedPricing(provider, model) {
3022
- const cacheKey = this.getCacheKey(provider, model);
3023
- const cached = this.cache.get(cacheKey);
3024
- if (cached && Date.now() - cached.fetchedAt < this.cacheTTL) return cached.pricing;
3025
- let pending = this.pendingFetches.get(cacheKey);
3026
- if (!pending) {
3027
- pending = this.fetchModelPricing(provider, model).then((pricing) => {
3028
- this.cache.set(cacheKey, {
3029
- pricing,
3030
- fetchedAt: Date.now()
3031
- });
3032
- return pricing;
3033
- }).finally(() => {
3034
- this.pendingFetches.delete(cacheKey);
3035
- });
3036
- this.pendingFetches.set(cacheKey, pending);
3037
- }
3038
- return pending;
3039
- }
3040
- /**
3041
- * Get pricing for a specific model.
3042
- *
3043
- * When the model name contains a slash (e.g. "google/gemini-2.5-flash"),
3044
- * it's likely an OpenRouter model ID. If the initial provider lookup fails,
3045
- * we automatically retry with "openrouter" as the provider.
3046
- */
3047
- async getModelPricing(provider, model) {
3048
- const pricing = await this.getCachedPricing(provider, model);
3049
- if (pricing) return pricing;
3050
- if (!pricing && model.includes("/") && provider.toLowerCase() !== "openrouter") {
3051
- logger.debug(`[Pricing] Retrying ${provider}/${model} as openrouter/${model}`);
3052
- return this.getCachedPricing("openrouter", model);
2633
+ if (validatedEvents.length === 0) return { count: 0 };
2634
+ const now = (/* @__PURE__ */ new Date()).toISOString();
2635
+ const values = validatedEvents.map((event) => ({
2636
+ id: randomUUID(),
2637
+ traceId: event.traceId,
2638
+ spanId: event.spanId,
2639
+ name: event.name,
2640
+ timestamp: event.timestamp.toISOString(),
2641
+ attributes: JSON.stringify(event.attributes),
2642
+ createdAt: now
2643
+ }));
2644
+ await db.insertInto("span_events").values(values).execute();
2645
+ return { count: values.length };
2646
+ },
2647
+ listTraces: async (params) => {
2648
+ const result = await listTracesSchema.safeParseAsync(params || {});
2649
+ if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2650
+ const { limit, offset, sessionId, userId, status, name, startDate, endDate, tags } = result.data;
2651
+ let baseQuery = db.selectFrom("traces");
2652
+ if (sessionId) baseQuery = baseQuery.where("sessionId", "=", sessionId);
2653
+ if (userId) baseQuery = baseQuery.where("userId", "=", userId);
2654
+ if (status) baseQuery = baseQuery.where("status", "=", status);
2655
+ if (name) baseQuery = baseQuery.where(sql`${col("name")} ILIKE ${"%" + name + "%"}`);
2656
+ if (startDate) baseQuery = baseQuery.where(sql`${col("startTime")} >= ${startDate.toISOString()}`);
2657
+ if (endDate) baseQuery = baseQuery.where(sql`${col("startTime")} <= ${endDate.toISOString()}`);
2658
+ if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
2659
+ if (values.length === 0) continue;
2660
+ if (values.length === 1) baseQuery = baseQuery.where(sql`${col("tags")}->>${key} = ${values[0]}`);
2661
+ else {
2662
+ const valueList = sql.join(values.map((v) => sql`${v}`));
2663
+ baseQuery = baseQuery.where(sql`${col("tags")}->>${key} IN (${valueList})`);
2664
+ }
2665
+ }
2666
+ const countResult = await baseQuery.select(sql`COUNT(*)`.as("total")).executeTakeFirst();
2667
+ const total = Number(countResult?.total ?? 0);
2668
+ return {
2669
+ data: await baseQuery.selectAll().orderBy("startTime", "desc").limit(limit).offset(offset).execute(),
2670
+ total,
2671
+ limit,
2672
+ offset
2673
+ };
2674
+ },
2675
+ getTraceWithSpans: async (traceId) => {
2676
+ const trace = await db.selectFrom("traces").selectAll().where("traceId", "=", traceId).executeTakeFirst();
2677
+ if (!trace) return void 0;
2678
+ return {
2679
+ trace,
2680
+ spans: await db.selectFrom("spans").selectAll().where("traceId", "=", traceId).orderBy("startTime", "asc").execute(),
2681
+ events: await db.selectFrom("span_events").selectAll().where("traceId", "=", traceId).orderBy("timestamp", "asc").execute()
2682
+ };
2683
+ },
2684
+ getTraceStats: async (params) => {
2685
+ const result = await traceStatsSchema.safeParseAsync(params);
2686
+ if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2687
+ const { startDate, endDate, sessionId, userId } = result.data;
2688
+ let query = db.selectFrom("traces").select([
2689
+ sql`COUNT(*)`.as("totalTraces"),
2690
+ sql`COALESCE(AVG(${col("durationMs")}), 0)`.as("avgDurationMs"),
2691
+ sql`COUNT(CASE WHEN ${col("status")} = 'error' THEN 1 END)`.as("errorCount"),
2692
+ sql`COALESCE(SUM(${col("totalCost")}), 0)`.as("totalCost"),
2693
+ sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
2694
+ sql`COALESCE(SUM(${col("spanCount")}), 0)`.as("totalSpans")
2695
+ ]).where(sql`${col("startTime")} >= ${startDate.toISOString()}`).where(sql`${col("startTime")} <= ${endDate.toISOString()}`);
2696
+ if (sessionId) query = query.where("sessionId", "=", sessionId);
2697
+ if (userId) query = query.where("userId", "=", userId);
2698
+ return query.executeTakeFirst();
3053
2699
  }
3054
- return pricing;
3055
- }
3056
- /**
3057
- * Force refresh the pricing cache (clears all cached entries)
3058
- */
3059
- async refreshCache() {
3060
- this.cache.clear();
3061
- }
3062
- /**
3063
- * Always ready — no bulk pre-fetch needed
3064
- */
3065
- isReady() {
3066
- return true;
3067
- }
3068
- /**
3069
- * Get the number of cached models (for debugging)
3070
- */
3071
- getCacheSize() {
3072
- return this.cache.size;
3073
- }
3074
- };
3075
- let defaultProvider = null;
3076
- /**
3077
- * Get the default pricing provider instance
3078
- */
3079
- function getDefaultPricingProvider() {
3080
- if (!defaultProvider) defaultProvider = new LLMOpsPricingProvider();
3081
- return defaultProvider;
2700
+ };
3082
2701
  }
3083
-
3084
- //#endregion
3085
- //#region src/telemetry/postgres.ts
3086
2702
  /**
3087
2703
  * Create a PostgreSQL-backed telemetry store.
3088
2704
  *
3089
2705
  * Usage:
3090
2706
  * ```ts
3091
- * import { llmops, pgStore } from '@llmops/sdk'
2707
+ * import { llmops } from '@llmops/sdk'
2708
+ * import { pgStore } from '@llmops/sdk/store/pg'
3092
2709
  *
3093
2710
  * const ops = llmops({
3094
2711
  * telemetry: pgStore(process.env.DATABASE_URL),
@@ -3109,139 +2726,13 @@ function createPgStore(connectionString, options) {
3109
2726
  await connection.executeQuery(CompiledQuery.raw(`SET search_path TO "${schema}"`));
3110
2727
  }
3111
2728
  }) });
3112
- const llmRequests = createLLMRequestsDataLayer(db);
3113
- const traces = createTracesDataLayer(db);
3114
2729
  logger.debug(`pgStore: initialized with schema "${schema}"`);
3115
2730
  return {
3116
- ...llmRequests,
3117
- ...traces,
2731
+ ...createLLMRequestsStore(db),
2732
+ ...createTracesStore(db),
3118
2733
  _db: db
3119
2734
  };
3120
2735
  }
3121
2736
 
3122
2737
  //#endregion
3123
- //#region src/manifest/builder.ts
3124
- /**
3125
- * Builds the gateway manifest from database
3126
- */
3127
- var ManifestBuilder = class {
3128
- constructor(db) {
3129
- this.db = db;
3130
- }
3131
- /**
3132
- * Build the manifest from database
3133
- */
3134
- async build() {
3135
- const [guardrailConfigs, providerGuardrailOverridesData] = await Promise.all([this.db.selectFrom("guardrail_configs").where("enabled", "=", true).selectAll().execute(), this.db.selectFrom("provider_guardrail_overrides").selectAll().execute()]);
3136
- const beforeRequestGuardrails = [];
3137
- const afterRequestGuardrails = [];
3138
- logger.info(`[ManifestBuilder] Found ${guardrailConfigs.length} enabled guardrail configs`);
3139
- for (const guardrail of guardrailConfigs) {
3140
- const parameters = typeof guardrail.parameters === "string" ? JSON.parse(guardrail.parameters) : guardrail.parameters;
3141
- const manifestGuardrail = {
3142
- id: guardrail.id,
3143
- name: guardrail.name,
3144
- pluginId: guardrail.pluginId,
3145
- functionId: guardrail.functionId,
3146
- hookType: guardrail.hookType,
3147
- parameters: parameters ?? {},
3148
- priority: guardrail.priority,
3149
- onFail: guardrail.onFail
3150
- };
3151
- if (guardrail.hookType === "beforeRequestHook") beforeRequestGuardrails.push(manifestGuardrail);
3152
- else afterRequestGuardrails.push(manifestGuardrail);
3153
- }
3154
- beforeRequestGuardrails.sort((a, b) => b.priority - a.priority);
3155
- afterRequestGuardrails.sort((a, b) => b.priority - a.priority);
3156
- const providerGuardrailOverrides = {};
3157
- for (const override of providerGuardrailOverridesData) {
3158
- const parameters = typeof override.parameters === "string" ? JSON.parse(override.parameters) : override.parameters;
3159
- const manifestOverride = {
3160
- id: override.id,
3161
- providerConfigId: override.providerConfigId,
3162
- guardrailConfigId: override.guardrailConfigId,
3163
- enabled: override.enabled,
3164
- parameters: parameters ?? null
3165
- };
3166
- if (!providerGuardrailOverrides[override.providerConfigId]) providerGuardrailOverrides[override.providerConfigId] = [];
3167
- providerGuardrailOverrides[override.providerConfigId].push(manifestOverride);
3168
- }
3169
- return {
3170
- version: Date.now(),
3171
- builtAt: (/* @__PURE__ */ new Date()).toISOString(),
3172
- guardrails: {
3173
- beforeRequestHook: beforeRequestGuardrails,
3174
- afterRequestHook: afterRequestGuardrails
3175
- },
3176
- providerGuardrailOverrides
3177
- };
3178
- }
3179
- };
3180
-
3181
- //#endregion
3182
- //#region src/manifest/service.ts
3183
- const MANIFEST_CACHE_KEY = "manifest";
3184
- const MANIFEST_NAMESPACE = "gateway";
3185
- const DEFAULT_TTL_MS = 300 * 1e3;
3186
- const log = logger.child({ module: "ManifestService" });
3187
- var ManifestService = class {
3188
- builder;
3189
- constructor(cache, db, ttlMs = DEFAULT_TTL_MS) {
3190
- this.cache = cache;
3191
- this.ttlMs = ttlMs;
3192
- this.builder = new ManifestBuilder(db);
3193
- log.debug({ ttlMs }, "ManifestService initialized");
3194
- }
3195
- /**
3196
- * Get the current manifest, building if necessary
3197
- */
3198
- async getManifest() {
3199
- log.debug("Getting manifest from cache or building");
3200
- const manifest = await this.cache.getOrSet(MANIFEST_CACHE_KEY, async () => {
3201
- log.info("Building new manifest");
3202
- const built = await this.builder.build();
3203
- log.info({ version: built.version }, "Manifest built successfully");
3204
- return built;
3205
- }, {
3206
- namespace: MANIFEST_NAMESPACE,
3207
- ttl: this.ttlMs
3208
- });
3209
- log.debug({ version: manifest.version }, "Manifest retrieved");
3210
- return manifest;
3211
- }
3212
- /**
3213
- * Force invalidate the manifest (called on mutations)
3214
- */
3215
- async invalidate() {
3216
- log.info("Invalidating manifest cache");
3217
- await this.cache.delete(MANIFEST_CACHE_KEY, MANIFEST_NAMESPACE);
3218
- }
3219
- /**
3220
- * Invalidate and immediately rebuild (atomic refresh)
3221
- */
3222
- async refresh() {
3223
- log.info("Refreshing manifest (invalidate + rebuild)");
3224
- await this.invalidate();
3225
- return this.getManifest();
3226
- }
3227
- /**
3228
- * Get manifest version without fetching full manifest
3229
- * Useful for checking if manifest is stale
3230
- */
3231
- async getVersion() {
3232
- const version = (await this.cache.get(MANIFEST_CACHE_KEY, MANIFEST_NAMESPACE))?.version ?? null;
3233
- log.debug({ version }, "Got manifest version");
3234
- return version;
3235
- }
3236
- /**
3237
- * Check if manifest exists in cache
3238
- */
3239
- async hasManifest() {
3240
- const exists = await this.cache.has(MANIFEST_CACHE_KEY, MANIFEST_NAMESPACE);
3241
- log.debug({ exists }, "Checked manifest existence");
3242
- return exists;
3243
- }
3244
- };
3245
-
3246
- //#endregion
3247
- export { COST_SUMMARY_GROUP_BY, CacheService, DEFAULT_PROVIDER_ENV_VARS, FileCacheBackend, LLMOPS_INTERNAL_HEADER, LLMOPS_REQUEST_ID_HEADER, LLMOPS_SESSION_ID_HEADER, LLMOPS_SPAN_ID_HEADER, LLMOPS_SPAN_NAME_HEADER, LLMOPS_TRACE_ID_HEADER, LLMOPS_TRACE_NAME_HEADER, LLMOPS_USER_ID_HEADER, LLMOpsPricingProvider, MS, ManifestBuilder, ManifestService, MemoryCacheBackend, SCHEMA_METADATA, SupportedProviders, calculateCacheAwareCost, calculateCost, chatCompletionCreateParamsBaseSchema, configVariantsSchema, configsSchema, createDataLayer, createDatabase, createDatabaseFromConnection, createDatasetsDataLayer, createGuardrailConfigsDataLayer, createLLMRequestsDataLayer, createNeonDialect, createPgStore, createPlaygroundDataLayer, createPlaygroundResultsDataLayer, createPlaygroundRunsDataLayer, createProviderConfigsDataLayer, createProviderGuardrailOverridesDataLayer, createTracesDataLayer, createWorkspaceSettingsDataLayer, datasetRecordsSchema, datasetVersionRecordsSchema, datasetVersionsSchema, datasetsSchema, detectDatabaseType, dollarsToMicroDollars, environmentSecretsSchema, environmentsSchema, executeWithSchema, formatCost, gateway, generateId, getDefaultPricingProvider, getDefaultProviders, getMigrations, guardrailConfigsSchema, llmRequestsSchema, llmopsConfigSchema, logger, matchType, mergeWithDefaultProviders, microDollarsToDollars, parsePartialTableData, parseTableData, playgroundColumnSchema, playgroundResultsSchema, playgroundRunsSchema, playgroundsSchema, providerConfigsSchema, providerGuardrailOverridesSchema, runAutoMigrations, schemas, spanEventsSchema, spansSchema, targetingRulesSchema, tracesSchema, validateLLMOpsConfig, validatePartialTableData, validateTableData, variantJsonDataSchema, variantVersionsSchema, variantsSchema, workspaceSettingsSchema };
2738
+ export { COST_SUMMARY_GROUP_BY, CacheService, DEFAULT_PROVIDER_ENV_VARS, FileCacheBackend, LLMOPS_INTERNAL_HEADER, LLMOPS_REQUEST_ID_HEADER, LLMOPS_SESSION_ID_HEADER, LLMOPS_SPAN_ID_HEADER, LLMOPS_SPAN_NAME_HEADER, LLMOPS_TRACE_ID_HEADER, LLMOPS_TRACE_NAME_HEADER, LLMOPS_USER_ID_HEADER, LLMOpsPricingProvider, MS, MemoryCacheBackend, SCHEMA_METADATA, SupportedProviders, calculateCacheAwareCost, calculateCost, chatCompletionCreateParamsBaseSchema, configVariantsSchema, configsSchema, createDataLayer, createDatabase, createDatabaseFromConnection, createDatasetsDataLayer, createNeonDialect, createPgStore, createPlaygroundDataLayer, createPlaygroundResultsDataLayer, createPlaygroundRunsDataLayer, createWorkspaceSettingsDataLayer, datasetRecordsSchema, datasetVersionRecordsSchema, datasetVersionsSchema, datasetsSchema, detectDatabaseType, dollarsToMicroDollars, environmentSecretsSchema, environmentsSchema, executeWithSchema, formatCost, gateway, generateId, getDefaultPricingProvider, getDefaultProviders, getMigrations, guardrailConfigsSchema, llmRequestsSchema, llmopsConfigSchema, logger, matchType, mergeWithDefaultProviders, microDollarsToDollars, parsePartialTableData, parseTableData, playgroundColumnSchema, playgroundResultsSchema, playgroundRunsSchema, playgroundsSchema, providerConfigsSchema, providerGuardrailOverridesSchema, runAutoMigrations, schemas, spanEventsSchema, spansSchema, targetingRulesSchema, tracesSchema, validateLLMOpsConfig, validatePartialTableData, validateTableData, variantJsonDataSchema, variantVersionsSchema, variantsSchema, workspaceSettingsSchema };