@llmops/core 1.0.0-beta.2 → 1.0.0-beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -1387,468 +1387,54 @@ const createDatasetsDataLayer = (db) => {
1387
1387
  };
1388
1388
 
1389
1389
  //#endregion
1390
- //#region src/datalayer/llmRequests.ts
1391
- /**
1392
- * Schema for individual guardrail result in telemetry
1393
- */
1394
- const guardrailResultSchema = require_db.zod_default.object({
1395
- checkId: require_db.zod_default.string(),
1396
- functionId: require_db.zod_default.string(),
1397
- hookType: require_db.zod_default.enum(["beforeRequestHook", "afterRequestHook"]),
1398
- verdict: require_db.zod_default.boolean(),
1399
- latencyMs: require_db.zod_default.number()
1400
- });
1401
- /**
1402
- * Schema for guardrail results aggregate
1403
- */
1404
- const guardrailResultsSchema = require_db.zod_default.object({
1405
- results: require_db.zod_default.array(guardrailResultSchema),
1406
- action: require_db.zod_default.enum([
1407
- "allowed",
1408
- "blocked",
1409
- "logged"
1410
- ]),
1411
- totalLatencyMs: require_db.zod_default.number()
1412
- });
1413
- /**
1414
- * Schema for inserting a new LLM request log
1415
- */
1416
- const insertLLMRequestSchema = require_db.zod_default.object({
1417
- requestId: require_db.zod_default.string().uuid(),
1418
- configId: require_db.zod_default.string().uuid().nullable().optional(),
1419
- variantId: require_db.zod_default.string().uuid().nullable().optional(),
1420
- environmentId: require_db.zod_default.string().uuid().nullable().optional(),
1421
- providerConfigId: require_db.zod_default.string().uuid().nullable().optional(),
1422
- provider: require_db.zod_default.string(),
1423
- model: require_db.zod_default.string(),
1424
- promptTokens: require_db.zod_default.number().int().default(0),
1425
- completionTokens: require_db.zod_default.number().int().default(0),
1426
- totalTokens: require_db.zod_default.number().int().default(0),
1427
- cachedTokens: require_db.zod_default.number().int().default(0),
1428
- cacheCreationTokens: require_db.zod_default.number().int().default(0),
1429
- cost: require_db.zod_default.number().int().default(0),
1430
- cacheSavings: require_db.zod_default.number().int().default(0),
1431
- inputCost: require_db.zod_default.number().int().default(0),
1432
- outputCost: require_db.zod_default.number().int().default(0),
1433
- endpoint: require_db.zod_default.string(),
1434
- statusCode: require_db.zod_default.number().int(),
1435
- latencyMs: require_db.zod_default.number().int().default(0),
1436
- isStreaming: require_db.zod_default.boolean().default(false),
1437
- userId: require_db.zod_default.string().nullable().optional(),
1438
- tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.string()).default({}),
1439
- guardrailResults: guardrailResultsSchema.nullable().optional(),
1440
- traceId: require_db.zod_default.string().nullable().optional(),
1441
- spanId: require_db.zod_default.string().nullable().optional(),
1442
- parentSpanId: require_db.zod_default.string().nullable().optional(),
1443
- sessionId: require_db.zod_default.string().nullable().optional()
1444
- });
1445
- /**
1446
- * Schema for listing LLM requests
1447
- */
1448
- const listRequestsSchema = require_db.zod_default.object({
1449
- limit: require_db.zod_default.number().int().positive().max(1e3).default(100),
1450
- offset: require_db.zod_default.number().int().nonnegative().default(0),
1451
- configId: require_db.zod_default.string().uuid().optional(),
1452
- variantId: require_db.zod_default.string().uuid().optional(),
1453
- environmentId: require_db.zod_default.string().uuid().optional(),
1454
- providerConfigId: require_db.zod_default.string().uuid().optional(),
1455
- provider: require_db.zod_default.string().optional(),
1456
- model: require_db.zod_default.string().optional(),
1457
- startDate: require_db.zod_default.date().optional(),
1458
- endDate: require_db.zod_default.date().optional(),
1459
- tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.array(require_db.zod_default.string())).optional()
1390
+ //#region src/datalayer/playgrounds.ts
1391
+ const createNewPlayground = require_db.zod_default.object({
1392
+ name: require_db.zod_default.string(),
1393
+ datasetId: require_db.zod_default.string().uuid().nullable().optional(),
1394
+ columns: require_db.zod_default.array(require_db.playgroundColumnSchema).nullable().optional()
1460
1395
  });
1461
- /**
1462
- * Schema for date range queries with optional filters
1463
- */
1464
- const dateRangeSchema = require_db.zod_default.object({
1465
- startDate: require_db.zod_default.date(),
1466
- endDate: require_db.zod_default.date(),
1467
- configId: require_db.zod_default.string().uuid().optional(),
1468
- variantId: require_db.zod_default.string().uuid().optional(),
1469
- environmentId: require_db.zod_default.string().uuid().optional(),
1470
- tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.array(require_db.zod_default.string())).optional()
1396
+ const updatePlayground = require_db.zod_default.object({
1397
+ playgroundId: require_db.zod_default.uuidv4(),
1398
+ name: require_db.zod_default.string().optional(),
1399
+ datasetId: require_db.zod_default.string().uuid().nullable().optional(),
1400
+ columns: require_db.zod_default.array(require_db.playgroundColumnSchema).nullable().optional()
1471
1401
  });
1472
- /**
1473
- * Valid groupBy values for cost summary queries
1474
- */
1475
- const COST_SUMMARY_GROUP_BY = [
1476
- "day",
1477
- "hour",
1478
- "model",
1479
- "provider",
1480
- "endpoint",
1481
- "tags"
1482
- ];
1483
- /**
1484
- * Schema for cost summary with grouping
1485
- */
1486
- const costSummarySchema = require_db.zod_default.object({
1487
- startDate: require_db.zod_default.date(),
1488
- endDate: require_db.zod_default.date(),
1489
- configId: require_db.zod_default.string().uuid().optional(),
1490
- variantId: require_db.zod_default.string().uuid().optional(),
1491
- environmentId: require_db.zod_default.string().uuid().optional(),
1492
- tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.array(require_db.zod_default.string())).optional(),
1493
- groupBy: require_db.zod_default.enum(COST_SUMMARY_GROUP_BY).optional(),
1494
- tagKeys: require_db.zod_default.array(require_db.zod_default.string()).optional()
1402
+ const getPlaygroundById = require_db.zod_default.object({ playgroundId: require_db.zod_default.uuidv4() });
1403
+ const deletePlayground = require_db.zod_default.object({ playgroundId: require_db.zod_default.uuidv4() });
1404
+ const listPlaygrounds = require_db.zod_default.object({
1405
+ limit: require_db.zod_default.number().int().positive().optional(),
1406
+ offset: require_db.zod_default.number().int().nonnegative().optional()
1495
1407
  });
1496
- /**
1497
- * Helper to create column reference for SQL
1498
- * Uses sql.ref() to properly quote column names for the database
1499
- */
1500
- const col$1 = (name) => kysely.sql.ref(name);
1501
- const createLLMRequestsDataLayer = (db) => {
1408
+ const createPlaygroundDataLayer = (db) => {
1502
1409
  return {
1503
- batchInsertRequests: async (requests) => {
1504
- if (requests.length === 0) return { count: 0 };
1505
- const validatedRequests = await Promise.all(requests.map(async (req) => {
1506
- const result = await insertLLMRequestSchema.safeParseAsync(req);
1507
- if (!result.success) throw new LLMOpsError(`Invalid request data: ${result.error.message}`);
1508
- return result.data;
1509
- }));
1510
- const now = (/* @__PURE__ */ new Date()).toISOString();
1511
- const values = validatedRequests.map((req) => ({
1512
- id: (0, node_crypto.randomUUID)(),
1513
- requestId: req.requestId,
1514
- configId: req.configId ?? null,
1515
- variantId: req.variantId ?? null,
1516
- environmentId: req.environmentId ?? null,
1517
- providerConfigId: req.providerConfigId ?? null,
1518
- provider: req.provider,
1519
- model: req.model,
1520
- promptTokens: req.promptTokens,
1521
- completionTokens: req.completionTokens,
1522
- totalTokens: req.totalTokens,
1523
- cachedTokens: req.cachedTokens,
1524
- cacheCreationTokens: req.cacheCreationTokens,
1525
- cost: req.cost,
1526
- cacheSavings: req.cacheSavings,
1527
- inputCost: req.inputCost,
1528
- outputCost: req.outputCost,
1529
- endpoint: req.endpoint,
1530
- statusCode: req.statusCode,
1531
- latencyMs: req.latencyMs,
1532
- isStreaming: req.isStreaming,
1533
- userId: req.userId ?? null,
1534
- tags: JSON.stringify(req.tags),
1535
- guardrailResults: req.guardrailResults ? JSON.stringify(req.guardrailResults) : null,
1536
- traceId: req.traceId ?? null,
1537
- spanId: req.spanId ?? null,
1538
- parentSpanId: req.parentSpanId ?? null,
1539
- sessionId: req.sessionId ?? null,
1540
- createdAt: now,
1541
- updatedAt: now
1542
- }));
1543
- await db.insertInto("llm_requests").values(values).execute();
1544
- return { count: values.length };
1545
- },
1546
- insertRequest: async (request) => {
1547
- const result = await insertLLMRequestSchema.safeParseAsync(request);
1548
- if (!result.success) throw new LLMOpsError(`Invalid request data: ${result.error.message}`);
1549
- const req = result.data;
1550
- const now = (/* @__PURE__ */ new Date()).toISOString();
1551
- return db.insertInto("llm_requests").values({
1410
+ createNewPlayground: async (params) => {
1411
+ const value = await createNewPlayground.safeParseAsync(params);
1412
+ if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
1413
+ const { name, datasetId, columns } = value.data;
1414
+ return db.insertInto("playgrounds").values({
1552
1415
  id: (0, node_crypto.randomUUID)(),
1553
- requestId: req.requestId,
1554
- configId: req.configId ?? null,
1555
- variantId: req.variantId ?? null,
1556
- environmentId: req.environmentId ?? null,
1557
- providerConfigId: req.providerConfigId ?? null,
1558
- provider: req.provider,
1559
- model: req.model,
1560
- promptTokens: req.promptTokens,
1561
- completionTokens: req.completionTokens,
1562
- totalTokens: req.totalTokens,
1563
- cachedTokens: req.cachedTokens,
1564
- cacheCreationTokens: req.cacheCreationTokens,
1565
- cost: req.cost,
1566
- cacheSavings: req.cacheSavings,
1567
- inputCost: req.inputCost,
1568
- outputCost: req.outputCost,
1569
- endpoint: req.endpoint,
1570
- statusCode: req.statusCode,
1571
- latencyMs: req.latencyMs,
1572
- isStreaming: req.isStreaming,
1573
- userId: req.userId ?? null,
1574
- tags: JSON.stringify(req.tags),
1575
- guardrailResults: req.guardrailResults ? JSON.stringify(req.guardrailResults) : null,
1576
- traceId: req.traceId ?? null,
1577
- spanId: req.spanId ?? null,
1578
- parentSpanId: req.parentSpanId ?? null,
1579
- sessionId: req.sessionId ?? null,
1580
- createdAt: now,
1581
- updatedAt: now
1416
+ name,
1417
+ datasetId: datasetId ?? null,
1418
+ columns: columns ? JSON.stringify(columns) : null,
1419
+ createdAt: (/* @__PURE__ */ new Date()).toISOString(),
1420
+ updatedAt: (/* @__PURE__ */ new Date()).toISOString()
1582
1421
  }).returningAll().executeTakeFirst();
1583
1422
  },
1584
- listRequests: async (params) => {
1585
- const result = await listRequestsSchema.safeParseAsync(params || {});
1586
- if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
1587
- const { limit, offset, configId, variantId, environmentId, providerConfigId, provider, model, startDate, endDate, tags } = result.data;
1588
- let baseQuery = db.selectFrom("llm_requests");
1589
- if (configId) baseQuery = baseQuery.where("configId", "=", configId);
1590
- if (variantId) baseQuery = baseQuery.where("variantId", "=", variantId);
1591
- if (environmentId) baseQuery = baseQuery.where("environmentId", "=", environmentId);
1592
- if (providerConfigId) baseQuery = baseQuery.where("providerConfigId", "=", providerConfigId);
1593
- if (provider) baseQuery = baseQuery.where("provider", "=", provider);
1594
- if (model) baseQuery = baseQuery.where("model", "=", model);
1595
- if (startDate) baseQuery = baseQuery.where(kysely.sql`${col$1("createdAt")} >= ${startDate.toISOString()}`);
1596
- if (endDate) baseQuery = baseQuery.where(kysely.sql`${col$1("createdAt")} <= ${endDate.toISOString()}`);
1597
- if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
1598
- if (values.length === 0) continue;
1599
- if (values.length === 1) baseQuery = baseQuery.where(kysely.sql`${col$1("tags")}->>${key} = ${values[0]}`);
1600
- else {
1601
- const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
1602
- baseQuery = baseQuery.where(kysely.sql`${col$1("tags")}->>${key} IN (${valueList})`);
1603
- }
1604
- }
1605
- const countResult = await baseQuery.select(kysely.sql`COUNT(*)`.as("total")).executeTakeFirst();
1606
- const total = Number(countResult?.total ?? 0);
1607
- return {
1608
- data: await baseQuery.selectAll().orderBy("createdAt", "desc").limit(limit).offset(offset).execute(),
1609
- total,
1610
- limit,
1611
- offset
1612
- };
1423
+ updatePlayground: async (params) => {
1424
+ const value = await updatePlayground.safeParseAsync(params);
1425
+ if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
1426
+ const { playgroundId, name, datasetId, columns } = value.data;
1427
+ const updateData = { updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
1428
+ if (name !== void 0) updateData.name = name;
1429
+ if (datasetId !== void 0) updateData.datasetId = datasetId;
1430
+ if (columns !== void 0) updateData.columns = columns ? JSON.stringify(columns) : null;
1431
+ return db.updateTable("playgrounds").set(updateData).where("id", "=", playgroundId).returningAll().executeTakeFirst();
1613
1432
  },
1614
- getRequestByRequestId: async (requestId) => {
1615
- return db.selectFrom("llm_requests").selectAll().where("requestId", "=", requestId).executeTakeFirst();
1616
- },
1617
- getTotalCost: async (params) => {
1618
- const result = await dateRangeSchema.safeParseAsync(params);
1619
- if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
1620
- const { startDate, endDate, configId, variantId, environmentId, tags } = result.data;
1621
- let query = db.selectFrom("llm_requests").select([
1622
- kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1623
- kysely.sql`COALESCE(SUM(${col$1("inputCost")}), 0)`.as("totalInputCost"),
1624
- kysely.sql`COALESCE(SUM(${col$1("outputCost")}), 0)`.as("totalOutputCost"),
1625
- kysely.sql`COALESCE(SUM(${col$1("promptTokens")}), 0)`.as("totalPromptTokens"),
1626
- kysely.sql`COALESCE(SUM(${col$1("completionTokens")}), 0)`.as("totalCompletionTokens"),
1627
- kysely.sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens"),
1628
- kysely.sql`COALESCE(SUM(${col$1("cachedTokens")}), 0)`.as("totalCachedTokens"),
1629
- kysely.sql`COALESCE(SUM(${col$1("cacheSavings")}), 0)`.as("totalCacheSavings"),
1630
- kysely.sql`COUNT(*)`.as("requestCount")
1631
- ]).where(kysely.sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col$1("createdAt")} <= ${endDate.toISOString()}`);
1632
- if (configId) query = query.where("configId", "=", configId);
1633
- if (variantId) query = query.where("variantId", "=", variantId);
1634
- if (environmentId) query = query.where("environmentId", "=", environmentId);
1635
- if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
1636
- if (values.length === 0) continue;
1637
- if (values.length === 1) query = query.where(kysely.sql`${col$1("tags")}->>${key} = ${values[0]}`);
1638
- else {
1639
- const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
1640
- query = query.where(kysely.sql`${col$1("tags")}->>${key} IN (${valueList})`);
1641
- }
1642
- }
1643
- return await query.executeTakeFirst();
1644
- },
1645
- getCostByModel: async (params) => {
1646
- const result = await dateRangeSchema.safeParseAsync(params);
1647
- if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
1648
- const { startDate, endDate } = result.data;
1649
- return db.selectFrom("llm_requests").select([
1650
- "provider",
1651
- "model",
1652
- kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1653
- kysely.sql`COALESCE(SUM(${col$1("inputCost")}), 0)`.as("totalInputCost"),
1654
- kysely.sql`COALESCE(SUM(${col$1("outputCost")}), 0)`.as("totalOutputCost"),
1655
- kysely.sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens"),
1656
- kysely.sql`COUNT(*)`.as("requestCount"),
1657
- kysely.sql`AVG(${col$1("latencyMs")})`.as("avgLatencyMs")
1658
- ]).where(kysely.sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col$1("createdAt")} <= ${endDate.toISOString()}`).groupBy(["provider", "model"]).orderBy(kysely.sql`SUM(${col$1("cost")})`, "desc").execute();
1659
- },
1660
- getCostByProvider: async (params) => {
1661
- const result = await dateRangeSchema.safeParseAsync(params);
1662
- if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
1663
- const { startDate, endDate } = result.data;
1664
- return db.selectFrom("llm_requests").select([
1665
- "provider",
1666
- kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1667
- kysely.sql`COALESCE(SUM(${col$1("inputCost")}), 0)`.as("totalInputCost"),
1668
- kysely.sql`COALESCE(SUM(${col$1("outputCost")}), 0)`.as("totalOutputCost"),
1669
- kysely.sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens"),
1670
- kysely.sql`COUNT(*)`.as("requestCount"),
1671
- kysely.sql`AVG(${col$1("latencyMs")})`.as("avgLatencyMs")
1672
- ]).where(kysely.sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col$1("createdAt")} <= ${endDate.toISOString()}`).groupBy("provider").orderBy(kysely.sql`SUM(${col$1("cost")})`, "desc").execute();
1673
- },
1674
- getDailyCosts: async (params) => {
1675
- const result = await dateRangeSchema.safeParseAsync(params);
1676
- if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
1677
- const { startDate, endDate } = result.data;
1678
- return db.selectFrom("llm_requests").select([
1679
- kysely.sql`DATE(${col$1("createdAt")})`.as("date"),
1680
- kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1681
- kysely.sql`COALESCE(SUM(${col$1("inputCost")}), 0)`.as("totalInputCost"),
1682
- kysely.sql`COALESCE(SUM(${col$1("outputCost")}), 0)`.as("totalOutputCost"),
1683
- kysely.sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens"),
1684
- kysely.sql`COUNT(*)`.as("requestCount")
1685
- ]).where(kysely.sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col$1("createdAt")} <= ${endDate.toISOString()}`).groupBy(kysely.sql`DATE(${col$1("createdAt")})`).orderBy(kysely.sql`DATE(${col$1("createdAt")})`, "asc").execute();
1686
- },
1687
- getCostSummary: async (params) => {
1688
- const result = await costSummarySchema.safeParseAsync(params);
1689
- if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
1690
- const { startDate, endDate, groupBy, configId, variantId, environmentId, tags, tagKeys } = result.data;
1691
- let baseQuery = db.selectFrom("llm_requests").where(kysely.sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col$1("createdAt")} <= ${endDate.toISOString()}`);
1692
- if (configId) baseQuery = baseQuery.where("configId", "=", configId);
1693
- if (variantId) baseQuery = baseQuery.where("variantId", "=", variantId);
1694
- if (environmentId) baseQuery = baseQuery.where("environmentId", "=", environmentId);
1695
- if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
1696
- if (values.length === 0) continue;
1697
- if (values.length === 1) baseQuery = baseQuery.where(kysely.sql`${col$1("tags")}->>${key} = ${values[0]}`);
1698
- else {
1699
- const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
1700
- baseQuery = baseQuery.where(kysely.sql`${col$1("tags")}->>${key} IN (${valueList})`);
1701
- }
1702
- }
1703
- switch (groupBy) {
1704
- case "day": return baseQuery.select([
1705
- kysely.sql`DATE(${col$1("createdAt")})`.as("groupKey"),
1706
- kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1707
- kysely.sql`COUNT(*)`.as("requestCount"),
1708
- kysely.sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens")
1709
- ]).groupBy(kysely.sql`DATE(${col$1("createdAt")})`).orderBy(kysely.sql`DATE(${col$1("createdAt")})`, "asc").execute();
1710
- case "hour": return baseQuery.select([
1711
- kysely.sql`DATE_TRUNC('hour', ${col$1("createdAt")})`.as("groupKey"),
1712
- kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1713
- kysely.sql`COUNT(*)`.as("requestCount"),
1714
- kysely.sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens")
1715
- ]).groupBy(kysely.sql`DATE_TRUNC('hour', ${col$1("createdAt")})`).orderBy(kysely.sql`DATE_TRUNC('hour', ${col$1("createdAt")})`, "asc").execute();
1716
- case "model": return baseQuery.select([
1717
- kysely.sql`${col$1("provider")} || '/' || ${col$1("model")}`.as("groupKey"),
1718
- kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1719
- kysely.sql`COUNT(*)`.as("requestCount")
1720
- ]).groupBy(["provider", "model"]).orderBy(kysely.sql`SUM(${col$1("cost")})`, "desc").execute();
1721
- case "provider": return baseQuery.select([
1722
- kysely.sql`${col$1("provider")}`.as("groupKey"),
1723
- kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1724
- kysely.sql`COUNT(*)`.as("requestCount")
1725
- ]).groupBy("provider").orderBy(kysely.sql`SUM(${col$1("cost")})`, "desc").execute();
1726
- case "endpoint": return baseQuery.select([
1727
- kysely.sql`COALESCE(${col$1("endpoint")}, 'unknown')`.as("groupKey"),
1728
- kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1729
- kysely.sql`COUNT(*)`.as("requestCount")
1730
- ]).groupBy("endpoint").orderBy(kysely.sql`SUM(${col$1("cost")})`, "desc").execute();
1731
- case "tags": {
1732
- const conditions = [kysely.sql`${col$1("createdAt")} >= ${startDate.toISOString()}`, kysely.sql`${col$1("createdAt")} <= ${endDate.toISOString()}`];
1733
- if (configId) conditions.push(kysely.sql`${col$1("configId")} = ${configId}`);
1734
- if (variantId) conditions.push(kysely.sql`${col$1("variantId")} = ${variantId}`);
1735
- if (environmentId) conditions.push(kysely.sql`${col$1("environmentId")} = ${environmentId}`);
1736
- if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
1737
- if (values.length === 0) continue;
1738
- if (values.length === 1) conditions.push(kysely.sql`${col$1("tags")}->>${key} = ${values[0]}`);
1739
- else {
1740
- const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
1741
- conditions.push(kysely.sql`${col$1("tags")}->>${key} IN (${valueList})`);
1742
- }
1743
- }
1744
- if (tagKeys && tagKeys.length > 0) {
1745
- const tagKeyList = kysely.sql.join(tagKeys.map((k) => kysely.sql`${k}`), kysely.sql`, `);
1746
- conditions.push(kysely.sql`t.key IN (${tagKeyList})`);
1747
- }
1748
- const whereClause = kysely.sql.join(conditions, kysely.sql` AND `);
1749
- return (await kysely.sql`
1750
- SELECT t.key || ':' || t.value as "groupKey",
1751
- COALESCE(SUM(${col$1("cost")}), 0) as "totalCost",
1752
- COUNT(*) as "requestCount"
1753
- FROM "llm_requests", jsonb_each_text(${col$1("tags")}) t
1754
- WHERE ${whereClause}
1755
- GROUP BY t.key, t.value
1756
- ORDER BY SUM(${col$1("cost")}) DESC
1757
- `.execute(db)).rows;
1758
- }
1759
- default: return baseQuery.select([
1760
- kysely.sql`'total'`.as("groupKey"),
1761
- kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1762
- kysely.sql`COUNT(*)`.as("requestCount")
1763
- ]).execute();
1764
- }
1765
- },
1766
- getRequestStats: async (params) => {
1767
- const result = await dateRangeSchema.safeParseAsync(params);
1768
- if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
1769
- const { startDate, endDate, configId, variantId, environmentId, tags } = result.data;
1770
- let query = db.selectFrom("llm_requests").select([
1771
- kysely.sql`COUNT(*)`.as("totalRequests"),
1772
- kysely.sql`COUNT(CASE WHEN ${col$1("statusCode")} >= 200 AND ${col$1("statusCode")} < 300 THEN 1 END)`.as("successfulRequests"),
1773
- kysely.sql`COUNT(CASE WHEN ${col$1("statusCode")} >= 400 THEN 1 END)`.as("failedRequests"),
1774
- kysely.sql`COUNT(CASE WHEN ${col$1("isStreaming")} = true THEN 1 END)`.as("streamingRequests"),
1775
- kysely.sql`AVG(${col$1("latencyMs")})`.as("avgLatencyMs"),
1776
- kysely.sql`MAX(${col$1("latencyMs")})`.as("maxLatencyMs"),
1777
- kysely.sql`MIN(${col$1("latencyMs")})`.as("minLatencyMs")
1778
- ]).where(kysely.sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col$1("createdAt")} <= ${endDate.toISOString()}`);
1779
- if (configId) query = query.where("configId", "=", configId);
1780
- if (variantId) query = query.where("variantId", "=", variantId);
1781
- if (environmentId) query = query.where("environmentId", "=", environmentId);
1782
- if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
1783
- if (values.length === 0) continue;
1784
- if (values.length === 1) query = query.where(kysely.sql`${col$1("tags")}->>${key} = ${values[0]}`);
1785
- else {
1786
- const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
1787
- query = query.where(kysely.sql`${col$1("tags")}->>${key} IN (${valueList})`);
1788
- }
1789
- }
1790
- return await query.executeTakeFirst();
1791
- },
1792
- getDistinctTags: async () => {
1793
- return (await kysely.sql`
1794
- SELECT DISTINCT key, value
1795
- FROM llm_requests, jsonb_each_text(tags) AS t(key, value)
1796
- WHERE tags != '{}'::jsonb
1797
- ORDER BY key, value
1798
- `.execute(db)).rows;
1799
- }
1800
- };
1801
- };
1802
-
1803
- //#endregion
1804
- //#region src/datalayer/playgrounds.ts
1805
- const createNewPlayground = require_db.zod_default.object({
1806
- name: require_db.zod_default.string(),
1807
- datasetId: require_db.zod_default.string().uuid().nullable().optional(),
1808
- columns: require_db.zod_default.array(require_db.playgroundColumnSchema).nullable().optional()
1809
- });
1810
- const updatePlayground = require_db.zod_default.object({
1811
- playgroundId: require_db.zod_default.uuidv4(),
1812
- name: require_db.zod_default.string().optional(),
1813
- datasetId: require_db.zod_default.string().uuid().nullable().optional(),
1814
- columns: require_db.zod_default.array(require_db.playgroundColumnSchema).nullable().optional()
1815
- });
1816
- const getPlaygroundById = require_db.zod_default.object({ playgroundId: require_db.zod_default.uuidv4() });
1817
- const deletePlayground = require_db.zod_default.object({ playgroundId: require_db.zod_default.uuidv4() });
1818
- const listPlaygrounds = require_db.zod_default.object({
1819
- limit: require_db.zod_default.number().int().positive().optional(),
1820
- offset: require_db.zod_default.number().int().nonnegative().optional()
1821
- });
1822
- const createPlaygroundDataLayer = (db) => {
1823
- return {
1824
- createNewPlayground: async (params) => {
1825
- const value = await createNewPlayground.safeParseAsync(params);
1826
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
1827
- const { name, datasetId, columns } = value.data;
1828
- return db.insertInto("playgrounds").values({
1829
- id: (0, node_crypto.randomUUID)(),
1830
- name,
1831
- datasetId: datasetId ?? null,
1832
- columns: columns ? JSON.stringify(columns) : null,
1833
- createdAt: (/* @__PURE__ */ new Date()).toISOString(),
1834
- updatedAt: (/* @__PURE__ */ new Date()).toISOString()
1835
- }).returningAll().executeTakeFirst();
1836
- },
1837
- updatePlayground: async (params) => {
1838
- const value = await updatePlayground.safeParseAsync(params);
1839
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
1840
- const { playgroundId, name, datasetId, columns } = value.data;
1841
- const updateData = { updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
1842
- if (name !== void 0) updateData.name = name;
1843
- if (datasetId !== void 0) updateData.datasetId = datasetId;
1844
- if (columns !== void 0) updateData.columns = columns ? JSON.stringify(columns) : null;
1845
- return db.updateTable("playgrounds").set(updateData).where("id", "=", playgroundId).returningAll().executeTakeFirst();
1846
- },
1847
- getPlaygroundById: async (params) => {
1848
- const value = await getPlaygroundById.safeParseAsync(params);
1849
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
1850
- const { playgroundId } = value.data;
1851
- return db.selectFrom("playgrounds").selectAll().where("id", "=", playgroundId).executeTakeFirst();
1433
+ getPlaygroundById: async (params) => {
1434
+ const value = await getPlaygroundById.safeParseAsync(params);
1435
+ if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
1436
+ const { playgroundId } = value.data;
1437
+ return db.selectFrom("playgrounds").selectAll().where("id", "=", playgroundId).executeTakeFirst();
1852
1438
  },
1853
1439
  deletePlayground: async (params) => {
1854
1440
  const value = await deletePlayground.safeParseAsync(params);
@@ -2095,292 +1681,13 @@ const createPlaygroundRunsDataLayer = (db) => {
2095
1681
  };
2096
1682
 
2097
1683
  //#endregion
2098
- //#region src/datalayer/traces.ts
2099
- const col = (name) => kysely.sql.ref(name);
2100
- /**
2101
- * Schema for upserting a trace
2102
- */
2103
- const upsertTraceSchema = require_db.zod_default.object({
2104
- traceId: require_db.zod_default.string(),
1684
+ //#region src/datalayer/workspaceSettings.ts
1685
+ const updateWorkspaceSettings = require_db.zod_default.object({
2105
1686
  name: require_db.zod_default.string().nullable().optional(),
2106
- sessionId: require_db.zod_default.string().nullable().optional(),
2107
- userId: require_db.zod_default.string().nullable().optional(),
2108
- status: require_db.zod_default.enum([
2109
- "unset",
2110
- "ok",
2111
- "error"
2112
- ]).default("unset"),
2113
- startTime: require_db.zod_default.date(),
2114
- endTime: require_db.zod_default.date().nullable().optional(),
2115
- durationMs: require_db.zod_default.number().int().nullable().optional(),
2116
- spanCount: require_db.zod_default.number().int().default(1),
2117
- totalInputTokens: require_db.zod_default.number().int().default(0),
2118
- totalOutputTokens: require_db.zod_default.number().int().default(0),
2119
- totalTokens: require_db.zod_default.number().int().default(0),
2120
- totalCost: require_db.zod_default.number().int().default(0),
2121
- tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.string()).default({}),
2122
- metadata: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.unknown()).default({})
2123
- });
2124
- /**
2125
- * Schema for inserting spans
2126
- */
2127
- const insertSpanSchema = require_db.zod_default.object({
2128
- traceId: require_db.zod_default.string(),
2129
- spanId: require_db.zod_default.string(),
2130
- parentSpanId: require_db.zod_default.string().nullable().optional(),
2131
- name: require_db.zod_default.string(),
2132
- kind: require_db.zod_default.number().int().default(1),
2133
- status: require_db.zod_default.number().int().default(0),
2134
- statusMessage: require_db.zod_default.string().nullable().optional(),
2135
- startTime: require_db.zod_default.date(),
2136
- endTime: require_db.zod_default.date().nullable().optional(),
2137
- durationMs: require_db.zod_default.number().int().nullable().optional(),
2138
- provider: require_db.zod_default.string().nullable().optional(),
2139
- model: require_db.zod_default.string().nullable().optional(),
2140
- promptTokens: require_db.zod_default.number().int().default(0),
2141
- completionTokens: require_db.zod_default.number().int().default(0),
2142
- totalTokens: require_db.zod_default.number().int().default(0),
2143
- cost: require_db.zod_default.number().int().default(0),
2144
- configId: require_db.zod_default.string().uuid().nullable().optional(),
2145
- variantId: require_db.zod_default.string().uuid().nullable().optional(),
2146
- environmentId: require_db.zod_default.string().uuid().nullable().optional(),
2147
- providerConfigId: require_db.zod_default.string().uuid().nullable().optional(),
2148
- requestId: require_db.zod_default.string().uuid().nullable().optional(),
2149
- source: require_db.zod_default.enum([
2150
- "gateway",
2151
- "otlp",
2152
- "langsmith"
2153
- ]).default("gateway"),
2154
- input: require_db.zod_default.unknown().nullable().optional(),
2155
- output: require_db.zod_default.unknown().nullable().optional(),
2156
- attributes: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.unknown()).default({})
2157
- });
2158
- /**
2159
- * Schema for inserting span events
2160
- */
2161
- const insertSpanEventSchema = require_db.zod_default.object({
2162
- traceId: require_db.zod_default.string(),
2163
- spanId: require_db.zod_default.string(),
2164
- name: require_db.zod_default.string(),
2165
- timestamp: require_db.zod_default.date(),
2166
- attributes: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.unknown()).default({})
2167
- });
2168
- /**
2169
- * Schema for listing traces
2170
- */
2171
- const listTracesSchema = require_db.zod_default.object({
2172
- limit: require_db.zod_default.number().int().positive().max(1e3).default(50),
2173
- offset: require_db.zod_default.number().int().nonnegative().default(0),
2174
- sessionId: require_db.zod_default.string().optional(),
2175
- userId: require_db.zod_default.string().optional(),
2176
- status: require_db.zod_default.enum([
2177
- "unset",
2178
- "ok",
2179
- "error"
2180
- ]).optional(),
2181
- name: require_db.zod_default.string().optional(),
2182
- startDate: require_db.zod_default.date().optional(),
2183
- endDate: require_db.zod_default.date().optional(),
2184
- tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.array(require_db.zod_default.string())).optional()
2185
- });
2186
- /**
2187
- * Schema for trace stats query
2188
- */
2189
- const traceStatsSchema = require_db.zod_default.object({
2190
- startDate: require_db.zod_default.date(),
2191
- endDate: require_db.zod_default.date(),
2192
- sessionId: require_db.zod_default.string().optional(),
2193
- userId: require_db.zod_default.string().optional()
1687
+ setupComplete: require_db.zod_default.boolean().optional(),
1688
+ superAdminId: require_db.zod_default.string().nullable().optional()
2194
1689
  });
2195
- const createTracesDataLayer = (db) => {
2196
- return {
2197
- upsertTrace: async (data) => {
2198
- const result = await upsertTraceSchema.safeParseAsync(data);
2199
- if (!result.success) throw new LLMOpsError(`Invalid trace data: ${result.error.message}`);
2200
- const trace = result.data;
2201
- const now = (/* @__PURE__ */ new Date()).toISOString();
2202
- await kysely.sql`
2203
- INSERT INTO "traces" (
2204
- "id", "traceId", "name", "sessionId", "userId", "status",
2205
- "startTime", "endTime", "durationMs", "spanCount",
2206
- "totalInputTokens", "totalOutputTokens", "totalTokens", "totalCost",
2207
- "tags", "metadata", "createdAt", "updatedAt"
2208
- ) VALUES (
2209
- ${(0, node_crypto.randomUUID)()}, ${trace.traceId}, ${trace.name ?? null}, ${trace.sessionId ?? null},
2210
- ${trace.userId ?? null}, ${trace.status},
2211
- ${trace.startTime.toISOString()}, ${trace.endTime?.toISOString() ?? null},
2212
- ${trace.durationMs ?? null}, ${trace.spanCount},
2213
- ${trace.totalInputTokens}, ${trace.totalOutputTokens},
2214
- ${trace.totalTokens}, ${trace.totalCost},
2215
- ${JSON.stringify(trace.tags)}::jsonb, ${JSON.stringify(trace.metadata)}::jsonb,
2216
- ${now}, ${now}
2217
- )
2218
- ON CONFLICT ("traceId") DO UPDATE SET
2219
- "name" = COALESCE(EXCLUDED."name", "traces"."name"),
2220
- "sessionId" = COALESCE(EXCLUDED."sessionId", "traces"."sessionId"),
2221
- "userId" = COALESCE(EXCLUDED."userId", "traces"."userId"),
2222
- "status" = CASE
2223
- WHEN EXCLUDED."status" = 'error' THEN 'error'
2224
- WHEN EXCLUDED."status" = 'ok' AND "traces"."status" != 'error' THEN 'ok'
2225
- ELSE "traces"."status"
2226
- END,
2227
- "startTime" = LEAST("traces"."startTime", EXCLUDED."startTime"),
2228
- "endTime" = GREATEST(
2229
- COALESCE("traces"."endTime", EXCLUDED."endTime"),
2230
- COALESCE(EXCLUDED."endTime", "traces"."endTime")
2231
- ),
2232
- "durationMs" = EXTRACT(EPOCH FROM (
2233
- GREATEST(
2234
- COALESCE("traces"."endTime", EXCLUDED."endTime"),
2235
- COALESCE(EXCLUDED."endTime", "traces"."endTime")
2236
- ) -
2237
- LEAST("traces"."startTime", EXCLUDED."startTime")
2238
- ))::integer * 1000,
2239
- "spanCount" = "traces"."spanCount" + EXCLUDED."spanCount",
2240
- "totalInputTokens" = "traces"."totalInputTokens" + EXCLUDED."totalInputTokens",
2241
- "totalOutputTokens" = "traces"."totalOutputTokens" + EXCLUDED."totalOutputTokens",
2242
- "totalTokens" = "traces"."totalTokens" + EXCLUDED."totalTokens",
2243
- "totalCost" = "traces"."totalCost" + EXCLUDED."totalCost",
2244
- "tags" = "traces"."tags" || EXCLUDED."tags",
2245
- "metadata" = "traces"."metadata" || EXCLUDED."metadata",
2246
- "updatedAt" = ${now}
2247
- `.execute(db);
2248
- },
2249
- batchInsertSpans: async (spans) => {
2250
- if (spans.length === 0) return { count: 0 };
2251
- const validatedSpans = [];
2252
- for (const span of spans) {
2253
- const result = await insertSpanSchema.safeParseAsync(span);
2254
- if (!result.success) {
2255
- require_db.logger.warn(`[batchInsertSpans] Skipping invalid span ${span.spanId}: ${result.error.message}`);
2256
- continue;
2257
- }
2258
- validatedSpans.push(result.data);
2259
- }
2260
- if (validatedSpans.length === 0) return { count: 0 };
2261
- const now = (/* @__PURE__ */ new Date()).toISOString();
2262
- const values = validatedSpans.map((span) => ({
2263
- id: (0, node_crypto.randomUUID)(),
2264
- traceId: span.traceId,
2265
- spanId: span.spanId,
2266
- parentSpanId: span.parentSpanId ?? null,
2267
- name: span.name,
2268
- kind: span.kind,
2269
- status: span.status,
2270
- statusMessage: span.statusMessage ?? null,
2271
- startTime: span.startTime.toISOString(),
2272
- endTime: span.endTime?.toISOString() ?? null,
2273
- durationMs: span.durationMs ?? null,
2274
- provider: span.provider ?? null,
2275
- model: span.model ?? null,
2276
- promptTokens: span.promptTokens,
2277
- completionTokens: span.completionTokens,
2278
- totalTokens: span.totalTokens,
2279
- cost: span.cost,
2280
- configId: span.configId ?? null,
2281
- variantId: span.variantId ?? null,
2282
- environmentId: span.environmentId ?? null,
2283
- providerConfigId: span.providerConfigId ?? null,
2284
- requestId: span.requestId ?? null,
2285
- source: span.source,
2286
- input: span.input != null ? JSON.stringify(span.input) : null,
2287
- output: span.output != null ? JSON.stringify(span.output) : null,
2288
- attributes: JSON.stringify(span.attributes),
2289
- createdAt: now,
2290
- updatedAt: now
2291
- }));
2292
- await db.insertInto("spans").values(values).onConflict((oc) => oc.column("spanId").doNothing()).execute();
2293
- return { count: values.length };
2294
- },
2295
- batchInsertSpanEvents: async (events) => {
2296
- if (events.length === 0) return { count: 0 };
2297
- const validatedEvents = [];
2298
- for (const event of events) {
2299
- const result = await insertSpanEventSchema.safeParseAsync(event);
2300
- if (!result.success) {
2301
- require_db.logger.warn(`[batchInsertSpanEvents] Skipping invalid event: ${result.error.message}`);
2302
- continue;
2303
- }
2304
- validatedEvents.push(result.data);
2305
- }
2306
- if (validatedEvents.length === 0) return { count: 0 };
2307
- const now = (/* @__PURE__ */ new Date()).toISOString();
2308
- const values = validatedEvents.map((event) => ({
2309
- id: (0, node_crypto.randomUUID)(),
2310
- traceId: event.traceId,
2311
- spanId: event.spanId,
2312
- name: event.name,
2313
- timestamp: event.timestamp.toISOString(),
2314
- attributes: JSON.stringify(event.attributes),
2315
- createdAt: now
2316
- }));
2317
- await db.insertInto("span_events").values(values).execute();
2318
- return { count: values.length };
2319
- },
2320
- listTraces: async (params) => {
2321
- const result = await listTracesSchema.safeParseAsync(params || {});
2322
- if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2323
- const { limit, offset, sessionId, userId, status, name, startDate, endDate, tags } = result.data;
2324
- let baseQuery = db.selectFrom("traces");
2325
- if (sessionId) baseQuery = baseQuery.where("sessionId", "=", sessionId);
2326
- if (userId) baseQuery = baseQuery.where("userId", "=", userId);
2327
- if (status) baseQuery = baseQuery.where("status", "=", status);
2328
- if (name) baseQuery = baseQuery.where(kysely.sql`${col("name")} ILIKE ${"%" + name + "%"}`);
2329
- if (startDate) baseQuery = baseQuery.where(kysely.sql`${col("startTime")} >= ${startDate.toISOString()}`);
2330
- if (endDate) baseQuery = baseQuery.where(kysely.sql`${col("startTime")} <= ${endDate.toISOString()}`);
2331
- if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
2332
- if (values.length === 0) continue;
2333
- if (values.length === 1) baseQuery = baseQuery.where(kysely.sql`${col("tags")}->>${key} = ${values[0]}`);
2334
- else {
2335
- const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
2336
- baseQuery = baseQuery.where(kysely.sql`${col("tags")}->>${key} IN (${valueList})`);
2337
- }
2338
- }
2339
- const countResult = await baseQuery.select(kysely.sql`COUNT(*)`.as("total")).executeTakeFirst();
2340
- const total = Number(countResult?.total ?? 0);
2341
- return {
2342
- data: await baseQuery.selectAll().orderBy("startTime", "desc").limit(limit).offset(offset).execute(),
2343
- total,
2344
- limit,
2345
- offset
2346
- };
2347
- },
2348
- getTraceWithSpans: async (traceId) => {
2349
- const trace = await db.selectFrom("traces").selectAll().where("traceId", "=", traceId).executeTakeFirst();
2350
- if (!trace) return void 0;
2351
- return {
2352
- trace,
2353
- spans: await db.selectFrom("spans").selectAll().where("traceId", "=", traceId).orderBy("startTime", "asc").execute(),
2354
- events: await db.selectFrom("span_events").selectAll().where("traceId", "=", traceId).orderBy("timestamp", "asc").execute()
2355
- };
2356
- },
2357
- getTraceStats: async (params) => {
2358
- const result = await traceStatsSchema.safeParseAsync(params);
2359
- if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2360
- const { startDate, endDate, sessionId, userId } = result.data;
2361
- let query = db.selectFrom("traces").select([
2362
- kysely.sql`COUNT(*)`.as("totalTraces"),
2363
- kysely.sql`COALESCE(AVG(${col("durationMs")}), 0)`.as("avgDurationMs"),
2364
- kysely.sql`COUNT(CASE WHEN ${col("status")} = 'error' THEN 1 END)`.as("errorCount"),
2365
- kysely.sql`COALESCE(SUM(${col("totalCost")}), 0)`.as("totalCost"),
2366
- kysely.sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
2367
- kysely.sql`COALESCE(SUM(${col("spanCount")}), 0)`.as("totalSpans")
2368
- ]).where(kysely.sql`${col("startTime")} >= ${startDate.toISOString()}`).where(kysely.sql`${col("startTime")} <= ${endDate.toISOString()}`);
2369
- if (sessionId) query = query.where("sessionId", "=", sessionId);
2370
- if (userId) query = query.where("userId", "=", userId);
2371
- return query.executeTakeFirst();
2372
- }
2373
- };
2374
- };
2375
-
2376
- //#endregion
2377
- //#region src/datalayer/workspaceSettings.ts
2378
- const updateWorkspaceSettings = require_db.zod_default.object({
2379
- name: require_db.zod_default.string().nullable().optional(),
2380
- setupComplete: require_db.zod_default.boolean().optional(),
2381
- superAdminId: require_db.zod_default.string().nullable().optional()
2382
- });
2383
- const createWorkspaceSettingsDataLayer = (db) => {
1690
+ const createWorkspaceSettingsDataLayer = (db) => {
2384
1691
  return {
2385
1692
  getWorkspaceSettings: async () => {
2386
1693
  let settings = await db.selectFrom("workspace_settings").selectAll().executeTakeFirst();
@@ -2466,11 +1773,9 @@ const createWorkspaceSettingsDataLayer = (db) => {
2466
1773
  function createDataLayer(db) {
2467
1774
  return {
2468
1775
  ...createDatasetsDataLayer(db),
2469
- ...createLLMRequestsDataLayer(db),
2470
1776
  ...createPlaygroundDataLayer(db),
2471
1777
  ...createPlaygroundResultsDataLayer(db),
2472
1778
  ...createPlaygroundRunsDataLayer(db),
2473
- ...createTracesDataLayer(db),
2474
1779
  ...createWorkspaceSettingsDataLayer(db)
2475
1780
  };
2476
1781
  }
@@ -2750,12 +2055,659 @@ function getDefaultPricingProvider() {
2750
2055
 
2751
2056
  //#endregion
2752
2057
  //#region src/telemetry/postgres.ts
2058
+ const guardrailResultSchema = require_db.zod_default.object({
2059
+ checkId: require_db.zod_default.string(),
2060
+ functionId: require_db.zod_default.string(),
2061
+ hookType: require_db.zod_default.enum(["beforeRequestHook", "afterRequestHook"]),
2062
+ verdict: require_db.zod_default.boolean(),
2063
+ latencyMs: require_db.zod_default.number()
2064
+ });
2065
+ const guardrailResultsSchema = require_db.zod_default.object({
2066
+ results: require_db.zod_default.array(guardrailResultSchema),
2067
+ action: require_db.zod_default.enum([
2068
+ "allowed",
2069
+ "blocked",
2070
+ "logged"
2071
+ ]),
2072
+ totalLatencyMs: require_db.zod_default.number()
2073
+ });
2074
+ const insertLLMRequestSchema = require_db.zod_default.object({
2075
+ requestId: require_db.zod_default.string().uuid(),
2076
+ configId: require_db.zod_default.string().uuid().nullable().optional(),
2077
+ variantId: require_db.zod_default.string().uuid().nullable().optional(),
2078
+ environmentId: require_db.zod_default.string().uuid().nullable().optional(),
2079
+ providerConfigId: require_db.zod_default.string().uuid().nullable().optional(),
2080
+ provider: require_db.zod_default.string(),
2081
+ model: require_db.zod_default.string(),
2082
+ promptTokens: require_db.zod_default.number().int().default(0),
2083
+ completionTokens: require_db.zod_default.number().int().default(0),
2084
+ totalTokens: require_db.zod_default.number().int().default(0),
2085
+ cachedTokens: require_db.zod_default.number().int().default(0),
2086
+ cacheCreationTokens: require_db.zod_default.number().int().default(0),
2087
+ cost: require_db.zod_default.number().int().default(0),
2088
+ cacheSavings: require_db.zod_default.number().int().default(0),
2089
+ inputCost: require_db.zod_default.number().int().default(0),
2090
+ outputCost: require_db.zod_default.number().int().default(0),
2091
+ endpoint: require_db.zod_default.string(),
2092
+ statusCode: require_db.zod_default.number().int(),
2093
+ latencyMs: require_db.zod_default.number().int().default(0),
2094
+ isStreaming: require_db.zod_default.boolean().default(false),
2095
+ userId: require_db.zod_default.string().nullable().optional(),
2096
+ tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.string()).default({}),
2097
+ guardrailResults: guardrailResultsSchema.nullable().optional(),
2098
+ traceId: require_db.zod_default.string().nullable().optional(),
2099
+ spanId: require_db.zod_default.string().nullable().optional(),
2100
+ parentSpanId: require_db.zod_default.string().nullable().optional(),
2101
+ sessionId: require_db.zod_default.string().nullable().optional()
2102
+ });
2103
+ const listRequestsSchema = require_db.zod_default.object({
2104
+ limit: require_db.zod_default.number().int().positive().max(1e3).default(100),
2105
+ offset: require_db.zod_default.number().int().nonnegative().default(0),
2106
+ configId: require_db.zod_default.string().uuid().optional(),
2107
+ variantId: require_db.zod_default.string().uuid().optional(),
2108
+ environmentId: require_db.zod_default.string().uuid().optional(),
2109
+ providerConfigId: require_db.zod_default.string().uuid().optional(),
2110
+ provider: require_db.zod_default.string().optional(),
2111
+ model: require_db.zod_default.string().optional(),
2112
+ startDate: require_db.zod_default.date().optional(),
2113
+ endDate: require_db.zod_default.date().optional(),
2114
+ tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.array(require_db.zod_default.string())).optional()
2115
+ });
2116
+ const dateRangeSchema = require_db.zod_default.object({
2117
+ startDate: require_db.zod_default.date(),
2118
+ endDate: require_db.zod_default.date(),
2119
+ configId: require_db.zod_default.string().uuid().optional(),
2120
+ variantId: require_db.zod_default.string().uuid().optional(),
2121
+ environmentId: require_db.zod_default.string().uuid().optional(),
2122
+ tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.array(require_db.zod_default.string())).optional()
2123
+ });
2124
+ const COST_SUMMARY_GROUP_BY = [
2125
+ "day",
2126
+ "hour",
2127
+ "model",
2128
+ "provider",
2129
+ "endpoint",
2130
+ "tags"
2131
+ ];
2132
+ const costSummarySchema = require_db.zod_default.object({
2133
+ startDate: require_db.zod_default.date(),
2134
+ endDate: require_db.zod_default.date(),
2135
+ configId: require_db.zod_default.string().uuid().optional(),
2136
+ variantId: require_db.zod_default.string().uuid().optional(),
2137
+ environmentId: require_db.zod_default.string().uuid().optional(),
2138
+ tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.array(require_db.zod_default.string())).optional(),
2139
+ groupBy: require_db.zod_default.enum(COST_SUMMARY_GROUP_BY).optional(),
2140
+ tagKeys: require_db.zod_default.array(require_db.zod_default.string()).optional()
2141
+ });
2142
+ const upsertTraceSchema = require_db.zod_default.object({
2143
+ traceId: require_db.zod_default.string(),
2144
+ name: require_db.zod_default.string().nullable().optional(),
2145
+ sessionId: require_db.zod_default.string().nullable().optional(),
2146
+ userId: require_db.zod_default.string().nullable().optional(),
2147
+ status: require_db.zod_default.enum([
2148
+ "unset",
2149
+ "ok",
2150
+ "error"
2151
+ ]).default("unset"),
2152
+ startTime: require_db.zod_default.date(),
2153
+ endTime: require_db.zod_default.date().nullable().optional(),
2154
+ durationMs: require_db.zod_default.number().int().nullable().optional(),
2155
+ spanCount: require_db.zod_default.number().int().default(1),
2156
+ totalInputTokens: require_db.zod_default.number().int().default(0),
2157
+ totalOutputTokens: require_db.zod_default.number().int().default(0),
2158
+ totalTokens: require_db.zod_default.number().int().default(0),
2159
+ totalCost: require_db.zod_default.number().int().default(0),
2160
+ tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.string()).default({}),
2161
+ metadata: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.unknown()).default({})
2162
+ });
2163
+ const insertSpanSchema = require_db.zod_default.object({
2164
+ traceId: require_db.zod_default.string(),
2165
+ spanId: require_db.zod_default.string(),
2166
+ parentSpanId: require_db.zod_default.string().nullable().optional(),
2167
+ name: require_db.zod_default.string(),
2168
+ kind: require_db.zod_default.number().int().default(1),
2169
+ status: require_db.zod_default.number().int().default(0),
2170
+ statusMessage: require_db.zod_default.string().nullable().optional(),
2171
+ startTime: require_db.zod_default.date(),
2172
+ endTime: require_db.zod_default.date().nullable().optional(),
2173
+ durationMs: require_db.zod_default.number().int().nullable().optional(),
2174
+ provider: require_db.zod_default.string().nullable().optional(),
2175
+ model: require_db.zod_default.string().nullable().optional(),
2176
+ promptTokens: require_db.zod_default.number().int().default(0),
2177
+ completionTokens: require_db.zod_default.number().int().default(0),
2178
+ totalTokens: require_db.zod_default.number().int().default(0),
2179
+ cost: require_db.zod_default.number().int().default(0),
2180
+ configId: require_db.zod_default.string().uuid().nullable().optional(),
2181
+ variantId: require_db.zod_default.string().uuid().nullable().optional(),
2182
+ environmentId: require_db.zod_default.string().uuid().nullable().optional(),
2183
+ providerConfigId: require_db.zod_default.string().uuid().nullable().optional(),
2184
+ requestId: require_db.zod_default.string().uuid().nullable().optional(),
2185
+ source: require_db.zod_default.enum([
2186
+ "gateway",
2187
+ "otlp",
2188
+ "langsmith"
2189
+ ]).default("gateway"),
2190
+ input: require_db.zod_default.unknown().nullable().optional(),
2191
+ output: require_db.zod_default.unknown().nullable().optional(),
2192
+ attributes: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.unknown()).default({})
2193
+ });
2194
+ const insertSpanEventSchema = require_db.zod_default.object({
2195
+ traceId: require_db.zod_default.string(),
2196
+ spanId: require_db.zod_default.string(),
2197
+ name: require_db.zod_default.string(),
2198
+ timestamp: require_db.zod_default.date(),
2199
+ attributes: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.unknown()).default({})
2200
+ });
2201
+ const listTracesSchema = require_db.zod_default.object({
2202
+ limit: require_db.zod_default.number().int().positive().max(1e3).default(50),
2203
+ offset: require_db.zod_default.number().int().nonnegative().default(0),
2204
+ sessionId: require_db.zod_default.string().optional(),
2205
+ userId: require_db.zod_default.string().optional(),
2206
+ status: require_db.zod_default.enum([
2207
+ "unset",
2208
+ "ok",
2209
+ "error"
2210
+ ]).optional(),
2211
+ name: require_db.zod_default.string().optional(),
2212
+ startDate: require_db.zod_default.date().optional(),
2213
+ endDate: require_db.zod_default.date().optional(),
2214
+ tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.array(require_db.zod_default.string())).optional()
2215
+ });
2216
+ const traceStatsSchema = require_db.zod_default.object({
2217
+ startDate: require_db.zod_default.date(),
2218
+ endDate: require_db.zod_default.date(),
2219
+ sessionId: require_db.zod_default.string().optional(),
2220
+ userId: require_db.zod_default.string().optional()
2221
+ });
2222
+ const col = (name) => kysely.sql.ref(name);
2223
+ function createLLMRequestsStore(db) {
2224
+ return {
2225
+ batchInsertRequests: async (requests) => {
2226
+ if (requests.length === 0) return { count: 0 };
2227
+ const validatedRequests = await Promise.all(requests.map(async (req) => {
2228
+ const result = await insertLLMRequestSchema.safeParseAsync(req);
2229
+ if (!result.success) throw new LLMOpsError(`Invalid request data: ${result.error.message}`);
2230
+ return result.data;
2231
+ }));
2232
+ const now = (/* @__PURE__ */ new Date()).toISOString();
2233
+ const values = validatedRequests.map((req) => ({
2234
+ id: (0, node_crypto.randomUUID)(),
2235
+ requestId: req.requestId,
2236
+ configId: req.configId ?? null,
2237
+ variantId: req.variantId ?? null,
2238
+ environmentId: req.environmentId ?? null,
2239
+ providerConfigId: req.providerConfigId ?? null,
2240
+ provider: req.provider,
2241
+ model: req.model,
2242
+ promptTokens: req.promptTokens,
2243
+ completionTokens: req.completionTokens,
2244
+ totalTokens: req.totalTokens,
2245
+ cachedTokens: req.cachedTokens,
2246
+ cacheCreationTokens: req.cacheCreationTokens,
2247
+ cost: req.cost,
2248
+ cacheSavings: req.cacheSavings,
2249
+ inputCost: req.inputCost,
2250
+ outputCost: req.outputCost,
2251
+ endpoint: req.endpoint,
2252
+ statusCode: req.statusCode,
2253
+ latencyMs: req.latencyMs,
2254
+ isStreaming: req.isStreaming,
2255
+ userId: req.userId ?? null,
2256
+ tags: JSON.stringify(req.tags),
2257
+ guardrailResults: req.guardrailResults ? JSON.stringify(req.guardrailResults) : null,
2258
+ traceId: req.traceId ?? null,
2259
+ spanId: req.spanId ?? null,
2260
+ parentSpanId: req.parentSpanId ?? null,
2261
+ sessionId: req.sessionId ?? null,
2262
+ createdAt: now,
2263
+ updatedAt: now
2264
+ }));
2265
+ await db.insertInto("llm_requests").values(values).execute();
2266
+ return { count: values.length };
2267
+ },
2268
+ insertRequest: async (request) => {
2269
+ const result = await insertLLMRequestSchema.safeParseAsync(request);
2270
+ if (!result.success) throw new LLMOpsError(`Invalid request data: ${result.error.message}`);
2271
+ const req = result.data;
2272
+ const now = (/* @__PURE__ */ new Date()).toISOString();
2273
+ return db.insertInto("llm_requests").values({
2274
+ id: (0, node_crypto.randomUUID)(),
2275
+ requestId: req.requestId,
2276
+ configId: req.configId ?? null,
2277
+ variantId: req.variantId ?? null,
2278
+ environmentId: req.environmentId ?? null,
2279
+ providerConfigId: req.providerConfigId ?? null,
2280
+ provider: req.provider,
2281
+ model: req.model,
2282
+ promptTokens: req.promptTokens,
2283
+ completionTokens: req.completionTokens,
2284
+ totalTokens: req.totalTokens,
2285
+ cachedTokens: req.cachedTokens,
2286
+ cacheCreationTokens: req.cacheCreationTokens,
2287
+ cost: req.cost,
2288
+ cacheSavings: req.cacheSavings,
2289
+ inputCost: req.inputCost,
2290
+ outputCost: req.outputCost,
2291
+ endpoint: req.endpoint,
2292
+ statusCode: req.statusCode,
2293
+ latencyMs: req.latencyMs,
2294
+ isStreaming: req.isStreaming,
2295
+ userId: req.userId ?? null,
2296
+ tags: JSON.stringify(req.tags),
2297
+ guardrailResults: req.guardrailResults ? JSON.stringify(req.guardrailResults) : null,
2298
+ traceId: req.traceId ?? null,
2299
+ spanId: req.spanId ?? null,
2300
+ parentSpanId: req.parentSpanId ?? null,
2301
+ sessionId: req.sessionId ?? null,
2302
+ createdAt: now,
2303
+ updatedAt: now
2304
+ }).returningAll().executeTakeFirst();
2305
+ },
2306
+ listRequests: async (params) => {
2307
+ const result = await listRequestsSchema.safeParseAsync(params || {});
2308
+ if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2309
+ const { limit, offset, configId, variantId, environmentId, providerConfigId, provider, model, startDate, endDate, tags } = result.data;
2310
+ let baseQuery = db.selectFrom("llm_requests");
2311
+ if (configId) baseQuery = baseQuery.where("configId", "=", configId);
2312
+ if (variantId) baseQuery = baseQuery.where("variantId", "=", variantId);
2313
+ if (environmentId) baseQuery = baseQuery.where("environmentId", "=", environmentId);
2314
+ if (providerConfigId) baseQuery = baseQuery.where("providerConfigId", "=", providerConfigId);
2315
+ if (provider) baseQuery = baseQuery.where("provider", "=", provider);
2316
+ if (model) baseQuery = baseQuery.where("model", "=", model);
2317
+ if (startDate) baseQuery = baseQuery.where(kysely.sql`${col("createdAt")} >= ${startDate.toISOString()}`);
2318
+ if (endDate) baseQuery = baseQuery.where(kysely.sql`${col("createdAt")} <= ${endDate.toISOString()}`);
2319
+ if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
2320
+ if (values.length === 0) continue;
2321
+ if (values.length === 1) baseQuery = baseQuery.where(kysely.sql`${col("tags")}->>${key} = ${values[0]}`);
2322
+ else {
2323
+ const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
2324
+ baseQuery = baseQuery.where(kysely.sql`${col("tags")}->>${key} IN (${valueList})`);
2325
+ }
2326
+ }
2327
+ const countResult = await baseQuery.select(kysely.sql`COUNT(*)`.as("total")).executeTakeFirst();
2328
+ const total = Number(countResult?.total ?? 0);
2329
+ return {
2330
+ data: await baseQuery.selectAll().orderBy("createdAt", "desc").limit(limit).offset(offset).execute(),
2331
+ total,
2332
+ limit,
2333
+ offset
2334
+ };
2335
+ },
2336
+ getRequestByRequestId: async (requestId) => {
2337
+ return db.selectFrom("llm_requests").selectAll().where("requestId", "=", requestId).executeTakeFirst();
2338
+ },
2339
+ getTotalCost: async (params) => {
2340
+ const result = await dateRangeSchema.safeParseAsync(params);
2341
+ if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2342
+ const { startDate, endDate, configId, variantId, environmentId, tags } = result.data;
2343
+ let query = db.selectFrom("llm_requests").select([
2344
+ kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2345
+ kysely.sql`COALESCE(SUM(${col("inputCost")}), 0)`.as("totalInputCost"),
2346
+ kysely.sql`COALESCE(SUM(${col("outputCost")}), 0)`.as("totalOutputCost"),
2347
+ kysely.sql`COALESCE(SUM(${col("promptTokens")}), 0)`.as("totalPromptTokens"),
2348
+ kysely.sql`COALESCE(SUM(${col("completionTokens")}), 0)`.as("totalCompletionTokens"),
2349
+ kysely.sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
2350
+ kysely.sql`COALESCE(SUM(${col("cachedTokens")}), 0)`.as("totalCachedTokens"),
2351
+ kysely.sql`COALESCE(SUM(${col("cacheSavings")}), 0)`.as("totalCacheSavings"),
2352
+ kysely.sql`COUNT(*)`.as("requestCount")
2353
+ ]).where(kysely.sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col("createdAt")} <= ${endDate.toISOString()}`);
2354
+ if (configId) query = query.where("configId", "=", configId);
2355
+ if (variantId) query = query.where("variantId", "=", variantId);
2356
+ if (environmentId) query = query.where("environmentId", "=", environmentId);
2357
+ if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
2358
+ if (values.length === 0) continue;
2359
+ if (values.length === 1) query = query.where(kysely.sql`${col("tags")}->>${key} = ${values[0]}`);
2360
+ else {
2361
+ const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
2362
+ query = query.where(kysely.sql`${col("tags")}->>${key} IN (${valueList})`);
2363
+ }
2364
+ }
2365
+ return query.executeTakeFirst();
2366
+ },
2367
+ getCostByModel: async (params) => {
2368
+ const result = await dateRangeSchema.safeParseAsync(params);
2369
+ if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2370
+ const { startDate, endDate } = result.data;
2371
+ return db.selectFrom("llm_requests").select([
2372
+ "provider",
2373
+ "model",
2374
+ kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2375
+ kysely.sql`COALESCE(SUM(${col("inputCost")}), 0)`.as("totalInputCost"),
2376
+ kysely.sql`COALESCE(SUM(${col("outputCost")}), 0)`.as("totalOutputCost"),
2377
+ kysely.sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
2378
+ kysely.sql`COUNT(*)`.as("requestCount"),
2379
+ kysely.sql`AVG(${col("latencyMs")})`.as("avgLatencyMs")
2380
+ ]).where(kysely.sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col("createdAt")} <= ${endDate.toISOString()}`).groupBy(["provider", "model"]).orderBy(kysely.sql`SUM(${col("cost")})`, "desc").execute();
2381
+ },
2382
+ getCostByProvider: async (params) => {
2383
+ const result = await dateRangeSchema.safeParseAsync(params);
2384
+ if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2385
+ const { startDate, endDate } = result.data;
2386
+ return db.selectFrom("llm_requests").select([
2387
+ "provider",
2388
+ kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2389
+ kysely.sql`COALESCE(SUM(${col("inputCost")}), 0)`.as("totalInputCost"),
2390
+ kysely.sql`COALESCE(SUM(${col("outputCost")}), 0)`.as("totalOutputCost"),
2391
+ kysely.sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
2392
+ kysely.sql`COUNT(*)`.as("requestCount"),
2393
+ kysely.sql`AVG(${col("latencyMs")})`.as("avgLatencyMs")
2394
+ ]).where(kysely.sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col("createdAt")} <= ${endDate.toISOString()}`).groupBy("provider").orderBy(kysely.sql`SUM(${col("cost")})`, "desc").execute();
2395
+ },
2396
+ getDailyCosts: async (params) => {
2397
+ const result = await dateRangeSchema.safeParseAsync(params);
2398
+ if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2399
+ const { startDate, endDate } = result.data;
2400
+ return db.selectFrom("llm_requests").select([
2401
+ kysely.sql`DATE(${col("createdAt")})`.as("date"),
2402
+ kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2403
+ kysely.sql`COALESCE(SUM(${col("inputCost")}), 0)`.as("totalInputCost"),
2404
+ kysely.sql`COALESCE(SUM(${col("outputCost")}), 0)`.as("totalOutputCost"),
2405
+ kysely.sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
2406
+ kysely.sql`COUNT(*)`.as("requestCount")
2407
+ ]).where(kysely.sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col("createdAt")} <= ${endDate.toISOString()}`).groupBy(kysely.sql`DATE(${col("createdAt")})`).orderBy(kysely.sql`DATE(${col("createdAt")})`, "asc").execute();
2408
+ },
2409
+ getCostSummary: async (params) => {
2410
+ const result = await costSummarySchema.safeParseAsync(params);
2411
+ if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2412
+ const { startDate, endDate, groupBy, configId, variantId, environmentId, tags, tagKeys } = result.data;
2413
+ let baseQuery = db.selectFrom("llm_requests").where(kysely.sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col("createdAt")} <= ${endDate.toISOString()}`);
2414
+ if (configId) baseQuery = baseQuery.where("configId", "=", configId);
2415
+ if (variantId) baseQuery = baseQuery.where("variantId", "=", variantId);
2416
+ if (environmentId) baseQuery = baseQuery.where("environmentId", "=", environmentId);
2417
+ if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
2418
+ if (values.length === 0) continue;
2419
+ if (values.length === 1) baseQuery = baseQuery.where(kysely.sql`${col("tags")}->>${key} = ${values[0]}`);
2420
+ else {
2421
+ const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
2422
+ baseQuery = baseQuery.where(kysely.sql`${col("tags")}->>${key} IN (${valueList})`);
2423
+ }
2424
+ }
2425
+ switch (groupBy) {
2426
+ case "day": return baseQuery.select([
2427
+ kysely.sql`DATE(${col("createdAt")})`.as("groupKey"),
2428
+ kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2429
+ kysely.sql`COUNT(*)`.as("requestCount"),
2430
+ kysely.sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens")
2431
+ ]).groupBy(kysely.sql`DATE(${col("createdAt")})`).orderBy(kysely.sql`DATE(${col("createdAt")})`, "asc").execute();
2432
+ case "hour": return baseQuery.select([
2433
+ kysely.sql`DATE_TRUNC('hour', ${col("createdAt")})`.as("groupKey"),
2434
+ kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2435
+ kysely.sql`COUNT(*)`.as("requestCount"),
2436
+ kysely.sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens")
2437
+ ]).groupBy(kysely.sql`DATE_TRUNC('hour', ${col("createdAt")})`).orderBy(kysely.sql`DATE_TRUNC('hour', ${col("createdAt")})`, "asc").execute();
2438
+ case "model": return baseQuery.select([
2439
+ kysely.sql`${col("provider")} || '/' || ${col("model")}`.as("groupKey"),
2440
+ kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2441
+ kysely.sql`COUNT(*)`.as("requestCount")
2442
+ ]).groupBy(["provider", "model"]).orderBy(kysely.sql`SUM(${col("cost")})`, "desc").execute();
2443
+ case "provider": return baseQuery.select([
2444
+ kysely.sql`${col("provider")}`.as("groupKey"),
2445
+ kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2446
+ kysely.sql`COUNT(*)`.as("requestCount")
2447
+ ]).groupBy("provider").orderBy(kysely.sql`SUM(${col("cost")})`, "desc").execute();
2448
+ case "endpoint": return baseQuery.select([
2449
+ kysely.sql`COALESCE(${col("endpoint")}, 'unknown')`.as("groupKey"),
2450
+ kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2451
+ kysely.sql`COUNT(*)`.as("requestCount")
2452
+ ]).groupBy("endpoint").orderBy(kysely.sql`SUM(${col("cost")})`, "desc").execute();
2453
+ case "tags": {
2454
+ const conditions = [kysely.sql`${col("createdAt")} >= ${startDate.toISOString()}`, kysely.sql`${col("createdAt")} <= ${endDate.toISOString()}`];
2455
+ if (configId) conditions.push(kysely.sql`${col("configId")} = ${configId}`);
2456
+ if (variantId) conditions.push(kysely.sql`${col("variantId")} = ${variantId}`);
2457
+ if (environmentId) conditions.push(kysely.sql`${col("environmentId")} = ${environmentId}`);
2458
+ if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
2459
+ if (values.length === 0) continue;
2460
+ if (values.length === 1) conditions.push(kysely.sql`${col("tags")}->>${key} = ${values[0]}`);
2461
+ else {
2462
+ const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
2463
+ conditions.push(kysely.sql`${col("tags")}->>${key} IN (${valueList})`);
2464
+ }
2465
+ }
2466
+ if (tagKeys && tagKeys.length > 0) {
2467
+ const tagKeyList = kysely.sql.join(tagKeys.map((k) => kysely.sql`${k}`), kysely.sql`, `);
2468
+ conditions.push(kysely.sql`t.key IN (${tagKeyList})`);
2469
+ }
2470
+ const whereClause = kysely.sql.join(conditions, kysely.sql` AND `);
2471
+ return (await kysely.sql`
2472
+ SELECT t.key || ':' || t.value as "groupKey",
2473
+ COALESCE(SUM(${col("cost")}), 0) as "totalCost",
2474
+ COUNT(*) as "requestCount"
2475
+ FROM "llm_requests", jsonb_each_text(${col("tags")}) t
2476
+ WHERE ${whereClause}
2477
+ GROUP BY t.key, t.value
2478
+ ORDER BY SUM(${col("cost")}) DESC
2479
+ `.execute(db)).rows;
2480
+ }
2481
+ default: return baseQuery.select([
2482
+ kysely.sql`'total'`.as("groupKey"),
2483
+ kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2484
+ kysely.sql`COUNT(*)`.as("requestCount")
2485
+ ]).execute();
2486
+ }
2487
+ },
2488
+ getRequestStats: async (params) => {
2489
+ const result = await dateRangeSchema.safeParseAsync(params);
2490
+ if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2491
+ const { startDate, endDate, configId, variantId, environmentId, tags } = result.data;
2492
+ let query = db.selectFrom("llm_requests").select([
2493
+ kysely.sql`COUNT(*)`.as("totalRequests"),
2494
+ kysely.sql`COUNT(CASE WHEN ${col("statusCode")} >= 200 AND ${col("statusCode")} < 300 THEN 1 END)`.as("successfulRequests"),
2495
+ kysely.sql`COUNT(CASE WHEN ${col("statusCode")} >= 400 THEN 1 END)`.as("failedRequests"),
2496
+ kysely.sql`COUNT(CASE WHEN ${col("isStreaming")} = true THEN 1 END)`.as("streamingRequests"),
2497
+ kysely.sql`AVG(${col("latencyMs")})`.as("avgLatencyMs"),
2498
+ kysely.sql`MAX(${col("latencyMs")})`.as("maxLatencyMs"),
2499
+ kysely.sql`MIN(${col("latencyMs")})`.as("minLatencyMs")
2500
+ ]).where(kysely.sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col("createdAt")} <= ${endDate.toISOString()}`);
2501
+ if (configId) query = query.where("configId", "=", configId);
2502
+ if (variantId) query = query.where("variantId", "=", variantId);
2503
+ if (environmentId) query = query.where("environmentId", "=", environmentId);
2504
+ if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
2505
+ if (values.length === 0) continue;
2506
+ if (values.length === 1) query = query.where(kysely.sql`${col("tags")}->>${key} = ${values[0]}`);
2507
+ else {
2508
+ const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
2509
+ query = query.where(kysely.sql`${col("tags")}->>${key} IN (${valueList})`);
2510
+ }
2511
+ }
2512
+ return query.executeTakeFirst();
2513
+ },
2514
+ getDistinctTags: async () => {
2515
+ return (await kysely.sql`
2516
+ SELECT DISTINCT key, value
2517
+ FROM llm_requests, jsonb_each_text(tags) AS t(key, value)
2518
+ WHERE tags != '{}'::jsonb
2519
+ ORDER BY key, value
2520
+ `.execute(db)).rows;
2521
+ }
2522
+ };
2523
+ }
2524
+ function createTracesStore(db) {
2525
+ return {
2526
+ upsertTrace: async (data) => {
2527
+ const result = await upsertTraceSchema.safeParseAsync(data);
2528
+ if (!result.success) throw new LLMOpsError(`Invalid trace data: ${result.error.message}`);
2529
+ const trace = result.data;
2530
+ const now = (/* @__PURE__ */ new Date()).toISOString();
2531
+ await kysely.sql`
2532
+ INSERT INTO "traces" (
2533
+ "id", "traceId", "name", "sessionId", "userId", "status",
2534
+ "startTime", "endTime", "durationMs", "spanCount",
2535
+ "totalInputTokens", "totalOutputTokens", "totalTokens", "totalCost",
2536
+ "tags", "metadata", "createdAt", "updatedAt"
2537
+ ) VALUES (
2538
+ ${(0, node_crypto.randomUUID)()}, ${trace.traceId}, ${trace.name ?? null}, ${trace.sessionId ?? null},
2539
+ ${trace.userId ?? null}, ${trace.status},
2540
+ ${trace.startTime.toISOString()}, ${trace.endTime?.toISOString() ?? null},
2541
+ ${trace.durationMs ?? null}, ${trace.spanCount},
2542
+ ${trace.totalInputTokens}, ${trace.totalOutputTokens},
2543
+ ${trace.totalTokens}, ${trace.totalCost},
2544
+ ${JSON.stringify(trace.tags)}::jsonb, ${JSON.stringify(trace.metadata)}::jsonb,
2545
+ ${now}, ${now}
2546
+ )
2547
+ ON CONFLICT ("traceId") DO UPDATE SET
2548
+ "name" = COALESCE(EXCLUDED."name", "traces"."name"),
2549
+ "sessionId" = COALESCE(EXCLUDED."sessionId", "traces"."sessionId"),
2550
+ "userId" = COALESCE(EXCLUDED."userId", "traces"."userId"),
2551
+ "status" = CASE
2552
+ WHEN EXCLUDED."status" = 'error' THEN 'error'
2553
+ WHEN EXCLUDED."status" = 'ok' AND "traces"."status" != 'error' THEN 'ok'
2554
+ ELSE "traces"."status"
2555
+ END,
2556
+ "startTime" = LEAST("traces"."startTime", EXCLUDED."startTime"),
2557
+ "endTime" = GREATEST(
2558
+ COALESCE("traces"."endTime", EXCLUDED."endTime"),
2559
+ COALESCE(EXCLUDED."endTime", "traces"."endTime")
2560
+ ),
2561
+ "durationMs" = EXTRACT(EPOCH FROM (
2562
+ GREATEST(
2563
+ COALESCE("traces"."endTime", EXCLUDED."endTime"),
2564
+ COALESCE(EXCLUDED."endTime", "traces"."endTime")
2565
+ ) -
2566
+ LEAST("traces"."startTime", EXCLUDED."startTime")
2567
+ ))::integer * 1000,
2568
+ "spanCount" = "traces"."spanCount" + EXCLUDED."spanCount",
2569
+ "totalInputTokens" = "traces"."totalInputTokens" + EXCLUDED."totalInputTokens",
2570
+ "totalOutputTokens" = "traces"."totalOutputTokens" + EXCLUDED."totalOutputTokens",
2571
+ "totalTokens" = "traces"."totalTokens" + EXCLUDED."totalTokens",
2572
+ "totalCost" = "traces"."totalCost" + EXCLUDED."totalCost",
2573
+ "tags" = "traces"."tags" || EXCLUDED."tags",
2574
+ "metadata" = "traces"."metadata" || EXCLUDED."metadata",
2575
+ "updatedAt" = ${now}
2576
+ `.execute(db);
2577
+ },
2578
+ batchInsertSpans: async (spans) => {
2579
+ if (spans.length === 0) return { count: 0 };
2580
+ const validatedSpans = [];
2581
+ for (const span of spans) {
2582
+ const result = await insertSpanSchema.safeParseAsync(span);
2583
+ if (!result.success) {
2584
+ require_db.logger.warn(`[batchInsertSpans] Skipping invalid span ${span.spanId}: ${result.error.message}`);
2585
+ continue;
2586
+ }
2587
+ validatedSpans.push(result.data);
2588
+ }
2589
+ if (validatedSpans.length === 0) return { count: 0 };
2590
+ const now = (/* @__PURE__ */ new Date()).toISOString();
2591
+ const values = validatedSpans.map((span) => ({
2592
+ id: (0, node_crypto.randomUUID)(),
2593
+ traceId: span.traceId,
2594
+ spanId: span.spanId,
2595
+ parentSpanId: span.parentSpanId ?? null,
2596
+ name: span.name,
2597
+ kind: span.kind,
2598
+ status: span.status,
2599
+ statusMessage: span.statusMessage ?? null,
2600
+ startTime: span.startTime.toISOString(),
2601
+ endTime: span.endTime?.toISOString() ?? null,
2602
+ durationMs: span.durationMs ?? null,
2603
+ provider: span.provider ?? null,
2604
+ model: span.model ?? null,
2605
+ promptTokens: span.promptTokens,
2606
+ completionTokens: span.completionTokens,
2607
+ totalTokens: span.totalTokens,
2608
+ cost: span.cost,
2609
+ configId: span.configId ?? null,
2610
+ variantId: span.variantId ?? null,
2611
+ environmentId: span.environmentId ?? null,
2612
+ providerConfigId: span.providerConfigId ?? null,
2613
+ requestId: span.requestId ?? null,
2614
+ source: span.source,
2615
+ input: span.input != null ? JSON.stringify(span.input) : null,
2616
+ output: span.output != null ? JSON.stringify(span.output) : null,
2617
+ attributes: JSON.stringify(span.attributes),
2618
+ createdAt: now,
2619
+ updatedAt: now
2620
+ }));
2621
+ await db.insertInto("spans").values(values).onConflict((oc) => oc.column("spanId").doNothing()).execute();
2622
+ return { count: values.length };
2623
+ },
2624
+ batchInsertSpanEvents: async (events) => {
2625
+ if (events.length === 0) return { count: 0 };
2626
+ const validatedEvents = [];
2627
+ for (const event of events) {
2628
+ const result = await insertSpanEventSchema.safeParseAsync(event);
2629
+ if (!result.success) {
2630
+ require_db.logger.warn(`[batchInsertSpanEvents] Skipping invalid event: ${result.error.message}`);
2631
+ continue;
2632
+ }
2633
+ validatedEvents.push(result.data);
2634
+ }
2635
+ if (validatedEvents.length === 0) return { count: 0 };
2636
+ const now = (/* @__PURE__ */ new Date()).toISOString();
2637
+ const values = validatedEvents.map((event) => ({
2638
+ id: (0, node_crypto.randomUUID)(),
2639
+ traceId: event.traceId,
2640
+ spanId: event.spanId,
2641
+ name: event.name,
2642
+ timestamp: event.timestamp.toISOString(),
2643
+ attributes: JSON.stringify(event.attributes),
2644
+ createdAt: now
2645
+ }));
2646
+ await db.insertInto("span_events").values(values).execute();
2647
+ return { count: values.length };
2648
+ },
2649
+ listTraces: async (params) => {
2650
+ const result = await listTracesSchema.safeParseAsync(params || {});
2651
+ if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2652
+ const { limit, offset, sessionId, userId, status, name, startDate, endDate, tags } = result.data;
2653
+ let baseQuery = db.selectFrom("traces");
2654
+ if (sessionId) baseQuery = baseQuery.where("sessionId", "=", sessionId);
2655
+ if (userId) baseQuery = baseQuery.where("userId", "=", userId);
2656
+ if (status) baseQuery = baseQuery.where("status", "=", status);
2657
+ if (name) baseQuery = baseQuery.where(kysely.sql`${col("name")} ILIKE ${"%" + name + "%"}`);
2658
+ if (startDate) baseQuery = baseQuery.where(kysely.sql`${col("startTime")} >= ${startDate.toISOString()}`);
2659
+ if (endDate) baseQuery = baseQuery.where(kysely.sql`${col("startTime")} <= ${endDate.toISOString()}`);
2660
+ if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
2661
+ if (values.length === 0) continue;
2662
+ if (values.length === 1) baseQuery = baseQuery.where(kysely.sql`${col("tags")}->>${key} = ${values[0]}`);
2663
+ else {
2664
+ const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
2665
+ baseQuery = baseQuery.where(kysely.sql`${col("tags")}->>${key} IN (${valueList})`);
2666
+ }
2667
+ }
2668
+ const countResult = await baseQuery.select(kysely.sql`COUNT(*)`.as("total")).executeTakeFirst();
2669
+ const total = Number(countResult?.total ?? 0);
2670
+ return {
2671
+ data: await baseQuery.selectAll().orderBy("startTime", "desc").limit(limit).offset(offset).execute(),
2672
+ total,
2673
+ limit,
2674
+ offset
2675
+ };
2676
+ },
2677
+ getTraceWithSpans: async (traceId) => {
2678
+ const trace = await db.selectFrom("traces").selectAll().where("traceId", "=", traceId).executeTakeFirst();
2679
+ if (!trace) return void 0;
2680
+ return {
2681
+ trace,
2682
+ spans: await db.selectFrom("spans").selectAll().where("traceId", "=", traceId).orderBy("startTime", "asc").execute(),
2683
+ events: await db.selectFrom("span_events").selectAll().where("traceId", "=", traceId).orderBy("timestamp", "asc").execute()
2684
+ };
2685
+ },
2686
+ getTraceStats: async (params) => {
2687
+ const result = await traceStatsSchema.safeParseAsync(params);
2688
+ if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2689
+ const { startDate, endDate, sessionId, userId } = result.data;
2690
+ let query = db.selectFrom("traces").select([
2691
+ kysely.sql`COUNT(*)`.as("totalTraces"),
2692
+ kysely.sql`COALESCE(AVG(${col("durationMs")}), 0)`.as("avgDurationMs"),
2693
+ kysely.sql`COUNT(CASE WHEN ${col("status")} = 'error' THEN 1 END)`.as("errorCount"),
2694
+ kysely.sql`COALESCE(SUM(${col("totalCost")}), 0)`.as("totalCost"),
2695
+ kysely.sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
2696
+ kysely.sql`COALESCE(SUM(${col("spanCount")}), 0)`.as("totalSpans")
2697
+ ]).where(kysely.sql`${col("startTime")} >= ${startDate.toISOString()}`).where(kysely.sql`${col("startTime")} <= ${endDate.toISOString()}`);
2698
+ if (sessionId) query = query.where("sessionId", "=", sessionId);
2699
+ if (userId) query = query.where("userId", "=", userId);
2700
+ return query.executeTakeFirst();
2701
+ }
2702
+ };
2703
+ }
2753
2704
  /**
2754
2705
  * Create a PostgreSQL-backed telemetry store.
2755
2706
  *
2756
2707
  * Usage:
2757
2708
  * ```ts
2758
- * import { llmops, pgStore } from '@llmops/sdk'
2709
+ * import { llmops } from '@llmops/sdk'
2710
+ * import { pgStore } from '@llmops/sdk/store/pg'
2759
2711
  *
2760
2712
  * const ops = llmops({
2761
2713
  * telemetry: pgStore(process.env.DATABASE_URL),
@@ -2776,12 +2728,10 @@ function createPgStore(connectionString, options) {
2776
2728
  await connection.executeQuery(kysely.CompiledQuery.raw(`SET search_path TO "${schema}"`));
2777
2729
  }
2778
2730
  }) });
2779
- const llmRequests = createLLMRequestsDataLayer(db);
2780
- const traces = createTracesDataLayer(db);
2781
2731
  require_db.logger.debug(`pgStore: initialized with schema "${schema}"`);
2782
2732
  return {
2783
- ...llmRequests,
2784
- ...traces,
2733
+ ...createLLMRequestsStore(db),
2734
+ ...createTracesStore(db),
2785
2735
  _db: db
2786
2736
  };
2787
2737
  }
@@ -2813,13 +2763,11 @@ exports.createDataLayer = createDataLayer;
2813
2763
  exports.createDatabase = require_db.createDatabase;
2814
2764
  exports.createDatabaseFromConnection = require_db.createDatabaseFromConnection;
2815
2765
  exports.createDatasetsDataLayer = createDatasetsDataLayer;
2816
- exports.createLLMRequestsDataLayer = createLLMRequestsDataLayer;
2817
2766
  exports.createNeonDialect = require_neon_dialect.createNeonDialect;
2818
2767
  exports.createPgStore = createPgStore;
2819
2768
  exports.createPlaygroundDataLayer = createPlaygroundDataLayer;
2820
2769
  exports.createPlaygroundResultsDataLayer = createPlaygroundResultsDataLayer;
2821
2770
  exports.createPlaygroundRunsDataLayer = createPlaygroundRunsDataLayer;
2822
- exports.createTracesDataLayer = createTracesDataLayer;
2823
2771
  exports.createWorkspaceSettingsDataLayer = createWorkspaceSettingsDataLayer;
2824
2772
  exports.datasetRecordsSchema = require_db.datasetRecordsSchema;
2825
2773
  exports.datasetVersionRecordsSchema = require_db.datasetVersionRecordsSchema;