@llmops/core 1.0.0-beta.2 → 1.0.0-beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -1385,468 +1385,54 @@ const createDatasetsDataLayer = (db) => {
1385
1385
  };
1386
1386
 
1387
1387
  //#endregion
1388
- //#region src/datalayer/llmRequests.ts
1389
- /**
1390
- * Schema for individual guardrail result in telemetry
1391
- */
1392
- const guardrailResultSchema = zod_default.object({
1393
- checkId: zod_default.string(),
1394
- functionId: zod_default.string(),
1395
- hookType: zod_default.enum(["beforeRequestHook", "afterRequestHook"]),
1396
- verdict: zod_default.boolean(),
1397
- latencyMs: zod_default.number()
1398
- });
1399
- /**
1400
- * Schema for guardrail results aggregate
1401
- */
1402
- const guardrailResultsSchema = zod_default.object({
1403
- results: zod_default.array(guardrailResultSchema),
1404
- action: zod_default.enum([
1405
- "allowed",
1406
- "blocked",
1407
- "logged"
1408
- ]),
1409
- totalLatencyMs: zod_default.number()
1410
- });
1411
- /**
1412
- * Schema for inserting a new LLM request log
1413
- */
1414
- const insertLLMRequestSchema = zod_default.object({
1415
- requestId: zod_default.string().uuid(),
1416
- configId: zod_default.string().uuid().nullable().optional(),
1417
- variantId: zod_default.string().uuid().nullable().optional(),
1418
- environmentId: zod_default.string().uuid().nullable().optional(),
1419
- providerConfigId: zod_default.string().uuid().nullable().optional(),
1420
- provider: zod_default.string(),
1421
- model: zod_default.string(),
1422
- promptTokens: zod_default.number().int().default(0),
1423
- completionTokens: zod_default.number().int().default(0),
1424
- totalTokens: zod_default.number().int().default(0),
1425
- cachedTokens: zod_default.number().int().default(0),
1426
- cacheCreationTokens: zod_default.number().int().default(0),
1427
- cost: zod_default.number().int().default(0),
1428
- cacheSavings: zod_default.number().int().default(0),
1429
- inputCost: zod_default.number().int().default(0),
1430
- outputCost: zod_default.number().int().default(0),
1431
- endpoint: zod_default.string(),
1432
- statusCode: zod_default.number().int(),
1433
- latencyMs: zod_default.number().int().default(0),
1434
- isStreaming: zod_default.boolean().default(false),
1435
- userId: zod_default.string().nullable().optional(),
1436
- tags: zod_default.record(zod_default.string(), zod_default.string()).default({}),
1437
- guardrailResults: guardrailResultsSchema.nullable().optional(),
1438
- traceId: zod_default.string().nullable().optional(),
1439
- spanId: zod_default.string().nullable().optional(),
1440
- parentSpanId: zod_default.string().nullable().optional(),
1441
- sessionId: zod_default.string().nullable().optional()
1442
- });
1443
- /**
1444
- * Schema for listing LLM requests
1445
- */
1446
- const listRequestsSchema = zod_default.object({
1447
- limit: zod_default.number().int().positive().max(1e3).default(100),
1448
- offset: zod_default.number().int().nonnegative().default(0),
1449
- configId: zod_default.string().uuid().optional(),
1450
- variantId: zod_default.string().uuid().optional(),
1451
- environmentId: zod_default.string().uuid().optional(),
1452
- providerConfigId: zod_default.string().uuid().optional(),
1453
- provider: zod_default.string().optional(),
1454
- model: zod_default.string().optional(),
1455
- startDate: zod_default.date().optional(),
1456
- endDate: zod_default.date().optional(),
1457
- tags: zod_default.record(zod_default.string(), zod_default.array(zod_default.string())).optional()
1388
+ //#region src/datalayer/playgrounds.ts
1389
+ const createNewPlayground = zod_default.object({
1390
+ name: zod_default.string(),
1391
+ datasetId: zod_default.string().uuid().nullable().optional(),
1392
+ columns: zod_default.array(playgroundColumnSchema).nullable().optional()
1458
1393
  });
1459
- /**
1460
- * Schema for date range queries with optional filters
1461
- */
1462
- const dateRangeSchema = zod_default.object({
1463
- startDate: zod_default.date(),
1464
- endDate: zod_default.date(),
1465
- configId: zod_default.string().uuid().optional(),
1466
- variantId: zod_default.string().uuid().optional(),
1467
- environmentId: zod_default.string().uuid().optional(),
1468
- tags: zod_default.record(zod_default.string(), zod_default.array(zod_default.string())).optional()
1394
+ const updatePlayground = zod_default.object({
1395
+ playgroundId: zod_default.uuidv4(),
1396
+ name: zod_default.string().optional(),
1397
+ datasetId: zod_default.string().uuid().nullable().optional(),
1398
+ columns: zod_default.array(playgroundColumnSchema).nullable().optional()
1469
1399
  });
1470
- /**
1471
- * Valid groupBy values for cost summary queries
1472
- */
1473
- const COST_SUMMARY_GROUP_BY = [
1474
- "day",
1475
- "hour",
1476
- "model",
1477
- "provider",
1478
- "endpoint",
1479
- "tags"
1480
- ];
1481
- /**
1482
- * Schema for cost summary with grouping
1483
- */
1484
- const costSummarySchema = zod_default.object({
1485
- startDate: zod_default.date(),
1486
- endDate: zod_default.date(),
1487
- configId: zod_default.string().uuid().optional(),
1488
- variantId: zod_default.string().uuid().optional(),
1489
- environmentId: zod_default.string().uuid().optional(),
1490
- tags: zod_default.record(zod_default.string(), zod_default.array(zod_default.string())).optional(),
1491
- groupBy: zod_default.enum(COST_SUMMARY_GROUP_BY).optional(),
1492
- tagKeys: zod_default.array(zod_default.string()).optional()
1400
+ const getPlaygroundById = zod_default.object({ playgroundId: zod_default.uuidv4() });
1401
+ const deletePlayground = zod_default.object({ playgroundId: zod_default.uuidv4() });
1402
+ const listPlaygrounds = zod_default.object({
1403
+ limit: zod_default.number().int().positive().optional(),
1404
+ offset: zod_default.number().int().nonnegative().optional()
1493
1405
  });
1494
- /**
1495
- * Helper to create column reference for SQL
1496
- * Uses sql.ref() to properly quote column names for the database
1497
- */
1498
- const col$1 = (name) => sql.ref(name);
1499
- const createLLMRequestsDataLayer = (db) => {
1406
+ const createPlaygroundDataLayer = (db) => {
1500
1407
  return {
1501
- batchInsertRequests: async (requests) => {
1502
- if (requests.length === 0) return { count: 0 };
1503
- const validatedRequests = await Promise.all(requests.map(async (req) => {
1504
- const result = await insertLLMRequestSchema.safeParseAsync(req);
1505
- if (!result.success) throw new LLMOpsError(`Invalid request data: ${result.error.message}`);
1506
- return result.data;
1507
- }));
1508
- const now = (/* @__PURE__ */ new Date()).toISOString();
1509
- const values = validatedRequests.map((req) => ({
1510
- id: randomUUID(),
1511
- requestId: req.requestId,
1512
- configId: req.configId ?? null,
1513
- variantId: req.variantId ?? null,
1514
- environmentId: req.environmentId ?? null,
1515
- providerConfigId: req.providerConfigId ?? null,
1516
- provider: req.provider,
1517
- model: req.model,
1518
- promptTokens: req.promptTokens,
1519
- completionTokens: req.completionTokens,
1520
- totalTokens: req.totalTokens,
1521
- cachedTokens: req.cachedTokens,
1522
- cacheCreationTokens: req.cacheCreationTokens,
1523
- cost: req.cost,
1524
- cacheSavings: req.cacheSavings,
1525
- inputCost: req.inputCost,
1526
- outputCost: req.outputCost,
1527
- endpoint: req.endpoint,
1528
- statusCode: req.statusCode,
1529
- latencyMs: req.latencyMs,
1530
- isStreaming: req.isStreaming,
1531
- userId: req.userId ?? null,
1532
- tags: JSON.stringify(req.tags),
1533
- guardrailResults: req.guardrailResults ? JSON.stringify(req.guardrailResults) : null,
1534
- traceId: req.traceId ?? null,
1535
- spanId: req.spanId ?? null,
1536
- parentSpanId: req.parentSpanId ?? null,
1537
- sessionId: req.sessionId ?? null,
1538
- createdAt: now,
1539
- updatedAt: now
1540
- }));
1541
- await db.insertInto("llm_requests").values(values).execute();
1542
- return { count: values.length };
1543
- },
1544
- insertRequest: async (request) => {
1545
- const result = await insertLLMRequestSchema.safeParseAsync(request);
1546
- if (!result.success) throw new LLMOpsError(`Invalid request data: ${result.error.message}`);
1547
- const req = result.data;
1548
- const now = (/* @__PURE__ */ new Date()).toISOString();
1549
- return db.insertInto("llm_requests").values({
1408
+ createNewPlayground: async (params) => {
1409
+ const value = await createNewPlayground.safeParseAsync(params);
1410
+ if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
1411
+ const { name, datasetId, columns } = value.data;
1412
+ return db.insertInto("playgrounds").values({
1550
1413
  id: randomUUID(),
1551
- requestId: req.requestId,
1552
- configId: req.configId ?? null,
1553
- variantId: req.variantId ?? null,
1554
- environmentId: req.environmentId ?? null,
1555
- providerConfigId: req.providerConfigId ?? null,
1556
- provider: req.provider,
1557
- model: req.model,
1558
- promptTokens: req.promptTokens,
1559
- completionTokens: req.completionTokens,
1560
- totalTokens: req.totalTokens,
1561
- cachedTokens: req.cachedTokens,
1562
- cacheCreationTokens: req.cacheCreationTokens,
1563
- cost: req.cost,
1564
- cacheSavings: req.cacheSavings,
1565
- inputCost: req.inputCost,
1566
- outputCost: req.outputCost,
1567
- endpoint: req.endpoint,
1568
- statusCode: req.statusCode,
1569
- latencyMs: req.latencyMs,
1570
- isStreaming: req.isStreaming,
1571
- userId: req.userId ?? null,
1572
- tags: JSON.stringify(req.tags),
1573
- guardrailResults: req.guardrailResults ? JSON.stringify(req.guardrailResults) : null,
1574
- traceId: req.traceId ?? null,
1575
- spanId: req.spanId ?? null,
1576
- parentSpanId: req.parentSpanId ?? null,
1577
- sessionId: req.sessionId ?? null,
1578
- createdAt: now,
1579
- updatedAt: now
1414
+ name,
1415
+ datasetId: datasetId ?? null,
1416
+ columns: columns ? JSON.stringify(columns) : null,
1417
+ createdAt: (/* @__PURE__ */ new Date()).toISOString(),
1418
+ updatedAt: (/* @__PURE__ */ new Date()).toISOString()
1580
1419
  }).returningAll().executeTakeFirst();
1581
1420
  },
1582
- listRequests: async (params) => {
1583
- const result = await listRequestsSchema.safeParseAsync(params || {});
1584
- if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
1585
- const { limit, offset, configId, variantId, environmentId, providerConfigId, provider, model, startDate, endDate, tags } = result.data;
1586
- let baseQuery = db.selectFrom("llm_requests");
1587
- if (configId) baseQuery = baseQuery.where("configId", "=", configId);
1588
- if (variantId) baseQuery = baseQuery.where("variantId", "=", variantId);
1589
- if (environmentId) baseQuery = baseQuery.where("environmentId", "=", environmentId);
1590
- if (providerConfigId) baseQuery = baseQuery.where("providerConfigId", "=", providerConfigId);
1591
- if (provider) baseQuery = baseQuery.where("provider", "=", provider);
1592
- if (model) baseQuery = baseQuery.where("model", "=", model);
1593
- if (startDate) baseQuery = baseQuery.where(sql`${col$1("createdAt")} >= ${startDate.toISOString()}`);
1594
- if (endDate) baseQuery = baseQuery.where(sql`${col$1("createdAt")} <= ${endDate.toISOString()}`);
1595
- if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
1596
- if (values.length === 0) continue;
1597
- if (values.length === 1) baseQuery = baseQuery.where(sql`${col$1("tags")}->>${key} = ${values[0]}`);
1598
- else {
1599
- const valueList = sql.join(values.map((v) => sql`${v}`));
1600
- baseQuery = baseQuery.where(sql`${col$1("tags")}->>${key} IN (${valueList})`);
1601
- }
1602
- }
1603
- const countResult = await baseQuery.select(sql`COUNT(*)`.as("total")).executeTakeFirst();
1604
- const total = Number(countResult?.total ?? 0);
1605
- return {
1606
- data: await baseQuery.selectAll().orderBy("createdAt", "desc").limit(limit).offset(offset).execute(),
1607
- total,
1608
- limit,
1609
- offset
1610
- };
1421
+ updatePlayground: async (params) => {
1422
+ const value = await updatePlayground.safeParseAsync(params);
1423
+ if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
1424
+ const { playgroundId, name, datasetId, columns } = value.data;
1425
+ const updateData = { updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
1426
+ if (name !== void 0) updateData.name = name;
1427
+ if (datasetId !== void 0) updateData.datasetId = datasetId;
1428
+ if (columns !== void 0) updateData.columns = columns ? JSON.stringify(columns) : null;
1429
+ return db.updateTable("playgrounds").set(updateData).where("id", "=", playgroundId).returningAll().executeTakeFirst();
1611
1430
  },
1612
- getRequestByRequestId: async (requestId) => {
1613
- return db.selectFrom("llm_requests").selectAll().where("requestId", "=", requestId).executeTakeFirst();
1614
- },
1615
- getTotalCost: async (params) => {
1616
- const result = await dateRangeSchema.safeParseAsync(params);
1617
- if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
1618
- const { startDate, endDate, configId, variantId, environmentId, tags } = result.data;
1619
- let query = db.selectFrom("llm_requests").select([
1620
- sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1621
- sql`COALESCE(SUM(${col$1("inputCost")}), 0)`.as("totalInputCost"),
1622
- sql`COALESCE(SUM(${col$1("outputCost")}), 0)`.as("totalOutputCost"),
1623
- sql`COALESCE(SUM(${col$1("promptTokens")}), 0)`.as("totalPromptTokens"),
1624
- sql`COALESCE(SUM(${col$1("completionTokens")}), 0)`.as("totalCompletionTokens"),
1625
- sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens"),
1626
- sql`COALESCE(SUM(${col$1("cachedTokens")}), 0)`.as("totalCachedTokens"),
1627
- sql`COALESCE(SUM(${col$1("cacheSavings")}), 0)`.as("totalCacheSavings"),
1628
- sql`COUNT(*)`.as("requestCount")
1629
- ]).where(sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col$1("createdAt")} <= ${endDate.toISOString()}`);
1630
- if (configId) query = query.where("configId", "=", configId);
1631
- if (variantId) query = query.where("variantId", "=", variantId);
1632
- if (environmentId) query = query.where("environmentId", "=", environmentId);
1633
- if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
1634
- if (values.length === 0) continue;
1635
- if (values.length === 1) query = query.where(sql`${col$1("tags")}->>${key} = ${values[0]}`);
1636
- else {
1637
- const valueList = sql.join(values.map((v) => sql`${v}`));
1638
- query = query.where(sql`${col$1("tags")}->>${key} IN (${valueList})`);
1639
- }
1640
- }
1641
- return await query.executeTakeFirst();
1642
- },
1643
- getCostByModel: async (params) => {
1644
- const result = await dateRangeSchema.safeParseAsync(params);
1645
- if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
1646
- const { startDate, endDate } = result.data;
1647
- return db.selectFrom("llm_requests").select([
1648
- "provider",
1649
- "model",
1650
- sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1651
- sql`COALESCE(SUM(${col$1("inputCost")}), 0)`.as("totalInputCost"),
1652
- sql`COALESCE(SUM(${col$1("outputCost")}), 0)`.as("totalOutputCost"),
1653
- sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens"),
1654
- sql`COUNT(*)`.as("requestCount"),
1655
- sql`AVG(${col$1("latencyMs")})`.as("avgLatencyMs")
1656
- ]).where(sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col$1("createdAt")} <= ${endDate.toISOString()}`).groupBy(["provider", "model"]).orderBy(sql`SUM(${col$1("cost")})`, "desc").execute();
1657
- },
1658
- getCostByProvider: async (params) => {
1659
- const result = await dateRangeSchema.safeParseAsync(params);
1660
- if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
1661
- const { startDate, endDate } = result.data;
1662
- return db.selectFrom("llm_requests").select([
1663
- "provider",
1664
- sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1665
- sql`COALESCE(SUM(${col$1("inputCost")}), 0)`.as("totalInputCost"),
1666
- sql`COALESCE(SUM(${col$1("outputCost")}), 0)`.as("totalOutputCost"),
1667
- sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens"),
1668
- sql`COUNT(*)`.as("requestCount"),
1669
- sql`AVG(${col$1("latencyMs")})`.as("avgLatencyMs")
1670
- ]).where(sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col$1("createdAt")} <= ${endDate.toISOString()}`).groupBy("provider").orderBy(sql`SUM(${col$1("cost")})`, "desc").execute();
1671
- },
1672
- getDailyCosts: async (params) => {
1673
- const result = await dateRangeSchema.safeParseAsync(params);
1674
- if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
1675
- const { startDate, endDate } = result.data;
1676
- return db.selectFrom("llm_requests").select([
1677
- sql`DATE(${col$1("createdAt")})`.as("date"),
1678
- sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1679
- sql`COALESCE(SUM(${col$1("inputCost")}), 0)`.as("totalInputCost"),
1680
- sql`COALESCE(SUM(${col$1("outputCost")}), 0)`.as("totalOutputCost"),
1681
- sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens"),
1682
- sql`COUNT(*)`.as("requestCount")
1683
- ]).where(sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col$1("createdAt")} <= ${endDate.toISOString()}`).groupBy(sql`DATE(${col$1("createdAt")})`).orderBy(sql`DATE(${col$1("createdAt")})`, "asc").execute();
1684
- },
1685
- getCostSummary: async (params) => {
1686
- const result = await costSummarySchema.safeParseAsync(params);
1687
- if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
1688
- const { startDate, endDate, groupBy, configId, variantId, environmentId, tags, tagKeys } = result.data;
1689
- let baseQuery = db.selectFrom("llm_requests").where(sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col$1("createdAt")} <= ${endDate.toISOString()}`);
1690
- if (configId) baseQuery = baseQuery.where("configId", "=", configId);
1691
- if (variantId) baseQuery = baseQuery.where("variantId", "=", variantId);
1692
- if (environmentId) baseQuery = baseQuery.where("environmentId", "=", environmentId);
1693
- if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
1694
- if (values.length === 0) continue;
1695
- if (values.length === 1) baseQuery = baseQuery.where(sql`${col$1("tags")}->>${key} = ${values[0]}`);
1696
- else {
1697
- const valueList = sql.join(values.map((v) => sql`${v}`));
1698
- baseQuery = baseQuery.where(sql`${col$1("tags")}->>${key} IN (${valueList})`);
1699
- }
1700
- }
1701
- switch (groupBy) {
1702
- case "day": return baseQuery.select([
1703
- sql`DATE(${col$1("createdAt")})`.as("groupKey"),
1704
- sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1705
- sql`COUNT(*)`.as("requestCount"),
1706
- sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens")
1707
- ]).groupBy(sql`DATE(${col$1("createdAt")})`).orderBy(sql`DATE(${col$1("createdAt")})`, "asc").execute();
1708
- case "hour": return baseQuery.select([
1709
- sql`DATE_TRUNC('hour', ${col$1("createdAt")})`.as("groupKey"),
1710
- sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1711
- sql`COUNT(*)`.as("requestCount"),
1712
- sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens")
1713
- ]).groupBy(sql`DATE_TRUNC('hour', ${col$1("createdAt")})`).orderBy(sql`DATE_TRUNC('hour', ${col$1("createdAt")})`, "asc").execute();
1714
- case "model": return baseQuery.select([
1715
- sql`${col$1("provider")} || '/' || ${col$1("model")}`.as("groupKey"),
1716
- sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1717
- sql`COUNT(*)`.as("requestCount")
1718
- ]).groupBy(["provider", "model"]).orderBy(sql`SUM(${col$1("cost")})`, "desc").execute();
1719
- case "provider": return baseQuery.select([
1720
- sql`${col$1("provider")}`.as("groupKey"),
1721
- sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1722
- sql`COUNT(*)`.as("requestCount")
1723
- ]).groupBy("provider").orderBy(sql`SUM(${col$1("cost")})`, "desc").execute();
1724
- case "endpoint": return baseQuery.select([
1725
- sql`COALESCE(${col$1("endpoint")}, 'unknown')`.as("groupKey"),
1726
- sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1727
- sql`COUNT(*)`.as("requestCount")
1728
- ]).groupBy("endpoint").orderBy(sql`SUM(${col$1("cost")})`, "desc").execute();
1729
- case "tags": {
1730
- const conditions = [sql`${col$1("createdAt")} >= ${startDate.toISOString()}`, sql`${col$1("createdAt")} <= ${endDate.toISOString()}`];
1731
- if (configId) conditions.push(sql`${col$1("configId")} = ${configId}`);
1732
- if (variantId) conditions.push(sql`${col$1("variantId")} = ${variantId}`);
1733
- if (environmentId) conditions.push(sql`${col$1("environmentId")} = ${environmentId}`);
1734
- if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
1735
- if (values.length === 0) continue;
1736
- if (values.length === 1) conditions.push(sql`${col$1("tags")}->>${key} = ${values[0]}`);
1737
- else {
1738
- const valueList = sql.join(values.map((v) => sql`${v}`));
1739
- conditions.push(sql`${col$1("tags")}->>${key} IN (${valueList})`);
1740
- }
1741
- }
1742
- if (tagKeys && tagKeys.length > 0) {
1743
- const tagKeyList = sql.join(tagKeys.map((k) => sql`${k}`), sql`, `);
1744
- conditions.push(sql`t.key IN (${tagKeyList})`);
1745
- }
1746
- const whereClause = sql.join(conditions, sql` AND `);
1747
- return (await sql`
1748
- SELECT t.key || ':' || t.value as "groupKey",
1749
- COALESCE(SUM(${col$1("cost")}), 0) as "totalCost",
1750
- COUNT(*) as "requestCount"
1751
- FROM "llm_requests", jsonb_each_text(${col$1("tags")}) t
1752
- WHERE ${whereClause}
1753
- GROUP BY t.key, t.value
1754
- ORDER BY SUM(${col$1("cost")}) DESC
1755
- `.execute(db)).rows;
1756
- }
1757
- default: return baseQuery.select([
1758
- sql`'total'`.as("groupKey"),
1759
- sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
1760
- sql`COUNT(*)`.as("requestCount")
1761
- ]).execute();
1762
- }
1763
- },
1764
- getRequestStats: async (params) => {
1765
- const result = await dateRangeSchema.safeParseAsync(params);
1766
- if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
1767
- const { startDate, endDate, configId, variantId, environmentId, tags } = result.data;
1768
- let query = db.selectFrom("llm_requests").select([
1769
- sql`COUNT(*)`.as("totalRequests"),
1770
- sql`COUNT(CASE WHEN ${col$1("statusCode")} >= 200 AND ${col$1("statusCode")} < 300 THEN 1 END)`.as("successfulRequests"),
1771
- sql`COUNT(CASE WHEN ${col$1("statusCode")} >= 400 THEN 1 END)`.as("failedRequests"),
1772
- sql`COUNT(CASE WHEN ${col$1("isStreaming")} = true THEN 1 END)`.as("streamingRequests"),
1773
- sql`AVG(${col$1("latencyMs")})`.as("avgLatencyMs"),
1774
- sql`MAX(${col$1("latencyMs")})`.as("maxLatencyMs"),
1775
- sql`MIN(${col$1("latencyMs")})`.as("minLatencyMs")
1776
- ]).where(sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col$1("createdAt")} <= ${endDate.toISOString()}`);
1777
- if (configId) query = query.where("configId", "=", configId);
1778
- if (variantId) query = query.where("variantId", "=", variantId);
1779
- if (environmentId) query = query.where("environmentId", "=", environmentId);
1780
- if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
1781
- if (values.length === 0) continue;
1782
- if (values.length === 1) query = query.where(sql`${col$1("tags")}->>${key} = ${values[0]}`);
1783
- else {
1784
- const valueList = sql.join(values.map((v) => sql`${v}`));
1785
- query = query.where(sql`${col$1("tags")}->>${key} IN (${valueList})`);
1786
- }
1787
- }
1788
- return await query.executeTakeFirst();
1789
- },
1790
- getDistinctTags: async () => {
1791
- return (await sql`
1792
- SELECT DISTINCT key, value
1793
- FROM llm_requests, jsonb_each_text(tags) AS t(key, value)
1794
- WHERE tags != '{}'::jsonb
1795
- ORDER BY key, value
1796
- `.execute(db)).rows;
1797
- }
1798
- };
1799
- };
1800
-
1801
- //#endregion
1802
- //#region src/datalayer/playgrounds.ts
1803
- const createNewPlayground = zod_default.object({
1804
- name: zod_default.string(),
1805
- datasetId: zod_default.string().uuid().nullable().optional(),
1806
- columns: zod_default.array(playgroundColumnSchema).nullable().optional()
1807
- });
1808
- const updatePlayground = zod_default.object({
1809
- playgroundId: zod_default.uuidv4(),
1810
- name: zod_default.string().optional(),
1811
- datasetId: zod_default.string().uuid().nullable().optional(),
1812
- columns: zod_default.array(playgroundColumnSchema).nullable().optional()
1813
- });
1814
- const getPlaygroundById = zod_default.object({ playgroundId: zod_default.uuidv4() });
1815
- const deletePlayground = zod_default.object({ playgroundId: zod_default.uuidv4() });
1816
- const listPlaygrounds = zod_default.object({
1817
- limit: zod_default.number().int().positive().optional(),
1818
- offset: zod_default.number().int().nonnegative().optional()
1819
- });
1820
- const createPlaygroundDataLayer = (db) => {
1821
- return {
1822
- createNewPlayground: async (params) => {
1823
- const value = await createNewPlayground.safeParseAsync(params);
1824
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
1825
- const { name, datasetId, columns } = value.data;
1826
- return db.insertInto("playgrounds").values({
1827
- id: randomUUID(),
1828
- name,
1829
- datasetId: datasetId ?? null,
1830
- columns: columns ? JSON.stringify(columns) : null,
1831
- createdAt: (/* @__PURE__ */ new Date()).toISOString(),
1832
- updatedAt: (/* @__PURE__ */ new Date()).toISOString()
1833
- }).returningAll().executeTakeFirst();
1834
- },
1835
- updatePlayground: async (params) => {
1836
- const value = await updatePlayground.safeParseAsync(params);
1837
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
1838
- const { playgroundId, name, datasetId, columns } = value.data;
1839
- const updateData = { updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
1840
- if (name !== void 0) updateData.name = name;
1841
- if (datasetId !== void 0) updateData.datasetId = datasetId;
1842
- if (columns !== void 0) updateData.columns = columns ? JSON.stringify(columns) : null;
1843
- return db.updateTable("playgrounds").set(updateData).where("id", "=", playgroundId).returningAll().executeTakeFirst();
1844
- },
1845
- getPlaygroundById: async (params) => {
1846
- const value = await getPlaygroundById.safeParseAsync(params);
1847
- if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
1848
- const { playgroundId } = value.data;
1849
- return db.selectFrom("playgrounds").selectAll().where("id", "=", playgroundId).executeTakeFirst();
1431
+ getPlaygroundById: async (params) => {
1432
+ const value = await getPlaygroundById.safeParseAsync(params);
1433
+ if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
1434
+ const { playgroundId } = value.data;
1435
+ return db.selectFrom("playgrounds").selectAll().where("id", "=", playgroundId).executeTakeFirst();
1850
1436
  },
1851
1437
  deletePlayground: async (params) => {
1852
1438
  const value = await deletePlayground.safeParseAsync(params);
@@ -2093,292 +1679,13 @@ const createPlaygroundRunsDataLayer = (db) => {
2093
1679
  };
2094
1680
 
2095
1681
  //#endregion
2096
- //#region src/datalayer/traces.ts
2097
- const col = (name) => sql.ref(name);
2098
- /**
2099
- * Schema for upserting a trace
2100
- */
2101
- const upsertTraceSchema = zod_default.object({
2102
- traceId: zod_default.string(),
1682
+ //#region src/datalayer/workspaceSettings.ts
1683
+ const updateWorkspaceSettings = zod_default.object({
2103
1684
  name: zod_default.string().nullable().optional(),
2104
- sessionId: zod_default.string().nullable().optional(),
2105
- userId: zod_default.string().nullable().optional(),
2106
- status: zod_default.enum([
2107
- "unset",
2108
- "ok",
2109
- "error"
2110
- ]).default("unset"),
2111
- startTime: zod_default.date(),
2112
- endTime: zod_default.date().nullable().optional(),
2113
- durationMs: zod_default.number().int().nullable().optional(),
2114
- spanCount: zod_default.number().int().default(1),
2115
- totalInputTokens: zod_default.number().int().default(0),
2116
- totalOutputTokens: zod_default.number().int().default(0),
2117
- totalTokens: zod_default.number().int().default(0),
2118
- totalCost: zod_default.number().int().default(0),
2119
- tags: zod_default.record(zod_default.string(), zod_default.string()).default({}),
2120
- metadata: zod_default.record(zod_default.string(), zod_default.unknown()).default({})
2121
- });
2122
- /**
2123
- * Schema for inserting spans
2124
- */
2125
- const insertSpanSchema = zod_default.object({
2126
- traceId: zod_default.string(),
2127
- spanId: zod_default.string(),
2128
- parentSpanId: zod_default.string().nullable().optional(),
2129
- name: zod_default.string(),
2130
- kind: zod_default.number().int().default(1),
2131
- status: zod_default.number().int().default(0),
2132
- statusMessage: zod_default.string().nullable().optional(),
2133
- startTime: zod_default.date(),
2134
- endTime: zod_default.date().nullable().optional(),
2135
- durationMs: zod_default.number().int().nullable().optional(),
2136
- provider: zod_default.string().nullable().optional(),
2137
- model: zod_default.string().nullable().optional(),
2138
- promptTokens: zod_default.number().int().default(0),
2139
- completionTokens: zod_default.number().int().default(0),
2140
- totalTokens: zod_default.number().int().default(0),
2141
- cost: zod_default.number().int().default(0),
2142
- configId: zod_default.string().uuid().nullable().optional(),
2143
- variantId: zod_default.string().uuid().nullable().optional(),
2144
- environmentId: zod_default.string().uuid().nullable().optional(),
2145
- providerConfigId: zod_default.string().uuid().nullable().optional(),
2146
- requestId: zod_default.string().uuid().nullable().optional(),
2147
- source: zod_default.enum([
2148
- "gateway",
2149
- "otlp",
2150
- "langsmith"
2151
- ]).default("gateway"),
2152
- input: zod_default.unknown().nullable().optional(),
2153
- output: zod_default.unknown().nullable().optional(),
2154
- attributes: zod_default.record(zod_default.string(), zod_default.unknown()).default({})
2155
- });
2156
- /**
2157
- * Schema for inserting span events
2158
- */
2159
- const insertSpanEventSchema = zod_default.object({
2160
- traceId: zod_default.string(),
2161
- spanId: zod_default.string(),
2162
- name: zod_default.string(),
2163
- timestamp: zod_default.date(),
2164
- attributes: zod_default.record(zod_default.string(), zod_default.unknown()).default({})
2165
- });
2166
- /**
2167
- * Schema for listing traces
2168
- */
2169
- const listTracesSchema = zod_default.object({
2170
- limit: zod_default.number().int().positive().max(1e3).default(50),
2171
- offset: zod_default.number().int().nonnegative().default(0),
2172
- sessionId: zod_default.string().optional(),
2173
- userId: zod_default.string().optional(),
2174
- status: zod_default.enum([
2175
- "unset",
2176
- "ok",
2177
- "error"
2178
- ]).optional(),
2179
- name: zod_default.string().optional(),
2180
- startDate: zod_default.date().optional(),
2181
- endDate: zod_default.date().optional(),
2182
- tags: zod_default.record(zod_default.string(), zod_default.array(zod_default.string())).optional()
2183
- });
2184
- /**
2185
- * Schema for trace stats query
2186
- */
2187
- const traceStatsSchema = zod_default.object({
2188
- startDate: zod_default.date(),
2189
- endDate: zod_default.date(),
2190
- sessionId: zod_default.string().optional(),
2191
- userId: zod_default.string().optional()
1685
+ setupComplete: zod_default.boolean().optional(),
1686
+ superAdminId: zod_default.string().nullable().optional()
2192
1687
  });
2193
- const createTracesDataLayer = (db) => {
2194
- return {
2195
- upsertTrace: async (data) => {
2196
- const result = await upsertTraceSchema.safeParseAsync(data);
2197
- if (!result.success) throw new LLMOpsError(`Invalid trace data: ${result.error.message}`);
2198
- const trace = result.data;
2199
- const now = (/* @__PURE__ */ new Date()).toISOString();
2200
- await sql`
2201
- INSERT INTO "traces" (
2202
- "id", "traceId", "name", "sessionId", "userId", "status",
2203
- "startTime", "endTime", "durationMs", "spanCount",
2204
- "totalInputTokens", "totalOutputTokens", "totalTokens", "totalCost",
2205
- "tags", "metadata", "createdAt", "updatedAt"
2206
- ) VALUES (
2207
- ${randomUUID()}, ${trace.traceId}, ${trace.name ?? null}, ${trace.sessionId ?? null},
2208
- ${trace.userId ?? null}, ${trace.status},
2209
- ${trace.startTime.toISOString()}, ${trace.endTime?.toISOString() ?? null},
2210
- ${trace.durationMs ?? null}, ${trace.spanCount},
2211
- ${trace.totalInputTokens}, ${trace.totalOutputTokens},
2212
- ${trace.totalTokens}, ${trace.totalCost},
2213
- ${JSON.stringify(trace.tags)}::jsonb, ${JSON.stringify(trace.metadata)}::jsonb,
2214
- ${now}, ${now}
2215
- )
2216
- ON CONFLICT ("traceId") DO UPDATE SET
2217
- "name" = COALESCE(EXCLUDED."name", "traces"."name"),
2218
- "sessionId" = COALESCE(EXCLUDED."sessionId", "traces"."sessionId"),
2219
- "userId" = COALESCE(EXCLUDED."userId", "traces"."userId"),
2220
- "status" = CASE
2221
- WHEN EXCLUDED."status" = 'error' THEN 'error'
2222
- WHEN EXCLUDED."status" = 'ok' AND "traces"."status" != 'error' THEN 'ok'
2223
- ELSE "traces"."status"
2224
- END,
2225
- "startTime" = LEAST("traces"."startTime", EXCLUDED."startTime"),
2226
- "endTime" = GREATEST(
2227
- COALESCE("traces"."endTime", EXCLUDED."endTime"),
2228
- COALESCE(EXCLUDED."endTime", "traces"."endTime")
2229
- ),
2230
- "durationMs" = EXTRACT(EPOCH FROM (
2231
- GREATEST(
2232
- COALESCE("traces"."endTime", EXCLUDED."endTime"),
2233
- COALESCE(EXCLUDED."endTime", "traces"."endTime")
2234
- ) -
2235
- LEAST("traces"."startTime", EXCLUDED."startTime")
2236
- ))::integer * 1000,
2237
- "spanCount" = "traces"."spanCount" + EXCLUDED."spanCount",
2238
- "totalInputTokens" = "traces"."totalInputTokens" + EXCLUDED."totalInputTokens",
2239
- "totalOutputTokens" = "traces"."totalOutputTokens" + EXCLUDED."totalOutputTokens",
2240
- "totalTokens" = "traces"."totalTokens" + EXCLUDED."totalTokens",
2241
- "totalCost" = "traces"."totalCost" + EXCLUDED."totalCost",
2242
- "tags" = "traces"."tags" || EXCLUDED."tags",
2243
- "metadata" = "traces"."metadata" || EXCLUDED."metadata",
2244
- "updatedAt" = ${now}
2245
- `.execute(db);
2246
- },
2247
- batchInsertSpans: async (spans) => {
2248
- if (spans.length === 0) return { count: 0 };
2249
- const validatedSpans = [];
2250
- for (const span of spans) {
2251
- const result = await insertSpanSchema.safeParseAsync(span);
2252
- if (!result.success) {
2253
- logger.warn(`[batchInsertSpans] Skipping invalid span ${span.spanId}: ${result.error.message}`);
2254
- continue;
2255
- }
2256
- validatedSpans.push(result.data);
2257
- }
2258
- if (validatedSpans.length === 0) return { count: 0 };
2259
- const now = (/* @__PURE__ */ new Date()).toISOString();
2260
- const values = validatedSpans.map((span) => ({
2261
- id: randomUUID(),
2262
- traceId: span.traceId,
2263
- spanId: span.spanId,
2264
- parentSpanId: span.parentSpanId ?? null,
2265
- name: span.name,
2266
- kind: span.kind,
2267
- status: span.status,
2268
- statusMessage: span.statusMessage ?? null,
2269
- startTime: span.startTime.toISOString(),
2270
- endTime: span.endTime?.toISOString() ?? null,
2271
- durationMs: span.durationMs ?? null,
2272
- provider: span.provider ?? null,
2273
- model: span.model ?? null,
2274
- promptTokens: span.promptTokens,
2275
- completionTokens: span.completionTokens,
2276
- totalTokens: span.totalTokens,
2277
- cost: span.cost,
2278
- configId: span.configId ?? null,
2279
- variantId: span.variantId ?? null,
2280
- environmentId: span.environmentId ?? null,
2281
- providerConfigId: span.providerConfigId ?? null,
2282
- requestId: span.requestId ?? null,
2283
- source: span.source,
2284
- input: span.input != null ? JSON.stringify(span.input) : null,
2285
- output: span.output != null ? JSON.stringify(span.output) : null,
2286
- attributes: JSON.stringify(span.attributes),
2287
- createdAt: now,
2288
- updatedAt: now
2289
- }));
2290
- await db.insertInto("spans").values(values).onConflict((oc) => oc.column("spanId").doNothing()).execute();
2291
- return { count: values.length };
2292
- },
2293
- batchInsertSpanEvents: async (events) => {
2294
- if (events.length === 0) return { count: 0 };
2295
- const validatedEvents = [];
2296
- for (const event of events) {
2297
- const result = await insertSpanEventSchema.safeParseAsync(event);
2298
- if (!result.success) {
2299
- logger.warn(`[batchInsertSpanEvents] Skipping invalid event: ${result.error.message}`);
2300
- continue;
2301
- }
2302
- validatedEvents.push(result.data);
2303
- }
2304
- if (validatedEvents.length === 0) return { count: 0 };
2305
- const now = (/* @__PURE__ */ new Date()).toISOString();
2306
- const values = validatedEvents.map((event) => ({
2307
- id: randomUUID(),
2308
- traceId: event.traceId,
2309
- spanId: event.spanId,
2310
- name: event.name,
2311
- timestamp: event.timestamp.toISOString(),
2312
- attributes: JSON.stringify(event.attributes),
2313
- createdAt: now
2314
- }));
2315
- await db.insertInto("span_events").values(values).execute();
2316
- return { count: values.length };
2317
- },
2318
- listTraces: async (params) => {
2319
- const result = await listTracesSchema.safeParseAsync(params || {});
2320
- if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2321
- const { limit, offset, sessionId, userId, status, name, startDate, endDate, tags } = result.data;
2322
- let baseQuery = db.selectFrom("traces");
2323
- if (sessionId) baseQuery = baseQuery.where("sessionId", "=", sessionId);
2324
- if (userId) baseQuery = baseQuery.where("userId", "=", userId);
2325
- if (status) baseQuery = baseQuery.where("status", "=", status);
2326
- if (name) baseQuery = baseQuery.where(sql`${col("name")} ILIKE ${"%" + name + "%"}`);
2327
- if (startDate) baseQuery = baseQuery.where(sql`${col("startTime")} >= ${startDate.toISOString()}`);
2328
- if (endDate) baseQuery = baseQuery.where(sql`${col("startTime")} <= ${endDate.toISOString()}`);
2329
- if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
2330
- if (values.length === 0) continue;
2331
- if (values.length === 1) baseQuery = baseQuery.where(sql`${col("tags")}->>${key} = ${values[0]}`);
2332
- else {
2333
- const valueList = sql.join(values.map((v) => sql`${v}`));
2334
- baseQuery = baseQuery.where(sql`${col("tags")}->>${key} IN (${valueList})`);
2335
- }
2336
- }
2337
- const countResult = await baseQuery.select(sql`COUNT(*)`.as("total")).executeTakeFirst();
2338
- const total = Number(countResult?.total ?? 0);
2339
- return {
2340
- data: await baseQuery.selectAll().orderBy("startTime", "desc").limit(limit).offset(offset).execute(),
2341
- total,
2342
- limit,
2343
- offset
2344
- };
2345
- },
2346
- getTraceWithSpans: async (traceId) => {
2347
- const trace = await db.selectFrom("traces").selectAll().where("traceId", "=", traceId).executeTakeFirst();
2348
- if (!trace) return void 0;
2349
- return {
2350
- trace,
2351
- spans: await db.selectFrom("spans").selectAll().where("traceId", "=", traceId).orderBy("startTime", "asc").execute(),
2352
- events: await db.selectFrom("span_events").selectAll().where("traceId", "=", traceId).orderBy("timestamp", "asc").execute()
2353
- };
2354
- },
2355
- getTraceStats: async (params) => {
2356
- const result = await traceStatsSchema.safeParseAsync(params);
2357
- if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2358
- const { startDate, endDate, sessionId, userId } = result.data;
2359
- let query = db.selectFrom("traces").select([
2360
- sql`COUNT(*)`.as("totalTraces"),
2361
- sql`COALESCE(AVG(${col("durationMs")}), 0)`.as("avgDurationMs"),
2362
- sql`COUNT(CASE WHEN ${col("status")} = 'error' THEN 1 END)`.as("errorCount"),
2363
- sql`COALESCE(SUM(${col("totalCost")}), 0)`.as("totalCost"),
2364
- sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
2365
- sql`COALESCE(SUM(${col("spanCount")}), 0)`.as("totalSpans")
2366
- ]).where(sql`${col("startTime")} >= ${startDate.toISOString()}`).where(sql`${col("startTime")} <= ${endDate.toISOString()}`);
2367
- if (sessionId) query = query.where("sessionId", "=", sessionId);
2368
- if (userId) query = query.where("userId", "=", userId);
2369
- return query.executeTakeFirst();
2370
- }
2371
- };
2372
- };
2373
-
2374
- //#endregion
2375
- //#region src/datalayer/workspaceSettings.ts
2376
- const updateWorkspaceSettings = zod_default.object({
2377
- name: zod_default.string().nullable().optional(),
2378
- setupComplete: zod_default.boolean().optional(),
2379
- superAdminId: zod_default.string().nullable().optional()
2380
- });
2381
- const createWorkspaceSettingsDataLayer = (db) => {
1688
+ const createWorkspaceSettingsDataLayer = (db) => {
2382
1689
  return {
2383
1690
  getWorkspaceSettings: async () => {
2384
1691
  let settings = await db.selectFrom("workspace_settings").selectAll().executeTakeFirst();
@@ -2464,11 +1771,9 @@ const createWorkspaceSettingsDataLayer = (db) => {
2464
1771
  function createDataLayer(db) {
2465
1772
  return {
2466
1773
  ...createDatasetsDataLayer(db),
2467
- ...createLLMRequestsDataLayer(db),
2468
1774
  ...createPlaygroundDataLayer(db),
2469
1775
  ...createPlaygroundResultsDataLayer(db),
2470
1776
  ...createPlaygroundRunsDataLayer(db),
2471
- ...createTracesDataLayer(db),
2472
1777
  ...createWorkspaceSettingsDataLayer(db)
2473
1778
  };
2474
1779
  }
@@ -2748,12 +2053,659 @@ function getDefaultPricingProvider() {
2748
2053
 
2749
2054
  //#endregion
2750
2055
  //#region src/telemetry/postgres.ts
2056
+ const guardrailResultSchema = zod_default.object({
2057
+ checkId: zod_default.string(),
2058
+ functionId: zod_default.string(),
2059
+ hookType: zod_default.enum(["beforeRequestHook", "afterRequestHook"]),
2060
+ verdict: zod_default.boolean(),
2061
+ latencyMs: zod_default.number()
2062
+ });
2063
+ const guardrailResultsSchema = zod_default.object({
2064
+ results: zod_default.array(guardrailResultSchema),
2065
+ action: zod_default.enum([
2066
+ "allowed",
2067
+ "blocked",
2068
+ "logged"
2069
+ ]),
2070
+ totalLatencyMs: zod_default.number()
2071
+ });
2072
+ const insertLLMRequestSchema = zod_default.object({
2073
+ requestId: zod_default.string().uuid(),
2074
+ configId: zod_default.string().uuid().nullable().optional(),
2075
+ variantId: zod_default.string().uuid().nullable().optional(),
2076
+ environmentId: zod_default.string().uuid().nullable().optional(),
2077
+ providerConfigId: zod_default.string().uuid().nullable().optional(),
2078
+ provider: zod_default.string(),
2079
+ model: zod_default.string(),
2080
+ promptTokens: zod_default.number().int().default(0),
2081
+ completionTokens: zod_default.number().int().default(0),
2082
+ totalTokens: zod_default.number().int().default(0),
2083
+ cachedTokens: zod_default.number().int().default(0),
2084
+ cacheCreationTokens: zod_default.number().int().default(0),
2085
+ cost: zod_default.number().int().default(0),
2086
+ cacheSavings: zod_default.number().int().default(0),
2087
+ inputCost: zod_default.number().int().default(0),
2088
+ outputCost: zod_default.number().int().default(0),
2089
+ endpoint: zod_default.string(),
2090
+ statusCode: zod_default.number().int(),
2091
+ latencyMs: zod_default.number().int().default(0),
2092
+ isStreaming: zod_default.boolean().default(false),
2093
+ userId: zod_default.string().nullable().optional(),
2094
+ tags: zod_default.record(zod_default.string(), zod_default.string()).default({}),
2095
+ guardrailResults: guardrailResultsSchema.nullable().optional(),
2096
+ traceId: zod_default.string().nullable().optional(),
2097
+ spanId: zod_default.string().nullable().optional(),
2098
+ parentSpanId: zod_default.string().nullable().optional(),
2099
+ sessionId: zod_default.string().nullable().optional()
2100
+ });
2101
+ const listRequestsSchema = zod_default.object({
2102
+ limit: zod_default.number().int().positive().max(1e3).default(100),
2103
+ offset: zod_default.number().int().nonnegative().default(0),
2104
+ configId: zod_default.string().uuid().optional(),
2105
+ variantId: zod_default.string().uuid().optional(),
2106
+ environmentId: zod_default.string().uuid().optional(),
2107
+ providerConfigId: zod_default.string().uuid().optional(),
2108
+ provider: zod_default.string().optional(),
2109
+ model: zod_default.string().optional(),
2110
+ startDate: zod_default.date().optional(),
2111
+ endDate: zod_default.date().optional(),
2112
+ tags: zod_default.record(zod_default.string(), zod_default.array(zod_default.string())).optional()
2113
+ });
2114
+ const dateRangeSchema = zod_default.object({
2115
+ startDate: zod_default.date(),
2116
+ endDate: zod_default.date(),
2117
+ configId: zod_default.string().uuid().optional(),
2118
+ variantId: zod_default.string().uuid().optional(),
2119
+ environmentId: zod_default.string().uuid().optional(),
2120
+ tags: zod_default.record(zod_default.string(), zod_default.array(zod_default.string())).optional()
2121
+ });
2122
+ const COST_SUMMARY_GROUP_BY = [
2123
+ "day",
2124
+ "hour",
2125
+ "model",
2126
+ "provider",
2127
+ "endpoint",
2128
+ "tags"
2129
+ ];
2130
+ const costSummarySchema = zod_default.object({
2131
+ startDate: zod_default.date(),
2132
+ endDate: zod_default.date(),
2133
+ configId: zod_default.string().uuid().optional(),
2134
+ variantId: zod_default.string().uuid().optional(),
2135
+ environmentId: zod_default.string().uuid().optional(),
2136
+ tags: zod_default.record(zod_default.string(), zod_default.array(zod_default.string())).optional(),
2137
+ groupBy: zod_default.enum(COST_SUMMARY_GROUP_BY).optional(),
2138
+ tagKeys: zod_default.array(zod_default.string()).optional()
2139
+ });
2140
+ const upsertTraceSchema = zod_default.object({
2141
+ traceId: zod_default.string(),
2142
+ name: zod_default.string().nullable().optional(),
2143
+ sessionId: zod_default.string().nullable().optional(),
2144
+ userId: zod_default.string().nullable().optional(),
2145
+ status: zod_default.enum([
2146
+ "unset",
2147
+ "ok",
2148
+ "error"
2149
+ ]).default("unset"),
2150
+ startTime: zod_default.date(),
2151
+ endTime: zod_default.date().nullable().optional(),
2152
+ durationMs: zod_default.number().int().nullable().optional(),
2153
+ spanCount: zod_default.number().int().default(1),
2154
+ totalInputTokens: zod_default.number().int().default(0),
2155
+ totalOutputTokens: zod_default.number().int().default(0),
2156
+ totalTokens: zod_default.number().int().default(0),
2157
+ totalCost: zod_default.number().int().default(0),
2158
+ tags: zod_default.record(zod_default.string(), zod_default.string()).default({}),
2159
+ metadata: zod_default.record(zod_default.string(), zod_default.unknown()).default({})
2160
+ });
2161
+ const insertSpanSchema = zod_default.object({
2162
+ traceId: zod_default.string(),
2163
+ spanId: zod_default.string(),
2164
+ parentSpanId: zod_default.string().nullable().optional(),
2165
+ name: zod_default.string(),
2166
+ kind: zod_default.number().int().default(1),
2167
+ status: zod_default.number().int().default(0),
2168
+ statusMessage: zod_default.string().nullable().optional(),
2169
+ startTime: zod_default.date(),
2170
+ endTime: zod_default.date().nullable().optional(),
2171
+ durationMs: zod_default.number().int().nullable().optional(),
2172
+ provider: zod_default.string().nullable().optional(),
2173
+ model: zod_default.string().nullable().optional(),
2174
+ promptTokens: zod_default.number().int().default(0),
2175
+ completionTokens: zod_default.number().int().default(0),
2176
+ totalTokens: zod_default.number().int().default(0),
2177
+ cost: zod_default.number().int().default(0),
2178
+ configId: zod_default.string().uuid().nullable().optional(),
2179
+ variantId: zod_default.string().uuid().nullable().optional(),
2180
+ environmentId: zod_default.string().uuid().nullable().optional(),
2181
+ providerConfigId: zod_default.string().uuid().nullable().optional(),
2182
+ requestId: zod_default.string().uuid().nullable().optional(),
2183
+ source: zod_default.enum([
2184
+ "gateway",
2185
+ "otlp",
2186
+ "langsmith"
2187
+ ]).default("gateway"),
2188
+ input: zod_default.unknown().nullable().optional(),
2189
+ output: zod_default.unknown().nullable().optional(),
2190
+ attributes: zod_default.record(zod_default.string(), zod_default.unknown()).default({})
2191
+ });
2192
+ const insertSpanEventSchema = zod_default.object({
2193
+ traceId: zod_default.string(),
2194
+ spanId: zod_default.string(),
2195
+ name: zod_default.string(),
2196
+ timestamp: zod_default.date(),
2197
+ attributes: zod_default.record(zod_default.string(), zod_default.unknown()).default({})
2198
+ });
2199
+ const listTracesSchema = zod_default.object({
2200
+ limit: zod_default.number().int().positive().max(1e3).default(50),
2201
+ offset: zod_default.number().int().nonnegative().default(0),
2202
+ sessionId: zod_default.string().optional(),
2203
+ userId: zod_default.string().optional(),
2204
+ status: zod_default.enum([
2205
+ "unset",
2206
+ "ok",
2207
+ "error"
2208
+ ]).optional(),
2209
+ name: zod_default.string().optional(),
2210
+ startDate: zod_default.date().optional(),
2211
+ endDate: zod_default.date().optional(),
2212
+ tags: zod_default.record(zod_default.string(), zod_default.array(zod_default.string())).optional()
2213
+ });
2214
+ const traceStatsSchema = zod_default.object({
2215
+ startDate: zod_default.date(),
2216
+ endDate: zod_default.date(),
2217
+ sessionId: zod_default.string().optional(),
2218
+ userId: zod_default.string().optional()
2219
+ });
2220
+ const col = (name) => sql.ref(name);
2221
+ function createLLMRequestsStore(db) {
2222
+ return {
2223
+ batchInsertRequests: async (requests) => {
2224
+ if (requests.length === 0) return { count: 0 };
2225
+ const validatedRequests = await Promise.all(requests.map(async (req) => {
2226
+ const result = await insertLLMRequestSchema.safeParseAsync(req);
2227
+ if (!result.success) throw new LLMOpsError(`Invalid request data: ${result.error.message}`);
2228
+ return result.data;
2229
+ }));
2230
+ const now = (/* @__PURE__ */ new Date()).toISOString();
2231
+ const values = validatedRequests.map((req) => ({
2232
+ id: randomUUID(),
2233
+ requestId: req.requestId,
2234
+ configId: req.configId ?? null,
2235
+ variantId: req.variantId ?? null,
2236
+ environmentId: req.environmentId ?? null,
2237
+ providerConfigId: req.providerConfigId ?? null,
2238
+ provider: req.provider,
2239
+ model: req.model,
2240
+ promptTokens: req.promptTokens,
2241
+ completionTokens: req.completionTokens,
2242
+ totalTokens: req.totalTokens,
2243
+ cachedTokens: req.cachedTokens,
2244
+ cacheCreationTokens: req.cacheCreationTokens,
2245
+ cost: req.cost,
2246
+ cacheSavings: req.cacheSavings,
2247
+ inputCost: req.inputCost,
2248
+ outputCost: req.outputCost,
2249
+ endpoint: req.endpoint,
2250
+ statusCode: req.statusCode,
2251
+ latencyMs: req.latencyMs,
2252
+ isStreaming: req.isStreaming,
2253
+ userId: req.userId ?? null,
2254
+ tags: JSON.stringify(req.tags),
2255
+ guardrailResults: req.guardrailResults ? JSON.stringify(req.guardrailResults) : null,
2256
+ traceId: req.traceId ?? null,
2257
+ spanId: req.spanId ?? null,
2258
+ parentSpanId: req.parentSpanId ?? null,
2259
+ sessionId: req.sessionId ?? null,
2260
+ createdAt: now,
2261
+ updatedAt: now
2262
+ }));
2263
+ await db.insertInto("llm_requests").values(values).execute();
2264
+ return { count: values.length };
2265
+ },
2266
+ insertRequest: async (request) => {
2267
+ const result = await insertLLMRequestSchema.safeParseAsync(request);
2268
+ if (!result.success) throw new LLMOpsError(`Invalid request data: ${result.error.message}`);
2269
+ const req = result.data;
2270
+ const now = (/* @__PURE__ */ new Date()).toISOString();
2271
+ return db.insertInto("llm_requests").values({
2272
+ id: randomUUID(),
2273
+ requestId: req.requestId,
2274
+ configId: req.configId ?? null,
2275
+ variantId: req.variantId ?? null,
2276
+ environmentId: req.environmentId ?? null,
2277
+ providerConfigId: req.providerConfigId ?? null,
2278
+ provider: req.provider,
2279
+ model: req.model,
2280
+ promptTokens: req.promptTokens,
2281
+ completionTokens: req.completionTokens,
2282
+ totalTokens: req.totalTokens,
2283
+ cachedTokens: req.cachedTokens,
2284
+ cacheCreationTokens: req.cacheCreationTokens,
2285
+ cost: req.cost,
2286
+ cacheSavings: req.cacheSavings,
2287
+ inputCost: req.inputCost,
2288
+ outputCost: req.outputCost,
2289
+ endpoint: req.endpoint,
2290
+ statusCode: req.statusCode,
2291
+ latencyMs: req.latencyMs,
2292
+ isStreaming: req.isStreaming,
2293
+ userId: req.userId ?? null,
2294
+ tags: JSON.stringify(req.tags),
2295
+ guardrailResults: req.guardrailResults ? JSON.stringify(req.guardrailResults) : null,
2296
+ traceId: req.traceId ?? null,
2297
+ spanId: req.spanId ?? null,
2298
+ parentSpanId: req.parentSpanId ?? null,
2299
+ sessionId: req.sessionId ?? null,
2300
+ createdAt: now,
2301
+ updatedAt: now
2302
+ }).returningAll().executeTakeFirst();
2303
+ },
2304
+ listRequests: async (params) => {
2305
+ const result = await listRequestsSchema.safeParseAsync(params || {});
2306
+ if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2307
+ const { limit, offset, configId, variantId, environmentId, providerConfigId, provider, model, startDate, endDate, tags } = result.data;
2308
+ let baseQuery = db.selectFrom("llm_requests");
2309
+ if (configId) baseQuery = baseQuery.where("configId", "=", configId);
2310
+ if (variantId) baseQuery = baseQuery.where("variantId", "=", variantId);
2311
+ if (environmentId) baseQuery = baseQuery.where("environmentId", "=", environmentId);
2312
+ if (providerConfigId) baseQuery = baseQuery.where("providerConfigId", "=", providerConfigId);
2313
+ if (provider) baseQuery = baseQuery.where("provider", "=", provider);
2314
+ if (model) baseQuery = baseQuery.where("model", "=", model);
2315
+ if (startDate) baseQuery = baseQuery.where(sql`${col("createdAt")} >= ${startDate.toISOString()}`);
2316
+ if (endDate) baseQuery = baseQuery.where(sql`${col("createdAt")} <= ${endDate.toISOString()}`);
2317
+ if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
2318
+ if (values.length === 0) continue;
2319
+ if (values.length === 1) baseQuery = baseQuery.where(sql`${col("tags")}->>${key} = ${values[0]}`);
2320
+ else {
2321
+ const valueList = sql.join(values.map((v) => sql`${v}`));
2322
+ baseQuery = baseQuery.where(sql`${col("tags")}->>${key} IN (${valueList})`);
2323
+ }
2324
+ }
2325
+ const countResult = await baseQuery.select(sql`COUNT(*)`.as("total")).executeTakeFirst();
2326
+ const total = Number(countResult?.total ?? 0);
2327
+ return {
2328
+ data: await baseQuery.selectAll().orderBy("createdAt", "desc").limit(limit).offset(offset).execute(),
2329
+ total,
2330
+ limit,
2331
+ offset
2332
+ };
2333
+ },
2334
+ getRequestByRequestId: async (requestId) => {
2335
+ return db.selectFrom("llm_requests").selectAll().where("requestId", "=", requestId).executeTakeFirst();
2336
+ },
2337
+ getTotalCost: async (params) => {
2338
+ const result = await dateRangeSchema.safeParseAsync(params);
2339
+ if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2340
+ const { startDate, endDate, configId, variantId, environmentId, tags } = result.data;
2341
+ let query = db.selectFrom("llm_requests").select([
2342
+ sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2343
+ sql`COALESCE(SUM(${col("inputCost")}), 0)`.as("totalInputCost"),
2344
+ sql`COALESCE(SUM(${col("outputCost")}), 0)`.as("totalOutputCost"),
2345
+ sql`COALESCE(SUM(${col("promptTokens")}), 0)`.as("totalPromptTokens"),
2346
+ sql`COALESCE(SUM(${col("completionTokens")}), 0)`.as("totalCompletionTokens"),
2347
+ sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
2348
+ sql`COALESCE(SUM(${col("cachedTokens")}), 0)`.as("totalCachedTokens"),
2349
+ sql`COALESCE(SUM(${col("cacheSavings")}), 0)`.as("totalCacheSavings"),
2350
+ sql`COUNT(*)`.as("requestCount")
2351
+ ]).where(sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col("createdAt")} <= ${endDate.toISOString()}`);
2352
+ if (configId) query = query.where("configId", "=", configId);
2353
+ if (variantId) query = query.where("variantId", "=", variantId);
2354
+ if (environmentId) query = query.where("environmentId", "=", environmentId);
2355
+ if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
2356
+ if (values.length === 0) continue;
2357
+ if (values.length === 1) query = query.where(sql`${col("tags")}->>${key} = ${values[0]}`);
2358
+ else {
2359
+ const valueList = sql.join(values.map((v) => sql`${v}`));
2360
+ query = query.where(sql`${col("tags")}->>${key} IN (${valueList})`);
2361
+ }
2362
+ }
2363
+ return query.executeTakeFirst();
2364
+ },
2365
+ getCostByModel: async (params) => {
2366
+ const result = await dateRangeSchema.safeParseAsync(params);
2367
+ if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2368
+ const { startDate, endDate } = result.data;
2369
+ return db.selectFrom("llm_requests").select([
2370
+ "provider",
2371
+ "model",
2372
+ sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2373
+ sql`COALESCE(SUM(${col("inputCost")}), 0)`.as("totalInputCost"),
2374
+ sql`COALESCE(SUM(${col("outputCost")}), 0)`.as("totalOutputCost"),
2375
+ sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
2376
+ sql`COUNT(*)`.as("requestCount"),
2377
+ sql`AVG(${col("latencyMs")})`.as("avgLatencyMs")
2378
+ ]).where(sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col("createdAt")} <= ${endDate.toISOString()}`).groupBy(["provider", "model"]).orderBy(sql`SUM(${col("cost")})`, "desc").execute();
2379
+ },
2380
+ getCostByProvider: async (params) => {
2381
+ const result = await dateRangeSchema.safeParseAsync(params);
2382
+ if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2383
+ const { startDate, endDate } = result.data;
2384
+ return db.selectFrom("llm_requests").select([
2385
+ "provider",
2386
+ sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2387
+ sql`COALESCE(SUM(${col("inputCost")}), 0)`.as("totalInputCost"),
2388
+ sql`COALESCE(SUM(${col("outputCost")}), 0)`.as("totalOutputCost"),
2389
+ sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
2390
+ sql`COUNT(*)`.as("requestCount"),
2391
+ sql`AVG(${col("latencyMs")})`.as("avgLatencyMs")
2392
+ ]).where(sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col("createdAt")} <= ${endDate.toISOString()}`).groupBy("provider").orderBy(sql`SUM(${col("cost")})`, "desc").execute();
2393
+ },
2394
+ getDailyCosts: async (params) => {
2395
+ const result = await dateRangeSchema.safeParseAsync(params);
2396
+ if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2397
+ const { startDate, endDate } = result.data;
2398
+ return db.selectFrom("llm_requests").select([
2399
+ sql`DATE(${col("createdAt")})`.as("date"),
2400
+ sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2401
+ sql`COALESCE(SUM(${col("inputCost")}), 0)`.as("totalInputCost"),
2402
+ sql`COALESCE(SUM(${col("outputCost")}), 0)`.as("totalOutputCost"),
2403
+ sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
2404
+ sql`COUNT(*)`.as("requestCount")
2405
+ ]).where(sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col("createdAt")} <= ${endDate.toISOString()}`).groupBy(sql`DATE(${col("createdAt")})`).orderBy(sql`DATE(${col("createdAt")})`, "asc").execute();
2406
+ },
2407
+ getCostSummary: async (params) => {
2408
+ const result = await costSummarySchema.safeParseAsync(params);
2409
+ if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2410
+ const { startDate, endDate, groupBy, configId, variantId, environmentId, tags, tagKeys } = result.data;
2411
+ let baseQuery = db.selectFrom("llm_requests").where(sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col("createdAt")} <= ${endDate.toISOString()}`);
2412
+ if (configId) baseQuery = baseQuery.where("configId", "=", configId);
2413
+ if (variantId) baseQuery = baseQuery.where("variantId", "=", variantId);
2414
+ if (environmentId) baseQuery = baseQuery.where("environmentId", "=", environmentId);
2415
+ if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
2416
+ if (values.length === 0) continue;
2417
+ if (values.length === 1) baseQuery = baseQuery.where(sql`${col("tags")}->>${key} = ${values[0]}`);
2418
+ else {
2419
+ const valueList = sql.join(values.map((v) => sql`${v}`));
2420
+ baseQuery = baseQuery.where(sql`${col("tags")}->>${key} IN (${valueList})`);
2421
+ }
2422
+ }
2423
+ switch (groupBy) {
2424
+ case "day": return baseQuery.select([
2425
+ sql`DATE(${col("createdAt")})`.as("groupKey"),
2426
+ sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2427
+ sql`COUNT(*)`.as("requestCount"),
2428
+ sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens")
2429
+ ]).groupBy(sql`DATE(${col("createdAt")})`).orderBy(sql`DATE(${col("createdAt")})`, "asc").execute();
2430
+ case "hour": return baseQuery.select([
2431
+ sql`DATE_TRUNC('hour', ${col("createdAt")})`.as("groupKey"),
2432
+ sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2433
+ sql`COUNT(*)`.as("requestCount"),
2434
+ sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens")
2435
+ ]).groupBy(sql`DATE_TRUNC('hour', ${col("createdAt")})`).orderBy(sql`DATE_TRUNC('hour', ${col("createdAt")})`, "asc").execute();
2436
+ case "model": return baseQuery.select([
2437
+ sql`${col("provider")} || '/' || ${col("model")}`.as("groupKey"),
2438
+ sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2439
+ sql`COUNT(*)`.as("requestCount")
2440
+ ]).groupBy(["provider", "model"]).orderBy(sql`SUM(${col("cost")})`, "desc").execute();
2441
+ case "provider": return baseQuery.select([
2442
+ sql`${col("provider")}`.as("groupKey"),
2443
+ sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2444
+ sql`COUNT(*)`.as("requestCount")
2445
+ ]).groupBy("provider").orderBy(sql`SUM(${col("cost")})`, "desc").execute();
2446
+ case "endpoint": return baseQuery.select([
2447
+ sql`COALESCE(${col("endpoint")}, 'unknown')`.as("groupKey"),
2448
+ sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2449
+ sql`COUNT(*)`.as("requestCount")
2450
+ ]).groupBy("endpoint").orderBy(sql`SUM(${col("cost")})`, "desc").execute();
2451
+ case "tags": {
2452
+ const conditions = [sql`${col("createdAt")} >= ${startDate.toISOString()}`, sql`${col("createdAt")} <= ${endDate.toISOString()}`];
2453
+ if (configId) conditions.push(sql`${col("configId")} = ${configId}`);
2454
+ if (variantId) conditions.push(sql`${col("variantId")} = ${variantId}`);
2455
+ if (environmentId) conditions.push(sql`${col("environmentId")} = ${environmentId}`);
2456
+ if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
2457
+ if (values.length === 0) continue;
2458
+ if (values.length === 1) conditions.push(sql`${col("tags")}->>${key} = ${values[0]}`);
2459
+ else {
2460
+ const valueList = sql.join(values.map((v) => sql`${v}`));
2461
+ conditions.push(sql`${col("tags")}->>${key} IN (${valueList})`);
2462
+ }
2463
+ }
2464
+ if (tagKeys && tagKeys.length > 0) {
2465
+ const tagKeyList = sql.join(tagKeys.map((k) => sql`${k}`), sql`, `);
2466
+ conditions.push(sql`t.key IN (${tagKeyList})`);
2467
+ }
2468
+ const whereClause = sql.join(conditions, sql` AND `);
2469
+ return (await sql`
2470
+ SELECT t.key || ':' || t.value as "groupKey",
2471
+ COALESCE(SUM(${col("cost")}), 0) as "totalCost",
2472
+ COUNT(*) as "requestCount"
2473
+ FROM "llm_requests", jsonb_each_text(${col("tags")}) t
2474
+ WHERE ${whereClause}
2475
+ GROUP BY t.key, t.value
2476
+ ORDER BY SUM(${col("cost")}) DESC
2477
+ `.execute(db)).rows;
2478
+ }
2479
+ default: return baseQuery.select([
2480
+ sql`'total'`.as("groupKey"),
2481
+ sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
2482
+ sql`COUNT(*)`.as("requestCount")
2483
+ ]).execute();
2484
+ }
2485
+ },
2486
+ getRequestStats: async (params) => {
2487
+ const result = await dateRangeSchema.safeParseAsync(params);
2488
+ if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2489
+ const { startDate, endDate, configId, variantId, environmentId, tags } = result.data;
2490
+ let query = db.selectFrom("llm_requests").select([
2491
+ sql`COUNT(*)`.as("totalRequests"),
2492
+ sql`COUNT(CASE WHEN ${col("statusCode")} >= 200 AND ${col("statusCode")} < 300 THEN 1 END)`.as("successfulRequests"),
2493
+ sql`COUNT(CASE WHEN ${col("statusCode")} >= 400 THEN 1 END)`.as("failedRequests"),
2494
+ sql`COUNT(CASE WHEN ${col("isStreaming")} = true THEN 1 END)`.as("streamingRequests"),
2495
+ sql`AVG(${col("latencyMs")})`.as("avgLatencyMs"),
2496
+ sql`MAX(${col("latencyMs")})`.as("maxLatencyMs"),
2497
+ sql`MIN(${col("latencyMs")})`.as("minLatencyMs")
2498
+ ]).where(sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col("createdAt")} <= ${endDate.toISOString()}`);
2499
+ if (configId) query = query.where("configId", "=", configId);
2500
+ if (variantId) query = query.where("variantId", "=", variantId);
2501
+ if (environmentId) query = query.where("environmentId", "=", environmentId);
2502
+ if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
2503
+ if (values.length === 0) continue;
2504
+ if (values.length === 1) query = query.where(sql`${col("tags")}->>${key} = ${values[0]}`);
2505
+ else {
2506
+ const valueList = sql.join(values.map((v) => sql`${v}`));
2507
+ query = query.where(sql`${col("tags")}->>${key} IN (${valueList})`);
2508
+ }
2509
+ }
2510
+ return query.executeTakeFirst();
2511
+ },
2512
+ getDistinctTags: async () => {
2513
+ return (await sql`
2514
+ SELECT DISTINCT key, value
2515
+ FROM llm_requests, jsonb_each_text(tags) AS t(key, value)
2516
+ WHERE tags != '{}'::jsonb
2517
+ ORDER BY key, value
2518
+ `.execute(db)).rows;
2519
+ }
2520
+ };
2521
+ }
2522
+ function createTracesStore(db) {
2523
+ return {
2524
+ upsertTrace: async (data) => {
2525
+ const result = await upsertTraceSchema.safeParseAsync(data);
2526
+ if (!result.success) throw new LLMOpsError(`Invalid trace data: ${result.error.message}`);
2527
+ const trace = result.data;
2528
+ const now = (/* @__PURE__ */ new Date()).toISOString();
2529
+ await sql`
2530
+ INSERT INTO "traces" (
2531
+ "id", "traceId", "name", "sessionId", "userId", "status",
2532
+ "startTime", "endTime", "durationMs", "spanCount",
2533
+ "totalInputTokens", "totalOutputTokens", "totalTokens", "totalCost",
2534
+ "tags", "metadata", "createdAt", "updatedAt"
2535
+ ) VALUES (
2536
+ ${randomUUID()}, ${trace.traceId}, ${trace.name ?? null}, ${trace.sessionId ?? null},
2537
+ ${trace.userId ?? null}, ${trace.status},
2538
+ ${trace.startTime.toISOString()}, ${trace.endTime?.toISOString() ?? null},
2539
+ ${trace.durationMs ?? null}, ${trace.spanCount},
2540
+ ${trace.totalInputTokens}, ${trace.totalOutputTokens},
2541
+ ${trace.totalTokens}, ${trace.totalCost},
2542
+ ${JSON.stringify(trace.tags)}::jsonb, ${JSON.stringify(trace.metadata)}::jsonb,
2543
+ ${now}, ${now}
2544
+ )
2545
+ ON CONFLICT ("traceId") DO UPDATE SET
2546
+ "name" = COALESCE(EXCLUDED."name", "traces"."name"),
2547
+ "sessionId" = COALESCE(EXCLUDED."sessionId", "traces"."sessionId"),
2548
+ "userId" = COALESCE(EXCLUDED."userId", "traces"."userId"),
2549
+ "status" = CASE
2550
+ WHEN EXCLUDED."status" = 'error' THEN 'error'
2551
+ WHEN EXCLUDED."status" = 'ok' AND "traces"."status" != 'error' THEN 'ok'
2552
+ ELSE "traces"."status"
2553
+ END,
2554
+ "startTime" = LEAST("traces"."startTime", EXCLUDED."startTime"),
2555
+ "endTime" = GREATEST(
2556
+ COALESCE("traces"."endTime", EXCLUDED."endTime"),
2557
+ COALESCE(EXCLUDED."endTime", "traces"."endTime")
2558
+ ),
2559
+ "durationMs" = EXTRACT(EPOCH FROM (
2560
+ GREATEST(
2561
+ COALESCE("traces"."endTime", EXCLUDED."endTime"),
2562
+ COALESCE(EXCLUDED."endTime", "traces"."endTime")
2563
+ ) -
2564
+ LEAST("traces"."startTime", EXCLUDED."startTime")
2565
+ ))::integer * 1000,
2566
+ "spanCount" = "traces"."spanCount" + EXCLUDED."spanCount",
2567
+ "totalInputTokens" = "traces"."totalInputTokens" + EXCLUDED."totalInputTokens",
2568
+ "totalOutputTokens" = "traces"."totalOutputTokens" + EXCLUDED."totalOutputTokens",
2569
+ "totalTokens" = "traces"."totalTokens" + EXCLUDED."totalTokens",
2570
+ "totalCost" = "traces"."totalCost" + EXCLUDED."totalCost",
2571
+ "tags" = "traces"."tags" || EXCLUDED."tags",
2572
+ "metadata" = "traces"."metadata" || EXCLUDED."metadata",
2573
+ "updatedAt" = ${now}
2574
+ `.execute(db);
2575
+ },
2576
+ batchInsertSpans: async (spans) => {
2577
+ if (spans.length === 0) return { count: 0 };
2578
+ const validatedSpans = [];
2579
+ for (const span of spans) {
2580
+ const result = await insertSpanSchema.safeParseAsync(span);
2581
+ if (!result.success) {
2582
+ logger.warn(`[batchInsertSpans] Skipping invalid span ${span.spanId}: ${result.error.message}`);
2583
+ continue;
2584
+ }
2585
+ validatedSpans.push(result.data);
2586
+ }
2587
+ if (validatedSpans.length === 0) return { count: 0 };
2588
+ const now = (/* @__PURE__ */ new Date()).toISOString();
2589
+ const values = validatedSpans.map((span) => ({
2590
+ id: randomUUID(),
2591
+ traceId: span.traceId,
2592
+ spanId: span.spanId,
2593
+ parentSpanId: span.parentSpanId ?? null,
2594
+ name: span.name,
2595
+ kind: span.kind,
2596
+ status: span.status,
2597
+ statusMessage: span.statusMessage ?? null,
2598
+ startTime: span.startTime.toISOString(),
2599
+ endTime: span.endTime?.toISOString() ?? null,
2600
+ durationMs: span.durationMs ?? null,
2601
+ provider: span.provider ?? null,
2602
+ model: span.model ?? null,
2603
+ promptTokens: span.promptTokens,
2604
+ completionTokens: span.completionTokens,
2605
+ totalTokens: span.totalTokens,
2606
+ cost: span.cost,
2607
+ configId: span.configId ?? null,
2608
+ variantId: span.variantId ?? null,
2609
+ environmentId: span.environmentId ?? null,
2610
+ providerConfigId: span.providerConfigId ?? null,
2611
+ requestId: span.requestId ?? null,
2612
+ source: span.source,
2613
+ input: span.input != null ? JSON.stringify(span.input) : null,
2614
+ output: span.output != null ? JSON.stringify(span.output) : null,
2615
+ attributes: JSON.stringify(span.attributes),
2616
+ createdAt: now,
2617
+ updatedAt: now
2618
+ }));
2619
+ await db.insertInto("spans").values(values).onConflict((oc) => oc.column("spanId").doNothing()).execute();
2620
+ return { count: values.length };
2621
+ },
2622
+ batchInsertSpanEvents: async (events) => {
2623
+ if (events.length === 0) return { count: 0 };
2624
+ const validatedEvents = [];
2625
+ for (const event of events) {
2626
+ const result = await insertSpanEventSchema.safeParseAsync(event);
2627
+ if (!result.success) {
2628
+ logger.warn(`[batchInsertSpanEvents] Skipping invalid event: ${result.error.message}`);
2629
+ continue;
2630
+ }
2631
+ validatedEvents.push(result.data);
2632
+ }
2633
+ if (validatedEvents.length === 0) return { count: 0 };
2634
+ const now = (/* @__PURE__ */ new Date()).toISOString();
2635
+ const values = validatedEvents.map((event) => ({
2636
+ id: randomUUID(),
2637
+ traceId: event.traceId,
2638
+ spanId: event.spanId,
2639
+ name: event.name,
2640
+ timestamp: event.timestamp.toISOString(),
2641
+ attributes: JSON.stringify(event.attributes),
2642
+ createdAt: now
2643
+ }));
2644
+ await db.insertInto("span_events").values(values).execute();
2645
+ return { count: values.length };
2646
+ },
2647
+ listTraces: async (params) => {
2648
+ const result = await listTracesSchema.safeParseAsync(params || {});
2649
+ if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2650
+ const { limit, offset, sessionId, userId, status, name, startDate, endDate, tags } = result.data;
2651
+ let baseQuery = db.selectFrom("traces");
2652
+ if (sessionId) baseQuery = baseQuery.where("sessionId", "=", sessionId);
2653
+ if (userId) baseQuery = baseQuery.where("userId", "=", userId);
2654
+ if (status) baseQuery = baseQuery.where("status", "=", status);
2655
+ if (name) baseQuery = baseQuery.where(sql`${col("name")} ILIKE ${"%" + name + "%"}`);
2656
+ if (startDate) baseQuery = baseQuery.where(sql`${col("startTime")} >= ${startDate.toISOString()}`);
2657
+ if (endDate) baseQuery = baseQuery.where(sql`${col("startTime")} <= ${endDate.toISOString()}`);
2658
+ if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
2659
+ if (values.length === 0) continue;
2660
+ if (values.length === 1) baseQuery = baseQuery.where(sql`${col("tags")}->>${key} = ${values[0]}`);
2661
+ else {
2662
+ const valueList = sql.join(values.map((v) => sql`${v}`));
2663
+ baseQuery = baseQuery.where(sql`${col("tags")}->>${key} IN (${valueList})`);
2664
+ }
2665
+ }
2666
+ const countResult = await baseQuery.select(sql`COUNT(*)`.as("total")).executeTakeFirst();
2667
+ const total = Number(countResult?.total ?? 0);
2668
+ return {
2669
+ data: await baseQuery.selectAll().orderBy("startTime", "desc").limit(limit).offset(offset).execute(),
2670
+ total,
2671
+ limit,
2672
+ offset
2673
+ };
2674
+ },
2675
+ getTraceWithSpans: async (traceId) => {
2676
+ const trace = await db.selectFrom("traces").selectAll().where("traceId", "=", traceId).executeTakeFirst();
2677
+ if (!trace) return void 0;
2678
+ return {
2679
+ trace,
2680
+ spans: await db.selectFrom("spans").selectAll().where("traceId", "=", traceId).orderBy("startTime", "asc").execute(),
2681
+ events: await db.selectFrom("span_events").selectAll().where("traceId", "=", traceId).orderBy("timestamp", "asc").execute()
2682
+ };
2683
+ },
2684
+ getTraceStats: async (params) => {
2685
+ const result = await traceStatsSchema.safeParseAsync(params);
2686
+ if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
2687
+ const { startDate, endDate, sessionId, userId } = result.data;
2688
+ let query = db.selectFrom("traces").select([
2689
+ sql`COUNT(*)`.as("totalTraces"),
2690
+ sql`COALESCE(AVG(${col("durationMs")}), 0)`.as("avgDurationMs"),
2691
+ sql`COUNT(CASE WHEN ${col("status")} = 'error' THEN 1 END)`.as("errorCount"),
2692
+ sql`COALESCE(SUM(${col("totalCost")}), 0)`.as("totalCost"),
2693
+ sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
2694
+ sql`COALESCE(SUM(${col("spanCount")}), 0)`.as("totalSpans")
2695
+ ]).where(sql`${col("startTime")} >= ${startDate.toISOString()}`).where(sql`${col("startTime")} <= ${endDate.toISOString()}`);
2696
+ if (sessionId) query = query.where("sessionId", "=", sessionId);
2697
+ if (userId) query = query.where("userId", "=", userId);
2698
+ return query.executeTakeFirst();
2699
+ }
2700
+ };
2701
+ }
2751
2702
  /**
2752
2703
  * Create a PostgreSQL-backed telemetry store.
2753
2704
  *
2754
2705
  * Usage:
2755
2706
  * ```ts
2756
- * import { llmops, pgStore } from '@llmops/sdk'
2707
+ * import { llmops } from '@llmops/sdk'
2708
+ * import { pgStore } from '@llmops/sdk/store/pg'
2757
2709
  *
2758
2710
  * const ops = llmops({
2759
2711
  * telemetry: pgStore(process.env.DATABASE_URL),
@@ -2774,15 +2726,13 @@ function createPgStore(connectionString, options) {
2774
2726
  await connection.executeQuery(CompiledQuery.raw(`SET search_path TO "${schema}"`));
2775
2727
  }
2776
2728
  }) });
2777
- const llmRequests = createLLMRequestsDataLayer(db);
2778
- const traces = createTracesDataLayer(db);
2779
2729
  logger.debug(`pgStore: initialized with schema "${schema}"`);
2780
2730
  return {
2781
- ...llmRequests,
2782
- ...traces,
2731
+ ...createLLMRequestsStore(db),
2732
+ ...createTracesStore(db),
2783
2733
  _db: db
2784
2734
  };
2785
2735
  }
2786
2736
 
2787
2737
  //#endregion
2788
- export { COST_SUMMARY_GROUP_BY, CacheService, DEFAULT_PROVIDER_ENV_VARS, FileCacheBackend, LLMOPS_INTERNAL_HEADER, LLMOPS_REQUEST_ID_HEADER, LLMOPS_SESSION_ID_HEADER, LLMOPS_SPAN_ID_HEADER, LLMOPS_SPAN_NAME_HEADER, LLMOPS_TRACE_ID_HEADER, LLMOPS_TRACE_NAME_HEADER, LLMOPS_USER_ID_HEADER, LLMOpsPricingProvider, MS, MemoryCacheBackend, SCHEMA_METADATA, SupportedProviders, calculateCacheAwareCost, calculateCost, chatCompletionCreateParamsBaseSchema, configVariantsSchema, configsSchema, createDataLayer, createDatabase, createDatabaseFromConnection, createDatasetsDataLayer, createLLMRequestsDataLayer, createNeonDialect, createPgStore, createPlaygroundDataLayer, createPlaygroundResultsDataLayer, createPlaygroundRunsDataLayer, createTracesDataLayer, createWorkspaceSettingsDataLayer, datasetRecordsSchema, datasetVersionRecordsSchema, datasetVersionsSchema, datasetsSchema, detectDatabaseType, dollarsToMicroDollars, environmentSecretsSchema, environmentsSchema, executeWithSchema, formatCost, gateway, generateId, getDefaultPricingProvider, getDefaultProviders, getMigrations, guardrailConfigsSchema, llmRequestsSchema, llmopsConfigSchema, logger, matchType, mergeWithDefaultProviders, microDollarsToDollars, parsePartialTableData, parseTableData, playgroundColumnSchema, playgroundResultsSchema, playgroundRunsSchema, playgroundsSchema, providerConfigsSchema, providerGuardrailOverridesSchema, runAutoMigrations, schemas, spanEventsSchema, spansSchema, targetingRulesSchema, tracesSchema, validateLLMOpsConfig, validatePartialTableData, validateTableData, variantJsonDataSchema, variantVersionsSchema, variantsSchema, workspaceSettingsSchema };
2738
+ export { COST_SUMMARY_GROUP_BY, CacheService, DEFAULT_PROVIDER_ENV_VARS, FileCacheBackend, LLMOPS_INTERNAL_HEADER, LLMOPS_REQUEST_ID_HEADER, LLMOPS_SESSION_ID_HEADER, LLMOPS_SPAN_ID_HEADER, LLMOPS_SPAN_NAME_HEADER, LLMOPS_TRACE_ID_HEADER, LLMOPS_TRACE_NAME_HEADER, LLMOPS_USER_ID_HEADER, LLMOpsPricingProvider, MS, MemoryCacheBackend, SCHEMA_METADATA, SupportedProviders, calculateCacheAwareCost, calculateCost, chatCompletionCreateParamsBaseSchema, configVariantsSchema, configsSchema, createDataLayer, createDatabase, createDatabaseFromConnection, createDatasetsDataLayer, createNeonDialect, createPgStore, createPlaygroundDataLayer, createPlaygroundResultsDataLayer, createPlaygroundRunsDataLayer, createWorkspaceSettingsDataLayer, datasetRecordsSchema, datasetVersionRecordsSchema, datasetVersionsSchema, datasetsSchema, detectDatabaseType, dollarsToMicroDollars, environmentSecretsSchema, environmentsSchema, executeWithSchema, formatCost, gateway, generateId, getDefaultPricingProvider, getDefaultProviders, getMigrations, guardrailConfigsSchema, llmRequestsSchema, llmopsConfigSchema, logger, matchType, mergeWithDefaultProviders, microDollarsToDollars, parsePartialTableData, parseTableData, playgroundColumnSchema, playgroundResultsSchema, playgroundRunsSchema, playgroundsSchema, providerConfigsSchema, providerGuardrailOverridesSchema, runAutoMigrations, schemas, spanEventsSchema, spansSchema, targetingRulesSchema, tracesSchema, validateLLMOpsConfig, validatePartialTableData, validateTableData, variantJsonDataSchema, variantVersionsSchema, variantsSchema, workspaceSettingsSchema };