@llmops/core 1.0.0-beta.2 → 1.0.0-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +695 -747
- package/dist/index.d.cts +832 -924
- package/dist/index.d.mts +832 -924
- package/dist/index.mjs +696 -746
- package/package.json +2 -2
package/dist/index.cjs
CHANGED
|
@@ -1387,468 +1387,54 @@ const createDatasetsDataLayer = (db) => {
|
|
|
1387
1387
|
};
|
|
1388
1388
|
|
|
1389
1389
|
//#endregion
|
|
1390
|
-
//#region src/datalayer/
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
checkId: require_db.zod_default.string(),
|
|
1396
|
-
functionId: require_db.zod_default.string(),
|
|
1397
|
-
hookType: require_db.zod_default.enum(["beforeRequestHook", "afterRequestHook"]),
|
|
1398
|
-
verdict: require_db.zod_default.boolean(),
|
|
1399
|
-
latencyMs: require_db.zod_default.number()
|
|
1400
|
-
});
|
|
1401
|
-
/**
|
|
1402
|
-
* Schema for guardrail results aggregate
|
|
1403
|
-
*/
|
|
1404
|
-
const guardrailResultsSchema = require_db.zod_default.object({
|
|
1405
|
-
results: require_db.zod_default.array(guardrailResultSchema),
|
|
1406
|
-
action: require_db.zod_default.enum([
|
|
1407
|
-
"allowed",
|
|
1408
|
-
"blocked",
|
|
1409
|
-
"logged"
|
|
1410
|
-
]),
|
|
1411
|
-
totalLatencyMs: require_db.zod_default.number()
|
|
1412
|
-
});
|
|
1413
|
-
/**
|
|
1414
|
-
* Schema for inserting a new LLM request log
|
|
1415
|
-
*/
|
|
1416
|
-
const insertLLMRequestSchema = require_db.zod_default.object({
|
|
1417
|
-
requestId: require_db.zod_default.string().uuid(),
|
|
1418
|
-
configId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
1419
|
-
variantId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
1420
|
-
environmentId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
1421
|
-
providerConfigId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
1422
|
-
provider: require_db.zod_default.string(),
|
|
1423
|
-
model: require_db.zod_default.string(),
|
|
1424
|
-
promptTokens: require_db.zod_default.number().int().default(0),
|
|
1425
|
-
completionTokens: require_db.zod_default.number().int().default(0),
|
|
1426
|
-
totalTokens: require_db.zod_default.number().int().default(0),
|
|
1427
|
-
cachedTokens: require_db.zod_default.number().int().default(0),
|
|
1428
|
-
cacheCreationTokens: require_db.zod_default.number().int().default(0),
|
|
1429
|
-
cost: require_db.zod_default.number().int().default(0),
|
|
1430
|
-
cacheSavings: require_db.zod_default.number().int().default(0),
|
|
1431
|
-
inputCost: require_db.zod_default.number().int().default(0),
|
|
1432
|
-
outputCost: require_db.zod_default.number().int().default(0),
|
|
1433
|
-
endpoint: require_db.zod_default.string(),
|
|
1434
|
-
statusCode: require_db.zod_default.number().int(),
|
|
1435
|
-
latencyMs: require_db.zod_default.number().int().default(0),
|
|
1436
|
-
isStreaming: require_db.zod_default.boolean().default(false),
|
|
1437
|
-
userId: require_db.zod_default.string().nullable().optional(),
|
|
1438
|
-
tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.string()).default({}),
|
|
1439
|
-
guardrailResults: guardrailResultsSchema.nullable().optional(),
|
|
1440
|
-
traceId: require_db.zod_default.string().nullable().optional(),
|
|
1441
|
-
spanId: require_db.zod_default.string().nullable().optional(),
|
|
1442
|
-
parentSpanId: require_db.zod_default.string().nullable().optional(),
|
|
1443
|
-
sessionId: require_db.zod_default.string().nullable().optional()
|
|
1444
|
-
});
|
|
1445
|
-
/**
|
|
1446
|
-
* Schema for listing LLM requests
|
|
1447
|
-
*/
|
|
1448
|
-
const listRequestsSchema = require_db.zod_default.object({
|
|
1449
|
-
limit: require_db.zod_default.number().int().positive().max(1e3).default(100),
|
|
1450
|
-
offset: require_db.zod_default.number().int().nonnegative().default(0),
|
|
1451
|
-
configId: require_db.zod_default.string().uuid().optional(),
|
|
1452
|
-
variantId: require_db.zod_default.string().uuid().optional(),
|
|
1453
|
-
environmentId: require_db.zod_default.string().uuid().optional(),
|
|
1454
|
-
providerConfigId: require_db.zod_default.string().uuid().optional(),
|
|
1455
|
-
provider: require_db.zod_default.string().optional(),
|
|
1456
|
-
model: require_db.zod_default.string().optional(),
|
|
1457
|
-
startDate: require_db.zod_default.date().optional(),
|
|
1458
|
-
endDate: require_db.zod_default.date().optional(),
|
|
1459
|
-
tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.array(require_db.zod_default.string())).optional()
|
|
1390
|
+
//#region src/datalayer/playgrounds.ts
|
|
1391
|
+
const createNewPlayground = require_db.zod_default.object({
|
|
1392
|
+
name: require_db.zod_default.string(),
|
|
1393
|
+
datasetId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
1394
|
+
columns: require_db.zod_default.array(require_db.playgroundColumnSchema).nullable().optional()
|
|
1460
1395
|
});
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
endDate: require_db.zod_default.date(),
|
|
1467
|
-
configId: require_db.zod_default.string().uuid().optional(),
|
|
1468
|
-
variantId: require_db.zod_default.string().uuid().optional(),
|
|
1469
|
-
environmentId: require_db.zod_default.string().uuid().optional(),
|
|
1470
|
-
tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.array(require_db.zod_default.string())).optional()
|
|
1396
|
+
const updatePlayground = require_db.zod_default.object({
|
|
1397
|
+
playgroundId: require_db.zod_default.uuidv4(),
|
|
1398
|
+
name: require_db.zod_default.string().optional(),
|
|
1399
|
+
datasetId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
1400
|
+
columns: require_db.zod_default.array(require_db.playgroundColumnSchema).nullable().optional()
|
|
1471
1401
|
});
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
"hour",
|
|
1478
|
-
"model",
|
|
1479
|
-
"provider",
|
|
1480
|
-
"endpoint",
|
|
1481
|
-
"tags"
|
|
1482
|
-
];
|
|
1483
|
-
/**
|
|
1484
|
-
* Schema for cost summary with grouping
|
|
1485
|
-
*/
|
|
1486
|
-
const costSummarySchema = require_db.zod_default.object({
|
|
1487
|
-
startDate: require_db.zod_default.date(),
|
|
1488
|
-
endDate: require_db.zod_default.date(),
|
|
1489
|
-
configId: require_db.zod_default.string().uuid().optional(),
|
|
1490
|
-
variantId: require_db.zod_default.string().uuid().optional(),
|
|
1491
|
-
environmentId: require_db.zod_default.string().uuid().optional(),
|
|
1492
|
-
tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.array(require_db.zod_default.string())).optional(),
|
|
1493
|
-
groupBy: require_db.zod_default.enum(COST_SUMMARY_GROUP_BY).optional(),
|
|
1494
|
-
tagKeys: require_db.zod_default.array(require_db.zod_default.string()).optional()
|
|
1402
|
+
const getPlaygroundById = require_db.zod_default.object({ playgroundId: require_db.zod_default.uuidv4() });
|
|
1403
|
+
const deletePlayground = require_db.zod_default.object({ playgroundId: require_db.zod_default.uuidv4() });
|
|
1404
|
+
const listPlaygrounds = require_db.zod_default.object({
|
|
1405
|
+
limit: require_db.zod_default.number().int().positive().optional(),
|
|
1406
|
+
offset: require_db.zod_default.number().int().nonnegative().optional()
|
|
1495
1407
|
});
|
|
1496
|
-
|
|
1497
|
-
* Helper to create column reference for SQL
|
|
1498
|
-
* Uses sql.ref() to properly quote column names for the database
|
|
1499
|
-
*/
|
|
1500
|
-
const col$1 = (name) => kysely.sql.ref(name);
|
|
1501
|
-
const createLLMRequestsDataLayer = (db) => {
|
|
1408
|
+
const createPlaygroundDataLayer = (db) => {
|
|
1502
1409
|
return {
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
return result.data;
|
|
1509
|
-
}));
|
|
1510
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1511
|
-
const values = validatedRequests.map((req) => ({
|
|
1512
|
-
id: (0, node_crypto.randomUUID)(),
|
|
1513
|
-
requestId: req.requestId,
|
|
1514
|
-
configId: req.configId ?? null,
|
|
1515
|
-
variantId: req.variantId ?? null,
|
|
1516
|
-
environmentId: req.environmentId ?? null,
|
|
1517
|
-
providerConfigId: req.providerConfigId ?? null,
|
|
1518
|
-
provider: req.provider,
|
|
1519
|
-
model: req.model,
|
|
1520
|
-
promptTokens: req.promptTokens,
|
|
1521
|
-
completionTokens: req.completionTokens,
|
|
1522
|
-
totalTokens: req.totalTokens,
|
|
1523
|
-
cachedTokens: req.cachedTokens,
|
|
1524
|
-
cacheCreationTokens: req.cacheCreationTokens,
|
|
1525
|
-
cost: req.cost,
|
|
1526
|
-
cacheSavings: req.cacheSavings,
|
|
1527
|
-
inputCost: req.inputCost,
|
|
1528
|
-
outputCost: req.outputCost,
|
|
1529
|
-
endpoint: req.endpoint,
|
|
1530
|
-
statusCode: req.statusCode,
|
|
1531
|
-
latencyMs: req.latencyMs,
|
|
1532
|
-
isStreaming: req.isStreaming,
|
|
1533
|
-
userId: req.userId ?? null,
|
|
1534
|
-
tags: JSON.stringify(req.tags),
|
|
1535
|
-
guardrailResults: req.guardrailResults ? JSON.stringify(req.guardrailResults) : null,
|
|
1536
|
-
traceId: req.traceId ?? null,
|
|
1537
|
-
spanId: req.spanId ?? null,
|
|
1538
|
-
parentSpanId: req.parentSpanId ?? null,
|
|
1539
|
-
sessionId: req.sessionId ?? null,
|
|
1540
|
-
createdAt: now,
|
|
1541
|
-
updatedAt: now
|
|
1542
|
-
}));
|
|
1543
|
-
await db.insertInto("llm_requests").values(values).execute();
|
|
1544
|
-
return { count: values.length };
|
|
1545
|
-
},
|
|
1546
|
-
insertRequest: async (request) => {
|
|
1547
|
-
const result = await insertLLMRequestSchema.safeParseAsync(request);
|
|
1548
|
-
if (!result.success) throw new LLMOpsError(`Invalid request data: ${result.error.message}`);
|
|
1549
|
-
const req = result.data;
|
|
1550
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1551
|
-
return db.insertInto("llm_requests").values({
|
|
1410
|
+
createNewPlayground: async (params) => {
|
|
1411
|
+
const value = await createNewPlayground.safeParseAsync(params);
|
|
1412
|
+
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1413
|
+
const { name, datasetId, columns } = value.data;
|
|
1414
|
+
return db.insertInto("playgrounds").values({
|
|
1552
1415
|
id: (0, node_crypto.randomUUID)(),
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
provider: req.provider,
|
|
1559
|
-
model: req.model,
|
|
1560
|
-
promptTokens: req.promptTokens,
|
|
1561
|
-
completionTokens: req.completionTokens,
|
|
1562
|
-
totalTokens: req.totalTokens,
|
|
1563
|
-
cachedTokens: req.cachedTokens,
|
|
1564
|
-
cacheCreationTokens: req.cacheCreationTokens,
|
|
1565
|
-
cost: req.cost,
|
|
1566
|
-
cacheSavings: req.cacheSavings,
|
|
1567
|
-
inputCost: req.inputCost,
|
|
1568
|
-
outputCost: req.outputCost,
|
|
1569
|
-
endpoint: req.endpoint,
|
|
1570
|
-
statusCode: req.statusCode,
|
|
1571
|
-
latencyMs: req.latencyMs,
|
|
1572
|
-
isStreaming: req.isStreaming,
|
|
1573
|
-
userId: req.userId ?? null,
|
|
1574
|
-
tags: JSON.stringify(req.tags),
|
|
1575
|
-
guardrailResults: req.guardrailResults ? JSON.stringify(req.guardrailResults) : null,
|
|
1576
|
-
traceId: req.traceId ?? null,
|
|
1577
|
-
spanId: req.spanId ?? null,
|
|
1578
|
-
parentSpanId: req.parentSpanId ?? null,
|
|
1579
|
-
sessionId: req.sessionId ?? null,
|
|
1580
|
-
createdAt: now,
|
|
1581
|
-
updatedAt: now
|
|
1416
|
+
name,
|
|
1417
|
+
datasetId: datasetId ?? null,
|
|
1418
|
+
columns: columns ? JSON.stringify(columns) : null,
|
|
1419
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1420
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1582
1421
|
}).returningAll().executeTakeFirst();
|
|
1583
1422
|
},
|
|
1584
|
-
|
|
1585
|
-
const
|
|
1586
|
-
if (!
|
|
1587
|
-
const {
|
|
1588
|
-
|
|
1589
|
-
if (
|
|
1590
|
-
if (
|
|
1591
|
-
if (
|
|
1592
|
-
|
|
1593
|
-
if (provider) baseQuery = baseQuery.where("provider", "=", provider);
|
|
1594
|
-
if (model) baseQuery = baseQuery.where("model", "=", model);
|
|
1595
|
-
if (startDate) baseQuery = baseQuery.where(kysely.sql`${col$1("createdAt")} >= ${startDate.toISOString()}`);
|
|
1596
|
-
if (endDate) baseQuery = baseQuery.where(kysely.sql`${col$1("createdAt")} <= ${endDate.toISOString()}`);
|
|
1597
|
-
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
1598
|
-
if (values.length === 0) continue;
|
|
1599
|
-
if (values.length === 1) baseQuery = baseQuery.where(kysely.sql`${col$1("tags")}->>${key} = ${values[0]}`);
|
|
1600
|
-
else {
|
|
1601
|
-
const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
|
|
1602
|
-
baseQuery = baseQuery.where(kysely.sql`${col$1("tags")}->>${key} IN (${valueList})`);
|
|
1603
|
-
}
|
|
1604
|
-
}
|
|
1605
|
-
const countResult = await baseQuery.select(kysely.sql`COUNT(*)`.as("total")).executeTakeFirst();
|
|
1606
|
-
const total = Number(countResult?.total ?? 0);
|
|
1607
|
-
return {
|
|
1608
|
-
data: await baseQuery.selectAll().orderBy("createdAt", "desc").limit(limit).offset(offset).execute(),
|
|
1609
|
-
total,
|
|
1610
|
-
limit,
|
|
1611
|
-
offset
|
|
1612
|
-
};
|
|
1423
|
+
updatePlayground: async (params) => {
|
|
1424
|
+
const value = await updatePlayground.safeParseAsync(params);
|
|
1425
|
+
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1426
|
+
const { playgroundId, name, datasetId, columns } = value.data;
|
|
1427
|
+
const updateData = { updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
1428
|
+
if (name !== void 0) updateData.name = name;
|
|
1429
|
+
if (datasetId !== void 0) updateData.datasetId = datasetId;
|
|
1430
|
+
if (columns !== void 0) updateData.columns = columns ? JSON.stringify(columns) : null;
|
|
1431
|
+
return db.updateTable("playgrounds").set(updateData).where("id", "=", playgroundId).returningAll().executeTakeFirst();
|
|
1613
1432
|
},
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
1620
|
-
const { startDate, endDate, configId, variantId, environmentId, tags } = result.data;
|
|
1621
|
-
let query = db.selectFrom("llm_requests").select([
|
|
1622
|
-
kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
|
|
1623
|
-
kysely.sql`COALESCE(SUM(${col$1("inputCost")}), 0)`.as("totalInputCost"),
|
|
1624
|
-
kysely.sql`COALESCE(SUM(${col$1("outputCost")}), 0)`.as("totalOutputCost"),
|
|
1625
|
-
kysely.sql`COALESCE(SUM(${col$1("promptTokens")}), 0)`.as("totalPromptTokens"),
|
|
1626
|
-
kysely.sql`COALESCE(SUM(${col$1("completionTokens")}), 0)`.as("totalCompletionTokens"),
|
|
1627
|
-
kysely.sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens"),
|
|
1628
|
-
kysely.sql`COALESCE(SUM(${col$1("cachedTokens")}), 0)`.as("totalCachedTokens"),
|
|
1629
|
-
kysely.sql`COALESCE(SUM(${col$1("cacheSavings")}), 0)`.as("totalCacheSavings"),
|
|
1630
|
-
kysely.sql`COUNT(*)`.as("requestCount")
|
|
1631
|
-
]).where(kysely.sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col$1("createdAt")} <= ${endDate.toISOString()}`);
|
|
1632
|
-
if (configId) query = query.where("configId", "=", configId);
|
|
1633
|
-
if (variantId) query = query.where("variantId", "=", variantId);
|
|
1634
|
-
if (environmentId) query = query.where("environmentId", "=", environmentId);
|
|
1635
|
-
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
1636
|
-
if (values.length === 0) continue;
|
|
1637
|
-
if (values.length === 1) query = query.where(kysely.sql`${col$1("tags")}->>${key} = ${values[0]}`);
|
|
1638
|
-
else {
|
|
1639
|
-
const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
|
|
1640
|
-
query = query.where(kysely.sql`${col$1("tags")}->>${key} IN (${valueList})`);
|
|
1641
|
-
}
|
|
1642
|
-
}
|
|
1643
|
-
return await query.executeTakeFirst();
|
|
1644
|
-
},
|
|
1645
|
-
getCostByModel: async (params) => {
|
|
1646
|
-
const result = await dateRangeSchema.safeParseAsync(params);
|
|
1647
|
-
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
1648
|
-
const { startDate, endDate } = result.data;
|
|
1649
|
-
return db.selectFrom("llm_requests").select([
|
|
1650
|
-
"provider",
|
|
1651
|
-
"model",
|
|
1652
|
-
kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
|
|
1653
|
-
kysely.sql`COALESCE(SUM(${col$1("inputCost")}), 0)`.as("totalInputCost"),
|
|
1654
|
-
kysely.sql`COALESCE(SUM(${col$1("outputCost")}), 0)`.as("totalOutputCost"),
|
|
1655
|
-
kysely.sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens"),
|
|
1656
|
-
kysely.sql`COUNT(*)`.as("requestCount"),
|
|
1657
|
-
kysely.sql`AVG(${col$1("latencyMs")})`.as("avgLatencyMs")
|
|
1658
|
-
]).where(kysely.sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col$1("createdAt")} <= ${endDate.toISOString()}`).groupBy(["provider", "model"]).orderBy(kysely.sql`SUM(${col$1("cost")})`, "desc").execute();
|
|
1659
|
-
},
|
|
1660
|
-
getCostByProvider: async (params) => {
|
|
1661
|
-
const result = await dateRangeSchema.safeParseAsync(params);
|
|
1662
|
-
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
1663
|
-
const { startDate, endDate } = result.data;
|
|
1664
|
-
return db.selectFrom("llm_requests").select([
|
|
1665
|
-
"provider",
|
|
1666
|
-
kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
|
|
1667
|
-
kysely.sql`COALESCE(SUM(${col$1("inputCost")}), 0)`.as("totalInputCost"),
|
|
1668
|
-
kysely.sql`COALESCE(SUM(${col$1("outputCost")}), 0)`.as("totalOutputCost"),
|
|
1669
|
-
kysely.sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens"),
|
|
1670
|
-
kysely.sql`COUNT(*)`.as("requestCount"),
|
|
1671
|
-
kysely.sql`AVG(${col$1("latencyMs")})`.as("avgLatencyMs")
|
|
1672
|
-
]).where(kysely.sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col$1("createdAt")} <= ${endDate.toISOString()}`).groupBy("provider").orderBy(kysely.sql`SUM(${col$1("cost")})`, "desc").execute();
|
|
1673
|
-
},
|
|
1674
|
-
getDailyCosts: async (params) => {
|
|
1675
|
-
const result = await dateRangeSchema.safeParseAsync(params);
|
|
1676
|
-
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
1677
|
-
const { startDate, endDate } = result.data;
|
|
1678
|
-
return db.selectFrom("llm_requests").select([
|
|
1679
|
-
kysely.sql`DATE(${col$1("createdAt")})`.as("date"),
|
|
1680
|
-
kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
|
|
1681
|
-
kysely.sql`COALESCE(SUM(${col$1("inputCost")}), 0)`.as("totalInputCost"),
|
|
1682
|
-
kysely.sql`COALESCE(SUM(${col$1("outputCost")}), 0)`.as("totalOutputCost"),
|
|
1683
|
-
kysely.sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens"),
|
|
1684
|
-
kysely.sql`COUNT(*)`.as("requestCount")
|
|
1685
|
-
]).where(kysely.sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col$1("createdAt")} <= ${endDate.toISOString()}`).groupBy(kysely.sql`DATE(${col$1("createdAt")})`).orderBy(kysely.sql`DATE(${col$1("createdAt")})`, "asc").execute();
|
|
1686
|
-
},
|
|
1687
|
-
getCostSummary: async (params) => {
|
|
1688
|
-
const result = await costSummarySchema.safeParseAsync(params);
|
|
1689
|
-
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
1690
|
-
const { startDate, endDate, groupBy, configId, variantId, environmentId, tags, tagKeys } = result.data;
|
|
1691
|
-
let baseQuery = db.selectFrom("llm_requests").where(kysely.sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col$1("createdAt")} <= ${endDate.toISOString()}`);
|
|
1692
|
-
if (configId) baseQuery = baseQuery.where("configId", "=", configId);
|
|
1693
|
-
if (variantId) baseQuery = baseQuery.where("variantId", "=", variantId);
|
|
1694
|
-
if (environmentId) baseQuery = baseQuery.where("environmentId", "=", environmentId);
|
|
1695
|
-
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
1696
|
-
if (values.length === 0) continue;
|
|
1697
|
-
if (values.length === 1) baseQuery = baseQuery.where(kysely.sql`${col$1("tags")}->>${key} = ${values[0]}`);
|
|
1698
|
-
else {
|
|
1699
|
-
const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
|
|
1700
|
-
baseQuery = baseQuery.where(kysely.sql`${col$1("tags")}->>${key} IN (${valueList})`);
|
|
1701
|
-
}
|
|
1702
|
-
}
|
|
1703
|
-
switch (groupBy) {
|
|
1704
|
-
case "day": return baseQuery.select([
|
|
1705
|
-
kysely.sql`DATE(${col$1("createdAt")})`.as("groupKey"),
|
|
1706
|
-
kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
|
|
1707
|
-
kysely.sql`COUNT(*)`.as("requestCount"),
|
|
1708
|
-
kysely.sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens")
|
|
1709
|
-
]).groupBy(kysely.sql`DATE(${col$1("createdAt")})`).orderBy(kysely.sql`DATE(${col$1("createdAt")})`, "asc").execute();
|
|
1710
|
-
case "hour": return baseQuery.select([
|
|
1711
|
-
kysely.sql`DATE_TRUNC('hour', ${col$1("createdAt")})`.as("groupKey"),
|
|
1712
|
-
kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
|
|
1713
|
-
kysely.sql`COUNT(*)`.as("requestCount"),
|
|
1714
|
-
kysely.sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens")
|
|
1715
|
-
]).groupBy(kysely.sql`DATE_TRUNC('hour', ${col$1("createdAt")})`).orderBy(kysely.sql`DATE_TRUNC('hour', ${col$1("createdAt")})`, "asc").execute();
|
|
1716
|
-
case "model": return baseQuery.select([
|
|
1717
|
-
kysely.sql`${col$1("provider")} || '/' || ${col$1("model")}`.as("groupKey"),
|
|
1718
|
-
kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
|
|
1719
|
-
kysely.sql`COUNT(*)`.as("requestCount")
|
|
1720
|
-
]).groupBy(["provider", "model"]).orderBy(kysely.sql`SUM(${col$1("cost")})`, "desc").execute();
|
|
1721
|
-
case "provider": return baseQuery.select([
|
|
1722
|
-
kysely.sql`${col$1("provider")}`.as("groupKey"),
|
|
1723
|
-
kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
|
|
1724
|
-
kysely.sql`COUNT(*)`.as("requestCount")
|
|
1725
|
-
]).groupBy("provider").orderBy(kysely.sql`SUM(${col$1("cost")})`, "desc").execute();
|
|
1726
|
-
case "endpoint": return baseQuery.select([
|
|
1727
|
-
kysely.sql`COALESCE(${col$1("endpoint")}, 'unknown')`.as("groupKey"),
|
|
1728
|
-
kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
|
|
1729
|
-
kysely.sql`COUNT(*)`.as("requestCount")
|
|
1730
|
-
]).groupBy("endpoint").orderBy(kysely.sql`SUM(${col$1("cost")})`, "desc").execute();
|
|
1731
|
-
case "tags": {
|
|
1732
|
-
const conditions = [kysely.sql`${col$1("createdAt")} >= ${startDate.toISOString()}`, kysely.sql`${col$1("createdAt")} <= ${endDate.toISOString()}`];
|
|
1733
|
-
if (configId) conditions.push(kysely.sql`${col$1("configId")} = ${configId}`);
|
|
1734
|
-
if (variantId) conditions.push(kysely.sql`${col$1("variantId")} = ${variantId}`);
|
|
1735
|
-
if (environmentId) conditions.push(kysely.sql`${col$1("environmentId")} = ${environmentId}`);
|
|
1736
|
-
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
1737
|
-
if (values.length === 0) continue;
|
|
1738
|
-
if (values.length === 1) conditions.push(kysely.sql`${col$1("tags")}->>${key} = ${values[0]}`);
|
|
1739
|
-
else {
|
|
1740
|
-
const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
|
|
1741
|
-
conditions.push(kysely.sql`${col$1("tags")}->>${key} IN (${valueList})`);
|
|
1742
|
-
}
|
|
1743
|
-
}
|
|
1744
|
-
if (tagKeys && tagKeys.length > 0) {
|
|
1745
|
-
const tagKeyList = kysely.sql.join(tagKeys.map((k) => kysely.sql`${k}`), kysely.sql`, `);
|
|
1746
|
-
conditions.push(kysely.sql`t.key IN (${tagKeyList})`);
|
|
1747
|
-
}
|
|
1748
|
-
const whereClause = kysely.sql.join(conditions, kysely.sql` AND `);
|
|
1749
|
-
return (await kysely.sql`
|
|
1750
|
-
SELECT t.key || ':' || t.value as "groupKey",
|
|
1751
|
-
COALESCE(SUM(${col$1("cost")}), 0) as "totalCost",
|
|
1752
|
-
COUNT(*) as "requestCount"
|
|
1753
|
-
FROM "llm_requests", jsonb_each_text(${col$1("tags")}) t
|
|
1754
|
-
WHERE ${whereClause}
|
|
1755
|
-
GROUP BY t.key, t.value
|
|
1756
|
-
ORDER BY SUM(${col$1("cost")}) DESC
|
|
1757
|
-
`.execute(db)).rows;
|
|
1758
|
-
}
|
|
1759
|
-
default: return baseQuery.select([
|
|
1760
|
-
kysely.sql`'total'`.as("groupKey"),
|
|
1761
|
-
kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
|
|
1762
|
-
kysely.sql`COUNT(*)`.as("requestCount")
|
|
1763
|
-
]).execute();
|
|
1764
|
-
}
|
|
1765
|
-
},
|
|
1766
|
-
getRequestStats: async (params) => {
|
|
1767
|
-
const result = await dateRangeSchema.safeParseAsync(params);
|
|
1768
|
-
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
1769
|
-
const { startDate, endDate, configId, variantId, environmentId, tags } = result.data;
|
|
1770
|
-
let query = db.selectFrom("llm_requests").select([
|
|
1771
|
-
kysely.sql`COUNT(*)`.as("totalRequests"),
|
|
1772
|
-
kysely.sql`COUNT(CASE WHEN ${col$1("statusCode")} >= 200 AND ${col$1("statusCode")} < 300 THEN 1 END)`.as("successfulRequests"),
|
|
1773
|
-
kysely.sql`COUNT(CASE WHEN ${col$1("statusCode")} >= 400 THEN 1 END)`.as("failedRequests"),
|
|
1774
|
-
kysely.sql`COUNT(CASE WHEN ${col$1("isStreaming")} = true THEN 1 END)`.as("streamingRequests"),
|
|
1775
|
-
kysely.sql`AVG(${col$1("latencyMs")})`.as("avgLatencyMs"),
|
|
1776
|
-
kysely.sql`MAX(${col$1("latencyMs")})`.as("maxLatencyMs"),
|
|
1777
|
-
kysely.sql`MIN(${col$1("latencyMs")})`.as("minLatencyMs")
|
|
1778
|
-
]).where(kysely.sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col$1("createdAt")} <= ${endDate.toISOString()}`);
|
|
1779
|
-
if (configId) query = query.where("configId", "=", configId);
|
|
1780
|
-
if (variantId) query = query.where("variantId", "=", variantId);
|
|
1781
|
-
if (environmentId) query = query.where("environmentId", "=", environmentId);
|
|
1782
|
-
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
1783
|
-
if (values.length === 0) continue;
|
|
1784
|
-
if (values.length === 1) query = query.where(kysely.sql`${col$1("tags")}->>${key} = ${values[0]}`);
|
|
1785
|
-
else {
|
|
1786
|
-
const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
|
|
1787
|
-
query = query.where(kysely.sql`${col$1("tags")}->>${key} IN (${valueList})`);
|
|
1788
|
-
}
|
|
1789
|
-
}
|
|
1790
|
-
return await query.executeTakeFirst();
|
|
1791
|
-
},
|
|
1792
|
-
getDistinctTags: async () => {
|
|
1793
|
-
return (await kysely.sql`
|
|
1794
|
-
SELECT DISTINCT key, value
|
|
1795
|
-
FROM llm_requests, jsonb_each_text(tags) AS t(key, value)
|
|
1796
|
-
WHERE tags != '{}'::jsonb
|
|
1797
|
-
ORDER BY key, value
|
|
1798
|
-
`.execute(db)).rows;
|
|
1799
|
-
}
|
|
1800
|
-
};
|
|
1801
|
-
};
|
|
1802
|
-
|
|
1803
|
-
//#endregion
|
|
1804
|
-
//#region src/datalayer/playgrounds.ts
|
|
1805
|
-
const createNewPlayground = require_db.zod_default.object({
|
|
1806
|
-
name: require_db.zod_default.string(),
|
|
1807
|
-
datasetId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
1808
|
-
columns: require_db.zod_default.array(require_db.playgroundColumnSchema).nullable().optional()
|
|
1809
|
-
});
|
|
1810
|
-
const updatePlayground = require_db.zod_default.object({
|
|
1811
|
-
playgroundId: require_db.zod_default.uuidv4(),
|
|
1812
|
-
name: require_db.zod_default.string().optional(),
|
|
1813
|
-
datasetId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
1814
|
-
columns: require_db.zod_default.array(require_db.playgroundColumnSchema).nullable().optional()
|
|
1815
|
-
});
|
|
1816
|
-
const getPlaygroundById = require_db.zod_default.object({ playgroundId: require_db.zod_default.uuidv4() });
|
|
1817
|
-
const deletePlayground = require_db.zod_default.object({ playgroundId: require_db.zod_default.uuidv4() });
|
|
1818
|
-
const listPlaygrounds = require_db.zod_default.object({
|
|
1819
|
-
limit: require_db.zod_default.number().int().positive().optional(),
|
|
1820
|
-
offset: require_db.zod_default.number().int().nonnegative().optional()
|
|
1821
|
-
});
|
|
1822
|
-
const createPlaygroundDataLayer = (db) => {
|
|
1823
|
-
return {
|
|
1824
|
-
createNewPlayground: async (params) => {
|
|
1825
|
-
const value = await createNewPlayground.safeParseAsync(params);
|
|
1826
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1827
|
-
const { name, datasetId, columns } = value.data;
|
|
1828
|
-
return db.insertInto("playgrounds").values({
|
|
1829
|
-
id: (0, node_crypto.randomUUID)(),
|
|
1830
|
-
name,
|
|
1831
|
-
datasetId: datasetId ?? null,
|
|
1832
|
-
columns: columns ? JSON.stringify(columns) : null,
|
|
1833
|
-
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1834
|
-
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1835
|
-
}).returningAll().executeTakeFirst();
|
|
1836
|
-
},
|
|
1837
|
-
updatePlayground: async (params) => {
|
|
1838
|
-
const value = await updatePlayground.safeParseAsync(params);
|
|
1839
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1840
|
-
const { playgroundId, name, datasetId, columns } = value.data;
|
|
1841
|
-
const updateData = { updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
1842
|
-
if (name !== void 0) updateData.name = name;
|
|
1843
|
-
if (datasetId !== void 0) updateData.datasetId = datasetId;
|
|
1844
|
-
if (columns !== void 0) updateData.columns = columns ? JSON.stringify(columns) : null;
|
|
1845
|
-
return db.updateTable("playgrounds").set(updateData).where("id", "=", playgroundId).returningAll().executeTakeFirst();
|
|
1846
|
-
},
|
|
1847
|
-
getPlaygroundById: async (params) => {
|
|
1848
|
-
const value = await getPlaygroundById.safeParseAsync(params);
|
|
1849
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1850
|
-
const { playgroundId } = value.data;
|
|
1851
|
-
return db.selectFrom("playgrounds").selectAll().where("id", "=", playgroundId).executeTakeFirst();
|
|
1433
|
+
getPlaygroundById: async (params) => {
|
|
1434
|
+
const value = await getPlaygroundById.safeParseAsync(params);
|
|
1435
|
+
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1436
|
+
const { playgroundId } = value.data;
|
|
1437
|
+
return db.selectFrom("playgrounds").selectAll().where("id", "=", playgroundId).executeTakeFirst();
|
|
1852
1438
|
},
|
|
1853
1439
|
deletePlayground: async (params) => {
|
|
1854
1440
|
const value = await deletePlayground.safeParseAsync(params);
|
|
@@ -2095,292 +1681,13 @@ const createPlaygroundRunsDataLayer = (db) => {
|
|
|
2095
1681
|
};
|
|
2096
1682
|
|
|
2097
1683
|
//#endregion
|
|
2098
|
-
//#region src/datalayer/
|
|
2099
|
-
const
|
|
2100
|
-
/**
|
|
2101
|
-
* Schema for upserting a trace
|
|
2102
|
-
*/
|
|
2103
|
-
const upsertTraceSchema = require_db.zod_default.object({
|
|
2104
|
-
traceId: require_db.zod_default.string(),
|
|
1684
|
+
//#region src/datalayer/workspaceSettings.ts
|
|
1685
|
+
const updateWorkspaceSettings = require_db.zod_default.object({
|
|
2105
1686
|
name: require_db.zod_default.string().nullable().optional(),
|
|
2106
|
-
|
|
2107
|
-
|
|
2108
|
-
status: require_db.zod_default.enum([
|
|
2109
|
-
"unset",
|
|
2110
|
-
"ok",
|
|
2111
|
-
"error"
|
|
2112
|
-
]).default("unset"),
|
|
2113
|
-
startTime: require_db.zod_default.date(),
|
|
2114
|
-
endTime: require_db.zod_default.date().nullable().optional(),
|
|
2115
|
-
durationMs: require_db.zod_default.number().int().nullable().optional(),
|
|
2116
|
-
spanCount: require_db.zod_default.number().int().default(1),
|
|
2117
|
-
totalInputTokens: require_db.zod_default.number().int().default(0),
|
|
2118
|
-
totalOutputTokens: require_db.zod_default.number().int().default(0),
|
|
2119
|
-
totalTokens: require_db.zod_default.number().int().default(0),
|
|
2120
|
-
totalCost: require_db.zod_default.number().int().default(0),
|
|
2121
|
-
tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.string()).default({}),
|
|
2122
|
-
metadata: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.unknown()).default({})
|
|
2123
|
-
});
|
|
2124
|
-
/**
|
|
2125
|
-
* Schema for inserting spans
|
|
2126
|
-
*/
|
|
2127
|
-
const insertSpanSchema = require_db.zod_default.object({
|
|
2128
|
-
traceId: require_db.zod_default.string(),
|
|
2129
|
-
spanId: require_db.zod_default.string(),
|
|
2130
|
-
parentSpanId: require_db.zod_default.string().nullable().optional(),
|
|
2131
|
-
name: require_db.zod_default.string(),
|
|
2132
|
-
kind: require_db.zod_default.number().int().default(1),
|
|
2133
|
-
status: require_db.zod_default.number().int().default(0),
|
|
2134
|
-
statusMessage: require_db.zod_default.string().nullable().optional(),
|
|
2135
|
-
startTime: require_db.zod_default.date(),
|
|
2136
|
-
endTime: require_db.zod_default.date().nullable().optional(),
|
|
2137
|
-
durationMs: require_db.zod_default.number().int().nullable().optional(),
|
|
2138
|
-
provider: require_db.zod_default.string().nullable().optional(),
|
|
2139
|
-
model: require_db.zod_default.string().nullable().optional(),
|
|
2140
|
-
promptTokens: require_db.zod_default.number().int().default(0),
|
|
2141
|
-
completionTokens: require_db.zod_default.number().int().default(0),
|
|
2142
|
-
totalTokens: require_db.zod_default.number().int().default(0),
|
|
2143
|
-
cost: require_db.zod_default.number().int().default(0),
|
|
2144
|
-
configId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2145
|
-
variantId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2146
|
-
environmentId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2147
|
-
providerConfigId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2148
|
-
requestId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2149
|
-
source: require_db.zod_default.enum([
|
|
2150
|
-
"gateway",
|
|
2151
|
-
"otlp",
|
|
2152
|
-
"langsmith"
|
|
2153
|
-
]).default("gateway"),
|
|
2154
|
-
input: require_db.zod_default.unknown().nullable().optional(),
|
|
2155
|
-
output: require_db.zod_default.unknown().nullable().optional(),
|
|
2156
|
-
attributes: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.unknown()).default({})
|
|
2157
|
-
});
|
|
2158
|
-
/**
|
|
2159
|
-
* Schema for inserting span events
|
|
2160
|
-
*/
|
|
2161
|
-
const insertSpanEventSchema = require_db.zod_default.object({
|
|
2162
|
-
traceId: require_db.zod_default.string(),
|
|
2163
|
-
spanId: require_db.zod_default.string(),
|
|
2164
|
-
name: require_db.zod_default.string(),
|
|
2165
|
-
timestamp: require_db.zod_default.date(),
|
|
2166
|
-
attributes: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.unknown()).default({})
|
|
2167
|
-
});
|
|
2168
|
-
/**
|
|
2169
|
-
* Schema for listing traces
|
|
2170
|
-
*/
|
|
2171
|
-
const listTracesSchema = require_db.zod_default.object({
|
|
2172
|
-
limit: require_db.zod_default.number().int().positive().max(1e3).default(50),
|
|
2173
|
-
offset: require_db.zod_default.number().int().nonnegative().default(0),
|
|
2174
|
-
sessionId: require_db.zod_default.string().optional(),
|
|
2175
|
-
userId: require_db.zod_default.string().optional(),
|
|
2176
|
-
status: require_db.zod_default.enum([
|
|
2177
|
-
"unset",
|
|
2178
|
-
"ok",
|
|
2179
|
-
"error"
|
|
2180
|
-
]).optional(),
|
|
2181
|
-
name: require_db.zod_default.string().optional(),
|
|
2182
|
-
startDate: require_db.zod_default.date().optional(),
|
|
2183
|
-
endDate: require_db.zod_default.date().optional(),
|
|
2184
|
-
tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.array(require_db.zod_default.string())).optional()
|
|
2185
|
-
});
|
|
2186
|
-
/**
|
|
2187
|
-
* Schema for trace stats query
|
|
2188
|
-
*/
|
|
2189
|
-
const traceStatsSchema = require_db.zod_default.object({
|
|
2190
|
-
startDate: require_db.zod_default.date(),
|
|
2191
|
-
endDate: require_db.zod_default.date(),
|
|
2192
|
-
sessionId: require_db.zod_default.string().optional(),
|
|
2193
|
-
userId: require_db.zod_default.string().optional()
|
|
1687
|
+
setupComplete: require_db.zod_default.boolean().optional(),
|
|
1688
|
+
superAdminId: require_db.zod_default.string().nullable().optional()
|
|
2194
1689
|
});
|
|
2195
|
-
const
|
|
2196
|
-
return {
|
|
2197
|
-
upsertTrace: async (data) => {
|
|
2198
|
-
const result = await upsertTraceSchema.safeParseAsync(data);
|
|
2199
|
-
if (!result.success) throw new LLMOpsError(`Invalid trace data: ${result.error.message}`);
|
|
2200
|
-
const trace = result.data;
|
|
2201
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
2202
|
-
await kysely.sql`
|
|
2203
|
-
INSERT INTO "traces" (
|
|
2204
|
-
"id", "traceId", "name", "sessionId", "userId", "status",
|
|
2205
|
-
"startTime", "endTime", "durationMs", "spanCount",
|
|
2206
|
-
"totalInputTokens", "totalOutputTokens", "totalTokens", "totalCost",
|
|
2207
|
-
"tags", "metadata", "createdAt", "updatedAt"
|
|
2208
|
-
) VALUES (
|
|
2209
|
-
${(0, node_crypto.randomUUID)()}, ${trace.traceId}, ${trace.name ?? null}, ${trace.sessionId ?? null},
|
|
2210
|
-
${trace.userId ?? null}, ${trace.status},
|
|
2211
|
-
${trace.startTime.toISOString()}, ${trace.endTime?.toISOString() ?? null},
|
|
2212
|
-
${trace.durationMs ?? null}, ${trace.spanCount},
|
|
2213
|
-
${trace.totalInputTokens}, ${trace.totalOutputTokens},
|
|
2214
|
-
${trace.totalTokens}, ${trace.totalCost},
|
|
2215
|
-
${JSON.stringify(trace.tags)}::jsonb, ${JSON.stringify(trace.metadata)}::jsonb,
|
|
2216
|
-
${now}, ${now}
|
|
2217
|
-
)
|
|
2218
|
-
ON CONFLICT ("traceId") DO UPDATE SET
|
|
2219
|
-
"name" = COALESCE(EXCLUDED."name", "traces"."name"),
|
|
2220
|
-
"sessionId" = COALESCE(EXCLUDED."sessionId", "traces"."sessionId"),
|
|
2221
|
-
"userId" = COALESCE(EXCLUDED."userId", "traces"."userId"),
|
|
2222
|
-
"status" = CASE
|
|
2223
|
-
WHEN EXCLUDED."status" = 'error' THEN 'error'
|
|
2224
|
-
WHEN EXCLUDED."status" = 'ok' AND "traces"."status" != 'error' THEN 'ok'
|
|
2225
|
-
ELSE "traces"."status"
|
|
2226
|
-
END,
|
|
2227
|
-
"startTime" = LEAST("traces"."startTime", EXCLUDED."startTime"),
|
|
2228
|
-
"endTime" = GREATEST(
|
|
2229
|
-
COALESCE("traces"."endTime", EXCLUDED."endTime"),
|
|
2230
|
-
COALESCE(EXCLUDED."endTime", "traces"."endTime")
|
|
2231
|
-
),
|
|
2232
|
-
"durationMs" = EXTRACT(EPOCH FROM (
|
|
2233
|
-
GREATEST(
|
|
2234
|
-
COALESCE("traces"."endTime", EXCLUDED."endTime"),
|
|
2235
|
-
COALESCE(EXCLUDED."endTime", "traces"."endTime")
|
|
2236
|
-
) -
|
|
2237
|
-
LEAST("traces"."startTime", EXCLUDED."startTime")
|
|
2238
|
-
))::integer * 1000,
|
|
2239
|
-
"spanCount" = "traces"."spanCount" + EXCLUDED."spanCount",
|
|
2240
|
-
"totalInputTokens" = "traces"."totalInputTokens" + EXCLUDED."totalInputTokens",
|
|
2241
|
-
"totalOutputTokens" = "traces"."totalOutputTokens" + EXCLUDED."totalOutputTokens",
|
|
2242
|
-
"totalTokens" = "traces"."totalTokens" + EXCLUDED."totalTokens",
|
|
2243
|
-
"totalCost" = "traces"."totalCost" + EXCLUDED."totalCost",
|
|
2244
|
-
"tags" = "traces"."tags" || EXCLUDED."tags",
|
|
2245
|
-
"metadata" = "traces"."metadata" || EXCLUDED."metadata",
|
|
2246
|
-
"updatedAt" = ${now}
|
|
2247
|
-
`.execute(db);
|
|
2248
|
-
},
|
|
2249
|
-
batchInsertSpans: async (spans) => {
|
|
2250
|
-
if (spans.length === 0) return { count: 0 };
|
|
2251
|
-
const validatedSpans = [];
|
|
2252
|
-
for (const span of spans) {
|
|
2253
|
-
const result = await insertSpanSchema.safeParseAsync(span);
|
|
2254
|
-
if (!result.success) {
|
|
2255
|
-
require_db.logger.warn(`[batchInsertSpans] Skipping invalid span ${span.spanId}: ${result.error.message}`);
|
|
2256
|
-
continue;
|
|
2257
|
-
}
|
|
2258
|
-
validatedSpans.push(result.data);
|
|
2259
|
-
}
|
|
2260
|
-
if (validatedSpans.length === 0) return { count: 0 };
|
|
2261
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
2262
|
-
const values = validatedSpans.map((span) => ({
|
|
2263
|
-
id: (0, node_crypto.randomUUID)(),
|
|
2264
|
-
traceId: span.traceId,
|
|
2265
|
-
spanId: span.spanId,
|
|
2266
|
-
parentSpanId: span.parentSpanId ?? null,
|
|
2267
|
-
name: span.name,
|
|
2268
|
-
kind: span.kind,
|
|
2269
|
-
status: span.status,
|
|
2270
|
-
statusMessage: span.statusMessage ?? null,
|
|
2271
|
-
startTime: span.startTime.toISOString(),
|
|
2272
|
-
endTime: span.endTime?.toISOString() ?? null,
|
|
2273
|
-
durationMs: span.durationMs ?? null,
|
|
2274
|
-
provider: span.provider ?? null,
|
|
2275
|
-
model: span.model ?? null,
|
|
2276
|
-
promptTokens: span.promptTokens,
|
|
2277
|
-
completionTokens: span.completionTokens,
|
|
2278
|
-
totalTokens: span.totalTokens,
|
|
2279
|
-
cost: span.cost,
|
|
2280
|
-
configId: span.configId ?? null,
|
|
2281
|
-
variantId: span.variantId ?? null,
|
|
2282
|
-
environmentId: span.environmentId ?? null,
|
|
2283
|
-
providerConfigId: span.providerConfigId ?? null,
|
|
2284
|
-
requestId: span.requestId ?? null,
|
|
2285
|
-
source: span.source,
|
|
2286
|
-
input: span.input != null ? JSON.stringify(span.input) : null,
|
|
2287
|
-
output: span.output != null ? JSON.stringify(span.output) : null,
|
|
2288
|
-
attributes: JSON.stringify(span.attributes),
|
|
2289
|
-
createdAt: now,
|
|
2290
|
-
updatedAt: now
|
|
2291
|
-
}));
|
|
2292
|
-
await db.insertInto("spans").values(values).onConflict((oc) => oc.column("spanId").doNothing()).execute();
|
|
2293
|
-
return { count: values.length };
|
|
2294
|
-
},
|
|
2295
|
-
batchInsertSpanEvents: async (events) => {
|
|
2296
|
-
if (events.length === 0) return { count: 0 };
|
|
2297
|
-
const validatedEvents = [];
|
|
2298
|
-
for (const event of events) {
|
|
2299
|
-
const result = await insertSpanEventSchema.safeParseAsync(event);
|
|
2300
|
-
if (!result.success) {
|
|
2301
|
-
require_db.logger.warn(`[batchInsertSpanEvents] Skipping invalid event: ${result.error.message}`);
|
|
2302
|
-
continue;
|
|
2303
|
-
}
|
|
2304
|
-
validatedEvents.push(result.data);
|
|
2305
|
-
}
|
|
2306
|
-
if (validatedEvents.length === 0) return { count: 0 };
|
|
2307
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
2308
|
-
const values = validatedEvents.map((event) => ({
|
|
2309
|
-
id: (0, node_crypto.randomUUID)(),
|
|
2310
|
-
traceId: event.traceId,
|
|
2311
|
-
spanId: event.spanId,
|
|
2312
|
-
name: event.name,
|
|
2313
|
-
timestamp: event.timestamp.toISOString(),
|
|
2314
|
-
attributes: JSON.stringify(event.attributes),
|
|
2315
|
-
createdAt: now
|
|
2316
|
-
}));
|
|
2317
|
-
await db.insertInto("span_events").values(values).execute();
|
|
2318
|
-
return { count: values.length };
|
|
2319
|
-
},
|
|
2320
|
-
listTraces: async (params) => {
|
|
2321
|
-
const result = await listTracesSchema.safeParseAsync(params || {});
|
|
2322
|
-
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2323
|
-
const { limit, offset, sessionId, userId, status, name, startDate, endDate, tags } = result.data;
|
|
2324
|
-
let baseQuery = db.selectFrom("traces");
|
|
2325
|
-
if (sessionId) baseQuery = baseQuery.where("sessionId", "=", sessionId);
|
|
2326
|
-
if (userId) baseQuery = baseQuery.where("userId", "=", userId);
|
|
2327
|
-
if (status) baseQuery = baseQuery.where("status", "=", status);
|
|
2328
|
-
if (name) baseQuery = baseQuery.where(kysely.sql`${col("name")} ILIKE ${"%" + name + "%"}`);
|
|
2329
|
-
if (startDate) baseQuery = baseQuery.where(kysely.sql`${col("startTime")} >= ${startDate.toISOString()}`);
|
|
2330
|
-
if (endDate) baseQuery = baseQuery.where(kysely.sql`${col("startTime")} <= ${endDate.toISOString()}`);
|
|
2331
|
-
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
2332
|
-
if (values.length === 0) continue;
|
|
2333
|
-
if (values.length === 1) baseQuery = baseQuery.where(kysely.sql`${col("tags")}->>${key} = ${values[0]}`);
|
|
2334
|
-
else {
|
|
2335
|
-
const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
|
|
2336
|
-
baseQuery = baseQuery.where(kysely.sql`${col("tags")}->>${key} IN (${valueList})`);
|
|
2337
|
-
}
|
|
2338
|
-
}
|
|
2339
|
-
const countResult = await baseQuery.select(kysely.sql`COUNT(*)`.as("total")).executeTakeFirst();
|
|
2340
|
-
const total = Number(countResult?.total ?? 0);
|
|
2341
|
-
return {
|
|
2342
|
-
data: await baseQuery.selectAll().orderBy("startTime", "desc").limit(limit).offset(offset).execute(),
|
|
2343
|
-
total,
|
|
2344
|
-
limit,
|
|
2345
|
-
offset
|
|
2346
|
-
};
|
|
2347
|
-
},
|
|
2348
|
-
getTraceWithSpans: async (traceId) => {
|
|
2349
|
-
const trace = await db.selectFrom("traces").selectAll().where("traceId", "=", traceId).executeTakeFirst();
|
|
2350
|
-
if (!trace) return void 0;
|
|
2351
|
-
return {
|
|
2352
|
-
trace,
|
|
2353
|
-
spans: await db.selectFrom("spans").selectAll().where("traceId", "=", traceId).orderBy("startTime", "asc").execute(),
|
|
2354
|
-
events: await db.selectFrom("span_events").selectAll().where("traceId", "=", traceId).orderBy("timestamp", "asc").execute()
|
|
2355
|
-
};
|
|
2356
|
-
},
|
|
2357
|
-
getTraceStats: async (params) => {
|
|
2358
|
-
const result = await traceStatsSchema.safeParseAsync(params);
|
|
2359
|
-
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2360
|
-
const { startDate, endDate, sessionId, userId } = result.data;
|
|
2361
|
-
let query = db.selectFrom("traces").select([
|
|
2362
|
-
kysely.sql`COUNT(*)`.as("totalTraces"),
|
|
2363
|
-
kysely.sql`COALESCE(AVG(${col("durationMs")}), 0)`.as("avgDurationMs"),
|
|
2364
|
-
kysely.sql`COUNT(CASE WHEN ${col("status")} = 'error' THEN 1 END)`.as("errorCount"),
|
|
2365
|
-
kysely.sql`COALESCE(SUM(${col("totalCost")}), 0)`.as("totalCost"),
|
|
2366
|
-
kysely.sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
|
|
2367
|
-
kysely.sql`COALESCE(SUM(${col("spanCount")}), 0)`.as("totalSpans")
|
|
2368
|
-
]).where(kysely.sql`${col("startTime")} >= ${startDate.toISOString()}`).where(kysely.sql`${col("startTime")} <= ${endDate.toISOString()}`);
|
|
2369
|
-
if (sessionId) query = query.where("sessionId", "=", sessionId);
|
|
2370
|
-
if (userId) query = query.where("userId", "=", userId);
|
|
2371
|
-
return query.executeTakeFirst();
|
|
2372
|
-
}
|
|
2373
|
-
};
|
|
2374
|
-
};
|
|
2375
|
-
|
|
2376
|
-
//#endregion
|
|
2377
|
-
//#region src/datalayer/workspaceSettings.ts
|
|
2378
|
-
const updateWorkspaceSettings = require_db.zod_default.object({
|
|
2379
|
-
name: require_db.zod_default.string().nullable().optional(),
|
|
2380
|
-
setupComplete: require_db.zod_default.boolean().optional(),
|
|
2381
|
-
superAdminId: require_db.zod_default.string().nullable().optional()
|
|
2382
|
-
});
|
|
2383
|
-
const createWorkspaceSettingsDataLayer = (db) => {
|
|
1690
|
+
const createWorkspaceSettingsDataLayer = (db) => {
|
|
2384
1691
|
return {
|
|
2385
1692
|
getWorkspaceSettings: async () => {
|
|
2386
1693
|
let settings = await db.selectFrom("workspace_settings").selectAll().executeTakeFirst();
|
|
@@ -2466,11 +1773,9 @@ const createWorkspaceSettingsDataLayer = (db) => {
|
|
|
2466
1773
|
function createDataLayer(db) {
|
|
2467
1774
|
return {
|
|
2468
1775
|
...createDatasetsDataLayer(db),
|
|
2469
|
-
...createLLMRequestsDataLayer(db),
|
|
2470
1776
|
...createPlaygroundDataLayer(db),
|
|
2471
1777
|
...createPlaygroundResultsDataLayer(db),
|
|
2472
1778
|
...createPlaygroundRunsDataLayer(db),
|
|
2473
|
-
...createTracesDataLayer(db),
|
|
2474
1779
|
...createWorkspaceSettingsDataLayer(db)
|
|
2475
1780
|
};
|
|
2476
1781
|
}
|
|
@@ -2750,12 +2055,659 @@ function getDefaultPricingProvider() {
|
|
|
2750
2055
|
|
|
2751
2056
|
//#endregion
|
|
2752
2057
|
//#region src/telemetry/postgres.ts
|
|
2058
|
+
const guardrailResultSchema = require_db.zod_default.object({
|
|
2059
|
+
checkId: require_db.zod_default.string(),
|
|
2060
|
+
functionId: require_db.zod_default.string(),
|
|
2061
|
+
hookType: require_db.zod_default.enum(["beforeRequestHook", "afterRequestHook"]),
|
|
2062
|
+
verdict: require_db.zod_default.boolean(),
|
|
2063
|
+
latencyMs: require_db.zod_default.number()
|
|
2064
|
+
});
|
|
2065
|
+
const guardrailResultsSchema = require_db.zod_default.object({
|
|
2066
|
+
results: require_db.zod_default.array(guardrailResultSchema),
|
|
2067
|
+
action: require_db.zod_default.enum([
|
|
2068
|
+
"allowed",
|
|
2069
|
+
"blocked",
|
|
2070
|
+
"logged"
|
|
2071
|
+
]),
|
|
2072
|
+
totalLatencyMs: require_db.zod_default.number()
|
|
2073
|
+
});
|
|
2074
|
+
const insertLLMRequestSchema = require_db.zod_default.object({
|
|
2075
|
+
requestId: require_db.zod_default.string().uuid(),
|
|
2076
|
+
configId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2077
|
+
variantId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2078
|
+
environmentId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2079
|
+
providerConfigId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2080
|
+
provider: require_db.zod_default.string(),
|
|
2081
|
+
model: require_db.zod_default.string(),
|
|
2082
|
+
promptTokens: require_db.zod_default.number().int().default(0),
|
|
2083
|
+
completionTokens: require_db.zod_default.number().int().default(0),
|
|
2084
|
+
totalTokens: require_db.zod_default.number().int().default(0),
|
|
2085
|
+
cachedTokens: require_db.zod_default.number().int().default(0),
|
|
2086
|
+
cacheCreationTokens: require_db.zod_default.number().int().default(0),
|
|
2087
|
+
cost: require_db.zod_default.number().int().default(0),
|
|
2088
|
+
cacheSavings: require_db.zod_default.number().int().default(0),
|
|
2089
|
+
inputCost: require_db.zod_default.number().int().default(0),
|
|
2090
|
+
outputCost: require_db.zod_default.number().int().default(0),
|
|
2091
|
+
endpoint: require_db.zod_default.string(),
|
|
2092
|
+
statusCode: require_db.zod_default.number().int(),
|
|
2093
|
+
latencyMs: require_db.zod_default.number().int().default(0),
|
|
2094
|
+
isStreaming: require_db.zod_default.boolean().default(false),
|
|
2095
|
+
userId: require_db.zod_default.string().nullable().optional(),
|
|
2096
|
+
tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.string()).default({}),
|
|
2097
|
+
guardrailResults: guardrailResultsSchema.nullable().optional(),
|
|
2098
|
+
traceId: require_db.zod_default.string().nullable().optional(),
|
|
2099
|
+
spanId: require_db.zod_default.string().nullable().optional(),
|
|
2100
|
+
parentSpanId: require_db.zod_default.string().nullable().optional(),
|
|
2101
|
+
sessionId: require_db.zod_default.string().nullable().optional()
|
|
2102
|
+
});
|
|
2103
|
+
const listRequestsSchema = require_db.zod_default.object({
|
|
2104
|
+
limit: require_db.zod_default.number().int().positive().max(1e3).default(100),
|
|
2105
|
+
offset: require_db.zod_default.number().int().nonnegative().default(0),
|
|
2106
|
+
configId: require_db.zod_default.string().uuid().optional(),
|
|
2107
|
+
variantId: require_db.zod_default.string().uuid().optional(),
|
|
2108
|
+
environmentId: require_db.zod_default.string().uuid().optional(),
|
|
2109
|
+
providerConfigId: require_db.zod_default.string().uuid().optional(),
|
|
2110
|
+
provider: require_db.zod_default.string().optional(),
|
|
2111
|
+
model: require_db.zod_default.string().optional(),
|
|
2112
|
+
startDate: require_db.zod_default.date().optional(),
|
|
2113
|
+
endDate: require_db.zod_default.date().optional(),
|
|
2114
|
+
tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.array(require_db.zod_default.string())).optional()
|
|
2115
|
+
});
|
|
2116
|
+
const dateRangeSchema = require_db.zod_default.object({
|
|
2117
|
+
startDate: require_db.zod_default.date(),
|
|
2118
|
+
endDate: require_db.zod_default.date(),
|
|
2119
|
+
configId: require_db.zod_default.string().uuid().optional(),
|
|
2120
|
+
variantId: require_db.zod_default.string().uuid().optional(),
|
|
2121
|
+
environmentId: require_db.zod_default.string().uuid().optional(),
|
|
2122
|
+
tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.array(require_db.zod_default.string())).optional()
|
|
2123
|
+
});
|
|
2124
|
+
const COST_SUMMARY_GROUP_BY = [
|
|
2125
|
+
"day",
|
|
2126
|
+
"hour",
|
|
2127
|
+
"model",
|
|
2128
|
+
"provider",
|
|
2129
|
+
"endpoint",
|
|
2130
|
+
"tags"
|
|
2131
|
+
];
|
|
2132
|
+
const costSummarySchema = require_db.zod_default.object({
|
|
2133
|
+
startDate: require_db.zod_default.date(),
|
|
2134
|
+
endDate: require_db.zod_default.date(),
|
|
2135
|
+
configId: require_db.zod_default.string().uuid().optional(),
|
|
2136
|
+
variantId: require_db.zod_default.string().uuid().optional(),
|
|
2137
|
+
environmentId: require_db.zod_default.string().uuid().optional(),
|
|
2138
|
+
tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.array(require_db.zod_default.string())).optional(),
|
|
2139
|
+
groupBy: require_db.zod_default.enum(COST_SUMMARY_GROUP_BY).optional(),
|
|
2140
|
+
tagKeys: require_db.zod_default.array(require_db.zod_default.string()).optional()
|
|
2141
|
+
});
|
|
2142
|
+
const upsertTraceSchema = require_db.zod_default.object({
|
|
2143
|
+
traceId: require_db.zod_default.string(),
|
|
2144
|
+
name: require_db.zod_default.string().nullable().optional(),
|
|
2145
|
+
sessionId: require_db.zod_default.string().nullable().optional(),
|
|
2146
|
+
userId: require_db.zod_default.string().nullable().optional(),
|
|
2147
|
+
status: require_db.zod_default.enum([
|
|
2148
|
+
"unset",
|
|
2149
|
+
"ok",
|
|
2150
|
+
"error"
|
|
2151
|
+
]).default("unset"),
|
|
2152
|
+
startTime: require_db.zod_default.date(),
|
|
2153
|
+
endTime: require_db.zod_default.date().nullable().optional(),
|
|
2154
|
+
durationMs: require_db.zod_default.number().int().nullable().optional(),
|
|
2155
|
+
spanCount: require_db.zod_default.number().int().default(1),
|
|
2156
|
+
totalInputTokens: require_db.zod_default.number().int().default(0),
|
|
2157
|
+
totalOutputTokens: require_db.zod_default.number().int().default(0),
|
|
2158
|
+
totalTokens: require_db.zod_default.number().int().default(0),
|
|
2159
|
+
totalCost: require_db.zod_default.number().int().default(0),
|
|
2160
|
+
tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.string()).default({}),
|
|
2161
|
+
metadata: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.unknown()).default({})
|
|
2162
|
+
});
|
|
2163
|
+
const insertSpanSchema = require_db.zod_default.object({
|
|
2164
|
+
traceId: require_db.zod_default.string(),
|
|
2165
|
+
spanId: require_db.zod_default.string(),
|
|
2166
|
+
parentSpanId: require_db.zod_default.string().nullable().optional(),
|
|
2167
|
+
name: require_db.zod_default.string(),
|
|
2168
|
+
kind: require_db.zod_default.number().int().default(1),
|
|
2169
|
+
status: require_db.zod_default.number().int().default(0),
|
|
2170
|
+
statusMessage: require_db.zod_default.string().nullable().optional(),
|
|
2171
|
+
startTime: require_db.zod_default.date(),
|
|
2172
|
+
endTime: require_db.zod_default.date().nullable().optional(),
|
|
2173
|
+
durationMs: require_db.zod_default.number().int().nullable().optional(),
|
|
2174
|
+
provider: require_db.zod_default.string().nullable().optional(),
|
|
2175
|
+
model: require_db.zod_default.string().nullable().optional(),
|
|
2176
|
+
promptTokens: require_db.zod_default.number().int().default(0),
|
|
2177
|
+
completionTokens: require_db.zod_default.number().int().default(0),
|
|
2178
|
+
totalTokens: require_db.zod_default.number().int().default(0),
|
|
2179
|
+
cost: require_db.zod_default.number().int().default(0),
|
|
2180
|
+
configId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2181
|
+
variantId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2182
|
+
environmentId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2183
|
+
providerConfigId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2184
|
+
requestId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2185
|
+
source: require_db.zod_default.enum([
|
|
2186
|
+
"gateway",
|
|
2187
|
+
"otlp",
|
|
2188
|
+
"langsmith"
|
|
2189
|
+
]).default("gateway"),
|
|
2190
|
+
input: require_db.zod_default.unknown().nullable().optional(),
|
|
2191
|
+
output: require_db.zod_default.unknown().nullable().optional(),
|
|
2192
|
+
attributes: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.unknown()).default({})
|
|
2193
|
+
});
|
|
2194
|
+
const insertSpanEventSchema = require_db.zod_default.object({
|
|
2195
|
+
traceId: require_db.zod_default.string(),
|
|
2196
|
+
spanId: require_db.zod_default.string(),
|
|
2197
|
+
name: require_db.zod_default.string(),
|
|
2198
|
+
timestamp: require_db.zod_default.date(),
|
|
2199
|
+
attributes: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.unknown()).default({})
|
|
2200
|
+
});
|
|
2201
|
+
const listTracesSchema = require_db.zod_default.object({
|
|
2202
|
+
limit: require_db.zod_default.number().int().positive().max(1e3).default(50),
|
|
2203
|
+
offset: require_db.zod_default.number().int().nonnegative().default(0),
|
|
2204
|
+
sessionId: require_db.zod_default.string().optional(),
|
|
2205
|
+
userId: require_db.zod_default.string().optional(),
|
|
2206
|
+
status: require_db.zod_default.enum([
|
|
2207
|
+
"unset",
|
|
2208
|
+
"ok",
|
|
2209
|
+
"error"
|
|
2210
|
+
]).optional(),
|
|
2211
|
+
name: require_db.zod_default.string().optional(),
|
|
2212
|
+
startDate: require_db.zod_default.date().optional(),
|
|
2213
|
+
endDate: require_db.zod_default.date().optional(),
|
|
2214
|
+
tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.array(require_db.zod_default.string())).optional()
|
|
2215
|
+
});
|
|
2216
|
+
const traceStatsSchema = require_db.zod_default.object({
|
|
2217
|
+
startDate: require_db.zod_default.date(),
|
|
2218
|
+
endDate: require_db.zod_default.date(),
|
|
2219
|
+
sessionId: require_db.zod_default.string().optional(),
|
|
2220
|
+
userId: require_db.zod_default.string().optional()
|
|
2221
|
+
});
|
|
2222
|
+
const col = (name) => kysely.sql.ref(name);
|
|
2223
|
+
function createLLMRequestsStore(db) {
|
|
2224
|
+
return {
|
|
2225
|
+
batchInsertRequests: async (requests) => {
|
|
2226
|
+
if (requests.length === 0) return { count: 0 };
|
|
2227
|
+
const validatedRequests = await Promise.all(requests.map(async (req) => {
|
|
2228
|
+
const result = await insertLLMRequestSchema.safeParseAsync(req);
|
|
2229
|
+
if (!result.success) throw new LLMOpsError(`Invalid request data: ${result.error.message}`);
|
|
2230
|
+
return result.data;
|
|
2231
|
+
}));
|
|
2232
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
2233
|
+
const values = validatedRequests.map((req) => ({
|
|
2234
|
+
id: (0, node_crypto.randomUUID)(),
|
|
2235
|
+
requestId: req.requestId,
|
|
2236
|
+
configId: req.configId ?? null,
|
|
2237
|
+
variantId: req.variantId ?? null,
|
|
2238
|
+
environmentId: req.environmentId ?? null,
|
|
2239
|
+
providerConfigId: req.providerConfigId ?? null,
|
|
2240
|
+
provider: req.provider,
|
|
2241
|
+
model: req.model,
|
|
2242
|
+
promptTokens: req.promptTokens,
|
|
2243
|
+
completionTokens: req.completionTokens,
|
|
2244
|
+
totalTokens: req.totalTokens,
|
|
2245
|
+
cachedTokens: req.cachedTokens,
|
|
2246
|
+
cacheCreationTokens: req.cacheCreationTokens,
|
|
2247
|
+
cost: req.cost,
|
|
2248
|
+
cacheSavings: req.cacheSavings,
|
|
2249
|
+
inputCost: req.inputCost,
|
|
2250
|
+
outputCost: req.outputCost,
|
|
2251
|
+
endpoint: req.endpoint,
|
|
2252
|
+
statusCode: req.statusCode,
|
|
2253
|
+
latencyMs: req.latencyMs,
|
|
2254
|
+
isStreaming: req.isStreaming,
|
|
2255
|
+
userId: req.userId ?? null,
|
|
2256
|
+
tags: JSON.stringify(req.tags),
|
|
2257
|
+
guardrailResults: req.guardrailResults ? JSON.stringify(req.guardrailResults) : null,
|
|
2258
|
+
traceId: req.traceId ?? null,
|
|
2259
|
+
spanId: req.spanId ?? null,
|
|
2260
|
+
parentSpanId: req.parentSpanId ?? null,
|
|
2261
|
+
sessionId: req.sessionId ?? null,
|
|
2262
|
+
createdAt: now,
|
|
2263
|
+
updatedAt: now
|
|
2264
|
+
}));
|
|
2265
|
+
await db.insertInto("llm_requests").values(values).execute();
|
|
2266
|
+
return { count: values.length };
|
|
2267
|
+
},
|
|
2268
|
+
insertRequest: async (request) => {
|
|
2269
|
+
const result = await insertLLMRequestSchema.safeParseAsync(request);
|
|
2270
|
+
if (!result.success) throw new LLMOpsError(`Invalid request data: ${result.error.message}`);
|
|
2271
|
+
const req = result.data;
|
|
2272
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
2273
|
+
return db.insertInto("llm_requests").values({
|
|
2274
|
+
id: (0, node_crypto.randomUUID)(),
|
|
2275
|
+
requestId: req.requestId,
|
|
2276
|
+
configId: req.configId ?? null,
|
|
2277
|
+
variantId: req.variantId ?? null,
|
|
2278
|
+
environmentId: req.environmentId ?? null,
|
|
2279
|
+
providerConfigId: req.providerConfigId ?? null,
|
|
2280
|
+
provider: req.provider,
|
|
2281
|
+
model: req.model,
|
|
2282
|
+
promptTokens: req.promptTokens,
|
|
2283
|
+
completionTokens: req.completionTokens,
|
|
2284
|
+
totalTokens: req.totalTokens,
|
|
2285
|
+
cachedTokens: req.cachedTokens,
|
|
2286
|
+
cacheCreationTokens: req.cacheCreationTokens,
|
|
2287
|
+
cost: req.cost,
|
|
2288
|
+
cacheSavings: req.cacheSavings,
|
|
2289
|
+
inputCost: req.inputCost,
|
|
2290
|
+
outputCost: req.outputCost,
|
|
2291
|
+
endpoint: req.endpoint,
|
|
2292
|
+
statusCode: req.statusCode,
|
|
2293
|
+
latencyMs: req.latencyMs,
|
|
2294
|
+
isStreaming: req.isStreaming,
|
|
2295
|
+
userId: req.userId ?? null,
|
|
2296
|
+
tags: JSON.stringify(req.tags),
|
|
2297
|
+
guardrailResults: req.guardrailResults ? JSON.stringify(req.guardrailResults) : null,
|
|
2298
|
+
traceId: req.traceId ?? null,
|
|
2299
|
+
spanId: req.spanId ?? null,
|
|
2300
|
+
parentSpanId: req.parentSpanId ?? null,
|
|
2301
|
+
sessionId: req.sessionId ?? null,
|
|
2302
|
+
createdAt: now,
|
|
2303
|
+
updatedAt: now
|
|
2304
|
+
}).returningAll().executeTakeFirst();
|
|
2305
|
+
},
|
|
2306
|
+
listRequests: async (params) => {
|
|
2307
|
+
const result = await listRequestsSchema.safeParseAsync(params || {});
|
|
2308
|
+
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2309
|
+
const { limit, offset, configId, variantId, environmentId, providerConfigId, provider, model, startDate, endDate, tags } = result.data;
|
|
2310
|
+
let baseQuery = db.selectFrom("llm_requests");
|
|
2311
|
+
if (configId) baseQuery = baseQuery.where("configId", "=", configId);
|
|
2312
|
+
if (variantId) baseQuery = baseQuery.where("variantId", "=", variantId);
|
|
2313
|
+
if (environmentId) baseQuery = baseQuery.where("environmentId", "=", environmentId);
|
|
2314
|
+
if (providerConfigId) baseQuery = baseQuery.where("providerConfigId", "=", providerConfigId);
|
|
2315
|
+
if (provider) baseQuery = baseQuery.where("provider", "=", provider);
|
|
2316
|
+
if (model) baseQuery = baseQuery.where("model", "=", model);
|
|
2317
|
+
if (startDate) baseQuery = baseQuery.where(kysely.sql`${col("createdAt")} >= ${startDate.toISOString()}`);
|
|
2318
|
+
if (endDate) baseQuery = baseQuery.where(kysely.sql`${col("createdAt")} <= ${endDate.toISOString()}`);
|
|
2319
|
+
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
2320
|
+
if (values.length === 0) continue;
|
|
2321
|
+
if (values.length === 1) baseQuery = baseQuery.where(kysely.sql`${col("tags")}->>${key} = ${values[0]}`);
|
|
2322
|
+
else {
|
|
2323
|
+
const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
|
|
2324
|
+
baseQuery = baseQuery.where(kysely.sql`${col("tags")}->>${key} IN (${valueList})`);
|
|
2325
|
+
}
|
|
2326
|
+
}
|
|
2327
|
+
const countResult = await baseQuery.select(kysely.sql`COUNT(*)`.as("total")).executeTakeFirst();
|
|
2328
|
+
const total = Number(countResult?.total ?? 0);
|
|
2329
|
+
return {
|
|
2330
|
+
data: await baseQuery.selectAll().orderBy("createdAt", "desc").limit(limit).offset(offset).execute(),
|
|
2331
|
+
total,
|
|
2332
|
+
limit,
|
|
2333
|
+
offset
|
|
2334
|
+
};
|
|
2335
|
+
},
|
|
2336
|
+
getRequestByRequestId: async (requestId) => {
|
|
2337
|
+
return db.selectFrom("llm_requests").selectAll().where("requestId", "=", requestId).executeTakeFirst();
|
|
2338
|
+
},
|
|
2339
|
+
getTotalCost: async (params) => {
|
|
2340
|
+
const result = await dateRangeSchema.safeParseAsync(params);
|
|
2341
|
+
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2342
|
+
const { startDate, endDate, configId, variantId, environmentId, tags } = result.data;
|
|
2343
|
+
let query = db.selectFrom("llm_requests").select([
|
|
2344
|
+
kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2345
|
+
kysely.sql`COALESCE(SUM(${col("inputCost")}), 0)`.as("totalInputCost"),
|
|
2346
|
+
kysely.sql`COALESCE(SUM(${col("outputCost")}), 0)`.as("totalOutputCost"),
|
|
2347
|
+
kysely.sql`COALESCE(SUM(${col("promptTokens")}), 0)`.as("totalPromptTokens"),
|
|
2348
|
+
kysely.sql`COALESCE(SUM(${col("completionTokens")}), 0)`.as("totalCompletionTokens"),
|
|
2349
|
+
kysely.sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
|
|
2350
|
+
kysely.sql`COALESCE(SUM(${col("cachedTokens")}), 0)`.as("totalCachedTokens"),
|
|
2351
|
+
kysely.sql`COALESCE(SUM(${col("cacheSavings")}), 0)`.as("totalCacheSavings"),
|
|
2352
|
+
kysely.sql`COUNT(*)`.as("requestCount")
|
|
2353
|
+
]).where(kysely.sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col("createdAt")} <= ${endDate.toISOString()}`);
|
|
2354
|
+
if (configId) query = query.where("configId", "=", configId);
|
|
2355
|
+
if (variantId) query = query.where("variantId", "=", variantId);
|
|
2356
|
+
if (environmentId) query = query.where("environmentId", "=", environmentId);
|
|
2357
|
+
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
2358
|
+
if (values.length === 0) continue;
|
|
2359
|
+
if (values.length === 1) query = query.where(kysely.sql`${col("tags")}->>${key} = ${values[0]}`);
|
|
2360
|
+
else {
|
|
2361
|
+
const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
|
|
2362
|
+
query = query.where(kysely.sql`${col("tags")}->>${key} IN (${valueList})`);
|
|
2363
|
+
}
|
|
2364
|
+
}
|
|
2365
|
+
return query.executeTakeFirst();
|
|
2366
|
+
},
|
|
2367
|
+
getCostByModel: async (params) => {
|
|
2368
|
+
const result = await dateRangeSchema.safeParseAsync(params);
|
|
2369
|
+
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2370
|
+
const { startDate, endDate } = result.data;
|
|
2371
|
+
return db.selectFrom("llm_requests").select([
|
|
2372
|
+
"provider",
|
|
2373
|
+
"model",
|
|
2374
|
+
kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2375
|
+
kysely.sql`COALESCE(SUM(${col("inputCost")}), 0)`.as("totalInputCost"),
|
|
2376
|
+
kysely.sql`COALESCE(SUM(${col("outputCost")}), 0)`.as("totalOutputCost"),
|
|
2377
|
+
kysely.sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
|
|
2378
|
+
kysely.sql`COUNT(*)`.as("requestCount"),
|
|
2379
|
+
kysely.sql`AVG(${col("latencyMs")})`.as("avgLatencyMs")
|
|
2380
|
+
]).where(kysely.sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col("createdAt")} <= ${endDate.toISOString()}`).groupBy(["provider", "model"]).orderBy(kysely.sql`SUM(${col("cost")})`, "desc").execute();
|
|
2381
|
+
},
|
|
2382
|
+
getCostByProvider: async (params) => {
|
|
2383
|
+
const result = await dateRangeSchema.safeParseAsync(params);
|
|
2384
|
+
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2385
|
+
const { startDate, endDate } = result.data;
|
|
2386
|
+
return db.selectFrom("llm_requests").select([
|
|
2387
|
+
"provider",
|
|
2388
|
+
kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2389
|
+
kysely.sql`COALESCE(SUM(${col("inputCost")}), 0)`.as("totalInputCost"),
|
|
2390
|
+
kysely.sql`COALESCE(SUM(${col("outputCost")}), 0)`.as("totalOutputCost"),
|
|
2391
|
+
kysely.sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
|
|
2392
|
+
kysely.sql`COUNT(*)`.as("requestCount"),
|
|
2393
|
+
kysely.sql`AVG(${col("latencyMs")})`.as("avgLatencyMs")
|
|
2394
|
+
]).where(kysely.sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col("createdAt")} <= ${endDate.toISOString()}`).groupBy("provider").orderBy(kysely.sql`SUM(${col("cost")})`, "desc").execute();
|
|
2395
|
+
},
|
|
2396
|
+
getDailyCosts: async (params) => {
|
|
2397
|
+
const result = await dateRangeSchema.safeParseAsync(params);
|
|
2398
|
+
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2399
|
+
const { startDate, endDate } = result.data;
|
|
2400
|
+
return db.selectFrom("llm_requests").select([
|
|
2401
|
+
kysely.sql`DATE(${col("createdAt")})`.as("date"),
|
|
2402
|
+
kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2403
|
+
kysely.sql`COALESCE(SUM(${col("inputCost")}), 0)`.as("totalInputCost"),
|
|
2404
|
+
kysely.sql`COALESCE(SUM(${col("outputCost")}), 0)`.as("totalOutputCost"),
|
|
2405
|
+
kysely.sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
|
|
2406
|
+
kysely.sql`COUNT(*)`.as("requestCount")
|
|
2407
|
+
]).where(kysely.sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col("createdAt")} <= ${endDate.toISOString()}`).groupBy(kysely.sql`DATE(${col("createdAt")})`).orderBy(kysely.sql`DATE(${col("createdAt")})`, "asc").execute();
|
|
2408
|
+
},
|
|
2409
|
+
getCostSummary: async (params) => {
|
|
2410
|
+
const result = await costSummarySchema.safeParseAsync(params);
|
|
2411
|
+
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2412
|
+
const { startDate, endDate, groupBy, configId, variantId, environmentId, tags, tagKeys } = result.data;
|
|
2413
|
+
let baseQuery = db.selectFrom("llm_requests").where(kysely.sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col("createdAt")} <= ${endDate.toISOString()}`);
|
|
2414
|
+
if (configId) baseQuery = baseQuery.where("configId", "=", configId);
|
|
2415
|
+
if (variantId) baseQuery = baseQuery.where("variantId", "=", variantId);
|
|
2416
|
+
if (environmentId) baseQuery = baseQuery.where("environmentId", "=", environmentId);
|
|
2417
|
+
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
2418
|
+
if (values.length === 0) continue;
|
|
2419
|
+
if (values.length === 1) baseQuery = baseQuery.where(kysely.sql`${col("tags")}->>${key} = ${values[0]}`);
|
|
2420
|
+
else {
|
|
2421
|
+
const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
|
|
2422
|
+
baseQuery = baseQuery.where(kysely.sql`${col("tags")}->>${key} IN (${valueList})`);
|
|
2423
|
+
}
|
|
2424
|
+
}
|
|
2425
|
+
switch (groupBy) {
|
|
2426
|
+
case "day": return baseQuery.select([
|
|
2427
|
+
kysely.sql`DATE(${col("createdAt")})`.as("groupKey"),
|
|
2428
|
+
kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2429
|
+
kysely.sql`COUNT(*)`.as("requestCount"),
|
|
2430
|
+
kysely.sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens")
|
|
2431
|
+
]).groupBy(kysely.sql`DATE(${col("createdAt")})`).orderBy(kysely.sql`DATE(${col("createdAt")})`, "asc").execute();
|
|
2432
|
+
case "hour": return baseQuery.select([
|
|
2433
|
+
kysely.sql`DATE_TRUNC('hour', ${col("createdAt")})`.as("groupKey"),
|
|
2434
|
+
kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2435
|
+
kysely.sql`COUNT(*)`.as("requestCount"),
|
|
2436
|
+
kysely.sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens")
|
|
2437
|
+
]).groupBy(kysely.sql`DATE_TRUNC('hour', ${col("createdAt")})`).orderBy(kysely.sql`DATE_TRUNC('hour', ${col("createdAt")})`, "asc").execute();
|
|
2438
|
+
case "model": return baseQuery.select([
|
|
2439
|
+
kysely.sql`${col("provider")} || '/' || ${col("model")}`.as("groupKey"),
|
|
2440
|
+
kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2441
|
+
kysely.sql`COUNT(*)`.as("requestCount")
|
|
2442
|
+
]).groupBy(["provider", "model"]).orderBy(kysely.sql`SUM(${col("cost")})`, "desc").execute();
|
|
2443
|
+
case "provider": return baseQuery.select([
|
|
2444
|
+
kysely.sql`${col("provider")}`.as("groupKey"),
|
|
2445
|
+
kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2446
|
+
kysely.sql`COUNT(*)`.as("requestCount")
|
|
2447
|
+
]).groupBy("provider").orderBy(kysely.sql`SUM(${col("cost")})`, "desc").execute();
|
|
2448
|
+
case "endpoint": return baseQuery.select([
|
|
2449
|
+
kysely.sql`COALESCE(${col("endpoint")}, 'unknown')`.as("groupKey"),
|
|
2450
|
+
kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2451
|
+
kysely.sql`COUNT(*)`.as("requestCount")
|
|
2452
|
+
]).groupBy("endpoint").orderBy(kysely.sql`SUM(${col("cost")})`, "desc").execute();
|
|
2453
|
+
case "tags": {
|
|
2454
|
+
const conditions = [kysely.sql`${col("createdAt")} >= ${startDate.toISOString()}`, kysely.sql`${col("createdAt")} <= ${endDate.toISOString()}`];
|
|
2455
|
+
if (configId) conditions.push(kysely.sql`${col("configId")} = ${configId}`);
|
|
2456
|
+
if (variantId) conditions.push(kysely.sql`${col("variantId")} = ${variantId}`);
|
|
2457
|
+
if (environmentId) conditions.push(kysely.sql`${col("environmentId")} = ${environmentId}`);
|
|
2458
|
+
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
2459
|
+
if (values.length === 0) continue;
|
|
2460
|
+
if (values.length === 1) conditions.push(kysely.sql`${col("tags")}->>${key} = ${values[0]}`);
|
|
2461
|
+
else {
|
|
2462
|
+
const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
|
|
2463
|
+
conditions.push(kysely.sql`${col("tags")}->>${key} IN (${valueList})`);
|
|
2464
|
+
}
|
|
2465
|
+
}
|
|
2466
|
+
if (tagKeys && tagKeys.length > 0) {
|
|
2467
|
+
const tagKeyList = kysely.sql.join(tagKeys.map((k) => kysely.sql`${k}`), kysely.sql`, `);
|
|
2468
|
+
conditions.push(kysely.sql`t.key IN (${tagKeyList})`);
|
|
2469
|
+
}
|
|
2470
|
+
const whereClause = kysely.sql.join(conditions, kysely.sql` AND `);
|
|
2471
|
+
return (await kysely.sql`
|
|
2472
|
+
SELECT t.key || ':' || t.value as "groupKey",
|
|
2473
|
+
COALESCE(SUM(${col("cost")}), 0) as "totalCost",
|
|
2474
|
+
COUNT(*) as "requestCount"
|
|
2475
|
+
FROM "llm_requests", jsonb_each_text(${col("tags")}) t
|
|
2476
|
+
WHERE ${whereClause}
|
|
2477
|
+
GROUP BY t.key, t.value
|
|
2478
|
+
ORDER BY SUM(${col("cost")}) DESC
|
|
2479
|
+
`.execute(db)).rows;
|
|
2480
|
+
}
|
|
2481
|
+
default: return baseQuery.select([
|
|
2482
|
+
kysely.sql`'total'`.as("groupKey"),
|
|
2483
|
+
kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2484
|
+
kysely.sql`COUNT(*)`.as("requestCount")
|
|
2485
|
+
]).execute();
|
|
2486
|
+
}
|
|
2487
|
+
},
|
|
2488
|
+
getRequestStats: async (params) => {
|
|
2489
|
+
const result = await dateRangeSchema.safeParseAsync(params);
|
|
2490
|
+
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2491
|
+
const { startDate, endDate, configId, variantId, environmentId, tags } = result.data;
|
|
2492
|
+
let query = db.selectFrom("llm_requests").select([
|
|
2493
|
+
kysely.sql`COUNT(*)`.as("totalRequests"),
|
|
2494
|
+
kysely.sql`COUNT(CASE WHEN ${col("statusCode")} >= 200 AND ${col("statusCode")} < 300 THEN 1 END)`.as("successfulRequests"),
|
|
2495
|
+
kysely.sql`COUNT(CASE WHEN ${col("statusCode")} >= 400 THEN 1 END)`.as("failedRequests"),
|
|
2496
|
+
kysely.sql`COUNT(CASE WHEN ${col("isStreaming")} = true THEN 1 END)`.as("streamingRequests"),
|
|
2497
|
+
kysely.sql`AVG(${col("latencyMs")})`.as("avgLatencyMs"),
|
|
2498
|
+
kysely.sql`MAX(${col("latencyMs")})`.as("maxLatencyMs"),
|
|
2499
|
+
kysely.sql`MIN(${col("latencyMs")})`.as("minLatencyMs")
|
|
2500
|
+
]).where(kysely.sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col("createdAt")} <= ${endDate.toISOString()}`);
|
|
2501
|
+
if (configId) query = query.where("configId", "=", configId);
|
|
2502
|
+
if (variantId) query = query.where("variantId", "=", variantId);
|
|
2503
|
+
if (environmentId) query = query.where("environmentId", "=", environmentId);
|
|
2504
|
+
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
2505
|
+
if (values.length === 0) continue;
|
|
2506
|
+
if (values.length === 1) query = query.where(kysely.sql`${col("tags")}->>${key} = ${values[0]}`);
|
|
2507
|
+
else {
|
|
2508
|
+
const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
|
|
2509
|
+
query = query.where(kysely.sql`${col("tags")}->>${key} IN (${valueList})`);
|
|
2510
|
+
}
|
|
2511
|
+
}
|
|
2512
|
+
return query.executeTakeFirst();
|
|
2513
|
+
},
|
|
2514
|
+
getDistinctTags: async () => {
|
|
2515
|
+
return (await kysely.sql`
|
|
2516
|
+
SELECT DISTINCT key, value
|
|
2517
|
+
FROM llm_requests, jsonb_each_text(tags) AS t(key, value)
|
|
2518
|
+
WHERE tags != '{}'::jsonb
|
|
2519
|
+
ORDER BY key, value
|
|
2520
|
+
`.execute(db)).rows;
|
|
2521
|
+
}
|
|
2522
|
+
};
|
|
2523
|
+
}
|
|
2524
|
+
function createTracesStore(db) {
|
|
2525
|
+
return {
|
|
2526
|
+
upsertTrace: async (data) => {
|
|
2527
|
+
const result = await upsertTraceSchema.safeParseAsync(data);
|
|
2528
|
+
if (!result.success) throw new LLMOpsError(`Invalid trace data: ${result.error.message}`);
|
|
2529
|
+
const trace = result.data;
|
|
2530
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
2531
|
+
await kysely.sql`
|
|
2532
|
+
INSERT INTO "traces" (
|
|
2533
|
+
"id", "traceId", "name", "sessionId", "userId", "status",
|
|
2534
|
+
"startTime", "endTime", "durationMs", "spanCount",
|
|
2535
|
+
"totalInputTokens", "totalOutputTokens", "totalTokens", "totalCost",
|
|
2536
|
+
"tags", "metadata", "createdAt", "updatedAt"
|
|
2537
|
+
) VALUES (
|
|
2538
|
+
${(0, node_crypto.randomUUID)()}, ${trace.traceId}, ${trace.name ?? null}, ${trace.sessionId ?? null},
|
|
2539
|
+
${trace.userId ?? null}, ${trace.status},
|
|
2540
|
+
${trace.startTime.toISOString()}, ${trace.endTime?.toISOString() ?? null},
|
|
2541
|
+
${trace.durationMs ?? null}, ${trace.spanCount},
|
|
2542
|
+
${trace.totalInputTokens}, ${trace.totalOutputTokens},
|
|
2543
|
+
${trace.totalTokens}, ${trace.totalCost},
|
|
2544
|
+
${JSON.stringify(trace.tags)}::jsonb, ${JSON.stringify(trace.metadata)}::jsonb,
|
|
2545
|
+
${now}, ${now}
|
|
2546
|
+
)
|
|
2547
|
+
ON CONFLICT ("traceId") DO UPDATE SET
|
|
2548
|
+
"name" = COALESCE(EXCLUDED."name", "traces"."name"),
|
|
2549
|
+
"sessionId" = COALESCE(EXCLUDED."sessionId", "traces"."sessionId"),
|
|
2550
|
+
"userId" = COALESCE(EXCLUDED."userId", "traces"."userId"),
|
|
2551
|
+
"status" = CASE
|
|
2552
|
+
WHEN EXCLUDED."status" = 'error' THEN 'error'
|
|
2553
|
+
WHEN EXCLUDED."status" = 'ok' AND "traces"."status" != 'error' THEN 'ok'
|
|
2554
|
+
ELSE "traces"."status"
|
|
2555
|
+
END,
|
|
2556
|
+
"startTime" = LEAST("traces"."startTime", EXCLUDED."startTime"),
|
|
2557
|
+
"endTime" = GREATEST(
|
|
2558
|
+
COALESCE("traces"."endTime", EXCLUDED."endTime"),
|
|
2559
|
+
COALESCE(EXCLUDED."endTime", "traces"."endTime")
|
|
2560
|
+
),
|
|
2561
|
+
"durationMs" = EXTRACT(EPOCH FROM (
|
|
2562
|
+
GREATEST(
|
|
2563
|
+
COALESCE("traces"."endTime", EXCLUDED."endTime"),
|
|
2564
|
+
COALESCE(EXCLUDED."endTime", "traces"."endTime")
|
|
2565
|
+
) -
|
|
2566
|
+
LEAST("traces"."startTime", EXCLUDED."startTime")
|
|
2567
|
+
))::integer * 1000,
|
|
2568
|
+
"spanCount" = "traces"."spanCount" + EXCLUDED."spanCount",
|
|
2569
|
+
"totalInputTokens" = "traces"."totalInputTokens" + EXCLUDED."totalInputTokens",
|
|
2570
|
+
"totalOutputTokens" = "traces"."totalOutputTokens" + EXCLUDED."totalOutputTokens",
|
|
2571
|
+
"totalTokens" = "traces"."totalTokens" + EXCLUDED."totalTokens",
|
|
2572
|
+
"totalCost" = "traces"."totalCost" + EXCLUDED."totalCost",
|
|
2573
|
+
"tags" = "traces"."tags" || EXCLUDED."tags",
|
|
2574
|
+
"metadata" = "traces"."metadata" || EXCLUDED."metadata",
|
|
2575
|
+
"updatedAt" = ${now}
|
|
2576
|
+
`.execute(db);
|
|
2577
|
+
},
|
|
2578
|
+
batchInsertSpans: async (spans) => {
|
|
2579
|
+
if (spans.length === 0) return { count: 0 };
|
|
2580
|
+
const validatedSpans = [];
|
|
2581
|
+
for (const span of spans) {
|
|
2582
|
+
const result = await insertSpanSchema.safeParseAsync(span);
|
|
2583
|
+
if (!result.success) {
|
|
2584
|
+
require_db.logger.warn(`[batchInsertSpans] Skipping invalid span ${span.spanId}: ${result.error.message}`);
|
|
2585
|
+
continue;
|
|
2586
|
+
}
|
|
2587
|
+
validatedSpans.push(result.data);
|
|
2588
|
+
}
|
|
2589
|
+
if (validatedSpans.length === 0) return { count: 0 };
|
|
2590
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
2591
|
+
const values = validatedSpans.map((span) => ({
|
|
2592
|
+
id: (0, node_crypto.randomUUID)(),
|
|
2593
|
+
traceId: span.traceId,
|
|
2594
|
+
spanId: span.spanId,
|
|
2595
|
+
parentSpanId: span.parentSpanId ?? null,
|
|
2596
|
+
name: span.name,
|
|
2597
|
+
kind: span.kind,
|
|
2598
|
+
status: span.status,
|
|
2599
|
+
statusMessage: span.statusMessage ?? null,
|
|
2600
|
+
startTime: span.startTime.toISOString(),
|
|
2601
|
+
endTime: span.endTime?.toISOString() ?? null,
|
|
2602
|
+
durationMs: span.durationMs ?? null,
|
|
2603
|
+
provider: span.provider ?? null,
|
|
2604
|
+
model: span.model ?? null,
|
|
2605
|
+
promptTokens: span.promptTokens,
|
|
2606
|
+
completionTokens: span.completionTokens,
|
|
2607
|
+
totalTokens: span.totalTokens,
|
|
2608
|
+
cost: span.cost,
|
|
2609
|
+
configId: span.configId ?? null,
|
|
2610
|
+
variantId: span.variantId ?? null,
|
|
2611
|
+
environmentId: span.environmentId ?? null,
|
|
2612
|
+
providerConfigId: span.providerConfigId ?? null,
|
|
2613
|
+
requestId: span.requestId ?? null,
|
|
2614
|
+
source: span.source,
|
|
2615
|
+
input: span.input != null ? JSON.stringify(span.input) : null,
|
|
2616
|
+
output: span.output != null ? JSON.stringify(span.output) : null,
|
|
2617
|
+
attributes: JSON.stringify(span.attributes),
|
|
2618
|
+
createdAt: now,
|
|
2619
|
+
updatedAt: now
|
|
2620
|
+
}));
|
|
2621
|
+
await db.insertInto("spans").values(values).onConflict((oc) => oc.column("spanId").doNothing()).execute();
|
|
2622
|
+
return { count: values.length };
|
|
2623
|
+
},
|
|
2624
|
+
batchInsertSpanEvents: async (events) => {
|
|
2625
|
+
if (events.length === 0) return { count: 0 };
|
|
2626
|
+
const validatedEvents = [];
|
|
2627
|
+
for (const event of events) {
|
|
2628
|
+
const result = await insertSpanEventSchema.safeParseAsync(event);
|
|
2629
|
+
if (!result.success) {
|
|
2630
|
+
require_db.logger.warn(`[batchInsertSpanEvents] Skipping invalid event: ${result.error.message}`);
|
|
2631
|
+
continue;
|
|
2632
|
+
}
|
|
2633
|
+
validatedEvents.push(result.data);
|
|
2634
|
+
}
|
|
2635
|
+
if (validatedEvents.length === 0) return { count: 0 };
|
|
2636
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
2637
|
+
const values = validatedEvents.map((event) => ({
|
|
2638
|
+
id: (0, node_crypto.randomUUID)(),
|
|
2639
|
+
traceId: event.traceId,
|
|
2640
|
+
spanId: event.spanId,
|
|
2641
|
+
name: event.name,
|
|
2642
|
+
timestamp: event.timestamp.toISOString(),
|
|
2643
|
+
attributes: JSON.stringify(event.attributes),
|
|
2644
|
+
createdAt: now
|
|
2645
|
+
}));
|
|
2646
|
+
await db.insertInto("span_events").values(values).execute();
|
|
2647
|
+
return { count: values.length };
|
|
2648
|
+
},
|
|
2649
|
+
listTraces: async (params) => {
|
|
2650
|
+
const result = await listTracesSchema.safeParseAsync(params || {});
|
|
2651
|
+
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2652
|
+
const { limit, offset, sessionId, userId, status, name, startDate, endDate, tags } = result.data;
|
|
2653
|
+
let baseQuery = db.selectFrom("traces");
|
|
2654
|
+
if (sessionId) baseQuery = baseQuery.where("sessionId", "=", sessionId);
|
|
2655
|
+
if (userId) baseQuery = baseQuery.where("userId", "=", userId);
|
|
2656
|
+
if (status) baseQuery = baseQuery.where("status", "=", status);
|
|
2657
|
+
if (name) baseQuery = baseQuery.where(kysely.sql`${col("name")} ILIKE ${"%" + name + "%"}`);
|
|
2658
|
+
if (startDate) baseQuery = baseQuery.where(kysely.sql`${col("startTime")} >= ${startDate.toISOString()}`);
|
|
2659
|
+
if (endDate) baseQuery = baseQuery.where(kysely.sql`${col("startTime")} <= ${endDate.toISOString()}`);
|
|
2660
|
+
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
2661
|
+
if (values.length === 0) continue;
|
|
2662
|
+
if (values.length === 1) baseQuery = baseQuery.where(kysely.sql`${col("tags")}->>${key} = ${values[0]}`);
|
|
2663
|
+
else {
|
|
2664
|
+
const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
|
|
2665
|
+
baseQuery = baseQuery.where(kysely.sql`${col("tags")}->>${key} IN (${valueList})`);
|
|
2666
|
+
}
|
|
2667
|
+
}
|
|
2668
|
+
const countResult = await baseQuery.select(kysely.sql`COUNT(*)`.as("total")).executeTakeFirst();
|
|
2669
|
+
const total = Number(countResult?.total ?? 0);
|
|
2670
|
+
return {
|
|
2671
|
+
data: await baseQuery.selectAll().orderBy("startTime", "desc").limit(limit).offset(offset).execute(),
|
|
2672
|
+
total,
|
|
2673
|
+
limit,
|
|
2674
|
+
offset
|
|
2675
|
+
};
|
|
2676
|
+
},
|
|
2677
|
+
getTraceWithSpans: async (traceId) => {
|
|
2678
|
+
const trace = await db.selectFrom("traces").selectAll().where("traceId", "=", traceId).executeTakeFirst();
|
|
2679
|
+
if (!trace) return void 0;
|
|
2680
|
+
return {
|
|
2681
|
+
trace,
|
|
2682
|
+
spans: await db.selectFrom("spans").selectAll().where("traceId", "=", traceId).orderBy("startTime", "asc").execute(),
|
|
2683
|
+
events: await db.selectFrom("span_events").selectAll().where("traceId", "=", traceId).orderBy("timestamp", "asc").execute()
|
|
2684
|
+
};
|
|
2685
|
+
},
|
|
2686
|
+
getTraceStats: async (params) => {
|
|
2687
|
+
const result = await traceStatsSchema.safeParseAsync(params);
|
|
2688
|
+
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2689
|
+
const { startDate, endDate, sessionId, userId } = result.data;
|
|
2690
|
+
let query = db.selectFrom("traces").select([
|
|
2691
|
+
kysely.sql`COUNT(*)`.as("totalTraces"),
|
|
2692
|
+
kysely.sql`COALESCE(AVG(${col("durationMs")}), 0)`.as("avgDurationMs"),
|
|
2693
|
+
kysely.sql`COUNT(CASE WHEN ${col("status")} = 'error' THEN 1 END)`.as("errorCount"),
|
|
2694
|
+
kysely.sql`COALESCE(SUM(${col("totalCost")}), 0)`.as("totalCost"),
|
|
2695
|
+
kysely.sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
|
|
2696
|
+
kysely.sql`COALESCE(SUM(${col("spanCount")}), 0)`.as("totalSpans")
|
|
2697
|
+
]).where(kysely.sql`${col("startTime")} >= ${startDate.toISOString()}`).where(kysely.sql`${col("startTime")} <= ${endDate.toISOString()}`);
|
|
2698
|
+
if (sessionId) query = query.where("sessionId", "=", sessionId);
|
|
2699
|
+
if (userId) query = query.where("userId", "=", userId);
|
|
2700
|
+
return query.executeTakeFirst();
|
|
2701
|
+
}
|
|
2702
|
+
};
|
|
2703
|
+
}
|
|
2753
2704
|
/**
|
|
2754
2705
|
* Create a PostgreSQL-backed telemetry store.
|
|
2755
2706
|
*
|
|
2756
2707
|
* Usage:
|
|
2757
2708
|
* ```ts
|
|
2758
|
-
* import { llmops
|
|
2709
|
+
* import { llmops } from '@llmops/sdk'
|
|
2710
|
+
* import { pgStore } from '@llmops/sdk/store/pg'
|
|
2759
2711
|
*
|
|
2760
2712
|
* const ops = llmops({
|
|
2761
2713
|
* telemetry: pgStore(process.env.DATABASE_URL),
|
|
@@ -2776,12 +2728,10 @@ function createPgStore(connectionString, options) {
|
|
|
2776
2728
|
await connection.executeQuery(kysely.CompiledQuery.raw(`SET search_path TO "${schema}"`));
|
|
2777
2729
|
}
|
|
2778
2730
|
}) });
|
|
2779
|
-
const llmRequests = createLLMRequestsDataLayer(db);
|
|
2780
|
-
const traces = createTracesDataLayer(db);
|
|
2781
2731
|
require_db.logger.debug(`pgStore: initialized with schema "${schema}"`);
|
|
2782
2732
|
return {
|
|
2783
|
-
...
|
|
2784
|
-
...
|
|
2733
|
+
...createLLMRequestsStore(db),
|
|
2734
|
+
...createTracesStore(db),
|
|
2785
2735
|
_db: db
|
|
2786
2736
|
};
|
|
2787
2737
|
}
|
|
@@ -2813,13 +2763,11 @@ exports.createDataLayer = createDataLayer;
|
|
|
2813
2763
|
exports.createDatabase = require_db.createDatabase;
|
|
2814
2764
|
exports.createDatabaseFromConnection = require_db.createDatabaseFromConnection;
|
|
2815
2765
|
exports.createDatasetsDataLayer = createDatasetsDataLayer;
|
|
2816
|
-
exports.createLLMRequestsDataLayer = createLLMRequestsDataLayer;
|
|
2817
2766
|
exports.createNeonDialect = require_neon_dialect.createNeonDialect;
|
|
2818
2767
|
exports.createPgStore = createPgStore;
|
|
2819
2768
|
exports.createPlaygroundDataLayer = createPlaygroundDataLayer;
|
|
2820
2769
|
exports.createPlaygroundResultsDataLayer = createPlaygroundResultsDataLayer;
|
|
2821
2770
|
exports.createPlaygroundRunsDataLayer = createPlaygroundRunsDataLayer;
|
|
2822
|
-
exports.createTracesDataLayer = createTracesDataLayer;
|
|
2823
2771
|
exports.createWorkspaceSettingsDataLayer = createWorkspaceSettingsDataLayer;
|
|
2824
2772
|
exports.datasetRecordsSchema = require_db.datasetRecordsSchema;
|
|
2825
2773
|
exports.datasetVersionRecordsSchema = require_db.datasetVersionRecordsSchema;
|