@llmops/core 1.0.0-beta.1 → 1.0.0-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{bun-sqlite-dialect-C3HVUfYl.cjs → bun-sqlite-dialect-CVUG5QUU.cjs} +1 -1
- package/dist/db/index.cjs +2 -2
- package/dist/db/index.d.cts +1 -1
- package/dist/db/index.d.mts +1 -1
- package/dist/db/index.mjs +1 -1
- package/dist/{db-CjwRQB4N.mjs → db-CZ8KtpL-.mjs} +3 -132
- package/dist/{db-D3WDjcvd.cjs → db-DZv0NtMm.cjs} +7 -136
- package/dist/{index-BoDvuqku.d.mts → index-BlFAMkmT.d.mts} +18 -416
- package/dist/{index-BmGf4wCW.d.cts → index-DjIHdwhi.d.cts} +18 -416
- package/dist/index.cjs +1132 -1648
- package/dist/index.d.cts +981 -1535
- package/dist/index.d.mts +981 -1535
- package/dist/index.mjs +1132 -1641
- package/dist/{neon-dialect-BSJpZ9YH.cjs → neon-dialect-BOnuygPe.cjs} +1 -1
- package/dist/{neon-dialect-DMClTHvw.cjs → neon-dialect-ByrFa9iy.cjs} +1 -1
- package/dist/{node-sqlite-dialect-BZGXfDHS.cjs → node-sqlite-dialect-fwmW40Ar.cjs} +1 -1
- package/package.json +2 -2
package/dist/index.cjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
const require_db = require('./db-
|
|
2
|
-
const require_neon_dialect = require('./neon-dialect-
|
|
1
|
+
const require_db = require('./db-DZv0NtMm.cjs');
|
|
2
|
+
const require_neon_dialect = require('./neon-dialect-ByrFa9iy.cjs');
|
|
3
3
|
let __llmops_gateway = require("@llmops/gateway");
|
|
4
4
|
__llmops_gateway = require_db.__toESM(__llmops_gateway);
|
|
5
5
|
let kysely = require("kysely");
|
|
@@ -1387,754 +1387,247 @@ const createDatasetsDataLayer = (db) => {
|
|
|
1387
1387
|
};
|
|
1388
1388
|
|
|
1389
1389
|
//#endregion
|
|
1390
|
-
//#region src/datalayer/
|
|
1391
|
-
const
|
|
1392
|
-
name: require_db.zod_default.string()
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
hookType: require_db.zod_default.enum(["beforeRequestHook", "afterRequestHook"]),
|
|
1396
|
-
parameters: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.unknown()).optional().default({}),
|
|
1397
|
-
enabled: require_db.zod_default.boolean().optional().default(true),
|
|
1398
|
-
priority: require_db.zod_default.number().int().optional().default(0),
|
|
1399
|
-
onFail: require_db.zod_default.enum(["block", "log"]).optional().default("block")
|
|
1390
|
+
//#region src/datalayer/playgrounds.ts
|
|
1391
|
+
const createNewPlayground = require_db.zod_default.object({
|
|
1392
|
+
name: require_db.zod_default.string(),
|
|
1393
|
+
datasetId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
1394
|
+
columns: require_db.zod_default.array(require_db.playgroundColumnSchema).nullable().optional()
|
|
1400
1395
|
});
|
|
1401
|
-
const
|
|
1402
|
-
|
|
1403
|
-
name: require_db.zod_default.string().
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
enabled: require_db.zod_default.boolean().optional(),
|
|
1407
|
-
priority: require_db.zod_default.number().int().optional(),
|
|
1408
|
-
onFail: require_db.zod_default.enum(["block", "log"]).optional()
|
|
1396
|
+
const updatePlayground = require_db.zod_default.object({
|
|
1397
|
+
playgroundId: require_db.zod_default.uuidv4(),
|
|
1398
|
+
name: require_db.zod_default.string().optional(),
|
|
1399
|
+
datasetId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
1400
|
+
columns: require_db.zod_default.array(require_db.playgroundColumnSchema).nullable().optional()
|
|
1409
1401
|
});
|
|
1410
|
-
const
|
|
1411
|
-
const
|
|
1412
|
-
const
|
|
1402
|
+
const getPlaygroundById = require_db.zod_default.object({ playgroundId: require_db.zod_default.uuidv4() });
|
|
1403
|
+
const deletePlayground = require_db.zod_default.object({ playgroundId: require_db.zod_default.uuidv4() });
|
|
1404
|
+
const listPlaygrounds = require_db.zod_default.object({
|
|
1413
1405
|
limit: require_db.zod_default.number().int().positive().optional(),
|
|
1414
|
-
offset: require_db.zod_default.number().int().nonnegative().optional()
|
|
1415
|
-
hookType: require_db.zod_default.enum(["beforeRequestHook", "afterRequestHook"]).optional(),
|
|
1416
|
-
enabled: require_db.zod_default.boolean().optional()
|
|
1406
|
+
offset: require_db.zod_default.number().int().nonnegative().optional()
|
|
1417
1407
|
});
|
|
1418
|
-
const
|
|
1408
|
+
const createPlaygroundDataLayer = (db) => {
|
|
1419
1409
|
return {
|
|
1420
|
-
|
|
1421
|
-
const value = await
|
|
1410
|
+
createNewPlayground: async (params) => {
|
|
1411
|
+
const value = await createNewPlayground.safeParseAsync(params);
|
|
1422
1412
|
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1423
|
-
const { name,
|
|
1424
|
-
return db.insertInto("
|
|
1413
|
+
const { name, datasetId, columns } = value.data;
|
|
1414
|
+
return db.insertInto("playgrounds").values({
|
|
1425
1415
|
id: (0, node_crypto.randomUUID)(),
|
|
1426
1416
|
name,
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
hookType,
|
|
1430
|
-
parameters: JSON.stringify(parameters),
|
|
1431
|
-
enabled,
|
|
1432
|
-
priority,
|
|
1433
|
-
onFail,
|
|
1417
|
+
datasetId: datasetId ?? null,
|
|
1418
|
+
columns: columns ? JSON.stringify(columns) : null,
|
|
1434
1419
|
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1435
1420
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1436
1421
|
}).returningAll().executeTakeFirst();
|
|
1437
1422
|
},
|
|
1438
|
-
|
|
1439
|
-
const value = await
|
|
1423
|
+
updatePlayground: async (params) => {
|
|
1424
|
+
const value = await updatePlayground.safeParseAsync(params);
|
|
1440
1425
|
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1441
|
-
const {
|
|
1426
|
+
const { playgroundId, name, datasetId, columns } = value.data;
|
|
1442
1427
|
const updateData = { updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
1443
1428
|
if (name !== void 0) updateData.name = name;
|
|
1444
|
-
if (
|
|
1445
|
-
if (
|
|
1446
|
-
|
|
1447
|
-
if (priority !== void 0) updateData.priority = priority;
|
|
1448
|
-
if (onFail !== void 0) updateData.onFail = onFail;
|
|
1449
|
-
return db.updateTable("guardrail_configs").set(updateData).where("id", "=", id).returningAll().executeTakeFirst();
|
|
1429
|
+
if (datasetId !== void 0) updateData.datasetId = datasetId;
|
|
1430
|
+
if (columns !== void 0) updateData.columns = columns ? JSON.stringify(columns) : null;
|
|
1431
|
+
return db.updateTable("playgrounds").set(updateData).where("id", "=", playgroundId).returningAll().executeTakeFirst();
|
|
1450
1432
|
},
|
|
1451
|
-
|
|
1452
|
-
const value = await
|
|
1433
|
+
getPlaygroundById: async (params) => {
|
|
1434
|
+
const value = await getPlaygroundById.safeParseAsync(params);
|
|
1453
1435
|
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1454
|
-
const {
|
|
1455
|
-
return db.selectFrom("
|
|
1436
|
+
const { playgroundId } = value.data;
|
|
1437
|
+
return db.selectFrom("playgrounds").selectAll().where("id", "=", playgroundId).executeTakeFirst();
|
|
1456
1438
|
},
|
|
1457
|
-
|
|
1458
|
-
const value = await
|
|
1439
|
+
deletePlayground: async (params) => {
|
|
1440
|
+
const value = await deletePlayground.safeParseAsync(params);
|
|
1459
1441
|
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1460
|
-
const {
|
|
1461
|
-
return db.deleteFrom("
|
|
1442
|
+
const { playgroundId } = value.data;
|
|
1443
|
+
return db.deleteFrom("playgrounds").where("id", "=", playgroundId).returningAll().executeTakeFirst();
|
|
1462
1444
|
},
|
|
1463
|
-
|
|
1464
|
-
const value = await
|
|
1445
|
+
listPlaygrounds: async (params) => {
|
|
1446
|
+
const value = await listPlaygrounds.safeParseAsync(params || {});
|
|
1465
1447
|
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1466
|
-
const { limit = 100, offset = 0
|
|
1467
|
-
|
|
1468
|
-
if (hookType !== void 0) query = query.where("hookType", "=", hookType);
|
|
1469
|
-
if (enabled !== void 0) query = query.where("enabled", "=", enabled);
|
|
1470
|
-
return query.execute();
|
|
1448
|
+
const { limit = 100, offset = 0 } = value.data;
|
|
1449
|
+
return db.selectFrom("playgrounds").selectAll().orderBy("createdAt", "desc").limit(limit).offset(offset).execute();
|
|
1471
1450
|
},
|
|
1472
|
-
|
|
1473
|
-
const result = await db.selectFrom("
|
|
1451
|
+
countPlaygrounds: async () => {
|
|
1452
|
+
const result = await db.selectFrom("playgrounds").select(db.fn.countAll().as("count")).executeTakeFirst();
|
|
1474
1453
|
return Number(result?.count ?? 0);
|
|
1475
|
-
},
|
|
1476
|
-
getEnabledGuardrailsByHookType: async (hookType) => {
|
|
1477
|
-
return db.selectFrom("guardrail_configs").selectAll().where("hookType", "=", hookType).where("enabled", "=", true).orderBy("priority", "desc").execute();
|
|
1478
1454
|
}
|
|
1479
1455
|
};
|
|
1480
1456
|
};
|
|
1481
1457
|
|
|
1482
1458
|
//#endregion
|
|
1483
|
-
//#region src/datalayer/
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
* Schema for guardrail results aggregate
|
|
1496
|
-
*/
|
|
1497
|
-
const guardrailResultsSchema = require_db.zod_default.object({
|
|
1498
|
-
results: require_db.zod_default.array(guardrailResultSchema),
|
|
1499
|
-
action: require_db.zod_default.enum([
|
|
1500
|
-
"allowed",
|
|
1501
|
-
"blocked",
|
|
1502
|
-
"logged"
|
|
1503
|
-
]),
|
|
1504
|
-
totalLatencyMs: require_db.zod_default.number()
|
|
1505
|
-
});
|
|
1506
|
-
/**
|
|
1507
|
-
* Schema for inserting a new LLM request log
|
|
1508
|
-
*/
|
|
1509
|
-
const insertLLMRequestSchema = require_db.zod_default.object({
|
|
1510
|
-
requestId: require_db.zod_default.string().uuid(),
|
|
1511
|
-
configId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
1512
|
-
variantId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
1513
|
-
environmentId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
1514
|
-
providerConfigId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
1515
|
-
provider: require_db.zod_default.string(),
|
|
1516
|
-
model: require_db.zod_default.string(),
|
|
1517
|
-
promptTokens: require_db.zod_default.number().int().default(0),
|
|
1518
|
-
completionTokens: require_db.zod_default.number().int().default(0),
|
|
1519
|
-
totalTokens: require_db.zod_default.number().int().default(0),
|
|
1520
|
-
cachedTokens: require_db.zod_default.number().int().default(0),
|
|
1521
|
-
cacheCreationTokens: require_db.zod_default.number().int().default(0),
|
|
1522
|
-
cost: require_db.zod_default.number().int().default(0),
|
|
1523
|
-
cacheSavings: require_db.zod_default.number().int().default(0),
|
|
1524
|
-
inputCost: require_db.zod_default.number().int().default(0),
|
|
1525
|
-
outputCost: require_db.zod_default.number().int().default(0),
|
|
1526
|
-
endpoint: require_db.zod_default.string(),
|
|
1527
|
-
statusCode: require_db.zod_default.number().int(),
|
|
1528
|
-
latencyMs: require_db.zod_default.number().int().default(0),
|
|
1529
|
-
isStreaming: require_db.zod_default.boolean().default(false),
|
|
1530
|
-
userId: require_db.zod_default.string().nullable().optional(),
|
|
1531
|
-
tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.string()).default({}),
|
|
1532
|
-
guardrailResults: guardrailResultsSchema.nullable().optional(),
|
|
1533
|
-
traceId: require_db.zod_default.string().nullable().optional(),
|
|
1534
|
-
spanId: require_db.zod_default.string().nullable().optional(),
|
|
1535
|
-
parentSpanId: require_db.zod_default.string().nullable().optional(),
|
|
1536
|
-
sessionId: require_db.zod_default.string().nullable().optional()
|
|
1537
|
-
});
|
|
1538
|
-
/**
|
|
1539
|
-
* Schema for listing LLM requests
|
|
1540
|
-
*/
|
|
1541
|
-
const listRequestsSchema = require_db.zod_default.object({
|
|
1542
|
-
limit: require_db.zod_default.number().int().positive().max(1e3).default(100),
|
|
1543
|
-
offset: require_db.zod_default.number().int().nonnegative().default(0),
|
|
1544
|
-
configId: require_db.zod_default.string().uuid().optional(),
|
|
1545
|
-
variantId: require_db.zod_default.string().uuid().optional(),
|
|
1546
|
-
environmentId: require_db.zod_default.string().uuid().optional(),
|
|
1547
|
-
providerConfigId: require_db.zod_default.string().uuid().optional(),
|
|
1548
|
-
provider: require_db.zod_default.string().optional(),
|
|
1549
|
-
model: require_db.zod_default.string().optional(),
|
|
1550
|
-
startDate: require_db.zod_default.date().optional(),
|
|
1551
|
-
endDate: require_db.zod_default.date().optional(),
|
|
1552
|
-
tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.array(require_db.zod_default.string())).optional()
|
|
1459
|
+
//#region src/datalayer/playgroundResults.ts
|
|
1460
|
+
const createPlaygroundResult = require_db.zod_default.object({
|
|
1461
|
+
runId: require_db.zod_default.string().uuid(),
|
|
1462
|
+
columnId: require_db.zod_default.string().uuid(),
|
|
1463
|
+
datasetRecordId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
1464
|
+
inputVariables: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.unknown()).default({}),
|
|
1465
|
+
status: require_db.zod_default.enum([
|
|
1466
|
+
"pending",
|
|
1467
|
+
"running",
|
|
1468
|
+
"completed",
|
|
1469
|
+
"failed"
|
|
1470
|
+
]).default("pending")
|
|
1553
1471
|
});
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1472
|
+
const createPlaygroundResultsBatch = require_db.zod_default.object({ results: require_db.zod_default.array(createPlaygroundResult) });
|
|
1473
|
+
const updatePlaygroundResult = require_db.zod_default.object({
|
|
1474
|
+
resultId: require_db.zod_default.string().uuid(),
|
|
1475
|
+
outputContent: require_db.zod_default.string().nullable().optional(),
|
|
1476
|
+
status: require_db.zod_default.enum([
|
|
1477
|
+
"pending",
|
|
1478
|
+
"running",
|
|
1479
|
+
"completed",
|
|
1480
|
+
"failed"
|
|
1481
|
+
]).optional(),
|
|
1482
|
+
error: require_db.zod_default.string().nullable().optional(),
|
|
1483
|
+
latencyMs: require_db.zod_default.number().int().nullable().optional(),
|
|
1484
|
+
promptTokens: require_db.zod_default.number().int().nullable().optional(),
|
|
1485
|
+
completionTokens: require_db.zod_default.number().int().nullable().optional(),
|
|
1486
|
+
totalTokens: require_db.zod_default.number().int().nullable().optional(),
|
|
1487
|
+
cost: require_db.zod_default.number().int().nullable().optional()
|
|
1564
1488
|
});
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
"model",
|
|
1572
|
-
"provider",
|
|
1573
|
-
"endpoint",
|
|
1574
|
-
"tags"
|
|
1575
|
-
];
|
|
1576
|
-
/**
|
|
1577
|
-
* Schema for cost summary with grouping
|
|
1578
|
-
*/
|
|
1579
|
-
const costSummarySchema = require_db.zod_default.object({
|
|
1580
|
-
startDate: require_db.zod_default.date(),
|
|
1581
|
-
endDate: require_db.zod_default.date(),
|
|
1582
|
-
configId: require_db.zod_default.string().uuid().optional(),
|
|
1583
|
-
variantId: require_db.zod_default.string().uuid().optional(),
|
|
1584
|
-
environmentId: require_db.zod_default.string().uuid().optional(),
|
|
1585
|
-
tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.array(require_db.zod_default.string())).optional(),
|
|
1586
|
-
groupBy: require_db.zod_default.enum(COST_SUMMARY_GROUP_BY).optional(),
|
|
1587
|
-
tagKeys: require_db.zod_default.array(require_db.zod_default.string()).optional()
|
|
1489
|
+
const getPlaygroundResultById = require_db.zod_default.object({ resultId: require_db.zod_default.string().uuid() });
|
|
1490
|
+
const listPlaygroundResults = require_db.zod_default.object({
|
|
1491
|
+
runId: require_db.zod_default.string().uuid(),
|
|
1492
|
+
columnId: require_db.zod_default.string().uuid().optional(),
|
|
1493
|
+
limit: require_db.zod_default.number().int().positive().optional(),
|
|
1494
|
+
offset: require_db.zod_default.number().int().nonnegative().optional()
|
|
1588
1495
|
});
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
* Uses sql.ref() to properly quote column names for the database
|
|
1592
|
-
*/
|
|
1593
|
-
const col$1 = (name) => kysely.sql.ref(name);
|
|
1594
|
-
const createLLMRequestsDataLayer = (db) => {
|
|
1496
|
+
const deletePlaygroundResultsByRunId = require_db.zod_default.object({ runId: require_db.zod_default.string().uuid() });
|
|
1497
|
+
const createPlaygroundResultsDataLayer = (db) => {
|
|
1595
1498
|
return {
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1499
|
+
createPlaygroundResult: async (params) => {
|
|
1500
|
+
const value = await createPlaygroundResult.safeParseAsync(params);
|
|
1501
|
+
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1502
|
+
const { runId, columnId, datasetRecordId, inputVariables, status } = value.data;
|
|
1503
|
+
return db.insertInto("playground_results").values({
|
|
1504
|
+
id: (0, node_crypto.randomUUID)(),
|
|
1505
|
+
runId,
|
|
1506
|
+
columnId,
|
|
1507
|
+
datasetRecordId: datasetRecordId ?? null,
|
|
1508
|
+
inputVariables: JSON.stringify(inputVariables),
|
|
1509
|
+
outputContent: null,
|
|
1510
|
+
status,
|
|
1511
|
+
error: null,
|
|
1512
|
+
latencyMs: null,
|
|
1513
|
+
promptTokens: null,
|
|
1514
|
+
completionTokens: null,
|
|
1515
|
+
totalTokens: null,
|
|
1516
|
+
cost: null,
|
|
1517
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1518
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1519
|
+
}).returningAll().executeTakeFirst();
|
|
1520
|
+
},
|
|
1521
|
+
createPlaygroundResultsBatch: async (params) => {
|
|
1522
|
+
const value = await createPlaygroundResultsBatch.safeParseAsync(params);
|
|
1523
|
+
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1524
|
+
const { results } = value.data;
|
|
1525
|
+
if (results.length === 0) return [];
|
|
1603
1526
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1604
|
-
const values =
|
|
1527
|
+
const values = results.map((result) => ({
|
|
1605
1528
|
id: (0, node_crypto.randomUUID)(),
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
cost: req.cost,
|
|
1619
|
-
cacheSavings: req.cacheSavings,
|
|
1620
|
-
inputCost: req.inputCost,
|
|
1621
|
-
outputCost: req.outputCost,
|
|
1622
|
-
endpoint: req.endpoint,
|
|
1623
|
-
statusCode: req.statusCode,
|
|
1624
|
-
latencyMs: req.latencyMs,
|
|
1625
|
-
isStreaming: req.isStreaming,
|
|
1626
|
-
userId: req.userId ?? null,
|
|
1627
|
-
tags: JSON.stringify(req.tags),
|
|
1628
|
-
guardrailResults: req.guardrailResults ? JSON.stringify(req.guardrailResults) : null,
|
|
1629
|
-
traceId: req.traceId ?? null,
|
|
1630
|
-
spanId: req.spanId ?? null,
|
|
1631
|
-
parentSpanId: req.parentSpanId ?? null,
|
|
1632
|
-
sessionId: req.sessionId ?? null,
|
|
1529
|
+
runId: result.runId,
|
|
1530
|
+
columnId: result.columnId,
|
|
1531
|
+
datasetRecordId: result.datasetRecordId ?? null,
|
|
1532
|
+
inputVariables: JSON.stringify(result.inputVariables),
|
|
1533
|
+
outputContent: null,
|
|
1534
|
+
status: result.status,
|
|
1535
|
+
error: null,
|
|
1536
|
+
latencyMs: null,
|
|
1537
|
+
promptTokens: null,
|
|
1538
|
+
completionTokens: null,
|
|
1539
|
+
totalTokens: null,
|
|
1540
|
+
cost: null,
|
|
1633
1541
|
createdAt: now,
|
|
1634
1542
|
updatedAt: now
|
|
1635
1543
|
}));
|
|
1636
|
-
|
|
1637
|
-
return { count: values.length };
|
|
1544
|
+
return db.insertInto("playground_results").values(values).returningAll().execute();
|
|
1638
1545
|
},
|
|
1639
|
-
|
|
1640
|
-
const
|
|
1641
|
-
if (!
|
|
1642
|
-
const
|
|
1643
|
-
const
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
promptTokens: req.promptTokens,
|
|
1654
|
-
completionTokens: req.completionTokens,
|
|
1655
|
-
totalTokens: req.totalTokens,
|
|
1656
|
-
cachedTokens: req.cachedTokens,
|
|
1657
|
-
cacheCreationTokens: req.cacheCreationTokens,
|
|
1658
|
-
cost: req.cost,
|
|
1659
|
-
cacheSavings: req.cacheSavings,
|
|
1660
|
-
inputCost: req.inputCost,
|
|
1661
|
-
outputCost: req.outputCost,
|
|
1662
|
-
endpoint: req.endpoint,
|
|
1663
|
-
statusCode: req.statusCode,
|
|
1664
|
-
latencyMs: req.latencyMs,
|
|
1665
|
-
isStreaming: req.isStreaming,
|
|
1666
|
-
userId: req.userId ?? null,
|
|
1667
|
-
tags: JSON.stringify(req.tags),
|
|
1668
|
-
guardrailResults: req.guardrailResults ? JSON.stringify(req.guardrailResults) : null,
|
|
1669
|
-
traceId: req.traceId ?? null,
|
|
1670
|
-
spanId: req.spanId ?? null,
|
|
1671
|
-
parentSpanId: req.parentSpanId ?? null,
|
|
1672
|
-
sessionId: req.sessionId ?? null,
|
|
1673
|
-
createdAt: now,
|
|
1674
|
-
updatedAt: now
|
|
1675
|
-
}).returningAll().executeTakeFirst();
|
|
1676
|
-
},
|
|
1677
|
-
listRequests: async (params) => {
|
|
1678
|
-
const result = await listRequestsSchema.safeParseAsync(params || {});
|
|
1679
|
-
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
1680
|
-
const { limit, offset, configId, variantId, environmentId, providerConfigId, provider, model, startDate, endDate, tags } = result.data;
|
|
1681
|
-
let baseQuery = db.selectFrom("llm_requests");
|
|
1682
|
-
if (configId) baseQuery = baseQuery.where("configId", "=", configId);
|
|
1683
|
-
if (variantId) baseQuery = baseQuery.where("variantId", "=", variantId);
|
|
1684
|
-
if (environmentId) baseQuery = baseQuery.where("environmentId", "=", environmentId);
|
|
1685
|
-
if (providerConfigId) baseQuery = baseQuery.where("providerConfigId", "=", providerConfigId);
|
|
1686
|
-
if (provider) baseQuery = baseQuery.where("provider", "=", provider);
|
|
1687
|
-
if (model) baseQuery = baseQuery.where("model", "=", model);
|
|
1688
|
-
if (startDate) baseQuery = baseQuery.where(kysely.sql`${col$1("createdAt")} >= ${startDate.toISOString()}`);
|
|
1689
|
-
if (endDate) baseQuery = baseQuery.where(kysely.sql`${col$1("createdAt")} <= ${endDate.toISOString()}`);
|
|
1690
|
-
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
1691
|
-
if (values.length === 0) continue;
|
|
1692
|
-
if (values.length === 1) baseQuery = baseQuery.where(kysely.sql`${col$1("tags")}->>${key} = ${values[0]}`);
|
|
1693
|
-
else {
|
|
1694
|
-
const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
|
|
1695
|
-
baseQuery = baseQuery.where(kysely.sql`${col$1("tags")}->>${key} IN (${valueList})`);
|
|
1696
|
-
}
|
|
1697
|
-
}
|
|
1698
|
-
const countResult = await baseQuery.select(kysely.sql`COUNT(*)`.as("total")).executeTakeFirst();
|
|
1699
|
-
const total = Number(countResult?.total ?? 0);
|
|
1700
|
-
return {
|
|
1701
|
-
data: await baseQuery.selectAll().orderBy("createdAt", "desc").limit(limit).offset(offset).execute(),
|
|
1702
|
-
total,
|
|
1703
|
-
limit,
|
|
1704
|
-
offset
|
|
1705
|
-
};
|
|
1706
|
-
},
|
|
1707
|
-
getRequestByRequestId: async (requestId) => {
|
|
1708
|
-
return db.selectFrom("llm_requests").selectAll().where("requestId", "=", requestId).executeTakeFirst();
|
|
1709
|
-
},
|
|
1710
|
-
getTotalCost: async (params) => {
|
|
1711
|
-
const result = await dateRangeSchema.safeParseAsync(params);
|
|
1712
|
-
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
1713
|
-
const { startDate, endDate, configId, variantId, environmentId, tags } = result.data;
|
|
1714
|
-
let query = db.selectFrom("llm_requests").select([
|
|
1715
|
-
kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
|
|
1716
|
-
kysely.sql`COALESCE(SUM(${col$1("inputCost")}), 0)`.as("totalInputCost"),
|
|
1717
|
-
kysely.sql`COALESCE(SUM(${col$1("outputCost")}), 0)`.as("totalOutputCost"),
|
|
1718
|
-
kysely.sql`COALESCE(SUM(${col$1("promptTokens")}), 0)`.as("totalPromptTokens"),
|
|
1719
|
-
kysely.sql`COALESCE(SUM(${col$1("completionTokens")}), 0)`.as("totalCompletionTokens"),
|
|
1720
|
-
kysely.sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens"),
|
|
1721
|
-
kysely.sql`COALESCE(SUM(${col$1("cachedTokens")}), 0)`.as("totalCachedTokens"),
|
|
1722
|
-
kysely.sql`COALESCE(SUM(${col$1("cacheSavings")}), 0)`.as("totalCacheSavings"),
|
|
1723
|
-
kysely.sql`COUNT(*)`.as("requestCount")
|
|
1724
|
-
]).where(kysely.sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col$1("createdAt")} <= ${endDate.toISOString()}`);
|
|
1725
|
-
if (configId) query = query.where("configId", "=", configId);
|
|
1726
|
-
if (variantId) query = query.where("variantId", "=", variantId);
|
|
1727
|
-
if (environmentId) query = query.where("environmentId", "=", environmentId);
|
|
1728
|
-
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
1729
|
-
if (values.length === 0) continue;
|
|
1730
|
-
if (values.length === 1) query = query.where(kysely.sql`${col$1("tags")}->>${key} = ${values[0]}`);
|
|
1731
|
-
else {
|
|
1732
|
-
const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
|
|
1733
|
-
query = query.where(kysely.sql`${col$1("tags")}->>${key} IN (${valueList})`);
|
|
1734
|
-
}
|
|
1735
|
-
}
|
|
1736
|
-
return await query.executeTakeFirst();
|
|
1737
|
-
},
|
|
1738
|
-
getCostByModel: async (params) => {
|
|
1739
|
-
const result = await dateRangeSchema.safeParseAsync(params);
|
|
1740
|
-
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
1741
|
-
const { startDate, endDate } = result.data;
|
|
1742
|
-
return db.selectFrom("llm_requests").select([
|
|
1743
|
-
"provider",
|
|
1744
|
-
"model",
|
|
1745
|
-
kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
|
|
1746
|
-
kysely.sql`COALESCE(SUM(${col$1("inputCost")}), 0)`.as("totalInputCost"),
|
|
1747
|
-
kysely.sql`COALESCE(SUM(${col$1("outputCost")}), 0)`.as("totalOutputCost"),
|
|
1748
|
-
kysely.sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens"),
|
|
1749
|
-
kysely.sql`COUNT(*)`.as("requestCount"),
|
|
1750
|
-
kysely.sql`AVG(${col$1("latencyMs")})`.as("avgLatencyMs")
|
|
1751
|
-
]).where(kysely.sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col$1("createdAt")} <= ${endDate.toISOString()}`).groupBy(["provider", "model"]).orderBy(kysely.sql`SUM(${col$1("cost")})`, "desc").execute();
|
|
1546
|
+
updatePlaygroundResult: async (params) => {
|
|
1547
|
+
const value = await updatePlaygroundResult.safeParseAsync(params);
|
|
1548
|
+
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1549
|
+
const { resultId, outputContent, status, error, latencyMs, promptTokens, completionTokens, totalTokens, cost } = value.data;
|
|
1550
|
+
const updateData = { updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
1551
|
+
if (outputContent !== void 0) updateData.outputContent = outputContent;
|
|
1552
|
+
if (status !== void 0) updateData.status = status;
|
|
1553
|
+
if (error !== void 0) updateData.error = error;
|
|
1554
|
+
if (latencyMs !== void 0) updateData.latencyMs = latencyMs;
|
|
1555
|
+
if (promptTokens !== void 0) updateData.promptTokens = promptTokens;
|
|
1556
|
+
if (completionTokens !== void 0) updateData.completionTokens = completionTokens;
|
|
1557
|
+
if (totalTokens !== void 0) updateData.totalTokens = totalTokens;
|
|
1558
|
+
if (cost !== void 0) updateData.cost = cost;
|
|
1559
|
+
return db.updateTable("playground_results").set(updateData).where("id", "=", resultId).returningAll().executeTakeFirst();
|
|
1752
1560
|
},
|
|
1753
|
-
|
|
1754
|
-
const
|
|
1755
|
-
if (!
|
|
1756
|
-
const {
|
|
1757
|
-
return db.selectFrom("
|
|
1758
|
-
"provider",
|
|
1759
|
-
kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
|
|
1760
|
-
kysely.sql`COALESCE(SUM(${col$1("inputCost")}), 0)`.as("totalInputCost"),
|
|
1761
|
-
kysely.sql`COALESCE(SUM(${col$1("outputCost")}), 0)`.as("totalOutputCost"),
|
|
1762
|
-
kysely.sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens"),
|
|
1763
|
-
kysely.sql`COUNT(*)`.as("requestCount"),
|
|
1764
|
-
kysely.sql`AVG(${col$1("latencyMs")})`.as("avgLatencyMs")
|
|
1765
|
-
]).where(kysely.sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col$1("createdAt")} <= ${endDate.toISOString()}`).groupBy("provider").orderBy(kysely.sql`SUM(${col$1("cost")})`, "desc").execute();
|
|
1561
|
+
getPlaygroundResultById: async (params) => {
|
|
1562
|
+
const value = await getPlaygroundResultById.safeParseAsync(params);
|
|
1563
|
+
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1564
|
+
const { resultId } = value.data;
|
|
1565
|
+
return db.selectFrom("playground_results").selectAll().where("id", "=", resultId).executeTakeFirst();
|
|
1766
1566
|
},
|
|
1767
|
-
|
|
1768
|
-
const
|
|
1769
|
-
if (!
|
|
1770
|
-
const {
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
kysely.sql`COALESCE(SUM(${col$1("inputCost")}), 0)`.as("totalInputCost"),
|
|
1775
|
-
kysely.sql`COALESCE(SUM(${col$1("outputCost")}), 0)`.as("totalOutputCost"),
|
|
1776
|
-
kysely.sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens"),
|
|
1777
|
-
kysely.sql`COUNT(*)`.as("requestCount")
|
|
1778
|
-
]).where(kysely.sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col$1("createdAt")} <= ${endDate.toISOString()}`).groupBy(kysely.sql`DATE(${col$1("createdAt")})`).orderBy(kysely.sql`DATE(${col$1("createdAt")})`, "asc").execute();
|
|
1567
|
+
listPlaygroundResults: async (params) => {
|
|
1568
|
+
const value = await listPlaygroundResults.safeParseAsync(params);
|
|
1569
|
+
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1570
|
+
const { runId, columnId, limit = 500, offset = 0 } = value.data;
|
|
1571
|
+
let query = db.selectFrom("playground_results").selectAll().where("runId", "=", runId);
|
|
1572
|
+
if (columnId) query = query.where("columnId", "=", columnId);
|
|
1573
|
+
return query.orderBy("createdAt", "asc").limit(limit).offset(offset).execute();
|
|
1779
1574
|
},
|
|
1780
|
-
|
|
1781
|
-
const
|
|
1782
|
-
if (!
|
|
1783
|
-
const {
|
|
1784
|
-
|
|
1785
|
-
if (configId) baseQuery = baseQuery.where("configId", "=", configId);
|
|
1786
|
-
if (variantId) baseQuery = baseQuery.where("variantId", "=", variantId);
|
|
1787
|
-
if (environmentId) baseQuery = baseQuery.where("environmentId", "=", environmentId);
|
|
1788
|
-
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
1789
|
-
if (values.length === 0) continue;
|
|
1790
|
-
if (values.length === 1) baseQuery = baseQuery.where(kysely.sql`${col$1("tags")}->>${key} = ${values[0]}`);
|
|
1791
|
-
else {
|
|
1792
|
-
const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
|
|
1793
|
-
baseQuery = baseQuery.where(kysely.sql`${col$1("tags")}->>${key} IN (${valueList})`);
|
|
1794
|
-
}
|
|
1795
|
-
}
|
|
1796
|
-
switch (groupBy) {
|
|
1797
|
-
case "day": return baseQuery.select([
|
|
1798
|
-
kysely.sql`DATE(${col$1("createdAt")})`.as("groupKey"),
|
|
1799
|
-
kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
|
|
1800
|
-
kysely.sql`COUNT(*)`.as("requestCount"),
|
|
1801
|
-
kysely.sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens")
|
|
1802
|
-
]).groupBy(kysely.sql`DATE(${col$1("createdAt")})`).orderBy(kysely.sql`DATE(${col$1("createdAt")})`, "asc").execute();
|
|
1803
|
-
case "hour": return baseQuery.select([
|
|
1804
|
-
kysely.sql`DATE_TRUNC('hour', ${col$1("createdAt")})`.as("groupKey"),
|
|
1805
|
-
kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
|
|
1806
|
-
kysely.sql`COUNT(*)`.as("requestCount"),
|
|
1807
|
-
kysely.sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens")
|
|
1808
|
-
]).groupBy(kysely.sql`DATE_TRUNC('hour', ${col$1("createdAt")})`).orderBy(kysely.sql`DATE_TRUNC('hour', ${col$1("createdAt")})`, "asc").execute();
|
|
1809
|
-
case "model": return baseQuery.select([
|
|
1810
|
-
kysely.sql`${col$1("provider")} || '/' || ${col$1("model")}`.as("groupKey"),
|
|
1811
|
-
kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
|
|
1812
|
-
kysely.sql`COUNT(*)`.as("requestCount")
|
|
1813
|
-
]).groupBy(["provider", "model"]).orderBy(kysely.sql`SUM(${col$1("cost")})`, "desc").execute();
|
|
1814
|
-
case "provider": return baseQuery.select([
|
|
1815
|
-
kysely.sql`${col$1("provider")}`.as("groupKey"),
|
|
1816
|
-
kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
|
|
1817
|
-
kysely.sql`COUNT(*)`.as("requestCount")
|
|
1818
|
-
]).groupBy("provider").orderBy(kysely.sql`SUM(${col$1("cost")})`, "desc").execute();
|
|
1819
|
-
case "endpoint": return baseQuery.select([
|
|
1820
|
-
kysely.sql`COALESCE(${col$1("endpoint")}, 'unknown')`.as("groupKey"),
|
|
1821
|
-
kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
|
|
1822
|
-
kysely.sql`COUNT(*)`.as("requestCount")
|
|
1823
|
-
]).groupBy("endpoint").orderBy(kysely.sql`SUM(${col$1("cost")})`, "desc").execute();
|
|
1824
|
-
case "tags": {
|
|
1825
|
-
const conditions = [kysely.sql`${col$1("createdAt")} >= ${startDate.toISOString()}`, kysely.sql`${col$1("createdAt")} <= ${endDate.toISOString()}`];
|
|
1826
|
-
if (configId) conditions.push(kysely.sql`${col$1("configId")} = ${configId}`);
|
|
1827
|
-
if (variantId) conditions.push(kysely.sql`${col$1("variantId")} = ${variantId}`);
|
|
1828
|
-
if (environmentId) conditions.push(kysely.sql`${col$1("environmentId")} = ${environmentId}`);
|
|
1829
|
-
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
1830
|
-
if (values.length === 0) continue;
|
|
1831
|
-
if (values.length === 1) conditions.push(kysely.sql`${col$1("tags")}->>${key} = ${values[0]}`);
|
|
1832
|
-
else {
|
|
1833
|
-
const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
|
|
1834
|
-
conditions.push(kysely.sql`${col$1("tags")}->>${key} IN (${valueList})`);
|
|
1835
|
-
}
|
|
1836
|
-
}
|
|
1837
|
-
if (tagKeys && tagKeys.length > 0) {
|
|
1838
|
-
const tagKeyList = kysely.sql.join(tagKeys.map((k) => kysely.sql`${k}`), kysely.sql`, `);
|
|
1839
|
-
conditions.push(kysely.sql`t.key IN (${tagKeyList})`);
|
|
1840
|
-
}
|
|
1841
|
-
const whereClause = kysely.sql.join(conditions, kysely.sql` AND `);
|
|
1842
|
-
return (await kysely.sql`
|
|
1843
|
-
SELECT t.key || ':' || t.value as "groupKey",
|
|
1844
|
-
COALESCE(SUM(${col$1("cost")}), 0) as "totalCost",
|
|
1845
|
-
COUNT(*) as "requestCount"
|
|
1846
|
-
FROM "llm_requests", jsonb_each_text(${col$1("tags")}) t
|
|
1847
|
-
WHERE ${whereClause}
|
|
1848
|
-
GROUP BY t.key, t.value
|
|
1849
|
-
ORDER BY SUM(${col$1("cost")}) DESC
|
|
1850
|
-
`.execute(db)).rows;
|
|
1851
|
-
}
|
|
1852
|
-
default: return baseQuery.select([
|
|
1853
|
-
kysely.sql`'total'`.as("groupKey"),
|
|
1854
|
-
kysely.sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
|
|
1855
|
-
kysely.sql`COUNT(*)`.as("requestCount")
|
|
1856
|
-
]).execute();
|
|
1857
|
-
}
|
|
1575
|
+
deletePlaygroundResultsByRunId: async (params) => {
|
|
1576
|
+
const value = await deletePlaygroundResultsByRunId.safeParseAsync(params);
|
|
1577
|
+
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1578
|
+
const { runId } = value.data;
|
|
1579
|
+
return db.deleteFrom("playground_results").where("runId", "=", runId).returningAll().execute();
|
|
1858
1580
|
},
|
|
1859
|
-
|
|
1860
|
-
const result = await
|
|
1861
|
-
|
|
1862
|
-
const { startDate, endDate, configId, variantId, environmentId, tags } = result.data;
|
|
1863
|
-
let query = db.selectFrom("llm_requests").select([
|
|
1864
|
-
kysely.sql`COUNT(*)`.as("totalRequests"),
|
|
1865
|
-
kysely.sql`COUNT(CASE WHEN ${col$1("statusCode")} >= 200 AND ${col$1("statusCode")} < 300 THEN 1 END)`.as("successfulRequests"),
|
|
1866
|
-
kysely.sql`COUNT(CASE WHEN ${col$1("statusCode")} >= 400 THEN 1 END)`.as("failedRequests"),
|
|
1867
|
-
kysely.sql`COUNT(CASE WHEN ${col$1("isStreaming")} = true THEN 1 END)`.as("streamingRequests"),
|
|
1868
|
-
kysely.sql`AVG(${col$1("latencyMs")})`.as("avgLatencyMs"),
|
|
1869
|
-
kysely.sql`MAX(${col$1("latencyMs")})`.as("maxLatencyMs"),
|
|
1870
|
-
kysely.sql`MIN(${col$1("latencyMs")})`.as("minLatencyMs")
|
|
1871
|
-
]).where(kysely.sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col$1("createdAt")} <= ${endDate.toISOString()}`);
|
|
1872
|
-
if (configId) query = query.where("configId", "=", configId);
|
|
1873
|
-
if (variantId) query = query.where("variantId", "=", variantId);
|
|
1874
|
-
if (environmentId) query = query.where("environmentId", "=", environmentId);
|
|
1875
|
-
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
1876
|
-
if (values.length === 0) continue;
|
|
1877
|
-
if (values.length === 1) query = query.where(kysely.sql`${col$1("tags")}->>${key} = ${values[0]}`);
|
|
1878
|
-
else {
|
|
1879
|
-
const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
|
|
1880
|
-
query = query.where(kysely.sql`${col$1("tags")}->>${key} IN (${valueList})`);
|
|
1881
|
-
}
|
|
1882
|
-
}
|
|
1883
|
-
return await query.executeTakeFirst();
|
|
1581
|
+
countPlaygroundResults: async (runId) => {
|
|
1582
|
+
const result = await db.selectFrom("playground_results").select(db.fn.countAll().as("count")).where("runId", "=", runId).executeTakeFirst();
|
|
1583
|
+
return Number(result?.count ?? 0);
|
|
1884
1584
|
},
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
FROM llm_requests, jsonb_each_text(tags) AS t(key, value)
|
|
1889
|
-
WHERE tags != '{}'::jsonb
|
|
1890
|
-
ORDER BY key, value
|
|
1891
|
-
`.execute(db)).rows;
|
|
1585
|
+
countCompletedPlaygroundResults: async (runId) => {
|
|
1586
|
+
const result = await db.selectFrom("playground_results").select(db.fn.countAll().as("count")).where("runId", "=", runId).where("status", "=", "completed").executeTakeFirst();
|
|
1587
|
+
return Number(result?.count ?? 0);
|
|
1892
1588
|
}
|
|
1893
1589
|
};
|
|
1894
1590
|
};
|
|
1895
1591
|
|
|
1896
1592
|
//#endregion
|
|
1897
|
-
//#region src/datalayer/
|
|
1898
|
-
const
|
|
1899
|
-
|
|
1593
|
+
//#region src/datalayer/playgroundRuns.ts
|
|
1594
|
+
const createPlaygroundRun = require_db.zod_default.object({
|
|
1595
|
+
playgroundId: require_db.zod_default.string().uuid(),
|
|
1900
1596
|
datasetId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
1901
|
-
|
|
1597
|
+
datasetVersionId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
1598
|
+
status: require_db.zod_default.enum([
|
|
1599
|
+
"pending",
|
|
1600
|
+
"running",
|
|
1601
|
+
"completed",
|
|
1602
|
+
"failed",
|
|
1603
|
+
"cancelled"
|
|
1604
|
+
]).default("pending"),
|
|
1605
|
+
totalRecords: require_db.zod_default.number().int().default(0)
|
|
1902
1606
|
});
|
|
1903
|
-
const
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1607
|
+
const updatePlaygroundRun = require_db.zod_default.object({
|
|
1608
|
+
runId: require_db.zod_default.string().uuid(),
|
|
1609
|
+
status: require_db.zod_default.enum([
|
|
1610
|
+
"pending",
|
|
1611
|
+
"running",
|
|
1612
|
+
"completed",
|
|
1613
|
+
"failed",
|
|
1614
|
+
"cancelled"
|
|
1615
|
+
]).optional(),
|
|
1616
|
+
startedAt: require_db.zod_default.date().nullable().optional(),
|
|
1617
|
+
completedAt: require_db.zod_default.date().nullable().optional(),
|
|
1618
|
+
completedRecords: require_db.zod_default.number().int().optional()
|
|
1908
1619
|
});
|
|
1909
|
-
const
|
|
1910
|
-
const
|
|
1911
|
-
|
|
1620
|
+
const getPlaygroundRunById = require_db.zod_default.object({ runId: require_db.zod_default.string().uuid() });
|
|
1621
|
+
const listPlaygroundRuns = require_db.zod_default.object({
|
|
1622
|
+
playgroundId: require_db.zod_default.string().uuid(),
|
|
1912
1623
|
limit: require_db.zod_default.number().int().positive().optional(),
|
|
1913
1624
|
offset: require_db.zod_default.number().int().nonnegative().optional()
|
|
1914
1625
|
});
|
|
1915
|
-
const
|
|
1626
|
+
const deletePlaygroundRun = require_db.zod_default.object({ runId: require_db.zod_default.string().uuid() });
|
|
1627
|
+
const createPlaygroundRunsDataLayer = (db) => {
|
|
1916
1628
|
return {
|
|
1917
|
-
|
|
1918
|
-
const value = await
|
|
1919
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1920
|
-
const { name, datasetId, columns } = value.data;
|
|
1921
|
-
return db.insertInto("playgrounds").values({
|
|
1922
|
-
id: (0, node_crypto.randomUUID)(),
|
|
1923
|
-
name,
|
|
1924
|
-
datasetId: datasetId ?? null,
|
|
1925
|
-
columns: columns ? JSON.stringify(columns) : null,
|
|
1926
|
-
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1927
|
-
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1928
|
-
}).returningAll().executeTakeFirst();
|
|
1929
|
-
},
|
|
1930
|
-
updatePlayground: async (params) => {
|
|
1931
|
-
const value = await updatePlayground.safeParseAsync(params);
|
|
1932
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1933
|
-
const { playgroundId, name, datasetId, columns } = value.data;
|
|
1934
|
-
const updateData = { updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
1935
|
-
if (name !== void 0) updateData.name = name;
|
|
1936
|
-
if (datasetId !== void 0) updateData.datasetId = datasetId;
|
|
1937
|
-
if (columns !== void 0) updateData.columns = columns ? JSON.stringify(columns) : null;
|
|
1938
|
-
return db.updateTable("playgrounds").set(updateData).where("id", "=", playgroundId).returningAll().executeTakeFirst();
|
|
1939
|
-
},
|
|
1940
|
-
getPlaygroundById: async (params) => {
|
|
1941
|
-
const value = await getPlaygroundById.safeParseAsync(params);
|
|
1942
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1943
|
-
const { playgroundId } = value.data;
|
|
1944
|
-
return db.selectFrom("playgrounds").selectAll().where("id", "=", playgroundId).executeTakeFirst();
|
|
1945
|
-
},
|
|
1946
|
-
deletePlayground: async (params) => {
|
|
1947
|
-
const value = await deletePlayground.safeParseAsync(params);
|
|
1948
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1949
|
-
const { playgroundId } = value.data;
|
|
1950
|
-
return db.deleteFrom("playgrounds").where("id", "=", playgroundId).returningAll().executeTakeFirst();
|
|
1951
|
-
},
|
|
1952
|
-
listPlaygrounds: async (params) => {
|
|
1953
|
-
const value = await listPlaygrounds.safeParseAsync(params || {});
|
|
1954
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1955
|
-
const { limit = 100, offset = 0 } = value.data;
|
|
1956
|
-
return db.selectFrom("playgrounds").selectAll().orderBy("createdAt", "desc").limit(limit).offset(offset).execute();
|
|
1957
|
-
},
|
|
1958
|
-
countPlaygrounds: async () => {
|
|
1959
|
-
const result = await db.selectFrom("playgrounds").select(db.fn.countAll().as("count")).executeTakeFirst();
|
|
1960
|
-
return Number(result?.count ?? 0);
|
|
1961
|
-
}
|
|
1962
|
-
};
|
|
1963
|
-
};
|
|
1964
|
-
|
|
1965
|
-
//#endregion
|
|
1966
|
-
//#region src/datalayer/playgroundResults.ts
|
|
1967
|
-
const createPlaygroundResult = require_db.zod_default.object({
|
|
1968
|
-
runId: require_db.zod_default.string().uuid(),
|
|
1969
|
-
columnId: require_db.zod_default.string().uuid(),
|
|
1970
|
-
datasetRecordId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
1971
|
-
inputVariables: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.unknown()).default({}),
|
|
1972
|
-
status: require_db.zod_default.enum([
|
|
1973
|
-
"pending",
|
|
1974
|
-
"running",
|
|
1975
|
-
"completed",
|
|
1976
|
-
"failed"
|
|
1977
|
-
]).default("pending")
|
|
1978
|
-
});
|
|
1979
|
-
const createPlaygroundResultsBatch = require_db.zod_default.object({ results: require_db.zod_default.array(createPlaygroundResult) });
|
|
1980
|
-
const updatePlaygroundResult = require_db.zod_default.object({
|
|
1981
|
-
resultId: require_db.zod_default.string().uuid(),
|
|
1982
|
-
outputContent: require_db.zod_default.string().nullable().optional(),
|
|
1983
|
-
status: require_db.zod_default.enum([
|
|
1984
|
-
"pending",
|
|
1985
|
-
"running",
|
|
1986
|
-
"completed",
|
|
1987
|
-
"failed"
|
|
1988
|
-
]).optional(),
|
|
1989
|
-
error: require_db.zod_default.string().nullable().optional(),
|
|
1990
|
-
latencyMs: require_db.zod_default.number().int().nullable().optional(),
|
|
1991
|
-
promptTokens: require_db.zod_default.number().int().nullable().optional(),
|
|
1992
|
-
completionTokens: require_db.zod_default.number().int().nullable().optional(),
|
|
1993
|
-
totalTokens: require_db.zod_default.number().int().nullable().optional(),
|
|
1994
|
-
cost: require_db.zod_default.number().int().nullable().optional()
|
|
1995
|
-
});
|
|
1996
|
-
const getPlaygroundResultById = require_db.zod_default.object({ resultId: require_db.zod_default.string().uuid() });
|
|
1997
|
-
const listPlaygroundResults = require_db.zod_default.object({
|
|
1998
|
-
runId: require_db.zod_default.string().uuid(),
|
|
1999
|
-
columnId: require_db.zod_default.string().uuid().optional(),
|
|
2000
|
-
limit: require_db.zod_default.number().int().positive().optional(),
|
|
2001
|
-
offset: require_db.zod_default.number().int().nonnegative().optional()
|
|
2002
|
-
});
|
|
2003
|
-
const deletePlaygroundResultsByRunId = require_db.zod_default.object({ runId: require_db.zod_default.string().uuid() });
|
|
2004
|
-
const createPlaygroundResultsDataLayer = (db) => {
|
|
2005
|
-
return {
|
|
2006
|
-
createPlaygroundResult: async (params) => {
|
|
2007
|
-
const value = await createPlaygroundResult.safeParseAsync(params);
|
|
2008
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2009
|
-
const { runId, columnId, datasetRecordId, inputVariables, status } = value.data;
|
|
2010
|
-
return db.insertInto("playground_results").values({
|
|
2011
|
-
id: (0, node_crypto.randomUUID)(),
|
|
2012
|
-
runId,
|
|
2013
|
-
columnId,
|
|
2014
|
-
datasetRecordId: datasetRecordId ?? null,
|
|
2015
|
-
inputVariables: JSON.stringify(inputVariables),
|
|
2016
|
-
outputContent: null,
|
|
2017
|
-
status,
|
|
2018
|
-
error: null,
|
|
2019
|
-
latencyMs: null,
|
|
2020
|
-
promptTokens: null,
|
|
2021
|
-
completionTokens: null,
|
|
2022
|
-
totalTokens: null,
|
|
2023
|
-
cost: null,
|
|
2024
|
-
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2025
|
-
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2026
|
-
}).returningAll().executeTakeFirst();
|
|
2027
|
-
},
|
|
2028
|
-
createPlaygroundResultsBatch: async (params) => {
|
|
2029
|
-
const value = await createPlaygroundResultsBatch.safeParseAsync(params);
|
|
2030
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2031
|
-
const { results } = value.data;
|
|
2032
|
-
if (results.length === 0) return [];
|
|
2033
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
2034
|
-
const values = results.map((result) => ({
|
|
2035
|
-
id: (0, node_crypto.randomUUID)(),
|
|
2036
|
-
runId: result.runId,
|
|
2037
|
-
columnId: result.columnId,
|
|
2038
|
-
datasetRecordId: result.datasetRecordId ?? null,
|
|
2039
|
-
inputVariables: JSON.stringify(result.inputVariables),
|
|
2040
|
-
outputContent: null,
|
|
2041
|
-
status: result.status,
|
|
2042
|
-
error: null,
|
|
2043
|
-
latencyMs: null,
|
|
2044
|
-
promptTokens: null,
|
|
2045
|
-
completionTokens: null,
|
|
2046
|
-
totalTokens: null,
|
|
2047
|
-
cost: null,
|
|
2048
|
-
createdAt: now,
|
|
2049
|
-
updatedAt: now
|
|
2050
|
-
}));
|
|
2051
|
-
return db.insertInto("playground_results").values(values).returningAll().execute();
|
|
2052
|
-
},
|
|
2053
|
-
updatePlaygroundResult: async (params) => {
|
|
2054
|
-
const value = await updatePlaygroundResult.safeParseAsync(params);
|
|
2055
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2056
|
-
const { resultId, outputContent, status, error, latencyMs, promptTokens, completionTokens, totalTokens, cost } = value.data;
|
|
2057
|
-
const updateData = { updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
2058
|
-
if (outputContent !== void 0) updateData.outputContent = outputContent;
|
|
2059
|
-
if (status !== void 0) updateData.status = status;
|
|
2060
|
-
if (error !== void 0) updateData.error = error;
|
|
2061
|
-
if (latencyMs !== void 0) updateData.latencyMs = latencyMs;
|
|
2062
|
-
if (promptTokens !== void 0) updateData.promptTokens = promptTokens;
|
|
2063
|
-
if (completionTokens !== void 0) updateData.completionTokens = completionTokens;
|
|
2064
|
-
if (totalTokens !== void 0) updateData.totalTokens = totalTokens;
|
|
2065
|
-
if (cost !== void 0) updateData.cost = cost;
|
|
2066
|
-
return db.updateTable("playground_results").set(updateData).where("id", "=", resultId).returningAll().executeTakeFirst();
|
|
2067
|
-
},
|
|
2068
|
-
getPlaygroundResultById: async (params) => {
|
|
2069
|
-
const value = await getPlaygroundResultById.safeParseAsync(params);
|
|
2070
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2071
|
-
const { resultId } = value.data;
|
|
2072
|
-
return db.selectFrom("playground_results").selectAll().where("id", "=", resultId).executeTakeFirst();
|
|
2073
|
-
},
|
|
2074
|
-
listPlaygroundResults: async (params) => {
|
|
2075
|
-
const value = await listPlaygroundResults.safeParseAsync(params);
|
|
2076
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2077
|
-
const { runId, columnId, limit = 500, offset = 0 } = value.data;
|
|
2078
|
-
let query = db.selectFrom("playground_results").selectAll().where("runId", "=", runId);
|
|
2079
|
-
if (columnId) query = query.where("columnId", "=", columnId);
|
|
2080
|
-
return query.orderBy("createdAt", "asc").limit(limit).offset(offset).execute();
|
|
2081
|
-
},
|
|
2082
|
-
deletePlaygroundResultsByRunId: async (params) => {
|
|
2083
|
-
const value = await deletePlaygroundResultsByRunId.safeParseAsync(params);
|
|
2084
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2085
|
-
const { runId } = value.data;
|
|
2086
|
-
return db.deleteFrom("playground_results").where("runId", "=", runId).returningAll().execute();
|
|
2087
|
-
},
|
|
2088
|
-
countPlaygroundResults: async (runId) => {
|
|
2089
|
-
const result = await db.selectFrom("playground_results").select(db.fn.countAll().as("count")).where("runId", "=", runId).executeTakeFirst();
|
|
2090
|
-
return Number(result?.count ?? 0);
|
|
2091
|
-
},
|
|
2092
|
-
countCompletedPlaygroundResults: async (runId) => {
|
|
2093
|
-
const result = await db.selectFrom("playground_results").select(db.fn.countAll().as("count")).where("runId", "=", runId).where("status", "=", "completed").executeTakeFirst();
|
|
2094
|
-
return Number(result?.count ?? 0);
|
|
2095
|
-
}
|
|
2096
|
-
};
|
|
2097
|
-
};
|
|
2098
|
-
|
|
2099
|
-
//#endregion
|
|
2100
|
-
//#region src/datalayer/playgroundRuns.ts
|
|
2101
|
-
const createPlaygroundRun = require_db.zod_default.object({
|
|
2102
|
-
playgroundId: require_db.zod_default.string().uuid(),
|
|
2103
|
-
datasetId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2104
|
-
datasetVersionId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2105
|
-
status: require_db.zod_default.enum([
|
|
2106
|
-
"pending",
|
|
2107
|
-
"running",
|
|
2108
|
-
"completed",
|
|
2109
|
-
"failed",
|
|
2110
|
-
"cancelled"
|
|
2111
|
-
]).default("pending"),
|
|
2112
|
-
totalRecords: require_db.zod_default.number().int().default(0)
|
|
2113
|
-
});
|
|
2114
|
-
const updatePlaygroundRun = require_db.zod_default.object({
|
|
2115
|
-
runId: require_db.zod_default.string().uuid(),
|
|
2116
|
-
status: require_db.zod_default.enum([
|
|
2117
|
-
"pending",
|
|
2118
|
-
"running",
|
|
2119
|
-
"completed",
|
|
2120
|
-
"failed",
|
|
2121
|
-
"cancelled"
|
|
2122
|
-
]).optional(),
|
|
2123
|
-
startedAt: require_db.zod_default.date().nullable().optional(),
|
|
2124
|
-
completedAt: require_db.zod_default.date().nullable().optional(),
|
|
2125
|
-
completedRecords: require_db.zod_default.number().int().optional()
|
|
2126
|
-
});
|
|
2127
|
-
const getPlaygroundRunById = require_db.zod_default.object({ runId: require_db.zod_default.string().uuid() });
|
|
2128
|
-
const listPlaygroundRuns = require_db.zod_default.object({
|
|
2129
|
-
playgroundId: require_db.zod_default.string().uuid(),
|
|
2130
|
-
limit: require_db.zod_default.number().int().positive().optional(),
|
|
2131
|
-
offset: require_db.zod_default.number().int().nonnegative().optional()
|
|
2132
|
-
});
|
|
2133
|
-
const deletePlaygroundRun = require_db.zod_default.object({ runId: require_db.zod_default.string().uuid() });
|
|
2134
|
-
const createPlaygroundRunsDataLayer = (db) => {
|
|
2135
|
-
return {
|
|
2136
|
-
createPlaygroundRun: async (params) => {
|
|
2137
|
-
const value = await createPlaygroundRun.safeParseAsync(params);
|
|
1629
|
+
createPlaygroundRun: async (params) => {
|
|
1630
|
+
const value = await createPlaygroundRun.safeParseAsync(params);
|
|
2138
1631
|
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2139
1632
|
const { playgroundId, datasetId, datasetVersionId, status, totalRecords } = value.data;
|
|
2140
1633
|
return db.insertInto("playground_runs").values({
|
|
@@ -2188,478 +1681,641 @@ const createPlaygroundRunsDataLayer = (db) => {
|
|
|
2188
1681
|
};
|
|
2189
1682
|
|
|
2190
1683
|
//#endregion
|
|
2191
|
-
//#region src/datalayer/
|
|
2192
|
-
|
|
2193
|
-
* Generate a unique slug for a provider config.
|
|
2194
|
-
* If the base slug already exists, appends -01, -02, etc.
|
|
2195
|
-
*/
|
|
2196
|
-
async function generateUniqueSlug(db, baseSlug) {
|
|
2197
|
-
const existing = await db.selectFrom("provider_configs").select("slug").where("slug", "like", `${baseSlug}%`).execute();
|
|
2198
|
-
if (existing.length === 0) return baseSlug;
|
|
2199
|
-
const existingSlugs = new Set(existing.map((e) => e.slug));
|
|
2200
|
-
if (!existingSlugs.has(baseSlug)) return baseSlug;
|
|
2201
|
-
let counter = 1;
|
|
2202
|
-
while (counter < 100) {
|
|
2203
|
-
const candidateSlug = `${baseSlug}-${counter.toString().padStart(2, "0")}`;
|
|
2204
|
-
if (!existingSlugs.has(candidateSlug)) return candidateSlug;
|
|
2205
|
-
counter++;
|
|
2206
|
-
}
|
|
2207
|
-
return `${baseSlug}-${(0, node_crypto.randomUUID)().slice(0, 8)}`;
|
|
2208
|
-
}
|
|
2209
|
-
const createProviderConfig = require_db.zod_default.object({
|
|
2210
|
-
providerId: require_db.zod_default.string().min(1),
|
|
2211
|
-
slug: require_db.zod_default.string().nullable().optional(),
|
|
2212
|
-
name: require_db.zod_default.string().nullable().optional(),
|
|
2213
|
-
config: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.unknown()),
|
|
2214
|
-
enabled: require_db.zod_default.boolean().optional().default(true)
|
|
2215
|
-
});
|
|
2216
|
-
const updateProviderConfig = require_db.zod_default.object({
|
|
2217
|
-
id: require_db.zod_default.uuidv4(),
|
|
2218
|
-
slug: require_db.zod_default.string().nullable().optional(),
|
|
1684
|
+
//#region src/datalayer/workspaceSettings.ts
|
|
1685
|
+
const updateWorkspaceSettings = require_db.zod_default.object({
|
|
2219
1686
|
name: require_db.zod_default.string().nullable().optional(),
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
});
|
|
2223
|
-
const getProviderConfigById = require_db.zod_default.object({ id: require_db.zod_default.uuidv4() });
|
|
2224
|
-
const getProviderConfigByProviderId = require_db.zod_default.object({ providerId: require_db.zod_default.string().min(1) });
|
|
2225
|
-
const getProviderConfigBySlug = require_db.zod_default.object({ slug: require_db.zod_default.string().min(1) });
|
|
2226
|
-
const deleteProviderConfig = require_db.zod_default.object({ id: require_db.zod_default.uuidv4() });
|
|
2227
|
-
const listProviderConfigs = require_db.zod_default.object({
|
|
2228
|
-
limit: require_db.zod_default.number().int().positive().optional(),
|
|
2229
|
-
offset: require_db.zod_default.number().int().nonnegative().optional()
|
|
1687
|
+
setupComplete: require_db.zod_default.boolean().optional(),
|
|
1688
|
+
superAdminId: require_db.zod_default.string().nullable().optional()
|
|
2230
1689
|
});
|
|
2231
|
-
const
|
|
1690
|
+
const createWorkspaceSettingsDataLayer = (db) => {
|
|
2232
1691
|
return {
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
if (!
|
|
2236
|
-
const { providerId, slug, name, config, enabled } = value.data;
|
|
2237
|
-
const finalSlug = slug ?? await generateUniqueSlug(db, providerId);
|
|
2238
|
-
return db.insertInto("provider_configs").values({
|
|
1692
|
+
getWorkspaceSettings: async () => {
|
|
1693
|
+
let settings = await db.selectFrom("workspace_settings").selectAll().executeTakeFirst();
|
|
1694
|
+
if (!settings) settings = await db.insertInto("workspace_settings").values({
|
|
2239
1695
|
id: (0, node_crypto.randomUUID)(),
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
name: name ?? null,
|
|
2243
|
-
config: JSON.stringify(config),
|
|
2244
|
-
enabled,
|
|
1696
|
+
name: null,
|
|
1697
|
+
setupComplete: false,
|
|
2245
1698
|
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2246
1699
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2247
1700
|
}).returningAll().executeTakeFirst();
|
|
1701
|
+
return settings;
|
|
2248
1702
|
},
|
|
2249
|
-
|
|
2250
|
-
const value = await
|
|
1703
|
+
updateWorkspaceSettings: async (params) => {
|
|
1704
|
+
const value = await updateWorkspaceSettings.safeParseAsync(params);
|
|
2251
1705
|
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2252
|
-
|
|
1706
|
+
let settings = await db.selectFrom("workspace_settings").selectAll().executeTakeFirst();
|
|
1707
|
+
if (!settings) return db.insertInto("workspace_settings").values({
|
|
1708
|
+
id: (0, node_crypto.randomUUID)(),
|
|
1709
|
+
name: value.data.name ?? null,
|
|
1710
|
+
setupComplete: value.data.setupComplete ?? false,
|
|
1711
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1712
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1713
|
+
}).returningAll().executeTakeFirst();
|
|
2253
1714
|
const updateData = { updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
2254
|
-
if (
|
|
2255
|
-
if (
|
|
2256
|
-
if (
|
|
2257
|
-
|
|
2258
|
-
return db.updateTable("provider_configs").set(updateData).where("id", "=", id).returningAll().executeTakeFirst();
|
|
2259
|
-
},
|
|
2260
|
-
getProviderConfigById: async (params) => {
|
|
2261
|
-
const value = await getProviderConfigById.safeParseAsync(params);
|
|
2262
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2263
|
-
const { id } = value.data;
|
|
2264
|
-
return db.selectFrom("provider_configs").selectAll().where("id", "=", id).executeTakeFirst();
|
|
1715
|
+
if (value.data.name !== void 0) updateData.name = value.data.name ?? null;
|
|
1716
|
+
if (value.data.setupComplete !== void 0) updateData.setupComplete = value.data.setupComplete;
|
|
1717
|
+
if (value.data.superAdminId !== void 0) updateData.superAdminId = value.data.superAdminId ?? null;
|
|
1718
|
+
return db.updateTable("workspace_settings").set(updateData).where("id", "=", settings.id).returningAll().executeTakeFirst();
|
|
2265
1719
|
},
|
|
2266
|
-
|
|
2267
|
-
|
|
2268
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2269
|
-
const { providerId } = value.data;
|
|
2270
|
-
return db.selectFrom("provider_configs").selectAll().where("providerId", "=", providerId).executeTakeFirst();
|
|
2271
|
-
},
|
|
2272
|
-
getProviderConfigBySlug: async (params) => {
|
|
2273
|
-
const value = await getProviderConfigBySlug.safeParseAsync(params);
|
|
2274
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2275
|
-
const { slug } = value.data;
|
|
2276
|
-
return db.selectFrom("provider_configs").selectAll().where("slug", "=", slug).executeTakeFirst();
|
|
2277
|
-
},
|
|
2278
|
-
deleteProviderConfig: async (params) => {
|
|
2279
|
-
const value = await deleteProviderConfig.safeParseAsync(params);
|
|
2280
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2281
|
-
const { id } = value.data;
|
|
2282
|
-
return db.deleteFrom("provider_configs").where("id", "=", id).returningAll().executeTakeFirst();
|
|
2283
|
-
},
|
|
2284
|
-
listProviderConfigs: async (params) => {
|
|
2285
|
-
const value = await listProviderConfigs.safeParseAsync(params || {});
|
|
2286
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2287
|
-
const { limit = 100, offset = 0 } = value.data;
|
|
2288
|
-
return db.selectFrom("provider_configs").selectAll().orderBy("createdAt", "desc").limit(limit).offset(offset).execute();
|
|
2289
|
-
},
|
|
2290
|
-
countProviderConfigs: async () => {
|
|
2291
|
-
const result = await db.selectFrom("provider_configs").select(db.fn.countAll().as("count")).executeTakeFirst();
|
|
2292
|
-
return Number(result?.count ?? 0);
|
|
1720
|
+
getSuperAdminId: async () => {
|
|
1721
|
+
return (await db.selectFrom("workspace_settings").select("superAdminId").executeTakeFirst())?.superAdminId ?? null;
|
|
2293
1722
|
},
|
|
2294
|
-
|
|
2295
|
-
|
|
2296
|
-
if (
|
|
2297
|
-
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
config: JSON.stringify(config),
|
|
2305
|
-
enabled,
|
|
1723
|
+
setSuperAdminId: async (userId) => {
|
|
1724
|
+
let settings = await db.selectFrom("workspace_settings").selectAll().executeTakeFirst();
|
|
1725
|
+
if (settings?.superAdminId) return false;
|
|
1726
|
+
if (!settings) {
|
|
1727
|
+
await db.insertInto("workspace_settings").values({
|
|
1728
|
+
id: (0, node_crypto.randomUUID)(),
|
|
1729
|
+
name: null,
|
|
1730
|
+
setupComplete: false,
|
|
1731
|
+
superAdminId: userId,
|
|
1732
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2306
1733
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2307
|
-
}).
|
|
1734
|
+
}).execute();
|
|
1735
|
+
return true;
|
|
2308
1736
|
}
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
id: (0, node_crypto.randomUUID)(),
|
|
2312
|
-
providerId,
|
|
2313
|
-
slug: finalSlug,
|
|
2314
|
-
name: name ?? null,
|
|
2315
|
-
config: JSON.stringify(config),
|
|
2316
|
-
enabled,
|
|
2317
|
-
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2318
|
-
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2319
|
-
}).returningAll().executeTakeFirst();
|
|
2320
|
-
}
|
|
2321
|
-
};
|
|
2322
|
-
};
|
|
2323
|
-
|
|
2324
|
-
//#endregion
|
|
2325
|
-
//#region src/datalayer/providerGuardrailOverrides.ts
|
|
2326
|
-
const createProviderGuardrailOverride = require_db.zod_default.object({
|
|
2327
|
-
providerConfigId: require_db.zod_default.string().uuid(),
|
|
2328
|
-
guardrailConfigId: require_db.zod_default.string().uuid(),
|
|
2329
|
-
enabled: require_db.zod_default.boolean().optional().default(true),
|
|
2330
|
-
parameters: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.unknown()).nullable().optional()
|
|
2331
|
-
});
|
|
2332
|
-
const updateProviderGuardrailOverride = require_db.zod_default.object({
|
|
2333
|
-
id: require_db.zod_default.string().uuid(),
|
|
2334
|
-
enabled: require_db.zod_default.boolean().optional(),
|
|
2335
|
-
parameters: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.unknown()).nullable().optional()
|
|
2336
|
-
});
|
|
2337
|
-
const getOverrideById = require_db.zod_default.object({ id: require_db.zod_default.string().uuid() });
|
|
2338
|
-
const deleteOverride = require_db.zod_default.object({ id: require_db.zod_default.string().uuid() });
|
|
2339
|
-
const getOverridesByProviderConfigId = require_db.zod_default.object({ providerConfigId: require_db.zod_default.string().uuid() });
|
|
2340
|
-
const getOverridesByGuardrailConfigId = require_db.zod_default.object({ guardrailConfigId: require_db.zod_default.string().uuid() });
|
|
2341
|
-
const getOverrideByProviderAndGuardrail = require_db.zod_default.object({
|
|
2342
|
-
providerConfigId: require_db.zod_default.string().uuid(),
|
|
2343
|
-
guardrailConfigId: require_db.zod_default.string().uuid()
|
|
2344
|
-
});
|
|
2345
|
-
const createProviderGuardrailOverridesDataLayer = (db) => {
|
|
2346
|
-
return {
|
|
2347
|
-
createProviderGuardrailOverride: async (params) => {
|
|
2348
|
-
const value = await createProviderGuardrailOverride.safeParseAsync(params);
|
|
2349
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2350
|
-
const { providerConfigId, guardrailConfigId, enabled, parameters } = value.data;
|
|
2351
|
-
return db.insertInto("provider_guardrail_overrides").values({
|
|
2352
|
-
id: (0, node_crypto.randomUUID)(),
|
|
2353
|
-
providerConfigId,
|
|
2354
|
-
guardrailConfigId,
|
|
2355
|
-
enabled,
|
|
2356
|
-
parameters: parameters ? JSON.stringify(parameters) : null,
|
|
2357
|
-
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1737
|
+
await db.updateTable("workspace_settings").set({
|
|
1738
|
+
superAdminId: userId,
|
|
2358
1739
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2359
|
-
}).
|
|
2360
|
-
|
|
2361
|
-
updateProviderGuardrailOverride: async (params) => {
|
|
2362
|
-
const value = await updateProviderGuardrailOverride.safeParseAsync(params);
|
|
2363
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2364
|
-
const { id, enabled, parameters } = value.data;
|
|
2365
|
-
const updateData = { updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
2366
|
-
if (enabled !== void 0) updateData.enabled = enabled;
|
|
2367
|
-
if (parameters !== void 0) updateData.parameters = parameters ? JSON.stringify(parameters) : null;
|
|
2368
|
-
return db.updateTable("provider_guardrail_overrides").set(updateData).where("id", "=", id).returningAll().executeTakeFirst();
|
|
2369
|
-
},
|
|
2370
|
-
getOverrideById: async (params) => {
|
|
2371
|
-
const value = await getOverrideById.safeParseAsync(params);
|
|
2372
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2373
|
-
const { id } = value.data;
|
|
2374
|
-
return db.selectFrom("provider_guardrail_overrides").selectAll().where("id", "=", id).executeTakeFirst();
|
|
2375
|
-
},
|
|
2376
|
-
deleteProviderGuardrailOverride: async (params) => {
|
|
2377
|
-
const value = await deleteOverride.safeParseAsync(params);
|
|
2378
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2379
|
-
const { id } = value.data;
|
|
2380
|
-
return db.deleteFrom("provider_guardrail_overrides").where("id", "=", id).returningAll().executeTakeFirst();
|
|
2381
|
-
},
|
|
2382
|
-
getOverridesByProviderConfigId: async (params) => {
|
|
2383
|
-
const value = await getOverridesByProviderConfigId.safeParseAsync(params);
|
|
2384
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2385
|
-
const { providerConfigId } = value.data;
|
|
2386
|
-
return db.selectFrom("provider_guardrail_overrides").selectAll().where("providerConfigId", "=", providerConfigId).execute();
|
|
2387
|
-
},
|
|
2388
|
-
getOverridesByGuardrailConfigId: async (params) => {
|
|
2389
|
-
const value = await getOverridesByGuardrailConfigId.safeParseAsync(params);
|
|
2390
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2391
|
-
const { guardrailConfigId } = value.data;
|
|
2392
|
-
return db.selectFrom("provider_guardrail_overrides").selectAll().where("guardrailConfigId", "=", guardrailConfigId).execute();
|
|
1740
|
+
}).where("id", "=", settings.id).execute();
|
|
1741
|
+
return true;
|
|
2393
1742
|
},
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
1743
|
+
isSetupComplete: async () => {
|
|
1744
|
+
try {
|
|
1745
|
+
return (await db.selectFrom("workspace_settings").select("setupComplete").executeTakeFirst())?.setupComplete ?? false;
|
|
1746
|
+
} catch {
|
|
1747
|
+
return false;
|
|
1748
|
+
}
|
|
2399
1749
|
},
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
if (!
|
|
2403
|
-
const { providerConfigId, guardrailConfigId, enabled, parameters } = value.data;
|
|
2404
|
-
const existing = await db.selectFrom("provider_guardrail_overrides").selectAll().where("providerConfigId", "=", providerConfigId).where("guardrailConfigId", "=", guardrailConfigId).executeTakeFirst();
|
|
2405
|
-
if (existing) return db.updateTable("provider_guardrail_overrides").set({
|
|
2406
|
-
enabled,
|
|
2407
|
-
parameters: parameters ? JSON.stringify(parameters) : null,
|
|
2408
|
-
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2409
|
-
}).where("id", "=", existing.id).returningAll().executeTakeFirst();
|
|
2410
|
-
return db.insertInto("provider_guardrail_overrides").values({
|
|
1750
|
+
markSetupComplete: async () => {
|
|
1751
|
+
let settings = await db.selectFrom("workspace_settings").selectAll().executeTakeFirst();
|
|
1752
|
+
if (!settings) return db.insertInto("workspace_settings").values({
|
|
2411
1753
|
id: (0, node_crypto.randomUUID)(),
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
enabled,
|
|
2415
|
-
parameters: parameters ? JSON.stringify(parameters) : null,
|
|
1754
|
+
name: null,
|
|
1755
|
+
setupComplete: true,
|
|
2416
1756
|
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2417
1757
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2418
1758
|
}).returningAll().executeTakeFirst();
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
const { guardrailConfigId } = value.data;
|
|
2424
|
-
return db.deleteFrom("provider_guardrail_overrides").where("guardrailConfigId", "=", guardrailConfigId).execute();
|
|
1759
|
+
return db.updateTable("workspace_settings").set({
|
|
1760
|
+
setupComplete: true,
|
|
1761
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1762
|
+
}).where("id", "=", settings.id).returningAll().executeTakeFirst();
|
|
2425
1763
|
}
|
|
2426
1764
|
};
|
|
2427
1765
|
};
|
|
2428
1766
|
|
|
2429
1767
|
//#endregion
|
|
2430
|
-
//#region src/datalayer/
|
|
2431
|
-
const col = (name) => kysely.sql.ref(name);
|
|
1768
|
+
//#region src/datalayer/create.ts
|
|
2432
1769
|
/**
|
|
2433
|
-
*
|
|
1770
|
+
* Create all datalayers from a Kysely database instance.
|
|
1771
|
+
* Returns a flat object with all datalayer methods spread together.
|
|
2434
1772
|
*/
|
|
2435
|
-
|
|
2436
|
-
|
|
2437
|
-
|
|
2438
|
-
|
|
2439
|
-
|
|
2440
|
-
|
|
2441
|
-
|
|
2442
|
-
|
|
2443
|
-
|
|
2444
|
-
|
|
2445
|
-
|
|
2446
|
-
|
|
2447
|
-
durationMs: require_db.zod_default.number().int().nullable().optional(),
|
|
2448
|
-
spanCount: require_db.zod_default.number().int().default(1),
|
|
2449
|
-
totalInputTokens: require_db.zod_default.number().int().default(0),
|
|
2450
|
-
totalOutputTokens: require_db.zod_default.number().int().default(0),
|
|
2451
|
-
totalTokens: require_db.zod_default.number().int().default(0),
|
|
2452
|
-
totalCost: require_db.zod_default.number().int().default(0),
|
|
2453
|
-
tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.string()).default({}),
|
|
2454
|
-
metadata: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.unknown()).default({})
|
|
2455
|
-
});
|
|
1773
|
+
function createDataLayer(db) {
|
|
1774
|
+
return {
|
|
1775
|
+
...createDatasetsDataLayer(db),
|
|
1776
|
+
...createPlaygroundDataLayer(db),
|
|
1777
|
+
...createPlaygroundResultsDataLayer(db),
|
|
1778
|
+
...createPlaygroundRunsDataLayer(db),
|
|
1779
|
+
...createWorkspaceSettingsDataLayer(db)
|
|
1780
|
+
};
|
|
1781
|
+
}
|
|
1782
|
+
|
|
1783
|
+
//#endregion
|
|
1784
|
+
//#region src/pricing/calculator.ts
|
|
2456
1785
|
/**
|
|
2457
|
-
*
|
|
1786
|
+
* Calculate the cost of an LLM request in micro-dollars
|
|
1787
|
+
*
|
|
1788
|
+
* Micro-dollars are used to avoid floating-point precision issues:
|
|
1789
|
+
* - 1 dollar = 1,000,000 micro-dollars
|
|
1790
|
+
* - $0.001 = 1,000 micro-dollars
|
|
1791
|
+
* - $0.000001 = 1 micro-dollar
|
|
1792
|
+
*
|
|
1793
|
+
* @param usage - Token usage data from the LLM response
|
|
1794
|
+
* @param pricing - Model pricing information
|
|
1795
|
+
* @returns Cost breakdown in micro-dollars
|
|
1796
|
+
*
|
|
1797
|
+
* @example
|
|
1798
|
+
* ```typescript
|
|
1799
|
+
* const usage = { promptTokens: 1000, completionTokens: 500 };
|
|
1800
|
+
* const pricing = { inputCostPer1M: 2.5, outputCostPer1M: 10.0 };
|
|
1801
|
+
* const cost = calculateCost(usage, pricing);
|
|
1802
|
+
* // cost = { inputCost: 2500, outputCost: 5000, totalCost: 7500 }
|
|
1803
|
+
* // In dollars: $0.0025 input + $0.005 output = $0.0075 total
|
|
1804
|
+
* ```
|
|
2458
1805
|
*/
|
|
2459
|
-
|
|
2460
|
-
|
|
2461
|
-
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
durationMs: require_db.zod_default.number().int().nullable().optional(),
|
|
2470
|
-
provider: require_db.zod_default.string().nullable().optional(),
|
|
2471
|
-
model: require_db.zod_default.string().nullable().optional(),
|
|
2472
|
-
promptTokens: require_db.zod_default.number().int().default(0),
|
|
2473
|
-
completionTokens: require_db.zod_default.number().int().default(0),
|
|
2474
|
-
totalTokens: require_db.zod_default.number().int().default(0),
|
|
2475
|
-
cost: require_db.zod_default.number().int().default(0),
|
|
2476
|
-
configId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2477
|
-
variantId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2478
|
-
environmentId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2479
|
-
providerConfigId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2480
|
-
requestId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2481
|
-
source: require_db.zod_default.enum([
|
|
2482
|
-
"gateway",
|
|
2483
|
-
"otlp",
|
|
2484
|
-
"langsmith"
|
|
2485
|
-
]).default("gateway"),
|
|
2486
|
-
input: require_db.zod_default.unknown().nullable().optional(),
|
|
2487
|
-
output: require_db.zod_default.unknown().nullable().optional(),
|
|
2488
|
-
attributes: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.unknown()).default({})
|
|
2489
|
-
});
|
|
1806
|
+
function calculateCost(usage, pricing) {
|
|
1807
|
+
const inputCost = Math.round(usage.promptTokens * pricing.inputCostPer1M);
|
|
1808
|
+
const outputCost = Math.round(usage.completionTokens * pricing.outputCostPer1M);
|
|
1809
|
+
return {
|
|
1810
|
+
inputCost,
|
|
1811
|
+
outputCost,
|
|
1812
|
+
totalCost: inputCost + outputCost,
|
|
1813
|
+
cacheSavings: 0
|
|
1814
|
+
};
|
|
1815
|
+
}
|
|
2490
1816
|
/**
|
|
2491
|
-
*
|
|
1817
|
+
* Get default cache read rate as a fraction of input cost per provider.
|
|
1818
|
+
* Used when models.dev doesn't provide cache pricing.
|
|
2492
1819
|
*/
|
|
2493
|
-
|
|
2494
|
-
|
|
2495
|
-
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
1820
|
+
function getDefaultCacheReadRate(provider, inputCostPer1M) {
|
|
1821
|
+
switch (provider?.toLowerCase()) {
|
|
1822
|
+
case "anthropic": return inputCostPer1M * .1;
|
|
1823
|
+
case "openai":
|
|
1824
|
+
case "azure-openai": return inputCostPer1M * .5;
|
|
1825
|
+
case "google":
|
|
1826
|
+
case "gemini":
|
|
1827
|
+
case "vertex_ai": return inputCostPer1M * .25;
|
|
1828
|
+
default: return inputCostPer1M * .5;
|
|
1829
|
+
}
|
|
1830
|
+
}
|
|
2500
1831
|
/**
|
|
2501
|
-
*
|
|
1832
|
+
* Get default cache write/creation rate as a fraction of input cost per provider.
|
|
1833
|
+
* Used when models.dev doesn't provide cache pricing.
|
|
2502
1834
|
*/
|
|
2503
|
-
|
|
2504
|
-
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
"unset",
|
|
2510
|
-
"ok",
|
|
2511
|
-
"error"
|
|
2512
|
-
]).optional(),
|
|
2513
|
-
name: require_db.zod_default.string().optional(),
|
|
2514
|
-
startDate: require_db.zod_default.date().optional(),
|
|
2515
|
-
endDate: require_db.zod_default.date().optional(),
|
|
2516
|
-
tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.array(require_db.zod_default.string())).optional()
|
|
2517
|
-
});
|
|
1835
|
+
function getDefaultCacheWriteRate(provider, inputCostPer1M) {
|
|
1836
|
+
switch (provider?.toLowerCase()) {
|
|
1837
|
+
case "anthropic": return inputCostPer1M * 1.25;
|
|
1838
|
+
default: return inputCostPer1M;
|
|
1839
|
+
}
|
|
1840
|
+
}
|
|
2518
1841
|
/**
|
|
2519
|
-
*
|
|
1842
|
+
* Calculate cache-aware cost of an LLM request in micro-dollars.
|
|
1843
|
+
*
|
|
1844
|
+
* Splits input tokens into uncached, cache-read, and cache-creation buckets,
|
|
1845
|
+
* each priced at different rates. Falls back to provider-specific multipliers
|
|
1846
|
+
* when models.dev doesn't provide cache pricing.
|
|
1847
|
+
*
|
|
1848
|
+
* @param usage - Token usage data (with cachedTokens and cacheCreationTokens)
|
|
1849
|
+
* @param pricing - Model pricing (may include cacheReadCostPer1M / cacheWriteCostPer1M)
|
|
1850
|
+
* @param provider - Provider name for fallback rate selection
|
|
1851
|
+
* @returns Cost breakdown in micro-dollars
|
|
2520
1852
|
*/
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
const
|
|
1853
|
+
function calculateCacheAwareCost(usage, pricing, provider) {
|
|
1854
|
+
const cachedTokens = usage.cachedTokens ?? 0;
|
|
1855
|
+
const cacheCreationTokens = usage.cacheCreationTokens ?? 0;
|
|
1856
|
+
if (cachedTokens === 0 && cacheCreationTokens === 0) return calculateCost(usage, pricing);
|
|
1857
|
+
const cacheReadRate = pricing.cacheReadCostPer1M ?? getDefaultCacheReadRate(provider, pricing.inputCostPer1M);
|
|
1858
|
+
const cacheWriteRate = pricing.cacheWriteCostPer1M ?? getDefaultCacheWriteRate(provider, pricing.inputCostPer1M);
|
|
1859
|
+
const uncachedInputTokens = Math.max(0, usage.promptTokens - cachedTokens - cacheCreationTokens);
|
|
1860
|
+
const regularInputCost = Math.round(uncachedInputTokens * pricing.inputCostPer1M);
|
|
1861
|
+
const cacheReadCost = Math.round(cachedTokens * cacheReadRate);
|
|
1862
|
+
const cacheWriteCost = Math.round(cacheCreationTokens * cacheWriteRate);
|
|
1863
|
+
const outputCost = Math.round(usage.completionTokens * pricing.outputCostPer1M);
|
|
1864
|
+
const inputCost = regularInputCost + cacheReadCost + cacheWriteCost;
|
|
2528
1865
|
return {
|
|
2529
|
-
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
|
|
2534
|
-
|
|
2535
|
-
|
|
2536
|
-
|
|
2537
|
-
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
|
|
2541
|
-
|
|
2542
|
-
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
|
|
2555
|
-
|
|
2556
|
-
|
|
2557
|
-
|
|
2558
|
-
|
|
2559
|
-
|
|
2560
|
-
|
|
2561
|
-
|
|
2562
|
-
|
|
2563
|
-
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
|
|
2567
|
-
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
2584
|
-
|
|
2585
|
-
|
|
2586
|
-
|
|
2587
|
-
|
|
2588
|
-
|
|
2589
|
-
|
|
2590
|
-
|
|
1866
|
+
inputCost,
|
|
1867
|
+
outputCost,
|
|
1868
|
+
totalCost: inputCost + outputCost,
|
|
1869
|
+
cacheSavings: Math.round((cachedTokens + cacheCreationTokens) * pricing.inputCostPer1M) - cacheReadCost - cacheWriteCost
|
|
1870
|
+
};
|
|
1871
|
+
}
|
|
1872
|
+
/**
|
|
1873
|
+
* Convert micro-dollars to dollars
|
|
1874
|
+
*
|
|
1875
|
+
* @param microDollars - Amount in micro-dollars
|
|
1876
|
+
* @returns Amount in dollars
|
|
1877
|
+
*
|
|
1878
|
+
* @example
|
|
1879
|
+
* ```typescript
|
|
1880
|
+
* microDollarsToDollars(7500); // 0.0075
|
|
1881
|
+
* microDollarsToDollars(1000000); // 1.0
|
|
1882
|
+
* ```
|
|
1883
|
+
*/
|
|
1884
|
+
function microDollarsToDollars(microDollars) {
|
|
1885
|
+
return microDollars / 1e6;
|
|
1886
|
+
}
|
|
1887
|
+
/**
|
|
1888
|
+
* Convert dollars to micro-dollars
|
|
1889
|
+
*
|
|
1890
|
+
* @param dollars - Amount in dollars
|
|
1891
|
+
* @returns Amount in micro-dollars (rounded to nearest integer)
|
|
1892
|
+
*
|
|
1893
|
+
* @example
|
|
1894
|
+
* ```typescript
|
|
1895
|
+
* dollarsToMicroDollars(0.0075); // 7500
|
|
1896
|
+
* dollarsToMicroDollars(1.0); // 1000000
|
|
1897
|
+
* ```
|
|
1898
|
+
*/
|
|
1899
|
+
function dollarsToMicroDollars(dollars) {
|
|
1900
|
+
return Math.round(dollars * 1e6);
|
|
1901
|
+
}
|
|
1902
|
+
/**
|
|
1903
|
+
* Format micro-dollars as a human-readable dollar string
|
|
1904
|
+
*
|
|
1905
|
+
* @param microDollars - Amount in micro-dollars
|
|
1906
|
+
* @param decimals - Number of decimal places (default: 6)
|
|
1907
|
+
* @returns Formatted dollar string
|
|
1908
|
+
*
|
|
1909
|
+
* @example
|
|
1910
|
+
* ```typescript
|
|
1911
|
+
* formatCost(7500); // "$0.007500"
|
|
1912
|
+
* formatCost(1234567, 2); // "$1.23"
|
|
1913
|
+
* ```
|
|
1914
|
+
*/
|
|
1915
|
+
function formatCost(microDollars, decimals = 6) {
|
|
1916
|
+
return `$${microDollarsToDollars(microDollars).toFixed(decimals)}`;
|
|
1917
|
+
}
|
|
1918
|
+
|
|
1919
|
+
//#endregion
|
|
1920
|
+
//#region src/pricing/provider.ts
|
|
1921
|
+
const LLMOPS_MODELS_API = "https://models.llmops.build";
|
|
1922
|
+
/**
|
|
1923
|
+
* Convert price from USD cents per token to dollars per 1M tokens.
|
|
1924
|
+
*
|
|
1925
|
+
* API returns cents/token. Our system uses dollars/1M tokens.
|
|
1926
|
+
* Formula: (centsPerToken / 100) * 1_000_000 = centsPerToken * 10_000
|
|
1927
|
+
*/
|
|
1928
|
+
function centsPerTokenToCostPer1M(centsPerToken) {
|
|
1929
|
+
return centsPerToken * 1e4;
|
|
1930
|
+
}
|
|
1931
|
+
/**
|
|
1932
|
+
* Pricing provider that fetches per-model data from the LLMOps Models API.
|
|
1933
|
+
*
|
|
1934
|
+
* Features:
|
|
1935
|
+
* - Per-model in-memory cache with configurable TTL (default 5 minutes)
|
|
1936
|
+
* - Deduplicates concurrent fetches for the same model
|
|
1937
|
+
* - Caches null results (404s) to avoid repeated lookups
|
|
1938
|
+
* - Falls back to stale cache on fetch errors
|
|
1939
|
+
*/
|
|
1940
|
+
var LLMOpsPricingProvider = class {
|
|
1941
|
+
cache = /* @__PURE__ */ new Map();
|
|
1942
|
+
pendingFetches = /* @__PURE__ */ new Map();
|
|
1943
|
+
cacheTTL;
|
|
1944
|
+
baseUrl;
|
|
1945
|
+
constructor(options) {
|
|
1946
|
+
this.cacheTTL = options?.cacheTTL ?? 300 * 1e3;
|
|
1947
|
+
this.baseUrl = options?.baseUrl ?? LLMOPS_MODELS_API;
|
|
1948
|
+
}
|
|
1949
|
+
getCacheKey(provider, model) {
|
|
1950
|
+
return `${provider.toLowerCase()}:${model.toLowerCase()}`;
|
|
1951
|
+
}
|
|
1952
|
+
/**
|
|
1953
|
+
* Fetch pricing for a single model from the API
|
|
1954
|
+
*/
|
|
1955
|
+
async fetchModelPricing(provider, model) {
|
|
1956
|
+
const url = `${this.baseUrl}/model-configs/pricing/${encodeURIComponent(provider)}/${model}`;
|
|
1957
|
+
try {
|
|
1958
|
+
require_db.logger.debug(`[Pricing] GET ${url}`);
|
|
1959
|
+
const startTime = Date.now();
|
|
1960
|
+
const response = await fetch(url);
|
|
1961
|
+
const elapsed = Date.now() - startTime;
|
|
1962
|
+
require_db.logger.debug(`[Pricing] GET ${url} -> ${response.status} (${elapsed}ms)`);
|
|
1963
|
+
if (response.status === 404) {
|
|
1964
|
+
require_db.logger.debug(`[Pricing] No pricing found for ${provider}/${model}`);
|
|
1965
|
+
return null;
|
|
2591
1966
|
}
|
|
2592
|
-
if (
|
|
1967
|
+
if (!response.ok) throw new Error(`API returned ${response.status}`);
|
|
1968
|
+
const data = await response.json();
|
|
1969
|
+
if (!data.pay_as_you_go) return null;
|
|
1970
|
+
const payg = data.pay_as_you_go;
|
|
1971
|
+
const pricing = {
|
|
1972
|
+
inputCostPer1M: centsPerTokenToCostPer1M(payg.request_token?.price ?? 0),
|
|
1973
|
+
outputCostPer1M: centsPerTokenToCostPer1M(payg.response_token?.price ?? 0),
|
|
1974
|
+
cacheReadCostPer1M: payg.cache_read_input_token?.price != null ? centsPerTokenToCostPer1M(payg.cache_read_input_token.price) : void 0,
|
|
1975
|
+
cacheWriteCostPer1M: payg.cache_write_input_token?.price != null ? centsPerTokenToCostPer1M(payg.cache_write_input_token.price) : void 0
|
|
1976
|
+
};
|
|
1977
|
+
require_db.logger.debug(`[Pricing] Cached pricing for ${provider}/${model}: input=$${pricing.inputCostPer1M}/1M, output=$${pricing.outputCostPer1M}/1M`);
|
|
1978
|
+
return pricing;
|
|
1979
|
+
} catch (error) {
|
|
1980
|
+
require_db.logger.error(`[Pricing] Failed to fetch pricing for ${provider}/${model}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1981
|
+
const cacheKey = this.getCacheKey(provider, model);
|
|
1982
|
+
const stale = this.cache.get(cacheKey);
|
|
1983
|
+
if (stale) {
|
|
1984
|
+
require_db.logger.debug(`[Pricing] Using stale cache for ${provider}/${model}`);
|
|
1985
|
+
return stale.pricing;
|
|
1986
|
+
}
|
|
1987
|
+
return null;
|
|
1988
|
+
}
|
|
1989
|
+
}
|
|
1990
|
+
/**
|
|
1991
|
+
* Internal: fetch with cache and deduplication for a specific provider+model
|
|
1992
|
+
*/
|
|
1993
|
+
async getCachedPricing(provider, model) {
|
|
1994
|
+
const cacheKey = this.getCacheKey(provider, model);
|
|
1995
|
+
const cached = this.cache.get(cacheKey);
|
|
1996
|
+
if (cached && Date.now() - cached.fetchedAt < this.cacheTTL) return cached.pricing;
|
|
1997
|
+
let pending = this.pendingFetches.get(cacheKey);
|
|
1998
|
+
if (!pending) {
|
|
1999
|
+
pending = this.fetchModelPricing(provider, model).then((pricing) => {
|
|
2000
|
+
this.cache.set(cacheKey, {
|
|
2001
|
+
pricing,
|
|
2002
|
+
fetchedAt: Date.now()
|
|
2003
|
+
});
|
|
2004
|
+
return pricing;
|
|
2005
|
+
}).finally(() => {
|
|
2006
|
+
this.pendingFetches.delete(cacheKey);
|
|
2007
|
+
});
|
|
2008
|
+
this.pendingFetches.set(cacheKey, pending);
|
|
2009
|
+
}
|
|
2010
|
+
return pending;
|
|
2011
|
+
}
|
|
2012
|
+
/**
|
|
2013
|
+
* Get pricing for a specific model.
|
|
2014
|
+
*
|
|
2015
|
+
* When the model name contains a slash (e.g. "google/gemini-2.5-flash"),
|
|
2016
|
+
* it's likely an OpenRouter model ID. If the initial provider lookup fails,
|
|
2017
|
+
* we automatically retry with "openrouter" as the provider.
|
|
2018
|
+
*/
|
|
2019
|
+
async getModelPricing(provider, model) {
|
|
2020
|
+
const pricing = await this.getCachedPricing(provider, model);
|
|
2021
|
+
if (pricing) return pricing;
|
|
2022
|
+
if (!pricing && model.includes("/") && provider.toLowerCase() !== "openrouter") {
|
|
2023
|
+
require_db.logger.debug(`[Pricing] Retrying ${provider}/${model} as openrouter/${model}`);
|
|
2024
|
+
return this.getCachedPricing("openrouter", model);
|
|
2025
|
+
}
|
|
2026
|
+
return pricing;
|
|
2027
|
+
}
|
|
2028
|
+
/**
|
|
2029
|
+
* Force refresh the pricing cache (clears all cached entries)
|
|
2030
|
+
*/
|
|
2031
|
+
async refreshCache() {
|
|
2032
|
+
this.cache.clear();
|
|
2033
|
+
}
|
|
2034
|
+
/**
|
|
2035
|
+
* Always ready — no bulk pre-fetch needed
|
|
2036
|
+
*/
|
|
2037
|
+
isReady() {
|
|
2038
|
+
return true;
|
|
2039
|
+
}
|
|
2040
|
+
/**
|
|
2041
|
+
* Get the number of cached models (for debugging)
|
|
2042
|
+
*/
|
|
2043
|
+
getCacheSize() {
|
|
2044
|
+
return this.cache.size;
|
|
2045
|
+
}
|
|
2046
|
+
};
|
|
2047
|
+
let defaultProvider = null;
|
|
2048
|
+
/**
|
|
2049
|
+
* Get the default pricing provider instance
|
|
2050
|
+
*/
|
|
2051
|
+
function getDefaultPricingProvider() {
|
|
2052
|
+
if (!defaultProvider) defaultProvider = new LLMOpsPricingProvider();
|
|
2053
|
+
return defaultProvider;
|
|
2054
|
+
}
|
|
2055
|
+
|
|
2056
|
+
//#endregion
|
|
2057
|
+
//#region src/telemetry/postgres.ts
|
|
2058
|
+
const guardrailResultSchema = require_db.zod_default.object({
|
|
2059
|
+
checkId: require_db.zod_default.string(),
|
|
2060
|
+
functionId: require_db.zod_default.string(),
|
|
2061
|
+
hookType: require_db.zod_default.enum(["beforeRequestHook", "afterRequestHook"]),
|
|
2062
|
+
verdict: require_db.zod_default.boolean(),
|
|
2063
|
+
latencyMs: require_db.zod_default.number()
|
|
2064
|
+
});
|
|
2065
|
+
const guardrailResultsSchema = require_db.zod_default.object({
|
|
2066
|
+
results: require_db.zod_default.array(guardrailResultSchema),
|
|
2067
|
+
action: require_db.zod_default.enum([
|
|
2068
|
+
"allowed",
|
|
2069
|
+
"blocked",
|
|
2070
|
+
"logged"
|
|
2071
|
+
]),
|
|
2072
|
+
totalLatencyMs: require_db.zod_default.number()
|
|
2073
|
+
});
|
|
2074
|
+
const insertLLMRequestSchema = require_db.zod_default.object({
|
|
2075
|
+
requestId: require_db.zod_default.string().uuid(),
|
|
2076
|
+
configId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2077
|
+
variantId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2078
|
+
environmentId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2079
|
+
providerConfigId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2080
|
+
provider: require_db.zod_default.string(),
|
|
2081
|
+
model: require_db.zod_default.string(),
|
|
2082
|
+
promptTokens: require_db.zod_default.number().int().default(0),
|
|
2083
|
+
completionTokens: require_db.zod_default.number().int().default(0),
|
|
2084
|
+
totalTokens: require_db.zod_default.number().int().default(0),
|
|
2085
|
+
cachedTokens: require_db.zod_default.number().int().default(0),
|
|
2086
|
+
cacheCreationTokens: require_db.zod_default.number().int().default(0),
|
|
2087
|
+
cost: require_db.zod_default.number().int().default(0),
|
|
2088
|
+
cacheSavings: require_db.zod_default.number().int().default(0),
|
|
2089
|
+
inputCost: require_db.zod_default.number().int().default(0),
|
|
2090
|
+
outputCost: require_db.zod_default.number().int().default(0),
|
|
2091
|
+
endpoint: require_db.zod_default.string(),
|
|
2092
|
+
statusCode: require_db.zod_default.number().int(),
|
|
2093
|
+
latencyMs: require_db.zod_default.number().int().default(0),
|
|
2094
|
+
isStreaming: require_db.zod_default.boolean().default(false),
|
|
2095
|
+
userId: require_db.zod_default.string().nullable().optional(),
|
|
2096
|
+
tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.string()).default({}),
|
|
2097
|
+
guardrailResults: guardrailResultsSchema.nullable().optional(),
|
|
2098
|
+
traceId: require_db.zod_default.string().nullable().optional(),
|
|
2099
|
+
spanId: require_db.zod_default.string().nullable().optional(),
|
|
2100
|
+
parentSpanId: require_db.zod_default.string().nullable().optional(),
|
|
2101
|
+
sessionId: require_db.zod_default.string().nullable().optional()
|
|
2102
|
+
});
|
|
2103
|
+
const listRequestsSchema = require_db.zod_default.object({
|
|
2104
|
+
limit: require_db.zod_default.number().int().positive().max(1e3).default(100),
|
|
2105
|
+
offset: require_db.zod_default.number().int().nonnegative().default(0),
|
|
2106
|
+
configId: require_db.zod_default.string().uuid().optional(),
|
|
2107
|
+
variantId: require_db.zod_default.string().uuid().optional(),
|
|
2108
|
+
environmentId: require_db.zod_default.string().uuid().optional(),
|
|
2109
|
+
providerConfigId: require_db.zod_default.string().uuid().optional(),
|
|
2110
|
+
provider: require_db.zod_default.string().optional(),
|
|
2111
|
+
model: require_db.zod_default.string().optional(),
|
|
2112
|
+
startDate: require_db.zod_default.date().optional(),
|
|
2113
|
+
endDate: require_db.zod_default.date().optional(),
|
|
2114
|
+
tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.array(require_db.zod_default.string())).optional()
|
|
2115
|
+
});
|
|
2116
|
+
const dateRangeSchema = require_db.zod_default.object({
|
|
2117
|
+
startDate: require_db.zod_default.date(),
|
|
2118
|
+
endDate: require_db.zod_default.date(),
|
|
2119
|
+
configId: require_db.zod_default.string().uuid().optional(),
|
|
2120
|
+
variantId: require_db.zod_default.string().uuid().optional(),
|
|
2121
|
+
environmentId: require_db.zod_default.string().uuid().optional(),
|
|
2122
|
+
tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.array(require_db.zod_default.string())).optional()
|
|
2123
|
+
});
|
|
2124
|
+
const COST_SUMMARY_GROUP_BY = [
|
|
2125
|
+
"day",
|
|
2126
|
+
"hour",
|
|
2127
|
+
"model",
|
|
2128
|
+
"provider",
|
|
2129
|
+
"endpoint",
|
|
2130
|
+
"tags"
|
|
2131
|
+
];
|
|
2132
|
+
const costSummarySchema = require_db.zod_default.object({
|
|
2133
|
+
startDate: require_db.zod_default.date(),
|
|
2134
|
+
endDate: require_db.zod_default.date(),
|
|
2135
|
+
configId: require_db.zod_default.string().uuid().optional(),
|
|
2136
|
+
variantId: require_db.zod_default.string().uuid().optional(),
|
|
2137
|
+
environmentId: require_db.zod_default.string().uuid().optional(),
|
|
2138
|
+
tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.array(require_db.zod_default.string())).optional(),
|
|
2139
|
+
groupBy: require_db.zod_default.enum(COST_SUMMARY_GROUP_BY).optional(),
|
|
2140
|
+
tagKeys: require_db.zod_default.array(require_db.zod_default.string()).optional()
|
|
2141
|
+
});
|
|
2142
|
+
const upsertTraceSchema = require_db.zod_default.object({
|
|
2143
|
+
traceId: require_db.zod_default.string(),
|
|
2144
|
+
name: require_db.zod_default.string().nullable().optional(),
|
|
2145
|
+
sessionId: require_db.zod_default.string().nullable().optional(),
|
|
2146
|
+
userId: require_db.zod_default.string().nullable().optional(),
|
|
2147
|
+
status: require_db.zod_default.enum([
|
|
2148
|
+
"unset",
|
|
2149
|
+
"ok",
|
|
2150
|
+
"error"
|
|
2151
|
+
]).default("unset"),
|
|
2152
|
+
startTime: require_db.zod_default.date(),
|
|
2153
|
+
endTime: require_db.zod_default.date().nullable().optional(),
|
|
2154
|
+
durationMs: require_db.zod_default.number().int().nullable().optional(),
|
|
2155
|
+
spanCount: require_db.zod_default.number().int().default(1),
|
|
2156
|
+
totalInputTokens: require_db.zod_default.number().int().default(0),
|
|
2157
|
+
totalOutputTokens: require_db.zod_default.number().int().default(0),
|
|
2158
|
+
totalTokens: require_db.zod_default.number().int().default(0),
|
|
2159
|
+
totalCost: require_db.zod_default.number().int().default(0),
|
|
2160
|
+
tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.string()).default({}),
|
|
2161
|
+
metadata: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.unknown()).default({})
|
|
2162
|
+
});
|
|
2163
|
+
const insertSpanSchema = require_db.zod_default.object({
|
|
2164
|
+
traceId: require_db.zod_default.string(),
|
|
2165
|
+
spanId: require_db.zod_default.string(),
|
|
2166
|
+
parentSpanId: require_db.zod_default.string().nullable().optional(),
|
|
2167
|
+
name: require_db.zod_default.string(),
|
|
2168
|
+
kind: require_db.zod_default.number().int().default(1),
|
|
2169
|
+
status: require_db.zod_default.number().int().default(0),
|
|
2170
|
+
statusMessage: require_db.zod_default.string().nullable().optional(),
|
|
2171
|
+
startTime: require_db.zod_default.date(),
|
|
2172
|
+
endTime: require_db.zod_default.date().nullable().optional(),
|
|
2173
|
+
durationMs: require_db.zod_default.number().int().nullable().optional(),
|
|
2174
|
+
provider: require_db.zod_default.string().nullable().optional(),
|
|
2175
|
+
model: require_db.zod_default.string().nullable().optional(),
|
|
2176
|
+
promptTokens: require_db.zod_default.number().int().default(0),
|
|
2177
|
+
completionTokens: require_db.zod_default.number().int().default(0),
|
|
2178
|
+
totalTokens: require_db.zod_default.number().int().default(0),
|
|
2179
|
+
cost: require_db.zod_default.number().int().default(0),
|
|
2180
|
+
configId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2181
|
+
variantId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2182
|
+
environmentId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2183
|
+
providerConfigId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2184
|
+
requestId: require_db.zod_default.string().uuid().nullable().optional(),
|
|
2185
|
+
source: require_db.zod_default.enum([
|
|
2186
|
+
"gateway",
|
|
2187
|
+
"otlp",
|
|
2188
|
+
"langsmith"
|
|
2189
|
+
]).default("gateway"),
|
|
2190
|
+
input: require_db.zod_default.unknown().nullable().optional(),
|
|
2191
|
+
output: require_db.zod_default.unknown().nullable().optional(),
|
|
2192
|
+
attributes: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.unknown()).default({})
|
|
2193
|
+
});
|
|
2194
|
+
const insertSpanEventSchema = require_db.zod_default.object({
|
|
2195
|
+
traceId: require_db.zod_default.string(),
|
|
2196
|
+
spanId: require_db.zod_default.string(),
|
|
2197
|
+
name: require_db.zod_default.string(),
|
|
2198
|
+
timestamp: require_db.zod_default.date(),
|
|
2199
|
+
attributes: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.unknown()).default({})
|
|
2200
|
+
});
|
|
2201
|
+
const listTracesSchema = require_db.zod_default.object({
|
|
2202
|
+
limit: require_db.zod_default.number().int().positive().max(1e3).default(50),
|
|
2203
|
+
offset: require_db.zod_default.number().int().nonnegative().default(0),
|
|
2204
|
+
sessionId: require_db.zod_default.string().optional(),
|
|
2205
|
+
userId: require_db.zod_default.string().optional(),
|
|
2206
|
+
status: require_db.zod_default.enum([
|
|
2207
|
+
"unset",
|
|
2208
|
+
"ok",
|
|
2209
|
+
"error"
|
|
2210
|
+
]).optional(),
|
|
2211
|
+
name: require_db.zod_default.string().optional(),
|
|
2212
|
+
startDate: require_db.zod_default.date().optional(),
|
|
2213
|
+
endDate: require_db.zod_default.date().optional(),
|
|
2214
|
+
tags: require_db.zod_default.record(require_db.zod_default.string(), require_db.zod_default.array(require_db.zod_default.string())).optional()
|
|
2215
|
+
});
|
|
2216
|
+
const traceStatsSchema = require_db.zod_default.object({
|
|
2217
|
+
startDate: require_db.zod_default.date(),
|
|
2218
|
+
endDate: require_db.zod_default.date(),
|
|
2219
|
+
sessionId: require_db.zod_default.string().optional(),
|
|
2220
|
+
userId: require_db.zod_default.string().optional()
|
|
2221
|
+
});
|
|
2222
|
+
const col = (name) => kysely.sql.ref(name);
|
|
2223
|
+
function createLLMRequestsStore(db) {
|
|
2224
|
+
return {
|
|
2225
|
+
batchInsertRequests: async (requests) => {
|
|
2226
|
+
if (requests.length === 0) return { count: 0 };
|
|
2227
|
+
const validatedRequests = await Promise.all(requests.map(async (req) => {
|
|
2228
|
+
const result = await insertLLMRequestSchema.safeParseAsync(req);
|
|
2229
|
+
if (!result.success) throw new LLMOpsError(`Invalid request data: ${result.error.message}`);
|
|
2230
|
+
return result.data;
|
|
2231
|
+
}));
|
|
2593
2232
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
2594
|
-
const values =
|
|
2233
|
+
const values = validatedRequests.map((req) => ({
|
|
2595
2234
|
id: (0, node_crypto.randomUUID)(),
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
|
|
2602
|
-
|
|
2603
|
-
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
|
-
|
|
2607
|
-
|
|
2608
|
-
|
|
2609
|
-
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
|
|
2620
|
-
|
|
2235
|
+
requestId: req.requestId,
|
|
2236
|
+
configId: req.configId ?? null,
|
|
2237
|
+
variantId: req.variantId ?? null,
|
|
2238
|
+
environmentId: req.environmentId ?? null,
|
|
2239
|
+
providerConfigId: req.providerConfigId ?? null,
|
|
2240
|
+
provider: req.provider,
|
|
2241
|
+
model: req.model,
|
|
2242
|
+
promptTokens: req.promptTokens,
|
|
2243
|
+
completionTokens: req.completionTokens,
|
|
2244
|
+
totalTokens: req.totalTokens,
|
|
2245
|
+
cachedTokens: req.cachedTokens,
|
|
2246
|
+
cacheCreationTokens: req.cacheCreationTokens,
|
|
2247
|
+
cost: req.cost,
|
|
2248
|
+
cacheSavings: req.cacheSavings,
|
|
2249
|
+
inputCost: req.inputCost,
|
|
2250
|
+
outputCost: req.outputCost,
|
|
2251
|
+
endpoint: req.endpoint,
|
|
2252
|
+
statusCode: req.statusCode,
|
|
2253
|
+
latencyMs: req.latencyMs,
|
|
2254
|
+
isStreaming: req.isStreaming,
|
|
2255
|
+
userId: req.userId ?? null,
|
|
2256
|
+
tags: JSON.stringify(req.tags),
|
|
2257
|
+
guardrailResults: req.guardrailResults ? JSON.stringify(req.guardrailResults) : null,
|
|
2258
|
+
traceId: req.traceId ?? null,
|
|
2259
|
+
spanId: req.spanId ?? null,
|
|
2260
|
+
parentSpanId: req.parentSpanId ?? null,
|
|
2261
|
+
sessionId: req.sessionId ?? null,
|
|
2621
2262
|
createdAt: now,
|
|
2622
2263
|
updatedAt: now
|
|
2623
2264
|
}));
|
|
2624
|
-
await db.insertInto("
|
|
2625
|
-
return { count: values.length };
|
|
2626
|
-
},
|
|
2627
|
-
batchInsertSpanEvents: async (events) => {
|
|
2628
|
-
if (events.length === 0) return { count: 0 };
|
|
2629
|
-
const validatedEvents = [];
|
|
2630
|
-
for (const event of events) {
|
|
2631
|
-
const result = await insertSpanEventSchema.safeParseAsync(event);
|
|
2632
|
-
if (!result.success) {
|
|
2633
|
-
require_db.logger.warn(`[batchInsertSpanEvents] Skipping invalid event: ${result.error.message}`);
|
|
2634
|
-
continue;
|
|
2635
|
-
}
|
|
2636
|
-
validatedEvents.push(result.data);
|
|
2637
|
-
}
|
|
2638
|
-
if (validatedEvents.length === 0) return { count: 0 };
|
|
2639
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
2640
|
-
const values = validatedEvents.map((event) => ({
|
|
2641
|
-
id: (0, node_crypto.randomUUID)(),
|
|
2642
|
-
traceId: event.traceId,
|
|
2643
|
-
spanId: event.spanId,
|
|
2644
|
-
name: event.name,
|
|
2645
|
-
timestamp: event.timestamp.toISOString(),
|
|
2646
|
-
attributes: JSON.stringify(event.attributes),
|
|
2647
|
-
createdAt: now
|
|
2648
|
-
}));
|
|
2649
|
-
await db.insertInto("span_events").values(values).execute();
|
|
2265
|
+
await db.insertInto("llm_requests").values(values).execute();
|
|
2650
2266
|
return { count: values.length };
|
|
2651
2267
|
},
|
|
2652
|
-
|
|
2653
|
-
const result = await
|
|
2268
|
+
insertRequest: async (request) => {
|
|
2269
|
+
const result = await insertLLMRequestSchema.safeParseAsync(request);
|
|
2270
|
+
if (!result.success) throw new LLMOpsError(`Invalid request data: ${result.error.message}`);
|
|
2271
|
+
const req = result.data;
|
|
2272
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
2273
|
+
return db.insertInto("llm_requests").values({
|
|
2274
|
+
id: (0, node_crypto.randomUUID)(),
|
|
2275
|
+
requestId: req.requestId,
|
|
2276
|
+
configId: req.configId ?? null,
|
|
2277
|
+
variantId: req.variantId ?? null,
|
|
2278
|
+
environmentId: req.environmentId ?? null,
|
|
2279
|
+
providerConfigId: req.providerConfigId ?? null,
|
|
2280
|
+
provider: req.provider,
|
|
2281
|
+
model: req.model,
|
|
2282
|
+
promptTokens: req.promptTokens,
|
|
2283
|
+
completionTokens: req.completionTokens,
|
|
2284
|
+
totalTokens: req.totalTokens,
|
|
2285
|
+
cachedTokens: req.cachedTokens,
|
|
2286
|
+
cacheCreationTokens: req.cacheCreationTokens,
|
|
2287
|
+
cost: req.cost,
|
|
2288
|
+
cacheSavings: req.cacheSavings,
|
|
2289
|
+
inputCost: req.inputCost,
|
|
2290
|
+
outputCost: req.outputCost,
|
|
2291
|
+
endpoint: req.endpoint,
|
|
2292
|
+
statusCode: req.statusCode,
|
|
2293
|
+
latencyMs: req.latencyMs,
|
|
2294
|
+
isStreaming: req.isStreaming,
|
|
2295
|
+
userId: req.userId ?? null,
|
|
2296
|
+
tags: JSON.stringify(req.tags),
|
|
2297
|
+
guardrailResults: req.guardrailResults ? JSON.stringify(req.guardrailResults) : null,
|
|
2298
|
+
traceId: req.traceId ?? null,
|
|
2299
|
+
spanId: req.spanId ?? null,
|
|
2300
|
+
parentSpanId: req.parentSpanId ?? null,
|
|
2301
|
+
sessionId: req.sessionId ?? null,
|
|
2302
|
+
createdAt: now,
|
|
2303
|
+
updatedAt: now
|
|
2304
|
+
}).returningAll().executeTakeFirst();
|
|
2305
|
+
},
|
|
2306
|
+
listRequests: async (params) => {
|
|
2307
|
+
const result = await listRequestsSchema.safeParseAsync(params || {});
|
|
2654
2308
|
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2655
|
-
const { limit, offset,
|
|
2656
|
-
let baseQuery = db.selectFrom("
|
|
2657
|
-
if (
|
|
2658
|
-
if (
|
|
2659
|
-
if (
|
|
2660
|
-
if (
|
|
2661
|
-
if (
|
|
2662
|
-
if (
|
|
2309
|
+
const { limit, offset, configId, variantId, environmentId, providerConfigId, provider, model, startDate, endDate, tags } = result.data;
|
|
2310
|
+
let baseQuery = db.selectFrom("llm_requests");
|
|
2311
|
+
if (configId) baseQuery = baseQuery.where("configId", "=", configId);
|
|
2312
|
+
if (variantId) baseQuery = baseQuery.where("variantId", "=", variantId);
|
|
2313
|
+
if (environmentId) baseQuery = baseQuery.where("environmentId", "=", environmentId);
|
|
2314
|
+
if (providerConfigId) baseQuery = baseQuery.where("providerConfigId", "=", providerConfigId);
|
|
2315
|
+
if (provider) baseQuery = baseQuery.where("provider", "=", provider);
|
|
2316
|
+
if (model) baseQuery = baseQuery.where("model", "=", model);
|
|
2317
|
+
if (startDate) baseQuery = baseQuery.where(kysely.sql`${col("createdAt")} >= ${startDate.toISOString()}`);
|
|
2318
|
+
if (endDate) baseQuery = baseQuery.where(kysely.sql`${col("createdAt")} <= ${endDate.toISOString()}`);
|
|
2663
2319
|
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
2664
2320
|
if (values.length === 0) continue;
|
|
2665
2321
|
if (values.length === 1) baseQuery = baseQuery.where(kysely.sql`${col("tags")}->>${key} = ${values[0]}`);
|
|
@@ -2671,426 +2327,387 @@ const createTracesDataLayer = (db) => {
|
|
|
2671
2327
|
const countResult = await baseQuery.select(kysely.sql`COUNT(*)`.as("total")).executeTakeFirst();
|
|
2672
2328
|
const total = Number(countResult?.total ?? 0);
|
|
2673
2329
|
return {
|
|
2674
|
-
data: await baseQuery.selectAll().orderBy("
|
|
2330
|
+
data: await baseQuery.selectAll().orderBy("createdAt", "desc").limit(limit).offset(offset).execute(),
|
|
2675
2331
|
total,
|
|
2676
2332
|
limit,
|
|
2677
2333
|
offset
|
|
2678
2334
|
};
|
|
2679
2335
|
},
|
|
2680
|
-
|
|
2681
|
-
|
|
2682
|
-
if (!trace) return void 0;
|
|
2683
|
-
return {
|
|
2684
|
-
trace,
|
|
2685
|
-
spans: await db.selectFrom("spans").selectAll().where("traceId", "=", traceId).orderBy("startTime", "asc").execute(),
|
|
2686
|
-
events: await db.selectFrom("span_events").selectAll().where("traceId", "=", traceId).orderBy("timestamp", "asc").execute()
|
|
2687
|
-
};
|
|
2336
|
+
getRequestByRequestId: async (requestId) => {
|
|
2337
|
+
return db.selectFrom("llm_requests").selectAll().where("requestId", "=", requestId).executeTakeFirst();
|
|
2688
2338
|
},
|
|
2689
|
-
|
|
2690
|
-
const result = await
|
|
2339
|
+
getTotalCost: async (params) => {
|
|
2340
|
+
const result = await dateRangeSchema.safeParseAsync(params);
|
|
2691
2341
|
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2692
|
-
const { startDate, endDate,
|
|
2693
|
-
let query = db.selectFrom("
|
|
2694
|
-
kysely.sql`
|
|
2695
|
-
kysely.sql`COALESCE(
|
|
2696
|
-
kysely.sql`
|
|
2697
|
-
kysely.sql`COALESCE(SUM(${col("
|
|
2342
|
+
const { startDate, endDate, configId, variantId, environmentId, tags } = result.data;
|
|
2343
|
+
let query = db.selectFrom("llm_requests").select([
|
|
2344
|
+
kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2345
|
+
kysely.sql`COALESCE(SUM(${col("inputCost")}), 0)`.as("totalInputCost"),
|
|
2346
|
+
kysely.sql`COALESCE(SUM(${col("outputCost")}), 0)`.as("totalOutputCost"),
|
|
2347
|
+
kysely.sql`COALESCE(SUM(${col("promptTokens")}), 0)`.as("totalPromptTokens"),
|
|
2348
|
+
kysely.sql`COALESCE(SUM(${col("completionTokens")}), 0)`.as("totalCompletionTokens"),
|
|
2698
2349
|
kysely.sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
|
|
2699
|
-
kysely.sql`COALESCE(SUM(${col("
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2350
|
+
kysely.sql`COALESCE(SUM(${col("cachedTokens")}), 0)`.as("totalCachedTokens"),
|
|
2351
|
+
kysely.sql`COALESCE(SUM(${col("cacheSavings")}), 0)`.as("totalCacheSavings"),
|
|
2352
|
+
kysely.sql`COUNT(*)`.as("requestCount")
|
|
2353
|
+
]).where(kysely.sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col("createdAt")} <= ${endDate.toISOString()}`);
|
|
2354
|
+
if (configId) query = query.where("configId", "=", configId);
|
|
2355
|
+
if (variantId) query = query.where("variantId", "=", variantId);
|
|
2356
|
+
if (environmentId) query = query.where("environmentId", "=", environmentId);
|
|
2357
|
+
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
2358
|
+
if (values.length === 0) continue;
|
|
2359
|
+
if (values.length === 1) query = query.where(kysely.sql`${col("tags")}->>${key} = ${values[0]}`);
|
|
2360
|
+
else {
|
|
2361
|
+
const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
|
|
2362
|
+
query = query.where(kysely.sql`${col("tags")}->>${key} IN (${valueList})`);
|
|
2363
|
+
}
|
|
2364
|
+
}
|
|
2703
2365
|
return query.executeTakeFirst();
|
|
2704
|
-
}
|
|
2705
|
-
};
|
|
2706
|
-
};
|
|
2707
|
-
|
|
2708
|
-
//#endregion
|
|
2709
|
-
//#region src/datalayer/workspaceSettings.ts
|
|
2710
|
-
const updateWorkspaceSettings = require_db.zod_default.object({
|
|
2711
|
-
name: require_db.zod_default.string().nullable().optional(),
|
|
2712
|
-
setupComplete: require_db.zod_default.boolean().optional(),
|
|
2713
|
-
superAdminId: require_db.zod_default.string().nullable().optional()
|
|
2714
|
-
});
|
|
2715
|
-
const createWorkspaceSettingsDataLayer = (db) => {
|
|
2716
|
-
return {
|
|
2717
|
-
getWorkspaceSettings: async () => {
|
|
2718
|
-
let settings = await db.selectFrom("workspace_settings").selectAll().executeTakeFirst();
|
|
2719
|
-
if (!settings) settings = await db.insertInto("workspace_settings").values({
|
|
2720
|
-
id: (0, node_crypto.randomUUID)(),
|
|
2721
|
-
name: null,
|
|
2722
|
-
setupComplete: false,
|
|
2723
|
-
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2724
|
-
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2725
|
-
}).returningAll().executeTakeFirst();
|
|
2726
|
-
return settings;
|
|
2727
2366
|
},
|
|
2728
|
-
|
|
2729
|
-
const
|
|
2730
|
-
if (!
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
2735
|
-
|
|
2736
|
-
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
|
|
2740
|
-
|
|
2741
|
-
|
|
2742
|
-
if (value.data.superAdminId !== void 0) updateData.superAdminId = value.data.superAdminId ?? null;
|
|
2743
|
-
return db.updateTable("workspace_settings").set(updateData).where("id", "=", settings.id).returningAll().executeTakeFirst();
|
|
2367
|
+
getCostByModel: async (params) => {
|
|
2368
|
+
const result = await dateRangeSchema.safeParseAsync(params);
|
|
2369
|
+
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2370
|
+
const { startDate, endDate } = result.data;
|
|
2371
|
+
return db.selectFrom("llm_requests").select([
|
|
2372
|
+
"provider",
|
|
2373
|
+
"model",
|
|
2374
|
+
kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2375
|
+
kysely.sql`COALESCE(SUM(${col("inputCost")}), 0)`.as("totalInputCost"),
|
|
2376
|
+
kysely.sql`COALESCE(SUM(${col("outputCost")}), 0)`.as("totalOutputCost"),
|
|
2377
|
+
kysely.sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
|
|
2378
|
+
kysely.sql`COUNT(*)`.as("requestCount"),
|
|
2379
|
+
kysely.sql`AVG(${col("latencyMs")})`.as("avgLatencyMs")
|
|
2380
|
+
]).where(kysely.sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col("createdAt")} <= ${endDate.toISOString()}`).groupBy(["provider", "model"]).orderBy(kysely.sql`SUM(${col("cost")})`, "desc").execute();
|
|
2744
2381
|
},
|
|
2745
|
-
|
|
2746
|
-
|
|
2382
|
+
getCostByProvider: async (params) => {
|
|
2383
|
+
const result = await dateRangeSchema.safeParseAsync(params);
|
|
2384
|
+
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2385
|
+
const { startDate, endDate } = result.data;
|
|
2386
|
+
return db.selectFrom("llm_requests").select([
|
|
2387
|
+
"provider",
|
|
2388
|
+
kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2389
|
+
kysely.sql`COALESCE(SUM(${col("inputCost")}), 0)`.as("totalInputCost"),
|
|
2390
|
+
kysely.sql`COALESCE(SUM(${col("outputCost")}), 0)`.as("totalOutputCost"),
|
|
2391
|
+
kysely.sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
|
|
2392
|
+
kysely.sql`COUNT(*)`.as("requestCount"),
|
|
2393
|
+
kysely.sql`AVG(${col("latencyMs")})`.as("avgLatencyMs")
|
|
2394
|
+
]).where(kysely.sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col("createdAt")} <= ${endDate.toISOString()}`).groupBy("provider").orderBy(kysely.sql`SUM(${col("cost")})`, "desc").execute();
|
|
2747
2395
|
},
|
|
2748
|
-
|
|
2749
|
-
|
|
2750
|
-
if (
|
|
2751
|
-
|
|
2752
|
-
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
|
|
2756
|
-
|
|
2757
|
-
|
|
2758
|
-
|
|
2759
|
-
|
|
2760
|
-
return true;
|
|
2761
|
-
}
|
|
2762
|
-
await db.updateTable("workspace_settings").set({
|
|
2763
|
-
superAdminId: userId,
|
|
2764
|
-
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2765
|
-
}).where("id", "=", settings.id).execute();
|
|
2766
|
-
return true;
|
|
2396
|
+
getDailyCosts: async (params) => {
|
|
2397
|
+
const result = await dateRangeSchema.safeParseAsync(params);
|
|
2398
|
+
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2399
|
+
const { startDate, endDate } = result.data;
|
|
2400
|
+
return db.selectFrom("llm_requests").select([
|
|
2401
|
+
kysely.sql`DATE(${col("createdAt")})`.as("date"),
|
|
2402
|
+
kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2403
|
+
kysely.sql`COALESCE(SUM(${col("inputCost")}), 0)`.as("totalInputCost"),
|
|
2404
|
+
kysely.sql`COALESCE(SUM(${col("outputCost")}), 0)`.as("totalOutputCost"),
|
|
2405
|
+
kysely.sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
|
|
2406
|
+
kysely.sql`COUNT(*)`.as("requestCount")
|
|
2407
|
+
]).where(kysely.sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col("createdAt")} <= ${endDate.toISOString()}`).groupBy(kysely.sql`DATE(${col("createdAt")})`).orderBy(kysely.sql`DATE(${col("createdAt")})`, "asc").execute();
|
|
2767
2408
|
},
|
|
2768
|
-
|
|
2769
|
-
|
|
2770
|
-
|
|
2771
|
-
}
|
|
2772
|
-
|
|
2409
|
+
getCostSummary: async (params) => {
|
|
2410
|
+
const result = await costSummarySchema.safeParseAsync(params);
|
|
2411
|
+
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2412
|
+
const { startDate, endDate, groupBy, configId, variantId, environmentId, tags, tagKeys } = result.data;
|
|
2413
|
+
let baseQuery = db.selectFrom("llm_requests").where(kysely.sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col("createdAt")} <= ${endDate.toISOString()}`);
|
|
2414
|
+
if (configId) baseQuery = baseQuery.where("configId", "=", configId);
|
|
2415
|
+
if (variantId) baseQuery = baseQuery.where("variantId", "=", variantId);
|
|
2416
|
+
if (environmentId) baseQuery = baseQuery.where("environmentId", "=", environmentId);
|
|
2417
|
+
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
2418
|
+
if (values.length === 0) continue;
|
|
2419
|
+
if (values.length === 1) baseQuery = baseQuery.where(kysely.sql`${col("tags")}->>${key} = ${values[0]}`);
|
|
2420
|
+
else {
|
|
2421
|
+
const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
|
|
2422
|
+
baseQuery = baseQuery.where(kysely.sql`${col("tags")}->>${key} IN (${valueList})`);
|
|
2423
|
+
}
|
|
2424
|
+
}
|
|
2425
|
+
switch (groupBy) {
|
|
2426
|
+
case "day": return baseQuery.select([
|
|
2427
|
+
kysely.sql`DATE(${col("createdAt")})`.as("groupKey"),
|
|
2428
|
+
kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2429
|
+
kysely.sql`COUNT(*)`.as("requestCount"),
|
|
2430
|
+
kysely.sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens")
|
|
2431
|
+
]).groupBy(kysely.sql`DATE(${col("createdAt")})`).orderBy(kysely.sql`DATE(${col("createdAt")})`, "asc").execute();
|
|
2432
|
+
case "hour": return baseQuery.select([
|
|
2433
|
+
kysely.sql`DATE_TRUNC('hour', ${col("createdAt")})`.as("groupKey"),
|
|
2434
|
+
kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2435
|
+
kysely.sql`COUNT(*)`.as("requestCount"),
|
|
2436
|
+
kysely.sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens")
|
|
2437
|
+
]).groupBy(kysely.sql`DATE_TRUNC('hour', ${col("createdAt")})`).orderBy(kysely.sql`DATE_TRUNC('hour', ${col("createdAt")})`, "asc").execute();
|
|
2438
|
+
case "model": return baseQuery.select([
|
|
2439
|
+
kysely.sql`${col("provider")} || '/' || ${col("model")}`.as("groupKey"),
|
|
2440
|
+
kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2441
|
+
kysely.sql`COUNT(*)`.as("requestCount")
|
|
2442
|
+
]).groupBy(["provider", "model"]).orderBy(kysely.sql`SUM(${col("cost")})`, "desc").execute();
|
|
2443
|
+
case "provider": return baseQuery.select([
|
|
2444
|
+
kysely.sql`${col("provider")}`.as("groupKey"),
|
|
2445
|
+
kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2446
|
+
kysely.sql`COUNT(*)`.as("requestCount")
|
|
2447
|
+
]).groupBy("provider").orderBy(kysely.sql`SUM(${col("cost")})`, "desc").execute();
|
|
2448
|
+
case "endpoint": return baseQuery.select([
|
|
2449
|
+
kysely.sql`COALESCE(${col("endpoint")}, 'unknown')`.as("groupKey"),
|
|
2450
|
+
kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2451
|
+
kysely.sql`COUNT(*)`.as("requestCount")
|
|
2452
|
+
]).groupBy("endpoint").orderBy(kysely.sql`SUM(${col("cost")})`, "desc").execute();
|
|
2453
|
+
case "tags": {
|
|
2454
|
+
const conditions = [kysely.sql`${col("createdAt")} >= ${startDate.toISOString()}`, kysely.sql`${col("createdAt")} <= ${endDate.toISOString()}`];
|
|
2455
|
+
if (configId) conditions.push(kysely.sql`${col("configId")} = ${configId}`);
|
|
2456
|
+
if (variantId) conditions.push(kysely.sql`${col("variantId")} = ${variantId}`);
|
|
2457
|
+
if (environmentId) conditions.push(kysely.sql`${col("environmentId")} = ${environmentId}`);
|
|
2458
|
+
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
2459
|
+
if (values.length === 0) continue;
|
|
2460
|
+
if (values.length === 1) conditions.push(kysely.sql`${col("tags")}->>${key} = ${values[0]}`);
|
|
2461
|
+
else {
|
|
2462
|
+
const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
|
|
2463
|
+
conditions.push(kysely.sql`${col("tags")}->>${key} IN (${valueList})`);
|
|
2464
|
+
}
|
|
2465
|
+
}
|
|
2466
|
+
if (tagKeys && tagKeys.length > 0) {
|
|
2467
|
+
const tagKeyList = kysely.sql.join(tagKeys.map((k) => kysely.sql`${k}`), kysely.sql`, `);
|
|
2468
|
+
conditions.push(kysely.sql`t.key IN (${tagKeyList})`);
|
|
2469
|
+
}
|
|
2470
|
+
const whereClause = kysely.sql.join(conditions, kysely.sql` AND `);
|
|
2471
|
+
return (await kysely.sql`
|
|
2472
|
+
SELECT t.key || ':' || t.value as "groupKey",
|
|
2473
|
+
COALESCE(SUM(${col("cost")}), 0) as "totalCost",
|
|
2474
|
+
COUNT(*) as "requestCount"
|
|
2475
|
+
FROM "llm_requests", jsonb_each_text(${col("tags")}) t
|
|
2476
|
+
WHERE ${whereClause}
|
|
2477
|
+
GROUP BY t.key, t.value
|
|
2478
|
+
ORDER BY SUM(${col("cost")}) DESC
|
|
2479
|
+
`.execute(db)).rows;
|
|
2480
|
+
}
|
|
2481
|
+
default: return baseQuery.select([
|
|
2482
|
+
kysely.sql`'total'`.as("groupKey"),
|
|
2483
|
+
kysely.sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2484
|
+
kysely.sql`COUNT(*)`.as("requestCount")
|
|
2485
|
+
]).execute();
|
|
2773
2486
|
}
|
|
2774
2487
|
},
|
|
2775
|
-
|
|
2776
|
-
|
|
2777
|
-
if (!
|
|
2778
|
-
|
|
2779
|
-
|
|
2780
|
-
|
|
2781
|
-
|
|
2782
|
-
|
|
2783
|
-
|
|
2784
|
-
|
|
2785
|
-
|
|
2786
|
-
|
|
2787
|
-
|
|
2488
|
+
getRequestStats: async (params) => {
|
|
2489
|
+
const result = await dateRangeSchema.safeParseAsync(params);
|
|
2490
|
+
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2491
|
+
const { startDate, endDate, configId, variantId, environmentId, tags } = result.data;
|
|
2492
|
+
let query = db.selectFrom("llm_requests").select([
|
|
2493
|
+
kysely.sql`COUNT(*)`.as("totalRequests"),
|
|
2494
|
+
kysely.sql`COUNT(CASE WHEN ${col("statusCode")} >= 200 AND ${col("statusCode")} < 300 THEN 1 END)`.as("successfulRequests"),
|
|
2495
|
+
kysely.sql`COUNT(CASE WHEN ${col("statusCode")} >= 400 THEN 1 END)`.as("failedRequests"),
|
|
2496
|
+
kysely.sql`COUNT(CASE WHEN ${col("isStreaming")} = true THEN 1 END)`.as("streamingRequests"),
|
|
2497
|
+
kysely.sql`AVG(${col("latencyMs")})`.as("avgLatencyMs"),
|
|
2498
|
+
kysely.sql`MAX(${col("latencyMs")})`.as("maxLatencyMs"),
|
|
2499
|
+
kysely.sql`MIN(${col("latencyMs")})`.as("minLatencyMs")
|
|
2500
|
+
]).where(kysely.sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(kysely.sql`${col("createdAt")} <= ${endDate.toISOString()}`);
|
|
2501
|
+
if (configId) query = query.where("configId", "=", configId);
|
|
2502
|
+
if (variantId) query = query.where("variantId", "=", variantId);
|
|
2503
|
+
if (environmentId) query = query.where("environmentId", "=", environmentId);
|
|
2504
|
+
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
2505
|
+
if (values.length === 0) continue;
|
|
2506
|
+
if (values.length === 1) query = query.where(kysely.sql`${col("tags")}->>${key} = ${values[0]}`);
|
|
2507
|
+
else {
|
|
2508
|
+
const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
|
|
2509
|
+
query = query.where(kysely.sql`${col("tags")}->>${key} IN (${valueList})`);
|
|
2510
|
+
}
|
|
2511
|
+
}
|
|
2512
|
+
return query.executeTakeFirst();
|
|
2513
|
+
},
|
|
2514
|
+
getDistinctTags: async () => {
|
|
2515
|
+
return (await kysely.sql`
|
|
2516
|
+
SELECT DISTINCT key, value
|
|
2517
|
+
FROM llm_requests, jsonb_each_text(tags) AS t(key, value)
|
|
2518
|
+
WHERE tags != '{}'::jsonb
|
|
2519
|
+
ORDER BY key, value
|
|
2520
|
+
`.execute(db)).rows;
|
|
2788
2521
|
}
|
|
2789
2522
|
};
|
|
2790
|
-
};
|
|
2791
|
-
|
|
2792
|
-
//#endregion
|
|
2793
|
-
//#region src/datalayer/create.ts
|
|
2794
|
-
/**
|
|
2795
|
-
* Create all datalayers from a Kysely database instance.
|
|
2796
|
-
* Returns a flat object with all datalayer methods spread together.
|
|
2797
|
-
*/
|
|
2798
|
-
function createDataLayer(db) {
|
|
2799
|
-
return {
|
|
2800
|
-
...createDatasetsDataLayer(db),
|
|
2801
|
-
...createGuardrailConfigsDataLayer(db),
|
|
2802
|
-
...createLLMRequestsDataLayer(db),
|
|
2803
|
-
...createPlaygroundDataLayer(db),
|
|
2804
|
-
...createPlaygroundResultsDataLayer(db),
|
|
2805
|
-
...createPlaygroundRunsDataLayer(db),
|
|
2806
|
-
...createProviderConfigsDataLayer(db),
|
|
2807
|
-
...createProviderGuardrailOverridesDataLayer(db),
|
|
2808
|
-
...createTracesDataLayer(db),
|
|
2809
|
-
...createWorkspaceSettingsDataLayer(db)
|
|
2810
|
-
};
|
|
2811
|
-
}
|
|
2812
|
-
|
|
2813
|
-
//#endregion
|
|
2814
|
-
//#region src/pricing/calculator.ts
|
|
2815
|
-
/**
|
|
2816
|
-
* Calculate the cost of an LLM request in micro-dollars
|
|
2817
|
-
*
|
|
2818
|
-
* Micro-dollars are used to avoid floating-point precision issues:
|
|
2819
|
-
* - 1 dollar = 1,000,000 micro-dollars
|
|
2820
|
-
* - $0.001 = 1,000 micro-dollars
|
|
2821
|
-
* - $0.000001 = 1 micro-dollar
|
|
2822
|
-
*
|
|
2823
|
-
* @param usage - Token usage data from the LLM response
|
|
2824
|
-
* @param pricing - Model pricing information
|
|
2825
|
-
* @returns Cost breakdown in micro-dollars
|
|
2826
|
-
*
|
|
2827
|
-
* @example
|
|
2828
|
-
* ```typescript
|
|
2829
|
-
* const usage = { promptTokens: 1000, completionTokens: 500 };
|
|
2830
|
-
* const pricing = { inputCostPer1M: 2.5, outputCostPer1M: 10.0 };
|
|
2831
|
-
* const cost = calculateCost(usage, pricing);
|
|
2832
|
-
* // cost = { inputCost: 2500, outputCost: 5000, totalCost: 7500 }
|
|
2833
|
-
* // In dollars: $0.0025 input + $0.005 output = $0.0075 total
|
|
2834
|
-
* ```
|
|
2835
|
-
*/
|
|
2836
|
-
function calculateCost(usage, pricing) {
|
|
2837
|
-
const inputCost = Math.round(usage.promptTokens * pricing.inputCostPer1M);
|
|
2838
|
-
const outputCost = Math.round(usage.completionTokens * pricing.outputCostPer1M);
|
|
2839
|
-
return {
|
|
2840
|
-
inputCost,
|
|
2841
|
-
outputCost,
|
|
2842
|
-
totalCost: inputCost + outputCost,
|
|
2843
|
-
cacheSavings: 0
|
|
2844
|
-
};
|
|
2845
|
-
}
|
|
2846
|
-
/**
|
|
2847
|
-
* Get default cache read rate as a fraction of input cost per provider.
|
|
2848
|
-
* Used when models.dev doesn't provide cache pricing.
|
|
2849
|
-
*/
|
|
2850
|
-
function getDefaultCacheReadRate(provider, inputCostPer1M) {
|
|
2851
|
-
switch (provider?.toLowerCase()) {
|
|
2852
|
-
case "anthropic": return inputCostPer1M * .1;
|
|
2853
|
-
case "openai":
|
|
2854
|
-
case "azure-openai": return inputCostPer1M * .5;
|
|
2855
|
-
case "google":
|
|
2856
|
-
case "gemini":
|
|
2857
|
-
case "vertex_ai": return inputCostPer1M * .25;
|
|
2858
|
-
default: return inputCostPer1M * .5;
|
|
2859
|
-
}
|
|
2860
|
-
}
|
|
2861
|
-
/**
|
|
2862
|
-
* Get default cache write/creation rate as a fraction of input cost per provider.
|
|
2863
|
-
* Used when models.dev doesn't provide cache pricing.
|
|
2864
|
-
*/
|
|
2865
|
-
function getDefaultCacheWriteRate(provider, inputCostPer1M) {
|
|
2866
|
-
switch (provider?.toLowerCase()) {
|
|
2867
|
-
case "anthropic": return inputCostPer1M * 1.25;
|
|
2868
|
-
default: return inputCostPer1M;
|
|
2869
|
-
}
|
|
2870
2523
|
}
|
|
2871
|
-
|
|
2872
|
-
* Calculate cache-aware cost of an LLM request in micro-dollars.
|
|
2873
|
-
*
|
|
2874
|
-
* Splits input tokens into uncached, cache-read, and cache-creation buckets,
|
|
2875
|
-
* each priced at different rates. Falls back to provider-specific multipliers
|
|
2876
|
-
* when models.dev doesn't provide cache pricing.
|
|
2877
|
-
*
|
|
2878
|
-
* @param usage - Token usage data (with cachedTokens and cacheCreationTokens)
|
|
2879
|
-
* @param pricing - Model pricing (may include cacheReadCostPer1M / cacheWriteCostPer1M)
|
|
2880
|
-
* @param provider - Provider name for fallback rate selection
|
|
2881
|
-
* @returns Cost breakdown in micro-dollars
|
|
2882
|
-
*/
|
|
2883
|
-
function calculateCacheAwareCost(usage, pricing, provider) {
|
|
2884
|
-
const cachedTokens = usage.cachedTokens ?? 0;
|
|
2885
|
-
const cacheCreationTokens = usage.cacheCreationTokens ?? 0;
|
|
2886
|
-
if (cachedTokens === 0 && cacheCreationTokens === 0) return calculateCost(usage, pricing);
|
|
2887
|
-
const cacheReadRate = pricing.cacheReadCostPer1M ?? getDefaultCacheReadRate(provider, pricing.inputCostPer1M);
|
|
2888
|
-
const cacheWriteRate = pricing.cacheWriteCostPer1M ?? getDefaultCacheWriteRate(provider, pricing.inputCostPer1M);
|
|
2889
|
-
const uncachedInputTokens = Math.max(0, usage.promptTokens - cachedTokens - cacheCreationTokens);
|
|
2890
|
-
const regularInputCost = Math.round(uncachedInputTokens * pricing.inputCostPer1M);
|
|
2891
|
-
const cacheReadCost = Math.round(cachedTokens * cacheReadRate);
|
|
2892
|
-
const cacheWriteCost = Math.round(cacheCreationTokens * cacheWriteRate);
|
|
2893
|
-
const outputCost = Math.round(usage.completionTokens * pricing.outputCostPer1M);
|
|
2894
|
-
const inputCost = regularInputCost + cacheReadCost + cacheWriteCost;
|
|
2524
|
+
function createTracesStore(db) {
|
|
2895
2525
|
return {
|
|
2896
|
-
|
|
2897
|
-
|
|
2898
|
-
|
|
2899
|
-
|
|
2900
|
-
|
|
2901
|
-
|
|
2902
|
-
|
|
2903
|
-
|
|
2904
|
-
|
|
2905
|
-
|
|
2906
|
-
|
|
2907
|
-
|
|
2908
|
-
|
|
2909
|
-
|
|
2910
|
-
|
|
2911
|
-
|
|
2912
|
-
|
|
2913
|
-
|
|
2914
|
-
|
|
2915
|
-
|
|
2916
|
-
|
|
2917
|
-
|
|
2918
|
-
|
|
2919
|
-
|
|
2920
|
-
|
|
2921
|
-
|
|
2922
|
-
|
|
2923
|
-
|
|
2924
|
-
|
|
2925
|
-
|
|
2926
|
-
|
|
2927
|
-
|
|
2928
|
-
|
|
2929
|
-
|
|
2930
|
-
|
|
2931
|
-
|
|
2932
|
-
|
|
2933
|
-
|
|
2934
|
-
|
|
2935
|
-
|
|
2936
|
-
|
|
2937
|
-
*
|
|
2938
|
-
|
|
2939
|
-
|
|
2940
|
-
|
|
2941
|
-
|
|
2942
|
-
|
|
2943
|
-
|
|
2944
|
-
|
|
2945
|
-
|
|
2946
|
-
|
|
2947
|
-
}
|
|
2948
|
-
|
|
2949
|
-
|
|
2950
|
-
|
|
2951
|
-
const
|
|
2952
|
-
|
|
2953
|
-
|
|
2954
|
-
|
|
2955
|
-
|
|
2956
|
-
|
|
2957
|
-
|
|
2958
|
-
function centsPerTokenToCostPer1M(centsPerToken) {
|
|
2959
|
-
return centsPerToken * 1e4;
|
|
2960
|
-
}
|
|
2961
|
-
/**
|
|
2962
|
-
* Pricing provider that fetches per-model data from the LLMOps Models API.
|
|
2963
|
-
*
|
|
2964
|
-
* Features:
|
|
2965
|
-
* - Per-model in-memory cache with configurable TTL (default 5 minutes)
|
|
2966
|
-
* - Deduplicates concurrent fetches for the same model
|
|
2967
|
-
* - Caches null results (404s) to avoid repeated lookups
|
|
2968
|
-
* - Falls back to stale cache on fetch errors
|
|
2969
|
-
*/
|
|
2970
|
-
var LLMOpsPricingProvider = class {
|
|
2971
|
-
cache = /* @__PURE__ */ new Map();
|
|
2972
|
-
pendingFetches = /* @__PURE__ */ new Map();
|
|
2973
|
-
cacheTTL;
|
|
2974
|
-
baseUrl;
|
|
2975
|
-
constructor(options) {
|
|
2976
|
-
this.cacheTTL = options?.cacheTTL ?? 300 * 1e3;
|
|
2977
|
-
this.baseUrl = options?.baseUrl ?? LLMOPS_MODELS_API;
|
|
2978
|
-
}
|
|
2979
|
-
getCacheKey(provider, model) {
|
|
2980
|
-
return `${provider.toLowerCase()}:${model.toLowerCase()}`;
|
|
2981
|
-
}
|
|
2982
|
-
/**
|
|
2983
|
-
* Fetch pricing for a single model from the API
|
|
2984
|
-
*/
|
|
2985
|
-
async fetchModelPricing(provider, model) {
|
|
2986
|
-
const url = `${this.baseUrl}/model-configs/pricing/${encodeURIComponent(provider)}/${model}`;
|
|
2987
|
-
try {
|
|
2988
|
-
require_db.logger.debug(`[Pricing] GET ${url}`);
|
|
2989
|
-
const startTime = Date.now();
|
|
2990
|
-
const response = await fetch(url);
|
|
2991
|
-
const elapsed = Date.now() - startTime;
|
|
2992
|
-
require_db.logger.debug(`[Pricing] GET ${url} -> ${response.status} (${elapsed}ms)`);
|
|
2993
|
-
if (response.status === 404) {
|
|
2994
|
-
require_db.logger.debug(`[Pricing] No pricing found for ${provider}/${model}`);
|
|
2995
|
-
return null;
|
|
2526
|
+
upsertTrace: async (data) => {
|
|
2527
|
+
const result = await upsertTraceSchema.safeParseAsync(data);
|
|
2528
|
+
if (!result.success) throw new LLMOpsError(`Invalid trace data: ${result.error.message}`);
|
|
2529
|
+
const trace = result.data;
|
|
2530
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
2531
|
+
await kysely.sql`
|
|
2532
|
+
INSERT INTO "traces" (
|
|
2533
|
+
"id", "traceId", "name", "sessionId", "userId", "status",
|
|
2534
|
+
"startTime", "endTime", "durationMs", "spanCount",
|
|
2535
|
+
"totalInputTokens", "totalOutputTokens", "totalTokens", "totalCost",
|
|
2536
|
+
"tags", "metadata", "createdAt", "updatedAt"
|
|
2537
|
+
) VALUES (
|
|
2538
|
+
${(0, node_crypto.randomUUID)()}, ${trace.traceId}, ${trace.name ?? null}, ${trace.sessionId ?? null},
|
|
2539
|
+
${trace.userId ?? null}, ${trace.status},
|
|
2540
|
+
${trace.startTime.toISOString()}, ${trace.endTime?.toISOString() ?? null},
|
|
2541
|
+
${trace.durationMs ?? null}, ${trace.spanCount},
|
|
2542
|
+
${trace.totalInputTokens}, ${trace.totalOutputTokens},
|
|
2543
|
+
${trace.totalTokens}, ${trace.totalCost},
|
|
2544
|
+
${JSON.stringify(trace.tags)}::jsonb, ${JSON.stringify(trace.metadata)}::jsonb,
|
|
2545
|
+
${now}, ${now}
|
|
2546
|
+
)
|
|
2547
|
+
ON CONFLICT ("traceId") DO UPDATE SET
|
|
2548
|
+
"name" = COALESCE(EXCLUDED."name", "traces"."name"),
|
|
2549
|
+
"sessionId" = COALESCE(EXCLUDED."sessionId", "traces"."sessionId"),
|
|
2550
|
+
"userId" = COALESCE(EXCLUDED."userId", "traces"."userId"),
|
|
2551
|
+
"status" = CASE
|
|
2552
|
+
WHEN EXCLUDED."status" = 'error' THEN 'error'
|
|
2553
|
+
WHEN EXCLUDED."status" = 'ok' AND "traces"."status" != 'error' THEN 'ok'
|
|
2554
|
+
ELSE "traces"."status"
|
|
2555
|
+
END,
|
|
2556
|
+
"startTime" = LEAST("traces"."startTime", EXCLUDED."startTime"),
|
|
2557
|
+
"endTime" = GREATEST(
|
|
2558
|
+
COALESCE("traces"."endTime", EXCLUDED."endTime"),
|
|
2559
|
+
COALESCE(EXCLUDED."endTime", "traces"."endTime")
|
|
2560
|
+
),
|
|
2561
|
+
"durationMs" = EXTRACT(EPOCH FROM (
|
|
2562
|
+
GREATEST(
|
|
2563
|
+
COALESCE("traces"."endTime", EXCLUDED."endTime"),
|
|
2564
|
+
COALESCE(EXCLUDED."endTime", "traces"."endTime")
|
|
2565
|
+
) -
|
|
2566
|
+
LEAST("traces"."startTime", EXCLUDED."startTime")
|
|
2567
|
+
))::integer * 1000,
|
|
2568
|
+
"spanCount" = "traces"."spanCount" + EXCLUDED."spanCount",
|
|
2569
|
+
"totalInputTokens" = "traces"."totalInputTokens" + EXCLUDED."totalInputTokens",
|
|
2570
|
+
"totalOutputTokens" = "traces"."totalOutputTokens" + EXCLUDED."totalOutputTokens",
|
|
2571
|
+
"totalTokens" = "traces"."totalTokens" + EXCLUDED."totalTokens",
|
|
2572
|
+
"totalCost" = "traces"."totalCost" + EXCLUDED."totalCost",
|
|
2573
|
+
"tags" = "traces"."tags" || EXCLUDED."tags",
|
|
2574
|
+
"metadata" = "traces"."metadata" || EXCLUDED."metadata",
|
|
2575
|
+
"updatedAt" = ${now}
|
|
2576
|
+
`.execute(db);
|
|
2577
|
+
},
|
|
2578
|
+
batchInsertSpans: async (spans) => {
|
|
2579
|
+
if (spans.length === 0) return { count: 0 };
|
|
2580
|
+
const validatedSpans = [];
|
|
2581
|
+
for (const span of spans) {
|
|
2582
|
+
const result = await insertSpanSchema.safeParseAsync(span);
|
|
2583
|
+
if (!result.success) {
|
|
2584
|
+
require_db.logger.warn(`[batchInsertSpans] Skipping invalid span ${span.spanId}: ${result.error.message}`);
|
|
2585
|
+
continue;
|
|
2586
|
+
}
|
|
2587
|
+
validatedSpans.push(result.data);
|
|
2996
2588
|
}
|
|
2997
|
-
if (
|
|
2998
|
-
const
|
|
2999
|
-
|
|
3000
|
-
|
|
3001
|
-
|
|
3002
|
-
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
|
|
3006
|
-
|
|
3007
|
-
|
|
3008
|
-
|
|
3009
|
-
|
|
3010
|
-
|
|
3011
|
-
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
2589
|
+
if (validatedSpans.length === 0) return { count: 0 };
|
|
2590
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
2591
|
+
const values = validatedSpans.map((span) => ({
|
|
2592
|
+
id: (0, node_crypto.randomUUID)(),
|
|
2593
|
+
traceId: span.traceId,
|
|
2594
|
+
spanId: span.spanId,
|
|
2595
|
+
parentSpanId: span.parentSpanId ?? null,
|
|
2596
|
+
name: span.name,
|
|
2597
|
+
kind: span.kind,
|
|
2598
|
+
status: span.status,
|
|
2599
|
+
statusMessage: span.statusMessage ?? null,
|
|
2600
|
+
startTime: span.startTime.toISOString(),
|
|
2601
|
+
endTime: span.endTime?.toISOString() ?? null,
|
|
2602
|
+
durationMs: span.durationMs ?? null,
|
|
2603
|
+
provider: span.provider ?? null,
|
|
2604
|
+
model: span.model ?? null,
|
|
2605
|
+
promptTokens: span.promptTokens,
|
|
2606
|
+
completionTokens: span.completionTokens,
|
|
2607
|
+
totalTokens: span.totalTokens,
|
|
2608
|
+
cost: span.cost,
|
|
2609
|
+
configId: span.configId ?? null,
|
|
2610
|
+
variantId: span.variantId ?? null,
|
|
2611
|
+
environmentId: span.environmentId ?? null,
|
|
2612
|
+
providerConfigId: span.providerConfigId ?? null,
|
|
2613
|
+
requestId: span.requestId ?? null,
|
|
2614
|
+
source: span.source,
|
|
2615
|
+
input: span.input != null ? JSON.stringify(span.input) : null,
|
|
2616
|
+
output: span.output != null ? JSON.stringify(span.output) : null,
|
|
2617
|
+
attributes: JSON.stringify(span.attributes),
|
|
2618
|
+
createdAt: now,
|
|
2619
|
+
updatedAt: now
|
|
2620
|
+
}));
|
|
2621
|
+
await db.insertInto("spans").values(values).onConflict((oc) => oc.column("spanId").doNothing()).execute();
|
|
2622
|
+
return { count: values.length };
|
|
2623
|
+
},
|
|
2624
|
+
batchInsertSpanEvents: async (events) => {
|
|
2625
|
+
if (events.length === 0) return { count: 0 };
|
|
2626
|
+
const validatedEvents = [];
|
|
2627
|
+
for (const event of events) {
|
|
2628
|
+
const result = await insertSpanEventSchema.safeParseAsync(event);
|
|
2629
|
+
if (!result.success) {
|
|
2630
|
+
require_db.logger.warn(`[batchInsertSpanEvents] Skipping invalid event: ${result.error.message}`);
|
|
2631
|
+
continue;
|
|
2632
|
+
}
|
|
2633
|
+
validatedEvents.push(result.data);
|
|
3016
2634
|
}
|
|
3017
|
-
return
|
|
3018
|
-
|
|
3019
|
-
|
|
3020
|
-
|
|
3021
|
-
|
|
3022
|
-
|
|
3023
|
-
|
|
3024
|
-
|
|
3025
|
-
|
|
3026
|
-
|
|
3027
|
-
|
|
3028
|
-
|
|
3029
|
-
|
|
3030
|
-
|
|
3031
|
-
|
|
3032
|
-
|
|
3033
|
-
|
|
3034
|
-
|
|
3035
|
-
|
|
3036
|
-
|
|
3037
|
-
|
|
3038
|
-
|
|
3039
|
-
|
|
3040
|
-
|
|
3041
|
-
|
|
3042
|
-
|
|
3043
|
-
|
|
3044
|
-
|
|
3045
|
-
|
|
3046
|
-
|
|
3047
|
-
|
|
3048
|
-
|
|
3049
|
-
|
|
3050
|
-
|
|
3051
|
-
|
|
3052
|
-
|
|
3053
|
-
|
|
3054
|
-
|
|
2635
|
+
if (validatedEvents.length === 0) return { count: 0 };
|
|
2636
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
2637
|
+
const values = validatedEvents.map((event) => ({
|
|
2638
|
+
id: (0, node_crypto.randomUUID)(),
|
|
2639
|
+
traceId: event.traceId,
|
|
2640
|
+
spanId: event.spanId,
|
|
2641
|
+
name: event.name,
|
|
2642
|
+
timestamp: event.timestamp.toISOString(),
|
|
2643
|
+
attributes: JSON.stringify(event.attributes),
|
|
2644
|
+
createdAt: now
|
|
2645
|
+
}));
|
|
2646
|
+
await db.insertInto("span_events").values(values).execute();
|
|
2647
|
+
return { count: values.length };
|
|
2648
|
+
},
|
|
2649
|
+
listTraces: async (params) => {
|
|
2650
|
+
const result = await listTracesSchema.safeParseAsync(params || {});
|
|
2651
|
+
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2652
|
+
const { limit, offset, sessionId, userId, status, name, startDate, endDate, tags } = result.data;
|
|
2653
|
+
let baseQuery = db.selectFrom("traces");
|
|
2654
|
+
if (sessionId) baseQuery = baseQuery.where("sessionId", "=", sessionId);
|
|
2655
|
+
if (userId) baseQuery = baseQuery.where("userId", "=", userId);
|
|
2656
|
+
if (status) baseQuery = baseQuery.where("status", "=", status);
|
|
2657
|
+
if (name) baseQuery = baseQuery.where(kysely.sql`${col("name")} ILIKE ${"%" + name + "%"}`);
|
|
2658
|
+
if (startDate) baseQuery = baseQuery.where(kysely.sql`${col("startTime")} >= ${startDate.toISOString()}`);
|
|
2659
|
+
if (endDate) baseQuery = baseQuery.where(kysely.sql`${col("startTime")} <= ${endDate.toISOString()}`);
|
|
2660
|
+
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
2661
|
+
if (values.length === 0) continue;
|
|
2662
|
+
if (values.length === 1) baseQuery = baseQuery.where(kysely.sql`${col("tags")}->>${key} = ${values[0]}`);
|
|
2663
|
+
else {
|
|
2664
|
+
const valueList = kysely.sql.join(values.map((v) => kysely.sql`${v}`));
|
|
2665
|
+
baseQuery = baseQuery.where(kysely.sql`${col("tags")}->>${key} IN (${valueList})`);
|
|
2666
|
+
}
|
|
2667
|
+
}
|
|
2668
|
+
const countResult = await baseQuery.select(kysely.sql`COUNT(*)`.as("total")).executeTakeFirst();
|
|
2669
|
+
const total = Number(countResult?.total ?? 0);
|
|
2670
|
+
return {
|
|
2671
|
+
data: await baseQuery.selectAll().orderBy("startTime", "desc").limit(limit).offset(offset).execute(),
|
|
2672
|
+
total,
|
|
2673
|
+
limit,
|
|
2674
|
+
offset
|
|
2675
|
+
};
|
|
2676
|
+
},
|
|
2677
|
+
getTraceWithSpans: async (traceId) => {
|
|
2678
|
+
const trace = await db.selectFrom("traces").selectAll().where("traceId", "=", traceId).executeTakeFirst();
|
|
2679
|
+
if (!trace) return void 0;
|
|
2680
|
+
return {
|
|
2681
|
+
trace,
|
|
2682
|
+
spans: await db.selectFrom("spans").selectAll().where("traceId", "=", traceId).orderBy("startTime", "asc").execute(),
|
|
2683
|
+
events: await db.selectFrom("span_events").selectAll().where("traceId", "=", traceId).orderBy("timestamp", "asc").execute()
|
|
2684
|
+
};
|
|
2685
|
+
},
|
|
2686
|
+
getTraceStats: async (params) => {
|
|
2687
|
+
const result = await traceStatsSchema.safeParseAsync(params);
|
|
2688
|
+
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2689
|
+
const { startDate, endDate, sessionId, userId } = result.data;
|
|
2690
|
+
let query = db.selectFrom("traces").select([
|
|
2691
|
+
kysely.sql`COUNT(*)`.as("totalTraces"),
|
|
2692
|
+
kysely.sql`COALESCE(AVG(${col("durationMs")}), 0)`.as("avgDurationMs"),
|
|
2693
|
+
kysely.sql`COUNT(CASE WHEN ${col("status")} = 'error' THEN 1 END)`.as("errorCount"),
|
|
2694
|
+
kysely.sql`COALESCE(SUM(${col("totalCost")}), 0)`.as("totalCost"),
|
|
2695
|
+
kysely.sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
|
|
2696
|
+
kysely.sql`COALESCE(SUM(${col("spanCount")}), 0)`.as("totalSpans")
|
|
2697
|
+
]).where(kysely.sql`${col("startTime")} >= ${startDate.toISOString()}`).where(kysely.sql`${col("startTime")} <= ${endDate.toISOString()}`);
|
|
2698
|
+
if (sessionId) query = query.where("sessionId", "=", sessionId);
|
|
2699
|
+
if (userId) query = query.where("userId", "=", userId);
|
|
2700
|
+
return query.executeTakeFirst();
|
|
3055
2701
|
}
|
|
3056
|
-
|
|
3057
|
-
}
|
|
3058
|
-
/**
|
|
3059
|
-
* Force refresh the pricing cache (clears all cached entries)
|
|
3060
|
-
*/
|
|
3061
|
-
async refreshCache() {
|
|
3062
|
-
this.cache.clear();
|
|
3063
|
-
}
|
|
3064
|
-
/**
|
|
3065
|
-
* Always ready — no bulk pre-fetch needed
|
|
3066
|
-
*/
|
|
3067
|
-
isReady() {
|
|
3068
|
-
return true;
|
|
3069
|
-
}
|
|
3070
|
-
/**
|
|
3071
|
-
* Get the number of cached models (for debugging)
|
|
3072
|
-
*/
|
|
3073
|
-
getCacheSize() {
|
|
3074
|
-
return this.cache.size;
|
|
3075
|
-
}
|
|
3076
|
-
};
|
|
3077
|
-
let defaultProvider = null;
|
|
3078
|
-
/**
|
|
3079
|
-
* Get the default pricing provider instance
|
|
3080
|
-
*/
|
|
3081
|
-
function getDefaultPricingProvider() {
|
|
3082
|
-
if (!defaultProvider) defaultProvider = new LLMOpsPricingProvider();
|
|
3083
|
-
return defaultProvider;
|
|
2702
|
+
};
|
|
3084
2703
|
}
|
|
3085
|
-
|
|
3086
|
-
//#endregion
|
|
3087
|
-
//#region src/telemetry/postgres.ts
|
|
3088
2704
|
/**
|
|
3089
2705
|
* Create a PostgreSQL-backed telemetry store.
|
|
3090
2706
|
*
|
|
3091
2707
|
* Usage:
|
|
3092
2708
|
* ```ts
|
|
3093
|
-
* import { llmops
|
|
2709
|
+
* import { llmops } from '@llmops/sdk'
|
|
2710
|
+
* import { pgStore } from '@llmops/sdk/store/pg'
|
|
3094
2711
|
*
|
|
3095
2712
|
* const ops = llmops({
|
|
3096
2713
|
* telemetry: pgStore(process.env.DATABASE_URL),
|
|
@@ -3111,140 +2728,14 @@ function createPgStore(connectionString, options) {
|
|
|
3111
2728
|
await connection.executeQuery(kysely.CompiledQuery.raw(`SET search_path TO "${schema}"`));
|
|
3112
2729
|
}
|
|
3113
2730
|
}) });
|
|
3114
|
-
const llmRequests = createLLMRequestsDataLayer(db);
|
|
3115
|
-
const traces = createTracesDataLayer(db);
|
|
3116
2731
|
require_db.logger.debug(`pgStore: initialized with schema "${schema}"`);
|
|
3117
2732
|
return {
|
|
3118
|
-
...
|
|
3119
|
-
...
|
|
2733
|
+
...createLLMRequestsStore(db),
|
|
2734
|
+
...createTracesStore(db),
|
|
3120
2735
|
_db: db
|
|
3121
2736
|
};
|
|
3122
2737
|
}
|
|
3123
2738
|
|
|
3124
|
-
//#endregion
|
|
3125
|
-
//#region src/manifest/builder.ts
|
|
3126
|
-
/**
|
|
3127
|
-
* Builds the gateway manifest from database
|
|
3128
|
-
*/
|
|
3129
|
-
var ManifestBuilder = class {
|
|
3130
|
-
constructor(db) {
|
|
3131
|
-
this.db = db;
|
|
3132
|
-
}
|
|
3133
|
-
/**
|
|
3134
|
-
* Build the manifest from database
|
|
3135
|
-
*/
|
|
3136
|
-
async build() {
|
|
3137
|
-
const [guardrailConfigs, providerGuardrailOverridesData] = await Promise.all([this.db.selectFrom("guardrail_configs").where("enabled", "=", true).selectAll().execute(), this.db.selectFrom("provider_guardrail_overrides").selectAll().execute()]);
|
|
3138
|
-
const beforeRequestGuardrails = [];
|
|
3139
|
-
const afterRequestGuardrails = [];
|
|
3140
|
-
require_db.logger.info(`[ManifestBuilder] Found ${guardrailConfigs.length} enabled guardrail configs`);
|
|
3141
|
-
for (const guardrail of guardrailConfigs) {
|
|
3142
|
-
const parameters = typeof guardrail.parameters === "string" ? JSON.parse(guardrail.parameters) : guardrail.parameters;
|
|
3143
|
-
const manifestGuardrail = {
|
|
3144
|
-
id: guardrail.id,
|
|
3145
|
-
name: guardrail.name,
|
|
3146
|
-
pluginId: guardrail.pluginId,
|
|
3147
|
-
functionId: guardrail.functionId,
|
|
3148
|
-
hookType: guardrail.hookType,
|
|
3149
|
-
parameters: parameters ?? {},
|
|
3150
|
-
priority: guardrail.priority,
|
|
3151
|
-
onFail: guardrail.onFail
|
|
3152
|
-
};
|
|
3153
|
-
if (guardrail.hookType === "beforeRequestHook") beforeRequestGuardrails.push(manifestGuardrail);
|
|
3154
|
-
else afterRequestGuardrails.push(manifestGuardrail);
|
|
3155
|
-
}
|
|
3156
|
-
beforeRequestGuardrails.sort((a, b) => b.priority - a.priority);
|
|
3157
|
-
afterRequestGuardrails.sort((a, b) => b.priority - a.priority);
|
|
3158
|
-
const providerGuardrailOverrides = {};
|
|
3159
|
-
for (const override of providerGuardrailOverridesData) {
|
|
3160
|
-
const parameters = typeof override.parameters === "string" ? JSON.parse(override.parameters) : override.parameters;
|
|
3161
|
-
const manifestOverride = {
|
|
3162
|
-
id: override.id,
|
|
3163
|
-
providerConfigId: override.providerConfigId,
|
|
3164
|
-
guardrailConfigId: override.guardrailConfigId,
|
|
3165
|
-
enabled: override.enabled,
|
|
3166
|
-
parameters: parameters ?? null
|
|
3167
|
-
};
|
|
3168
|
-
if (!providerGuardrailOverrides[override.providerConfigId]) providerGuardrailOverrides[override.providerConfigId] = [];
|
|
3169
|
-
providerGuardrailOverrides[override.providerConfigId].push(manifestOverride);
|
|
3170
|
-
}
|
|
3171
|
-
return {
|
|
3172
|
-
version: Date.now(),
|
|
3173
|
-
builtAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
3174
|
-
guardrails: {
|
|
3175
|
-
beforeRequestHook: beforeRequestGuardrails,
|
|
3176
|
-
afterRequestHook: afterRequestGuardrails
|
|
3177
|
-
},
|
|
3178
|
-
providerGuardrailOverrides
|
|
3179
|
-
};
|
|
3180
|
-
}
|
|
3181
|
-
};
|
|
3182
|
-
|
|
3183
|
-
//#endregion
|
|
3184
|
-
//#region src/manifest/service.ts
|
|
3185
|
-
const MANIFEST_CACHE_KEY = "manifest";
|
|
3186
|
-
const MANIFEST_NAMESPACE = "gateway";
|
|
3187
|
-
const DEFAULT_TTL_MS = 300 * 1e3;
|
|
3188
|
-
const log = require_db.logger.child({ module: "ManifestService" });
|
|
3189
|
-
var ManifestService = class {
|
|
3190
|
-
builder;
|
|
3191
|
-
constructor(cache, db, ttlMs = DEFAULT_TTL_MS) {
|
|
3192
|
-
this.cache = cache;
|
|
3193
|
-
this.ttlMs = ttlMs;
|
|
3194
|
-
this.builder = new ManifestBuilder(db);
|
|
3195
|
-
log.debug({ ttlMs }, "ManifestService initialized");
|
|
3196
|
-
}
|
|
3197
|
-
/**
|
|
3198
|
-
* Get the current manifest, building if necessary
|
|
3199
|
-
*/
|
|
3200
|
-
async getManifest() {
|
|
3201
|
-
log.debug("Getting manifest from cache or building");
|
|
3202
|
-
const manifest = await this.cache.getOrSet(MANIFEST_CACHE_KEY, async () => {
|
|
3203
|
-
log.info("Building new manifest");
|
|
3204
|
-
const built = await this.builder.build();
|
|
3205
|
-
log.info({ version: built.version }, "Manifest built successfully");
|
|
3206
|
-
return built;
|
|
3207
|
-
}, {
|
|
3208
|
-
namespace: MANIFEST_NAMESPACE,
|
|
3209
|
-
ttl: this.ttlMs
|
|
3210
|
-
});
|
|
3211
|
-
log.debug({ version: manifest.version }, "Manifest retrieved");
|
|
3212
|
-
return manifest;
|
|
3213
|
-
}
|
|
3214
|
-
/**
|
|
3215
|
-
* Force invalidate the manifest (called on mutations)
|
|
3216
|
-
*/
|
|
3217
|
-
async invalidate() {
|
|
3218
|
-
log.info("Invalidating manifest cache");
|
|
3219
|
-
await this.cache.delete(MANIFEST_CACHE_KEY, MANIFEST_NAMESPACE);
|
|
3220
|
-
}
|
|
3221
|
-
/**
|
|
3222
|
-
* Invalidate and immediately rebuild (atomic refresh)
|
|
3223
|
-
*/
|
|
3224
|
-
async refresh() {
|
|
3225
|
-
log.info("Refreshing manifest (invalidate + rebuild)");
|
|
3226
|
-
await this.invalidate();
|
|
3227
|
-
return this.getManifest();
|
|
3228
|
-
}
|
|
3229
|
-
/**
|
|
3230
|
-
* Get manifest version without fetching full manifest
|
|
3231
|
-
* Useful for checking if manifest is stale
|
|
3232
|
-
*/
|
|
3233
|
-
async getVersion() {
|
|
3234
|
-
const version = (await this.cache.get(MANIFEST_CACHE_KEY, MANIFEST_NAMESPACE))?.version ?? null;
|
|
3235
|
-
log.debug({ version }, "Got manifest version");
|
|
3236
|
-
return version;
|
|
3237
|
-
}
|
|
3238
|
-
/**
|
|
3239
|
-
* Check if manifest exists in cache
|
|
3240
|
-
*/
|
|
3241
|
-
async hasManifest() {
|
|
3242
|
-
const exists = await this.cache.has(MANIFEST_CACHE_KEY, MANIFEST_NAMESPACE);
|
|
3243
|
-
log.debug({ exists }, "Checked manifest existence");
|
|
3244
|
-
return exists;
|
|
3245
|
-
}
|
|
3246
|
-
};
|
|
3247
|
-
|
|
3248
2739
|
//#endregion
|
|
3249
2740
|
exports.COST_SUMMARY_GROUP_BY = COST_SUMMARY_GROUP_BY;
|
|
3250
2741
|
exports.CacheService = CacheService;
|
|
@@ -3260,8 +2751,6 @@ exports.LLMOPS_TRACE_NAME_HEADER = LLMOPS_TRACE_NAME_HEADER;
|
|
|
3260
2751
|
exports.LLMOPS_USER_ID_HEADER = LLMOPS_USER_ID_HEADER;
|
|
3261
2752
|
exports.LLMOpsPricingProvider = LLMOpsPricingProvider;
|
|
3262
2753
|
exports.MS = MS;
|
|
3263
|
-
exports.ManifestBuilder = ManifestBuilder;
|
|
3264
|
-
exports.ManifestService = ManifestService;
|
|
3265
2754
|
exports.MemoryCacheBackend = MemoryCacheBackend;
|
|
3266
2755
|
exports.SCHEMA_METADATA = require_db.SCHEMA_METADATA;
|
|
3267
2756
|
exports.SupportedProviders = SupportedProviders;
|
|
@@ -3274,16 +2763,11 @@ exports.createDataLayer = createDataLayer;
|
|
|
3274
2763
|
exports.createDatabase = require_db.createDatabase;
|
|
3275
2764
|
exports.createDatabaseFromConnection = require_db.createDatabaseFromConnection;
|
|
3276
2765
|
exports.createDatasetsDataLayer = createDatasetsDataLayer;
|
|
3277
|
-
exports.createGuardrailConfigsDataLayer = createGuardrailConfigsDataLayer;
|
|
3278
|
-
exports.createLLMRequestsDataLayer = createLLMRequestsDataLayer;
|
|
3279
2766
|
exports.createNeonDialect = require_neon_dialect.createNeonDialect;
|
|
3280
2767
|
exports.createPgStore = createPgStore;
|
|
3281
2768
|
exports.createPlaygroundDataLayer = createPlaygroundDataLayer;
|
|
3282
2769
|
exports.createPlaygroundResultsDataLayer = createPlaygroundResultsDataLayer;
|
|
3283
2770
|
exports.createPlaygroundRunsDataLayer = createPlaygroundRunsDataLayer;
|
|
3284
|
-
exports.createProviderConfigsDataLayer = createProviderConfigsDataLayer;
|
|
3285
|
-
exports.createProviderGuardrailOverridesDataLayer = createProviderGuardrailOverridesDataLayer;
|
|
3286
|
-
exports.createTracesDataLayer = createTracesDataLayer;
|
|
3287
2771
|
exports.createWorkspaceSettingsDataLayer = createWorkspaceSettingsDataLayer;
|
|
3288
2772
|
exports.datasetRecordsSchema = require_db.datasetRecordsSchema;
|
|
3289
2773
|
exports.datasetVersionRecordsSchema = require_db.datasetVersionRecordsSchema;
|