@llmops/core 1.0.0-beta.1 → 1.0.0-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{bun-sqlite-dialect-C3HVUfYl.cjs → bun-sqlite-dialect-CVUG5QUU.cjs} +1 -1
- package/dist/db/index.cjs +2 -2
- package/dist/db/index.d.cts +1 -1
- package/dist/db/index.d.mts +1 -1
- package/dist/db/index.mjs +1 -1
- package/dist/{db-CjwRQB4N.mjs → db-CZ8KtpL-.mjs} +3 -132
- package/dist/{db-D3WDjcvd.cjs → db-DZv0NtMm.cjs} +7 -136
- package/dist/{index-BoDvuqku.d.mts → index-BlFAMkmT.d.mts} +18 -416
- package/dist/{index-BmGf4wCW.d.cts → index-DjIHdwhi.d.cts} +18 -416
- package/dist/index.cjs +1132 -1648
- package/dist/index.d.cts +981 -1535
- package/dist/index.d.mts +981 -1535
- package/dist/index.mjs +1132 -1641
- package/dist/{neon-dialect-BSJpZ9YH.cjs → neon-dialect-BOnuygPe.cjs} +1 -1
- package/dist/{neon-dialect-DMClTHvw.cjs → neon-dialect-ByrFa9iy.cjs} +1 -1
- package/dist/{node-sqlite-dialect-BZGXfDHS.cjs → node-sqlite-dialect-fwmW40Ar.cjs} +1 -1
- package/package.json +2 -2
package/dist/index.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { A as spanEventsSchema, B as array, C as playgroundColumnSchema, D as providerConfigsSchema, E as playgroundsSchema, F as variantsSchema, G as record, H as literal, I as workspaceSettingsSchema, J as unknown, K as string, L as zod_default, M as targetingRulesSchema, N as tracesSchema, O as providerGuardrailOverridesSchema, P as variantVersionsSchema, R as _enum, S as llmRequestsSchema, T as playgroundRunsSchema, U as number, V as boolean, W as object, Y as __require, _ as datasetVersionsSchema, a as matchType, b as environmentsSchema, c as parsePartialTableData, d as validateTableData, f as SCHEMA_METADATA, g as datasetVersionRecordsSchema, h as datasetRecordsSchema, i as getMigrations, j as spansSchema, k as schemas, l as parseTableData, m as configsSchema, n as createDatabaseFromConnection, o as runAutoMigrations, p as configVariantsSchema, q as union, r as detectDatabaseType, s as logger, t as createDatabase, u as validatePartialTableData, v as datasetsSchema, w as playgroundResultsSchema, x as guardrailConfigsSchema, y as environmentSecretsSchema, z as any } from "./db-
|
|
1
|
+
import { A as spanEventsSchema, B as array, C as playgroundColumnSchema, D as providerConfigsSchema, E as playgroundsSchema, F as variantsSchema, G as record, H as literal, I as workspaceSettingsSchema, J as unknown, K as string, L as zod_default, M as targetingRulesSchema, N as tracesSchema, O as providerGuardrailOverridesSchema, P as variantVersionsSchema, R as _enum, S as llmRequestsSchema, T as playgroundRunsSchema, U as number, V as boolean, W as object, Y as __require, _ as datasetVersionsSchema, a as matchType, b as environmentsSchema, c as parsePartialTableData, d as validateTableData, f as SCHEMA_METADATA, g as datasetVersionRecordsSchema, h as datasetRecordsSchema, i as getMigrations, j as spansSchema, k as schemas, l as parseTableData, m as configsSchema, n as createDatabaseFromConnection, o as runAutoMigrations, p as configVariantsSchema, q as union, r as detectDatabaseType, s as logger, t as createDatabase, u as validatePartialTableData, v as datasetsSchema, w as playgroundResultsSchema, x as guardrailConfigsSchema, y as environmentSecretsSchema, z as any } from "./db-CZ8KtpL-.mjs";
|
|
2
2
|
import { n as executeWithSchema, t as createNeonDialect } from "./neon-dialect-DySGBYUi.mjs";
|
|
3
3
|
import gateway from "@llmops/gateway";
|
|
4
4
|
import { CompiledQuery, Kysely, PostgresDialect, sql } from "kysely";
|
|
@@ -1385,754 +1385,247 @@ const createDatasetsDataLayer = (db) => {
|
|
|
1385
1385
|
};
|
|
1386
1386
|
|
|
1387
1387
|
//#endregion
|
|
1388
|
-
//#region src/datalayer/
|
|
1389
|
-
const
|
|
1390
|
-
name: zod_default.string()
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
hookType: zod_default.enum(["beforeRequestHook", "afterRequestHook"]),
|
|
1394
|
-
parameters: zod_default.record(zod_default.string(), zod_default.unknown()).optional().default({}),
|
|
1395
|
-
enabled: zod_default.boolean().optional().default(true),
|
|
1396
|
-
priority: zod_default.number().int().optional().default(0),
|
|
1397
|
-
onFail: zod_default.enum(["block", "log"]).optional().default("block")
|
|
1388
|
+
//#region src/datalayer/playgrounds.ts
|
|
1389
|
+
const createNewPlayground = zod_default.object({
|
|
1390
|
+
name: zod_default.string(),
|
|
1391
|
+
datasetId: zod_default.string().uuid().nullable().optional(),
|
|
1392
|
+
columns: zod_default.array(playgroundColumnSchema).nullable().optional()
|
|
1398
1393
|
});
|
|
1399
|
-
const
|
|
1400
|
-
|
|
1401
|
-
name: zod_default.string().
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
enabled: zod_default.boolean().optional(),
|
|
1405
|
-
priority: zod_default.number().int().optional(),
|
|
1406
|
-
onFail: zod_default.enum(["block", "log"]).optional()
|
|
1394
|
+
const updatePlayground = zod_default.object({
|
|
1395
|
+
playgroundId: zod_default.uuidv4(),
|
|
1396
|
+
name: zod_default.string().optional(),
|
|
1397
|
+
datasetId: zod_default.string().uuid().nullable().optional(),
|
|
1398
|
+
columns: zod_default.array(playgroundColumnSchema).nullable().optional()
|
|
1407
1399
|
});
|
|
1408
|
-
const
|
|
1409
|
-
const
|
|
1410
|
-
const
|
|
1400
|
+
const getPlaygroundById = zod_default.object({ playgroundId: zod_default.uuidv4() });
|
|
1401
|
+
const deletePlayground = zod_default.object({ playgroundId: zod_default.uuidv4() });
|
|
1402
|
+
const listPlaygrounds = zod_default.object({
|
|
1411
1403
|
limit: zod_default.number().int().positive().optional(),
|
|
1412
|
-
offset: zod_default.number().int().nonnegative().optional()
|
|
1413
|
-
hookType: zod_default.enum(["beforeRequestHook", "afterRequestHook"]).optional(),
|
|
1414
|
-
enabled: zod_default.boolean().optional()
|
|
1404
|
+
offset: zod_default.number().int().nonnegative().optional()
|
|
1415
1405
|
});
|
|
1416
|
-
const
|
|
1406
|
+
const createPlaygroundDataLayer = (db) => {
|
|
1417
1407
|
return {
|
|
1418
|
-
|
|
1419
|
-
const value = await
|
|
1408
|
+
createNewPlayground: async (params) => {
|
|
1409
|
+
const value = await createNewPlayground.safeParseAsync(params);
|
|
1420
1410
|
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1421
|
-
const { name,
|
|
1422
|
-
return db.insertInto("
|
|
1411
|
+
const { name, datasetId, columns } = value.data;
|
|
1412
|
+
return db.insertInto("playgrounds").values({
|
|
1423
1413
|
id: randomUUID(),
|
|
1424
1414
|
name,
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
hookType,
|
|
1428
|
-
parameters: JSON.stringify(parameters),
|
|
1429
|
-
enabled,
|
|
1430
|
-
priority,
|
|
1431
|
-
onFail,
|
|
1415
|
+
datasetId: datasetId ?? null,
|
|
1416
|
+
columns: columns ? JSON.stringify(columns) : null,
|
|
1432
1417
|
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1433
1418
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1434
1419
|
}).returningAll().executeTakeFirst();
|
|
1435
1420
|
},
|
|
1436
|
-
|
|
1437
|
-
const value = await
|
|
1421
|
+
updatePlayground: async (params) => {
|
|
1422
|
+
const value = await updatePlayground.safeParseAsync(params);
|
|
1438
1423
|
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1439
|
-
const {
|
|
1424
|
+
const { playgroundId, name, datasetId, columns } = value.data;
|
|
1440
1425
|
const updateData = { updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
1441
1426
|
if (name !== void 0) updateData.name = name;
|
|
1442
|
-
if (
|
|
1443
|
-
if (
|
|
1444
|
-
|
|
1445
|
-
if (priority !== void 0) updateData.priority = priority;
|
|
1446
|
-
if (onFail !== void 0) updateData.onFail = onFail;
|
|
1447
|
-
return db.updateTable("guardrail_configs").set(updateData).where("id", "=", id).returningAll().executeTakeFirst();
|
|
1427
|
+
if (datasetId !== void 0) updateData.datasetId = datasetId;
|
|
1428
|
+
if (columns !== void 0) updateData.columns = columns ? JSON.stringify(columns) : null;
|
|
1429
|
+
return db.updateTable("playgrounds").set(updateData).where("id", "=", playgroundId).returningAll().executeTakeFirst();
|
|
1448
1430
|
},
|
|
1449
|
-
|
|
1450
|
-
const value = await
|
|
1431
|
+
getPlaygroundById: async (params) => {
|
|
1432
|
+
const value = await getPlaygroundById.safeParseAsync(params);
|
|
1451
1433
|
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1452
|
-
const {
|
|
1453
|
-
return db.selectFrom("
|
|
1434
|
+
const { playgroundId } = value.data;
|
|
1435
|
+
return db.selectFrom("playgrounds").selectAll().where("id", "=", playgroundId).executeTakeFirst();
|
|
1454
1436
|
},
|
|
1455
|
-
|
|
1456
|
-
const value = await
|
|
1437
|
+
deletePlayground: async (params) => {
|
|
1438
|
+
const value = await deletePlayground.safeParseAsync(params);
|
|
1457
1439
|
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1458
|
-
const {
|
|
1459
|
-
return db.deleteFrom("
|
|
1440
|
+
const { playgroundId } = value.data;
|
|
1441
|
+
return db.deleteFrom("playgrounds").where("id", "=", playgroundId).returningAll().executeTakeFirst();
|
|
1460
1442
|
},
|
|
1461
|
-
|
|
1462
|
-
const value = await
|
|
1443
|
+
listPlaygrounds: async (params) => {
|
|
1444
|
+
const value = await listPlaygrounds.safeParseAsync(params || {});
|
|
1463
1445
|
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1464
|
-
const { limit = 100, offset = 0
|
|
1465
|
-
|
|
1466
|
-
if (hookType !== void 0) query = query.where("hookType", "=", hookType);
|
|
1467
|
-
if (enabled !== void 0) query = query.where("enabled", "=", enabled);
|
|
1468
|
-
return query.execute();
|
|
1446
|
+
const { limit = 100, offset = 0 } = value.data;
|
|
1447
|
+
return db.selectFrom("playgrounds").selectAll().orderBy("createdAt", "desc").limit(limit).offset(offset).execute();
|
|
1469
1448
|
},
|
|
1470
|
-
|
|
1471
|
-
const result = await db.selectFrom("
|
|
1449
|
+
countPlaygrounds: async () => {
|
|
1450
|
+
const result = await db.selectFrom("playgrounds").select(db.fn.countAll().as("count")).executeTakeFirst();
|
|
1472
1451
|
return Number(result?.count ?? 0);
|
|
1473
|
-
},
|
|
1474
|
-
getEnabledGuardrailsByHookType: async (hookType) => {
|
|
1475
|
-
return db.selectFrom("guardrail_configs").selectAll().where("hookType", "=", hookType).where("enabled", "=", true).orderBy("priority", "desc").execute();
|
|
1476
1452
|
}
|
|
1477
1453
|
};
|
|
1478
1454
|
};
|
|
1479
1455
|
|
|
1480
1456
|
//#endregion
|
|
1481
|
-
//#region src/datalayer/
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
* Schema for guardrail results aggregate
|
|
1494
|
-
*/
|
|
1495
|
-
const guardrailResultsSchema = zod_default.object({
|
|
1496
|
-
results: zod_default.array(guardrailResultSchema),
|
|
1497
|
-
action: zod_default.enum([
|
|
1498
|
-
"allowed",
|
|
1499
|
-
"blocked",
|
|
1500
|
-
"logged"
|
|
1501
|
-
]),
|
|
1502
|
-
totalLatencyMs: zod_default.number()
|
|
1503
|
-
});
|
|
1504
|
-
/**
|
|
1505
|
-
* Schema for inserting a new LLM request log
|
|
1506
|
-
*/
|
|
1507
|
-
const insertLLMRequestSchema = zod_default.object({
|
|
1508
|
-
requestId: zod_default.string().uuid(),
|
|
1509
|
-
configId: zod_default.string().uuid().nullable().optional(),
|
|
1510
|
-
variantId: zod_default.string().uuid().nullable().optional(),
|
|
1511
|
-
environmentId: zod_default.string().uuid().nullable().optional(),
|
|
1512
|
-
providerConfigId: zod_default.string().uuid().nullable().optional(),
|
|
1513
|
-
provider: zod_default.string(),
|
|
1514
|
-
model: zod_default.string(),
|
|
1515
|
-
promptTokens: zod_default.number().int().default(0),
|
|
1516
|
-
completionTokens: zod_default.number().int().default(0),
|
|
1517
|
-
totalTokens: zod_default.number().int().default(0),
|
|
1518
|
-
cachedTokens: zod_default.number().int().default(0),
|
|
1519
|
-
cacheCreationTokens: zod_default.number().int().default(0),
|
|
1520
|
-
cost: zod_default.number().int().default(0),
|
|
1521
|
-
cacheSavings: zod_default.number().int().default(0),
|
|
1522
|
-
inputCost: zod_default.number().int().default(0),
|
|
1523
|
-
outputCost: zod_default.number().int().default(0),
|
|
1524
|
-
endpoint: zod_default.string(),
|
|
1525
|
-
statusCode: zod_default.number().int(),
|
|
1526
|
-
latencyMs: zod_default.number().int().default(0),
|
|
1527
|
-
isStreaming: zod_default.boolean().default(false),
|
|
1528
|
-
userId: zod_default.string().nullable().optional(),
|
|
1529
|
-
tags: zod_default.record(zod_default.string(), zod_default.string()).default({}),
|
|
1530
|
-
guardrailResults: guardrailResultsSchema.nullable().optional(),
|
|
1531
|
-
traceId: zod_default.string().nullable().optional(),
|
|
1532
|
-
spanId: zod_default.string().nullable().optional(),
|
|
1533
|
-
parentSpanId: zod_default.string().nullable().optional(),
|
|
1534
|
-
sessionId: zod_default.string().nullable().optional()
|
|
1535
|
-
});
|
|
1536
|
-
/**
|
|
1537
|
-
* Schema for listing LLM requests
|
|
1538
|
-
*/
|
|
1539
|
-
const listRequestsSchema = zod_default.object({
|
|
1540
|
-
limit: zod_default.number().int().positive().max(1e3).default(100),
|
|
1541
|
-
offset: zod_default.number().int().nonnegative().default(0),
|
|
1542
|
-
configId: zod_default.string().uuid().optional(),
|
|
1543
|
-
variantId: zod_default.string().uuid().optional(),
|
|
1544
|
-
environmentId: zod_default.string().uuid().optional(),
|
|
1545
|
-
providerConfigId: zod_default.string().uuid().optional(),
|
|
1546
|
-
provider: zod_default.string().optional(),
|
|
1547
|
-
model: zod_default.string().optional(),
|
|
1548
|
-
startDate: zod_default.date().optional(),
|
|
1549
|
-
endDate: zod_default.date().optional(),
|
|
1550
|
-
tags: zod_default.record(zod_default.string(), zod_default.array(zod_default.string())).optional()
|
|
1457
|
+
//#region src/datalayer/playgroundResults.ts
|
|
1458
|
+
const createPlaygroundResult = zod_default.object({
|
|
1459
|
+
runId: zod_default.string().uuid(),
|
|
1460
|
+
columnId: zod_default.string().uuid(),
|
|
1461
|
+
datasetRecordId: zod_default.string().uuid().nullable().optional(),
|
|
1462
|
+
inputVariables: zod_default.record(zod_default.string(), zod_default.unknown()).default({}),
|
|
1463
|
+
status: zod_default.enum([
|
|
1464
|
+
"pending",
|
|
1465
|
+
"running",
|
|
1466
|
+
"completed",
|
|
1467
|
+
"failed"
|
|
1468
|
+
]).default("pending")
|
|
1551
1469
|
});
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1470
|
+
const createPlaygroundResultsBatch = zod_default.object({ results: zod_default.array(createPlaygroundResult) });
|
|
1471
|
+
const updatePlaygroundResult = zod_default.object({
|
|
1472
|
+
resultId: zod_default.string().uuid(),
|
|
1473
|
+
outputContent: zod_default.string().nullable().optional(),
|
|
1474
|
+
status: zod_default.enum([
|
|
1475
|
+
"pending",
|
|
1476
|
+
"running",
|
|
1477
|
+
"completed",
|
|
1478
|
+
"failed"
|
|
1479
|
+
]).optional(),
|
|
1480
|
+
error: zod_default.string().nullable().optional(),
|
|
1481
|
+
latencyMs: zod_default.number().int().nullable().optional(),
|
|
1482
|
+
promptTokens: zod_default.number().int().nullable().optional(),
|
|
1483
|
+
completionTokens: zod_default.number().int().nullable().optional(),
|
|
1484
|
+
totalTokens: zod_default.number().int().nullable().optional(),
|
|
1485
|
+
cost: zod_default.number().int().nullable().optional()
|
|
1562
1486
|
});
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
"model",
|
|
1570
|
-
"provider",
|
|
1571
|
-
"endpoint",
|
|
1572
|
-
"tags"
|
|
1573
|
-
];
|
|
1574
|
-
/**
|
|
1575
|
-
* Schema for cost summary with grouping
|
|
1576
|
-
*/
|
|
1577
|
-
const costSummarySchema = zod_default.object({
|
|
1578
|
-
startDate: zod_default.date(),
|
|
1579
|
-
endDate: zod_default.date(),
|
|
1580
|
-
configId: zod_default.string().uuid().optional(),
|
|
1581
|
-
variantId: zod_default.string().uuid().optional(),
|
|
1582
|
-
environmentId: zod_default.string().uuid().optional(),
|
|
1583
|
-
tags: zod_default.record(zod_default.string(), zod_default.array(zod_default.string())).optional(),
|
|
1584
|
-
groupBy: zod_default.enum(COST_SUMMARY_GROUP_BY).optional(),
|
|
1585
|
-
tagKeys: zod_default.array(zod_default.string()).optional()
|
|
1487
|
+
const getPlaygroundResultById = zod_default.object({ resultId: zod_default.string().uuid() });
|
|
1488
|
+
const listPlaygroundResults = zod_default.object({
|
|
1489
|
+
runId: zod_default.string().uuid(),
|
|
1490
|
+
columnId: zod_default.string().uuid().optional(),
|
|
1491
|
+
limit: zod_default.number().int().positive().optional(),
|
|
1492
|
+
offset: zod_default.number().int().nonnegative().optional()
|
|
1586
1493
|
});
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
* Uses sql.ref() to properly quote column names for the database
|
|
1590
|
-
*/
|
|
1591
|
-
const col$1 = (name) => sql.ref(name);
|
|
1592
|
-
const createLLMRequestsDataLayer = (db) => {
|
|
1494
|
+
const deletePlaygroundResultsByRunId = zod_default.object({ runId: zod_default.string().uuid() });
|
|
1495
|
+
const createPlaygroundResultsDataLayer = (db) => {
|
|
1593
1496
|
return {
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1497
|
+
createPlaygroundResult: async (params) => {
|
|
1498
|
+
const value = await createPlaygroundResult.safeParseAsync(params);
|
|
1499
|
+
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1500
|
+
const { runId, columnId, datasetRecordId, inputVariables, status } = value.data;
|
|
1501
|
+
return db.insertInto("playground_results").values({
|
|
1502
|
+
id: randomUUID(),
|
|
1503
|
+
runId,
|
|
1504
|
+
columnId,
|
|
1505
|
+
datasetRecordId: datasetRecordId ?? null,
|
|
1506
|
+
inputVariables: JSON.stringify(inputVariables),
|
|
1507
|
+
outputContent: null,
|
|
1508
|
+
status,
|
|
1509
|
+
error: null,
|
|
1510
|
+
latencyMs: null,
|
|
1511
|
+
promptTokens: null,
|
|
1512
|
+
completionTokens: null,
|
|
1513
|
+
totalTokens: null,
|
|
1514
|
+
cost: null,
|
|
1515
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1516
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1517
|
+
}).returningAll().executeTakeFirst();
|
|
1518
|
+
},
|
|
1519
|
+
createPlaygroundResultsBatch: async (params) => {
|
|
1520
|
+
const value = await createPlaygroundResultsBatch.safeParseAsync(params);
|
|
1521
|
+
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1522
|
+
const { results } = value.data;
|
|
1523
|
+
if (results.length === 0) return [];
|
|
1601
1524
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1602
|
-
const values =
|
|
1525
|
+
const values = results.map((result) => ({
|
|
1603
1526
|
id: randomUUID(),
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
cost: req.cost,
|
|
1617
|
-
cacheSavings: req.cacheSavings,
|
|
1618
|
-
inputCost: req.inputCost,
|
|
1619
|
-
outputCost: req.outputCost,
|
|
1620
|
-
endpoint: req.endpoint,
|
|
1621
|
-
statusCode: req.statusCode,
|
|
1622
|
-
latencyMs: req.latencyMs,
|
|
1623
|
-
isStreaming: req.isStreaming,
|
|
1624
|
-
userId: req.userId ?? null,
|
|
1625
|
-
tags: JSON.stringify(req.tags),
|
|
1626
|
-
guardrailResults: req.guardrailResults ? JSON.stringify(req.guardrailResults) : null,
|
|
1627
|
-
traceId: req.traceId ?? null,
|
|
1628
|
-
spanId: req.spanId ?? null,
|
|
1629
|
-
parentSpanId: req.parentSpanId ?? null,
|
|
1630
|
-
sessionId: req.sessionId ?? null,
|
|
1527
|
+
runId: result.runId,
|
|
1528
|
+
columnId: result.columnId,
|
|
1529
|
+
datasetRecordId: result.datasetRecordId ?? null,
|
|
1530
|
+
inputVariables: JSON.stringify(result.inputVariables),
|
|
1531
|
+
outputContent: null,
|
|
1532
|
+
status: result.status,
|
|
1533
|
+
error: null,
|
|
1534
|
+
latencyMs: null,
|
|
1535
|
+
promptTokens: null,
|
|
1536
|
+
completionTokens: null,
|
|
1537
|
+
totalTokens: null,
|
|
1538
|
+
cost: null,
|
|
1631
1539
|
createdAt: now,
|
|
1632
1540
|
updatedAt: now
|
|
1633
1541
|
}));
|
|
1634
|
-
|
|
1635
|
-
return { count: values.length };
|
|
1542
|
+
return db.insertInto("playground_results").values(values).returningAll().execute();
|
|
1636
1543
|
},
|
|
1637
|
-
|
|
1638
|
-
const
|
|
1639
|
-
if (!
|
|
1640
|
-
const
|
|
1641
|
-
const
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
promptTokens: req.promptTokens,
|
|
1652
|
-
completionTokens: req.completionTokens,
|
|
1653
|
-
totalTokens: req.totalTokens,
|
|
1654
|
-
cachedTokens: req.cachedTokens,
|
|
1655
|
-
cacheCreationTokens: req.cacheCreationTokens,
|
|
1656
|
-
cost: req.cost,
|
|
1657
|
-
cacheSavings: req.cacheSavings,
|
|
1658
|
-
inputCost: req.inputCost,
|
|
1659
|
-
outputCost: req.outputCost,
|
|
1660
|
-
endpoint: req.endpoint,
|
|
1661
|
-
statusCode: req.statusCode,
|
|
1662
|
-
latencyMs: req.latencyMs,
|
|
1663
|
-
isStreaming: req.isStreaming,
|
|
1664
|
-
userId: req.userId ?? null,
|
|
1665
|
-
tags: JSON.stringify(req.tags),
|
|
1666
|
-
guardrailResults: req.guardrailResults ? JSON.stringify(req.guardrailResults) : null,
|
|
1667
|
-
traceId: req.traceId ?? null,
|
|
1668
|
-
spanId: req.spanId ?? null,
|
|
1669
|
-
parentSpanId: req.parentSpanId ?? null,
|
|
1670
|
-
sessionId: req.sessionId ?? null,
|
|
1671
|
-
createdAt: now,
|
|
1672
|
-
updatedAt: now
|
|
1673
|
-
}).returningAll().executeTakeFirst();
|
|
1674
|
-
},
|
|
1675
|
-
listRequests: async (params) => {
|
|
1676
|
-
const result = await listRequestsSchema.safeParseAsync(params || {});
|
|
1677
|
-
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
1678
|
-
const { limit, offset, configId, variantId, environmentId, providerConfigId, provider, model, startDate, endDate, tags } = result.data;
|
|
1679
|
-
let baseQuery = db.selectFrom("llm_requests");
|
|
1680
|
-
if (configId) baseQuery = baseQuery.where("configId", "=", configId);
|
|
1681
|
-
if (variantId) baseQuery = baseQuery.where("variantId", "=", variantId);
|
|
1682
|
-
if (environmentId) baseQuery = baseQuery.where("environmentId", "=", environmentId);
|
|
1683
|
-
if (providerConfigId) baseQuery = baseQuery.where("providerConfigId", "=", providerConfigId);
|
|
1684
|
-
if (provider) baseQuery = baseQuery.where("provider", "=", provider);
|
|
1685
|
-
if (model) baseQuery = baseQuery.where("model", "=", model);
|
|
1686
|
-
if (startDate) baseQuery = baseQuery.where(sql`${col$1("createdAt")} >= ${startDate.toISOString()}`);
|
|
1687
|
-
if (endDate) baseQuery = baseQuery.where(sql`${col$1("createdAt")} <= ${endDate.toISOString()}`);
|
|
1688
|
-
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
1689
|
-
if (values.length === 0) continue;
|
|
1690
|
-
if (values.length === 1) baseQuery = baseQuery.where(sql`${col$1("tags")}->>${key} = ${values[0]}`);
|
|
1691
|
-
else {
|
|
1692
|
-
const valueList = sql.join(values.map((v) => sql`${v}`));
|
|
1693
|
-
baseQuery = baseQuery.where(sql`${col$1("tags")}->>${key} IN (${valueList})`);
|
|
1694
|
-
}
|
|
1695
|
-
}
|
|
1696
|
-
const countResult = await baseQuery.select(sql`COUNT(*)`.as("total")).executeTakeFirst();
|
|
1697
|
-
const total = Number(countResult?.total ?? 0);
|
|
1698
|
-
return {
|
|
1699
|
-
data: await baseQuery.selectAll().orderBy("createdAt", "desc").limit(limit).offset(offset).execute(),
|
|
1700
|
-
total,
|
|
1701
|
-
limit,
|
|
1702
|
-
offset
|
|
1703
|
-
};
|
|
1704
|
-
},
|
|
1705
|
-
getRequestByRequestId: async (requestId) => {
|
|
1706
|
-
return db.selectFrom("llm_requests").selectAll().where("requestId", "=", requestId).executeTakeFirst();
|
|
1707
|
-
},
|
|
1708
|
-
getTotalCost: async (params) => {
|
|
1709
|
-
const result = await dateRangeSchema.safeParseAsync(params);
|
|
1710
|
-
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
1711
|
-
const { startDate, endDate, configId, variantId, environmentId, tags } = result.data;
|
|
1712
|
-
let query = db.selectFrom("llm_requests").select([
|
|
1713
|
-
sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
|
|
1714
|
-
sql`COALESCE(SUM(${col$1("inputCost")}), 0)`.as("totalInputCost"),
|
|
1715
|
-
sql`COALESCE(SUM(${col$1("outputCost")}), 0)`.as("totalOutputCost"),
|
|
1716
|
-
sql`COALESCE(SUM(${col$1("promptTokens")}), 0)`.as("totalPromptTokens"),
|
|
1717
|
-
sql`COALESCE(SUM(${col$1("completionTokens")}), 0)`.as("totalCompletionTokens"),
|
|
1718
|
-
sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens"),
|
|
1719
|
-
sql`COALESCE(SUM(${col$1("cachedTokens")}), 0)`.as("totalCachedTokens"),
|
|
1720
|
-
sql`COALESCE(SUM(${col$1("cacheSavings")}), 0)`.as("totalCacheSavings"),
|
|
1721
|
-
sql`COUNT(*)`.as("requestCount")
|
|
1722
|
-
]).where(sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col$1("createdAt")} <= ${endDate.toISOString()}`);
|
|
1723
|
-
if (configId) query = query.where("configId", "=", configId);
|
|
1724
|
-
if (variantId) query = query.where("variantId", "=", variantId);
|
|
1725
|
-
if (environmentId) query = query.where("environmentId", "=", environmentId);
|
|
1726
|
-
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
1727
|
-
if (values.length === 0) continue;
|
|
1728
|
-
if (values.length === 1) query = query.where(sql`${col$1("tags")}->>${key} = ${values[0]}`);
|
|
1729
|
-
else {
|
|
1730
|
-
const valueList = sql.join(values.map((v) => sql`${v}`));
|
|
1731
|
-
query = query.where(sql`${col$1("tags")}->>${key} IN (${valueList})`);
|
|
1732
|
-
}
|
|
1733
|
-
}
|
|
1734
|
-
return await query.executeTakeFirst();
|
|
1735
|
-
},
|
|
1736
|
-
getCostByModel: async (params) => {
|
|
1737
|
-
const result = await dateRangeSchema.safeParseAsync(params);
|
|
1738
|
-
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
1739
|
-
const { startDate, endDate } = result.data;
|
|
1740
|
-
return db.selectFrom("llm_requests").select([
|
|
1741
|
-
"provider",
|
|
1742
|
-
"model",
|
|
1743
|
-
sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
|
|
1744
|
-
sql`COALESCE(SUM(${col$1("inputCost")}), 0)`.as("totalInputCost"),
|
|
1745
|
-
sql`COALESCE(SUM(${col$1("outputCost")}), 0)`.as("totalOutputCost"),
|
|
1746
|
-
sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens"),
|
|
1747
|
-
sql`COUNT(*)`.as("requestCount"),
|
|
1748
|
-
sql`AVG(${col$1("latencyMs")})`.as("avgLatencyMs")
|
|
1749
|
-
]).where(sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col$1("createdAt")} <= ${endDate.toISOString()}`).groupBy(["provider", "model"]).orderBy(sql`SUM(${col$1("cost")})`, "desc").execute();
|
|
1544
|
+
updatePlaygroundResult: async (params) => {
|
|
1545
|
+
const value = await updatePlaygroundResult.safeParseAsync(params);
|
|
1546
|
+
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1547
|
+
const { resultId, outputContent, status, error, latencyMs, promptTokens, completionTokens, totalTokens, cost } = value.data;
|
|
1548
|
+
const updateData = { updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
1549
|
+
if (outputContent !== void 0) updateData.outputContent = outputContent;
|
|
1550
|
+
if (status !== void 0) updateData.status = status;
|
|
1551
|
+
if (error !== void 0) updateData.error = error;
|
|
1552
|
+
if (latencyMs !== void 0) updateData.latencyMs = latencyMs;
|
|
1553
|
+
if (promptTokens !== void 0) updateData.promptTokens = promptTokens;
|
|
1554
|
+
if (completionTokens !== void 0) updateData.completionTokens = completionTokens;
|
|
1555
|
+
if (totalTokens !== void 0) updateData.totalTokens = totalTokens;
|
|
1556
|
+
if (cost !== void 0) updateData.cost = cost;
|
|
1557
|
+
return db.updateTable("playground_results").set(updateData).where("id", "=", resultId).returningAll().executeTakeFirst();
|
|
1750
1558
|
},
|
|
1751
|
-
|
|
1752
|
-
const
|
|
1753
|
-
if (!
|
|
1754
|
-
const {
|
|
1755
|
-
return db.selectFrom("
|
|
1756
|
-
"provider",
|
|
1757
|
-
sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
|
|
1758
|
-
sql`COALESCE(SUM(${col$1("inputCost")}), 0)`.as("totalInputCost"),
|
|
1759
|
-
sql`COALESCE(SUM(${col$1("outputCost")}), 0)`.as("totalOutputCost"),
|
|
1760
|
-
sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens"),
|
|
1761
|
-
sql`COUNT(*)`.as("requestCount"),
|
|
1762
|
-
sql`AVG(${col$1("latencyMs")})`.as("avgLatencyMs")
|
|
1763
|
-
]).where(sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col$1("createdAt")} <= ${endDate.toISOString()}`).groupBy("provider").orderBy(sql`SUM(${col$1("cost")})`, "desc").execute();
|
|
1559
|
+
getPlaygroundResultById: async (params) => {
|
|
1560
|
+
const value = await getPlaygroundResultById.safeParseAsync(params);
|
|
1561
|
+
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1562
|
+
const { resultId } = value.data;
|
|
1563
|
+
return db.selectFrom("playground_results").selectAll().where("id", "=", resultId).executeTakeFirst();
|
|
1764
1564
|
},
|
|
1765
|
-
|
|
1766
|
-
const
|
|
1767
|
-
if (!
|
|
1768
|
-
const {
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
sql`COALESCE(SUM(${col$1("inputCost")}), 0)`.as("totalInputCost"),
|
|
1773
|
-
sql`COALESCE(SUM(${col$1("outputCost")}), 0)`.as("totalOutputCost"),
|
|
1774
|
-
sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens"),
|
|
1775
|
-
sql`COUNT(*)`.as("requestCount")
|
|
1776
|
-
]).where(sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col$1("createdAt")} <= ${endDate.toISOString()}`).groupBy(sql`DATE(${col$1("createdAt")})`).orderBy(sql`DATE(${col$1("createdAt")})`, "asc").execute();
|
|
1565
|
+
listPlaygroundResults: async (params) => {
|
|
1566
|
+
const value = await listPlaygroundResults.safeParseAsync(params);
|
|
1567
|
+
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1568
|
+
const { runId, columnId, limit = 500, offset = 0 } = value.data;
|
|
1569
|
+
let query = db.selectFrom("playground_results").selectAll().where("runId", "=", runId);
|
|
1570
|
+
if (columnId) query = query.where("columnId", "=", columnId);
|
|
1571
|
+
return query.orderBy("createdAt", "asc").limit(limit).offset(offset).execute();
|
|
1777
1572
|
},
|
|
1778
|
-
|
|
1779
|
-
const
|
|
1780
|
-
if (!
|
|
1781
|
-
const {
|
|
1782
|
-
|
|
1783
|
-
if (configId) baseQuery = baseQuery.where("configId", "=", configId);
|
|
1784
|
-
if (variantId) baseQuery = baseQuery.where("variantId", "=", variantId);
|
|
1785
|
-
if (environmentId) baseQuery = baseQuery.where("environmentId", "=", environmentId);
|
|
1786
|
-
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
1787
|
-
if (values.length === 0) continue;
|
|
1788
|
-
if (values.length === 1) baseQuery = baseQuery.where(sql`${col$1("tags")}->>${key} = ${values[0]}`);
|
|
1789
|
-
else {
|
|
1790
|
-
const valueList = sql.join(values.map((v) => sql`${v}`));
|
|
1791
|
-
baseQuery = baseQuery.where(sql`${col$1("tags")}->>${key} IN (${valueList})`);
|
|
1792
|
-
}
|
|
1793
|
-
}
|
|
1794
|
-
switch (groupBy) {
|
|
1795
|
-
case "day": return baseQuery.select([
|
|
1796
|
-
sql`DATE(${col$1("createdAt")})`.as("groupKey"),
|
|
1797
|
-
sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
|
|
1798
|
-
sql`COUNT(*)`.as("requestCount"),
|
|
1799
|
-
sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens")
|
|
1800
|
-
]).groupBy(sql`DATE(${col$1("createdAt")})`).orderBy(sql`DATE(${col$1("createdAt")})`, "asc").execute();
|
|
1801
|
-
case "hour": return baseQuery.select([
|
|
1802
|
-
sql`DATE_TRUNC('hour', ${col$1("createdAt")})`.as("groupKey"),
|
|
1803
|
-
sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
|
|
1804
|
-
sql`COUNT(*)`.as("requestCount"),
|
|
1805
|
-
sql`COALESCE(SUM(${col$1("totalTokens")}), 0)`.as("totalTokens")
|
|
1806
|
-
]).groupBy(sql`DATE_TRUNC('hour', ${col$1("createdAt")})`).orderBy(sql`DATE_TRUNC('hour', ${col$1("createdAt")})`, "asc").execute();
|
|
1807
|
-
case "model": return baseQuery.select([
|
|
1808
|
-
sql`${col$1("provider")} || '/' || ${col$1("model")}`.as("groupKey"),
|
|
1809
|
-
sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
|
|
1810
|
-
sql`COUNT(*)`.as("requestCount")
|
|
1811
|
-
]).groupBy(["provider", "model"]).orderBy(sql`SUM(${col$1("cost")})`, "desc").execute();
|
|
1812
|
-
case "provider": return baseQuery.select([
|
|
1813
|
-
sql`${col$1("provider")}`.as("groupKey"),
|
|
1814
|
-
sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
|
|
1815
|
-
sql`COUNT(*)`.as("requestCount")
|
|
1816
|
-
]).groupBy("provider").orderBy(sql`SUM(${col$1("cost")})`, "desc").execute();
|
|
1817
|
-
case "endpoint": return baseQuery.select([
|
|
1818
|
-
sql`COALESCE(${col$1("endpoint")}, 'unknown')`.as("groupKey"),
|
|
1819
|
-
sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
|
|
1820
|
-
sql`COUNT(*)`.as("requestCount")
|
|
1821
|
-
]).groupBy("endpoint").orderBy(sql`SUM(${col$1("cost")})`, "desc").execute();
|
|
1822
|
-
case "tags": {
|
|
1823
|
-
const conditions = [sql`${col$1("createdAt")} >= ${startDate.toISOString()}`, sql`${col$1("createdAt")} <= ${endDate.toISOString()}`];
|
|
1824
|
-
if (configId) conditions.push(sql`${col$1("configId")} = ${configId}`);
|
|
1825
|
-
if (variantId) conditions.push(sql`${col$1("variantId")} = ${variantId}`);
|
|
1826
|
-
if (environmentId) conditions.push(sql`${col$1("environmentId")} = ${environmentId}`);
|
|
1827
|
-
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
1828
|
-
if (values.length === 0) continue;
|
|
1829
|
-
if (values.length === 1) conditions.push(sql`${col$1("tags")}->>${key} = ${values[0]}`);
|
|
1830
|
-
else {
|
|
1831
|
-
const valueList = sql.join(values.map((v) => sql`${v}`));
|
|
1832
|
-
conditions.push(sql`${col$1("tags")}->>${key} IN (${valueList})`);
|
|
1833
|
-
}
|
|
1834
|
-
}
|
|
1835
|
-
if (tagKeys && tagKeys.length > 0) {
|
|
1836
|
-
const tagKeyList = sql.join(tagKeys.map((k) => sql`${k}`), sql`, `);
|
|
1837
|
-
conditions.push(sql`t.key IN (${tagKeyList})`);
|
|
1838
|
-
}
|
|
1839
|
-
const whereClause = sql.join(conditions, sql` AND `);
|
|
1840
|
-
return (await sql`
|
|
1841
|
-
SELECT t.key || ':' || t.value as "groupKey",
|
|
1842
|
-
COALESCE(SUM(${col$1("cost")}), 0) as "totalCost",
|
|
1843
|
-
COUNT(*) as "requestCount"
|
|
1844
|
-
FROM "llm_requests", jsonb_each_text(${col$1("tags")}) t
|
|
1845
|
-
WHERE ${whereClause}
|
|
1846
|
-
GROUP BY t.key, t.value
|
|
1847
|
-
ORDER BY SUM(${col$1("cost")}) DESC
|
|
1848
|
-
`.execute(db)).rows;
|
|
1849
|
-
}
|
|
1850
|
-
default: return baseQuery.select([
|
|
1851
|
-
sql`'total'`.as("groupKey"),
|
|
1852
|
-
sql`COALESCE(SUM(${col$1("cost")}), 0)`.as("totalCost"),
|
|
1853
|
-
sql`COUNT(*)`.as("requestCount")
|
|
1854
|
-
]).execute();
|
|
1855
|
-
}
|
|
1573
|
+
deletePlaygroundResultsByRunId: async (params) => {
|
|
1574
|
+
const value = await deletePlaygroundResultsByRunId.safeParseAsync(params);
|
|
1575
|
+
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1576
|
+
const { runId } = value.data;
|
|
1577
|
+
return db.deleteFrom("playground_results").where("runId", "=", runId).returningAll().execute();
|
|
1856
1578
|
},
|
|
1857
|
-
|
|
1858
|
-
const result = await
|
|
1859
|
-
|
|
1860
|
-
const { startDate, endDate, configId, variantId, environmentId, tags } = result.data;
|
|
1861
|
-
let query = db.selectFrom("llm_requests").select([
|
|
1862
|
-
sql`COUNT(*)`.as("totalRequests"),
|
|
1863
|
-
sql`COUNT(CASE WHEN ${col$1("statusCode")} >= 200 AND ${col$1("statusCode")} < 300 THEN 1 END)`.as("successfulRequests"),
|
|
1864
|
-
sql`COUNT(CASE WHEN ${col$1("statusCode")} >= 400 THEN 1 END)`.as("failedRequests"),
|
|
1865
|
-
sql`COUNT(CASE WHEN ${col$1("isStreaming")} = true THEN 1 END)`.as("streamingRequests"),
|
|
1866
|
-
sql`AVG(${col$1("latencyMs")})`.as("avgLatencyMs"),
|
|
1867
|
-
sql`MAX(${col$1("latencyMs")})`.as("maxLatencyMs"),
|
|
1868
|
-
sql`MIN(${col$1("latencyMs")})`.as("minLatencyMs")
|
|
1869
|
-
]).where(sql`${col$1("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col$1("createdAt")} <= ${endDate.toISOString()}`);
|
|
1870
|
-
if (configId) query = query.where("configId", "=", configId);
|
|
1871
|
-
if (variantId) query = query.where("variantId", "=", variantId);
|
|
1872
|
-
if (environmentId) query = query.where("environmentId", "=", environmentId);
|
|
1873
|
-
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
1874
|
-
if (values.length === 0) continue;
|
|
1875
|
-
if (values.length === 1) query = query.where(sql`${col$1("tags")}->>${key} = ${values[0]}`);
|
|
1876
|
-
else {
|
|
1877
|
-
const valueList = sql.join(values.map((v) => sql`${v}`));
|
|
1878
|
-
query = query.where(sql`${col$1("tags")}->>${key} IN (${valueList})`);
|
|
1879
|
-
}
|
|
1880
|
-
}
|
|
1881
|
-
return await query.executeTakeFirst();
|
|
1579
|
+
countPlaygroundResults: async (runId) => {
|
|
1580
|
+
const result = await db.selectFrom("playground_results").select(db.fn.countAll().as("count")).where("runId", "=", runId).executeTakeFirst();
|
|
1581
|
+
return Number(result?.count ?? 0);
|
|
1882
1582
|
},
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
FROM llm_requests, jsonb_each_text(tags) AS t(key, value)
|
|
1887
|
-
WHERE tags != '{}'::jsonb
|
|
1888
|
-
ORDER BY key, value
|
|
1889
|
-
`.execute(db)).rows;
|
|
1583
|
+
countCompletedPlaygroundResults: async (runId) => {
|
|
1584
|
+
const result = await db.selectFrom("playground_results").select(db.fn.countAll().as("count")).where("runId", "=", runId).where("status", "=", "completed").executeTakeFirst();
|
|
1585
|
+
return Number(result?.count ?? 0);
|
|
1890
1586
|
}
|
|
1891
1587
|
};
|
|
1892
1588
|
};
|
|
1893
1589
|
|
|
1894
1590
|
//#endregion
|
|
1895
|
-
//#region src/datalayer/
|
|
1896
|
-
const
|
|
1897
|
-
|
|
1591
|
+
//#region src/datalayer/playgroundRuns.ts
|
|
1592
|
+
const createPlaygroundRun = zod_default.object({
|
|
1593
|
+
playgroundId: zod_default.string().uuid(),
|
|
1898
1594
|
datasetId: zod_default.string().uuid().nullable().optional(),
|
|
1899
|
-
|
|
1595
|
+
datasetVersionId: zod_default.string().uuid().nullable().optional(),
|
|
1596
|
+
status: zod_default.enum([
|
|
1597
|
+
"pending",
|
|
1598
|
+
"running",
|
|
1599
|
+
"completed",
|
|
1600
|
+
"failed",
|
|
1601
|
+
"cancelled"
|
|
1602
|
+
]).default("pending"),
|
|
1603
|
+
totalRecords: zod_default.number().int().default(0)
|
|
1900
1604
|
});
|
|
1901
|
-
const
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1605
|
+
const updatePlaygroundRun = zod_default.object({
|
|
1606
|
+
runId: zod_default.string().uuid(),
|
|
1607
|
+
status: zod_default.enum([
|
|
1608
|
+
"pending",
|
|
1609
|
+
"running",
|
|
1610
|
+
"completed",
|
|
1611
|
+
"failed",
|
|
1612
|
+
"cancelled"
|
|
1613
|
+
]).optional(),
|
|
1614
|
+
startedAt: zod_default.date().nullable().optional(),
|
|
1615
|
+
completedAt: zod_default.date().nullable().optional(),
|
|
1616
|
+
completedRecords: zod_default.number().int().optional()
|
|
1906
1617
|
});
|
|
1907
|
-
const
|
|
1908
|
-
const
|
|
1909
|
-
|
|
1618
|
+
const getPlaygroundRunById = zod_default.object({ runId: zod_default.string().uuid() });
|
|
1619
|
+
const listPlaygroundRuns = zod_default.object({
|
|
1620
|
+
playgroundId: zod_default.string().uuid(),
|
|
1910
1621
|
limit: zod_default.number().int().positive().optional(),
|
|
1911
1622
|
offset: zod_default.number().int().nonnegative().optional()
|
|
1912
1623
|
});
|
|
1913
|
-
const
|
|
1624
|
+
const deletePlaygroundRun = zod_default.object({ runId: zod_default.string().uuid() });
|
|
1625
|
+
const createPlaygroundRunsDataLayer = (db) => {
|
|
1914
1626
|
return {
|
|
1915
|
-
|
|
1916
|
-
const value = await
|
|
1917
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1918
|
-
const { name, datasetId, columns } = value.data;
|
|
1919
|
-
return db.insertInto("playgrounds").values({
|
|
1920
|
-
id: randomUUID(),
|
|
1921
|
-
name,
|
|
1922
|
-
datasetId: datasetId ?? null,
|
|
1923
|
-
columns: columns ? JSON.stringify(columns) : null,
|
|
1924
|
-
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1925
|
-
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1926
|
-
}).returningAll().executeTakeFirst();
|
|
1927
|
-
},
|
|
1928
|
-
updatePlayground: async (params) => {
|
|
1929
|
-
const value = await updatePlayground.safeParseAsync(params);
|
|
1930
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1931
|
-
const { playgroundId, name, datasetId, columns } = value.data;
|
|
1932
|
-
const updateData = { updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
1933
|
-
if (name !== void 0) updateData.name = name;
|
|
1934
|
-
if (datasetId !== void 0) updateData.datasetId = datasetId;
|
|
1935
|
-
if (columns !== void 0) updateData.columns = columns ? JSON.stringify(columns) : null;
|
|
1936
|
-
return db.updateTable("playgrounds").set(updateData).where("id", "=", playgroundId).returningAll().executeTakeFirst();
|
|
1937
|
-
},
|
|
1938
|
-
getPlaygroundById: async (params) => {
|
|
1939
|
-
const value = await getPlaygroundById.safeParseAsync(params);
|
|
1940
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1941
|
-
const { playgroundId } = value.data;
|
|
1942
|
-
return db.selectFrom("playgrounds").selectAll().where("id", "=", playgroundId).executeTakeFirst();
|
|
1943
|
-
},
|
|
1944
|
-
deletePlayground: async (params) => {
|
|
1945
|
-
const value = await deletePlayground.safeParseAsync(params);
|
|
1946
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1947
|
-
const { playgroundId } = value.data;
|
|
1948
|
-
return db.deleteFrom("playgrounds").where("id", "=", playgroundId).returningAll().executeTakeFirst();
|
|
1949
|
-
},
|
|
1950
|
-
listPlaygrounds: async (params) => {
|
|
1951
|
-
const value = await listPlaygrounds.safeParseAsync(params || {});
|
|
1952
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
1953
|
-
const { limit = 100, offset = 0 } = value.data;
|
|
1954
|
-
return db.selectFrom("playgrounds").selectAll().orderBy("createdAt", "desc").limit(limit).offset(offset).execute();
|
|
1955
|
-
},
|
|
1956
|
-
countPlaygrounds: async () => {
|
|
1957
|
-
const result = await db.selectFrom("playgrounds").select(db.fn.countAll().as("count")).executeTakeFirst();
|
|
1958
|
-
return Number(result?.count ?? 0);
|
|
1959
|
-
}
|
|
1960
|
-
};
|
|
1961
|
-
};
|
|
1962
|
-
|
|
1963
|
-
//#endregion
|
|
1964
|
-
//#region src/datalayer/playgroundResults.ts
|
|
1965
|
-
const createPlaygroundResult = zod_default.object({
|
|
1966
|
-
runId: zod_default.string().uuid(),
|
|
1967
|
-
columnId: zod_default.string().uuid(),
|
|
1968
|
-
datasetRecordId: zod_default.string().uuid().nullable().optional(),
|
|
1969
|
-
inputVariables: zod_default.record(zod_default.string(), zod_default.unknown()).default({}),
|
|
1970
|
-
status: zod_default.enum([
|
|
1971
|
-
"pending",
|
|
1972
|
-
"running",
|
|
1973
|
-
"completed",
|
|
1974
|
-
"failed"
|
|
1975
|
-
]).default("pending")
|
|
1976
|
-
});
|
|
1977
|
-
const createPlaygroundResultsBatch = zod_default.object({ results: zod_default.array(createPlaygroundResult) });
|
|
1978
|
-
const updatePlaygroundResult = zod_default.object({
|
|
1979
|
-
resultId: zod_default.string().uuid(),
|
|
1980
|
-
outputContent: zod_default.string().nullable().optional(),
|
|
1981
|
-
status: zod_default.enum([
|
|
1982
|
-
"pending",
|
|
1983
|
-
"running",
|
|
1984
|
-
"completed",
|
|
1985
|
-
"failed"
|
|
1986
|
-
]).optional(),
|
|
1987
|
-
error: zod_default.string().nullable().optional(),
|
|
1988
|
-
latencyMs: zod_default.number().int().nullable().optional(),
|
|
1989
|
-
promptTokens: zod_default.number().int().nullable().optional(),
|
|
1990
|
-
completionTokens: zod_default.number().int().nullable().optional(),
|
|
1991
|
-
totalTokens: zod_default.number().int().nullable().optional(),
|
|
1992
|
-
cost: zod_default.number().int().nullable().optional()
|
|
1993
|
-
});
|
|
1994
|
-
const getPlaygroundResultById = zod_default.object({ resultId: zod_default.string().uuid() });
|
|
1995
|
-
const listPlaygroundResults = zod_default.object({
|
|
1996
|
-
runId: zod_default.string().uuid(),
|
|
1997
|
-
columnId: zod_default.string().uuid().optional(),
|
|
1998
|
-
limit: zod_default.number().int().positive().optional(),
|
|
1999
|
-
offset: zod_default.number().int().nonnegative().optional()
|
|
2000
|
-
});
|
|
2001
|
-
const deletePlaygroundResultsByRunId = zod_default.object({ runId: zod_default.string().uuid() });
|
|
2002
|
-
const createPlaygroundResultsDataLayer = (db) => {
|
|
2003
|
-
return {
|
|
2004
|
-
createPlaygroundResult: async (params) => {
|
|
2005
|
-
const value = await createPlaygroundResult.safeParseAsync(params);
|
|
2006
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2007
|
-
const { runId, columnId, datasetRecordId, inputVariables, status } = value.data;
|
|
2008
|
-
return db.insertInto("playground_results").values({
|
|
2009
|
-
id: randomUUID(),
|
|
2010
|
-
runId,
|
|
2011
|
-
columnId,
|
|
2012
|
-
datasetRecordId: datasetRecordId ?? null,
|
|
2013
|
-
inputVariables: JSON.stringify(inputVariables),
|
|
2014
|
-
outputContent: null,
|
|
2015
|
-
status,
|
|
2016
|
-
error: null,
|
|
2017
|
-
latencyMs: null,
|
|
2018
|
-
promptTokens: null,
|
|
2019
|
-
completionTokens: null,
|
|
2020
|
-
totalTokens: null,
|
|
2021
|
-
cost: null,
|
|
2022
|
-
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2023
|
-
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2024
|
-
}).returningAll().executeTakeFirst();
|
|
2025
|
-
},
|
|
2026
|
-
createPlaygroundResultsBatch: async (params) => {
|
|
2027
|
-
const value = await createPlaygroundResultsBatch.safeParseAsync(params);
|
|
2028
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2029
|
-
const { results } = value.data;
|
|
2030
|
-
if (results.length === 0) return [];
|
|
2031
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
2032
|
-
const values = results.map((result) => ({
|
|
2033
|
-
id: randomUUID(),
|
|
2034
|
-
runId: result.runId,
|
|
2035
|
-
columnId: result.columnId,
|
|
2036
|
-
datasetRecordId: result.datasetRecordId ?? null,
|
|
2037
|
-
inputVariables: JSON.stringify(result.inputVariables),
|
|
2038
|
-
outputContent: null,
|
|
2039
|
-
status: result.status,
|
|
2040
|
-
error: null,
|
|
2041
|
-
latencyMs: null,
|
|
2042
|
-
promptTokens: null,
|
|
2043
|
-
completionTokens: null,
|
|
2044
|
-
totalTokens: null,
|
|
2045
|
-
cost: null,
|
|
2046
|
-
createdAt: now,
|
|
2047
|
-
updatedAt: now
|
|
2048
|
-
}));
|
|
2049
|
-
return db.insertInto("playground_results").values(values).returningAll().execute();
|
|
2050
|
-
},
|
|
2051
|
-
updatePlaygroundResult: async (params) => {
|
|
2052
|
-
const value = await updatePlaygroundResult.safeParseAsync(params);
|
|
2053
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2054
|
-
const { resultId, outputContent, status, error, latencyMs, promptTokens, completionTokens, totalTokens, cost } = value.data;
|
|
2055
|
-
const updateData = { updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
2056
|
-
if (outputContent !== void 0) updateData.outputContent = outputContent;
|
|
2057
|
-
if (status !== void 0) updateData.status = status;
|
|
2058
|
-
if (error !== void 0) updateData.error = error;
|
|
2059
|
-
if (latencyMs !== void 0) updateData.latencyMs = latencyMs;
|
|
2060
|
-
if (promptTokens !== void 0) updateData.promptTokens = promptTokens;
|
|
2061
|
-
if (completionTokens !== void 0) updateData.completionTokens = completionTokens;
|
|
2062
|
-
if (totalTokens !== void 0) updateData.totalTokens = totalTokens;
|
|
2063
|
-
if (cost !== void 0) updateData.cost = cost;
|
|
2064
|
-
return db.updateTable("playground_results").set(updateData).where("id", "=", resultId).returningAll().executeTakeFirst();
|
|
2065
|
-
},
|
|
2066
|
-
getPlaygroundResultById: async (params) => {
|
|
2067
|
-
const value = await getPlaygroundResultById.safeParseAsync(params);
|
|
2068
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2069
|
-
const { resultId } = value.data;
|
|
2070
|
-
return db.selectFrom("playground_results").selectAll().where("id", "=", resultId).executeTakeFirst();
|
|
2071
|
-
},
|
|
2072
|
-
listPlaygroundResults: async (params) => {
|
|
2073
|
-
const value = await listPlaygroundResults.safeParseAsync(params);
|
|
2074
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2075
|
-
const { runId, columnId, limit = 500, offset = 0 } = value.data;
|
|
2076
|
-
let query = db.selectFrom("playground_results").selectAll().where("runId", "=", runId);
|
|
2077
|
-
if (columnId) query = query.where("columnId", "=", columnId);
|
|
2078
|
-
return query.orderBy("createdAt", "asc").limit(limit).offset(offset).execute();
|
|
2079
|
-
},
|
|
2080
|
-
deletePlaygroundResultsByRunId: async (params) => {
|
|
2081
|
-
const value = await deletePlaygroundResultsByRunId.safeParseAsync(params);
|
|
2082
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2083
|
-
const { runId } = value.data;
|
|
2084
|
-
return db.deleteFrom("playground_results").where("runId", "=", runId).returningAll().execute();
|
|
2085
|
-
},
|
|
2086
|
-
countPlaygroundResults: async (runId) => {
|
|
2087
|
-
const result = await db.selectFrom("playground_results").select(db.fn.countAll().as("count")).where("runId", "=", runId).executeTakeFirst();
|
|
2088
|
-
return Number(result?.count ?? 0);
|
|
2089
|
-
},
|
|
2090
|
-
countCompletedPlaygroundResults: async (runId) => {
|
|
2091
|
-
const result = await db.selectFrom("playground_results").select(db.fn.countAll().as("count")).where("runId", "=", runId).where("status", "=", "completed").executeTakeFirst();
|
|
2092
|
-
return Number(result?.count ?? 0);
|
|
2093
|
-
}
|
|
2094
|
-
};
|
|
2095
|
-
};
|
|
2096
|
-
|
|
2097
|
-
//#endregion
|
|
2098
|
-
//#region src/datalayer/playgroundRuns.ts
|
|
2099
|
-
const createPlaygroundRun = zod_default.object({
|
|
2100
|
-
playgroundId: zod_default.string().uuid(),
|
|
2101
|
-
datasetId: zod_default.string().uuid().nullable().optional(),
|
|
2102
|
-
datasetVersionId: zod_default.string().uuid().nullable().optional(),
|
|
2103
|
-
status: zod_default.enum([
|
|
2104
|
-
"pending",
|
|
2105
|
-
"running",
|
|
2106
|
-
"completed",
|
|
2107
|
-
"failed",
|
|
2108
|
-
"cancelled"
|
|
2109
|
-
]).default("pending"),
|
|
2110
|
-
totalRecords: zod_default.number().int().default(0)
|
|
2111
|
-
});
|
|
2112
|
-
const updatePlaygroundRun = zod_default.object({
|
|
2113
|
-
runId: zod_default.string().uuid(),
|
|
2114
|
-
status: zod_default.enum([
|
|
2115
|
-
"pending",
|
|
2116
|
-
"running",
|
|
2117
|
-
"completed",
|
|
2118
|
-
"failed",
|
|
2119
|
-
"cancelled"
|
|
2120
|
-
]).optional(),
|
|
2121
|
-
startedAt: zod_default.date().nullable().optional(),
|
|
2122
|
-
completedAt: zod_default.date().nullable().optional(),
|
|
2123
|
-
completedRecords: zod_default.number().int().optional()
|
|
2124
|
-
});
|
|
2125
|
-
const getPlaygroundRunById = zod_default.object({ runId: zod_default.string().uuid() });
|
|
2126
|
-
const listPlaygroundRuns = zod_default.object({
|
|
2127
|
-
playgroundId: zod_default.string().uuid(),
|
|
2128
|
-
limit: zod_default.number().int().positive().optional(),
|
|
2129
|
-
offset: zod_default.number().int().nonnegative().optional()
|
|
2130
|
-
});
|
|
2131
|
-
const deletePlaygroundRun = zod_default.object({ runId: zod_default.string().uuid() });
|
|
2132
|
-
const createPlaygroundRunsDataLayer = (db) => {
|
|
2133
|
-
return {
|
|
2134
|
-
createPlaygroundRun: async (params) => {
|
|
2135
|
-
const value = await createPlaygroundRun.safeParseAsync(params);
|
|
1627
|
+
createPlaygroundRun: async (params) => {
|
|
1628
|
+
const value = await createPlaygroundRun.safeParseAsync(params);
|
|
2136
1629
|
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2137
1630
|
const { playgroundId, datasetId, datasetVersionId, status, totalRecords } = value.data;
|
|
2138
1631
|
return db.insertInto("playground_runs").values({
|
|
@@ -2186,478 +1679,641 @@ const createPlaygroundRunsDataLayer = (db) => {
|
|
|
2186
1679
|
};
|
|
2187
1680
|
|
|
2188
1681
|
//#endregion
|
|
2189
|
-
//#region src/datalayer/
|
|
2190
|
-
|
|
2191
|
-
* Generate a unique slug for a provider config.
|
|
2192
|
-
* If the base slug already exists, appends -01, -02, etc.
|
|
2193
|
-
*/
|
|
2194
|
-
async function generateUniqueSlug(db, baseSlug) {
|
|
2195
|
-
const existing = await db.selectFrom("provider_configs").select("slug").where("slug", "like", `${baseSlug}%`).execute();
|
|
2196
|
-
if (existing.length === 0) return baseSlug;
|
|
2197
|
-
const existingSlugs = new Set(existing.map((e) => e.slug));
|
|
2198
|
-
if (!existingSlugs.has(baseSlug)) return baseSlug;
|
|
2199
|
-
let counter = 1;
|
|
2200
|
-
while (counter < 100) {
|
|
2201
|
-
const candidateSlug = `${baseSlug}-${counter.toString().padStart(2, "0")}`;
|
|
2202
|
-
if (!existingSlugs.has(candidateSlug)) return candidateSlug;
|
|
2203
|
-
counter++;
|
|
2204
|
-
}
|
|
2205
|
-
return `${baseSlug}-${randomUUID().slice(0, 8)}`;
|
|
2206
|
-
}
|
|
2207
|
-
const createProviderConfig = zod_default.object({
|
|
2208
|
-
providerId: zod_default.string().min(1),
|
|
2209
|
-
slug: zod_default.string().nullable().optional(),
|
|
2210
|
-
name: zod_default.string().nullable().optional(),
|
|
2211
|
-
config: zod_default.record(zod_default.string(), zod_default.unknown()),
|
|
2212
|
-
enabled: zod_default.boolean().optional().default(true)
|
|
2213
|
-
});
|
|
2214
|
-
const updateProviderConfig = zod_default.object({
|
|
2215
|
-
id: zod_default.uuidv4(),
|
|
2216
|
-
slug: zod_default.string().nullable().optional(),
|
|
1682
|
+
//#region src/datalayer/workspaceSettings.ts
|
|
1683
|
+
const updateWorkspaceSettings = zod_default.object({
|
|
2217
1684
|
name: zod_default.string().nullable().optional(),
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
});
|
|
2221
|
-
const getProviderConfigById = zod_default.object({ id: zod_default.uuidv4() });
|
|
2222
|
-
const getProviderConfigByProviderId = zod_default.object({ providerId: zod_default.string().min(1) });
|
|
2223
|
-
const getProviderConfigBySlug = zod_default.object({ slug: zod_default.string().min(1) });
|
|
2224
|
-
const deleteProviderConfig = zod_default.object({ id: zod_default.uuidv4() });
|
|
2225
|
-
const listProviderConfigs = zod_default.object({
|
|
2226
|
-
limit: zod_default.number().int().positive().optional(),
|
|
2227
|
-
offset: zod_default.number().int().nonnegative().optional()
|
|
1685
|
+
setupComplete: zod_default.boolean().optional(),
|
|
1686
|
+
superAdminId: zod_default.string().nullable().optional()
|
|
2228
1687
|
});
|
|
2229
|
-
const
|
|
1688
|
+
const createWorkspaceSettingsDataLayer = (db) => {
|
|
2230
1689
|
return {
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
if (!
|
|
2234
|
-
const { providerId, slug, name, config, enabled } = value.data;
|
|
2235
|
-
const finalSlug = slug ?? await generateUniqueSlug(db, providerId);
|
|
2236
|
-
return db.insertInto("provider_configs").values({
|
|
1690
|
+
getWorkspaceSettings: async () => {
|
|
1691
|
+
let settings = await db.selectFrom("workspace_settings").selectAll().executeTakeFirst();
|
|
1692
|
+
if (!settings) settings = await db.insertInto("workspace_settings").values({
|
|
2237
1693
|
id: randomUUID(),
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
name: name ?? null,
|
|
2241
|
-
config: JSON.stringify(config),
|
|
2242
|
-
enabled,
|
|
1694
|
+
name: null,
|
|
1695
|
+
setupComplete: false,
|
|
2243
1696
|
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2244
1697
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2245
1698
|
}).returningAll().executeTakeFirst();
|
|
1699
|
+
return settings;
|
|
2246
1700
|
},
|
|
2247
|
-
|
|
2248
|
-
const value = await
|
|
1701
|
+
updateWorkspaceSettings: async (params) => {
|
|
1702
|
+
const value = await updateWorkspaceSettings.safeParseAsync(params);
|
|
2249
1703
|
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2250
|
-
|
|
1704
|
+
let settings = await db.selectFrom("workspace_settings").selectAll().executeTakeFirst();
|
|
1705
|
+
if (!settings) return db.insertInto("workspace_settings").values({
|
|
1706
|
+
id: randomUUID(),
|
|
1707
|
+
name: value.data.name ?? null,
|
|
1708
|
+
setupComplete: value.data.setupComplete ?? false,
|
|
1709
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1710
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1711
|
+
}).returningAll().executeTakeFirst();
|
|
2251
1712
|
const updateData = { updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
2252
|
-
if (
|
|
2253
|
-
if (
|
|
2254
|
-
if (
|
|
2255
|
-
|
|
2256
|
-
return db.updateTable("provider_configs").set(updateData).where("id", "=", id).returningAll().executeTakeFirst();
|
|
2257
|
-
},
|
|
2258
|
-
getProviderConfigById: async (params) => {
|
|
2259
|
-
const value = await getProviderConfigById.safeParseAsync(params);
|
|
2260
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2261
|
-
const { id } = value.data;
|
|
2262
|
-
return db.selectFrom("provider_configs").selectAll().where("id", "=", id).executeTakeFirst();
|
|
1713
|
+
if (value.data.name !== void 0) updateData.name = value.data.name ?? null;
|
|
1714
|
+
if (value.data.setupComplete !== void 0) updateData.setupComplete = value.data.setupComplete;
|
|
1715
|
+
if (value.data.superAdminId !== void 0) updateData.superAdminId = value.data.superAdminId ?? null;
|
|
1716
|
+
return db.updateTable("workspace_settings").set(updateData).where("id", "=", settings.id).returningAll().executeTakeFirst();
|
|
2263
1717
|
},
|
|
2264
|
-
|
|
2265
|
-
|
|
2266
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2267
|
-
const { providerId } = value.data;
|
|
2268
|
-
return db.selectFrom("provider_configs").selectAll().where("providerId", "=", providerId).executeTakeFirst();
|
|
2269
|
-
},
|
|
2270
|
-
getProviderConfigBySlug: async (params) => {
|
|
2271
|
-
const value = await getProviderConfigBySlug.safeParseAsync(params);
|
|
2272
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2273
|
-
const { slug } = value.data;
|
|
2274
|
-
return db.selectFrom("provider_configs").selectAll().where("slug", "=", slug).executeTakeFirst();
|
|
2275
|
-
},
|
|
2276
|
-
deleteProviderConfig: async (params) => {
|
|
2277
|
-
const value = await deleteProviderConfig.safeParseAsync(params);
|
|
2278
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2279
|
-
const { id } = value.data;
|
|
2280
|
-
return db.deleteFrom("provider_configs").where("id", "=", id).returningAll().executeTakeFirst();
|
|
2281
|
-
},
|
|
2282
|
-
listProviderConfigs: async (params) => {
|
|
2283
|
-
const value = await listProviderConfigs.safeParseAsync(params || {});
|
|
2284
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2285
|
-
const { limit = 100, offset = 0 } = value.data;
|
|
2286
|
-
return db.selectFrom("provider_configs").selectAll().orderBy("createdAt", "desc").limit(limit).offset(offset).execute();
|
|
2287
|
-
},
|
|
2288
|
-
countProviderConfigs: async () => {
|
|
2289
|
-
const result = await db.selectFrom("provider_configs").select(db.fn.countAll().as("count")).executeTakeFirst();
|
|
2290
|
-
return Number(result?.count ?? 0);
|
|
1718
|
+
getSuperAdminId: async () => {
|
|
1719
|
+
return (await db.selectFrom("workspace_settings").select("superAdminId").executeTakeFirst())?.superAdminId ?? null;
|
|
2291
1720
|
},
|
|
2292
|
-
|
|
2293
|
-
|
|
2294
|
-
if (
|
|
2295
|
-
|
|
2296
|
-
|
|
2297
|
-
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
config: JSON.stringify(config),
|
|
2303
|
-
enabled,
|
|
1721
|
+
setSuperAdminId: async (userId) => {
|
|
1722
|
+
let settings = await db.selectFrom("workspace_settings").selectAll().executeTakeFirst();
|
|
1723
|
+
if (settings?.superAdminId) return false;
|
|
1724
|
+
if (!settings) {
|
|
1725
|
+
await db.insertInto("workspace_settings").values({
|
|
1726
|
+
id: randomUUID(),
|
|
1727
|
+
name: null,
|
|
1728
|
+
setupComplete: false,
|
|
1729
|
+
superAdminId: userId,
|
|
1730
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2304
1731
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2305
|
-
}).
|
|
1732
|
+
}).execute();
|
|
1733
|
+
return true;
|
|
2306
1734
|
}
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
id: randomUUID(),
|
|
2310
|
-
providerId,
|
|
2311
|
-
slug: finalSlug,
|
|
2312
|
-
name: name ?? null,
|
|
2313
|
-
config: JSON.stringify(config),
|
|
2314
|
-
enabled,
|
|
2315
|
-
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2316
|
-
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2317
|
-
}).returningAll().executeTakeFirst();
|
|
2318
|
-
}
|
|
2319
|
-
};
|
|
2320
|
-
};
|
|
2321
|
-
|
|
2322
|
-
//#endregion
|
|
2323
|
-
//#region src/datalayer/providerGuardrailOverrides.ts
|
|
2324
|
-
const createProviderGuardrailOverride = zod_default.object({
|
|
2325
|
-
providerConfigId: zod_default.string().uuid(),
|
|
2326
|
-
guardrailConfigId: zod_default.string().uuid(),
|
|
2327
|
-
enabled: zod_default.boolean().optional().default(true),
|
|
2328
|
-
parameters: zod_default.record(zod_default.string(), zod_default.unknown()).nullable().optional()
|
|
2329
|
-
});
|
|
2330
|
-
const updateProviderGuardrailOverride = zod_default.object({
|
|
2331
|
-
id: zod_default.string().uuid(),
|
|
2332
|
-
enabled: zod_default.boolean().optional(),
|
|
2333
|
-
parameters: zod_default.record(zod_default.string(), zod_default.unknown()).nullable().optional()
|
|
2334
|
-
});
|
|
2335
|
-
const getOverrideById = zod_default.object({ id: zod_default.string().uuid() });
|
|
2336
|
-
const deleteOverride = zod_default.object({ id: zod_default.string().uuid() });
|
|
2337
|
-
const getOverridesByProviderConfigId = zod_default.object({ providerConfigId: zod_default.string().uuid() });
|
|
2338
|
-
const getOverridesByGuardrailConfigId = zod_default.object({ guardrailConfigId: zod_default.string().uuid() });
|
|
2339
|
-
const getOverrideByProviderAndGuardrail = zod_default.object({
|
|
2340
|
-
providerConfigId: zod_default.string().uuid(),
|
|
2341
|
-
guardrailConfigId: zod_default.string().uuid()
|
|
2342
|
-
});
|
|
2343
|
-
const createProviderGuardrailOverridesDataLayer = (db) => {
|
|
2344
|
-
return {
|
|
2345
|
-
createProviderGuardrailOverride: async (params) => {
|
|
2346
|
-
const value = await createProviderGuardrailOverride.safeParseAsync(params);
|
|
2347
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2348
|
-
const { providerConfigId, guardrailConfigId, enabled, parameters } = value.data;
|
|
2349
|
-
return db.insertInto("provider_guardrail_overrides").values({
|
|
2350
|
-
id: randomUUID(),
|
|
2351
|
-
providerConfigId,
|
|
2352
|
-
guardrailConfigId,
|
|
2353
|
-
enabled,
|
|
2354
|
-
parameters: parameters ? JSON.stringify(parameters) : null,
|
|
2355
|
-
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1735
|
+
await db.updateTable("workspace_settings").set({
|
|
1736
|
+
superAdminId: userId,
|
|
2356
1737
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2357
|
-
}).
|
|
2358
|
-
|
|
2359
|
-
updateProviderGuardrailOverride: async (params) => {
|
|
2360
|
-
const value = await updateProviderGuardrailOverride.safeParseAsync(params);
|
|
2361
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2362
|
-
const { id, enabled, parameters } = value.data;
|
|
2363
|
-
const updateData = { updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
2364
|
-
if (enabled !== void 0) updateData.enabled = enabled;
|
|
2365
|
-
if (parameters !== void 0) updateData.parameters = parameters ? JSON.stringify(parameters) : null;
|
|
2366
|
-
return db.updateTable("provider_guardrail_overrides").set(updateData).where("id", "=", id).returningAll().executeTakeFirst();
|
|
2367
|
-
},
|
|
2368
|
-
getOverrideById: async (params) => {
|
|
2369
|
-
const value = await getOverrideById.safeParseAsync(params);
|
|
2370
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2371
|
-
const { id } = value.data;
|
|
2372
|
-
return db.selectFrom("provider_guardrail_overrides").selectAll().where("id", "=", id).executeTakeFirst();
|
|
2373
|
-
},
|
|
2374
|
-
deleteProviderGuardrailOverride: async (params) => {
|
|
2375
|
-
const value = await deleteOverride.safeParseAsync(params);
|
|
2376
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2377
|
-
const { id } = value.data;
|
|
2378
|
-
return db.deleteFrom("provider_guardrail_overrides").where("id", "=", id).returningAll().executeTakeFirst();
|
|
2379
|
-
},
|
|
2380
|
-
getOverridesByProviderConfigId: async (params) => {
|
|
2381
|
-
const value = await getOverridesByProviderConfigId.safeParseAsync(params);
|
|
2382
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2383
|
-
const { providerConfigId } = value.data;
|
|
2384
|
-
return db.selectFrom("provider_guardrail_overrides").selectAll().where("providerConfigId", "=", providerConfigId).execute();
|
|
2385
|
-
},
|
|
2386
|
-
getOverridesByGuardrailConfigId: async (params) => {
|
|
2387
|
-
const value = await getOverridesByGuardrailConfigId.safeParseAsync(params);
|
|
2388
|
-
if (!value.success) throw new LLMOpsError(`Invalid parameters: ${value.error.message}`);
|
|
2389
|
-
const { guardrailConfigId } = value.data;
|
|
2390
|
-
return db.selectFrom("provider_guardrail_overrides").selectAll().where("guardrailConfigId", "=", guardrailConfigId).execute();
|
|
1738
|
+
}).where("id", "=", settings.id).execute();
|
|
1739
|
+
return true;
|
|
2391
1740
|
},
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
1741
|
+
isSetupComplete: async () => {
|
|
1742
|
+
try {
|
|
1743
|
+
return (await db.selectFrom("workspace_settings").select("setupComplete").executeTakeFirst())?.setupComplete ?? false;
|
|
1744
|
+
} catch {
|
|
1745
|
+
return false;
|
|
1746
|
+
}
|
|
2397
1747
|
},
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
if (!
|
|
2401
|
-
const { providerConfigId, guardrailConfigId, enabled, parameters } = value.data;
|
|
2402
|
-
const existing = await db.selectFrom("provider_guardrail_overrides").selectAll().where("providerConfigId", "=", providerConfigId).where("guardrailConfigId", "=", guardrailConfigId).executeTakeFirst();
|
|
2403
|
-
if (existing) return db.updateTable("provider_guardrail_overrides").set({
|
|
2404
|
-
enabled,
|
|
2405
|
-
parameters: parameters ? JSON.stringify(parameters) : null,
|
|
2406
|
-
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2407
|
-
}).where("id", "=", existing.id).returningAll().executeTakeFirst();
|
|
2408
|
-
return db.insertInto("provider_guardrail_overrides").values({
|
|
1748
|
+
markSetupComplete: async () => {
|
|
1749
|
+
let settings = await db.selectFrom("workspace_settings").selectAll().executeTakeFirst();
|
|
1750
|
+
if (!settings) return db.insertInto("workspace_settings").values({
|
|
2409
1751
|
id: randomUUID(),
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
enabled,
|
|
2413
|
-
parameters: parameters ? JSON.stringify(parameters) : null,
|
|
1752
|
+
name: null,
|
|
1753
|
+
setupComplete: true,
|
|
2414
1754
|
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2415
1755
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2416
1756
|
}).returningAll().executeTakeFirst();
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
const { guardrailConfigId } = value.data;
|
|
2422
|
-
return db.deleteFrom("provider_guardrail_overrides").where("guardrailConfigId", "=", guardrailConfigId).execute();
|
|
1757
|
+
return db.updateTable("workspace_settings").set({
|
|
1758
|
+
setupComplete: true,
|
|
1759
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1760
|
+
}).where("id", "=", settings.id).returningAll().executeTakeFirst();
|
|
2423
1761
|
}
|
|
2424
1762
|
};
|
|
2425
1763
|
};
|
|
2426
1764
|
|
|
2427
1765
|
//#endregion
|
|
2428
|
-
//#region src/datalayer/
|
|
2429
|
-
const col = (name) => sql.ref(name);
|
|
1766
|
+
//#region src/datalayer/create.ts
|
|
2430
1767
|
/**
|
|
2431
|
-
*
|
|
1768
|
+
* Create all datalayers from a Kysely database instance.
|
|
1769
|
+
* Returns a flat object with all datalayer methods spread together.
|
|
2432
1770
|
*/
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
|
|
2437
|
-
|
|
2438
|
-
|
|
2439
|
-
|
|
2440
|
-
|
|
2441
|
-
|
|
2442
|
-
|
|
2443
|
-
|
|
2444
|
-
|
|
2445
|
-
durationMs: zod_default.number().int().nullable().optional(),
|
|
2446
|
-
spanCount: zod_default.number().int().default(1),
|
|
2447
|
-
totalInputTokens: zod_default.number().int().default(0),
|
|
2448
|
-
totalOutputTokens: zod_default.number().int().default(0),
|
|
2449
|
-
totalTokens: zod_default.number().int().default(0),
|
|
2450
|
-
totalCost: zod_default.number().int().default(0),
|
|
2451
|
-
tags: zod_default.record(zod_default.string(), zod_default.string()).default({}),
|
|
2452
|
-
metadata: zod_default.record(zod_default.string(), zod_default.unknown()).default({})
|
|
2453
|
-
});
|
|
1771
|
+
function createDataLayer(db) {
|
|
1772
|
+
return {
|
|
1773
|
+
...createDatasetsDataLayer(db),
|
|
1774
|
+
...createPlaygroundDataLayer(db),
|
|
1775
|
+
...createPlaygroundResultsDataLayer(db),
|
|
1776
|
+
...createPlaygroundRunsDataLayer(db),
|
|
1777
|
+
...createWorkspaceSettingsDataLayer(db)
|
|
1778
|
+
};
|
|
1779
|
+
}
|
|
1780
|
+
|
|
1781
|
+
//#endregion
|
|
1782
|
+
//#region src/pricing/calculator.ts
|
|
2454
1783
|
/**
|
|
2455
|
-
*
|
|
1784
|
+
* Calculate the cost of an LLM request in micro-dollars
|
|
1785
|
+
*
|
|
1786
|
+
* Micro-dollars are used to avoid floating-point precision issues:
|
|
1787
|
+
* - 1 dollar = 1,000,000 micro-dollars
|
|
1788
|
+
* - $0.001 = 1,000 micro-dollars
|
|
1789
|
+
* - $0.000001 = 1 micro-dollar
|
|
1790
|
+
*
|
|
1791
|
+
* @param usage - Token usage data from the LLM response
|
|
1792
|
+
* @param pricing - Model pricing information
|
|
1793
|
+
* @returns Cost breakdown in micro-dollars
|
|
1794
|
+
*
|
|
1795
|
+
* @example
|
|
1796
|
+
* ```typescript
|
|
1797
|
+
* const usage = { promptTokens: 1000, completionTokens: 500 };
|
|
1798
|
+
* const pricing = { inputCostPer1M: 2.5, outputCostPer1M: 10.0 };
|
|
1799
|
+
* const cost = calculateCost(usage, pricing);
|
|
1800
|
+
* // cost = { inputCost: 2500, outputCost: 5000, totalCost: 7500 }
|
|
1801
|
+
* // In dollars: $0.0025 input + $0.005 output = $0.0075 total
|
|
1802
|
+
* ```
|
|
2456
1803
|
*/
|
|
2457
|
-
|
|
2458
|
-
|
|
2459
|
-
|
|
2460
|
-
|
|
2461
|
-
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
durationMs: zod_default.number().int().nullable().optional(),
|
|
2468
|
-
provider: zod_default.string().nullable().optional(),
|
|
2469
|
-
model: zod_default.string().nullable().optional(),
|
|
2470
|
-
promptTokens: zod_default.number().int().default(0),
|
|
2471
|
-
completionTokens: zod_default.number().int().default(0),
|
|
2472
|
-
totalTokens: zod_default.number().int().default(0),
|
|
2473
|
-
cost: zod_default.number().int().default(0),
|
|
2474
|
-
configId: zod_default.string().uuid().nullable().optional(),
|
|
2475
|
-
variantId: zod_default.string().uuid().nullable().optional(),
|
|
2476
|
-
environmentId: zod_default.string().uuid().nullable().optional(),
|
|
2477
|
-
providerConfigId: zod_default.string().uuid().nullable().optional(),
|
|
2478
|
-
requestId: zod_default.string().uuid().nullable().optional(),
|
|
2479
|
-
source: zod_default.enum([
|
|
2480
|
-
"gateway",
|
|
2481
|
-
"otlp",
|
|
2482
|
-
"langsmith"
|
|
2483
|
-
]).default("gateway"),
|
|
2484
|
-
input: zod_default.unknown().nullable().optional(),
|
|
2485
|
-
output: zod_default.unknown().nullable().optional(),
|
|
2486
|
-
attributes: zod_default.record(zod_default.string(), zod_default.unknown()).default({})
|
|
2487
|
-
});
|
|
1804
|
+
function calculateCost(usage, pricing) {
|
|
1805
|
+
const inputCost = Math.round(usage.promptTokens * pricing.inputCostPer1M);
|
|
1806
|
+
const outputCost = Math.round(usage.completionTokens * pricing.outputCostPer1M);
|
|
1807
|
+
return {
|
|
1808
|
+
inputCost,
|
|
1809
|
+
outputCost,
|
|
1810
|
+
totalCost: inputCost + outputCost,
|
|
1811
|
+
cacheSavings: 0
|
|
1812
|
+
};
|
|
1813
|
+
}
|
|
2488
1814
|
/**
|
|
2489
|
-
*
|
|
1815
|
+
* Get default cache read rate as a fraction of input cost per provider.
|
|
1816
|
+
* Used when models.dev doesn't provide cache pricing.
|
|
2490
1817
|
*/
|
|
2491
|
-
|
|
2492
|
-
|
|
2493
|
-
|
|
2494
|
-
|
|
2495
|
-
|
|
2496
|
-
|
|
2497
|
-
|
|
1818
|
+
function getDefaultCacheReadRate(provider, inputCostPer1M) {
|
|
1819
|
+
switch (provider?.toLowerCase()) {
|
|
1820
|
+
case "anthropic": return inputCostPer1M * .1;
|
|
1821
|
+
case "openai":
|
|
1822
|
+
case "azure-openai": return inputCostPer1M * .5;
|
|
1823
|
+
case "google":
|
|
1824
|
+
case "gemini":
|
|
1825
|
+
case "vertex_ai": return inputCostPer1M * .25;
|
|
1826
|
+
default: return inputCostPer1M * .5;
|
|
1827
|
+
}
|
|
1828
|
+
}
|
|
2498
1829
|
/**
|
|
2499
|
-
*
|
|
1830
|
+
* Get default cache write/creation rate as a fraction of input cost per provider.
|
|
1831
|
+
* Used when models.dev doesn't provide cache pricing.
|
|
2500
1832
|
*/
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
"unset",
|
|
2508
|
-
"ok",
|
|
2509
|
-
"error"
|
|
2510
|
-
]).optional(),
|
|
2511
|
-
name: zod_default.string().optional(),
|
|
2512
|
-
startDate: zod_default.date().optional(),
|
|
2513
|
-
endDate: zod_default.date().optional(),
|
|
2514
|
-
tags: zod_default.record(zod_default.string(), zod_default.array(zod_default.string())).optional()
|
|
2515
|
-
});
|
|
1833
|
+
function getDefaultCacheWriteRate(provider, inputCostPer1M) {
|
|
1834
|
+
switch (provider?.toLowerCase()) {
|
|
1835
|
+
case "anthropic": return inputCostPer1M * 1.25;
|
|
1836
|
+
default: return inputCostPer1M;
|
|
1837
|
+
}
|
|
1838
|
+
}
|
|
2516
1839
|
/**
|
|
2517
|
-
*
|
|
1840
|
+
* Calculate cache-aware cost of an LLM request in micro-dollars.
|
|
1841
|
+
*
|
|
1842
|
+
* Splits input tokens into uncached, cache-read, and cache-creation buckets,
|
|
1843
|
+
* each priced at different rates. Falls back to provider-specific multipliers
|
|
1844
|
+
* when models.dev doesn't provide cache pricing.
|
|
1845
|
+
*
|
|
1846
|
+
* @param usage - Token usage data (with cachedTokens and cacheCreationTokens)
|
|
1847
|
+
* @param pricing - Model pricing (may include cacheReadCostPer1M / cacheWriteCostPer1M)
|
|
1848
|
+
* @param provider - Provider name for fallback rate selection
|
|
1849
|
+
* @returns Cost breakdown in micro-dollars
|
|
2518
1850
|
*/
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
const
|
|
1851
|
+
function calculateCacheAwareCost(usage, pricing, provider) {
|
|
1852
|
+
const cachedTokens = usage.cachedTokens ?? 0;
|
|
1853
|
+
const cacheCreationTokens = usage.cacheCreationTokens ?? 0;
|
|
1854
|
+
if (cachedTokens === 0 && cacheCreationTokens === 0) return calculateCost(usage, pricing);
|
|
1855
|
+
const cacheReadRate = pricing.cacheReadCostPer1M ?? getDefaultCacheReadRate(provider, pricing.inputCostPer1M);
|
|
1856
|
+
const cacheWriteRate = pricing.cacheWriteCostPer1M ?? getDefaultCacheWriteRate(provider, pricing.inputCostPer1M);
|
|
1857
|
+
const uncachedInputTokens = Math.max(0, usage.promptTokens - cachedTokens - cacheCreationTokens);
|
|
1858
|
+
const regularInputCost = Math.round(uncachedInputTokens * pricing.inputCostPer1M);
|
|
1859
|
+
const cacheReadCost = Math.round(cachedTokens * cacheReadRate);
|
|
1860
|
+
const cacheWriteCost = Math.round(cacheCreationTokens * cacheWriteRate);
|
|
1861
|
+
const outputCost = Math.round(usage.completionTokens * pricing.outputCostPer1M);
|
|
1862
|
+
const inputCost = regularInputCost + cacheReadCost + cacheWriteCost;
|
|
2526
1863
|
return {
|
|
2527
|
-
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
|
|
2534
|
-
|
|
2535
|
-
|
|
2536
|
-
|
|
2537
|
-
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
|
|
2541
|
-
|
|
2542
|
-
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
|
|
2555
|
-
|
|
2556
|
-
|
|
2557
|
-
|
|
2558
|
-
|
|
2559
|
-
|
|
2560
|
-
|
|
2561
|
-
|
|
2562
|
-
|
|
2563
|
-
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
|
|
2567
|
-
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
2584
|
-
|
|
2585
|
-
|
|
2586
|
-
|
|
2587
|
-
|
|
2588
|
-
|
|
1864
|
+
inputCost,
|
|
1865
|
+
outputCost,
|
|
1866
|
+
totalCost: inputCost + outputCost,
|
|
1867
|
+
cacheSavings: Math.round((cachedTokens + cacheCreationTokens) * pricing.inputCostPer1M) - cacheReadCost - cacheWriteCost
|
|
1868
|
+
};
|
|
1869
|
+
}
|
|
1870
|
+
/**
|
|
1871
|
+
* Convert micro-dollars to dollars
|
|
1872
|
+
*
|
|
1873
|
+
* @param microDollars - Amount in micro-dollars
|
|
1874
|
+
* @returns Amount in dollars
|
|
1875
|
+
*
|
|
1876
|
+
* @example
|
|
1877
|
+
* ```typescript
|
|
1878
|
+
* microDollarsToDollars(7500); // 0.0075
|
|
1879
|
+
* microDollarsToDollars(1000000); // 1.0
|
|
1880
|
+
* ```
|
|
1881
|
+
*/
|
|
1882
|
+
function microDollarsToDollars(microDollars) {
|
|
1883
|
+
return microDollars / 1e6;
|
|
1884
|
+
}
|
|
1885
|
+
/**
|
|
1886
|
+
* Convert dollars to micro-dollars
|
|
1887
|
+
*
|
|
1888
|
+
* @param dollars - Amount in dollars
|
|
1889
|
+
* @returns Amount in micro-dollars (rounded to nearest integer)
|
|
1890
|
+
*
|
|
1891
|
+
* @example
|
|
1892
|
+
* ```typescript
|
|
1893
|
+
* dollarsToMicroDollars(0.0075); // 7500
|
|
1894
|
+
* dollarsToMicroDollars(1.0); // 1000000
|
|
1895
|
+
* ```
|
|
1896
|
+
*/
|
|
1897
|
+
function dollarsToMicroDollars(dollars) {
|
|
1898
|
+
return Math.round(dollars * 1e6);
|
|
1899
|
+
}
|
|
1900
|
+
/**
|
|
1901
|
+
* Format micro-dollars as a human-readable dollar string
|
|
1902
|
+
*
|
|
1903
|
+
* @param microDollars - Amount in micro-dollars
|
|
1904
|
+
* @param decimals - Number of decimal places (default: 6)
|
|
1905
|
+
* @returns Formatted dollar string
|
|
1906
|
+
*
|
|
1907
|
+
* @example
|
|
1908
|
+
* ```typescript
|
|
1909
|
+
* formatCost(7500); // "$0.007500"
|
|
1910
|
+
* formatCost(1234567, 2); // "$1.23"
|
|
1911
|
+
* ```
|
|
1912
|
+
*/
|
|
1913
|
+
function formatCost(microDollars, decimals = 6) {
|
|
1914
|
+
return `$${microDollarsToDollars(microDollars).toFixed(decimals)}`;
|
|
1915
|
+
}
|
|
1916
|
+
|
|
1917
|
+
//#endregion
|
|
1918
|
+
//#region src/pricing/provider.ts
|
|
1919
|
+
const LLMOPS_MODELS_API = "https://models.llmops.build";
|
|
1920
|
+
/**
|
|
1921
|
+
* Convert price from USD cents per token to dollars per 1M tokens.
|
|
1922
|
+
*
|
|
1923
|
+
* API returns cents/token. Our system uses dollars/1M tokens.
|
|
1924
|
+
* Formula: (centsPerToken / 100) * 1_000_000 = centsPerToken * 10_000
|
|
1925
|
+
*/
|
|
1926
|
+
function centsPerTokenToCostPer1M(centsPerToken) {
|
|
1927
|
+
return centsPerToken * 1e4;
|
|
1928
|
+
}
|
|
1929
|
+
/**
|
|
1930
|
+
* Pricing provider that fetches per-model data from the LLMOps Models API.
|
|
1931
|
+
*
|
|
1932
|
+
* Features:
|
|
1933
|
+
* - Per-model in-memory cache with configurable TTL (default 5 minutes)
|
|
1934
|
+
* - Deduplicates concurrent fetches for the same model
|
|
1935
|
+
* - Caches null results (404s) to avoid repeated lookups
|
|
1936
|
+
* - Falls back to stale cache on fetch errors
|
|
1937
|
+
*/
|
|
1938
|
+
var LLMOpsPricingProvider = class {
|
|
1939
|
+
cache = /* @__PURE__ */ new Map();
|
|
1940
|
+
pendingFetches = /* @__PURE__ */ new Map();
|
|
1941
|
+
cacheTTL;
|
|
1942
|
+
baseUrl;
|
|
1943
|
+
constructor(options) {
|
|
1944
|
+
this.cacheTTL = options?.cacheTTL ?? 300 * 1e3;
|
|
1945
|
+
this.baseUrl = options?.baseUrl ?? LLMOPS_MODELS_API;
|
|
1946
|
+
}
|
|
1947
|
+
getCacheKey(provider, model) {
|
|
1948
|
+
return `${provider.toLowerCase()}:${model.toLowerCase()}`;
|
|
1949
|
+
}
|
|
1950
|
+
/**
|
|
1951
|
+
* Fetch pricing for a single model from the API
|
|
1952
|
+
*/
|
|
1953
|
+
async fetchModelPricing(provider, model) {
|
|
1954
|
+
const url = `${this.baseUrl}/model-configs/pricing/${encodeURIComponent(provider)}/${model}`;
|
|
1955
|
+
try {
|
|
1956
|
+
logger.debug(`[Pricing] GET ${url}`);
|
|
1957
|
+
const startTime = Date.now();
|
|
1958
|
+
const response = await fetch(url);
|
|
1959
|
+
const elapsed = Date.now() - startTime;
|
|
1960
|
+
logger.debug(`[Pricing] GET ${url} -> ${response.status} (${elapsed}ms)`);
|
|
1961
|
+
if (response.status === 404) {
|
|
1962
|
+
logger.debug(`[Pricing] No pricing found for ${provider}/${model}`);
|
|
1963
|
+
return null;
|
|
2589
1964
|
}
|
|
2590
|
-
if (
|
|
1965
|
+
if (!response.ok) throw new Error(`API returned ${response.status}`);
|
|
1966
|
+
const data = await response.json();
|
|
1967
|
+
if (!data.pay_as_you_go) return null;
|
|
1968
|
+
const payg = data.pay_as_you_go;
|
|
1969
|
+
const pricing = {
|
|
1970
|
+
inputCostPer1M: centsPerTokenToCostPer1M(payg.request_token?.price ?? 0),
|
|
1971
|
+
outputCostPer1M: centsPerTokenToCostPer1M(payg.response_token?.price ?? 0),
|
|
1972
|
+
cacheReadCostPer1M: payg.cache_read_input_token?.price != null ? centsPerTokenToCostPer1M(payg.cache_read_input_token.price) : void 0,
|
|
1973
|
+
cacheWriteCostPer1M: payg.cache_write_input_token?.price != null ? centsPerTokenToCostPer1M(payg.cache_write_input_token.price) : void 0
|
|
1974
|
+
};
|
|
1975
|
+
logger.debug(`[Pricing] Cached pricing for ${provider}/${model}: input=$${pricing.inputCostPer1M}/1M, output=$${pricing.outputCostPer1M}/1M`);
|
|
1976
|
+
return pricing;
|
|
1977
|
+
} catch (error) {
|
|
1978
|
+
logger.error(`[Pricing] Failed to fetch pricing for ${provider}/${model}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1979
|
+
const cacheKey = this.getCacheKey(provider, model);
|
|
1980
|
+
const stale = this.cache.get(cacheKey);
|
|
1981
|
+
if (stale) {
|
|
1982
|
+
logger.debug(`[Pricing] Using stale cache for ${provider}/${model}`);
|
|
1983
|
+
return stale.pricing;
|
|
1984
|
+
}
|
|
1985
|
+
return null;
|
|
1986
|
+
}
|
|
1987
|
+
}
|
|
1988
|
+
/**
|
|
1989
|
+
* Internal: fetch with cache and deduplication for a specific provider+model
|
|
1990
|
+
*/
|
|
1991
|
+
async getCachedPricing(provider, model) {
|
|
1992
|
+
const cacheKey = this.getCacheKey(provider, model);
|
|
1993
|
+
const cached = this.cache.get(cacheKey);
|
|
1994
|
+
if (cached && Date.now() - cached.fetchedAt < this.cacheTTL) return cached.pricing;
|
|
1995
|
+
let pending = this.pendingFetches.get(cacheKey);
|
|
1996
|
+
if (!pending) {
|
|
1997
|
+
pending = this.fetchModelPricing(provider, model).then((pricing) => {
|
|
1998
|
+
this.cache.set(cacheKey, {
|
|
1999
|
+
pricing,
|
|
2000
|
+
fetchedAt: Date.now()
|
|
2001
|
+
});
|
|
2002
|
+
return pricing;
|
|
2003
|
+
}).finally(() => {
|
|
2004
|
+
this.pendingFetches.delete(cacheKey);
|
|
2005
|
+
});
|
|
2006
|
+
this.pendingFetches.set(cacheKey, pending);
|
|
2007
|
+
}
|
|
2008
|
+
return pending;
|
|
2009
|
+
}
|
|
2010
|
+
/**
|
|
2011
|
+
* Get pricing for a specific model.
|
|
2012
|
+
*
|
|
2013
|
+
* When the model name contains a slash (e.g. "google/gemini-2.5-flash"),
|
|
2014
|
+
* it's likely an OpenRouter model ID. If the initial provider lookup fails,
|
|
2015
|
+
* we automatically retry with "openrouter" as the provider.
|
|
2016
|
+
*/
|
|
2017
|
+
async getModelPricing(provider, model) {
|
|
2018
|
+
const pricing = await this.getCachedPricing(provider, model);
|
|
2019
|
+
if (pricing) return pricing;
|
|
2020
|
+
if (!pricing && model.includes("/") && provider.toLowerCase() !== "openrouter") {
|
|
2021
|
+
logger.debug(`[Pricing] Retrying ${provider}/${model} as openrouter/${model}`);
|
|
2022
|
+
return this.getCachedPricing("openrouter", model);
|
|
2023
|
+
}
|
|
2024
|
+
return pricing;
|
|
2025
|
+
}
|
|
2026
|
+
/**
|
|
2027
|
+
* Force refresh the pricing cache (clears all cached entries)
|
|
2028
|
+
*/
|
|
2029
|
+
async refreshCache() {
|
|
2030
|
+
this.cache.clear();
|
|
2031
|
+
}
|
|
2032
|
+
/**
|
|
2033
|
+
* Always ready — no bulk pre-fetch needed
|
|
2034
|
+
*/
|
|
2035
|
+
isReady() {
|
|
2036
|
+
return true;
|
|
2037
|
+
}
|
|
2038
|
+
/**
|
|
2039
|
+
* Get the number of cached models (for debugging)
|
|
2040
|
+
*/
|
|
2041
|
+
getCacheSize() {
|
|
2042
|
+
return this.cache.size;
|
|
2043
|
+
}
|
|
2044
|
+
};
|
|
2045
|
+
let defaultProvider = null;
|
|
2046
|
+
/**
|
|
2047
|
+
* Get the default pricing provider instance
|
|
2048
|
+
*/
|
|
2049
|
+
function getDefaultPricingProvider() {
|
|
2050
|
+
if (!defaultProvider) defaultProvider = new LLMOpsPricingProvider();
|
|
2051
|
+
return defaultProvider;
|
|
2052
|
+
}
|
|
2053
|
+
|
|
2054
|
+
//#endregion
|
|
2055
|
+
//#region src/telemetry/postgres.ts
|
|
2056
|
+
const guardrailResultSchema = zod_default.object({
|
|
2057
|
+
checkId: zod_default.string(),
|
|
2058
|
+
functionId: zod_default.string(),
|
|
2059
|
+
hookType: zod_default.enum(["beforeRequestHook", "afterRequestHook"]),
|
|
2060
|
+
verdict: zod_default.boolean(),
|
|
2061
|
+
latencyMs: zod_default.number()
|
|
2062
|
+
});
|
|
2063
|
+
const guardrailResultsSchema = zod_default.object({
|
|
2064
|
+
results: zod_default.array(guardrailResultSchema),
|
|
2065
|
+
action: zod_default.enum([
|
|
2066
|
+
"allowed",
|
|
2067
|
+
"blocked",
|
|
2068
|
+
"logged"
|
|
2069
|
+
]),
|
|
2070
|
+
totalLatencyMs: zod_default.number()
|
|
2071
|
+
});
|
|
2072
|
+
const insertLLMRequestSchema = zod_default.object({
|
|
2073
|
+
requestId: zod_default.string().uuid(),
|
|
2074
|
+
configId: zod_default.string().uuid().nullable().optional(),
|
|
2075
|
+
variantId: zod_default.string().uuid().nullable().optional(),
|
|
2076
|
+
environmentId: zod_default.string().uuid().nullable().optional(),
|
|
2077
|
+
providerConfigId: zod_default.string().uuid().nullable().optional(),
|
|
2078
|
+
provider: zod_default.string(),
|
|
2079
|
+
model: zod_default.string(),
|
|
2080
|
+
promptTokens: zod_default.number().int().default(0),
|
|
2081
|
+
completionTokens: zod_default.number().int().default(0),
|
|
2082
|
+
totalTokens: zod_default.number().int().default(0),
|
|
2083
|
+
cachedTokens: zod_default.number().int().default(0),
|
|
2084
|
+
cacheCreationTokens: zod_default.number().int().default(0),
|
|
2085
|
+
cost: zod_default.number().int().default(0),
|
|
2086
|
+
cacheSavings: zod_default.number().int().default(0),
|
|
2087
|
+
inputCost: zod_default.number().int().default(0),
|
|
2088
|
+
outputCost: zod_default.number().int().default(0),
|
|
2089
|
+
endpoint: zod_default.string(),
|
|
2090
|
+
statusCode: zod_default.number().int(),
|
|
2091
|
+
latencyMs: zod_default.number().int().default(0),
|
|
2092
|
+
isStreaming: zod_default.boolean().default(false),
|
|
2093
|
+
userId: zod_default.string().nullable().optional(),
|
|
2094
|
+
tags: zod_default.record(zod_default.string(), zod_default.string()).default({}),
|
|
2095
|
+
guardrailResults: guardrailResultsSchema.nullable().optional(),
|
|
2096
|
+
traceId: zod_default.string().nullable().optional(),
|
|
2097
|
+
spanId: zod_default.string().nullable().optional(),
|
|
2098
|
+
parentSpanId: zod_default.string().nullable().optional(),
|
|
2099
|
+
sessionId: zod_default.string().nullable().optional()
|
|
2100
|
+
});
|
|
2101
|
+
const listRequestsSchema = zod_default.object({
|
|
2102
|
+
limit: zod_default.number().int().positive().max(1e3).default(100),
|
|
2103
|
+
offset: zod_default.number().int().nonnegative().default(0),
|
|
2104
|
+
configId: zod_default.string().uuid().optional(),
|
|
2105
|
+
variantId: zod_default.string().uuid().optional(),
|
|
2106
|
+
environmentId: zod_default.string().uuid().optional(),
|
|
2107
|
+
providerConfigId: zod_default.string().uuid().optional(),
|
|
2108
|
+
provider: zod_default.string().optional(),
|
|
2109
|
+
model: zod_default.string().optional(),
|
|
2110
|
+
startDate: zod_default.date().optional(),
|
|
2111
|
+
endDate: zod_default.date().optional(),
|
|
2112
|
+
tags: zod_default.record(zod_default.string(), zod_default.array(zod_default.string())).optional()
|
|
2113
|
+
});
|
|
2114
|
+
const dateRangeSchema = zod_default.object({
|
|
2115
|
+
startDate: zod_default.date(),
|
|
2116
|
+
endDate: zod_default.date(),
|
|
2117
|
+
configId: zod_default.string().uuid().optional(),
|
|
2118
|
+
variantId: zod_default.string().uuid().optional(),
|
|
2119
|
+
environmentId: zod_default.string().uuid().optional(),
|
|
2120
|
+
tags: zod_default.record(zod_default.string(), zod_default.array(zod_default.string())).optional()
|
|
2121
|
+
});
|
|
2122
|
+
const COST_SUMMARY_GROUP_BY = [
|
|
2123
|
+
"day",
|
|
2124
|
+
"hour",
|
|
2125
|
+
"model",
|
|
2126
|
+
"provider",
|
|
2127
|
+
"endpoint",
|
|
2128
|
+
"tags"
|
|
2129
|
+
];
|
|
2130
|
+
const costSummarySchema = zod_default.object({
|
|
2131
|
+
startDate: zod_default.date(),
|
|
2132
|
+
endDate: zod_default.date(),
|
|
2133
|
+
configId: zod_default.string().uuid().optional(),
|
|
2134
|
+
variantId: zod_default.string().uuid().optional(),
|
|
2135
|
+
environmentId: zod_default.string().uuid().optional(),
|
|
2136
|
+
tags: zod_default.record(zod_default.string(), zod_default.array(zod_default.string())).optional(),
|
|
2137
|
+
groupBy: zod_default.enum(COST_SUMMARY_GROUP_BY).optional(),
|
|
2138
|
+
tagKeys: zod_default.array(zod_default.string()).optional()
|
|
2139
|
+
});
|
|
2140
|
+
const upsertTraceSchema = zod_default.object({
|
|
2141
|
+
traceId: zod_default.string(),
|
|
2142
|
+
name: zod_default.string().nullable().optional(),
|
|
2143
|
+
sessionId: zod_default.string().nullable().optional(),
|
|
2144
|
+
userId: zod_default.string().nullable().optional(),
|
|
2145
|
+
status: zod_default.enum([
|
|
2146
|
+
"unset",
|
|
2147
|
+
"ok",
|
|
2148
|
+
"error"
|
|
2149
|
+
]).default("unset"),
|
|
2150
|
+
startTime: zod_default.date(),
|
|
2151
|
+
endTime: zod_default.date().nullable().optional(),
|
|
2152
|
+
durationMs: zod_default.number().int().nullable().optional(),
|
|
2153
|
+
spanCount: zod_default.number().int().default(1),
|
|
2154
|
+
totalInputTokens: zod_default.number().int().default(0),
|
|
2155
|
+
totalOutputTokens: zod_default.number().int().default(0),
|
|
2156
|
+
totalTokens: zod_default.number().int().default(0),
|
|
2157
|
+
totalCost: zod_default.number().int().default(0),
|
|
2158
|
+
tags: zod_default.record(zod_default.string(), zod_default.string()).default({}),
|
|
2159
|
+
metadata: zod_default.record(zod_default.string(), zod_default.unknown()).default({})
|
|
2160
|
+
});
|
|
2161
|
+
const insertSpanSchema = zod_default.object({
|
|
2162
|
+
traceId: zod_default.string(),
|
|
2163
|
+
spanId: zod_default.string(),
|
|
2164
|
+
parentSpanId: zod_default.string().nullable().optional(),
|
|
2165
|
+
name: zod_default.string(),
|
|
2166
|
+
kind: zod_default.number().int().default(1),
|
|
2167
|
+
status: zod_default.number().int().default(0),
|
|
2168
|
+
statusMessage: zod_default.string().nullable().optional(),
|
|
2169
|
+
startTime: zod_default.date(),
|
|
2170
|
+
endTime: zod_default.date().nullable().optional(),
|
|
2171
|
+
durationMs: zod_default.number().int().nullable().optional(),
|
|
2172
|
+
provider: zod_default.string().nullable().optional(),
|
|
2173
|
+
model: zod_default.string().nullable().optional(),
|
|
2174
|
+
promptTokens: zod_default.number().int().default(0),
|
|
2175
|
+
completionTokens: zod_default.number().int().default(0),
|
|
2176
|
+
totalTokens: zod_default.number().int().default(0),
|
|
2177
|
+
cost: zod_default.number().int().default(0),
|
|
2178
|
+
configId: zod_default.string().uuid().nullable().optional(),
|
|
2179
|
+
variantId: zod_default.string().uuid().nullable().optional(),
|
|
2180
|
+
environmentId: zod_default.string().uuid().nullable().optional(),
|
|
2181
|
+
providerConfigId: zod_default.string().uuid().nullable().optional(),
|
|
2182
|
+
requestId: zod_default.string().uuid().nullable().optional(),
|
|
2183
|
+
source: zod_default.enum([
|
|
2184
|
+
"gateway",
|
|
2185
|
+
"otlp",
|
|
2186
|
+
"langsmith"
|
|
2187
|
+
]).default("gateway"),
|
|
2188
|
+
input: zod_default.unknown().nullable().optional(),
|
|
2189
|
+
output: zod_default.unknown().nullable().optional(),
|
|
2190
|
+
attributes: zod_default.record(zod_default.string(), zod_default.unknown()).default({})
|
|
2191
|
+
});
|
|
2192
|
+
const insertSpanEventSchema = zod_default.object({
|
|
2193
|
+
traceId: zod_default.string(),
|
|
2194
|
+
spanId: zod_default.string(),
|
|
2195
|
+
name: zod_default.string(),
|
|
2196
|
+
timestamp: zod_default.date(),
|
|
2197
|
+
attributes: zod_default.record(zod_default.string(), zod_default.unknown()).default({})
|
|
2198
|
+
});
|
|
2199
|
+
const listTracesSchema = zod_default.object({
|
|
2200
|
+
limit: zod_default.number().int().positive().max(1e3).default(50),
|
|
2201
|
+
offset: zod_default.number().int().nonnegative().default(0),
|
|
2202
|
+
sessionId: zod_default.string().optional(),
|
|
2203
|
+
userId: zod_default.string().optional(),
|
|
2204
|
+
status: zod_default.enum([
|
|
2205
|
+
"unset",
|
|
2206
|
+
"ok",
|
|
2207
|
+
"error"
|
|
2208
|
+
]).optional(),
|
|
2209
|
+
name: zod_default.string().optional(),
|
|
2210
|
+
startDate: zod_default.date().optional(),
|
|
2211
|
+
endDate: zod_default.date().optional(),
|
|
2212
|
+
tags: zod_default.record(zod_default.string(), zod_default.array(zod_default.string())).optional()
|
|
2213
|
+
});
|
|
2214
|
+
const traceStatsSchema = zod_default.object({
|
|
2215
|
+
startDate: zod_default.date(),
|
|
2216
|
+
endDate: zod_default.date(),
|
|
2217
|
+
sessionId: zod_default.string().optional(),
|
|
2218
|
+
userId: zod_default.string().optional()
|
|
2219
|
+
});
|
|
2220
|
+
const col = (name) => sql.ref(name);
|
|
2221
|
+
function createLLMRequestsStore(db) {
|
|
2222
|
+
return {
|
|
2223
|
+
batchInsertRequests: async (requests) => {
|
|
2224
|
+
if (requests.length === 0) return { count: 0 };
|
|
2225
|
+
const validatedRequests = await Promise.all(requests.map(async (req) => {
|
|
2226
|
+
const result = await insertLLMRequestSchema.safeParseAsync(req);
|
|
2227
|
+
if (!result.success) throw new LLMOpsError(`Invalid request data: ${result.error.message}`);
|
|
2228
|
+
return result.data;
|
|
2229
|
+
}));
|
|
2591
2230
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
2592
|
-
const values =
|
|
2231
|
+
const values = validatedRequests.map((req) => ({
|
|
2593
2232
|
id: randomUUID(),
|
|
2594
|
-
|
|
2595
|
-
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
|
|
2602
|
-
|
|
2603
|
-
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
|
-
|
|
2607
|
-
|
|
2608
|
-
|
|
2609
|
-
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
2233
|
+
requestId: req.requestId,
|
|
2234
|
+
configId: req.configId ?? null,
|
|
2235
|
+
variantId: req.variantId ?? null,
|
|
2236
|
+
environmentId: req.environmentId ?? null,
|
|
2237
|
+
providerConfigId: req.providerConfigId ?? null,
|
|
2238
|
+
provider: req.provider,
|
|
2239
|
+
model: req.model,
|
|
2240
|
+
promptTokens: req.promptTokens,
|
|
2241
|
+
completionTokens: req.completionTokens,
|
|
2242
|
+
totalTokens: req.totalTokens,
|
|
2243
|
+
cachedTokens: req.cachedTokens,
|
|
2244
|
+
cacheCreationTokens: req.cacheCreationTokens,
|
|
2245
|
+
cost: req.cost,
|
|
2246
|
+
cacheSavings: req.cacheSavings,
|
|
2247
|
+
inputCost: req.inputCost,
|
|
2248
|
+
outputCost: req.outputCost,
|
|
2249
|
+
endpoint: req.endpoint,
|
|
2250
|
+
statusCode: req.statusCode,
|
|
2251
|
+
latencyMs: req.latencyMs,
|
|
2252
|
+
isStreaming: req.isStreaming,
|
|
2253
|
+
userId: req.userId ?? null,
|
|
2254
|
+
tags: JSON.stringify(req.tags),
|
|
2255
|
+
guardrailResults: req.guardrailResults ? JSON.stringify(req.guardrailResults) : null,
|
|
2256
|
+
traceId: req.traceId ?? null,
|
|
2257
|
+
spanId: req.spanId ?? null,
|
|
2258
|
+
parentSpanId: req.parentSpanId ?? null,
|
|
2259
|
+
sessionId: req.sessionId ?? null,
|
|
2619
2260
|
createdAt: now,
|
|
2620
2261
|
updatedAt: now
|
|
2621
2262
|
}));
|
|
2622
|
-
await db.insertInto("
|
|
2623
|
-
return { count: values.length };
|
|
2624
|
-
},
|
|
2625
|
-
batchInsertSpanEvents: async (events) => {
|
|
2626
|
-
if (events.length === 0) return { count: 0 };
|
|
2627
|
-
const validatedEvents = [];
|
|
2628
|
-
for (const event of events) {
|
|
2629
|
-
const result = await insertSpanEventSchema.safeParseAsync(event);
|
|
2630
|
-
if (!result.success) {
|
|
2631
|
-
logger.warn(`[batchInsertSpanEvents] Skipping invalid event: ${result.error.message}`);
|
|
2632
|
-
continue;
|
|
2633
|
-
}
|
|
2634
|
-
validatedEvents.push(result.data);
|
|
2635
|
-
}
|
|
2636
|
-
if (validatedEvents.length === 0) return { count: 0 };
|
|
2637
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
2638
|
-
const values = validatedEvents.map((event) => ({
|
|
2639
|
-
id: randomUUID(),
|
|
2640
|
-
traceId: event.traceId,
|
|
2641
|
-
spanId: event.spanId,
|
|
2642
|
-
name: event.name,
|
|
2643
|
-
timestamp: event.timestamp.toISOString(),
|
|
2644
|
-
attributes: JSON.stringify(event.attributes),
|
|
2645
|
-
createdAt: now
|
|
2646
|
-
}));
|
|
2647
|
-
await db.insertInto("span_events").values(values).execute();
|
|
2263
|
+
await db.insertInto("llm_requests").values(values).execute();
|
|
2648
2264
|
return { count: values.length };
|
|
2649
2265
|
},
|
|
2650
|
-
|
|
2651
|
-
const result = await
|
|
2266
|
+
insertRequest: async (request) => {
|
|
2267
|
+
const result = await insertLLMRequestSchema.safeParseAsync(request);
|
|
2268
|
+
if (!result.success) throw new LLMOpsError(`Invalid request data: ${result.error.message}`);
|
|
2269
|
+
const req = result.data;
|
|
2270
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
2271
|
+
return db.insertInto("llm_requests").values({
|
|
2272
|
+
id: randomUUID(),
|
|
2273
|
+
requestId: req.requestId,
|
|
2274
|
+
configId: req.configId ?? null,
|
|
2275
|
+
variantId: req.variantId ?? null,
|
|
2276
|
+
environmentId: req.environmentId ?? null,
|
|
2277
|
+
providerConfigId: req.providerConfigId ?? null,
|
|
2278
|
+
provider: req.provider,
|
|
2279
|
+
model: req.model,
|
|
2280
|
+
promptTokens: req.promptTokens,
|
|
2281
|
+
completionTokens: req.completionTokens,
|
|
2282
|
+
totalTokens: req.totalTokens,
|
|
2283
|
+
cachedTokens: req.cachedTokens,
|
|
2284
|
+
cacheCreationTokens: req.cacheCreationTokens,
|
|
2285
|
+
cost: req.cost,
|
|
2286
|
+
cacheSavings: req.cacheSavings,
|
|
2287
|
+
inputCost: req.inputCost,
|
|
2288
|
+
outputCost: req.outputCost,
|
|
2289
|
+
endpoint: req.endpoint,
|
|
2290
|
+
statusCode: req.statusCode,
|
|
2291
|
+
latencyMs: req.latencyMs,
|
|
2292
|
+
isStreaming: req.isStreaming,
|
|
2293
|
+
userId: req.userId ?? null,
|
|
2294
|
+
tags: JSON.stringify(req.tags),
|
|
2295
|
+
guardrailResults: req.guardrailResults ? JSON.stringify(req.guardrailResults) : null,
|
|
2296
|
+
traceId: req.traceId ?? null,
|
|
2297
|
+
spanId: req.spanId ?? null,
|
|
2298
|
+
parentSpanId: req.parentSpanId ?? null,
|
|
2299
|
+
sessionId: req.sessionId ?? null,
|
|
2300
|
+
createdAt: now,
|
|
2301
|
+
updatedAt: now
|
|
2302
|
+
}).returningAll().executeTakeFirst();
|
|
2303
|
+
},
|
|
2304
|
+
listRequests: async (params) => {
|
|
2305
|
+
const result = await listRequestsSchema.safeParseAsync(params || {});
|
|
2652
2306
|
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2653
|
-
const { limit, offset,
|
|
2654
|
-
let baseQuery = db.selectFrom("
|
|
2655
|
-
if (
|
|
2656
|
-
if (
|
|
2657
|
-
if (
|
|
2658
|
-
if (
|
|
2659
|
-
if (
|
|
2660
|
-
if (
|
|
2307
|
+
const { limit, offset, configId, variantId, environmentId, providerConfigId, provider, model, startDate, endDate, tags } = result.data;
|
|
2308
|
+
let baseQuery = db.selectFrom("llm_requests");
|
|
2309
|
+
if (configId) baseQuery = baseQuery.where("configId", "=", configId);
|
|
2310
|
+
if (variantId) baseQuery = baseQuery.where("variantId", "=", variantId);
|
|
2311
|
+
if (environmentId) baseQuery = baseQuery.where("environmentId", "=", environmentId);
|
|
2312
|
+
if (providerConfigId) baseQuery = baseQuery.where("providerConfigId", "=", providerConfigId);
|
|
2313
|
+
if (provider) baseQuery = baseQuery.where("provider", "=", provider);
|
|
2314
|
+
if (model) baseQuery = baseQuery.where("model", "=", model);
|
|
2315
|
+
if (startDate) baseQuery = baseQuery.where(sql`${col("createdAt")} >= ${startDate.toISOString()}`);
|
|
2316
|
+
if (endDate) baseQuery = baseQuery.where(sql`${col("createdAt")} <= ${endDate.toISOString()}`);
|
|
2661
2317
|
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
2662
2318
|
if (values.length === 0) continue;
|
|
2663
2319
|
if (values.length === 1) baseQuery = baseQuery.where(sql`${col("tags")}->>${key} = ${values[0]}`);
|
|
@@ -2669,426 +2325,387 @@ const createTracesDataLayer = (db) => {
|
|
|
2669
2325
|
const countResult = await baseQuery.select(sql`COUNT(*)`.as("total")).executeTakeFirst();
|
|
2670
2326
|
const total = Number(countResult?.total ?? 0);
|
|
2671
2327
|
return {
|
|
2672
|
-
data: await baseQuery.selectAll().orderBy("
|
|
2328
|
+
data: await baseQuery.selectAll().orderBy("createdAt", "desc").limit(limit).offset(offset).execute(),
|
|
2673
2329
|
total,
|
|
2674
2330
|
limit,
|
|
2675
2331
|
offset
|
|
2676
2332
|
};
|
|
2677
2333
|
},
|
|
2678
|
-
|
|
2679
|
-
|
|
2680
|
-
if (!trace) return void 0;
|
|
2681
|
-
return {
|
|
2682
|
-
trace,
|
|
2683
|
-
spans: await db.selectFrom("spans").selectAll().where("traceId", "=", traceId).orderBy("startTime", "asc").execute(),
|
|
2684
|
-
events: await db.selectFrom("span_events").selectAll().where("traceId", "=", traceId).orderBy("timestamp", "asc").execute()
|
|
2685
|
-
};
|
|
2334
|
+
getRequestByRequestId: async (requestId) => {
|
|
2335
|
+
return db.selectFrom("llm_requests").selectAll().where("requestId", "=", requestId).executeTakeFirst();
|
|
2686
2336
|
},
|
|
2687
|
-
|
|
2688
|
-
const result = await
|
|
2337
|
+
getTotalCost: async (params) => {
|
|
2338
|
+
const result = await dateRangeSchema.safeParseAsync(params);
|
|
2689
2339
|
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2690
|
-
const { startDate, endDate,
|
|
2691
|
-
let query = db.selectFrom("
|
|
2692
|
-
sql`
|
|
2693
|
-
sql`COALESCE(
|
|
2694
|
-
sql`
|
|
2695
|
-
sql`COALESCE(SUM(${col("
|
|
2340
|
+
const { startDate, endDate, configId, variantId, environmentId, tags } = result.data;
|
|
2341
|
+
let query = db.selectFrom("llm_requests").select([
|
|
2342
|
+
sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2343
|
+
sql`COALESCE(SUM(${col("inputCost")}), 0)`.as("totalInputCost"),
|
|
2344
|
+
sql`COALESCE(SUM(${col("outputCost")}), 0)`.as("totalOutputCost"),
|
|
2345
|
+
sql`COALESCE(SUM(${col("promptTokens")}), 0)`.as("totalPromptTokens"),
|
|
2346
|
+
sql`COALESCE(SUM(${col("completionTokens")}), 0)`.as("totalCompletionTokens"),
|
|
2696
2347
|
sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
|
|
2697
|
-
sql`COALESCE(SUM(${col("
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2348
|
+
sql`COALESCE(SUM(${col("cachedTokens")}), 0)`.as("totalCachedTokens"),
|
|
2349
|
+
sql`COALESCE(SUM(${col("cacheSavings")}), 0)`.as("totalCacheSavings"),
|
|
2350
|
+
sql`COUNT(*)`.as("requestCount")
|
|
2351
|
+
]).where(sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col("createdAt")} <= ${endDate.toISOString()}`);
|
|
2352
|
+
if (configId) query = query.where("configId", "=", configId);
|
|
2353
|
+
if (variantId) query = query.where("variantId", "=", variantId);
|
|
2354
|
+
if (environmentId) query = query.where("environmentId", "=", environmentId);
|
|
2355
|
+
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
2356
|
+
if (values.length === 0) continue;
|
|
2357
|
+
if (values.length === 1) query = query.where(sql`${col("tags")}->>${key} = ${values[0]}`);
|
|
2358
|
+
else {
|
|
2359
|
+
const valueList = sql.join(values.map((v) => sql`${v}`));
|
|
2360
|
+
query = query.where(sql`${col("tags")}->>${key} IN (${valueList})`);
|
|
2361
|
+
}
|
|
2362
|
+
}
|
|
2701
2363
|
return query.executeTakeFirst();
|
|
2702
|
-
}
|
|
2703
|
-
};
|
|
2704
|
-
};
|
|
2705
|
-
|
|
2706
|
-
//#endregion
|
|
2707
|
-
//#region src/datalayer/workspaceSettings.ts
|
|
2708
|
-
const updateWorkspaceSettings = zod_default.object({
|
|
2709
|
-
name: zod_default.string().nullable().optional(),
|
|
2710
|
-
setupComplete: zod_default.boolean().optional(),
|
|
2711
|
-
superAdminId: zod_default.string().nullable().optional()
|
|
2712
|
-
});
|
|
2713
|
-
const createWorkspaceSettingsDataLayer = (db) => {
|
|
2714
|
-
return {
|
|
2715
|
-
getWorkspaceSettings: async () => {
|
|
2716
|
-
let settings = await db.selectFrom("workspace_settings").selectAll().executeTakeFirst();
|
|
2717
|
-
if (!settings) settings = await db.insertInto("workspace_settings").values({
|
|
2718
|
-
id: randomUUID(),
|
|
2719
|
-
name: null,
|
|
2720
|
-
setupComplete: false,
|
|
2721
|
-
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2722
|
-
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2723
|
-
}).returningAll().executeTakeFirst();
|
|
2724
|
-
return settings;
|
|
2725
2364
|
},
|
|
2726
|
-
|
|
2727
|
-
const
|
|
2728
|
-
if (!
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
2735
|
-
|
|
2736
|
-
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
|
|
2740
|
-
if (value.data.superAdminId !== void 0) updateData.superAdminId = value.data.superAdminId ?? null;
|
|
2741
|
-
return db.updateTable("workspace_settings").set(updateData).where("id", "=", settings.id).returningAll().executeTakeFirst();
|
|
2365
|
+
getCostByModel: async (params) => {
|
|
2366
|
+
const result = await dateRangeSchema.safeParseAsync(params);
|
|
2367
|
+
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2368
|
+
const { startDate, endDate } = result.data;
|
|
2369
|
+
return db.selectFrom("llm_requests").select([
|
|
2370
|
+
"provider",
|
|
2371
|
+
"model",
|
|
2372
|
+
sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2373
|
+
sql`COALESCE(SUM(${col("inputCost")}), 0)`.as("totalInputCost"),
|
|
2374
|
+
sql`COALESCE(SUM(${col("outputCost")}), 0)`.as("totalOutputCost"),
|
|
2375
|
+
sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
|
|
2376
|
+
sql`COUNT(*)`.as("requestCount"),
|
|
2377
|
+
sql`AVG(${col("latencyMs")})`.as("avgLatencyMs")
|
|
2378
|
+
]).where(sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col("createdAt")} <= ${endDate.toISOString()}`).groupBy(["provider", "model"]).orderBy(sql`SUM(${col("cost")})`, "desc").execute();
|
|
2742
2379
|
},
|
|
2743
|
-
|
|
2744
|
-
|
|
2380
|
+
getCostByProvider: async (params) => {
|
|
2381
|
+
const result = await dateRangeSchema.safeParseAsync(params);
|
|
2382
|
+
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2383
|
+
const { startDate, endDate } = result.data;
|
|
2384
|
+
return db.selectFrom("llm_requests").select([
|
|
2385
|
+
"provider",
|
|
2386
|
+
sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2387
|
+
sql`COALESCE(SUM(${col("inputCost")}), 0)`.as("totalInputCost"),
|
|
2388
|
+
sql`COALESCE(SUM(${col("outputCost")}), 0)`.as("totalOutputCost"),
|
|
2389
|
+
sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
|
|
2390
|
+
sql`COUNT(*)`.as("requestCount"),
|
|
2391
|
+
sql`AVG(${col("latencyMs")})`.as("avgLatencyMs")
|
|
2392
|
+
]).where(sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col("createdAt")} <= ${endDate.toISOString()}`).groupBy("provider").orderBy(sql`SUM(${col("cost")})`, "desc").execute();
|
|
2745
2393
|
},
|
|
2746
|
-
|
|
2747
|
-
|
|
2748
|
-
if (
|
|
2749
|
-
|
|
2750
|
-
|
|
2751
|
-
|
|
2752
|
-
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
|
|
2756
|
-
|
|
2757
|
-
|
|
2758
|
-
return true;
|
|
2759
|
-
}
|
|
2760
|
-
await db.updateTable("workspace_settings").set({
|
|
2761
|
-
superAdminId: userId,
|
|
2762
|
-
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2763
|
-
}).where("id", "=", settings.id).execute();
|
|
2764
|
-
return true;
|
|
2394
|
+
getDailyCosts: async (params) => {
|
|
2395
|
+
const result = await dateRangeSchema.safeParseAsync(params);
|
|
2396
|
+
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2397
|
+
const { startDate, endDate } = result.data;
|
|
2398
|
+
return db.selectFrom("llm_requests").select([
|
|
2399
|
+
sql`DATE(${col("createdAt")})`.as("date"),
|
|
2400
|
+
sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2401
|
+
sql`COALESCE(SUM(${col("inputCost")}), 0)`.as("totalInputCost"),
|
|
2402
|
+
sql`COALESCE(SUM(${col("outputCost")}), 0)`.as("totalOutputCost"),
|
|
2403
|
+
sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
|
|
2404
|
+
sql`COUNT(*)`.as("requestCount")
|
|
2405
|
+
]).where(sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col("createdAt")} <= ${endDate.toISOString()}`).groupBy(sql`DATE(${col("createdAt")})`).orderBy(sql`DATE(${col("createdAt")})`, "asc").execute();
|
|
2765
2406
|
},
|
|
2766
|
-
|
|
2767
|
-
|
|
2768
|
-
|
|
2769
|
-
}
|
|
2770
|
-
|
|
2407
|
+
getCostSummary: async (params) => {
|
|
2408
|
+
const result = await costSummarySchema.safeParseAsync(params);
|
|
2409
|
+
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2410
|
+
const { startDate, endDate, groupBy, configId, variantId, environmentId, tags, tagKeys } = result.data;
|
|
2411
|
+
let baseQuery = db.selectFrom("llm_requests").where(sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col("createdAt")} <= ${endDate.toISOString()}`);
|
|
2412
|
+
if (configId) baseQuery = baseQuery.where("configId", "=", configId);
|
|
2413
|
+
if (variantId) baseQuery = baseQuery.where("variantId", "=", variantId);
|
|
2414
|
+
if (environmentId) baseQuery = baseQuery.where("environmentId", "=", environmentId);
|
|
2415
|
+
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
2416
|
+
if (values.length === 0) continue;
|
|
2417
|
+
if (values.length === 1) baseQuery = baseQuery.where(sql`${col("tags")}->>${key} = ${values[0]}`);
|
|
2418
|
+
else {
|
|
2419
|
+
const valueList = sql.join(values.map((v) => sql`${v}`));
|
|
2420
|
+
baseQuery = baseQuery.where(sql`${col("tags")}->>${key} IN (${valueList})`);
|
|
2421
|
+
}
|
|
2422
|
+
}
|
|
2423
|
+
switch (groupBy) {
|
|
2424
|
+
case "day": return baseQuery.select([
|
|
2425
|
+
sql`DATE(${col("createdAt")})`.as("groupKey"),
|
|
2426
|
+
sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2427
|
+
sql`COUNT(*)`.as("requestCount"),
|
|
2428
|
+
sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens")
|
|
2429
|
+
]).groupBy(sql`DATE(${col("createdAt")})`).orderBy(sql`DATE(${col("createdAt")})`, "asc").execute();
|
|
2430
|
+
case "hour": return baseQuery.select([
|
|
2431
|
+
sql`DATE_TRUNC('hour', ${col("createdAt")})`.as("groupKey"),
|
|
2432
|
+
sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2433
|
+
sql`COUNT(*)`.as("requestCount"),
|
|
2434
|
+
sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens")
|
|
2435
|
+
]).groupBy(sql`DATE_TRUNC('hour', ${col("createdAt")})`).orderBy(sql`DATE_TRUNC('hour', ${col("createdAt")})`, "asc").execute();
|
|
2436
|
+
case "model": return baseQuery.select([
|
|
2437
|
+
sql`${col("provider")} || '/' || ${col("model")}`.as("groupKey"),
|
|
2438
|
+
sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2439
|
+
sql`COUNT(*)`.as("requestCount")
|
|
2440
|
+
]).groupBy(["provider", "model"]).orderBy(sql`SUM(${col("cost")})`, "desc").execute();
|
|
2441
|
+
case "provider": return baseQuery.select([
|
|
2442
|
+
sql`${col("provider")}`.as("groupKey"),
|
|
2443
|
+
sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2444
|
+
sql`COUNT(*)`.as("requestCount")
|
|
2445
|
+
]).groupBy("provider").orderBy(sql`SUM(${col("cost")})`, "desc").execute();
|
|
2446
|
+
case "endpoint": return baseQuery.select([
|
|
2447
|
+
sql`COALESCE(${col("endpoint")}, 'unknown')`.as("groupKey"),
|
|
2448
|
+
sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2449
|
+
sql`COUNT(*)`.as("requestCount")
|
|
2450
|
+
]).groupBy("endpoint").orderBy(sql`SUM(${col("cost")})`, "desc").execute();
|
|
2451
|
+
case "tags": {
|
|
2452
|
+
const conditions = [sql`${col("createdAt")} >= ${startDate.toISOString()}`, sql`${col("createdAt")} <= ${endDate.toISOString()}`];
|
|
2453
|
+
if (configId) conditions.push(sql`${col("configId")} = ${configId}`);
|
|
2454
|
+
if (variantId) conditions.push(sql`${col("variantId")} = ${variantId}`);
|
|
2455
|
+
if (environmentId) conditions.push(sql`${col("environmentId")} = ${environmentId}`);
|
|
2456
|
+
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
2457
|
+
if (values.length === 0) continue;
|
|
2458
|
+
if (values.length === 1) conditions.push(sql`${col("tags")}->>${key} = ${values[0]}`);
|
|
2459
|
+
else {
|
|
2460
|
+
const valueList = sql.join(values.map((v) => sql`${v}`));
|
|
2461
|
+
conditions.push(sql`${col("tags")}->>${key} IN (${valueList})`);
|
|
2462
|
+
}
|
|
2463
|
+
}
|
|
2464
|
+
if (tagKeys && tagKeys.length > 0) {
|
|
2465
|
+
const tagKeyList = sql.join(tagKeys.map((k) => sql`${k}`), sql`, `);
|
|
2466
|
+
conditions.push(sql`t.key IN (${tagKeyList})`);
|
|
2467
|
+
}
|
|
2468
|
+
const whereClause = sql.join(conditions, sql` AND `);
|
|
2469
|
+
return (await sql`
|
|
2470
|
+
SELECT t.key || ':' || t.value as "groupKey",
|
|
2471
|
+
COALESCE(SUM(${col("cost")}), 0) as "totalCost",
|
|
2472
|
+
COUNT(*) as "requestCount"
|
|
2473
|
+
FROM "llm_requests", jsonb_each_text(${col("tags")}) t
|
|
2474
|
+
WHERE ${whereClause}
|
|
2475
|
+
GROUP BY t.key, t.value
|
|
2476
|
+
ORDER BY SUM(${col("cost")}) DESC
|
|
2477
|
+
`.execute(db)).rows;
|
|
2478
|
+
}
|
|
2479
|
+
default: return baseQuery.select([
|
|
2480
|
+
sql`'total'`.as("groupKey"),
|
|
2481
|
+
sql`COALESCE(SUM(${col("cost")}), 0)`.as("totalCost"),
|
|
2482
|
+
sql`COUNT(*)`.as("requestCount")
|
|
2483
|
+
]).execute();
|
|
2771
2484
|
}
|
|
2772
2485
|
},
|
|
2773
|
-
|
|
2774
|
-
|
|
2775
|
-
if (!
|
|
2776
|
-
|
|
2777
|
-
|
|
2778
|
-
|
|
2779
|
-
|
|
2780
|
-
|
|
2781
|
-
|
|
2782
|
-
|
|
2783
|
-
|
|
2784
|
-
|
|
2785
|
-
|
|
2486
|
+
getRequestStats: async (params) => {
|
|
2487
|
+
const result = await dateRangeSchema.safeParseAsync(params);
|
|
2488
|
+
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2489
|
+
const { startDate, endDate, configId, variantId, environmentId, tags } = result.data;
|
|
2490
|
+
let query = db.selectFrom("llm_requests").select([
|
|
2491
|
+
sql`COUNT(*)`.as("totalRequests"),
|
|
2492
|
+
sql`COUNT(CASE WHEN ${col("statusCode")} >= 200 AND ${col("statusCode")} < 300 THEN 1 END)`.as("successfulRequests"),
|
|
2493
|
+
sql`COUNT(CASE WHEN ${col("statusCode")} >= 400 THEN 1 END)`.as("failedRequests"),
|
|
2494
|
+
sql`COUNT(CASE WHEN ${col("isStreaming")} = true THEN 1 END)`.as("streamingRequests"),
|
|
2495
|
+
sql`AVG(${col("latencyMs")})`.as("avgLatencyMs"),
|
|
2496
|
+
sql`MAX(${col("latencyMs")})`.as("maxLatencyMs"),
|
|
2497
|
+
sql`MIN(${col("latencyMs")})`.as("minLatencyMs")
|
|
2498
|
+
]).where(sql`${col("createdAt")} >= ${startDate.toISOString()}`).where(sql`${col("createdAt")} <= ${endDate.toISOString()}`);
|
|
2499
|
+
if (configId) query = query.where("configId", "=", configId);
|
|
2500
|
+
if (variantId) query = query.where("variantId", "=", variantId);
|
|
2501
|
+
if (environmentId) query = query.where("environmentId", "=", environmentId);
|
|
2502
|
+
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
2503
|
+
if (values.length === 0) continue;
|
|
2504
|
+
if (values.length === 1) query = query.where(sql`${col("tags")}->>${key} = ${values[0]}`);
|
|
2505
|
+
else {
|
|
2506
|
+
const valueList = sql.join(values.map((v) => sql`${v}`));
|
|
2507
|
+
query = query.where(sql`${col("tags")}->>${key} IN (${valueList})`);
|
|
2508
|
+
}
|
|
2509
|
+
}
|
|
2510
|
+
return query.executeTakeFirst();
|
|
2511
|
+
},
|
|
2512
|
+
getDistinctTags: async () => {
|
|
2513
|
+
return (await sql`
|
|
2514
|
+
SELECT DISTINCT key, value
|
|
2515
|
+
FROM llm_requests, jsonb_each_text(tags) AS t(key, value)
|
|
2516
|
+
WHERE tags != '{}'::jsonb
|
|
2517
|
+
ORDER BY key, value
|
|
2518
|
+
`.execute(db)).rows;
|
|
2786
2519
|
}
|
|
2787
2520
|
};
|
|
2788
|
-
};
|
|
2789
|
-
|
|
2790
|
-
//#endregion
|
|
2791
|
-
//#region src/datalayer/create.ts
|
|
2792
|
-
/**
|
|
2793
|
-
* Create all datalayers from a Kysely database instance.
|
|
2794
|
-
* Returns a flat object with all datalayer methods spread together.
|
|
2795
|
-
*/
|
|
2796
|
-
function createDataLayer(db) {
|
|
2797
|
-
return {
|
|
2798
|
-
...createDatasetsDataLayer(db),
|
|
2799
|
-
...createGuardrailConfigsDataLayer(db),
|
|
2800
|
-
...createLLMRequestsDataLayer(db),
|
|
2801
|
-
...createPlaygroundDataLayer(db),
|
|
2802
|
-
...createPlaygroundResultsDataLayer(db),
|
|
2803
|
-
...createPlaygroundRunsDataLayer(db),
|
|
2804
|
-
...createProviderConfigsDataLayer(db),
|
|
2805
|
-
...createProviderGuardrailOverridesDataLayer(db),
|
|
2806
|
-
...createTracesDataLayer(db),
|
|
2807
|
-
...createWorkspaceSettingsDataLayer(db)
|
|
2808
|
-
};
|
|
2809
|
-
}
|
|
2810
|
-
|
|
2811
|
-
//#endregion
|
|
2812
|
-
//#region src/pricing/calculator.ts
|
|
2813
|
-
/**
|
|
2814
|
-
* Calculate the cost of an LLM request in micro-dollars
|
|
2815
|
-
*
|
|
2816
|
-
* Micro-dollars are used to avoid floating-point precision issues:
|
|
2817
|
-
* - 1 dollar = 1,000,000 micro-dollars
|
|
2818
|
-
* - $0.001 = 1,000 micro-dollars
|
|
2819
|
-
* - $0.000001 = 1 micro-dollar
|
|
2820
|
-
*
|
|
2821
|
-
* @param usage - Token usage data from the LLM response
|
|
2822
|
-
* @param pricing - Model pricing information
|
|
2823
|
-
* @returns Cost breakdown in micro-dollars
|
|
2824
|
-
*
|
|
2825
|
-
* @example
|
|
2826
|
-
* ```typescript
|
|
2827
|
-
* const usage = { promptTokens: 1000, completionTokens: 500 };
|
|
2828
|
-
* const pricing = { inputCostPer1M: 2.5, outputCostPer1M: 10.0 };
|
|
2829
|
-
* const cost = calculateCost(usage, pricing);
|
|
2830
|
-
* // cost = { inputCost: 2500, outputCost: 5000, totalCost: 7500 }
|
|
2831
|
-
* // In dollars: $0.0025 input + $0.005 output = $0.0075 total
|
|
2832
|
-
* ```
|
|
2833
|
-
*/
|
|
2834
|
-
function calculateCost(usage, pricing) {
|
|
2835
|
-
const inputCost = Math.round(usage.promptTokens * pricing.inputCostPer1M);
|
|
2836
|
-
const outputCost = Math.round(usage.completionTokens * pricing.outputCostPer1M);
|
|
2837
|
-
return {
|
|
2838
|
-
inputCost,
|
|
2839
|
-
outputCost,
|
|
2840
|
-
totalCost: inputCost + outputCost,
|
|
2841
|
-
cacheSavings: 0
|
|
2842
|
-
};
|
|
2843
|
-
}
|
|
2844
|
-
/**
|
|
2845
|
-
* Get default cache read rate as a fraction of input cost per provider.
|
|
2846
|
-
* Used when models.dev doesn't provide cache pricing.
|
|
2847
|
-
*/
|
|
2848
|
-
function getDefaultCacheReadRate(provider, inputCostPer1M) {
|
|
2849
|
-
switch (provider?.toLowerCase()) {
|
|
2850
|
-
case "anthropic": return inputCostPer1M * .1;
|
|
2851
|
-
case "openai":
|
|
2852
|
-
case "azure-openai": return inputCostPer1M * .5;
|
|
2853
|
-
case "google":
|
|
2854
|
-
case "gemini":
|
|
2855
|
-
case "vertex_ai": return inputCostPer1M * .25;
|
|
2856
|
-
default: return inputCostPer1M * .5;
|
|
2857
|
-
}
|
|
2858
|
-
}
|
|
2859
|
-
/**
|
|
2860
|
-
* Get default cache write/creation rate as a fraction of input cost per provider.
|
|
2861
|
-
* Used when models.dev doesn't provide cache pricing.
|
|
2862
|
-
*/
|
|
2863
|
-
function getDefaultCacheWriteRate(provider, inputCostPer1M) {
|
|
2864
|
-
switch (provider?.toLowerCase()) {
|
|
2865
|
-
case "anthropic": return inputCostPer1M * 1.25;
|
|
2866
|
-
default: return inputCostPer1M;
|
|
2867
|
-
}
|
|
2868
2521
|
}
|
|
2869
|
-
|
|
2870
|
-
* Calculate cache-aware cost of an LLM request in micro-dollars.
|
|
2871
|
-
*
|
|
2872
|
-
* Splits input tokens into uncached, cache-read, and cache-creation buckets,
|
|
2873
|
-
* each priced at different rates. Falls back to provider-specific multipliers
|
|
2874
|
-
* when models.dev doesn't provide cache pricing.
|
|
2875
|
-
*
|
|
2876
|
-
* @param usage - Token usage data (with cachedTokens and cacheCreationTokens)
|
|
2877
|
-
* @param pricing - Model pricing (may include cacheReadCostPer1M / cacheWriteCostPer1M)
|
|
2878
|
-
* @param provider - Provider name for fallback rate selection
|
|
2879
|
-
* @returns Cost breakdown in micro-dollars
|
|
2880
|
-
*/
|
|
2881
|
-
function calculateCacheAwareCost(usage, pricing, provider) {
|
|
2882
|
-
const cachedTokens = usage.cachedTokens ?? 0;
|
|
2883
|
-
const cacheCreationTokens = usage.cacheCreationTokens ?? 0;
|
|
2884
|
-
if (cachedTokens === 0 && cacheCreationTokens === 0) return calculateCost(usage, pricing);
|
|
2885
|
-
const cacheReadRate = pricing.cacheReadCostPer1M ?? getDefaultCacheReadRate(provider, pricing.inputCostPer1M);
|
|
2886
|
-
const cacheWriteRate = pricing.cacheWriteCostPer1M ?? getDefaultCacheWriteRate(provider, pricing.inputCostPer1M);
|
|
2887
|
-
const uncachedInputTokens = Math.max(0, usage.promptTokens - cachedTokens - cacheCreationTokens);
|
|
2888
|
-
const regularInputCost = Math.round(uncachedInputTokens * pricing.inputCostPer1M);
|
|
2889
|
-
const cacheReadCost = Math.round(cachedTokens * cacheReadRate);
|
|
2890
|
-
const cacheWriteCost = Math.round(cacheCreationTokens * cacheWriteRate);
|
|
2891
|
-
const outputCost = Math.round(usage.completionTokens * pricing.outputCostPer1M);
|
|
2892
|
-
const inputCost = regularInputCost + cacheReadCost + cacheWriteCost;
|
|
2522
|
+
function createTracesStore(db) {
|
|
2893
2523
|
return {
|
|
2894
|
-
|
|
2895
|
-
|
|
2896
|
-
|
|
2897
|
-
|
|
2898
|
-
|
|
2899
|
-
|
|
2900
|
-
|
|
2901
|
-
|
|
2902
|
-
|
|
2903
|
-
|
|
2904
|
-
|
|
2905
|
-
|
|
2906
|
-
|
|
2907
|
-
|
|
2908
|
-
|
|
2909
|
-
|
|
2910
|
-
|
|
2911
|
-
|
|
2912
|
-
|
|
2913
|
-
|
|
2914
|
-
|
|
2915
|
-
|
|
2916
|
-
|
|
2917
|
-
|
|
2918
|
-
|
|
2919
|
-
|
|
2920
|
-
|
|
2921
|
-
|
|
2922
|
-
|
|
2923
|
-
|
|
2924
|
-
|
|
2925
|
-
|
|
2926
|
-
|
|
2927
|
-
|
|
2928
|
-
|
|
2929
|
-
|
|
2930
|
-
|
|
2931
|
-
|
|
2932
|
-
|
|
2933
|
-
|
|
2934
|
-
|
|
2935
|
-
*
|
|
2936
|
-
|
|
2937
|
-
|
|
2938
|
-
|
|
2939
|
-
|
|
2940
|
-
|
|
2941
|
-
|
|
2942
|
-
|
|
2943
|
-
|
|
2944
|
-
|
|
2945
|
-
}
|
|
2946
|
-
|
|
2947
|
-
|
|
2948
|
-
|
|
2949
|
-
const
|
|
2950
|
-
|
|
2951
|
-
|
|
2952
|
-
|
|
2953
|
-
|
|
2954
|
-
|
|
2955
|
-
|
|
2956
|
-
function centsPerTokenToCostPer1M(centsPerToken) {
|
|
2957
|
-
return centsPerToken * 1e4;
|
|
2958
|
-
}
|
|
2959
|
-
/**
|
|
2960
|
-
* Pricing provider that fetches per-model data from the LLMOps Models API.
|
|
2961
|
-
*
|
|
2962
|
-
* Features:
|
|
2963
|
-
* - Per-model in-memory cache with configurable TTL (default 5 minutes)
|
|
2964
|
-
* - Deduplicates concurrent fetches for the same model
|
|
2965
|
-
* - Caches null results (404s) to avoid repeated lookups
|
|
2966
|
-
* - Falls back to stale cache on fetch errors
|
|
2967
|
-
*/
|
|
2968
|
-
var LLMOpsPricingProvider = class {
|
|
2969
|
-
cache = /* @__PURE__ */ new Map();
|
|
2970
|
-
pendingFetches = /* @__PURE__ */ new Map();
|
|
2971
|
-
cacheTTL;
|
|
2972
|
-
baseUrl;
|
|
2973
|
-
constructor(options) {
|
|
2974
|
-
this.cacheTTL = options?.cacheTTL ?? 300 * 1e3;
|
|
2975
|
-
this.baseUrl = options?.baseUrl ?? LLMOPS_MODELS_API;
|
|
2976
|
-
}
|
|
2977
|
-
getCacheKey(provider, model) {
|
|
2978
|
-
return `${provider.toLowerCase()}:${model.toLowerCase()}`;
|
|
2979
|
-
}
|
|
2980
|
-
/**
|
|
2981
|
-
* Fetch pricing for a single model from the API
|
|
2982
|
-
*/
|
|
2983
|
-
async fetchModelPricing(provider, model) {
|
|
2984
|
-
const url = `${this.baseUrl}/model-configs/pricing/${encodeURIComponent(provider)}/${model}`;
|
|
2985
|
-
try {
|
|
2986
|
-
logger.debug(`[Pricing] GET ${url}`);
|
|
2987
|
-
const startTime = Date.now();
|
|
2988
|
-
const response = await fetch(url);
|
|
2989
|
-
const elapsed = Date.now() - startTime;
|
|
2990
|
-
logger.debug(`[Pricing] GET ${url} -> ${response.status} (${elapsed}ms)`);
|
|
2991
|
-
if (response.status === 404) {
|
|
2992
|
-
logger.debug(`[Pricing] No pricing found for ${provider}/${model}`);
|
|
2993
|
-
return null;
|
|
2524
|
+
upsertTrace: async (data) => {
|
|
2525
|
+
const result = await upsertTraceSchema.safeParseAsync(data);
|
|
2526
|
+
if (!result.success) throw new LLMOpsError(`Invalid trace data: ${result.error.message}`);
|
|
2527
|
+
const trace = result.data;
|
|
2528
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
2529
|
+
await sql`
|
|
2530
|
+
INSERT INTO "traces" (
|
|
2531
|
+
"id", "traceId", "name", "sessionId", "userId", "status",
|
|
2532
|
+
"startTime", "endTime", "durationMs", "spanCount",
|
|
2533
|
+
"totalInputTokens", "totalOutputTokens", "totalTokens", "totalCost",
|
|
2534
|
+
"tags", "metadata", "createdAt", "updatedAt"
|
|
2535
|
+
) VALUES (
|
|
2536
|
+
${randomUUID()}, ${trace.traceId}, ${trace.name ?? null}, ${trace.sessionId ?? null},
|
|
2537
|
+
${trace.userId ?? null}, ${trace.status},
|
|
2538
|
+
${trace.startTime.toISOString()}, ${trace.endTime?.toISOString() ?? null},
|
|
2539
|
+
${trace.durationMs ?? null}, ${trace.spanCount},
|
|
2540
|
+
${trace.totalInputTokens}, ${trace.totalOutputTokens},
|
|
2541
|
+
${trace.totalTokens}, ${trace.totalCost},
|
|
2542
|
+
${JSON.stringify(trace.tags)}::jsonb, ${JSON.stringify(trace.metadata)}::jsonb,
|
|
2543
|
+
${now}, ${now}
|
|
2544
|
+
)
|
|
2545
|
+
ON CONFLICT ("traceId") DO UPDATE SET
|
|
2546
|
+
"name" = COALESCE(EXCLUDED."name", "traces"."name"),
|
|
2547
|
+
"sessionId" = COALESCE(EXCLUDED."sessionId", "traces"."sessionId"),
|
|
2548
|
+
"userId" = COALESCE(EXCLUDED."userId", "traces"."userId"),
|
|
2549
|
+
"status" = CASE
|
|
2550
|
+
WHEN EXCLUDED."status" = 'error' THEN 'error'
|
|
2551
|
+
WHEN EXCLUDED."status" = 'ok' AND "traces"."status" != 'error' THEN 'ok'
|
|
2552
|
+
ELSE "traces"."status"
|
|
2553
|
+
END,
|
|
2554
|
+
"startTime" = LEAST("traces"."startTime", EXCLUDED."startTime"),
|
|
2555
|
+
"endTime" = GREATEST(
|
|
2556
|
+
COALESCE("traces"."endTime", EXCLUDED."endTime"),
|
|
2557
|
+
COALESCE(EXCLUDED."endTime", "traces"."endTime")
|
|
2558
|
+
),
|
|
2559
|
+
"durationMs" = EXTRACT(EPOCH FROM (
|
|
2560
|
+
GREATEST(
|
|
2561
|
+
COALESCE("traces"."endTime", EXCLUDED."endTime"),
|
|
2562
|
+
COALESCE(EXCLUDED."endTime", "traces"."endTime")
|
|
2563
|
+
) -
|
|
2564
|
+
LEAST("traces"."startTime", EXCLUDED."startTime")
|
|
2565
|
+
))::integer * 1000,
|
|
2566
|
+
"spanCount" = "traces"."spanCount" + EXCLUDED."spanCount",
|
|
2567
|
+
"totalInputTokens" = "traces"."totalInputTokens" + EXCLUDED."totalInputTokens",
|
|
2568
|
+
"totalOutputTokens" = "traces"."totalOutputTokens" + EXCLUDED."totalOutputTokens",
|
|
2569
|
+
"totalTokens" = "traces"."totalTokens" + EXCLUDED."totalTokens",
|
|
2570
|
+
"totalCost" = "traces"."totalCost" + EXCLUDED."totalCost",
|
|
2571
|
+
"tags" = "traces"."tags" || EXCLUDED."tags",
|
|
2572
|
+
"metadata" = "traces"."metadata" || EXCLUDED."metadata",
|
|
2573
|
+
"updatedAt" = ${now}
|
|
2574
|
+
`.execute(db);
|
|
2575
|
+
},
|
|
2576
|
+
batchInsertSpans: async (spans) => {
|
|
2577
|
+
if (spans.length === 0) return { count: 0 };
|
|
2578
|
+
const validatedSpans = [];
|
|
2579
|
+
for (const span of spans) {
|
|
2580
|
+
const result = await insertSpanSchema.safeParseAsync(span);
|
|
2581
|
+
if (!result.success) {
|
|
2582
|
+
logger.warn(`[batchInsertSpans] Skipping invalid span ${span.spanId}: ${result.error.message}`);
|
|
2583
|
+
continue;
|
|
2584
|
+
}
|
|
2585
|
+
validatedSpans.push(result.data);
|
|
2994
2586
|
}
|
|
2995
|
-
if (
|
|
2996
|
-
const
|
|
2997
|
-
|
|
2998
|
-
|
|
2999
|
-
|
|
3000
|
-
|
|
3001
|
-
|
|
3002
|
-
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
|
|
3006
|
-
|
|
3007
|
-
|
|
3008
|
-
|
|
3009
|
-
|
|
3010
|
-
|
|
3011
|
-
|
|
3012
|
-
|
|
3013
|
-
|
|
2587
|
+
if (validatedSpans.length === 0) return { count: 0 };
|
|
2588
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
2589
|
+
const values = validatedSpans.map((span) => ({
|
|
2590
|
+
id: randomUUID(),
|
|
2591
|
+
traceId: span.traceId,
|
|
2592
|
+
spanId: span.spanId,
|
|
2593
|
+
parentSpanId: span.parentSpanId ?? null,
|
|
2594
|
+
name: span.name,
|
|
2595
|
+
kind: span.kind,
|
|
2596
|
+
status: span.status,
|
|
2597
|
+
statusMessage: span.statusMessage ?? null,
|
|
2598
|
+
startTime: span.startTime.toISOString(),
|
|
2599
|
+
endTime: span.endTime?.toISOString() ?? null,
|
|
2600
|
+
durationMs: span.durationMs ?? null,
|
|
2601
|
+
provider: span.provider ?? null,
|
|
2602
|
+
model: span.model ?? null,
|
|
2603
|
+
promptTokens: span.promptTokens,
|
|
2604
|
+
completionTokens: span.completionTokens,
|
|
2605
|
+
totalTokens: span.totalTokens,
|
|
2606
|
+
cost: span.cost,
|
|
2607
|
+
configId: span.configId ?? null,
|
|
2608
|
+
variantId: span.variantId ?? null,
|
|
2609
|
+
environmentId: span.environmentId ?? null,
|
|
2610
|
+
providerConfigId: span.providerConfigId ?? null,
|
|
2611
|
+
requestId: span.requestId ?? null,
|
|
2612
|
+
source: span.source,
|
|
2613
|
+
input: span.input != null ? JSON.stringify(span.input) : null,
|
|
2614
|
+
output: span.output != null ? JSON.stringify(span.output) : null,
|
|
2615
|
+
attributes: JSON.stringify(span.attributes),
|
|
2616
|
+
createdAt: now,
|
|
2617
|
+
updatedAt: now
|
|
2618
|
+
}));
|
|
2619
|
+
await db.insertInto("spans").values(values).onConflict((oc) => oc.column("spanId").doNothing()).execute();
|
|
2620
|
+
return { count: values.length };
|
|
2621
|
+
},
|
|
2622
|
+
batchInsertSpanEvents: async (events) => {
|
|
2623
|
+
if (events.length === 0) return { count: 0 };
|
|
2624
|
+
const validatedEvents = [];
|
|
2625
|
+
for (const event of events) {
|
|
2626
|
+
const result = await insertSpanEventSchema.safeParseAsync(event);
|
|
2627
|
+
if (!result.success) {
|
|
2628
|
+
logger.warn(`[batchInsertSpanEvents] Skipping invalid event: ${result.error.message}`);
|
|
2629
|
+
continue;
|
|
2630
|
+
}
|
|
2631
|
+
validatedEvents.push(result.data);
|
|
3014
2632
|
}
|
|
3015
|
-
return
|
|
3016
|
-
|
|
3017
|
-
|
|
3018
|
-
|
|
3019
|
-
|
|
3020
|
-
|
|
3021
|
-
|
|
3022
|
-
|
|
3023
|
-
|
|
3024
|
-
|
|
3025
|
-
|
|
3026
|
-
|
|
3027
|
-
|
|
3028
|
-
|
|
3029
|
-
|
|
3030
|
-
|
|
3031
|
-
|
|
3032
|
-
|
|
3033
|
-
|
|
3034
|
-
|
|
3035
|
-
|
|
3036
|
-
|
|
3037
|
-
|
|
3038
|
-
|
|
3039
|
-
|
|
3040
|
-
|
|
3041
|
-
|
|
3042
|
-
|
|
3043
|
-
|
|
3044
|
-
|
|
3045
|
-
|
|
3046
|
-
|
|
3047
|
-
|
|
3048
|
-
|
|
3049
|
-
|
|
3050
|
-
|
|
3051
|
-
|
|
3052
|
-
|
|
2633
|
+
if (validatedEvents.length === 0) return { count: 0 };
|
|
2634
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
2635
|
+
const values = validatedEvents.map((event) => ({
|
|
2636
|
+
id: randomUUID(),
|
|
2637
|
+
traceId: event.traceId,
|
|
2638
|
+
spanId: event.spanId,
|
|
2639
|
+
name: event.name,
|
|
2640
|
+
timestamp: event.timestamp.toISOString(),
|
|
2641
|
+
attributes: JSON.stringify(event.attributes),
|
|
2642
|
+
createdAt: now
|
|
2643
|
+
}));
|
|
2644
|
+
await db.insertInto("span_events").values(values).execute();
|
|
2645
|
+
return { count: values.length };
|
|
2646
|
+
},
|
|
2647
|
+
listTraces: async (params) => {
|
|
2648
|
+
const result = await listTracesSchema.safeParseAsync(params || {});
|
|
2649
|
+
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2650
|
+
const { limit, offset, sessionId, userId, status, name, startDate, endDate, tags } = result.data;
|
|
2651
|
+
let baseQuery = db.selectFrom("traces");
|
|
2652
|
+
if (sessionId) baseQuery = baseQuery.where("sessionId", "=", sessionId);
|
|
2653
|
+
if (userId) baseQuery = baseQuery.where("userId", "=", userId);
|
|
2654
|
+
if (status) baseQuery = baseQuery.where("status", "=", status);
|
|
2655
|
+
if (name) baseQuery = baseQuery.where(sql`${col("name")} ILIKE ${"%" + name + "%"}`);
|
|
2656
|
+
if (startDate) baseQuery = baseQuery.where(sql`${col("startTime")} >= ${startDate.toISOString()}`);
|
|
2657
|
+
if (endDate) baseQuery = baseQuery.where(sql`${col("startTime")} <= ${endDate.toISOString()}`);
|
|
2658
|
+
if (tags && Object.keys(tags).length > 0) for (const [key, values] of Object.entries(tags)) {
|
|
2659
|
+
if (values.length === 0) continue;
|
|
2660
|
+
if (values.length === 1) baseQuery = baseQuery.where(sql`${col("tags")}->>${key} = ${values[0]}`);
|
|
2661
|
+
else {
|
|
2662
|
+
const valueList = sql.join(values.map((v) => sql`${v}`));
|
|
2663
|
+
baseQuery = baseQuery.where(sql`${col("tags")}->>${key} IN (${valueList})`);
|
|
2664
|
+
}
|
|
2665
|
+
}
|
|
2666
|
+
const countResult = await baseQuery.select(sql`COUNT(*)`.as("total")).executeTakeFirst();
|
|
2667
|
+
const total = Number(countResult?.total ?? 0);
|
|
2668
|
+
return {
|
|
2669
|
+
data: await baseQuery.selectAll().orderBy("startTime", "desc").limit(limit).offset(offset).execute(),
|
|
2670
|
+
total,
|
|
2671
|
+
limit,
|
|
2672
|
+
offset
|
|
2673
|
+
};
|
|
2674
|
+
},
|
|
2675
|
+
getTraceWithSpans: async (traceId) => {
|
|
2676
|
+
const trace = await db.selectFrom("traces").selectAll().where("traceId", "=", traceId).executeTakeFirst();
|
|
2677
|
+
if (!trace) return void 0;
|
|
2678
|
+
return {
|
|
2679
|
+
trace,
|
|
2680
|
+
spans: await db.selectFrom("spans").selectAll().where("traceId", "=", traceId).orderBy("startTime", "asc").execute(),
|
|
2681
|
+
events: await db.selectFrom("span_events").selectAll().where("traceId", "=", traceId).orderBy("timestamp", "asc").execute()
|
|
2682
|
+
};
|
|
2683
|
+
},
|
|
2684
|
+
getTraceStats: async (params) => {
|
|
2685
|
+
const result = await traceStatsSchema.safeParseAsync(params);
|
|
2686
|
+
if (!result.success) throw new LLMOpsError(`Invalid parameters: ${result.error.message}`);
|
|
2687
|
+
const { startDate, endDate, sessionId, userId } = result.data;
|
|
2688
|
+
let query = db.selectFrom("traces").select([
|
|
2689
|
+
sql`COUNT(*)`.as("totalTraces"),
|
|
2690
|
+
sql`COALESCE(AVG(${col("durationMs")}), 0)`.as("avgDurationMs"),
|
|
2691
|
+
sql`COUNT(CASE WHEN ${col("status")} = 'error' THEN 1 END)`.as("errorCount"),
|
|
2692
|
+
sql`COALESCE(SUM(${col("totalCost")}), 0)`.as("totalCost"),
|
|
2693
|
+
sql`COALESCE(SUM(${col("totalTokens")}), 0)`.as("totalTokens"),
|
|
2694
|
+
sql`COALESCE(SUM(${col("spanCount")}), 0)`.as("totalSpans")
|
|
2695
|
+
]).where(sql`${col("startTime")} >= ${startDate.toISOString()}`).where(sql`${col("startTime")} <= ${endDate.toISOString()}`);
|
|
2696
|
+
if (sessionId) query = query.where("sessionId", "=", sessionId);
|
|
2697
|
+
if (userId) query = query.where("userId", "=", userId);
|
|
2698
|
+
return query.executeTakeFirst();
|
|
3053
2699
|
}
|
|
3054
|
-
|
|
3055
|
-
}
|
|
3056
|
-
/**
|
|
3057
|
-
* Force refresh the pricing cache (clears all cached entries)
|
|
3058
|
-
*/
|
|
3059
|
-
async refreshCache() {
|
|
3060
|
-
this.cache.clear();
|
|
3061
|
-
}
|
|
3062
|
-
/**
|
|
3063
|
-
* Always ready — no bulk pre-fetch needed
|
|
3064
|
-
*/
|
|
3065
|
-
isReady() {
|
|
3066
|
-
return true;
|
|
3067
|
-
}
|
|
3068
|
-
/**
|
|
3069
|
-
* Get the number of cached models (for debugging)
|
|
3070
|
-
*/
|
|
3071
|
-
getCacheSize() {
|
|
3072
|
-
return this.cache.size;
|
|
3073
|
-
}
|
|
3074
|
-
};
|
|
3075
|
-
let defaultProvider = null;
|
|
3076
|
-
/**
|
|
3077
|
-
* Get the default pricing provider instance
|
|
3078
|
-
*/
|
|
3079
|
-
function getDefaultPricingProvider() {
|
|
3080
|
-
if (!defaultProvider) defaultProvider = new LLMOpsPricingProvider();
|
|
3081
|
-
return defaultProvider;
|
|
2700
|
+
};
|
|
3082
2701
|
}
|
|
3083
|
-
|
|
3084
|
-
//#endregion
|
|
3085
|
-
//#region src/telemetry/postgres.ts
|
|
3086
2702
|
/**
|
|
3087
2703
|
* Create a PostgreSQL-backed telemetry store.
|
|
3088
2704
|
*
|
|
3089
2705
|
* Usage:
|
|
3090
2706
|
* ```ts
|
|
3091
|
-
* import { llmops
|
|
2707
|
+
* import { llmops } from '@llmops/sdk'
|
|
2708
|
+
* import { pgStore } from '@llmops/sdk/store/pg'
|
|
3092
2709
|
*
|
|
3093
2710
|
* const ops = llmops({
|
|
3094
2711
|
* telemetry: pgStore(process.env.DATABASE_URL),
|
|
@@ -3109,139 +2726,13 @@ function createPgStore(connectionString, options) {
|
|
|
3109
2726
|
await connection.executeQuery(CompiledQuery.raw(`SET search_path TO "${schema}"`));
|
|
3110
2727
|
}
|
|
3111
2728
|
}) });
|
|
3112
|
-
const llmRequests = createLLMRequestsDataLayer(db);
|
|
3113
|
-
const traces = createTracesDataLayer(db);
|
|
3114
2729
|
logger.debug(`pgStore: initialized with schema "${schema}"`);
|
|
3115
2730
|
return {
|
|
3116
|
-
...
|
|
3117
|
-
...
|
|
2731
|
+
...createLLMRequestsStore(db),
|
|
2732
|
+
...createTracesStore(db),
|
|
3118
2733
|
_db: db
|
|
3119
2734
|
};
|
|
3120
2735
|
}
|
|
3121
2736
|
|
|
3122
2737
|
//#endregion
|
|
3123
|
-
|
|
3124
|
-
/**
|
|
3125
|
-
* Builds the gateway manifest from database
|
|
3126
|
-
*/
|
|
3127
|
-
var ManifestBuilder = class {
|
|
3128
|
-
constructor(db) {
|
|
3129
|
-
this.db = db;
|
|
3130
|
-
}
|
|
3131
|
-
/**
|
|
3132
|
-
* Build the manifest from database
|
|
3133
|
-
*/
|
|
3134
|
-
async build() {
|
|
3135
|
-
const [guardrailConfigs, providerGuardrailOverridesData] = await Promise.all([this.db.selectFrom("guardrail_configs").where("enabled", "=", true).selectAll().execute(), this.db.selectFrom("provider_guardrail_overrides").selectAll().execute()]);
|
|
3136
|
-
const beforeRequestGuardrails = [];
|
|
3137
|
-
const afterRequestGuardrails = [];
|
|
3138
|
-
logger.info(`[ManifestBuilder] Found ${guardrailConfigs.length} enabled guardrail configs`);
|
|
3139
|
-
for (const guardrail of guardrailConfigs) {
|
|
3140
|
-
const parameters = typeof guardrail.parameters === "string" ? JSON.parse(guardrail.parameters) : guardrail.parameters;
|
|
3141
|
-
const manifestGuardrail = {
|
|
3142
|
-
id: guardrail.id,
|
|
3143
|
-
name: guardrail.name,
|
|
3144
|
-
pluginId: guardrail.pluginId,
|
|
3145
|
-
functionId: guardrail.functionId,
|
|
3146
|
-
hookType: guardrail.hookType,
|
|
3147
|
-
parameters: parameters ?? {},
|
|
3148
|
-
priority: guardrail.priority,
|
|
3149
|
-
onFail: guardrail.onFail
|
|
3150
|
-
};
|
|
3151
|
-
if (guardrail.hookType === "beforeRequestHook") beforeRequestGuardrails.push(manifestGuardrail);
|
|
3152
|
-
else afterRequestGuardrails.push(manifestGuardrail);
|
|
3153
|
-
}
|
|
3154
|
-
beforeRequestGuardrails.sort((a, b) => b.priority - a.priority);
|
|
3155
|
-
afterRequestGuardrails.sort((a, b) => b.priority - a.priority);
|
|
3156
|
-
const providerGuardrailOverrides = {};
|
|
3157
|
-
for (const override of providerGuardrailOverridesData) {
|
|
3158
|
-
const parameters = typeof override.parameters === "string" ? JSON.parse(override.parameters) : override.parameters;
|
|
3159
|
-
const manifestOverride = {
|
|
3160
|
-
id: override.id,
|
|
3161
|
-
providerConfigId: override.providerConfigId,
|
|
3162
|
-
guardrailConfigId: override.guardrailConfigId,
|
|
3163
|
-
enabled: override.enabled,
|
|
3164
|
-
parameters: parameters ?? null
|
|
3165
|
-
};
|
|
3166
|
-
if (!providerGuardrailOverrides[override.providerConfigId]) providerGuardrailOverrides[override.providerConfigId] = [];
|
|
3167
|
-
providerGuardrailOverrides[override.providerConfigId].push(manifestOverride);
|
|
3168
|
-
}
|
|
3169
|
-
return {
|
|
3170
|
-
version: Date.now(),
|
|
3171
|
-
builtAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
3172
|
-
guardrails: {
|
|
3173
|
-
beforeRequestHook: beforeRequestGuardrails,
|
|
3174
|
-
afterRequestHook: afterRequestGuardrails
|
|
3175
|
-
},
|
|
3176
|
-
providerGuardrailOverrides
|
|
3177
|
-
};
|
|
3178
|
-
}
|
|
3179
|
-
};
|
|
3180
|
-
|
|
3181
|
-
//#endregion
|
|
3182
|
-
//#region src/manifest/service.ts
|
|
3183
|
-
const MANIFEST_CACHE_KEY = "manifest";
|
|
3184
|
-
const MANIFEST_NAMESPACE = "gateway";
|
|
3185
|
-
const DEFAULT_TTL_MS = 300 * 1e3;
|
|
3186
|
-
const log = logger.child({ module: "ManifestService" });
|
|
3187
|
-
var ManifestService = class {
|
|
3188
|
-
builder;
|
|
3189
|
-
constructor(cache, db, ttlMs = DEFAULT_TTL_MS) {
|
|
3190
|
-
this.cache = cache;
|
|
3191
|
-
this.ttlMs = ttlMs;
|
|
3192
|
-
this.builder = new ManifestBuilder(db);
|
|
3193
|
-
log.debug({ ttlMs }, "ManifestService initialized");
|
|
3194
|
-
}
|
|
3195
|
-
/**
|
|
3196
|
-
* Get the current manifest, building if necessary
|
|
3197
|
-
*/
|
|
3198
|
-
async getManifest() {
|
|
3199
|
-
log.debug("Getting manifest from cache or building");
|
|
3200
|
-
const manifest = await this.cache.getOrSet(MANIFEST_CACHE_KEY, async () => {
|
|
3201
|
-
log.info("Building new manifest");
|
|
3202
|
-
const built = await this.builder.build();
|
|
3203
|
-
log.info({ version: built.version }, "Manifest built successfully");
|
|
3204
|
-
return built;
|
|
3205
|
-
}, {
|
|
3206
|
-
namespace: MANIFEST_NAMESPACE,
|
|
3207
|
-
ttl: this.ttlMs
|
|
3208
|
-
});
|
|
3209
|
-
log.debug({ version: manifest.version }, "Manifest retrieved");
|
|
3210
|
-
return manifest;
|
|
3211
|
-
}
|
|
3212
|
-
/**
|
|
3213
|
-
* Force invalidate the manifest (called on mutations)
|
|
3214
|
-
*/
|
|
3215
|
-
async invalidate() {
|
|
3216
|
-
log.info("Invalidating manifest cache");
|
|
3217
|
-
await this.cache.delete(MANIFEST_CACHE_KEY, MANIFEST_NAMESPACE);
|
|
3218
|
-
}
|
|
3219
|
-
/**
|
|
3220
|
-
* Invalidate and immediately rebuild (atomic refresh)
|
|
3221
|
-
*/
|
|
3222
|
-
async refresh() {
|
|
3223
|
-
log.info("Refreshing manifest (invalidate + rebuild)");
|
|
3224
|
-
await this.invalidate();
|
|
3225
|
-
return this.getManifest();
|
|
3226
|
-
}
|
|
3227
|
-
/**
|
|
3228
|
-
* Get manifest version without fetching full manifest
|
|
3229
|
-
* Useful for checking if manifest is stale
|
|
3230
|
-
*/
|
|
3231
|
-
async getVersion() {
|
|
3232
|
-
const version = (await this.cache.get(MANIFEST_CACHE_KEY, MANIFEST_NAMESPACE))?.version ?? null;
|
|
3233
|
-
log.debug({ version }, "Got manifest version");
|
|
3234
|
-
return version;
|
|
3235
|
-
}
|
|
3236
|
-
/**
|
|
3237
|
-
* Check if manifest exists in cache
|
|
3238
|
-
*/
|
|
3239
|
-
async hasManifest() {
|
|
3240
|
-
const exists = await this.cache.has(MANIFEST_CACHE_KEY, MANIFEST_NAMESPACE);
|
|
3241
|
-
log.debug({ exists }, "Checked manifest existence");
|
|
3242
|
-
return exists;
|
|
3243
|
-
}
|
|
3244
|
-
};
|
|
3245
|
-
|
|
3246
|
-
//#endregion
|
|
3247
|
-
export { COST_SUMMARY_GROUP_BY, CacheService, DEFAULT_PROVIDER_ENV_VARS, FileCacheBackend, LLMOPS_INTERNAL_HEADER, LLMOPS_REQUEST_ID_HEADER, LLMOPS_SESSION_ID_HEADER, LLMOPS_SPAN_ID_HEADER, LLMOPS_SPAN_NAME_HEADER, LLMOPS_TRACE_ID_HEADER, LLMOPS_TRACE_NAME_HEADER, LLMOPS_USER_ID_HEADER, LLMOpsPricingProvider, MS, ManifestBuilder, ManifestService, MemoryCacheBackend, SCHEMA_METADATA, SupportedProviders, calculateCacheAwareCost, calculateCost, chatCompletionCreateParamsBaseSchema, configVariantsSchema, configsSchema, createDataLayer, createDatabase, createDatabaseFromConnection, createDatasetsDataLayer, createGuardrailConfigsDataLayer, createLLMRequestsDataLayer, createNeonDialect, createPgStore, createPlaygroundDataLayer, createPlaygroundResultsDataLayer, createPlaygroundRunsDataLayer, createProviderConfigsDataLayer, createProviderGuardrailOverridesDataLayer, createTracesDataLayer, createWorkspaceSettingsDataLayer, datasetRecordsSchema, datasetVersionRecordsSchema, datasetVersionsSchema, datasetsSchema, detectDatabaseType, dollarsToMicroDollars, environmentSecretsSchema, environmentsSchema, executeWithSchema, formatCost, gateway, generateId, getDefaultPricingProvider, getDefaultProviders, getMigrations, guardrailConfigsSchema, llmRequestsSchema, llmopsConfigSchema, logger, matchType, mergeWithDefaultProviders, microDollarsToDollars, parsePartialTableData, parseTableData, playgroundColumnSchema, playgroundResultsSchema, playgroundRunsSchema, playgroundsSchema, providerConfigsSchema, providerGuardrailOverridesSchema, runAutoMigrations, schemas, spanEventsSchema, spansSchema, targetingRulesSchema, tracesSchema, validateLLMOpsConfig, validatePartialTableData, validateTableData, variantJsonDataSchema, variantVersionsSchema, variantsSchema, workspaceSettingsSchema };
|
|
2738
|
+
export { COST_SUMMARY_GROUP_BY, CacheService, DEFAULT_PROVIDER_ENV_VARS, FileCacheBackend, LLMOPS_INTERNAL_HEADER, LLMOPS_REQUEST_ID_HEADER, LLMOPS_SESSION_ID_HEADER, LLMOPS_SPAN_ID_HEADER, LLMOPS_SPAN_NAME_HEADER, LLMOPS_TRACE_ID_HEADER, LLMOPS_TRACE_NAME_HEADER, LLMOPS_USER_ID_HEADER, LLMOpsPricingProvider, MS, MemoryCacheBackend, SCHEMA_METADATA, SupportedProviders, calculateCacheAwareCost, calculateCost, chatCompletionCreateParamsBaseSchema, configVariantsSchema, configsSchema, createDataLayer, createDatabase, createDatabaseFromConnection, createDatasetsDataLayer, createNeonDialect, createPgStore, createPlaygroundDataLayer, createPlaygroundResultsDataLayer, createPlaygroundRunsDataLayer, createWorkspaceSettingsDataLayer, datasetRecordsSchema, datasetVersionRecordsSchema, datasetVersionsSchema, datasetsSchema, detectDatabaseType, dollarsToMicroDollars, environmentSecretsSchema, environmentsSchema, executeWithSchema, formatCost, gateway, generateId, getDefaultPricingProvider, getDefaultProviders, getMigrations, guardrailConfigsSchema, llmRequestsSchema, llmopsConfigSchema, logger, matchType, mergeWithDefaultProviders, microDollarsToDollars, parsePartialTableData, parseTableData, playgroundColumnSchema, playgroundResultsSchema, playgroundRunsSchema, playgroundsSchema, providerConfigsSchema, providerGuardrailOverridesSchema, runAutoMigrations, schemas, spanEventsSchema, spansSchema, targetingRulesSchema, tracesSchema, validateLLMOpsConfig, validatePartialTableData, validateTableData, variantJsonDataSchema, variantVersionsSchema, variantsSchema, workspaceSettingsSchema };
|