@dudousxd/nestjs-agent-telescope 0.7.1 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -1
- package/dist/index.cjs +705 -12
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +223 -16
- package/dist/index.d.ts +223 -16
- package/dist/index.js +678 -9
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -492,7 +492,180 @@ function agentDashboard(opts = {}) {
|
|
|
492
492
|
]
|
|
493
493
|
}
|
|
494
494
|
]
|
|
495
|
-
}
|
|
495
|
+
},
|
|
496
|
+
{
|
|
497
|
+
// RAG. Fed by the `aviary:rag:retrieval` telemetry `@dudousxd/nestjs-agent-rag` emits, NOT
|
|
498
|
+
// derived from tool-call rows: a tool call that retrieved nothing and one that retrieved
|
|
499
|
+
// five perfect passages are the same row with the same `ok` status, so every number here
|
|
500
|
+
// would have been unanswerable from that side.
|
|
501
|
+
title: "Retrieval",
|
|
502
|
+
cols: 3,
|
|
503
|
+
panels: [
|
|
504
|
+
{
|
|
505
|
+
kind: "stat",
|
|
506
|
+
title: "Retrievals",
|
|
507
|
+
data: {
|
|
508
|
+
provider: "agent.rag.retrievals"
|
|
509
|
+
}
|
|
510
|
+
},
|
|
511
|
+
{
|
|
512
|
+
kind: "stat",
|
|
513
|
+
title: "Zero-hit rate",
|
|
514
|
+
data: {
|
|
515
|
+
provider: "agent.rag.zeroHitRate"
|
|
516
|
+
},
|
|
517
|
+
format: "percent"
|
|
518
|
+
},
|
|
519
|
+
{
|
|
520
|
+
kind: "stat",
|
|
521
|
+
title: "Passages per retrieval",
|
|
522
|
+
data: {
|
|
523
|
+
provider: "agent.rag.chunks"
|
|
524
|
+
},
|
|
525
|
+
format: "number"
|
|
526
|
+
},
|
|
527
|
+
{
|
|
528
|
+
// A real histogram, unlike the run-duration pair above: retrieval events carry the raw
|
|
529
|
+
// per-call duration, so there ARE samples to bucket and markers to place.
|
|
530
|
+
kind: "distribution",
|
|
531
|
+
title: "Retrieval latency",
|
|
532
|
+
data: {
|
|
533
|
+
provider: "agent.rag.latency"
|
|
534
|
+
},
|
|
535
|
+
markers: [
|
|
536
|
+
"p50",
|
|
537
|
+
"p95",
|
|
538
|
+
"p99"
|
|
539
|
+
],
|
|
540
|
+
format: "duration"
|
|
541
|
+
},
|
|
542
|
+
{
|
|
543
|
+
// Bound to ONE retriever kind on purpose — a cosine similarity, a BM25 score and an RRF
|
|
544
|
+
// rank score share no scale, so a histogram over all of them has bins that mean a
|
|
545
|
+
// different thing per bar. See `toScoreDistribution`.
|
|
546
|
+
kind: "distribution",
|
|
547
|
+
title: "Top score (dense)",
|
|
548
|
+
data: {
|
|
549
|
+
provider: "agent.rag.scores",
|
|
550
|
+
query: {
|
|
551
|
+
retriever: "embedding"
|
|
552
|
+
}
|
|
553
|
+
},
|
|
554
|
+
markers: [
|
|
555
|
+
"p50",
|
|
556
|
+
"p95"
|
|
557
|
+
],
|
|
558
|
+
format: "number"
|
|
559
|
+
},
|
|
560
|
+
{
|
|
561
|
+
kind: "timeseries",
|
|
562
|
+
title: "Retrievals & zero-hits",
|
|
563
|
+
data: {
|
|
564
|
+
provider: "agent.rag.trend"
|
|
565
|
+
},
|
|
566
|
+
series: [
|
|
567
|
+
"retrievals",
|
|
568
|
+
"zeroHits"
|
|
569
|
+
],
|
|
570
|
+
style: "stacked"
|
|
571
|
+
}
|
|
572
|
+
]
|
|
573
|
+
},
|
|
574
|
+
{
|
|
575
|
+
title: "Retrieval sources",
|
|
576
|
+
cols: 2,
|
|
577
|
+
panels: [
|
|
578
|
+
{
|
|
579
|
+
kind: "breakdown",
|
|
580
|
+
title: "Retrievals by store",
|
|
581
|
+
data: {
|
|
582
|
+
provider: "agent.rag.byStore"
|
|
583
|
+
},
|
|
584
|
+
style: "donut"
|
|
585
|
+
},
|
|
586
|
+
{
|
|
587
|
+
kind: "breakdown",
|
|
588
|
+
title: "Retrievals by retriever",
|
|
589
|
+
data: {
|
|
590
|
+
provider: "agent.rag.byRetriever"
|
|
591
|
+
},
|
|
592
|
+
style: "donut"
|
|
593
|
+
},
|
|
594
|
+
{
|
|
595
|
+
kind: "table",
|
|
596
|
+
title: "By collection",
|
|
597
|
+
data: {
|
|
598
|
+
provider: "agent.rag.byCollection"
|
|
599
|
+
},
|
|
600
|
+
columns: [
|
|
601
|
+
{
|
|
602
|
+
key: "collection",
|
|
603
|
+
label: "Collection"
|
|
604
|
+
},
|
|
605
|
+
{
|
|
606
|
+
key: "store",
|
|
607
|
+
label: "Store"
|
|
608
|
+
},
|
|
609
|
+
{
|
|
610
|
+
key: "retrievals",
|
|
611
|
+
label: "Retrievals"
|
|
612
|
+
},
|
|
613
|
+
{
|
|
614
|
+
key: "zeroHits",
|
|
615
|
+
label: "Zero-hits"
|
|
616
|
+
},
|
|
617
|
+
{
|
|
618
|
+
key: "p95Ms",
|
|
619
|
+
label: "p95 (ms)"
|
|
620
|
+
},
|
|
621
|
+
{
|
|
622
|
+
key: "meanTopScore",
|
|
623
|
+
label: "Mean top score"
|
|
624
|
+
}
|
|
625
|
+
]
|
|
626
|
+
},
|
|
627
|
+
{
|
|
628
|
+
kind: "table",
|
|
629
|
+
title: "Slowest retrievals",
|
|
630
|
+
data: {
|
|
631
|
+
provider: "agent.rag.slowest"
|
|
632
|
+
},
|
|
633
|
+
// Seven columns, matching the recent-runs table's width: this is the widest a table can
|
|
634
|
+
// get here before it overflows its card.
|
|
635
|
+
columns: [
|
|
636
|
+
{
|
|
637
|
+
key: "at",
|
|
638
|
+
label: "When"
|
|
639
|
+
},
|
|
640
|
+
{
|
|
641
|
+
key: "retriever",
|
|
642
|
+
label: "Retriever"
|
|
643
|
+
},
|
|
644
|
+
{
|
|
645
|
+
key: "store",
|
|
646
|
+
label: "Store"
|
|
647
|
+
},
|
|
648
|
+
{
|
|
649
|
+
key: "collection",
|
|
650
|
+
label: "Collection"
|
|
651
|
+
},
|
|
652
|
+
{
|
|
653
|
+
key: "chunks",
|
|
654
|
+
label: "Passages"
|
|
655
|
+
},
|
|
656
|
+
{
|
|
657
|
+
key: "topScore",
|
|
658
|
+
label: "Top score"
|
|
659
|
+
},
|
|
660
|
+
{
|
|
661
|
+
key: "durationMs",
|
|
662
|
+
label: "Duration (ms)"
|
|
663
|
+
}
|
|
664
|
+
]
|
|
665
|
+
}
|
|
666
|
+
]
|
|
667
|
+
},
|
|
668
|
+
...opts.sections ?? []
|
|
496
669
|
]
|
|
497
670
|
};
|
|
498
671
|
}
|
|
@@ -913,9 +1086,15 @@ function agentRunsTrendProvider() {
|
|
|
913
1086
|
__name(agentRunsTrendProvider, "agentRunsTrendProvider");
|
|
914
1087
|
var DEFAULT_RECENT_LIMIT = 50;
|
|
915
1088
|
var MAX_RECENT_LIMIT = 500;
|
|
1089
|
+
function readPositiveNumber(raw) {
|
|
1090
|
+
if (typeof raw === "number") return Number.isFinite(raw) && raw > 0 ? raw : void 0;
|
|
1091
|
+
if (typeof raw !== "string" || raw.trim() === "") return void 0;
|
|
1092
|
+
const parsed = Number(raw);
|
|
1093
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : void 0;
|
|
1094
|
+
}
|
|
1095
|
+
__name(readPositiveNumber, "readPositiveNumber");
|
|
916
1096
|
function resolveLimit(query, fallback) {
|
|
917
|
-
const
|
|
918
|
-
const value = typeof raw === "number" && Number.isFinite(raw) ? raw : fallback;
|
|
1097
|
+
const value = readPositiveNumber(query?.limit) ?? fallback;
|
|
919
1098
|
return Math.min(MAX_RECENT_LIMIT, Math.max(1, Math.trunc(value)));
|
|
920
1099
|
}
|
|
921
1100
|
__name(resolveLimit, "resolveLimit");
|
|
@@ -931,8 +1110,7 @@ function governanceLimitProvider(name, defaultLimit, fetch, format) {
|
|
|
931
1110
|
}
|
|
932
1111
|
__name(governanceLimitProvider, "governanceLimitProvider");
|
|
933
1112
|
function resolvePage(query) {
|
|
934
|
-
const
|
|
935
|
-
const value = typeof raw === "number" && Number.isFinite(raw) ? raw : 1;
|
|
1113
|
+
const value = readPositiveNumber(query?.page) ?? 1;
|
|
936
1114
|
return Math.max(1, Math.trunc(value));
|
|
937
1115
|
}
|
|
938
1116
|
__name(resolvePage, "resolvePage");
|
|
@@ -1066,22 +1244,478 @@ var AgentTelescopeWatcher = class {
|
|
|
1066
1244
|
}
|
|
1067
1245
|
};
|
|
1068
1246
|
|
|
1247
|
+
// src/rag-data-providers.ts
|
|
1248
|
+
import { TELESCOPE_STORAGE as TELESCOPE_STORAGE2 } from "@dudousxd/nestjs-telescope";
|
|
1249
|
+
|
|
1250
|
+
// src/rag-telescope.watcher.ts
|
|
1251
|
+
import { subscribe as subscribe2, unsubscribe as unsubscribe2 } from "node:diagnostics_channel";
|
|
1252
|
+
import { channelName as channelName2, claimDiagnostics as claimDiagnostics2 } from "@dudousxd/nestjs-diagnostics";
|
|
1253
|
+
var RAG_LIB = "rag";
|
|
1254
|
+
var RETRIEVAL_EVENT = "retrieval";
|
|
1255
|
+
var RAG_ENTRY_TYPE = "agent-rag";
|
|
1256
|
+
function isEnvelope(message) {
|
|
1257
|
+
return typeof message === "object" && message !== null && typeof Reflect.get(message, "payload") === "object" && Reflect.get(message, "payload") !== null;
|
|
1258
|
+
}
|
|
1259
|
+
__name(isEnvelope, "isEnvelope");
|
|
1260
|
+
var RagTelescopeWatcher = class {
|
|
1261
|
+
static {
|
|
1262
|
+
__name(this, "RagTelescopeWatcher");
|
|
1263
|
+
}
|
|
1264
|
+
type = RAG_ENTRY_TYPE;
|
|
1265
|
+
disposers = [];
|
|
1266
|
+
register(ctx) {
|
|
1267
|
+
this.disposers.push(claimDiagnostics2(RAG_LIB, [
|
|
1268
|
+
RETRIEVAL_EVENT
|
|
1269
|
+
]));
|
|
1270
|
+
const channel = channelName2(RAG_LIB, RETRIEVAL_EVENT);
|
|
1271
|
+
const onMessage = /* @__PURE__ */ __name((message) => {
|
|
1272
|
+
if (!isEnvelope(message)) {
|
|
1273
|
+
return;
|
|
1274
|
+
}
|
|
1275
|
+
ctx.record({
|
|
1276
|
+
type: RAG_ENTRY_TYPE,
|
|
1277
|
+
content: {
|
|
1278
|
+
event: RETRIEVAL_EVENT,
|
|
1279
|
+
...message.payload,
|
|
1280
|
+
...message.durationMs !== void 0 ? {
|
|
1281
|
+
durationMs: message.durationMs
|
|
1282
|
+
} : {}
|
|
1283
|
+
},
|
|
1284
|
+
// Tagged by store and retriever kind so the Entries screen can filter to "every retrieval
|
|
1285
|
+
// pgvector served" without a provider — the same tag-per-dimension the agent watcher uses.
|
|
1286
|
+
tags: tagsOf(message.payload)
|
|
1287
|
+
});
|
|
1288
|
+
}, "onMessage");
|
|
1289
|
+
subscribe2(channel, onMessage);
|
|
1290
|
+
this.disposers.push(() => unsubscribe2(channel, onMessage));
|
|
1291
|
+
}
|
|
1292
|
+
/** Detach the subscription and release the diagnostics claim (e.g. on module destroy). */
|
|
1293
|
+
dispose() {
|
|
1294
|
+
while (this.disposers.length) this.disposers.pop()?.();
|
|
1295
|
+
}
|
|
1296
|
+
};
|
|
1297
|
+
function tagsOf(payload) {
|
|
1298
|
+
const tags = [
|
|
1299
|
+
RETRIEVAL_EVENT
|
|
1300
|
+
];
|
|
1301
|
+
if (typeof payload.retriever === "string") {
|
|
1302
|
+
tags.push(`retriever:${payload.retriever}`);
|
|
1303
|
+
}
|
|
1304
|
+
if (typeof payload.store === "string") {
|
|
1305
|
+
tags.push(`store:${payload.store}`);
|
|
1306
|
+
}
|
|
1307
|
+
if (payload.zeroHit === true) {
|
|
1308
|
+
tags.push("zero-hit");
|
|
1309
|
+
}
|
|
1310
|
+
if (payload.failed === true) {
|
|
1311
|
+
tags.push("failed");
|
|
1312
|
+
}
|
|
1313
|
+
return tags;
|
|
1314
|
+
}
|
|
1315
|
+
__name(tagsOf, "tagsOf");
|
|
1316
|
+
|
|
1317
|
+
// src/rag-data-providers.ts
|
|
1318
|
+
var RETRIEVAL_WINDOW = 5e3;
|
|
1319
|
+
var TABLE_LIMIT = 10;
|
|
1320
|
+
var TREND_HOURS = 24;
|
|
1321
|
+
var NO_VALUE2 = "\u2014";
|
|
1322
|
+
function isRecord2(value) {
|
|
1323
|
+
return typeof value === "object" && value !== null;
|
|
1324
|
+
}
|
|
1325
|
+
__name(isRecord2, "isRecord");
|
|
1326
|
+
function readString(source, key) {
|
|
1327
|
+
const value = source[key];
|
|
1328
|
+
return typeof value === "string" ? value : null;
|
|
1329
|
+
}
|
|
1330
|
+
__name(readString, "readString");
|
|
1331
|
+
function readNumber(source, key) {
|
|
1332
|
+
const value = source[key];
|
|
1333
|
+
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
1334
|
+
}
|
|
1335
|
+
__name(readNumber, "readNumber");
|
|
1336
|
+
function isEntryStorage(value) {
|
|
1337
|
+
return isRecord2(value) && typeof value.get === "function";
|
|
1338
|
+
}
|
|
1339
|
+
__name(isEntryStorage, "isEntryStorage");
|
|
1340
|
+
function toRagRetrieval(entry) {
|
|
1341
|
+
if (!isRecord2(entry) || !isRecord2(entry.content)) {
|
|
1342
|
+
return null;
|
|
1343
|
+
}
|
|
1344
|
+
const content = entry.content;
|
|
1345
|
+
const retriever = readString(content, "retriever");
|
|
1346
|
+
if (retriever === null) {
|
|
1347
|
+
return null;
|
|
1348
|
+
}
|
|
1349
|
+
const chunks = readNumber(content, "chunks") ?? 0;
|
|
1350
|
+
const createdAt = entry.createdAt;
|
|
1351
|
+
return {
|
|
1352
|
+
at: createdAt instanceof Date ? createdAt : /* @__PURE__ */ new Date(0),
|
|
1353
|
+
retriever,
|
|
1354
|
+
store: readString(content, "store"),
|
|
1355
|
+
collection: readString(content, "collection"),
|
|
1356
|
+
chunks,
|
|
1357
|
+
zeroHit: content.zeroHit === true || chunks === 0,
|
|
1358
|
+
topScore: readNumber(content, "topScore"),
|
|
1359
|
+
durationMs: readNumber(content, "durationMs"),
|
|
1360
|
+
failed: content.failed === true
|
|
1361
|
+
};
|
|
1362
|
+
}
|
|
1363
|
+
__name(toRagRetrieval, "toRagRetrieval");
|
|
1364
|
+
async function readRetrievals(storage) {
|
|
1365
|
+
if (!isEntryStorage(storage)) {
|
|
1366
|
+
return [];
|
|
1367
|
+
}
|
|
1368
|
+
const page = await storage.get({
|
|
1369
|
+
type: RAG_ENTRY_TYPE,
|
|
1370
|
+
limit: RETRIEVAL_WINDOW
|
|
1371
|
+
});
|
|
1372
|
+
const retrievals = [];
|
|
1373
|
+
for (const entry of page.data) {
|
|
1374
|
+
const retrieval = toRagRetrieval(entry);
|
|
1375
|
+
if (retrieval !== null) {
|
|
1376
|
+
retrievals.push(retrieval);
|
|
1377
|
+
}
|
|
1378
|
+
}
|
|
1379
|
+
return retrievals;
|
|
1380
|
+
}
|
|
1381
|
+
__name(readRetrievals, "readRetrievals");
|
|
1382
|
+
function retrievalStorage(ctx) {
|
|
1383
|
+
try {
|
|
1384
|
+
return ctx.moduleRef.get(TELESCOPE_STORAGE2, {
|
|
1385
|
+
strict: false
|
|
1386
|
+
});
|
|
1387
|
+
} catch {
|
|
1388
|
+
return void 0;
|
|
1389
|
+
}
|
|
1390
|
+
}
|
|
1391
|
+
__name(retrievalStorage, "retrievalStorage");
|
|
1392
|
+
function percentile(samples, fraction) {
|
|
1393
|
+
if (samples.length === 0) {
|
|
1394
|
+
return null;
|
|
1395
|
+
}
|
|
1396
|
+
const sorted = [
|
|
1397
|
+
...samples
|
|
1398
|
+
].sort((a, b) => a - b);
|
|
1399
|
+
const rank = Math.ceil(fraction * sorted.length);
|
|
1400
|
+
return sorted[Math.min(Math.max(rank, 1), sorted.length) - 1] ?? null;
|
|
1401
|
+
}
|
|
1402
|
+
__name(percentile, "percentile");
|
|
1403
|
+
function round(value, places) {
|
|
1404
|
+
const factor = 10 ** places;
|
|
1405
|
+
return Math.round(value * factor) / factor;
|
|
1406
|
+
}
|
|
1407
|
+
__name(round, "round");
|
|
1408
|
+
function zeroHitRate(retrievals) {
|
|
1409
|
+
if (retrievals.length === 0) {
|
|
1410
|
+
return 0;
|
|
1411
|
+
}
|
|
1412
|
+
const zeroHits = retrievals.filter((retrieval) => retrieval.zeroHit).length;
|
|
1413
|
+
return round(zeroHits / retrievals.length, 4);
|
|
1414
|
+
}
|
|
1415
|
+
__name(zeroHitRate, "zeroHitRate");
|
|
1416
|
+
function meanChunks(retrievals) {
|
|
1417
|
+
if (retrievals.length === 0) {
|
|
1418
|
+
return 0;
|
|
1419
|
+
}
|
|
1420
|
+
const total = retrievals.reduce((sum, retrieval) => sum + retrieval.chunks, 0);
|
|
1421
|
+
return round(total / retrievals.length, 2);
|
|
1422
|
+
}
|
|
1423
|
+
__name(meanChunks, "meanChunks");
|
|
1424
|
+
var LATENCY_EDGES = [
|
|
1425
|
+
10,
|
|
1426
|
+
25,
|
|
1427
|
+
50,
|
|
1428
|
+
100,
|
|
1429
|
+
250,
|
|
1430
|
+
500,
|
|
1431
|
+
1e3,
|
|
1432
|
+
2500,
|
|
1433
|
+
5e3
|
|
1434
|
+
];
|
|
1435
|
+
function latencyLabel(index) {
|
|
1436
|
+
const upper = LATENCY_EDGES[index];
|
|
1437
|
+
if (upper === void 0) {
|
|
1438
|
+
return `\u2265${LATENCY_EDGES[LATENCY_EDGES.length - 1] ?? 0}ms`;
|
|
1439
|
+
}
|
|
1440
|
+
return `${index === 0 ? 0 : LATENCY_EDGES[index - 1] ?? 0}\u2013${upper}ms`;
|
|
1441
|
+
}
|
|
1442
|
+
__name(latencyLabel, "latencyLabel");
|
|
1443
|
+
function toLatencyDistribution(retrievals) {
|
|
1444
|
+
const samples = [];
|
|
1445
|
+
const counts = new Array(LATENCY_EDGES.length + 1).fill(0);
|
|
1446
|
+
for (const { durationMs } of retrievals) {
|
|
1447
|
+
if (durationMs === null) {
|
|
1448
|
+
continue;
|
|
1449
|
+
}
|
|
1450
|
+
samples.push(durationMs);
|
|
1451
|
+
const found = LATENCY_EDGES.findIndex((edge) => durationMs < edge);
|
|
1452
|
+
const index = found === -1 ? LATENCY_EDGES.length : found;
|
|
1453
|
+
counts[index] = (counts[index] ?? 0) + 1;
|
|
1454
|
+
}
|
|
1455
|
+
const buckets = counts.map((count, index) => ({
|
|
1456
|
+
label: latencyLabel(index),
|
|
1457
|
+
count
|
|
1458
|
+
}));
|
|
1459
|
+
const p50 = percentile(samples, 0.5);
|
|
1460
|
+
const p95 = percentile(samples, 0.95);
|
|
1461
|
+
const p99 = percentile(samples, 0.99);
|
|
1462
|
+
return {
|
|
1463
|
+
buckets,
|
|
1464
|
+
...p50 !== null ? {
|
|
1465
|
+
p50
|
|
1466
|
+
} : {},
|
|
1467
|
+
...p95 !== null ? {
|
|
1468
|
+
p95
|
|
1469
|
+
} : {},
|
|
1470
|
+
...p99 !== null ? {
|
|
1471
|
+
p99
|
|
1472
|
+
} : {}
|
|
1473
|
+
};
|
|
1474
|
+
}
|
|
1475
|
+
__name(toLatencyDistribution, "toLatencyDistribution");
|
|
1476
|
+
var SCORE_BINS = 10;
|
|
1477
|
+
function toScoreDistribution(retrievals, retriever) {
|
|
1478
|
+
const samples = [];
|
|
1479
|
+
const counts = new Array(SCORE_BINS).fill(0);
|
|
1480
|
+
for (const retrieval of retrievals) {
|
|
1481
|
+
if (retrieval.retriever !== retriever || retrieval.topScore === null) {
|
|
1482
|
+
continue;
|
|
1483
|
+
}
|
|
1484
|
+
samples.push(retrieval.topScore);
|
|
1485
|
+
const clamped = Math.min(Math.max(retrieval.topScore, 0), 1);
|
|
1486
|
+
const index = Math.min(Math.floor(clamped * SCORE_BINS), SCORE_BINS - 1);
|
|
1487
|
+
counts[index] = (counts[index] ?? 0) + 1;
|
|
1488
|
+
}
|
|
1489
|
+
const buckets = counts.map((count, index) => ({
|
|
1490
|
+
label: `${round(index / SCORE_BINS, 1)}\u2013${round((index + 1) / SCORE_BINS, 1)}`,
|
|
1491
|
+
count
|
|
1492
|
+
}));
|
|
1493
|
+
const p50 = percentile(samples, 0.5);
|
|
1494
|
+
const p95 = percentile(samples, 0.95);
|
|
1495
|
+
return {
|
|
1496
|
+
buckets,
|
|
1497
|
+
...p50 !== null ? {
|
|
1498
|
+
p50: round(p50, 4)
|
|
1499
|
+
} : {},
|
|
1500
|
+
...p95 !== null ? {
|
|
1501
|
+
p95: round(p95, 4)
|
|
1502
|
+
} : {}
|
|
1503
|
+
};
|
|
1504
|
+
}
|
|
1505
|
+
__name(toScoreDistribution, "toScoreDistribution");
|
|
1506
|
+
function hourStart(at) {
|
|
1507
|
+
return Math.floor(at.getTime() / 36e5) * 36e5;
|
|
1508
|
+
}
|
|
1509
|
+
__name(hourStart, "hourStart");
|
|
1510
|
+
function toRetrievalTrendRows(retrievals, now = /* @__PURE__ */ new Date()) {
|
|
1511
|
+
const latest = hourStart(now);
|
|
1512
|
+
const earliest = latest - (TREND_HOURS - 1) * 36e5;
|
|
1513
|
+
const buckets = /* @__PURE__ */ new Map();
|
|
1514
|
+
for (let hour = earliest; hour <= latest; hour += 36e5) {
|
|
1515
|
+
buckets.set(hour, {
|
|
1516
|
+
retrievals: 0,
|
|
1517
|
+
zeroHits: 0
|
|
1518
|
+
});
|
|
1519
|
+
}
|
|
1520
|
+
for (const retrieval of retrievals) {
|
|
1521
|
+
const bucket = buckets.get(hourStart(retrieval.at));
|
|
1522
|
+
if (bucket === void 0) {
|
|
1523
|
+
continue;
|
|
1524
|
+
}
|
|
1525
|
+
bucket.retrievals += 1;
|
|
1526
|
+
if (retrieval.zeroHit) {
|
|
1527
|
+
bucket.zeroHits += 1;
|
|
1528
|
+
}
|
|
1529
|
+
}
|
|
1530
|
+
return [
|
|
1531
|
+
...buckets.entries()
|
|
1532
|
+
].map(([hour, counts]) => ({
|
|
1533
|
+
label: `${new Date(hour).toISOString().slice(11, 13)}:00`,
|
|
1534
|
+
retrievals: counts.retrievals,
|
|
1535
|
+
zeroHits: counts.zeroHits
|
|
1536
|
+
}));
|
|
1537
|
+
}
|
|
1538
|
+
__name(toRetrievalTrendRows, "toRetrievalTrendRows");
|
|
1539
|
+
function segmentsBy(retrievals, pick) {
|
|
1540
|
+
const counts = /* @__PURE__ */ new Map();
|
|
1541
|
+
for (const retrieval of retrievals) {
|
|
1542
|
+
const label = pick(retrieval) ?? NO_VALUE2;
|
|
1543
|
+
counts.set(label, (counts.get(label) ?? 0) + 1);
|
|
1544
|
+
}
|
|
1545
|
+
return [
|
|
1546
|
+
...counts.entries()
|
|
1547
|
+
].map(([label, value]) => ({
|
|
1548
|
+
label,
|
|
1549
|
+
value
|
|
1550
|
+
})).sort((a, b) => b.value - a.value);
|
|
1551
|
+
}
|
|
1552
|
+
__name(segmentsBy, "segmentsBy");
|
|
1553
|
+
function toStoreSegments(retrievals) {
|
|
1554
|
+
return segmentsBy(retrievals, (retrieval) => retrieval.store);
|
|
1555
|
+
}
|
|
1556
|
+
__name(toStoreSegments, "toStoreSegments");
|
|
1557
|
+
function toRetrieverSegments(retrievals) {
|
|
1558
|
+
return segmentsBy(retrievals, (retrieval) => retrieval.retriever);
|
|
1559
|
+
}
|
|
1560
|
+
__name(toRetrieverSegments, "toRetrieverSegments");
|
|
1561
|
+
function toCollectionRows(retrievals, limit = TABLE_LIMIT) {
|
|
1562
|
+
const groups = /* @__PURE__ */ new Map();
|
|
1563
|
+
for (const retrieval of retrievals) {
|
|
1564
|
+
const key = retrieval.collection ?? NO_VALUE2;
|
|
1565
|
+
const group = groups.get(key);
|
|
1566
|
+
if (group === void 0) {
|
|
1567
|
+
groups.set(key, [
|
|
1568
|
+
retrieval
|
|
1569
|
+
]);
|
|
1570
|
+
} else {
|
|
1571
|
+
group.push(retrieval);
|
|
1572
|
+
}
|
|
1573
|
+
}
|
|
1574
|
+
return [
|
|
1575
|
+
...groups.entries()
|
|
1576
|
+
].map(([collection, group]) => {
|
|
1577
|
+
const durations = [];
|
|
1578
|
+
const scores = [];
|
|
1579
|
+
for (const retrieval of group) {
|
|
1580
|
+
if (retrieval.durationMs !== null) {
|
|
1581
|
+
durations.push(retrieval.durationMs);
|
|
1582
|
+
}
|
|
1583
|
+
if (retrieval.topScore !== null) {
|
|
1584
|
+
scores.push(retrieval.topScore);
|
|
1585
|
+
}
|
|
1586
|
+
}
|
|
1587
|
+
const p95 = percentile(durations, 0.95);
|
|
1588
|
+
const meanScore = scores.length === 0 ? null : scores.reduce((sum, score) => sum + score, 0) / scores.length;
|
|
1589
|
+
return {
|
|
1590
|
+
collection,
|
|
1591
|
+
store: group[0]?.store ?? NO_VALUE2,
|
|
1592
|
+
retrievals: group.length,
|
|
1593
|
+
zeroHits: group.filter((retrieval) => retrieval.zeroHit).length,
|
|
1594
|
+
p95Ms: p95 === null ? NO_VALUE2 : Math.round(p95),
|
|
1595
|
+
meanTopScore: meanScore === null ? NO_VALUE2 : round(meanScore, 3)
|
|
1596
|
+
};
|
|
1597
|
+
}).sort((a, b) => b.retrievals - a.retrievals).slice(0, limit);
|
|
1598
|
+
}
|
|
1599
|
+
__name(toCollectionRows, "toCollectionRows");
|
|
1600
|
+
function toSlowestRows(retrievals, limit = TABLE_LIMIT) {
|
|
1601
|
+
return retrievals.filter((retrieval) => retrieval.durationMs !== null).sort((a, b) => (b.durationMs ?? 0) - (a.durationMs ?? 0)).slice(0, limit).map((retrieval) => ({
|
|
1602
|
+
at: retrieval.at.toISOString(),
|
|
1603
|
+
retriever: retrieval.retriever,
|
|
1604
|
+
store: retrieval.store ?? NO_VALUE2,
|
|
1605
|
+
collection: retrieval.collection ?? NO_VALUE2,
|
|
1606
|
+
chunks: retrieval.chunks,
|
|
1607
|
+
topScore: retrieval.topScore === null ? NO_VALUE2 : round(retrieval.topScore, 3),
|
|
1608
|
+
durationMs: retrieval.durationMs === null ? NO_VALUE2 : Math.round(retrieval.durationMs)
|
|
1609
|
+
}));
|
|
1610
|
+
}
|
|
1611
|
+
__name(toSlowestRows, "toSlowestRows");
|
|
1612
|
+
function retrievalProvider(name, format) {
|
|
1613
|
+
return {
|
|
1614
|
+
name,
|
|
1615
|
+
async resolve(query, ctx) {
|
|
1616
|
+
return format(await readRetrievals(retrievalStorage(ctx)), query);
|
|
1617
|
+
}
|
|
1618
|
+
};
|
|
1619
|
+
}
|
|
1620
|
+
__name(retrievalProvider, "retrievalProvider");
|
|
1621
|
+
function ragRetrievalsProvider() {
|
|
1622
|
+
return retrievalProvider("agent.rag.retrievals", (retrievals) => ({
|
|
1623
|
+
value: retrievals.length
|
|
1624
|
+
}));
|
|
1625
|
+
}
|
|
1626
|
+
__name(ragRetrievalsProvider, "ragRetrievalsProvider");
|
|
1627
|
+
function ragZeroHitRateProvider() {
|
|
1628
|
+
return retrievalProvider("agent.rag.zeroHitRate", (retrievals) => ({
|
|
1629
|
+
value: zeroHitRate(retrievals)
|
|
1630
|
+
}));
|
|
1631
|
+
}
|
|
1632
|
+
__name(ragZeroHitRateProvider, "ragZeroHitRateProvider");
|
|
1633
|
+
function ragChunksProvider() {
|
|
1634
|
+
return retrievalProvider("agent.rag.chunks", (retrievals) => ({
|
|
1635
|
+
value: meanChunks(retrievals)
|
|
1636
|
+
}));
|
|
1637
|
+
}
|
|
1638
|
+
__name(ragChunksProvider, "ragChunksProvider");
|
|
1639
|
+
function ragLatencyProvider() {
|
|
1640
|
+
return retrievalProvider("agent.rag.latency", toLatencyDistribution);
|
|
1641
|
+
}
|
|
1642
|
+
__name(ragLatencyProvider, "ragLatencyProvider");
|
|
1643
|
+
function ragScoresProvider() {
|
|
1644
|
+
return retrievalProvider("agent.rag.scores", (retrievals, query) => {
|
|
1645
|
+
const requested = query?.retriever;
|
|
1646
|
+
return toScoreDistribution(retrievals, typeof requested === "string" ? requested : "embedding");
|
|
1647
|
+
});
|
|
1648
|
+
}
|
|
1649
|
+
__name(ragScoresProvider, "ragScoresProvider");
|
|
1650
|
+
function ragTrendProvider() {
|
|
1651
|
+
return retrievalProvider("agent.rag.trend", (retrievals) => ({
|
|
1652
|
+
rows: toRetrievalTrendRows(retrievals)
|
|
1653
|
+
}));
|
|
1654
|
+
}
|
|
1655
|
+
__name(ragTrendProvider, "ragTrendProvider");
|
|
1656
|
+
function ragStoreBreakdownProvider() {
|
|
1657
|
+
return retrievalProvider("agent.rag.byStore", (retrievals) => ({
|
|
1658
|
+
segments: toStoreSegments(retrievals)
|
|
1659
|
+
}));
|
|
1660
|
+
}
|
|
1661
|
+
__name(ragStoreBreakdownProvider, "ragStoreBreakdownProvider");
|
|
1662
|
+
function ragRetrieverBreakdownProvider() {
|
|
1663
|
+
return retrievalProvider("agent.rag.byRetriever", (retrievals) => ({
|
|
1664
|
+
segments: toRetrieverSegments(retrievals)
|
|
1665
|
+
}));
|
|
1666
|
+
}
|
|
1667
|
+
__name(ragRetrieverBreakdownProvider, "ragRetrieverBreakdownProvider");
|
|
1668
|
+
function ragCollectionsTableProvider() {
|
|
1669
|
+
return retrievalProvider("agent.rag.byCollection", (retrievals) => ({
|
|
1670
|
+
rows: toCollectionRows(retrievals)
|
|
1671
|
+
}));
|
|
1672
|
+
}
|
|
1673
|
+
__name(ragCollectionsTableProvider, "ragCollectionsTableProvider");
|
|
1674
|
+
function ragSlowestTableProvider() {
|
|
1675
|
+
return retrievalProvider("agent.rag.slowest", (retrievals) => ({
|
|
1676
|
+
rows: toSlowestRows(retrievals)
|
|
1677
|
+
}));
|
|
1678
|
+
}
|
|
1679
|
+
__name(ragSlowestTableProvider, "ragSlowestTableProvider");
|
|
1680
|
+
|
|
1069
1681
|
// src/agent-telescope.extension.ts
|
|
1682
|
+
var RESERVED_PROVIDER_PREFIX = "agent.";
|
|
1070
1683
|
function agentTelescopeExtension(opts = {}) {
|
|
1684
|
+
const hostProviders = opts.providers ?? [];
|
|
1685
|
+
const reserved = hostProviders.filter((provider) => provider.name.startsWith(RESERVED_PROVIDER_PREFIX));
|
|
1686
|
+
if (reserved.length > 0) {
|
|
1687
|
+
throw new Error(`agentTelescopeExtension: host-contributed data providers may not use the reserved "${RESERVED_PROVIDER_PREFIX}" prefix \u2014 rename ${reserved.map((provider) => `"${provider.name}"`).join(", ")} to your own namespace.`);
|
|
1688
|
+
}
|
|
1071
1689
|
return defineTelescopeExtension({
|
|
1072
1690
|
name: "agent",
|
|
1073
1691
|
watchers: /* @__PURE__ */ __name(() => [
|
|
1074
|
-
new AgentTelescopeWatcher()
|
|
1692
|
+
new AgentTelescopeWatcher(),
|
|
1693
|
+
new RagTelescopeWatcher()
|
|
1075
1694
|
], "watchers"),
|
|
1076
1695
|
entryTypes: /* @__PURE__ */ __name(() => [
|
|
1077
1696
|
{
|
|
1078
1697
|
id: "agent",
|
|
1079
1698
|
label: "Agent",
|
|
1080
1699
|
dot: "bg-violet-400"
|
|
1700
|
+
},
|
|
1701
|
+
{
|
|
1702
|
+
id: "agent-rag",
|
|
1703
|
+
label: "RAG",
|
|
1704
|
+
dot: "bg-emerald-400"
|
|
1081
1705
|
}
|
|
1082
1706
|
], "entryTypes"),
|
|
1083
1707
|
dashboards: /* @__PURE__ */ __name(() => [
|
|
1084
|
-
agentDashboard(
|
|
1708
|
+
agentDashboard({
|
|
1709
|
+
...opts.threadHref !== void 0 ? {
|
|
1710
|
+
threadHref: opts.threadHref
|
|
1711
|
+
} : {},
|
|
1712
|
+
...opts.runHref !== void 0 ? {
|
|
1713
|
+
runHref: opts.runHref
|
|
1714
|
+
} : {},
|
|
1715
|
+
...opts.sections !== void 0 ? {
|
|
1716
|
+
sections: opts.sections
|
|
1717
|
+
} : {}
|
|
1718
|
+
})
|
|
1085
1719
|
], "dashboards"),
|
|
1086
1720
|
dataProviders: /* @__PURE__ */ __name(() => [
|
|
1087
1721
|
agentRunsProvider(),
|
|
@@ -1111,13 +1745,26 @@ function agentTelescopeExtension(opts = {}) {
|
|
|
1111
1745
|
agentThreadsPagedTableProvider(),
|
|
1112
1746
|
agentPendingApprovalsCountProvider(),
|
|
1113
1747
|
agentPendingApprovalsTableProvider(),
|
|
1114
|
-
agentToolStatsTableProvider()
|
|
1748
|
+
agentToolStatsTableProvider(),
|
|
1749
|
+
ragRetrievalsProvider(),
|
|
1750
|
+
ragZeroHitRateProvider(),
|
|
1751
|
+
ragChunksProvider(),
|
|
1752
|
+
ragLatencyProvider(),
|
|
1753
|
+
ragScoresProvider(),
|
|
1754
|
+
ragTrendProvider(),
|
|
1755
|
+
ragStoreBreakdownProvider(),
|
|
1756
|
+
ragRetrieverBreakdownProvider(),
|
|
1757
|
+
ragCollectionsTableProvider(),
|
|
1758
|
+
ragSlowestTableProvider(),
|
|
1759
|
+
...hostProviders
|
|
1115
1760
|
], "dataProviders")
|
|
1116
1761
|
});
|
|
1117
1762
|
}
|
|
1118
1763
|
__name(agentTelescopeExtension, "agentTelescopeExtension");
|
|
1119
1764
|
export {
|
|
1120
1765
|
AgentTelescopeWatcher,
|
|
1766
|
+
RAG_ENTRY_TYPE,
|
|
1767
|
+
RagTelescopeWatcher,
|
|
1121
1768
|
agentActorSpendTableProvider,
|
|
1122
1769
|
agentDashboard,
|
|
1123
1770
|
agentModelSpendTableProvider,
|
|
@@ -1147,24 +1794,46 @@ export {
|
|
|
1147
1794
|
agentTopThreadsTableProvider,
|
|
1148
1795
|
agentUsageTrendProvider,
|
|
1149
1796
|
capErrorMessage,
|
|
1797
|
+
meanChunks,
|
|
1798
|
+
percentile,
|
|
1799
|
+
ragChunksProvider,
|
|
1800
|
+
ragCollectionsTableProvider,
|
|
1801
|
+
ragLatencyProvider,
|
|
1802
|
+
ragRetrievalsProvider,
|
|
1803
|
+
ragRetrieverBreakdownProvider,
|
|
1804
|
+
ragScoresProvider,
|
|
1805
|
+
ragSlowestTableProvider,
|
|
1806
|
+
ragStoreBreakdownProvider,
|
|
1807
|
+
ragTrendProvider,
|
|
1808
|
+
ragZeroHitRateProvider,
|
|
1809
|
+
readRetrievals,
|
|
1150
1810
|
resolveRange,
|
|
1151
1811
|
shiftUtcDay,
|
|
1152
1812
|
shortPromptHash,
|
|
1153
1813
|
toActorSpendRows,
|
|
1154
1814
|
toActorSpendSegments,
|
|
1815
|
+
toCollectionRows,
|
|
1816
|
+
toLatencyDistribution,
|
|
1155
1817
|
toModelSpendRows,
|
|
1156
1818
|
toModelSpendSegments,
|
|
1157
1819
|
toPendingApprovalTableRows,
|
|
1820
|
+
toRagRetrieval,
|
|
1158
1821
|
toRecentRunTableRows,
|
|
1159
1822
|
toRecentThreadTableRows,
|
|
1160
1823
|
toRecentToolCallRows,
|
|
1824
|
+
toRetrievalTrendRows,
|
|
1825
|
+
toRetrieverSegments,
|
|
1161
1826
|
toRunAgentTableRows,
|
|
1162
1827
|
toRunErrorSegments,
|
|
1163
1828
|
toRunTrendRows,
|
|
1829
|
+
toScoreDistribution,
|
|
1830
|
+
toSlowestRows,
|
|
1831
|
+
toStoreSegments,
|
|
1164
1832
|
toThreadSpendRows,
|
|
1165
1833
|
toToolStatTableRows,
|
|
1166
1834
|
toUsageTrendRows,
|
|
1167
1835
|
totalCostUsd,
|
|
1168
|
-
totalTokens
|
|
1836
|
+
totalTokens,
|
|
1837
|
+
zeroHitRate
|
|
1169
1838
|
};
|
|
1170
1839
|
//# sourceMappingURL=index.js.map
|