@dudousxd/nestjs-agent-telescope 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -1
- package/dist/index.cjs +696 -8
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +223 -16
- package/dist/index.d.ts +223 -16
- package/dist/index.js +669 -5
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
package/dist/index.cjs
CHANGED
|
@@ -22,6 +22,8 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
22
22
|
var index_exports = {};
|
|
23
23
|
__export(index_exports, {
|
|
24
24
|
AgentTelescopeWatcher: () => AgentTelescopeWatcher,
|
|
25
|
+
RAG_ENTRY_TYPE: () => RAG_ENTRY_TYPE,
|
|
26
|
+
RagTelescopeWatcher: () => RagTelescopeWatcher,
|
|
25
27
|
agentActorSpendTableProvider: () => agentActorSpendTableProvider,
|
|
26
28
|
agentDashboard: () => agentDashboard,
|
|
27
29
|
agentModelSpendTableProvider: () => agentModelSpendTableProvider,
|
|
@@ -51,30 +53,52 @@ __export(index_exports, {
|
|
|
51
53
|
agentTopThreadsTableProvider: () => agentTopThreadsTableProvider,
|
|
52
54
|
agentUsageTrendProvider: () => agentUsageTrendProvider,
|
|
53
55
|
capErrorMessage: () => capErrorMessage,
|
|
56
|
+
meanChunks: () => meanChunks,
|
|
57
|
+
percentile: () => percentile,
|
|
58
|
+
ragChunksProvider: () => ragChunksProvider,
|
|
59
|
+
ragCollectionsTableProvider: () => ragCollectionsTableProvider,
|
|
60
|
+
ragLatencyProvider: () => ragLatencyProvider,
|
|
61
|
+
ragRetrievalsProvider: () => ragRetrievalsProvider,
|
|
62
|
+
ragRetrieverBreakdownProvider: () => ragRetrieverBreakdownProvider,
|
|
63
|
+
ragScoresProvider: () => ragScoresProvider,
|
|
64
|
+
ragSlowestTableProvider: () => ragSlowestTableProvider,
|
|
65
|
+
ragStoreBreakdownProvider: () => ragStoreBreakdownProvider,
|
|
66
|
+
ragTrendProvider: () => ragTrendProvider,
|
|
67
|
+
ragZeroHitRateProvider: () => ragZeroHitRateProvider,
|
|
68
|
+
readRetrievals: () => readRetrievals,
|
|
54
69
|
resolveRange: () => resolveRange,
|
|
55
70
|
shiftUtcDay: () => shiftUtcDay,
|
|
56
71
|
shortPromptHash: () => shortPromptHash,
|
|
57
72
|
toActorSpendRows: () => toActorSpendRows,
|
|
58
73
|
toActorSpendSegments: () => toActorSpendSegments,
|
|
74
|
+
toCollectionRows: () => toCollectionRows,
|
|
75
|
+
toLatencyDistribution: () => toLatencyDistribution,
|
|
59
76
|
toModelSpendRows: () => toModelSpendRows,
|
|
60
77
|
toModelSpendSegments: () => toModelSpendSegments,
|
|
61
78
|
toPendingApprovalTableRows: () => toPendingApprovalTableRows,
|
|
79
|
+
toRagRetrieval: () => toRagRetrieval,
|
|
62
80
|
toRecentRunTableRows: () => toRecentRunTableRows,
|
|
63
81
|
toRecentThreadTableRows: () => toRecentThreadTableRows,
|
|
64
82
|
toRecentToolCallRows: () => toRecentToolCallRows,
|
|
83
|
+
toRetrievalTrendRows: () => toRetrievalTrendRows,
|
|
84
|
+
toRetrieverSegments: () => toRetrieverSegments,
|
|
65
85
|
toRunAgentTableRows: () => toRunAgentTableRows,
|
|
66
86
|
toRunErrorSegments: () => toRunErrorSegments,
|
|
67
87
|
toRunTrendRows: () => toRunTrendRows,
|
|
88
|
+
toScoreDistribution: () => toScoreDistribution,
|
|
89
|
+
toSlowestRows: () => toSlowestRows,
|
|
90
|
+
toStoreSegments: () => toStoreSegments,
|
|
68
91
|
toThreadSpendRows: () => toThreadSpendRows,
|
|
69
92
|
toToolStatTableRows: () => toToolStatTableRows,
|
|
70
93
|
toUsageTrendRows: () => toUsageTrendRows,
|
|
71
94
|
totalCostUsd: () => totalCostUsd,
|
|
72
|
-
totalTokens: () => totalTokens
|
|
95
|
+
totalTokens: () => totalTokens,
|
|
96
|
+
zeroHitRate: () => zeroHitRate
|
|
73
97
|
});
|
|
74
98
|
module.exports = __toCommonJS(index_exports);
|
|
75
99
|
|
|
76
100
|
// src/agent-telescope.extension.ts
|
|
77
|
-
var
|
|
101
|
+
var import_nestjs_telescope3 = require("@dudousxd/nestjs-telescope");
|
|
78
102
|
|
|
79
103
|
// src/agent-dashboard.spec-data.ts
|
|
80
104
|
function col(key, label, href) {
|
|
@@ -564,7 +588,180 @@ function agentDashboard(opts = {}) {
|
|
|
564
588
|
]
|
|
565
589
|
}
|
|
566
590
|
]
|
|
567
|
-
}
|
|
591
|
+
},
|
|
592
|
+
{
|
|
593
|
+
// RAG. Fed by the `aviary:rag:retrieval` telemetry `@dudousxd/nestjs-agent-rag` emits, NOT
|
|
594
|
+
// derived from tool-call rows: a tool call that retrieved nothing and one that retrieved
|
|
595
|
+
// five perfect passages are the same row with the same `ok` status, so every number here
|
|
596
|
+
// would have been unanswerable from that side.
|
|
597
|
+
title: "Retrieval",
|
|
598
|
+
cols: 3,
|
|
599
|
+
panels: [
|
|
600
|
+
{
|
|
601
|
+
kind: "stat",
|
|
602
|
+
title: "Retrievals",
|
|
603
|
+
data: {
|
|
604
|
+
provider: "agent.rag.retrievals"
|
|
605
|
+
}
|
|
606
|
+
},
|
|
607
|
+
{
|
|
608
|
+
kind: "stat",
|
|
609
|
+
title: "Zero-hit rate",
|
|
610
|
+
data: {
|
|
611
|
+
provider: "agent.rag.zeroHitRate"
|
|
612
|
+
},
|
|
613
|
+
format: "percent"
|
|
614
|
+
},
|
|
615
|
+
{
|
|
616
|
+
kind: "stat",
|
|
617
|
+
title: "Passages per retrieval",
|
|
618
|
+
data: {
|
|
619
|
+
provider: "agent.rag.chunks"
|
|
620
|
+
},
|
|
621
|
+
format: "number"
|
|
622
|
+
},
|
|
623
|
+
{
|
|
624
|
+
// A real histogram, unlike the run-duration pair above: retrieval events carry the raw
|
|
625
|
+
// per-call duration, so there ARE samples to bucket and markers to place.
|
|
626
|
+
kind: "distribution",
|
|
627
|
+
title: "Retrieval latency",
|
|
628
|
+
data: {
|
|
629
|
+
provider: "agent.rag.latency"
|
|
630
|
+
},
|
|
631
|
+
markers: [
|
|
632
|
+
"p50",
|
|
633
|
+
"p95",
|
|
634
|
+
"p99"
|
|
635
|
+
],
|
|
636
|
+
format: "duration"
|
|
637
|
+
},
|
|
638
|
+
{
|
|
639
|
+
// Bound to ONE retriever kind on purpose — a cosine similarity, a BM25 score and an RRF
|
|
640
|
+
// rank score share no scale, so a histogram over all of them has bins that mean a
|
|
641
|
+
// different thing per bar. See `toScoreDistribution`.
|
|
642
|
+
kind: "distribution",
|
|
643
|
+
title: "Top score (dense)",
|
|
644
|
+
data: {
|
|
645
|
+
provider: "agent.rag.scores",
|
|
646
|
+
query: {
|
|
647
|
+
retriever: "embedding"
|
|
648
|
+
}
|
|
649
|
+
},
|
|
650
|
+
markers: [
|
|
651
|
+
"p50",
|
|
652
|
+
"p95"
|
|
653
|
+
],
|
|
654
|
+
format: "number"
|
|
655
|
+
},
|
|
656
|
+
{
|
|
657
|
+
kind: "timeseries",
|
|
658
|
+
title: "Retrievals & zero-hits",
|
|
659
|
+
data: {
|
|
660
|
+
provider: "agent.rag.trend"
|
|
661
|
+
},
|
|
662
|
+
series: [
|
|
663
|
+
"retrievals",
|
|
664
|
+
"zeroHits"
|
|
665
|
+
],
|
|
666
|
+
style: "stacked"
|
|
667
|
+
}
|
|
668
|
+
]
|
|
669
|
+
},
|
|
670
|
+
{
|
|
671
|
+
title: "Retrieval sources",
|
|
672
|
+
cols: 2,
|
|
673
|
+
panels: [
|
|
674
|
+
{
|
|
675
|
+
kind: "breakdown",
|
|
676
|
+
title: "Retrievals by store",
|
|
677
|
+
data: {
|
|
678
|
+
provider: "agent.rag.byStore"
|
|
679
|
+
},
|
|
680
|
+
style: "donut"
|
|
681
|
+
},
|
|
682
|
+
{
|
|
683
|
+
kind: "breakdown",
|
|
684
|
+
title: "Retrievals by retriever",
|
|
685
|
+
data: {
|
|
686
|
+
provider: "agent.rag.byRetriever"
|
|
687
|
+
},
|
|
688
|
+
style: "donut"
|
|
689
|
+
},
|
|
690
|
+
{
|
|
691
|
+
kind: "table",
|
|
692
|
+
title: "By collection",
|
|
693
|
+
data: {
|
|
694
|
+
provider: "agent.rag.byCollection"
|
|
695
|
+
},
|
|
696
|
+
columns: [
|
|
697
|
+
{
|
|
698
|
+
key: "collection",
|
|
699
|
+
label: "Collection"
|
|
700
|
+
},
|
|
701
|
+
{
|
|
702
|
+
key: "store",
|
|
703
|
+
label: "Store"
|
|
704
|
+
},
|
|
705
|
+
{
|
|
706
|
+
key: "retrievals",
|
|
707
|
+
label: "Retrievals"
|
|
708
|
+
},
|
|
709
|
+
{
|
|
710
|
+
key: "zeroHits",
|
|
711
|
+
label: "Zero-hits"
|
|
712
|
+
},
|
|
713
|
+
{
|
|
714
|
+
key: "p95Ms",
|
|
715
|
+
label: "p95 (ms)"
|
|
716
|
+
},
|
|
717
|
+
{
|
|
718
|
+
key: "meanTopScore",
|
|
719
|
+
label: "Mean top score"
|
|
720
|
+
}
|
|
721
|
+
]
|
|
722
|
+
},
|
|
723
|
+
{
|
|
724
|
+
kind: "table",
|
|
725
|
+
title: "Slowest retrievals",
|
|
726
|
+
data: {
|
|
727
|
+
provider: "agent.rag.slowest"
|
|
728
|
+
},
|
|
729
|
+
// Seven columns, matching the recent-runs table's width: this is the widest a table can
|
|
730
|
+
// get here before it overflows its card.
|
|
731
|
+
columns: [
|
|
732
|
+
{
|
|
733
|
+
key: "at",
|
|
734
|
+
label: "When"
|
|
735
|
+
},
|
|
736
|
+
{
|
|
737
|
+
key: "retriever",
|
|
738
|
+
label: "Retriever"
|
|
739
|
+
},
|
|
740
|
+
{
|
|
741
|
+
key: "store",
|
|
742
|
+
label: "Store"
|
|
743
|
+
},
|
|
744
|
+
{
|
|
745
|
+
key: "collection",
|
|
746
|
+
label: "Collection"
|
|
747
|
+
},
|
|
748
|
+
{
|
|
749
|
+
key: "chunks",
|
|
750
|
+
label: "Passages"
|
|
751
|
+
},
|
|
752
|
+
{
|
|
753
|
+
key: "topScore",
|
|
754
|
+
label: "Top score"
|
|
755
|
+
},
|
|
756
|
+
{
|
|
757
|
+
key: "durationMs",
|
|
758
|
+
label: "Duration (ms)"
|
|
759
|
+
}
|
|
760
|
+
]
|
|
761
|
+
}
|
|
762
|
+
]
|
|
763
|
+
},
|
|
764
|
+
...opts.sections ?? []
|
|
568
765
|
]
|
|
569
766
|
};
|
|
570
767
|
}
|
|
@@ -1138,22 +1335,478 @@ var AgentTelescopeWatcher = class {
|
|
|
1138
1335
|
}
|
|
1139
1336
|
};
|
|
1140
1337
|
|
|
1338
|
+
// src/rag-data-providers.ts
|
|
1339
|
+
var import_nestjs_telescope2 = require("@dudousxd/nestjs-telescope");
|
|
1340
|
+
|
|
1341
|
+
// src/rag-telescope.watcher.ts
|
|
1342
|
+
var import_node_diagnostics_channel2 = require("diagnostics_channel");
|
|
1343
|
+
var import_nestjs_diagnostics2 = require("@dudousxd/nestjs-diagnostics");
|
|
1344
|
+
var RAG_LIB = "rag";
|
|
1345
|
+
var RETRIEVAL_EVENT = "retrieval";
|
|
1346
|
+
var RAG_ENTRY_TYPE = "agent-rag";
|
|
1347
|
+
function isEnvelope(message) {
|
|
1348
|
+
return typeof message === "object" && message !== null && typeof Reflect.get(message, "payload") === "object" && Reflect.get(message, "payload") !== null;
|
|
1349
|
+
}
|
|
1350
|
+
__name(isEnvelope, "isEnvelope");
|
|
1351
|
+
var RagTelescopeWatcher = class {
|
|
1352
|
+
static {
|
|
1353
|
+
__name(this, "RagTelescopeWatcher");
|
|
1354
|
+
}
|
|
1355
|
+
type = RAG_ENTRY_TYPE;
|
|
1356
|
+
disposers = [];
|
|
1357
|
+
register(ctx) {
|
|
1358
|
+
this.disposers.push((0, import_nestjs_diagnostics2.claimDiagnostics)(RAG_LIB, [
|
|
1359
|
+
RETRIEVAL_EVENT
|
|
1360
|
+
]));
|
|
1361
|
+
const channel = (0, import_nestjs_diagnostics2.channelName)(RAG_LIB, RETRIEVAL_EVENT);
|
|
1362
|
+
const onMessage = /* @__PURE__ */ __name((message) => {
|
|
1363
|
+
if (!isEnvelope(message)) {
|
|
1364
|
+
return;
|
|
1365
|
+
}
|
|
1366
|
+
ctx.record({
|
|
1367
|
+
type: RAG_ENTRY_TYPE,
|
|
1368
|
+
content: {
|
|
1369
|
+
event: RETRIEVAL_EVENT,
|
|
1370
|
+
...message.payload,
|
|
1371
|
+
...message.durationMs !== void 0 ? {
|
|
1372
|
+
durationMs: message.durationMs
|
|
1373
|
+
} : {}
|
|
1374
|
+
},
|
|
1375
|
+
// Tagged by store and retriever kind so the Entries screen can filter to "every retrieval
|
|
1376
|
+
// pgvector served" without a provider — the same tag-per-dimension the agent watcher uses.
|
|
1377
|
+
tags: tagsOf(message.payload)
|
|
1378
|
+
});
|
|
1379
|
+
}, "onMessage");
|
|
1380
|
+
(0, import_node_diagnostics_channel2.subscribe)(channel, onMessage);
|
|
1381
|
+
this.disposers.push(() => (0, import_node_diagnostics_channel2.unsubscribe)(channel, onMessage));
|
|
1382
|
+
}
|
|
1383
|
+
/** Detach the subscription and release the diagnostics claim (e.g. on module destroy). */
|
|
1384
|
+
dispose() {
|
|
1385
|
+
while (this.disposers.length) this.disposers.pop()?.();
|
|
1386
|
+
}
|
|
1387
|
+
};
|
|
1388
|
+
function tagsOf(payload) {
|
|
1389
|
+
const tags = [
|
|
1390
|
+
RETRIEVAL_EVENT
|
|
1391
|
+
];
|
|
1392
|
+
if (typeof payload.retriever === "string") {
|
|
1393
|
+
tags.push(`retriever:${payload.retriever}`);
|
|
1394
|
+
}
|
|
1395
|
+
if (typeof payload.store === "string") {
|
|
1396
|
+
tags.push(`store:${payload.store}`);
|
|
1397
|
+
}
|
|
1398
|
+
if (payload.zeroHit === true) {
|
|
1399
|
+
tags.push("zero-hit");
|
|
1400
|
+
}
|
|
1401
|
+
if (payload.failed === true) {
|
|
1402
|
+
tags.push("failed");
|
|
1403
|
+
}
|
|
1404
|
+
return tags;
|
|
1405
|
+
}
|
|
1406
|
+
__name(tagsOf, "tagsOf");
|
|
1407
|
+
|
|
1408
|
+
// src/rag-data-providers.ts
|
|
1409
|
+
var RETRIEVAL_WINDOW = 5e3;
|
|
1410
|
+
var TABLE_LIMIT = 10;
|
|
1411
|
+
var TREND_HOURS = 24;
|
|
1412
|
+
var NO_VALUE2 = "\u2014";
|
|
1413
|
+
function isRecord2(value) {
|
|
1414
|
+
return typeof value === "object" && value !== null;
|
|
1415
|
+
}
|
|
1416
|
+
__name(isRecord2, "isRecord");
|
|
1417
|
+
function readString(source, key) {
|
|
1418
|
+
const value = source[key];
|
|
1419
|
+
return typeof value === "string" ? value : null;
|
|
1420
|
+
}
|
|
1421
|
+
__name(readString, "readString");
|
|
1422
|
+
function readNumber(source, key) {
|
|
1423
|
+
const value = source[key];
|
|
1424
|
+
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
1425
|
+
}
|
|
1426
|
+
__name(readNumber, "readNumber");
|
|
1427
|
+
function isEntryStorage(value) {
|
|
1428
|
+
return isRecord2(value) && typeof value.get === "function";
|
|
1429
|
+
}
|
|
1430
|
+
__name(isEntryStorage, "isEntryStorage");
|
|
1431
|
+
function toRagRetrieval(entry) {
|
|
1432
|
+
if (!isRecord2(entry) || !isRecord2(entry.content)) {
|
|
1433
|
+
return null;
|
|
1434
|
+
}
|
|
1435
|
+
const content = entry.content;
|
|
1436
|
+
const retriever = readString(content, "retriever");
|
|
1437
|
+
if (retriever === null) {
|
|
1438
|
+
return null;
|
|
1439
|
+
}
|
|
1440
|
+
const chunks = readNumber(content, "chunks") ?? 0;
|
|
1441
|
+
const createdAt = entry.createdAt;
|
|
1442
|
+
return {
|
|
1443
|
+
at: createdAt instanceof Date ? createdAt : /* @__PURE__ */ new Date(0),
|
|
1444
|
+
retriever,
|
|
1445
|
+
store: readString(content, "store"),
|
|
1446
|
+
collection: readString(content, "collection"),
|
|
1447
|
+
chunks,
|
|
1448
|
+
zeroHit: content.zeroHit === true || chunks === 0,
|
|
1449
|
+
topScore: readNumber(content, "topScore"),
|
|
1450
|
+
durationMs: readNumber(content, "durationMs"),
|
|
1451
|
+
failed: content.failed === true
|
|
1452
|
+
};
|
|
1453
|
+
}
|
|
1454
|
+
__name(toRagRetrieval, "toRagRetrieval");
|
|
1455
|
+
async function readRetrievals(storage) {
|
|
1456
|
+
if (!isEntryStorage(storage)) {
|
|
1457
|
+
return [];
|
|
1458
|
+
}
|
|
1459
|
+
const page = await storage.get({
|
|
1460
|
+
type: RAG_ENTRY_TYPE,
|
|
1461
|
+
limit: RETRIEVAL_WINDOW
|
|
1462
|
+
});
|
|
1463
|
+
const retrievals = [];
|
|
1464
|
+
for (const entry of page.data) {
|
|
1465
|
+
const retrieval = toRagRetrieval(entry);
|
|
1466
|
+
if (retrieval !== null) {
|
|
1467
|
+
retrievals.push(retrieval);
|
|
1468
|
+
}
|
|
1469
|
+
}
|
|
1470
|
+
return retrievals;
|
|
1471
|
+
}
|
|
1472
|
+
__name(readRetrievals, "readRetrievals");
|
|
1473
|
+
function retrievalStorage(ctx) {
|
|
1474
|
+
try {
|
|
1475
|
+
return ctx.moduleRef.get(import_nestjs_telescope2.TELESCOPE_STORAGE, {
|
|
1476
|
+
strict: false
|
|
1477
|
+
});
|
|
1478
|
+
} catch {
|
|
1479
|
+
return void 0;
|
|
1480
|
+
}
|
|
1481
|
+
}
|
|
1482
|
+
__name(retrievalStorage, "retrievalStorage");
|
|
1483
|
+
function percentile(samples, fraction) {
|
|
1484
|
+
if (samples.length === 0) {
|
|
1485
|
+
return null;
|
|
1486
|
+
}
|
|
1487
|
+
const sorted = [
|
|
1488
|
+
...samples
|
|
1489
|
+
].sort((a, b) => a - b);
|
|
1490
|
+
const rank = Math.ceil(fraction * sorted.length);
|
|
1491
|
+
return sorted[Math.min(Math.max(rank, 1), sorted.length) - 1] ?? null;
|
|
1492
|
+
}
|
|
1493
|
+
__name(percentile, "percentile");
|
|
1494
|
+
function round(value, places) {
|
|
1495
|
+
const factor = 10 ** places;
|
|
1496
|
+
return Math.round(value * factor) / factor;
|
|
1497
|
+
}
|
|
1498
|
+
__name(round, "round");
|
|
1499
|
+
function zeroHitRate(retrievals) {
|
|
1500
|
+
if (retrievals.length === 0) {
|
|
1501
|
+
return 0;
|
|
1502
|
+
}
|
|
1503
|
+
const zeroHits = retrievals.filter((retrieval) => retrieval.zeroHit).length;
|
|
1504
|
+
return round(zeroHits / retrievals.length, 4);
|
|
1505
|
+
}
|
|
1506
|
+
__name(zeroHitRate, "zeroHitRate");
|
|
1507
|
+
function meanChunks(retrievals) {
|
|
1508
|
+
if (retrievals.length === 0) {
|
|
1509
|
+
return 0;
|
|
1510
|
+
}
|
|
1511
|
+
const total = retrievals.reduce((sum, retrieval) => sum + retrieval.chunks, 0);
|
|
1512
|
+
return round(total / retrievals.length, 2);
|
|
1513
|
+
}
|
|
1514
|
+
__name(meanChunks, "meanChunks");
|
|
1515
|
+
var LATENCY_EDGES = [
|
|
1516
|
+
10,
|
|
1517
|
+
25,
|
|
1518
|
+
50,
|
|
1519
|
+
100,
|
|
1520
|
+
250,
|
|
1521
|
+
500,
|
|
1522
|
+
1e3,
|
|
1523
|
+
2500,
|
|
1524
|
+
5e3
|
|
1525
|
+
];
|
|
1526
|
+
function latencyLabel(index) {
|
|
1527
|
+
const upper = LATENCY_EDGES[index];
|
|
1528
|
+
if (upper === void 0) {
|
|
1529
|
+
return `\u2265${LATENCY_EDGES[LATENCY_EDGES.length - 1] ?? 0}ms`;
|
|
1530
|
+
}
|
|
1531
|
+
return `${index === 0 ? 0 : LATENCY_EDGES[index - 1] ?? 0}\u2013${upper}ms`;
|
|
1532
|
+
}
|
|
1533
|
+
__name(latencyLabel, "latencyLabel");
|
|
1534
|
+
function toLatencyDistribution(retrievals) {
|
|
1535
|
+
const samples = [];
|
|
1536
|
+
const counts = new Array(LATENCY_EDGES.length + 1).fill(0);
|
|
1537
|
+
for (const { durationMs } of retrievals) {
|
|
1538
|
+
if (durationMs === null) {
|
|
1539
|
+
continue;
|
|
1540
|
+
}
|
|
1541
|
+
samples.push(durationMs);
|
|
1542
|
+
const found = LATENCY_EDGES.findIndex((edge) => durationMs < edge);
|
|
1543
|
+
const index = found === -1 ? LATENCY_EDGES.length : found;
|
|
1544
|
+
counts[index] = (counts[index] ?? 0) + 1;
|
|
1545
|
+
}
|
|
1546
|
+
const buckets = counts.map((count, index) => ({
|
|
1547
|
+
label: latencyLabel(index),
|
|
1548
|
+
count
|
|
1549
|
+
}));
|
|
1550
|
+
const p50 = percentile(samples, 0.5);
|
|
1551
|
+
const p95 = percentile(samples, 0.95);
|
|
1552
|
+
const p99 = percentile(samples, 0.99);
|
|
1553
|
+
return {
|
|
1554
|
+
buckets,
|
|
1555
|
+
...p50 !== null ? {
|
|
1556
|
+
p50
|
|
1557
|
+
} : {},
|
|
1558
|
+
...p95 !== null ? {
|
|
1559
|
+
p95
|
|
1560
|
+
} : {},
|
|
1561
|
+
...p99 !== null ? {
|
|
1562
|
+
p99
|
|
1563
|
+
} : {}
|
|
1564
|
+
};
|
|
1565
|
+
}
|
|
1566
|
+
__name(toLatencyDistribution, "toLatencyDistribution");
|
|
1567
|
+
var SCORE_BINS = 10;
|
|
1568
|
+
function toScoreDistribution(retrievals, retriever) {
|
|
1569
|
+
const samples = [];
|
|
1570
|
+
const counts = new Array(SCORE_BINS).fill(0);
|
|
1571
|
+
for (const retrieval of retrievals) {
|
|
1572
|
+
if (retrieval.retriever !== retriever || retrieval.topScore === null) {
|
|
1573
|
+
continue;
|
|
1574
|
+
}
|
|
1575
|
+
samples.push(retrieval.topScore);
|
|
1576
|
+
const clamped = Math.min(Math.max(retrieval.topScore, 0), 1);
|
|
1577
|
+
const index = Math.min(Math.floor(clamped * SCORE_BINS), SCORE_BINS - 1);
|
|
1578
|
+
counts[index] = (counts[index] ?? 0) + 1;
|
|
1579
|
+
}
|
|
1580
|
+
const buckets = counts.map((count, index) => ({
|
|
1581
|
+
label: `${round(index / SCORE_BINS, 1)}\u2013${round((index + 1) / SCORE_BINS, 1)}`,
|
|
1582
|
+
count
|
|
1583
|
+
}));
|
|
1584
|
+
const p50 = percentile(samples, 0.5);
|
|
1585
|
+
const p95 = percentile(samples, 0.95);
|
|
1586
|
+
return {
|
|
1587
|
+
buckets,
|
|
1588
|
+
...p50 !== null ? {
|
|
1589
|
+
p50: round(p50, 4)
|
|
1590
|
+
} : {},
|
|
1591
|
+
...p95 !== null ? {
|
|
1592
|
+
p95: round(p95, 4)
|
|
1593
|
+
} : {}
|
|
1594
|
+
};
|
|
1595
|
+
}
|
|
1596
|
+
__name(toScoreDistribution, "toScoreDistribution");
|
|
1597
|
+
function hourStart(at) {
|
|
1598
|
+
return Math.floor(at.getTime() / 36e5) * 36e5;
|
|
1599
|
+
}
|
|
1600
|
+
__name(hourStart, "hourStart");
|
|
1601
|
+
function toRetrievalTrendRows(retrievals, now = /* @__PURE__ */ new Date()) {
|
|
1602
|
+
const latest = hourStart(now);
|
|
1603
|
+
const earliest = latest - (TREND_HOURS - 1) * 36e5;
|
|
1604
|
+
const buckets = /* @__PURE__ */ new Map();
|
|
1605
|
+
for (let hour = earliest; hour <= latest; hour += 36e5) {
|
|
1606
|
+
buckets.set(hour, {
|
|
1607
|
+
retrievals: 0,
|
|
1608
|
+
zeroHits: 0
|
|
1609
|
+
});
|
|
1610
|
+
}
|
|
1611
|
+
for (const retrieval of retrievals) {
|
|
1612
|
+
const bucket = buckets.get(hourStart(retrieval.at));
|
|
1613
|
+
if (bucket === void 0) {
|
|
1614
|
+
continue;
|
|
1615
|
+
}
|
|
1616
|
+
bucket.retrievals += 1;
|
|
1617
|
+
if (retrieval.zeroHit) {
|
|
1618
|
+
bucket.zeroHits += 1;
|
|
1619
|
+
}
|
|
1620
|
+
}
|
|
1621
|
+
return [
|
|
1622
|
+
...buckets.entries()
|
|
1623
|
+
].map(([hour, counts]) => ({
|
|
1624
|
+
label: `${new Date(hour).toISOString().slice(11, 13)}:00`,
|
|
1625
|
+
retrievals: counts.retrievals,
|
|
1626
|
+
zeroHits: counts.zeroHits
|
|
1627
|
+
}));
|
|
1628
|
+
}
|
|
1629
|
+
__name(toRetrievalTrendRows, "toRetrievalTrendRows");
|
|
1630
|
+
function segmentsBy(retrievals, pick) {
|
|
1631
|
+
const counts = /* @__PURE__ */ new Map();
|
|
1632
|
+
for (const retrieval of retrievals) {
|
|
1633
|
+
const label = pick(retrieval) ?? NO_VALUE2;
|
|
1634
|
+
counts.set(label, (counts.get(label) ?? 0) + 1);
|
|
1635
|
+
}
|
|
1636
|
+
return [
|
|
1637
|
+
...counts.entries()
|
|
1638
|
+
].map(([label, value]) => ({
|
|
1639
|
+
label,
|
|
1640
|
+
value
|
|
1641
|
+
})).sort((a, b) => b.value - a.value);
|
|
1642
|
+
}
|
|
1643
|
+
__name(segmentsBy, "segmentsBy");
|
|
1644
|
+
function toStoreSegments(retrievals) {
|
|
1645
|
+
return segmentsBy(retrievals, (retrieval) => retrieval.store);
|
|
1646
|
+
}
|
|
1647
|
+
__name(toStoreSegments, "toStoreSegments");
|
|
1648
|
+
function toRetrieverSegments(retrievals) {
|
|
1649
|
+
return segmentsBy(retrievals, (retrieval) => retrieval.retriever);
|
|
1650
|
+
}
|
|
1651
|
+
__name(toRetrieverSegments, "toRetrieverSegments");
|
|
1652
|
+
function toCollectionRows(retrievals, limit = TABLE_LIMIT) {
|
|
1653
|
+
const groups = /* @__PURE__ */ new Map();
|
|
1654
|
+
for (const retrieval of retrievals) {
|
|
1655
|
+
const key = retrieval.collection ?? NO_VALUE2;
|
|
1656
|
+
const group = groups.get(key);
|
|
1657
|
+
if (group === void 0) {
|
|
1658
|
+
groups.set(key, [
|
|
1659
|
+
retrieval
|
|
1660
|
+
]);
|
|
1661
|
+
} else {
|
|
1662
|
+
group.push(retrieval);
|
|
1663
|
+
}
|
|
1664
|
+
}
|
|
1665
|
+
return [
|
|
1666
|
+
...groups.entries()
|
|
1667
|
+
].map(([collection, group]) => {
|
|
1668
|
+
const durations = [];
|
|
1669
|
+
const scores = [];
|
|
1670
|
+
for (const retrieval of group) {
|
|
1671
|
+
if (retrieval.durationMs !== null) {
|
|
1672
|
+
durations.push(retrieval.durationMs);
|
|
1673
|
+
}
|
|
1674
|
+
if (retrieval.topScore !== null) {
|
|
1675
|
+
scores.push(retrieval.topScore);
|
|
1676
|
+
}
|
|
1677
|
+
}
|
|
1678
|
+
const p95 = percentile(durations, 0.95);
|
|
1679
|
+
const meanScore = scores.length === 0 ? null : scores.reduce((sum, score) => sum + score, 0) / scores.length;
|
|
1680
|
+
return {
|
|
1681
|
+
collection,
|
|
1682
|
+
store: group[0]?.store ?? NO_VALUE2,
|
|
1683
|
+
retrievals: group.length,
|
|
1684
|
+
zeroHits: group.filter((retrieval) => retrieval.zeroHit).length,
|
|
1685
|
+
p95Ms: p95 === null ? NO_VALUE2 : Math.round(p95),
|
|
1686
|
+
meanTopScore: meanScore === null ? NO_VALUE2 : round(meanScore, 3)
|
|
1687
|
+
};
|
|
1688
|
+
}).sort((a, b) => b.retrievals - a.retrievals).slice(0, limit);
|
|
1689
|
+
}
|
|
1690
|
+
__name(toCollectionRows, "toCollectionRows");
|
|
1691
|
+
function toSlowestRows(retrievals, limit = TABLE_LIMIT) {
|
|
1692
|
+
return retrievals.filter((retrieval) => retrieval.durationMs !== null).sort((a, b) => (b.durationMs ?? 0) - (a.durationMs ?? 0)).slice(0, limit).map((retrieval) => ({
|
|
1693
|
+
at: retrieval.at.toISOString(),
|
|
1694
|
+
retriever: retrieval.retriever,
|
|
1695
|
+
store: retrieval.store ?? NO_VALUE2,
|
|
1696
|
+
collection: retrieval.collection ?? NO_VALUE2,
|
|
1697
|
+
chunks: retrieval.chunks,
|
|
1698
|
+
topScore: retrieval.topScore === null ? NO_VALUE2 : round(retrieval.topScore, 3),
|
|
1699
|
+
durationMs: retrieval.durationMs === null ? NO_VALUE2 : Math.round(retrieval.durationMs)
|
|
1700
|
+
}));
|
|
1701
|
+
}
|
|
1702
|
+
__name(toSlowestRows, "toSlowestRows");
|
|
1703
|
+
function retrievalProvider(name, format) {
|
|
1704
|
+
return {
|
|
1705
|
+
name,
|
|
1706
|
+
async resolve(query, ctx) {
|
|
1707
|
+
return format(await readRetrievals(retrievalStorage(ctx)), query);
|
|
1708
|
+
}
|
|
1709
|
+
};
|
|
1710
|
+
}
|
|
1711
|
+
__name(retrievalProvider, "retrievalProvider");
|
|
1712
|
+
function ragRetrievalsProvider() {
|
|
1713
|
+
return retrievalProvider("agent.rag.retrievals", (retrievals) => ({
|
|
1714
|
+
value: retrievals.length
|
|
1715
|
+
}));
|
|
1716
|
+
}
|
|
1717
|
+
__name(ragRetrievalsProvider, "ragRetrievalsProvider");
|
|
1718
|
+
function ragZeroHitRateProvider() {
|
|
1719
|
+
return retrievalProvider("agent.rag.zeroHitRate", (retrievals) => ({
|
|
1720
|
+
value: zeroHitRate(retrievals)
|
|
1721
|
+
}));
|
|
1722
|
+
}
|
|
1723
|
+
__name(ragZeroHitRateProvider, "ragZeroHitRateProvider");
|
|
1724
|
+
function ragChunksProvider() {
|
|
1725
|
+
return retrievalProvider("agent.rag.chunks", (retrievals) => ({
|
|
1726
|
+
value: meanChunks(retrievals)
|
|
1727
|
+
}));
|
|
1728
|
+
}
|
|
1729
|
+
__name(ragChunksProvider, "ragChunksProvider");
|
|
1730
|
+
function ragLatencyProvider() {
|
|
1731
|
+
return retrievalProvider("agent.rag.latency", toLatencyDistribution);
|
|
1732
|
+
}
|
|
1733
|
+
__name(ragLatencyProvider, "ragLatencyProvider");
|
|
1734
|
+
function ragScoresProvider() {
|
|
1735
|
+
return retrievalProvider("agent.rag.scores", (retrievals, query) => {
|
|
1736
|
+
const requested = query?.retriever;
|
|
1737
|
+
return toScoreDistribution(retrievals, typeof requested === "string" ? requested : "embedding");
|
|
1738
|
+
});
|
|
1739
|
+
}
|
|
1740
|
+
__name(ragScoresProvider, "ragScoresProvider");
|
|
1741
|
+
function ragTrendProvider() {
|
|
1742
|
+
return retrievalProvider("agent.rag.trend", (retrievals) => ({
|
|
1743
|
+
rows: toRetrievalTrendRows(retrievals)
|
|
1744
|
+
}));
|
|
1745
|
+
}
|
|
1746
|
+
__name(ragTrendProvider, "ragTrendProvider");
|
|
1747
|
+
function ragStoreBreakdownProvider() {
|
|
1748
|
+
return retrievalProvider("agent.rag.byStore", (retrievals) => ({
|
|
1749
|
+
segments: toStoreSegments(retrievals)
|
|
1750
|
+
}));
|
|
1751
|
+
}
|
|
1752
|
+
__name(ragStoreBreakdownProvider, "ragStoreBreakdownProvider");
|
|
1753
|
+
function ragRetrieverBreakdownProvider() {
|
|
1754
|
+
return retrievalProvider("agent.rag.byRetriever", (retrievals) => ({
|
|
1755
|
+
segments: toRetrieverSegments(retrievals)
|
|
1756
|
+
}));
|
|
1757
|
+
}
|
|
1758
|
+
__name(ragRetrieverBreakdownProvider, "ragRetrieverBreakdownProvider");
|
|
1759
|
+
function ragCollectionsTableProvider() {
|
|
1760
|
+
return retrievalProvider("agent.rag.byCollection", (retrievals) => ({
|
|
1761
|
+
rows: toCollectionRows(retrievals)
|
|
1762
|
+
}));
|
|
1763
|
+
}
|
|
1764
|
+
__name(ragCollectionsTableProvider, "ragCollectionsTableProvider");
|
|
1765
|
+
function ragSlowestTableProvider() {
|
|
1766
|
+
return retrievalProvider("agent.rag.slowest", (retrievals) => ({
|
|
1767
|
+
rows: toSlowestRows(retrievals)
|
|
1768
|
+
}));
|
|
1769
|
+
}
|
|
1770
|
+
__name(ragSlowestTableProvider, "ragSlowestTableProvider");
|
|
1771
|
+
|
|
1141
1772
|
// src/agent-telescope.extension.ts
|
|
1773
|
+
var RESERVED_PROVIDER_PREFIX = "agent.";
|
|
1142
1774
|
function agentTelescopeExtension(opts = {}) {
|
|
1143
|
-
|
|
1775
|
+
const hostProviders = opts.providers ?? [];
|
|
1776
|
+
const reserved = hostProviders.filter((provider) => provider.name.startsWith(RESERVED_PROVIDER_PREFIX));
|
|
1777
|
+
if (reserved.length > 0) {
|
|
1778
|
+
throw new Error(`agentTelescopeExtension: host-contributed data providers may not use the reserved "${RESERVED_PROVIDER_PREFIX}" prefix \u2014 rename ${reserved.map((provider) => `"${provider.name}"`).join(", ")} to your own namespace.`);
|
|
1779
|
+
}
|
|
1780
|
+
return (0, import_nestjs_telescope3.defineTelescopeExtension)({
|
|
1144
1781
|
name: "agent",
|
|
1145
1782
|
watchers: /* @__PURE__ */ __name(() => [
|
|
1146
|
-
new AgentTelescopeWatcher()
|
|
1783
|
+
new AgentTelescopeWatcher(),
|
|
1784
|
+
new RagTelescopeWatcher()
|
|
1147
1785
|
], "watchers"),
|
|
1148
1786
|
entryTypes: /* @__PURE__ */ __name(() => [
|
|
1149
1787
|
{
|
|
1150
1788
|
id: "agent",
|
|
1151
1789
|
label: "Agent",
|
|
1152
1790
|
dot: "bg-violet-400"
|
|
1791
|
+
},
|
|
1792
|
+
{
|
|
1793
|
+
id: "agent-rag",
|
|
1794
|
+
label: "RAG",
|
|
1795
|
+
dot: "bg-emerald-400"
|
|
1153
1796
|
}
|
|
1154
1797
|
], "entryTypes"),
|
|
1155
1798
|
dashboards: /* @__PURE__ */ __name(() => [
|
|
1156
|
-
agentDashboard(
|
|
1799
|
+
agentDashboard({
|
|
1800
|
+
...opts.threadHref !== void 0 ? {
|
|
1801
|
+
threadHref: opts.threadHref
|
|
1802
|
+
} : {},
|
|
1803
|
+
...opts.runHref !== void 0 ? {
|
|
1804
|
+
runHref: opts.runHref
|
|
1805
|
+
} : {},
|
|
1806
|
+
...opts.sections !== void 0 ? {
|
|
1807
|
+
sections: opts.sections
|
|
1808
|
+
} : {}
|
|
1809
|
+
})
|
|
1157
1810
|
], "dashboards"),
|
|
1158
1811
|
dataProviders: /* @__PURE__ */ __name(() => [
|
|
1159
1812
|
agentRunsProvider(),
|
|
@@ -1183,7 +1836,18 @@ function agentTelescopeExtension(opts = {}) {
|
|
|
1183
1836
|
agentThreadsPagedTableProvider(),
|
|
1184
1837
|
agentPendingApprovalsCountProvider(),
|
|
1185
1838
|
agentPendingApprovalsTableProvider(),
|
|
1186
|
-
agentToolStatsTableProvider()
|
|
1839
|
+
agentToolStatsTableProvider(),
|
|
1840
|
+
ragRetrievalsProvider(),
|
|
1841
|
+
ragZeroHitRateProvider(),
|
|
1842
|
+
ragChunksProvider(),
|
|
1843
|
+
ragLatencyProvider(),
|
|
1844
|
+
ragScoresProvider(),
|
|
1845
|
+
ragTrendProvider(),
|
|
1846
|
+
ragStoreBreakdownProvider(),
|
|
1847
|
+
ragRetrieverBreakdownProvider(),
|
|
1848
|
+
ragCollectionsTableProvider(),
|
|
1849
|
+
ragSlowestTableProvider(),
|
|
1850
|
+
...hostProviders
|
|
1187
1851
|
], "dataProviders")
|
|
1188
1852
|
});
|
|
1189
1853
|
}
|
|
@@ -1191,6 +1855,8 @@ __name(agentTelescopeExtension, "agentTelescopeExtension");
|
|
|
1191
1855
|
// Annotate the CommonJS export names for ESM import in node:
|
|
1192
1856
|
0 && (module.exports = {
|
|
1193
1857
|
AgentTelescopeWatcher,
|
|
1858
|
+
RAG_ENTRY_TYPE,
|
|
1859
|
+
RagTelescopeWatcher,
|
|
1194
1860
|
agentActorSpendTableProvider,
|
|
1195
1861
|
agentDashboard,
|
|
1196
1862
|
agentModelSpendTableProvider,
|
|
@@ -1220,24 +1886,46 @@ __name(agentTelescopeExtension, "agentTelescopeExtension");
|
|
|
1220
1886
|
agentTopThreadsTableProvider,
|
|
1221
1887
|
agentUsageTrendProvider,
|
|
1222
1888
|
capErrorMessage,
|
|
1889
|
+
meanChunks,
|
|
1890
|
+
percentile,
|
|
1891
|
+
ragChunksProvider,
|
|
1892
|
+
ragCollectionsTableProvider,
|
|
1893
|
+
ragLatencyProvider,
|
|
1894
|
+
ragRetrievalsProvider,
|
|
1895
|
+
ragRetrieverBreakdownProvider,
|
|
1896
|
+
ragScoresProvider,
|
|
1897
|
+
ragSlowestTableProvider,
|
|
1898
|
+
ragStoreBreakdownProvider,
|
|
1899
|
+
ragTrendProvider,
|
|
1900
|
+
ragZeroHitRateProvider,
|
|
1901
|
+
readRetrievals,
|
|
1223
1902
|
resolveRange,
|
|
1224
1903
|
shiftUtcDay,
|
|
1225
1904
|
shortPromptHash,
|
|
1226
1905
|
toActorSpendRows,
|
|
1227
1906
|
toActorSpendSegments,
|
|
1907
|
+
toCollectionRows,
|
|
1908
|
+
toLatencyDistribution,
|
|
1228
1909
|
toModelSpendRows,
|
|
1229
1910
|
toModelSpendSegments,
|
|
1230
1911
|
toPendingApprovalTableRows,
|
|
1912
|
+
toRagRetrieval,
|
|
1231
1913
|
toRecentRunTableRows,
|
|
1232
1914
|
toRecentThreadTableRows,
|
|
1233
1915
|
toRecentToolCallRows,
|
|
1916
|
+
toRetrievalTrendRows,
|
|
1917
|
+
toRetrieverSegments,
|
|
1234
1918
|
toRunAgentTableRows,
|
|
1235
1919
|
toRunErrorSegments,
|
|
1236
1920
|
toRunTrendRows,
|
|
1921
|
+
toScoreDistribution,
|
|
1922
|
+
toSlowestRows,
|
|
1923
|
+
toStoreSegments,
|
|
1237
1924
|
toThreadSpendRows,
|
|
1238
1925
|
toToolStatTableRows,
|
|
1239
1926
|
toUsageTrendRows,
|
|
1240
1927
|
totalCostUsd,
|
|
1241
|
-
totalTokens
|
|
1928
|
+
totalTokens,
|
|
1929
|
+
zeroHitRate
|
|
1242
1930
|
});
|
|
1243
1931
|
//# sourceMappingURL=index.cjs.map
|