@dudousxd/nestjs-agent-telescope 0.7.1 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -1
- package/dist/index.cjs +705 -12
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +223 -16
- package/dist/index.d.ts +223 -16
- package/dist/index.js +678 -9
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -22,6 +22,8 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
22
22
|
var index_exports = {};
|
|
23
23
|
__export(index_exports, {
|
|
24
24
|
AgentTelescopeWatcher: () => AgentTelescopeWatcher,
|
|
25
|
+
RAG_ENTRY_TYPE: () => RAG_ENTRY_TYPE,
|
|
26
|
+
RagTelescopeWatcher: () => RagTelescopeWatcher,
|
|
25
27
|
agentActorSpendTableProvider: () => agentActorSpendTableProvider,
|
|
26
28
|
agentDashboard: () => agentDashboard,
|
|
27
29
|
agentModelSpendTableProvider: () => agentModelSpendTableProvider,
|
|
@@ -51,30 +53,52 @@ __export(index_exports, {
|
|
|
51
53
|
agentTopThreadsTableProvider: () => agentTopThreadsTableProvider,
|
|
52
54
|
agentUsageTrendProvider: () => agentUsageTrendProvider,
|
|
53
55
|
capErrorMessage: () => capErrorMessage,
|
|
56
|
+
meanChunks: () => meanChunks,
|
|
57
|
+
percentile: () => percentile,
|
|
58
|
+
ragChunksProvider: () => ragChunksProvider,
|
|
59
|
+
ragCollectionsTableProvider: () => ragCollectionsTableProvider,
|
|
60
|
+
ragLatencyProvider: () => ragLatencyProvider,
|
|
61
|
+
ragRetrievalsProvider: () => ragRetrievalsProvider,
|
|
62
|
+
ragRetrieverBreakdownProvider: () => ragRetrieverBreakdownProvider,
|
|
63
|
+
ragScoresProvider: () => ragScoresProvider,
|
|
64
|
+
ragSlowestTableProvider: () => ragSlowestTableProvider,
|
|
65
|
+
ragStoreBreakdownProvider: () => ragStoreBreakdownProvider,
|
|
66
|
+
ragTrendProvider: () => ragTrendProvider,
|
|
67
|
+
ragZeroHitRateProvider: () => ragZeroHitRateProvider,
|
|
68
|
+
readRetrievals: () => readRetrievals,
|
|
54
69
|
resolveRange: () => resolveRange,
|
|
55
70
|
shiftUtcDay: () => shiftUtcDay,
|
|
56
71
|
shortPromptHash: () => shortPromptHash,
|
|
57
72
|
toActorSpendRows: () => toActorSpendRows,
|
|
58
73
|
toActorSpendSegments: () => toActorSpendSegments,
|
|
74
|
+
toCollectionRows: () => toCollectionRows,
|
|
75
|
+
toLatencyDistribution: () => toLatencyDistribution,
|
|
59
76
|
toModelSpendRows: () => toModelSpendRows,
|
|
60
77
|
toModelSpendSegments: () => toModelSpendSegments,
|
|
61
78
|
toPendingApprovalTableRows: () => toPendingApprovalTableRows,
|
|
79
|
+
toRagRetrieval: () => toRagRetrieval,
|
|
62
80
|
toRecentRunTableRows: () => toRecentRunTableRows,
|
|
63
81
|
toRecentThreadTableRows: () => toRecentThreadTableRows,
|
|
64
82
|
toRecentToolCallRows: () => toRecentToolCallRows,
|
|
83
|
+
toRetrievalTrendRows: () => toRetrievalTrendRows,
|
|
84
|
+
toRetrieverSegments: () => toRetrieverSegments,
|
|
65
85
|
toRunAgentTableRows: () => toRunAgentTableRows,
|
|
66
86
|
toRunErrorSegments: () => toRunErrorSegments,
|
|
67
87
|
toRunTrendRows: () => toRunTrendRows,
|
|
88
|
+
toScoreDistribution: () => toScoreDistribution,
|
|
89
|
+
toSlowestRows: () => toSlowestRows,
|
|
90
|
+
toStoreSegments: () => toStoreSegments,
|
|
68
91
|
toThreadSpendRows: () => toThreadSpendRows,
|
|
69
92
|
toToolStatTableRows: () => toToolStatTableRows,
|
|
70
93
|
toUsageTrendRows: () => toUsageTrendRows,
|
|
71
94
|
totalCostUsd: () => totalCostUsd,
|
|
72
|
-
totalTokens: () => totalTokens
|
|
95
|
+
totalTokens: () => totalTokens,
|
|
96
|
+
zeroHitRate: () => zeroHitRate
|
|
73
97
|
});
|
|
74
98
|
module.exports = __toCommonJS(index_exports);
|
|
75
99
|
|
|
76
100
|
// src/agent-telescope.extension.ts
|
|
77
|
-
var
|
|
101
|
+
var import_nestjs_telescope3 = require("@dudousxd/nestjs-telescope");
|
|
78
102
|
|
|
79
103
|
// src/agent-dashboard.spec-data.ts
|
|
80
104
|
function col(key, label, href) {
|
|
@@ -564,7 +588,180 @@ function agentDashboard(opts = {}) {
|
|
|
564
588
|
]
|
|
565
589
|
}
|
|
566
590
|
]
|
|
567
|
-
}
|
|
591
|
+
},
|
|
592
|
+
{
|
|
593
|
+
// RAG. Fed by the `aviary:rag:retrieval` telemetry `@dudousxd/nestjs-agent-rag` emits, NOT
|
|
594
|
+
// derived from tool-call rows: a tool call that retrieved nothing and one that retrieved
|
|
595
|
+
// five perfect passages are the same row with the same `ok` status, so every number here
|
|
596
|
+
// would have been unanswerable from that side.
|
|
597
|
+
title: "Retrieval",
|
|
598
|
+
cols: 3,
|
|
599
|
+
panels: [
|
|
600
|
+
{
|
|
601
|
+
kind: "stat",
|
|
602
|
+
title: "Retrievals",
|
|
603
|
+
data: {
|
|
604
|
+
provider: "agent.rag.retrievals"
|
|
605
|
+
}
|
|
606
|
+
},
|
|
607
|
+
{
|
|
608
|
+
kind: "stat",
|
|
609
|
+
title: "Zero-hit rate",
|
|
610
|
+
data: {
|
|
611
|
+
provider: "agent.rag.zeroHitRate"
|
|
612
|
+
},
|
|
613
|
+
format: "percent"
|
|
614
|
+
},
|
|
615
|
+
{
|
|
616
|
+
kind: "stat",
|
|
617
|
+
title: "Passages per retrieval",
|
|
618
|
+
data: {
|
|
619
|
+
provider: "agent.rag.chunks"
|
|
620
|
+
},
|
|
621
|
+
format: "number"
|
|
622
|
+
},
|
|
623
|
+
{
|
|
624
|
+
// A real histogram, unlike the run-duration pair above: retrieval events carry the raw
|
|
625
|
+
// per-call duration, so there ARE samples to bucket and markers to place.
|
|
626
|
+
kind: "distribution",
|
|
627
|
+
title: "Retrieval latency",
|
|
628
|
+
data: {
|
|
629
|
+
provider: "agent.rag.latency"
|
|
630
|
+
},
|
|
631
|
+
markers: [
|
|
632
|
+
"p50",
|
|
633
|
+
"p95",
|
|
634
|
+
"p99"
|
|
635
|
+
],
|
|
636
|
+
format: "duration"
|
|
637
|
+
},
|
|
638
|
+
{
|
|
639
|
+
// Bound to ONE retriever kind on purpose — a cosine similarity, a BM25 score and an RRF
|
|
640
|
+
// rank score share no scale, so a histogram over all of them has bins that mean a
|
|
641
|
+
// different thing per bar. See `toScoreDistribution`.
|
|
642
|
+
kind: "distribution",
|
|
643
|
+
title: "Top score (dense)",
|
|
644
|
+
data: {
|
|
645
|
+
provider: "agent.rag.scores",
|
|
646
|
+
query: {
|
|
647
|
+
retriever: "embedding"
|
|
648
|
+
}
|
|
649
|
+
},
|
|
650
|
+
markers: [
|
|
651
|
+
"p50",
|
|
652
|
+
"p95"
|
|
653
|
+
],
|
|
654
|
+
format: "number"
|
|
655
|
+
},
|
|
656
|
+
{
|
|
657
|
+
kind: "timeseries",
|
|
658
|
+
title: "Retrievals & zero-hits",
|
|
659
|
+
data: {
|
|
660
|
+
provider: "agent.rag.trend"
|
|
661
|
+
},
|
|
662
|
+
series: [
|
|
663
|
+
"retrievals",
|
|
664
|
+
"zeroHits"
|
|
665
|
+
],
|
|
666
|
+
style: "stacked"
|
|
667
|
+
}
|
|
668
|
+
]
|
|
669
|
+
},
|
|
670
|
+
{
|
|
671
|
+
title: "Retrieval sources",
|
|
672
|
+
cols: 2,
|
|
673
|
+
panels: [
|
|
674
|
+
{
|
|
675
|
+
kind: "breakdown",
|
|
676
|
+
title: "Retrievals by store",
|
|
677
|
+
data: {
|
|
678
|
+
provider: "agent.rag.byStore"
|
|
679
|
+
},
|
|
680
|
+
style: "donut"
|
|
681
|
+
},
|
|
682
|
+
{
|
|
683
|
+
kind: "breakdown",
|
|
684
|
+
title: "Retrievals by retriever",
|
|
685
|
+
data: {
|
|
686
|
+
provider: "agent.rag.byRetriever"
|
|
687
|
+
},
|
|
688
|
+
style: "donut"
|
|
689
|
+
},
|
|
690
|
+
{
|
|
691
|
+
kind: "table",
|
|
692
|
+
title: "By collection",
|
|
693
|
+
data: {
|
|
694
|
+
provider: "agent.rag.byCollection"
|
|
695
|
+
},
|
|
696
|
+
columns: [
|
|
697
|
+
{
|
|
698
|
+
key: "collection",
|
|
699
|
+
label: "Collection"
|
|
700
|
+
},
|
|
701
|
+
{
|
|
702
|
+
key: "store",
|
|
703
|
+
label: "Store"
|
|
704
|
+
},
|
|
705
|
+
{
|
|
706
|
+
key: "retrievals",
|
|
707
|
+
label: "Retrievals"
|
|
708
|
+
},
|
|
709
|
+
{
|
|
710
|
+
key: "zeroHits",
|
|
711
|
+
label: "Zero-hits"
|
|
712
|
+
},
|
|
713
|
+
{
|
|
714
|
+
key: "p95Ms",
|
|
715
|
+
label: "p95 (ms)"
|
|
716
|
+
},
|
|
717
|
+
{
|
|
718
|
+
key: "meanTopScore",
|
|
719
|
+
label: "Mean top score"
|
|
720
|
+
}
|
|
721
|
+
]
|
|
722
|
+
},
|
|
723
|
+
{
|
|
724
|
+
kind: "table",
|
|
725
|
+
title: "Slowest retrievals",
|
|
726
|
+
data: {
|
|
727
|
+
provider: "agent.rag.slowest"
|
|
728
|
+
},
|
|
729
|
+
// Seven columns, matching the recent-runs table's width: this is the widest a table can
|
|
730
|
+
// get here before it overflows its card.
|
|
731
|
+
columns: [
|
|
732
|
+
{
|
|
733
|
+
key: "at",
|
|
734
|
+
label: "When"
|
|
735
|
+
},
|
|
736
|
+
{
|
|
737
|
+
key: "retriever",
|
|
738
|
+
label: "Retriever"
|
|
739
|
+
},
|
|
740
|
+
{
|
|
741
|
+
key: "store",
|
|
742
|
+
label: "Store"
|
|
743
|
+
},
|
|
744
|
+
{
|
|
745
|
+
key: "collection",
|
|
746
|
+
label: "Collection"
|
|
747
|
+
},
|
|
748
|
+
{
|
|
749
|
+
key: "chunks",
|
|
750
|
+
label: "Passages"
|
|
751
|
+
},
|
|
752
|
+
{
|
|
753
|
+
key: "topScore",
|
|
754
|
+
label: "Top score"
|
|
755
|
+
},
|
|
756
|
+
{
|
|
757
|
+
key: "durationMs",
|
|
758
|
+
label: "Duration (ms)"
|
|
759
|
+
}
|
|
760
|
+
]
|
|
761
|
+
}
|
|
762
|
+
]
|
|
763
|
+
},
|
|
764
|
+
...opts.sections ?? []
|
|
568
765
|
]
|
|
569
766
|
};
|
|
570
767
|
}
|
|
@@ -985,9 +1182,15 @@ function agentRunsTrendProvider() {
|
|
|
985
1182
|
__name(agentRunsTrendProvider, "agentRunsTrendProvider");
|
|
986
1183
|
var DEFAULT_RECENT_LIMIT = 50;
|
|
987
1184
|
var MAX_RECENT_LIMIT = 500;
|
|
1185
|
+
function readPositiveNumber(raw) {
|
|
1186
|
+
if (typeof raw === "number") return Number.isFinite(raw) && raw > 0 ? raw : void 0;
|
|
1187
|
+
if (typeof raw !== "string" || raw.trim() === "") return void 0;
|
|
1188
|
+
const parsed = Number(raw);
|
|
1189
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : void 0;
|
|
1190
|
+
}
|
|
1191
|
+
__name(readPositiveNumber, "readPositiveNumber");
|
|
988
1192
|
function resolveLimit(query, fallback) {
|
|
989
|
-
const
|
|
990
|
-
const value = typeof raw === "number" && Number.isFinite(raw) ? raw : fallback;
|
|
1193
|
+
const value = readPositiveNumber(query?.limit) ?? fallback;
|
|
991
1194
|
return Math.min(MAX_RECENT_LIMIT, Math.max(1, Math.trunc(value)));
|
|
992
1195
|
}
|
|
993
1196
|
__name(resolveLimit, "resolveLimit");
|
|
@@ -1003,8 +1206,7 @@ function governanceLimitProvider(name, defaultLimit, fetch, format) {
|
|
|
1003
1206
|
}
|
|
1004
1207
|
__name(governanceLimitProvider, "governanceLimitProvider");
|
|
1005
1208
|
function resolvePage(query) {
|
|
1006
|
-
const
|
|
1007
|
-
const value = typeof raw === "number" && Number.isFinite(raw) ? raw : 1;
|
|
1209
|
+
const value = readPositiveNumber(query?.page) ?? 1;
|
|
1008
1210
|
return Math.max(1, Math.trunc(value));
|
|
1009
1211
|
}
|
|
1010
1212
|
__name(resolvePage, "resolvePage");
|
|
@@ -1138,22 +1340,478 @@ var AgentTelescopeWatcher = class {
|
|
|
1138
1340
|
}
|
|
1139
1341
|
};
|
|
1140
1342
|
|
|
1343
|
+
// src/rag-data-providers.ts
|
|
1344
|
+
var import_nestjs_telescope2 = require("@dudousxd/nestjs-telescope");
|
|
1345
|
+
|
|
1346
|
+
// src/rag-telescope.watcher.ts
|
|
1347
|
+
var import_node_diagnostics_channel2 = require("diagnostics_channel");
|
|
1348
|
+
var import_nestjs_diagnostics2 = require("@dudousxd/nestjs-diagnostics");
|
|
1349
|
+
var RAG_LIB = "rag";
|
|
1350
|
+
var RETRIEVAL_EVENT = "retrieval";
|
|
1351
|
+
var RAG_ENTRY_TYPE = "agent-rag";
|
|
1352
|
+
function isEnvelope(message) {
|
|
1353
|
+
return typeof message === "object" && message !== null && typeof Reflect.get(message, "payload") === "object" && Reflect.get(message, "payload") !== null;
|
|
1354
|
+
}
|
|
1355
|
+
__name(isEnvelope, "isEnvelope");
|
|
1356
|
+
var RagTelescopeWatcher = class {
|
|
1357
|
+
static {
|
|
1358
|
+
__name(this, "RagTelescopeWatcher");
|
|
1359
|
+
}
|
|
1360
|
+
type = RAG_ENTRY_TYPE;
|
|
1361
|
+
disposers = [];
|
|
1362
|
+
register(ctx) {
|
|
1363
|
+
this.disposers.push((0, import_nestjs_diagnostics2.claimDiagnostics)(RAG_LIB, [
|
|
1364
|
+
RETRIEVAL_EVENT
|
|
1365
|
+
]));
|
|
1366
|
+
const channel = (0, import_nestjs_diagnostics2.channelName)(RAG_LIB, RETRIEVAL_EVENT);
|
|
1367
|
+
const onMessage = /* @__PURE__ */ __name((message) => {
|
|
1368
|
+
if (!isEnvelope(message)) {
|
|
1369
|
+
return;
|
|
1370
|
+
}
|
|
1371
|
+
ctx.record({
|
|
1372
|
+
type: RAG_ENTRY_TYPE,
|
|
1373
|
+
content: {
|
|
1374
|
+
event: RETRIEVAL_EVENT,
|
|
1375
|
+
...message.payload,
|
|
1376
|
+
...message.durationMs !== void 0 ? {
|
|
1377
|
+
durationMs: message.durationMs
|
|
1378
|
+
} : {}
|
|
1379
|
+
},
|
|
1380
|
+
// Tagged by store and retriever kind so the Entries screen can filter to "every retrieval
|
|
1381
|
+
// pgvector served" without a provider — the same tag-per-dimension the agent watcher uses.
|
|
1382
|
+
tags: tagsOf(message.payload)
|
|
1383
|
+
});
|
|
1384
|
+
}, "onMessage");
|
|
1385
|
+
(0, import_node_diagnostics_channel2.subscribe)(channel, onMessage);
|
|
1386
|
+
this.disposers.push(() => (0, import_node_diagnostics_channel2.unsubscribe)(channel, onMessage));
|
|
1387
|
+
}
|
|
1388
|
+
/** Detach the subscription and release the diagnostics claim (e.g. on module destroy). */
|
|
1389
|
+
dispose() {
|
|
1390
|
+
while (this.disposers.length) this.disposers.pop()?.();
|
|
1391
|
+
}
|
|
1392
|
+
};
|
|
1393
|
+
function tagsOf(payload) {
|
|
1394
|
+
const tags = [
|
|
1395
|
+
RETRIEVAL_EVENT
|
|
1396
|
+
];
|
|
1397
|
+
if (typeof payload.retriever === "string") {
|
|
1398
|
+
tags.push(`retriever:${payload.retriever}`);
|
|
1399
|
+
}
|
|
1400
|
+
if (typeof payload.store === "string") {
|
|
1401
|
+
tags.push(`store:${payload.store}`);
|
|
1402
|
+
}
|
|
1403
|
+
if (payload.zeroHit === true) {
|
|
1404
|
+
tags.push("zero-hit");
|
|
1405
|
+
}
|
|
1406
|
+
if (payload.failed === true) {
|
|
1407
|
+
tags.push("failed");
|
|
1408
|
+
}
|
|
1409
|
+
return tags;
|
|
1410
|
+
}
|
|
1411
|
+
__name(tagsOf, "tagsOf");
|
|
1412
|
+
|
|
1413
|
+
// src/rag-data-providers.ts
|
|
1414
|
+
var RETRIEVAL_WINDOW = 5e3;
|
|
1415
|
+
var TABLE_LIMIT = 10;
|
|
1416
|
+
var TREND_HOURS = 24;
|
|
1417
|
+
var NO_VALUE2 = "\u2014";
|
|
1418
|
+
function isRecord2(value) {
|
|
1419
|
+
return typeof value === "object" && value !== null;
|
|
1420
|
+
}
|
|
1421
|
+
__name(isRecord2, "isRecord");
|
|
1422
|
+
function readString(source, key) {
|
|
1423
|
+
const value = source[key];
|
|
1424
|
+
return typeof value === "string" ? value : null;
|
|
1425
|
+
}
|
|
1426
|
+
__name(readString, "readString");
|
|
1427
|
+
function readNumber(source, key) {
|
|
1428
|
+
const value = source[key];
|
|
1429
|
+
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
1430
|
+
}
|
|
1431
|
+
__name(readNumber, "readNumber");
|
|
1432
|
+
function isEntryStorage(value) {
|
|
1433
|
+
return isRecord2(value) && typeof value.get === "function";
|
|
1434
|
+
}
|
|
1435
|
+
__name(isEntryStorage, "isEntryStorage");
|
|
1436
|
+
function toRagRetrieval(entry) {
|
|
1437
|
+
if (!isRecord2(entry) || !isRecord2(entry.content)) {
|
|
1438
|
+
return null;
|
|
1439
|
+
}
|
|
1440
|
+
const content = entry.content;
|
|
1441
|
+
const retriever = readString(content, "retriever");
|
|
1442
|
+
if (retriever === null) {
|
|
1443
|
+
return null;
|
|
1444
|
+
}
|
|
1445
|
+
const chunks = readNumber(content, "chunks") ?? 0;
|
|
1446
|
+
const createdAt = entry.createdAt;
|
|
1447
|
+
return {
|
|
1448
|
+
at: createdAt instanceof Date ? createdAt : /* @__PURE__ */ new Date(0),
|
|
1449
|
+
retriever,
|
|
1450
|
+
store: readString(content, "store"),
|
|
1451
|
+
collection: readString(content, "collection"),
|
|
1452
|
+
chunks,
|
|
1453
|
+
zeroHit: content.zeroHit === true || chunks === 0,
|
|
1454
|
+
topScore: readNumber(content, "topScore"),
|
|
1455
|
+
durationMs: readNumber(content, "durationMs"),
|
|
1456
|
+
failed: content.failed === true
|
|
1457
|
+
};
|
|
1458
|
+
}
|
|
1459
|
+
__name(toRagRetrieval, "toRagRetrieval");
|
|
1460
|
+
async function readRetrievals(storage) {
|
|
1461
|
+
if (!isEntryStorage(storage)) {
|
|
1462
|
+
return [];
|
|
1463
|
+
}
|
|
1464
|
+
const page = await storage.get({
|
|
1465
|
+
type: RAG_ENTRY_TYPE,
|
|
1466
|
+
limit: RETRIEVAL_WINDOW
|
|
1467
|
+
});
|
|
1468
|
+
const retrievals = [];
|
|
1469
|
+
for (const entry of page.data) {
|
|
1470
|
+
const retrieval = toRagRetrieval(entry);
|
|
1471
|
+
if (retrieval !== null) {
|
|
1472
|
+
retrievals.push(retrieval);
|
|
1473
|
+
}
|
|
1474
|
+
}
|
|
1475
|
+
return retrievals;
|
|
1476
|
+
}
|
|
1477
|
+
__name(readRetrievals, "readRetrievals");
|
|
1478
|
+
function retrievalStorage(ctx) {
|
|
1479
|
+
try {
|
|
1480
|
+
return ctx.moduleRef.get(import_nestjs_telescope2.TELESCOPE_STORAGE, {
|
|
1481
|
+
strict: false
|
|
1482
|
+
});
|
|
1483
|
+
} catch {
|
|
1484
|
+
return void 0;
|
|
1485
|
+
}
|
|
1486
|
+
}
|
|
1487
|
+
__name(retrievalStorage, "retrievalStorage");
|
|
1488
|
+
function percentile(samples, fraction) {
|
|
1489
|
+
if (samples.length === 0) {
|
|
1490
|
+
return null;
|
|
1491
|
+
}
|
|
1492
|
+
const sorted = [
|
|
1493
|
+
...samples
|
|
1494
|
+
].sort((a, b) => a - b);
|
|
1495
|
+
const rank = Math.ceil(fraction * sorted.length);
|
|
1496
|
+
return sorted[Math.min(Math.max(rank, 1), sorted.length) - 1] ?? null;
|
|
1497
|
+
}
|
|
1498
|
+
__name(percentile, "percentile");
|
|
1499
|
+
function round(value, places) {
|
|
1500
|
+
const factor = 10 ** places;
|
|
1501
|
+
return Math.round(value * factor) / factor;
|
|
1502
|
+
}
|
|
1503
|
+
__name(round, "round");
|
|
1504
|
+
function zeroHitRate(retrievals) {
|
|
1505
|
+
if (retrievals.length === 0) {
|
|
1506
|
+
return 0;
|
|
1507
|
+
}
|
|
1508
|
+
const zeroHits = retrievals.filter((retrieval) => retrieval.zeroHit).length;
|
|
1509
|
+
return round(zeroHits / retrievals.length, 4);
|
|
1510
|
+
}
|
|
1511
|
+
__name(zeroHitRate, "zeroHitRate");
|
|
1512
|
+
function meanChunks(retrievals) {
|
|
1513
|
+
if (retrievals.length === 0) {
|
|
1514
|
+
return 0;
|
|
1515
|
+
}
|
|
1516
|
+
const total = retrievals.reduce((sum, retrieval) => sum + retrieval.chunks, 0);
|
|
1517
|
+
return round(total / retrievals.length, 2);
|
|
1518
|
+
}
|
|
1519
|
+
__name(meanChunks, "meanChunks");
|
|
1520
|
+
var LATENCY_EDGES = [
|
|
1521
|
+
10,
|
|
1522
|
+
25,
|
|
1523
|
+
50,
|
|
1524
|
+
100,
|
|
1525
|
+
250,
|
|
1526
|
+
500,
|
|
1527
|
+
1e3,
|
|
1528
|
+
2500,
|
|
1529
|
+
5e3
|
|
1530
|
+
];
|
|
1531
|
+
function latencyLabel(index) {
|
|
1532
|
+
const upper = LATENCY_EDGES[index];
|
|
1533
|
+
if (upper === void 0) {
|
|
1534
|
+
return `\u2265${LATENCY_EDGES[LATENCY_EDGES.length - 1] ?? 0}ms`;
|
|
1535
|
+
}
|
|
1536
|
+
return `${index === 0 ? 0 : LATENCY_EDGES[index - 1] ?? 0}\u2013${upper}ms`;
|
|
1537
|
+
}
|
|
1538
|
+
__name(latencyLabel, "latencyLabel");
|
|
1539
|
+
function toLatencyDistribution(retrievals) {
|
|
1540
|
+
const samples = [];
|
|
1541
|
+
const counts = new Array(LATENCY_EDGES.length + 1).fill(0);
|
|
1542
|
+
for (const { durationMs } of retrievals) {
|
|
1543
|
+
if (durationMs === null) {
|
|
1544
|
+
continue;
|
|
1545
|
+
}
|
|
1546
|
+
samples.push(durationMs);
|
|
1547
|
+
const found = LATENCY_EDGES.findIndex((edge) => durationMs < edge);
|
|
1548
|
+
const index = found === -1 ? LATENCY_EDGES.length : found;
|
|
1549
|
+
counts[index] = (counts[index] ?? 0) + 1;
|
|
1550
|
+
}
|
|
1551
|
+
const buckets = counts.map((count, index) => ({
|
|
1552
|
+
label: latencyLabel(index),
|
|
1553
|
+
count
|
|
1554
|
+
}));
|
|
1555
|
+
const p50 = percentile(samples, 0.5);
|
|
1556
|
+
const p95 = percentile(samples, 0.95);
|
|
1557
|
+
const p99 = percentile(samples, 0.99);
|
|
1558
|
+
return {
|
|
1559
|
+
buckets,
|
|
1560
|
+
...p50 !== null ? {
|
|
1561
|
+
p50
|
|
1562
|
+
} : {},
|
|
1563
|
+
...p95 !== null ? {
|
|
1564
|
+
p95
|
|
1565
|
+
} : {},
|
|
1566
|
+
...p99 !== null ? {
|
|
1567
|
+
p99
|
|
1568
|
+
} : {}
|
|
1569
|
+
};
|
|
1570
|
+
}
|
|
1571
|
+
__name(toLatencyDistribution, "toLatencyDistribution");
|
|
1572
|
+
var SCORE_BINS = 10;
|
|
1573
|
+
function toScoreDistribution(retrievals, retriever) {
|
|
1574
|
+
const samples = [];
|
|
1575
|
+
const counts = new Array(SCORE_BINS).fill(0);
|
|
1576
|
+
for (const retrieval of retrievals) {
|
|
1577
|
+
if (retrieval.retriever !== retriever || retrieval.topScore === null) {
|
|
1578
|
+
continue;
|
|
1579
|
+
}
|
|
1580
|
+
samples.push(retrieval.topScore);
|
|
1581
|
+
const clamped = Math.min(Math.max(retrieval.topScore, 0), 1);
|
|
1582
|
+
const index = Math.min(Math.floor(clamped * SCORE_BINS), SCORE_BINS - 1);
|
|
1583
|
+
counts[index] = (counts[index] ?? 0) + 1;
|
|
1584
|
+
}
|
|
1585
|
+
const buckets = counts.map((count, index) => ({
|
|
1586
|
+
label: `${round(index / SCORE_BINS, 1)}\u2013${round((index + 1) / SCORE_BINS, 1)}`,
|
|
1587
|
+
count
|
|
1588
|
+
}));
|
|
1589
|
+
const p50 = percentile(samples, 0.5);
|
|
1590
|
+
const p95 = percentile(samples, 0.95);
|
|
1591
|
+
return {
|
|
1592
|
+
buckets,
|
|
1593
|
+
...p50 !== null ? {
|
|
1594
|
+
p50: round(p50, 4)
|
|
1595
|
+
} : {},
|
|
1596
|
+
...p95 !== null ? {
|
|
1597
|
+
p95: round(p95, 4)
|
|
1598
|
+
} : {}
|
|
1599
|
+
};
|
|
1600
|
+
}
|
|
1601
|
+
__name(toScoreDistribution, "toScoreDistribution");
|
|
1602
|
+
function hourStart(at) {
|
|
1603
|
+
return Math.floor(at.getTime() / 36e5) * 36e5;
|
|
1604
|
+
}
|
|
1605
|
+
__name(hourStart, "hourStart");
|
|
1606
|
+
function toRetrievalTrendRows(retrievals, now = /* @__PURE__ */ new Date()) {
|
|
1607
|
+
const latest = hourStart(now);
|
|
1608
|
+
const earliest = latest - (TREND_HOURS - 1) * 36e5;
|
|
1609
|
+
const buckets = /* @__PURE__ */ new Map();
|
|
1610
|
+
for (let hour = earliest; hour <= latest; hour += 36e5) {
|
|
1611
|
+
buckets.set(hour, {
|
|
1612
|
+
retrievals: 0,
|
|
1613
|
+
zeroHits: 0
|
|
1614
|
+
});
|
|
1615
|
+
}
|
|
1616
|
+
for (const retrieval of retrievals) {
|
|
1617
|
+
const bucket = buckets.get(hourStart(retrieval.at));
|
|
1618
|
+
if (bucket === void 0) {
|
|
1619
|
+
continue;
|
|
1620
|
+
}
|
|
1621
|
+
bucket.retrievals += 1;
|
|
1622
|
+
if (retrieval.zeroHit) {
|
|
1623
|
+
bucket.zeroHits += 1;
|
|
1624
|
+
}
|
|
1625
|
+
}
|
|
1626
|
+
return [
|
|
1627
|
+
...buckets.entries()
|
|
1628
|
+
].map(([hour, counts]) => ({
|
|
1629
|
+
label: `${new Date(hour).toISOString().slice(11, 13)}:00`,
|
|
1630
|
+
retrievals: counts.retrievals,
|
|
1631
|
+
zeroHits: counts.zeroHits
|
|
1632
|
+
}));
|
|
1633
|
+
}
|
|
1634
|
+
__name(toRetrievalTrendRows, "toRetrievalTrendRows");
|
|
1635
|
+
function segmentsBy(retrievals, pick) {
|
|
1636
|
+
const counts = /* @__PURE__ */ new Map();
|
|
1637
|
+
for (const retrieval of retrievals) {
|
|
1638
|
+
const label = pick(retrieval) ?? NO_VALUE2;
|
|
1639
|
+
counts.set(label, (counts.get(label) ?? 0) + 1);
|
|
1640
|
+
}
|
|
1641
|
+
return [
|
|
1642
|
+
...counts.entries()
|
|
1643
|
+
].map(([label, value]) => ({
|
|
1644
|
+
label,
|
|
1645
|
+
value
|
|
1646
|
+
})).sort((a, b) => b.value - a.value);
|
|
1647
|
+
}
|
|
1648
|
+
__name(segmentsBy, "segmentsBy");
|
|
1649
|
+
function toStoreSegments(retrievals) {
|
|
1650
|
+
return segmentsBy(retrievals, (retrieval) => retrieval.store);
|
|
1651
|
+
}
|
|
1652
|
+
__name(toStoreSegments, "toStoreSegments");
|
|
1653
|
+
function toRetrieverSegments(retrievals) {
|
|
1654
|
+
return segmentsBy(retrievals, (retrieval) => retrieval.retriever);
|
|
1655
|
+
}
|
|
1656
|
+
__name(toRetrieverSegments, "toRetrieverSegments");
|
|
1657
|
+
function toCollectionRows(retrievals, limit = TABLE_LIMIT) {
|
|
1658
|
+
const groups = /* @__PURE__ */ new Map();
|
|
1659
|
+
for (const retrieval of retrievals) {
|
|
1660
|
+
const key = retrieval.collection ?? NO_VALUE2;
|
|
1661
|
+
const group = groups.get(key);
|
|
1662
|
+
if (group === void 0) {
|
|
1663
|
+
groups.set(key, [
|
|
1664
|
+
retrieval
|
|
1665
|
+
]);
|
|
1666
|
+
} else {
|
|
1667
|
+
group.push(retrieval);
|
|
1668
|
+
}
|
|
1669
|
+
}
|
|
1670
|
+
return [
|
|
1671
|
+
...groups.entries()
|
|
1672
|
+
].map(([collection, group]) => {
|
|
1673
|
+
const durations = [];
|
|
1674
|
+
const scores = [];
|
|
1675
|
+
for (const retrieval of group) {
|
|
1676
|
+
if (retrieval.durationMs !== null) {
|
|
1677
|
+
durations.push(retrieval.durationMs);
|
|
1678
|
+
}
|
|
1679
|
+
if (retrieval.topScore !== null) {
|
|
1680
|
+
scores.push(retrieval.topScore);
|
|
1681
|
+
}
|
|
1682
|
+
}
|
|
1683
|
+
const p95 = percentile(durations, 0.95);
|
|
1684
|
+
const meanScore = scores.length === 0 ? null : scores.reduce((sum, score) => sum + score, 0) / scores.length;
|
|
1685
|
+
return {
|
|
1686
|
+
collection,
|
|
1687
|
+
store: group[0]?.store ?? NO_VALUE2,
|
|
1688
|
+
retrievals: group.length,
|
|
1689
|
+
zeroHits: group.filter((retrieval) => retrieval.zeroHit).length,
|
|
1690
|
+
p95Ms: p95 === null ? NO_VALUE2 : Math.round(p95),
|
|
1691
|
+
meanTopScore: meanScore === null ? NO_VALUE2 : round(meanScore, 3)
|
|
1692
|
+
};
|
|
1693
|
+
}).sort((a, b) => b.retrievals - a.retrievals).slice(0, limit);
|
|
1694
|
+
}
|
|
1695
|
+
__name(toCollectionRows, "toCollectionRows");
|
|
1696
|
+
function toSlowestRows(retrievals, limit = TABLE_LIMIT) {
|
|
1697
|
+
return retrievals.filter((retrieval) => retrieval.durationMs !== null).sort((a, b) => (b.durationMs ?? 0) - (a.durationMs ?? 0)).slice(0, limit).map((retrieval) => ({
|
|
1698
|
+
at: retrieval.at.toISOString(),
|
|
1699
|
+
retriever: retrieval.retriever,
|
|
1700
|
+
store: retrieval.store ?? NO_VALUE2,
|
|
1701
|
+
collection: retrieval.collection ?? NO_VALUE2,
|
|
1702
|
+
chunks: retrieval.chunks,
|
|
1703
|
+
topScore: retrieval.topScore === null ? NO_VALUE2 : round(retrieval.topScore, 3),
|
|
1704
|
+
durationMs: retrieval.durationMs === null ? NO_VALUE2 : Math.round(retrieval.durationMs)
|
|
1705
|
+
}));
|
|
1706
|
+
}
|
|
1707
|
+
__name(toSlowestRows, "toSlowestRows");
|
|
1708
|
+
function retrievalProvider(name, format) {
|
|
1709
|
+
return {
|
|
1710
|
+
name,
|
|
1711
|
+
async resolve(query, ctx) {
|
|
1712
|
+
return format(await readRetrievals(retrievalStorage(ctx)), query);
|
|
1713
|
+
}
|
|
1714
|
+
};
|
|
1715
|
+
}
|
|
1716
|
+
__name(retrievalProvider, "retrievalProvider");
|
|
1717
|
+
function ragRetrievalsProvider() {
|
|
1718
|
+
return retrievalProvider("agent.rag.retrievals", (retrievals) => ({
|
|
1719
|
+
value: retrievals.length
|
|
1720
|
+
}));
|
|
1721
|
+
}
|
|
1722
|
+
__name(ragRetrievalsProvider, "ragRetrievalsProvider");
|
|
1723
|
+
function ragZeroHitRateProvider() {
|
|
1724
|
+
return retrievalProvider("agent.rag.zeroHitRate", (retrievals) => ({
|
|
1725
|
+
value: zeroHitRate(retrievals)
|
|
1726
|
+
}));
|
|
1727
|
+
}
|
|
1728
|
+
__name(ragZeroHitRateProvider, "ragZeroHitRateProvider");
|
|
1729
|
+
function ragChunksProvider() {
|
|
1730
|
+
return retrievalProvider("agent.rag.chunks", (retrievals) => ({
|
|
1731
|
+
value: meanChunks(retrievals)
|
|
1732
|
+
}));
|
|
1733
|
+
}
|
|
1734
|
+
__name(ragChunksProvider, "ragChunksProvider");
|
|
1735
|
+
function ragLatencyProvider() {
|
|
1736
|
+
return retrievalProvider("agent.rag.latency", toLatencyDistribution);
|
|
1737
|
+
}
|
|
1738
|
+
__name(ragLatencyProvider, "ragLatencyProvider");
|
|
1739
|
+
function ragScoresProvider() {
|
|
1740
|
+
return retrievalProvider("agent.rag.scores", (retrievals, query) => {
|
|
1741
|
+
const requested = query?.retriever;
|
|
1742
|
+
return toScoreDistribution(retrievals, typeof requested === "string" ? requested : "embedding");
|
|
1743
|
+
});
|
|
1744
|
+
}
|
|
1745
|
+
__name(ragScoresProvider, "ragScoresProvider");
|
|
1746
|
+
function ragTrendProvider() {
|
|
1747
|
+
return retrievalProvider("agent.rag.trend", (retrievals) => ({
|
|
1748
|
+
rows: toRetrievalTrendRows(retrievals)
|
|
1749
|
+
}));
|
|
1750
|
+
}
|
|
1751
|
+
__name(ragTrendProvider, "ragTrendProvider");
|
|
1752
|
+
function ragStoreBreakdownProvider() {
|
|
1753
|
+
return retrievalProvider("agent.rag.byStore", (retrievals) => ({
|
|
1754
|
+
segments: toStoreSegments(retrievals)
|
|
1755
|
+
}));
|
|
1756
|
+
}
|
|
1757
|
+
__name(ragStoreBreakdownProvider, "ragStoreBreakdownProvider");
|
|
1758
|
+
function ragRetrieverBreakdownProvider() {
|
|
1759
|
+
return retrievalProvider("agent.rag.byRetriever", (retrievals) => ({
|
|
1760
|
+
segments: toRetrieverSegments(retrievals)
|
|
1761
|
+
}));
|
|
1762
|
+
}
|
|
1763
|
+
__name(ragRetrieverBreakdownProvider, "ragRetrieverBreakdownProvider");
|
|
1764
|
+
function ragCollectionsTableProvider() {
|
|
1765
|
+
return retrievalProvider("agent.rag.byCollection", (retrievals) => ({
|
|
1766
|
+
rows: toCollectionRows(retrievals)
|
|
1767
|
+
}));
|
|
1768
|
+
}
|
|
1769
|
+
__name(ragCollectionsTableProvider, "ragCollectionsTableProvider");
|
|
1770
|
+
function ragSlowestTableProvider() {
|
|
1771
|
+
return retrievalProvider("agent.rag.slowest", (retrievals) => ({
|
|
1772
|
+
rows: toSlowestRows(retrievals)
|
|
1773
|
+
}));
|
|
1774
|
+
}
|
|
1775
|
+
__name(ragSlowestTableProvider, "ragSlowestTableProvider");
|
|
1776
|
+
|
|
1141
1777
|
// src/agent-telescope.extension.ts
|
|
1778
|
+
var RESERVED_PROVIDER_PREFIX = "agent.";
|
|
1142
1779
|
function agentTelescopeExtension(opts = {}) {
|
|
1143
|
-
|
|
1780
|
+
const hostProviders = opts.providers ?? [];
|
|
1781
|
+
const reserved = hostProviders.filter((provider) => provider.name.startsWith(RESERVED_PROVIDER_PREFIX));
|
|
1782
|
+
if (reserved.length > 0) {
|
|
1783
|
+
throw new Error(`agentTelescopeExtension: host-contributed data providers may not use the reserved "${RESERVED_PROVIDER_PREFIX}" prefix \u2014 rename ${reserved.map((provider) => `"${provider.name}"`).join(", ")} to your own namespace.`);
|
|
1784
|
+
}
|
|
1785
|
+
return (0, import_nestjs_telescope3.defineTelescopeExtension)({
|
|
1144
1786
|
name: "agent",
|
|
1145
1787
|
watchers: /* @__PURE__ */ __name(() => [
|
|
1146
|
-
new AgentTelescopeWatcher()
|
|
1788
|
+
new AgentTelescopeWatcher(),
|
|
1789
|
+
new RagTelescopeWatcher()
|
|
1147
1790
|
], "watchers"),
|
|
1148
1791
|
entryTypes: /* @__PURE__ */ __name(() => [
|
|
1149
1792
|
{
|
|
1150
1793
|
id: "agent",
|
|
1151
1794
|
label: "Agent",
|
|
1152
1795
|
dot: "bg-violet-400"
|
|
1796
|
+
},
|
|
1797
|
+
{
|
|
1798
|
+
id: "agent-rag",
|
|
1799
|
+
label: "RAG",
|
|
1800
|
+
dot: "bg-emerald-400"
|
|
1153
1801
|
}
|
|
1154
1802
|
], "entryTypes"),
|
|
1155
1803
|
dashboards: /* @__PURE__ */ __name(() => [
|
|
1156
|
-
agentDashboard(
|
|
1804
|
+
agentDashboard({
|
|
1805
|
+
...opts.threadHref !== void 0 ? {
|
|
1806
|
+
threadHref: opts.threadHref
|
|
1807
|
+
} : {},
|
|
1808
|
+
...opts.runHref !== void 0 ? {
|
|
1809
|
+
runHref: opts.runHref
|
|
1810
|
+
} : {},
|
|
1811
|
+
...opts.sections !== void 0 ? {
|
|
1812
|
+
sections: opts.sections
|
|
1813
|
+
} : {}
|
|
1814
|
+
})
|
|
1157
1815
|
], "dashboards"),
|
|
1158
1816
|
dataProviders: /* @__PURE__ */ __name(() => [
|
|
1159
1817
|
agentRunsProvider(),
|
|
@@ -1183,7 +1841,18 @@ function agentTelescopeExtension(opts = {}) {
|
|
|
1183
1841
|
agentThreadsPagedTableProvider(),
|
|
1184
1842
|
agentPendingApprovalsCountProvider(),
|
|
1185
1843
|
agentPendingApprovalsTableProvider(),
|
|
1186
|
-
agentToolStatsTableProvider()
|
|
1844
|
+
agentToolStatsTableProvider(),
|
|
1845
|
+
ragRetrievalsProvider(),
|
|
1846
|
+
ragZeroHitRateProvider(),
|
|
1847
|
+
ragChunksProvider(),
|
|
1848
|
+
ragLatencyProvider(),
|
|
1849
|
+
ragScoresProvider(),
|
|
1850
|
+
ragTrendProvider(),
|
|
1851
|
+
ragStoreBreakdownProvider(),
|
|
1852
|
+
ragRetrieverBreakdownProvider(),
|
|
1853
|
+
ragCollectionsTableProvider(),
|
|
1854
|
+
ragSlowestTableProvider(),
|
|
1855
|
+
...hostProviders
|
|
1187
1856
|
], "dataProviders")
|
|
1188
1857
|
});
|
|
1189
1858
|
}
|
|
@@ -1191,6 +1860,8 @@ __name(agentTelescopeExtension, "agentTelescopeExtension");
|
|
|
1191
1860
|
// Annotate the CommonJS export names for ESM import in node:
|
|
1192
1861
|
0 && (module.exports = {
|
|
1193
1862
|
AgentTelescopeWatcher,
|
|
1863
|
+
RAG_ENTRY_TYPE,
|
|
1864
|
+
RagTelescopeWatcher,
|
|
1194
1865
|
agentActorSpendTableProvider,
|
|
1195
1866
|
agentDashboard,
|
|
1196
1867
|
agentModelSpendTableProvider,
|
|
@@ -1220,24 +1891,46 @@ __name(agentTelescopeExtension, "agentTelescopeExtension");
|
|
|
1220
1891
|
agentTopThreadsTableProvider,
|
|
1221
1892
|
agentUsageTrendProvider,
|
|
1222
1893
|
capErrorMessage,
|
|
1894
|
+
meanChunks,
|
|
1895
|
+
percentile,
|
|
1896
|
+
ragChunksProvider,
|
|
1897
|
+
ragCollectionsTableProvider,
|
|
1898
|
+
ragLatencyProvider,
|
|
1899
|
+
ragRetrievalsProvider,
|
|
1900
|
+
ragRetrieverBreakdownProvider,
|
|
1901
|
+
ragScoresProvider,
|
|
1902
|
+
ragSlowestTableProvider,
|
|
1903
|
+
ragStoreBreakdownProvider,
|
|
1904
|
+
ragTrendProvider,
|
|
1905
|
+
ragZeroHitRateProvider,
|
|
1906
|
+
readRetrievals,
|
|
1223
1907
|
resolveRange,
|
|
1224
1908
|
shiftUtcDay,
|
|
1225
1909
|
shortPromptHash,
|
|
1226
1910
|
toActorSpendRows,
|
|
1227
1911
|
toActorSpendSegments,
|
|
1912
|
+
toCollectionRows,
|
|
1913
|
+
toLatencyDistribution,
|
|
1228
1914
|
toModelSpendRows,
|
|
1229
1915
|
toModelSpendSegments,
|
|
1230
1916
|
toPendingApprovalTableRows,
|
|
1917
|
+
toRagRetrieval,
|
|
1231
1918
|
toRecentRunTableRows,
|
|
1232
1919
|
toRecentThreadTableRows,
|
|
1233
1920
|
toRecentToolCallRows,
|
|
1921
|
+
toRetrievalTrendRows,
|
|
1922
|
+
toRetrieverSegments,
|
|
1234
1923
|
toRunAgentTableRows,
|
|
1235
1924
|
toRunErrorSegments,
|
|
1236
1925
|
toRunTrendRows,
|
|
1926
|
+
toScoreDistribution,
|
|
1927
|
+
toSlowestRows,
|
|
1928
|
+
toStoreSegments,
|
|
1237
1929
|
toThreadSpendRows,
|
|
1238
1930
|
toToolStatTableRows,
|
|
1239
1931
|
toUsageTrendRows,
|
|
1240
1932
|
totalCostUsd,
|
|
1241
|
-
totalTokens
|
|
1933
|
+
totalTokens,
|
|
1934
|
+
zeroHitRate
|
|
1242
1935
|
});
|
|
1243
1936
|
//# sourceMappingURL=index.cjs.map
|