@dudousxd/nestjs-agent-telescope 0.7.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -1
- package/dist/index.cjs +696 -8
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +223 -16
- package/dist/index.d.ts +223 -16
- package/dist/index.js +669 -5
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.d.cts
CHANGED
|
@@ -1,14 +1,49 @@
|
|
|
1
1
|
import * as _dudousxd_nestjs_telescope from '@dudousxd/nestjs-telescope';
|
|
2
|
-
import { Watcher, WatcherContext, DashboardSpec
|
|
2
|
+
import { DataProvider, DashboardSection, Watcher, WatcherContext, DashboardSpec } from '@dudousxd/nestjs-telescope';
|
|
3
3
|
import { GovernanceRange, ActorSpendRow, ModelSpendRow, PendingApprovalRow, RecentRunRow, ThreadActivityRow, ToolCallActivityRow, RunAgentBreakdownRow, RunErrorBreakdownRow, RunTrendPoint, ThreadSpendRow, ToolStatRow, UsageTrendPoint } from '@dudousxd/nestjs-agent-core';
|
|
4
4
|
|
|
5
|
+
interface AgentTelescopeExtensionOptions {
|
|
6
|
+
/** Deep-link template for a `{threadId}` cell, e.g. `'/admin/threads/{threadId}'`. */
|
|
7
|
+
threadHref?: string;
|
|
8
|
+
/** Deep-link template for a `{runId}` cell. Defaults to the in-app trace waterfall. */
|
|
9
|
+
runHref?: string;
|
|
10
|
+
/**
|
|
11
|
+
* HOST-contributed data providers, registered alongside the built-in ones.
|
|
12
|
+
*
|
|
13
|
+
* This exists because a panel on this dashboard **cannot** bind to a provider contributed by a
|
|
14
|
+
* different extension, and the reason is structural rather than a policy we could relax: the UI
|
|
15
|
+
* derives the request path from the dashboard id (`agent.overview` → `GET /ext/agent/data/:name`),
|
|
16
|
+
* and the controller 404s when `providerOwner(name) !== ext` so the URL namespace cannot be
|
|
17
|
+
* spoofed. A host that registers its own `TelescopeModule` extension therefore gets its own tab,
|
|
18
|
+
* never a panel on this one. Passing the providers through here makes THIS extension their owner,
|
|
19
|
+
* which is what makes a host section on this page resolve at all.
|
|
20
|
+
*
|
|
21
|
+
* Name them under your own prefix (`myapp.rag.collections`); anything starting with `agent.` is
|
|
22
|
+
* refused at boot, because a collision there would surface as Telescope's generic "contributed by
|
|
23
|
+
* both agent and agent" error, which names the same extension twice and says nothing useful.
|
|
24
|
+
*/
|
|
25
|
+
providers?: DataProvider[];
|
|
26
|
+
/**
|
|
27
|
+
* HOST-contributed dashboard sections, appended after the built-in ones. Bind their panels to the
|
|
28
|
+
* providers passed above (or to any built-in `agent.*` provider).
|
|
29
|
+
*
|
|
30
|
+
* Size each section's panel count to an exact multiple of its `cols`, for the same reason every
|
|
31
|
+
* built-in section is: the renderer lays a section out as a fixed `grid-cols-N` grid with no
|
|
32
|
+
* `colSpan`, so an orphan panel leaves a visible hole beside it. That is a layout convention, not
|
|
33
|
+
* something validated here — a gap is cosmetic, and failing a host's boot over cosmetics would be
|
|
34
|
+
* a worse trade than the gap.
|
|
35
|
+
*/
|
|
36
|
+
sections?: DashboardSection[];
|
|
37
|
+
}
|
|
5
38
|
/**
|
|
6
|
-
* The first-class Telescope extension for nestjs-agent: an "Agent" tab fed by
|
|
7
|
-
* the `aviary:agent:*` diagnostics channel (live runs, tool calls) via the watcher,
|
|
39
|
+
* The first-class Telescope extension for nestjs-agent: an "Agent" tab fed by three sources —
|
|
40
|
+
* the `aviary:agent:*` diagnostics channel (live runs, tool calls) via the watcher, the
|
|
8
41
|
* authoritative `AGENT_GOVERNANCE_QUERIES` read-model (historical spend/usage, run reliability,
|
|
9
|
-
* tool activity, the approvals inbox) via the governance providers
|
|
10
|
-
*
|
|
11
|
-
*
|
|
42
|
+
* tool activity, the approvals inbox) via the governance providers, and the `aviary:rag:retrieval`
|
|
43
|
+
* channel (retrieval latency, chunk counts, score distribution, store/collection) via the RAG
|
|
44
|
+
* watcher and providers. The extension `name`, entry-type ids, dashboard id, and every built-in
|
|
45
|
+
* provider name share the `agent` prefix so the registry's global-uniqueness namespaces never
|
|
46
|
+
* collide with sibling extensions.
|
|
12
47
|
*
|
|
13
48
|
* `threadHref` deep-links a table row's `threadId` cell out to the HOST's own thread viewer —
|
|
14
49
|
* passed straight through to {@link agentDashboard}, mirroring `durableTelescopeExtension`'s
|
|
@@ -22,12 +57,11 @@ import { GovernanceRange, ActorSpendRow, ModelSpendRow, PendingApprovalRow, Rece
|
|
|
22
57
|
* (e.g. `store-mikro-orm` / `store-drizzle` / `testing`) — in the same module that registers
|
|
23
58
|
* `TelescopeModule.forRoot({ extensions: [agentTelescopeExtension()] })`. If the binding is
|
|
24
59
|
* absent, those panels render an empty state; the live watcher-fed panels (Runs/Tokens stats and
|
|
25
|
-
* the Tool-call status breakdown) keep working regardless.
|
|
60
|
+
* the Tool-call status breakdown) keep working regardless. The RAG panels need no binding at all —
|
|
61
|
+
* they read Telescope's own storage — but they stay empty until something emits retrieval telemetry
|
|
62
|
+
* (`createRetrievalTool` does by default; `instrumentRetriever` for every other call path).
|
|
26
63
|
*/
|
|
27
|
-
declare function agentTelescopeExtension(opts?:
|
|
28
|
-
threadHref?: string;
|
|
29
|
-
runHref?: string;
|
|
30
|
-
}): _dudousxd_nestjs_telescope.TelescopeExtension;
|
|
64
|
+
declare function agentTelescopeExtension(opts?: AgentTelescopeExtensionOptions): _dudousxd_nestjs_telescope.TelescopeExtension;
|
|
31
65
|
|
|
32
66
|
/**
|
|
33
67
|
* Records `aviary:agent:*` diagnostics events as Telescope entries of type `agent`. It depends
|
|
@@ -52,6 +86,39 @@ declare class AgentTelescopeWatcher implements Watcher {
|
|
|
52
86
|
dispose(): void;
|
|
53
87
|
}
|
|
54
88
|
|
|
89
|
+
/** The Telescope entry `type` retrieval events are recorded as — see the class doc. */
|
|
90
|
+
declare const RAG_ENTRY_TYPE = "agent-rag";
|
|
91
|
+
/**
|
|
92
|
+
* Records `aviary:rag:retrieval` events as Telescope entries of type `agent-rag` — the source every
|
|
93
|
+
* RAG panel on the Agent dashboard reads.
|
|
94
|
+
*
|
|
95
|
+
* **Its own entry type, not `agent`.** Retrieval is per-tool-call where a run is per-conversation, so
|
|
96
|
+
* a busy hour produces an order of magnitude more retrieval events than agent events. Sharing the
|
|
97
|
+
* `agent` type would put them in one storage window: the providers page 5k entries of a type, and
|
|
98
|
+
* retrieval traffic would push `run.finished` out of that window, quietly zeroing the Runs and Tokens
|
|
99
|
+
* stats that have nothing to do with RAG. Separate types make the two independent, and give the
|
|
100
|
+
* Entries screen a "RAG" filter that lists retrievals and only retrievals.
|
|
101
|
+
*
|
|
102
|
+
* **The duration is lifted off the ENVELOPE into the entry content.** The emitter puts it there
|
|
103
|
+
* (`emit`'s `opts.durationMs`) because that is the field Telescope's OTel exporter turns into a
|
|
104
|
+
* latency histogram; the panels need it as an ordinary field they can bucket, and an entry's content
|
|
105
|
+
* is all a `DataProvider` gets. Recording it in both places would mean two sources of truth for the
|
|
106
|
+
* same number, so it is lifted rather than duplicated at the emitter.
|
|
107
|
+
*
|
|
108
|
+
* Claims `rag:retrieval` (refcounted, released in `dispose()`) so
|
|
109
|
+
* `@dudousxd/nestjs-diagnostics-telescope`'s generic bridge skips it instead of recording every
|
|
110
|
+
* retrieval a second time as a generic `diagnostic` entry.
|
|
111
|
+
*
|
|
112
|
+
* Registered by `agentTelescopeExtension()`; also usable on its own in a host's `watchers` list.
|
|
113
|
+
*/
|
|
114
|
+
declare class RagTelescopeWatcher implements Watcher {
|
|
115
|
+
readonly type = "agent-rag";
|
|
116
|
+
private readonly disposers;
|
|
117
|
+
register(ctx: WatcherContext): void;
|
|
118
|
+
/** Detach the subscription and release the diagnostics claim (e.g. on module destroy). */
|
|
119
|
+
dispose(): void;
|
|
120
|
+
}
|
|
121
|
+
|
|
55
122
|
/**
|
|
56
123
|
* The "Agent" overview dashboard. Panels bind to the `agent.*` data providers.
|
|
57
124
|
*
|
|
@@ -63,16 +130,156 @@ declare class AgentTelescopeWatcher implements Watcher {
|
|
|
63
130
|
* still override it. Every table whose rows carry a `threadId`/`runId` gets a `Column.link` for it
|
|
64
131
|
* (via {@link col}); `threadHref` left unset leaves that column plain text.
|
|
65
132
|
*
|
|
66
|
-
* Layout:
|
|
67
|
-
* so its panel count is an exact multiple of its `cols` —
|
|
68
|
-
* `ExtensionDashboardPage` renders a section as a
|
|
69
|
-
* `colSpan`, so a lone table in an otherwise-empty
|
|
133
|
+
* Layout: eight sections (Overview lean; Spend; Reliability; Activity; Approvals; Tools;
|
|
134
|
+
* Retrieval; Retrieval sources), each sized so its panel count is an exact multiple of its `cols` —
|
|
135
|
+
* no half-empty row. `nestjs-telescope`'s `ExtensionDashboardPage` renders a section as a
|
|
136
|
+
* `grid-cols-N` grid with one panel per cell and no `colSpan`, so a lone table in an otherwise-empty
|
|
137
|
+
* row would leave a visible gap next to it.
|
|
138
|
+
*
|
|
139
|
+
* `sections` appends HOST-contributed sections after the built-in ones — see
|
|
140
|
+
* {@link import('./agent-telescope.extension.js').agentTelescopeExtension}'s `sections`/`providers`
|
|
141
|
+
* options, which is the supported way an application puts its own panels (its knowledge-base
|
|
142
|
+
* collections, its ingestion activity) on this page. They are appended rather than interleaved so a
|
|
143
|
+
* host's layout can never push a built-in section out of the row it was sized for.
|
|
70
144
|
*/
|
|
71
145
|
declare function agentDashboard(opts?: {
|
|
72
146
|
threadHref?: string;
|
|
73
147
|
runHref?: string;
|
|
148
|
+
sections?: DashboardSection[];
|
|
74
149
|
}): DashboardSpec;
|
|
75
150
|
|
|
151
|
+
/** One retrieval, normalized out of an entry's untyped `content`. */
|
|
152
|
+
interface RagRetrieval {
|
|
153
|
+
at: Date;
|
|
154
|
+
retriever: string;
|
|
155
|
+
store: string | null;
|
|
156
|
+
collection: string | null;
|
|
157
|
+
chunks: number;
|
|
158
|
+
zeroHit: boolean;
|
|
159
|
+
topScore: number | null;
|
|
160
|
+
durationMs: number | null;
|
|
161
|
+
failed: boolean;
|
|
162
|
+
}
|
|
163
|
+
interface BreakdownSegment$1 {
|
|
164
|
+
label: string;
|
|
165
|
+
value: number;
|
|
166
|
+
}
|
|
167
|
+
interface DistributionBucket {
|
|
168
|
+
label: string;
|
|
169
|
+
count: number;
|
|
170
|
+
}
|
|
171
|
+
/** One row of the per-collection table. */
|
|
172
|
+
interface CollectionRow {
|
|
173
|
+
collection: string;
|
|
174
|
+
store: string;
|
|
175
|
+
retrievals: number;
|
|
176
|
+
zeroHits: number;
|
|
177
|
+
p95Ms: number | string;
|
|
178
|
+
meanTopScore: number | string;
|
|
179
|
+
}
|
|
180
|
+
/** One row of the slowest-retrievals table. */
|
|
181
|
+
interface SlowestRow {
|
|
182
|
+
at: string;
|
|
183
|
+
retriever: string;
|
|
184
|
+
store: string;
|
|
185
|
+
collection: string;
|
|
186
|
+
chunks: number;
|
|
187
|
+
topScore: number | string;
|
|
188
|
+
durationMs: number | string;
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* Normalize one storage entry into a {@link RagRetrieval}, or `null` when it is not a retrieval this
|
|
192
|
+
* shaping understands. Field-by-field `typeof` checks rather than a cast: entries are persisted JSON
|
|
193
|
+
* written by a possibly older producer, so "the field is there and is a number" is a runtime
|
|
194
|
+
* question, and a missing `durationMs` must land as `null` (excluded from percentiles) rather than as
|
|
195
|
+
* a `NaN` that silently poisons every aggregate it touches.
|
|
196
|
+
*/
|
|
197
|
+
declare function toRagRetrieval(entry: unknown): RagRetrieval | null;
|
|
198
|
+
/**
|
|
199
|
+
* Every retrieval in the window, newest first. Takes the resolved binding as `unknown` and narrows it
|
|
200
|
+
* here, so a host whose Telescope storage is missing or unrecognised gets empty panels rather than a
|
|
201
|
+
* 502 on every RAG card — the same degrade-don't-throw posture the governance providers take.
|
|
202
|
+
*/
|
|
203
|
+
declare function readRetrievals(storage: unknown): Promise<RagRetrieval[]>;
|
|
204
|
+
/** Nearest-rank percentile over an unsorted sample. `null` for an empty sample. */
|
|
205
|
+
declare function percentile(samples: number[], fraction: number): number | null;
|
|
206
|
+
/** Fraction (0–1) of retrievals that came back with nothing. `0` over an empty window. */
|
|
207
|
+
declare function zeroHitRate(retrievals: RagRetrieval[]): number;
|
|
208
|
+
/** Mean passages returned per retrieval — `topK` minus this is how often the corpus ran out. */
|
|
209
|
+
declare function meanChunks(retrievals: RagRetrieval[]): number;
|
|
210
|
+
/** Latency histogram over the ladder above, plus p50/p95/p99 of the raw samples. */
|
|
211
|
+
declare function toLatencyDistribution(retrievals: RagRetrieval[]): {
|
|
212
|
+
buckets: DistributionBucket[];
|
|
213
|
+
p50?: number;
|
|
214
|
+
p95?: number;
|
|
215
|
+
p99?: number;
|
|
216
|
+
};
|
|
217
|
+
/**
|
|
218
|
+
* Top-score histogram — **for ONE retriever kind at a time**, which is the whole reason this takes a
|
|
219
|
+
* `retriever` argument instead of bucketing everything it was handed.
|
|
220
|
+
*
|
|
221
|
+
* A `Passage.score` has no shared meaning across strategies: `EmbeddingRetriever` returns a
|
|
222
|
+
* calibrated cosine similarity in `[0, 1]`, a BM25 leg returns an unbounded relevance score in the
|
|
223
|
+
* tens, and `HybridRetriever`'s RRF returns a rank-derived number that lives in a band about one
|
|
224
|
+
* part in fifty wide and says nothing about quality at all (see the score section on
|
|
225
|
+
* `HybridRetriever`). Pouring those into one histogram produces a chart where the bins mean a
|
|
226
|
+
* different thing per bar — and the reading it invites, "our scores collapsed", would be a change in
|
|
227
|
+
* traffic MIX rather than in retrieval quality.
|
|
228
|
+
*
|
|
229
|
+
* So the panel binds `query.retriever: 'embedding'`: the dense leg is the only population whose
|
|
230
|
+
* score is comparable across queries, which is also why `minScore` lives on that retriever alone.
|
|
231
|
+
* Scores outside `[0, 1]` are clamped into the end bins rather than dropped, so a store that reports
|
|
232
|
+
* a distance-derived score still shows up as an obviously-pinned bar instead of an empty chart.
|
|
233
|
+
*/
|
|
234
|
+
declare function toScoreDistribution(retrievals: RagRetrieval[], retriever: string): {
|
|
235
|
+
buckets: DistributionBucket[];
|
|
236
|
+
p50?: number;
|
|
237
|
+
p95?: number;
|
|
238
|
+
};
|
|
239
|
+
/**
|
|
240
|
+
* Retrievals and zero-hits per UTC hour, oldest bucket first, over the last {@link TREND_HOURS}.
|
|
241
|
+
* Empty hours are emitted as zeroes so a quiet stretch reads as a gap in the line rather than
|
|
242
|
+
* disappearing and making the outage look like it never happened.
|
|
243
|
+
*/
|
|
244
|
+
declare function toRetrievalTrendRows(retrievals: RagRetrieval[], now?: Date): Array<{
|
|
245
|
+
label: string;
|
|
246
|
+
retrievals: number;
|
|
247
|
+
zeroHits: number;
|
|
248
|
+
}>;
|
|
249
|
+
/** Retrievals by vector store (`memory`/`pg`/`redis`, or `—` for an in-process retriever). */
|
|
250
|
+
declare function toStoreSegments(retrievals: RagRetrieval[]): BreakdownSegment$1[];
|
|
251
|
+
/** Retrievals by strategy — the composed kind, since that is what answered. */
|
|
252
|
+
declare function toRetrieverSegments(retrievals: RagRetrieval[]): BreakdownSegment$1[];
|
|
253
|
+
/** Per-collection rollup, busiest first. `p95Ms`/`meanTopScore` fall back to `—` when unmeasured. */
|
|
254
|
+
declare function toCollectionRows(retrievals: RagRetrieval[], limit?: number): CollectionRow[];
|
|
255
|
+
/**
|
|
256
|
+
* The slowest retrievals in the window, worst first — the table an operator opens after the p95
|
|
257
|
+
* moves. Sorted by duration rather than by recency on purpose: "recent retrievals" would be a list
|
|
258
|
+
* of whatever just happened, which the trend already shows in aggregate, whereas the tail is the
|
|
259
|
+
* only place a single pathological query is visible at all.
|
|
260
|
+
*/
|
|
261
|
+
declare function toSlowestRows(retrievals: RagRetrieval[], limit?: number): SlowestRow[];
|
|
262
|
+
/** stat → retrievals in the window. */
|
|
263
|
+
declare function ragRetrievalsProvider(): DataProvider;
|
|
264
|
+
/** stat (percent) → share of retrievals that returned nothing. */
|
|
265
|
+
declare function ragZeroHitRateProvider(): DataProvider;
|
|
266
|
+
/** stat → mean passages returned per retrieval. */
|
|
267
|
+
declare function ragChunksProvider(): DataProvider;
|
|
268
|
+
/** distribution → retrieval latency, with p50/p95/p99 markers. */
|
|
269
|
+
declare function ragLatencyProvider(): DataProvider;
|
|
270
|
+
/** distribution → top score for ONE retriever kind (`query.retriever`, default `embedding`). */
|
|
271
|
+
declare function ragScoresProvider(): DataProvider;
|
|
272
|
+
/** timeseries → retrievals and zero-hits per hour. */
|
|
273
|
+
declare function ragTrendProvider(): DataProvider;
|
|
274
|
+
/** breakdown → retrievals by vector store. */
|
|
275
|
+
declare function ragStoreBreakdownProvider(): DataProvider;
|
|
276
|
+
/** breakdown → retrievals by retriever strategy. */
|
|
277
|
+
declare function ragRetrieverBreakdownProvider(): DataProvider;
|
|
278
|
+
/** table → per-collection rollup, busiest first. */
|
|
279
|
+
declare function ragCollectionsTableProvider(): DataProvider;
|
|
280
|
+
/** table → the slowest retrievals in the window. */
|
|
281
|
+
declare function ragSlowestTableProvider(): DataProvider;
|
|
282
|
+
|
|
76
283
|
/** stat → total agent runs (run.finished entries). */
|
|
77
284
|
declare function agentRunsProvider(): DataProvider;
|
|
78
285
|
/** stat → total tokens across all finished runs. */
|
|
@@ -296,4 +503,4 @@ declare function agentPendingApprovalsTableProvider(): DataProvider;
|
|
|
296
503
|
/** table → per-tool call/failure/rejection/latency rollup over the range. */
|
|
297
504
|
declare function agentToolStatsTableProvider(): DataProvider;
|
|
298
505
|
|
|
299
|
-
export { AgentTelescopeWatcher, agentActorSpendTableProvider, agentDashboard, agentModelSpendTableProvider, agentPendingApprovalsCountProvider, agentPendingApprovalsTableProvider, agentRecentRunsTableProvider, agentRecentThreadsTableProvider, agentRecentToolCallsTableProvider, agentRunErrorsProvider, agentRunsByAgentTableProvider, agentRunsDurationProvider, agentRunsFailedProvider, agentRunsProvider, agentRunsRetriesProvider, agentRunsSuccessRateProvider, agentRunsTotalProvider, agentRunsTrendProvider, agentSpendByActorProvider, agentSpendByModelProvider, agentSpendTotalProvider, agentTelescopeExtension, agentTokensProvider, agentTokensTotalProvider, agentToolStatsTableProvider, agentToolStatusProvider, agentToolsProvider, agentTopThreadsTableProvider, agentUsageTrendProvider, capErrorMessage, resolveRange, shiftUtcDay, shortPromptHash, toActorSpendRows, toActorSpendSegments, toModelSpendRows, toModelSpendSegments, toPendingApprovalTableRows, toRecentRunTableRows, toRecentThreadTableRows, toRecentToolCallRows, toRunAgentTableRows, toRunErrorSegments, toRunTrendRows, toThreadSpendRows, toToolStatTableRows, toUsageTrendRows, totalCostUsd, totalTokens };
|
|
506
|
+
export { type AgentTelescopeExtensionOptions, AgentTelescopeWatcher, RAG_ENTRY_TYPE, type RagRetrieval, RagTelescopeWatcher, agentActorSpendTableProvider, agentDashboard, agentModelSpendTableProvider, agentPendingApprovalsCountProvider, agentPendingApprovalsTableProvider, agentRecentRunsTableProvider, agentRecentThreadsTableProvider, agentRecentToolCallsTableProvider, agentRunErrorsProvider, agentRunsByAgentTableProvider, agentRunsDurationProvider, agentRunsFailedProvider, agentRunsProvider, agentRunsRetriesProvider, agentRunsSuccessRateProvider, agentRunsTotalProvider, agentRunsTrendProvider, agentSpendByActorProvider, agentSpendByModelProvider, agentSpendTotalProvider, agentTelescopeExtension, agentTokensProvider, agentTokensTotalProvider, agentToolStatsTableProvider, agentToolStatusProvider, agentToolsProvider, agentTopThreadsTableProvider, agentUsageTrendProvider, capErrorMessage, meanChunks, percentile, ragChunksProvider, ragCollectionsTableProvider, ragLatencyProvider, ragRetrievalsProvider, ragRetrieverBreakdownProvider, ragScoresProvider, ragSlowestTableProvider, ragStoreBreakdownProvider, ragTrendProvider, ragZeroHitRateProvider, readRetrievals, resolveRange, shiftUtcDay, shortPromptHash, toActorSpendRows, toActorSpendSegments, toCollectionRows, toLatencyDistribution, toModelSpendRows, toModelSpendSegments, toPendingApprovalTableRows, toRagRetrieval, toRecentRunTableRows, toRecentThreadTableRows, toRecentToolCallRows, toRetrievalTrendRows, toRetrieverSegments, toRunAgentTableRows, toRunErrorSegments, toRunTrendRows, toScoreDistribution, toSlowestRows, toStoreSegments, toThreadSpendRows, toToolStatTableRows, toUsageTrendRows, totalCostUsd, totalTokens, zeroHitRate };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,14 +1,49 @@
|
|
|
1
1
|
import * as _dudousxd_nestjs_telescope from '@dudousxd/nestjs-telescope';
|
|
2
|
-
import { Watcher, WatcherContext, DashboardSpec
|
|
2
|
+
import { DataProvider, DashboardSection, Watcher, WatcherContext, DashboardSpec } from '@dudousxd/nestjs-telescope';
|
|
3
3
|
import { GovernanceRange, ActorSpendRow, ModelSpendRow, PendingApprovalRow, RecentRunRow, ThreadActivityRow, ToolCallActivityRow, RunAgentBreakdownRow, RunErrorBreakdownRow, RunTrendPoint, ThreadSpendRow, ToolStatRow, UsageTrendPoint } from '@dudousxd/nestjs-agent-core';
|
|
4
4
|
|
|
5
|
+
interface AgentTelescopeExtensionOptions {
|
|
6
|
+
/** Deep-link template for a `{threadId}` cell, e.g. `'/admin/threads/{threadId}'`. */
|
|
7
|
+
threadHref?: string;
|
|
8
|
+
/** Deep-link template for a `{runId}` cell. Defaults to the in-app trace waterfall. */
|
|
9
|
+
runHref?: string;
|
|
10
|
+
/**
|
|
11
|
+
* HOST-contributed data providers, registered alongside the built-in ones.
|
|
12
|
+
*
|
|
13
|
+
* This exists because a panel on this dashboard **cannot** bind to a provider contributed by a
|
|
14
|
+
* different extension, and the reason is structural rather than a policy we could relax: the UI
|
|
15
|
+
* derives the request path from the dashboard id (`agent.overview` → `GET /ext/agent/data/:name`),
|
|
16
|
+
* and the controller 404s when `providerOwner(name) !== ext` so the URL namespace cannot be
|
|
17
|
+
* spoofed. A host that registers its own `TelescopeModule` extension therefore gets its own tab,
|
|
18
|
+
* never a panel on this one. Passing the providers through here makes THIS extension their owner,
|
|
19
|
+
* which is what makes a host section on this page resolve at all.
|
|
20
|
+
*
|
|
21
|
+
* Name them under your own prefix (`myapp.rag.collections`); anything starting with `agent.` is
|
|
22
|
+
* refused at boot, because a collision there would surface as Telescope's generic "contributed by
|
|
23
|
+
* both agent and agent" error, which names the same extension twice and says nothing useful.
|
|
24
|
+
*/
|
|
25
|
+
providers?: DataProvider[];
|
|
26
|
+
/**
|
|
27
|
+
* HOST-contributed dashboard sections, appended after the built-in ones. Bind their panels to the
|
|
28
|
+
* providers passed above (or to any built-in `agent.*` provider).
|
|
29
|
+
*
|
|
30
|
+
* Size each section's panel count to an exact multiple of its `cols`, for the same reason every
|
|
31
|
+
* built-in section is: the renderer lays a section out as a fixed `grid-cols-N` grid with no
|
|
32
|
+
* `colSpan`, so an orphan panel leaves a visible hole beside it. That is a layout convention, not
|
|
33
|
+
* something validated here — a gap is cosmetic, and failing a host's boot over cosmetics would be
|
|
34
|
+
* a worse trade than the gap.
|
|
35
|
+
*/
|
|
36
|
+
sections?: DashboardSection[];
|
|
37
|
+
}
|
|
5
38
|
/**
|
|
6
|
-
* The first-class Telescope extension for nestjs-agent: an "Agent" tab fed by
|
|
7
|
-
* the `aviary:agent:*` diagnostics channel (live runs, tool calls) via the watcher,
|
|
39
|
+
* The first-class Telescope extension for nestjs-agent: an "Agent" tab fed by three sources —
|
|
40
|
+
* the `aviary:agent:*` diagnostics channel (live runs, tool calls) via the watcher, the
|
|
8
41
|
* authoritative `AGENT_GOVERNANCE_QUERIES` read-model (historical spend/usage, run reliability,
|
|
9
|
-
* tool activity, the approvals inbox) via the governance providers
|
|
10
|
-
*
|
|
11
|
-
*
|
|
42
|
+
* tool activity, the approvals inbox) via the governance providers, and the `aviary:rag:retrieval`
|
|
43
|
+
* channel (retrieval latency, chunk counts, score distribution, store/collection) via the RAG
|
|
44
|
+
* watcher and providers. The extension `name`, entry-type ids, dashboard id, and every built-in
|
|
45
|
+
* provider name share the `agent` prefix so the registry's global-uniqueness namespaces never
|
|
46
|
+
* collide with sibling extensions.
|
|
12
47
|
*
|
|
13
48
|
* `threadHref` deep-links a table row's `threadId` cell out to the HOST's own thread viewer —
|
|
14
49
|
* passed straight through to {@link agentDashboard}, mirroring `durableTelescopeExtension`'s
|
|
@@ -22,12 +57,11 @@ import { GovernanceRange, ActorSpendRow, ModelSpendRow, PendingApprovalRow, Rece
|
|
|
22
57
|
* (e.g. `store-mikro-orm` / `store-drizzle` / `testing`) — in the same module that registers
|
|
23
58
|
* `TelescopeModule.forRoot({ extensions: [agentTelescopeExtension()] })`. If the binding is
|
|
24
59
|
* absent, those panels render an empty state; the live watcher-fed panels (Runs/Tokens stats and
|
|
25
|
-
* the Tool-call status breakdown) keep working regardless.
|
|
60
|
+
* the Tool-call status breakdown) keep working regardless. The RAG panels need no binding at all —
|
|
61
|
+
* they read Telescope's own storage — but they stay empty until something emits retrieval telemetry
|
|
62
|
+
* (`createRetrievalTool` does by default; `instrumentRetriever` for every other call path).
|
|
26
63
|
*/
|
|
27
|
-
declare function agentTelescopeExtension(opts?:
|
|
28
|
-
threadHref?: string;
|
|
29
|
-
runHref?: string;
|
|
30
|
-
}): _dudousxd_nestjs_telescope.TelescopeExtension;
|
|
64
|
+
declare function agentTelescopeExtension(opts?: AgentTelescopeExtensionOptions): _dudousxd_nestjs_telescope.TelescopeExtension;
|
|
31
65
|
|
|
32
66
|
/**
|
|
33
67
|
* Records `aviary:agent:*` diagnostics events as Telescope entries of type `agent`. It depends
|
|
@@ -52,6 +86,39 @@ declare class AgentTelescopeWatcher implements Watcher {
|
|
|
52
86
|
dispose(): void;
|
|
53
87
|
}
|
|
54
88
|
|
|
89
|
+
/** The Telescope entry `type` retrieval events are recorded as — see the class doc. */
|
|
90
|
+
declare const RAG_ENTRY_TYPE = "agent-rag";
|
|
91
|
+
/**
|
|
92
|
+
* Records `aviary:rag:retrieval` events as Telescope entries of type `agent-rag` — the source every
|
|
93
|
+
* RAG panel on the Agent dashboard reads.
|
|
94
|
+
*
|
|
95
|
+
* **Its own entry type, not `agent`.** Retrieval is per-tool-call where a run is per-conversation, so
|
|
96
|
+
* a busy hour produces an order of magnitude more retrieval events than agent events. Sharing the
|
|
97
|
+
* `agent` type would put them in one storage window: the providers page 5k entries of a type, and
|
|
98
|
+
* retrieval traffic would push `run.finished` out of that window, quietly zeroing the Runs and Tokens
|
|
99
|
+
* stats that have nothing to do with RAG. Separate types make the two independent, and give the
|
|
100
|
+
* Entries screen a "RAG" filter that lists retrievals and only retrievals.
|
|
101
|
+
*
|
|
102
|
+
* **The duration is lifted off the ENVELOPE into the entry content.** The emitter puts it there
|
|
103
|
+
* (`emit`'s `opts.durationMs`) because that is the field Telescope's OTel exporter turns into a
|
|
104
|
+
* latency histogram; the panels need it as an ordinary field they can bucket, and an entry's content
|
|
105
|
+
* is all a `DataProvider` gets. Recording it in both places would mean two sources of truth for the
|
|
106
|
+
* same number, so it is lifted rather than duplicated at the emitter.
|
|
107
|
+
*
|
|
108
|
+
* Claims `rag:retrieval` (refcounted, released in `dispose()`) so
|
|
109
|
+
* `@dudousxd/nestjs-diagnostics-telescope`'s generic bridge skips it instead of recording every
|
|
110
|
+
* retrieval a second time as a generic `diagnostic` entry.
|
|
111
|
+
*
|
|
112
|
+
* Registered by `agentTelescopeExtension()`; also usable on its own in a host's `watchers` list.
|
|
113
|
+
*/
|
|
114
|
+
declare class RagTelescopeWatcher implements Watcher {
|
|
115
|
+
readonly type = "agent-rag";
|
|
116
|
+
private readonly disposers;
|
|
117
|
+
register(ctx: WatcherContext): void;
|
|
118
|
+
/** Detach the subscription and release the diagnostics claim (e.g. on module destroy). */
|
|
119
|
+
dispose(): void;
|
|
120
|
+
}
|
|
121
|
+
|
|
55
122
|
/**
|
|
56
123
|
* The "Agent" overview dashboard. Panels bind to the `agent.*` data providers.
|
|
57
124
|
*
|
|
@@ -63,16 +130,156 @@ declare class AgentTelescopeWatcher implements Watcher {
|
|
|
63
130
|
* still override it. Every table whose rows carry a `threadId`/`runId` gets a `Column.link` for it
|
|
64
131
|
* (via {@link col}); `threadHref` left unset leaves that column plain text.
|
|
65
132
|
*
|
|
66
|
-
* Layout:
|
|
67
|
-
* so its panel count is an exact multiple of its `cols` —
|
|
68
|
-
* `ExtensionDashboardPage` renders a section as a
|
|
69
|
-
* `colSpan`, so a lone table in an otherwise-empty
|
|
133
|
+
* Layout: eight sections (Overview lean; Spend; Reliability; Activity; Approvals; Tools;
|
|
134
|
+
* Retrieval; Retrieval sources), each sized so its panel count is an exact multiple of its `cols` —
|
|
135
|
+
* no half-empty row. `nestjs-telescope`'s `ExtensionDashboardPage` renders a section as a
|
|
136
|
+
* `grid-cols-N` grid with one panel per cell and no `colSpan`, so a lone table in an otherwise-empty
|
|
137
|
+
* row would leave a visible gap next to it.
|
|
138
|
+
*
|
|
139
|
+
* `sections` appends HOST-contributed sections after the built-in ones — see
|
|
140
|
+
* {@link import('./agent-telescope.extension.js').agentTelescopeExtension}'s `sections`/`providers`
|
|
141
|
+
* options, which is the supported way an application puts its own panels (its knowledge-base
|
|
142
|
+
* collections, its ingestion activity) on this page. They are appended rather than interleaved so a
|
|
143
|
+
* host's layout can never push a built-in section out of the row it was sized for.
|
|
70
144
|
*/
|
|
71
145
|
declare function agentDashboard(opts?: {
|
|
72
146
|
threadHref?: string;
|
|
73
147
|
runHref?: string;
|
|
148
|
+
sections?: DashboardSection[];
|
|
74
149
|
}): DashboardSpec;
|
|
75
150
|
|
|
151
|
+
/** One retrieval, normalized out of an entry's untyped `content`. */
|
|
152
|
+
interface RagRetrieval {
|
|
153
|
+
at: Date;
|
|
154
|
+
retriever: string;
|
|
155
|
+
store: string | null;
|
|
156
|
+
collection: string | null;
|
|
157
|
+
chunks: number;
|
|
158
|
+
zeroHit: boolean;
|
|
159
|
+
topScore: number | null;
|
|
160
|
+
durationMs: number | null;
|
|
161
|
+
failed: boolean;
|
|
162
|
+
}
|
|
163
|
+
interface BreakdownSegment$1 {
|
|
164
|
+
label: string;
|
|
165
|
+
value: number;
|
|
166
|
+
}
|
|
167
|
+
interface DistributionBucket {
|
|
168
|
+
label: string;
|
|
169
|
+
count: number;
|
|
170
|
+
}
|
|
171
|
+
/** One row of the per-collection table. */
|
|
172
|
+
interface CollectionRow {
|
|
173
|
+
collection: string;
|
|
174
|
+
store: string;
|
|
175
|
+
retrievals: number;
|
|
176
|
+
zeroHits: number;
|
|
177
|
+
p95Ms: number | string;
|
|
178
|
+
meanTopScore: number | string;
|
|
179
|
+
}
|
|
180
|
+
/** One row of the slowest-retrievals table. */
|
|
181
|
+
interface SlowestRow {
|
|
182
|
+
at: string;
|
|
183
|
+
retriever: string;
|
|
184
|
+
store: string;
|
|
185
|
+
collection: string;
|
|
186
|
+
chunks: number;
|
|
187
|
+
topScore: number | string;
|
|
188
|
+
durationMs: number | string;
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* Normalize one storage entry into a {@link RagRetrieval}, or `null` when it is not a retrieval this
|
|
192
|
+
* shaping understands. Field-by-field `typeof` checks rather than a cast: entries are persisted JSON
|
|
193
|
+
* written by a possibly older producer, so "the field is there and is a number" is a runtime
|
|
194
|
+
* question, and a missing `durationMs` must land as `null` (excluded from percentiles) rather than as
|
|
195
|
+
* a `NaN` that silently poisons every aggregate it touches.
|
|
196
|
+
*/
|
|
197
|
+
declare function toRagRetrieval(entry: unknown): RagRetrieval | null;
|
|
198
|
+
/**
|
|
199
|
+
* Every retrieval in the window, newest first. Takes the resolved binding as `unknown` and narrows it
|
|
200
|
+
* here, so a host whose Telescope storage is missing or unrecognised gets empty panels rather than a
|
|
201
|
+
* 502 on every RAG card — the same degrade-don't-throw posture the governance providers take.
|
|
202
|
+
*/
|
|
203
|
+
declare function readRetrievals(storage: unknown): Promise<RagRetrieval[]>;
|
|
204
|
+
/** Nearest-rank percentile over an unsorted sample. `null` for an empty sample. */
|
|
205
|
+
declare function percentile(samples: number[], fraction: number): number | null;
|
|
206
|
+
/** Fraction (0–1) of retrievals that came back with nothing. `0` over an empty window. */
|
|
207
|
+
declare function zeroHitRate(retrievals: RagRetrieval[]): number;
|
|
208
|
+
/** Mean passages returned per retrieval — `topK` minus this is how often the corpus ran out. */
|
|
209
|
+
declare function meanChunks(retrievals: RagRetrieval[]): number;
|
|
210
|
+
/** Latency histogram over the ladder above, plus p50/p95/p99 of the raw samples. */
|
|
211
|
+
declare function toLatencyDistribution(retrievals: RagRetrieval[]): {
|
|
212
|
+
buckets: DistributionBucket[];
|
|
213
|
+
p50?: number;
|
|
214
|
+
p95?: number;
|
|
215
|
+
p99?: number;
|
|
216
|
+
};
|
|
217
|
+
/**
|
|
218
|
+
* Top-score histogram — **for ONE retriever kind at a time**, which is the whole reason this takes a
|
|
219
|
+
* `retriever` argument instead of bucketing everything it was handed.
|
|
220
|
+
*
|
|
221
|
+
* A `Passage.score` has no shared meaning across strategies: `EmbeddingRetriever` returns a
|
|
222
|
+
* calibrated cosine similarity in `[0, 1]`, a BM25 leg returns an unbounded relevance score in the
|
|
223
|
+
* tens, and `HybridRetriever`'s RRF returns a rank-derived number that lives in a band about one
|
|
224
|
+
* part in fifty wide and says nothing about quality at all (see the score section on
|
|
225
|
+
* `HybridRetriever`). Pouring those into one histogram produces a chart where the bins mean a
|
|
226
|
+
* different thing per bar — and the reading it invites, "our scores collapsed", would be a change in
|
|
227
|
+
* traffic MIX rather than in retrieval quality.
|
|
228
|
+
*
|
|
229
|
+
* So the panel binds `query.retriever: 'embedding'`: the dense leg is the only population whose
|
|
230
|
+
* score is comparable across queries, which is also why `minScore` lives on that retriever alone.
|
|
231
|
+
* Scores outside `[0, 1]` are clamped into the end bins rather than dropped, so a store that reports
|
|
232
|
+
* a distance-derived score still shows up as an obviously-pinned bar instead of an empty chart.
|
|
233
|
+
*/
|
|
234
|
+
declare function toScoreDistribution(retrievals: RagRetrieval[], retriever: string): {
|
|
235
|
+
buckets: DistributionBucket[];
|
|
236
|
+
p50?: number;
|
|
237
|
+
p95?: number;
|
|
238
|
+
};
|
|
239
|
+
/**
|
|
240
|
+
* Retrievals and zero-hits per UTC hour, oldest bucket first, over the last {@link TREND_HOURS}.
|
|
241
|
+
* Empty hours are emitted as zeroes so a quiet stretch reads as a gap in the line rather than
|
|
242
|
+
* disappearing and making the outage look like it never happened.
|
|
243
|
+
*/
|
|
244
|
+
declare function toRetrievalTrendRows(retrievals: RagRetrieval[], now?: Date): Array<{
|
|
245
|
+
label: string;
|
|
246
|
+
retrievals: number;
|
|
247
|
+
zeroHits: number;
|
|
248
|
+
}>;
|
|
249
|
+
/** Retrievals by vector store (`memory`/`pg`/`redis`, or `—` for an in-process retriever). */
|
|
250
|
+
declare function toStoreSegments(retrievals: RagRetrieval[]): BreakdownSegment$1[];
|
|
251
|
+
/** Retrievals by strategy — the composed kind, since that is what answered. */
|
|
252
|
+
declare function toRetrieverSegments(retrievals: RagRetrieval[]): BreakdownSegment$1[];
|
|
253
|
+
/** Per-collection rollup, busiest first. `p95Ms`/`meanTopScore` fall back to `—` when unmeasured. */
|
|
254
|
+
declare function toCollectionRows(retrievals: RagRetrieval[], limit?: number): CollectionRow[];
|
|
255
|
+
/**
|
|
256
|
+
* The slowest retrievals in the window, worst first — the table an operator opens after the p95
|
|
257
|
+
* moves. Sorted by duration rather than by recency on purpose: "recent retrievals" would be a list
|
|
258
|
+
* of whatever just happened, which the trend already shows in aggregate, whereas the tail is the
|
|
259
|
+
* only place a single pathological query is visible at all.
|
|
260
|
+
*/
|
|
261
|
+
declare function toSlowestRows(retrievals: RagRetrieval[], limit?: number): SlowestRow[];
|
|
262
|
+
/** stat → retrievals in the window. */
|
|
263
|
+
declare function ragRetrievalsProvider(): DataProvider;
|
|
264
|
+
/** stat (percent) → share of retrievals that returned nothing. */
|
|
265
|
+
declare function ragZeroHitRateProvider(): DataProvider;
|
|
266
|
+
/** stat → mean passages returned per retrieval. */
|
|
267
|
+
declare function ragChunksProvider(): DataProvider;
|
|
268
|
+
/** distribution → retrieval latency, with p50/p95/p99 markers. */
|
|
269
|
+
declare function ragLatencyProvider(): DataProvider;
|
|
270
|
+
/** distribution → top score for ONE retriever kind (`query.retriever`, default `embedding`). */
|
|
271
|
+
declare function ragScoresProvider(): DataProvider;
|
|
272
|
+
/** timeseries → retrievals and zero-hits per hour. */
|
|
273
|
+
declare function ragTrendProvider(): DataProvider;
|
|
274
|
+
/** breakdown → retrievals by vector store. */
|
|
275
|
+
declare function ragStoreBreakdownProvider(): DataProvider;
|
|
276
|
+
/** breakdown → retrievals by retriever strategy. */
|
|
277
|
+
declare function ragRetrieverBreakdownProvider(): DataProvider;
|
|
278
|
+
/** table → per-collection rollup, busiest first. */
|
|
279
|
+
declare function ragCollectionsTableProvider(): DataProvider;
|
|
280
|
+
/** table → the slowest retrievals in the window. */
|
|
281
|
+
declare function ragSlowestTableProvider(): DataProvider;
|
|
282
|
+
|
|
76
283
|
/** stat → total agent runs (run.finished entries). */
|
|
77
284
|
declare function agentRunsProvider(): DataProvider;
|
|
78
285
|
/** stat → total tokens across all finished runs. */
|
|
@@ -296,4 +503,4 @@ declare function agentPendingApprovalsTableProvider(): DataProvider;
|
|
|
296
503
|
/** table → per-tool call/failure/rejection/latency rollup over the range. */
|
|
297
504
|
declare function agentToolStatsTableProvider(): DataProvider;
|
|
298
505
|
|
|
299
|
-
export { AgentTelescopeWatcher, agentActorSpendTableProvider, agentDashboard, agentModelSpendTableProvider, agentPendingApprovalsCountProvider, agentPendingApprovalsTableProvider, agentRecentRunsTableProvider, agentRecentThreadsTableProvider, agentRecentToolCallsTableProvider, agentRunErrorsProvider, agentRunsByAgentTableProvider, agentRunsDurationProvider, agentRunsFailedProvider, agentRunsProvider, agentRunsRetriesProvider, agentRunsSuccessRateProvider, agentRunsTotalProvider, agentRunsTrendProvider, agentSpendByActorProvider, agentSpendByModelProvider, agentSpendTotalProvider, agentTelescopeExtension, agentTokensProvider, agentTokensTotalProvider, agentToolStatsTableProvider, agentToolStatusProvider, agentToolsProvider, agentTopThreadsTableProvider, agentUsageTrendProvider, capErrorMessage, resolveRange, shiftUtcDay, shortPromptHash, toActorSpendRows, toActorSpendSegments, toModelSpendRows, toModelSpendSegments, toPendingApprovalTableRows, toRecentRunTableRows, toRecentThreadTableRows, toRecentToolCallRows, toRunAgentTableRows, toRunErrorSegments, toRunTrendRows, toThreadSpendRows, toToolStatTableRows, toUsageTrendRows, totalCostUsd, totalTokens };
|
|
506
|
+
export { type AgentTelescopeExtensionOptions, AgentTelescopeWatcher, RAG_ENTRY_TYPE, type RagRetrieval, RagTelescopeWatcher, agentActorSpendTableProvider, agentDashboard, agentModelSpendTableProvider, agentPendingApprovalsCountProvider, agentPendingApprovalsTableProvider, agentRecentRunsTableProvider, agentRecentThreadsTableProvider, agentRecentToolCallsTableProvider, agentRunErrorsProvider, agentRunsByAgentTableProvider, agentRunsDurationProvider, agentRunsFailedProvider, agentRunsProvider, agentRunsRetriesProvider, agentRunsSuccessRateProvider, agentRunsTotalProvider, agentRunsTrendProvider, agentSpendByActorProvider, agentSpendByModelProvider, agentSpendTotalProvider, agentTelescopeExtension, agentTokensProvider, agentTokensTotalProvider, agentToolStatsTableProvider, agentToolStatusProvider, agentToolsProvider, agentTopThreadsTableProvider, agentUsageTrendProvider, capErrorMessage, meanChunks, percentile, ragChunksProvider, ragCollectionsTableProvider, ragLatencyProvider, ragRetrievalsProvider, ragRetrieverBreakdownProvider, ragScoresProvider, ragSlowestTableProvider, ragStoreBreakdownProvider, ragTrendProvider, ragZeroHitRateProvider, readRetrievals, resolveRange, shiftUtcDay, shortPromptHash, toActorSpendRows, toActorSpendSegments, toCollectionRows, toLatencyDistribution, toModelSpendRows, toModelSpendSegments, toPendingApprovalTableRows, toRagRetrieval, toRecentRunTableRows, toRecentThreadTableRows, toRecentToolCallRows, toRetrievalTrendRows, toRetrieverSegments, toRunAgentTableRows, toRunErrorSegments, toRunTrendRows, toScoreDistribution, toSlowestRows, toStoreSegments, toThreadSpendRows, toToolStatTableRows, toUsageTrendRows, totalCostUsd, totalTokens, zeroHitRate };
|