@dudousxd/nestjs-agent-telescope 0.7.0 → 0.8.0

package/dist/index.js

@@ -492,7 +492,180 @@ function agentDashboard(opts = {}) {
             ]
           }
         ]
-      }
+      },
+      {
+        // RAG. Fed by the `aviary:rag:retrieval` telemetry `@dudousxd/nestjs-agent-rag` emits, NOT
+        // derived from tool-call rows: a tool call that retrieved nothing and one that retrieved
+        // five perfect passages are the same row with the same `ok` status, so every number here
+        // would have been unanswerable from that side.
+        title: "Retrieval",
+        cols: 3,
+        panels: [
+          {
+            kind: "stat",
+            title: "Retrievals",
+            data: {
+              provider: "agent.rag.retrievals"
+            }
+          },
+          {
+            kind: "stat",
+            title: "Zero-hit rate",
+            data: {
+              provider: "agent.rag.zeroHitRate"
+            },
+            format: "percent"
+          },
+          {
+            kind: "stat",
+            title: "Passages per retrieval",
+            data: {
+              provider: "agent.rag.chunks"
+            },
+            format: "number"
+          },
+          {
+            // A real histogram, unlike the run-duration pair above: retrieval events carry the raw
+            // per-call duration, so there ARE samples to bucket and markers to place.
+            kind: "distribution",
+            title: "Retrieval latency",
+            data: {
+              provider: "agent.rag.latency"
+            },
+            markers: [
+              "p50",
+              "p95",
+              "p99"
+            ],
+            format: "duration"
+          },
+          {
+            // Bound to ONE retriever kind on purpose — a cosine similarity, a BM25 score and an RRF
+            // rank score share no scale, so a histogram over all of them has bins that mean a
+            // different thing per bar. See `toScoreDistribution`.
+            kind: "distribution",
+            title: "Top score (dense)",
+            data: {
+              provider: "agent.rag.scores",
+              query: {
+                retriever: "embedding"
+              }
+            },
+            markers: [
+              "p50",
+              "p95"
+            ],
+            format: "number"
+          },
+          {
+            kind: "timeseries",
+            title: "Retrievals & zero-hits",
+            data: {
+              provider: "agent.rag.trend"
+            },
+            series: [
+              "retrievals",
+              "zeroHits"
+            ],
+            style: "stacked"
+          }
+        ]
+      },
+      {
+        title: "Retrieval sources",
+        cols: 2,
+        panels: [
+          {
+            kind: "breakdown",
+            title: "Retrievals by store",
+            data: {
+              provider: "agent.rag.byStore"
+            },
+            style: "donut"
+          },
+          {
+            kind: "breakdown",
+            title: "Retrievals by retriever",
+            data: {
+              provider: "agent.rag.byRetriever"
+            },
+            style: "donut"
+          },
+          {
+            kind: "table",
+            title: "By collection",
+            data: {
+              provider: "agent.rag.byCollection"
+            },
+            columns: [
+              {
+                key: "collection",
+                label: "Collection"
+              },
+              {
+                key: "store",
+                label: "Store"
+              },
+              {
+                key: "retrievals",
+                label: "Retrievals"
+              },
+              {
+                key: "zeroHits",
+                label: "Zero-hits"
+              },
+              {
+                key: "p95Ms",
+                label: "p95 (ms)"
+              },
+              {
+                key: "meanTopScore",
+                label: "Mean top score"
+              }
+            ]
+          },
+          {
+            kind: "table",
+            title: "Slowest retrievals",
+            data: {
+              provider: "agent.rag.slowest"
+            },
+            // Seven columns, matching the recent-runs table's width: this is the widest a table can
+            // get here before it overflows its card.
+            columns: [
+              {
+                key: "at",
+                label: "When"
+              },
+              {
+                key: "retriever",
+                label: "Retriever"
+              },
+              {
+                key: "store",
+                label: "Store"
+              },
+              {
+                key: "collection",
+                label: "Collection"
+              },
+              {
+                key: "chunks",
+                label: "Passages"
+              },
+              {
+                key: "topScore",
+                label: "Top score"
+              },
+              {
+                key: "durationMs",
+                label: "Duration (ms)"
+              }
+            ]
+          }
+        ]
+      },
+      ...opts.sections ?? []
     ]
   };
 }
@@ -1066,22 +1239,478 @@ var AgentTelescopeWatcher = class {
   }
 };
 
+// src/rag-data-providers.ts
+import { TELESCOPE_STORAGE as TELESCOPE_STORAGE2 } from "@dudousxd/nestjs-telescope";
+
+// src/rag-telescope.watcher.ts
+import { subscribe as subscribe2, unsubscribe as unsubscribe2 } from "node:diagnostics_channel";
+import { channelName as channelName2, claimDiagnostics as claimDiagnostics2 } from "@dudousxd/nestjs-diagnostics";
+var RAG_LIB = "rag";
+var RETRIEVAL_EVENT = "retrieval";
+var RAG_ENTRY_TYPE = "agent-rag";
+function isEnvelope(message) {
+  return typeof message === "object" && message !== null && typeof Reflect.get(message, "payload") === "object" && Reflect.get(message, "payload") !== null;
+}
+__name(isEnvelope, "isEnvelope");
+var RagTelescopeWatcher = class {
+  static {
+    __name(this, "RagTelescopeWatcher");
+  }
+  type = RAG_ENTRY_TYPE;
+  disposers = [];
+  register(ctx) {
+    this.disposers.push(claimDiagnostics2(RAG_LIB, [
+      RETRIEVAL_EVENT
+    ]));
+    const channel = channelName2(RAG_LIB, RETRIEVAL_EVENT);
+    const onMessage = /* @__PURE__ */ __name((message) => {
+      if (!isEnvelope(message)) {
+        return;
+      }
+      ctx.record({
+        type: RAG_ENTRY_TYPE,
+        content: {
+          event: RETRIEVAL_EVENT,
+          ...message.payload,
+          ...message.durationMs !== void 0 ? {
+            durationMs: message.durationMs
+          } : {}
+        },
+        // Tagged by store and retriever kind so the Entries screen can filter to "every retrieval
+        // pgvector served" without a provider — the same tag-per-dimension the agent watcher uses.
+        tags: tagsOf(message.payload)
+      });
+    }, "onMessage");
+    subscribe2(channel, onMessage);
+    this.disposers.push(() => unsubscribe2(channel, onMessage));
+  }
+  /** Detach the subscription and release the diagnostics claim (e.g. on module destroy). */
+  dispose() {
+    while (this.disposers.length) this.disposers.pop()?.();
+  }
+};
+function tagsOf(payload) {
+  const tags = [
+    RETRIEVAL_EVENT
+  ];
+  if (typeof payload.retriever === "string") {
+    tags.push(`retriever:${payload.retriever}`);
+  }
+  if (typeof payload.store === "string") {
+    tags.push(`store:${payload.store}`);
+  }
+  if (payload.zeroHit === true) {
+    tags.push("zero-hit");
+  }
+  if (payload.failed === true) {
+    tags.push("failed");
+  }
+  return tags;
+}
+__name(tagsOf, "tagsOf");
+
+// src/rag-data-providers.ts
+var RETRIEVAL_WINDOW = 5e3;
+var TABLE_LIMIT = 10;
+var TREND_HOURS = 24;
+var NO_VALUE2 = "\u2014";
+function isRecord2(value) {
+  return typeof value === "object" && value !== null;
+}
+__name(isRecord2, "isRecord");
+function readString(source, key) {
+  const value = source[key];
+  return typeof value === "string" ? value : null;
+}
+__name(readString, "readString");
+function readNumber(source, key) {
+  const value = source[key];
+  return typeof value === "number" && Number.isFinite(value) ? value : null;
+}
+__name(readNumber, "readNumber");
+function isEntryStorage(value) {
+  return isRecord2(value) && typeof value.get === "function";
+}
+__name(isEntryStorage, "isEntryStorage");
+function toRagRetrieval(entry) {
+  if (!isRecord2(entry) || !isRecord2(entry.content)) {
+    return null;
+  }
+  const content = entry.content;
+  const retriever = readString(content, "retriever");
+  if (retriever === null) {
+    return null;
+  }
+  const chunks = readNumber(content, "chunks") ?? 0;
+  const createdAt = entry.createdAt;
+  return {
+    at: createdAt instanceof Date ? createdAt : /* @__PURE__ */ new Date(0),
+    retriever,
+    store: readString(content, "store"),
+    collection: readString(content, "collection"),
+    chunks,
+    zeroHit: content.zeroHit === true || chunks === 0,
+    topScore: readNumber(content, "topScore"),
+    durationMs: readNumber(content, "durationMs"),
+    failed: content.failed === true
+  };
+}
+__name(toRagRetrieval, "toRagRetrieval");
+async function readRetrievals(storage) {
+  if (!isEntryStorage(storage)) {
+    return [];
+  }
+  const page = await storage.get({
+    type: RAG_ENTRY_TYPE,
+    limit: RETRIEVAL_WINDOW
+  });
+  const retrievals = [];
+  for (const entry of page.data) {
+    const retrieval = toRagRetrieval(entry);
+    if (retrieval !== null) {
+      retrievals.push(retrieval);
+    }
+  }
+  return retrievals;
+}
+__name(readRetrievals, "readRetrievals");
+function retrievalStorage(ctx) {
+  try {
+    return ctx.moduleRef.get(TELESCOPE_STORAGE2, {
+      strict: false
+    });
+  } catch {
+    return void 0;
+  }
+}
+__name(retrievalStorage, "retrievalStorage");
+function percentile(samples, fraction) {
+  if (samples.length === 0) {
+    return null;
+  }
+  const sorted = [
+    ...samples
+  ].sort((a, b) => a - b);
+  const rank = Math.ceil(fraction * sorted.length);
+  return sorted[Math.min(Math.max(rank, 1), sorted.length) - 1] ?? null;
+}
+__name(percentile, "percentile");
+function round(value, places) {
+  const factor = 10 ** places;
+  return Math.round(value * factor) / factor;
+}
+__name(round, "round");
+function zeroHitRate(retrievals) {
+  if (retrievals.length === 0) {
+    return 0;
+  }
+  const zeroHits = retrievals.filter((retrieval) => retrieval.zeroHit).length;
+  return round(zeroHits / retrievals.length, 4);
+}
+__name(zeroHitRate, "zeroHitRate");
+function meanChunks(retrievals) {
+  if (retrievals.length === 0) {
+    return 0;
+  }
+  const total = retrievals.reduce((sum, retrieval) => sum + retrieval.chunks, 0);
+  return round(total / retrievals.length, 2);
+}
+__name(meanChunks, "meanChunks");
+var LATENCY_EDGES = [
+  10,
+  25,
+  50,
+  100,
+  250,
+  500,
+  1e3,
+  2500,
+  5e3
+];
+function latencyLabel(index) {
+  const upper = LATENCY_EDGES[index];
+  if (upper === void 0) {
+    return `\u2265${LATENCY_EDGES[LATENCY_EDGES.length - 1] ?? 0}ms`;
+  }
+  return `${index === 0 ? 0 : LATENCY_EDGES[index - 1] ?? 0}\u2013${upper}ms`;
+}
+__name(latencyLabel, "latencyLabel");
+function toLatencyDistribution(retrievals) {
+  const samples = [];
+  const counts = new Array(LATENCY_EDGES.length + 1).fill(0);
+  for (const { durationMs } of retrievals) {
+    if (durationMs === null) {
+      continue;
+    }
+    samples.push(durationMs);
+    const found = LATENCY_EDGES.findIndex((edge) => durationMs < edge);
+    const index = found === -1 ? LATENCY_EDGES.length : found;
+    counts[index] = (counts[index] ?? 0) + 1;
+  }
+  const buckets = counts.map((count, index) => ({
+    label: latencyLabel(index),
+    count
+  }));
+  const p50 = percentile(samples, 0.5);
+  const p95 = percentile(samples, 0.95);
+  const p99 = percentile(samples, 0.99);
+  return {
+    buckets,
+    ...p50 !== null ? {
+      p50
+    } : {},
+    ...p95 !== null ? {
+      p95
+    } : {},
+    ...p99 !== null ? {
+      p99
+    } : {}
+  };
+}
+__name(toLatencyDistribution, "toLatencyDistribution");
+var SCORE_BINS = 10;
+function toScoreDistribution(retrievals, retriever) {
+  const samples = [];
+  const counts = new Array(SCORE_BINS).fill(0);
+  for (const retrieval of retrievals) {
+    if (retrieval.retriever !== retriever || retrieval.topScore === null) {
+      continue;
+    }
+    samples.push(retrieval.topScore);
+    const clamped = Math.min(Math.max(retrieval.topScore, 0), 1);
+    const index = Math.min(Math.floor(clamped * SCORE_BINS), SCORE_BINS - 1);
+    counts[index] = (counts[index] ?? 0) + 1;
+  }
+  const buckets = counts.map((count, index) => ({
+    label: `${round(index / SCORE_BINS, 1)}\u2013${round((index + 1) / SCORE_BINS, 1)}`,
+    count
+  }));
+  const p50 = percentile(samples, 0.5);
+  const p95 = percentile(samples, 0.95);
+  return {
+    buckets,
+    ...p50 !== null ? {
+      p50: round(p50, 4)
+    } : {},
+    ...p95 !== null ? {
+      p95: round(p95, 4)
+    } : {}
+  };
+}
+__name(toScoreDistribution, "toScoreDistribution");
+function hourStart(at) {
+  return Math.floor(at.getTime() / 36e5) * 36e5;
+}
+__name(hourStart, "hourStart");
+function toRetrievalTrendRows(retrievals, now = /* @__PURE__ */ new Date()) {
+  const latest = hourStart(now);
+  const earliest = latest - (TREND_HOURS - 1) * 36e5;
+  const buckets = /* @__PURE__ */ new Map();
+  for (let hour = earliest; hour <= latest; hour += 36e5) {
+    buckets.set(hour, {
+      retrievals: 0,
+      zeroHits: 0
+    });
+  }
+  for (const retrieval of retrievals) {
+    const bucket = buckets.get(hourStart(retrieval.at));
+    if (bucket === void 0) {
+      continue;
+    }
+    bucket.retrievals += 1;
+    if (retrieval.zeroHit) {
+      bucket.zeroHits += 1;
+    }
+  }
+  return [
+    ...buckets.entries()
+  ].map(([hour, counts]) => ({
+    label: `${new Date(hour).toISOString().slice(11, 13)}:00`,
+    retrievals: counts.retrievals,
+    zeroHits: counts.zeroHits
+  }));
+}
+__name(toRetrievalTrendRows, "toRetrievalTrendRows");
+function segmentsBy(retrievals, pick) {
+  const counts = /* @__PURE__ */ new Map();
+  for (const retrieval of retrievals) {
+    const label = pick(retrieval) ?? NO_VALUE2;
+    counts.set(label, (counts.get(label) ?? 0) + 1);
+  }
+  return [
+    ...counts.entries()
+  ].map(([label, value]) => ({
+    label,
+    value
+  })).sort((a, b) => b.value - a.value);
+}
+__name(segmentsBy, "segmentsBy");
+function toStoreSegments(retrievals) {
+  return segmentsBy(retrievals, (retrieval) => retrieval.store);
+}
+__name(toStoreSegments, "toStoreSegments");
+function toRetrieverSegments(retrievals) {
+  return segmentsBy(retrievals, (retrieval) => retrieval.retriever);
+}
+__name(toRetrieverSegments, "toRetrieverSegments");
+function toCollectionRows(retrievals, limit = TABLE_LIMIT) {
+  const groups = /* @__PURE__ */ new Map();
+  for (const retrieval of retrievals) {
+    const key = retrieval.collection ?? NO_VALUE2;
+    const group = groups.get(key);
+    if (group === void 0) {
+      groups.set(key, [
+        retrieval
+      ]);
+    } else {
+      group.push(retrieval);
+    }
+  }
+  return [
+    ...groups.entries()
+  ].map(([collection, group]) => {
+    const durations = [];
+    const scores = [];
+    for (const retrieval of group) {
+      if (retrieval.durationMs !== null) {
+        durations.push(retrieval.durationMs);
+      }
+      if (retrieval.topScore !== null) {
+        scores.push(retrieval.topScore);
+      }
+    }
+    const p95 = percentile(durations, 0.95);
+    const meanScore = scores.length === 0 ? null : scores.reduce((sum, score) => sum + score, 0) / scores.length;
+    return {
+      collection,
+      store: group[0]?.store ?? NO_VALUE2,
+      retrievals: group.length,
+      zeroHits: group.filter((retrieval) => retrieval.zeroHit).length,
+      p95Ms: p95 === null ? NO_VALUE2 : Math.round(p95),
+      meanTopScore: meanScore === null ? NO_VALUE2 : round(meanScore, 3)
+    };
+  }).sort((a, b) => b.retrievals - a.retrievals).slice(0, limit);
+}
+__name(toCollectionRows, "toCollectionRows");
+function toSlowestRows(retrievals, limit = TABLE_LIMIT) {
+  return retrievals.filter((retrieval) => retrieval.durationMs !== null).sort((a, b) => (b.durationMs ?? 0) - (a.durationMs ?? 0)).slice(0, limit).map((retrieval) => ({
+    at: retrieval.at.toISOString(),
+    retriever: retrieval.retriever,
+    store: retrieval.store ?? NO_VALUE2,
+    collection: retrieval.collection ?? NO_VALUE2,
+    chunks: retrieval.chunks,
+    topScore: retrieval.topScore === null ? NO_VALUE2 : round(retrieval.topScore, 3),
+    durationMs: retrieval.durationMs === null ? NO_VALUE2 : Math.round(retrieval.durationMs)
+  }));
+}
+__name(toSlowestRows, "toSlowestRows");
+function retrievalProvider(name, format) {
+  return {
+    name,
+    async resolve(query, ctx) {
+      return format(await readRetrievals(retrievalStorage(ctx)), query);
+    }
+  };
+}
+__name(retrievalProvider, "retrievalProvider");
+function ragRetrievalsProvider() {
+  return retrievalProvider("agent.rag.retrievals", (retrievals) => ({
+    value: retrievals.length
+  }));
+}
+__name(ragRetrievalsProvider, "ragRetrievalsProvider");
+function ragZeroHitRateProvider() {
+  return retrievalProvider("agent.rag.zeroHitRate", (retrievals) => ({
+    value: zeroHitRate(retrievals)
+  }));
+}
+__name(ragZeroHitRateProvider, "ragZeroHitRateProvider");
+function ragChunksProvider() {
+  return retrievalProvider("agent.rag.chunks", (retrievals) => ({
+    value: meanChunks(retrievals)
+  }));
+}
+__name(ragChunksProvider, "ragChunksProvider");
+function ragLatencyProvider() {
+  return retrievalProvider("agent.rag.latency", toLatencyDistribution);
+}
+__name(ragLatencyProvider, "ragLatencyProvider");
+function ragScoresProvider() {
+  return retrievalProvider("agent.rag.scores", (retrievals, query) => {
+    const requested = query?.retriever;
+    return toScoreDistribution(retrievals, typeof requested === "string" ? requested : "embedding");
+  });
+}
+__name(ragScoresProvider, "ragScoresProvider");
+function ragTrendProvider() {
+  return retrievalProvider("agent.rag.trend", (retrievals) => ({
+    rows: toRetrievalTrendRows(retrievals)
+  }));
+}
+__name(ragTrendProvider, "ragTrendProvider");
+function ragStoreBreakdownProvider() {
+  return retrievalProvider("agent.rag.byStore", (retrievals) => ({
+    segments: toStoreSegments(retrievals)
+  }));
+}
+__name(ragStoreBreakdownProvider, "ragStoreBreakdownProvider");
+function ragRetrieverBreakdownProvider() {
+  return retrievalProvider("agent.rag.byRetriever", (retrievals) => ({
+    segments: toRetrieverSegments(retrievals)
+  }));
+}
+__name(ragRetrieverBreakdownProvider, "ragRetrieverBreakdownProvider");
+function ragCollectionsTableProvider() {
+  return retrievalProvider("agent.rag.byCollection", (retrievals) => ({
+    rows: toCollectionRows(retrievals)
+  }));
+}
+__name(ragCollectionsTableProvider, "ragCollectionsTableProvider");
+function ragSlowestTableProvider() {
+  return retrievalProvider("agent.rag.slowest", (retrievals) => ({
+    rows: toSlowestRows(retrievals)
+  }));
+}
+__name(ragSlowestTableProvider, "ragSlowestTableProvider");
+
 // src/agent-telescope.extension.ts
+var RESERVED_PROVIDER_PREFIX = "agent.";
 function agentTelescopeExtension(opts = {}) {
+  const hostProviders = opts.providers ?? [];
+  const reserved = hostProviders.filter((provider) => provider.name.startsWith(RESERVED_PROVIDER_PREFIX));
+  if (reserved.length > 0) {
+    throw new Error(`agentTelescopeExtension: host-contributed data providers may not use the reserved "${RESERVED_PROVIDER_PREFIX}" prefix \u2014 rename ${reserved.map((provider) => `"${provider.name}"`).join(", ")} to your own namespace.`);
+  }
   return defineTelescopeExtension({
     name: "agent",
     watchers: /* @__PURE__ */ __name(() => [
-      new AgentTelescopeWatcher()
+      new AgentTelescopeWatcher(),
+      new RagTelescopeWatcher()
     ], "watchers"),
     entryTypes: /* @__PURE__ */ __name(() => [
       {
         id: "agent",
         label: "Agent",
         dot: "bg-violet-400"
+      },
+      {
+        id: "agent-rag",
+        label: "RAG",
+        dot: "bg-emerald-400"
       }
     ], "entryTypes"),
     dashboards: /* @__PURE__ */ __name(() => [
-      agentDashboard(opts)
+      agentDashboard({
+        ...opts.threadHref !== void 0 ? {
+          threadHref: opts.threadHref
+        } : {},
+        ...opts.runHref !== void 0 ? {
+          runHref: opts.runHref
+        } : {},
+        ...opts.sections !== void 0 ? {
+          sections: opts.sections
+        } : {}
+      })
     ], "dashboards"),
     dataProviders: /* @__PURE__ */ __name(() => [
       agentRunsProvider(),
@@ -1111,13 +1740,26 @@ function agentTelescopeExtension(opts = {}) {
       agentThreadsPagedTableProvider(),
       agentPendingApprovalsCountProvider(),
       agentPendingApprovalsTableProvider(),
-      agentToolStatsTableProvider()
+      agentToolStatsTableProvider(),
+      ragRetrievalsProvider(),
+      ragZeroHitRateProvider(),
+      ragChunksProvider(),
+      ragLatencyProvider(),
+      ragScoresProvider(),
+      ragTrendProvider(),
+      ragStoreBreakdownProvider(),
+      ragRetrieverBreakdownProvider(),
+      ragCollectionsTableProvider(),
+      ragSlowestTableProvider(),
+      ...hostProviders
     ], "dataProviders")
   });
 }
 __name(agentTelescopeExtension, "agentTelescopeExtension");
 export {
   AgentTelescopeWatcher,
+  RAG_ENTRY_TYPE,
+  RagTelescopeWatcher,
   agentActorSpendTableProvider,
   agentDashboard,
   agentModelSpendTableProvider,
@@ -1147,24 +1789,46 @@ export {
   agentTopThreadsTableProvider,
   agentUsageTrendProvider,
   capErrorMessage,
+  meanChunks,
+  percentile,
+  ragChunksProvider,
+  ragCollectionsTableProvider,
+  ragLatencyProvider,
+  ragRetrievalsProvider,
+  ragRetrieverBreakdownProvider,
+  ragScoresProvider,
+  ragSlowestTableProvider,
+  ragStoreBreakdownProvider,
+  ragTrendProvider,
+  ragZeroHitRateProvider,
+  readRetrievals,
   resolveRange,
   shiftUtcDay,
   shortPromptHash,
   toActorSpendRows,
   toActorSpendSegments,
+  toCollectionRows,
+  toLatencyDistribution,
   toModelSpendRows,
   toModelSpendSegments,
   toPendingApprovalTableRows,
+  toRagRetrieval,
   toRecentRunTableRows,
   toRecentThreadTableRows,
   toRecentToolCallRows,
+  toRetrievalTrendRows,
+  toRetrieverSegments,
   toRunAgentTableRows,
   toRunErrorSegments,
   toRunTrendRows,
+  toScoreDistribution,
+  toSlowestRows,
+  toStoreSegments,
   toThreadSpendRows,
   toToolStatTableRows,
   toUsageTrendRows,
   totalCostUsd,
-  totalTokens
+  totalTokens,
+  zeroHitRate
 };
 //# sourceMappingURL=index.js.map