@mcp-graph-workflow/mcp-graph 5.6.1 → 5.7.0

package/dist/core/rag/post-retrieval.js

@@ -0,0 +1,119 @@
+/**
+ * Post-Retrieval Pipeline — processes search results after initial retrieval.
+ *
+ * Pipeline stages:
+ * 1. Deduplication — remove results with identical content
+ * 2. Reranking — boost results with higher query keyword overlap
+ * 3. Chunk stitching — merge adjacent chunks from the same source
+ * 4. Limit — enforce maxResults
+ */
+import { tokenize } from "../search/tokenizer.js";
+import { logger } from "../utils/logger.js";
+/**
+ * Remove results with identical content, keeping the highest-scored one.
+ */
+export function deduplicateResults(results) {
+    const seen = new Map();
+    for (const result of results) {
+        const key = result.content.trim().toLowerCase();
+        const existing = seen.get(key);
+        if (!existing || result.score > existing.score) {
+            seen.set(key, result);
+        }
+    }
+    return Array.from(seen.values()).sort((a, b) => b.score - a.score);
+}
+/**
+ * Rerank results by keyword overlap with the query.
+ * Combines original score with keyword overlap boost.
+ */
+export function rerankByKeywordOverlap(results, query) {
+    const queryTokens = new Set(tokenize(query));
+    if (queryTokens.size === 0)
+        return results;
+    const scored = results.map((r) => {
+        const contentTokens = tokenize(r.content);
+        const overlap = contentTokens.filter((t) => queryTokens.has(t)).length;
+        const overlapRatio = contentTokens.length > 0 ? overlap / queryTokens.size : 0;
+        // Combine: 70% original score + 30% keyword overlap
+        const combinedScore = r.score * 0.7 + overlapRatio * 0.3;
+        return { ...r, score: Math.round(combinedScore * 10000) / 10000 };
+    });
+    return scored.sort((a, b) => b.score - a.score);
+}
+/**
+ * Merge adjacent chunks from the same source document.
+ * Chunks are considered adjacent if their chunk indices differ by 1.
+ */
+export function stitchAdjacentChunks(results, chunkMeta) {
+    if (results.length <= 1 || chunkMeta.size === 0)
+        return results;
+    // Group by sourceId
+    const groups = new Map();
+    const ungrouped = [];
+    for (const r of results) {
+        const chunkIdx = chunkMeta.get(r.id);
+        if (chunkIdx === undefined) {
+            ungrouped.push(r);
+            continue;
+        }
+        const group = groups.get(r.sourceId) ?? [];
+        group.push(r);
+        groups.set(r.sourceId, group);
+    }
+    const stitched = [];
+    for (const [_sourceId, group] of groups) {
+        // Sort by chunk index
+        group.sort((a, b) => (chunkMeta.get(a.id) ?? 0) - (chunkMeta.get(b.id) ?? 0));
+        let current = group[0];
+        let currentIdx = chunkMeta.get(current.id) ?? 0;
+        for (let i = 1; i < group.length; i++) {
+            const next = group[i];
+            const nextIdx = chunkMeta.get(next.id) ?? 0;
+            if (nextIdx === currentIdx + 1) {
+                // Stitch: merge content, keep higher score
+                current = {
+                    ...current,
+                    content: current.content + "\n\n" + next.content,
+                    score: Math.max(current.score, next.score),
+                    strategies: [...new Set([...current.strategies, ...next.strategies])],
+                };
+                currentIdx = nextIdx;
+            }
+            else {
+                stitched.push(current);
+                current = next;
+                currentIdx = nextIdx;
+            }
+        }
+        stitched.push(current);
+    }
+    const final = [...stitched, ...ungrouped].sort((a, b) => b.score - a.score);
+    return final;
+}
+/**
+ * Full post-retrieval pipeline.
+ */
+export function postRetrievalPipeline(options) {
+    const { query, results, maxResults, chunkMeta } = options;
+    // Stage 1: Deduplication
+    const deduped = deduplicateResults(results);
+    const deduplicated = results.length - deduped.length;
+    // Stage 2: Reranking
+    const reranked = rerankByKeywordOverlap(deduped, query);
+    // Stage 3: Chunk stitching (if metadata available)
+    const stitched = chunkMeta
+        ? stitchAdjacentChunks(reranked, chunkMeta)
+        : reranked;
+    const stitchedChunks = reranked.length - stitched.length;
+    // Stage 4: Limit
+    const limited = stitched.slice(0, maxResults);
+    logger.debug("Post-retrieval pipeline complete", {
+        input: results.length,
+        deduplicated,
+        stitchedChunks,
+        output: limited.length,
+    });
+    return { results: limited, deduplicated, stitchedChunks };
+}
+//# sourceMappingURL=post-retrieval.js.map

package/dist/core/rag/post-retrieval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"post-retrieval.js","sourceRoot":"","sources":["../../../src/core/rag/post-retrieval.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAe5C;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAAuB;IACxD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAwB,CAAC;IAE7C,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAChD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;YAC/C,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;AACrE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CACpC,OAAuB,EACvB,KAAa;IAEb,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;IAC7C,IAAI,WAAW,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC;IAE3C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC/B,MAAM,aAAa,GAAG,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QACvE,MAAM,YAAY,GAAG,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/E,oDAAoD;QACpD,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,GAAG,GAAG,GAAG,YAAY,GAAG,GAAG,CAAC;QACzD,OAAO,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa,GAAG,KAAK,CAAC,GAAG,KAAK,EAAE,CAAC;IACpE,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;AAClD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAClC,OAAuB,EACvB,SAA8B;IAE9B,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,SAAS,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC;IAEhE,oBAAoB;IACpB,MAAM,MAAM,GAAG,IAAI,GAAG,EAA0B,CAAC;IACjD,MAAM,SAAS,GAAmB,EAAE,CAAC;IAErC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACrC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,SAAS;QACX,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC3C,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACd,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,MAAM,QAAQ,GAAmB,EAAE,CAAC;IAEpC,KAAK,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;QACxC,sBAAsB;QACtB,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAE9E,IAAI,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACvB,IAAI,UAAU,GAAG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;QAEhD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;YAE5C,IAAI,OAAO,KAAK,UAAU,GAAG,CAAC,EAAE,CAAC;gBAC/B,2CAA2C;gBAC3C,OAAO,GAAG;oBACR,GAAG,OAAO;oBACV,OAAO,EAAE,OAAO,CAAC,OAAO,GAAG,MAAM,GAAG,IAAI,CAAC,OAAO;oBAChD,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC;oBAC1C,UAAU,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;iBACtE,CAAC;gBACF,UAAU,GAAG,OAAO,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACN,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvB,OAAO,GAAG,IAAI,CAAC;gBACf,UAAU,GAAG,OAAO,CAAC;YACvB,CAAC;QACH,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACzB,CAAC;IAED,MAAM,KAAK,GAAG,CAAC,GAAG,QAAQ,EAAE,GAAG,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAC5E,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,OAA6B;IACjE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;IAE1D,yBAAyB;IACzB,MAAM,OAAO,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC5C,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAErD,qBAAqB;IACrB,MAAM,QAAQ,GAAG,sBAAsB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IAExD,mDAAmD;IACnD,MAAM,QAAQ,GAAG,SAAS;QACxB,CAAC,CAAC,oBAAoB,CAAC,QAAQ,EAAE,SAAS,CAAC;QAC3C,CAAC,CAAC,QAAQ,CAAC;IACb,MAAM,cAAc,GAAG,QAAQ,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;IAEzD,iBAAiB;IACjB,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IAE9C,MAAM,CAAC,KAAK,CAAC,kCAAkC,EAAE;QAC/C,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,YAAY;QACZ,cAAc;QACd,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC,CAAC;IAEH,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,CAAC;AAC5D,CAAC"}

package/dist/core/rag/query-cache.d.ts

@@ -0,0 +1,54 @@
+/**
+ * Query Cache — in-memory cache for RAG search results.
+ *
+ * Features:
+ * - TTL-based expiration
+ * - LRU eviction when maxSize exceeded
+ * - Query normalization (case-insensitive, trimmed)
+ * - Hit/miss statistics
+ * - Bulk invalidation (e.g., when knowledge store changes)
+ */
+import type { RankedResult } from "./multi-strategy-retrieval.js";
+export interface CacheOptions {
+    ttlMs: number;
+    maxSize: number;
+}
+export interface CacheStats {
+    size: number;
+    hits: number;
+    misses: number;
+    evictions: number;
+}
+/**
+ * In-memory query cache with TTL and LRU eviction.
+ */
+export declare class QueryCache {
+    private entries;
+    private ttlMs;
+    private maxSize;
+    private hits;
+    private misses;
+    private evictions;
+    constructor(options: CacheOptions);
+    /**
+     * Retrieve cached results for a query, or undefined if miss/expired.
+     */
+    get(query: string): RankedResult[] | undefined;
+    /**
+     * Store results for a query.
+     */
+    set(query: string, results: RankedResult[]): void;
+    /**
+     * Invalidate all cached entries.
+     */
+    invalidateAll(): void;
+    /**
+     * Get cache statistics.
+     */
+    getStats(): CacheStats;
+    /**
+     * Evict the oldest (by creation time) entry.
+     */
+    private evictOldest;
+}
+//# sourceMappingURL=query-cache.d.ts.map

package/dist/core/rag/query-cache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"query-cache.d.ts","sourceRoot":"","sources":["../../../src/core/rag/query-cache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAGlE,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;CACjB;AAQD,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;CACnB;AASD;;GAEG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,OAAO,CAAsC;IACrD,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,IAAI,CAAa;IACzB,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,SAAS,CAAa;gBAElB,OAAO,EAAE,YAAY;IAKjC;;OAEG;IACH,GAAG,CAAC,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,GAAG,SAAS;IAqB9C;;OAEG;IACH,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,IAAI;IAejD;;OAEG;IACH,aAAa,IAAI,IAAI;IAMrB;;OAEG;IACH,QAAQ,IAAI,UAAU;IAStB;;OAEG;IACH,OAAO,CAAC,WAAW;CAgBpB"}

package/dist/core/rag/query-cache.js

@@ -0,0 +1,104 @@
+/**
+ * Query Cache — in-memory cache for RAG search results.
+ *
+ * Features:
+ * - TTL-based expiration
+ * - LRU eviction when maxSize exceeded
+ * - Query normalization (case-insensitive, trimmed)
+ * - Hit/miss statistics
+ * - Bulk invalidation (e.g., when knowledge store changes)
+ */
+import { logger } from "../utils/logger.js";
+/**
+ * Normalize query for consistent cache keys.
+ */
+function normalizeKey(query) {
+    return query.trim().toLowerCase();
+}
+/**
+ * In-memory query cache with TTL and LRU eviction.
+ */
+export class QueryCache {
+    entries = new Map();
+    ttlMs;
+    maxSize;
+    hits = 0;
+    misses = 0;
+    evictions = 0;
+    constructor(options) {
+        this.ttlMs = options.ttlMs;
+        this.maxSize = options.maxSize;
+    }
+    /**
+     * Retrieve cached results for a query, or undefined if miss/expired.
+     */
+    get(query) {
+        const key = normalizeKey(query);
+        const entry = this.entries.get(key);
+        if (!entry) {
+            this.misses++;
+            return undefined;
+        }
+        // Check TTL
+        if (Date.now() - entry.createdAt > this.ttlMs) {
+            this.entries.delete(key);
+            this.misses++;
+            return undefined;
+        }
+        entry.lastAccessedAt = Date.now();
+        this.hits++;
+        return entry.results;
+    }
+    /**
+     * Store results for a query.
+     */
+    set(query, results) {
+        const key = normalizeKey(query);
+        // Evict if at capacity
+        if (this.entries.size >= this.maxSize && !this.entries.has(key)) {
+            this.evictOldest();
+        }
+        this.entries.set(key, {
+            results,
+            createdAt: Date.now(),
+            lastAccessedAt: Date.now(),
+        });
+    }
+    /**
+     * Invalidate all cached entries.
+     */
+    invalidateAll() {
+        const count = this.entries.size;
+        this.entries.clear();
+        logger.debug("Query cache invalidated", { entriesCleared: count });
+    }
+    /**
+     * Get cache statistics.
+     */
+    getStats() {
+        return {
+            size: this.entries.size,
+            hits: this.hits,
+            misses: this.misses,
+            evictions: this.evictions,
+        };
+    }
+    /**
+     * Evict the oldest (by creation time) entry.
+     */
+    evictOldest() {
+        let oldestKey;
+        let oldestTime = Infinity;
+        for (const [key, entry] of this.entries) {
+            if (entry.createdAt < oldestTime) {
+                oldestTime = entry.createdAt;
+                oldestKey = key;
+            }
+        }
+        if (oldestKey) {
+            this.entries.delete(oldestKey);
+            this.evictions++;
+        }
+    }
+}
+//# sourceMappingURL=query-cache.js.map

package/dist/core/rag/query-cache.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"query-cache.js","sourceRoot":"","sources":["../../../src/core/rag/query-cache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAoB5C;;GAEG;AACH,SAAS,YAAY,CAAC,KAAa;IACjC,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AACpC,CAAC;AAED;;GAEG;AACH,MAAM,OAAO,UAAU;IACb,OAAO,GAA4B,IAAI,GAAG,EAAE,CAAC;IAC7C,KAAK,CAAS;IACd,OAAO,CAAS;IAChB,IAAI,GAAW,CAAC,CAAC;IACjB,MAAM,GAAW,CAAC,CAAC;IACnB,SAAS,GAAW,CAAC,CAAC;IAE9B,YAAY,OAAqB;QAC/B,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC3B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,KAAa;QACf,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;QAChC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEpC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,YAAY;QACZ,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;YAC9C,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACzB,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,KAAK,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAClC,IAAI,CAAC,IAAI,EAAE,CAAC;QACZ,OAAO,KAAK,CAAC,OAAO,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,KAAa,EAAE,OAAuB;QACxC,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;QAEhC,uBAAuB;QACvB,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAChE,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE;YACpB,OAAO;YACP,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,cAAc,EAAE,IAAI,CAAC,GAAG,EAAE;SAC3B,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,aAAa;QACX,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;QAChC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACrB,MAAM,CAAC,KAAK,CAAC,yBAAyB,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;IACrE,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI;YACvB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,WAAW;QACjB,IAAI,SAA6B,CAAC;QAClC,IAAI,UAAU,GAAG,QAAQ,CAAC;QAE1B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACxC,IAAI,KAAK,CAAC,SAAS,GAAG,UAAU,EAAE,CAAC;gBACjC,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC;gBAC7B,SAAS,GAAG,GAAG,CAAC;YAClB,CAAC;QACH,CAAC;QAED,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAC/B,IAAI,CAAC,SAAS,EAAE,CAAC;QACnB,CAAC;IACH,CAAC;CACF"}

package/dist/core/rag/query-understanding.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Query Understanding — preprocesses user queries before retrieval.
+ *
+ * Provides:
+ * - Intent detection (search, how_to, status, debug, compare, history)
+ * - Entity extraction (PascalCase, camelCase from query)
+ * - Source type filtering (detect "in the PRD", "in code", etc.)
+ * - Query expansion with related terms
+ * - Query rewriting for FTS5 optimization
+ */
+export type QueryIntent = "search" | "how_to" | "status" | "debug" | "compare" | "history";
+export interface UnderstandingResult {
+    originalQuery: string;
+    rewrittenQuery: string;
+    entities: string[];
+    intent: QueryIntent;
+    expandedTerms: string[];
+    sourceTypeFilter: string[];
+}
+/**
+ * Detect the intent of a query from keyword patterns.
+ */
+export declare function detectIntent(query: string): QueryIntent;
+/**
+ * Detect source type filters from query content.
+ */
+export declare function detectSourceFilter(query: string): string[];
+/**
+ * Expand query tokens with related technical terms.
+ */
+export declare function expandQuery(query: string): string[];
+/**
+ * Full query understanding pipeline.
+ * Analyzes a natural language query and returns structured understanding.
+ */
+export declare function understandQuery(query: string): UnderstandingResult;
+//# sourceMappingURL=query-understanding.d.ts.map

package/dist/core/rag/query-understanding.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"query-understanding.d.ts","sourceRoot":"","sources":["../../../src/core/rag/query-understanding.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAMH,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,GAAG,SAAS,GAAG,SAAS,CAAC;AAE3F,MAAM,WAAW,mBAAmB;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,EAAE,WAAW,CAAC;IACpB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC5B;AAYD;;GAEG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW,CAKvD;AAiBD;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,CAQ1D;AAmBD;;GAEG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,CAYnD;AAUD;;;GAGG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,mBAAmB,CAuBlE"}

package/dist/core/rag/query-understanding.js

@@ -0,0 +1,123 @@
+/**
+ * Query Understanding — preprocesses user queries before retrieval.
+ *
+ * Provides:
+ * - Intent detection (search, how_to, status, debug, compare, history)
+ * - Entity extraction (PascalCase, camelCase from query)
+ * - Source type filtering (detect "in the PRD", "in code", etc.)
+ * - Query expansion with related terms
+ * - Query rewriting for FTS5 optimization
+ */
+import { tokenize } from "../search/tokenizer.js";
+import { extractEntities } from "./enrichment-pipeline.js";
+import { logger } from "../utils/logger.js";
+// ── Intent detection patterns ────────────────────────────
+const INTENT_PATTERNS = [
+    { pattern: /^how\b|como\s+(fazer|funciona|implementar)/i, intent: "how_to" },
+    { pattern: /\b(status|progress|progresso|sprint|burndown|velocity)\b/i, intent: "status" },
+    { pattern: /\b(error|erro|bug|fail|falha|why\s+does|por\s*que)\b/i, intent: "debug" },
+    { pattern: /\b(compare|compar|vs\.?|versus|difference|diferença|diferenca)\b/i, intent: "compare" },
+    { pattern: /\b(history|histórico|historico|changelog|when\s+was|quando)\b/i, intent: "history" },
+];
+/**
+ * Detect the intent of a query from keyword patterns.
+ */
+export function detectIntent(query) {
+    for (const { pattern, intent } of INTENT_PATTERNS) {
+        if (pattern.test(query))
+            return intent;
+    }
+    return "search";
+}
+// ── Source type detection patterns ────────────────────────
+const SOURCE_PATTERNS = [
+    { pattern: /\b(code|código|codigo|symbol|function|class|módulo|modulo)\b/i, sourceTypes: ["code_context"] },
+    { pattern: /\b(prd|requisito|requirement|user\s+stor)/i, sourceTypes: ["prd"] },
+    { pattern: /\b(journey|jornada|screen|tela|fluxo|flow)\b/i, sourceTypes: ["journey"] },
+    { pattern: /\b(skills?|habilidade)\b/i, sourceTypes: ["skill"] },
+    { pattern: /\b(doc|docs|documentation|documentação|documentacao|library|lib)\b/i, sourceTypes: ["docs"] },
+    { pattern: /\b(memory|memória|memoria|decision|decisão|decisao)\b/i, sourceTypes: ["memory", "ai_decision"] },
+    { pattern: /\b(siebel|sif|composer)\b/i, sourceTypes: ["siebel_sif", "siebel_composer", "siebel_docs"] },
+    { pattern: /\b(benchmark|performance|latency|latência|token\s+economy)\b/i, sourceTypes: ["benchmark"] },
+    { pattern: /\b(capture|screenshot|web\s+page|página|pagina)\b/i, sourceTypes: ["web_capture"] },
+    { pattern: /\b(swagger|api\s+spec|openapi|endpoint)\b/i, sourceTypes: ["swagger"] },
+];
+/**
+ * Detect source type filters from query content.
+ */
+export function detectSourceFilter(query) {
+    const filters = new Set();
+    for (const { pattern, sourceTypes } of SOURCE_PATTERNS) {
+        if (pattern.test(query)) {
+            for (const st of sourceTypes)
+                filters.add(st);
+        }
+    }
+    return Array.from(filters);
+}
+// ── Query expansion ──────────────────────────────────────
+const EXPANSION_MAP = {
+    database: ["sqlite", "store", "migration", "schema"],
+    migration: ["database", "schema", "version", "upgrade"],
+    search: ["fts5", "bm25", "query", "ranking"],
+    context: ["rag", "token", "budget", "tiered"],
+    graph: ["node", "edge", "dependency", "traversal"],
+    parser: ["segment", "classify", "extract", "normalize"],
+    chunk: ["split", "overlap", "token", "sentence"],
+    embedding: ["tfidf", "vector", "cosine", "similarity"],
+    knowledge: ["store", "document", "index", "rag"],
+    node: ["task", "epic", "status", "dependency"],
+    test: ["vitest", "tdd", "assertion", "mock"],
+    skill: ["phase", "lifecycle", "recommend"],
+};
+/**
+ * Expand query tokens with related technical terms.
+ */
+export function expandQuery(query) {
+    const tokens = tokenize(query);
+    if (tokens.length === 0)
+        return [];
+    const expanded = new Set(tokens);
+    for (const token of tokens) {
+        const related = EXPANSION_MAP[token];
+        if (related) {
+            for (const term of related)
+                expanded.add(term);
+        }
+    }
+    return Array.from(expanded);
+}
+/**
+ * Rewrite query for FTS5 — clean, tokenize, rejoin.
+ */
+function rewriteQuery(query) {
+    const tokens = tokenize(query);
+    return tokens.join(" ");
+}
+/**
+ * Full query understanding pipeline.
+ * Analyzes a natural language query and returns structured understanding.
+ */
+export function understandQuery(query) {
+    const originalQuery = query;
+    const intent = detectIntent(query);
+    const entities = extractEntities(query);
+    const sourceTypeFilter = detectSourceFilter(query);
+    const expandedTerms = expandQuery(query);
+    const rewrittenQuery = rewriteQuery(query);
+    logger.debug("Query understood", {
+        intent,
+        entityCount: entities.length,
+        sourceFilters: sourceTypeFilter.length,
+        expandedTermCount: expandedTerms.length,
+    });
+    return {
+        originalQuery,
+        rewrittenQuery,
+        entities,
+        intent,
+        expandedTerms,
+        sourceTypeFilter,
+    };
+}
+//# sourceMappingURL=query-understanding.js.map

package/dist/core/rag/query-understanding.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"query-understanding.js","sourceRoot":"","sources":["../../../src/core/rag/query-understanding.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAa5C,4DAA4D;AAE5D,MAAM,eAAe,GAAoD;IACvE,EAAE,OAAO,EAAE,6CAA6C,EAAE,MAAM,EAAE,QAAQ,EAAE;IAC5E,EAAE,OAAO,EAAE,2DAA2D,EAAE,MAAM,EAAE,QAAQ,EAAE;IAC1F,EAAE,OAAO,EAAE,uDAAuD,EAAE,MAAM,EAAE,OAAO,EAAE;IACrF,EAAE,OAAO,EAAE,mEAAmE,EAAE,MAAM,EAAE,SAAS,EAAE;IACnG,EAAE,OAAO,EAAE,gEAAgE,EAAE,MAAM,EAAE,SAAS,EAAE;CACjG,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,KAAa;IACxC,KAAK,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,eAAe,EAAE,CAAC;QAClD,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,MAAM,CAAC;IACzC,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,6DAA6D;AAE7D,MAAM,eAAe,GAAsD;IACzE,EAAE,OAAO,EAAE,+DAA+D,EAAE,WAAW,EAAE,CAAC,cAAc,CAAC,EAAE;IAC3G,EAAE,OAAO,EAAE,4CAA4C,EAAE,WAAW,EAAE,CAAC,KAAK,CAAC,EAAE;IAC/E,EAAE,OAAO,EAAE,+CAA+C,EAAE,WAAW,EAAE,CAAC,SAAS,CAAC,EAAE;IACtF,EAAE,OAAO,EAAE,2BAA2B,EAAE,WAAW,EAAE,CAAC,OAAO,CAAC,EAAE;IAChE,EAAE,OAAO,EAAE,qEAAqE,EAAE,WAAW,EAAE,CAAC,MAAM,CAAC,EAAE;IACzG,EAAE,OAAO,EAAE,wDAAwD,EAAE,WAAW,EAAE,CAAC,QAAQ,EAAE,aAAa,CAAC,EAAE;IAC7G,EAAE,OAAO,EAAE,4BAA4B,EAAE,WAAW,EAAE,CAAC,YAAY,EAAE,iBAAiB,EAAE,aAAa,CAAC,EAAE;IACxG,EAAE,OAAO,EAAE,+DAA+D,EAAE,WAAW,EAAE,CAAC,WAAW,CAAC,EAAE;IACxG,EAAE,OAAO,EAAE,oDAAoD,EAAE,WAAW,EAAE,CAAC,aAAa,CAAC,EAAE;IAC/F,EAAE,OAAO,EAAE,4CAA4C,EAAE,WAAW,EAAE,CAAC,SAAS,CAAC,EAAE;CACpF,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAAa;IAC9C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,KAAK,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,eAAe,EAAE,CAAC;QACvD,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACxB,KAAK,MAAM,EAAE,IAAI,WAAW;gBAAE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAC7B,CAAC;AAED,4DAA4D;AAE5D,MAAM,aAAa,GAA6B;IAC9C,QAAQ,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,WAAW,EAAE,QAAQ,CAAC;IACpD,SAAS,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC;IACvD,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,CAAC;IAC5C,OAAO,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC;IAC7C,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,WAAW,CAAC;IAClD,MAAM,EAAE,CAAC,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,CAAC;IACvD,KAAK,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,UAAU,CAAC;IAChD,SAAS,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY,CAAC;IACtD,SAAS,EAAE,CAAC,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,CAAC;IAChD,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,YAAY,CAAC;IAC9C,IAAI,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,CAAC;IAC5C,KAAK,EAAE,CAAC,OAAO,EAAE,WAAW,EAAE,WAAW,CAAC;CAC3C,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,KAAa;IACvC,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC/B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEnC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IACjC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;QACrC,IAAI,OAAO,EAAE,CAAC;YACZ,KAAK,MAAM,IAAI,IAAI,OAAO;gBAAE,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AAC9B,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,KAAa;IACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC/B,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC1B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,KAAa;IAC3C,MAAM,aAAa,GAAG,KAAK,CAAC;IAC5B,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IACxC,MAAM,gBAAgB,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;IACnD,MAAM,aAAa,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,cAAc,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IAE3C,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE;QAC/B,MAAM;QACN,WAAW,EAAE,QAAQ,CAAC,MAAM;QAC5B,aAAa,EAAE,gBAAgB,CAAC,MAAM;QACtC,iBAAiB,EAAE,aAAa,CAAC,MAAM;KACxC,CAAC,CAAC;IAEH,OAAO;QACL,aAAa;QACb,cAAc;QACd,QAAQ;QACR,MAAM;QACN,aAAa;QACb,gBAAgB;KACjB,CAAC;AACJ,CAAC"}

package/dist/core/rag/rag-trace.d.ts

@@ -0,0 +1,67 @@
+/**
+ * RAG Trace — observability for the RAG pipeline.
+ *
+ * Instruments each stage (query understanding, retrieval, post-retrieval,
+ * assembly, citation) with timing, input/output counts, and source contributions.
+ */
+export type RagStageName = "query_understanding" | "retrieval" | "post_retrieval" | "assembly" | "citation";
+export interface RagStageTrace {
+    stage: RagStageName;
+    latencyMs: number;
+    inputCount: number;
+    outputCount: number;
+    details: Record<string, unknown>;
+}
+export interface RagTrace {
+    traceId: string;
+    query: string;
+    timestamp: string;
+    stages: RagStageTrace[];
+    totalLatencyMs: number;
+    totalTokensUsed: number;
+    sourcesContributed: Record<string, number>;
+    citationCount: number;
+}
+/**
+ * Tracer instance — tracks stages during a single RAG pipeline execution.
+ */
+export declare class RagTracer {
+    private traceId;
+    private query;
+    private timestamp;
+    private stages;
+    private stageStartTimes;
+    private sourcesContributed;
+    private citationCount;
+    private totalTokensUsed;
+    constructor(query: string);
+    /**
+     * Mark the start of a pipeline stage.
+     */
+    startStage(stage: RagStageName): void;
+    /**
+     * Mark the end of a pipeline stage with input/output counts.
+     */
+    endStage(stage: RagStageName, counts: {
+        inputCount: number;
+        outputCount: number;
+        details?: Record<string, unknown>;
+    }): void;
+    /**
+     * Record how many results came from a specific source type.
+     */
+    recordSourceContribution(sourceType: string, count: number): void;
+    /**
+     * Set the number of citations produced.
+     */
+    setCitationCount(count: number): void;
+    /**
+     * Set total tokens consumed by the assembled context.
+     */
+    setTokensUsed(tokens: number): void;
+    /**
+     * Finalize the trace and return the complete record.
+     */
+    finalize(): RagTrace;
+}
+//# sourceMappingURL=rag-trace.d.ts.map

package/dist/core/rag/rag-trace.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rag-trace.d.ts","sourceRoot":"","sources":["../../../src/core/rag/rag-trace.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,MAAM,MAAM,YAAY,GACpB,qBAAqB,GACrB,WAAW,GACX,gBAAgB,GAChB,UAAU,GACV,UAAU,CAAC;AAEf,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,YAAY,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,QAAQ;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,aAAa,EAAE,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,kBAAkB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3C,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,qBAAa,SAAS;IACpB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,eAAe,CAAkC;IACzD,OAAO,CAAC,kBAAkB,CAA8B;IACxD,OAAO,CAAC,aAAa,CAAa;IAClC,OAAO,CAAC,eAAe,CAAa;gBAExB,KAAK,EAAE,MAAM;IAMzB;;OAEG;IACH,UAAU,CAAC,KAAK,EAAE,YAAY,GAAG,IAAI;IAIrC;;OAEG;IACH,QAAQ,CACN,KAAK,EAAE,YAAY,EACnB,MAAM,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAE,GACrF,IAAI;IAeP;;OAEG;IACH,wBAAwB,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAIjE;;OAEG;IACH,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAIrC;;OAEG;IACH,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAInC;;OAEG;IACH,QAAQ,IAAI,QAAQ;CAcrB"}

package/dist/core/rag/rag-trace.js

@@ -0,0 +1,82 @@
+/**
+ * RAG Trace — observability for the RAG pipeline.
+ *
+ * Instruments each stage (query understanding, retrieval, post-retrieval,
+ * assembly, citation) with timing, input/output counts, and source contributions.
+ */
+import { generateId } from "../utils/id.js";
+import { now } from "../utils/time.js";
+/**
+ * Tracer instance — tracks stages during a single RAG pipeline execution.
+ */
+export class RagTracer {
+    traceId;
+    query;
+    timestamp;
+    stages = [];
+    stageStartTimes = new Map();
+    sourcesContributed = {};
+    citationCount = 0;
+    totalTokensUsed = 0;
+    constructor(query) {
+        this.traceId = generateId("trace");
+        this.query = query;
+        this.timestamp = now();
+    }
+    /**
+     * Mark the start of a pipeline stage.
+     */
+    startStage(stage) {
+        this.stageStartTimes.set(stage, performance.now());
+    }
+    /**
+     * Mark the end of a pipeline stage with input/output counts.
+     */
+    endStage(stage, counts) {
+        const startTime = this.stageStartTimes.get(stage) ?? performance.now();
+        const latencyMs = Math.round(performance.now() - startTime);
+        this.stages.push({
+            stage,
+            latencyMs,
+            inputCount: counts.inputCount,
+            outputCount: counts.outputCount,
+            details: counts.details ?? {},
+        });
+        this.stageStartTimes.delete(stage);
+    }
+    /**
+     * Record how many results came from a specific source type.
+     */
+    recordSourceContribution(sourceType, count) {
+        this.sourcesContributed[sourceType] = (this.sourcesContributed[sourceType] ?? 0) + count;
+    }
+    /**
+     * Set the number of citations produced.
+     */
+    setCitationCount(count) {
+        this.citationCount = count;
+    }
+    /**
+     * Set total tokens consumed by the assembled context.
+     */
+    setTokensUsed(tokens) {
+        this.totalTokensUsed = tokens;
+    }
+    /**
+     * Finalize the trace and return the complete record.
+     */
+    finalize() {
+        const totalLatencyMs = this.stages.reduce((sum, s) => sum + s.latencyMs, 0);
+        return {
+            traceId: this.traceId,
+            query: this.query,
+            timestamp: this.timestamp,
+            stages: [...this.stages],
+            totalLatencyMs,
+            totalTokensUsed: this.totalTokensUsed,
+            sourcesContributed: { ...this.sourcesContributed },
+            citationCount: this.citationCount,
+        };
+    }
+}
+//# sourceMappingURL=rag-trace.js.map

package/dist/core/rag/rag-trace.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rag-trace.js","sourceRoot":"","sources":["../../../src/core/rag/rag-trace.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAC;AA4BvC;;GAEG;AACH,MAAM,OAAO,SAAS;IACZ,OAAO,CAAS;IAChB,KAAK,CAAS;IACd,SAAS,CAAS;IAClB,MAAM,GAAoB,EAAE,CAAC;IAC7B,eAAe,GAAwB,IAAI,GAAG,EAAE,CAAC;IACjD,kBAAkB,GAA2B,EAAE,CAAC;IAChD,aAAa,GAAW,CAAC,CAAC;IAC1B,eAAe,GAAW,CAAC,CAAC;IAEpC,YAAY,KAAa;QACvB,IAAI,CAAC,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;QACnC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,SAAS,GAAG,GAAG,EAAE,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,KAAmB;QAC5B,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC;IACrD,CAAC;IAED;;OAEG;IACH,QAAQ,CACN,KAAmB,EACnB,MAAsF;QAEtF,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,WAAW,CAAC,GAAG,EAAE,CAAC;QACvE,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC;QAE5D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;YACf,KAAK;YACL,SAAS;YACT,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;SAC9B,CAAC,CAAC;QAEH,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,wBAAwB,CAAC,UAAkB,EAAE,KAAa;QACxD,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC;IAC3F,CAAC;IAED;;OAEG;IACH,gBAAgB,CAAC,KAAa;QAC5B,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,MAAc;QAC1B,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QAE5E,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,MAAM,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;YACxB,cAAc;YACd,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,kBAAkB,EAAE,EAAE,GAAG,IAAI,CAAC,kBAAkB,EAAE;YAClD,aAAa,EAAE,IAAI,CAAC,aAAa;SAClC,CAAC;IACJ,CAAC;CACF"}

package/dist/core/rag/source-contribution.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Source Contribution — measures how much each knowledge source
+ * contributes to RAG quality.
+ *
+ * Aggregates data from traces and usage logs to compute per-source metrics.
+ */
+export interface TraceAggregation {
+    sourceType: string;
+    retrievalCount: number;
+    totalRelevanceScore: number;
+    helpfulCount: number;
+    unhelpfulCount: number;
+    totalTokens: number;
+}
+export interface SourceContribution {
+    sourceType: string;
+    documentCount: number;
+    retrievalHitRate: number;
+    avgRelevanceScore: number;
+    helpfulFeedbackRate: number;
+    tokenContribution: number;
+}
+export interface UnderutilizedSource {
+    sourceType: string;
+    reason: string;
+    documentCount: number;
+    retrievalHitRate: number;
+}
+/**
+ * Calculate contribution metrics for each source type.
+ */
+export declare function calculateSourceContributions(traces: TraceAggregation[], totalQueries: number): SourceContribution[];
+/**
+ * Identify sources that are indexed but rarely retrieved.
+ */
+export declare function identifyUnderutilizedSources(contributions: SourceContribution[]): UnderutilizedSource[];
+//# sourceMappingURL=source-contribution.d.ts.map