@aperdomoll90/ledger-ai 1.4.0 → 1.4.2

package/dist/lib/search/ai-search.js

@@ -0,0 +1,396 @@
+// ai-search.ts
+// AI-powered search — vector (meaning), keyword (exact words), hybrid (both combined).
+// Each function calls a Postgres RPC function that does the actual search.
+// TypeScript's job: generate the query embedding, then call the right function.
+import { getOrCacheQueryEmbedding, toVectorString } from './embeddings.js';
+import { rerankResults } from './reranker.js';
+import { buildSearchParams, extractSourceDocIds, SEMANTIC_CACHE_MODEL_ID, SEMANTIC_CACHE_THRESHOLD, } from './semantic-cache.js';
+import { runSearchTrace, startSpan, recordChildSpan, withActiveSpan } from '../observability.js';
+// =============================================================================
+// Search evaluation logging
+// =============================================================================
+/**
+ * Log a search to the search_evaluations table.
+ * Called after every search — silently records what was searched,
+ * what came back, and how long it took. This is the raw data
+ * that powers all evaluation, quality tracking, and improvement.
+ *
+ * Fire-and-forget: we don't await this. If logging fails,
+ * the search still returns results. The user never waits for logging.
+ */
+function logSearchEvaluation(supabase, params) {
+    // Extract unique document_types and source_types from results
+    // These tell us which types of documents search finds well vs poorly
+    const documentTypes = [...new Set(params.results.map(result => result.document_type))];
+    // Build the results JSONB array — just IDs and scores, not full content
+    const resultsSummary = params.results.map(result => ({
+        id: result.id,
+        score: result.similarity ?? result.rank ?? result.score ?? null,
+        document_type: result.document_type,
+    }));
+    // Fire and forget — don't await, don't block the search response
+    supabase
+        .from('search_evaluations')
+        .insert({
+        query_text: params.query,
+        search_mode: params.searchMode,
+        result_count: params.results.length,
+        results: resultsSummary,
+        document_types: documentTypes,
+        response_time_ms: params.responseTimeMs,
+    })
+        .then(() => { })
+        .catch((logError) => {
+        process.stderr.write(`[ledger] search evaluation logging failed: ${logError.message ?? 'unknown error'}\n`);
+    });
+}
+// =============================================================================
+// Search functions
+// =============================================================================
+/**
+ * Search by meaning — "how does auth work?" finds documents about OAuth.
+ *
+ * Flow:
+ * 1. Convert query text to an embedding (array of 1,536 numbers) via OpenAI
+ * 2. Check the query_cache first to avoid repeat API calls
+ * 3. Call match_documents RPC — Postgres compares the query embedding
+ *    against every chunk's embedding using cosine similarity
+ * 4. Return matching documents sorted by similarity
+ */
+export async function searchByVector(clients, props) {
+    const startTime = Date.now();
+    return runSearchTrace({
+        mode: 'vector',
+        query: props.query,
+        environment: clients.observabilityEnvironment,
+        sessionId: clients.sessionId,
+        input: {
+            query: props.query,
+            filters: { domain: props.domain, project: props.project, document_type: props.document_type },
+        },
+        metadata: { threshold: props.threshold ?? 0.38, limit: props.limit ?? 10 },
+    }, async (trace) => {
+        const queryEmbedding = await getOrCacheQueryEmbedding(clients, props.query);
+        const embeddingString = toVectorString(queryEmbedding);
+        // Semantic cache lookup (layer 2)
+        const searchParams = buildSearchParams({
+            threshold: props.threshold ?? 0.38,
+            limit: props.limit ?? 10,
+            domain: props.domain,
+            document_type: props.document_type,
+            project: props.project,
+        });
+        const cacheSpan = startSpan('semantic-cache-lookup');
+        const { data: cachedResults } = await clients.supabase.rpc('semantic_cache_lookup', {
+            p_query_embedding: embeddingString,
+            p_search_mode: 'vector',
+            p_search_params: searchParams,
+            p_embedding_model_id: SEMANTIC_CACHE_MODEL_ID,
+            p_similarity_threshold: SEMANTIC_CACHE_THRESHOLD,
+        });
+        const cacheHit = !!(cachedResults && cachedResults.length > 0);
+        cacheSpan.update({ output: { hit: cacheHit } });
+        cacheSpan.end();
+        if (cacheHit) {
+            const results = cachedResults;
+            trace.update({
+                output: {
+                    resultCount: results.length,
+                    topResultIds: results.slice(0, 3).map(result => result.id),
+                    cacheHit: true,
+                },
+            });
+            logSearchEvaluation(clients.supabase, {
+                query: props.query,
+                searchMode: 'vector',
+                results,
+                responseTimeMs: Date.now() - startTime,
+            });
+            return results;
+        }
+        // Cache miss: run full search pipeline
+        const retrieveSpan = startSpan('retrieve');
+        const { data, error } = await clients.supabase.rpc('match_documents', {
+            q_emb: embeddingString,
+            p_threshold: props.threshold ?? 0.38,
+            p_max_results: props.limit ?? 10,
+            p_domain: props.domain ?? null,
+            p_document_type: props.document_type ?? null,
+            p_project: props.project ?? null,
+        });
+        if (error) {
+            retrieveSpan.update({ output: { error: error.message } });
+            retrieveSpan.end();
+            trace.update({ output: { error: error.message } });
+            throw new Error(`Vector search failed for "${props.query}": ${error.message}`);
+        }
+        const results = (data ?? []);
+        retrieveSpan.update({ output: { rowCount: results.length } });
+        retrieveSpan.end();
+        // Store in semantic cache (non-blocking)
+        if (results.length > 0) {
+            const sourceDocIds = extractSourceDocIds(results);
+            const storeSpan = startSpan('semantic-cache-store');
+            Promise.resolve(clients.supabase.rpc('semantic_cache_store', {
+                p_query_text: props.query,
+                p_query_embedding: embeddingString,
+                p_search_mode: 'vector',
+                p_search_params: searchParams,
+                p_cached_results: results,
+                p_source_doc_ids: sourceDocIds,
+                p_embedding_model_id: SEMANTIC_CACHE_MODEL_ID,
+            })).then(() => {
+                storeSpan.end();
+            }).catch((cacheStoreError) => {
+                storeSpan.update({ output: { error: cacheStoreError.message ?? 'unknown' } });
+                storeSpan.end();
+                process.stderr.write(`[ledger] semantic cache store failed: ${cacheStoreError.message ?? 'unknown'}\n`);
+            });
+        }
+        trace.update({
+            output: {
+                resultCount: results.length,
+                topResultIds: results.slice(0, 3).map(result => result.id),
+                cacheHit: false,
+            },
+        });
+        logSearchEvaluation(clients.supabase, {
+            query: props.query,
+            searchMode: 'vector',
+            results,
+            responseTimeMs: Date.now() - startTime,
+        });
+        return results;
+    });
+}
+/**
+ * Search by exact words — "pgvector HNSW" finds documents containing those words.
+ *
+ * No embedding needed — Postgres uses the search_vector column (GIN index)
+ * to match words directly. Good for code identifiers, proper nouns, error messages.
+ */
+export async function searchByKeyword(clients, props) {
+    const startTime = Date.now();
+    return runSearchTrace({
+        mode: 'keyword',
+        query: props.query,
+        environment: clients.observabilityEnvironment,
+        sessionId: clients.sessionId,
+        input: {
+            query: props.query,
+            filters: { domain: props.domain, project: props.project, document_type: props.document_type },
+        },
+        metadata: { limit: props.limit ?? 10 },
+    }, async (trace) => {
+        const { data, error } = await clients.supabase.rpc('match_documents_keyword', {
+            p_query: props.query,
+            p_max_results: props.limit ?? 10,
+            p_domain: props.domain ?? null,
+            p_document_type: props.document_type ?? null,
+            p_project: props.project ?? null,
+        });
+        if (error) {
+            trace.update({ output: { error: error.message } });
+            throw new Error(`Keyword search failed for "${props.query}": ${error.message}`);
+        }
+        const results = (data ?? []);
+        trace.update({
+            output: {
+                resultCount: results.length,
+                topResultIds: results.slice(0, 3).map(result => result.id),
+                cacheHit: false,
+            },
+        });
+        logSearchEvaluation(clients.supabase, {
+            query: props.query,
+            searchMode: 'keyword',
+            results,
+            responseTimeMs: Date.now() - startTime,
+        });
+        return results;
+    });
+}
+/**
+ * Combined search — runs both vector AND keyword, merges results with RRF fusion.
+ *
+ * Documents found by both methods rank highest. This is the default search mode
+ * because it handles both meaning-based queries ("how does auth work?") and
+ * exact-term queries ("pgvector HNSW") well.
+ *
+ * RRF (Reciprocal Rank Fusion) formula:
+ *   score = 1/(k + vector_rank) + 1/(k + keyword_rank)
+ *   k=60 is a smoothing constant that prevents the #1 result from dominating.
+ */
+export async function searchHybrid(clients, props) {
+    const startTime = Date.now();
+    // When reranking, fetch more candidates so the reranker has a bigger pool.
+    // The reranker will select the best N from this larger set.
+    const useReranker = props.reranker === 'cohere' && clients.cohereApiKey;
+    const desiredLimit = props.limit ?? 10;
+    const requestLimit = useReranker ? desiredLimit * 2 : desiredLimit;
+    return runSearchTrace({
+        mode: useReranker ? 'hybrid+rerank' : 'hybrid',
+        query: props.query,
+        environment: clients.observabilityEnvironment,
+        sessionId: clients.sessionId,
+        input: {
+            query: props.query,
+            filters: { domain: props.domain, project: props.project, document_type: props.document_type },
+        },
+        metadata: {
+            threshold: props.threshold ?? 0.38,
+            limit: desiredLimit,
+            rerankerEnabled: !!useReranker,
+            reciprocalRankFusionK: props.reciprocalRankFusionK ?? 60,
+        },
+    }, async (trace) => {
+        const queryEmbedding = await getOrCacheQueryEmbedding(clients, props.query);
+        const embeddingString = toVectorString(queryEmbedding);
+        // Semantic cache lookup (layer 2)
+        // Skip cache when reranker is enabled (reranker produces different ordering)
+        const searchParams = buildSearchParams({
+            threshold: props.threshold ?? 0.38,
+            limit: requestLimit,
+            domain: props.domain,
+            document_type: props.document_type,
+            project: props.project,
+        });
+        if (!useReranker) {
+            const cacheSpan = startSpan('semantic-cache-lookup');
+            const { data: cachedResults } = await clients.supabase.rpc('semantic_cache_lookup', {
+                p_query_embedding: embeddingString,
+                p_search_mode: 'hybrid',
+                p_search_params: searchParams,
+                p_embedding_model_id: SEMANTIC_CACHE_MODEL_ID,
+                p_similarity_threshold: SEMANTIC_CACHE_THRESHOLD,
+            });
+            const cacheHit = !!(cachedResults && cachedResults.length > 0);
+            cacheSpan.update({ output: { hit: cacheHit } });
+            cacheSpan.end();
+            if (cacheHit) {
+                const cachedRows = cachedResults;
+                trace.update({
+                    output: {
+                        resultCount: cachedRows.length,
+                        topResultIds: cachedRows.slice(0, 3).map(result => result.id),
+                        cacheHit: true,
+                    },
+                });
+                logSearchEvaluation(clients.supabase, {
+                    query: props.query,
+                    searchMode: 'hybrid',
+                    results: cachedRows,
+                    responseTimeMs: Date.now() - startTime,
+                });
+                return cachedRows;
+            }
+        }
+        // Cache miss: run full search pipeline
+        const retrieveSpan = startSpan('retrieve');
+        const retrieveStart = Date.now();
+        const { data, error } = await clients.supabase.rpc('match_documents_hybrid', {
+            q_emb: embeddingString,
+            q_text: props.query,
+            p_threshold: props.threshold ?? 0.38,
+            p_max_results: requestLimit,
+            p_domain: props.domain ?? null,
+            p_document_type: props.document_type ?? null,
+            p_project: props.project ?? null,
+            p_rrf_k: props.reciprocalRankFusionK ?? 60,
+        });
+        if (error) {
+            retrieveSpan.update({ output: { error: error.message } });
+            retrieveSpan.end();
+            trace.update({ output: { error: error.message } });
+            throw new Error(`Hybrid search failed for "${props.query}": ${error.message}`);
+        }
+        const rows = (data ?? []);
+        const timing = rows[0]?.timing;
+        retrieveSpan.update({ output: { rowCount: rows.length, timing } });
+        retrieveSpan.end();
+        // Emit three child spans from the Postgres timing sidecar.
+        // Spans are backdated from retrieveStart using the measured ms deltas.
+        if (timing) {
+            let cursor = retrieveStart;
+            recordChildSpan('retrieve.vector', cursor, cursor + timing.vector_ms, { durationMs: timing.vector_ms });
+            cursor += timing.vector_ms;
+            recordChildSpan('retrieve.keyword', cursor, cursor + timing.keyword_ms, { durationMs: timing.keyword_ms });
+            cursor += timing.keyword_ms;
+            recordChildSpan('retrieve.fusion', cursor, cursor + timing.fusion_ms, { durationMs: timing.fusion_ms });
+        }
+        // Strip timing from rows before exposing to callers (internal sidecar only).
+        let results = rows.map(({ timing: _timing, ...rest }) => rest);
+        // Rerank: send candidates to Cohere cross-encoder for re-scoring.
+        // If reranking fails, results are returned unchanged (graceful degradation).
+        if (useReranker && results.length > 0) {
+            const rerankSpan = startSpan('rerank');
+            const inputCount = results.length;
+            results = await withActiveSpan(rerankSpan, async () => {
+                return rerankResults(props.query, results, {
+                    apiKey: clients.cohereApiKey,
+                    topN: desiredLimit,
+                });
+            });
+            rerankSpan.update({ output: { inputCount, outputCount: results.length } });
+            rerankSpan.end();
+        }
+        // Store in semantic cache (non-blocking, skip if reranker was used)
+        if (results.length > 0 && !useReranker) {
+            const sourceDocIds = extractSourceDocIds(results);
+            const storeSpan = startSpan('semantic-cache-store');
+            Promise.resolve(clients.supabase.rpc('semantic_cache_store', {
+                p_query_text: props.query,
+                p_query_embedding: embeddingString,
+                p_search_mode: 'hybrid',
+                p_search_params: searchParams,
+                p_cached_results: results,
+                p_source_doc_ids: sourceDocIds,
+                p_embedding_model_id: SEMANTIC_CACHE_MODEL_ID,
+            })).then(() => {
+                storeSpan.end();
+            }).catch((cacheStoreError) => {
+                storeSpan.update({ output: { error: cacheStoreError.message ?? 'unknown' } });
+                storeSpan.end();
+                process.stderr.write(`[ledger] semantic cache store failed: ${cacheStoreError.message ?? 'unknown'}\n`);
+            });
+        }
+        trace.update({
+            output: {
+                resultCount: results.length,
+                topResultIds: results.slice(0, 3).map(result => result.id),
+                cacheHit: false,
+            },
+        });
+        logSearchEvaluation(clients.supabase, {
+            query: props.query,
+            searchMode: useReranker ? 'hybrid+rerank' : 'hybrid',
+            results,
+            responseTimeMs: Date.now() - startTime,
+        });
+        return results;
+    });
+}
+/**
+ * Smart retrieval — decide how much content to send to the LLM.
+ *
+ * After search finds a matching document, this decides:
+ * - Small document (under context_window chars) → return full content
+ * - Large document → return only the matched chunk + neighbors
+ *
+ * Why: sending a 50,000-char document to the LLM when only one section
+ * is relevant wastes tokens and money. But sending only a 500-char chunk
+ * might miss context. This finds the balance.
+ */
+export async function retrieveContext(supabase, props) {
+    const { data, error } = await supabase.rpc('retrieve_context', {
+        p_document_id: props.document_id,
+        p_matched_chunk_index: props.matched_chunk_index,
+        p_context_window: props.context_window ?? 4000,
+        p_neighbor_count: props.neighbor_count ?? 1,
+    });
+    if (error)
+        throw new Error(`Context retrieval failed for document #${props.document_id}, chunk ${props.matched_chunk_index}: ${error.message}`);
+    if (!data || (Array.isArray(data) && data.length === 0))
+        return null;
+    return (Array.isArray(data) ? data[0] : data);
+}

package/dist/lib/search/chunk-context-enrichment.js

@@ -0,0 +1,155 @@
+// chunk-context-enrichment.ts
+// Pre-embedding enrichment — generates context summaries per chunk using an LLM.
+//
+// Implements the "Contextual Retrieval" technique (Anthropic, 2024).
+// We call it "chunk context enrichment" because the code operates at ingestion
+// time, enriching chunks with document context before embedding — not at
+// retrieval time. The industry name describes the goal (better retrieval),
+// not the action.
+//
+// Optimized pipeline (S38):
+// 1. Generate a document summary (one LLM call, processes full document once)
+// 2. For each chunk, build context from: summary + header path + neighbor chunks
+// 3. Fire all LLM calls in parallel (rate limiter controls concurrency)
+// 4. Each call processes ~1K tokens instead of ~18K (95% token reduction)
+//
+// This reduces ingestion of a 73K document from ~12 minutes to ~30 seconds.
+// The key insight: truncated context (summary + neighbors instead of full doc)
+// reduces per-call tokens enough to unblock parallelism without hitting
+// the TPM (Tokens Per Minute) limit.
+import { openaiLimiter } from '../rate-limiter.js';
+const CONTEXT_ENRICHMENT_MODEL = 'gpt-4o-mini';
+const SUMMARY_PROMPT = `Summarize this document in 150-200 words. Focus on: what the document is about, its structure, and the key topics it covers. Be factual and concise.
+
+<document>
+{DOCUMENT_CONTENT}
+</document>`;
+const CONTEXT_PROMPT = `Here is a summary of the document:
+<document_summary>
+{DOCUMENT_SUMMARY}
+</document_summary>
+
+Here is the section this chunk belongs to (header path):
+<section>
+{HEADER_PATH}
+</section>
+
+Here are the neighboring chunks for context:
+<previous_chunk>
+{PREV_CHUNK}
+</previous_chunk>
+
+<chunk>
+{CHUNK_CONTENT}
+</chunk>
+
+<next_chunk>
+{NEXT_CHUNK}
+</next_chunk>
+
+Write a short context (2-3 sentences) that situates this chunk within the document. Include the document's topic and what specific information this chunk covers. Be concise and factual.`;
+// =============================================================================
+// Pure functions
+// =============================================================================
+/**
+ * Estimate token count from character length.
+ * Standard approximation for English text with GPT tokenizers: ~4 chars per token.
+ * Used for token budgeting in search results (e.g., limiting chunks to fit a context window).
+ */
+export function estimateTokenCount(text) {
+    return Math.ceil(text.length / 4);
+}
+/**
+ * Find the markdown header hierarchy for a chunk's position in the document.
+ * Returns a path like "Database > Caching > semantic_cache".
+ * Uses string matching, no LLM call needed.
+ */
+export function findHeaderPath(documentContent, chunkContent) {
+    const lines = documentContent.split('\n');
+    const headers = [];
+    let foundChunk = false;
+    for (const line of lines) {
+        if (/^#{1,6}\s/.test(line)) {
+            const level = line.match(/^(#+)/)?.[1].length ?? 1;
+            while (headers.length >= level)
+                headers.pop();
+            headers.push(line.replace(/^#+\s*/, '').trim());
+        }
+        if (line.includes(chunkContent.slice(0, 50))) {
+            foundChunk = true;
+            break;
+        }
+    }
+    return foundChunk ? headers.join(' > ') : '';
+}
+// =============================================================================
+// LLM functions
+// =============================================================================
+/**
+ * Generate context summaries for each chunk using an LLM.
+ *
+ * Optimized pipeline:
+ * 1. Generate a document summary (one LLM call)
+ * 2. For each chunk, send summary + header path + neighbors (not the full document)
+ * 3. All chunk enrichment calls run in parallel via the rate limiter
+ *
+ * This produces context summaries of equivalent quality while using 95% fewer
+ * tokens and completing 25x faster for large documents.
+ */
+export async function generateContextSummaries(openai, chunks, documentContent) {
+    if (chunks.length === 0)
+        return [];
+    // Step 1: Generate document summary (one LLM call, full document)
+    const summaryPrompt = SUMMARY_PROMPT.replace('{DOCUMENT_CONTENT}', documentContent);
+    const summaryResponse = await openaiLimiter.schedule(() => openai.chat.completions.create({
+        model: CONTEXT_ENRICHMENT_MODEL,
+        messages: [
+            { role: 'system', content: 'You are a precise technical writer. Output only the summary, nothing else.' },
+            { role: 'user', content: summaryPrompt },
+        ],
+        max_tokens: 300,
+        temperature: 0,
+    }));
+    const docSummary = (summaryResponse.choices[0].message.content ?? '').trim();
+    // Step 2: Parallel enrichment with truncated context
+    const promises = chunks.map((chunk, chunkIndex) => {
+        const prevChunk = chunkIndex > 0 ? chunks[chunkIndex - 1].content : '(start of document)';
+        const nextChunk = chunkIndex < chunks.length - 1 ? chunks[chunkIndex + 1].content : '(end of document)';
+        const headerPath = findHeaderPath(documentContent, chunk.content);
+        const prompt = CONTEXT_PROMPT
+            .replace('{DOCUMENT_SUMMARY}', docSummary)
+            .replace('{HEADER_PATH}', headerPath || '(unknown section)')
+            .replace('{PREV_CHUNK}', prevChunk)
+            .replace('{CHUNK_CONTENT}', chunk.content)
+            .replace('{NEXT_CHUNK}', nextChunk);
+        return openaiLimiter.schedule({ id: `enrich-${chunkIndex}` }, async () => {
+            try {
+                const response = await openai.chat.completions.create({
+                    model: CONTEXT_ENRICHMENT_MODEL,
+                    messages: [
+                        { role: 'system', content: 'You are a precise technical writer. Output only the context summary, nothing else.' },
+                        { role: 'user', content: prompt },
+                    ],
+                    max_tokens: 150,
+                    temperature: 0,
+                });
+                return {
+                    index: chunkIndex,
+                    summary: (response.choices[0].message.content ?? '').trim(),
+                    tokenCount: estimateTokenCount(chunk.content),
+                };
+            }
+            catch (error) {
+                const preview = chunk.content.slice(0, 60).replace(/\n/g, ' ');
+                throw new Error(`Context summary failed for chunk ${chunk.chunk_index} ("${preview}..."): ${error instanceof Error ? error.message : String(error)}`);
+            }
+        });
+    });
+    const results = await Promise.all(promises);
+    // Sort back to original order (parallel execution may complete out of order)
+    results.sort((first, second) => first.index - second.index);
+    return results.map(enrichmentResult => ({
+        summary: enrichmentResult.summary,
+        tokenCount: enrichmentResult.tokenCount,
+    }));
+}