@aperdomoll90/ledger-ai 1.3.0 → 1.4.2

package/dist/lib/search/embeddings.js

@@ -0,0 +1,293 @@
+// embeddings.ts
+// Prepares data for the database: generate embeddings, chunk text, format vectors.
+// The database can't call OpenAI or split text — that's TypeScript's job.
+import { createHash } from 'crypto';
+import { openaiLimiter } from '../rate-limiter.js';
+// =============================================================================
+// Constants
+// =============================================================================
+const EMBEDDING_MODEL = 'text-embedding-3-small';
+const DEFAULT_CHUNK_CONFIG = {
+    maxChunkSize: 1000,
+    overlapChars: 200,
+    strategy: 'recursive',
+};
+// Split separators — ordered from coarsest to finest.
+// The recursive chunker tries each level in order until chunks fit.
+const SPLIT_SEPARATORS = [
+    /^#{1,6}\s/m, // Level 0: Markdown headers
+    /\n\n+/, // Level 1: Double newlines (paragraphs)
+    /\n/, // Level 2: Single newlines
+    /(?<=[.!?])\s+/, // Level 3: Sentence boundaries
+];
+// =============================================================================
+// Pure functions — no API calls, no database, fully testable
+// =============================================================================
+/**
+ * SHA-256 hash of text content.
+ * Used for change detection: "has this document's content changed since last sync?"
+ * Same algorithm used in Postgres via pgcrypto.
+ */
+export function contentHash(text) {
+    return createHash('sha256').update(text, 'utf-8').digest('hex');
+}
+/**
+ * Format a number[] embedding as a Postgres vector string.
+ * Supabase RPC can't send number[] as vector(1536) — it needs this string format.
+ * Example: [0.021, -0.007, 0.045] → "[0.021,-0.007,0.045]"
+ */
+export function toVectorString(embedding) {
+    return `[${embedding.join(',')}]`;
+}
+/**
+ * Parse a Postgres vector back into a number[].
+ * Supabase REST API returns vector(1536) columns as strings like "[0.021,-0.007,0.045]".
+ * If the value is already a number[] (e.g. from a mock in tests), it passes through unchanged.
+ */
+export function parseVector(raw) {
+    if (Array.isArray(raw))
+        return raw;
+    if (typeof raw === 'string')
+        return JSON.parse(raw);
+    throw new Error(`Cannot parse vector: expected string or number[], got ${typeof raw}`);
+}
+/**
+ * Split text into chunks using a recursive hierarchical strategy.
+ *
+ * Implements chunk context enrichment pipeline step 1 (chunking).
+ * Based on the recursive character splitting pattern used in production
+ * RAG systems (LangChain, LlamaIndex).
+ *
+ * Split hierarchy (coarsest to finest):
+ *   1. Markdown headers (^#{1,6}\s)
+ *   2. Double newlines (paragraph boundaries)
+ *   3. Single newlines (line breaks)
+ *   4. Sentence boundaries (after . ! ?)
+ *   5. Character-level force split (fallback)
+ *
+ * If text fits within maxChunkSize, returns it as a single chunk.
+ * Otherwise, splits at the coarsest level possible. If any resulting
+ * section still exceeds maxChunkSize, recurses to the next finer level.
+ *
+ * Overlap is applied between adjacent chunks at levels 1-4.
+ * Header-level splits (level 0) do NOT overlap — sections are
+ * semantically distinct.
+ */
+export function chunkText(text, config) {
+    const resolvedConfig = { ...DEFAULT_CHUNK_CONFIG, ...config };
+    const { maxChunkSize, strategy } = resolvedConfig;
+    // Short text = one chunk
+    if (text.length <= maxChunkSize) {
+        return [{
+                content: text,
+                chunk_index: 0,
+                content_type: 'text',
+                strategy,
+                overlap_chars: 0,
+            }];
+    }
+    const rawChunks = recursiveSplit(text, 0, resolvedConfig);
+    // Assign sequential chunk_index across all chunks
+    return rawChunks.map((chunk, index) => ({
+        ...chunk,
+        chunk_index: index,
+    }));
+}
+/**
+ * Core recursive splitting logic.
+ * Tries the separator at `level`. If a section still exceeds maxChunkSize,
+ * recurses to level + 1. At the bottom level, force-splits at character positions.
+ */
+function recursiveSplit(text, level, config) {
+    const { maxChunkSize, overlapChars, strategy } = config;
+    // Base case: text fits
+    if (text.length <= maxChunkSize) {
+        return [{
+                content: text,
+                chunk_index: 0, // reassigned by caller
+                content_type: 'text',
+                strategy,
+                overlap_chars: 0,
+            }];
+    }
+    // Bottom level: force-split at character boundaries
+    if (level >= SPLIT_SEPARATORS.length) {
+        return forceCharSplit(text, maxChunkSize, overlapChars);
+    }
+    const separator = SPLIT_SEPARATORS[level];
+    const isHeaderLevel = level === 0;
+    const sections = splitKeepingSeparator(text, separator, isHeaderLevel);
+    // If splitting produced only 1 section (separator not found), try next level
+    if (sections.length <= 1) {
+        return recursiveSplit(text, level + 1, config);
+    }
+    // Pack sections into chunks, recurse oversized ones
+    const chunks = [];
+    let currentContent = '';
+    for (const section of sections) {
+        const wouldExceed = (currentContent + section).length > maxChunkSize;
+        if (wouldExceed && currentContent.length > 0) {
+            // Flush current accumulated content as chunk(s)
+            chunks.push(...recursiveSplit(currentContent.trim(), level + 1, config));
+            // Apply overlap (except at header boundaries)
+            if (!isHeaderLevel && overlapChars > 0) {
+                const overlap = currentContent.slice(-overlapChars);
+                currentContent = overlap + section;
+            }
+            else {
+                currentContent = section;
+            }
+        }
+        else {
+            currentContent = currentContent + section;
+        }
+    }
+    // Flush remaining content
+    if (currentContent.trim().length > 0) {
+        chunks.push(...recursiveSplit(currentContent.trim(), level + 1, config));
+    }
+    // Mark overlap on chunks (first chunk has 0)
+    return chunks.map((chunk, index) => ({
+        ...chunk,
+        overlap_chars: index > 0 && !isHeaderLevel ? Math.min(overlapChars, chunk.content.length) : 0,
+    }));
+}
+/**
+ * Split text by a regex separator.
+ * For header-level splits, the separator (e.g. "# Title") is kept at the
+ * start of the section it belongs to. For other levels, the separator
+ * is consumed (it's whitespace/newlines anyway).
+ */
+function splitKeepingSeparator(text, separator, keepSeparator) {
+    if (keepSeparator) {
+        // Header split: split just before each header, keep header in its section
+        const parts = text.split(new RegExp(`(?=${separator.source})`, 'm'));
+        return parts.filter(part => part.length > 0);
+    }
+    return text.split(separator).filter(part => part.trim().length > 0);
+}
+/**
+ * Force-split at character boundaries as a last resort.
+ * Handles text with no structural separators (JSON blobs, base64, walls of text).
+ */
+function forceCharSplit(text, maxChunkSize, overlapChars) {
+    const chunks = [];
+    const step = Math.max(1, maxChunkSize - overlapChars);
+    for (let offset = 0; offset < text.length; offset += step) {
+        chunks.push({
+            content: text.slice(offset, offset + maxChunkSize),
+            chunk_index: 0, // reassigned by caller
+            content_type: 'text',
+            strategy: 'forced',
+            overlap_chars: offset > 0 ? overlapChars : 0,
+        });
+    }
+    return chunks;
+}
+// =============================================================================
+// API functions — call OpenAI and/or database
+// =============================================================================
+/**
+ * Call OpenAI to convert text into an array of 1,536 numbers.
+ * These numbers represent the "meaning" of the text in a mathematical space.
+ * Similar texts produce similar numbers — that's how search works.
+ */
+export async function generateEmbedding(openai, text) {
+    try {
+        return await openaiLimiter.schedule(async () => {
+            const result = await openai.embeddings.create({
+                model: EMBEDDING_MODEL,
+                input: text,
+            });
+            return result.data[0].embedding;
+        });
+    }
+    catch (error) {
+        const preview = text.slice(0, 80).replace(/\n/g, ' ');
+        throw new Error(`Embedding generation failed for text "${preview}...": ${error instanceof Error ? error.message : String(error)}`);
+    }
+}
+/**
+ * Batch-generate embeddings for multiple texts in a single API call.
+ * OpenAI accepts an array of inputs and returns all embeddings at once.
+ * For 151 chunks, this is 2 API calls (batches of 100) instead of 151.
+ */
+export async function generateEmbeddingsBatch(openai, texts) {
+    if (texts.length === 0)
+        return [];
+    const BATCH_SIZE = 100;
+    const allEmbeddings = [];
+    for (let batchStart = 0; batchStart < texts.length; batchStart += BATCH_SIZE) {
+        const batch = texts.slice(batchStart, batchStart + BATCH_SIZE);
+        try {
+            const embeddings = await openaiLimiter.schedule(async () => {
+                const result = await openai.embeddings.create({
+                    model: EMBEDDING_MODEL,
+                    input: batch,
+                });
+                return result.data.map(entry => entry.embedding);
+            });
+            allEmbeddings.push(...embeddings);
+        }
+        catch (error) {
+            const batchNumber = Math.floor(batchStart / BATCH_SIZE) + 1;
+            throw new Error(`Batch embedding failed (batch ${batchNumber}, ${batch.length} texts): ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    return allEmbeddings;
+}
+/**
+ * Get an embedding for a search query, using the cache to avoid repeat API calls.
+ *
+ * Flow:
+ * 1. Check query_cache table for this exact query text
+ * 2. If cached: return the cached embedding, update hit_count
+ * 3. If not cached: call OpenAI, save to cache, return embedding
+ *
+ * Why cache: each OpenAI embedding call costs money. If you search
+ * "how does auth work" three times, the cache saves 2 API calls.
+ */
+export async function getOrCacheQueryEmbedding(clients, query) {
+    // Normalize query to avoid cache misses from capitalization/whitespace differences
+    const normalizedQuery = query.toLowerCase().trim();
+    // Check cache
+    const { data: cached, error: cacheError } = await clients.supabase
+        .from('query_cache')
+        .select('embedding, hit_count')
+        .eq('query_text', normalizedQuery)
+        .single();
+    if (cacheError && cacheError.code !== 'PGRST116') {
+        // PGRST116 = "not found" (expected for cache miss). Any other error is real.
+        process.stderr.write(`[ledger] query cache lookup failed: ${cacheError.message}\n`);
+    }
+    if (cached?.embedding) {
+        // Update cache stats (non-blocking, non-fatal)
+        const { error: updateError } = await clients.supabase
+            .from('query_cache')
+            .update({
+            hit_count: cached.hit_count + 1,
+            last_used_at: new Date().toISOString(),
+        })
+            .eq('query_text', normalizedQuery);
+        if (updateError) {
+            process.stderr.write(`[ledger] query cache hit_count update failed: ${updateError.message}\n`);
+        }
+        return parseVector(cached.embedding);
+    }
+    // Generate and cache. Send original query to OpenAI (preserves meaning),
+    // but store under normalized key (so "Auth" and "auth" share one cache entry)
+    const embedding = await generateEmbedding(clients.openai, query);
+    // Cache insert is non-blocking, non-fatal. A failed insert means the next
+    // identical query will hit OpenAI again, but search still works.
+    const { error: insertError } = await clients.supabase
+        .from('query_cache')
+        .insert({
+        query_text: normalizedQuery,
+        embedding: toVectorString(embedding),
+        embedding_model_id: 'openai/text-embedding-3-small',
+    });
+    if (insertError) {
+        process.stderr.write(`[ledger] query cache insert failed for "${normalizedQuery}": ${insertError.message}\n`);
+    }
+    return embedding;
+}

package/dist/lib/search/reranker.js

@@ -0,0 +1,120 @@
+// reranker.ts
+// Cross-encoder reranking via Cohere Rerank API.
+//
+// After hybrid search returns candidates, this module re-scores each one
+// by sending (query, document) pairs to a cross-encoder model. The model
+// reads query and document together — much more accurate than embedding
+// similarity, which encodes them separately.
+//
+// Uses native fetch — no Cohere SDK dependency. The API is one endpoint,
+// one request shape, one response shape.
+//
+// Graceful degradation: if the API fails, returns original results unchanged.
+// Search should never break because reranking failed.
+import { cohereLimiter } from '../rate-limiter.js';
+import { startSpan } from '../observability.js';
+const COHERE_RERANK_URL = 'https://api.cohere.com/v2/rerank';
+const COHERE_RERANK_MODEL = 'rerank-v3.5';
+// =============================================================================
+// rerankResults
+// =============================================================================
+/**
+ * Re-rank search results using Cohere's cross-encoder model.
+ *
+ * Sends each result's content + the query to Cohere, which scores
+ * how well each document answers the query (0 to 1). Results are
+ * re-sorted by this relevance score (highest first).
+ *
+ * The score field on each result is replaced with the Cohere
+ * relevance score — this is intentional. The original RRF score
+ * is a ranking position, not a quality signal. The reranker score
+ * IS a quality signal (how relevant is this document to this query).
+ *
+ * Security: API key is sent only in the Authorization header,
+ * never in the request body. Document content IS sent to Cohere
+ * for scoring — same data flow as OpenAI embeddings.
+ */
+export async function rerankResults(query, searchResults, options) {
+    if (searchResults.length === 0)
+        return [];
+    const topN = options.topN ?? searchResults.length;
+    const model = options.model ?? COHERE_RERANK_MODEL;
+    // --- rerank.prepare ---
+    const prepareSpan = startSpan('rerank.prepare');
+    const documents = searchResults.map(searchResult => ({
+        text: searchResult.content,
+    }));
+    const totalContentLength = documents.reduce((sum, document) => sum + document.text.length, 0);
+    prepareSpan.update({ output: { documentCount: documents.length, totalContentLength } });
+    prepareSpan.end();
+    // --- rerank.queue-wait + rerank.api-call ---
+    const queueSpan = startSpan('rerank.queue-wait');
+    let response;
+    try {
+        response = await cohereLimiter.schedule(() => {
+            queueSpan.end();
+            const apiSpan = startSpan('rerank.api-call');
+            apiSpan.update({ input: { model, topN, documentCount: documents.length } });
+            const apiStartMs = Date.now();
+            return fetch(COHERE_RERANK_URL, {
+                method: 'POST',
+                headers: {
+                    'Authorization': `Bearer ${options.apiKey}`,
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify({
+                    model,
+                    query,
+                    documents,
+                    top_n: topN,
+                }),
+            }).then(fetchResponse => {
+                apiSpan.update({
+                    output: {
+                        statusCode: fetchResponse.status,
+                        latencyMs: Date.now() - apiStartMs,
+                    },
+                });
+                apiSpan.end();
+                return fetchResponse;
+            }).catch(fetchError => {
+                apiSpan.update({
+                    output: {
+                        error: fetchError.message,
+                        errorType: 'network',
+                        latencyMs: Date.now() - apiStartMs,
+                    },
+                });
+                apiSpan.end();
+                throw fetchError;
+            });
+        });
+    }
+    catch (_networkError) {
+        // Network failure or limiter error — return originals unchanged
+        return searchResults;
+    }
+    if (!response.ok) {
+        // API error (rate limit, bad key, server error) — return originals unchanged
+        return searchResults;
+    }
+    let cohereResponse;
+    try {
+        cohereResponse = (await response.json());
+    }
+    catch (_parseError) {
+        // Malformed JSON from Cohere — return originals unchanged
+        return searchResults;
+    }
+    if (!cohereResponse.results || !Array.isArray(cohereResponse.results)) {
+        return searchResults;
+    }
+    // Map Cohere results back to our search results, re-sorted by relevance.
+    // Cohere returns results sorted by relevance_score (highest first).
+    // Each result has an 'index' pointing to the original position in our input array.
+    const rerankedResults = cohereResponse.results.map((cohereResult) => ({
+        ...searchResults[cohereResult.index],
+        score: cohereResult.relevance_score,
+    }));
+    return rerankedResults;
+}

package/dist/lib/search/semantic-cache.js

@@ -0,0 +1,53 @@
+// semantic-cache.ts
+// Helpers for the semantic cache (layer 2).
+// Handles serialization, deserialization, and parameter normalization
+// for cache lookup and store operations.
+//
+// The actual cache logic (HNSW lookup, store, invalidation) lives in Postgres
+// RPC functions. This module prepares data for those calls.
+const EMBEDDING_MODEL_ID = 'openai/text-embedding-3-small';
+const SIMILARITY_THRESHOLD = 0.90;
+export { EMBEDDING_MODEL_ID as SEMANTIC_CACHE_MODEL_ID };
+export { SIMILARITY_THRESHOLD as SEMANTIC_CACHE_THRESHOLD };
+/**
+ * Build a normalized search_params object for cache key matching.
+ * Keys are sorted alphabetically so JSONB equality works regardless
+ * of the order properties were passed in.
+ * Undefined values are omitted (not set to null) to avoid
+ * mismatches between {domain: null} and {}.
+ */
+export function buildSearchParams(input) {
+    const params = {};
+    // Alphabetical order for consistent JSONB serialization
+    if (input.document_type !== undefined)
+        params.document_type = input.document_type;
+    if (input.domain !== undefined)
+        params.domain = input.domain;
+    if (input.limit !== undefined)
+        params.limit = input.limit;
+    if (input.project !== undefined)
+        params.project = input.project;
+    if (input.threshold !== undefined)
+        params.threshold = input.threshold;
+    return params;
+}
+// =============================================================================
+// Result serialization
+// =============================================================================
+/**
+ * Parse cached_results JSONB from Postgres into typed array.
+ * Returns the array directly since we cache full ISearchResultProps objects.
+ */
+export function parseCachedResults(jsonb) {
+    if (!jsonb || jsonb.length === 0)
+        return [];
+    return jsonb;
+}
+/**
+ * Extract unique document IDs from search results for the reverse index.
+ * These are stored in source_doc_ids so document_update/delete can
+ * invalidate affected cache entries.
+ */
+export function extractSourceDocIds(results) {
+    return [...new Set(results.map(result => result.id))];
+}

package/dist/lib/type-registry.test.js

@@ -6,7 +6,7 @@ vi.mock('./config.js', () => ({
     saveConfigFile: (config) => { mockConfigState.current = config; },
 }));
 // Import AFTER mock setup so modules pick up the mocked config
-const { BUILTIN_TYPES, getTypeRegistry, inferDelivery, getRegisteredTypes, isRegisteredType, registerType, validateTypeName, checkMetadataCompleteness, } = await import('./notes.js');
+const { BUILTIN_TYPES, getTypeRegistry, inferDelivery, inferDomain, getRegisteredTypes, isRegisteredType, registerType, validateTypeName, checkMetadataCompleteness, } = await import('./notes.js');
 // --- Helpers ---
 function setUserTypes(types) {
     mockConfigState.current = { types };
@@ -178,7 +178,7 @@ describe('validateTypeName', () => {
     });
 });
 // ============================================================
-// 7. checkMetadataCompleteness (dynamic delivery check)
+// 7. checkMetadataCompleteness (dynamic domain check)
 // ============================================================
 describe('checkMetadataCompleteness', () => {
     it('returns null when all fields are present for project type', () => {
@@ -189,20 +189,20 @@ describe('checkMetadataCompleteness', () => {
         const result = checkMetadataCompleteness({ description: 'test', upsert_key: 'test-key' }, 'code-craft');
         expect(result).toBeNull();
     });
-    it('prompts for status on project-delivery types', () => {
+    it('prompts for status on project-domain types', () => {
         const result = checkMetadataCompleteness({ description: 'test', upsert_key: 'test-key' }, 'architecture-decision');
         expect(result).toContain('status');
     });
-    it('does NOT prompt for status on persona-delivery types', () => {
+    it('does NOT prompt for status on persona-domain types', () => {
         const result = checkMetadataCompleteness({ description: 'test', upsert_key: 'test-key' }, 'persona-rule');
         expect(result).toBeNull();
     });
-    it('uses inferDelivery for custom types — project custom type requires status', () => {
+    it('uses type registry for custom types — project custom type requires status', () => {
         setUserTypes({ 'wine-log': 'project' });
         const result = checkMetadataCompleteness({ description: 'test', upsert_key: 'test-key' }, 'wine-log');
         expect(result).toContain('status');
     });
-    it('uses inferDelivery for custom types — knowledge custom type skips status', () => {
+    it('uses type registry for custom types — knowledge custom type skips status', () => {
         setUserTypes({ 'recipe': 'knowledge' });
         const result = checkMetadataCompleteness({ description: 'test', upsert_key: 'test-key' }, 'recipe');
         expect(result).toBeNull();