@luckydraw/cumulus 0.5.1

package/dist/lib/content-store.js

@@ -0,0 +1,955 @@
+import * as crypto from 'crypto';
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { nanoid } from 'nanoid';
+import { generateHeuristicSummary } from './content-detector.js';
+import { estimateTokens } from './context-budget.js';
+import { cosineSimilarity, embeddingsAvailable, getEmbeddingProvider } from './embeddings.js';
+/** Default chunk size in tokens */
+const DEFAULT_CHUNK_SIZE = 800;
+/** Minimum structural unit size before merging with sibling */
+const MIN_UNIT_TOKENS = 100;
+/** Maximum snippet length for search results */
+const SNIPPET_LENGTH = 300;
+/** Minimum similarity threshold for semantic search */
+const SEMANTIC_THRESHOLD = 0.3;
+/**
+ * ContentStore manages externalized content storage and retrieval.
+ *
+ * Content is stored in a thread-specific directory:
+ * - index.jsonl: Metadata for all stored content
+ * - chunks/: Directory containing actual content chunks
+ */
+export class ContentStore {
+    basePath;
+    indexPath;
+    chunksDir;
+    constructor(basePath) {
+        this.basePath = basePath;
+        this.indexPath = path.join(basePath, 'index.jsonl');
+        this.chunksDir = path.join(basePath, 'chunks');
+    }
+    /**
+     * Ensure storage directories exist.
+     */
+    async ensureDirectories() {
+        await fs.mkdir(this.chunksDir, { recursive: true });
+    }
+    /**
+     * Get path to the images directory for cached image files.
+     */
+    getImagesDir() {
+        return path.join(this.basePath, 'images');
+    }
+    /**
+     * Ensure the images directory exists.
+     */
+    async ensureImagesDir() {
+        const imagesDir = this.getImagesDir();
+        await fs.mkdir(imagesDir, { recursive: true });
+        return imagesDir;
+    }
+    /**
+     * Compute SHA-256 hash of content for deduplication.
+     */
+    computeHash(content) {
+        return crypto.createHash('sha256').update(content).digest('hex');
+    }
+    /**
+     * Split content into chunks of approximately chunkSize tokens.
+     */
+    createChunks(content, chunkSize = DEFAULT_CHUNK_SIZE) {
+        const lines = content.split('\n');
+        const chunks = [];
+        let currentChunk = [];
+        let currentTokens = 0;
+        for (const line of lines) {
+            const lineTokens = estimateTokens(line);
+            // If adding this line would exceed chunk size, finalize current chunk
+            if (currentTokens + lineTokens > chunkSize && currentChunk.length > 0) {
+                const chunkContent = currentChunk.join('\n');
+                chunks.push({
+                    index: chunks.length,
+                    content: chunkContent,
+                    tokenEstimate: currentTokens,
+                });
+                currentChunk = [];
+                currentTokens = 0;
+            }
+            currentChunk.push(line);
+            currentTokens += lineTokens;
+        }
+        // Don't forget the last chunk
+        if (currentChunk.length > 0) {
+            const chunkContent = currentChunk.join('\n');
+            chunks.push({
+                index: chunks.length,
+                content: chunkContent,
+                tokenEstimate: currentTokens,
+            });
+        }
+        return chunks;
+    }
+    /**
+     * Detect structural boundaries in non-code content and split into structural units.
+     * Each unit is a coherent block: a headed section, table, code fence, paragraph, etc.
+     */
+    detectStructuralUnits(lines) {
+        const units = [];
+        let current = [];
+        let inCodeFence = false;
+        let inTable = false;
+        const flushCurrent = () => {
+            if (current.length > 0) {
+                units.push(current);
+                current = [];
+            }
+        };
+        for (let i = 0; i < lines.length; i++) {
+            const line = lines[i];
+            const trimmed = line.trimStart();
+            // Code fence toggle
+            if (trimmed.startsWith('```') || trimmed.startsWith('~~~')) {
+                if (!inCodeFence) {
+                    // Starting a code fence — flush anything before it
+                    flushCurrent();
+                    inCodeFence = true;
+                    current.push(line);
+                }
+                else {
+                    // Closing a code fence — include closing line, flush as atomic unit
+                    inCodeFence = false;
+                    current.push(line);
+                    flushCurrent();
+                }
+                continue;
+            }
+            // Inside a code fence — accumulate without any splitting
+            if (inCodeFence) {
+                current.push(line);
+                continue;
+            }
+            // Table detection: lines starting with |
+            const isTableLine = trimmed.startsWith('|') && trimmed.includes('|', 1);
+            if (isTableLine) {
+                if (!inTable) {
+                    // Starting a table — flush anything before it
+                    flushCurrent();
+                    inTable = true;
+                }
+                current.push(line);
+                continue;
+            }
+            else if (inTable) {
+                // Leaving a table — flush the table as atomic unit
+                inTable = false;
+                flushCurrent();
+            }
+            // Markdown heading — starts a new section
+            if (/^#{1,6}\s/.test(trimmed)) {
+                flushCurrent();
+                current.push(line);
+                continue;
+            }
+            // Horizontal rule
+            if (/^(?:---+|___+|\*\*\*+)\s*$/.test(trimmed)) {
+                flushCurrent();
+                current.push(line);
+                flushCurrent();
+                continue;
+            }
+            // Blank line — paragraph boundary
+            if (trimmed === '') {
+                if (current.length > 0) {
+                    // End of a paragraph/section
+                    flushCurrent();
+                }
+                // Skip blank lines (don't include in any unit)
+                continue;
+            }
+            // Regular line — accumulate into current unit
+            current.push(line);
+        }
+        // Flush remaining (including unclosed code fences)
+        flushCurrent();
+        return units;
+    }
+    /**
+     * Split non-code content into structure-aware chunks.
+     * Respects headers, tables, code fences, and paragraph boundaries.
+     * Merges small units and splits oversized ones.
+     */
+    createStructuralChunks(content, chunkSize = DEFAULT_CHUNK_SIZE) {
+        const lines = content.split('\n');
+        const units = this.detectStructuralUnits(lines);
+        // Merge tiny units with their next sibling
+        const mergedUnits = [];
+        let pendingUnit = null;
+        let pendingTokens = 0;
+        for (const unit of units) {
+            const unitContent = unit.join('\n');
+            const unitTokens = estimateTokens(unitContent);
+            if (pendingUnit) {
+                // Merge pending tiny unit with this one
+                pendingUnit.push('', ...unit); // blank line separator
+                pendingTokens += unitTokens;
+                if (pendingTokens >= MIN_UNIT_TOKENS) {
+                    mergedUnits.push(pendingUnit);
+                    pendingUnit = null;
+                    pendingTokens = 0;
+                }
+            }
+            else if (unitTokens < MIN_UNIT_TOKENS) {
+                pendingUnit = [...unit];
+                pendingTokens = unitTokens;
+            }
+            else {
+                mergedUnits.push(unit);
+            }
+        }
+        // Flush any remaining pending unit
+        if (pendingUnit) {
+            if (mergedUnits.length > 0) {
+                // Append to previous unit
+                const last = mergedUnits[mergedUnits.length - 1];
+                last.push('', ...pendingUnit);
+            }
+            else {
+                mergedUnits.push(pendingUnit);
+            }
+        }
+        // Pack units into chunks
+        const chunks = [];
+        let chunkLines = [];
+        let chunkTokens = 0;
+        for (const unit of mergedUnits) {
+            const unitContent = unit.join('\n');
+            const unitTokens = estimateTokens(unitContent);
+            // If a single unit exceeds chunk size, split it with the fallback line-based splitter
+            if (unitTokens > chunkSize) {
+                // Flush accumulated chunk first
+                if (chunkLines.length > 0) {
+                    const cc = chunkLines.join('\n');
+                    chunks.push({ index: chunks.length, content: cc, tokenEstimate: chunkTokens });
+                    chunkLines = [];
+                    chunkTokens = 0;
+                }
+                // Split the oversized unit at line boundaries
+                const subChunks = this.createChunks(unitContent, chunkSize);
+                for (const sub of subChunks) {
+                    chunks.push({
+                        index: chunks.length,
+                        content: sub.content,
+                        tokenEstimate: sub.tokenEstimate,
+                    });
+                }
+                continue;
+            }
+            // If adding this unit exceeds chunk size, finalize current chunk
+            if (chunkTokens + unitTokens > chunkSize && chunkLines.length > 0) {
+                const cc = chunkLines.join('\n');
+                chunks.push({ index: chunks.length, content: cc, tokenEstimate: chunkTokens });
+                chunkLines = [];
+                chunkTokens = 0;
+            }
+            // Add blank line separator between units within a chunk
+            if (chunkLines.length > 0) {
+                chunkLines.push('');
+            }
+            chunkLines.push(...unit);
+            chunkTokens += unitTokens;
+        }
+        // Final chunk
+        if (chunkLines.length > 0) {
+            const cc = chunkLines.join('\n');
+            chunks.push({ index: chunks.length, content: cc, tokenEstimate: chunkTokens });
+        }
+        return chunks;
+    }
+    /**
+     * Regex patterns that identify top-level code block boundaries.
+     */
+    static CODE_BLOCK_PATTERNS = [
+        // JavaScript/TypeScript
+        /^(?:export\s+)?(?:async\s+)?function\s+\w+/,
+        /^(?:export\s+)?(?:const|let|var)\s+\w+\s*=\s*(?:async\s*)?\(/,
+        /^(?:export\s+)?class\s+\w+/,
+        /^(?:export\s+)?(?:interface|type|enum)\s+\w+/,
+        // Python
+        /^(?:async\s+)?def\s+\w+/,
+        // Rust
+        /^(?:pub\s+)?(?:async\s+)?fn\s+\w+/,
+        /^(?:pub\s+)?(?:struct|enum|trait)\s+/,
+        /^impl\s+/,
+        // Go
+        /^func\s+/,
+        /^type\s+\w+\s+(?:struct|interface)/,
+    ];
+    /**
+     * Pattern matching import/use/include lines.
+     */
+    static IMPORT_PATTERN = /^(?:import\s|from\s|require\(|use\s|#include\s|include\s)/;
+    /**
+     * Check if a line is a code block boundary (top-level declaration).
+     */
+    isBlockBoundary(line) {
+        return ContentStore.CODE_BLOCK_PATTERNS.some(p => p.test(line));
+    }
+    /**
+     * Split code content into chunks at function/class/type boundaries.
+     * Falls back to line-based splitting for oversized single blocks.
+     */
+    createCodeChunks(content, chunkSize = DEFAULT_CHUNK_SIZE) {
+        const lines = content.split('\n');
+        const blocks = [];
+        let currentBlock = [];
+        let inImportBlock = false;
+        for (const line of lines) {
+            const isImport = ContentStore.IMPORT_PATTERN.test(line);
+            const isBoundary = this.isBlockBoundary(line);
+            if (isImport) {
+                // Group consecutive imports together
+                if (!inImportBlock && currentBlock.length > 0) {
+                    blocks.push(currentBlock);
+                    currentBlock = [];
+                }
+                inImportBlock = true;
+                currentBlock.push(line);
+            }
+            else if (isBoundary) {
+                // New top-level declaration starts a new block
+                if (currentBlock.length > 0) {
+                    blocks.push(currentBlock);
+                }
+                inImportBlock = false;
+                currentBlock = [line];
+            }
+            else {
+                // Continue current block (or start a new implicit block after imports)
+                if (inImportBlock && line.trim() !== '') {
+                    blocks.push(currentBlock);
+                    currentBlock = [line];
+                    inImportBlock = false;
+                }
+                else {
+                    currentBlock.push(line);
+                }
+            }
+        }
+        if (currentBlock.length > 0) {
+            blocks.push(currentBlock);
+        }
+        // Now accumulate blocks into chunks, respecting size limits
+        const chunks = [];
+        let chunkLines = [];
+        let chunkTokens = 0;
+        for (const block of blocks) {
+            const blockContent = block.join('\n');
+            const blockTokens = estimateTokens(blockContent);
+            // If a single block exceeds chunk size, fall back to line-based splitting
+            if (blockTokens > chunkSize) {
+                // First, flush any accumulated lines
+                if (chunkLines.length > 0) {
+                    const content = chunkLines.join('\n');
+                    chunks.push({ index: chunks.length, content, tokenEstimate: chunkTokens });
+                    chunkLines = [];
+                    chunkTokens = 0;
+                }
+                // Split the oversized block by lines
+                const subChunks = this.createChunks(blockContent, chunkSize);
+                for (const sub of subChunks) {
+                    chunks.push({
+                        index: chunks.length,
+                        content: sub.content,
+                        tokenEstimate: sub.tokenEstimate,
+                    });
+                }
+                continue;
+            }
+            // If adding this block would exceed chunk size, finalize current chunk
+            if (chunkTokens + blockTokens > chunkSize && chunkLines.length > 0) {
+                const content = chunkLines.join('\n');
+                chunks.push({ index: chunks.length, content, tokenEstimate: chunkTokens });
+                chunkLines = [];
+                chunkTokens = 0;
+            }
+            chunkLines.push(...block);
+            chunkTokens += blockTokens;
+        }
+        // Final chunk
+        if (chunkLines.length > 0) {
+            const content = chunkLines.join('\n');
+            chunks.push({ index: chunks.length, content, tokenEstimate: chunkTokens });
+        }
+        return chunks;
+    }
+    /**
+     * Generate a brief summary of content.
+     * Delegates to the shared heuristic summary generator in content-detector.
+     */
+    generateSimpleSummary(content, contentType) {
+        return generateHeuristicSummary(content, contentType, 500);
+    }
+    /**
+     * Store content externally and return metadata.
+     * Returns existing content ID if content hash matches (deduplication).
+     */
+    async store(content, options) {
+        await this.ensureDirectories();
+        const contentHash = this.computeHash(content);
+        // Check for existing content with same hash
+        const existing = await this.findByHash(contentHash);
+        if (existing) {
+            return existing;
+        }
+        const id = `cnt_${nanoid(10)}`;
+        const contentType = options.contentType ?? this.detectContentType(content);
+        const chunks = contentType === 'code'
+            ? this.createCodeChunks(content)
+            : this.createStructuralChunks(content);
+        const summary = options.summary ?? this.generateSimpleSummary(content, contentType);
+        // Context propagation: prepend document-level context to chunks 1..N
+        if (chunks.length > 1) {
+            const sourceName = options.metadata?.filePath ??
+                options.metadata?.command ??
+                options.sourceTool ??
+                options.sourceType;
+            const contextHeader = `[Source: ${sourceName} | ${contentType} | ${summary.slice(0, 150)}]`;
+            const headerTokens = estimateTokens(contextHeader);
+            for (let i = 1; i < chunks.length; i++) {
+                chunks[i].content = contextHeader + '\n\n' + chunks[i].content;
+                chunks[i].tokenEstimate += headerTokens;
+            }
+        }
+        const meta = {
+            id,
+            timestamp: Date.now(),
+            sourceType: options.sourceType,
+            sourceTool: options.sourceTool,
+            originalSize: content.length,
+            tokenEstimate: estimateTokens(content),
+            contentType,
+            summary,
+            chunkCount: chunks.length,
+            contentHash,
+            metadata: options.metadata ?? {},
+        };
+        // Write chunks to files
+        for (const chunk of chunks) {
+            const chunkPath = path.join(this.chunksDir, `${id}_${chunk.index}.txt`);
+            await fs.writeFile(chunkPath, chunk.content, 'utf-8');
+        }
+        // Append metadata to index
+        const indexLine = JSON.stringify(meta) + '\n';
+        await fs.appendFile(this.indexPath, indexLine, 'utf-8');
+        return meta;
+    }
+    /**
+     * Simple content type detection based on heuristics.
+     */
+    detectContentType(content) {
+        const lines = content.split('\n');
+        const sample = content.slice(0, 2000);
+        // Check for JSON
+        if (sample.trim().startsWith('{') || sample.trim().startsWith('[')) {
+            try {
+                JSON.parse(content);
+                return 'json';
+            }
+            catch {
+                // Not valid JSON
+            }
+        }
+        // Check for code patterns
+        const codePatterns = [
+            /^import\s+/m,
+            /^export\s+/m,
+            /^(?:const|let|var)\s+\w+\s*=/m,
+            /^(?:function|def|fn|func)\s+\w+/m,
+            /^(?:class|interface|struct)\s+\w+/m,
+            /^(?:public|private|protected)\s+/m,
+            /^\s*(?:if|for|while|switch)\s*\(/m,
+        ];
+        const codeScore = codePatterns.filter(p => p.test(sample)).length;
+        if (codeScore >= 2) {
+            return 'code';
+        }
+        // Check for log patterns
+        const logPatterns = [
+            /^\d{4}-\d{2}-\d{2}/m, // Date stamps
+            /^\[\w+\]/m, // [INFO], [ERROR], etc.
+            /^(?:DEBUG|INFO|WARN|ERROR|FATAL):/m,
+            /^\d+:\d+:\d+/m, // Time stamps
+        ];
+        const logScore = logPatterns.filter(p => p.test(sample)).length;
+        if (logScore >= 2) {
+            return 'logs';
+        }
+        // Check for prose (sentences, paragraphs)
+        const avgLineLength = content.length / Math.max(lines.length, 1);
+        const hasLongLines = avgLineLength > 60;
+        const hasPunctuation = /[.!?]\s+[A-Z]/.test(sample);
+        if (hasLongLines && hasPunctuation) {
+            return 'prose';
+        }
+        return 'mixed';
+    }
+    /**
+     * Find content by hash (for deduplication).
+     */
+    async findByHash(hash) {
+        const index = await this.loadIndex();
+        return index.find(meta => meta.contentHash === hash) ?? null;
+    }
+    /**
+     * Load the content index.
+     */
+    async loadIndex() {
+        try {
+            const content = await fs.readFile(this.indexPath, 'utf-8');
+            if (!content.trim()) {
+                return [];
+            }
+            return content
+                .trim()
+                .split('\n')
+                .filter(line => line.trim())
+                .map(line => JSON.parse(line));
+        }
+        catch (error) {
+            if (error.code === 'ENOENT') {
+                return [];
+            }
+            throw error;
+        }
+    }
+    /**
+     * Retrieve content by ID.
+     * @param id - Content ID
+     * @param chunkIndex - Optional specific chunk index
+     * @returns Full content or specific chunk
+     */
+    async retrieve(id, chunkIndex) {
+        const index = await this.loadIndex();
+        const meta = index.find(m => m.id === id);
+        if (!meta) {
+            return null;
+        }
+        if (chunkIndex !== undefined) {
+            if (chunkIndex < 0 || chunkIndex >= meta.chunkCount) {
+                return null;
+            }
+            const chunkPath = path.join(this.chunksDir, `${id}_${chunkIndex}.txt`);
+            try {
+                return await fs.readFile(chunkPath, 'utf-8');
+            }
+            catch {
+                return null;
+            }
+        }
+        // Retrieve all chunks
+        const chunks = [];
+        for (let i = 0; i < meta.chunkCount; i++) {
+            const chunkPath = path.join(this.chunksDir, `${id}_${i}.txt`);
+            try {
+                const chunk = await fs.readFile(chunkPath, 'utf-8');
+                chunks.push(chunk);
+            }
+            catch {
+                // Skip missing chunks
+            }
+        }
+        return chunks.join('\n');
+    }
+    /**
+     * Get metadata for stored content.
+     */
+    async getMeta(id) {
+        const index = await this.loadIndex();
+        return index.find(m => m.id === id) ?? null;
+    }
+    /**
+     * Get previews of all chunks for a stored content item.
+     * Returns a brief preview of each chunk for navigation (table of contents).
+     */
+    async getChunkPreviews(id) {
+        const meta = await this.getMeta(id);
+        if (!meta)
+            return null;
+        const previews = [];
+        for (let i = 0; i < meta.chunkCount; i++) {
+            const chunkPath = path.join(this.chunksDir, `${id}_${i}.txt`);
+            try {
+                const content = await fs.readFile(chunkPath, 'utf-8');
+                const lines = content.split('\n');
+                // Skip context propagation header (starts with [Source:)
+                let previewLine = '';
+                for (const line of lines) {
+                    const trimmed = line.trim();
+                    if (trimmed === '' || trimmed.startsWith('[Source:'))
+                        continue;
+                    previewLine = trimmed;
+                    break;
+                }
+                // Truncate to 120 chars
+                const preview = previewLine.length > 120 ? previewLine.slice(0, 120) + '...' : previewLine;
+                const tokens = estimateTokens(content);
+                previews.push({ index: i, tokens, preview });
+            }
+            catch {
+                previews.push({ index: i, tokens: 0, preview: '(chunk not found)' });
+            }
+        }
+        return previews;
+    }
+    /**
+     * List all stored content.
+     */
+    async list(options = {}) {
+        let index = await this.loadIndex();
+        if (options.sourceTypes && options.sourceTypes.length > 0) {
+            index = index.filter(m => options.sourceTypes.includes(m.sourceType));
+        }
+        // Sort by timestamp descending (most recent first)
+        index.sort((a, b) => b.timestamp - a.timestamp);
+        if (options.limit) {
+            index = index.slice(0, options.limit);
+        }
+        return index;
+    }
+    /**
+     * Delete stored content by ID.
+     * Removes from index, chunks, and embeddings.
+     * @returns true if content was found and deleted, false if not found
+     */
+    async delete(id) {
+        const index = await this.loadIndex();
+        const metaIndex = index.findIndex(m => m.id === id);
+        if (metaIndex === -1) {
+            return false;
+        }
+        const meta = index[metaIndex];
+        // Remove chunk files
+        for (let i = 0; i < meta.chunkCount; i++) {
+            const chunkPath = path.join(this.chunksDir, `${id}_${i}.txt`);
+            try {
+                await fs.unlink(chunkPath);
+            }
+            catch {
+                // Ignore if chunk file doesn't exist
+            }
+        }
+        // Remove from index
+        index.splice(metaIndex, 1);
+        await this.saveIndex(index);
+        // Remove from embeddings
+        await this.deleteEmbeddings(id);
+        return true;
+    }
+    /**
+     * Delete embeddings for a content ID.
+     */
+    async deleteEmbeddings(contentId) {
+        const embeddingsPath = this.getEmbeddingsPath();
+        try {
+            const content = await fs.readFile(embeddingsPath, 'utf-8');
+            const data = JSON.parse(content);
+            // Filter out entries for this content ID
+            data.entries = data.entries.filter(e => e.contentId !== contentId);
+            await fs.writeFile(embeddingsPath, JSON.stringify(data), 'utf-8');
+        }
+        catch {
+            // No embeddings file or parse error - nothing to delete
+        }
+    }
+    /**
+     * Save the index file.
+     */
+    async saveIndex(index) {
+        await this.ensureDirectories();
+        const content = index.map(m => JSON.stringify(m)).join('\n');
+        await fs.writeFile(this.indexPath, content + (content ? '\n' : ''), 'utf-8');
+    }
+    /**
+     * Get path to embeddings file.
+     */
+    getEmbeddingsPath() {
+        return path.join(this.basePath, 'embeddings.json');
+    }
+    /**
+     * Get embeddings for all chunks of a specific content item.
+     * Generates missing embeddings first if needed.
+     * Returns a Map keyed by chunk index.
+     */
+    async getEmbeddingsForContent(contentId) {
+        const meta = await this.getMeta(contentId);
+        if (!meta)
+            return null;
+        await this.generateMissingEmbeddings();
+        const allEmbeddings = await this.loadContentEmbeddings();
+        const result = new Map();
+        for (let i = 0; i < meta.chunkCount; i++) {
+            const embedding = allEmbeddings.get(`${contentId}:${i}`);
+            if (embedding) {
+                result.set(i, embedding);
+            }
+        }
+        return result;
+    }
+    /**
+     * Load content embeddings.
+     */
+    async loadContentEmbeddings() {
+        const embeddingsPath = this.getEmbeddingsPath();
+        try {
+            const content = await fs.readFile(embeddingsPath, 'utf-8');
+            const data = JSON.parse(content);
+            const map = new Map();
+            for (const entry of data.entries) {
+                // Key is contentId:chunkIndex
+                map.set(`${entry.contentId}:${entry.chunkIndex}`, entry.embedding);
+            }
+            return map;
+        }
+        catch (error) {
+            if (error.code === 'ENOENT') {
+                return new Map();
+            }
+            throw error;
+        }
+    }
+    /**
+     * Save content embeddings.
+     */
+    async saveContentEmbeddings(entries) {
+        const embeddingsPath = this.getEmbeddingsPath();
+        const provider = getEmbeddingProvider();
+        let existingData;
+        try {
+            const content = await fs.readFile(embeddingsPath, 'utf-8');
+            existingData = JSON.parse(content);
+        }
+        catch {
+            existingData = {
+                version: 1,
+                model: provider.name,
+                dimensions: provider.dimensions,
+                entries: [],
+            };
+        }
+        existingData.entries.push(...entries);
+        await fs.mkdir(this.basePath, { recursive: true });
+        await fs.writeFile(embeddingsPath, JSON.stringify(existingData), 'utf-8');
+    }
+    /**
+     * Generate embeddings for content that doesn't have them yet.
+     */
+    async generateMissingEmbeddings() {
+        if (!(await embeddingsAvailable())) {
+            return 0;
+        }
+        const index = await this.loadIndex();
+        const existingEmbeddings = await this.loadContentEmbeddings();
+        const provider = getEmbeddingProvider();
+        const newEntries = [];
+        for (const meta of index) {
+            for (let i = 0; i < meta.chunkCount; i++) {
+                const key = `${meta.id}:${i}`;
+                if (existingEmbeddings.has(key)) {
+                    continue;
+                }
+                const chunkContent = await this.retrieve(meta.id, i);
+                if (!chunkContent)
+                    continue;
+                const [embedding] = await provider.embed([chunkContent]);
+                if (embedding) {
+                    newEntries.push({
+                        contentId: meta.id,
+                        chunkIndex: i,
+                        embedding,
+                        timestamp: new Date().toISOString(),
+                    });
+                }
+            }
+        }
+        if (newEntries.length > 0) {
+            await this.saveContentEmbeddings(newEntries);
+        }
+        return newEntries.length;
+    }
+    /**
+     * Search stored content with keyword, semantic, or hybrid search.
+     */
+    async search(query, options = {}) {
+        const index = await this.loadIndex();
+        const queryLower = query.toLowerCase();
+        // Use pre-expanded terms if provided, otherwise split query into terms
+        const queryTerms = options.expandedTerms ?? queryLower.split(/\s+/).filter(t => t.length > 2);
+        const limit = options.limit ?? 10;
+        const mode = options.mode ?? 'hybrid';
+        // Keyword scores
+        const keywordResults = new Map();
+        for (const meta of index) {
+            // Apply filters
+            if (options.sourceTypes && !options.sourceTypes.includes(meta.sourceType)) {
+                continue;
+            }
+            if (options.contentTypes && !options.contentTypes.includes(meta.contentType)) {
+                continue;
+            }
+            const summaryLower = meta.summary.toLowerCase();
+            let summaryScore = 0;
+            for (const term of queryTerms) {
+                if (summaryLower.includes(term)) {
+                    summaryScore++;
+                }
+            }
+            // Check summary for keyword match (any term matches)
+            if (summaryScore > 0) {
+                keywordResults.set(meta.id, {
+                    meta,
+                    score: Math.min(0.5 * (summaryScore / queryTerms.length), 0.5),
+                    snippet: meta.summary.slice(0, SNIPPET_LENGTH),
+                });
+                continue;
+            }
+            // Check content chunks for keyword match
+            const content = await this.retrieve(meta.id);
+            if (content) {
+                const contentLower = content.toLowerCase();
+                let contentScore = 0;
+                let bestMatchIndex = -1;
+                for (const term of queryTerms) {
+                    if (contentLower.includes(term)) {
+                        contentScore++;
+                        // Track first significant term match for snippet
+                        if (bestMatchIndex < 0 && term.length > 3) {
+                            bestMatchIndex = contentLower.indexOf(term);
+                        }
+                    }
+                }
+                if (contentScore > 0) {
+                    // Generate snippet around the best match
+                    const matchIndex = bestMatchIndex >= 0 ? bestMatchIndex : 0;
+                    const start = Math.max(0, matchIndex - 50);
+                    const end = Math.min(content.length, matchIndex + 250);
+                    const snippet = (start > 0 ? '...' : '') +
+                        content.slice(start, end) +
+                        (end < content.length ? '...' : '');
+                    keywordResults.set(meta.id, {
+                        meta,
+                        score: Math.min(0.8 * (contentScore / queryTerms.length), 0.8),
+                        snippet,
+                    });
+                }
+            }
+        }
+        // If keyword-only mode, return keyword results
+        if (mode === 'keyword') {
+            const results = Array.from(keywordResults.values());
+            results.sort((a, b) => b.score - a.score);
+            return results.slice(0, limit);
+        }
+        // Semantic search
+        const semanticResults = new Map();
+        if (await embeddingsAvailable()) {
+            try {
+                // Generate embeddings for any content that doesn't have them
+                await this.generateMissingEmbeddings();
+                const embeddings = await this.loadContentEmbeddings();
+                const provider = getEmbeddingProvider();
+                const [queryEmbedding] = await provider.embed([query]);
+                if (queryEmbedding) {
+                    for (const meta of index) {
+                        // Apply filters
+                        if (options.sourceTypes && !options.sourceTypes.includes(meta.sourceType)) {
+                            continue;
+                        }
+                        if (options.contentTypes && !options.contentTypes.includes(meta.contentType)) {
+                            continue;
+                        }
+                        // Find best matching chunk
+                        let bestScore = 0;
+                        let bestChunkIndex = 0;
+                        for (let i = 0; i < meta.chunkCount; i++) {
+                            const embedding = embeddings.get(`${meta.id}:${i}`);
+                            if (!embedding)
+                                continue;
+                            const similarity = cosineSimilarity(queryEmbedding, embedding);
+                            if (similarity > bestScore) {
+                                bestScore = similarity;
+                                bestChunkIndex = i;
+                            }
+                        }
+                        if (bestScore > SEMANTIC_THRESHOLD) {
+                            const chunkContent = await this.retrieve(meta.id, bestChunkIndex);
+                            const snippet = chunkContent?.slice(0, SNIPPET_LENGTH) + '...' || meta.summary;
+                            semanticResults.set(meta.id, {
+                                meta,
+                                score: bestScore,
+                                snippet,
+                            });
+                        }
+                    }
+                }
+            }
+            catch {
+                // Fall back to keyword-only if semantic search fails
+            }
+        }
+        // If semantic-only mode, return semantic results
+        if (mode === 'semantic') {
+            const results = Array.from(semanticResults.values());
+            results.sort((a, b) => b.score - a.score);
+            return results.slice(0, limit);
+        }
+        // Hybrid mode: combine scores
+        const combinedResults = new Map();
+        for (const [id, result] of keywordResults) {
+            combinedResults.set(id, result);
+        }
+        for (const [id, result] of semanticResults) {
+            const existing = combinedResults.get(id);
+            if (existing) {
+                // Combine scores: weight keyword 0.3, semantic 0.7, boost if both match
+                const combinedScore = (existing.score * 0.3 + result.score * 0.7) * 1.2;
+                combinedResults.set(id, {
+                    meta: result.meta,
+                    score: Math.min(combinedScore, 1),
+                    snippet: existing.snippet, // Keep keyword snippet as it's more targeted
+                });
+            }
+            else {
+                combinedResults.set(id, {
+                    meta: result.meta,
+                    score: result.score * 0.7, // Semantic-only gets weighted down
+                    snippet: result.snippet,
+                });
+            }
+        }
+        const results = Array.from(combinedResults.values());
+        results.sort((a, b) => b.score - a.score);
+        return results.slice(0, limit);
+    }
+    /**
+     * Get statistics about stored content.
+     */
+    async getStats() {
+        const index = await this.loadIndex();
+        const stats = {
+            totalItems: index.length,
+            totalSize: 0,
+            totalTokens: 0,
+            bySourceType: {},
+            byContentType: {},
+        };
+        for (const meta of index) {
+            stats.totalSize += meta.originalSize;
+            stats.totalTokens += meta.tokenEstimate;
+            stats.bySourceType[meta.sourceType] = (stats.bySourceType[meta.sourceType] ?? 0) + 1;
+            stats.byContentType[meta.contentType] = (stats.byContentType[meta.contentType] ?? 0) + 1;
+        }
+        return stats;
+    }
+}
+//# sourceMappingURL=content-store.js.map