agent-orcha 0.0.4 → 0.0.7

package/dist/lib/knowledge/knowledge-store.js

@@ -0,0 +1,670 @@
+import * as fs from 'fs/promises';
+import * as crypto from 'crypto';
+import * as path from 'path';
+import { glob } from 'glob';
+import { parse as parseYaml } from 'yaml';
+import { SqliteStore } from "./sqlite-store.js";
+import { DirectMapper } from "./direct-mapper.js";
+import { KnowledgeConfigSchema } from "./types.js";
+import { KnowledgeMetadataManager, createDefaultMetadata } from "./knowledge-store-metadata.js";
+import { CharacterTextSplitter, RecursiveCharacterTextSplitter } from "../types/text-splitters.js";
+import { OpenAIEmbeddingsProvider } from "../llm/providers/openai-embeddings.js";
+import { GeminiEmbeddingsProvider } from "../llm/providers/gemini-embeddings.js";
+import { getEmbeddingConfig, resolveApiKey } from "../llm/llm-config.js";
+import { detectProvider } from "../llm/provider-detector.js";
+import { DatabaseLoader, WebLoader, TextLoader, JSONLoader, CSVLoader, PDFLoader } from "./loaders/index.js";
+import cron from 'node-cron';
+import { substituteEnvVars } from "../utils/env-substitution.js";
+import { createLogger } from "../logger.js";
+const logger = createLogger('KnowledgeStore');
+const searchLogger = createLogger('KnowledgeSearch');
+/**
+ * Unified knowledge store. Replaces KnowledgeStoreManager + KnowledgeStoreFactory + GraphRagFactory.
+ * All stores use SQLite as persistence — no caching layers.
+ * If config.graph exists, entity extraction is enabled; otherwise chunks-only.
+ */
+export class KnowledgeStore {
+    knowledgeDir;
+    workspaceRoot;
+    stores = new Map();
+    configs = new Map();
+    sqliteStores = new Map();
+    metadataManager;
+    activeIndexing = new Map();
+    reindexTasks = new Map();
+    pathToName = new Map();
+    constructor(knowledgeDir, workspaceRoot) {
+        this.knowledgeDir = knowledgeDir;
+        this.workspaceRoot = workspaceRoot;
+        this.metadataManager = new KnowledgeMetadataManager(path.join(workspaceRoot, '.knowledge-cache'));
+    }
+    // --- Config Loading ---
+    async loadAll() {
+        const files = await glob('**/*.knowledge.yaml', { cwd: this.knowledgeDir });
+        for (const file of files) {
+            const filePath = path.join(this.knowledgeDir, file);
+            try {
+                await this.loadOne(filePath);
+            }
+            catch (err) {
+                const message = err instanceof Error ? err.message : String(err);
+                logger.warn(`Skipping invalid knowledge file "${file}": ${message}`);
+            }
+        }
+        const names = Array.from(this.configs.keys());
+        await this.metadataManager.resetStaleIndexing(names);
+        await this.restoreIndexedStores();
+        this.startReindexCrons();
+    }
+    async loadOne(filePath) {
+        const content = await fs.readFile(filePath, 'utf-8');
+        const parsed = parseYaml(substituteEnvVars(content));
+        const config = KnowledgeConfigSchema.parse(parsed);
+        // Resolve sqlite:// paths relative to workspaceRoot so they work regardless of cwd
+        if (config.source.type === 'database' && config.source.connectionString.startsWith('sqlite://')) {
+            const filePart = config.source.connectionString.replace(/^sqlite:\/\//, '');
+            if (!path.isAbsolute(filePart)) {
+                config.source.connectionString = `sqlite://${path.resolve(this.workspaceRoot, filePart)}`;
+            }
+        }
+        this.configs.set(config.name, config);
+        this.pathToName.set(path.resolve(filePath), config.name);
+        return config;
+    }
+    async restoreIndexedStores() {
+        const statuses = await this.getAllStatuses();
+        for (const [name, metadata] of statuses) {
+            if (metadata.status !== 'indexed')
+                continue;
+            if (this.stores.has(name))
+                continue;
+            try {
+                logger.info(`Restoring "${name}" from SQLite...`);
+                await this.initialize(name);
+            }
+            catch (error) {
+                logger.warn(`Failed to restore "${name}": ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+    }
+    startReindexCrons() {
+        for (const [name, config] of this.configs) {
+            if (!config.reindex?.schedule)
+                continue;
+            if (!cron.validate(config.reindex.schedule)) {
+                logger.warn(`Invalid reindex cron "${config.reindex.schedule}" for "${name}", skipping`);
+                continue;
+            }
+            this.reindexTasks.get(name)?.stop();
+            const task = cron.schedule(config.reindex.schedule, async () => {
+                if (!this.stores.has(name)) {
+                    logger.info(`Reindex cron fired for "${name}" but store is not initialized, skipping`);
+                    return;
+                }
+                if (this.activeIndexing.has(name)) {
+                    logger.info(`Reindex cron fired for "${name}" but indexing already in progress, skipping`);
+                    return;
+                }
+                try {
+                    logger.info(`Reindex cron firing for "${name}"...`);
+                    await this.refresh(name);
+                    logger.info(`Reindex cron completed for "${name}"`);
+                }
+                catch (error) {
+                    logger.error(`Reindex cron failed for "${name}":`, error);
+                }
+            });
+            this.reindexTasks.set(name, task);
+            logger.info(`Scheduled reindex cron for "${name}" [${config.reindex.schedule}]`);
+        }
+    }
+    evict(name) {
+        // Stop reindex cron
+        const cronTask = this.reindexTasks.get(name);
+        if (cronTask) {
+            cronTask.stop();
+            this.reindexTasks.delete(name);
+        }
+        // Close and remove SQLite store
+        const sqliteStore = this.sqliteStores.get(name);
+        if (sqliteStore) {
+            sqliteStore.close();
+            this.sqliteStores.delete(name);
+        }
+        // Remove from all maps
+        this.stores.delete(name);
+        this.configs.delete(name);
+        this.activeIndexing.delete(name);
+        logger.info(`Evicted knowledge store "${name}"`);
+    }
+    nameForPath(absolutePath) {
+        return this.pathToName.get(absolutePath);
+    }
+    close() {
+        for (const [, task] of this.reindexTasks) {
+            task.stop();
+        }
+        this.reindexTasks.clear();
+    }
+    // --- Initialization ---
+    async initialize(name, onProgress) {
+        const existing = this.stores.get(name);
+        if (existing) {
+            logger.info(`"${name}" already initialized`);
+            return existing;
+        }
+        // Prevent concurrent indexing of the same store
+        const activePromise = this.activeIndexing.get(name);
+        if (activePromise) {
+            logger.info(`"${name}" is already being indexed, waiting...`);
+            return activePromise;
+        }
+        const config = this.configs.get(name);
+        if (!config) {
+            throw new Error(`Knowledge config not found: ${name}`);
+        }
+        const promise = this.doInitialize(name, config, onProgress);
+        this.activeIndexing.set(name, promise);
+        try {
+            const store = await promise;
+            return store;
+        }
+        finally {
+            this.activeIndexing.delete(name);
+        }
+    }
+    async doInitialize(name, config, onProgress) {
+        const hasGraph = !!config.graph;
+        logger.info(`Initializing "${name}" (graph: ${hasGraph})...`);
+        if (config.source.type === 'directory' || config.source.type === 'file') {
+            logger.info(`Source: ${config.source.path}`);
+        }
+        else if (config.source.type === 'database') {
+            logger.info(`Source: database (${config.source.connectionString.split('@')[1] || 'unknown'})`);
+        }
+        else if (config.source.type === 'web') {
+            logger.info(`Source: web (${config.source.url})`);
+        }
+        const metadata = createDefaultMetadata(name, hasGraph ? 'graph-rag' : 'vector');
+        metadata.embeddingModel = config.embedding;
+        metadata.status = 'indexing';
+        await this.metadataManager.save(name, metadata);
+        onProgress?.({ name, phase: 'loading', progress: 0, message: 'Starting initialization...' });
+        const startTime = Date.now();
+        try {
+            // 1. Create embeddings and test to get dimensions
+            const embeddings = KnowledgeStore.createEmbeddings(config.embedding);
+            const testEmbedding = await embeddings.embedQuery('dimension test');
+            const dimensions = testEmbedding.length;
+            logger.info(`Embedding dimensions: ${dimensions}`);
+            // 2. Determine DB path and open SqliteStore
+            const dbPath = path.join(this.workspaceRoot, '.knowledge-data', `${name}.db`);
+            // Check dimension mismatch
+            if (!SqliteStore.validateDimensions(dbPath, dimensions)) {
+                logger.warn(`Dimension mismatch for "${name}", will re-index`);
+                // Delete old DB
+                try {
+                    await fs.unlink(dbPath);
+                }
+                catch { /* may not exist */ }
+            }
+            const sqliteStore = new SqliteStore(dbPath, dimensions);
+            // 3. Check if DB already has valid data
+            const storedHashes = sqliteStore.getMeta('sourceHashes');
+            const currentHashes = await KnowledgeStore.computeFileHashes(config, this.workspaceRoot);
+            const currentHashStr = JSON.stringify(currentHashes);
+            const isUpToDate = sqliteStore.hasData() && storedHashes === currentHashStr;
+            if (isUpToDate) {
+                logger.info(`"${name}" restored from SQLite (${sqliteStore.getChunkCount()} chunks, ${sqliteStore.getEntityCount()} entities)`);
+                onProgress?.({ name, phase: 'done', progress: 100, message: 'Restored from SQLite' });
+                metadata.documentCount = sqliteStore.getChunkCount();
+                metadata.chunkCount = sqliteStore.getChunkCount();
+                metadata.entityCount = sqliteStore.getEntityCount();
+                metadata.edgeCount = sqliteStore.getRelationshipCount();
+                metadata.sourceHashes = currentHashes;
+            }
+            else {
+                // Full indexing pipeline
+                sqliteStore.clear();
+                onProgress?.({ name, phase: 'loading', progress: 10, message: 'Loading documents...' });
+                const documents = await KnowledgeStore.loadDocuments(config, this.workspaceRoot);
+                logger.info(`Loaded ${documents.length} document(s)`);
+                metadata.documentCount = documents.length;
+                onProgress?.({ name, phase: 'splitting', progress: 20, message: `Splitting ${documents.length} documents...` });
+                const splitDocs = await KnowledgeStore.splitDocuments(config, documents);
+                logger.info(`Split into ${splitDocs.length} chunk(s)`);
+                metadata.chunkCount = splitDocs.length;
+                // Embed and insert chunks
+                onProgress?.({ name, phase: 'embedding', progress: 35, message: `Embedding ${splitDocs.length} chunks...` });
+                const chunkTexts = splitDocs.map(d => d.pageContent);
+                const chunkEmbeddings = await embeddings.embedDocuments(chunkTexts);
+                const chunkInserts = splitDocs.map((doc, i) => ({
+                    content: doc.pageContent,
+                    metadata: doc.metadata,
+                    source: doc.metadata?.source ?? '',
+                    embedding: chunkEmbeddings[i],
+                }));
+                onProgress?.({ name, phase: 'building', progress: 55, message: 'Inserting chunks...' });
+                sqliteStore.insertChunks(chunkInserts);
+                // Entity extraction (if graph config present)
+                if (config.graph) {
+                    onProgress?.({ name, phase: 'extracting', progress: 60, message: 'Extracting entities...' });
+                    if (!config.graph.directMapping) {
+                        throw new Error(`Graph config for "${name}" requires a directMapping configuration`);
+                    }
+                    const { entities, relationships } = DirectMapper.mapQueryResults(documents, config.graph.directMapping);
+                    logger.info(`Extracted ${entities.length} entities, ${relationships.length} relationships`);
+                    if (entities.length > 0) {
+                        // Embed entities
+                        onProgress?.({ name, phase: 'embedding', progress: 75, message: `Embedding ${entities.length} entities...` });
+                        const entityDescs = entities.map(e => `${e.name}: ${e.description}`);
+                        const entityEmbeddings = await embeddings.embedDocuments(entityDescs);
+                        const entityInserts = entities.map((entity, i) => ({
+                            id: normalizeId(entity.name, entity.type),
+                            type: entity.type,
+                            name: entity.name,
+                            description: entity.description,
+                            properties: entity.properties,
+                            sourceChunkIds: entity.properties.sourceChunkIds ?? [],
+                            embedding: entityEmbeddings[i],
+                        }));
+                        onProgress?.({ name, phase: 'building', progress: 85, message: 'Inserting entities and relationships...' });
+                        sqliteStore.insertEntities(entityInserts);
+                        // Build and insert relationships
+                        const relInserts = buildRelationships(relationships, entityInserts.map(e => ({ id: e.id, name: e.name, type: e.type })));
+                        if (relInserts.length > 0) {
+                            sqliteStore.insertRelationships(relInserts);
+                        }
+                    }
+                    metadata.entityCount = entities.length;
+                    metadata.edgeCount = relationships.length;
+                }
+                // Store source hashes
+                sqliteStore.setMeta('sourceHashes', currentHashStr);
+                metadata.sourceHashes = currentHashes;
+                onProgress?.({ name, phase: 'caching', progress: 95, message: 'Finalizing...' });
+            }
+            // Register stores
+            this.sqliteStores.set(name, sqliteStore);
+            // Create the KnowledgeStoreInstance
+            const instance = this.createInstance(name, config, sqliteStore, embeddings, metadata);
+            this.stores.set(name, instance);
+            // Update metadata on success
+            const storeMetadata = instance.getMetadata();
+            storeMetadata.status = 'indexed';
+            storeMetadata.lastIndexedAt = new Date().toISOString();
+            storeMetadata.lastIndexDurationMs = Date.now() - startTime;
+            storeMetadata.errorMessage = null;
+            await this.metadataManager.save(name, storeMetadata);
+            onProgress?.({ name, phase: 'done', progress: 100, message: 'Initialization complete' });
+            logger.info(`"${name}" initialized successfully (${storeMetadata.lastIndexDurationMs}ms)`);
+            return instance;
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            metadata.status = 'error';
+            metadata.errorMessage = errorMessage;
+            metadata.lastIndexDurationMs = Date.now() - startTime;
+            await this.metadataManager.save(name, metadata);
+            onProgress?.({ name, phase: 'error', progress: 0, message: errorMessage });
+            logger.error(`Failed to initialize "${name}":`, error);
+            throw error;
+        }
+    }
+    createInstance(name, config, sqliteStore, embeddings, metadata) {
+        return {
+            config,
+            search: async (query, k) => {
+                const numResults = k ?? config.search?.defaultK ?? 4;
+                searchLogger.info(`Searching "${name}" for: "${query.substring(0, 50)}..." (k=${numResults})`);
+                try {
+                    const queryEmbedding = await embeddings.embedQuery(query);
+                    // Chunk similarity search
+                    const chunkResults = sqliteStore.searchChunks(queryEmbedding, numResults);
+                    // Entity neighborhood search (if entities exist)
+                    const entityResults = [];
+                    if (sqliteStore.getEntityCount() > 0) {
+                        const topEntities = sqliteStore.searchEntities(queryEmbedding, Math.min(numResults, 5));
+                        for (const entity of topEntities) {
+                            const { entities, relationships } = sqliteStore.getNeighborhood(entity.id, 2);
+                            const neighborText = formatNeighborhood(entity, entities, relationships);
+                            entityResults.push({
+                                content: neighborText,
+                                metadata: { type: 'entity_neighborhood', entityId: entity.id, entityName: entity.name },
+                                score: entity.score,
+                            });
+                        }
+                    }
+                    // Merge all results by score, return top k
+                    const allResults = [
+                        ...chunkResults.map(r => ({
+                            content: r.content,
+                            metadata: r.metadata,
+                            score: r.score,
+                        })),
+                        ...entityResults,
+                    ];
+                    allResults.sort((a, b) => b.score - a.score);
+                    const threshold = config.search?.scoreThreshold;
+                    const filtered = threshold !== undefined
+                        ? allResults.filter(r => r.score >= threshold)
+                        : allResults;
+                    const final = filtered.slice(0, numResults);
+                    searchLogger.info(`Results: ${chunkResults.length} chunks, ${entityResults.length} entities → ${final.length} total`);
+                    return final;
+                }
+                catch (error) {
+                    const errorMessage = error instanceof Error ? error.message : String(error);
+                    searchLogger.error(`Error during search: ${errorMessage}`, error);
+                    return [];
+                }
+            },
+            addDocuments: async (docs) => {
+                const texts = docs.map(d => d.content);
+                const vectors = await embeddings.embedDocuments(texts);
+                sqliteStore.insertChunks(docs.map((d, i) => ({
+                    content: d.content,
+                    metadata: d.metadata ?? {},
+                    source: '',
+                    embedding: vectors[i],
+                })));
+                metadata.chunkCount = sqliteStore.getChunkCount();
+            },
+            refresh: async (refreshOnProgress) => {
+                refreshOnProgress?.({ name, phase: 'loading', progress: 10, message: 'Checking for changes...' });
+                const newSourceHashes = await KnowledgeStore.computeFileHashes(config, this.workspaceRoot);
+                const currentHashStr = JSON.stringify(newSourceHashes);
+                const storedHashes = sqliteStore.getMeta('sourceHashes');
+                if (currentHashStr === storedHashes) {
+                    logger.info(`No changes detected for "${name}", skipping refresh`);
+                    return;
+                }
+                logger.info(`Changes detected for "${name}", re-indexing...`);
+                // Remove current store references, re-initialize
+                this.stores.delete(name);
+                this.sqliteStores.delete(name);
+                sqliteStore.close();
+                // Delete old DB file to force full re-index
+                try {
+                    await fs.unlink(sqliteStore.getDbPath());
+                }
+                catch { /* */ }
+                await this.doInitialize(name, config, refreshOnProgress);
+            },
+            getMetadata: () => ({ ...metadata }),
+        };
+    }
+    // --- Public Accessors ---
+    get(name) {
+        return this.stores.get(name);
+    }
+    getConfig(name) {
+        return this.configs.get(name);
+    }
+    getSqliteStore(name) {
+        return this.sqliteStores.get(name);
+    }
+    list() {
+        return Array.from(this.stores.values());
+    }
+    listConfigs() {
+        return Array.from(this.configs.values());
+    }
+    async getStatus(name) {
+        return this.metadataManager.load(name);
+    }
+    async getAllStatuses() {
+        const names = Array.from(this.configs.keys());
+        return this.metadataManager.getAll(names);
+    }
+    getMetadataManager() {
+        return this.metadataManager;
+    }
+    isIndexing(name) {
+        return this.activeIndexing.has(name);
+    }
+    async refresh(name, onProgress) {
+        const store = this.stores.get(name);
+        if (!store)
+            return;
+        const config = this.configs.get(name);
+        if (config) {
+            const metadata = await this.metadataManager.load(name);
+            if (metadata) {
+                metadata.status = 'indexing';
+                await this.metadataManager.save(name, metadata);
+            }
+        }
+        const refreshPromise = (async () => {
+            const startTime = Date.now();
+            onProgress?.({ name, phase: 'loading', progress: 0, message: 'Starting refresh...' });
+            try {
+                await store.refresh(onProgress);
+                const storeMetadata = store.getMetadata();
+                storeMetadata.status = 'indexed';
+                storeMetadata.lastIndexedAt = new Date().toISOString();
+                storeMetadata.lastIndexDurationMs = Date.now() - startTime;
+                storeMetadata.errorMessage = null;
+                await this.metadataManager.save(name, storeMetadata);
+                onProgress?.({ name, phase: 'done', progress: 100, message: 'Refresh complete' });
+                return store;
+            }
+            catch (error) {
+                const errorMessage = error instanceof Error ? error.message : String(error);
+                const metadata = await this.metadataManager.load(name);
+                if (metadata) {
+                    metadata.status = 'error';
+                    metadata.errorMessage = errorMessage;
+                    await this.metadataManager.save(name, metadata);
+                }
+                onProgress?.({ name, phase: 'error', progress: 0, message: errorMessage });
+                throw error;
+            }
+        })();
+        this.activeIndexing.set(name, refreshPromise);
+        try {
+            await refreshPromise;
+        }
+        finally {
+            this.activeIndexing.delete(name);
+        }
+    }
+    // --- Static Helpers (reused from old KnowledgeStoreFactory) ---
+    static createEmbeddings(configName) {
+        const embeddingConfig = getEmbeddingConfig(configName);
+        const provider = detectProvider(embeddingConfig);
+        const eosToken = embeddingConfig.eosToken;
+        logger.info(`Embedding model: ${embeddingConfig.model} (provider: ${provider})${embeddingConfig.baseUrl ? `, URL: ${embeddingConfig.baseUrl}` : ''}`);
+        let baseEmbeddings;
+        switch (provider) {
+            case 'gemini':
+                baseEmbeddings = new GeminiEmbeddingsProvider({
+                    modelName: embeddingConfig.model,
+                    apiKey: resolveApiKey('gemini', embeddingConfig.apiKey),
+                });
+                break;
+            case 'openai':
+            case 'local':
+            case 'anthropic':
+            default: {
+                const apiKey = resolveApiKey(provider, embeddingConfig.apiKey);
+                baseEmbeddings = new OpenAIEmbeddingsProvider({
+                    modelName: embeddingConfig.model,
+                    apiKey,
+                    baseURL: embeddingConfig.baseUrl,
+                    dimensions: embeddingConfig.dimensions,
+                });
+                break;
+            }
+        }
+        return this.wrapWithValidation(baseEmbeddings, eosToken);
+    }
+    static wrapWithValidation(embeddings, eosToken) {
+        const appendToken = (text) => {
+            if (!eosToken || text.endsWith(eosToken))
+                return text;
+            return `${text}${eosToken}`;
+        };
+        const validateEmbedding = (result, context) => {
+            if (!Array.isArray(result) || result.length === 0) {
+                throw new Error(`${context}: Embedding returned invalid format`);
+            }
+            if (result.some(v => !isFinite(v))) {
+                throw new Error(`${context}: Embedding contains NaN or Infinity values`);
+            }
+            if (result.every(v => v === 0)) {
+                throw new Error(`${context}: Embedding returned a zero vector`);
+            }
+            return result;
+        };
+        const originalEmbedQuery = embeddings.embedQuery.bind(embeddings);
+        const originalEmbedDocuments = embeddings.embedDocuments.bind(embeddings);
+        return {
+            embedQuery: async (text) => {
+                try {
+                    return validateEmbedding(await originalEmbedQuery(appendToken(text)), 'embedQuery');
+                }
+                catch (error) {
+                    const msg = error instanceof Error ? error.message : String(error);
+                    throw new Error(`Embedding query failed: ${msg}`);
+                }
+            },
+            embedDocuments: async (texts) => {
+                try {
+                    const result = await originalEmbedDocuments(texts.map(appendToken));
+                    return result.map((emb, i) => validateEmbedding(emb, `embedDocuments[${i}]`));
+                }
+                catch (error) {
+                    const msg = error instanceof Error ? error.message : String(error);
+                    throw new Error(`Embedding documents failed: ${msg}`);
+                }
+            },
+        };
+    }
+    static async loadDocuments(config, workspaceRoot) {
+        if (config.source.type === 'database') {
+            const dbLoader = new DatabaseLoader(config.source);
+            return dbLoader.load();
+        }
+        if (config.source.type === 'web') {
+            const loaderType = config.loader?.type ?? 'html';
+            const webLoader = new WebLoader(config.source, loaderType);
+            return webLoader.load();
+        }
+        const loaderType = config.loader?.type ?? 'text';
+        const sourcePath = path.join(workspaceRoot, config.source.path);
+        if (config.source.type === 'directory') {
+            const pattern = config.source.pattern ?? '*';
+            const files = await glob(pattern, { cwd: sourcePath, absolute: true });
+            logger.info(`Found ${files.length} file(s) in ${sourcePath}`);
+            const allDocs = [];
+            for (const file of files) {
+                const loader = this.createLoader(loaderType, file);
+                const docs = await loader.load();
+                allDocs.push(...docs);
+            }
+            return allDocs;
+        }
+        if (config.source.type === 'file') {
+            const loader = this.createLoader(loaderType, sourcePath);
+            return loader.load();
+        }
+        throw new Error(`Unknown source type: ${config.source.type}`);
+    }
+    static createLoader(type, filePath) {
+        switch (type) {
+            case 'pdf': return new PDFLoader(filePath);
+            case 'csv': return new CSVLoader(filePath);
+            case 'json': return new JSONLoader(filePath);
+            case 'text':
+            case 'markdown':
+            default: return new TextLoader(filePath);
+        }
+    }
+    static async splitDocuments(config, documents) {
+        const splitterConfig = {
+            chunkSize: config.splitter.chunkSize,
+            chunkOverlap: config.splitter.chunkOverlap,
+            separator: config.splitter.separator,
+        };
+        const splitter = config.splitter.type === 'recursive'
+            ? new RecursiveCharacterTextSplitter(splitterConfig)
+            : new CharacterTextSplitter(splitterConfig);
+        return splitter.splitDocuments(documents);
+    }
+    static async computeFileHashes(config, workspaceRoot) {
+        const hashes = {};
+        if (config.source.type === 'directory') {
+            const sourcePath = path.join(workspaceRoot, config.source.path);
+            const pattern = config.source.pattern ?? '*';
+            const files = await glob(pattern, { cwd: sourcePath, absolute: true });
+            for (const file of files) {
+                const content = await fs.readFile(file);
+                hashes[file] = crypto.createHash('sha256').update(content).digest('hex');
+            }
+        }
+        else if (config.source.type === 'file') {
+            const sourcePath = path.join(workspaceRoot, config.source.path);
+            const content = await fs.readFile(sourcePath);
+            hashes[sourcePath] = crypto.createHash('sha256').update(content).digest('hex');
+        }
+        else if (config.source.type === 'database') {
+            hashes['database:query'] = crypto.createHash('sha256').update(config.source.query).digest('hex');
+        }
+        else if (config.source.type === 'web') {
+            hashes['web:url'] = crypto.createHash('sha256').update(config.source.url).digest('hex');
+        }
+        return hashes;
+    }
+}
+// --- Helpers ---
+function normalizeId(name, type) {
+    return `${type}::${name}`
+        .toLowerCase()
+        .replace(/[^a-z0-9:]+/g, '-')
+        .replace(/^-|-$/g, '');
+}
+function buildRelationships(relationships, entities) {
+    const nodeIdMap = new Map();
+    for (const entity of entities) {
+        const key = normalizeId(entity.name, entity.type);
+        nodeIdMap.set(key, entity.id);
+    }
+    const result = [];
+    for (let i = 0; i < relationships.length; i++) {
+        const rel = relationships[i];
+        const sourceKey = normalizeId(rel.sourceName, rel.sourceType);
+        const targetKey = normalizeId(rel.targetName, rel.targetType);
+        const sourceId = nodeIdMap.get(sourceKey);
+        const targetId = nodeIdMap.get(targetKey);
+        if (!sourceId || !targetId)
+            continue;
+        result.push({
+            id: `edge-${i}-${rel.type}`,
+            type: rel.type,
+            sourceId,
+            targetId,
+            description: rel.description,
+            weight: rel.weight,
+            properties: {},
+        });
+    }
+    return result;
+}
+function formatNeighborhood(entity, neighbors, relationships) {
+    const lines = [];
+    lines.push(`[${entity.type}] ${entity.name}: ${entity.description}`);
+    if (relationships.length > 0) {
+        lines.push('Relationships:');
+        for (const rel of relationships.slice(0, 10)) {
+            const other = rel.source_id === entity.id
+                ? neighbors.find(n => n.id === rel.target_id)
+                : neighbors.find(n => n.id === rel.source_id);
+            if (other) {
+                lines.push(`  -[${rel.type}]-> [${other.type}] ${other.name}`);
+            }
+        }
+    }
+    return lines.join('\n');
+}
+//# sourceMappingURL=knowledge-store.js.map