@openturtle/cli 0.3.2 → 0.3.4

package/dist/local-knowledge/index.js

@@ -0,0 +1,398 @@
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+import { localKnowledgeStoreRoot } from './root.js';
+const MAX_INDEXED_FILE_BYTES = 20 * 1024 * 1024;
+const MAX_CHUNK_CHARACTERS = 6_000;
+function indexPath(userDataPath = localKnowledgeStoreRoot()) {
+    return path.join(userDataPath, 'knowledge', 'index.db');
+}
+async function openIndex(userDataPath) {
+    const filePath = indexPath(userDataPath);
+    fs.mkdirSync(path.dirname(filePath), { recursive: true });
+    const importBuiltin = new Function('specifier', 'return import(specifier)');
+    const sqlite = await importBuiltin('node:sqlite');
+    if (!sqlite.DatabaseSync)
+        throw new Error('当前 Node.js 运行时不支持本地知识索引');
+    const db = new sqlite.DatabaseSync(filePath);
+    db.exec(`
+    PRAGMA busy_timeout = 5000;
+    PRAGMA journal_mode = WAL;
+    PRAGMA synchronous = NORMAL;
+    CREATE TABLE IF NOT EXISTS indexed_assets (
+      asset_id TEXT PRIMARY KEY,
+      content_hash TEXT NOT NULL,
+      index_status TEXT NOT NULL,
+      indexed_at TEXT NOT NULL
+    );
+    CREATE VIRTUAL TABLE IF NOT EXISTS knowledge_chunks_fts USING fts5(
+      asset_id UNINDEXED,
+      chunk_index UNINDEXED,
+      line_start UNINDEXED,
+      line_end UNINDEXED,
+      content,
+      tokenize='trigram'
+    );
+  `);
+    return db;
+}
+function isIndexableText(asset) {
+    return (asset.mimeType.startsWith('text/') ||
+        /\.(?:md|markdown|txt|log|csv|jsonl?|ya?ml|xml|html?|tsx?|jsx?|py|sh|sql|toml|ini|conf)$/i.test(asset.name));
+}
+function expectedIndexStatus(asset) {
+    if (asset.syncState === 'missing')
+        return 'missing';
+    if (!isIndexableText(asset) || asset.sizeBytes > MAX_INDEXED_FILE_BYTES)
+        return 'unsupported';
+    return 'indexed';
+}
+function indexFingerprint(asset) {
+    return crypto
+        .createHash('sha256')
+        .update(`${asset.contentHash}\n${asset.entry?.title || ''}\n${asset.entry?.kind || ''}\n${asset.entry?.projectId || ''}`)
+        .digest('hex');
+}
+function textChunks(content) {
+    const lines = content.replace(/\r\n/g, '\n').split('\n');
+    const chunks = [];
+    let start = 0;
+    while (start < lines.length) {
+        let end = start;
+        let size = 0;
+        while (end < lines.length) {
+            const nextSize = lines[end].length + 1;
+            if (end > start && size + nextSize > MAX_CHUNK_CHARACTERS)
+                break;
+            size += nextSize;
+            end += 1;
+        }
+        const chunk = lines.slice(start, end).join('\n').trim();
+        if (chunk)
+            chunks.push({ content: chunk, lineStart: start + 1, lineEnd: end });
+        start = end;
+    }
+    return chunks;
+}
+function prepareIndexStatements(db) {
+    return {
+        deleteChunks: db.prepare('DELETE FROM knowledge_chunks_fts WHERE asset_id = ?'),
+        deleteAsset: db.prepare('DELETE FROM indexed_assets WHERE asset_id = ?'),
+        insertChunk: db.prepare('INSERT INTO knowledge_chunks_fts(asset_id, chunk_index, line_start, line_end, content) VALUES (?, ?, ?, ?, ?)'),
+        upsertStatus: db.prepare(`INSERT INTO indexed_assets(asset_id, content_hash, index_status, indexed_at)
+       VALUES (?, ?, ?, ?)
+       ON CONFLICT(asset_id) DO UPDATE SET
+         content_hash = excluded.content_hash,
+         index_status = excluded.index_status,
+         indexed_at = excluded.indexed_at`),
+    };
+}
+function removeIndexedAsset(statements, assetId) {
+    statements.deleteChunks.run(assetId);
+    statements.deleteAsset.run(assetId);
+}
+function setIndexStatus(statements, asset, status) {
+    statements.upsertStatus.run(asset.id, indexFingerprint(asset), status, new Date().toISOString());
+}
+async function indexAsset(asset, existing, statements) {
+    if (existing?.contentHash === indexFingerprint(asset) && existing.indexStatus === expectedIndexStatus(asset))
+        return;
+    removeIndexedAsset(statements, asset.id);
+    if (asset.syncState === 'missing') {
+        setIndexStatus(statements, asset, 'missing');
+        return;
+    }
+    if (!isIndexableText(asset) || asset.sizeBytes > MAX_INDEXED_FILE_BYTES) {
+        setIndexStatus(statements, asset, 'unsupported');
+        return;
+    }
+    try {
+        const bytes = await fs.promises.readFile(asset.path);
+        if (bytes.includes(0)) {
+            setIndexStatus(statements, asset, 'unsupported');
+            return;
+        }
+        const content = bytes.toString('utf8');
+        const chunks = textChunks(asset.entry ? `${asset.entry.title}\n\n${content}` : content);
+        chunks.forEach((chunk, index) => statements.insertChunk.run(asset.id, index, chunk.lineStart, chunk.lineEnd, chunk.content));
+        setIndexStatus(statements, asset, 'indexed');
+    }
+    catch {
+        removeIndexedAsset(statements, asset.id);
+        setIndexStatus(statements, asset, 'failed');
+    }
+}
+export async function syncLocalKnowledgeIndex(assets, userDataPath) {
+    const db = await openIndex(userDataPath);
+    try {
+        const statements = prepareIndexStatements(db);
+        const assetIds = new Set(assets.map((asset) => asset.id));
+        const indexed = new Map();
+        const staleAssetIds = [];
+        for (const row of db.prepare('SELECT asset_id, content_hash, index_status FROM indexed_assets').all()) {
+            const assetId = String(row.asset_id || '');
+            if (!assetId)
+                continue;
+            if (!assetIds.has(assetId)) {
+                staleAssetIds.push(assetId);
+                continue;
+            }
+            indexed.set(assetId, {
+                contentHash: String(row.content_hash || ''),
+                indexStatus: String(row.index_status || ''),
+            });
+        }
+        const changedAssets = assets.filter((asset) => {
+            const existing = indexed.get(asset.id);
+            return existing?.contentHash !== indexFingerprint(asset) || existing.indexStatus !== expectedIndexStatus(asset);
+        });
+        if (staleAssetIds.length === 0 && changedAssets.length === 0)
+            return;
+        db.exec('BEGIN IMMEDIATE');
+        try {
+            for (const assetId of staleAssetIds)
+                removeIndexedAsset(statements, assetId);
+            for (const asset of changedAssets)
+                await indexAsset(asset, indexed.get(asset.id), statements);
+            db.exec('COMMIT');
+        }
+        catch (error) {
+            db.exec('ROLLBACK');
+            throw error;
+        }
+    }
+    finally {
+        db.close();
+    }
+}
+function excerpt(content, query, maxCharacters = 240) {
+    const normalized = content.toLocaleLowerCase('zh-CN');
+    const index = normalized.indexOf(query.toLocaleLowerCase('zh-CN'));
+    if (index < 0 && content.length <= maxCharacters)
+        return content;
+    const start = Math.max(0, index < 0 ? 0 : index - Math.floor(maxCharacters / 3));
+    const value = content
+        .slice(start, start + maxCharacters)
+        .replace(/\s+/g, ' ')
+        .trim();
+    return `${start > 0 ? '…' : ''}${value}${start + maxCharacters < content.length ? '…' : ''}`;
+}
+function searchQueries(value, matchMode) {
+    const values = Array.isArray(value) ? value : [value];
+    const queries = [];
+    const seen = new Set();
+    for (const candidate of values) {
+        const query = candidate.trim();
+        const key = query.toLocaleLowerCase('zh-CN');
+        if (!query || seen.has(key))
+            continue;
+        seen.add(key);
+        queries.push(query);
+        if (queries.length >= 12)
+            break;
+    }
+    return matchMode === 'exact' && queries.length > 1 ? [queries.join(' ')] : queries;
+}
+function escapedLikePattern(value) {
+    return `%${value.replace(/([%_\\])/g, '\\$1')}%`;
+}
+function lineNumberAt(content, index) {
+    return content.slice(0, Math.max(0, index)).split('\n').length;
+}
+export async function searchLocalKnowledgeIndex(assets, query, limit, userDataPath, matchMode = 'any') {
+    await syncLocalKnowledgeIndex(assets, userDataPath);
+    const queries = searchQueries(query, matchMode);
+    const normalizedQueries = queries.map((item) => item.toLocaleLowerCase('zh-CN'));
+    if (queries.length === 0) {
+        return assets.slice(0, limit).map((asset) => ({
+            assetId: asset.id,
+            matchedQueries: [],
+            matches: [],
+        }));
+    }
+    const db = await openIndex(userDataPath);
+    try {
+        const byAsset = new Map();
+        const stateFor = (assetId, index) => {
+            const existing = byAsset.get(assetId);
+            if (existing)
+                return existing;
+            const created = {
+                index,
+                labelQueries: new Set(),
+                matchedQueries: new Set(),
+                matches: [],
+            };
+            byAsset.set(assetId, created);
+            return created;
+        };
+        const assetIndex = new Map(assets.map((asset, index) => [asset.id, index]));
+        for (const [index, asset] of assets.entries()) {
+            const label = `${asset.name}\n${asset.relativePath}`.toLocaleLowerCase('zh-CN');
+            normalizedQueries.forEach((needle, queryIndex) => {
+                if (!label.includes(needle))
+                    return;
+                const state = stateFor(asset.id, index);
+                state.matchedQueries.add(queries[queryIndex]);
+                state.labelQueries.add(queries[queryIndex]);
+            });
+        }
+        const visibleAssets = new Set(assets.map((asset) => asset.id));
+        const candidateLimit = Math.min(2_000, Math.max(limit * 40, 200));
+        const longQueries = normalizedQueries.filter((item) => Array.from(item).length >= 3);
+        const shortQueries = normalizedQueries.filter((item) => Array.from(item).length < 3);
+        const rows = [];
+        if (longQueries.length > 0) {
+            const expression = longQueries.map((item) => `"${item.replaceAll('"', '""')}"`).join(' OR ');
+            rows.push(...db
+                .prepare(`SELECT asset_id, line_start, line_end, content
+             FROM knowledge_chunks_fts
+             WHERE knowledge_chunks_fts MATCH ?
+             ORDER BY bm25(knowledge_chunks_fts)
+             LIMIT ?`)
+                .all(expression, candidateLimit));
+        }
+        const globallySearchShortQueries = shortQueries.length > 0 && (matchMode !== 'all' || longQueries.length === 0);
+        if (globallySearchShortQueries) {
+            const predicates = shortQueries.map(() => "lower(content) LIKE ? ESCAPE '\\'").join(' OR ');
+            rows.push(...db
+                .prepare(`SELECT asset_id, line_start, line_end, content
+             FROM knowledge_chunks_fts
+             WHERE ${predicates}
+             LIMIT ?`)
+                .all(...shortQueries.map(escapedLikePattern), candidateLimit));
+        }
+        const seenMatches = new Set();
+        for (const row of rows) {
+            const assetId = String(row.asset_id || '');
+            if (!visibleAssets.has(assetId))
+                continue;
+            const content = String(row.content || '');
+            const normalizedContent = content.toLocaleLowerCase('zh-CN');
+            const matchedIndexes = normalizedQueries.flatMap((needle, index) => normalizedContent.includes(needle) ? [index] : []);
+            if (matchedIndexes.length === 0)
+                continue;
+            const state = stateFor(assetId, assetIndex.get(assetId) || 0);
+            for (const queryIndex of matchedIndexes) {
+                const matchedQuery = queries[queryIndex];
+                state.matchedQueries.add(matchedQuery);
+                const matchKey = `${assetId}:${row.line_start}:${row.line_end}:${matchedQuery}`;
+                if (state.matches.length >= 3 || seenMatches.has(matchKey))
+                    continue;
+                seenMatches.add(matchKey);
+                state.matches.push({
+                    query: matchedQuery,
+                    excerpt: excerpt(content, matchedQuery),
+                    lineStart: Number(row.line_start || 0),
+                    lineEnd: Number(row.line_end || 0),
+                });
+            }
+        }
+        if (matchMode === 'all' && longQueries.length > 0 && shortQueries.length > 0) {
+            const longOriginalQueries = queries.filter((_, index) => Array.from(normalizedQueries[index]).length >= 3);
+            for (const [assetId, state] of byAsset) {
+                if (!longOriginalQueries.every((item) => state.matchedQueries.has(item)))
+                    continue;
+                const asset = assets[assetIndex.get(assetId) ?? -1];
+                if (!asset || expectedIndexStatus(asset) !== 'indexed')
+                    continue;
+                let content;
+                try {
+                    content = fs.readFileSync(asset.path, 'utf8');
+                }
+                catch {
+                    continue;
+                }
+                const normalizedContent = content.toLocaleLowerCase('zh-CN');
+                normalizedQueries.forEach((needle, queryIndex) => {
+                    if (Array.from(needle).length >= 3 || !normalizedContent.includes(needle))
+                        return;
+                    const matchedQuery = queries[queryIndex];
+                    state.matchedQueries.add(matchedQuery);
+                    if (state.matches.length >= 3)
+                        return;
+                    const matchIndex = normalizedContent.indexOf(needle);
+                    const lineNumber = lineNumberAt(content, matchIndex);
+                    state.matches.push({
+                        query: matchedQuery,
+                        excerpt: excerpt(content, matchedQuery),
+                        lineStart: lineNumber,
+                        lineEnd: lineNumber,
+                    });
+                });
+            }
+        }
+        return [...byAsset.entries()]
+            .filter(([, state]) => matchMode === 'all' ? state.matchedQueries.size === queries.length : state.matchedQueries.size > 0)
+            .sort(([, left], [, right]) => {
+            const leftScore = left.matchedQueries.size * 100 + left.labelQueries.size * 10 + left.matches.length;
+            const rightScore = right.matchedQueries.size * 100 + right.labelQueries.size * 10 + right.matches.length;
+            return rightScore - leftScore || left.index - right.index;
+        })
+            .slice(0, limit)
+            .map(([assetId, state]) => ({
+            assetId,
+            matchedQueries: [...state.matchedQueries],
+            matches: state.matches,
+        }));
+    }
+    finally {
+        db.close();
+    }
+}
+export async function readLocalKnowledgeIndexedContent(asset, truncate, userDataPath) {
+    const db = await openIndex(userDataPath);
+    try {
+        const statements = prepareIndexStatements(db);
+        const existingRow = db
+            .prepare('SELECT content_hash, index_status FROM indexed_assets WHERE asset_id = ?')
+            .get(asset.id);
+        const existing = existingRow
+            ? {
+                contentHash: String(existingRow.content_hash || ''),
+                indexStatus: String(existingRow.index_status || ''),
+            }
+            : undefined;
+        if (existing?.contentHash !== indexFingerprint(asset) || existing.indexStatus !== expectedIndexStatus(asset)) {
+            db.exec('BEGIN IMMEDIATE');
+            try {
+                await indexAsset(asset, existing, statements);
+                db.exec('COMMIT');
+            }
+            catch (error) {
+                db.exec('ROLLBACK');
+                throw error;
+            }
+        }
+        const statusRow = db.prepare('SELECT index_status FROM indexed_assets WHERE asset_id = ?').get(asset.id);
+        const indexStatus = String(statusRow?.index_status || 'failed');
+        if (indexStatus !== 'indexed') {
+            return { content: null, contentAvailable: false, truncated: false, indexStatus };
+        }
+        if (asset.entry) {
+            const content = await fs.promises.readFile(asset.path, 'utf8');
+            return {
+                content: content.slice(0, truncate),
+                contentAvailable: true,
+                truncated: content.length > truncate,
+                indexStatus,
+            };
+        }
+        const rows = db
+            .prepare('SELECT content FROM knowledge_chunks_fts WHERE asset_id = ? ORDER BY CAST(chunk_index AS INTEGER)')
+            .all(asset.id);
+        const content = rows
+            .map((row) => String(row.content || ''))
+            .filter(Boolean)
+            .join('\n\n');
+        return {
+            content: content.slice(0, truncate),
+            contentAvailable: true,
+            truncated: content.length > truncate,
+            indexStatus,
+        };
+    }
+    finally {
+        db.close();
+    }
+}