@mars167/git-ai 2.3.0

package/dist/src/core/indexerIncremental.js

@@ -0,0 +1,303 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IncrementalIndexerV2 = void 0;
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const path_1 = __importDefault(require("path"));
+const simple_git_1 = __importDefault(require("simple-git"));
+const crypto_1 = require("./crypto");
+const lancedb_1 = require("./lancedb");
+const embedding_1 = require("./embedding");
+const sq8_1 = require("./sq8");
+const paths_1 = require("./paths");
+const astGraph_1 = require("./astGraph");
+const snapshotParser_1 = require("./dsr/snapshotParser");
+const git_1 = require("./git");
+function buildChunkText(file, symbol) {
+    return `file:${file}\nkind:${symbol.kind}\nname:${symbol.name}\nsignature:${symbol.signature}`;
+}
+function inferIndexLang(file) {
+    if (file.endsWith('.md') || file.endsWith('.mdx'))
+        return 'markdown';
+    if (file.endsWith('.yml') || file.endsWith('.yaml'))
+        return 'yaml';
+    if (file.endsWith('.java'))
+        return 'java';
+    if (file.endsWith('.c') || file.endsWith('.h'))
+        return 'c';
+    if (file.endsWith('.go'))
+        return 'go';
+    if (file.endsWith('.py'))
+        return 'python';
+    if (file.endsWith('.rs'))
+        return 'rust';
+    return 'ts';
+}
+function isIndexableFile(file) {
+    return /\.(ts|tsx|js|jsx|java|c|h|go|py|rs|md|mdx|yml|yaml)$/i.test(file);
+}
+function escapeQuotes(s) {
+    return s.replace(/'/g, "''");
+}
+async function readStagedFile(repoRoot, filePosix) {
+    const git = (0, simple_git_1.default)(repoRoot);
+    try {
+        return await git.raw(['show', `:${filePosix}`]);
+    }
+    catch {
+        return null;
+    }
+}
+async function readWorktreeFile(scanRoot, filePosix) {
+    const abs = path_1.default.join(scanRoot, filePosix);
+    try {
+        return await fs_extra_1.default.readFile(abs, 'utf-8');
+    }
+    catch {
+        return null;
+    }
+}
+async function deleteRefsForFile(byLang, filePosix) {
+    const safe = escapeQuotes(filePosix);
+    const pred = `file == '${safe}'`;
+    const langs = Object.keys(byLang);
+    for (const lang of langs) {
+        const t = byLang[lang];
+        if (!t)
+            continue;
+        if (typeof t.refs.delete === 'function') {
+            await t.refs.delete(pred);
+        }
+    }
+}
+class IncrementalIndexerV2 {
+    constructor(options) {
+        this.repoRoot = path_1.default.resolve(options.repoRoot);
+        this.scanRoot = path_1.default.resolve(options.scanRoot ?? options.repoRoot);
+        this.dim = options.dim;
+        this.source = options.source;
+        this.changes = options.changes;
+        this.onProgress = options.onProgress;
+        this.parser = new snapshotParser_1.SnapshotCodeParser();
+    }
+    async run() {
+        const gitAiDir = path_1.default.join(this.repoRoot, '.git-ai');
+        await fs_extra_1.default.ensureDir(gitAiDir);
+        const dbDir = (0, lancedb_1.defaultDbDir)(this.repoRoot);
+        const { byLang } = await (0, lancedb_1.openTablesByLang)({
+            dbDir,
+            dim: this.dim,
+            mode: 'create_if_missing',
+            languages: lancedb_1.ALL_INDEX_LANGS,
+        });
+        const chunkRowsByLang = {};
+        const refRowsByLang = {};
+        const candidateChunksByLang = {};
+        const astFiles = [];
+        const astSymbols = [];
+        const astContains = [];
+        const astExtendsName = [];
+        const astImplementsName = [];
+        const astRefsName = [];
+        const astCallsName = [];
+        const neededHashByLang = {};
+        const totalFiles = this.changes.length;
+        this.onProgress?.({ totalFiles, processedFiles: 0 });
+        let processed = 0;
+        for (const ch of this.changes) {
+            processed++;
+            const filePosix = (0, paths_1.toPosixPath)(ch.path);
+            this.onProgress?.({ totalFiles, processedFiles: processed, currentFile: filePosix });
+            if (ch.status === 'R' && ch.oldPath) {
+                const oldFile = (0, paths_1.toPosixPath)(ch.oldPath);
+                await deleteRefsForFile(byLang, oldFile);
+                await (0, astGraph_1.removeFileFromAstGraph)(this.repoRoot, oldFile);
+            }
+            if (ch.status === 'D') {
+                await deleteRefsForFile(byLang, filePosix);
+                await (0, astGraph_1.removeFileFromAstGraph)(this.repoRoot, filePosix);
+                continue;
+            }
+            await deleteRefsForFile(byLang, filePosix);
+            await (0, astGraph_1.removeFileFromAstGraph)(this.repoRoot, filePosix);
+            if (!isIndexableFile(filePosix))
+                continue;
+            const lang = inferIndexLang(filePosix);
+            if (!chunkRowsByLang[lang])
+                chunkRowsByLang[lang] = [];
+            if (!refRowsByLang[lang])
+                refRowsByLang[lang] = [];
+            if (!candidateChunksByLang[lang])
+                candidateChunksByLang[lang] = new Map();
+            if (!neededHashByLang[lang])
+                neededHashByLang[lang] = new Set();
+            const content = this.source === 'staged'
+                ? await readStagedFile(this.repoRoot, filePosix)
+                : await readWorktreeFile(this.scanRoot, filePosix);
+            if (content == null)
+                continue;
+            const parsed = this.parser.parseContent(filePosix, content);
+            const symbols = parsed.symbols;
+            const fileRefs = parsed.refs;
+            const fileId = (0, crypto_1.sha256Hex)(`file:${filePosix}`);
+            astFiles.push([fileId, filePosix, lang]);
+            const callableScopes = [];
+            for (const s of symbols) {
+                const text = buildChunkText(filePosix, s);
+                const contentHash = (0, crypto_1.sha256Hex)(text);
+                const refId = (0, crypto_1.sha256Hex)(`${filePosix}:${s.name}:${s.kind}:${s.startLine}:${s.endLine}:${contentHash}`);
+                astSymbols.push([refId, filePosix, lang, s.name, s.kind, s.signature, s.startLine, s.endLine]);
+                if (s.kind === 'function' || s.kind === 'method') {
+                    callableScopes.push({ refId, startLine: s.startLine, endLine: s.endLine });
+                }
+                let parentId = fileId;
+                if (s.container) {
+                    const cText = buildChunkText(filePosix, s.container);
+                    const cHash = (0, crypto_1.sha256Hex)(cText);
+                    parentId = (0, crypto_1.sha256Hex)(`${filePosix}:${s.container.name}:${s.container.kind}:${s.container.startLine}:${s.container.endLine}:${cHash}`);
+                }
+                astContains.push([parentId, refId]);
+                if (s.kind === 'class') {
+                    if (s.extends)
+                        for (const superName of s.extends)
+                            astExtendsName.push([refId, superName]);
+                    if (s.implements)
+                        for (const ifaceName of s.implements)
+                            astImplementsName.push([refId, ifaceName]);
+                }
+                neededHashByLang[lang].add(contentHash);
+                candidateChunksByLang[lang].set(contentHash, text);
+                const refRow = {
+                    ref_id: refId,
+                    content_hash: contentHash,
+                    file: filePosix,
+                    symbol: s.name,
+                    kind: s.kind,
+                    signature: s.signature,
+                    start_line: s.startLine,
+                    end_line: s.endLine,
+                };
+                refRowsByLang[lang].push(refRow);
+            }
+            const pickScope = (line) => {
+                let best = null;
+                for (const s of callableScopes) {
+                    if (line < s.startLine || line > s.endLine)
+                        continue;
+                    const span = s.endLine - s.startLine;
+                    if (!best || span < best.span)
+                        best = { refId: s.refId, span };
+                }
+                return best ? best.refId : fileId;
+            };
+            for (const r of fileRefs) {
+                const fromId = pickScope(r.line);
+                astRefsName.push([fromId, lang, r.name, r.refKind, filePosix, r.line, r.column]);
+                if (r.refKind === 'call' || r.refKind === 'new') {
+                    astCallsName.push([fromId, lang, r.name, filePosix, r.line, r.column]);
+                }
+            }
+        }
+        const existingChunkIdsByLang = {};
+        for (const lang of Object.keys(neededHashByLang)) {
+            const t = byLang[lang];
+            if (!t)
+                continue;
+            const needed = Array.from(neededHashByLang[lang] ?? []);
+            const existing = new Set();
+            for (let i = 0; i < needed.length; i += 400) {
+                const chunk = needed.slice(i, i + 400);
+                if (chunk.length === 0)
+                    continue;
+                const pred = `content_hash IN (${chunk.map((h) => `'${escapeQuotes(h)}'`).join(',')})`;
+                const rows = await t.chunks.query().where(pred).select(['content_hash']).limit(chunk.length).toArray();
+                for (const row of rows) {
+                    const id = String(row.content_hash ?? '');
+                    if (id)
+                        existing.add(id);
+                }
+            }
+            existingChunkIdsByLang[lang] = existing;
+        }
+        for (const lang of Object.keys(candidateChunksByLang)) {
+            const t = byLang[lang];
+            if (!t)
+                continue;
+            const existing = existingChunkIdsByLang[lang] ?? new Set();
+            const chunkRows = [];
+            const candidates = candidateChunksByLang[lang];
+            for (const [contentHash, text] of candidates.entries()) {
+                if (!contentHash || !text)
+                    continue;
+                if (existing.has(contentHash))
+                    continue;
+                const vec = (0, embedding_1.hashEmbedding)(text, { dim: this.dim });
+                const q = (0, sq8_1.quantizeSQ8)(vec);
+                chunkRows.push({
+                    content_hash: contentHash,
+                    text,
+                    dim: q.dim,
+                    scale: q.scale,
+                    qvec_b64: Buffer.from(q.q).toString('base64'),
+                });
+                existing.add(contentHash);
+            }
+            chunkRowsByLang[lang] = chunkRows;
+        }
+        const addedByLang = {};
+        for (const lang of lancedb_1.ALL_INDEX_LANGS) {
+            const t = byLang[lang];
+            if (!t)
+                continue;
+            const chunkRows = chunkRowsByLang[lang] ?? [];
+            const refRows = refRowsByLang[lang] ?? [];
+            if (chunkRows.length > 0)
+                await t.chunks.add(chunkRows);
+            if (refRows.length > 0)
+                await t.refs.add(refRows);
+            if (chunkRows.length > 0 || refRows.length > 0) {
+                addedByLang[lang] = { chunksAdded: chunkRows.length, refsAdded: refRows.length };
+            }
+        }
+        const astGraph = await (0, astGraph_1.writeAstGraphToCozo)(this.repoRoot, {
+            files: astFiles,
+            symbols: astSymbols,
+            contains: astContains,
+            extends_name: astExtendsName,
+            implements_name: astImplementsName,
+            refs_name: astRefsName,
+            calls_name: astCallsName,
+        }, { mode: 'put' });
+        const metaPath = path_1.default.join(gitAiDir, 'meta.json');
+        const prev = await fs_extra_1.default.readJSON(metaPath).catch(() => null);
+        const commitHash = await (0, git_1.getCurrentCommitHash)(this.repoRoot);
+        const meta = {
+            ...(prev && typeof prev === 'object' ? prev : {}),
+            version: '2.1',
+            index_schema_version: 3,
+            dim: this.dim,
+            dbDir: path_1.default.relative(this.repoRoot, dbDir),
+            scanRoot: path_1.default.relative(this.repoRoot, this.scanRoot),
+            languages: lancedb_1.ALL_INDEX_LANGS,
+            byLang: addedByLang,
+            ...(commitHash ? { commit_hash: commitHash } : {}),
+            astGraph: astGraph.enabled
+                ? {
+                    backend: 'cozo',
+                    engine: astGraph.engine,
+                    dbPath: astGraph.dbPath ? path_1.default.relative(this.repoRoot, astGraph.dbPath) : undefined,
+                    counts: astGraph.counts,
+                }
+                : {
+                    backend: 'cozo',
+                    enabled: false,
+                    skippedReason: astGraph.skippedReason,
+                },
+        };
+        await fs_extra_1.default.writeJSON(metaPath, meta, { spaces: 2 });
+        return { processed: this.changes.length, addedByLang };
+    }
+}
+exports.IncrementalIndexerV2 = IncrementalIndexerV2;

package/dist/src/core/indexing/config.js

@@ -0,0 +1,51 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.defaultIndexingConfig = defaultIndexingConfig;
+exports.defaultErrorHandlingConfig = defaultErrorHandlingConfig;
+exports.defaultIndexingRuntimeConfig = defaultIndexingRuntimeConfig;
+exports.mergeRuntimeConfig = mergeRuntimeConfig;
+const os_1 = __importDefault(require("os"));
+const hnsw_1 = require("./hnsw");
+function defaultIndexingConfig() {
+    const cpuCount = Math.max(1, os_1.default.cpus()?.length ?? 1);
+    return {
+        workerCount: Math.max(1, cpuCount - 1),
+        batchSize: 32,
+        memoryBudgetMb: 4096,
+        hnswConfig: (0, hnsw_1.clampHnswParameters)({
+            M: 16,
+            efConstruction: 200,
+            efSearch: 100,
+            quantizationBits: 8,
+        }),
+    };
+}
+function defaultErrorHandlingConfig() {
+    return {
+        parseFailureFallback: 'text-only',
+        largeFileThreshold: 1000000,
+        maxChunkSize: 10000,
+        memoryWarningThreshold: 0.8,
+        memoryCriticalThreshold: 0.95,
+    };
+}
+function defaultIndexingRuntimeConfig() {
+    return {
+        indexing: defaultIndexingConfig(),
+        errorHandling: defaultErrorHandlingConfig(),
+    };
+}
+function mergeRuntimeConfig(overrides) {
+    const defaults = defaultIndexingRuntimeConfig();
+    if (!overrides)
+        return defaults;
+    const merged = {
+        indexing: { ...defaults.indexing, ...overrides.indexing },
+        errorHandling: { ...defaults.errorHandling, ...overrides.errorHandling },
+    };
+    merged.indexing.hnswConfig = (0, hnsw_1.clampHnswParameters)(merged.indexing.hnswConfig);
+    return merged;
+}