@mars167/git-ai 2.3.0

package/dist/src/core/embedding.js

@@ -0,0 +1,36 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.hashEmbedding = hashEmbedding;
+const crypto_1 = require("./crypto");
+function tokenise(text) {
+    return (text || '')
+        .toLowerCase()
+        .split(/[^a-z0-9_]+/g)
+        .filter(Boolean);
+}
+function hashToUint32(hex) {
+    return parseInt(hex.slice(0, 8), 16) >>> 0;
+}
+function hashEmbedding(text, options) {
+    const dim = options.dim;
+    const vec = new Float32Array(dim);
+    const tokens = tokenise(text);
+    if (tokens.length === 0)
+        return Array.from(vec);
+    for (const t of tokens) {
+        const h = (0, crypto_1.sha256Hex)(t);
+        const u = hashToUint32(h);
+        const idx = u % dim;
+        const sign = (u & 1) === 0 ? 1 : -1;
+        vec[idx] += sign;
+    }
+    let norm = 0;
+    for (let i = 0; i < dim; i++)
+        norm += vec[i] * vec[i];
+    norm = Math.sqrt(norm);
+    if (norm > 0) {
+        for (let i = 0; i < dim; i++)
+            vec[i] = vec[i] / norm;
+    }
+    return Array.from(vec);
+}

package/dist/src/core/git.js

@@ -0,0 +1,49 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolveGitRoot = resolveGitRoot;
+exports.inferScanRoot = inferScanRoot;
+exports.inferWorkspaceRoot = inferWorkspaceRoot;
+exports.getCurrentCommitHash = getCurrentCommitHash;
+const simple_git_1 = __importDefault(require("simple-git"));
+const path_1 = __importDefault(require("path"));
+async function resolveGitRoot(startDir) {
+    const resolved = path_1.default.resolve(startDir);
+    try {
+        const git = (0, simple_git_1.default)(resolved);
+        const root = await git.raw(['rev-parse', '--show-toplevel']);
+        return root.trim();
+    }
+    catch {
+        return resolved;
+    }
+}
+function inferScanRoot(repoRoot) {
+    return path_1.default.resolve(repoRoot);
+}
+function inferWorkspaceRoot(repoRoot) {
+    const resolved = path_1.default.resolve(repoRoot);
+    const parts = resolved.split(path_1.default.sep).filter(Boolean);
+    const repoIdx = parts.lastIndexOf('.repo');
+    if (repoIdx >= 0 && repoIdx + 1 < parts.length) {
+        const next = parts[repoIdx + 1];
+        if (next === 'manifests' || next === 'manifests.git') {
+            const prefixParts = parts.slice(0, repoIdx);
+            const prefix = prefixParts.length === 0 ? path_1.default.parse(resolved).root : path_1.default.join(path_1.default.parse(resolved).root, ...prefixParts);
+            return prefix;
+        }
+    }
+    return null;
+}
+async function getCurrentCommitHash(repoRoot) {
+    try {
+        const git = (0, simple_git_1.default)(repoRoot);
+        const hash = await git.raw(['rev-parse', 'HEAD']);
+        return hash.trim();
+    }
+    catch {
+        return null;
+    }
+}

package/dist/src/core/gitDiff.js

@@ -0,0 +1,73 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getStagedNameStatus = getStagedNameStatus;
+exports.getWorktreeNameStatus = getWorktreeNameStatus;
+const simple_git_1 = __importDefault(require("simple-git"));
+function splitZ(raw) {
+    return raw.split('\0').map((s) => s.trim()).filter(Boolean);
+}
+async function getStagedNameStatus(repoRoot) {
+    const git = (0, simple_git_1.default)(repoRoot);
+    const raw = await git.raw(['diff', '--cached', '--name-status', '-z', '--find-renames']);
+    const parts = splitZ(raw);
+    const out = [];
+    for (let i = 0; i < parts.length; i++) {
+        const entry = parts[i] ?? '';
+        if (!entry)
+            continue;
+        const tabIdx = entry.indexOf('\t');
+        const statusRaw = tabIdx >= 0 ? entry.slice(0, tabIdx) : entry;
+        const rest = tabIdx >= 0 ? entry.slice(tabIdx + 1) : '';
+        const statusLetter = statusRaw[0] ?? '';
+        if (statusLetter === 'R') {
+            const oldPath = rest;
+            const newPath = parts[i + 1] ?? '';
+            i += 1;
+            if (oldPath && newPath)
+                out.push({ status: 'R', oldPath, path: newPath });
+            continue;
+        }
+        if (statusLetter === 'A' || statusLetter === 'M' || statusLetter === 'D') {
+            const p = rest;
+            if (p)
+                out.push({ status: statusLetter, path: p });
+            continue;
+        }
+    }
+    out.sort((a, b) => `${a.status}\t${a.oldPath ?? ''}\t${a.path}`.localeCompare(`${b.status}\t${b.oldPath ?? ''}\t${b.path}`));
+    return out;
+}
+async function getWorktreeNameStatus(repoRoot) {
+    const git = (0, simple_git_1.default)(repoRoot);
+    const raw = await git.raw(['diff', '--name-status', '-z', '--find-renames']);
+    const parts = splitZ(raw);
+    const out = [];
+    for (let i = 0; i < parts.length; i++) {
+        const entry = parts[i] ?? '';
+        if (!entry)
+            continue;
+        const tabIdx = entry.indexOf('\t');
+        const statusRaw = tabIdx >= 0 ? entry.slice(0, tabIdx) : entry;
+        const rest = tabIdx >= 0 ? entry.slice(tabIdx + 1) : '';
+        const statusLetter = statusRaw[0] ?? '';
+        if (statusLetter === 'R') {
+            const oldPath = rest;
+            const newPath = parts[i + 1] ?? '';
+            i += 1;
+            if (oldPath && newPath)
+                out.push({ status: 'R', oldPath, path: newPath });
+            continue;
+        }
+        if (statusLetter === 'A' || statusLetter === 'M' || statusLetter === 'D') {
+            const p = rest;
+            if (p)
+                out.push({ status: statusLetter, path: p });
+            continue;
+        }
+    }
+    out.sort((a, b) => `${a.status}\t${a.oldPath ?? ''}\t${a.path}`.localeCompare(`${b.status}\t${b.oldPath ?? ''}\t${b.path}`));
+    return out;
+}

package/dist/src/core/indexCheck.js

@@ -0,0 +1,131 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EXPECTED_INDEX_SCHEMA_VERSION = void 0;
+exports.resolveLangs = resolveLangs;
+exports.checkIndex = checkIndex;
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const path_1 = __importDefault(require("path"));
+const lancedb = __importStar(require("@lancedb/lancedb"));
+const lancedb_1 = require("./lancedb");
+const git_1 = require("./git");
+exports.EXPECTED_INDEX_SCHEMA_VERSION = 3;
+function requiredTablesForLang(lang) {
+    return [`chunks_${lang}`, `refs_${lang}`];
+}
+function resolveLangs(meta, selector) {
+    const raw = Array.isArray(meta?.languages) ? meta.languages.map((v) => String(v)) : [];
+    const filtered = raw.filter((l) => lancedb_1.ALL_INDEX_LANGS.includes(l));
+    const available = filtered.length > 0 ? filtered : ['java', 'ts'];
+    if (selector === 'all')
+        return available;
+    if (selector !== 'auto')
+        return available.includes(selector) ? [selector] : [];
+    const preferred = ['java', 'ts', 'python', 'go', 'rust', 'c', 'markdown', 'yaml'];
+    for (const lang of preferred) {
+        if (available.includes(lang))
+            return [lang];
+    }
+    return available.slice(0, 1);
+}
+async function checkIndex(repoRoot) {
+    const metaPath = path_1.default.join(repoRoot, '.git-ai', 'meta.json');
+    const lancedbDir = path_1.default.join(repoRoot, '.git-ai', 'lancedb');
+    const astGraphDbPath = path_1.default.join(repoRoot, '.git-ai', 'ast-graph.sqlite');
+    const problems = [];
+    const warnings = [];
+    const meta = await fs_extra_1.default.pathExists(metaPath) ? await fs_extra_1.default.readJSON(metaPath).catch(() => null) : null;
+    const currentCommitHash = await (0, git_1.getCurrentCommitHash)(repoRoot);
+    if (!meta) {
+        problems.push('missing_or_unreadable_meta_json');
+    }
+    else if (meta.index_schema_version !== exports.EXPECTED_INDEX_SCHEMA_VERSION) {
+        problems.push(`index_schema_version_mismatch(found=${String(meta.index_schema_version ?? 'null')}, expected=${exports.EXPECTED_INDEX_SCHEMA_VERSION})`);
+    }
+    // Check if index commit hash matches current HEAD
+    if (meta && currentCommitHash) {
+        if (meta.commit_hash && meta.commit_hash !== currentCommitHash) {
+            warnings.push(`index_commit_mismatch(index=${meta.commit_hash.slice(0, 7)}, head=${currentCommitHash.slice(0, 7)})`);
+        }
+        else if (!meta.commit_hash) {
+            warnings.push('index_commit_hash_not_recorded');
+        }
+    }
+    let lancedbTables;
+    if (await fs_extra_1.default.pathExists(lancedbDir)) {
+        try {
+            const db = await lancedb.connect(lancedbDir);
+            lancedbTables = await db.tableNames();
+        }
+        catch {
+            problems.push('lancedb_open_failed');
+        }
+    }
+    else {
+        problems.push('missing_lancedb_dir');
+    }
+    const astGraphDbExists = await fs_extra_1.default.pathExists(astGraphDbPath);
+    if (!astGraphDbExists)
+        problems.push('missing_ast_graph_db');
+    if (meta && lancedbTables) {
+        const raw = Array.isArray(meta.languages) ? meta.languages.map((v) => String(v)) : [];
+        const langs = raw.filter((l) => lancedb_1.ALL_INDEX_LANGS.includes(l));
+        const expectedTables = langs.flatMap(requiredTablesForLang);
+        for (const t of expectedTables) {
+            if (!lancedbTables.includes(t))
+                problems.push(`missing_lancedb_table(${t})`);
+        }
+    }
+    const ok = problems.length === 0;
+    return {
+        ok,
+        problems,
+        warnings,
+        expected: { index_schema_version: exports.EXPECTED_INDEX_SCHEMA_VERSION },
+        found: {
+            metaPath,
+            meta,
+            lancedbDir,
+            lancedbTables,
+            astGraphDbPath,
+            astGraphDbExists,
+            currentCommitHash,
+        },
+        hint: ok ? 'ok' : 'Rebuild index: git-ai ai index --overwrite',
+    };
+}

package/dist/src/core/indexer.js

@@ -0,0 +1,185 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IndexerV2 = void 0;
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const path_1 = __importDefault(require("path"));
+const glob_1 = require("glob");
+const config_1 = require("./indexing/config");
+const parallel_1 = require("./indexing/parallel");
+const lancedb_1 = require("./lancedb");
+const astGraph_1 = require("./astGraph");
+const git_1 = require("./git");
+async function loadIgnorePatterns(repoRoot, fileName) {
+    const ignorePath = path_1.default.join(repoRoot, fileName);
+    if (!await fs_extra_1.default.pathExists(ignorePath))
+        return [];
+    const raw = await fs_extra_1.default.readFile(ignorePath, 'utf-8');
+    return raw
+        .split('\n')
+        .map(l => l.trim())
+        .map((l) => {
+        if (l.length === 0)
+            return null;
+        if (l.startsWith('#'))
+            return null;
+        if (l.startsWith('!'))
+            return null;
+        const withoutLeadingSlash = l.startsWith('/') ? l.slice(1) : l;
+        if (withoutLeadingSlash.endsWith('/'))
+            return `${withoutLeadingSlash}**`;
+        return withoutLeadingSlash;
+    })
+        .filter((l) => Boolean(l));
+}
+function inferIndexLang(file) {
+    if (file.endsWith('.md') || file.endsWith('.mdx'))
+        return 'markdown';
+    if (file.endsWith('.yml') || file.endsWith('.yaml'))
+        return 'yaml';
+    if (file.endsWith('.java'))
+        return 'java';
+    if (file.endsWith('.c') || file.endsWith('.h'))
+        return 'c';
+    if (file.endsWith('.go'))
+        return 'go';
+    if (file.endsWith('.py'))
+        return 'python';
+    if (file.endsWith('.rs'))
+        return 'rust';
+    return 'ts';
+}
+class IndexerV2 {
+    constructor(options) {
+        this.repoRoot = path_1.default.resolve(options.repoRoot);
+        this.scanRoot = path_1.default.resolve(options.scanRoot ?? options.repoRoot);
+        this.dim = options.dim;
+        this.overwrite = options.overwrite;
+        this.onProgress = options.onProgress;
+        this.config = (0, config_1.mergeRuntimeConfig)(options.config);
+    }
+    async run() {
+        const gitAiDir = path_1.default.join(this.repoRoot, '.git-ai');
+        await fs_extra_1.default.ensureDir(gitAiDir);
+        const dbDir = (0, lancedb_1.defaultDbDir)(this.repoRoot);
+        const aiIgnore = await loadIgnorePatterns(this.repoRoot, '.aiignore');
+        const gitIgnore = await loadIgnorePatterns(this.repoRoot, '.gitignore');
+        const files = await (0, glob_1.glob)('**/*.{ts,tsx,js,jsx,java,c,h,go,py,rs,md,mdx,yml,yaml}', {
+            cwd: this.scanRoot,
+            nodir: true,
+            ignore: [
+                'node_modules/**',
+                '**/node_modules/**',
+                '.git/**',
+                '**/.git/**',
+                '.git-ai/**',
+                '**/.git-ai/**',
+                '.repo/**',
+                '**/.repo/**',
+                'dist/**',
+                'target/**',
+                '**/target/**',
+                'build/**',
+                '**/build/**',
+                '.gradle/**',
+                '**/.gradle/**',
+                ...aiIgnore,
+                ...gitIgnore,
+            ],
+        });
+        const languages = Array.from(new Set(files.map(inferIndexLang)));
+        const { byLang } = await (0, lancedb_1.openTablesByLang)({
+            dbDir,
+            dim: this.dim,
+            mode: this.overwrite ? 'overwrite' : 'create_if_missing',
+            languages,
+        });
+        const existingChunkIdsByLang = {};
+        if (!this.overwrite) {
+            for (const lang of languages) {
+                const t = byLang[lang];
+                if (!t)
+                    continue;
+                const set = new Set();
+                const existing = await t.chunks.query().select(['content_hash']).limit(1000000).toArray();
+                for (const row of existing) {
+                    const id = String(row.content_hash ?? '');
+                    if (id)
+                        set.add(id);
+                }
+                existingChunkIdsByLang[lang] = set;
+            }
+        }
+        const parallelResult = await (0, parallel_1.runParallelIndexing)({
+            repoRoot: this.repoRoot,
+            scanRoot: this.scanRoot,
+            dim: this.dim,
+            files,
+            indexing: this.config.indexing,
+            errorHandling: this.config.errorHandling,
+            existingChunkIdsByLang,
+            onProgress: this.onProgress,
+        });
+        const chunkRowsByLang = parallelResult.chunkRowsByLang;
+        const refRowsByLang = parallelResult.refRowsByLang;
+        const astFiles = parallelResult.astFiles;
+        const astSymbols = parallelResult.astSymbols;
+        const astContains = parallelResult.astContains;
+        const astExtendsName = parallelResult.astExtendsName;
+        const astImplementsName = parallelResult.astImplementsName;
+        const astRefsName = parallelResult.astRefsName;
+        const astCallsName = parallelResult.astCallsName;
+        const addedByLang = {};
+        for (const lang of languages) {
+            const t = byLang[lang];
+            if (!t)
+                continue;
+            const chunkRows = chunkRowsByLang[lang] ?? [];
+            const refRows = refRowsByLang[lang] ?? [];
+            if (chunkRows.length > 0)
+                await t.chunks.add(chunkRows);
+            if (refRows.length > 0)
+                await t.refs.add(refRows);
+            addedByLang[lang] = { chunksAdded: chunkRows.length, refsAdded: refRows.length };
+        }
+        const astGraph = await (0, astGraph_1.writeAstGraphToCozo)(this.repoRoot, {
+            files: astFiles,
+            symbols: astSymbols,
+            contains: astContains,
+            extends_name: astExtendsName,
+            implements_name: astImplementsName,
+            refs_name: astRefsName,
+            calls_name: astCallsName,
+        });
+        const commitHash = await (0, git_1.getCurrentCommitHash)(this.repoRoot);
+        const meta = {
+            version: '2.1',
+            index_schema_version: 3,
+            dim: this.dim,
+            files: files.length,
+            chunksAdded: Object.values(addedByLang).reduce((a, b) => a + b.chunksAdded, 0),
+            refsAdded: Object.values(addedByLang).reduce((a, b) => a + b.refsAdded, 0),
+            byLang: addedByLang,
+            languages,
+            dbDir: path_1.default.relative(this.repoRoot, dbDir),
+            scanRoot: path_1.default.relative(this.repoRoot, this.scanRoot),
+            ...(commitHash ? { commit_hash: commitHash } : {}),
+            astGraph: astGraph.enabled
+                ? {
+                    backend: 'cozo',
+                    engine: astGraph.engine,
+                    dbPath: astGraph.dbPath ? path_1.default.relative(this.repoRoot, astGraph.dbPath) : undefined,
+                    counts: astGraph.counts,
+                }
+                : {
+                    backend: 'cozo',
+                    enabled: false,
+                    skippedReason: astGraph.skippedReason,
+                },
+        };
+        await fs_extra_1.default.writeJSON(path_1.default.join(gitAiDir, 'meta.json'), meta, { spaces: 2 });
+    }
+}
+exports.IndexerV2 = IndexerV2;