@opencodehub/ingestion 0.1.0

package/dist/pipeline/phases/scan.d.ts

@@ -0,0 +1,54 @@
+/**
+ * Scan phase — walks the repository, emits one record per kept file.
+ *
+ * Responsibilities:
+ *  - Honor hardcoded directory ignores (`node_modules`, `.git`, etc.) and
+ *    the repository-root `.gitignore` file.
+ *  - Skip binary files (first 8 KB contains NUL → heuristic binary).
+ *  - Skip files exceeding `options.byteCapPerFile` (default 10 MB).
+ *  - Cap total file count at `options.maxTotalFiles` (default 100k).
+ *  - Compute `sha256` for each kept file (enables future incremental skip).
+ *  - Optionally capture `git rev-parse HEAD` when not in `skipGit` mode.
+ *
+ * This phase reads but does not write the graph. Later phases build nodes
+ * and edges from the scan output.
+ */
+import type { LanguageId } from "../../parse/types.js";
+import type { PipelinePhase } from "../types.js";
+/** One row of the scan phase output. */
+export interface ScannedFile {
+    readonly absPath: string;
+    /** POSIX-separated path relative to repo root. */
+    readonly relPath: string;
+    readonly byteSize: number;
+    /** Lowercase hex sha256 of file contents. */
+    readonly sha256: string;
+    /** Detected language, if any; consumers may skip unknown files. */
+    readonly language?: LanguageId;
+    /**
+     * Content-addressed grammar fingerprint for the detected language — the
+     * second half of theparse-cache composite key. `null` when the
+     * language is unknown, its grammar package is not installed, or its
+     * `package.json` could not be read. The field participates in no sort
+     * (it is a pure function of `language`), so it does not affect scan
+     * determinism beyond what the existing `relPath` sort already gives.
+     */
+    readonly grammarSha: string | null;
+}
+export interface ScanOutput {
+    readonly files: readonly ScannedFile[];
+    /** `undefined` when `skipGit` is set or the repo is not a git checkout. */
+    readonly gitHead?: string;
+    readonly totalBytes: number;
+    /**
+     * Paths (relative to `repoPath`, POSIX-separated, no trailing slash) of every
+     * git submodule registered in this repo. Populated via `git ls-tree -r -z
+     * HEAD` filtered for tree-entry mode `160000` (the Linguist canonical
+     * pattern). Falls back to textual parsing of `.gitmodules` when `skipGit`
+     * is set. Empty when the repo has no submodules or is not a git checkout.
+     */
+    readonly submodulePaths: readonly string[];
+}
+export declare const SCAN_PHASE_NAME = "scan";
+export declare const scanPhase: PipelinePhase<ScanOutput>;
+//# sourceMappingURL=scan.d.ts.map

package/dist/pipeline/phases/scan.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scan.d.ts","sourceRoot":"","sources":["../../../src/pipeline/phases/scan.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAQH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAOvD,OAAO,KAAK,EAAmB,aAAa,EAAE,MAAM,aAAa,CAAC;AAMlE,wCAAwC;AACxC,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,kDAAkD;IAClD,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,6CAA6C;IAC7C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,mEAAmE;IACnE,QAAQ,CAAC,QAAQ,CAAC,EAAE,UAAU,CAAC;IAC/B;;;;;;;OAOG;IACH,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,KAAK,EAAE,SAAS,WAAW,EAAE,CAAC;IACvC,2EAA2E;IAC3E,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B;;;;;;OAMG;IACH,QAAQ,CAAC,cAAc,EAAE,SAAS,MAAM,EAAE,CAAC;CAC5C;AAED,eAAO,MAAM,eAAe,SAAS,CAAC;AAEtC,eAAO,MAAM,SAAS,EAAE,aAAa,CAAC,UAAU,CAM/C,CAAC"}

package/dist/pipeline/phases/scan.js

@@ -0,0 +1,340 @@
+/**
+ * Scan phase — walks the repository, emits one record per kept file.
+ *
+ * Responsibilities:
+ *  - Honor hardcoded directory ignores (`node_modules`, `.git`, etc.) and
+ *    the repository-root `.gitignore` file.
+ *  - Skip binary files (first 8 KB contains NUL → heuristic binary).
+ *  - Skip files exceeding `options.byteCapPerFile` (default 10 MB).
+ *  - Cap total file count at `options.maxTotalFiles` (default 100k).
+ *  - Compute `sha256` for each kept file (enables future incremental skip).
+ *  - Optionally capture `git rev-parse HEAD` when not in `skipGit` mode.
+ *
+ * This phase reads but does not write the graph. Later phases build nodes
+ * and edges from the scan output.
+ */
+import { spawn } from "node:child_process";
+import { promises as fs } from "node:fs";
+import path from "node:path";
+import { sha256Hex } from "@opencodehub/core-types";
+import { getGrammarSha } from "../../parse/grammar-registry.js";
+import { detectLanguage } from "../../parse/language-detector.js";
+import { HARDCODED_IGNORES, loadGitignoreChain, shouldIgnore, } from "../gitignore.js";
+const DEFAULT_BYTE_CAP_PER_FILE = 10 * 1024 * 1024; // 10 MB
+const DEFAULT_MAX_TOTAL_FILES = 100_000;
+const BINARY_PROBE_BYTES = 8 * 1024; // 8 KB — same probe git itself uses.
+export const SCAN_PHASE_NAME = "scan";
+export const scanPhase = {
+    name: SCAN_PHASE_NAME,
+    deps: [],
+    async run(ctx) {
+        return runScan(ctx);
+    },
+};
+async function runScan(ctx) {
+    const byteCapPerFile = ctx.options.byteCapPerFile ?? DEFAULT_BYTE_CAP_PER_FILE;
+    const maxTotalFiles = ctx.options.maxTotalFiles ?? DEFAULT_MAX_TOTAL_FILES;
+    // Layered gitignore chain — nested `.gitignore` files stack from repo
+    // root downward; deeper layers override shallower ones.
+    const chain = await loadGitignoreChain(ctx.repoPath);
+    const hardcoded = new Set(HARDCODED_IGNORES);
+    const collected = [];
+    let totalBytes = 0;
+    let capHit = false;
+    await walk(ctx.repoPath, "", {
+        chain,
+        hardcoded,
+        byteCapPerFile,
+        maxTotalFiles,
+        collected,
+        onTotalBytes: (b) => {
+            totalBytes += b;
+        },
+        onCapHit: () => {
+            capHit = true;
+        },
+        onWarn: (msg) => {
+            ctx.onProgress?.({ phase: SCAN_PHASE_NAME, kind: "warn", message: msg });
+        },
+    });
+    if (capHit && ctx.onProgress) {
+        ctx.onProgress({
+            phase: SCAN_PHASE_NAME,
+            kind: "warn",
+            message: `scan: reached maxTotalFiles=${maxTotalFiles}; remaining files were skipped`,
+        });
+    }
+    // Deterministic output: sort by relPath ascending.
+    collected.sort((a, b) => (a.relPath < b.relPath ? -1 : a.relPath > b.relPath ? 1 : 0));
+    let gitHead;
+    let submodulePaths = [];
+    if (ctx.options.skipGit !== true) {
+        gitHead = await tryGitHead(ctx.repoPath);
+        submodulePaths = await listGitSubmodules(ctx.repoPath);
+    }
+    else {
+        submodulePaths = await parseGitmodulesSubmodules(ctx.repoPath);
+    }
+    return {
+        files: collected,
+        ...(gitHead !== undefined ? { gitHead } : {}),
+        totalBytes,
+        submodulePaths,
+    };
+}
+async function walk(repoRoot, relDir, p) {
+    if (p.collected.length >= p.maxTotalFiles) {
+        p.onCapHit();
+        return;
+    }
+    const absDir = path.join(repoRoot, relDir);
+    let entries;
+    try {
+        entries = await fs.readdir(absDir, { withFileTypes: true });
+    }
+    catch (err) {
+        p.onWarn(`scan: cannot read directory ${absDir}: ${err.message}`);
+        return;
+    }
+    // Sort by name so walk order is deterministic.
+    entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+    for (const entry of entries) {
+        if (p.collected.length >= p.maxTotalFiles) {
+            p.onCapHit();
+            return;
+        }
+        const name = entry.name;
+        if (p.hardcoded.has(name))
+            continue;
+        const relPath = relDir === "" ? name : `${relDir}/${name}`;
+        const isDir = entry.isDirectory();
+        const ignored = shouldIgnore(relPath, p.chain, { isDirectory: isDir });
+        if (ignored)
+            continue;
+        if (isDir) {
+            await walk(repoRoot, relPath, p);
+            continue;
+        }
+        // Skip symlinks and non-regular files — they are rare in source repos
+        // and their semantics are ambiguous for a content-addressed hash.
+        if (!entry.isFile())
+            continue;
+        const absPath = path.join(absDir, name);
+        // Open once and stat through the handle so the size check and the read
+        // operate on the same file descriptor — eliminates the TOCTOU window
+        // (js/file-system-race) that a path-based `stat` then `readFile` opens.
+        let buf;
+        let handle;
+        try {
+            handle = await fs.open(absPath, "r");
+            const stat = await handle.stat();
+            if (stat.size > p.byteCapPerFile) {
+                p.onWarn(`scan: skipping ${relPath} (${stat.size} bytes > cap ${p.byteCapPerFile})`);
+                continue;
+            }
+            buf = await handle.readFile();
+        }
+        catch (err) {
+            p.onWarn(`scan: cannot read ${absPath}: ${err.message}`);
+            continue;
+        }
+        finally {
+            if (handle !== undefined)
+                await handle.close().catch(() => undefined);
+        }
+        if (looksBinary(buf))
+            continue;
+        const sha256 = sha256Of(buf);
+        const language = detectLanguage(relPath, firstLine(buf));
+        // Resolve the content-addressed grammar fingerprint now so downstream
+        // phases (parse, content-cache) do not need to re-touch the grammar
+        // registry per file. `getGrammarSha` is memoized per-process, so the
+        // per-file cost is a single Map lookup after the first resolution per
+        // language. Null when the language is unknown or the grammar package
+        // is not installed — cache lookups simply miss in that case.
+        const grammarSha = language !== undefined ? await getGrammarSha(language) : null;
+        p.collected.push({
+            absPath,
+            relPath,
+            byteSize: buf.byteLength,
+            sha256,
+            ...(language !== undefined ? { language } : {}),
+            grammarSha,
+        });
+        p.onTotalBytes(buf.byteLength);
+    }
+}
+function looksBinary(buf) {
+    const probeLen = Math.min(buf.byteLength, BINARY_PROBE_BYTES);
+    for (let i = 0; i < probeLen; i += 1) {
+        if (buf[i] === 0)
+            return true;
+    }
+    return false;
+}
+function sha256Of(buf) {
+    // Normalize to a Uint8Array — the `@types/node` Buffer shape shipped
+    // with older @types/node declarations is not assignable to the crypto
+    // `BinaryLike` constraint on newer Node runtimes.
+    return sha256Hex(new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength));
+}
+function firstLine(buf) {
+    const probeLen = Math.min(buf.byteLength, 256);
+    const slice = buf.subarray(0, probeLen).toString("utf8");
+    const nl = slice.indexOf("\n");
+    if (nl === -1) {
+        if (probeLen < buf.byteLength)
+            return undefined;
+        return slice;
+    }
+    return slice.slice(0, nl);
+}
+/**
+ * Enumerate submodule paths by asking git for the tree at HEAD and filtering
+ * for gitlink entries (mode `160000`). This is the canonical pattern GitHub
+ * Linguist uses — it works on bare repos, detached worktrees, and
+ * partially-initialised submodules alike (does not depend on `.git/config`
+ * being populated via `git submodule init`).
+ *
+ * Output paths are POSIX-separated relative paths, no trailing slash.
+ * Returns an empty array when `git` is unavailable, the repo is not a git
+ * checkout, or there are no submodules.
+ */
+async function listGitSubmodules(repoPath) {
+    return new Promise((resolveProm) => {
+        let stdout = "";
+        let settled = false;
+        const child = spawn("git", ["ls-tree", "-r", "-z", "HEAD"], {
+            cwd: repoPath,
+            stdio: ["ignore", "pipe", "ignore"],
+        });
+        child.stdout.setEncoding("utf8");
+        child.stdout.on("data", (chunk) => {
+            stdout += chunk;
+        });
+        child.on("error", () => {
+            if (!settled) {
+                settled = true;
+                resolveProm([]);
+            }
+        });
+        child.on("close", (code) => {
+            if (settled)
+                return;
+            settled = true;
+            if (code !== 0) {
+                resolveProm([]);
+                return;
+            }
+            resolveProm(parseLsTreeSubmodules(stdout));
+        });
+    });
+}
+/**
+ * Parse the NUL-delimited output of `git ls-tree -r -z HEAD`. Each record is
+ * `<mode> SP <type> SP <sha> TAB <path>`. We keep records whose mode is
+ * exactly `160000` (gitlink) — that is the submodule marker in git's tree
+ * format. All other entries (blobs, trees, symlinks) are dropped.
+ */
+function parseLsTreeSubmodules(raw) {
+    const out = [];
+    for (const record of raw.split("\0")) {
+        if (record.length === 0)
+            continue;
+        const tabIdx = record.indexOf("\t");
+        if (tabIdx < 0)
+            continue;
+        const header = record.slice(0, tabIdx);
+        const path = record.slice(tabIdx + 1);
+        // header: "<mode> <type> <sha>"
+        const firstSpace = header.indexOf(" ");
+        if (firstSpace < 0)
+            continue;
+        const mode = header.slice(0, firstSpace);
+        if (mode !== "160000")
+            continue;
+        out.push(normalizeSubmodulePath(path));
+    }
+    return out;
+}
+/**
+ * Textual `.gitmodules` fallback, used when `skipGit` is true and we cannot
+ * spawn `git`. The format is an INI-like file with stanzas like:
+ *
+ *     [submodule "fixtures/python/sample"]
+ *         path = fixtures/python/sample
+ *         url = https://github.com/...
+ *
+ * We tolerate leading whitespace and trailing CR bytes; unrecognised lines
+ * are ignored. Missing file → empty list.
+ */
+async function parseGitmodulesSubmodules(repoPath) {
+    let raw;
+    try {
+        raw = await fs.readFile(path.join(repoPath, ".gitmodules"), "utf8");
+    }
+    catch {
+        return [];
+    }
+    const out = [];
+    for (const line of raw.split(/\r?\n/)) {
+        const trimmed = line.trim();
+        if (!trimmed.startsWith("path"))
+            continue;
+        const eqIdx = trimmed.indexOf("=");
+        if (eqIdx < 0)
+            continue;
+        const key = trimmed.slice(0, eqIdx).trim();
+        if (key !== "path")
+            continue;
+        const value = trimmed.slice(eqIdx + 1).trim();
+        if (value.length === 0)
+            continue;
+        out.push(normalizeSubmodulePath(value));
+    }
+    return out;
+}
+function normalizeSubmodulePath(p) {
+    // Normalise to POSIX separators and strip any trailing slash; never emit a
+    // leading "./" prefix. `git ls-tree` already emits POSIX-separated paths,
+    // but `.gitmodules` can contain arbitrary text — be defensive.
+    let out = p.replace(/\\/g, "/");
+    while (out.endsWith("/"))
+        out = out.slice(0, -1);
+    if (out.startsWith("./"))
+        out = out.slice(2);
+    return out;
+}
+async function tryGitHead(repoPath) {
+    return new Promise((resolve) => {
+        let stdout = "";
+        let settled = false;
+        const child = spawn("git", ["rev-parse", "HEAD"], {
+            cwd: repoPath,
+            stdio: ["ignore", "pipe", "ignore"],
+        });
+        child.stdout.setEncoding("utf8");
+        child.stdout.on("data", (chunk) => {
+            stdout += chunk;
+        });
+        child.on("error", () => {
+            if (!settled) {
+                settled = true;
+                resolve(undefined);
+            }
+        });
+        child.on("close", (code) => {
+            if (settled)
+                return;
+            settled = true;
+            if (code === 0) {
+                const trimmed = stdout.trim();
+                resolve(trimmed.length > 0 ? trimmed : undefined);
+            }
+            else {
+                resolve(undefined);
+            }
+        });
+    });
+}
+//# sourceMappingURL=scan.js.map

package/dist/pipeline/phases/scan.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scan.js","sourceRoot":"","sources":["../../../src/pipeline/phases/scan.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAC;AAElE,OAAO,EACL,iBAAiB,EAEjB,kBAAkB,EAClB,YAAY,GACb,MAAM,iBAAiB,CAAC;AAGzB,MAAM,yBAAyB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,QAAQ;AAC5D,MAAM,uBAAuB,GAAG,OAAO,CAAC;AACxC,MAAM,kBAAkB,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,qCAAqC;AAsC1E,MAAM,CAAC,MAAM,eAAe,GAAG,MAAM,CAAC;AAEtC,MAAM,CAAC,MAAM,SAAS,GAA8B;IAClD,IAAI,EAAE,eAAe;IACrB,IAAI,EAAE,EAAE;IACR,KAAK,CAAC,GAAG,CAAC,GAAG;QACX,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC;IACtB,CAAC;CACF,CAAC;AAEF,KAAK,UAAU,OAAO,CAAC,GAAoB;IACzC,MAAM,cAAc,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,IAAI,yBAAyB,CAAC;IAC/E,MAAM,aAAa,GAAG,GAAG,CAAC,OAAO,CAAC,aAAa,IAAI,uBAAuB,CAAC;IAE3E,sEAAsE;IACtE,wDAAwD;IACxD,MAAM,KAAK,GAAG,MAAM,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAErD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAS,iBAAiB,CAAC,CAAC;IACrD,MAAM,SAAS,GAAkB,EAAE,CAAC;IACpC,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,MAAM,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,EAAE;QAC3B,KAAK;QACL,SAAS;QACT,cAAc;QACd,aAAa;QACb,SAAS;QACT,YAAY,EAAE,CAAC,CAAC,EAAE,EAAE;YAClB,UAAU,IAAI,CAAC,CAAC;QAClB,CAAC;QACD,QAAQ,EAAE,GAAG,EAAE;YACb,MAAM,GAAG,IAAI,CAAC;QAChB,CAAC;QACD,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE;YACd,GAAG,CAAC,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QAC3E,CAAC;KACF,CAAC,CAAC;IAEH,IAAI,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;QAC7B,GAAG,CAAC,UAAU,CAAC;YACb,KAAK,EAAE,eAAe;YACtB,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE,+BAA+B,aAAa,gCAAgC;SACtF,CAAC,CAAC;IACL,CAAC;IAED,mDAAmD;IACnD,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEvF,IAAI,OAA2B,CAAC;IAChC,IAAI,cAAc,GAAsB,EAAE,CAAC;IAC3C,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;QACjC,OAAO,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACzC,cAAc,GAAG,MAAM,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACzD,CAAC;SAAM,CAAC;QACN,cAAc,GAAG,MAAM,yBAAyB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACjE,CAAC;IAED,OAAO;QACL,KAAK,EAAE,SAAS;QAChB,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7C,UAAU;QACV,cAAc;KACf,CAAC;AACJ,CAAC;AAaD,KAAK,UAAU,IAAI,CAAC,QAAgB,EAAE,MAAc,EAAE,CAAa;IACjE,IAAI,CAAC,CAAC,SAAS,CAAC,MAAM,IAAI,CAAC,CAAC,aAAa,EAAE,CAAC;QAC1C,CAAC,CAAC,QAAQ,EAAE,CAAC;QACb,OAAO;IACT,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC3C,IAAI,OAAmC,CAAC;IACxC,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9D,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,CAAC,CAAC,MAAM,CAAC,+BAA+B,MAAM,KAAM,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7E,OAAO;IACT,CAAC;IAED,+CAA+C;IAC/C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEzE,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,CAAC,SAAS,CAAC,MAAM,IAAI,CAAC,CAAC,aAAa,EAAE,CAAC;YAC1C,CAAC,CAAC,QAAQ,EAAE,CAAC;YACb,OAAO;QACT,CAAC;QACD,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;QACxB,IAAI,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,SAAS;QAEpC,MAAM,OAAO,GAAG,MAAM,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,IAAI,EAAE,CAAC;QAC3D,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;QACvE,IAAI,OAAO;YAAE,SAAS;QAEtB,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;YACjC,SAAS;QACX,CAAC;QAED,sEAAsE;QACtE,kEAAkE;QAClE,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE;YAAE,SAAS;QAE9B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACxC,uEAAuE;QACvE,qEAAqE;QACrE,wEAAwE;QACxE,IAAI,GAAW,CAAC;QAChB,IAAI,MAAyD,CAAC;QAC9D,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YACrC,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,cAAc,EAAE,CAAC;gBACjC,CAAC,CAAC,MAAM,CAAC,kBAAkB,OAAO,KAAK,IAAI,CAAC,IAAI,gBAAgB,CAAC,CAAC,cAAc,GAAG,CAAC,CAAC;gBACrF,SAAS;YACX,CAAC;YACD,GAAG,GAAG,MAAM,MAAM,CAAC,QAAQ,EAAE,CAAC;QAChC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,CAAC,CAAC,MAAM,CAAC,qBAAqB,OAAO,KAAM,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;YACpE,SAAS;QACX,CAAC;gBAAS,CAAC;YACT,IAAI,MAAM,KAAK,SAAS;gBAAE,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACxE,CAAC;QAED,IAAI,WAAW,CAAC,GAAG,CAAC;YAAE,SAAS;QAE/B,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;QAC7B,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;QAEzD,sEAAsE;QACtE,oEAAoE;QACpE,qEAAqE;QACrE,sEAAsE;QACtE,qEAAqE;QACrE,6DAA6D;QAC7D,MAAM,UAAU,GAAG,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEjF,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC;YACf,OAAO;YACP,OAAO;YACP,QAAQ,EAAE,GAAG,CAAC,UAAU;YACxB,MAAM;YACN,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/C,UAAU;SACX,CAAC,CAAC;QACH,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,GAAW;IAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAC;IAC9D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACrC,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;IAChC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,QAAQ,CAAC,GAAW;IAC3B,qEAAqE;IACrE,sEAAsE;IACtE,kDAAkD;IAClD,OAAO,SAAS,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;AAC/E,CAAC;AAED,SAAS,SAAS,CAAC,GAAW;IAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;IAC/C,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACzD,MAAM,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC;QACd,IAAI,QAAQ,GAAG,GAAG,CAAC,UAAU;YAAE,OAAO,SAAS,CAAC;QAChD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC5B,CAAC;AAED;;;;;;;;;;GAUG;AACH,KAAK,UAAU,iBAAiB,CAAC,QAAgB;IAC/C,OAAO,IAAI,OAAO,CAAC,CAAC,WAAW,EAAE,EAAE;QACjC,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,EAAE;YAC1D,GAAG,EAAE,QAAQ;YACb,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC;SACpC,CAAC,CAAC;QACH,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACjC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;YAChC,MAAM,IAAI,KAAK,CAAC;QAClB,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACrB,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO,GAAG,IAAI,CAAC;gBACf,WAAW,CAAC,EAAE,CAAC,CAAC;YAClB,CAAC;QACH,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;YACzB,IAAI,OAAO;gBAAE,OAAO;YACpB,OAAO,GAAG,IAAI,CAAC;YACf,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBACf,WAAW,CAAC,EAAE,CAAC,CAAC;gBAChB,OAAO;YACT,CAAC;YACD,WAAW,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,SAAS,qBAAqB,CAAC,GAAW;IACxC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACrC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAClC,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,MAAM,GAAG,CAAC;YAAE,SAAS;QACzB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QACvC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACtC,gCAAgC;QAChC,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACvC,IAAI,UAAU,GAAG,CAAC;YAAE,SAAS;QAC7B,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;QACzC,IAAI,IAAI,KAAK,QAAQ;YAAE,SAAS;QAChC,GAAG,CAAC,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;;;GAUG;AACH,KAAK,UAAU,yBAAyB,CAAC,QAAgB;IACvD,IAAI,GAAW,CAAC;IAChB,IAAI,CAAC;QACH,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,EAAE,MAAM,CAAC,CAAC;IACtE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QACtC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC;YAAE,SAAS;QAC1C,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,KAAK,GAAG,CAAC;YAAE,SAAS;QACxB,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3C,IAAI,GAAG,KAAK,MAAM;YAAE,SAAS;QAC7B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC9C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QACjC,GAAG,CAAC,IAAI,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,sBAAsB,CAAC,CAAS;IACvC,2EAA2E;IAC3E,0EAA0E;IAC1E,+DAA+D;IAC/D,IAAI,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAChC,OAAO,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACjD,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC7C,OAAO,GAAG,CAAC;AACb,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,QAAgB;IACxC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,EAAE;YAChD,GAAG,EAAE,QAAQ;YACb,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC;SACpC,CAAC,CAAC;QACH,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACjC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;YAChC,MAAM,IAAI,KAAK,CAAC;QAClB,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACrB,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO,GAAG,IAAI,CAAC;gBACf,OAAO,CAAC,SAAS,CAAC,CAAC;YACrB,CAAC;QACH,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;YACzB,IAAI,OAAO;gBAAE,OAAO;YACpB,OAAO,GAAG,IAAI,CAAC;YACf,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBACf,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC9B,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YACpD,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,SAAS,CAAC,CAAC;YACrB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/pipeline/phases/scip-index.d.ts

@@ -0,0 +1,54 @@
+/**
+ * `scip-index` phase — replaces the four per-language LSP upgrade
+ * phases (lsp-python / lsp-typescript / lsp-go / lsp-rust) with a
+ * single pass over SCIP (https://scip-code.org) indexes.
+ *
+ * For every language detected in the profile we:
+ *   1. Run the appropriate SCIP indexer into
+ *      `.codehub/scip/<lang>.scip`, unless the artifact is fresh
+ *      (mtime newer than every source file for that language).
+ *   2. Parse the index with `@opencodehub/scip-ingest`, derive caller
+ *      -> callee edges via innermost-enclosing-range attribution.
+ *   3. Map each SCIP call site (document, line) back to the tightest
+ *      OpenCodeHub symbol node via the same file+line lookup the LSP
+ *      phases used.
+ *   4. Emit CodeRelation edges with `confidence = 1.0` and
+ *      `reason = scip:<indexer>@<version>` so the downstream
+ *      `confidence-demote`, `summarize`, `mcp/confidence`, and
+ *      `cli/analyze` consumers keep treating SCIP edges as oracle-
+ *      confirmed (see `SCIP_PROVENANCE_PREFIXES`).
+ *
+ * Skip semantics:
+ *   - `CODEHUB_DISABLE_SCIP=1`       -> entire phase no-op.
+ *   - Indexer binary missing         -> per-language skip with a warn
+ *                                       ProgressEvent; tree-sitter
+ *                                       heuristic tier keeps its
+ *                                       low-confidence edges.
+ *   - `options.offline && !cached`   -> per-language skip (no network
+ *                                       install path).
+ */
+import type { GraphNode } from "@opencodehub/core-types";
+import type { IndexerKind } from "@opencodehub/scip-ingest";
+import type { PipelinePhase } from "../types.js";
+export declare const SCIP_INDEX_PHASE_NAME = "scip-index";
+export interface ScipIndexPerLanguage {
+    readonly kind: IndexerKind;
+    readonly tool: string;
+    readonly version: string;
+    readonly skipped: boolean;
+    readonly skipReason?: string;
+    readonly scipPath: string;
+    readonly edgesAdded: number;
+    readonly edgesUpgraded: number;
+}
+export interface ScipIndexOutput {
+    readonly enabled: boolean;
+    readonly skippedReason?: string;
+    readonly languages: readonly ScipIndexPerLanguage[];
+    readonly totalEdgesAdded: number;
+    readonly totalEdgesUpgraded: number;
+    readonly durationMs: number;
+}
+export declare const scipIndexPhase: PipelinePhase<ScipIndexOutput>;
+export type _NodeShape = GraphNode;
+//# sourceMappingURL=scip-index.d.ts.map

package/dist/pipeline/phases/scip-index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scip-index.d.ts","sourceRoot":"","sources":["../../../src/pipeline/phases/scip-index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAKH,OAAO,KAAK,EAAE,SAAS,EAAU,MAAM,yBAAyB,CAAC;AACjE,OAAO,KAAK,EAGV,WAAW,EAGZ,MAAM,0BAA0B,CAAC;AAUlC,OAAO,KAAK,EAAmB,aAAa,EAAE,MAAM,aAAa,CAAC;AAOlE,eAAO,MAAM,qBAAqB,eAAe,CAAC;AAIlD,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;CAChC;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,SAAS,EAAE,SAAS,oBAAoB,EAAE,CAAC;IACpD,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED,eAAO,MAAM,cAAc,EAAE,aAAa,CAAC,eAAe,CAYzD,CAAC;AA4dF,MAAM,MAAM,UAAU,GAAG,SAAS,CAAC"}