@shrkcrft/graph 0.1.0-alpha.10

package/dist/indexer/incremental-updater.js

@@ -0,0 +1,395 @@
+import { createHash } from 'node:crypto';
+import { existsSync, readdirSync, statSync } from 'node:fs';
+import * as nodePath from 'node:path';
+import { execSync } from 'node:child_process';
+import { EdgeKind } from "../schema/edge-kind.js";
+import { NodeKind } from "../schema/node-kind.js";
+import { fingerprintFile } from "../store/file-fingerprint.js";
+import { GraphStore } from "../store/graph-store.js";
+import { summarizeCycles } from "../query/cycle-detection.js";
+import { summarizeUnresolvedImports } from "./unresolved-imports.js";
+import { detectWorkspacePackages, } from "./detect-workspace.js";
+import { EXTRACT_TS_FILE_SOURCE, extractTsFile, stitchPerFileReferences, } from "./extract-ts-file.js";
+import { extractPythonFile } from "./extract-python-file.js";
+import { extractGoFile } from "./extract-go-file.js";
+import { extractJavaFile } from "./extract-java-file.js";
+import { extractRustFile } from "./extract-rust-file.js";
+import { extractKotlinFile } from "./extract-kotlin-file.js";
+import { extractRubyFile } from "./extract-ruby-file.js";
+import { extractCsharpFile } from "./extract-csharp-file.js";
+import { extractElixirFile } from "./extract-elixir-file.js";
+import { extractPhpFile } from "./extract-php-file.js";
+import { extractDartFile } from "./extract-dart-file.js";
+import { extractSwiftFile } from "./extract-swift-file.js";
+import { createImportResolverContext, ImportResolution, resolveImport, } from "./resolve-imports.js";
+const SOURCE_EXTS = new Set([
+    '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.mts', '.cts',
+    '.vue', '.svelte', '.astro', '.py', '.go', '.java', '.rs', '.kt', '.kts',
+    '.rb', '.cs', '.csx', '.ex', '.exs', '.php', '.dart', '.swift',
+    '.graphql', '.gql',
+]);
+/**
+ * Apply a delta to the on-disk index.
+ *
+ * Strategy (MVP): load the snapshot, mutate in memory, rewrite the full
+ * store. Cheap for SharkCraft-sized indexes (a few MB on disk). Per-kind
+ * append/compact is the optimisation when the cold-rewrite cost is felt.
+ */
+export function updateChanged(options) {
+    const start = Date.now();
+    const { projectRoot } = options;
+    const store = new GraphStore(projectRoot);
+    if (!store.exists()) {
+        throw new Error(`code-graph store missing under ${store.storeDir}. Run 'shrk graph index' first.`);
+    }
+    const snap = store.loadSnapshot();
+    const nodes = new Map(snap.nodes);
+    const edges = new Map(snap.edges);
+    const files = new Map(snap.files);
+    const workspaces = detectWorkspacePackages(projectRoot);
+    const resolverCtx = createImportResolverContext(projectRoot, workspaces);
+    const packageDirIndex = buildPackageDirIndex(workspaces);
+    // Make sure the package nodes match the current workspace state. Adds
+    // new packages; doesn't drop existing ones (rare to lose a package
+    // mid-session).
+    for (const p of workspaces) {
+        const id = `package:${p.name}`;
+        if (!nodes.has(id)) {
+            nodes.set(id, {
+                id,
+                kind: NodeKind.Package,
+                label: p.name,
+                path: p.dir,
+                ...(p.entry ? { data: { entry: p.entry } } : {}),
+            });
+        }
+    }
+    const deleted = [];
+    for (const rel of options.deletedFiles ?? []) {
+        const fileId = `file:${rel}`;
+        if (!nodes.has(fileId))
+            continue;
+        removeFileFromGraph(rel, nodes, edges, files);
+        deleted.push(rel);
+    }
+    const updated = [];
+    const skipped = [];
+    // For each file we actually re-extract, retain extracted + resolved
+    // spec so the stitcher pass can produce reference / call edges.
+    const reExtracted = new Map();
+    for (const rel of options.changedFiles ?? []) {
+        const abs = nodePath.resolve(projectRoot, rel);
+        if (!existsSync(abs) || !isFile(abs)) {
+            // Treat as deletion if the file no longer exists.
+            if (nodes.has(`file:${rel}`)) {
+                removeFileFromGraph(rel, nodes, edges, files);
+                deleted.push(rel);
+            }
+            continue;
+        }
+        if (!SOURCE_EXTS.has(nodePath.extname(rel).toLowerCase()))
+            continue;
+        const newFp = fingerprintFile(abs, projectRoot);
+        const oldFp = files.get(newFp.path);
+        if (oldFp && oldFp.sha1 === newFp.sha1 && oldFp.mtime === newFp.mtime) {
+            skipped.push(newFp.path);
+            continue;
+        }
+        // Remove the file's previous contribution.
+        if (oldFp)
+            removeFileFromGraph(newFp.path, nodes, edges, files);
+        // Re-extract.
+        files.set(newFp.path, newFp);
+        const extracted = newFp.language === 'python' ? extractPythonFile(newFp, abs)
+            : newFp.language === 'go' ? extractGoFile(newFp, abs)
+                : newFp.language === 'java' ? extractJavaFile(newFp, abs)
+                    : newFp.language === 'rust' ? extractRustFile(newFp, abs)
+                        : newFp.language === 'kotlin' ? extractKotlinFile(newFp, abs)
+                            : newFp.language === 'ruby' ? extractRubyFile(newFp, abs)
+                                : newFp.language === 'csharp' ? extractCsharpFile(newFp, abs)
+                                    : newFp.language === 'elixir' ? extractElixirFile(newFp, abs)
+                                        : newFp.language === 'php' ? extractPhpFile(newFp, abs)
+                                            : newFp.language === 'dart' ? extractDartFile(newFp, abs)
+                                                : newFp.language === 'swift' ? extractSwiftFile(newFp, abs)
+                                                    : extractTsFile(newFp, abs);
+        nodes.set(extracted.fileNode.id, extracted.fileNode);
+        for (const sym of extracted.symbolNodes)
+            nodes.set(sym.id, sym);
+        for (const e of extracted.edges)
+            edges.set(e.id, e);
+        const pkg = findOwningPackage(newFp.path, packageDirIndex);
+        if (pkg) {
+            const e = buildEdge(newFp.nodeId, `package:${pkg.name}`, EdgeKind.BelongsToPackage, EXTRACT_TS_FILE_SOURCE);
+            edges.set(e.id, e);
+        }
+        const resolvedSpec = new Map();
+        for (const raw of extracted.rawImportSpecifiers) {
+            const r = resolveImport(raw.specifier, abs, resolverCtx);
+            resolvedSpec.set(raw.specifier, r.targetPath);
+            const data = {
+                specifier: r.specifier,
+                line: raw.line,
+                importKind: raw.kind,
+                resolutionKind: r.kind,
+            };
+            const targetId = r.targetPath
+                ? `file:${r.targetPath}`
+                : r.kind === ImportResolution.External
+                    ? `external:${r.specifier}`
+                    : `unresolved:${r.specifier}`;
+            const e = buildEdge(newFp.nodeId, targetId, EdgeKind.ImportsFile, EXTRACT_TS_FILE_SOURCE, data);
+            edges.set(e.id, e);
+        }
+        reExtracted.set(newFp.path, { extracted, resolvedSpec });
+        updated.push(newFp.path);
+    }
+    // Stitch references / calls for re-extracted files. Build the default
+    // export name map from the current node table (covers both
+    // re-extracted and untouched files).
+    if (reExtracted.size > 0) {
+        const defaultExportNameByPath = new Map();
+        for (const n of nodes.values()) {
+            if (n.kind !== NodeKind.File || !n.path)
+                continue;
+            defaultExportNameByPath.set(n.path, n.data?.['defaultExportName'] ?? undefined);
+        }
+        for (const [path, { extracted, resolvedSpec }] of reExtracted) {
+            const localNames = new Map();
+            for (const sym of extracted.symbolNodes)
+                localNames.set(sym.label, sym.id);
+            const refEdges = stitchPerFileReferences({
+                fileNodeId: extracted.fileNode.id,
+                extracted,
+                resolvedSpec,
+                defaultExportNameByPath,
+                localSymbolNamesInThisFile: localNames,
+            });
+            for (const e of refEdges)
+                edges.set(e.id, e);
+        }
+    }
+    // Rebuild package-depends-on aggregates from the (now-updated)
+    // imports-file edges. Drop the old ones first.
+    for (const id of [...edges.keys()]) {
+        const e = edges.get(id);
+        if (e.kind === EdgeKind.PackageDependsOn)
+            edges.delete(id);
+    }
+    for (const e of collectPackageDependsOn([...edges.values()], packageDirIndex)) {
+        edges.set(e.id, e);
+    }
+    const nodeList = [...nodes.values()];
+    const edgeList = [...edges.values()];
+    const cycles = summarizeCycles(nodeList, edgeList);
+    const unresolved = summarizeUnresolvedImports(edgeList);
+    const manifest = store.writeSnapshot(nodeList, edgeList, [...files.values()], {
+        projectRoot,
+        lastIndexedAt: new Date().toISOString(),
+        lastIndexDurationMs: Date.now() - start,
+        filesIndexed: files.size,
+        workspacePackages: workspaces.map((w) => w.name),
+        nodesByKind: {},
+        edgesByKind: {},
+        cycleCount: cycles.cycleCount,
+        largestCycleSize: cycles.largestCycleSize,
+        filesInCycles: cycles.filesInCycles,
+        unresolvedImportCount: unresolved.unresolvedImportCount,
+        filesWithUnresolvedImports: unresolved.filesWithUnresolvedImports,
+        unresolvedImportSamples: unresolved.unresolvedImportSamples,
+    });
+    return {
+        manifest,
+        durationMs: Date.now() - start,
+        updated,
+        deleted,
+        skipped,
+    };
+}
+/**
+ * Helper: detect changed files by walking the project and comparing
+ * fingerprints against the stored snapshot. Used by
+ * `shrk graph index --changed` when no explicit file list is provided.
+ */
+export function detectChangedAndDeleted(projectRoot) {
+    const store = new GraphStore(projectRoot);
+    if (!store.exists()) {
+        return { changed: [], deleted: [] };
+    }
+    const snap = store.loadSnapshot();
+    const seen = new Set();
+    const changed = [];
+    const fsStack = [projectRoot];
+    const skip = new Set([
+        'node_modules',
+        'dist',
+        'build',
+        'coverage',
+        '.git',
+        '.sharkcraft',
+        '.next',
+        '.cache',
+        '.tmp-pack',
+        'out',
+        'target',
+    ]);
+    while (fsStack.length > 0) {
+        const dir = fsStack.pop();
+        let entries;
+        try {
+            entries = readdirSync(dir);
+        }
+        catch {
+            continue;
+        }
+        for (const name of entries) {
+            if (skip.has(name))
+                continue;
+            if (name.startsWith('.') && name !== '.')
+                continue;
+            const full = nodePath.join(dir, name);
+            let st;
+            try {
+                st = statSync(full);
+            }
+            catch {
+                continue;
+            }
+            if (st.isDirectory()) {
+                fsStack.push(full);
+                continue;
+            }
+            if (!st.isFile())
+                continue;
+            if (!SOURCE_EXTS.has(nodePath.extname(full).toLowerCase()))
+                continue;
+            const rel = nodePath
+                .relative(projectRoot, full)
+                .split(nodePath.sep)
+                .join('/');
+            seen.add(rel);
+            const oldFp = snap.files.get(rel);
+            if (!oldFp) {
+                changed.push(rel);
+                continue;
+            }
+            // Cheap check: mtime first. If equal, trust it (assumes mtime
+            // bumps on writes — true on local FS, sometimes wrong inside
+            // bind mounts). Otherwise recompute the sha and compare.
+            if (Math.floor(st.mtimeMs) === oldFp.mtime && st.size === oldFp.sizeBytes)
+                continue;
+            const newFp = fingerprintFile(full, projectRoot);
+            if (newFp.sha1 !== oldFp.sha1)
+                changed.push(rel);
+        }
+    }
+    const deleted = [];
+    for (const path of snap.files.keys()) {
+        if (!seen.has(path))
+            deleted.push(path);
+    }
+    return { changed, deleted };
+}
+/**
+ * Get the list of files changed since a git ref (e.g. `main`, `HEAD~5`,
+ * a tag). Returns project-relative POSIX paths. Errors → empty list.
+ */
+export function changedFilesSince(projectRoot, ref) {
+    try {
+        const raw = execSync(`git diff --name-only ${ref}`, {
+            cwd: projectRoot,
+            encoding: 'utf8',
+            stdio: ['ignore', 'pipe', 'ignore'],
+        });
+        return raw
+            .split('\n')
+            .map((s) => s.trim())
+            .filter((s) => s.length > 0);
+    }
+    catch {
+        return [];
+    }
+}
+// ── helpers ───────────────────────────────────────────────────────────
+function removeFileFromGraph(rel, nodes, edges, files) {
+    const fileId = `file:${rel}`;
+    const symbolPrefix = `symbol:${rel}#`;
+    files.delete(rel);
+    // Drop the file node + all symbol nodes declared in this file.
+    for (const id of [...nodes.keys()]) {
+        if (id === fileId || id.startsWith(symbolPrefix)) {
+            nodes.delete(id);
+        }
+    }
+    // Drop every edge that touches this file or any of its symbols.
+    for (const id of [...edges.keys()]) {
+        const e = edges.get(id);
+        if (e.from === fileId ||
+            e.to === fileId ||
+            e.from.startsWith(symbolPrefix) ||
+            e.to.startsWith(symbolPrefix)) {
+            edges.delete(id);
+        }
+    }
+}
+function isFile(p) {
+    try {
+        return statSync(p).isFile();
+    }
+    catch {
+        return false;
+    }
+}
+function buildPackageDirIndex(packages) {
+    const entries = [...packages]
+        .map((p) => ({ dir: p.dir.replace(/\/+$/, ''), name: p.name }))
+        .sort((a, b) => b.dir.length - a.dir.length);
+    return { entries };
+}
+function findOwningPackage(filePath, index) {
+    for (const e of index.entries) {
+        if (filePath === e.dir || filePath.startsWith(e.dir + '/'))
+            return e;
+    }
+    return undefined;
+}
+function collectPackageDependsOn(allEdges, index) {
+    const pairs = new Map();
+    for (const e of allEdges) {
+        if (e.kind !== EdgeKind.ImportsFile)
+            continue;
+        const fromFile = stripPrefix(e.from, 'file:');
+        const toFile = stripPrefix(e.to, 'file:');
+        if (!fromFile || !toFile)
+            continue;
+        const fromPkg = findOwningPackage(fromFile, index);
+        const toPkg = findOwningPackage(toFile, index);
+        if (!fromPkg || !toPkg)
+            continue;
+        if (fromPkg.name === toPkg.name)
+            continue;
+        const k = `${fromPkg.name}|${toPkg.name}`;
+        const cur = pairs.get(k);
+        if (cur)
+            cur.count += 1;
+        else
+            pairs.set(k, { from: fromPkg.name, to: toPkg.name, count: 1 });
+    }
+    const out = [];
+    for (const { from, to, count } of pairs.values()) {
+        out.push(buildEdge(`package:${from}`, `package:${to}`, EdgeKind.PackageDependsOn, 'incremental-updater@v1', { count }));
+    }
+    return out;
+}
+function buildEdge(from, to, kind, source, data) {
+    return {
+        id: createHash('sha1').update(`${from}|${to}|${kind}`).digest('hex'),
+        from,
+        to,
+        kind,
+        source,
+        ...(data ? { data } : {}),
+    };
+}
+function stripPrefix(id, prefix) {
+    return id.startsWith(prefix) ? id.slice(prefix.length) : undefined;
+}

package/dist/indexer/index-builder.d.ts

@@ -0,0 +1,23 @@
+import type { IGraphManifest } from '../schema/manifest.js';
+export interface IIndexBuilderOptions {
+    projectRoot: string;
+    /** Override the ignore set. Adds to defaults; does not replace them. */
+    extraIgnore?: readonly string[];
+    /** Cap the number of files indexed. Useful for tests; 0 = unlimited. */
+    maxFiles?: number;
+}
+export interface IFullIndexResult {
+    manifest: IGraphManifest;
+    durationMs: number;
+}
+/**
+ * Build a full graph index and write it to disk. Overwrites any
+ * pre-existing store under `<root>/.sharkcraft/graph/`.
+ *
+ * Single-process, no worker pool yet. The compiler-API per-file extractor
+ * is fast enough at SharkCraft's size; worker-pool parallelism is a
+ * later optimisation tied to measured budgets (see code-intelligence.md
+ * §7).
+ */
+export declare function buildFullIndex(options: IIndexBuilderOptions): IFullIndexResult;
+//# sourceMappingURL=index-builder.d.ts.map

package/dist/indexer/index-builder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index-builder.d.ts","sourceRoot":"","sources":["../../src/indexer/index-builder.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AA6D5D,MAAM,WAAW,oBAAoB;IACnC,WAAW,EAAE,MAAM,CAAC;IACpB,wEAAwE;IACxE,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAChC,wEAAwE;IACxE,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,cAAc,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,oBAAoB,GAAG,gBAAgB,CA0I9E"}