cgraphx 2.0.2 → 2.0.6

package/dist/core/code/engine/storage/scope-index-store.js

@@ -0,0 +1,247 @@
+"use strict";
+/**
+ * Disk-backed scope store + lazy `ScopeTree` (out-of-core scope index — kernel scope-resolution
+ * out-of-core index).
+ *
+ * ## Why this exists
+ *
+ * The scope-resolution resident floor is dominated by the per-`Scope` binding
+ * payload (`Scope.bindings` / `typeBindings` / `ownedDefs`) — ~17-20 GB for one
+ * language on the Linux kernel — held in `preExtractedByPath` AND aliased by the
+ * finalize `scopeTree` through the whole emit phase. The emit passes never read
+ * `parsed.scopes` directly; they reach scopes EXCLUSIVELY through
+ * `scopeTree.getScope(id)` (a point lookup) and `getChildren(id)`. So the heavy
+ * scope payload can move to disk behind that point lookup without changing what
+ * any consumer observes: every consumer reads a `Scope` BY VALUE (its `bindings`
+ * / `ownedDefs` contents, `parent`, `range`), never by object identity.
+ *
+ * {@link persistScopeShards} writes the scopes to per-file JSON shards (one file
+ * = one shard, so a file's whole parent-chain — which never crosses filePath —
+ * stays within one shard) using the same `mapReplacer` + def-interning reviver
+ * the ParsedFile store proved byte-identical (#1983 / def-object interning). {@link DiskBackedScopeTree}
+ * serves `getScope` from a bounded LRU of decoded shards plus a small resident
+ * skeleton (`scopeId -> {shard, childIds, parent}`), so only the working set of
+ * scopes is resident, not all of them.
+ *
+ * Default-off: only constructed when `CGRAPH_DISK_SCOPE_INDEX` is set. The
+ * resident `buildScopeTree` path is untouched otherwise.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TransitionalScopeTree = exports.DiskBackedScopeTree = exports.persistScopeShards = exports.clearScopeIndexStore = exports.getScopeIndexStoreDir = void 0;
+const node_fs_1 = require("node:fs");
+const node_path_1 = __importDefault(require("node:path"));
+const index_js_1 = require("../../shared/index.js");
+const parse_cache_js_1 = require("./parse-cache.js");
+const parsedfile_store_js_1 = require("./parsedfile-store.js");
+const STORE_DIRNAME = 'scope-index-store';
+const getScopeIndexStoreDir = (storagePath) => node_path_1.default.join(storagePath, STORE_DIRNAME);
+exports.getScopeIndexStoreDir = getScopeIndexStoreDir;
+/**
+ * Remove any prior seal's scope shards so a fresh seal starts clean. The shard
+ * names are sequential (`s<n>.json`) and the index resets per `persistScopeShards`
+ * call, so without this a seal that writes FEWER shards than a previous one (a
+ * later language with fewer files, or a later run of a shrunken repo) would leave
+ * stale tail shards on disk indefinitely — pure garbage the disk-backed tree
+ * never reads, but multi-GB on kernel-scale repos. Idempotent; synchronous to
+ * fit the main-thread seal path. Safe to call before each seal: the previously
+ * sealed language has finished emit and been released before the next seal runs,
+ * so its `DiskBackedScopeTree` never reads these shards again.
+ */
+const clearScopeIndexStore = (storagePath) => {
+    (0, node_fs_1.rmSync)((0, exports.getScopeIndexStoreDir)(storagePath), { recursive: true, force: true });
+};
+exports.clearScopeIndexStore = clearScopeIndexStore;
+const EMPTY = Object.freeze([]);
+/**
+ * Persist `scopes` to per-file shards under `<storagePath>/scope-index-store/`
+ * and return the resident skeleton the {@link DiskBackedScopeTree} needs. Sharded
+ * by `filePath` (sequential `s<n>.json` names — no filePath→filename encoding),
+ * so one file's full scope subtree round-trips together. Synchronous: pass-A runs
+ * on the main thread and this replaces an in-heap `buildScopeTree`.
+ */
+const persistScopeShards = (storagePath, scopes) => {
+    const dir = (0, exports.getScopeIndexStoreDir)(storagePath);
+    // Clear first so a seal that writes fewer shards than the previous one (a
+    // later language with fewer files, or a shrunken repo on re-run) leaves no
+    // stale `s<n>.json` tail behind. See {@link clearScopeIndexStore}.
+    (0, exports.clearScopeIndexStore)(storagePath);
+    (0, node_fs_1.mkdirSync)(dir, { recursive: true });
+    // Group scope objects by filePath, preserving input order within a file.
+    const byFile = new Map();
+    for (const s of scopes) {
+        const bucket = byFile.get(s.filePath);
+        if (bucket)
+            bucket.push(s);
+        else
+            byFile.set(s.filePath, [s]);
+    }
+    // childIds in input order (mirrors buildScopeTree's children buckets).
+    const childIds = new Map();
+    for (const s of scopes) {
+        if (s.parent !== null) {
+            const b = childIds.get(s.parent);
+            if (b)
+                b.push(s.id);
+            else
+                childIds.set(s.parent, [s.id]);
+        }
+    }
+    const skeleton = new Map();
+    let shardIndex = 0;
+    for (const [, fileScopes] of byFile) {
+        const shard = `s${shardIndex++}.json`;
+        (0, node_fs_1.writeFileSync)(node_path_1.default.join(dir, shard), JSON.stringify(fileScopes, parse_cache_js_1.mapReplacer), 'utf-8');
+        for (const s of fileScopes) {
+            skeleton.set(s.id, {
+                shard,
+                parent: s.parent,
+                childIds: childIds.get(s.id) ?? [],
+            });
+        }
+    }
+    return skeleton;
+};
+exports.persistScopeShards = persistScopeShards;
+/**
+ * A `ScopeTree` that serves `getScope` from disk shards via a bounded LRU,
+ * holding only `maxResidentShards` decoded shards + the resident skeleton. Drop-in
+ * for the in-heap `buildScopeTree` result for the methods scope-resolution
+ * actually calls — `getScope` (the hot one), `getChildren`, `getParent`,
+ * `getAncestors`, `has`, `size`. `byId` (a full materialization) is only used by
+ * the `INGESTION_EMIT_SCOPES=1` debug path and is unsupported here — the two flags
+ * are mutually exclusive.
+ */
+class DiskBackedScopeTree {
+    dir;
+    skeleton;
+    maxResidentShards;
+    /** Decoded shards, most-recently-used last (Map preserves insertion order). */
+    lru = new Map();
+    constructor(storagePath, skeleton, maxResidentShards = 64) {
+        this.dir = (0, exports.getScopeIndexStoreDir)(storagePath);
+        this.skeleton = skeleton;
+        this.maxResidentShards = Math.max(1, maxResidentShards);
+    }
+    get size() {
+        return this.skeleton.size;
+    }
+    get byId() {
+        throw new Error('DiskBackedScopeTree.byId is unsupported (INGESTION_EMIT_SCOPES is incompatible with CGRAPH_DISK_SCOPE_INDEX).');
+    }
+    has(id) {
+        return this.skeleton.has(id);
+    }
+    getChildren(id) {
+        return this.skeleton.get(id)?.childIds ?? EMPTY;
+    }
+    getScope(id) {
+        const meta = this.skeleton.get(id);
+        if (meta === undefined)
+            return undefined;
+        return this.loadShard(meta.shard).get(id);
+    }
+    getParent(id) {
+        const parent = this.skeleton.get(id)?.parent;
+        return parent === null || parent === undefined ? undefined : this.getScope(parent);
+    }
+    getAncestors(id) {
+        const out = [];
+        let cur = this.skeleton.get(id)?.parent ?? null;
+        const seen = new Set();
+        while (cur !== null && !seen.has(cur)) {
+            seen.add(cur);
+            out.push(cur);
+            cur = this.skeleton.get(cur)?.parent ?? null;
+        }
+        return out;
+    }
+    /** Load + decode a shard, touch it as MRU, evict the LRU tail past the cap. */
+    loadShard(shard) {
+        const cached = this.lru.get(shard);
+        if (cached !== undefined) {
+            // Touch: move to MRU end.
+            this.lru.delete(shard);
+            this.lru.set(shard, cached);
+            return cached;
+        }
+        const pool = new Map();
+        const defPool = new Map();
+        const raw = (0, node_fs_1.readFileSync)(node_path_1.default.join(this.dir, shard), 'utf-8');
+        const fileScopes = JSON.parse(raw, (0, parsedfile_store_js_1.makeInterningReviver)(pool, defPool));
+        const decoded = new Map();
+        for (const s of fileScopes)
+            decoded.set(s.id, s);
+        this.lru.set(shard, decoded);
+        if (this.lru.size > this.maxResidentShards) {
+            const oldest = this.lru.keys().next().value;
+            if (oldest !== undefined)
+                this.lru.delete(oldest);
+        }
+        return decoded;
+    }
+}
+exports.DiskBackedScopeTree = DiskBackedScopeTree;
+/**
+ * A `ScopeTree` that starts fully resident (validated by `buildScopeTree`, used
+ * by finalize / propagate / resolve, which may mutate `Scope.typeBindings` in
+ * place) and then {@link seal}s to a {@link DiskBackedScopeTree} just before emit.
+ *
+ * Sealing is what actually frees the ~17-20 GB of `Scope.bindings`: the resident
+ * tree is held BY THE MODEL'S frozen index bundle, so it cannot be swapped out
+ * from the outside. This wrapper IS that held object — `seal()` mutates its own
+ * (non-frozen) internal field to null the resident tree from the inside, after
+ * persisting it, so the model's reference now points at the disk-backed serving
+ * path and the heavy scopes become collectible (once the emit-side ParsedFiles
+ * also drop their `scopes`). Idempotent; a no-op before `seal()` keeps today's
+ * fully-resident behavior byte-identical.
+ */
+class TransitionalScopeTree {
+    resident;
+    disk = null;
+    constructor(scopes) {
+        this.resident = (0, index_js_1.buildScopeTree)(scopes); // validates invariants + serves the resident phase
+    }
+    /** Persist the resident scopes, switch to disk-backed serving, and drop the
+     *  resident tree so its scope/binding payload can be reclaimed. Idempotent. */
+    seal(storagePath, maxResidentShards) {
+        if (this.disk !== null || this.resident === null)
+            return;
+        const skeleton = (0, exports.persistScopeShards)(storagePath, [...this.resident.byId.values()]);
+        this.disk = new DiskBackedScopeTree(storagePath, skeleton, maxResidentShards);
+        this.resident = null;
+    }
+    get sealed() {
+        return this.disk !== null;
+    }
+    get active() {
+        const t = this.disk ?? this.resident;
+        if (t === null)
+            throw new Error('TransitionalScopeTree has no active backing (resident dropped without seal).');
+        return t;
+    }
+    get size() {
+        return this.active.size;
+    }
+    get byId() {
+        return this.active.byId; // throws in disk mode (DiskBackedScopeTree.byId)
+    }
+    has(id) {
+        return this.active.has(id);
+    }
+    getScope(id) {
+        return this.active.getScope(id);
+    }
+    getParent(id) {
+        return this.active.getParent(id);
+    }
+    getChildren(id) {
+        return this.active.getChildren(id);
+    }
+    getAncestors(id) {
+        return this.active.getAncestors(id);
+    }
+}
+exports.TransitionalScopeTree = TransitionalScopeTree;

package/dist/core/code/engine/types/pipeline.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/core/code/scripts/install-duckdb-extension.mjs

@@ -0,0 +1,125 @@
+#!/usr/bin/env node
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { createRequire } from 'node:module';
+import { pathToFileURL } from 'node:url';
+
+const EXTENSION_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_]*$/;
+
+// Positive on-disk-corruption signatures. `FORCE INSTALL` re-downloads even when
+// a file is already present; we only want that when the LOAD error proves the
+// existing file is bad (truncated/wrong-platform, #2374). For everything else —
+// a missing file (plain INSTALL downloads it), or a permanent non-file failure a
+// re-download can never fix (missing runtime dep: "cannot open shared object") —
+// plain INSTALL avoids re-downloading ~2 MB on every analyze run forever.
+// Exported so a parity test keeps this byte-identical to the copy in
+// src/core/lbug/extension-load-error.ts (this `.mjs` cannot import that `.ts`), #2383 F5b.
+export const FILE_CORRUPTION_SIGNATURES = [
+  /invalid elf/i,
+  /file too short/i,
+  /not a valid/i,
+  /bad magic/i,
+  /wrong architecture/i,
+  /mach-o/i,
+  /truncat/i,
+];
+
+/**
+ * Decide the install verb from the LOAD error that triggered this install.
+ * `FORCE INSTALL` only when the error positively indicates file-level breakage;
+ * otherwise plain `INSTALL` (missing file, missing-dependency dlopen failure,
+ * or unknown/absent error).
+ */
+export function chooseInstallVerb(loadError) {
+  if (loadError && FILE_CORRUPTION_SIGNATURES.some((re) => re.test(loadError))) {
+    return 'FORCE INSTALL';
+  }
+  return 'INSTALL';
+}
+
+function parseLbugMaxDbSize(raw) {
+  const parsed = raw ? Number(raw) : NaN;
+  if (!Number.isFinite(parsed) || parsed <= 0) {
+    throw new Error(`Invalid LadybugDB max DB size for extension installer: ${raw ?? '<missing>'}`);
+  }
+  return Math.floor(parsed);
+}
+
+function resolveMaxDbSize() {
+  // argv[3] is the optional positional size; ignore it when it is actually a
+  // flag token (e.g. `--verify-only`) and fall back to the env default.
+  const sizeArg =
+    process.argv[3] && !process.argv[3].startsWith('--') ? process.argv[3] : undefined;
+  return parseLbugMaxDbSize(sizeArg ?? process.env.CGRAPH_LBUG_MAX_DB_SIZE);
+}
+
+/** Open a scratch LadybugDB and return its connection plus a disposer. */
+async function defaultConnect(lbugMaxDbSize) {
+  const require = createRequire(import.meta.url);
+  const lbugModule = require('@ladybugdb/core');
+  const lbug = lbugModule.default ?? lbugModule;
+
+  const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-ext-install-'));
+  const dbPath = path.join(tmpDir, 'install.lbug');
+  const db = new lbug.Database(dbPath, 0, false, false, lbugMaxDbSize);
+  const conn = new lbug.Connection(db);
+  return {
+    conn,
+    dispose: async () => {
+      await conn.close().catch(() => {});
+      await db.close().catch(() => {});
+      await fs.rm(tmpDir, { recursive: true, force: true }).catch(() => {});
+    },
+  };
+}
+
+/**
+ * Install (or verify) an optional LadybugDB extension in this short-lived process.
+ *
+ * @param {string} extensionName
+ * @param {object} [options]
+ * @param {boolean} [options.verifyOnly] LOAD-only Docker build gate — no install.
+ * @param {string} [options.loadError] The parent's LOAD failure; selects the verb.
+ * @param {(size: number) => Promise<{conn: {query: (sql: string) => Promise<unknown>}, dispose: () => Promise<void>}>} [options.connect]
+ *        Connection factory; injectable for offline unit tests.
+ */
+export async function installDuckDbExtension(extensionName, options = {}) {
+  const { verifyOnly = false, loadError, connect } = options;
+  if (!extensionName || !EXTENSION_NAME_PATTERN.test(extensionName)) {
+    throw new Error(`Invalid DuckDB extension name: ${extensionName ?? '<missing>'}`);
+  }
+
+  const makeConnection = connect ?? (() => defaultConnect(resolveMaxDbSize()));
+  const { conn, dispose } = await makeConnection();
+
+  try {
+    if (verifyOnly) {
+      // Prove a previously-baked extension is resolvable by a FRESH process
+      // under the current HOME (the runtime `LOAD EXTENSION` path) — no INSTALL,
+      // no network. Used as a Docker build-time gate so a HOME/extension-dir
+      // mismatch fails the build instead of silently degrading search at runtime.
+      await conn.query(`LOAD EXTENSION ${extensionName}`);
+      console.log(
+        `[install-ext] LOAD-only verify OK for '${extensionName}' (HOME=${process.env.HOME})`,
+      );
+    } else {
+      // Plain INSTALL is a no-op when the file already exists; escalate to FORCE
+      // only when the LOAD error proves the on-disk file is broken (#2374).
+      await conn.query(`${chooseInstallVerb(loadError)} ${extensionName}`);
+    }
+  } finally {
+    await dispose();
+  }
+}
+
+// Only run when executed directly — imported (e.g. by unit tests) it stays inert.
+if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) {
+  installDuckDbExtension(process.argv[2] ?? process.env.CGRAPH_LBUG_EXTENSION_NAME, {
+    verifyOnly: process.argv.includes('--verify-only'),
+    loadError: process.env.CGRAPH_LBUG_EXTENSION_LOAD_ERROR,
+  }).catch((err) => {
+    console.error(err instanceof Error ? (err.stack ?? err.message) : String(err));
+    process.exitCode = 1;
+  });
+}