arceus-s 1.6.4

package/dist/core/lbug/extension-loader.js

@@ -0,0 +1,186 @@
+import { spawn } from 'child_process';
+import { fileURLToPath } from 'node:url';
+import { LBUG_MAX_DB_SIZE } from './lbug-config.js';
+import { logger } from '../logger.js';
+const DEFAULT_EXTENSION_INSTALL_TIMEOUT_MS = 15_000;
+const EXTENSION_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_]*$/;
+const alreadyAvailable = (message) => message.includes('already loaded') ||
+    message.includes('already installed') ||
+    message.includes('already exists');
+const resolvePolicyFromEnv = () => {
+    const raw = process.env.ARC_LBUG_EXTENSION_INSTALL;
+    if (raw === 'load-only' || raw === 'never' || raw === 'auto')
+        return raw;
+    return 'auto';
+};
+export const getExtensionInstallTimeoutMs = () => {
+    const raw = process.env.ARC_LBUG_EXTENSION_INSTALL_TIMEOUT_MS;
+    const parsed = raw ? Number(raw) : NaN;
+    return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_EXTENSION_INSTALL_TIMEOUT_MS;
+};
+export const getExtensionInstallChildProcessArgs = (extensionName, maxDbSize = LBUG_MAX_DB_SIZE) => {
+    const childScript = new URL('../../../scripts/install-duckdb-extension.mjs', import.meta.url);
+    return [fileURLToPath(childScript), extensionName, String(maxDbSize)];
+};
+/**
+ * Run `INSTALL <extension>` in a short-lived child Node process so the parent
+ * event loop is never blocked by DuckDB's synchronous network call.
+ *
+ * The child opens its own scratch LadybugDB, executes the install, and exits.
+ * If the child exceeds `timeoutMs` the parent kills it with SIGKILL and
+ * resolves with `timedOut: true`.
+ */
+export const installDuckDbExtensionOutOfProcess = async (extensionName, timeoutMs = getExtensionInstallTimeoutMs()) => {
+    if (!EXTENSION_NAME_PATTERN.test(extensionName)) {
+        throw new Error(`Invalid DuckDB extension name: ${extensionName}`);
+    }
+    return await new Promise((resolve) => {
+        const child = spawn(process.execPath, getExtensionInstallChildProcessArgs(extensionName), {
+            env: {
+                ...process.env,
+                ARC_LBUG_EXTENSION_NAME: extensionName,
+            },
+            stdio: ['ignore', 'ignore', 'pipe'],
+            windowsHide: true,
+        });
+        let stderr = '';
+        child.stderr?.setEncoding('utf8');
+        child.stderr?.on('data', (chunk) => {
+            stderr = (stderr + chunk).slice(-4000);
+        });
+        let settled = false;
+        const timer = setTimeout(() => {
+            if (settled)
+                return;
+            settled = true;
+            child.kill('SIGKILL');
+            resolve({
+                success: false,
+                timedOut: true,
+                message: `INSTALL ${extensionName} timed out after ${timeoutMs}ms`,
+            });
+        }, timeoutMs);
+        child.on('error', (err) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            resolve({ success: false, timedOut: false, message: err.message });
+        });
+        child.on('exit', (code, signal) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            resolve({
+                success: code === 0,
+                timedOut: false,
+                message: code === 0
+                    ? `INSTALL ${extensionName} completed`
+                    : `INSTALL ${extensionName} failed with ${signal ?? `exit code ${code}`}${stderr ? `: ${stderr.trim()}` : ''}`,
+            });
+        });
+    });
+};
+/**
+ * Centralized lifecycle manager for optional LadybugDB extensions.
+ *
+ * Always tries `LOAD EXTENSION <name>` first — it is per-connection,
+ * idempotent, and never touches the network. If `LOAD` fails and the active
+ * policy permits, the manager runs a single bounded out-of-process `INSTALL`
+ * attempt per process and retries `LOAD`. Capability outcomes are cached so
+ * unavailable extensions degrade search features without ever blocking
+ * subsequent analyze or query calls.
+ *
+ * Policy precedence (most specific wins):
+ *   per-call `opts.policy` → constructor `options.policy` → env → `auto`
+ */
+export class ExtensionManager {
+    options;
+    capabilities = new Map();
+    installAttempted = new Map();
+    warnedKeys = new Set();
+    constructor(options = {}) {
+        this.options = options;
+    }
+    /** Reset cached capability and install state. Test-only. */
+    reset() {
+        this.capabilities.clear();
+        this.installAttempted.clear();
+        this.warnedKeys.clear();
+    }
+    /** Snapshot of currently-known optional extension capabilities. */
+    getCapabilities() {
+        return Array.from(this.capabilities.values());
+    }
+    /**
+     * Ensure an optional extension is loaded on the supplied connection.
+     *
+     * Returns `true` when the extension is usable on `query`, `false` when it
+     * is unavailable. Never throws on install failure — analyze and query
+     * paths are expected to degrade gracefully.
+     */
+    async ensure(query, name, label, opts = {}) {
+        if (!EXTENSION_NAME_PATTERN.test(name)) {
+            throw new Error(`Invalid DuckDB extension name: ${name}`);
+        }
+        const policy = opts.policy ?? this.options.policy ?? resolvePolicyFromEnv();
+        const timeoutMs = opts.installTimeoutMs ?? this.options.installTimeoutMs ?? getExtensionInstallTimeoutMs();
+        const warn = this.options.warn ?? ((msg) => logger.warn(msg));
+        if (policy === 'never') {
+            this.markUnavailable(name, label, 'extension install policy is "never"', warn);
+            return false;
+        }
+        if (await this.tryLoad(query, name)) {
+            this.markLoaded(name);
+            return true;
+        }
+        if (policy === 'load-only') {
+            this.markUnavailable(name, label, 'load-only policy: extension not pre-installed', warn);
+            return false;
+        }
+        let install = this.installAttempted.get(name);
+        if (!install) {
+            const installFn = this.options.installExtension ?? installDuckDbExtensionOutOfProcess;
+            install = await installFn(name, timeoutMs);
+            this.installAttempted.set(name, install);
+        }
+        if (!install.success) {
+            this.markUnavailable(name, label, install.message, warn);
+            return false;
+        }
+        if (await this.tryLoad(query, name)) {
+            this.markLoaded(name);
+            return true;
+        }
+        this.markUnavailable(name, label, `LOAD ${name} failed after successful INSTALL`, warn);
+        return false;
+    }
+    async tryLoad(query, name) {
+        try {
+            await query(`LOAD EXTENSION ${name}`);
+            return true;
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            return alreadyAvailable(msg);
+        }
+    }
+    markLoaded(name) {
+        this.capabilities.set(name, { name, loaded: true });
+    }
+    markUnavailable(name, label, reason, warn) {
+        this.capabilities.set(name, { name, loaded: false, reason });
+        const key = `${name}:${reason}`;
+        if (this.warnedKeys.has(key))
+            return;
+        this.warnedKeys.add(key);
+        warn(`Arceus: ${label} extension unavailable; continuing without ${label} features. ${reason}`);
+    }
+}
+/** Process-wide singleton shared by core and pool adapters. */
+export const extensionManager = new ExtensionManager();
+/** Snapshot of which optional DuckDB extensions are loaded in this process. */
+export const getExtensionCapabilities = () => extensionManager.getCapabilities();
+/** Test-only: clear the singleton's cached capability and install state. */
+export const resetExtensionState = () => extensionManager.reset();

package/dist/core/lbug/lbug-adapter.d.ts

@@ -0,0 +1,243 @@
+import lbug from '@ladybugdb/core';
+import { KnowledgeGraph } from '../graph/types.js';
+import type { CachedEmbedding } from '../embeddings/types.js';
+import { type ExtensionEnsureOptions } from './extension-loader.js';
+/** Factory for creating WriteStreams — injectable for testing. */
+export type WriteStreamFactory = (filePath: string) => import('fs').WriteStream;
+/** Result of splitting the relationship CSV into per-label-pair files. */
+export interface RelCsvSplitResult {
+    relHeader: string;
+    relsByPairMeta: Map<string, {
+        csvPath: string;
+        rows: number;
+    }>;
+    pairWriteStreams: Map<string, import('fs').WriteStream>;
+    skippedRels: number;
+    totalValidRels: number;
+}
+/**
+ * Split a relationship CSV into per-label-pair files on disk.
+ *
+ * Streams the CSV line-by-line, routing each relationship to a file named
+ * `rel_{fromLabel}_{toLabel}.csv`. Handles backpressure correctly: only one
+ * drain listener per stream at a time, and readline resumes only when ALL
+ * backpressured streams have drained.
+ *
+ * @param csvPath       Path to the combined relationship CSV
+ * @param csvDir        Directory to write per-pair CSV files
+ * @param validTables   Set of valid node table names
+ * @param getNodeLabel  Function to extract the label from a node ID
+ * @param wsFactory     Optional WriteStream factory (defaults to fs.createWriteStream)
+ */
+export declare const splitRelCsvByLabelPair: (csvPath: string, csvDir: string, validTables: Set<string>, getNodeLabel: (id: string) => string, wsFactory?: WriteStreamFactory) => Promise<RelCsvSplitResult>;
+/** Expose the current Database for pool adapter reuse in tests. */
+export declare const getDatabase: () => lbug.Database | null;
+/**
+ * Return true when the error message indicates a write was attempted against
+ * a read-only LadybugDB connection. The MCP query pool opens DBs read-only,
+ * so any path that calls a `CREATE_*` procedure there will surface this
+ * (e.g. defensive `ensureFTSIndex` calls). Owners of the writable analyze
+ * path should ignore this error — index creation is owned by `arc
+ * analyze` and either already happened or will happen on the next run.
+ */
+export declare const isReadOnlyDbError: (err: unknown) => boolean;
+export declare const initLbug: (dbPath: string) => Promise<{
+    db: lbug.Database;
+    conn: lbug.Connection;
+}>;
+/**
+ * Execute multiple queries against one repo DB atomically.
+ * While the callback runs, no other request can switch the active DB.
+ *
+ * Automatically retries up to DB_LOCK_RETRY_ATTEMPTS times when the
+ * database is busy (e.g. `arc analyze` holds the write lock).
+ * Each retry waits DB_LOCK_RETRY_DELAY_MS * attempt milliseconds.
+ */
+export declare const withLbugDb: <T>(dbPath: string, operation: () => Promise<T>) => Promise<T>;
+export type LbugProgressCallback = (message: string) => void;
+export declare const loadGraphToLbug: (graph: KnowledgeGraph, repoPath: string, storagePath: string, onProgress?: LbugProgressCallback) => Promise<{
+    success: boolean;
+    insertedRels: number;
+    skippedRels: number;
+    warnings: string[];
+}>;
+/**
+ * Insert a single node to LadybugDB
+ * @param label - Node type (File, Function, Class, etc.)
+ * @param properties - Node properties
+ * @param dbPath - Path to LadybugDB database (optional if already initialized)
+ */
+export declare const insertNodeToLbug: (label: string, properties: Record<string, any>, dbPath?: string) => Promise<boolean>;
+/**
+ * Batch insert multiple nodes to LadybugDB using a single connection
+ * @param nodes - Array of {label, properties} to insert
+ * @param dbPath - Path to LadybugDB database
+ * @returns Object with success count and error count
+ */
+export declare const batchInsertNodesToLbug: (nodes: Array<{
+    label: string;
+    properties: Record<string, any>;
+}>, dbPath: string) => Promise<{
+    inserted: number;
+    failed: number;
+}>;
+export declare const executeQuery: (cypher: string) => Promise<any[]>;
+export declare const streamQuery: (cypher: string, onRow: (row: any) => void | Promise<void>) => Promise<number>;
+/**
+ * Execute a single parameterized query (prepare/execute pattern).
+ * Prevents Cypher injection by binding values as parameters.
+ */
+export declare const executePrepared: (cypher: string, params: Record<string, any>) => Promise<any[]>;
+export declare const executeWithReusedStatement: (cypher: string, paramsList: Array<Record<string, any>>) => Promise<void>;
+export declare const getLbugStats: () => Promise<{
+    nodes: number;
+    edges: number;
+}>;
+/**
+ * Load cached embeddings from LadybugDB before a rebuild.
+ * Returns all embedding vectors so they can be re-inserted after the graph is reloaded,
+ * avoiding expensive re-embedding of unchanged nodes.
+ *
+ * Detects old schema (no chunkIndex column) and returns empty cache to trigger rebuild.
+ */
+export declare const loadCachedEmbeddings: () => Promise<{
+    embeddingNodeIds: Set<string>;
+    embeddings: CachedEmbedding[];
+}>;
+/**
+ * Fetch existing embedding hashes from CodeEmbedding table for incremental embedding.
+ * Returns a Map<nodeId, contentHash> suitable for passing to `runEmbeddingPipeline`.
+ * Handles legacy DBs without the `contentHash` column (all rows treated as stale with empty hash).
+ * Returns undefined if the CodeEmbedding table does not exist.
+ *
+ * @param execQuery - Cypher query executor (typically pool-adapter's `executeQuery`)
+ */
+export declare const fetchExistingEmbeddingHashes: (execQuery: (cypher: string) => Promise<any[]>) => Promise<Map<string, string> | undefined>;
+/**
+ * Flush the WAL so all pending writes are visible to subsequent readers.
+ *
+ * Best-effort: swallows errors from older LadybugDB versions or schemaless
+ * databases that do not support the CHECKPOINT command.  A no-op when there
+ * is nothing pending, so safe (and cheap) to call unconditionally after any
+ * write path.
+ *
+ * Use this instead of safeClose when the connection must stay open
+ * (e.g. the /api/embed handler that keeps serving queries after flushing).
+ *
+ * @see safeClose — CHECKPOINT + connection/database close
+ */
+export declare const flushWAL: () => Promise<void>;
+/**
+ * Flush the WAL and close the connection and database handles.
+ *
+ * Consolidates the CHECKPOINT + close pattern into a single function so
+ * callers never call conn.close() or db.close() directly (#1376).
+ * An ESLint no-restricted-syntax rule enforces this — see eslint.config.mjs.
+ *
+ * @see flushWAL — CHECKPOINT-only (connection stays open)
+ * @see closeLbug — safeClose + module state reset (full teardown)
+ */
+export declare const safeClose: () => Promise<void>;
+export declare const closeLbug: () => Promise<void>;
+export declare const isLbugReady: () => boolean;
+/**
+ * Delete all nodes (and their relationships) for a specific file from LadybugDB
+ * @param filePath - The file path to delete nodes for
+ * @param dbPath - Optional path to LadybugDB for per-query connection
+ * @returns Object with counts of deleted nodes
+ */
+export declare const deleteNodesForFile: (filePath: string, dbPath?: string) => Promise<{
+    deletedNodes: number;
+}>;
+export declare const getEmbeddingTableName: () => string;
+/**
+ * Return the distinct repo-relative paths of files that import
+ * `targetFilePath` according to the IMPORTS edges currently in the
+ * DB. Used by the incremental writeback path to expand the
+ * "files-to-rewrite" set so that files importing a changed file get
+ * their edges (which may have been refined by cross-file resolution)
+ * re-emitted, rather than left stale in the DB.
+ *
+ * The DB query reads the *previous* run's state — pre-pipeline, before
+ * any nodes are deleted — so the returned importers are "files that
+ * USED TO import the target". That's the right set to invalidate:
+ * those are the files whose edges in the DB might no longer match
+ * what cross-file resolution produces given the changed file's new
+ * exports.
+ */
+export declare const queryImporters: (targetFilePath: string) => Promise<string[]>;
+/**
+ * Drop every Community and Process node (and their MEMBER_OF /
+ * STEP_IN_PROCESS edges via DETACH DELETE). Used at the start of an
+ * incremental run so the communities and processes phases regenerate
+ * them from scratch on the merged graph — required for the
+ * "Leiden runs on the FULL graph" correctness invariant.
+ */
+export declare const deleteAllCommunitiesAndProcesses: () => Promise<{
+    nodesDeleted: number;
+}>;
+/**
+ * Load the FTS extension on the supplied connection (or the singleton
+ * writable connection when none is given).
+ *
+ * Delegates to the shared `ExtensionManager` so install policy (auto /
+ * load-only / never), out-of-process bounded INSTALL, and capability
+ * caching are owned in one place. The module-level `ftsLoaded` flag is
+ * kept purely as a per-call short-circuit on the singleton writable
+ * connection so repeated callers (e.g. createFTSIndex) avoid an extra
+ * `LOAD` round-trip per invocation. Pool adapter callers pass
+ * `{ policy: 'load-only' }` so query paths never block on a network install.
+ */
+export declare const loadFTSExtension: (targetConn?: lbug.Connection, opts?: ExtensionEnsureOptions) => Promise<boolean>;
+/**
+ * Load the VECTOR extension on the supplied connection (or the singleton
+ * writable connection when none is given). Returns false when VECTOR is
+ * unavailable so semantic search can fall back to exact scan.
+ */
+export declare const loadVectorExtension: (targetConn?: lbug.Connection, opts?: ExtensionEnsureOptions) => Promise<boolean>;
+/**
+ * Create a full-text search index on a table
+ * @param tableName - The node table name (e.g., 'File', 'CodeSymbol')
+ * @param indexName - Name for the FTS index
+ * @param properties - List of properties to index (e.g., ['name', 'code'])
+ * @param stemmer - Stemming algorithm (default: 'porter')
+ */
+export declare const createFTSIndex: (tableName: string, indexName: string, properties: string[], stemmer?: string) => Promise<void>;
+/**
+ * Lazy-create an FTS index, caching the fact in-process.
+ *
+ * Kept for writable maintenance paths that need to lazily materialize an
+ * index. Read-only query paths must not call this; production analysis owns
+ * creating the configured search indexes before the database is served.
+ *
+ * Safe to call repeatedly — the in-process Set guarantees only the first
+ * call hits LadybugDB. `closeLbug` clears the cache so re-init starts fresh.
+ *
+ * Defense in depth: if the active connection is read-only (e.g. the MCP
+ * pool adapter), `CREATE_FTS_INDEX` will fail with "Cannot execute write
+ * operations in a read-only database". Treat that as a no-op and cache
+ * the key so callers don't loop on a path that can never succeed here —
+ * the index is owned by `arc analyze` (writable) and either already
+ * exists or will be created on the next analyze.
+ */
+export declare const ensureFTSIndex: (tableName: string, indexName: string, properties: string[], stemmer?: string) => Promise<void>;
+/**
+ * Query a full-text search index
+ * @param tableName - The node table name
+ * @param indexName - FTS index name
+ * @param query - Search query string
+ * @param limit - Maximum results
+ * @param conjunctive - If true, all terms must match (AND); if false, any term matches (OR)
+ * @returns Array of { node properties, score }
+ */
+export declare const queryFTS: (tableName: string, indexName: string, query: string, limit?: number, conjunctive?: boolean) => Promise<Array<{
+    nodeId: string;
+    name: string;
+    filePath: string;
+    score: number;
+    [key: string]: any;
+}>>;
+/**
+ * Drop an FTS index
+ */
+export declare const dropFTSIndex: (tableName: string, indexName: string) => Promise<void>;