arceus-s 1.6.4

package/dist/core/lbug/csv-generator.js

@@ -0,0 +1,463 @@
+/**
+ * CSV Generator for LadybugDB Hybrid Schema
+ *
+ * Streams CSV rows directly to disk files in a single pass over graph nodes.
+ * File contents are lazy-read from disk per-node to avoid holding the entire
+ * repo in RAM. Rows are buffered (FLUSH_EVERY) before writing to minimize
+ * per-row Promise overhead.
+ *
+ * RFC 4180 Compliant:
+ * - Fields containing commas, double quotes, or newlines are enclosed in double quotes
+ * - Double quotes within fields are escaped by doubling them ("")
+ * - All fields are consistently quoted for safety with code content
+ */
+import fs from 'fs/promises';
+import { createWriteStream } from 'fs';
+import path from 'path';
+/** Flush buffered rows to disk every N rows */
+const FLUSH_EVERY = 500;
+// ============================================================================
+// CSV ESCAPE UTILITIES
+// ============================================================================
+export const sanitizeUTF8 = (str) => {
+    return str
+        .replace(/\r\n/g, '\n')
+        .replace(/\r/g, '\n')
+        .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, '')
+        .replace(/[\uD800-\uDFFF]/g, '')
+        .replace(/[\uFFFE\uFFFF]/g, '');
+};
+export const escapeCSVField = (value) => {
+    if (value === undefined || value === null)
+        return '""';
+    let str = String(value);
+    str = sanitizeUTF8(str);
+    return `"${str.replace(/"/g, '""')}"`;
+};
+export const escapeCSVNumber = (value, defaultValue = -1) => {
+    if (value === undefined || value === null)
+        return String(defaultValue);
+    return String(value);
+};
+// ============================================================================
+// CONTENT EXTRACTION (lazy — reads from disk on demand)
+// ============================================================================
+export const isBinaryContent = (content) => {
+    if (!content || content.length === 0)
+        return false;
+    const sample = content.slice(0, 1000);
+    let nonPrintable = 0;
+    for (let i = 0; i < sample.length; i++) {
+        const code = sample.charCodeAt(i);
+        if (code < 9 || (code > 13 && code < 32) || code === 127)
+            nonPrintable++;
+    }
+    return nonPrintable / sample.length > 0.1;
+};
+/**
+ * LRU content cache — avoids re-reading the same source file for every
+ * symbol defined in it. Sized generously so most files stay cached during
+ * the single-pass node iteration.
+ */
+class FileContentCache {
+    cache = new Map();
+    accessOrder = [];
+    maxSize;
+    repoPath;
+    constructor(repoPath, maxSize = 3000) {
+        this.repoPath = repoPath;
+        this.maxSize = maxSize;
+    }
+    async get(relativePath) {
+        if (!relativePath)
+            return '';
+        const cached = this.cache.get(relativePath);
+        if (cached !== undefined) {
+            // Move to end of accessOrder (LRU promotion)
+            const idx = this.accessOrder.indexOf(relativePath);
+            if (idx !== -1) {
+                this.accessOrder.splice(idx, 1);
+                this.accessOrder.push(relativePath);
+            }
+            return cached;
+        }
+        try {
+            const fullPath = path.join(this.repoPath, relativePath);
+            const content = await fs.readFile(fullPath, 'utf-8');
+            this.set(relativePath, content);
+            return content;
+        }
+        catch {
+            this.set(relativePath, '');
+            return '';
+        }
+    }
+    set(key, value) {
+        if (this.cache.size >= this.maxSize) {
+            const oldest = this.accessOrder.shift();
+            if (oldest)
+                this.cache.delete(oldest);
+        }
+        this.cache.set(key, value);
+        this.accessOrder.push(key);
+    }
+}
+const extractContent = async (node, contentCache) => {
+    const filePath = node.properties.filePath;
+    const content = await contentCache.get(filePath);
+    if (!content)
+        return '';
+    if (node.label === 'Folder')
+        return '';
+    if (isBinaryContent(content))
+        return '[Binary file - content not stored]';
+    if (node.label === 'File') {
+        const MAX_FILE_CONTENT = 10000;
+        return content.length > MAX_FILE_CONTENT
+            ? content.slice(0, MAX_FILE_CONTENT) + '\n... [truncated]'
+            : content;
+    }
+    const startLine = node.properties.startLine;
+    const endLine = node.properties.endLine;
+    if (startLine === undefined || endLine === undefined)
+        return '';
+    const lines = content.split('\n');
+    const start = Math.max(0, startLine - 2);
+    const end = Math.min(lines.length - 1, endLine + 2);
+    const snippet = lines.slice(start, end + 1).join('\n');
+    const MAX_SNIPPET = 5000;
+    return snippet.length > MAX_SNIPPET
+        ? snippet.slice(0, MAX_SNIPPET) + '\n... [truncated]'
+        : snippet;
+};
+// ============================================================================
+// BUFFERED CSV WRITER
+// ============================================================================
+class BufferedCSVWriter {
+    ws;
+    buffer = [];
+    rows = 0;
+    constructor(filePath, header) {
+        this.ws = createWriteStream(filePath, 'utf-8');
+        // Large repos flush many times — raise listener cap to avoid MaxListenersExceededWarning
+        this.ws.setMaxListeners(50);
+        this.buffer.push(header);
+    }
+    addRow(row) {
+        this.buffer.push(row);
+        this.rows++;
+        if (this.buffer.length >= FLUSH_EVERY) {
+            return this.flush();
+        }
+        return Promise.resolve();
+    }
+    flush() {
+        if (this.buffer.length === 0)
+            return Promise.resolve();
+        const chunk = this.buffer.join('\n') + '\n';
+        this.buffer.length = 0;
+        return new Promise((resolve, reject) => {
+            this.ws.once('error', reject);
+            const ok = this.ws.write(chunk);
+            if (ok) {
+                this.ws.removeListener('error', reject);
+                resolve();
+            }
+            else {
+                this.ws.once('drain', () => {
+                    this.ws.removeListener('error', reject);
+                    resolve();
+                });
+            }
+        });
+    }
+    async finish() {
+        await this.flush();
+        return new Promise((resolve, reject) => {
+            this.ws.end(() => resolve());
+            this.ws.on('error', reject);
+        });
+    }
+}
+/**
+ * Stream all CSV data directly to disk files.
+ * Iterates graph nodes exactly ONCE — routes each node to the right writer.
+ * File contents are lazy-read from disk with a generous LRU cache.
+ */
+export const streamAllCSVsToDisk = async (graph, repoPath, csvDir) => {
+    // Remove stale CSVs from previous crashed runs, then recreate
+    try {
+        await fs.rm(csvDir, { recursive: true, force: true });
+    }
+    catch { }
+    await fs.mkdir(csvDir, { recursive: true });
+    // We open ~30 concurrent write-streams; raise process limit to suppress
+    // MaxListenersExceededWarning (restored after all streams finish).
+    const prevMax = process.getMaxListeners();
+    process.setMaxListeners(prevMax + 40);
+    const contentCache = new FileContentCache(repoPath);
+    // Create writers for every node type up-front
+    const fileWriter = new BufferedCSVWriter(path.join(csvDir, 'file.csv'), 'id,name,filePath,content');
+    const folderWriter = new BufferedCSVWriter(path.join(csvDir, 'folder.csv'), 'id,name,filePath');
+    const codeElementHeader = 'id,name,filePath,startLine,endLine,isExported,content,description';
+    const functionWriter = new BufferedCSVWriter(path.join(csvDir, 'function.csv'), codeElementHeader);
+    const classWriter = new BufferedCSVWriter(path.join(csvDir, 'class.csv'), codeElementHeader);
+    const interfaceWriter = new BufferedCSVWriter(path.join(csvDir, 'interface.csv'), codeElementHeader);
+    const methodHeader = 'id,name,filePath,startLine,endLine,isExported,content,description,parameterCount,returnType';
+    const methodWriter = new BufferedCSVWriter(path.join(csvDir, 'method.csv'), methodHeader);
+    const codeElemWriter = new BufferedCSVWriter(path.join(csvDir, 'codeelement.csv'), codeElementHeader);
+    const communityWriter = new BufferedCSVWriter(path.join(csvDir, 'community.csv'), 'id,label,heuristicLabel,keywords,description,enrichedBy,cohesion,symbolCount');
+    const processWriter = new BufferedCSVWriter(path.join(csvDir, 'process.csv'), 'id,label,heuristicLabel,processType,stepCount,communities,entryPointId,terminalId');
+    // Section nodes have an extra 'level' column
+    const sectionWriter = new BufferedCSVWriter(path.join(csvDir, 'section.csv'), 'id,name,filePath,startLine,endLine,level,content,description');
+    // Route nodes for API endpoint mapping
+    const routeWriter = new BufferedCSVWriter(path.join(csvDir, 'route.csv'), 'id,name,filePath,responseKeys,errorKeys,middleware');
+    // Tool nodes for MCP tool definitions
+    const toolWriter = new BufferedCSVWriter(path.join(csvDir, 'tool.csv'), 'id,name,filePath,description');
+    // Multi-language node types share the same CSV shape (no isExported column)
+    const multiLangHeader = 'id,name,filePath,startLine,endLine,content,description';
+    const MULTI_LANG_TYPES = [
+        'Struct',
+        'Enum',
+        'Macro',
+        'Typedef',
+        'Union',
+        'Namespace',
+        'Trait',
+        'Impl',
+        'TypeAlias',
+        'Const',
+        'Static',
+        'Variable',
+        'Property',
+        'Record',
+        'Delegate',
+        'Annotation',
+        'Constructor',
+        'Template',
+        'Module',
+    ];
+    const propertyHeader = 'id,name,filePath,startLine,endLine,content,description,declaredType';
+    const multiLangWriters = new Map();
+    for (const t of MULTI_LANG_TYPES) {
+        multiLangWriters.set(t, new BufferedCSVWriter(path.join(csvDir, `${t.toLowerCase()}.csv`), t === 'Property' ? propertyHeader : multiLangHeader));
+    }
+    const codeWriterMap = {
+        Function: functionWriter,
+        Class: classWriter,
+        Interface: interfaceWriter,
+        CodeElement: codeElemWriter,
+    };
+    // Deduplicate all node types — the pipeline can produce duplicate IDs across
+    // all symbol types (Class, Method, Function, etc.), not just File nodes.
+    // A single Set covering every label prevents PK violations on COPY.
+    const seenNodeIds = new Set();
+    // --- SINGLE PASS over all nodes ---
+    for (const node of graph.iterNodes()) {
+        if (seenNodeIds.has(node.id))
+            continue;
+        seenNodeIds.add(node.id);
+        switch (node.label) {
+            case 'File': {
+                const content = await extractContent(node, contentCache);
+                await fileWriter.addRow([
+                    escapeCSVField(node.id),
+                    escapeCSVField(node.properties.name || ''),
+                    escapeCSVField(node.properties.filePath || ''),
+                    escapeCSVField(content),
+                ].join(','));
+                break;
+            }
+            case 'Folder':
+                await folderWriter.addRow([
+                    escapeCSVField(node.id),
+                    escapeCSVField(node.properties.name || ''),
+                    escapeCSVField(node.properties.filePath || ''),
+                ].join(','));
+                break;
+            case 'Community': {
+                const keywords = node.properties.keywords || [];
+                const keywordsStr = `[${keywords.map((k) => `'${k.replace(/\\/g, '\\\\').replace(/'/g, "''").replace(/,/g, '\\,')}'`).join(',')}]`;
+                await communityWriter.addRow([
+                    escapeCSVField(node.id),
+                    escapeCSVField(node.properties.name || ''),
+                    escapeCSVField(node.properties.heuristicLabel || ''),
+                    keywordsStr,
+                    escapeCSVField(node.properties.description || ''),
+                    escapeCSVField(node.properties.enrichedBy || 'heuristic'),
+                    escapeCSVNumber(node.properties.cohesion, 0),
+                    escapeCSVNumber(node.properties.symbolCount, 0),
+                ].join(','));
+                break;
+            }
+            case 'Process': {
+                const communities = node.properties.communities || [];
+                const communitiesStr = `[${communities.map((c) => `'${c.replace(/'/g, "''")}'`).join(',')}]`;
+                await processWriter.addRow([
+                    escapeCSVField(node.id),
+                    escapeCSVField(node.properties.name || ''),
+                    escapeCSVField(node.properties.heuristicLabel || ''),
+                    escapeCSVField(node.properties.processType || ''),
+                    escapeCSVNumber(node.properties.stepCount, 0),
+                    escapeCSVField(communitiesStr),
+                    escapeCSVField(node.properties.entryPointId || ''),
+                    escapeCSVField(node.properties.terminalId || ''),
+                ].join(','));
+                break;
+            }
+            case 'Method': {
+                const content = await extractContent(node, contentCache);
+                await methodWriter.addRow([
+                    escapeCSVField(node.id),
+                    escapeCSVField(node.properties.name || ''),
+                    escapeCSVField(node.properties.filePath || ''),
+                    escapeCSVNumber(node.properties.startLine, -1),
+                    escapeCSVNumber(node.properties.endLine, -1),
+                    node.properties.isExported ? 'true' : 'false',
+                    escapeCSVField(content),
+                    escapeCSVField(node.properties.description || ''),
+                    escapeCSVNumber(node.properties.parameterCount, 0),
+                    escapeCSVField(node.properties.returnType || ''),
+                ].join(','));
+                break;
+            }
+            case 'Section': {
+                const content = await extractContent(node, contentCache);
+                await sectionWriter.addRow([
+                    escapeCSVField(node.id),
+                    escapeCSVField(node.properties.name || ''),
+                    escapeCSVField(node.properties.filePath || ''),
+                    escapeCSVNumber(node.properties.startLine, -1),
+                    escapeCSVNumber(node.properties.endLine, -1),
+                    escapeCSVNumber(node.properties.level, 1),
+                    escapeCSVField(content),
+                    escapeCSVField(node.properties.description || ''),
+                ].join(','));
+                break;
+            }
+            case 'Route': {
+                const responseKeys = node.properties.responseKeys || [];
+                // LadybugDB array literal inside a quoted CSV field: escapeCSVField wraps in "..."
+                // and the array uses single-quoted elements
+                const keysStr = `[${responseKeys.map((k) => `'${k.replace(/'/g, "''")}'`).join(',')}]`;
+                const errorKeys = node.properties.errorKeys || [];
+                const errorKeysStr = `[${errorKeys.map((k) => `'${k.replace(/'/g, "''")}'`).join(',')}]`;
+                const middleware = node.properties.middleware || [];
+                const middlewareStr = `[${middleware.map((m) => `'${m.replace(/'/g, "''")}'`).join(',')}]`;
+                await routeWriter.addRow([
+                    escapeCSVField(node.id),
+                    escapeCSVField(node.properties.name || ''),
+                    escapeCSVField(node.properties.filePath || ''),
+                    escapeCSVField(keysStr),
+                    escapeCSVField(errorKeysStr),
+                    escapeCSVField(middlewareStr),
+                ].join(','));
+                break;
+            }
+            case 'Tool':
+                await toolWriter.addRow([
+                    escapeCSVField(node.id),
+                    escapeCSVField(node.properties.name || ''),
+                    escapeCSVField(node.properties.filePath || ''),
+                    escapeCSVField(node.properties.description || ''),
+                ].join(','));
+                break;
+            default: {
+                // Code element nodes (Function, Class, Interface, CodeElement)
+                const writer = codeWriterMap[node.label];
+                if (writer) {
+                    const content = await extractContent(node, contentCache);
+                    await writer.addRow([
+                        escapeCSVField(node.id),
+                        escapeCSVField(node.properties.name || ''),
+                        escapeCSVField(node.properties.filePath || ''),
+                        escapeCSVNumber(node.properties.startLine, -1),
+                        escapeCSVNumber(node.properties.endLine, -1),
+                        node.properties.isExported ? 'true' : 'false',
+                        escapeCSVField(content),
+                        escapeCSVField(node.properties.description || ''),
+                    ].join(','));
+                }
+                else {
+                    // Multi-language node types (Struct, Impl, Trait, Macro, etc.)
+                    const mlWriter = multiLangWriters.get(node.label);
+                    if (mlWriter) {
+                        const content = await extractContent(node, contentCache);
+                        await mlWriter.addRow([
+                            escapeCSVField(node.id),
+                            escapeCSVField(node.properties.name || ''),
+                            escapeCSVField(node.properties.filePath || ''),
+                            escapeCSVNumber(node.properties.startLine, -1),
+                            escapeCSVNumber(node.properties.endLine, -1),
+                            escapeCSVField(content),
+                            escapeCSVField(node.properties.description || ''),
+                            ...(node.label === 'Property'
+                                ? [escapeCSVField(node.properties.declaredType || '')]
+                                : []),
+                        ].join(','));
+                    }
+                }
+                break;
+            }
+        }
+    }
+    // Finish all node writers
+    const allWriters = [
+        fileWriter,
+        folderWriter,
+        functionWriter,
+        classWriter,
+        interfaceWriter,
+        methodWriter,
+        codeElemWriter,
+        communityWriter,
+        processWriter,
+        sectionWriter,
+        routeWriter,
+        toolWriter,
+        ...multiLangWriters.values(),
+    ];
+    await Promise.all(allWriters.map((w) => w.finish()));
+    // --- Stream relationship CSV ---
+    const relCsvPath = path.join(csvDir, 'relations.csv');
+    const relWriter = new BufferedCSVWriter(relCsvPath, 'from,to,type,confidence,reason,step');
+    for (const rel of graph.iterRelationships()) {
+        await relWriter.addRow([
+            escapeCSVField(rel.sourceId),
+            escapeCSVField(rel.targetId),
+            escapeCSVField(rel.type),
+            escapeCSVNumber(rel.confidence, 1.0),
+            escapeCSVField(rel.reason),
+            escapeCSVNumber(rel.step, 0),
+        ].join(','));
+    }
+    await relWriter.finish();
+    // Build result map — only include tables that have rows
+    const nodeFiles = new Map();
+    const tableMap = [
+        ['File', fileWriter],
+        ['Folder', folderWriter],
+        ['Function', functionWriter],
+        ['Class', classWriter],
+        ['Interface', interfaceWriter],
+        ['Method', methodWriter],
+        ['CodeElement', codeElemWriter],
+        ['Community', communityWriter],
+        ['Process', processWriter],
+        ['Section', sectionWriter],
+        ['Route', routeWriter],
+        ['Tool', toolWriter],
+        ...Array.from(multiLangWriters.entries()).map(([name, w]) => [name, w]),
+    ];
+    for (const [name, writer] of tableMap) {
+        if (writer.rows > 0) {
+            nodeFiles.set(name, {
+                csvPath: path.join(csvDir, `${name.toLowerCase()}.csv`),
+                rows: writer.rows,
+            });
+        }
+    }
+    // Restore original process listener limit
+    process.setMaxListeners(prevMax);
+    return { nodeFiles, relCsvPath, relRows: relWriter.rows };
+};

package/dist/core/lbug/extension-loader.d.ts

@@ -0,0 +1,86 @@
+/**
+ * Lifecycle policy for an optional DuckDB extension.
+ *
+ * - `auto`     — try `LOAD`, fall back to one bounded out-of-process `INSTALL`
+ *                attempt per process if `LOAD` fails. Default for analyze.
+ * - `load-only`— try `LOAD` only; never spawn an installer. Used by serve/MCP
+ *                read paths so user queries never block on a network install.
+ * - `never`    — skip the extension entirely. Operators can use this to
+ *                forcibly disable optional search features.
+ */
+export type ExtensionInstallPolicy = 'auto' | 'load-only' | 'never';
+export interface ExtensionInstallResult {
+    success: boolean;
+    timedOut: boolean;
+    message: string;
+}
+/** Snapshot of one optional extension's resolved capability state. */
+export interface ExtensionCapability {
+    name: string;
+    loaded: boolean;
+    /** Human-readable reason when `loaded` is false. */
+    reason?: string;
+}
+/** Per-call overrides applied on top of `ExtensionManager` defaults. */
+export interface ExtensionEnsureOptions {
+    policy?: ExtensionInstallPolicy;
+    installTimeoutMs?: number;
+}
+export interface ExtensionManagerOptions {
+    policy?: ExtensionInstallPolicy;
+    installTimeoutMs?: number;
+    installExtension?: (extensionName: string, timeoutMs: number) => Promise<ExtensionInstallResult>;
+    warn?: (message: string) => void;
+}
+export declare const getExtensionInstallTimeoutMs: () => number;
+export declare const getExtensionInstallChildProcessArgs: (extensionName: string, maxDbSize?: number) => string[];
+/**
+ * Run `INSTALL <extension>` in a short-lived child Node process so the parent
+ * event loop is never blocked by DuckDB's synchronous network call.
+ *
+ * The child opens its own scratch LadybugDB, executes the install, and exits.
+ * If the child exceeds `timeoutMs` the parent kills it with SIGKILL and
+ * resolves with `timedOut: true`.
+ */
+export declare const installDuckDbExtensionOutOfProcess: (extensionName: string, timeoutMs?: number) => Promise<ExtensionInstallResult>;
+/**
+ * Centralized lifecycle manager for optional LadybugDB extensions.
+ *
+ * Always tries `LOAD EXTENSION <name>` first — it is per-connection,
+ * idempotent, and never touches the network. If `LOAD` fails and the active
+ * policy permits, the manager runs a single bounded out-of-process `INSTALL`
+ * attempt per process and retries `LOAD`. Capability outcomes are cached so
+ * unavailable extensions degrade search features without ever blocking
+ * subsequent analyze or query calls.
+ *
+ * Policy precedence (most specific wins):
+ *   per-call `opts.policy` → constructor `options.policy` → env → `auto`
+ */
+export declare class ExtensionManager {
+    private readonly options;
+    private readonly capabilities;
+    private readonly installAttempted;
+    private readonly warnedKeys;
+    constructor(options?: ExtensionManagerOptions);
+    /** Reset cached capability and install state. Test-only. */
+    reset(): void;
+    /** Snapshot of currently-known optional extension capabilities. */
+    getCapabilities(): ExtensionCapability[];
+    /**
+     * Ensure an optional extension is loaded on the supplied connection.
+     *
+     * Returns `true` when the extension is usable on `query`, `false` when it
+     * is unavailable. Never throws on install failure — analyze and query
+     * paths are expected to degrade gracefully.
+     */
+    ensure(query: (sql: string) => Promise<unknown>, name: string, label: string, opts?: ExtensionEnsureOptions): Promise<boolean>;
+    private tryLoad;
+    private markLoaded;
+    private markUnavailable;
+}
+/** Process-wide singleton shared by core and pool adapters. */
+export declare const extensionManager: ExtensionManager;
+/** Snapshot of which optional DuckDB extensions are loaded in this process. */
+export declare const getExtensionCapabilities: () => ExtensionCapability[];
+/** Test-only: clear the singleton's cached capability and install state. */
+export declare const resetExtensionState: () => void;