arceus-s 1.6.4

package/dist/core/wiki/prompts.js

@@ -0,0 +1,181 @@
+/**
+ * LLM Prompt Templates for Wiki Generation
+ *
+ * All prompts produce deterministic, source-grounded documentation.
+ * Templates use {{PLACEHOLDER}} substitution.
+ */
+// ─── Grouping Prompt ──────────────────────────────────────────────────
+export const GROUPING_SYSTEM_PROMPT = `You are a documentation architect. Given a list of source files with their exported symbols, group them into logical documentation modules.
+
+Rules:
+- Each module should represent a cohesive feature, layer, or domain
+- Every file must appear in exactly one module
+- Module names should be human-readable (e.g. "Authentication", "Database Layer", "API Routes")
+- Aim for 5-15 modules for a typical project. Fewer for small projects, more for large ones
+- Group by functionality, not by file type or directory structure alone
+- Do NOT create modules for tests, configs, or non-source files`;
+export const GROUPING_USER_PROMPT = `Group these source files into documentation modules.
+
+**Files and their exports:**
+{{FILE_LIST}}
+
+**Directory structure:**
+{{DIRECTORY_TREE}}
+
+Respond with ONLY a JSON object mapping module names to file path arrays. No markdown, no explanation.
+Example format:
+{
+  "Authentication": ["src/auth/login.ts", "src/auth/session.ts"],
+  "Database": ["src/db/connection.ts", "src/db/models.ts"]
+}`;
+// ─── Leaf Module Prompt ───────────────────────────────────────────────
+export const MODULE_SYSTEM_PROMPT = `You are a technical documentation writer. Write clear, developer-focused documentation for a code module.
+
+Rules:
+- Output ONLY the documentation content — no meta-commentary like "I've written...", "Here's the documentation...", "The documentation covers...", or similar
+- Start directly with the module heading and content
+- Reference actual function names, class names, and code patterns — do NOT invent APIs
+- Use the call graph and execution flow data for accuracy, but do NOT mechanically list every edge
+- Include Mermaid diagrams only when they genuinely help understanding. Keep them small (5-10 nodes max)
+- Structure the document however makes sense for this module — there is no mandatory format
+- Write for a developer who needs to understand and contribute to this code`;
+export const MODULE_USER_PROMPT = `Write documentation for the **{{MODULE_NAME}}** module.
+
+## Source Code
+
+{{SOURCE_CODE}}
+
+## Call Graph & Execution Flows (reference for accuracy)
+
+Internal calls: {{INTRA_CALLS}}
+Outgoing calls: {{OUTGOING_CALLS}}
+Incoming calls: {{INCOMING_CALLS}}
+Execution flows: {{PROCESSES}}
+
+---
+
+Write comprehensive documentation for this module. Cover its purpose, how it works, its key components, and how it connects to the rest of the codebase. Use whatever structure best fits this module — you decide the sections and headings. Include a Mermaid diagram only if it genuinely clarifies the architecture.`;
+// ─── Parent Module Prompt ─────────────────────────────────────────────
+export const PARENT_SYSTEM_PROMPT = `You are a technical documentation writer. Write a summary page for a module that contains sub-modules. Synthesize the children's documentation — do not re-read source code.
+
+Rules:
+- Output ONLY the documentation content — no meta-commentary like "I've written...", "Here's the documentation...", "The documentation covers...", or similar
+- Start directly with the module heading and content
+- Reference actual components from the child modules
+- Focus on how the sub-modules work together, not repeating their individual docs
+- Keep it concise — the reader can click through to child pages for detail
+- Include a Mermaid diagram only if it genuinely clarifies how the sub-modules relate`;
+export const PARENT_USER_PROMPT = `Write documentation for the **{{MODULE_NAME}}** module, which contains these sub-modules:
+
+{{CHILDREN_DOCS}}
+
+Cross-module calls: {{CROSS_MODULE_CALLS}}
+Shared execution flows: {{CROSS_PROCESSES}}
+
+---
+
+Write a concise overview of this module group. Explain its purpose, how the sub-modules fit together, and the key workflows that span them. Link to sub-module pages (e.g. \`[Sub-module Name](sub-module-slug.md)\`) rather than repeating their content. Use whatever structure fits best.`;
+// ─── Overview Prompt ──────────────────────────────────────────────────
+export const OVERVIEW_SYSTEM_PROMPT = `You are a technical documentation writer. Write the top-level overview page for a repository wiki. This is the first page a new developer sees.
+
+Rules:
+- Output ONLY the documentation content — no meta-commentary like "I've written...", "Here's the documentation...", "The page has been rewritten...", or similar
+- Start directly with the project heading and content
+- Be clear and welcoming — this is the entry point to the entire codebase
+- Reference actual module names so readers can navigate to their docs
+- Include a high-level Mermaid architecture diagram showing only the most important modules and their relationships (max 10 nodes). A new dev should grasp it in 10 seconds
+- Do NOT create module index tables or list every module with descriptions — just link to module pages naturally within the text
+- Use the inter-module edges and execution flow data for accuracy, but do NOT dump them raw`;
+export const OVERVIEW_USER_PROMPT = `Write the overview page for this repository's wiki.
+
+## Project Info
+
+{{PROJECT_INFO}}
+
+## Module Summaries
+
+{{MODULE_SUMMARIES}}
+
+## Reference Data (for accuracy — do not reproduce verbatim)
+
+Inter-module call edges: {{MODULE_EDGES}}
+Key system flows: {{TOP_PROCESSES}}
+
+---
+
+Write a clear overview of this project: what it does, how it's architected, and the key end-to-end flows. Include a simple Mermaid architecture diagram (max 10 nodes, big-picture only). Link to module pages (e.g. \`[Module Name](module-slug.md)\`) naturally in the text rather than listing them in a table. If project config was provided, include brief setup instructions. Structure the page however reads best.`;
+// ─── Template Substitution Helper ─────────────────────────────────────
+/**
+ * Replace {{PLACEHOLDER}} tokens in a template string.
+ */
+export function fillTemplate(template, vars) {
+    let result = template;
+    for (const [key, value] of Object.entries(vars)) {
+        result = result.replaceAll(`{{${key}}}`, value);
+    }
+    return result;
+}
+// ─── Formatting Helpers ───────────────────────────────────────────────
+/**
+ * Format file list with exports for the grouping prompt.
+ */
+export function formatFileListForGrouping(files) {
+    return files
+        .map((f) => {
+        const exports = f.symbols.length > 0
+            ? f.symbols.map((s) => `${s.name} (${s.type})`).join(', ')
+            : 'no exports';
+        return `- ${f.filePath}: ${exports}`;
+    })
+        .join('\n');
+}
+/**
+ * Build a directory tree string from file paths.
+ */
+export function formatDirectoryTree(filePaths) {
+    const dirs = new Set();
+    for (const fp of filePaths) {
+        const parts = fp.replace(/\\/g, '/').split('/');
+        for (let i = 1; i < parts.length; i++) {
+            dirs.add(parts.slice(0, i).join('/'));
+        }
+    }
+    const sorted = Array.from(dirs).sort();
+    if (sorted.length === 0)
+        return '(flat structure)';
+    return (sorted.slice(0, 50).join('\n') +
+        (sorted.length > 50 ? `\n... and ${sorted.length - 50} more directories` : ''));
+}
+/**
+ * Format call edges as readable text.
+ */
+export function formatCallEdges(edges) {
+    if (edges.length === 0)
+        return 'None';
+    return edges
+        .slice(0, 30)
+        .map((e) => `${e.fromName} (${shortPath(e.fromFile)}) → ${e.toName} (${shortPath(e.toFile)})`)
+        .join('\n');
+}
+/**
+ * Format process traces as readable text.
+ */
+export function formatProcesses(processes) {
+    if (processes.length === 0)
+        return 'No execution flows detected for this module.';
+    return processes
+        .map((p) => {
+        const stepsText = p.steps
+            .map((s) => `  ${s.step}. ${s.name} (${shortPath(s.filePath)})`)
+            .join('\n');
+        return `**${p.label}** (${p.type}):\n${stepsText}`;
+    })
+        .join('\n\n');
+}
+/**
+ * Shorten a file path for readability.
+ */
+function shortPath(fp) {
+    const parts = fp.replace(/\\/g, '/').split('/');
+    return parts.length > 3 ? parts.slice(-3).join('/') : fp;
+}

package/dist/lib/utils.d.ts

@@ -0,0 +1 @@
+export declare const generateId: (label: string, name: string) => string;

package/dist/lib/utils.js

@@ -0,0 +1,3 @@
+export const generateId = (label, name) => {
+    return `${label}:${name}`;
+};

package/dist/mcp/compatible-stdio-transport.d.ts

@@ -0,0 +1,25 @@
+import type { Transport, TransportSendOptions } from '@modelcontextprotocol/sdk/shared/transport.js';
+import { type JSONRPCMessage } from '@modelcontextprotocol/sdk/types.js';
+export type StdioFraming = 'content-length' | 'newline';
+export declare class CompatibleStdioServerTransport implements Transport {
+    private readonly _stdin;
+    private readonly _stdout;
+    private _readBuffer;
+    private _started;
+    private _framing;
+    onmessage?: (message: JSONRPCMessage) => void;
+    onerror?: (error: Error) => void;
+    onclose?: () => void;
+    constructor(_stdin?: NodeJS.ReadableStream, _stdout?: NodeJS.WritableStream);
+    private readonly _ondata;
+    private readonly _onerror;
+    start(): Promise<void>;
+    private detectFraming;
+    private discardBufferedInput;
+    private readContentLengthMessage;
+    private readNewlineMessage;
+    private readMessage;
+    private processReadBuffer;
+    close(): Promise<void>;
+    send(message: JSONRPCMessage, _options?: TransportSendOptions): Promise<void>;
+}

package/dist/mcp/compatible-stdio-transport.js

@@ -0,0 +1,206 @@
+import process from 'node:process';
+import { JSONRPCMessageSchema } from '@modelcontextprotocol/sdk/types.js';
+import { withMcpWrite } from './stdio-context.js';
+function deserializeMessage(raw) {
+    return JSONRPCMessageSchema.parse(JSON.parse(raw));
+}
+function serializeNewlineMessage(message) {
+    return `${JSON.stringify(message)}\n`;
+}
+function serializeContentLengthMessage(message) {
+    const body = JSON.stringify(message);
+    return `Content-Length: ${Buffer.byteLength(body, 'utf8')}\r\n\r\n${body}`;
+}
+function findHeaderEnd(buffer) {
+    const crlfEnd = buffer.indexOf('\r\n\r\n');
+    if (crlfEnd !== -1) {
+        return { index: crlfEnd, separatorLength: 4 };
+    }
+    const lfEnd = buffer.indexOf('\n\n');
+    if (lfEnd !== -1) {
+        return { index: lfEnd, separatorLength: 2 };
+    }
+    return null;
+}
+function looksLikeContentLength(buffer) {
+    if (buffer.length < 14) {
+        return false;
+    }
+    const probe = buffer.toString('utf8', 0, Math.min(buffer.length, 32));
+    return /^content-length\s*:/i.test(probe);
+}
+const MAX_BUFFER_SIZE = 10 * 1024 * 1024; // 10 MB — generous for JSON-RPC
+export class CompatibleStdioServerTransport {
+    _stdin;
+    _stdout;
+    _readBuffer;
+    _started = false;
+    _framing = null;
+    onmessage;
+    onerror;
+    onclose;
+    constructor(_stdin = process.stdin, _stdout = process.stdout) {
+        this._stdin = _stdin;
+        this._stdout = _stdout;
+    }
+    _ondata = (chunk) => {
+        this._readBuffer = this._readBuffer ? Buffer.concat([this._readBuffer, chunk]) : chunk;
+        if (this._readBuffer.length > MAX_BUFFER_SIZE) {
+            this.onerror?.(new Error(`Read buffer exceeded maximum size (${MAX_BUFFER_SIZE} bytes)`));
+            this.discardBufferedInput();
+            return;
+        }
+        this.processReadBuffer();
+    };
+    _onerror = (error) => {
+        this.onerror?.(error);
+    };
+    async start() {
+        if (this._started) {
+            throw new Error('CompatibleStdioServerTransport already started!');
+        }
+        this._started = true;
+        this._stdin.on('data', this._ondata);
+        this._stdin.on('error', this._onerror);
+    }
+    detectFraming() {
+        if (!this._readBuffer || this._readBuffer.length === 0) {
+            return null;
+        }
+        const firstByte = this._readBuffer[0];
+        if (firstByte === 0x7b || firstByte === 0x5b) {
+            return 'newline';
+        }
+        if (looksLikeContentLength(this._readBuffer)) {
+            return 'content-length';
+        }
+        return null;
+    }
+    discardBufferedInput() {
+        this._readBuffer = undefined;
+        this._framing = null;
+    }
+    readContentLengthMessage() {
+        if (!this._readBuffer) {
+            return null;
+        }
+        const header = findHeaderEnd(this._readBuffer);
+        if (header === null) {
+            return null;
+        }
+        const headerText = this._readBuffer
+            .toString('utf8', 0, header.index)
+            .replace(/\r\n/g, '\n')
+            .replace(/\r/g, '\n');
+        const match = headerText.match(/(?:^|\n)content-length\s*:\s*(\d+)/i);
+        if (!match) {
+            this.discardBufferedInput();
+            throw new Error('Missing Content-Length header from MCP client');
+        }
+        const contentLength = Number.parseInt(match[1], 10);
+        if (!Number.isFinite(contentLength) || contentLength < 0) {
+            this.discardBufferedInput();
+            throw new Error('Invalid Content-Length header from MCP client');
+        }
+        if (contentLength > MAX_BUFFER_SIZE) {
+            this.discardBufferedInput();
+            throw new Error(`Content-Length ${contentLength} exceeds maximum allowed size (${MAX_BUFFER_SIZE} bytes)`);
+        }
+        const bodyStart = header.index + header.separatorLength;
+        const bodyEnd = bodyStart + contentLength;
+        if (this._readBuffer.length < bodyEnd) {
+            return null;
+        }
+        const body = this._readBuffer.toString('utf8', bodyStart, bodyEnd);
+        this._readBuffer = this._readBuffer.subarray(bodyEnd);
+        return deserializeMessage(body);
+    }
+    readNewlineMessage() {
+        if (!this._readBuffer) {
+            return null;
+        }
+        while (true) {
+            const newlineIndex = this._readBuffer.indexOf('\n');
+            if (newlineIndex === -1) {
+                return null;
+            }
+            const line = this._readBuffer.toString('utf8', 0, newlineIndex).replace(/\r$/, '');
+            this._readBuffer = this._readBuffer.subarray(newlineIndex + 1);
+            if (line.trim().length === 0) {
+                continue;
+            }
+            return deserializeMessage(line);
+        }
+    }
+    readMessage() {
+        if (!this._readBuffer || this._readBuffer.length === 0) {
+            return null;
+        }
+        if (this._framing === null) {
+            this._framing = this.detectFraming();
+            if (this._framing === null) {
+                return null;
+            }
+        }
+        return this._framing === 'content-length'
+            ? this.readContentLengthMessage()
+            : this.readNewlineMessage();
+    }
+    processReadBuffer() {
+        while (true) {
+            try {
+                const message = this.readMessage();
+                if (message === null) {
+                    break;
+                }
+                this.onmessage?.(message);
+            }
+            catch (error) {
+                this.onerror?.(error);
+                break;
+            }
+        }
+    }
+    async close() {
+        this._stdin.off('data', this._ondata);
+        this._stdin.off('error', this._onerror);
+        const remainingDataListeners = this._stdin.listenerCount('data');
+        if (remainingDataListeners === 0) {
+            this._stdin.pause();
+        }
+        this._started = false;
+        this._readBuffer = undefined;
+        this.onclose?.();
+    }
+    send(message, _options) {
+        return new Promise((resolve, reject) => {
+            if (!this._started) {
+                reject(new Error('Transport is closed'));
+                return;
+            }
+            const payload = this._framing === 'newline'
+                ? serializeNewlineMessage(message)
+                : serializeContentLengthMessage(message);
+            const onError = (error) => {
+                this._stdout.removeListener('error', onError);
+                reject(error);
+            };
+            this._stdout.on('error', onError);
+            // Tag the write with the MCP transport context so the sentinel
+            // (server.ts createStdoutSentinel Proxy) recognizes it as a legitimate
+            // JSON-RPC frame and passes it through to the real stdout instead of
+            // redirecting to stderr.
+            const writeOk = withMcpWrite(() => this._stdout.write(payload));
+            if (writeOk) {
+                this._stdout.removeListener('error', onError);
+                resolve();
+            }
+            else {
+                this._stdout.once('drain', () => {
+                    this._stdout.removeListener('error', onError);
+                    resolve();
+                });
+            }
+        });
+    }
+}

package/dist/mcp/core/embedder.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Embedder Module (Read-Only)
+ *
+ * Singleton factory for transformers.js embedding pipeline.
+ * For MCP, we only need to compute query embeddings, not batch embed.
+ */
+import { type FeatureExtractionPipeline } from '@huggingface/transformers';
+/**
+ * Initialize the embedding model (lazy, on first search)
+ */
+export declare const initEmbedder: () => Promise<FeatureExtractionPipeline>;
+/**
+ * Check if embedder is ready
+ */
+export declare const isEmbedderReady: () => boolean;
+/**
+ * Embed a query text for semantic search
+ */
+export declare const embedQuery: (query: string) => Promise<number[]>;
+/**
+ * Get embedding dimensions
+ */
+export declare const getEmbeddingDims: () => number;
+/**
+ * Cleanup embedder
+ */
+export declare const disposeEmbedder: () => Promise<void>;

package/dist/mcp/core/embedder.js

@@ -0,0 +1,145 @@
+/**
+ * Embedder Module (Read-Only)
+ *
+ * Singleton factory for transformers.js embedding pipeline.
+ * For MCP, we only need to compute query embeddings, not batch embed.
+ */
+import { pipeline, env } from '@huggingface/transformers';
+import { isHttpMode, getHttpDimensions, httpEmbedQuery, } from '../../core/embeddings/http-client.js';
+import { resolveEmbeddingConfig } from '../../core/embeddings/config.js';
+import { applyHfEnvOverrides, isHfDownloadFailure, withHfDownloadRetry, } from '../../core/embeddings/hf-env.js';
+import { silenceStdout, restoreStdout, realStderrWrite } from '../../core/lbug/pool-adapter.js';
+import { logger } from '../../core/logger.js';
+// Model config
+const MODEL_ID = 'Snowflake/snowflake-arctic-embed-xs';
+// Module-level state for singleton pattern
+let embedderInstance = null;
+let isInitializing = false;
+let initPromise = null;
+/**
+ * Initialize the embedding model (lazy, on first search)
+ */
+export const initEmbedder = async () => {
+    if (isHttpMode()) {
+        throw new Error('initEmbedder() should not be called in HTTP mode.');
+    }
+    if (embedderInstance) {
+        return embedderInstance;
+    }
+    if (isInitializing && initPromise) {
+        return initPromise;
+    }
+    isInitializing = true;
+    initPromise = (async () => {
+        try {
+            env.allowLocalModels = false;
+            // Bridge user-controlled env vars to transformers.js: HF_HOME →
+            // env.cacheDir, HF_ENDPOINT → env.remoteHost (#1205). Centralised in
+            // applyHfEnvOverrides so this MCP entry point behaves identically to
+            // the analyze pipeline embedder.
+            applyHfEnvOverrides(env);
+            const embeddingConfig = resolveEmbeddingConfig();
+            logger.info('Arceus: Loading embedding model (first search may take a moment)...');
+            const devicesToTry = embeddingConfig.device === 'dml' || embeddingConfig.device === 'cuda'
+                ? [embeddingConfig.device, 'cpu']
+                : ['cpu'];
+            for (const device of devicesToTry) {
+                try {
+                    // Silence stdout and stderr during model load — ONNX Runtime and transformers.js
+                    // may write progress/init messages that corrupt MCP stdio protocol or produce
+                    // noisy warnings (e.g. node assignment to execution providers).
+                    // Use the centralized silenceStdout() to avoid conflicts with pool-adapter's
+                    // own stdout patching (independent patching caused restore-order bugs).
+                    silenceStdout();
+                    process.stderr.write = (() => true);
+                    try {
+                        embedderInstance = await withHfDownloadRetry(() => pipeline('feature-extraction', MODEL_ID, {
+                            device: device,
+                            dtype: 'fp32',
+                            session_options: {
+                                logSeverityLevel: 3,
+                                intraOpNumThreads: embeddingConfig.threads,
+                                interOpNumThreads: 1,
+                                executionMode: 'sequential',
+                            },
+                        }));
+                    }
+                    finally {
+                        restoreStdout();
+                        process.stderr.write = realStderrWrite;
+                    }
+                    logger.info({ device }, 'Arceus: Embedding model loaded');
+                    return embedderInstance;
+                }
+                catch (deviceError) {
+                    // Network errors and circuit-open errors are not device-specific —
+                    // they will fail the same way on every device. Rethrow immediately
+                    // with actionable HF_ENDPOINT guidance rather than silently falling
+                    // back to the next device.
+                    const errMsg = deviceError instanceof Error ? deviceError.message : String(deviceError);
+                    if (isHfDownloadFailure(errMsg)) {
+                        const endpointHint = process.env.HF_ENDPOINT
+                            ? `The configured endpoint (${process.env.HF_ENDPOINT}) may be unreachable.`
+                            : `huggingface.co may be unreachable from your network.\n` +
+                                `  Set HF_ENDPOINT to a mirror and retry:\n` +
+                                `    HF_ENDPOINT=https://hf-mirror.com npx arc analyze --embeddings\n` +
+                                `    (Windows: set HF_ENDPOINT=https://hf-mirror.com && npx arc analyze --embeddings)`;
+                        throw new Error(`Failed to download embedding model: ${errMsg}\n  ${endpointHint}`);
+                    }
+                    if (device === 'cpu')
+                        throw new Error('Failed to load embedding model');
+                }
+            }
+            throw new Error('No suitable device found');
+        }
+        catch (error) {
+            isInitializing = false;
+            initPromise = null;
+            embedderInstance = null;
+            throw error;
+        }
+        finally {
+            isInitializing = false;
+        }
+    })();
+    return initPromise;
+};
+/**
+ * Check if embedder is ready
+ */
+export const isEmbedderReady = () => isHttpMode() || embedderInstance !== null;
+/**
+ * Embed a query text for semantic search
+ */
+export const embedQuery = async (query) => {
+    if (isHttpMode()) {
+        return httpEmbedQuery(query);
+    }
+    const embedder = await initEmbedder();
+    const result = await embedder(query, {
+        pooling: 'mean',
+        normalize: true,
+    });
+    return Array.from(result.data);
+};
+/**
+ * Get embedding dimensions
+ */
+export const getEmbeddingDims = () => {
+    return getHttpDimensions() ?? 384;
+};
+/**
+ * Cleanup embedder
+ */
+export const disposeEmbedder = async () => {
+    if (embedderInstance) {
+        try {
+            if ('dispose' in embedderInstance && typeof embedderInstance.dispose === 'function') {
+                await embedderInstance.dispose();
+            }
+        }
+        catch { }
+        embedderInstance = null;
+        initPromise = null;
+    }
+};

package/dist/mcp/core/lbug-adapter.d.ts

@@ -0,0 +1,11 @@
+/**
+ * LadybugDB connection pool — re-exported from core.
+ *
+ * KEEP THIS FILE. It is intentionally a shim re-export of
+ * `../../core/lbug/pool-adapter.js`. The MCP test suite uses this path as
+ * a vi.mock seam so unit tests can stub LadybugDB without affecting other
+ * importers of `core/lbug/pool-adapter.js` (which is shared with the
+ * analyze pipeline). New non-test code MAY import from `pool-adapter.js`
+ * directly, but the shim must continue to exist for the mock seam to work.
+ */
+export * from '../../core/lbug/pool-adapter.js';

package/dist/mcp/core/lbug-adapter.js

@@ -0,0 +1,11 @@
+/**
+ * LadybugDB connection pool — re-exported from core.
+ *
+ * KEEP THIS FILE. It is intentionally a shim re-export of
+ * `../../core/lbug/pool-adapter.js`. The MCP test suite uses this path as
+ * a vi.mock seam so unit tests can stub LadybugDB without affecting other
+ * importers of `core/lbug/pool-adapter.js` (which is shared with the
+ * analyze pipeline). New non-test code MAY import from `pool-adapter.js`
+ * directly, but the shim must continue to exist for the mock seam to work.
+ */
+export * from '../../core/lbug/pool-adapter.js';