arceus-s 1.6.4

package/dist/core/tree-sitter/parser-loader.js

@@ -0,0 +1,189 @@
+import Parser from 'tree-sitter';
+import { createRequire } from 'node:module';
+import { SupportedLanguages } from '../../_shared/index.js';
+import { logger } from '../logger.js';
+const _require = createRequire(import.meta.url);
+const ISSUES_URL = 'https://github.com/Sirius6907/Arceus/issues';
+const SOURCES = {
+    [SupportedLanguages.JavaScript]: {
+        load: () => _require('tree-sitter-javascript'),
+        unavailableNote: 'JavaScript parsing requires `tree-sitter-javascript`. ' +
+            'Check that the package and its native binding installed cleanly (`npm ci`).',
+    },
+    [SupportedLanguages.TypeScript]: {
+        load: () => _require('tree-sitter-typescript').typescript,
+        unavailableNote: 'TypeScript parsing requires `tree-sitter-typescript`. ' +
+            'Check that the package and its native binding installed cleanly (`npm ci`).',
+    },
+    [`${SupportedLanguages.TypeScript}:tsx`]: {
+        load: () => _require('tree-sitter-typescript').tsx,
+        unavailableNote: 'TSX parsing requires `tree-sitter-typescript` (re-uses the same native binding as TS).',
+    },
+    [SupportedLanguages.Python]: {
+        load: () => _require('tree-sitter-python'),
+        unavailableNote: 'Python parsing requires `tree-sitter-python`. Check the install and native binding.',
+    },
+    [SupportedLanguages.Java]: {
+        load: () => _require('tree-sitter-java'),
+        unavailableNote: 'Java parsing requires `tree-sitter-java`. Check the install and native binding.',
+    },
+    // tree-sitter-c-sharp declares `type: "module"` with `main: "bindings/node"`
+    // (no extension) and no `exports` field, which triggers Node 22's DEP0151
+    // deprecation warning on the bare-package import. The explicit subpath
+    // bypasses the deprecated ESM main-field resolution. (#1013)
+    [SupportedLanguages.CSharp]: {
+        load: () => _require('tree-sitter-c-sharp/bindings/node/index.js'),
+        unavailableNote: 'C# parsing requires `tree-sitter-c-sharp/bindings/node/index.js`. ' +
+            `If the subpath is missing, see ${ISSUES_URL}/1013.`,
+    },
+    [SupportedLanguages.CPlusPlus]: {
+        load: () => _require('tree-sitter-cpp'),
+        unavailableNote: 'C++ parsing requires `tree-sitter-cpp`. Check the install and native binding.',
+    },
+    [SupportedLanguages.Go]: {
+        load: () => _require('tree-sitter-go'),
+        unavailableNote: 'Go parsing requires `tree-sitter-go`. Check the install and native binding.',
+    },
+    [SupportedLanguages.Rust]: {
+        load: () => _require('tree-sitter-rust'),
+        unavailableNote: 'Rust parsing requires `tree-sitter-rust`. Check the install and native binding.',
+    },
+    [SupportedLanguages.PHP]: {
+        load: () => _require('tree-sitter-php').php_only,
+        unavailableNote: 'PHP parsing requires `tree-sitter-php` (the `php_only` export). ' +
+            'Check the install and native binding.',
+    },
+    [SupportedLanguages.Ruby]: {
+        load: () => _require('tree-sitter-ruby'),
+        unavailableNote: 'Ruby parsing requires `tree-sitter-ruby`. Check the install and native binding.',
+    },
+    [SupportedLanguages.Vue]: {
+        load: () => _require('tree-sitter-typescript').typescript,
+        unavailableNote: 'Vue parsing piggybacks on `tree-sitter-typescript`. Check the install and native binding.',
+    },
+    // tree-sitter-c is a required dependency, but its native binding has
+    // historically been ABI-incompatible with the bundled tree-sitter@0.21.1
+    // runtime on some platforms (#1242, #858). Loading it through the
+    // optional machinery turns a would-be segfault into a clean degradation
+    // while preserving every other language's analysis. Severity is pinned
+    // to `error` because the package is in `dependencies`: a failure here
+    // is always an install/platform problem the user needs to see, never an
+    // expected "user opted out" condition like Swift/Dart/Kotlin.
+    [SupportedLanguages.C]: {
+        load: () => _require('tree-sitter-c'),
+        optional: true,
+        severity: 'error',
+        unavailableNote: 'C parsing disabled: `tree-sitter-c` could not be loaded. ' +
+            'This package is in `dependencies` and prebuilds ship for all supported ' +
+            'platforms (win32/darwin/linux x64+arm64, Node 18/20/22), so this ' +
+            'usually indicates a corrupted install, an unsupported Node version, ' +
+            'or a native ABI mismatch with the bundled tree-sitter runtime. ' +
+            'Try `npm rebuild tree-sitter-c` or reinstalling, then re-run analyze. ' +
+            `If the failure persists, file details at ${ISSUES_URL}/1242.`,
+    },
+    // optionalDependencies — may be absent on platforms without prebuilds
+    // or when users skip optional installs.
+    [SupportedLanguages.Swift]: {
+        load: () => _require('tree-sitter-swift'),
+        optional: true,
+        unavailableNote: 'Swift parsing disabled: vendored `tree-sitter-swift` (under ' +
+            '`arc/vendor/tree-sitter-swift`) failed to load. ' +
+            'Likely cause: no prebuilt `.node` for this platform/architecture. ' +
+            `See ${ISSUES_URL}/1130.`,
+    },
+    [SupportedLanguages.Dart]: {
+        load: () => _require('tree-sitter-dart'),
+        optional: true,
+        unavailableNote: 'Dart parsing disabled: vendored `tree-sitter-dart` (under ' +
+            '`arc/vendor/tree-sitter-dart`) failed to load. ' +
+            'Likely cause: native compile failed at install (missing python3/make/g++). ' +
+            `See ${ISSUES_URL}/1125.`,
+    },
+    [SupportedLanguages.Kotlin]: {
+        load: () => _require('tree-sitter-kotlin'),
+        optional: true,
+        unavailableNote: 'Kotlin parsing disabled: `tree-sitter-kotlin` is an optionalDependency ' +
+            'and is not installed (or its native binding failed to build).',
+    },
+};
+const loadCache = new Map();
+const logged = new Set();
+const logFailure = (key, result) => {
+    if (result.ok === true)
+        return;
+    if (logged.has(key))
+        return;
+    logged.add(key);
+    const message = `[arc] ${result.note} (${result.error.message})`;
+    // Severity routes to the correct pino level. Both go to stderr (pino's
+    // default destination), so MCP stdio framing is preserved either way —
+    // the level tag drives log filtering, not channel selection.
+    if (result.severity === 'error') {
+        logger.error(message);
+    }
+    else {
+        logger.warn(message);
+    }
+};
+export const resolveLanguageKey = (language, filePath) => language === SupportedLanguages.TypeScript && filePath?.endsWith('.tsx')
+    ? `${language}:tsx`
+    : language;
+const loadGrammar = (key) => {
+    const cached = loadCache.get(key);
+    if (cached)
+        return cached;
+    const source = SOURCES[key];
+    if (!source) {
+        const result = {
+            ok: false,
+            error: new Error(`Unsupported language: ${key}`),
+            note: `No grammar registered for language key \`${key}\`. Add a row to SOURCES.`,
+            fatal: true,
+            severity: 'error',
+        };
+        loadCache.set(key, result);
+        return result;
+    }
+    let result;
+    try {
+        result = { ok: true, grammar: source.load() };
+    }
+    catch (err) {
+        const fatal = !source.optional;
+        result = {
+            ok: false,
+            error: err,
+            note: source.unavailableNote,
+            fatal,
+            severity: source.severity ?? (fatal ? 'error' : 'warn'),
+        };
+    }
+    loadCache.set(key, result);
+    if (result.ok === false)
+        logFailure(key, result);
+    return result;
+};
+export const isLanguageAvailable = (language, filePath) => loadGrammar(resolveLanguageKey(language, filePath)).ok;
+export const getLanguageGrammar = (language, filePath) => {
+    const key = resolveLanguageKey(language, filePath);
+    const result = loadGrammar(key);
+    if (result.ok === true)
+        return result.grammar;
+    // Fatal failures throw the original underlying error (preserving stack)
+    // after the note has been logged. Optional failures fall through to the
+    // standard "Unsupported language" message that callers already handle.
+    if (result.fatal)
+        throw result.error;
+    throw new Error(`Unsupported language: ${language}`);
+};
+let sharedParser = null;
+export const loadParser = async () => (sharedParser ??= new Parser());
+export const loadLanguage = async (language, filePath) => {
+    const parser = await loadParser();
+    parser.setLanguage(getLanguageGrammar(language, filePath));
+};
+export const createParserForLanguage = async (language, filePath) => {
+    const parser = new Parser();
+    parser.setLanguage(getLanguageGrammar(language, filePath));
+    return parser;
+};

package/dist/core/tree-sitter/safe-parse.d.ts

@@ -0,0 +1,6 @@
+import type Parser from 'tree-sitter';
+/**
+ * Parse `sourceText` safely on every platform. See {@link SAFE_PARSE_CHUNK_CHARS}
+ * for the underlying tree-sitter binding bug this works around.
+ */
+export declare function parseSourceSafe(parser: Parser, sourceText: string, oldTree?: Parser.Tree, options?: Parser.Options): Parser.Tree;

package/dist/core/tree-sitter/safe-parse.js

@@ -0,0 +1,32 @@
+/**
+ * tree-sitter 0.21.x's Node native binding crashes (SIGSEGV) on Windows when
+ * `parser.parse(string, …)` is handed a JS string longer than 32 767 chars.
+ * The crash happens inside the binding's V8 string-to-buffer conversion and
+ * cannot be intercepted from JavaScript. The callback (`Parser.Input`) overload
+ * pulls source in fixed-size chunks via repeated callback invocations and
+ * bypasses that conversion path entirely.
+ *
+ * Chunk size is comfortably below the boundary; any value < 32 767 works.
+ */
+const SAFE_PARSE_CHUNK_CHARS = 16 * 1024;
+/**
+ * Files at or below this length skip the callback machinery and use the
+ * direct string overload — the bug only manifests above the int16 boundary,
+ * so small inputs save the cost of N callback invocations per parse.
+ */
+const DIRECT_PARSE_LIMIT_CHARS = 16 * 1024;
+/**
+ * Parse `sourceText` safely on every platform. See {@link SAFE_PARSE_CHUNK_CHARS}
+ * for the underlying tree-sitter binding bug this works around.
+ */
+export function parseSourceSafe(parser, sourceText, oldTree, options) {
+    if (sourceText.length <= DIRECT_PARSE_LIMIT_CHARS) {
+        return parser.parse(sourceText, oldTree, options);
+    }
+    const input = (index) => {
+        if (index >= sourceText.length)
+            return null;
+        return sourceText.slice(index, index + SAFE_PARSE_CHUNK_CHARS);
+    };
+    return parser.parse(input, oldTree, options);
+}

package/dist/core/wiki/cursor-client.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Cursor CLI Client for Wiki Generation
+ *
+ * Wrapper for the Cursor headless CLI (`agent` command).
+ * Uses print mode for non-interactive LLM calls.
+ *
+ * Docs: https://cursor.com/docs/cli/headless
+ */
+import type { LLMResponse, CallLLMOptions } from './llm-client.js';
+export interface CursorConfig {
+    model?: string;
+    workingDirectory?: string;
+}
+/**
+ * Detect if Cursor CLI is available in PATH.
+ * Returns the binary name if found ('agent'), null otherwise.
+ * Result is cached after the first call.
+ */
+export declare function detectCursorCLI(): string | null;
+/**
+ * Resolve Cursor CLI configuration.
+ * Model is optional - if not provided, Cursor CLI uses its default (auto).
+ */
+export declare function resolveCursorConfig(overrides?: Partial<CursorConfig>): CursorConfig;
+/**
+ * Call the Cursor CLI in print mode.
+ *
+ * Uses `agent -p --output-format text` for clean non-streaming output.
+ * The prompt is passed as the final CLI argument.
+ */
+export declare function callCursorLLM(prompt: string, config: CursorConfig, systemPrompt?: string, options?: CallLLMOptions): Promise<LLMResponse>;

package/dist/core/wiki/cursor-client.js

@@ -0,0 +1,123 @@
+/**
+ * Cursor CLI Client for Wiki Generation
+ *
+ * Wrapper for the Cursor headless CLI (`agent` command).
+ * Uses print mode for non-interactive LLM calls.
+ *
+ * Docs: https://cursor.com/docs/cli/headless
+ */
+import { spawn, execSync } from 'child_process';
+import { logger } from '../logger.js';
+function isVerbose() {
+    return process.env.ARC_VERBOSE === '1';
+}
+function verboseLog(...args) {
+    if (isVerbose()) {
+        logger.info({ args }, '[cursor-cli]');
+    }
+}
+let cachedCursorBin;
+/**
+ * Detect if Cursor CLI is available in PATH.
+ * Returns the binary name if found ('agent'), null otherwise.
+ * Result is cached after the first call.
+ */
+export function detectCursorCLI() {
+    if (cachedCursorBin !== undefined)
+        return cachedCursorBin;
+    try {
+        execSync('agent --version', { stdio: 'ignore' });
+        cachedCursorBin = 'agent';
+    }
+    catch {
+        cachedCursorBin = null;
+    }
+    return cachedCursorBin;
+}
+/**
+ * Resolve Cursor CLI configuration.
+ * Model is optional - if not provided, Cursor CLI uses its default (auto).
+ */
+export function resolveCursorConfig(overrides) {
+    return {
+        model: overrides?.model,
+        workingDirectory: overrides?.workingDirectory,
+    };
+}
+/**
+ * Call the Cursor CLI in print mode.
+ *
+ * Uses `agent -p --output-format text` for clean non-streaming output.
+ * The prompt is passed as the final CLI argument.
+ */
+export async function callCursorLLM(prompt, config, systemPrompt, options) {
+    const cursorBin = detectCursorCLI();
+    if (!cursorBin) {
+        throw new Error('Cursor CLI not found. Install it from https://cursor.com/docs/cli/installation');
+    }
+    // Always use text format to get clean output without agent narration/thinking.
+    // stream-json captures assistant messages which include "Let me explore..." narration
+    // that pollutes the actual content when using thinking models.
+    const fullPrompt = systemPrompt ? `${systemPrompt}\n\n---\n\n${prompt}` : prompt;
+    const args = ['-p', '--output-format', 'text'];
+    if (config.model) {
+        args.push('--model', config.model);
+    }
+    // Add the prompt as the final argument
+    args.push(fullPrompt);
+    verboseLog('Spawning:', cursorBin, args.slice(0, -1).join(' '), '[prompt length:', fullPrompt.length, 'chars]');
+    verboseLog('Working directory:', config.workingDirectory || process.cwd());
+    if (config.model) {
+        verboseLog('Model:', config.model);
+    }
+    else {
+        verboseLog('Model: auto (default)');
+    }
+    const startTime = Date.now();
+    return new Promise((resolve, reject) => {
+        const child = spawn(cursorBin, args, {
+            cwd: config.workingDirectory || process.cwd(),
+            stdio: ['pipe', 'pipe', 'pipe'],
+            env: {
+                ...process.env,
+                // Ensure non-interactive mode
+                CI: '1',
+            },
+        });
+        verboseLog('Process spawned with PID:', child.pid);
+        let stdout = '';
+        let stderr = '';
+        // Text mode - collect all output, report progress based on output size
+        child.stdout.on('data', (chunk) => {
+            const chunkStr = chunk.toString();
+            stdout += chunkStr;
+            verboseLog(`[stdout] received ${chunkStr.length} chars, total: ${stdout.length}`);
+            // Report progress if callback provided
+            if (options?.onChunk) {
+                options.onChunk(stdout.length);
+            }
+        });
+        child.on('close', (code) => {
+            const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+            verboseLog(`Process exited with code ${code} after ${elapsed}s`);
+            verboseLog(`stdout length: ${stdout.length} chars`);
+            if (code !== 0) {
+                verboseLog('stderr:', stderr);
+                reject(new Error(`Cursor CLI exited with code ${code}: ${stderr}`));
+                return;
+            }
+            resolve({ content: stdout.trim() });
+        });
+        child.stderr.on('data', (chunk) => {
+            const chunkStr = chunk.toString();
+            stderr += chunkStr;
+            verboseLog('[stderr]', chunkStr.trim());
+        });
+        child.on('error', (err) => {
+            verboseLog('Spawn error:', err.message);
+            reject(new Error(`Failed to spawn Cursor CLI: ${err.message}`));
+        });
+        // Close stdin immediately since we pass prompt as argument
+        child.stdin.end();
+    });
+}

package/dist/core/wiki/generator.d.ts

@@ -0,0 +1,129 @@
+/**
+ * Wiki Generator
+ *
+ * Orchestrates the full wiki generation pipeline:
+ *   Phase 0: Validate prerequisites + gather graph structure
+ *   Phase 1: Build module tree (one LLM call)
+ *   Phase 2: Generate module pages (one LLM call per module, bottom-up)
+ *   Phase 3: Generate overview page
+ *
+ * Supports incremental updates via git diff + module-file mapping.
+ */
+import { type LLMConfig } from './llm-client.js';
+export interface WikiOptions {
+    force?: boolean;
+    maxTokensPerModule?: number;
+    concurrency?: number;
+    /** If true, stop after building module tree for user review */
+    reviewOnly?: boolean;
+}
+export interface WikiMeta {
+    fromCommit: string;
+    generatedAt: string;
+    model: string;
+    moduleFiles: Record<string, string[]>;
+    moduleTree: ModuleTreeNode[];
+}
+export interface ModuleTreeNode {
+    name: string;
+    slug: string;
+    files: string[];
+    children?: ModuleTreeNode[];
+}
+export type ProgressCallback = (phase: string, percent: number, detail?: string) => void;
+export interface WikiRunResult {
+    pagesGenerated: number;
+    mode: 'full' | 'incremental' | 'up-to-date';
+    failedModules: string[];
+    moduleTree?: ModuleTreeNode[];
+}
+export declare class WikiGenerator {
+    private repoPath;
+    private storagePath;
+    private wikiDir;
+    private lbugPath;
+    private llmConfig;
+    private maxTokensPerModule;
+    private concurrency;
+    private options;
+    private onProgress;
+    private failedModules;
+    constructor(repoPath: string, storagePath: string, lbugPath: string, llmConfig: LLMConfig, options?: WikiOptions, onProgress?: ProgressCallback);
+    private lastPercent;
+    /**
+     * Create streaming options that report LLM progress to the progress bar.
+     *
+     * Progress calculation:
+     * - If fixedPercent is provided, we show incremental progress within that phase
+     *   based on token generation (e.g., grouping at 15% → 15-28%)
+     * - If fixedPercent is NOT provided, we only update the label with token count
+     *   but keep the current percentage (avoids fluctuation during module generation)
+     *
+     * Also touches the DB connection periodically to prevent idle timeout.
+     */
+    private streamOpts;
+    /**
+     * Route LLM call to the appropriate provider (OpenAI-compatible or Cursor CLI).
+     */
+    private invokeLLM;
+    /**
+     * Main entry point. Runs the full pipeline or incremental update.
+     */
+    run(): Promise<WikiRunResult>;
+    private ensureHTMLViewer;
+    private fullGeneration;
+    private buildModuleTree;
+    /**
+     * Parse LLM grouping response. Validates all files are assigned.
+     */
+    private parseGroupingResponse;
+    /**
+     * Fallback grouping by top-level directory when LLM parsing fails.
+     */
+    private fallbackGrouping;
+    /**
+     * Split a large module into sub-modules by subdirectory.
+     * Uses the full subDir path for naming to avoid slug collisions
+     * (e.g., "synapse-screen/src" vs "synapse-core/src").
+     */
+    private splitBySubdirectory;
+    /**
+     * Generate a leaf module page from source code + graph data.
+     */
+    private generateLeafPage;
+    /**
+     * Generate a parent module page from children's documentation.
+     */
+    private generateParentPage;
+    private generateOverview;
+    private incrementalUpdate;
+    private getCurrentCommit;
+    /**
+     * Check if fromCommit is an ancestor of toCommit (reachable in git history).
+     * Returns false if commits are on divergent branches or fromCommit doesn't exist.
+     */
+    private isCommitReachable;
+    private getChangedFiles;
+    private readSourceFiles;
+    private truncateSource;
+    private estimateModuleTokens;
+    private readProjectInfo;
+    private extractModuleFiles;
+    private countModules;
+    /**
+     * Flatten the module tree into leaf nodes and parent nodes.
+     * Leaves can be processed in parallel; parents must wait for children.
+     */
+    private flattenModuleTree;
+    /**
+     * Run async tasks in parallel with a concurrency limit and adaptive rate limiting.
+     * If a 429 rate limit is hit, concurrency is temporarily reduced.
+     */
+    private runParallel;
+    private findNodeBySlug;
+    private slugify;
+    private fileExists;
+    private loadWikiMeta;
+    private saveWikiMeta;
+    private saveModuleTree;
+}