cmp-standards 3.1.2 → 3.3.1

package/dist/services/CodeWikiIndexer.js

@@ -0,0 +1,664 @@
+/**
+ * @file CodeWiki Indexer Service - cmp-standards v3.3
+ * @description Automated code indexing service for knowledge graph generation
+ *
+ * Features:
+ * - Scans project files and extracts metadata
+ * - Parses TypeScript/JavaScript for symbols, imports, exports
+ * - Detects dependencies and relationships
+ * - Stores indexed data in Turso database
+ * - Supports incremental indexing (only changed files)
+ */
+import { createHash } from 'crypto';
+import { existsSync, statSync, readFileSync } from 'fs';
+import { relative, basename, extname, resolve, dirname } from 'path';
+import { glob } from 'glob';
+import { turso } from '../db/turso-client.js';
+import { getLogger } from '../utils/logger.js';
+import { createFileMetadata, createCodeStructure, createCodeDependency, createIndexRun, detectLanguage, isTestFile, isTypeDefinitionFile, } from '../schema/codewiki-types.js';
+const logger = getLogger();
+// =============================================================================
+// Constants
+// =============================================================================
+const DEFAULT_INCLUDE = [
+    '**/*.ts',
+    '**/*.tsx',
+    '**/*.js',
+    '**/*.jsx',
+    '**/*.mjs',
+    '**/*.cjs',
+];
+const DEFAULT_EXCLUDE = [
+    '**/node_modules/**',
+    '**/dist/**',
+    '**/build/**',
+    '**/.git/**',
+    '**/coverage/**',
+    '**/*.min.js',
+    '**/*.map',
+    '**/package-lock.json',
+    '**/yarn.lock',
+    '**/pnpm-lock.yaml',
+];
+const FILE_LIMITS = {
+    quick: 50,
+    standard: 200,
+    thorough: 500,
+};
+// =============================================================================
+// CodeWiki Indexer Service
+// =============================================================================
+export class CodeWikiIndexer {
+    system;
+    rootDir;
+    fileMetadataCache = new Map();
+    constructor(system, rootDir) {
+        this.system = system;
+        this.rootDir = rootDir ?? process.cwd();
+    }
+    /**
+     * Run full indexing process
+     */
+    async index(options) {
+        const startTime = Date.now();
+        const runId = `run_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
+        const depth = options.depth ?? 'standard';
+        const paths = options.paths ?? ['.'];
+        const rootDir = options.rootDir ?? this.rootDir;
+        // Create index run record
+        const indexRun = createIndexRun(options.projectId, paths, depth);
+        await this.saveIndexRun(runId, indexRun);
+        const errors = [];
+        try {
+            // Load existing file metadata cache
+            await this.loadMetadataCache();
+            // Collect files to index
+            const files = await this.collectFiles(rootDir, {
+                include: options.include ?? DEFAULT_INCLUDE,
+                exclude: options.exclude ?? DEFAULT_EXCLUDE,
+                limit: FILE_LIMITS[depth],
+                paths,
+            });
+            indexRun.stats.filesScanned = files.length;
+            logger.info('CodeWiki indexing started', {
+                projectId: options.projectId,
+                files: files.length,
+                depth,
+            });
+            // Process each file
+            for (const file of files) {
+                try {
+                    const result = await this.processFile(file, runId, options.force ?? false);
+                    if (result.indexed) {
+                        indexRun.stats.filesIndexed++;
+                        if (result.changed) {
+                            indexRun.stats.filesChanged++;
+                        }
+                        indexRun.stats.symbolsExtracted += result.symbolCount;
+                        indexRun.stats.dependenciesFound += result.dependencyCount;
+                    }
+                    else {
+                        indexRun.stats.filesSkipped++;
+                    }
+                }
+                catch (err) {
+                    const error = err instanceof Error ? err.message : String(err);
+                    errors.push({ file: file.relativePath, error });
+                    indexRun.stats.errorsEncountered++;
+                    logger.warn('Failed to index file', { file: file.relativePath, error });
+                }
+            }
+            // Update run status
+            indexRun.status = 'completed';
+            indexRun.completedAt = new Date().toISOString();
+            indexRun.durationMs = Date.now() - startTime;
+            indexRun.errors = errors;
+            await this.saveIndexRun(runId, indexRun);
+            logger.info('CodeWiki indexing completed', {
+                projectId: options.projectId,
+                filesIndexed: indexRun.stats.filesIndexed,
+                symbolsExtracted: indexRun.stats.symbolsExtracted,
+                durationMs: indexRun.durationMs,
+            });
+            return {
+                runId,
+                status: indexRun.status,
+                stats: indexRun.stats,
+                durationMs: indexRun.durationMs ?? 0,
+                errors,
+            };
+        }
+        catch (err) {
+            indexRun.status = 'blocked';
+            indexRun.completedAt = new Date().toISOString();
+            indexRun.durationMs = Date.now() - startTime;
+            indexRun.errors = errors;
+            await this.saveIndexRun(runId, indexRun);
+            throw err;
+        }
+    }
+    /**
+     * Collect files matching patterns
+     */
+    async collectFiles(rootDir, options) {
+        const files = [];
+        for (const basePath of options.paths) {
+            const searchDir = resolve(rootDir, basePath);
+            if (!existsSync(searchDir)) {
+                logger.warn('Path does not exist', { path: searchDir });
+                continue;
+            }
+            for (const pattern of options.include) {
+                const matches = await glob(pattern, {
+                    cwd: searchDir,
+                    ignore: options.exclude,
+                    nodir: true,
+                    absolute: true,
+                });
+                for (const match of matches) {
+                    if (files.length >= options.limit)
+                        break;
+                    try {
+                        const stat = statSync(match);
+                        files.push({
+                            absolutePath: match,
+                            relativePath: relative(rootDir, match),
+                            fileName: basename(match),
+                            extension: extname(match).slice(1),
+                            size: stat.size,
+                            mtime: stat.mtime,
+                        });
+                    }
+                    catch {
+                        // Skip files that can't be stat'd
+                    }
+                }
+                if (files.length >= options.limit)
+                    break;
+            }
+        }
+        return files;
+    }
+    /**
+     * Process a single file
+     */
+    async processFile(file, runId, force) {
+        // Read file content
+        const content = readFileSync(file.absolutePath, 'utf-8');
+        const hash = this.hashContent(content);
+        // Check if file has changed
+        const cached = this.fileMetadataCache.get(file.relativePath);
+        if (!force && cached && cached.hash === hash) {
+            return { indexed: false, changed: false, symbolCount: 0, dependencyCount: 0 };
+        }
+        const changed = cached !== undefined;
+        const language = detectLanguage(file.extension);
+        // Create file metadata
+        const metadata = this.createMetadata(file, content, hash, language);
+        const metadataId = await this.saveFileMetadata(metadata, runId);
+        // Update cache
+        this.fileMetadataCache.set(file.relativePath, { id: metadataId, hash });
+        // Parse code structure for supported languages
+        let symbolCount = 0;
+        let dependencyCount = 0;
+        if (this.isParseableLanguage(language)) {
+            const structure = await this.parseCodeStructure(file, content, metadataId, language);
+            symbolCount = structure.symbols.length;
+            dependencyCount = structure.imports.length;
+            await this.saveCodeStructure(structure, metadataId, runId);
+            // Extract dependencies
+            const dependencies = await this.extractDependencies(file, structure, metadataId);
+            for (const dep of dependencies) {
+                await this.saveCodeDependency(dep, runId);
+            }
+            dependencyCount = dependencies.length;
+        }
+        return { indexed: true, changed, symbolCount, dependencyCount };
+    }
+    /**
+     * Create file metadata from file info
+     */
+    createMetadata(file, content, hash, language) {
+        const lines = content.split('\n');
+        const nonEmptyLines = lines.filter(l => l.trim().length > 0).length;
+        const commentLines = this.countCommentLines(content, language);
+        return createFileMetadata(file.absolutePath, {
+            relativePath: file.relativePath,
+            fileName: file.fileName,
+            extension: file.extension,
+            language,
+            size: file.size,
+            loc: lines.length,
+            locNonEmpty: nonEmptyLines,
+            locComments: commentLines,
+            hash,
+            lastModified: file.mtime.toISOString(),
+            projectId: this.system,
+        });
+    }
+    /**
+     * Parse code structure from content
+     */
+    async parseCodeStructure(file, content, metadataId, language) {
+        const symbols = [];
+        const imports = [];
+        const exports = [];
+        // Simple regex-based parsing (Phase 1)
+        // TODO: Phase 2 will use proper AST parsing with typescript compiler API
+        const lines = content.split('\n');
+        let lineNum = 0;
+        for (const line of lines) {
+            lineNum++;
+            const trimmed = line.trim();
+            // Parse imports
+            const importMatch = trimmed.match(/^import\s+(.+)\s+from\s+['"](.+)['"]/);
+            if (importMatch) {
+                const importSpec = importMatch[1];
+                const source = importMatch[2];
+                const isExternal = !source.startsWith('.') && !source.startsWith('/');
+                imports.push({
+                    source: source ?? '',
+                    isExternal,
+                    isTypeOnly: trimmed.startsWith('import type'),
+                    isNamespace: importSpec?.includes('* as') ?? false,
+                    isDefault: !importSpec?.startsWith('{') && !importSpec?.includes('* as'),
+                    names: this.parseImportNames(importSpec ?? ''),
+                    line: lineNum,
+                });
+                continue;
+            }
+            // Parse exports
+            if (trimmed.startsWith('export ')) {
+                const isDefault = trimmed.includes('export default');
+                const isType = trimmed.startsWith('export type');
+                const isReExport = trimmed.includes(' from ');
+                let name = 'default';
+                if (!isDefault) {
+                    const nameMatch = trimmed.match(/export\s+(?:const|let|var|function|class|type|interface|enum)\s+(\w+)/);
+                    name = nameMatch?.[1] ?? 'unknown';
+                }
+                exports.push({
+                    name,
+                    isDefault,
+                    isType,
+                    isReExport,
+                    source: isReExport ? trimmed.match(/from\s+['"](.+)['"]/)?.[1] : undefined,
+                    line: lineNum,
+                });
+            }
+            // Parse symbols (basic patterns)
+            this.parseSymbolFromLine(trimmed, lineNum, symbols);
+        }
+        return createCodeStructure(file.absolutePath, metadataId, {
+            symbols,
+            imports,
+            exports,
+            isTest: isTestFile(file.relativePath),
+            isTypeDefinition: isTypeDefinitionFile(file.relativePath),
+            isComponent: this.detectReactComponent(content),
+            framework: this.detectFramework(content),
+        });
+    }
+    /**
+     * Parse import names from import specifier
+     */
+    parseImportNames(spec) {
+        const names = [];
+        // Default import
+        if (!spec.startsWith('{') && !spec.includes('* as')) {
+            const defaultName = spec.split(',')[0]?.trim();
+            if (defaultName) {
+                names.push({ name: defaultName });
+            }
+        }
+        // Namespace import
+        const nsMatch = spec.match(/\*\s+as\s+(\w+)/);
+        if (nsMatch) {
+            names.push({ name: '*', alias: nsMatch[1] });
+        }
+        // Named imports
+        const namedMatch = spec.match(/\{([^}]+)\}/);
+        if (namedMatch) {
+            const namedParts = namedMatch[1]?.split(',') ?? [];
+            for (const part of namedParts) {
+                const aliasMatch = part.trim().match(/(\w+)\s+as\s+(\w+)/);
+                if (aliasMatch) {
+                    names.push({ name: aliasMatch[1] ?? '', alias: aliasMatch[2] });
+                }
+                else {
+                    const name = part.trim();
+                    if (name) {
+                        names.push({ name });
+                    }
+                }
+            }
+        }
+        return names;
+    }
+    /**
+     * Parse symbol from a single line
+     */
+    parseSymbolFromLine(line, lineNum, symbols) {
+        // Function declarations
+        const funcMatch = line.match(/^(?:export\s+)?(?:async\s+)?function\s+(\w+)/);
+        if (funcMatch) {
+            symbols.push({
+                name: funcMatch[1] ?? '',
+                kind: 'function',
+                lineStart: lineNum,
+                lineEnd: lineNum, // Basic - doesn't track end
+                isExported: line.startsWith('export'),
+                isDefault: false,
+                isAsync: line.includes('async '),
+                generics: [],
+                decorators: [],
+                modifiers: [],
+            });
+            return;
+        }
+        // Arrow functions (const x = () => or const x = async () =>)
+        const arrowMatch = line.match(/^(?:export\s+)?const\s+(\w+)\s*=\s*(?:async\s+)?\([^)]*\)\s*(?::\s*\w+)?\s*=>/);
+        if (arrowMatch) {
+            symbols.push({
+                name: arrowMatch[1] ?? '',
+                kind: 'arrow_function',
+                lineStart: lineNum,
+                lineEnd: lineNum,
+                isExported: line.startsWith('export'),
+                isDefault: false,
+                isAsync: line.includes('async'),
+                generics: [],
+                decorators: [],
+                modifiers: [],
+            });
+            return;
+        }
+        // Class declarations
+        const classMatch = line.match(/^(?:export\s+)?(?:abstract\s+)?class\s+(\w+)/);
+        if (classMatch) {
+            symbols.push({
+                name: classMatch[1] ?? '',
+                kind: 'class',
+                lineStart: lineNum,
+                lineEnd: lineNum,
+                isExported: line.startsWith('export'),
+                isDefault: false,
+                modifiers: line.includes('abstract') ? ['abstract'] : [],
+                generics: [],
+                decorators: [],
+            });
+            return;
+        }
+        // Interface declarations
+        const ifaceMatch = line.match(/^(?:export\s+)?interface\s+(\w+)/);
+        if (ifaceMatch) {
+            symbols.push({
+                name: ifaceMatch[1] ?? '',
+                kind: 'interface',
+                lineStart: lineNum,
+                lineEnd: lineNum,
+                isExported: line.startsWith('export'),
+                isDefault: false,
+                generics: [],
+                decorators: [],
+                modifiers: [],
+            });
+            return;
+        }
+        // Type aliases
+        const typeMatch = line.match(/^(?:export\s+)?type\s+(\w+)/);
+        if (typeMatch) {
+            symbols.push({
+                name: typeMatch[1] ?? '',
+                kind: 'type',
+                lineStart: lineNum,
+                lineEnd: lineNum,
+                isExported: line.startsWith('export'),
+                isDefault: false,
+                generics: [],
+                decorators: [],
+                modifiers: [],
+            });
+            return;
+        }
+        // Enum declarations
+        const enumMatch = line.match(/^(?:export\s+)?(?:const\s+)?enum\s+(\w+)/);
+        if (enumMatch) {
+            symbols.push({
+                name: enumMatch[1] ?? '',
+                kind: 'enum',
+                lineStart: lineNum,
+                lineEnd: lineNum,
+                isExported: line.startsWith('export'),
+                isDefault: false,
+                generics: [],
+                decorators: [],
+                modifiers: [],
+            });
+            return;
+        }
+        // Const declarations (not arrow functions)
+        const constMatch = line.match(/^(?:export\s+)?const\s+(\w+)\s*[=:]/);
+        if (constMatch && !arrowMatch) {
+            symbols.push({
+                name: constMatch[1] ?? '',
+                kind: 'const',
+                lineStart: lineNum,
+                lineEnd: lineNum,
+                isExported: line.startsWith('export'),
+                isDefault: false,
+                generics: [],
+                decorators: [],
+                modifiers: [],
+            });
+        }
+    }
+    /**
+     * Extract dependencies from code structure
+     */
+    async extractDependencies(file, structure, metadataId) {
+        const dependencies = [];
+        for (const imp of structure.imports) {
+            let targetFile = imp.source;
+            let targetFileId;
+            if (!imp.isExternal) {
+                // Resolve relative import
+                targetFile = this.resolveImport(file.absolutePath, imp.source);
+                // Try to find the target in our cache
+                const cached = this.fileMetadataCache.get(relative(this.rootDir, targetFile));
+                targetFileId = cached?.id;
+            }
+            dependencies.push(createCodeDependency(file.absolutePath, targetFile, {
+                sourceFileId: metadataId,
+                targetFileId,
+                isExternal: imp.isExternal,
+                packageName: imp.isExternal ? this.getPackageName(imp.source) : undefined,
+                dependencyType: imp.isTypeOnly ? 'type_only' : 'import',
+                symbols: imp.names.map(n => n.alias ?? n.name),
+                line: imp.line,
+            }));
+        }
+        return dependencies;
+    }
+    /**
+     * Resolve relative import to absolute path
+     */
+    resolveImport(fromFile, importPath) {
+        const dir = dirname(fromFile);
+        let resolved = resolve(dir, importPath);
+        // Try common extensions if not specified
+        const extensions = ['.ts', '.tsx', '.js', '.jsx', '/index.ts', '/index.tsx', '/index.js'];
+        for (const ext of extensions) {
+            const withExt = resolved + ext;
+            if (existsSync(withExt)) {
+                return withExt;
+            }
+        }
+        return resolved;
+    }
+    /**
+     * Get package name from import source
+     */
+    getPackageName(source) {
+        // Handle scoped packages (@scope/package)
+        if (source.startsWith('@')) {
+            const parts = source.split('/');
+            return `${parts[0]}/${parts[1]}`;
+        }
+        // Handle regular packages (package or package/subpath)
+        return source.split('/')[0] ?? source;
+    }
+    /**
+     * Detect if content contains a React component
+     */
+    detectReactComponent(content) {
+        return (content.includes('React.') ||
+            content.includes('from \'react\'') ||
+            content.includes('from "react"') ||
+            /return\s+\(?\s*</.test(content));
+    }
+    /**
+     * Detect framework from content
+     */
+    detectFramework(content) {
+        if (content.includes('from \'next') || content.includes('from "next')) {
+            return 'next';
+        }
+        if (content.includes('from \'react') || content.includes('from "react')) {
+            return 'react';
+        }
+        if (content.includes('from \'express') || content.includes('from "express')) {
+            return 'express';
+        }
+        if (content.includes('from \'fastify') || content.includes('from "fastify')) {
+            return 'fastify';
+        }
+        return undefined;
+    }
+    /**
+     * Count comment lines in content
+     */
+    countCommentLines(content, language) {
+        if (!['typescript', 'javascript', 'tsx', 'jsx'].includes(language)) {
+            return 0;
+        }
+        let count = 0;
+        let inBlockComment = false;
+        for (const line of content.split('\n')) {
+            const trimmed = line.trim();
+            if (inBlockComment) {
+                count++;
+                if (trimmed.includes('*/')) {
+                    inBlockComment = false;
+                }
+                continue;
+            }
+            if (trimmed.startsWith('//')) {
+                count++;
+                continue;
+            }
+            if (trimmed.startsWith('/*')) {
+                count++;
+                inBlockComment = !trimmed.includes('*/');
+                continue;
+            }
+        }
+        return count;
+    }
+    /**
+     * Check if language is parseable
+     */
+    isParseableLanguage(language) {
+        return ['typescript', 'javascript', 'tsx', 'jsx'].includes(language);
+    }
+    /**
+     * Hash content for change detection
+     */
+    hashContent(content) {
+        return createHash('sha256').update(content).digest('hex').slice(0, 16);
+    }
+    // ===========================================================================
+    // Database Operations
+    // ===========================================================================
+    /**
+     * Load existing file metadata cache
+     */
+    async loadMetadataCache() {
+        try {
+            const items = await turso.query({
+                type: 'file_metadata',
+                system: this.system,
+                status: 'active',
+                limit: 500,
+            });
+            for (const item of items) {
+                try {
+                    const content = item.content;
+                    this.fileMetadataCache.set(content.relativePath, {
+                        id: item.id,
+                        hash: content.hash,
+                    });
+                }
+                catch {
+                    // Skip invalid entries
+                }
+            }
+            logger.debug('Loaded file metadata cache', { count: this.fileMetadataCache.size });
+        }
+        catch (err) {
+            logger.warn('Failed to load metadata cache', { error: String(err) });
+        }
+    }
+    /**
+     * Save file metadata to database
+     */
+    async saveFileMetadata(metadata, _runId) {
+        const id = await turso.create('file_metadata', this.system, metadata, 'active');
+        return id;
+    }
+    /**
+     * Save code structure to database
+     */
+    async saveCodeStructure(structure, _metadataId, _runId) {
+        await turso.create('code_structure', this.system, structure, 'active');
+    }
+    /**
+     * Save code dependency to database
+     */
+    async saveCodeDependency(dependency, _runId) {
+        await turso.create('code_dependency', this.system, dependency, 'active');
+    }
+    /**
+     * Save index run to database
+     */
+    async saveIndexRun(_runId, run) {
+        await turso.create('index_run', this.system, run, run.status);
+    }
+}
+// =============================================================================
+// Singleton & Factory
+// =============================================================================
+let indexerInstance = null;
+/**
+ * Get CodeWiki indexer instance
+ */
+export function getCodeWikiIndexer(system, rootDir) {
+    if (!indexerInstance || indexerInstance['system'] !== system) {
+        indexerInstance = new CodeWikiIndexer(system, rootDir);
+    }
+    return indexerInstance;
+}
+/**
+ * Create new CodeWiki indexer
+ */
+export function createCodeWikiIndexer(system, rootDir) {
+    return new CodeWikiIndexer(system, rootDir);
+}
+/**
+ * Reset indexer instance
+ */
+export function resetCodeWikiIndexer() {
+    indexerInstance = null;
+}
+export default CodeWikiIndexer;
+//# sourceMappingURL=CodeWikiIndexer.js.map