cgraphx 2.0.1 → 2.0.3

package/dist/core/code/engine/core/ingestion/field-extractors/generic.js

@@ -0,0 +1,147 @@
+"use strict";
+// cgraph/src/core/ingestion/field-extractors/generic.ts
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createFieldExtractor = createFieldExtractor;
+const field_extractor_js_1 = require("../field-extractor.js");
+// ---------------------------------------------------------------------------
+// Factory
+// ---------------------------------------------------------------------------
+/**
+ * Create a FieldExtractor from a declarative config.
+ */
+function createFieldExtractor(config) {
+    const typeDeclarationSet = new Set(config.typeDeclarationNodes);
+    const fieldNodeSet = new Set(config.fieldNodeTypes);
+    const bodyNodeSet = new Set(config.bodyNodeTypes);
+    class GenericFieldExtractor extends field_extractor_js_1.BaseFieldExtractor {
+        language = config.language;
+        isTypeDeclaration(node) {
+            return typeDeclarationSet.has(node.type);
+        }
+        extractVisibility(node) {
+            return config.extractVisibility(node);
+        }
+        extract(node, context) {
+            if (!this.isTypeDeclaration(node))
+                return null;
+            const ownerFqn = config.extractOwnerName?.(node) ?? node.childForFieldName('name')?.text;
+            if (!ownerFqn)
+                return null;
+            const fields = [];
+            // Find body container(s)
+            const bodies = this.findBodies(node);
+            for (const body of bodies) {
+                this.extractFieldsFromBody(body, context, fields);
+            }
+            // Extract fields from primary constructor parameters (e.g. C# records)
+            if (config.extractPrimaryFields) {
+                const primaryFields = config.extractPrimaryFields(node, context);
+                for (const f of primaryFields)
+                    fields.push(f);
+            }
+            return { ownerFqn, fields, nestedTypes: [] };
+        }
+        // ------------------------------------------------------------------
+        // private helpers
+        // ------------------------------------------------------------------
+        findBodies(node) {
+            if (config.findBodyNodes)
+                return config.findBodyNodes(node);
+            const result = [];
+            // Try named 'body' field first
+            const bodyField = node.childForFieldName('body');
+            if (bodyField && bodyNodeSet.has(bodyField.type)) {
+                result.push(bodyField);
+                return result;
+            }
+            // Walk immediate children for matching body node types
+            for (let i = 0; i < node.namedChildCount; i++) {
+                const child = node.namedChild(i);
+                if (child && bodyNodeSet.has(child.type)) {
+                    result.push(child);
+                }
+            }
+            // Fallback: use the body field even if its type is not in bodyNodeSet
+            if (result.length === 0 && bodyField) {
+                result.push(bodyField);
+            }
+            return result;
+        }
+        extractFieldsFromBody(body, context, out) {
+            for (let i = 0; i < body.namedChildCount; i++) {
+                const child = body.namedChild(i);
+                if (!child)
+                    continue;
+                if (fieldNodeSet.has(child.type)) {
+                    if (config.extractNames) {
+                        // Multi-name path: one node may declare several fields (e.g. Ruby attr_accessor)
+                        const names = config.extractNames(child);
+                        for (const name of names) {
+                            const field = this.buildField(child, name, context);
+                            if (field)
+                                out.push(field);
+                        }
+                    }
+                    else {
+                        const field = this.extractSingleField(child, context);
+                        if (field)
+                            out.push(field);
+                    }
+                }
+            }
+        }
+        extractSingleField(node, context) {
+            const name = config.extractName(node);
+            if (!name)
+                return null;
+            return this.buildField(node, name, context);
+        }
+        buildField(node, name, context) {
+            if (!name)
+                return null;
+            let type = config.extractType(node) ?? null;
+            if (type) {
+                type = this.normalizeType(type);
+                const resolved = this.resolveType(type, context);
+                if (resolved)
+                    type = resolved;
+            }
+            // Raw declared type deliberately bypasses normalizeType/resolveType —
+            // it is the verbatim source text (generics preserved).
+            let rawDeclaredType;
+            try {
+                rawDeclaredType = config.extractRawType?.(node);
+            }
+            catch {
+                // A throw here (an unexpected tree-sitter node shape, a config bug)
+                // must NOT propagate — it would escape processFileGroup to the
+                // language-group catch, which treats any throw as "parser unavailable"
+                // and silently drops every remaining file in the group. Degrade to a
+                // field without the raw type instead. Mirrors the descriptionExtractor
+                // / extractTemplateConstraints guards in parse-worker.ts (#2286 review).
+                rawDeclaredType = undefined;
+            }
+            let annotations;
+            try {
+                annotations = config.extractAnnotations?.(node);
+            }
+            catch {
+                // Same group-drop rationale as the extractRawType guard above —
+                // degrade to a field without annotations (#2286 review).
+                annotations = undefined;
+            }
+            return {
+                name,
+                type,
+                ...(rawDeclaredType !== undefined ? { rawDeclaredType } : {}),
+                ...(annotations !== undefined && annotations.length > 0 ? { annotations } : {}),
+                visibility: config.extractVisibilityForName?.(node, name) ?? config.extractVisibility(node),
+                isStatic: config.isStatic(node),
+                isReadonly: config.isReadonly(node),
+                sourceFile: context.filePath,
+                line: node.startPosition.row + 1,
+            };
+        }
+    }
+    return new GenericFieldExtractor();
+}

package/dist/core/code/engine/core/ingestion/field-extractors/typescript.js

@@ -0,0 +1,266 @@
+"use strict";
+// cgraph/src/core/ingestion/field-extractors/typescript.ts
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.typescriptFieldExtractor = exports.TypeScriptFieldExtractor = void 0;
+const index_js_1 = require("../../../../shared/index.js");
+const field_extractor_js_1 = require("../field-extractor.js");
+/**
+ * Hand-written TypeScript field extractor.
+ *
+ * This exists alongside the config-based extractor in configs/typescript-javascript.ts
+ * (used for JavaScript) because TypeScript has unique requirements:
+ * 1. type_alias_declaration with object type literals (e.g., type Config = { key: string })
+ * 2. Optional property detection appending '| undefined' to types
+ * 3. Nested type discovery within class/interface bodies
+ *
+ * The config-based extractor cannot express these TS-specific capabilities.
+ * JavaScript uses the config-based version since it lacks type syntax.
+ */
+class TypeScriptFieldExtractor extends field_extractor_js_1.BaseFieldExtractor {
+    language = index_js_1.SupportedLanguages.TypeScript;
+    /**
+     * Node types that represent type declarations with fields in TypeScript
+     */
+    static TYPE_DECLARATION_NODES = new Set([
+        'class_declaration',
+        'interface_declaration',
+        'abstract_class_declaration',
+        'type_alias_declaration', // for object type literals
+    ]);
+    /**
+     * Node types that contain field definitions within class bodies
+     */
+    static FIELD_NODE_TYPES = new Set([
+        'public_field_definition', // class field: private users: User[]
+        'property_signature', // interface property: name: string
+        'field_definition', // fallback field type
+    ]);
+    /**
+     * Visibility modifiers in TypeScript
+     */
+    static VISIBILITY_MODIFIERS = new Set([
+        'public',
+        'private',
+        'protected',
+    ]);
+    /**
+     * Check if this node represents a type declaration with fields
+     */
+    isTypeDeclaration(node) {
+        return TypeScriptFieldExtractor.TYPE_DECLARATION_NODES.has(node.type);
+    }
+    /**
+     * Extract visibility modifier from a field node
+     */
+    extractVisibility(node) {
+        // Check for accessibility_modifier named child (tree-sitter typescript)
+        for (let i = 0; i < node.namedChildCount; i++) {
+            const child = node.namedChild(i);
+            if (child && child.type === 'accessibility_modifier') {
+                const text = child.text.trim();
+                if (TypeScriptFieldExtractor.VISIBILITY_MODIFIERS.has(text)) {
+                    return text;
+                }
+            }
+        }
+        // Check for modifiers in the field's unnamed children (fallback)
+        for (let i = 0; i < node.childCount; i++) {
+            const child = node.child(i);
+            if (child && !child.isNamed) {
+                const text = child.text.trim();
+                if (TypeScriptFieldExtractor.VISIBILITY_MODIFIERS.has(text)) {
+                    return text;
+                }
+            }
+        }
+        // TypeScript class members are public by default
+        return 'public';
+    }
+    /**
+     * Check if a field has the static modifier
+     */
+    isStatic(node) {
+        for (let i = 0; i < node.childCount; i++) {
+            const child = node.child(i);
+            if (child && !child.isNamed && child.text.trim() === 'static') {
+                return true;
+            }
+        }
+        return false;
+    }
+    /**
+     * Check if a field has the readonly modifier
+     */
+    isReadonly(node) {
+        for (let i = 0; i < node.childCount; i++) {
+            const child = node.child(i);
+            if (child && !child.isNamed && child.text.trim() === 'readonly') {
+                return true;
+            }
+        }
+        return false;
+    }
+    /**
+     * Check if a property is optional (has ?: syntax)
+     */
+    isOptional(node) {
+        // Look for the optional marker '?' in unnamed children
+        for (let i = 0; i < node.childCount; i++) {
+            const child = node.child(i);
+            if (child && !child.isNamed && child.text === '?') {
+                return true;
+            }
+        }
+        // Also check for optional_property_signature or marker in type
+        const kind = node.childForFieldName('kind');
+        if (kind && kind.text === '?') {
+            return true;
+        }
+        return false;
+    }
+    /**
+     * Extract the full type text, handling complex generic types.
+     *
+     * type_annotation nodes wrap the literal ': SomeType' — only that branch
+     * needs special handling to unwrap the inner child and skip the colon.
+     * All other node kinds are already the type text itself, so normalizeType
+     * is applied directly.
+     */
+    extractFullType(typeNode) {
+        if (!typeNode)
+            return null;
+        if (typeNode.type === 'type_annotation') {
+            const innerType = typeNode.firstNamedChild;
+            return innerType ? this.normalizeType(innerType.text) : null;
+        }
+        return this.normalizeType(typeNode.text);
+    }
+    /**
+     * Extract a single field from a field definition node
+     */
+    extractField(node, context) {
+        // Get the field name
+        const nameNode = node.childForFieldName('name') ?? node.childForFieldName('property');
+        if (!nameNode)
+            return null;
+        const name = nameNode.text;
+        if (!name)
+            return null;
+        // Get the type annotation
+        const typeNode = node.childForFieldName('type');
+        let type = this.extractFullType(typeNode);
+        // Try to resolve the type using the context
+        if (type) {
+            const resolvedType = this.resolveType(type, context);
+            type = resolvedType ?? type;
+        }
+        return {
+            name,
+            type,
+            visibility: this.extractVisibility(node),
+            isStatic: this.isStatic(node),
+            isReadonly: this.isReadonly(node),
+            sourceFile: context.filePath,
+            line: node.startPosition.row + 1,
+        };
+    }
+    /**
+     * Extract fields from a class body or interface body
+     */
+    extractFieldsFromBody(bodyNode, context) {
+        const fields = [];
+        // Find all field definition nodes within the body
+        for (let i = 0; i < bodyNode.namedChildCount; i++) {
+            const child = bodyNode.namedChild(i);
+            if (!child)
+                continue;
+            if (TypeScriptFieldExtractor.FIELD_NODE_TYPES.has(child.type)) {
+                const field = this.extractField(child, context);
+                if (field) {
+                    fields.push(field);
+                }
+            }
+        }
+        return fields;
+    }
+    /**
+     * Extract fields from an object type (used in type aliases)
+     */
+    extractFieldsFromObjectType(objectTypeNode, context) {
+        const fields = [];
+        // Find all property_signature nodes within the object type
+        const propertySignatures = objectTypeNode.descendantsOfType('property_signature');
+        for (const propNode of propertySignatures) {
+            const field = this.extractField(propNode, context);
+            if (field) {
+                // Mark optional properties
+                if (this.isOptional(propNode) && field.type) {
+                    field.type = field.type + ' | undefined';
+                }
+                fields.push(field);
+            }
+        }
+        return fields;
+    }
+    /**
+     * Extract fields from a class or interface declaration
+     */
+    extract(node, context) {
+        if (!this.isTypeDeclaration(node))
+            return null;
+        // Get the type name
+        const nameNode = node.childForFieldName('name');
+        if (!nameNode)
+            return null;
+        const typeName = nameNode.text;
+        const ownerFqn = typeName;
+        const fields = [];
+        const nestedTypes = [];
+        // Handle different declaration types
+        if (node.type === 'class_declaration' || node.type === 'abstract_class_declaration') {
+            // Find the class body
+            const bodyNode = node.childForFieldName('body');
+            if (bodyNode) {
+                const extractedFields = this.extractFieldsFromBody(bodyNode, context);
+                fields.push(...extractedFields);
+            }
+        }
+        else if (node.type === 'interface_declaration') {
+            // Find the interface body
+            const bodyNode = node.childForFieldName('body');
+            if (bodyNode) {
+                const extractedFields = this.extractFieldsFromBody(bodyNode, context);
+                fields.push(...extractedFields);
+            }
+        }
+        else if (node.type === 'type_alias_declaration') {
+            // Handle type aliases with object types
+            const valueNode = node.childForFieldName('value');
+            if (valueNode && valueNode.type === 'object_type') {
+                const extractedFields = this.extractFieldsFromObjectType(valueNode, context);
+                fields.push(...extractedFields);
+            }
+        }
+        // Find nested type declarations
+        const nestedClasses = node.descendantsOfType('class_declaration');
+        const nestedInterfaces = node.descendantsOfType('interface_declaration');
+        const nestedDeclarations = [...nestedClasses, ...nestedInterfaces];
+        for (const nested of nestedDeclarations) {
+            // Skip the current node itself
+            if (nested === node)
+                continue;
+            const nestedName = nested.childForFieldName('name');
+            if (nestedName) {
+                nestedTypes.push(nestedName.text);
+            }
+        }
+        return {
+            ownerFqn,
+            fields,
+            nestedTypes,
+        };
+    }
+}
+exports.TypeScriptFieldExtractor = TypeScriptFieldExtractor;
+// Export a singleton instance for registration
+exports.typescriptFieldExtractor = new TypeScriptFieldExtractor();

package/dist/core/code/engine/core/ingestion/field-types.js

@@ -0,0 +1,3 @@
+"use strict";
+// cgraph/src/core/ingestion/field-types.ts
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/core/code/engine/core/ingestion/filesystem-walker.js

@@ -0,0 +1,136 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.readFileContents = exports.walkRepositoryPaths = void 0;
+const verbose_js_1 = require("./utils/verbose.js");
+const max_file_size_js_1 = require("./utils/max-file-size.js");
+const promises_1 = __importDefault(require("fs/promises"));
+const path_1 = __importDefault(require("path"));
+const glob_1 = require("glob");
+const ignore_service_js_1 = require("../../config/ignore-service.js");
+const workspace_config_js_1 = require("./workspace-config.js");
+const logger_js_1 = require("../logger.js");
+const READ_CONCURRENCY = 32;
+const ANALYZE_PROGRESS_ACTIVE_ENV = 'CGRAPH_ANALYZE_PROGRESS_ACTIVE';
+const warnLargeFileSkip = (message) => {
+    if (process.env[ANALYZE_PROGRESS_ACTIVE_ENV] === '1') {
+        // analyze.ts routes console.warn through the progress bar logger while
+        // the bar is active. Emitting the operator-facing large-file notice there
+        // avoids raw pino NDJSON corrupting the one-line progress display in the
+        // heap-respawn child, whose stderr is intentionally piped for crash
+        // classification.
+        // eslint-disable-next-line no-console -- intentionally routed by analyze progress UI
+        console.warn(message);
+        return;
+    }
+    logger_js_1.logger.warn(message);
+};
+/**
+ * Phase 1: Scan repository — stat files to get paths + sizes, no content loaded.
+ * Memory: ~10MB for 100K files vs ~1GB+ with content.
+ */
+const walkRepositoryPaths = async (repoPath, onProgress) => {
+    // spec §4 / §5.1: declared workspace subproject directories must not be
+    // pruned by .gitignore so multi-git-subproject aggregation roots (hx mode)
+    // are scanned in full. `loadWorkspaceConfig` returns null when no
+    // cgraphx-workspace.yaml is present → workspaceDirs undefined → ignore
+    // semantics unchanged. A malformed config throws WorkspaceConfigError here
+    // (spec §9: analyze aborts) — but run-analyze validates it earlier, so the
+    // cached result is returned and no new throw path is introduced.
+    const workspaceConfig = (0, workspace_config_js_1.loadWorkspaceConfig)(repoPath);
+    const ignoreFilter = await (0, ignore_service_js_1.createIgnoreFilter)(repoPath, {
+        workspaceDirs: workspaceConfig?.services,
+    });
+    const maxFileSizeBytes = (0, max_file_size_js_1.getMaxFileSizeBytes)();
+    const filtered = await (0, glob_1.glob)('**/*', {
+        cwd: repoPath,
+        nodir: true,
+        dot: false,
+        ignore: ignoreFilter,
+    });
+    const entries = [];
+    let processed = 0;
+    let skippedLarge = 0;
+    const skippedLargePaths = [];
+    for (let start = 0; start < filtered.length; start += READ_CONCURRENCY) {
+        const batch = filtered.slice(start, start + READ_CONCURRENCY);
+        const results = await Promise.allSettled(batch.map(async (relativePath) => {
+            const fullPath = path_1.default.join(repoPath, relativePath);
+            const stat = await promises_1.default.stat(fullPath);
+            if (stat.size > maxFileSizeBytes) {
+                skippedLarge++;
+                skippedLargePaths.push(relativePath.replace(/\\/g, '/'));
+                return null;
+            }
+            return { path: relativePath.replace(/\\/g, '/'), size: stat.size };
+        }));
+        for (const result of results) {
+            processed++;
+            if (result.status === 'fulfilled' && result.value !== null) {
+                entries.push(result.value);
+                onProgress?.(processed, filtered.length, result.value.path);
+            }
+            else {
+                onProgress?.(processed, filtered.length, batch[results.indexOf(result)]);
+            }
+        }
+    }
+    // Filesystem/glob traversal order is not stable across filesystems or repeated
+    // scans. Canonicalize once at the scan boundary so every downstream phase sees
+    // the same repository order.
+    entries.sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0));
+    if (skippedLarge > 0) {
+        const isDefault = maxFileSizeBytes === max_file_size_js_1.DEFAULT_MAX_FILE_SIZE_BYTES;
+        const isOverrideUnset = !process.env.CGRAPH_MAX_FILE_SIZE;
+        const suffix = isDefault ? ', likely generated/vendored' : '';
+        warnLargeFileSkip(`  Skipped ${skippedLarge} large files (>${maxFileSizeBytes / 1024}KB${suffix})`);
+        // Always show at least the first few paths so users can diagnose why
+        // edges are missing from a specific file (issue #1659). The full list is
+        // gated behind CGRAPH_VERBOSE=1 to avoid flooding output on repos with
+        // many generated/vendored blobs. Sort before slicing so the preview is
+        // stable across runs (fs.stat callbacks race within each batch).
+        skippedLargePaths.sort();
+        const SKIPPED_PREVIEW_CAP = 5;
+        const showAll = (0, verbose_js_1.isVerboseIngestionEnabled)() || skippedLargePaths.length <= SKIPPED_PREVIEW_CAP;
+        const preview = showAll ? skippedLargePaths : skippedLargePaths.slice(0, SKIPPED_PREVIEW_CAP);
+        for (const p of preview) {
+            warnLargeFileSkip(`  - ${p}`);
+        }
+        if (!showAll) {
+            const remaining = skippedLargePaths.length - SKIPPED_PREVIEW_CAP;
+            warnLargeFileSkip(`  ...and ${remaining} more (set CGRAPH_VERBOSE=1 to list them all)`);
+        }
+        // Only hint about the env var when the user has not set it at all. An
+        // explicit CGRAPH_MAX_FILE_SIZE=512 happens to resolve to the same
+        // bytes as the default but the operator clearly already knows the knob.
+        if (isDefault && isOverrideUnset) {
+            warnLargeFileSkip(`  Set CGRAPH_MAX_FILE_SIZE=<KB> to include files above the default cap.`);
+        }
+    }
+    return entries;
+};
+exports.walkRepositoryPaths = walkRepositoryPaths;
+/**
+ * Phase 2: Read file contents for a specific set of relative paths.
+ * Returns a Map for O(1) lookup. Silently skips files that fail to read.
+ */
+const readFileContents = async (repoPath, relativePaths) => {
+    const contents = new Map();
+    for (let start = 0; start < relativePaths.length; start += READ_CONCURRENCY) {
+        const batch = relativePaths.slice(start, start + READ_CONCURRENCY);
+        const results = await Promise.allSettled(batch.map(async (relativePath) => {
+            const fullPath = path_1.default.join(repoPath, relativePath);
+            const content = await promises_1.default.readFile(fullPath, 'utf-8');
+            return { path: relativePath, content };
+        }));
+        for (const result of results) {
+            if (result.status === 'fulfilled') {
+                contents.set(result.value.path, result.value.content);
+            }
+        }
+    }
+    return contents;
+};
+exports.readFileContents = readFileContents;