kodingo-cli 1.0.38 → 1.1.1

package/dist/utils/tree-sitter-parser.js

@@ -0,0 +1,281 @@
+"use strict";
+/**
+ * tree-sitter-parser.ts
+ *
+ * Real AST-based symbol extraction for non-TS/JS languages, using web-tree-sitter
+ * with prebuilt WASM grammars (tree-sitter-wasms) — no native compilation required,
+ * works identically on any developer machine.
+ *
+ * This replaces regex/brace-counting heuristics (parseStructural, parsePython in
+ * ast-parser.ts) with genuine parse trees. Call and declaration extraction is still
+ * heuristic in the sense that it isn't type-checked (that's Tier 1, TS/JS only via
+ * the compiler), but it is now grounded in a real AST rather than pattern matching
+ * over raw text — eliminating false positives from strings/comments that merely
+ * look like declarations or calls.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseWithTreeSitter = parseWithTreeSitter;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const web_tree_sitter_1 = __importDefault(require("web-tree-sitter"));
+const ast_parser_1 = require("./ast-parser");
+const LANGUAGE_CONFIGS = {
+    python: {
+        wasmFile: "tree-sitter-python.wasm",
+        functionTypes: ["function_definition"],
+        classTypes: ["class_definition"],
+        callTypes: ["call"],
+        calleeFieldCandidates: ["function"],
+        nameFieldCandidates: ["name"],
+        importTypes: ["import_statement", "import_from_statement"],
+    },
+    go: {
+        wasmFile: "tree-sitter-go.wasm",
+        functionTypes: ["function_declaration", "method_declaration"],
+        classTypes: [],
+        callTypes: ["call_expression"],
+        calleeFieldCandidates: ["function"],
+        nameFieldCandidates: ["name"],
+        importTypes: ["import_spec"],
+    },
+    rust: {
+        wasmFile: "tree-sitter-rust.wasm",
+        functionTypes: ["function_item"],
+        classTypes: ["impl_item"],
+        callTypes: ["call_expression"],
+        calleeFieldCandidates: ["function"],
+        nameFieldCandidates: ["name", "type"],
+        importTypes: ["use_declaration"],
+    },
+    java: {
+        wasmFile: "tree-sitter-java.wasm",
+        functionTypes: ["method_declaration", "constructor_declaration"],
+        classTypes: ["class_declaration", "interface_declaration"],
+        callTypes: ["method_invocation"],
+        calleeFieldCandidates: ["name"],
+        nameFieldCandidates: ["name"],
+        importTypes: ["import_declaration"],
+    },
+    kotlin: {
+        wasmFile: "tree-sitter-kotlin.wasm",
+        functionTypes: ["function_declaration"],
+        classTypes: ["class_declaration"],
+        callTypes: ["call_expression"],
+        calleeFieldCandidates: ["function"],
+        nameFieldCandidates: ["name"],
+        importTypes: ["import_header"],
+    },
+    ruby: {
+        wasmFile: "tree-sitter-ruby.wasm",
+        functionTypes: ["method", "singleton_method"],
+        classTypes: ["class", "module"],
+        callTypes: ["call", "method_call"],
+        calleeFieldCandidates: ["method"],
+        nameFieldCandidates: ["name"],
+        importTypes: [],
+    },
+    php: {
+        wasmFile: "tree-sitter-php.wasm",
+        functionTypes: ["function_definition", "method_declaration"],
+        classTypes: ["class_declaration"],
+        callTypes: ["function_call_expression", "member_call_expression", "scoped_call_expression"],
+        calleeFieldCandidates: ["function", "name"],
+        nameFieldCandidates: ["name"],
+        importTypes: ["namespace_use_declaration"],
+    },
+    c: {
+        wasmFile: "tree-sitter-c.wasm",
+        functionTypes: ["function_definition"],
+        classTypes: [],
+        callTypes: ["call_expression"],
+        calleeFieldCandidates: ["function"],
+        nameFieldCandidates: [], // handled via declarator fallback
+        importTypes: ["preproc_include"],
+    },
+    cpp: {
+        wasmFile: "tree-sitter-cpp.wasm",
+        functionTypes: ["function_definition"],
+        classTypes: ["class_specifier", "struct_specifier"],
+        callTypes: ["call_expression"],
+        calleeFieldCandidates: ["function"],
+        nameFieldCandidates: [], // handled via declarator fallback
+        importTypes: ["preproc_include"],
+    },
+};
+const EXT_TO_LANGUAGE = {
+    ".py": "python",
+    ".go": "go",
+    ".rs": "rust",
+    ".java": "java",
+    ".kt": "kotlin",
+    ".rb": "ruby",
+    ".php": "php",
+    ".c": "c",
+    ".cpp": "cpp",
+};
+let initialised = false;
+const languageCache = new Map();
+async function ensureInit() {
+    if (!initialised) {
+        await web_tree_sitter_1.default.init();
+        initialised = true;
+    }
+}
+async function loadLanguage(key, config) {
+    const cached = languageCache.get(key);
+    if (cached)
+        return cached;
+    const wasmDir = path_1.default.join(path_1.default.dirname(require.resolve("tree-sitter-wasms/package.json")), "out");
+    const wasmPath = path_1.default.join(wasmDir, config.wasmFile);
+    const wasmBuffer = fs_1.default.readFileSync(wasmPath);
+    const lang = await web_tree_sitter_1.default.Language.load(wasmBuffer);
+    languageCache.set(key, lang);
+    return lang;
+}
+// ── Node helpers ──────────────────────────────────────────────────────────────
+function getFieldOrFallback(node, candidates) {
+    for (const field of candidates) {
+        const child = node.childForFieldName(field);
+        if (child)
+            return child;
+    }
+    return null;
+}
+/** Descends through nested declarators (C/C++ function name pattern) to find the innermost identifier. */
+function extractDeclaratorName(node) {
+    let current = node.childForFieldName("declarator");
+    let guard = 0;
+    while (current && guard < 10) {
+        guard++;
+        if (current.type === "identifier" || current.type === "field_identifier") {
+            return current.text;
+        }
+        const nested = current.childForFieldName("declarator");
+        if (!nested)
+            break;
+        current = nested;
+    }
+    return null;
+}
+function getNodeName(node, config) {
+    const viaField = getFieldOrFallback(node, config.nameFieldCandidates);
+    if (viaField)
+        return viaField.text;
+    const viaDeclarator = extractDeclaratorName(node);
+    if (viaDeclarator)
+        return viaDeclarator;
+    // Last resort: first identifier-like named child
+    for (let i = 0; i < node.namedChildCount; i++) {
+        const child = node.namedChild(i);
+        if (child && (child.type === "identifier" || child.type === "type_identifier" || child.type === "simple_identifier")) {
+            return child.text;
+        }
+    }
+    return null;
+}
+/** Extracts a trailing identifier from a callee expression (e.g. "$this->service->method" -> "method"). */
+function extractTrailingIdentifier(text) {
+    const match = text.match(/([A-Za-z_][A-Za-z0-9_]*)$/);
+    return match ? match[1] : null;
+}
+function extractCallNamesTs(node, config) {
+    const calls = new Set();
+    function walk(n) {
+        if (config.callTypes.includes(n.type)) {
+            const calleeNode = getFieldOrFallback(n, config.calleeFieldCandidates) ?? n.namedChild(0);
+            if (calleeNode) {
+                const name = extractTrailingIdentifier(calleeNode.text);
+                if (name)
+                    calls.add(name);
+            }
+        }
+        for (let i = 0; i < n.namedChildCount; i++) {
+            const child = n.namedChild(i);
+            if (child)
+                walk(child);
+        }
+    }
+    walk(node);
+    return Array.from(calls);
+}
+function extractImportsTs(root, config) {
+    const imports = [];
+    if (config.importTypes.length === 0)
+        return imports;
+    function walk(n) {
+        if (config.importTypes.includes(n.type)) {
+            imports.push({ imported: "", source: n.text.trim(), isDefault: false });
+        }
+        for (let i = 0; i < n.namedChildCount; i++) {
+            const child = n.namedChild(i);
+            if (child)
+                walk(child);
+        }
+    }
+    walk(root);
+    return imports;
+}
+// ── Main extraction ───────────────────────────────────────────────────────────
+async function parseWithTreeSitter(filePath, source) {
+    const ext = path_1.default.extname(filePath).toLowerCase();
+    const langKey = EXT_TO_LANGUAGE[ext];
+    if (!langKey)
+        return [];
+    const config = LANGUAGE_CONFIGS[langKey];
+    if (!config)
+        return [];
+    try {
+        await ensureInit();
+        const language = await loadLanguage(langKey, config);
+        const parser = new web_tree_sitter_1.default();
+        parser.setLanguage(language);
+        const tree = parser.parse(source);
+        const root = tree.rootNode;
+        const fileImports = extractImportsTs(root, config);
+        const symbols = [];
+        function visit(node, enclosingClassName) {
+            if (config.classTypes.includes(node.type)) {
+                const className = getNodeName(node, config);
+                for (let i = 0; i < node.namedChildCount; i++) {
+                    const child = node.namedChild(i);
+                    if (child)
+                        visit(child, className ?? undefined);
+                }
+                return;
+            }
+            if (config.functionTypes.includes(node.type)) {
+                const rawName = getNodeName(node, config);
+                if (rawName) {
+                    const qualifiedName = enclosingClassName ? enclosingClassName + "." + rawName : rawName;
+                    const body = node.text;
+                    symbols.push({
+                        name: qualifiedName,
+                        type: enclosingClassName ? "method" : "function",
+                        file: filePath,
+                        startLine: node.startPosition.row + 1,
+                        endLine: node.endPosition.row + 1,
+                        body,
+                        fingerprint: (0, ast_parser_1.computeFingerprint)(body),
+                        calls: extractCallNamesTs(node, config),
+                        imports: fileImports,
+                    });
+                }
+                // still recurse in case of nested closures/functions
+            }
+            for (let i = 0; i < node.namedChildCount; i++) {
+                const child = node.namedChild(i);
+                if (child)
+                    visit(child, enclosingClassName);
+            }
+        }
+        visit(root);
+        return symbols;
+    }
+    catch {
+        // Parsing failures must never crash a full-repo scan — return empty for this file
+        return [];
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kodingo-cli",
-  "version": "1.0.38",
+  "version": "1.1.1",
   "description": "Kodingo CLI",
   "license": "MIT",
   "private": false,
@@ -28,12 +28,14 @@
     "kodingo-core": "^0.2.1",
     "open": "^11.0.0",
     "simple-git": "^3.30.0",
-    "uuid": "^10.0.0"
+    "tree-sitter-wasms": "^0.1.13",
+    "typescript": "^5.9.3",
+    "uuid": "^10.0.0",
+    "web-tree-sitter": "^0.20.8"
   },
   "devDependencies": {
     "@types/babel__traverse": "^7.28.0",
     "@types/node": "^22.19.20",
-    "@types/uuid": "^10.0.0",
-    "typescript": "^5.6.0"
+    "@types/uuid": "^10.0.0"
   }
 }