@oisheek_c/codeforge 1.0.1

package/packages/retrieval/parser/languages.js

@@ -0,0 +1,218 @@
+/**
+ * packages/retrieval/parser/languages.js
+ *
+ * Language registry.
+ * Maps language identifiers to Tree-sitter grammars.
+ */
+
+import JavaScript from "tree-sitter-javascript";
+import TypeScript from "tree-sitter-typescript";
+import Python from "tree-sitter-python";
+import Go from "tree-sitter-go";
+import Rust from "tree-sitter-rust";
+import Java from "tree-sitter-java";
+import C from "tree-sitter-c";
+import CPP from "tree-sitter-cpp";
+import CSharp from "tree-sitter-c-sharp";
+import PHP from "tree-sitter-php";
+import Ruby from "tree-sitter-ruby";
+import Swift from "tree-sitter-swift";
+
+const LANGUAGES = new Map([
+    [
+        "javascript",
+        {
+            id: "javascript",
+            grammar: JavaScript,
+            extensions: [".js", ".mjs", ".cjs"],
+            aliases: ["javascript", "js"],
+        },
+    ],
+
+    [
+        "jsx",
+        {
+            id: "jsx",
+            grammar: JavaScript,
+            extensions: [".jsx"],
+            aliases: ["jsx"],
+        },
+    ],
+
+    [
+        "typescript",
+        {
+            id: "typescript",
+            grammar: TypeScript.typescript,
+            extensions: [".ts", ".mts", ".cts"],
+            aliases: ["typescript", "ts"],
+        },
+    ],
+
+    [
+        "tsx",
+        {
+            id: "tsx",
+            grammar: TypeScript.tsx,
+            extensions: [".tsx"],
+            aliases: ["tsx"],
+        },
+    ],
+
+    [
+        "python",
+        {
+            id: "python",
+            grammar: Python,
+            extensions: [".py"],
+            aliases: ["python", "py"],
+        },
+    ],
+
+    [
+        "go",
+        {
+            id: "go",
+            grammar: Go,
+            extensions: [".go"],
+            aliases: ["go", "golang"],
+        },
+    ],
+
+    [
+        "rust",
+        {
+            id: "rust",
+            grammar: Rust,
+            extensions: [".rs"],
+            aliases: ["rust", "rs"],
+        },
+    ],
+
+    [
+        "java",
+        {
+            id: "java",
+            grammar: Java,
+            extensions: [".java"],
+            aliases: ["java"],
+        },
+    ],
+
+    [
+        "c",
+        {
+            id: "c",
+            grammar: C,
+            extensions: [".c", ".h"],
+            aliases: ["c"],
+        },
+    ],
+
+    [
+        "cpp",
+        {
+            id: "cpp",
+            grammar: CPP,
+            extensions: [
+                ".cpp",
+                ".cc",
+                ".cxx",
+                ".hpp",
+                ".hh",
+                ".hxx",
+            ],
+            aliases: ["cpp", "c++"],
+        },
+    ],
+
+    [
+        "csharp",
+        {
+            id: "csharp",
+            grammar: CSharp,
+            extensions: [".cs"],
+            aliases: ["csharp", "c#"],
+        },
+    ],
+
+    [
+        "php",
+        {
+            id: "php",
+            grammar: PHP,
+            extensions: [".php"],
+            aliases: ["php"],
+        },
+    ],
+
+    [
+        "ruby",
+        {
+            id: "ruby",
+            grammar: Ruby,
+            extensions: [".rb"],
+            aliases: ["ruby", "rb"],
+        },
+    ],
+
+    [
+        "swift",
+        {
+            id: "swift",
+            grammar: Swift,
+            extensions: [".swift"],
+            aliases: ["swift"],
+        },
+    ],
+]);
+
+const EXTENSIONS = new Map();
+
+for (const language of LANGUAGES.values()) {
+    for (const extension of language.extensions) {
+        EXTENSIONS.set(extension.toLowerCase(), language.id);
+    }
+}
+
+/**
+ * Returns language metadata.
+ */
+export function getLanguage(name) {
+    return LANGUAGES.get(name) ?? null;
+}
+
+/**
+ * Returns the Tree-sitter grammar.
+ */
+export function getGrammar(name) {
+    return getLanguage(name)?.grammar ?? null;
+}
+
+/**
+ * Returns whether a language is supported.
+ */
+export function supportsLanguage(name) {
+    return LANGUAGES.has(name);
+}
+
+/**
+ * Returns all registered languages.
+ */
+export function getLanguages() {
+    return [...LANGUAGES.values()];
+}
+
+/**
+ * Returns all language identifiers.
+ */
+export function getLanguageNames() {
+    return [...LANGUAGES.keys()];
+}
+
+/**
+ * Returns the language identifier for a file extension.
+ */
+export function getLanguageFromExtension(extension) {
+    return EXTENSIONS.get(extension.toLowerCase()) ?? null;
+}

package/packages/retrieval/parser/parseFile.js

@@ -0,0 +1,79 @@
+/**
+ * packages/retrieval/parser/parseFile.js
+ */
+
+import fs from "node:fs/promises";
+import path from "node:path";
+import crypto from "node:crypto";
+
+import { getLanguageFromExtension } from "./languages.js";
+import { getParser, getQueries } from "./registry.js";
+import { traverse } from "./traverse.js";
+import { createParsedFile } from "./types.js";
+
+export async function parseFile(filePath) {
+    const source = await fs.readFile(filePath, "utf8");
+
+    const extension = path.extname(filePath);
+
+    const language = getLanguageFromExtension(extension);
+
+    if (!language) {
+        return createParsedFile({
+            path: filePath,
+            name: path.basename(filePath),
+            extension,
+            supported: false,
+            source,
+        });
+    }
+
+
+    const parser = getParser(language);
+
+
+    const tree = parser.parse(source);
+
+    const queries = getQueries(language);
+
+
+    const result = traverse({
+        tree,
+        source,
+        file: filePath,
+        language,
+        queries,
+    });
+ 
+
+    return createParsedFile({
+        path: filePath,
+
+        name: path.basename(filePath),
+
+        extension,
+
+        language,
+
+        hash: crypto
+            .createHash("sha256")
+            .update(source)
+            .digest("hex"),
+
+        size: Buffer.byteLength(source),
+
+        source,
+
+        ast: tree,
+
+        imports: result.imports,
+        exports: result.exports,
+        symbols: result.symbols,
+        calls: result.calls,
+        comments: result.comments,
+        todos: result.todos,
+
+        diagnostics: result.diagnostics,
+        metrics: result.metrics,
+    });
+}

package/packages/retrieval/parser/parseRepository.js

@@ -0,0 +1,78 @@
+/**
+ * packages/retrieval/parser/parseRepository.js
+ */
+
+import { parseFile } from "./parseFile.js";
+
+/**
+ * Parse an indexed repository.
+ *
+ * @param {Array} repositoryIndex
+ * @param {Object|null} cache
+ * @returns {Promise<Object>}
+ */
+export async function parseRepository(
+    repositoryIndex,
+    cache = null
+) {
+    const files = [];
+
+    for (const file of repositoryIndex) {
+    const filePath =
+        typeof file === "string"
+            ? file
+            : file.path;
+
+    const fileHash =
+        typeof file === "string"
+            ? null
+            : file.hash;
+
+    try {
+        // Cache hit
+        if (
+            cache &&
+            fileHash &&
+            cache.has(filePath, fileHash)
+        ) {
+            const cached = cache.get(filePath);
+
+            if (cached?.parsed) {
+                files.push(cached.parsed);
+                continue;
+            }
+        }
+
+        const parsed = await parseFile(filePath);
+
+        if (cache && fileHash) {
+            cache.set(
+                filePath,
+                fileHash,
+                {
+                    parsed,
+                }
+            );
+        }
+
+        files.push(parsed);
+    } catch (error) {
+       console.error(`Failed to parse ${filePath}`);
+console.error(error.stack);
+    }
+}
+
+    if (cache) {
+        await cache.save();
+    }
+
+    return {
+        files,
+
+        metrics: {
+            total: repositoryIndex.length,
+            parsed: files.filter(f => f.supported).length,
+            skipped: files.filter(f => !f.supported).length,
+        },
+    };
+}

package/packages/retrieval/parser/queries/c.js

@@ -0,0 +1,148 @@
+/**
+ * Tree-sitter queries for C.
+ *
+ * Stable capture namespaces:
+ *
+ *  @symbol.struct
+ *  @symbol.union
+ *  @symbol.enum
+ *  @symbol.typedef
+ *  @symbol.function
+ *  @symbol.prototype
+ *  @symbol.variable
+ *  @symbol.constant
+ *  @symbol.macro
+ *
+ *  @import
+ *  @export
+ *  @call
+ *  @comment
+ *  @todo
+ */
+
+export const SYMBOLS = `
+;; ============================================================
+;; Structs
+;; ============================================================
+
+(struct_specifier
+  name: (type_identifier) @symbol.struct)
+
+;; ============================================================
+;; Unions
+;; ============================================================
+
+(union_specifier
+  name: (type_identifier) @symbol.union)
+
+;; ============================================================
+;; Enums
+;; ============================================================
+
+(enum_specifier
+  name: (type_identifier) @symbol.enum)
+
+;; ============================================================
+;; Typedefs
+;; ============================================================
+
+(type_definition
+  declarator: (type_identifier) @symbol.typedef)
+
+(type_definition
+  declarator: (identifier) @symbol.typedef)
+
+;; ============================================================
+;; Function Definitions
+;; ============================================================
+
+(function_definition
+  declarator:
+    (function_declarator
+      declarator: (identifier) @symbol.function))
+
+;; ============================================================
+;; Function Prototypes
+;; ============================================================
+
+(declaration
+  declarator:
+    (function_declarator
+      declarator: (identifier) @symbol.prototype))
+
+;; ============================================================
+;; Global Variables
+;; ============================================================
+
+(declaration
+  declarator:
+    (init_declarator
+      declarator: (identifier) @symbol.variable))
+
+(declaration
+  declarator:
+    (identifier) @symbol.variable)
+
+;; ============================================================
+;; Enum Constants
+;; ============================================================
+
+(enumerator
+  name: (identifier) @symbol.constant)
+
+;; ============================================================
+;; Macros
+;; ============================================================
+
+(preproc_def
+  name: (identifier) @symbol.macro)
+
+(preproc_function_def
+  name: (identifier) @symbol.macro)
+`;
+
+export const IMPORTS = `
+;; ============================================================
+;; #include
+;; ============================================================
+
+(preproc_include) @import
+`;
+
+export const EXPORTS = `
+`;
+export const CALLS = `
+;; ============================================================
+;; foo()
+;; ============================================================
+
+(call_expression
+  function: (identifier) @call)
+
+;; ============================================================
+;; ptr->foo()
+;; obj.foo()
+;; ============================================================
+
+(call_expression
+  function:
+    (field_expression
+      field: (field_identifier) @call))
+
+;; ============================================================
+;; (*fp)()
+;; ============================================================
+
+(call_expression
+  function:
+    (pointer_expression) @call)
+`;
+
+export const COMMENTS = `
+(comment) @comment
+`;
+
+export const TODOS = `
+(comment) @todo
+(#match? @todo "(?i)(TODO|FIXME|BUG|HACK|XXX|NOTE)")
+`;

package/packages/retrieval/parser/queries/cpp.js

@@ -0,0 +1,198 @@
+/**
+ * Tree-sitter queries for C++.
+ *
+ * Stable capture namespaces:
+ *
+ *  @symbol.namespace
+ *  @symbol.class
+ *  @symbol.struct
+ *  @symbol.union
+ *  @symbol.enum
+ *  @symbol.typedef
+ *  @symbol.alias
+ *  @symbol.template
+ *  @symbol.function
+ *  @symbol.constructor
+ *  @symbol.destructor
+ *  @symbol.method
+ *  @symbol.operator
+ *  @symbol.field
+ *  @symbol.variable
+ *  @symbol.constant
+ *  @symbol.macro
+ *
+ *  @import
+ *  @export
+ *  @call
+ *  @comment
+ *  @todo
+ */
+
+export const SYMBOLS = `
+;; ============================================================
+;; Namespaces
+;; ============================================================
+
+(namespace_definition
+  name: (namespace_identifier) @symbol.namespace)
+
+;; ============================================================
+;; Classes
+;; ============================================================
+
+(class_specifier
+  name: (type_identifier) @symbol.class)
+
+;; ============================================================
+;; Structs
+;; ============================================================
+
+(struct_specifier
+  name: (type_identifier) @symbol.struct)
+
+;; ============================================================
+;; Unions
+;; ============================================================
+
+(union_specifier
+  name: (type_identifier) @symbol.union)
+
+;; ============================================================
+;; Enums
+;; ============================================================
+
+(enum_specifier
+  name: (type_identifier) @symbol.enum)
+
+;; ============================================================
+;; Typedefs
+;; ============================================================
+
+(type_definition
+  declarator: (_) @symbol.typedef)
+
+;; ============================================================
+;; Using aliases
+;; ============================================================
+
+(alias_declaration
+  name: (type_identifier) @symbol.alias)
+
+;; ============================================================
+;; Templates
+;; ============================================================
+
+(template_declaration
+  declaration: (_) @symbol.template)
+
+;; ============================================================
+;; Constructors
+;; ============================================================
+
+(function_definition
+  declarator:
+    (function_declarator
+      declarator: (identifier) @symbol.constructor)
+  (#match? @symbol.constructor "^[A-Z]"))
+
+;; ============================================================
+;; Destructors
+;; ============================================================
+
+(function_definition
+  declarator:
+    (function_declarator
+      declarator: (destructor_name) @symbol.destructor))
+
+;; ============================================================
+;; Functions
+;; ============================================================
+
+(function_definition
+  declarator:
+    (function_declarator
+      declarator: (identifier) @symbol.function))
+
+;; ============================================================
+;; Methods
+;; ============================================================
+
+(field_declaration
+  declarator:
+    (function_declarator
+      declarator: (field_identifier) @symbol.method))
+
+;; ============================================================
+;; Operator overloads
+;; ============================================================
+
+(function_definition
+  declarator:
+    (function_declarator
+      declarator: (operator_name) @symbol.operator))
+
+;; ============================================================
+;; Fields
+;; ============================================================
+
+(field_declaration
+  declarator: (_) @symbol.field)
+
+;; ============================================================
+;; Variables
+;; ============================================================
+
+(declaration
+  declarator: (_) @symbol.variable)
+
+;; ============================================================
+;; Enum constants
+;; ============================================================
+
+(enumerator
+  name: (identifier) @symbol.constant)
+
+;; ============================================================
+;; Macros
+;; ============================================================
+
+(preproc_def
+  name: (identifier) @symbol.macro)
+
+(preproc_function_def
+  name: (identifier) @symbol.macro)
+`;
+
+export const IMPORTS = `
+(preproc_include) @import
+`;
+
+export const EXPORTS = `
+`;
+
+export const CALLS = `
+;; foo()
+
+(call_expression
+  function: (identifier) @call)
+
+;; obj.foo()
+
+(call_expression
+  function:
+    (field_expression
+      field: (field_identifier) @call))
+
+;; ns::foo()
+
+(call_expression
+  function:
+    (qualified_identifier
+      name: (identifier) @call))
+
+;; (*fp)()
+
+(call_expression
+  function:
+    (pointer_expression) @call)
+`;