@oisheek_c/codeforge 1.0.1

package/packages/retrieval/parser/queries/typescript.js

@@ -0,0 +1,202 @@
+/**
+ * Tree-sitter queries for JavaScript.
+ *
+ * Stable capture namespaces:
+ *
+ *  @symbol.class
+ *  @symbol.function
+ *  @symbol.method
+ *  @symbol.field
+ *  @symbol.variable
+ *
+ *  @import
+ *  @export
+ *  @call
+ *  @comment
+ *  @todo
+ */
+
+export const SYMBOLS = `
+;; ============================================================
+;; Classes
+;; ============================================================
+
+(class_declaration
+  name: (identifier) @symbol.class)
+
+;; ============================================================
+;; Functions
+;; ============================================================
+
+(function_declaration
+  name: (identifier) @symbol.function)
+
+(generator_function_declaration
+  name: (identifier) @symbol.function)
+
+;; Arrow functions
+
+(variable_declarator
+  name: (identifier) @symbol.function
+  value: (arrow_function))
+
+;; Function expressions
+
+(variable_declarator
+  name: (identifier) @symbol.function
+  value: (function))
+
+;; ============================================================
+;; Methods
+;; ============================================================
+
+(method_definition
+  name: (property_identifier) @symbol.method)
+
+;; ============================================================
+;; Class fields (modern JavaScript)
+;; ============================================================
+
+(public_field_definition
+  name: (property_identifier) @symbol.field)
+
+;; ============================================================
+;; Variables
+;; ============================================================
+
+(variable_declarator
+  name: (identifier) @symbol.variable)
+`;
+
+export const IMPORTS = `
+;; ============================================================
+;; ES Modules
+;; ============================================================
+
+(import_statement) @import
+
+;; ============================================================
+;; require(...)
+;; ============================================================
+
+(call_expression
+  function: (identifier) @_require
+  (#eq? @_require "require"))
+@import
+
+;; ============================================================
+;; Dynamic import(...)
+;; ============================================================
+
+(import_expression) @import
+`;
+
+export const EXPORTS = `
+;; ============================================================
+;; Named declaration exports
+;; ============================================================
+
+(export_statement
+  declaration:
+    (function_declaration
+      name: (identifier) @export.named))
+
+(export_statement
+  declaration:
+    (generator_function_declaration
+      name: (identifier) @export.named))
+
+(export_statement
+  declaration:
+    (class_declaration
+      name: (type_identifier) @export.named))
+
+(export_statement
+  declaration:
+    (lexical_declaration
+      (variable_declarator
+        name: (identifier) @export.named)))
+
+;; ============================================================
+;; Named export specifiers
+;; ============================================================
+
+(export_specifier
+  alias: (identifier) @export.named)
+
+(export_specifier
+  name: (identifier) @export.named)
+
+;; ============================================================
+;; Default named declarations
+;; ============================================================
+
+(export_statement
+  declaration:
+    (function_declaration
+      name: (identifier) @export.default))
+
+(export_statement
+  declaration:
+    (class_declaration
+      name: (type_identifier) @export.default))
+
+;; ============================================================
+;; CommonJS: module.exports = ...
+;; ============================================================
+
+(assignment_expression
+  left:
+    (member_expression
+      object: (identifier) @_module
+      property: (property_identifier) @export.commonjs)
+  (#eq? @_module "module")
+  (#eq? @export.commonjs "exports"))
+
+;; ============================================================
+;; CommonJS: exports.foo = ...
+;; ============================================================
+
+(assignment_expression
+  left:
+    (member_expression
+      object: (identifier) @_exports
+      property: (property_identifier) @export.commonjs)
+  (#eq? @_exports "exports"))
+`;
+
+export const CALLS = `
+;; foo()
+
+(call_expression
+  function: (identifier) @call)
+
+;; obj.foo()
+
+(call_expression
+  function:
+    (member_expression
+      property: (property_identifier) @call))
+
+;; obj["foo"]()
+
+(call_expression
+  function:
+    (subscript_expression) @call)
+
+;; obj?.foo()
+
+(optional_call_expression
+  function:
+    (member_expression
+      property: (property_identifier) @call))
+`;
+
+export const COMMENTS = `
+(comment) @comment
+`;
+
+export const TODOS = `
+(comment) @todo
+(#match? @todo "(?i)(TODO|FIXME|BUG|HACK|XXX|NOTE)")
+`;

package/packages/retrieval/parser/registry.js

@@ -0,0 +1,107 @@
+/**
+ * packages/retrieval/parser/registry.js
+ */
+
+import Parser from "tree-sitter";
+
+import * as languages from "./languages.js";
+import { getQuerySet } from "./queries/index.js";
+
+const parserCache = new Map();
+const queryCache = new Map();
+
+export function supportedLanguages() {
+    return [
+        "javascript",
+        "typescript",
+        "python",
+        "java",
+        "go",
+        "rust",
+        "c",
+        "cpp",
+        "csharp",
+        "php",
+        "ruby",
+        "swift",
+    ];
+}
+
+/* -------------------------------------------------------------------------- */
+/* Parser                                                                      */
+/* -------------------------------------------------------------------------- */
+
+export function getParser(language) {
+    if (parserCache.has(language)) {
+        return parserCache.get(language);
+    }
+
+    const grammar = languages.getGrammar(language);
+
+    if (!grammar) {
+        throw new Error(`Unsupported language: ${language}`);
+    }
+
+    const parser = new Parser();
+
+    parser.setLanguage(grammar);
+
+    parserCache.set(language, parser);
+
+    return parser;
+}
+
+/* -------------------------------------------------------------------------- */
+/* Queries                                                                     */
+/* -------------------------------------------------------------------------- */
+
+export function getQueries(language) {
+    if (queryCache.has(language)) {
+        return queryCache.get(language);
+    }
+
+    const grammar = languages.getGrammar(language);
+    const querySet = getQuerySet(language);
+
+    if (!grammar) {
+        throw new Error(`Unsupported language: ${language}`);
+    }
+
+    if (!querySet) {
+        throw new Error(`No query set registered for language: ${language}`);
+    }
+
+    const compiled = {};
+
+    for (const [name, source] of Object.entries(querySet)) {
+    if (!source?.trim()) {
+        compiled[name] = null;
+        continue;
+    }
+
+    try {
+        compiled[name] = new Parser.Query(grammar, source);
+    } catch (err) {
+        console.error("\n========================================");
+        console.error(`Failed language : ${language}`);
+        console.error(`Failed query    : ${name}`);
+        console.error("========================================");
+        console.error(source);
+        console.error("========================================\n");
+        throw err;
+    }
+}
+
+    queryCache.set(language, compiled);
+
+    return compiled;
+}
+
+/* -------------------------------------------------------------------------- */
+/* Cache                                                                       */
+/* -------------------------------------------------------------------------- */
+
+export function clearRegistryCache() {
+    parserCache.clear();
+    queryCache.clear();
+}

package/packages/retrieval/parser/traverse.js

@@ -0,0 +1,82 @@
+/**
+ * packages/retrieval/parser/traverse.js
+ */
+
+import extractSymbols from "./extractors/symbols.js";
+import extractImports from "./extractors/imports.js";
+import extractExports from "./extractors/exports.js";
+import extractCalls from "./extractors/calls.js";
+import extractComments from "./extractors/comments.js";
+import extractTodos from "./extractors/todos.js";
+import extractDiagnostics from "./extractors/diagnostics.js";
+import extractMetrics from "./extractors/metrics.js";
+
+export function traverse({
+    tree,
+    source,
+    file,
+    language,
+    queries,
+}) {
+    const symbols = extractSymbols({
+        query: queries.SYMBOLS,
+        tree,
+        source,
+        file,
+        language,
+    });
+
+    const imports = extractImports({
+        query: queries.IMPORTS,
+        tree,
+        source,
+    });
+
+    const exportsList = extractExports({
+        query: queries.EXPORTS,
+        tree,
+        source,
+    });
+
+    const calls = extractCalls({
+        query: queries.CALLS,
+        tree,
+        source,
+    });
+
+    const comments = extractComments({
+        query: queries.COMMENTS,
+        tree,
+        source,
+    });
+
+    const todos = extractTodos({
+        query: queries.TODOS,
+        tree,
+        source,
+    });
+
+    const metrics = extractMetrics({
+        source,
+        symbols,
+        imports,
+        exports: exportsList,
+        todos,
+    });
+
+    const diagnostics = extractDiagnostics({
+        tree,
+        source,
+    });
+
+    return {
+        symbols,
+        imports,
+        exports: exportsList,
+        calls,
+        comments,
+        todos,
+        metrics,
+        diagnostics,
+    };
+}

package/packages/retrieval/parser/types.js

@@ -0,0 +1,239 @@
+/**
+ * packages/retrieval/parser/types.js
+ *
+ * Normalized parser data contracts.
+ * Every extractor should create objects using these factories.
+ */
+
+import crypto from "node:crypto";
+
+/* -------------------------------------------------------------------------- */
+/* Location                                                                    */
+/* -------------------------------------------------------------------------- */
+
+export function createLocation(data = {}) {
+    return {
+        startLine: data.startLine ?? 0,
+        startColumn: data.startColumn ?? 0,
+
+        endLine: data.endLine ?? 0,
+        endColumn: data.endColumn ?? 0,
+    };
+}
+
+/* -------------------------------------------------------------------------- */
+/* Parsed File                                                                 */
+/* -------------------------------------------------------------------------- */
+
+export function createParsedFile(data = {}) {
+    return {
+        path: data.path ?? "",
+        name: data.name ?? "",
+        extension: data.extension ?? "",
+        language: data.language ?? "",
+
+        hash: data.hash ?? "",
+        size: data.size ?? 0,
+
+        supported: data.supported ?? true,
+
+        parser: data.parser ?? "tree-sitter",
+        parserVersion: data.parserVersion ?? 1,
+        grammarVersion: data.grammarVersion ?? 1,
+
+        source: data.source ?? "",
+        ast: data.ast ?? null,
+
+        imports: data.imports ?? [],
+        exports: data.exports ?? [],
+        symbols: data.symbols ?? [],
+        calls: data.calls ?? [],
+        comments: data.comments ?? [],
+        todos: data.todos ?? [],
+
+        diagnostics: data.diagnostics ?? [],
+        metrics: data.metrics ?? createMetrics(),
+    };
+}
+
+/* -------------------------------------------------------------------------- */
+/* Symbol                                                                      */
+/* -------------------------------------------------------------------------- */
+
+export function createSymbol(data = {}) {
+    return {
+        id: data.id ?? crypto.randomUUID(),
+
+        name: data.name ?? "anonymous",
+        qualifiedName: data.qualifiedName ?? null,
+
+        kind: data.kind ?? "unknown",
+
+        language: data.language ?? "",
+        file: data.file ?? "",
+
+        parent: data.parent ?? null,
+
+        exported: data.exported ?? false,
+
+        signature: data.signature ?? null,
+
+        modifiers: data.modifiers ?? [],
+
+        metadata: data.metadata ?? {},
+
+        location: data.location ?? createLocation(),
+    };
+}
+
+/* -------------------------------------------------------------------------- */
+/* Import                                                                      */
+/* -------------------------------------------------------------------------- */
+
+export function createImport(data = {}) {
+    return {
+        source: data.source ?? "",
+
+        default: data.default ?? null,
+        namespace: data.namespace ?? null,
+        specifiers: data.specifiers ?? [],
+
+        alias: data.alias ?? null,
+
+        typeOnly: data.typeOnly ?? false,
+        dynamic: data.dynamic ?? false,
+
+        metadata: data.metadata ?? {},
+
+        location: data.location ?? createLocation(),
+    };
+}
+
+/* -------------------------------------------------------------------------- */
+/* Export                                                                      */
+/* -------------------------------------------------------------------------- */
+
+export function createExport(data = {}) {
+    return {
+        name: data.name ?? "",
+
+        kind: data.kind ?? "named",
+
+        source: data.source ?? null,
+
+        default: data.default ?? false,
+
+        exported: data.exported ?? null,
+
+        metadata: data.metadata ?? {},
+
+        location: data.location ?? createLocation(),
+    };
+}
+
+/* -------------------------------------------------------------------------- */
+/* Call                                                                        */
+/* -------------------------------------------------------------------------- */
+
+export function createCall(data = {}) {
+    return {
+        name: data.name ?? "",
+
+        kind: data.kind ?? "function",
+
+        receiver: data.receiver ?? null,
+
+        arguments: data.arguments ?? 0,
+
+        async: data.async ?? false,
+
+        metadata: data.metadata ?? {},
+
+        location: data.location ?? createLocation(),
+    };
+}
+/* -------------------------------------------------------------------------- */
+/* Comment                                                                     */
+/* -------------------------------------------------------------------------- */
+
+export function createComment(data = {}) {
+    return {
+        text: data.text ?? "",
+
+        block: data.block ?? false,
+
+        documentation: data.documentation ?? false,
+
+        line: data.line ?? 0,
+
+        metadata: data.metadata ?? {},
+
+        location: data.location ?? createLocation(),
+    };
+}
+
+/* -------------------------------------------------------------------------- */
+/* TODO                                                                        */
+/* -------------------------------------------------------------------------- */
+
+export function createTodo(data = {}) {
+    return {
+        type: data.type ?? "TODO",
+
+        text: data.text ?? "",
+
+        line: data.line ?? 0,
+
+        metadata: data.metadata ?? {},
+
+        location: data.location ?? createLocation(),
+    };
+}
+
+/* -------------------------------------------------------------------------- */
+/* Diagnostic                                                                  */
+/* -------------------------------------------------------------------------- */
+
+export function createDiagnostic(data = {}) {
+    return {
+        severity: data.severity ?? "error",
+
+        type: data.type ?? "syntax",
+
+        message: data.message ?? "",
+
+        location: data.location ?? createLocation(),
+    };
+}
+
+/* -------------------------------------------------------------------------- */
+/* Metrics                                                                     */
+/* -------------------------------------------------------------------------- */
+
+export function createMetrics(data = {}) {
+    return {
+        files: data.files ?? 1,
+
+        lines: data.lines ?? 0,
+        code: data.code ?? 0,
+        comments: data.comments ?? 0,
+        blanks: data.blanks ?? 0,
+
+        statements: data.statements ?? 0,
+
+        functions: data.functions ?? 0,
+        classes: data.classes ?? 0,
+        methods: data.methods ?? 0,
+
+        imports: data.imports ?? 0,
+        exports: data.exports ?? 0,
+
+        todos: data.todos ?? 0,
+
+        branches: data.branches ?? 0,
+
+        depth: data.depth ?? 0,
+
+        complexity: data.complexity ?? 0,
+    };
+}

package/packages/retrieval/parser.js

@@ -0,0 +1,78 @@
+/**
+ * packages/retrieval/parser.js
+ *
+ * Builds the complete retrieval pipeline for a repository.
+ */
+
+import fs from "node:fs/promises";
+import path from "node:path";
+
+import { DEFAULT_IGNORE } from "../scanner/ignore.js";
+
+import { parseRepository } from "./parser/index.js";
+
+import { buildSymbolIndex } from "./symbols.js";
+import { buildImportIndex } from "./imports.js";
+import { buildRepositoryGraph } from "./graph.js";
+import { createRetriever } from "./retriever.js";
+
+async function collectFiles(root) {
+    const files = [];
+
+    async function walk(dir) {
+        const entries = await fs.readdir(dir, {
+            withFileTypes: true,
+        });
+
+        for (const entry of entries) {
+            if (DEFAULT_IGNORE.includes(entry.name)) {
+                continue;
+            }
+
+            const fullPath = path.join(dir, entry.name);
+
+            if (entry.isDirectory()) {
+                await walk(fullPath);
+            } else {
+                files.push(fullPath);
+            }
+        }
+    }
+
+    await walk(root);
+
+    return files;
+}
+
+/**
+ * Build complete retrieval pipeline.
+ */
+export async function buildRepository(repositoryRoot, cache = null) {
+    const fileIndex = await collectFiles(repositoryRoot);
+
+    const repository = await parseRepository(fileIndex, cache);
+
+    const symbols = buildSymbolIndex(repository);
+    const imports = buildImportIndex(repository);
+
+    const graph = buildRepositoryGraph(
+        repository,
+        symbols,
+        imports
+    );
+
+    const retriever = createRetriever({
+        repository,
+        symbols,
+        imports,
+        graph,
+    });
+
+    return {
+        repository,
+        symbols,
+        imports,
+        graph,
+        retriever,
+    };
+}