@oisheek_c/codeforge 1.0.1

package/packages/retrieval/budget.js

@@ -0,0 +1,92 @@
+/**
+ * packages/retrieval/budget.js
+ *
+ * Context budget manager.
+ */
+
+export class ContextBudget {
+    constructor(limit = 16000) {
+        this.limit = limit;
+    }
+
+    /**
+     * Estimate token count.
+     *
+     * Default heuristic:
+     * 1 token ≈ 4 characters.
+     */
+    estimate(text = "") {
+        return Math.ceil(text.length / 4);
+    }
+
+    /**
+     * Select as many items as fit within the budget.
+     */
+    select(items) {
+        let used = 0;
+
+        const selected = [];
+
+        for (const item of items) {
+            const text =
+                item.text ??
+                item.metadata?.text ??
+                "";
+
+            const cost = this.estimate(text);
+
+            if (used + cost > this.limit) {
+                continue;
+            }
+
+            used += cost;
+            selected.push(item);
+        }
+
+        return {
+            items: selected,
+            used,
+            remaining: this.limit - used,
+            limit: this.limit,
+            utilization:
+                this.limit === 0
+                    ? 0
+                    : used / this.limit,
+        };
+    }
+
+    /**
+     * Check whether a single item fits.
+     */
+    fits(item) {
+        const text =
+            item.text ??
+            item.metadata?.text ??
+            "";
+
+        return this.estimate(text) <= this.limit;
+    }
+
+    /**
+     * Remaining budget after using some tokens.
+     */
+    remaining(used) {
+        return Math.max(0, this.limit - used);
+    }
+
+    /**
+     * Budget statistics.
+     */
+    stats() {
+        return {
+            limit: this.limit,
+        };
+    }
+}
+
+/**
+ * Factory.
+ */
+export function createContextBudget(limit) {
+    return new ContextBudget(limit);
+}

package/packages/retrieval/cache.js

@@ -0,0 +1,84 @@
+/**
+ * packages/retrieval/cache.js
+ *
+ * Simple in-memory cache for retrieval artifacts.
+ */
+
+export class RetrievalCache {
+    constructor() {
+        this.clear();
+    }
+
+    clear() {
+        this.cache = new Map();
+    }
+
+    /**
+     * Check if a key exists.
+     */
+    has(key) {
+        return this.cache.has(key);
+    }
+
+    /**
+     * Retrieve a cached value.
+     */
+    get(key) {
+        const entry = this.cache.get(key);
+
+        if (!entry) {
+            return null;
+        }
+
+        return entry.value;
+    }
+
+    /**
+     * Store a value.
+     */
+    set(key, value) {
+        this.cache.set(key, {
+            value,
+            createdAt: Date.now(),
+        });
+
+        return value;
+    }
+
+    /**
+     * Remove one cache entry.
+     */
+    delete(key) {
+        return this.cache.delete(key);
+    }
+
+    /**
+     * Number of cached entries.
+     */
+    size() {
+        return this.cache.size;
+    }
+
+    /**
+     * List all cache keys.
+     */
+    keys() {
+        return [...this.cache.keys()];
+    }
+
+    /**
+     * Cache statistics.
+     */
+    stats() {
+        return {
+            entries: this.cache.size,
+        };
+    }
+}
+
+/**
+ * Factory.
+ */
+export function createRetrievalCache() {
+    return new RetrievalCache();
+}

package/packages/retrieval/embeddings.js

@@ -0,0 +1,147 @@
+/**
+ * packages/retrieval/embeddings.js
+ *
+ * Embedding provider abstraction and vector store.
+ */
+
+/**
+ * Base embedding provider.
+ */
+export class EmbeddingProvider {
+    async embed(_text) {
+        throw new Error("embed() not implemented");
+    }
+
+    async embedBatch(texts) {
+        return Promise.all(
+            texts.map(text => this.embed(text))
+        );
+    }
+}
+
+/**
+ * Normalize a vector.
+ */
+export function normalizeVector(vector) {
+    const magnitude = Math.sqrt(
+        vector.reduce((sum, value) => sum + value * value, 0)
+    );
+
+    if (magnitude === 0) {
+        return [...vector];
+    }
+
+    return vector.map(value => value / magnitude);
+}
+
+/**
+ * In-memory embedding store.
+ */
+export class EmbeddingStore {
+    constructor() {
+        this.clear();
+    }
+
+    clear() {
+        this.vectors = new Map();
+    }
+
+    set(key, embedding, metadata = {}) {
+        this.vectors.set(key, {
+            embedding: normalizeVector(embedding),
+            metadata,
+            createdAt: Date.now(),
+        });
+    }
+
+    setMany(entries) {
+        for (const entry of entries) {
+            this.set(
+                entry.key,
+                entry.embedding,
+                entry.metadata
+            );
+        }
+    }
+
+    get(key) {
+        return this.vectors.get(key) ?? null;
+    }
+
+    has(key) {
+        return this.vectors.has(key);
+    }
+
+    delete(key) {
+        return this.vectors.delete(key);
+    }
+
+    entries() {
+        return [...this.vectors.entries()];
+    }
+
+    size() {
+        return this.vectors.size;
+    }
+
+    stats() {
+        return {
+            vectors: this.vectors.size,
+        };
+    }
+}
+
+/**
+ * Cosine similarity.
+ */
+export function cosineSimilarity(a, b) {
+    if (
+        !Array.isArray(a) ||
+        !Array.isArray(b) ||
+        a.length !== b.length ||
+        a.length === 0
+    ) {
+        return 0;
+    }
+
+    let dot = 0;
+
+    let magA = 0;
+    let magB = 0;
+
+    for (let i = 0; i < a.length; i++) {
+        dot += a[i] * b[i];
+        magA += a[i] * a[i];
+        magB += b[i] * b[i];
+    }
+
+    if (magA === 0 || magB === 0) {
+        return 0;
+    }
+
+    return dot / (Math.sqrt(magA) * Math.sqrt(magB));
+}
+
+/**
+ * Search embeddings.
+ */
+export function searchEmbeddings(queryVector, store) {
+    const query = normalizeVector(queryVector);
+
+    return store
+        .entries()
+        .map(([key, value]) => ({
+            key,
+            score: cosineSimilarity(
+                query,
+                value.embedding
+            ),
+            metadata: value.metadata,
+        }))
+        .sort((a, b) => b.score - a.score);
+}
+
+/**
+ * Default singleton store.
+ */
+export const embeddingStore = new EmbeddingStore();

package/packages/retrieval/graph.js

@@ -0,0 +1,151 @@
+/**
+ * packages/retrieval/graph.js
+ *
+ * Repository dependency graph.
+ */
+
+export class RepositoryGraph {
+    constructor() {
+        this.clear();
+    }
+
+    clear() {
+        this.files = new Map();
+
+        // Graph edges
+        this.dependencies = new Map();
+        this.dependents = new Map();
+
+        // External indexes
+        this.symbolIndex = null;
+        this.importIndex = null;
+    }
+
+    /**
+     * Build graph from repository indexes.
+     */
+    build(repository, symbolIndex, importIndex) {
+        this.clear();
+
+        this.symbolIndex = symbolIndex;
+        this.importIndex = importIndex;
+
+        const files = repository.files ?? repository;
+
+        for (const file of files) {
+            this.addFile(file);
+        }
+
+        return this;
+    }
+
+    /**
+     * Add one parsed file.
+     */
+    addFile(parsedFile) {
+        this.files.set(parsedFile.path, parsedFile);
+
+        const edges = new Set();
+
+        const imports = this.importIndex?.file(parsedFile.path) ?? [];
+
+        for (const imp of imports) {
+            edges.add(imp.source);
+
+            if (!this.dependents.has(imp.source)) {
+                this.dependents.set(imp.source, new Set());
+            }
+
+            this.dependents
+                .get(imp.source)
+                .add(parsedFile.path);
+        }
+
+        this.dependencies.set(parsedFile.path, edges);
+    }
+
+    /**
+     * Lookup a parsed file.
+     */
+    getFile(path) {
+        return this.files.get(path) ?? null;
+    }
+
+    /**
+     * All parsed files.
+     */
+    getFiles() {
+        return [...this.files.values()];
+    }
+
+    /**
+     * File dependencies.
+     */
+    getDependencies(path) {
+        return [...(this.dependencies.get(path) ?? [])];
+    }
+
+    /**
+     * Reverse dependencies.
+     */
+    getDependents(module) {
+        return [...(this.dependents.get(module) ?? [])];
+    }
+
+    /**
+     * Symbol lookup.
+     *
+     * Delegates to SymbolIndex.
+     */
+    findSymbol(name) {
+        return this.symbolIndex?.find(name) ?? [];
+    }
+
+    /**
+     * File exists.
+     */
+    hasFile(path) {
+        return this.files.has(path);
+    }
+
+    /**
+     * Repository statistics.
+     */
+    stats() {
+        let edgeCount = 0;
+
+        for (const deps of this.dependencies.values()) {
+            edgeCount += deps.size;
+        }
+
+        return {
+            files: this.files.size,
+            edges: edgeCount,
+            symbols: this.symbolIndex?.stats().total ?? 0,
+            modules: this.importIndex?.stats().modules ?? 0,
+        };
+    }
+
+    fileCount() {
+        return this.files.size;
+    }
+
+    symbolCount() {
+        return this.symbolIndex?.stats().total ?? 0;
+    }
+}
+
+/**
+ * Convenience builder.
+ */
+export function buildRepositoryGraph(
+    repository,
+    symbolIndex,
+    importIndex
+) {
+    return new RepositoryGraph().build(
+        repository,
+        symbolIndex,
+        importIndex
+    );
+}

package/packages/retrieval/imports.js

@@ -0,0 +1,74 @@
+/**
+ * packages/retrieval/imports.js
+ *
+ * Repository-wide import index.
+ */
+
+export class ImportIndex {
+    constructor() {
+        this.clear();
+    }
+
+    clear() {
+        this.byFile = new Map();
+        this.bySource = new Map();
+    }
+
+    add(file, imp) {
+        if (!this.byFile.has(file)) {
+            this.byFile.set(file, []);
+        }
+
+        this.byFile.get(file).push(imp);
+
+        if (!this.bySource.has(imp.source)) {
+            this.bySource.set(imp.source, []);
+        }
+
+        this.bySource.get(imp.source).push({
+            file,
+            import: imp,
+        });
+    }
+
+    addFile(parsedFile) {
+        if (!parsedFile?.imports) {
+            return;
+        }
+
+        for (const imp of parsedFile.imports) {
+            this.add(parsedFile.path, imp);
+        }
+    }
+
+    build(repository) {
+        this.clear();
+
+        const files = repository.files ?? repository;
+
+        for (const file of files) {
+            this.addFile(file);
+        }
+
+        return this;
+    }
+
+    file(path) {
+        return this.byFile.get(path) ?? [];
+    }
+
+    source(module) {
+        return this.bySource.get(module) ?? [];
+    }
+
+    stats() {
+        return {
+            files: this.byFile.size,
+            modules: this.bySource.size,
+        };
+    }
+}
+
+export function buildImportIndex(repository) {
+    return new ImportIndex().build(repository);
+}

package/packages/retrieval/index.js

@@ -0,0 +1,9 @@
+export * from "./budget.js";
+export * from "./cache.js";
+export * from "./embeddings.js";
+export * from "./graph.js";
+export * from "./imports.js";
+export * from "./parser.js";
+export * from "./ranker.js";
+export * from "./retriever.js";
+export * from "./symbols.js";

package/packages/retrieval/parser/extractors/calls.js

@@ -0,0 +1,43 @@
+/**
+ * packages/retrieval/parser/extractors/calls.js
+ */
+
+import {
+    iterateCaptures,
+    getNodeLocation,
+    getNodeText,
+} from "../helpers.js";
+
+import { createCall } from "../types.js";
+
+export default function extractCalls({
+    query,
+    tree,
+    source,
+}) {
+    const calls = [];
+
+    if (!query) {
+        return calls;
+    }
+
+    for (const { node, name } of iterateCaptures(query, tree)) {
+        calls.push(
+            createCall({
+                // Query should capture the called identifier/member.
+                name: node.text,
+
+                // e.g. call.function, call.method, call.constructor
+                kind: name.replace(/^call\./, ""),
+
+                location: getNodeLocation(node),
+
+                metadata: {
+                    text: getNodeText(node, source),
+                },
+            })
+        );
+    }
+
+    return calls;
+}

package/packages/retrieval/parser/extractors/comments.js

@@ -0,0 +1,49 @@
+/**
+ * packages/retrieval/parser/extractors/comments.js
+ */
+
+import {
+    iterateCaptures,
+    getNodeLocation,
+    getNodeText,
+} from "../helpers.js";
+
+import { createComment } from "../types.js";
+
+export default function extractComments({
+    query,
+    tree,
+    source,
+}) {
+    const comments = [];
+
+    if (!query) {
+        return comments;
+    }
+
+    for (const { node } of iterateCaptures(query, tree)) {
+        const text = getNodeText(node, source);
+
+        comments.push(
+            createComment({
+                text,
+
+                block: text.startsWith("/*"),
+
+                documentation:
+                    text.startsWith("/**") ||
+                    text.startsWith("///"),
+
+                line: node.startPosition.row + 1,
+
+                metadata: {
+                    raw: text,
+                },
+
+                location: getNodeLocation(node),
+            })
+        );
+    }
+
+    return comments;
+}

package/packages/retrieval/parser/extractors/diagnostics.js

@@ -0,0 +1,21 @@
+/**
+ * packages/retrieval/parser/extractors/diagnostics.js
+ */
+
+import { createDiagnostic } from "../types.js";
+
+export default function extractDiagnostics({
+    tree,
+}) {
+    if (!tree.rootNode.hasError) {
+        return [];
+    }
+
+    return [
+        createDiagnostic({
+            severity: "error",
+            type: "syntax",
+            message: "Tree-sitter detected syntax errors.",
+        }),
+    ];
+}