@unerr-ai/unerr 0.2.1 → 0.2.2

package/dist/intelligence/search-index.js

@@ -1,173 +0,0 @@
-/**
- * Local search index using CozoDB relations.
- *
- * Tokenizes entity names (camelCase, snake_case, PascalCase) and stores
- * as search_tokens relation for fast local text search.
- *
- * Sprint 6.6: IDF-weighted scoring — rare tokens rank higher than common ones.
- * Pre-computes IDF weights during buildSearchIndex() for O(1) lookup at query time.
- */
-/**
- * Tokenize an entity name into searchable tokens.
- * Handles camelCase, PascalCase, snake_case, and kebab-case.
- */
-export function tokenize(name) {
-    // Split on non-alphanumeric, then split camelCase
-    const parts = name
-        .replace(/([a-z])([A-Z])/g, "$1 $2")
-        .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2")
-        .split(/[^a-zA-Z0-9]+/)
-        .filter((p) => p.length > 0)
-        .map((p) => p.toLowerCase());
-    return [...new Set(parts)];
-}
-/**
- * Build search index from entities already loaded in CozoDB.
- * Reads all entities, tokenizes names, populates search_tokens,
- * and computes IDF weights in token_doc_frequency.
- */
-export async function buildSearchIndex(db) {
-    // Read all entities
-    let result;
-    try {
-        result = await db.run("?[key, name] := *entities{key, name}");
-    }
-    catch (err) {
-        process.stderr.write(`[unerr:search-index] Failed to read entities: ${err instanceof Error ? err.message : JSON.stringify(err)}\n`);
-        return; // Cannot build search index without entities
-    }
-    if (!result?.rows)
-        return;
-    const totalEntities = result.rows.length;
-    process.stderr.write(`[unerr:search-index] Building index for ${totalEntities} entities\n`);
-    // Track document frequency: how many entities contain each token
-    const tokenDocCount = new Map();
-    for (const row of result.rows) {
-        const [key, name] = row;
-        const tokens = tokenize(name);
-        for (const token of tokens) {
-            tokenDocCount.set(token, (tokenDocCount.get(token) ?? 0) + 1);
-            try {
-                await db.run("?[token, entity_key] <- [[$token, $key]] :put search_tokens { token, entity_key }", { token, key });
-            }
-            catch {
-                // Duplicate — ignore
-            }
-        }
-    }
-    // Store document frequencies + pre-computed IDF weights
-    let idfErrors = 0;
-    for (const [token, docCount] of tokenDocCount) {
-        const idf = totalEntities > 0 ? Math.log(totalEntities / Math.max(docCount, 1)) : 0;
-        try {
-            await db.run("?[token, doc_count, idf] <- [[$token, $dc, $idf]] :put token_doc_frequency { token => doc_count, idf }", { token, dc: docCount, idf });
-        }
-        catch {
-            idfErrors++;
-        }
-    }
-    process.stderr.write(`[unerr:search-index] Done: ${tokenDocCount.size} tokens indexed, ${idfErrors} IDF errors\n`);
-}
-/**
- * Incrementally update search index for specific entities.
- * Removes old tokens for given keys, re-tokenizes, and updates IDF weights.
- */
-export async function updateSearchIndexIncremental(db, changedKeys, deletedKeys) {
-    if (changedKeys.size === 0 && deletedKeys.size === 0)
-        return;
-    const allKeys = new Set([...changedKeys, ...deletedKeys]);
-    // Remove old tokens for all affected keys
-    for (const key of allKeys) {
-        try {
-            await db.run("?[token, entity_key] := *search_tokens{token, entity_key}, entity_key = $key :rm search_tokens { token, entity_key }", { key });
-        }
-        catch {
-            /* safe */
-        }
-    }
-    // Re-tokenize changed entities (not deleted ones)
-    if (changedKeys.size === 0)
-        return;
-    // Fetch names for changed keys
-    const keyRows = [...changedKeys]
-        .map((k) => `["${k.replace(/"/g, '\\"')}"]`)
-        .join(", ");
-    let result;
-    try {
-        result = await db.run(`
-      keys[k] <- [${keyRows}]
-      ?[key, name] := keys[k], *entities{key: k, name}, key = k
-    `);
-    }
-    catch {
-        return;
-    }
-    if (!result?.rows)
-        return;
-    // Insert new tokens
-    for (const row of result.rows) {
-        const [key, name] = row;
-        const tokens = tokenize(name);
-        for (const token of tokens) {
-            try {
-                await db.run("?[token, entity_key] <- [[$token, $key]] :put search_tokens { token, entity_key }", { token, key });
-            }
-            catch {
-                /* safe */
-            }
-        }
-    }
-    // Recompute IDF for all tokens (lightweight — just counts + math)
-    try {
-        const totalResult = await db.run("?[count(key)] := *entities{key}");
-        const totalEntities = totalResult.rows?.[0]?.[0] ?? 1;
-        const tokenResult = await db.run("?[token, count(entity_key)] := *search_tokens{token, entity_key}");
-        if (tokenResult.rows) {
-            for (const row of tokenResult.rows) {
-                const [token, docCount] = row;
-                const idf = Math.log(totalEntities / Math.max(docCount, 1));
-                try {
-                    await db.run("?[token, doc_count, idf] <- [[$token, $dc, $idf]] :put token_doc_frequency { token => doc_count, idf }", { token, dc: docCount, idf });
-                }
-                catch {
-                    /* safe */
-                }
-            }
-        }
-    }
-    catch {
-        /* IDF update failed — search still works, just with stale weights */
-    }
-}
-/**
- * Search local entities by query string.
- * Tokenizes query, finds matching entities via token intersection,
- * ranks by IDF-weighted score (rare tokens contribute more to score).
- */
-export async function searchLocal(db, query, limit = 20) {
-    const queryTokens = tokenize(query);
-    if (queryTokens.length === 0)
-        return [];
-    // Find entities that match ANY token, sum IDF weights per entity
-    const tokenRows = queryTokens.map((t) => `["${t}"]`).join(", ");
-    let result;
-    try {
-        result = await db.run(`
-      tokens[t] <- [${tokenRows}]
-      matched[ek, sum(w)] := tokens[t], *search_tokens[t, ek], *token_doc_frequency[t, _, w]
-      ?[ek, score, name, kind, fp] := matched[ek, score],
-        *entities{key: ek, kind, name, file_path: fp}
-      :order -score
-      :limit ${limit}
-    `);
-    }
-    catch {
-        return [];
-    }
-    if (!result?.rows)
-        return [];
-    return result.rows.map((row) => {
-        const [key, score, name, kind, file_path] = row;
-        return { key, name, kind, file_path, score };
-    });
-}

package/dist/intelligence/semantic/docstring-extractor.js

@@ -1,67 +0,0 @@
-/**
- * Docstring/Comment Extractor — extracts documentation annotations from entities.
- *
- * Reads JSDoc, Python docstrings, Go doc comments, etc. from the source
- * and associates them with the closest entity declaration.
- */
-/**
- * Extract doc comment text from raw source at a specific line.
- * Looks backward from the entity start line for comment blocks.
- */
-export function extractDocComment(source, entityStartLine) {
-    const lines = source.split("\n");
-    const lineIdx = entityStartLine - 1;
-    if (lineIdx <= 0 || lineIdx >= lines.length)
-        return null;
-    const commentLines = [];
-    let i = lineIdx - 1;
-    while (i >= 0) {
-        const line = lines[i]?.trim();
-        if (line === undefined)
-            break;
-        if (line.startsWith("*") || line.startsWith("//") || line.startsWith("#")) {
-            commentLines.unshift(line);
-            i--;
-        }
-        else if (line === "/**" || line === "*/") {
-            commentLines.unshift(line);
-            i--;
-        }
-        else if (line.startsWith("/*")) {
-            commentLines.unshift(line);
-            break;
-        }
-        else if (line === '"""' || line === "'''") {
-            commentLines.unshift(line);
-            break;
-        }
-        else {
-            break;
-        }
-    }
-    if (commentLines.length === 0)
-        return null;
-    const text = commentLines
-        .join("\n")
-        .replace(/^\/\*\*?\s*|\s*\*\/$/g, "")
-        .replace(/^\s*\*\s?/gm, "")
-        .replace(/^\/\/\s?/gm, "")
-        .replace(/^#\s?/gm, "")
-        .replace(/^"""|'''|"""|'''/gm, "")
-        .trim();
-    return text.length > 0 ? text.slice(0, 500) : null;
-}
-/**
- * Extract JSDoc tags (@param, @returns, @deprecated, etc.)
- */
-export function extractDocTags(docstring) {
-    const tagPattern = /@(\w+)/g;
-    const tags = [];
-    let match;
-    match = tagPattern.exec(docstring);
-    while (match !== null) {
-        tags.push(match[1]);
-        match = tagPattern.exec(docstring);
-    }
-    return [...new Set(tags)];
-}

package/dist/intelligence/semantic/embedding-store.js

@@ -1,52 +0,0 @@
-/**
- * CozoDB Embedding Store — stores Float32Array embeddings inline.
- *
- * Each entity gets a 128-dim embedding (512 bytes) stored as a
- * base64-encoded string in CozoDB. Provides cosine similarity
- * queries in <5ms.
- */
-/**
- * Encode a Float32Array to a base64 string for CozoDB storage.
- */
-export function encodeEmbedding(embedding) {
-    const buffer = Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength);
-    return buffer.toString("base64");
-}
-/**
- * Decode a base64 string back to Float32Array.
- */
-export function decodeEmbedding(encoded) {
-    const buffer = Buffer.from(encoded, "base64");
-    return new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4);
-}
-/**
- * Compute cosine similarity between two embeddings.
- */
-export function cosineSimilarity(a, b) {
-    if (a.length !== b.length)
-        return 0;
-    let dot = 0;
-    let normA = 0;
-    let normB = 0;
-    for (let i = 0; i < a.length; i++) {
-        dot += a[i] * b[i];
-        normA += a[i] * a[i];
-        normB += b[i] * b[i];
-    }
-    const denom = Math.sqrt(normA) * Math.sqrt(normB);
-    return denom > 0 ? dot / denom : 0;
-}
-/**
- * Find the top-K most similar entities to a query embedding.
- */
-export function findSimilar(query, allEmbeddings, topK = 10, excludeKey) {
-    const results = [];
-    for (const [key, embedding] of allEmbeddings) {
-        if (key === excludeKey)
-            continue;
-        const sim = cosineSimilarity(query, embedding);
-        results.push({ entityKey: key, similarity: sim });
-    }
-    results.sort((a, b) => b.similarity - a.similarity);
-    return results.slice(0, topK);
-}

package/dist/intelligence/semantic/enrichment-orchestrator.js

@@ -1,48 +0,0 @@
-/**
- * Semantic Enrichment Orchestrator — runs after Phase 1 indexing.
- *
- * Pipeline: tokenize → TF-IDF → Node2Vec → git mine → combine → store.
- * Runs in background, non-blocking to MCP proxy startup.
- */
-import { createModuleLogger } from "../../utils/logger.js";
-import { tokenizeIdentifier } from "./identifier-tokenizer.js";
-import { trainEmbeddings } from "./node2vec-embeddings.js";
-import { buildAdjacencyFromEdges, generateWalks } from "./node2vec-walks.js";
-import { getPrimaryDomain } from "./path-domain-inference.js";
-import { buildCorpus, computeTfIdfVector } from "./tfidf-vectors.js";
-const log = createModuleLogger("semantic-enrichment");
-/**
- * Run the full semantic enrichment pipeline.
- */
-export async function runEnrichment(input) {
-    const start = performance.now();
-    const { entities, edges } = input;
-    if (entities.length === 0) {
-        return { embeddings: new Map(), entityCount: 0, durationMs: 0 };
-    }
-    log.info(`Starting semantic enrichment for ${entities.length} entities`);
-    const identifiers = entities.map((e) => e.name);
-    const corpus = buildCorpus(identifiers);
-    const adjacency = buildAdjacencyFromEdges(edges);
-    const walks = generateWalks(adjacency, {
-        seed: 42,
-        walksPerNode: 5,
-        walkLength: 20,
-    });
-    const structuralEmbeddings = trainEmbeddings(walks);
-    const embeddings = new Map();
-    for (const entity of entities) {
-        const lexical = computeTfIdfVector(entity.name, corpus);
-        const structural = structuralEmbeddings.get(entity.key) ?? new Float32Array(64);
-        const domain = getPrimaryDomain(entity.file_path);
-        embeddings.set(entity.key, {
-            lexical,
-            structural,
-            domain,
-            keywords: tokenizeIdentifier(entity.name),
-        });
-    }
-    const durationMs = performance.now() - start;
-    log.info(`Semantic enrichment complete: ${entities.length} entities in ${Math.round(durationMs)}ms`);
-    return { embeddings, entityCount: entities.length, durationMs };
-}

package/dist/intelligence/semantic/git-message-miner.js

@@ -1,114 +0,0 @@
-/**
- * Git Commit Message Miner — maps commit messages to entities via blame.
- *
- * Extracts semantic context from git history:
- *   - Which commits touched which entities (via file + line intersection)
- *   - Commit message keywords → entity semantic labels
- */
-import { gitQuery } from "../../utils/exec.js";
-/**
- * Get recent commits that touched a specific file.
- */
-export async function getFileCommits(filePath, cwd, maxCount = 20) {
-    const output = await gitQuery(["log", `--max-count=${maxCount}`, "--format=%H|%s|%ci", "--", filePath], cwd);
-    if (!output)
-        return [];
-    return output
-        .split("\n")
-        .filter(Boolean)
-        .map((line) => {
-        const [hash, message, date] = line.split("|");
-        return { hash: hash ?? "", message: message ?? "", date: date ?? "" };
-    })
-        .filter((c) => c.hash.length > 0);
-}
-/**
- * Extract keywords from commit messages (for semantic context).
- */
-export function extractKeywords(messages) {
-    const stopWords = new Set([
-        "a",
-        "an",
-        "the",
-        "is",
-        "are",
-        "was",
-        "were",
-        "be",
-        "been",
-        "being",
-        "have",
-        "has",
-        "had",
-        "do",
-        "does",
-        "did",
-        "will",
-        "would",
-        "could",
-        "should",
-        "may",
-        "might",
-        "shall",
-        "can",
-        "need",
-        "dare",
-        "ought",
-        "and",
-        "or",
-        "but",
-        "not",
-        "no",
-        "nor",
-        "for",
-        "to",
-        "in",
-        "on",
-        "at",
-        "by",
-        "from",
-        "with",
-        "of",
-        "it",
-        "its",
-        "this",
-        "that",
-    ]);
-    const keywords = new Map();
-    for (const msg of messages) {
-        const words = msg
-            .toLowerCase()
-            .replace(/[^a-z0-9\s]/g, " ")
-            .split(/\s+/)
-            .filter((w) => w.length > 2 && !stopWords.has(w));
-        for (const word of words) {
-            keywords.set(word, (keywords.get(word) ?? 0) + 1);
-        }
-    }
-    return [...keywords.entries()]
-        .sort((a, b) => b[1] - a[1])
-        .slice(0, 20)
-        .map(([word]) => word);
-}
-/**
- * Mine git context for a set of entities.
- */
-export async function mineEntityContext(entities, cwd) {
-    const contexts = new Map();
-    const fileCache = new Map();
-    for (const entity of entities) {
-        let commits = fileCache.get(entity.file_path);
-        if (!commits) {
-            commits = await getFileCommits(entity.file_path, cwd);
-            fileCache.set(entity.file_path, commits);
-        }
-        const messages = commits.map((c) => c.message);
-        const keywords = extractKeywords(messages);
-        contexts.set(entity.key, {
-            entityKey: entity.key,
-            commitMessages: messages.slice(0, 5),
-            keywords,
-        });
-    }
-    return contexts;
-}

package/dist/intelligence/semantic/identifier-tokenizer.js

@@ -1,51 +0,0 @@
-/**
- * Identifier Tokenizer — splits code identifiers into semantic tokens.
- *
- * Handles: camelCase, PascalCase, snake_case, kebab-case, SCREAMING_CASE,
- * acronyms (e.g., HTMLParser → ["html", "parser"]), numbers.
- *
- * Performance: <1ms per entity (pure string splitting).
- */
-export function tokenizeIdentifier(name) {
-    if (!name || name.length === 0)
-        return [];
-    const normalized = name
-        .replace(/[-_./\\]/g, " ")
-        .replace(/([a-z\d])([A-Z])/g, "$1 $2")
-        .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2");
-    const raw = normalized.split(/\s+/).filter(Boolean);
-    const tokens = [];
-    for (const part of raw) {
-        const lower = part.toLowerCase();
-        if (lower.length <= 1 && /\d/.test(lower))
-            continue;
-        if (lower.length > 0)
-            tokens.push(lower);
-    }
-    return tokens;
-}
-/**
- * Tokenize a file path into semantic tokens.
- */
-export function tokenizeFilePath(filePath) {
-    const parts = filePath.split(/[/\\]/).filter(Boolean);
-    const tokens = [];
-    for (const part of parts) {
-        const withoutExt = part.replace(/\.[^.]+$/, "");
-        tokens.push(...tokenizeIdentifier(withoutExt));
-    }
-    return tokens;
-}
-/**
- * Build a token frequency map from an array of identifiers.
- */
-export function buildTokenFrequency(identifiers) {
-    const freq = new Map();
-    for (const id of identifiers) {
-        const tokens = tokenizeIdentifier(id);
-        for (const token of tokens) {
-            freq.set(token, (freq.get(token) ?? 0) + 1);
-        }
-    }
-    return freq;
-}

package/dist/intelligence/semantic/node2vec-embeddings.js

@@ -1,71 +0,0 @@
-/**
- * Node2Vec Embedding Trainer — Skip-gram on walk sequences.
- *
- * Converts random walks into 64-dimensional structural embeddings.
- * Uses a simplified Skip-gram model: for each (center, context) pair
- * in the walk window, accumulate co-occurrence → reduce to fixed dims.
- *
- * Performance: <2s for 1K entities (10 walks × 40 steps).
- */
-const STRUCTURAL_DIM = 64;
-/**
- * Train Node2Vec embeddings from random walks.
- * Returns a map from entity key to 64-dim Float32Array.
- */
-export function trainEmbeddings(walks, windowSize = 5) {
-    const cooccurrence = new Map();
-    for (const walk of walks) {
-        for (let i = 0; i < walk.length; i++) {
-            const center = walk[i];
-            if (!cooccurrence.has(center)) {
-                cooccurrence.set(center, new Map());
-            }
-            const centerMap = cooccurrence.get(center);
-            const start = Math.max(0, i - windowSize);
-            const end = Math.min(walk.length - 1, i + windowSize);
-            for (let j = start; j <= end; j++) {
-                if (j === i)
-                    continue;
-                const context = walk[j];
-                centerMap.set(context, (centerMap.get(context) ?? 0) + 1);
-            }
-        }
-    }
-    const allNodes = [...cooccurrence.keys()];
-    const nodeIndex = new Map(allNodes.map((n, i) => [n, i]));
-    const embeddings = new Map();
-    for (const node of allNodes) {
-        const vec = new Float32Array(STRUCTURAL_DIM);
-        const neighbors = cooccurrence.get(node) ?? new Map();
-        for (const [neighbor, weight] of neighbors) {
-            const nIdx = nodeIndex.get(neighbor) ?? 0;
-            const hashIdx = (nIdx * 2654435761) % STRUCTURAL_DIM;
-            vec[Math.abs(hashIdx)] += weight;
-        }
-        normalize(vec);
-        embeddings.set(node, vec);
-    }
-    return embeddings;
-}
-/**
- * Combine lexical (TF-IDF) and structural (Node2Vec) into 128-dim embedding.
- */
-export function combineEmbeddings(lexical, structural) {
-    const combined = new Float32Array(128);
-    combined.set(lexical, 0);
-    combined.set(structural, 64);
-    return combined;
-}
-function normalize(vector) {
-    let norm = 0;
-    for (let i = 0; i < vector.length; i++) {
-        norm += vector[i] * vector[i];
-    }
-    norm = Math.sqrt(norm);
-    if (norm > 0) {
-        for (let i = 0; i < vector.length; i++) {
-            vector[i] /= norm;
-        }
-    }
-}
-export { STRUCTURAL_DIM };

package/dist/intelligence/semantic/node2vec-walks.js

@@ -1,103 +0,0 @@
-/**
- * Node2Vec Random Walk Generator — seeded walks on the entity graph.
- *
- * Produces random walk sequences that capture graph structure.
- * Uses biased random walks (Node2Vec with p/q parameters) to balance
- * between BFS-like (local) and DFS-like (exploration) behavior.
- *
- * Seeded for deterministic, reproducible walks.
- */
-const DEFAULT_CONFIG = {
-    walkLength: 40,
-    walksPerNode: 10,
-    p: 1.0,
-    q: 1.0,
-    seed: 42,
-};
-/**
- * Seeded PRNG (xorshift32). Deterministic given same seed.
- */
-function createRng(seed) {
-    let state = seed | 0;
-    if (state === 0)
-        state = 1;
-    return () => {
-        state ^= state << 13;
-        state ^= state >> 17;
-        state ^= state << 5;
-        return (state >>> 0) / 0xffffffff;
-    };
-}
-/**
- * Generate random walks from a graph.
- * Returns array of walks, where each walk is a sequence of entity keys.
- */
-export function generateWalks(adjacency, config = {}) {
-    const cfg = { ...DEFAULT_CONFIG, ...config };
-    const rng = createRng(cfg.seed);
-    const nodes = [...adjacency.keys()];
-    const walks = [];
-    for (let w = 0; w < cfg.walksPerNode; w++) {
-        for (const startNode of nodes) {
-            const walk = [startNode];
-            let current = startNode;
-            let prev = null;
-            for (let step = 1; step < cfg.walkLength; step++) {
-                const neighbors = adjacency.get(current) ?? [];
-                if (neighbors.length === 0)
-                    break;
-                if (prev === null) {
-                    const idx = Math.floor(rng() * neighbors.length);
-                    const next = neighbors[idx];
-                    walk.push(next);
-                    prev = current;
-                    current = next;
-                }
-                else {
-                    const weights = [];
-                    for (const neighbor of neighbors) {
-                        if (neighbor === prev) {
-                            weights.push(1 / cfg.p);
-                        }
-                        else if ((adjacency.get(prev) ?? []).includes(neighbor)) {
-                            weights.push(1);
-                        }
-                        else {
-                            weights.push(1 / cfg.q);
-                        }
-                    }
-                    const totalWeight = weights.reduce((a, b) => a + b, 0);
-                    let r = rng() * totalWeight;
-                    let chosen = neighbors[0];
-                    for (let i = 0; i < weights.length; i++) {
-                        r -= weights[i];
-                        if (r <= 0) {
-                            chosen = neighbors[i];
-                            break;
-                        }
-                    }
-                    walk.push(chosen);
-                    prev = current;
-                    current = chosen;
-                }
-            }
-            walks.push(walk);
-        }
-    }
-    return walks;
-}
-/**
- * Build adjacency list from edge data.
- */
-export function buildAdjacencyFromEdges(edges) {
-    const adj = new Map();
-    for (const edge of edges) {
-        if (!adj.has(edge.from_key))
-            adj.set(edge.from_key, []);
-        if (!adj.has(edge.to_key))
-            adj.set(edge.to_key, []);
-        adj.get(edge.from_key)?.push(edge.to_key);
-        adj.get(edge.to_key)?.push(edge.from_key);
-    }
-    return adj;
-}