gitnexus 1.6.2-rc.8 → 1.6.2

package/dist/core/embeddings/structural-extractor.js

@@ -0,0 +1,58 @@
+/**
+ * Structural Extractor Module
+ *
+ * Reuses ingestion pipeline's AST-based MethodExtractor / FieldExtractor
+ * to extract method and field names for embedding text generation.
+ */
+import { getProviderForFile } from '../ingestion/languages/index.js';
+import { buildTypeEnv } from '../ingestion/type-env.js';
+import { ensureAndParse, findDeclarationNode } from './ast-utils.js';
+const NOOP_SYMBOL_TABLE = {
+    lookupExactAll: () => [],
+    lookupExact: () => undefined,
+    lookupExactFull: () => undefined,
+};
+/**
+ * Extract method and field names from a class/struct/interface node
+ * using the ingestion pipeline's AST extractors.
+ */
+export const extractStructuralNames = async (content, filePath) => {
+    const provider = getProviderForFile(filePath);
+    if (!provider)
+        return { methodNames: [], fieldNames: [] };
+    const tree = await ensureAndParse(content, filePath);
+    if (!tree)
+        return { methodNames: [], fieldNames: [] };
+    // Parse node.content (a snippet) — find declaration directly, not by range
+    const classNode = findDeclarationNode(tree.rootNode);
+    if (!classNode)
+        return { methodNames: [], fieldNames: [] };
+    const language = provider.id;
+    const methodNames = extractMethodNames(classNode, provider, filePath, language);
+    const fieldNames = extractFieldNames(classNode, provider, tree, filePath, language);
+    return { methodNames, fieldNames };
+};
+function extractMethodNames(classNode, provider, filePath, language) {
+    if (!provider.methodExtractor)
+        return [];
+    const context = { filePath, language };
+    const result = provider.methodExtractor.extract(classNode, context);
+    if (!result?.methods?.length)
+        return [];
+    return result.methods.map((m) => m.name);
+}
+function extractFieldNames(classNode, provider, tree, filePath, language) {
+    if (!provider.fieldExtractor)
+        return [];
+    const typeEnv = buildTypeEnv(tree, language);
+    const context = {
+        typeEnv,
+        symbolTable: NOOP_SYMBOL_TABLE,
+        filePath,
+        language,
+    };
+    const result = provider.fieldExtractor.extract(classNode, context);
+    if (!result?.fields?.length)
+        return [];
+    return result.fields.map((f) => f.name);
+}

package/dist/core/embeddings/text-generator.d.ts

@@ -1,24 +1,31 @@
 /**
  * Text Generator Module
  *
- * Pure functions to generate embedding text from code nodes.
- * Combines node metadata with code snippets for semantic matching.
+ * Generates enriched embedding text from code nodes with metadata.
+ * Supports chunkable labels (Function/Method with AST chunking),
+ * Class-specific structural text, and short-node direct embed.
+ *
+ * Method/field names for Class nodes are extracted by the ingestion
+ * pipeline's AST extractors and passed via node.methodNames/node.fieldNames.
  */
 import type { EmbeddableNode, EmbeddingConfig } from './types.js';
+/**
+ * Truncate description to max length at sentence/word boundary
+ */
+declare const truncateDescription: (text: string, maxLength: number) => string;
+/**
+ * Extract class/interface/struct declaration lines, skipping method bodies.
+ * - Brace-based languages: detects method signatures (lines with `(` and `{`)
+ *   and skips until depth returns to class body level.
+ * - Non-brace languages (Python/Ruby): returns empty string (patterns handle extraction).
+ */
+export declare const extractDeclarationOnly: (content: string) => string;
 /**
  * Generate embedding text for any embeddable node
  * Dispatches to the appropriate generator based on node label
- *
- * @param node - The node to generate text for
- * @param config - Optional configuration for max snippet length
- * @returns Text suitable for embedding
  */
-export declare const generateEmbeddingText: (node: EmbeddableNode, config?: Partial<EmbeddingConfig>) => string;
+export declare const generateEmbeddingText: (node: EmbeddableNode, codeBody: string, config?: Partial<EmbeddingConfig>) => string;
 /**
- * Generate embedding texts for a batch of nodes
- *
- * @param nodes - Array of nodes to generate text for
- * @param config - Optional configuration
- * @returns Array of texts in the same order as input nodes
+ * Export truncation helper for testing
  */
-export declare const generateBatchEmbeddingTexts: (nodes: EmbeddableNode[], config?: Partial<EmbeddingConfig>) => string[];
+export { truncateDescription };

package/dist/core/embeddings/text-generator.js

@@ -1,167 +1,199 @@
 /**
  * Text Generator Module
  *
- * Pure functions to generate embedding text from code nodes.
- * Combines node metadata with code snippets for semantic matching.
- */
-import { DEFAULT_EMBEDDING_CONFIG } from './types.js';
-/**
- * Extract the filename from a file path
- */
-const getFileName = (filePath) => {
-    const parts = filePath.split('/');
-    return parts[parts.length - 1] || filePath;
-};
-/**
- * Extract the directory path from a file path
+ * Generates enriched embedding text from code nodes with metadata.
+ * Supports chunkable labels (Function/Method with AST chunking),
+ * Class-specific structural text, and short-node direct embed.
+ *
+ * Method/field names for Class nodes are extracted by the ingestion
+ * pipeline's AST extractors and passed via node.methodNames/node.fieldNames.
  */
-const getDirectory = (filePath) => {
-    const parts = filePath.split('/');
-    parts.pop();
-    return parts.join('/') || '';
-};
+import { DEFAULT_EMBEDDING_CONFIG, isShortLabel } from './types.js';
 /**
- * Truncate content to max length, preserving word boundaries
+ * Truncate description to max length at sentence/word boundary
  */
-const truncateContent = (content, maxLength) => {
-    if (content.length <= maxLength) {
-        return content;
+const truncateDescription = (text, maxLength) => {
+    if (text.length <= maxLength)
+        return text;
+    const truncated = text.slice(0, maxLength);
+    // Try sentence boundary (. ! ?)
+    const sentenceEnd = Math.max(truncated.lastIndexOf('. '), truncated.lastIndexOf('! '), truncated.lastIndexOf('? '));
+    if (sentenceEnd > maxLength * 0.5) {
+        return truncated.slice(0, sentenceEnd + 1);
     }
-    // Find last space before maxLength to avoid cutting words
-    const truncated = content.slice(0, maxLength);
+    // Try word boundary
     const lastSpace = truncated.lastIndexOf(' ');
-    if (lastSpace > maxLength * 0.8) {
-        return truncated.slice(0, lastSpace) + '...';
+    if (lastSpace > maxLength * 0.5) {
+        return truncated.slice(0, lastSpace);
     }
-    return truncated + '...';
+    return truncated;
 };
 /**
  * Clean code content for embedding
- * Removes excessive whitespace while preserving structure
  */
 const cleanContent = (content) => {
-    return (content
-        // Normalize line endings
+    return content
         .replace(/\r\n/g, '\n')
-        // Remove excessive blank lines (more than 2)
         .replace(/\n{3,}/g, '\n\n')
-        // Trim each line
         .split('\n')
         .map((line) => line.trimEnd())
         .join('\n')
-        .trim());
+        .trim();
 };
 /**
- * Generate embedding text for a Function node
+ * Build metadata header for a node
  */
-const generateFunctionText = (node, maxSnippetLength) => {
-    const parts = [`Function: ${node.name}`, `File: ${getFileName(node.filePath)}`];
-    const dir = getDirectory(node.filePath);
-    if (dir) {
-        parts.push(`Directory: ${dir}`);
+const buildMetadataHeader = (node, config) => {
+    const parts = [];
+    // Label + name
+    parts.push(`${node.label}: ${node.name}`);
+    // Repo name
+    if (node.repoName) {
+        parts.push(`Repo: ${node.repoName}`);
     }
-    if (node.content) {
-        const cleanedContent = cleanContent(node.content);
-        const snippet = truncateContent(cleanedContent, maxSnippetLength);
-        parts.push('', snippet);
+    // Server name (optional)
+    if (node.serverName) {
+        parts.push(`Server: ${node.serverName}`);
     }
-    return parts.join('\n');
-};
-/**
- * Generate embedding text for a Class node
- */
-const generateClassText = (node, maxSnippetLength) => {
-    const parts = [`Class: ${node.name}`, `File: ${getFileName(node.filePath)}`];
-    const dir = getDirectory(node.filePath);
-    if (dir) {
-        parts.push(`Directory: ${dir}`);
+    // Full file path
+    parts.push(`Path: ${node.filePath}`);
+    // Export status
+    if (node.isExported !== undefined) {
+        parts.push(`Export: ${node.isExported}`);
     }
-    if (node.content) {
-        const cleanedContent = cleanContent(node.content);
-        const snippet = truncateContent(cleanedContent, maxSnippetLength);
-        parts.push('', snippet);
+    // Description (truncated)
+    if (node.description) {
+        const maxLen = config.maxDescriptionLength ?? DEFAULT_EMBEDDING_CONFIG.maxDescriptionLength;
+        const truncated = truncateDescription(node.description, maxLen);
+        if (truncated) {
+            parts.push(truncated);
+        }
     }
     return parts.join('\n');
 };
+const generateCodeBodyText = (node, codeBody, config) => {
+    const header = buildMetadataHeader(node, config);
+    const cleaned = cleanContent(codeBody);
+    return `${header}\n\n${cleaned}`;
+};
 /**
- * Generate embedding text for a Method node
+ * Generate embedding text for Class nodes
+ * Signature + properties + method name list only (no method bodies)
+ * Method/field names come from AST extractors via node.methodNames/node.fieldNames.
  */
-const generateMethodText = (node, maxSnippetLength) => {
-    const parts = [`Method: ${node.name}`, `File: ${getFileName(node.filePath)}`];
-    const dir = getDirectory(node.filePath);
-    if (dir) {
-        parts.push(`Directory: ${dir}`);
+const generateClassText = (node, codeBody, config) => {
+    return generateStructuralTypeText(node, codeBody, config);
+};
+const generateStructuralTypeText = (node, codeBody, config) => {
+    const header = buildMetadataHeader(node, config);
+    const parts = [header];
+    if (node.methodNames?.length) {
+        parts.push(`Methods: ${node.methodNames.join(', ')}`);
     }
-    if (node.content) {
-        const cleanedContent = cleanContent(node.content);
-        const snippet = truncateContent(cleanedContent, maxSnippetLength);
-        parts.push('', snippet);
+    if (node.fieldNames?.length) {
+        parts.push(`Properties: ${node.fieldNames.join(', ')}`);
     }
-    return parts.join('\n');
-};
-/**
- * Generate embedding text for an Interface node
- */
-const generateInterfaceText = (node, maxSnippetLength) => {
-    const parts = [`Interface: ${node.name}`, `File: ${getFileName(node.filePath)}`];
-    const dir = getDirectory(node.filePath);
-    if (dir) {
-        parts.push(`Directory: ${dir}`);
+    const declarationOnly = extractDeclarationOnly(cleanContent(node.content));
+    if (declarationOnly) {
+        parts.push('', declarationOnly);
     }
-    if (node.content) {
-        const cleanedContent = cleanContent(node.content);
-        const snippet = truncateContent(cleanedContent, maxSnippetLength);
-        parts.push('', snippet);
+    const cleanedChunk = cleanContent(codeBody);
+    if (cleanedChunk && cleanedChunk !== cleanContent(node.content)) {
+        parts.push('', cleanedChunk);
     }
     return parts.join('\n');
 };
+const DECL_START_RE = /^(?:(?:export|pub|data|abstract)\s+)*(?:type\s+\w+\s+struct|(?:class|struct|enum|interface)\s)/;
 /**
- * Generate embedding text for a File node
- * Uses file name and first N characters of content
+ * Extract class/interface/struct declaration lines, skipping method bodies.
+ * - Brace-based languages: detects method signatures (lines with `(` and `{`)
+ *   and skips until depth returns to class body level.
+ * - Non-brace languages (Python/Ruby): returns empty string (patterns handle extraction).
  */
-const generateFileText = (node, maxSnippetLength) => {
-    const parts = [`File: ${node.name}`, `Path: ${node.filePath}`];
-    if (node.content) {
-        const cleanedContent = cleanContent(node.content);
-        // For files, use a shorter snippet since they can be very long
-        const snippet = truncateContent(cleanedContent, Math.min(maxSnippetLength, 300));
-        parts.push('', snippet);
+export const extractDeclarationOnly = (content) => {
+    const lines = content.split('\n');
+    const declLines = [];
+    let depth = 0;
+    let started = false;
+    let classDepth = 0;
+    let skipDepth = 0;
+    for (const [idx, line] of lines.entries()) {
+        const trimmed = line.trim();
+        if (!started) {
+            if (DECL_START_RE.test(trimmed)) {
+                // Non-brace language check: current line or next 3 lines must have `{`
+                const nextLines = lines.slice(idx + 1, idx + 4);
+                if (!trimmed.includes('{') && !nextLines.some((l) => l.includes('{'))) {
+                    return '';
+                }
+                started = true;
+                declLines.push(trimmed);
+                for (const ch of trimmed) {
+                    if (ch === '{')
+                        depth++;
+                    else if (ch === '}')
+                        depth--;
+                }
+                if (depth > 0)
+                    classDepth = depth;
+            }
+            continue;
+        }
+        // Always update depth (even when skipping)
+        const opens = (trimmed.match(/{/g) || []).length;
+        const closes = (trimmed.match(/}/g) || []).length;
+        const prevDepth = depth;
+        depth += opens - closes;
+        if (skipDepth > 0) {
+            if (depth <= classDepth) {
+                skipDepth = 0;
+                // Closing brace of class
+                if (depth <= 0) {
+                    declLines.push(trimmed);
+                    break;
+                }
+            }
+            continue;
+        }
+        // Detect method signature: line has both `(` and `{` and goes deeper than class body
+        const hasParens = trimmed.includes('(');
+        const hasOpenBrace = opens > 0;
+        if (hasParens && hasOpenBrace && prevDepth + opens > classDepth) {
+            if (opens === closes && trimmed.endsWith(';')) {
+                // Property with function/object initializer like `config = { timeout: 5000 };` — keep
+                declLines.push(trimmed);
+            }
+            // else: single-line or multi-line method — skip entirely
+            if (opens !== closes) {
+                skipDepth = classDepth;
+            }
+            continue;
+        }
+        declLines.push(trimmed);
+        if (depth <= 0 && declLines.length > 1)
+            break;
     }
-    return parts.join('\n');
+    return declLines.join('\n').trim();
 };
 /**
  * Generate embedding text for any embeddable node
  * Dispatches to the appropriate generator based on node label
- *
- * @param node - The node to generate text for
- * @param config - Optional configuration for max snippet length
- * @returns Text suitable for embedding
  */
-export const generateEmbeddingText = (node, config = {}) => {
-    const maxSnippetLength = config.maxSnippetLength ?? DEFAULT_EMBEDDING_CONFIG.maxSnippetLength;
-    switch (node.label) {
-        case 'Function':
-            return generateFunctionText(node, maxSnippetLength);
-        case 'Class':
-            return generateClassText(node, maxSnippetLength);
-        case 'Method':
-            return generateMethodText(node, maxSnippetLength);
-        case 'Interface':
-            return generateInterfaceText(node, maxSnippetLength);
-        case 'File':
-            return generateFileText(node, maxSnippetLength);
-        default:
-            // Fallback for any other embeddable type
-            return `${node.label}: ${node.name}\nPath: ${node.filePath}`;
+export const generateEmbeddingText = (node, codeBody, config = {}) => {
+    if (isShortLabel(node.label)) {
+        const header = buildMetadataHeader(node, config);
+        const cleaned = cleanContent(node.content);
+        return `${header}\n\n${cleaned}`;
+    }
+    if (node.label === 'Class') {
+        return generateClassText(node, codeBody, config);
+    }
+    if (node.label === 'Interface') {
+        return generateStructuralTypeText(node, codeBody, config);
     }
+    return generateCodeBodyText(node, codeBody, config);
 };
 /**
- * Generate embedding texts for a batch of nodes
- *
- * @param nodes - Array of nodes to generate text for
- * @param config - Optional configuration
- * @returns Array of texts in the same order as input nodes
+ * Export truncation helper for testing
  */
-export const generateBatchEmbeddingTexts = (nodes, config = {}) => {
-    return nodes.map((node) => generateEmbeddingText(node, config));
-};
+export { truncateDescription };

package/dist/core/embeddings/types.d.ts

@@ -4,15 +4,38 @@
  * Type definitions for the embedding generation and semantic search system.
  */
 /**
- * Node labels that should be embedded for semantic search
- * These are code elements that benefit from semantic matching
+ * Node labels that need chunking (have code body, potentially long)
  */
-export declare const EMBEDDABLE_LABELS: readonly ["Function", "Class", "Method", "Interface", "File"];
+export declare const CHUNKABLE_LABELS: readonly ["Function", "Method", "Constructor", "Class", "Interface", "Struct", "Enum", "Trait", "Impl", "Macro", "Namespace"];
+/**
+ * Node labels that are short (no chunking needed, embed directly)
+ */
+export declare const SHORT_LABELS: readonly ["TypeAlias", "Typedef", "Const", "Property", "Record", "Union", "Static", "Variable"];
+/**
+ * All embeddable labels (union of CHUNKABLE + SHORT)
+ */
+export declare const EMBEDDABLE_LABELS: readonly ["Function", "Method", "Constructor", "Class", "Interface", "Struct", "Enum", "Trait", "Impl", "Macro", "Namespace", "TypeAlias", "Typedef", "Const", "Property", "Record", "Union", "Static", "Variable"];
 export type EmbeddableLabel = (typeof EMBEDDABLE_LABELS)[number];
 /**
  * Check if a label should be embedded
  */
 export declare const isEmbeddableLabel: (label: string) => label is EmbeddableLabel;
+/**
+ * Check if a label needs chunking
+ */
+export declare const isChunkableLabel: (label: string) => boolean;
+/**
+ * Check if a label is a short type (no chunking)
+ */
+export declare const isShortLabel: (label: string) => boolean;
+/**
+ * Node labels that have structural names (methods/fields) extractable via AST
+ */
+export declare const STRUCTURAL_LABELS: ReadonlySet<string>;
+/**
+ * Node labels that have isExported column in their schema
+ */
+export declare const LABELS_WITH_EXPORTED: ReadonlySet<string>;
 /**
  * Embedding pipeline phases
  */
@@ -44,6 +67,12 @@ export interface EmbeddingConfig {
     device: 'auto' | 'dml' | 'cuda' | 'cpu' | 'wasm';
     /** Maximum characters of code snippet to include */
     maxSnippetLength: number;
+    /** Maximum code chunk size in characters (for chunking long code) */
+    chunkSize: number;
+    /** Overlap between chunks in characters */
+    overlap: number;
+    /** Maximum description length in characters */
+    maxDescriptionLength: number;
 }
 /**
  * Default embedding configuration
@@ -74,6 +103,32 @@ export interface EmbeddableNode {
     content: string;
     startLine?: number;
     endLine?: number;
+    isExported?: boolean;
+    description?: string;
+    parameterCount?: number;
+    returnType?: string;
+    repoName?: string;
+    serverName?: string;
+    methodNames?: string[];
+    fieldNames?: string[];
+}
+/**
+ * Cached embedding entry restored from LadybugDB before a graph rebuild
+ */
+export interface CachedEmbedding {
+    nodeId: string;
+    chunkIndex: number;
+    startLine: number;
+    endLine: number;
+    embedding: number[];
+    contentHash?: string;
+}
+/**
+ * Context info for embedding pipeline (repo/server metadata enrichment)
+ */
+export interface EmbeddingContext {
+    repoName?: string;
+    serverName?: string;
 }
 /**
  * Model download progress from transformers.js
@@ -85,3 +140,26 @@ export interface ModelProgress {
     loaded?: number;
     total?: number;
 }
+export interface ChunkSearchRow {
+    nodeId: string;
+    chunkIndex: number;
+    startLine: number;
+    endLine: number;
+    distance: number;
+}
+export interface BestChunkMatch {
+    chunkIndex: number;
+    startLine: number;
+    endLine: number;
+    distance: number;
+}
+/**
+ * Deduplicate vector search chunk results by nodeId,
+ * keeping the chunk with smallest distance for each node.
+ */
+export declare const dedupBestChunks: (rows: ChunkSearchRow[], limit?: number) => Map<string, BestChunkMatch>;
+/**
+ * Fetch vector-search chunks until we have enough unique nodeIds
+ * or can tell the result set is exhausted.
+ */
+export declare const collectBestChunks: (limit: number, fetchRows: (fetchLimit: number) => Promise<ChunkSearchRow[]>, maxFetch?: number) => Promise<Map<string, BestChunkMatch>>;

package/dist/core/embeddings/types.js

@@ -4,14 +4,69 @@
  * Type definitions for the embedding generation and semantic search system.
  */
 /**
- * Node labels that should be embedded for semantic search
- * These are code elements that benefit from semantic matching
+ * Node labels that need chunking (have code body, potentially long)
  */
-export const EMBEDDABLE_LABELS = ['Function', 'Class', 'Method', 'Interface', 'File'];
+export const CHUNKABLE_LABELS = [
+    'Function',
+    'Method',
+    'Constructor',
+    'Class',
+    'Interface',
+    'Struct',
+    'Enum',
+    'Trait',
+    'Impl',
+    'Macro',
+    'Namespace',
+];
+/**
+ * Node labels that are short (no chunking needed, embed directly)
+ */
+export const SHORT_LABELS = [
+    'TypeAlias',
+    'Typedef',
+    'Const',
+    'Property',
+    'Record',
+    'Union',
+    'Static',
+    'Variable',
+];
+/**
+ * All embeddable labels (union of CHUNKABLE + SHORT)
+ */
+export const EMBEDDABLE_LABELS = [...CHUNKABLE_LABELS, ...SHORT_LABELS];
 /**
  * Check if a label should be embedded
  */
 export const isEmbeddableLabel = (label) => EMBEDDABLE_LABELS.includes(label);
+/**
+ * Check if a label needs chunking
+ */
+export const isChunkableLabel = (label) => CHUNKABLE_LABELS.includes(label);
+/**
+ * Check if a label is a short type (no chunking)
+ */
+export const isShortLabel = (label) => SHORT_LABELS.includes(label);
+/**
+ * Node labels that have structural names (methods/fields) extractable via AST
+ */
+export const STRUCTURAL_LABELS = new Set([
+    'Class',
+    'Struct',
+    'Interface',
+    'Enum',
+]);
+/**
+ * Node labels that have isExported column in their schema
+ */
+export const LABELS_WITH_EXPORTED = new Set([
+    'Function',
+    'Class',
+    'Interface',
+    'Method',
+    'CodeElement',
+]);
 /**
  * Default embedding configuration
  * Uses snowflake-arctic-embed-xs for browser efficiency
@@ -23,4 +78,51 @@ export const DEFAULT_EMBEDDING_CONFIG = {
     dimensions: 384,
     device: 'auto',
     maxSnippetLength: 500,
+    chunkSize: 1200,
+    overlap: 120,
+    maxDescriptionLength: 150,
+};
+/**
+ * Deduplicate vector search chunk results by nodeId,
+ * keeping the chunk with smallest distance for each node.
+ */
+export const dedupBestChunks = (rows, limit) => {
+    const best = new Map();
+    for (const row of rows) {
+        const existing = best.get(row.nodeId);
+        if (!existing || row.distance < existing.distance) {
+            best.set(row.nodeId, {
+                chunkIndex: row.chunkIndex,
+                startLine: row.startLine,
+                endLine: row.endLine,
+                distance: row.distance,
+            });
+        }
+        if (limit !== undefined && best.size >= limit)
+            break;
+    }
+    return best;
+};
+const DEFAULT_FETCH_MULTIPLIER = 4;
+const DEFAULT_FETCH_BUFFER = 8;
+const DEFAULT_MAX_FETCH = 200;
+/**
+ * Fetch vector-search chunks until we have enough unique nodeIds
+ * or can tell the result set is exhausted.
+ */
+export const collectBestChunks = async (limit, fetchRows, maxFetch = DEFAULT_MAX_FETCH) => {
+    if (limit <= 0)
+        return new Map();
+    let fetchLimit = Math.max(limit * DEFAULT_FETCH_MULTIPLIER, limit + DEFAULT_FETCH_BUFFER);
+    let previousFetchLimit = 0;
+    while (fetchLimit > previousFetchLimit) {
+        const rows = await fetchRows(fetchLimit);
+        const bestChunks = dedupBestChunks(rows, limit);
+        if (bestChunks.size >= limit || rows.length < fetchLimit) {
+            return bestChunks;
+        }
+        previousFetchLimit = fetchLimit;
+        fetchLimit = fetchLimit >= maxFetch ? fetchLimit * 2 : Math.min(maxFetch, fetchLimit * 2);
+    }
+    return new Map();
 };