arceus-s 1.6.4

package/dist/core/embeddings/server-mapping.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Server Mapping Configuration
+ *
+ * Reads ~/.arc/server-mapping.json to map repo names to service names.
+ * Used in embedding text to enrich metadata with microservice context.
+ */
+/**
+ * Read the server mapping file and return the serverName for a given repoName.
+ * Returns undefined if no mapping exists.
+ */
+export declare const readServerMapping: (repoName: string) => Promise<string | undefined>;
+/**
+ * Clear the cached mapping (useful for testing or after file changes)
+ */
+export declare const clearServerMappingCache: () => void;

package/dist/core/embeddings/server-mapping.js

@@ -0,0 +1,33 @@
+/**
+ * Server Mapping Configuration
+ *
+ * Reads ~/.arc/server-mapping.json to map repo names to service names.
+ * Used in embedding text to enrich metadata with microservice context.
+ */
+import fs from 'fs/promises';
+import path from 'path';
+import os from 'os';
+const MAPPING_FILE = path.join(os.homedir(), '.arc', 'server-mapping.json');
+let cachedMapping = null;
+/**
+ * Read the server mapping file and return the serverName for a given repoName.
+ * Returns undefined if no mapping exists.
+ */
+export const readServerMapping = async (repoName) => {
+    try {
+        if (!cachedMapping) {
+            const raw = await fs.readFile(MAPPING_FILE, 'utf-8');
+            cachedMapping = JSON.parse(raw);
+        }
+        return cachedMapping[repoName];
+    }
+    catch {
+        return undefined;
+    }
+};
+/**
+ * Clear the cached mapping (useful for testing or after file changes)
+ */
+export const clearServerMappingCache = () => {
+    cachedMapping = null;
+};

package/dist/core/embeddings/structural-extractor.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Structural Extractor Module
+ *
+ * Reuses ingestion pipeline's AST-based MethodExtractor / FieldExtractor
+ * to extract method and field names for embedding text generation.
+ */
+export interface StructuralNames {
+    methodNames: string[];
+    fieldNames: string[];
+}
+/**
+ * Extract method and field names from a class/struct/interface node
+ * using the ingestion pipeline's AST extractors.
+ */
+export declare const extractStructuralNames: (content: string, filePath: string) => Promise<StructuralNames>;

package/dist/core/embeddings/structural-extractor.js

@@ -0,0 +1,58 @@
+/**
+ * Structural Extractor Module
+ *
+ * Reuses ingestion pipeline's AST-based MethodExtractor / FieldExtractor
+ * to extract method and field names for embedding text generation.
+ */
+import { getProviderForFile } from '../ingestion/languages/index.js';
+import { buildTypeEnv } from '../ingestion/type-env.js';
+import { ensureAndParse, findDeclarationNode } from './ast-utils.js';
+const NOOP_SYMBOL_TABLE = {
+    lookupExactAll: () => [],
+    lookupExact: () => undefined,
+    lookupExactFull: () => undefined,
+};
+/**
+ * Extract method and field names from a class/struct/interface node
+ * using the ingestion pipeline's AST extractors.
+ */
+export const extractStructuralNames = async (content, filePath) => {
+    const provider = getProviderForFile(filePath);
+    if (!provider)
+        return { methodNames: [], fieldNames: [] };
+    const tree = await ensureAndParse(content, filePath);
+    if (!tree)
+        return { methodNames: [], fieldNames: [] };
+    // Parse node.content (a snippet) — find declaration directly, not by range
+    const classNode = findDeclarationNode(tree.rootNode);
+    if (!classNode)
+        return { methodNames: [], fieldNames: [] };
+    const language = provider.id;
+    const methodNames = extractMethodNames(classNode, provider, filePath, language);
+    const fieldNames = extractFieldNames(classNode, provider, tree, filePath, language);
+    return { methodNames, fieldNames };
+};
+function extractMethodNames(classNode, provider, filePath, language) {
+    if (!provider.methodExtractor)
+        return [];
+    const context = { filePath, language };
+    const result = provider.methodExtractor.extract(classNode, context);
+    if (!result?.methods?.length)
+        return [];
+    return result.methods.map((m) => m.name);
+}
+function extractFieldNames(classNode, provider, tree, filePath, language) {
+    if (!provider.fieldExtractor)
+        return [];
+    const typeEnv = buildTypeEnv(tree, language);
+    const context = {
+        typeEnv,
+        symbolTable: NOOP_SYMBOL_TABLE,
+        filePath,
+        language,
+    };
+    const result = provider.fieldExtractor.extract(classNode, context);
+    if (!result?.fields?.length)
+        return [];
+    return result.fields.map((f) => f.name);
+}

package/dist/core/embeddings/text-generator.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Text Generator Module
+ *
+ * Generates enriched embedding text from code nodes with metadata.
+ * Supports chunkable labels (Function/Method with AST chunking),
+ * Class-specific structural text, and short-node direct embed.
+ *
+ * Method/field names for Class nodes are extracted by the ingestion
+ * pipeline's AST extractors and passed via node.methodNames/node.fieldNames.
+ */
+import type { EmbeddableNode, EmbeddingConfig } from './types.js';
+/**
+ * Truncate description to max length at sentence/word boundary
+ */
+declare const truncateDescription: (text: string, maxLength: number) => string;
+/**
+ * Extract class/interface/struct declaration lines, skipping method bodies.
+ * - Brace-based languages: detects method signatures (lines with `(` and `{`)
+ *   and skips until depth returns to class body level.
+ * - Non-brace languages (Python/Ruby): returns empty string (patterns handle extraction).
+ */
+export declare const extractDeclarationOnly: (content: string) => string;
+/**
+ * Generate embedding text for any embeddable node
+ * Dispatches to the appropriate generator based on node label
+ */
+export declare const generateEmbeddingText: (node: EmbeddableNode, codeBody: string, config?: Partial<EmbeddingConfig>, chunkIndex?: number, prevTail?: string) => string;
+/**
+ * Export truncation helper for testing
+ */
+export { truncateDescription };

package/dist/core/embeddings/text-generator.js

@@ -0,0 +1,208 @@
+/**
+ * Text Generator Module
+ *
+ * Generates enriched embedding text from code nodes with metadata.
+ * Supports chunkable labels (Function/Method with AST chunking),
+ * Class-specific structural text, and short-node direct embed.
+ *
+ * Method/field names for Class nodes are extracted by the ingestion
+ * pipeline's AST extractors and passed via node.methodNames/node.fieldNames.
+ */
+import { CHUNKING_RULES, DEFAULT_EMBEDDING_CONFIG, STRUCTURAL_TEXT_MODE_DECLARATION, isShortLabel, } from './types.js';
+/**
+ * Truncate description to max length at sentence/word boundary
+ */
+const truncateDescription = (text, maxLength) => {
+    if (text.length <= maxLength)
+        return text;
+    const truncated = text.slice(0, maxLength);
+    // Try sentence boundary (. ! ?)
+    const sentenceEnd = Math.max(truncated.lastIndexOf('. '), truncated.lastIndexOf('! '), truncated.lastIndexOf('? '));
+    if (sentenceEnd > maxLength * 0.5) {
+        return truncated.slice(0, sentenceEnd + 1);
+    }
+    // Try word boundary
+    const lastSpace = truncated.lastIndexOf(' ');
+    if (lastSpace > maxLength * 0.5) {
+        return truncated.slice(0, lastSpace);
+    }
+    return truncated;
+};
+/**
+ * Clean code content for embedding
+ */
+const cleanContent = (content) => {
+    return content
+        .replace(/\r\n/g, '\n')
+        .replace(/\n{3,}/g, '\n\n')
+        .split('\n')
+        .map((line) => line.trimEnd())
+        .join('\n')
+        .trim();
+};
+/**
+ * Build metadata header for a node
+ */
+const buildMetadataHeader = (node, config) => {
+    const parts = [];
+    // Label + name
+    parts.push(`${node.label}: ${node.name}`);
+    // Repo name
+    if (node.repoName) {
+        parts.push(`Repo: ${node.repoName}`);
+    }
+    // Server name (optional)
+    if (node.serverName) {
+        parts.push(`Server: ${node.serverName}`);
+    }
+    // Full file path
+    parts.push(`Path: ${node.filePath}`);
+    // Export status
+    if (node.isExported !== undefined) {
+        parts.push(`Export: ${node.isExported}`);
+    }
+    // Description (truncated)
+    if (node.description) {
+        const maxLen = config.maxDescriptionLength ?? DEFAULT_EMBEDDING_CONFIG.maxDescriptionLength;
+        const truncated = truncateDescription(node.description, maxLen);
+        if (truncated) {
+            parts.push(truncated);
+        }
+    }
+    return parts.join('\n');
+};
+const generateCodeBodyText = (node, codeBody, config, prevTail) => {
+    const header = buildMetadataHeader(node, config);
+    const parts = [header];
+    if (prevTail) {
+        parts.push(`[preceding context]: ...${cleanContent(prevTail)}`);
+    }
+    parts.push('', cleanContent(codeBody));
+    return parts.join('\n');
+};
+const getCompactContainerContext = (cleanedContent, declarationOnly) => {
+    const source = declarationOnly || cleanedContent;
+    const nlIdx = source.indexOf('\n');
+    const firstLine = (nlIdx === -1 ? source : source.substring(0, nlIdx)).trim();
+    return firstLine ? `Container: ${firstLine}` : undefined;
+};
+const generateStructuralTypeText = (node, codeBody, config, chunkIndex, prevTail) => {
+    const header = buildMetadataHeader(node, config);
+    const parts = [header];
+    const isFirstChunk = chunkIndex === undefined || chunkIndex === 0;
+    const cleanedContent = cleanContent(node.content);
+    const declarationOnly = extractDeclarationOnly(cleanedContent);
+    const compactContainerContext = getCompactContainerContext(cleanedContent, declarationOnly);
+    if (compactContainerContext) {
+        parts.push(compactContainerContext);
+    }
+    if (prevTail) {
+        parts.push(`[preceding context]: ...${cleanContent(prevTail)}`);
+    }
+    if (isFirstChunk && node.methodNames?.length) {
+        parts.push(`Methods: ${node.methodNames.join(', ')}`);
+    }
+    if (isFirstChunk && node.fieldNames?.length) {
+        parts.push(`Properties: ${node.fieldNames.join(', ')}`);
+    }
+    if (isFirstChunk && declarationOnly) {
+        parts.push('', declarationOnly);
+    }
+    const cleanedChunk = cleanContent(codeBody);
+    if (cleanedChunk && cleanedChunk !== cleanedContent) {
+        parts.push('', cleanedChunk);
+    }
+    return parts.join('\n');
+};
+const DECL_START_RE = /^(?:(?:export|pub|data|abstract)\s+)*(?:type\s+\w+\s+struct|(?:class|struct|enum|interface)\s)/;
+/**
+ * Extract class/interface/struct declaration lines, skipping method bodies.
+ * - Brace-based languages: detects method signatures (lines with `(` and `{`)
+ *   and skips until depth returns to class body level.
+ * - Non-brace languages (Python/Ruby): returns empty string (patterns handle extraction).
+ */
+export const extractDeclarationOnly = (content) => {
+    const lines = content.split('\n');
+    const declLines = [];
+    let depth = 0;
+    let started = false;
+    let classDepth = 0;
+    let skipDepth = 0;
+    for (const [idx, line] of lines.entries()) {
+        const trimmed = line.trim();
+        if (!started) {
+            if (DECL_START_RE.test(trimmed)) {
+                // Non-brace language check: current line or next 3 lines must have `{`
+                const nextLines = lines.slice(idx + 1, idx + 4);
+                if (!trimmed.includes('{') && !nextLines.some((l) => l.includes('{'))) {
+                    return '';
+                }
+                started = true;
+                declLines.push(trimmed);
+                for (const ch of trimmed) {
+                    if (ch === '{')
+                        depth++;
+                    else if (ch === '}')
+                        depth--;
+                }
+                if (depth > 0)
+                    classDepth = depth;
+            }
+            continue;
+        }
+        // Always update depth (even when skipping)
+        const opens = (trimmed.match(/{/g) || []).length;
+        const closes = (trimmed.match(/}/g) || []).length;
+        const prevDepth = depth;
+        depth += opens - closes;
+        if (skipDepth > 0) {
+            if (depth <= classDepth) {
+                skipDepth = 0;
+                // Closing brace of class
+                if (depth <= 0) {
+                    declLines.push(trimmed);
+                    break;
+                }
+            }
+            continue;
+        }
+        // Detect method signature: line has both `(` and `{` and goes deeper than class body
+        const hasParens = trimmed.includes('(');
+        const hasOpenBrace = opens > 0;
+        if (hasParens && hasOpenBrace && prevDepth + opens > classDepth) {
+            if (opens === closes && trimmed.endsWith(';')) {
+                // Property with function/object initializer like `config = { timeout: 5000 };` — keep
+                declLines.push(trimmed);
+            }
+            // else: single-line or multi-line method — skip entirely
+            if (opens !== closes) {
+                skipDepth = classDepth;
+            }
+            continue;
+        }
+        declLines.push(trimmed);
+        if (depth <= 0 && declLines.length > 1)
+            break;
+    }
+    return declLines.join('\n').trim();
+};
+/**
+ * Generate embedding text for any embeddable node
+ * Dispatches to the appropriate generator based on node label
+ */
+export const generateEmbeddingText = (node, codeBody, config = {}, chunkIndex, prevTail) => {
+    if (isShortLabel(node.label)) {
+        const header = buildMetadataHeader(node, config);
+        const cleaned = cleanContent(node.content);
+        return `${header}\n\n${cleaned}`;
+    }
+    const chunkingRule = CHUNKING_RULES[node.label];
+    if (chunkingRule?.structuralTextMode === STRUCTURAL_TEXT_MODE_DECLARATION) {
+        return generateStructuralTypeText(node, codeBody, config, chunkIndex, prevTail);
+    }
+    return generateCodeBodyText(node, codeBody, config, prevTail);
+};
+/**
+ * Export truncation helper for testing
+ */
+export { truncateDescription };

package/dist/core/embeddings/types.d.ts

@@ -0,0 +1,211 @@
+/**
+ * Embedding Pipeline Types
+ *
+ * Type definitions for the embedding generation and semantic search system.
+ */
+export declare const LABEL_FUNCTION: "Function";
+export declare const LABEL_METHOD: "Method";
+export declare const LABEL_CONSTRUCTOR: "Constructor";
+export declare const LABEL_CLASS: "Class";
+export declare const LABEL_INTERFACE: "Interface";
+export declare const LABEL_STRUCT: "Struct";
+export declare const LABEL_ENUM: "Enum";
+export declare const LABEL_TRAIT: "Trait";
+export declare const LABEL_IMPL: "Impl";
+export declare const LABEL_MACRO: "Macro";
+export declare const LABEL_NAMESPACE: "Namespace";
+export declare const LABEL_TYPE_ALIAS: "TypeAlias";
+export declare const LABEL_TYPEDEF: "Typedef";
+export declare const LABEL_CONST: "Const";
+export declare const LABEL_PROPERTY: "Property";
+export declare const LABEL_RECORD: "Record";
+export declare const LABEL_UNION: "Union";
+export declare const LABEL_STATIC: "Static";
+export declare const LABEL_VARIABLE: "Variable";
+export declare const LABEL_CODE_ELEMENT: "CodeElement";
+export declare const CHUNK_MODE_AST_FUNCTION: "ast-function";
+export declare const CHUNK_MODE_AST_DECLARATION: "ast-declaration";
+export declare const CHUNK_MODE_CHARACTER: "character";
+export declare const STRUCTURAL_TEXT_MODE_NONE: "none";
+export declare const STRUCTURAL_TEXT_MODE_DECLARATION: "declaration";
+export interface ChunkingRule {
+    mode: typeof CHUNK_MODE_AST_FUNCTION | typeof CHUNK_MODE_AST_DECLARATION | typeof CHUNK_MODE_CHARACTER;
+    includePrefix: boolean;
+    includeSuffix: boolean;
+    groupFields: boolean;
+    structuralTextMode: typeof STRUCTURAL_TEXT_MODE_NONE | typeof STRUCTURAL_TEXT_MODE_DECLARATION;
+}
+/**
+ * Node labels that need chunking (have code body, potentially long)
+ */
+export declare const CHUNKABLE_LABELS: readonly ["Function", "Method", "Constructor", "Class", "Interface", "Struct", "Enum", "Trait", "Impl", "Macro", "Namespace"];
+/**
+ * Node labels that are short (no chunking needed, embed directly)
+ */
+export declare const SHORT_LABELS: readonly ["TypeAlias", "Typedef", "Const", "Property", "Record", "Union", "Static", "Variable"];
+/**
+ * All embeddable labels (union of CHUNKABLE + SHORT)
+ */
+export declare const EMBEDDABLE_LABELS: readonly ["Function", "Method", "Constructor", "Class", "Interface", "Struct", "Enum", "Trait", "Impl", "Macro", "Namespace", "TypeAlias", "Typedef", "Const", "Property", "Record", "Union", "Static", "Variable"];
+export type EmbeddableLabel = (typeof EMBEDDABLE_LABELS)[number];
+/**
+ * Check if a label should be embedded
+ */
+export declare const isEmbeddableLabel: (label: string) => label is EmbeddableLabel;
+/**
+ * Check if a label needs chunking
+ */
+export declare const isChunkableLabel: (label: string) => boolean;
+/**
+ * Check if a label is a short type (no chunking)
+ */
+export declare const isShortLabel: (label: string) => boolean;
+/**
+ * Node labels that have structural names (methods/fields) extractable via AST.
+ * Only labels that consume methodNames/fieldNames in their embedding text should
+ * be listed here — extra entries trigger wasted AST parses with no effect on output.
+ */
+export declare const STRUCTURAL_LABELS: ReadonlySet<string>;
+/**
+ * Node labels that have isExported column in their schema
+ */
+export declare const LABELS_WITH_EXPORTED: ReadonlySet<string>;
+/**
+ * Labels that need special chunking and/or structural text semantics.
+ * Any chunkable label omitted here intentionally falls back to characterChunk
+ * plus generateCodeBodyText (for example Enum/Trait/Impl/Macro/Namespace).
+ */
+type ChunkableLabel = (typeof CHUNKABLE_LABELS)[number];
+export declare const CHUNKING_RULES: Readonly<Partial<Record<ChunkableLabel, ChunkingRule>>>;
+/**
+ * Embedding pipeline phases
+ */
+export type EmbeddingPhase = 'idle' | 'loading-model' | 'embedding' | 'indexing' | 'ready' | 'error';
+/**
+ * Progress information for the embedding pipeline
+ */
+export interface EmbeddingProgress {
+    phase: EmbeddingPhase;
+    percent: number;
+    modelDownloadPercent?: number;
+    nodesProcessed?: number;
+    totalNodes?: number;
+    currentBatch?: number;
+    totalBatches?: number;
+    error?: string;
+}
+/**
+ * Configuration for the embedding pipeline
+ */
+export interface EmbeddingConfig {
+    /** Model identifier for transformers.js (local) or the HTTP endpoint model name */
+    modelId: string;
+    /** Number of nodes to embed in each batch */
+    batchSize: number;
+    /** Number of chunks passed to one local/HTTP embedding call */
+    subBatchSize: number;
+    /** Maximum ONNX Runtime CPU threads for local inference */
+    threads: number;
+    /** Embedding vector dimensions */
+    dimensions: number;
+    /** Device to use for inference: 'auto' tries GPU first (DirectML on Windows, CUDA on Linux), falls back to CPU */
+    device: 'auto' | 'dml' | 'cuda' | 'cpu' | 'wasm';
+    /** Maximum characters of code snippet to include */
+    maxSnippetLength: number;
+    /** Maximum code chunk size in characters (for chunking long code) */
+    chunkSize: number;
+    /** Overlap between chunks in characters */
+    overlap: number;
+    /** Maximum description length in characters */
+    maxDescriptionLength: number;
+}
+/**
+ * Default embedding configuration
+ * Uses snowflake-arctic-embed-xs for browser efficiency
+ * Tries WebGPU first (fast), user can choose WASM fallback if unavailable
+ */
+export declare const DEFAULT_EMBEDDING_CONFIG: EmbeddingConfig;
+/**
+ * Result from semantic search
+ */
+export interface SemanticSearchResult {
+    nodeId: string;
+    name: string;
+    label: string;
+    filePath: string;
+    distance: number;
+    startLine?: number;
+    endLine?: number;
+}
+/**
+ * Node data for embedding (minimal structure from LadybugDB query)
+ */
+export interface EmbeddableNode {
+    id: string;
+    name: string;
+    label: string;
+    filePath: string;
+    content: string;
+    startLine?: number;
+    endLine?: number;
+    isExported?: boolean;
+    description?: string;
+    parameterCount?: number;
+    returnType?: string;
+    repoName?: string;
+    serverName?: string;
+    methodNames?: string[];
+    fieldNames?: string[];
+}
+/**
+ * Cached embedding entry restored from LadybugDB before a graph rebuild
+ */
+export interface CachedEmbedding {
+    nodeId: string;
+    chunkIndex: number;
+    startLine: number;
+    endLine: number;
+    embedding: number[];
+    contentHash?: string;
+}
+/**
+ * Context info for embedding pipeline (repo/server metadata enrichment)
+ */
+export interface EmbeddingContext {
+    repoName?: string;
+    serverName?: string;
+}
+/**
+ * Model download progress from transformers.js
+ */
+export interface ModelProgress {
+    status: 'initiate' | 'download' | 'progress' | 'done' | 'ready';
+    file?: string;
+    progress?: number;
+    loaded?: number;
+    total?: number;
+}
+export interface ChunkSearchRow {
+    nodeId: string;
+    chunkIndex: number;
+    startLine: number;
+    endLine: number;
+    distance: number;
+}
+export interface BestChunkMatch {
+    chunkIndex: number;
+    startLine: number;
+    endLine: number;
+    distance: number;
+}
+/**
+ * Deduplicate vector search chunk results by nodeId,
+ * keeping the chunk with smallest distance for each node.
+ */
+export declare const dedupBestChunks: (rows: ChunkSearchRow[], limit?: number) => Map<string, BestChunkMatch>;
+/**
+ * Fetch vector-search chunks until we have enough unique nodeIds
+ * or can tell the result set is exhausted.
+ */
+export declare const collectBestChunks: (limit: number, fetchRows: (fetchLimit: number) => Promise<ChunkSearchRow[]>, maxFetch?: number) => Promise<Map<string, BestChunkMatch>>;
+export {};