codebaxing 0.1.0

package/dist/indexing/embedding-service.js

@@ -0,0 +1,271 @@
+/**
+ * Embedding service using @huggingface/transformers (Transformers.js).
+ *
+ * Runs ONNX models locally in Node.js — no external server needed.
+ * Default model: Xenova/all-MiniLM-L6-v2 (384 dims, fast, well-supported)
+ *
+ * Device Support:
+ *   Set CODEBAXING_DEVICE environment variable:
+ *   - 'cpu' (default): CPU inference, works everywhere
+ *   - 'webgpu': WebGPU backend (experimental, uses Metal on macOS)
+ *   - 'cuda': NVIDIA GPU (Linux/Windows only, requires CUDA drivers)
+ *   - 'auto': Auto-detect best available device
+ *
+ * Note: macOS does not support CUDA. Use 'webgpu' for GPU acceleration on Mac.
+ */
+import { EmbeddingError } from '../core/exceptions.js';
+const VALID_DEVICES = ['cpu', 'cuda', 'webgpu', 'auto'];
+/**
+ * Get the configured device from environment variable.
+ * Defaults to 'cpu' for maximum compatibility.
+ */
+export function getConfiguredDevice() {
+    const envDevice = process.env.CODEBAXING_DEVICE?.toLowerCase();
+    if (envDevice && VALID_DEVICES.includes(envDevice)) {
+        return envDevice;
+    }
+    return 'cpu';
+}
+// Lazy import transformers (heavy dependency)
+// Using any types for transformers.js as its types are complex and version-dependent
+/* eslint-disable @typescript-eslint/no-explicit-any */
+let pipeline;
+let env;
+let transformersLoaded = false;
+/* eslint-enable @typescript-eslint/no-explicit-any */
+async function importTransformers() {
+    if (transformersLoaded)
+        return;
+    const transformers = await import('@huggingface/transformers');
+    pipeline = transformers.pipeline;
+    env = transformers.env;
+    // Disable remote model downloads warning
+    env.allowLocalModels = true;
+    env.allowRemoteModels = true;
+    transformersLoaded = true;
+}
+export const EMBEDDING_MODELS = {
+    'all-MiniLM-L6-v2': {
+        modelId: 'Xenova/all-MiniLM-L6-v2',
+        dimensions: 384,
+        maxSeqLength: 256,
+        queryPrefix: '',
+        documentPrefix: '',
+    },
+    'coderankembed': {
+        modelId: 'nomic-ai/CodeRankEmbed',
+        dimensions: 768,
+        maxSeqLength: 8192,
+        queryPrefix: 'Represent this query for searching relevant code: ',
+        documentPrefix: '',
+    },
+};
+export const DEFAULT_MODEL = 'all-MiniLM-L6-v2';
+// ─── EmbeddingService ────────────────────────────────────────────────────────
+export class EmbeddingService {
+    modelName;
+    config;
+    device;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    extractor = null;
+    loading = null;
+    showProgress;
+    // LRU cache
+    cache = new Map();
+    cacheMaxSize = 5000;
+    // Stats
+    stats = {
+        totalEmbeddings: 0,
+        totalBatches: 0,
+        totalTime: 0,
+        cacheHits: 0,
+        cacheMisses: 0,
+    };
+    constructor(modelName = DEFAULT_MODEL, options = {}) {
+        this.modelName = modelName;
+        this.showProgress = options.showProgress ?? false;
+        this.device = options.device ?? getConfiguredDevice();
+        if (modelName in EMBEDDING_MODELS) {
+            this.config = { ...EMBEDDING_MODELS[modelName] };
+        }
+        else {
+            // Assume it's a HuggingFace model ID
+            this.config = {
+                modelId: modelName,
+                dimensions: 384,
+                maxSeqLength: 512,
+                queryPrefix: '',
+                documentPrefix: '',
+            };
+        }
+    }
+    /** Get the device being used for inference */
+    getDevice() {
+        return this.device;
+    }
+    get dimensions() {
+        return this.config.dimensions;
+    }
+    async loadModel() {
+        if (this.extractor)
+            return;
+        if (this.loading)
+            return this.loading;
+        this.loading = (async () => {
+            await importTransformers();
+            const deviceLabel = this.device === 'cpu' ? 'CPU' : this.device.toUpperCase();
+            if (this.showProgress) {
+                console.log(`Loading embedding model: ${this.config.modelId} (device: ${deviceLabel})`);
+            }
+            // Build pipeline options
+            const pipelineOptions = {
+                quantized: true,
+            };
+            // Configure device (Transformers.js uses 'device' option)
+            // Note: 'auto' lets Transformers.js pick the best available
+            if (this.device !== 'cpu') {
+                pipelineOptions.device = this.device;
+            }
+            try {
+                this.extractor = await pipeline('feature-extraction', this.config.modelId, pipelineOptions);
+            }
+            catch (e) {
+                const errorMsg = e.message;
+                // If GPU failed, try falling back to CPU
+                if (this.device !== 'cpu' && (errorMsg.includes('GPU') || errorMsg.includes('CUDA') || errorMsg.includes('WebGPU'))) {
+                    console.warn(`Failed to use ${deviceLabel}: ${errorMsg}. Falling back to CPU.`);
+                    this.device = 'cpu';
+                    this.extractor = await pipeline('feature-extraction', this.config.modelId, {
+                        quantized: true,
+                    });
+                }
+                else if (this.modelName !== DEFAULT_MODEL) {
+                    // Fall back to default model if custom model fails
+                    console.warn(`Failed to load model ${this.config.modelId}: ${errorMsg}. ` +
+                        `Falling back to ${DEFAULT_MODEL}`);
+                    this.config = { ...EMBEDDING_MODELS[DEFAULT_MODEL] };
+                    this.extractor = await pipeline('feature-extraction', this.config.modelId, {
+                        quantized: true,
+                    });
+                }
+                else {
+                    throw new EmbeddingError(`Failed to load embedding model: ${errorMsg}`);
+                }
+            }
+            if (this.showProgress) {
+                const actualDevice = this.device === 'cpu' ? 'CPU' : this.device.toUpperCase();
+                console.log(`Model loaded: ${this.config.modelId} (${this.config.dimensions} dims, ${actualDevice})`);
+            }
+            this.loading = null;
+        })();
+        return this.loading;
+    }
+    async embed(text, isQuery = false) {
+        // Check cache
+        const cacheKey = `${isQuery ? 'q:' : 'd:'}${text}`;
+        const cached = this.cache.get(cacheKey);
+        if (cached) {
+            this.stats.cacheHits++;
+            return cached;
+        }
+        await this.loadModel();
+        const prefix = isQuery ? this.config.queryPrefix : this.config.documentPrefix;
+        const inputText = prefix ? prefix + text : text;
+        const output = await this.extractor(inputText, {
+            pooling: 'mean',
+            normalize: true,
+        });
+        const embedding = Array.from(output.data);
+        // Update cache
+        this.stats.cacheMisses++;
+        this.stats.totalEmbeddings++;
+        if (this.cache.size >= this.cacheMaxSize) {
+            // Remove oldest entry
+            const firstKey = this.cache.keys().next().value;
+            this.cache.delete(firstKey);
+        }
+        this.cache.set(cacheKey, embedding);
+        return embedding;
+    }
+    async embedBatch(texts, isQuery = false) {
+        if (texts.length === 0)
+            return [];
+        await this.loadModel();
+        const startTime = performance.now();
+        const prefix = isQuery ? this.config.queryPrefix : this.config.documentPrefix;
+        const inputTexts = prefix ? texts.map(t => prefix + t) : texts;
+        // Process in sub-batches to manage memory
+        const batchSize = 32;
+        const allEmbeddings = [];
+        for (let i = 0; i < inputTexts.length; i += batchSize) {
+            const batch = inputTexts.slice(i, i + batchSize);
+            // Process each text individually (Transformers.js handles batching internally)
+            for (const text of batch) {
+                try {
+                    const output = await this.extractor(text, {
+                        pooling: 'mean',
+                        normalize: true,
+                    });
+                    allEmbeddings.push(Array.from(output.data));
+                }
+                catch (e) {
+                    console.warn(`Embedding error for text: ${e.message}. Skipping.`);
+                    // Push zero vector as fallback
+                    allEmbeddings.push(new Array(this.config.dimensions).fill(0));
+                }
+            }
+        }
+        const elapsed = (performance.now() - startTime) / 1000;
+        this.stats.totalEmbeddings += texts.length;
+        this.stats.totalBatches++;
+        this.stats.totalTime += elapsed;
+        return allEmbeddings;
+    }
+    getStats() {
+        return {
+            ...this.stats,
+            device: this.device,
+            embeddingsPerSecond: this.stats.totalTime > 0
+                ? this.stats.totalEmbeddings / this.stats.totalTime
+                : 0,
+        };
+    }
+    clearCache() {
+        this.cache.clear();
+        this.stats.cacheHits = 0;
+        this.stats.cacheMisses = 0;
+    }
+    async unload() {
+        if (this.extractor) {
+            this.extractor = null;
+            this.cache.clear();
+        }
+    }
+}
+// ─── Singleton Management ────────────────────────────────────────────────────
+const embeddingServices = new Map();
+export function getEmbeddingService(modelName = DEFAULT_MODEL, options = {}) {
+    // Include device in cache key to support different devices per model
+    const device = options.device ?? getConfiguredDevice();
+    const cacheKey = `${modelName}:${device}`;
+    if (!embeddingServices.has(cacheKey)) {
+        embeddingServices.set(cacheKey, new EmbeddingService(modelName, { ...options, device }));
+    }
+    return embeddingServices.get(cacheKey);
+}
+export async function resetEmbeddingService(modelName) {
+    if (modelName) {
+        const service = embeddingServices.get(modelName);
+        if (service) {
+            await service.unload();
+            embeddingServices.delete(modelName);
+        }
+    }
+    else {
+        for (const service of embeddingServices.values()) {
+            await service.unload();
+        }
+        embeddingServices.clear();
+    }
+}
+//# sourceMappingURL=embedding-service.js.map

package/dist/indexing/embedding-service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedding-service.js","sourceRoot":"","sources":["../../src/indexing/embedding-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAMvD,MAAM,aAAa,GAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;AAEtE;;;GAGG;AACH,MAAM,UAAU,mBAAmB;IACjC,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,WAAW,EAAE,CAAC;IAC/D,IAAI,SAAS,IAAI,aAAa,CAAC,QAAQ,CAAC,SAAuB,CAAC,EAAE,CAAC;QACjE,OAAO,SAAuB,CAAC;IACjC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,8CAA8C;AAC9C,qFAAqF;AACrF,uDAAuD;AACvD,IAAI,QAAa,CAAC;AAClB,IAAI,GAAQ,CAAC;AACb,IAAI,kBAAkB,GAAG,KAAK,CAAC;AAC/B,sDAAsD;AAEtD,KAAK,UAAU,kBAAkB;IAC/B,IAAI,kBAAkB;QAAE,OAAO;IAC/B,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;IAC/D,QAAQ,GAAG,YAAY,CAAC,QAAQ,CAAC;IACjC,GAAG,GAAG,YAAY,CAAC,GAAG,CAAC;IAEvB,yCAAyC;IACzC,GAAG,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAC5B,GAAG,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAC7B,kBAAkB,GAAG,IAAI,CAAC;AAC5B,CAAC;AAYD,MAAM,CAAC,MAAM,gBAAgB,GAAyC;IACpE,kBAAkB,EAAE;QAClB,OAAO,EAAE,yBAAyB;QAClC,UAAU,EAAE,GAAG;QACf,YAAY,EAAE,GAAG;QACjB,WAAW,EAAE,EAAE;QACf,cAAc,EAAE,EAAE;KACnB;IACD,eAAe,EAAE;QACf,OAAO,EAAE,wBAAwB;QACjC,UAAU,EAAE,GAAG;QACf,YAAY,EAAE,IAAI;QAClB,WAAW,EAAE,oDAAoD;QACjE,cAAc,EAAE,EAAE;KACnB;CACF,CAAC;AAEF,MAAM,CAAC,MAAM,aAAa,GAAG,kBAAkB,CAAC;AAEhD,gFAAgF;AAEhF,MAAM,OAAO,gBAAgB;IACnB,SAAS,CAAS;IAClB,MAAM,CAAuB;IAC7B,MAAM,CAAa;IAC3B,8DAA8D;IACtD,SAAS,GAAQ,IAAI,CAAC;IACtB,OAAO,GAAyB,IAAI,CAAC;IACrC,YAAY,CAAU;IAE9B,YAAY;IACJ,KAAK,GAA0B,IAAI,GAAG,EAAE,CAAC;IACzC,YAAY,GAAG,IAAI,CAAC;IAE5B,QAAQ;IACR,KAAK,GAAG;QACN,eAAe,EAAE,CAAC;QAClB,YAAY,EAAE,CAAC;QACf,SAAS,EAAE,CAAC;QACZ,SAAS,EAAE,CAAC;QACZ,WAAW,EAAE,CAAC;KACf,CAAC;IAEF,YACE,YAAoB,aAAa,EACjC,UAA2D,EAAE;QAE7D,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,KAAK,CAAC;QAClD,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,mBAAmB,EAAE,CAAC;QAEtD,IAAI,SAAS,IAAI,gBAAgB,EAAE,CAAC;YAClC,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,gBAAgB,CAAC,SAAS,CAAC,EAAE,CAAC;QACnD,CAAC;aAAM,CAAC;YACN,qCAAqC;YACrC,IAAI,CAAC,MAAM,GAAG;gBACZ,OAAO,EAAE,SAAS;gBAClB,UAAU,EAAE,GAAG;gBACf,YAAY,EAAE,GAAG;gBACjB,WAAW,EAAE,EAAE;gBACf,cAAc,EAAE,EAAE;aACnB,CAAC;QACJ,CAAC;IACH,CAAC;IAED,8CAA8C;IAC9C,SAAS;QACP,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;IAChC,CAAC;IAEO,KAAK,CAAC,SAAS;QACrB,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO;QAC3B,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC,OAAO,CAAC;QAEtC,IAAI,CAAC,OAAO,GAAG,CAAC,KAAK,IAAI,EAAE;YACzB,MAAM,kBAAkB,EAAE,CAAC;YAE3B,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAC9E,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACtB,OAAO,CAAC,GAAG,CAAC,4BAA4B,IAAI,CAAC,MAAM,CAAC,OAAO,aAAa,WAAW,GAAG,CAAC,CAAC;YAC1F,CAAC;YAED,yBAAyB;YACzB,MAAM,eAAe,GAA4B;gBAC/C,SAAS,EAAE,IAAI;aAChB,CAAC;YAEF,0DAA0D;YAC1D,4DAA4D;YAC5D,IAAI,IAAI,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;gBAC1B,eAAe,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;YACvC,CAAC;YAED,IAAI,CAAC;gBACH,IAAI,CAAC,SAAS,GAAG,MAAM,QAAQ,CAAC,oBAAoB,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;YAC9F,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,QAAQ,GAAI,CAAW,CAAC,OAAO,CAAC;gBAEtC,yCAAyC;gBACzC,IAAI,IAAI,CAAC,MAAM,KAAK,KAAK,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;oBACpH,OAAO,CAAC,IAAI,CACV,iBAAiB,WAAW,KAAK,QAAQ,wBAAwB,CAClE,CAAC;oBACF,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;oBACpB,IAAI,CAAC,SAAS,GAAG,MAAM,QAAQ,CAAC,oBAAoB,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE;wBACzE,SAAS,EAAE,IAAI;qBAChB,CAAC,CAAC;gBACL,CAAC;qBAAM,IAAI,IAAI,CAAC,SAAS,KAAK,aAAa,EAAE,CAAC;oBAC5C,mDAAmD;oBACnD,OAAO,CAAC,IAAI,CACV,wBAAwB,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,QAAQ,IAAI;wBAC5D,mBAAmB,aAAa,EAAE,CACnC,CAAC;oBACF,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,gBAAgB,CAAC,aAAa,CAAC,EAAE,CAAC;oBACrD,IAAI,CAAC,SAAS,GAAG,MAAM,QAAQ,CAAC,oBAAoB,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE;wBACzE,SAAS,EAAE,IAAI;qBAChB,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,MAAM,IAAI,cAAc,CAAC,mCAAmC,QAAQ,EAAE,CAAC,CAAC;gBAC1E,CAAC;YACH,CAAC;YAED,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACtB,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;gBAC/E,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,IAAI,CAAC,MAAM,CAAC,UAAU,UAAU,YAAY,GAAG,CAAC,CAAC;YACxG,CAAC;YACD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC,CAAC,EAAE,CAAC;QAEL,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,UAAmB,KAAK;QAChD,cAAc;QACd,MAAM,QAAQ,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,IAAI,EAAE,CAAC;QACnD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACxC,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;YACvB,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QAEvB,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC;QAC9E,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;QAEhD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE;YAC7C,OAAO,EAAE,MAAM;YACf,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;QAEH,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAa,CAAC;QAEtD,eAAe;QACf,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;QAC7B,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACzC,sBAAsB;YACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAM,CAAC;YACjD,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC9B,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAEpC,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAe,EAAE,UAAmB,KAAK;QACxD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAElC,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QAEvB,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC;QAC9E,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAE/D,0CAA0C;QAC1C,MAAM,SAAS,GAAG,EAAE,CAAC;QACrB,MAAM,aAAa,GAAe,EAAE,CAAC;QAErC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACtD,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAEjD,+EAA+E;YAC/E,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE;wBACxC,OAAO,EAAE,MAAM;wBACf,SAAS,EAAE,IAAI;qBAChB,CAAC,CAAC;oBACH,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAa,CAAC,CAAC;gBAC1D,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,OAAO,CAAC,IAAI,CAAC,6BAA8B,CAAW,CAAC,OAAO,aAAa,CAAC,CAAC;oBAC7E,+BAA+B;oBAC/B,aAAa,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;gBAChE,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC;QACvD,IAAI,CAAC,KAAK,CAAC,eAAe,IAAI,KAAK,CAAC,MAAM,CAAC;QAC3C,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;QAC1B,IAAI,CAAC,KAAK,CAAC,SAAS,IAAI,OAAO,CAAC;QAEhC,OAAO,aAAa,CAAC;IACvB,CAAC;IAED,QAAQ;QACN,OAAO;YACL,GAAG,IAAI,CAAC,KAAK;YACb,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,mBAAmB,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC;gBAC3C,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS;gBACnD,CAAC,CAAC,CAAC;SACN,CAAC;IACJ,CAAC;IAED,UAAU;QACR,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,MAAM;QACV,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;YACtB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;CACF;AAED,gFAAgF;AAEhF,MAAM,iBAAiB,GAAkC,IAAI,GAAG,EAAE,CAAC;AAEnE,MAAM,UAAU,mBAAmB,CACjC,YAAoB,aAAa,EACjC,UAA2D,EAAE;IAE7D,qEAAqE;IACrE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,mBAAmB,EAAE,CAAC;IACvD,MAAM,QAAQ,GAAG,GAAG,SAAS,IAAI,MAAM,EAAE,CAAC;IAE1C,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QACrC,iBAAiB,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,gBAAgB,CAAC,SAAS,EAAE,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;IAC3F,CAAC;IACD,OAAO,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;AAC1C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,SAAkB;IAC5D,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,OAAO,GAAG,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACjD,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC;YACvB,iBAAiB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;SAAM,CAAC;QACN,KAAK,MAAM,OAAO,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YACjD,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC;QACzB,CAAC;QACD,iBAAiB,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;AACH,CAAC"}

package/dist/indexing/memory-retriever.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Memory retriever for semantic memory search.
+ *
+ * Provides semantic search over memories (conversations, status,
+ * decisions, etc.) using the same embedding infrastructure as code search.
+ */
+import { EmbeddingService } from './embedding-service.js';
+import { Memory } from '../core/models.js';
+export declare class MemoryRetriever {
+    static readonly COLLECTION_NAME = "memories";
+    static readonly METADATA_FILE = "memory_metadata.json";
+    private projectPath;
+    private projectId;
+    private embeddingService;
+    private chromaClient;
+    private collection;
+    private persistPath;
+    private verbose;
+    private initialized;
+    stats: Record<string, unknown>;
+    constructor(options: {
+        projectPath: string;
+        embeddingModel?: string;
+        verbose?: boolean;
+        persistPath?: string;
+        embeddingService?: EmbeddingService;
+    });
+    private ensureInitialized;
+    private log;
+    private loadStats;
+    private saveStats;
+    remember(options: {
+        content: string;
+        memoryType: string;
+        tags?: string[];
+        ttl?: string;
+        source?: string;
+        metadata?: Record<string, unknown>;
+    }): Promise<Memory>;
+    recall(options: {
+        query: string;
+        memoryType?: string;
+        tags?: string[];
+        nResults?: number;
+        timeRange?: string;
+        minRelevance?: number;
+    }): Promise<Record<string, unknown>[]>;
+    forget(options: {
+        memoryId?: string;
+        memoryType?: string;
+        tags?: string[];
+        olderThan?: string;
+    }): Promise<{
+        deleted: number;
+    }>;
+    getStats(): Promise<Record<string, unknown>>;
+}
+//# sourceMappingURL=memory-retriever.d.ts.map

package/dist/indexing/memory-retriever.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"memory-retriever.d.ts","sourceRoot":"","sources":["../../src/indexing/memory-retriever.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH,OAAO,EAAE,gBAAgB,EAAuB,MAAM,wBAAwB,CAAC;AAC/E,OAAO,EAAE,MAAM,EAA8C,MAAM,mBAAmB,CAAC;AAWvF,qBAAa,eAAe;IAC1B,MAAM,CAAC,QAAQ,CAAC,eAAe,cAAc;IAC7C,MAAM,CAAC,QAAQ,CAAC,aAAa,0BAA0B;IAEvD,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,gBAAgB,CAAmB;IAC3C,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,UAAU,CAAc;IAChC,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,OAAO,CAAU;IACzB,OAAO,CAAC,WAAW,CAAS;IAE5B,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAI5B;gBAEU,OAAO,EAAE;QACnB,WAAW,EAAE,MAAM,CAAC;QACpB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;KACrC;YAuBa,iBAAiB;IAY/B,OAAO,CAAC,GAAG;IAIX,OAAO,CAAC,SAAS;IAYjB,OAAO,CAAC,SAAS;IASX,QAAQ,CAAC,OAAO,EAAE;QACtB,OAAO,EAAE,MAAM,CAAC;QAChB,UAAU,EAAE,MAAM,CAAC;QACnB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;QAChB,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,GAAG,OAAO,CAAC,MAAM,CAAC;IA6Cb,MAAM,CAAC,OAAO,EAAE;QACpB,KAAK,EAAE,MAAM,CAAC;QACd,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;QAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAyGhC,MAAM,CAAC,OAAO,EAAE;QACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;QAChB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAmF1B,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CA6BnD"}

package/dist/indexing/memory-retriever.js

@@ -0,0 +1,327 @@
+/**
+ * Memory retriever for semantic memory search.
+ *
+ * Provides semantic search over memories (conversations, status,
+ * decisions, etc.) using the same embedding infrastructure as code search.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { ChromaClient, IncludeEnum } from 'chromadb';
+import { getEmbeddingService } from './embedding-service.js';
+import { Memory, MemoryType, memoryTypeFromString } from '../core/models.js';
+// TTL duration mappings (in milliseconds)
+const TTL_DURATIONS = {
+    session: 24 * 60 * 60 * 1000, // 24 hours
+    day: 24 * 60 * 60 * 1000, // 1 day
+    week: 7 * 24 * 60 * 60 * 1000, // 1 week
+    month: 30 * 24 * 60 * 60 * 1000, // 30 days
+    permanent: null, // Never expires
+};
+export class MemoryRetriever {
+    static COLLECTION_NAME = 'memories';
+    static METADATA_FILE = 'memory_metadata.json';
+    projectPath;
+    projectId;
+    embeddingService;
+    chromaClient;
+    collection;
+    persistPath;
+    verbose;
+    initialized = false;
+    stats = {
+        totalMemories: 0,
+        byType: {},
+        lastCleanup: null,
+    };
+    constructor(options) {
+        this.projectPath = path.resolve(options.projectPath);
+        this.projectId = this.projectPath;
+        this.verbose = options.verbose ?? false;
+        this.persistPath = options.persistPath;
+        this.embeddingService = options.embeddingService ?? getEmbeddingService(options.embeddingModel ?? 'all-MiniLM-L6-v2', { showProgress: false });
+        if (this.persistPath) {
+            fs.mkdirSync(this.persistPath, { recursive: true });
+        }
+        // ChromaDB Node.js client connects to a server.
+        // Set CHROMADB_URL env var to connect to a running ChromaDB server.
+        const chromaUrl = process.env.CHROMADB_URL;
+        this.chromaClient = chromaUrl
+            ? new ChromaClient({ path: chromaUrl })
+            : new ChromaClient();
+    }
+    async ensureInitialized() {
+        if (this.initialized)
+            return;
+        this.collection = await this.chromaClient.getOrCreateCollection({
+            name: MemoryRetriever.COLLECTION_NAME,
+            metadata: { description: 'Memory storage for conversations, status, decisions' },
+        });
+        this.loadStats();
+        this.initialized = true;
+    }
+    log(message) {
+        if (this.verbose)
+            console.log(`[Memory] ${message}`);
+    }
+    loadStats() {
+        if (this.persistPath) {
+            const metadataPath = path.join(path.dirname(this.persistPath), MemoryRetriever.METADATA_FILE);
+            try {
+                if (fs.existsSync(metadataPath)) {
+                    const data = JSON.parse(fs.readFileSync(metadataPath, 'utf-8'));
+                    this.stats = data.stats ?? this.stats;
+                }
+            }
+            catch { /* ignore */ }
+        }
+    }
+    saveStats() {
+        if (this.persistPath) {
+            const metadataPath = path.join(path.dirname(this.persistPath), MemoryRetriever.METADATA_FILE);
+            try {
+                fs.writeFileSync(metadataPath, JSON.stringify({ stats: this.stats }, null, 2));
+            }
+            catch { /* ignore */ }
+        }
+    }
+    async remember(options) {
+        await this.ensureInitialized();
+        const memory = new Memory({
+            id: crypto.randomUUID(),
+            content: options.content,
+            memoryType: memoryTypeFromString(options.memoryType),
+            project: this.projectId,
+            tags: options.tags ?? [],
+            ttl: options.ttl ?? 'permanent',
+            source: options.source ?? 'user',
+            metadata: options.metadata ?? {},
+        });
+        const embedding = await this.embeddingService.embed(options.content);
+        // Flatten metadata for ChromaDB
+        const chromaMetadata = {
+            memory_type: memory.memoryType,
+            project: memory.project,
+            tags: memory.tags.join(','),
+            created_at: memory.createdAt,
+            accessed_at: memory.accessedAt,
+            ttl: memory.ttl,
+            source: memory.source,
+        };
+        await this.collection.add({
+            ids: [memory.id],
+            embeddings: [embedding],
+            documents: [memory.content],
+            metadatas: [chromaMetadata],
+        });
+        // Update stats
+        const count = await this.collection.count();
+        this.stats.totalMemories = count;
+        const byType = this.stats.byType;
+        byType[memory.memoryType] = (byType[memory.memoryType] ?? 0) + 1;
+        this.saveStats();
+        this.log(`Stored memory: ${memory.id.substring(0, 8)}... (${options.memoryType})`);
+        return memory;
+    }
+    async recall(options) {
+        await this.ensureInitialized();
+        const count = await this.collection.count();
+        if (count === 0)
+            return [];
+        const nResults = options.nResults ?? 5;
+        const queryEmbedding = await this.embeddingService.embed(options.query, true);
+        // Parse time range
+        let timeCutoff = null;
+        if (options.timeRange && options.timeRange !== 'all') {
+            const now = new Date();
+            const ranges = {
+                today: 24 * 60 * 60 * 1000,
+                week: 7 * 24 * 60 * 60 * 1000,
+                month: 30 * 24 * 60 * 60 * 1000,
+            };
+            if (options.timeRange in ranges) {
+                timeCutoff = new Date(now.getTime() - ranges[options.timeRange]);
+            }
+        }
+        // Build filter
+        const whereClauses = [{ project: this.projectId }];
+        if (options.memoryType) {
+            whereClauses.push({ memory_type: options.memoryType });
+        }
+        const chromaWhere = whereClauses.length > 1
+            ? { $and: whereClauses }
+            : whereClauses[0];
+        try {
+            const results = await this.collection.query({
+                queryEmbeddings: [queryEmbedding],
+                nResults: Math.min(nResults * 2, 50),
+                // ChromaDB types don't fully support $and operator typing
+                where: chromaWhere,
+                include: [IncludeEnum.Documents, IncludeEnum.Metadatas, IncludeEnum.Distances],
+            });
+            if (!results.ids?.[0]?.length)
+                return [];
+            const memories = [];
+            const ids = results.ids[0];
+            const docs = results.documents?.[0] ?? [];
+            const metas = results.metadatas?.[0] ?? [];
+            const distances = results.distances?.[0] ?? [];
+            for (let i = 0; i < ids.length; i++) {
+                const distance = distances[i];
+                const relevance = 1.0 / (1.0 + distance);
+                const minRelevance = options.minRelevance ?? 0;
+                if (relevance < minRelevance)
+                    continue;
+                const metadata = metas[i];
+                // Time range filter
+                if (timeCutoff && metadata.created_at) {
+                    try {
+                        const created = new Date(metadata.created_at);
+                        if (created < timeCutoff)
+                            continue;
+                    }
+                    catch { /* skip */ }
+                }
+                // Tags filter
+                if (options.tags?.length) {
+                    const storedTags = new Set(metadata.tags?.split(',') ?? []);
+                    if (!options.tags.some(t => storedTags.has(t)))
+                        continue;
+                }
+                memories.push({
+                    id: ids[i],
+                    content: docs[i],
+                    memoryType: metadata.memory_type,
+                    tags: metadata.tags ? metadata.tags.split(',').filter(Boolean) : [],
+                    createdAt: metadata.created_at,
+                    relevance: Math.round(relevance * 1000) / 1000,
+                    source: metadata.source ?? 'unknown',
+                });
+                if (memories.length >= nResults)
+                    break;
+            }
+            // Update accessed_at for returned memories
+            const now = new Date().toISOString();
+            for (const mem of memories) {
+                try {
+                    await this.collection.update({
+                        ids: [mem.id],
+                        metadatas: [{ accessed_at: now }],
+                    });
+                }
+                catch { /* non-critical */ }
+            }
+            this.log(`Recalled ${memories.length} memories for query: ${options.query.substring(0, 50)}...`);
+            return memories;
+        }
+        catch (e) {
+            this.log(`Query error: ${e.message}`);
+            return [];
+        }
+    }
+    async forget(options) {
+        await this.ensureInitialized();
+        let deleted = 0;
+        if (options.memoryId) {
+            try {
+                const existing = await this.collection.get({ ids: [options.memoryId] });
+                if (existing.ids.length > 0) {
+                    await this.collection.delete({ ids: [options.memoryId] });
+                    deleted = 1;
+                }
+            }
+            catch { /* ignore */ }
+        }
+        else if (options.olderThan || options.tags?.length) {
+            // Fetch all and filter
+            const allMemories = await this.collection.get({
+                where: { project: this.projectId },
+                include: [IncludeEnum.Metadatas],
+            });
+            const idsToDelete = [];
+            const durationMap = {
+                '1d': 24 * 60 * 60 * 1000,
+                '7d': 7 * 24 * 60 * 60 * 1000,
+                '30d': 30 * 24 * 60 * 60 * 1000,
+                '1y': 365 * 24 * 60 * 60 * 1000,
+            };
+            for (let i = 0; i < allMemories.ids.length; i++) {
+                const metadata = allMemories.metadatas?.[i];
+                let shouldDelete = false;
+                if (options.olderThan && options.olderThan in durationMap) {
+                    const cutoff = Date.now() - durationMap[options.olderThan];
+                    const created = new Date(metadata.created_at).getTime();
+                    if (created < cutoff)
+                        shouldDelete = true;
+                }
+                if (options.tags?.length) {
+                    const storedTags = new Set(metadata.tags?.split(',') ?? []);
+                    if (options.tags.some(t => storedTags.has(t)))
+                        shouldDelete = true;
+                }
+                if (options.memoryType && metadata.memory_type !== options.memoryType) {
+                    shouldDelete = false;
+                }
+                if (shouldDelete)
+                    idsToDelete.push(allMemories.ids[i]);
+            }
+            if (idsToDelete.length > 0) {
+                await this.collection.delete({ ids: idsToDelete });
+                deleted = idsToDelete.length;
+            }
+        }
+        else if (options.memoryType) {
+            try {
+                const toDelete = await this.collection.get({
+                    where: {
+                        $and: [
+                            { memory_type: options.memoryType },
+                            { project: this.projectId },
+                        ],
+                    },
+                });
+                if (toDelete.ids.length > 0) {
+                    await this.collection.delete({ ids: toDelete.ids });
+                    deleted = toDelete.ids.length;
+                }
+            }
+            catch (e) {
+                this.log(`Delete error: ${e.message}`);
+            }
+        }
+        // Update stats
+        const count = await this.collection.count();
+        this.stats.totalMemories = count;
+        this.saveStats();
+        this.log(`Forgot ${deleted} memories`);
+        return { deleted };
+    }
+    async getStats() {
+        await this.ensureInitialized();
+        const total = await this.collection.count();
+        const byType = {};
+        for (const memType of Object.values(MemoryType)) {
+            try {
+                const result = await this.collection.get({
+                    where: {
+                        $and: [
+                            { memory_type: memType },
+                            { project: this.projectId },
+                        ],
+                    },
+                });
+                if (result.ids.length > 0) {
+                    byType[memType] = result.ids.length;
+                }
+            }
+            catch { /* ignore */ }
+        }
+        return {
+            totalMemories: total,
+            byType,
+            project: this.projectId,
+            lastCleanup: this.stats.lastCleanup,
+        };
+    }
+}
+//# sourceMappingURL=memory-retriever.js.map