@easynet/agent-memory 0.0.2

package/dist/core/agent-memory.js

@@ -0,0 +1,297 @@
+"use strict";
+/**
+ * AgentMemory - Main class for agent memory management
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AgentMemory = void 0;
+const config_1 = require("../config");
+const adapters_1 = require("../adapters");
+const gate_1 = require("../gate");
+const recall_engine_1 = require("../recall/recall-engine");
+const context_builder_1 = require("../context/context-builder");
+const document_processor_1 = require("../processing/document/document-processor");
+const persistence_1 = require("../persistence");
+const indexing_1 = require("../indexing");
+const embedding_1 = require("../embedding");
+const boot_1 = require("../boot");
+class AgentMemory {
+    constructor(config) {
+        this.store = null;
+        this.retriever = null;
+        this.recallEngine = null;
+        // Route A (in-memory) components
+        this.repo = null;
+        this.index = null;
+        this.embedder = null;
+        this.rebuilder = null;
+        this.isInMemoryMode = false;
+        this.config = config;
+        this.gate = new gate_1.MemoryGate(config);
+        this.documentProcessor = new document_processor_1.DocumentProcessor();
+        const mem0Mode = config.store.mem0.mode || "platform";
+        this.isInMemoryMode = mem0Mode === "in_memory";
+        if (this.isInMemoryMode) {
+            // Route A: Initialize SQLite + InMemoryIndex
+            this.initInMemoryMode();
+        }
+        else {
+            // Original: Platform API mode
+            this.store = (0, adapters_1.createMemoryStore)(config);
+            this.retriever = (0, adapters_1.createDocRetriever)(config);
+            this.recallEngine = new recall_engine_1.RecallEngine(this.store, this.retriever, config);
+        }
+    }
+    initInMemoryMode() {
+        if (!this.config.persistence?.sqlite) {
+            throw new Error("persistence.sqlite is required for in_memory mode");
+        }
+        if (!this.config.embedding) {
+            throw new Error("embedding configuration is required for in_memory mode");
+        }
+        // Initialize SQLite repository
+        this.repo = new persistence_1.SqliteMemoryRepo({
+            path: this.config.persistence.sqlite.path,
+            wal: this.config.persistence.sqlite.wal !== false,
+        });
+        // Initialize in-memory index
+        this.index = new indexing_1.InMemoryIndex();
+        // Initialize embedder
+        this.embedder = (0, embedding_1.createEmbedder)(this.config.embedding);
+        // Initialize boot rebuilder
+        if (this.config.boot?.rebuild?.enabled !== false) {
+            this.rebuilder = new boot_1.BootRebuilder(this.repo, this.index, this.embedder, {
+                batchSize: this.config.boot?.rebuild?.batch_size,
+                onModelMismatch: this.config.boot?.rebuild?.on_model_mismatch,
+            });
+        }
+    }
+    /**
+     * Create AgentMemory instance from YAML configuration file
+     */
+    static async fromYaml(configPath) {
+        const config = await (0, config_1.loadConfig)(configPath);
+        const instance = new AgentMemory(config);
+        // If in-memory mode, rebuild index on startup
+        if (instance.isInMemoryMode && instance.rebuilder) {
+            await instance.rebuilder.rebuild();
+        }
+        return instance;
+    }
+    /**
+     * Remember - Store memories from messages or text
+     */
+    async remember(options) {
+        // Process through MemoryGate
+        const items = await this.gate.process({
+            messages: options.messages,
+            text: options.text,
+            actor: options.actor,
+            scope: options.scope,
+            hint: options.hint,
+        });
+        if (items.length === 0) {
+            return;
+        }
+        if (this.isInMemoryMode) {
+            // Route A: Write to SQLite first, then index
+            await this.rememberInMemory(items, options);
+        }
+        else {
+            // Original: Write to platform API
+            if (!this.store) {
+                throw new Error("Store not initialized");
+            }
+            await this.store.add(items, {
+                actor: options.actor,
+                scope: options.scope,
+            });
+        }
+    }
+    async rememberInMemory(items, options) {
+        if (!this.repo || !this.index || !this.embedder) {
+            throw new Error("In-memory components not initialized");
+        }
+        // 1. Write to SQLite (source of truth)
+        const toIndex = [];
+        for (const item of items) {
+            const { inserted } = await this.repo.upsertMemory(item);
+            if (inserted) {
+                toIndex.push(item);
+            }
+        }
+        if (toIndex.length === 0) {
+            return; // All items were duplicates
+        }
+        // 2. Compute embeddings asynchronously (don't block)
+        this.persistEmbeddingsAsync(toIndex).catch((error) => {
+            console.error("Failed to persist embeddings:", error);
+        });
+        // 3. Add to in-memory index (for immediate recall)
+        await this.index.add(toIndex);
+    }
+    async persistEmbeddingsAsync(items) {
+        if (!this.repo || !this.index || !this.embedder || items.length === 0) {
+            return;
+        }
+        try {
+            const texts = items.map((item) => item.content);
+            const embeddings = await this.embedder.embed(texts);
+            for (let i = 0; i < items.length; i++) {
+                const item = items[i];
+                const embedding = embeddings[i];
+                // Save to SQLite
+                await this.repo.saveEmbedding(item.id, this.embedder.model, this.embedder.dim, embedding);
+                // Add to index
+                await this.index.addEmbedding(item.id, embedding);
+            }
+        }
+        catch (error) {
+            console.error("Failed to persist embeddings:", error);
+            // Mark as failed in SQLite (optional)
+        }
+    }
+    /**
+     * Recall - Retrieve memories and documents
+     */
+    async recall(options) {
+        if (this.isInMemoryMode) {
+            return this.recallInMemory(options);
+        }
+        else {
+            if (!this.recallEngine) {
+                throw new Error("Recall engine not initialized");
+            }
+            return this.recallEngine.recall(options);
+        }
+    }
+    async recallInMemory(options) {
+        if (!this.repo || !this.index || !this.embedder) {
+            throw new Error("In-memory components not initialized");
+        }
+        const topK = options.topK || 10;
+        // 1. Embed query
+        const queryEmbedding = (await this.embedder.embed([options.query]))[0];
+        // 2. Search in-memory index
+        const { ids } = await this.index.search(queryEmbedding, topK, {
+            actor: options.actor,
+            scope: options.scope,
+        });
+        // 3. Load full items from SQLite (source of truth)
+        const memory = await this.repo.loadByIds(ids);
+        // 4. Build context
+        const contextBuilder = new context_builder_1.ContextBuilder(this.config.recall?.context || {
+            enabled: true,
+            template: "default",
+            max_tokens: 1400,
+        });
+        // Get docs if enabled
+        let docs = [];
+        if (options.includeDocs !== false && this.retriever) {
+            try {
+                docs = await this.retriever.search(options.query, {
+                    scope: options.scope,
+                    topK,
+                });
+            }
+            catch (error) {
+                console.error("Failed to retrieve documents:", error);
+            }
+        }
+        const context = contextBuilder.build(memory, docs);
+        return {
+            memory,
+            docs,
+            context,
+            traceId: this.generateTraceId(),
+        };
+    }
+    generateTraceId() {
+        return "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, function (c) {
+            const r = (Math.random() * 16) | 0;
+            const v = c === "x" ? r : (r & 0x3) | 0x8;
+            return v.toString(16);
+        });
+    }
+    /**
+     * Remember documents - Store document content as memories
+     * Supports multi-modal documents (text, PDF, images, etc.)
+     */
+    async rememberDocument(options) {
+        const { document, actor, scope, autoChunk = true, chunkSize, chunkOverlap, extractMetadata = false, tags = [], } = options;
+        // Process documents into chunks
+        const chunks = autoChunk
+            ? this.documentProcessor.processDocument(document, {
+                chunkSize,
+                chunkOverlap,
+            })
+            : Array.isArray(document)
+                ? document
+                : [document];
+        // Convert chunks to memory items
+        const memoryItems = [];
+        for (const chunk of chunks) {
+            // Convert chunk to memory items
+            const items = this.documentProcessor.chunksToMemoryItems([chunk], actor, scope, tags);
+            memoryItems.push(...items);
+            // Extract metadata if requested
+            if (extractMetadata) {
+                const metaItems = this.documentProcessor.extractMetadataAsMemories(chunk, actor, scope);
+                memoryItems.push(...metaItems);
+            }
+        }
+        if (memoryItems.length === 0) {
+            return;
+        }
+        // Process through MemoryGate for redaction and deduplication
+        // Documents already have structure, so we apply light processing
+        // Use gate's internal components for redaction and deduplication only
+        const gateInternal = this.gate;
+        const redactor = gateInternal.redactor;
+        const deduplicator = gateInternal.deduplicator;
+        // Apply redaction to content
+        const redactedItems = memoryItems.map((item) => ({
+            ...item,
+            content: redactor.redact(item.content),
+        }));
+        // Filter out items with too little content after redaction
+        const validItems = redactedItems.filter((item) => item.content.trim().length >= 10);
+        // Apply deduplication
+        const finalItems = deduplicator.dedupe(validItems);
+        if (finalItems.length === 0) {
+            return;
+        }
+        // Store memories
+        if (this.isInMemoryMode) {
+            await this.rememberInMemory(finalItems, {
+                actor,
+                scope,
+            });
+        }
+        else {
+            if (!this.store) {
+                throw new Error("Store not initialized");
+            }
+            await this.store.add(finalItems, {
+                actor,
+                scope,
+            });
+        }
+    }
+    /**
+     * Feedback - Provide feedback for memory utility learning (reserved for v0.2+)
+     */
+    async feedback(options) {
+        // Placeholder for future implementation
+        console.warn("feedback() is not yet implemented in v0.1");
+    }
+    /**
+     * Close resources (SQLite connection, etc.)
+     */
+    close() {
+        if (this.repo) {
+            this.repo.close();
+        }
+    }
+}
+exports.AgentMemory = AgentMemory;
+//# sourceMappingURL=agent-memory.js.map

package/dist/core/agent-memory.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-memory.js","sourceRoot":"","sources":["../../src/core/agent-memory.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAEH,sCAAuC;AAEvC,0CAAoE;AAEpE,kCAAqC;AACrC,2DAAuD;AACvD,gEAA4D;AAS5D,kFAA8E;AAC9E,gDAAkD;AAClD,0CAA4C;AAC5C,4CAAwD;AACxD,kCAAwC;AAExC,MAAa,WAAW;IAiBtB,YAAoB,MAAc;QAf1B,UAAK,GAAuB,IAAI,CAAC;QACjC,cAAS,GAAwB,IAAI,CAAC;QAEtC,iBAAY,GAAwB,IAAI,CAAC;QAEjD,iCAAiC;QACzB,SAAI,GAA4B,IAAI,CAAC;QACrC,UAAK,GAAyB,IAAI,CAAC;QACnC,aAAQ,GAAoB,IAAI,CAAC;QACjC,cAAS,GAAyB,IAAI,CAAC;QACvC,mBAAc,GAAY,KAAK,CAAC;QAMtC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,IAAI,iBAAU,CAAC,MAAM,CAAC,CAAC;QACnC,IAAI,CAAC,iBAAiB,GAAG,IAAI,sCAAiB,EAAE,CAAC;QAEjD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,UAAU,CAAC;QACtD,IAAI,CAAC,cAAc,GAAG,QAAQ,KAAK,WAAW,CAAC;QAE/C,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,6CAA6C;YAC7C,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1B,CAAC;aAAM,CAAC;YACN,8BAA8B;YAC9B,IAAI,CAAC,KAAK,GAAG,IAAA,4BAAiB,EAAC,MAAM,CAAC,CAAC;YACvC,IAAI,CAAC,SAAS,GAAG,IAAA,6BAAkB,EAAC,MAAM,CAAC,CAAC;YAC5C,IAAI,CAAC,YAAY,GAAG,IAAI,4BAAY,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;IAEO,gBAAgB;QACtB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;QACvE,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC5E,CAAC;QAED,+BAA+B;QAC/B,IAAI,CAAC,IAAI,GAAG,IAAI,8BAAgB,CAAC;YAC/B,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI;YACzC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,KAAK,KAAK;SAClD,CAAC,CAAC;QAEH,6BAA6B;QAC7B,IAAI,CAAC,KAAK,GAAG,IAAI,wBAAa,EAAE,CAAC;QAEjC,sBAAsB;QACtB,IAAI,CAAC,QAAQ,GAAG,IAAA,0BAAc,EAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAEtD,4BAA4B;QAC5B,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,KAAK,KAAK,EAAE,CAAC;YACjD,IAAI,CAAC,SAAS,GAAG,IAAI,oBAAa,CAChC,IAAI,CAAC,IAAI,EACT,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,QAAQ,EACb;gBACE,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,UAAU;gBAChD,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,iBAAiB;aAC9D,CACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAkB;QACtC,MAAM,MAAM,GAAG,MAAM,IAAA,mBAAU,EAAC,UAAU,CAAC,CAAC;QAC5C,MAAM,QAAQ,GAAG,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC;QAEzC,8CAA8C;QAC9C,IAAI,QAAQ,CAAC,cAAc,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;YAClD,MAAM,QAAQ,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QACrC,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,OAAwB;QACrC,6BAA6B;QAC7B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;YACpC,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,IAAI,EAAE,OAAO,CAAC,IAAI;SACnB,CAAC,CAAC;QAEH,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,6CAA6C;YAC7C,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC9C,CAAC;aAAM,CAAC;YACN,kCAAkC;YAClC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBAChB,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;YAC3C,CAAC;YACD,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE;gBAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,KAAK,EAAE,OAAO,CAAC,KAAK;aACrB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,KAAmB,EAAE,OAAwB;QAC1E,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChD,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;QAED,uCAAuC;QACvC,MAAM,OAAO,GAAiB,EAAE,CAAC;QACjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YACxD,IAAI,QAAQ,EAAE,CAAC;gBACb,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,4BAA4B;QACtC,CAAC;QAED,qDAAqD;QACrD,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACnD,OAAO,CAAC,KAAK,CAAC,+BAA+B,EAAE,KAAK,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,mDAAmD;QACnD,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAAC,KAAmB;QACtD,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtE,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAChD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAEpD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACtB,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;gBAEhC,iBAAiB;gBACjB,MAAM,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;gBAE1F,eAAe;gBACf,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;YACpD,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,+BAA+B,EAAE,KAAK,CAAC,CAAC;YACtD,sCAAsC;QACxC,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,OAAsB;QACjC,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,OAAO,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;gBACvB,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;YACnD,CAAC;YACD,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,OAAsB;QACjD,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChD,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;QAED,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;QAEhC,iBAAiB;QACjB,MAAM,cAAc,GAAG,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEvE,4BAA4B;QAC5B,MAAM,EAAE,GAAG,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,EAAE,IAAI,EAAE;YAC5D,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,KAAK,EAAE,OAAO,CAAC,KAAK;SACrB,CAAC,CAAC;QAEH,mDAAmD;QACnD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAE9C,mBAAmB;QACnB,MAAM,cAAc,GAAG,IAAI,gCAAc,CACvC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,IAAI;YAC7B,OAAO,EAAE,IAAI;YACb,QAAQ,EAAE,SAAS;YACnB,UAAU,EAAE,IAAI;SACjB,CACF,CAAC;QAEF,sBAAsB;QACtB,IAAI,IAAI,GAAU,EAAE,CAAC;QACrB,IAAI,OAAO,CAAC,WAAW,KAAK,KAAK,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACpD,IAAI,CAAC;gBACH,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE;oBAChD,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,IAAI;iBACL,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,+BAA+B,EAAE,KAAK,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAEnD,OAAO;YACL,MAAM;YACN,IAAI;YACJ,OAAO;YACP,OAAO,EAAE,IAAI,CAAC,eAAe,EAAE;SAChC,CAAC;IACJ,CAAC;IAEO,eAAe;QACrB,OAAO,sCAAsC,CAAC,OAAO,CAAC,OAAO,EAAE,UAAU,CAAC;YACxE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;YACnC,MAAM,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;YAC1C,OAAO,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,gBAAgB,CAAC,OAA8B;QACnD,MAAM,EACJ,QAAQ,EACR,KAAK,EACL,KAAK,EACL,SAAS,GAAG,IAAI,EAChB,SAAS,EACT,YAAY,EACZ,eAAe,GAAG,KAAK,EACvB,IAAI,GAAG,EAAE,GACV,GAAG,OAAO,CAAC;QAEZ,gCAAgC;QAChC,MAAM,MAAM,GAAG,SAAS;YACtB,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,QAAQ,EAAE;gBAC/C,SAAS;gBACT,YAAY;aACb,CAAC;YACJ,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;gBACzB,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QAEf,iCAAiC;QACjC,MAAM,WAAW,GAAiB,EAAE,CAAC;QAErC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,gCAAgC;YAChC,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,mBAAmB,CACtD,CAAC,KAAK,CAAC,EACP,KAAK,EACL,KAAK,EACL,IAAI,CACL,CAAC;YACF,WAAW,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;YAE3B,gCAAgC;YAChC,IAAI,eAAe,EAAE,CAAC;gBACpB,MAAM,SAAS,GAAG,IAAI,CAAC,iBAAiB,CAAC,yBAAyB,CAChE,KAAK,EACL,KAAK,EACL,KAAK,CACN,CAAC;gBACF,WAAW,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;QAED,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO;QACT,CAAC;QAED,6DAA6D;QAC7D,iEAAiE;QACjE,sEAAsE;QACtE,MAAM,YAAY,GAAG,IAAI,CAAC,IAAW,CAAC;QACtC,MAAM,QAAQ,GAAG,YAAY,CAAC,QAAQ,CAAC;QACvC,MAAM,YAAY,GAAG,YAAY,CAAC,YAAY,CAAC;QAE/C,6BAA6B;QAC7B,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC/C,GAAG,IAAI;YACP,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;SACvC,CAAC,CAAC,CAAC;QAEJ,2DAA2D;QAC3D,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CACrC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,IAAI,EAAE,CAC3C,CAAC;QAEF,sBAAsB;QACtB,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAEnD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO;QACT,CAAC;QAED,iBAAiB;QACjB,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE;gBACtC,KAAK;gBACL,KAAK;aACN,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBAChB,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;YAC3C,CAAC;YACD,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE;gBAC/B,KAAK;gBACL,KAAK;aACN,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,OAAwB;QACrC,wCAAwC;QACxC,OAAO,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;IAC5D,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QACpB,CAAC;IACH,CAAC;CACF;AAlWD,kCAkWC"}

package/dist/embedding/embedder.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Embedding provider interface and implementations
+ */
+export interface Embedder {
+    model: string;
+    dim: number;
+    embed(texts: string[]): Promise<Float32Array[]>;
+}
+export interface EmbedderConfig {
+    provider: "ollama" | "openai";
+    model: string;
+    dim: number;
+    base_url?: string;
+    api_key?: string;
+    concurrency?: number;
+}
+/**
+ * Ollama Embedder
+ */
+export declare class OllamaEmbedder implements Embedder {
+    readonly model: string;
+    readonly dim: number;
+    private baseUrl;
+    private concurrency;
+    private queue;
+    private running;
+    constructor(config: EmbedderConfig);
+    embed(texts: string[]): Promise<Float32Array[]>;
+    private embedOne;
+}
+/**
+ * OpenAI Embedder
+ */
+export declare class OpenAIEmbedder implements Embedder {
+    readonly model: string;
+    readonly dim: number;
+    private apiKey;
+    private baseUrl;
+    private concurrency;
+    private running;
+    constructor(config: EmbedderConfig);
+    embed(texts: string[]): Promise<Float32Array[]>;
+    private embedBatch;
+}
+/**
+ * Create embedder from config
+ */
+export declare function createEmbedder(config: EmbedderConfig): Embedder;
+//# sourceMappingURL=embedder.d.ts.map

package/dist/embedding/embedder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../../src/embedding/embedder.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;CACjD;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,qBAAa,cAAe,YAAW,QAAQ;IAC7C,SAAgB,KAAK,EAAE,MAAM,CAAC;IAC9B,SAAgB,GAAG,EAAE,MAAM,CAAC;IAC5B,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,KAAK,CAAkC;IAC/C,OAAO,CAAC,OAAO,CAAK;gBAER,MAAM,EAAE,cAAc;IAO5B,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;YAevC,QAAQ;CAoCvB;AAED;;GAEG;AACH,qBAAa,cAAe,YAAW,QAAQ;IAC7C,SAAgB,KAAK,EAAE,MAAM,CAAC;IAC9B,SAAgB,GAAG,EAAE,MAAM,CAAC;IAC5B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,OAAO,CAAK;gBAER,MAAM,EAAE,cAAc;IAW5B,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;YA4BvC,UAAU;CA6BzB;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,cAAc,GAAG,QAAQ,CAS/D"}

package/dist/embedding/embedder.js

@@ -0,0 +1,146 @@
+"use strict";
+/**
+ * Embedding provider interface and implementations
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OpenAIEmbedder = exports.OllamaEmbedder = void 0;
+exports.createEmbedder = createEmbedder;
+/**
+ * Ollama Embedder
+ */
+class OllamaEmbedder {
+    constructor(config) {
+        this.queue = [];
+        this.running = 0;
+        this.model = config.model;
+        this.dim = config.dim;
+        this.baseUrl = config.base_url || "http://localhost:11434";
+        this.concurrency = config.concurrency || 4;
+    }
+    async embed(texts) {
+        const results = [];
+        const promises = [];
+        for (const text of texts) {
+            const promise = this.embedOne(text).then((vector) => {
+                results.push(vector);
+            });
+            promises.push(promise);
+        }
+        await Promise.all(promises);
+        return results;
+    }
+    async embedOne(text) {
+        // Wait for available slot
+        while (this.running >= this.concurrency) {
+            await new Promise((resolve) => setTimeout(resolve, 10));
+        }
+        this.running++;
+        try {
+            const response = await fetch(`${this.baseUrl}/api/embeddings`, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                },
+                body: JSON.stringify({
+                    model: this.model,
+                    prompt: text,
+                }),
+            });
+            if (!response.ok) {
+                const error = await response.text();
+                throw new Error(`Ollama embedding error: ${response.status} ${error}`);
+            }
+            const result = await response.json();
+            const embedding = result.embedding || result.embedding || [];
+            if (!Array.isArray(embedding) || embedding.length !== this.dim) {
+                throw new Error(`Invalid embedding dimension: expected ${this.dim}, got ${embedding.length}`);
+            }
+            return new Float32Array(embedding);
+        }
+        finally {
+            this.running--;
+        }
+    }
+}
+exports.OllamaEmbedder = OllamaEmbedder;
+/**
+ * OpenAI Embedder
+ */
+class OpenAIEmbedder {
+    constructor(config) {
+        this.running = 0;
+        if (!config.api_key) {
+            throw new Error("OpenAI API key is required");
+        }
+        this.model = config.model;
+        this.dim = config.dim;
+        this.apiKey = config.api_key;
+        this.baseUrl = config.base_url || "https://api.openai.com/v1";
+        this.concurrency = config.concurrency || 4;
+    }
+    async embed(texts) {
+        const results = [];
+        const batches = [];
+        const batchSize = 100; // OpenAI allows up to 2048 inputs per request
+        // Batch texts
+        for (let i = 0; i < texts.length; i += batchSize) {
+            batches.push(texts.slice(i, i + batchSize));
+        }
+        // Process batches with concurrency control
+        for (const batch of batches) {
+            while (this.running >= this.concurrency) {
+                await new Promise((resolve) => setTimeout(resolve, 10));
+            }
+            this.running++;
+            try {
+                const vectors = await this.embedBatch(batch);
+                results.push(...vectors);
+            }
+            finally {
+                this.running--;
+            }
+        }
+        return results;
+    }
+    async embedBatch(texts) {
+        const response = await fetch(`${this.baseUrl}/embeddings`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                Authorization: `Bearer ${this.apiKey}`,
+            },
+            body: JSON.stringify({
+                model: this.model,
+                input: texts,
+            }),
+        });
+        if (!response.ok) {
+            const error = await response.text();
+            throw new Error(`OpenAI embedding error: ${response.status} ${error}`);
+        }
+        const result = await response.json();
+        const embeddings = result.data || [];
+        return embeddings.map((item) => {
+            const embedding = item.embedding || [];
+            if (!Array.isArray(embedding) || embedding.length !== this.dim) {
+                throw new Error(`Invalid embedding dimension: expected ${this.dim}, got ${embedding.length}`);
+            }
+            return new Float32Array(embedding);
+        });
+    }
+}
+exports.OpenAIEmbedder = OpenAIEmbedder;
+/**
+ * Create embedder from config
+ */
+function createEmbedder(config) {
+    switch (config.provider) {
+        case "ollama":
+            return new OllamaEmbedder(config);
+        case "openai":
+            return new OpenAIEmbedder(config);
+        default:
+            throw new Error(`Unsupported embedding provider: ${config.provider}`);
+    }
+}
+//# sourceMappingURL=embedder.js.map

package/dist/embedding/embedder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../../src/embedding/embedder.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AA4KH,wCASC;AApKD;;GAEG;AACH,MAAa,cAAc;IAQzB,YAAY,MAAsB;QAH1B,UAAK,GAA+B,EAAE,CAAC;QACvC,YAAO,GAAG,CAAC,CAAC;QAGlB,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QAC1B,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;QACtB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,QAAQ,IAAI,wBAAwB,CAAC;QAC3D,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAe;QACzB,MAAM,OAAO,GAAmB,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAoB,EAAE,CAAC;QAErC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;gBAClD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC,CAAC,CAAC;YACH,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC;QAED,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC5B,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,IAAY;QACjC,0BAA0B;QAC1B,OAAO,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACxC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;QAC1D,CAAC;QAED,IAAI,CAAC,OAAO,EAAE,CAAC;QACf,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,iBAAiB,EAAE;gBAC7D,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;iBACnC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,MAAM,EAAE,IAAI;iBACb,CAAC;aACH,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACpC,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC,CAAC;YACzE,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAS,CAAC;YAC5C,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;YAE7D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC,MAAM,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC/D,MAAM,IAAI,KAAK,CAAC,yCAAyC,IAAI,CAAC,GAAG,SAAS,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;YAChG,CAAC;YAED,OAAO,IAAI,YAAY,CAAC,SAAS,CAAC,CAAC;QACrC,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,OAAO,EAAE,CAAC;QACjB,CAAC;IACH,CAAC;CACF;AAlED,wCAkEC;AAED;;GAEG;AACH,MAAa,cAAc;IAQzB,YAAY,MAAsB;QAF1B,YAAO,GAAG,CAAC,CAAC;QAGlB,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAChD,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QAC1B,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;QACtB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC;QAC7B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,QAAQ,IAAI,2BAA2B,CAAC;QAC9D,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAe;QACzB,MAAM,OAAO,GAAmB,EAAE,CAAC;QACnC,MAAM,OAAO,GAAe,EAAE,CAAC;QAC/B,MAAM,SAAS,GAAG,GAAG,CAAC,CAAC,8CAA8C;QAErE,cAAc;QACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACjD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;QAC9C,CAAC;QAED,2CAA2C;QAC3C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;gBACxC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;YAC1D,CAAC;YAED,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;gBAC7C,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;YAC3B,CAAC;oBAAS,CAAC;gBACT,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,KAAe;QACtC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,aAAa,EAAE;YACzD,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;aACvC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,KAAK,EAAE,KAAK;aACb,CAAC;SACH,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC,CAAC;QACzE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAS,CAAC;QAC5C,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QAErC,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE;YAClC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;YACvC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC,MAAM,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC/D,MAAM,IAAI,KAAK,CAAC,yCAAyC,IAAI,CAAC,GAAG,SAAS,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;YAChG,CAAC;YACD,OAAO,IAAI,YAAY,CAAC,SAAS,CAAC,CAAC;QACrC,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AA5ED,wCA4EC;AAED;;GAEG;AACH,SAAgB,cAAc,CAAC,MAAsB;IACnD,QAAQ,MAAM,CAAC,QAAQ,EAAE,CAAC;QACxB,KAAK,QAAQ;YACX,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;QACpC,KAAK,QAAQ;YACX,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;QACpC;YACE,MAAM,IAAI,KAAK,CAAC,mCAAmC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC1E,CAAC;AACH,CAAC"}

package/dist/embedding/index.d.ts

@@ -0,0 +1,3 @@
+export { Embedder, OllamaEmbedder, OpenAIEmbedder, createEmbedder } from "./embedder";
+export type { EmbedderConfig } from "./embedder";
+//# sourceMappingURL=index.d.ts.map

package/dist/embedding/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/embedding/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACtF,YAAY,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC"}

package/dist/embedding/index.js

@@ -0,0 +1,8 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createEmbedder = exports.OpenAIEmbedder = exports.OllamaEmbedder = void 0;
+var embedder_1 = require("./embedder");
+Object.defineProperty(exports, "OllamaEmbedder", { enumerable: true, get: function () { return embedder_1.OllamaEmbedder; } });
+Object.defineProperty(exports, "OpenAIEmbedder", { enumerable: true, get: function () { return embedder_1.OpenAIEmbedder; } });
+Object.defineProperty(exports, "createEmbedder", { enumerable: true, get: function () { return embedder_1.createEmbedder; } });
+//# sourceMappingURL=index.js.map

package/dist/embedding/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/embedding/index.ts"],"names":[],"mappings":";;;AAAA,uCAAsF;AAAnE,0GAAA,cAAc,OAAA;AAAE,0GAAA,cAAc,OAAA;AAAE,0GAAA,cAAc,OAAA"}

package/dist/gate/deduplicator.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Deduplication - Remove similar/duplicate memories
+ */
+import { MemoryItem } from "../types";
+export interface DeduplicationConfig {
+    enabled: boolean;
+    similarity_threshold: number;
+}
+export declare class Deduplicator {
+    private config;
+    constructor(config: DeduplicationConfig);
+    /**
+     * Simple text similarity using Jaccard similarity on words
+     * For production, consider using embeddings
+     */
+    private similarity;
+    /**
+     * Remove duplicates from a list of memory items
+     */
+    dedupe(items: MemoryItem[]): MemoryItem[];
+}
+//# sourceMappingURL=deduplicator.d.ts.map

package/dist/gate/deduplicator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"deduplicator.d.ts","sourceRoot":"","sources":["../../src/gate/deduplicator.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAEtC,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,OAAO,CAAC;IACjB,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAED,qBAAa,YAAY;IACvB,OAAO,CAAC,MAAM,CAAsB;gBAExB,MAAM,EAAE,mBAAmB;IAIvC;;;OAGG;IACH,OAAO,CAAC,UAAU;IAUlB;;OAEG;IACH,MAAM,CAAC,KAAK,EAAE,UAAU,EAAE,GAAG,UAAU,EAAE;CAuD1C"}

package/dist/gate/deduplicator.js

@@ -0,0 +1,74 @@
+"use strict";
+/**
+ * Deduplication - Remove similar/duplicate memories
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Deduplicator = void 0;
+class Deduplicator {
+    constructor(config) {
+        this.config = config;
+    }
+    /**
+     * Simple text similarity using Jaccard similarity on words
+     * For production, consider using embeddings
+     */
+    similarity(text1, text2) {
+        const words1 = new Set(text1.toLowerCase().split(/\s+/));
+        const words2 = new Set(text2.toLowerCase().split(/\s+/));
+        const intersection = new Set([...words1].filter(x => words2.has(x)));
+        const union = new Set([...words1, ...words2]);
+        return intersection.size / union.size;
+    }
+    /**
+     * Remove duplicates from a list of memory items
+     */
+    dedupe(items) {
+        if (!this.config.enabled || items.length <= 1) {
+            return items;
+        }
+        const result = [];
+        const seen = new Set();
+        for (let i = 0; i < items.length; i++) {
+            if (seen.has(i)) {
+                continue;
+            }
+            const item = items[i];
+            const duplicates = [];
+            // Find similar items
+            for (let j = i + 1; j < items.length; j++) {
+                if (seen.has(j)) {
+                    continue;
+                }
+                const other = items[j];
+                // Only dedupe items of the same type
+                if (item.type === other.type) {
+                    const sim = this.similarity(item.content, other.content);
+                    if (sim >= this.config.similarity_threshold) {
+                        duplicates.push(j);
+                    }
+                }
+            }
+            // Keep the item with higher confidence, or the first one
+            if (duplicates.length > 0) {
+                const allSimilar = [i, ...duplicates];
+                allSimilar.sort((a, b) => {
+                    const confA = items[a].confidence || 0;
+                    const confB = items[b].confidence || 0;
+                    return confB - confA;
+                });
+                // Keep the best one
+                result.push(items[allSimilar[0]]);
+                // Mark others as seen
+                for (let idx of allSimilar.slice(1)) {
+                    seen.add(idx);
+                }
+            }
+            else {
+                result.push(item);
+            }
+        }
+        return result;
+    }
+}
+exports.Deduplicator = Deduplicator;
+//# sourceMappingURL=deduplicator.js.map

package/dist/gate/deduplicator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"deduplicator.js","sourceRoot":"","sources":["../../src/gate/deduplicator.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AASH,MAAa,YAAY;IAGvB,YAAY,MAA2B;QACrC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED;;;OAGG;IACK,UAAU,CAAC,KAAa,EAAE,KAAa;QAC7C,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QACzD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QAEzD,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC;QAE9C,OAAO,YAAY,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;IACxC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAmB;QACxB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAC9C,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,MAAM,GAAiB,EAAE,CAAC;QAChC,MAAM,IAAI,GAAgB,IAAI,GAAG,EAAE,CAAC;QAEpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAChB,SAAS;YACX,CAAC;YAED,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,UAAU,GAAa,EAAE,CAAC;YAEhC,qBAAqB;YACrB,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC1C,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;oBAChB,SAAS;gBACX,CAAC;gBAED,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACvB,qCAAqC;gBACrC,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,EAAE,CAAC;oBAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;oBACzD,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,oBAAoB,EAAE,CAAC;wBAC5C,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;oBACrB,CAAC;gBACH,CAAC;YACH,CAAC;YAED,yDAAyD;YACzD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,UAAU,GAAG,CAAC,CAAC,EAAE,GAAG,UAAU,CAAC,CAAC;gBACtC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;oBACvB,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC;oBACvC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC;oBACvC,OAAO,KAAK,GAAG,KAAK,CAAC;gBACvB,CAAC,CAAC,CAAC;gBAEH,oBAAoB;gBACpB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAElC,sBAAsB;gBACtB,KAAK,IAAI,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;oBACpC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAChB,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AA/ED,oCA+EC"}