cozo-memory 1.0.4 → 1.0.6

package/dist/cli.js

@@ -0,0 +1,410 @@
+#!/usr/bin/env node
+"use strict";
+/**
+ * Pure CLI for CozoDB Memory
+ * Usage: cozo-memory <command> [options]
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const commander_1 = require("commander");
+const cli_commands_js_1 = require("./cli-commands.js");
+const chalk_1 = __importDefault(require("chalk"));
+const fs = __importStar(require("fs"));
+const program = new commander_1.Command();
+const cli = new cli_commands_js_1.CLICommands();
+// Helper to format output
+function formatOutput(data, format = 'pretty') {
+    if (format === 'json') {
+        console.log(JSON.stringify(data, null, 2));
+    }
+    else {
+        console.log(chalk_1.default.cyan(JSON.stringify(data, null, 2)));
+    }
+}
+// Helper to handle errors
+function handleError(error) {
+    console.error(chalk_1.default.red('Error:'), error.message || error);
+    process.exit(1);
+}
+program
+    .name('cozo-memory')
+    .description('CLI for CozoDB Memory - Local-first persistent memory for AI agents')
+    .version('1.0.6');
+// Entity commands
+const entity = program.command('entity').description('Entity operations');
+entity
+    .command('create')
+    .description('Create a new entity')
+    .requiredOption('-n, --name <name>', 'Entity name')
+    .requiredOption('-t, --type <type>', 'Entity type')
+    .option('-m, --metadata <json>', 'Metadata as JSON string')
+    .option('-f, --format <format>', 'Output format (json|pretty)', 'pretty')
+    .action(async (options) => {
+    try {
+        await cli.init();
+        const metadata = options.metadata ? JSON.parse(options.metadata) : undefined;
+        const result = await cli.createEntity(options.name, options.type, metadata);
+        formatOutput(result, options.format);
+        await cli.close();
+    }
+    catch (error) {
+        handleError(error);
+    }
+});
+entity
+    .command('get')
+    .description('Get entity details')
+    .requiredOption('-i, --id <id>', 'Entity ID')
+    .option('-f, --format <format>', 'Output format (json|pretty)', 'pretty')
+    .action(async (options) => {
+    try {
+        await cli.init();
+        const result = await cli.getEntity(options.id);
+        formatOutput(result, options.format);
+        await cli.close();
+    }
+    catch (error) {
+        handleError(error);
+    }
+});
+entity
+    .command('delete')
+    .description('Delete an entity')
+    .requiredOption('-i, --id <id>', 'Entity ID')
+    .option('-f, --format <format>', 'Output format (json|pretty)', 'pretty')
+    .action(async (options) => {
+    try {
+        await cli.init();
+        const result = await cli.deleteEntity(options.id);
+        formatOutput(result, options.format);
+        await cli.close();
+    }
+    catch (error) {
+        handleError(error);
+    }
+});
+// Observation commands
+const observation = program.command('observation').alias('obs').description('Observation operations');
+observation
+    .command('add')
+    .description('Add observation to entity')
+    .requiredOption('-i, --entity-id <id>', 'Entity ID')
+    .requiredOption('-t, --text <text>', 'Observation text')
+    .option('-m, --metadata <json>', 'Metadata as JSON string')
+    .option('-f, --format <format>', 'Output format (json|pretty)', 'pretty')
+    .action(async (options) => {
+    try {
+        await cli.init();
+        const metadata = options.metadata ? JSON.parse(options.metadata) : undefined;
+        const result = await cli.addObservation(options.entityId, options.text, metadata);
+        formatOutput(result, options.format);
+        await cli.close();
+    }
+    catch (error) {
+        handleError(error);
+    }
+});
+// Relation commands
+const relation = program.command('relation').alias('rel').description('Relation operations');
+relation
+    .command('create')
+    .description('Create relation between entities')
+    .requiredOption('--from <id>', 'From entity ID')
+    .requiredOption('--to <id>', 'To entity ID')
+    .requiredOption('--type <type>', 'Relation type')
+    .option('-s, --strength <number>', 'Relation strength (0-1)', parseFloat)
+    .option('-m, --metadata <json>', 'Metadata as JSON string')
+    .option('-f, --format <format>', 'Output format (json|pretty)', 'pretty')
+    .action(async (options) => {
+    try {
+        await cli.init();
+        const metadata = options.metadata ? JSON.parse(options.metadata) : undefined;
+        const result = await cli.createRelation(options.from, options.to, options.type, options.strength, metadata);
+        formatOutput(result, options.format);
+        await cli.close();
+    }
+    catch (error) {
+        handleError(error);
+    }
+});
+// Search commands
+const search = program.command('search').description('Search operations');
+search
+    .command('query')
+    .description('Search memory')
+    .requiredOption('-q, --query <query>', 'Search query')
+    .option('-l, --limit <number>', 'Result limit', parseInt, 10)
+    .option('-t, --types <types>', 'Entity types (comma-separated)')
+    .option('--no-entities', 'Exclude entities')
+    .option('--no-observations', 'Exclude observations')
+    .option('-f, --format <format>', 'Output format (json|pretty)', 'pretty')
+    .action(async (options) => {
+    try {
+        await cli.init();
+        const entityTypes = options.types ? options.types.split(',') : undefined;
+        const result = await cli.search(options.query, options.limit, entityTypes, options.entities, options.observations);
+        formatOutput(result, options.format);
+        await cli.close();
+    }
+    catch (error) {
+        handleError(error);
+    }
+});
+search
+    .command('context')
+    .description('Get contextual information')
+    .requiredOption('-q, --query <query>', 'Context query')
+    .option('-w, --window <number>', 'Context window', parseInt)
+    .option('-h, --hours <number>', 'Time range in hours', parseInt)
+    .option('-f, --format <format>', 'Output format (json|pretty)', 'pretty')
+    .action(async (options) => {
+    try {
+        await cli.init();
+        const result = await cli.context(options.query, options.window, options.hours);
+        formatOutput(result, options.format);
+        await cli.close();
+    }
+    catch (error) {
+        handleError(error);
+    }
+});
+// Graph commands
+const graph = program.command('graph').description('Graph operations');
+graph
+    .command('explore')
+    .description('Explore graph from entity')
+    .requiredOption('-s, --start <id>', 'Start entity ID')
+    .option('-e, --end <id>', 'End entity ID (for path finding)')
+    .option('-h, --hops <number>', 'Max hops', parseInt, 3)
+    .option('-t, --types <types>', 'Relation types (comma-separated)')
+    .option('-f, --format <format>', 'Output format (json|pretty)', 'pretty')
+    .action(async (options) => {
+    try {
+        await cli.init();
+        const relationTypes = options.types ? options.types.split(',') : undefined;
+        const result = await cli.explore(options.start, options.end, options.hops, relationTypes);
+        formatOutput(result, options.format);
+        await cli.close();
+    }
+    catch (error) {
+        handleError(error);
+    }
+});
+graph
+    .command('pagerank')
+    .description('Calculate PageRank')
+    .option('-f, --format <format>', 'Output format (json|pretty)', 'pretty')
+    .action(async (options) => {
+    try {
+        await cli.init();
+        const result = await cli.pagerank();
+        formatOutput(result, options.format);
+        await cli.close();
+    }
+    catch (error) {
+        handleError(error);
+    }
+});
+graph
+    .command('communities')
+    .description('Detect communities')
+    .option('-f, --format <format>', 'Output format (json|pretty)', 'pretty')
+    .action(async (options) => {
+    try {
+        await cli.init();
+        const result = await cli.communities();
+        formatOutput(result, options.format);
+        await cli.close();
+    }
+    catch (error) {
+        handleError(error);
+    }
+});
+// System commands
+const system = program.command('system').alias('sys').description('System operations');
+system
+    .command('health')
+    .description('Check system health')
+    .option('-f, --format <format>', 'Output format (json|pretty)', 'pretty')
+    .action(async (options) => {
+    try {
+        await cli.init();
+        const result = await cli.health();
+        formatOutput(result, options.format);
+        await cli.close();
+    }
+    catch (error) {
+        handleError(error);
+    }
+});
+system
+    .command('metrics')
+    .description('Get system metrics')
+    .option('-f, --format <format>', 'Output format (json|pretty)', 'pretty')
+    .action(async (options) => {
+    try {
+        await cli.init();
+        const result = await cli.metrics();
+        formatOutput(result, options.format);
+        await cli.close();
+    }
+    catch (error) {
+        handleError(error);
+    }
+});
+// Export/Import commands
+const exportCmd = program.command('export').description('Export memory');
+exportCmd
+    .command('json')
+    .description('Export as JSON')
+    .option('-o, --output <file>', 'Output file')
+    .option('--include-metadata', 'Include metadata')
+    .option('--include-relationships', 'Include relationships')
+    .option('--include-observations', 'Include observations')
+    .action(async (options) => {
+    try {
+        await cli.init();
+        const result = await cli.exportMemory('json', {
+            includeMetadata: options.includeMetadata,
+            includeRelationships: options.includeRelationships,
+            includeObservations: options.includeObservations
+        });
+        const jsonData = typeof result.data === 'string' ? result.data : JSON.stringify(result.data, null, 2);
+        if (options.output) {
+            fs.writeFileSync(options.output, jsonData);
+            console.log(chalk_1.default.green(`✓ Exported to ${options.output}`));
+        }
+        else {
+            console.log(jsonData);
+        }
+        await cli.close();
+    }
+    catch (error) {
+        handleError(error);
+    }
+});
+exportCmd
+    .command('markdown')
+    .description('Export as Markdown')
+    .option('-o, --output <file>', 'Output file')
+    .action(async (options) => {
+    try {
+        await cli.init();
+        const result = await cli.exportMemory('markdown');
+        if (options.output) {
+            fs.writeFileSync(options.output, result.data);
+            console.log(chalk_1.default.green(`✓ Exported to ${options.output}`));
+        }
+        else {
+            console.log(result.data);
+        }
+        await cli.close();
+    }
+    catch (error) {
+        handleError(error);
+    }
+});
+exportCmd
+    .command('obsidian')
+    .description('Export as Obsidian ZIP')
+    .requiredOption('-o, --output <file>', 'Output ZIP file')
+    .action(async (options) => {
+    try {
+        await cli.init();
+        const result = await cli.exportMemory('obsidian');
+        // Obsidian export returns zipBuffer, not data
+        const buffer = result.zipBuffer || result.data;
+        if (!buffer) {
+            throw new Error('No buffer returned from export');
+        }
+        fs.writeFileSync(options.output, buffer);
+        console.log(chalk_1.default.green(`✓ Exported to ${options.output}`));
+        await cli.close();
+    }
+    catch (error) {
+        handleError(error);
+    }
+});
+const importCmd = program.command('import').description('Import memory');
+importCmd
+    .command('file')
+    .description('Import from file')
+    .requiredOption('-i, --input <file>', 'Input file')
+    .requiredOption('-f, --format <format>', 'Source format (cozo|mem0|memgpt|markdown)')
+    .option('-s, --strategy <strategy>', 'Merge strategy (skip|overwrite|merge)', 'skip')
+    .action(async (options) => {
+    try {
+        await cli.init();
+        const data = fs.readFileSync(options.input, 'utf-8');
+        const result = await cli.importMemory(data, options.format, {
+            mergeStrategy: options.strategy
+        });
+        formatOutput(result, 'pretty');
+        await cli.close();
+    }
+    catch (error) {
+        handleError(error);
+    }
+});
+// Ingest commands
+const ingest = program.command('ingest').description('Ingest files');
+ingest
+    .command('file')
+    .description('Ingest file into entity')
+    .requiredOption('-i, --entity-id <id>', 'Entity ID')
+    .requiredOption('-p, --path <path>', 'File path')
+    .requiredOption('-f, --format <format>', 'File format (markdown|json|pdf)')
+    .option('-c, --chunking <type>', 'Chunking type (none|paragraphs)', 'paragraphs')
+    .option('-m, --max <number>', 'Max observations', parseInt)
+    .option('--no-deduplicate', 'Disable deduplication')
+    .action(async (options) => {
+    try {
+        await cli.init();
+        const result = await cli.ingestFile(options.entityId, options.format, options.path, undefined, {
+            chunking: options.chunking,
+            maxObservations: options.max,
+            deduplicate: options.deduplicate
+        });
+        formatOutput(result, 'pretty');
+        await cli.close();
+    }
+    catch (error) {
+        handleError(error);
+    }
+});
+program.parse();

package/dist/download-model.js

@@ -33,12 +33,14 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
+require("dotenv/config"); // Load .env file first
 const transformers_1 = require("@xenova/transformers");
 const path = __importStar(require("path"));
 // Configure cache path
 const CACHE_DIR = path.resolve('./.cache');
 transformers_1.env.cacheDir = CACHE_DIR;
-const MODEL_ID = "Xenova/bge-m3";
+// Read model from environment variable or use default
+const MODEL_ID = process.env.EMBEDDING_MODEL || "Xenova/bge-m3";
 async function downloadModel() {
     console.log(`Downloading FP32 model for ${MODEL_ID}...`);
     // quantized: false forces FP32 model download

package/dist/embedding-service.js

@@ -34,6 +34,7 @@ var __importStar = (this && this.__importStar) || (function () {
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.EmbeddingService = void 0;
+require("dotenv/config"); // Load .env file first
 const transformers_1 = require("@xenova/transformers");
 const ort = require('onnxruntime-node');
 const path = __importStar(require("path"));
@@ -91,11 +92,27 @@ class EmbeddingService {
     cache;
     session = null;
     tokenizer = null;
-    modelId = "Xenova/bge-m3";
-    dimensions = 1024;
+    modelId;
+    dimensions;
     queue = Promise.resolve();
     constructor() {
         this.cache = new LRUCache(1000, 3600000); // 1000 entries, 1h TTL
+        // Support multiple embedding models via environment variable
+        this.modelId = process.env.EMBEDDING_MODEL || "Xenova/bge-m3";
+        // Set dimensions based on model
+        const dimensionMap = {
+            "Xenova/bge-m3": 1024,
+            "Xenova/all-MiniLM-L6-v2": 384,
+            "Xenova/bge-small-en-v1.5": 384,
+            "Xenova/nomic-embed-text-v1": 768,
+            "onnx-community/Qwen3-Embedding-0.6B-ONNX": 1024,
+        };
+        this.dimensions = dimensionMap[this.modelId] || 1024;
+        console.error(`[EmbeddingService] Using model: ${this.modelId} (${this.dimensions} dimensions)`);
+    }
+    // Public getter for dimensions
+    getDimensions() {
+        return this.dimensions;
     }
     // Serializes embedding execution to avoid event loop blocking
     async runSerialized(task) {
@@ -109,21 +126,38 @@ class EmbeddingService {
         if (this.session && this.tokenizer)
             return;
         try {
-            // 1. Load Tokenizer
+            // 1. Check if model needs to be downloaded
+            // Extract namespace and model name from modelId (e.g., "Xenova/bge-m3" or "onnx-community/Qwen3-Embedding-0.6B-ONNX")
+            const parts = this.modelId.split('/');
+            const namespace = parts[0];
+            const modelName = parts[1];
+            // Try both possible cache locations
+            let baseDir = path.join(transformers_1.env.cacheDir, namespace, modelName, 'onnx');
+            let fp32Path = path.join(baseDir, 'model.onnx');
+            let quantizedPath = path.join(baseDir, 'model_quantized.onnx');
+            // If ONNX model files don't exist, download them
+            if (!fs.existsSync(fp32Path) && !fs.existsSync(quantizedPath)) {
+                console.log(`[EmbeddingService] Model not found, downloading ${this.modelId}...`);
+                console.log(`[EmbeddingService] This may take a few minutes on first run.`);
+                // Import AutoModel dynamically to trigger download
+                const { AutoModel } = await import("@xenova/transformers");
+                await AutoModel.from_pretrained(this.modelId, { quantized: false });
+                console.log(`[EmbeddingService] Model download completed.`);
+            }
+            // 2. Load Tokenizer
             if (!this.tokenizer) {
                 this.tokenizer = await transformers_1.AutoTokenizer.from_pretrained(this.modelId);
             }
-            // 2. Determine model path
-            const baseDir = path.join(transformers_1.env.cacheDir, 'Xenova', 'bge-m3', 'onnx');
+            // 3. Determine model path
             // Priority: FP32 (model.onnx) > Quantized (model_quantized.onnx)
-            let modelPath = path.join(baseDir, 'model.onnx');
+            let modelPath = fp32Path;
             if (!fs.existsSync(modelPath)) {
-                modelPath = path.join(baseDir, 'model_quantized.onnx');
+                modelPath = quantizedPath;
             }
             if (!fs.existsSync(modelPath)) {
-                throw new Error(`Model file not found at: ${modelPath}`);
+                throw new Error(`Model file not found at: ${modelPath}. Download may have failed.`);
             }
-            // 3. Create Session
+            // 4. Create Session
             if (!this.session) {
                 const options = {
                     executionProviders: ['cpu'], // Use CPU backend to avoid native conflicts
@@ -139,7 +173,15 @@ class EmbeddingService {
     }
     async embed(text) {
         return this.runSerialized(async () => {
-            const textStr = String(text || "");
+            let textStr = String(text || "");
+            // For Qwen3-Embedding models, add instruction prefix for better results
+            // (only for queries, not for documents being indexed)
+            if (this.modelId.includes('Qwen3-Embedding')) {
+                // Add instruction prefix if not already present
+                if (!textStr.startsWith('Instruct:')) {
+                    textStr = `Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery: ${textStr}`;
+                }
+            }
             // 1. Cache lookup
             const cached = this.cache.get(textStr);
             if (cached) {
@@ -171,14 +213,22 @@ class EmbeddingService {
                 const results = await this.session.run(feeds);
                 // 5. Pooling & Normalization
                 // Output name usually 'last_hidden_state' or 'logits'
-                // For BGE-M3, the first output is usually the hidden states [batch, seq_len, hidden_size]
                 const outputName = this.session.outputNames[0];
                 const outputTensor = results[outputName];
                 // Ensure we have data
                 if (!outputTensor || !attentionMaskData) {
                     throw new Error("No output data or attention mask available");
                 }
-                const embedding = this.meanPooling(outputTensor.data, attentionMaskData, outputTensor.dims);
+                // Choose pooling strategy based on model
+                let embedding;
+                if (this.modelId.includes('Qwen3-Embedding')) {
+                    // Qwen3-Embedding uses last token pooling
+                    embedding = this.lastTokenPooling(outputTensor.data, attentionMaskData, outputTensor.dims);
+                }
+                else {
+                    // BGE and other models use mean pooling
+                    embedding = this.meanPooling(outputTensor.data, attentionMaskData, outputTensor.dims);
+                }
                 // Normalize
                 const normalized = this.normalize(embedding);
                 this.cache.set(textStr, normalized);
@@ -200,6 +250,25 @@ class EmbeddingService {
         }
         return results;
     }
+    lastTokenPooling(data, attentionMask, dims) {
+        // dims: [batch_size, seq_len, hidden_size]
+        // Extract the last valid token's hidden state
+        const [batchSize, seqLen, hiddenSize] = dims;
+        // Find last valid token position
+        let lastValidIdx = seqLen - 1;
+        for (let i = seqLen - 1; i >= 0; i--) {
+            if (attentionMask[i] === 1n) {
+                lastValidIdx = i;
+                break;
+            }
+        }
+        // Extract embedding at last valid position
+        const embedding = new Float32Array(hiddenSize);
+        for (let j = 0; j < hiddenSize; j++) {
+            embedding[j] = data[lastValidIdx * hiddenSize + j];
+        }
+        return Array.from(embedding);
+    }
     meanPooling(data, attentionMask, dims) {
         // dims: [batch_size, seq_len, hidden_size]
         // We assume batch_size = 1 for single embedding call

package/dist/hybrid-search.js

@@ -184,7 +184,7 @@ class HybridSearch {
         console.error('--- DEBUG: Cozo Datalog Query ---');
         console.error(datalogQuery);
         console.error('--- DEBUG: Params ---');
-        console.dir(params, { depth: null });
+        console.error(JSON.stringify(params, null, 2));
         try {
             const results = await this.db.run(datalogQuery, params);
             let searchResults = results.rows.map((r) => ({
@@ -234,8 +234,13 @@ class HybridSearch {
         const { limit: queryLimit = 10, filters, graphConstraints, vectorParams } = options;
         // @ts-ignore
         const { topk = 5, efSearch = 50 } = vectorParams || {};
-        // Fallback Mock
-        return [];
+        // Use advancedSearch as the default implementation
+        return this.advancedSearch({
+            ...options,
+            vectorParams: {
+                efSearch: 100
+            }
+        });
     }
     async graphRag(options) {
         console.error("[HybridSearch] Starting graphRag with options:", JSON.stringify(options, null, 2));