cozo-memory 1.0.4 → 1.0.6

package/dist/index.js

@@ -12,12 +12,12 @@ const cozo_node_1 = require("cozo-node");
 const zod_1 = require("zod");
 const uuid_1 = require("uuid");
 const path_1 = __importDefault(require("path"));
+const fs_1 = __importDefault(require("fs"));
+const pdf_mjs_1 = require("pdfjs-dist/legacy/build/pdf.mjs");
 const hybrid_search_1 = require("./hybrid-search");
 const inference_engine_1 = require("./inference-engine");
 exports.DB_PATH = path_1.default.resolve(__dirname, "..", "memory_db.cozo");
 const DB_ENGINE = process.env.DB_ENGINE || "sqlite"; // "sqlite" or "rocksdb"
-const EMBEDDING_MODEL = "Xenova/bge-m3";
-const EMBEDDING_DIM = 1024;
 exports.USER_ENTITY_ID = "global_user_profile";
 exports.USER_ENTITY_NAME = "The User";
 exports.USER_ENTITY_TYPE = "User";
@@ -484,6 +484,9 @@ class MemoryServer {
     async setupSchema() {
         try {
             console.error("[Schema] Initializing schema...");
+            // Get embedding dimensions from service
+            const EMBEDDING_DIM = this.embeddingService.getDimensions();
+            console.error(`[Schema] Using embedding dimensions: ${EMBEDDING_DIM}`);
             const existingRelations = await this.db.run("::relations");
             const relations = existingRelations.rows.map((r) => r[0]);
             // Entity Table
@@ -1029,7 +1032,8 @@ class MemoryServer {
         [$id, [${now}, true], $name, $type, $embedding, $name_embedding, $metadata]
       ] :insert entity {id, created_at => name, type, embedding, name_embedding, metadata}
     `, { id, name, type, embedding, name_embedding, metadata: metadata || {} });
-        return { id, name, type, status: "Entity created" };
+        const created_at_iso = new Date(Math.floor(now / 1000)).toISOString();
+        return { id, name, type, created_at: now, created_at_iso, status: "Entity created" };
     }
     async initUserProfile() {
         try {
@@ -1188,9 +1192,12 @@ class MemoryServer {
             // Optional: Automatic inference after new observation (in background)
             const suggestionsRaw = await this.inferenceEngine.inferRelations(entityId);
             const suggestions = await this.formatInferredRelationsForContext(suggestionsRaw);
+            const created_at_iso = new Date(Math.floor(now / 1000)).toISOString();
             return {
                 id,
                 entity_id: entityId,
+                created_at: now,
+                created_at_iso,
                 status: "Observation saved",
                 inferred_suggestions: suggestions
             };
@@ -1244,7 +1251,15 @@ class MemoryServer {
             strength: args.strength ?? 1.0,
             metadata: args.metadata || {}
         });
-        return { status: "Relationship created" };
+        const created_at_iso = new Date(Math.floor(now / 1000)).toISOString();
+        return {
+            from_id: args.from_id,
+            to_id: args.to_id,
+            relation_type: args.relation_type,
+            created_at: now,
+            created_at_iso,
+            status: "Relationship created"
+        };
     }
     async exploreGraph(args) {
         await this.initPromise;
@@ -1664,7 +1679,39 @@ ids[id] <- $ids
     async ingestFile(args) {
         await this.initPromise;
         try {
-            const content = (args.content ?? "").trim();
+            // Check that either file_path or content is provided
+            if (!args.file_path && !args.content) {
+                return { error: "Either file_path or content must be provided" };
+            }
+            // Read content from file if file_path is provided
+            let content;
+            if (args.file_path) {
+                try {
+                    if (args.format === "pdf") {
+                        // Read PDF file and extract text using pdfjs-dist
+                        const data = new Uint8Array(fs_1.default.readFileSync(args.file_path));
+                        const loadingTask = (0, pdf_mjs_1.getDocument)({ data });
+                        const pdf = await loadingTask.promise;
+                        const numPages = pdf.numPages;
+                        const pageTextPromises = Array.from({ length: numPages }, async (_, i) => {
+                            const page = await pdf.getPage(i + 1);
+                            const textContent = await page.getTextContent();
+                            return textContent.items.map((item) => item.str).join(' ');
+                        });
+                        const pageTexts = await Promise.all(pageTextPromises);
+                        content = pageTexts.join('\n').trim();
+                    }
+                    else {
+                        content = fs_1.default.readFileSync(args.file_path, 'utf-8').trim();
+                    }
+                }
+                catch (error) {
+                    return { error: `Failed to read file: ${error.message}` };
+                }
+            }
+            else {
+                content = (args.content ?? "").trim();
+            }
             if (!content)
                 return { error: "Content must not be empty" };
             let entityId = undefined;
@@ -1701,7 +1748,7 @@ ids[id] <- $ids
             const deduplicate = args.deduplicate ?? true;
             const chunking = args.chunking ?? "none";
             const observations = [];
-            if (args.format === "markdown") {
+            if (args.format === "markdown" || args.format === "pdf") {
                 if (chunking === "paragraphs") {
                     const parts = content
                         .split(/\r?\n\s*\r?\n+/g)
@@ -2230,9 +2277,10 @@ ids[id] <- $ids
                 entity_id: zod_1.z.string().optional().describe("ID of the target entity"),
                 entity_name: zod_1.z.string().optional().describe("Name of the target entity (will be created if not exists)"),
                 entity_type: zod_1.z.string().optional().default("Document").describe("Type of the target entity (only when creating)"),
-                format: zod_1.z.enum(["markdown", "json"]).describe("Input format"),
+                format: zod_1.z.enum(["markdown", "json", "pdf"]).describe("Input format"),
                 chunking: zod_1.z.enum(["none", "paragraphs"]).optional().default("none").describe("Chunking for Markdown"),
-                content: zod_1.z.string().describe("File content (or LLM summary)"),
+                file_path: zod_1.z.string().optional().describe("Path to file on disk (alternative to content parameter)"),
+                content: zod_1.z.string().optional().describe("File content (or LLM summary) - required if file_path not provided"),
                 metadata: MetadataSchema.optional().describe("Metadata for entity creation"),
                 observation_metadata: MetadataSchema.optional().describe("Metadata applied to all observations"),
                 deduplicate: zod_1.z.boolean().optional().default(true).describe("Skip exact duplicates"),
@@ -2240,6 +2288,9 @@ ids[id] <- $ids
             }).refine((v) => Boolean(v.entity_id) || Boolean(v.entity_name), {
                 message: "entity_id or entity_name is required for ingest_file",
                 path: ["entity_id"],
+            }).refine((v) => Boolean(v.file_path) || Boolean(v.content), {
+                message: "file_path or content is required for ingest_file",
+                path: ["file_path"],
             }),
         ]);
         const MutateMemoryParameters = zod_1.z.object({
@@ -2254,9 +2305,10 @@ ids[id] <- $ids
             entity_type: zod_1.z.string().optional().describe("Only when entity_name is used and entity is created new"),
             text: zod_1.z.string().optional().describe("For add_observation (required)"),
             datalog: zod_1.z.string().optional().describe("For add_inference_rule (required)"),
-            format: zod_1.z.enum(["markdown", "json"]).optional().describe("For ingest_file (required)"),
+            format: zod_1.z.enum(["markdown", "json", "pdf"]).optional().describe("For ingest_file (required)"),
             chunking: zod_1.z.enum(["none", "paragraphs"]).optional().describe("Optional for ingest_file (for markdown)"),
-            content: zod_1.z.string().optional().describe("For ingest_file (required)"),
+            file_path: zod_1.z.string().optional().describe("For ingest_file - path to file on disk (alternative to content)"),
+            content: zod_1.z.string().optional().describe("For ingest_file - file content (required if file_path not provided)"),
             observation_metadata: MetadataSchema.optional().describe("Optional for ingest_file"),
             deduplicate: zod_1.z.boolean().optional().describe("Optional for ingest_file and add_observation"),
             max_observations: zod_1.z.number().optional().describe("Optional for ingest_file"),

package/dist/memory-service.js

@@ -1,7 +1,42 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MemoryService = void 0;
 const uuid_1 = require("uuid");
+const pdf_mjs_1 = require("pdfjs-dist/legacy/build/pdf.mjs");
+const fs = __importStar(require("fs"));
 class MemoryService {
     db;
     embeddings;
@@ -169,7 +204,7 @@ class MemoryService {
         console.error('[MemoryService] Snapshot created:', snapshotId, stats);
         return snapshotId;
     }
-    async ingestFile(content, format, entityName, entityType = 'Document', chunking = 'paragraphs') {
+    async ingestFile(content, format, entityName, entityType = 'Document', chunking = 'paragraphs', filePath) {
         const searchResults = await this.search(entityName, 1);
         let entity;
         if (searchResults.length > 0 && searchResults[0].entity.name.toLowerCase() === entityName.toLowerCase()) {
@@ -179,16 +214,64 @@ class MemoryService {
             entity = await this.createEntity(entityName, entityType, { format: format });
         }
         let chunks = [];
-        if (format === 'markdown' && chunking === 'paragraphs') {
-            chunks = content.split(/\n\s*\n/).filter((c) => c.trim().length > 0);
+        if (format === 'pdf') {
+            try {
+                let data;
+                // If filePath is provided, read from file
+                if (filePath) {
+                    data = new Uint8Array(fs.readFileSync(filePath));
+                }
+                else {
+                    // Otherwise, assume content is base64
+                    const buffer = Buffer.from(content, 'base64');
+                    data = new Uint8Array(buffer);
+                }
+                const loadingTask = (0, pdf_mjs_1.getDocument)({ data });
+                const pdf = await loadingTask.promise;
+                const numPages = pdf.numPages;
+                const pageTextPromises = Array.from({ length: numPages }, async (_, i) => {
+                    const page = await pdf.getPage(i + 1);
+                    const textContent = await page.getTextContent();
+                    return textContent.items.map((item) => item.str).join(' ');
+                });
+                const pageTexts = await Promise.all(pageTextPromises);
+                const text = pageTexts.join('\n');
+                if (chunking === 'paragraphs') {
+                    chunks = text.split(/\n\s*\n/).filter((c) => c.trim().length > 0);
+                }
+                else {
+                    chunks = [text];
+                }
+            }
+            catch (e) {
+                console.error('[MemoryService] PDF parsing error:', e);
+                throw new Error(`Failed to parse PDF: ${e instanceof Error ? e.message : String(e)}`);
+            }
+        }
+        else if (format === 'markdown') {
+            // For markdown, also support file path
+            let textContent = content;
+            if (filePath) {
+                textContent = fs.readFileSync(filePath, 'utf-8');
+            }
+            if (chunking === 'paragraphs') {
+                chunks = textContent.split(/\n\s*\n/).filter((c) => c.trim().length > 0);
+            }
+            else {
+                chunks = [textContent];
+            }
         }
         else if (format === 'json') {
+            let textContent = content;
+            if (filePath) {
+                textContent = fs.readFileSync(filePath, 'utf-8');
+            }
             try {
-                const data = JSON.parse(content);
+                const data = JSON.parse(textContent);
                 chunks = [JSON.stringify(data, null, 2)];
             }
             catch (e) {
-                chunks = [content];
+                chunks = [textContent];
             }
         }
         else {

package/dist/temporal-normalizer.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/test-hybrid-debug.js

@@ -0,0 +1,52 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const index_1 = require("./index");
+async function debugHybridSearch() {
+    console.log('Initializing MemoryServer...');
+    const server = new index_1.MemoryServer();
+    await server.initPromise;
+    // Check database content
+    console.log('\n=== Database Content ===');
+    const entities = await server.db.run('?[id, name, type] := *entity{id, name, type, @ "NOW"}');
+    console.log(`Entities: ${entities.rows.length}`);
+    entities.rows.slice(0, 3).forEach((row) => {
+        console.log(`  - ${row[1]} (${row[2]})`);
+    });
+    const observations = await server.db.run('?[id, text] := *observation{id, text, @ "NOW"}');
+    console.log(`\nObservations: ${observations.rows.length}`);
+    observations.rows.slice(0, 3).forEach((row) => {
+        console.log(`  - ${row[1].substring(0, 60)}...`);
+    });
+    // Test embedding
+    console.log('\n=== Test Embedding ===');
+    const testEmbedding = await server.embeddingService.embed('Alice');
+    console.log(`Embedding dimensions: ${testEmbedding.length}`);
+    console.log(`First 5 values: ${testEmbedding.slice(0, 5)}`);
+    // Test HNSW search directly
+    console.log('\n=== Test HNSW Search Directly ===');
+    try {
+        const hnswQuery = `
+      ?[id, name, type, dist] := 
+        ~entity:name_semantic{id | query: vec([${testEmbedding.join(',')}]), k: 5, bind_distance: dist},
+        *entity{id, name, type, @ "NOW"}
+    `;
+        const hnswResult = await server.db.run(hnswQuery);
+        console.log(`HNSW results: ${hnswResult.rows.length}`);
+        hnswResult.rows.forEach((row) => {
+            console.log(`  - ${row[1]} (${row[2]}) - distance: ${row[3]}`);
+        });
+    }
+    catch (e) {
+        console.error('HNSW error:', e.message);
+    }
+    // Test hybridSearch
+    console.log('\n=== Test HybridSearch ===');
+    const searchResult = await server.hybridSearch.search({
+        query: 'Alice',
+        limit: 5,
+        includeEntities: true,
+        includeObservations: true
+    });
+    console.log('HybridSearch result:', JSON.stringify(searchResult, null, 2));
+}
+debugHybridSearch().catch(console.error);

package/dist/test-mcp-search.js

@@ -0,0 +1,47 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const index_1 = require("./index");
+async function testMCPSearch() {
+    console.log('Initializing MemoryServer...');
+    const server = new index_1.MemoryServer();
+    await server.initPromise;
+    console.log('\n=== Test 1: Create Entity via MCP mutate_memory ===');
+    const createResult = await server.mutateMemory({
+        action: 'create_entity',
+        name: 'Test Search Entity',
+        type: 'test',
+        metadata: { purpose: 'search_test' }
+    });
+    console.log('Created:', createResult);
+    const entityId = createResult.id;
+    console.log('\n=== Test 2: Add Observation via MCP mutate_memory ===');
+    const obsResult = await server.mutateMemory({
+        action: 'add_observation',
+        entity_id: entityId,
+        text: 'This is a test observation for searching with keywords like authentication and OAuth',
+        metadata: { test: true }
+    });
+    console.log('Added observation:', obsResult);
+    console.log('\n=== Test 3: Search via MCP query_memory ===');
+    const searchResult = await server.queryMemory({
+        action: 'search',
+        query: 'authentication',
+        limit: 5
+    });
+    console.log('Search result:', JSON.stringify(searchResult, null, 2));
+    console.log('\n=== Test 4: Search for "Alice" ===');
+    const aliceResult = await server.queryMemory({
+        action: 'search',
+        query: 'Alice',
+        limit: 5
+    });
+    console.log('Alice search result:', JSON.stringify(aliceResult, null, 2));
+    console.log('\n=== Test 5: Search for "TypeScript" ===');
+    const tsResult = await server.queryMemory({
+        action: 'search',
+        query: 'TypeScript',
+        limit: 5
+    });
+    console.log('TypeScript search result:', JSON.stringify(tsResult, null, 2));
+}
+testMCPSearch().catch(console.error);

package/dist/test-pdf-ingest.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/test-qwen3-bilingual.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/test-search-simple.js

@@ -0,0 +1,27 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const index_1 = require("./index");
+async function testSearch() {
+    console.log('Initializing MemoryServer...');
+    const server = new index_1.MemoryServer();
+    await server.initPromise;
+    console.log('\n=== Testing Search ===');
+    // Test simple search
+    const result = await server.hybridSearch.search({
+        query: 'Alice',
+        limit: 5,
+        includeEntities: true,
+        includeObservations: true
+    });
+    console.log('Search result:', JSON.stringify(result, null, 2));
+    // Check if result is array or object
+    console.log('\nResult type:', typeof result);
+    console.log('Is array:', Array.isArray(result));
+    if (Array.isArray(result)) {
+        console.log(`Found ${result.length} results`);
+    }
+    else {
+        console.log('Result keys:', Object.keys(result));
+    }
+}
+testSearch().catch(console.error);

package/dist/timestamp-utils.js

@@ -0,0 +1,44 @@
+"use strict";
+/**
+ * Utility functions for timestamp handling
+ * Provides both Unix microsecond timestamps and ISO 8601 strings
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.toDualTimestamp = toDualTimestamp;
+exports.nowDual = nowDual;
+exports.parseToDual = parseToDual;
+/**
+ * Convert CozoDB microsecond timestamp to dual format
+ */
+function toDualTimestamp(microseconds) {
+    const milliseconds = Math.floor(microseconds / 1000);
+    const date = new Date(milliseconds);
+    return {
+        timestamp: microseconds,
+        iso: date.toISOString()
+    };
+}
+/**
+ * Get current time in dual format
+ */
+function nowDual() {
+    const now = Date.now();
+    return {
+        timestamp: now * 1000, // Convert to microseconds
+        iso: new Date(now).toISOString()
+    };
+}
+/**
+ * Parse ISO string or Unix timestamp to dual format
+ */
+function parseToDual(input) {
+    if (typeof input === 'number') {
+        return toDualTimestamp(input);
+    }
+    const date = new Date(input);
+    const milliseconds = date.getTime();
+    return {
+        timestamp: milliseconds * 1000, // Convert to microseconds
+        iso: date.toISOString()
+    };
+}