cozo-memory 1.0.3 → 1.0.5

package/dist/index.js

@@ -6,17 +6,18 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MemoryServer = exports.USER_ENTITY_TYPE = exports.USER_ENTITY_NAME = exports.USER_ENTITY_ID = exports.DB_PATH = void 0;
 const embedding_service_1 = require("./embedding-service");
+const export_import_service_1 = require("./export-import-service");
 const fastmcp_1 = require("fastmcp");
 const cozo_node_1 = require("cozo-node");
 const zod_1 = require("zod");
 const uuid_1 = require("uuid");
 const path_1 = __importDefault(require("path"));
+const fs_1 = __importDefault(require("fs"));
+const pdf_mjs_1 = require("pdfjs-dist/legacy/build/pdf.mjs");
 const hybrid_search_1 = require("./hybrid-search");
 const inference_engine_1 = require("./inference-engine");
 exports.DB_PATH = path_1.default.resolve(__dirname, "..", "memory_db.cozo");
 const DB_ENGINE = process.env.DB_ENGINE || "sqlite"; // "sqlite" or "rocksdb"
-const EMBEDDING_MODEL = "Xenova/bge-m3";
-const EMBEDDING_DIM = 1024;
 exports.USER_ENTITY_ID = "global_user_profile";
 exports.USER_ENTITY_NAME = "The User";
 exports.USER_ENTITY_TYPE = "User";
@@ -27,6 +28,47 @@ class MemoryServer {
     hybridSearch;
     inferenceEngine;
     initPromise;
+    // Metrics tracking
+    metrics = {
+        operations: {
+            create_entity: 0,
+            update_entity: 0,
+            delete_entity: 0,
+            add_observation: 0,
+            create_relation: 0,
+            search: 0,
+            graph_operations: 0
+        },
+        errors: {
+            total: 0,
+            by_operation: {}
+        },
+        performance: {
+            last_operation_ms: 0,
+            avg_operation_ms: 0,
+            total_operations: 0
+        }
+    };
+    trackOperation(operation, startTime) {
+        const duration = Date.now() - startTime;
+        this.metrics.performance.last_operation_ms = duration;
+        this.metrics.performance.total_operations++;
+        // Calculate running average
+        const prevAvg = this.metrics.performance.avg_operation_ms;
+        const n = this.metrics.performance.total_operations;
+        this.metrics.performance.avg_operation_ms = (prevAvg * (n - 1) + duration) / n;
+        // Track operation count
+        if (operation in this.metrics.operations) {
+            this.metrics.operations[operation]++;
+        }
+    }
+    trackError(operation) {
+        this.metrics.errors.total++;
+        if (!this.metrics.errors.by_operation[operation]) {
+            this.metrics.errors.by_operation[operation] = 0;
+        }
+        this.metrics.errors.by_operation[operation]++;
+    }
     constructor(dbPath = exports.DB_PATH) {
         const fullDbPath = DB_ENGINE === "sqlite" ? dbPath + ".db" : dbPath;
         this.db = new cozo_node_1.CozoDb(DB_ENGINE, fullDbPath);
@@ -442,6 +484,9 @@ class MemoryServer {
     async setupSchema() {
         try {
             console.error("[Schema] Initializing schema...");
+            // Get embedding dimensions from service
+            const EMBEDDING_DIM = this.embeddingService.getDimensions();
+            console.error(`[Schema] Using embedding dimensions: ${EMBEDDING_DIM}`);
             const existingRelations = await this.db.run("::relations");
             const relations = existingRelations.rows.map((r) => r[0]);
             // Entity Table
@@ -934,6 +979,7 @@ class MemoryServer {
         });
     }
     async createEntity(args) {
+        const startTime = Date.now();
         try {
             if (!args.name || args.name.trim() === "") {
                 return { error: "Entity name must not be empty" };
@@ -957,9 +1003,12 @@ class MemoryServer {
                 }
             }
             const id = (0, uuid_1.v4)();
-            return this.createEntityWithId(id, args.name, args.type, args.metadata);
+            const result = await this.createEntityWithId(id, args.name, args.type, args.metadata);
+            this.trackOperation('create_entity', startTime);
+            return result;
         }
         catch (error) {
+            this.trackError('create_entity');
             console.error("Error in create_entity:", error);
             if (error.display) {
                 console.error("CozoDB Error Details:", error.display);
@@ -1168,6 +1217,28 @@ class MemoryServer {
         if (toEntity.rows.length === 0) {
             return { error: `Target entity with ID '${args.to_id}' not found` };
         }
+        // FIX: Check for duplicate relations
+        try {
+            const existingRel = await this.db.run('?[from_id, to_id, relation_type, strength, metadata] := *relationship{from_id, to_id, relation_type, strength, metadata, @ "NOW"}, from_id = $from, to_id = $to, relation_type = $type', { from: args.from_id, to: args.to_id, type: args.relation_type });
+            if (existingRel.rows.length > 0) {
+                // Update existing relation instead of creating duplicate
+                const now = Date.now() * 1000;
+                await this.db.run(`
+          ?[from_id, to_id, relation_type, created_at, strength, metadata] <- [[$from_id, $to_id, $relation_type, [${now}, true], $strength, $metadata]] 
+          :put relationship {from_id, to_id, relation_type, created_at => strength, metadata}
+        `, {
+                    from_id: args.from_id,
+                    to_id: args.to_id,
+                    relation_type: args.relation_type,
+                    strength: args.strength ?? 1.0,
+                    metadata: args.metadata || {}
+                });
+                return { status: "Relationship updated (was duplicate)" };
+            }
+        }
+        catch (e) {
+            console.error('[Relation] Duplicate check failed, proceeding with insert:', e);
+        }
         const now = Date.now() * 1000;
         await this.db.run(`?[from_id, to_id, relation_type, created_at, strength, metadata] <- [[$from_id, $to_id, $relation_type, [${now}, true], $strength, $metadata]] :insert relationship {from_id, to_id, relation_type, created_at => strength, metadata}`, {
             from_id: args.from_id,
@@ -1596,7 +1667,39 @@ ids[id] <- $ids
     async ingestFile(args) {
         await this.initPromise;
         try {
-            const content = (args.content ?? "").trim();
+            // Check that either file_path or content is provided
+            if (!args.file_path && !args.content) {
+                return { error: "Either file_path or content must be provided" };
+            }
+            // Read content from file if file_path is provided
+            let content;
+            if (args.file_path) {
+                try {
+                    if (args.format === "pdf") {
+                        // Read PDF file and extract text using pdfjs-dist
+                        const data = new Uint8Array(fs_1.default.readFileSync(args.file_path));
+                        const loadingTask = (0, pdf_mjs_1.getDocument)({ data });
+                        const pdf = await loadingTask.promise;
+                        const numPages = pdf.numPages;
+                        const pageTextPromises = Array.from({ length: numPages }, async (_, i) => {
+                            const page = await pdf.getPage(i + 1);
+                            const textContent = await page.getTextContent();
+                            return textContent.items.map((item) => item.str).join(' ');
+                        });
+                        const pageTexts = await Promise.all(pageTextPromises);
+                        content = pageTexts.join('\n').trim();
+                    }
+                    else {
+                        content = fs_1.default.readFileSync(args.file_path, 'utf-8').trim();
+                    }
+                }
+                catch (error) {
+                    return { error: `Failed to read file: ${error.message}` };
+                }
+            }
+            else {
+                content = (args.content ?? "").trim();
+            }
             if (!content)
                 return { error: "Content must not be empty" };
             let entityId = undefined;
@@ -1633,7 +1736,7 @@ ids[id] <- $ids
             const deduplicate = args.deduplicate ?? true;
             const chunking = args.chunking ?? "none";
             const observations = [];
-            if (args.format === "markdown") {
+            if (args.format === "markdown" || args.format === "pdf") {
                 if (chunking === "paragraphs") {
                     const parts = content
                         .split(/\r?\n\s*\r?\n+/g)
@@ -1707,23 +1810,54 @@ ids[id] <- $ids
         }
     }
     async deleteEntity(args) {
+        const startTime = Date.now();
         try {
+            console.error(`[Delete] Starting deletion for entity: ${args.entity_id}`);
             // 1. Check if entity exists
             const entityRes = await this.db.run('?[name] := *entity{id: $id, name, @ "NOW"}', { id: args.entity_id });
             if (entityRes.rows.length === 0) {
+                console.error(`[Delete] Entity not found: ${args.entity_id}`);
                 return { error: `Entity with ID '${args.entity_id}' not found` };
             }
-            // 2. Delete all related data in a transaction (block)
+            console.error(`[Delete] Entity found: ${entityRes.rows[0][0]}`);
+            // 2. Count related data before deletion
+            const obsCount = await this.db.run('?[count(id)] := *observation{id, entity_id, @ "NOW"}, entity_id = $id', { id: args.entity_id });
+            const relOutCount = await this.db.run('?[count(from_id)] := *relationship{from_id, to_id, @ "NOW"}, from_id = $id', { id: args.entity_id });
+            const relInCount = await this.db.run('?[count(to_id)] := *relationship{from_id, to_id, @ "NOW"}, to_id = $id', { id: args.entity_id });
+            console.error(`[Delete] Related data: ${obsCount.rows[0][0]} observations, ${relOutCount.rows[0][0]} outgoing relations, ${relInCount.rows[0][0]} incoming relations`);
+            // 3. Delete all related data in a transaction (block)
+            console.error(`[Delete] Executing deletion transaction...`);
             await this.db.run(`
         { ?[id, created_at] := *observation{id, entity_id, created_at}, entity_id = $target_id :rm observation {id, created_at} }
         { ?[from_id, to_id, relation_type, created_at] := *relationship{from_id, to_id, relation_type, created_at}, from_id = $target_id :rm relationship {from_id, to_id, relation_type, created_at} }
         { ?[from_id, to_id, relation_type, created_at] := *relationship{from_id, to_id, relation_type, created_at}, to_id = $target_id :rm relationship {from_id, to_id, relation_type, created_at} }
         { ?[id, created_at] := *entity{id, created_at}, id = $target_id :rm entity {id, created_at} }
       `, { target_id: args.entity_id });
-            return { status: "Entity and all related data deleted" };
+            // 4. Verify deletion
+            const verifyEntity = await this.db.run('?[id] := *entity{id, @ "NOW"}, id = $id', { id: args.entity_id });
+            const verifyObs = await this.db.run('?[count(id)] := *observation{id, entity_id, @ "NOW"}, entity_id = $id', { id: args.entity_id });
+            const verifyRelOut = await this.db.run('?[count(from_id)] := *relationship{from_id, @ "NOW"}, from_id = $id', { id: args.entity_id });
+            const verifyRelIn = await this.db.run('?[count(to_id)] := *relationship{to_id, @ "NOW"}, to_id = $id', { id: args.entity_id });
+            console.error(`[Delete] Verification: entity exists=${verifyEntity.rows.length > 0}, observations=${verifyObs.rows[0][0]}, outgoing_relations=${verifyRelOut.rows[0][0]}, incoming_relations=${verifyRelIn.rows[0][0]}`);
+            if (verifyEntity.rows.length > 0) {
+                console.error(`[Delete] WARNING: Entity still visible after deletion!`);
+            }
+            else {
+                console.error(`[Delete] ✓ Entity successfully deleted`);
+            }
+            this.trackOperation('delete_entity', startTime);
+            return {
+                status: "Entity and all related data deleted",
+                deleted: {
+                    observations: obsCount.rows[0][0],
+                    outgoing_relations: relOutCount.rows[0][0],
+                    incoming_relations: relInCount.rows[0][0]
+                }
+            };
         }
         catch (error) {
-            console.error("Error during deletion:", error);
+            console.error("[Delete] Error during deletion:", error);
+            this.trackError('delete_entity');
             return { error: "Deletion failed", message: error.message };
         }
     }
@@ -1865,6 +1999,26 @@ ids[id] <- $ids
                                 to_id = existingId;
                             }
                         }
+                        // FIX: Validate that both entities exist before creating relation
+                        try {
+                            const [fromExists, toExists] = await Promise.all([
+                                this.db.run('?[id, name] := *entity{id, name, @ "NOW"}, id = $id', { id: from_id }),
+                                this.db.run('?[id, name] := *entity{id, name, @ "NOW"}, id = $id', { id: to_id })
+                            ]);
+                            if (fromExists.rows.length === 0) {
+                                return { error: `Transaction failed: Source entity '${from_id}' not found in operation ${i}` };
+                            }
+                            if (toExists.rows.length === 0) {
+                                return { error: `Transaction failed: Target entity '${to_id}' not found in operation ${i}` };
+                            }
+                        }
+                        catch (e) {
+                            return { error: `Transaction failed: Entity validation error in operation ${i}: ${e.message}` };
+                        }
+                        // FIX: Check for self-reference
+                        if (from_id === to_id) {
+                            return { error: `Transaction failed: Self-references not allowed in operation ${i}` };
+                        }
                         const now = Date.now() * 1000;
                         allParams[`rel_from${suffix}`] = from_id;
                         allParams[`rel_to${suffix}`] = to_id;
@@ -1991,6 +2145,22 @@ ids[id] <- $ids
         await this.initPromise;
         return this.hybridSearch.graphRag(args);
     }
+    async exportMemory(args) {
+        await this.initPromise;
+        const dbService = { run: (query, params) => this.db.run(query, params) };
+        const exportService = new export_import_service_1.ExportImportService(dbService);
+        return exportService.exportMemory(args);
+    }
+    async importMemory(args) {
+        await this.initPromise;
+        const dbService = { run: (query, params) => this.db.run(query, params) };
+        const exportService = new export_import_service_1.ExportImportService(dbService);
+        return exportService.importMemory(args.data, {
+            sourceFormat: args.sourceFormat,
+            mergeStrategy: args.mergeStrategy,
+            defaultEntityType: args.defaultEntityType
+        });
+    }
     registerTools() {
         const MetadataSchema = zod_1.z.record(zod_1.z.string(), zod_1.z.any());
         const MutateMemorySchema = zod_1.z.discriminatedUnion("action", [
@@ -2095,9 +2265,10 @@ ids[id] <- $ids
                 entity_id: zod_1.z.string().optional().describe("ID of the target entity"),
                 entity_name: zod_1.z.string().optional().describe("Name of the target entity (will be created if not exists)"),
                 entity_type: zod_1.z.string().optional().default("Document").describe("Type of the target entity (only when creating)"),
-                format: zod_1.z.enum(["markdown", "json"]).describe("Input format"),
+                format: zod_1.z.enum(["markdown", "json", "pdf"]).describe("Input format"),
                 chunking: zod_1.z.enum(["none", "paragraphs"]).optional().default("none").describe("Chunking for Markdown"),
-                content: zod_1.z.string().describe("File content (or LLM summary)"),
+                file_path: zod_1.z.string().optional().describe("Path to file on disk (alternative to content parameter)"),
+                content: zod_1.z.string().optional().describe("File content (or LLM summary) - required if file_path not provided"),
                 metadata: MetadataSchema.optional().describe("Metadata for entity creation"),
                 observation_metadata: MetadataSchema.optional().describe("Metadata applied to all observations"),
                 deduplicate: zod_1.z.boolean().optional().default(true).describe("Skip exact duplicates"),
@@ -2105,6 +2276,9 @@ ids[id] <- $ids
             }).refine((v) => Boolean(v.entity_id) || Boolean(v.entity_name), {
                 message: "entity_id or entity_name is required for ingest_file",
                 path: ["entity_id"],
+            }).refine((v) => Boolean(v.file_path) || Boolean(v.content), {
+                message: "file_path or content is required for ingest_file",
+                path: ["file_path"],
             }),
         ]);
         const MutateMemoryParameters = zod_1.z.object({
@@ -2119,9 +2293,10 @@ ids[id] <- $ids
             entity_type: zod_1.z.string().optional().describe("Only when entity_name is used and entity is created new"),
             text: zod_1.z.string().optional().describe("For add_observation (required)"),
             datalog: zod_1.z.string().optional().describe("For add_inference_rule (required)"),
-            format: zod_1.z.enum(["markdown", "json"]).optional().describe("For ingest_file (required)"),
+            format: zod_1.z.enum(["markdown", "json", "pdf"]).optional().describe("For ingest_file (required)"),
             chunking: zod_1.z.enum(["none", "paragraphs"]).optional().describe("Optional for ingest_file (for markdown)"),
-            content: zod_1.z.string().optional().describe("For ingest_file (required)"),
+            file_path: zod_1.z.string().optional().describe("For ingest_file - path to file on disk (alternative to content)"),
+            content: zod_1.z.string().optional().describe("For ingest_file - file content (required if file_path not provided)"),
             observation_metadata: MetadataSchema.optional().describe("Optional for ingest_file"),
             deduplicate: zod_1.z.boolean().optional().describe("Optional for ingest_file and add_observation"),
             max_observations: zod_1.z.number().optional().describe("Optional for ingest_file"),
@@ -2881,6 +3056,23 @@ Supported actions:
         });
         const ManageSystemSchema = zod_1.z.discriminatedUnion("action", [
             zod_1.z.object({ action: zod_1.z.literal("health") }),
+            zod_1.z.object({ action: zod_1.z.literal("metrics") }),
+            zod_1.z.object({
+                action: zod_1.z.literal("export_memory"),
+                format: zod_1.z.enum(["json", "markdown", "obsidian"]).describe("Export format"),
+                includeMetadata: zod_1.z.boolean().optional().describe("Include metadata in export"),
+                includeRelationships: zod_1.z.boolean().optional().describe("Include relationships in export"),
+                includeObservations: zod_1.z.boolean().optional().describe("Include observations in export"),
+                entityTypes: zod_1.z.array(zod_1.z.string()).optional().describe("Filter by entity types"),
+                since: zod_1.z.number().optional().describe("Unix timestamp (ms) - only export data created after this time"),
+            }),
+            zod_1.z.object({
+                action: zod_1.z.literal("import_memory"),
+                data: zod_1.z.union([zod_1.z.string(), zod_1.z.any()]).describe("Import data (JSON string or object)"),
+                sourceFormat: zod_1.z.enum(["mem0", "memgpt", "markdown", "cozo"]).describe("Source format"),
+                mergeStrategy: zod_1.z.enum(["skip", "overwrite", "merge"]).optional().default("skip").describe("How to handle existing entities"),
+                defaultEntityType: zod_1.z.string().optional().default("Note").describe("Default type for imported entities"),
+            }),
             zod_1.z.object({
                 action: zod_1.z.literal("snapshot_create"),
                 metadata: MetadataSchema.optional().describe("Additional metadata for the snapshot"),
@@ -2911,8 +3103,18 @@ Supported actions:
         ]);
         const ManageSystemParameters = zod_1.z.object({
             action: zod_1.z
-                .enum(["health", "snapshot_create", "snapshot_list", "snapshot_diff", "cleanup", "reflect", "clear_memory"])
+                .enum(["health", "metrics", "export_memory", "import_memory", "snapshot_create", "snapshot_list", "snapshot_diff", "cleanup", "reflect", "clear_memory"])
                 .describe("Action (determines which fields are required)"),
+            format: zod_1.z.enum(["json", "markdown", "obsidian"]).optional().describe("Export format (for export_memory)"),
+            includeMetadata: zod_1.z.boolean().optional().describe("Include metadata (for export_memory)"),
+            includeRelationships: zod_1.z.boolean().optional().describe("Include relationships (for export_memory)"),
+            includeObservations: zod_1.z.boolean().optional().describe("Include observations (for export_memory)"),
+            entityTypes: zod_1.z.array(zod_1.z.string()).optional().describe("Filter by entity types (for export_memory)"),
+            since: zod_1.z.number().optional().describe("Unix timestamp in ms (for export_memory)"),
+            data: zod_1.z.union([zod_1.z.string(), zod_1.z.any()]).optional().describe("Import data (for import_memory)"),
+            sourceFormat: zod_1.z.enum(["mem0", "memgpt", "markdown", "cozo"]).optional().describe("Source format (for import_memory)"),
+            mergeStrategy: zod_1.z.enum(["skip", "overwrite", "merge"]).optional().describe("Merge strategy (for import_memory)"),
+            defaultEntityType: zod_1.z.string().optional().describe("Default entity type (for import_memory)"),
             snapshot_id_a: zod_1.z.string().optional().describe("Required for snapshot_diff"),
             snapshot_id_b: zod_1.z.string().optional().describe("Required for snapshot_diff"),
             metadata: MetadataSchema.optional().describe("Optional for snapshot_create"),
@@ -2927,7 +3129,17 @@ Supported actions:
             name: "manage_system",
             description: `System maintenance and memory management. Select operation via 'action'.
 Supported actions:
-- 'health': Status check. Returns DB counts and embedding cache statistics.
+- 'health': Status check. Returns DB counts, embedding cache statistics, and performance metrics.
+- 'metrics': Detailed metrics. Returns operation counts, error statistics, and performance data.
+- 'export_memory': Export memory to various formats. Params: { format: 'json'|'markdown'|'obsidian', includeMetadata?: boolean, includeRelationships?: boolean, includeObservations?: boolean, entityTypes?: string[], since?: number }.
+  * format='json': Native Cozo format (re-importable)
+  * format='markdown': Human-readable markdown document
+  * format='obsidian': ZIP archive with wiki-links, ready for Obsidian vault
+- 'import_memory': Import memory from external sources. Params: { data: string|object, sourceFormat: 'mem0'|'memgpt'|'markdown'|'cozo', mergeStrategy?: 'skip'|'overwrite'|'merge', defaultEntityType?: string }.
+  * sourceFormat='mem0': Import from Mem0 format (user_id becomes entity)
+  * sourceFormat='memgpt': Import from MemGPT archival/recall memory
+  * sourceFormat='markdown': Parse markdown sections as entities
+  * sourceFormat='cozo': Import from native Cozo JSON export
 - 'snapshot_create': Creates a backup point. Params: { metadata?: object }.
 - 'snapshot_list': Lists all available snapshots.
 - 'snapshot_diff': Compares two snapshots. Params: { snapshot_id_a: string, snapshot_id_b: string }.
@@ -2957,7 +3169,16 @@ Supported actions:
                                 observations: obsResult.rows.length,
                                 relations: relResult.rows.length,
                             },
-                            performance: { embedding_cache: this.embeddingService.getCacheStats() },
+                            performance: {
+                                embedding_cache: this.embeddingService.getCacheStats(),
+                                last_operation_ms: this.metrics.performance.last_operation_ms,
+                                avg_operation_ms: this.metrics.performance.avg_operation_ms,
+                                total_operations: this.metrics.performance.total_operations
+                            },
+                            metrics: {
+                                operations: this.metrics.operations,
+                                errors: this.metrics.errors
+                            },
                             timestamp: new Date().toISOString(),
                         });
                     }
@@ -2965,6 +3186,60 @@ Supported actions:
                         return JSON.stringify({ status: "error", error: error.message, timestamp: new Date().toISOString() });
                     }
                 }
+                if (input.action === "metrics") {
+                    try {
+                        return JSON.stringify({
+                            operations: this.metrics.operations,
+                            errors: this.metrics.errors,
+                            performance: this.metrics.performance,
+                            embedding_cache: this.embeddingService.getCacheStats(),
+                            timestamp: new Date().toISOString()
+                        });
+                    }
+                    catch (error) {
+                        return JSON.stringify({ error: "Failed to retrieve metrics", message: error.message });
+                    }
+                }
+                if (input.action === "export_memory") {
+                    try {
+                        const result = await this.exportMemory({
+                            format: input.format,
+                            includeMetadata: input.includeMetadata,
+                            includeRelationships: input.includeRelationships,
+                            includeObservations: input.includeObservations,
+                            entityTypes: input.entityTypes,
+                            since: input.since
+                        });
+                        if (result.format === 'obsidian' && result.zipBuffer) {
+                            // For Obsidian ZIP, return base64 encoded buffer
+                            return JSON.stringify({
+                                format: 'obsidian',
+                                data: result.zipBuffer.toString('base64'),
+                                stats: result.stats,
+                                encoding: 'base64',
+                                filename: `memory_export_${Date.now()}.zip`
+                            });
+                        }
+                        return JSON.stringify(result);
+                    }
+                    catch (error) {
+                        return JSON.stringify({ error: "Export failed", message: error.message });
+                    }
+                }
+                if (input.action === "import_memory") {
+                    try {
+                        const result = await this.importMemory({
+                            data: input.data,
+                            sourceFormat: input.sourceFormat,
+                            mergeStrategy: input.mergeStrategy,
+                            defaultEntityType: input.defaultEntityType
+                        });
+                        return JSON.stringify(result);
+                    }
+                    catch (error) {
+                        return JSON.stringify({ error: "Import failed", message: error.message });
+                    }
+                }
                 if (input.action === "snapshot_create") {
                     try {
                         // Optimization: Sequential execution and count aggregation instead of full fetch

package/dist/inference-engine.js

@@ -324,9 +324,16 @@ class InferenceEngine {
                     for (const r of res.rows) {
                         if (!r || r.length < 5)
                             continue;
+                        const fromId = String(r[0]);
+                        const toId = String(r[1]);
+                        // FIX: Filter out self-references
+                        if (fromId === toId) {
+                            console.error(`[Inference] Skipping self-reference in custom rule ${ruleName}: ${fromId} -> ${toId}`);
+                            continue;
+                        }
                         results.push({
-                            from_id: String(r[0]),
-                            to_id: String(r[1]),
+                            from_id: fromId,
+                            to_id: toId,
                             relation_type: String(r[2]),
                             confidence: Number(r[3]),
                             reason: String(r[4]),

package/dist/memory-service.js

@@ -1,7 +1,42 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MemoryService = void 0;
 const uuid_1 = require("uuid");
+const pdf_mjs_1 = require("pdfjs-dist/legacy/build/pdf.mjs");
+const fs = __importStar(require("fs"));
 class MemoryService {
     db;
     embeddings;
@@ -169,7 +204,7 @@ class MemoryService {
         console.error('[MemoryService] Snapshot created:', snapshotId, stats);
         return snapshotId;
     }
-    async ingestFile(content, format, entityName, entityType = 'Document', chunking = 'paragraphs') {
+    async ingestFile(content, format, entityName, entityType = 'Document', chunking = 'paragraphs', filePath) {
         const searchResults = await this.search(entityName, 1);
         let entity;
         if (searchResults.length > 0 && searchResults[0].entity.name.toLowerCase() === entityName.toLowerCase()) {
@@ -179,16 +214,64 @@ class MemoryService {
             entity = await this.createEntity(entityName, entityType, { format: format });
         }
         let chunks = [];
-        if (format === 'markdown' && chunking === 'paragraphs') {
-            chunks = content.split(/\n\s*\n/).filter((c) => c.trim().length > 0);
+        if (format === 'pdf') {
+            try {
+                let data;
+                // If filePath is provided, read from file
+                if (filePath) {
+                    data = new Uint8Array(fs.readFileSync(filePath));
+                }
+                else {
+                    // Otherwise, assume content is base64
+                    const buffer = Buffer.from(content, 'base64');
+                    data = new Uint8Array(buffer);
+                }
+                const loadingTask = (0, pdf_mjs_1.getDocument)({ data });
+                const pdf = await loadingTask.promise;
+                const numPages = pdf.numPages;
+                const pageTextPromises = Array.from({ length: numPages }, async (_, i) => {
+                    const page = await pdf.getPage(i + 1);
+                    const textContent = await page.getTextContent();
+                    return textContent.items.map((item) => item.str).join(' ');
+                });
+                const pageTexts = await Promise.all(pageTextPromises);
+                const text = pageTexts.join('\n');
+                if (chunking === 'paragraphs') {
+                    chunks = text.split(/\n\s*\n/).filter((c) => c.trim().length > 0);
+                }
+                else {
+                    chunks = [text];
+                }
+            }
+            catch (e) {
+                console.error('[MemoryService] PDF parsing error:', e);
+                throw new Error(`Failed to parse PDF: ${e instanceof Error ? e.message : String(e)}`);
+            }
+        }
+        else if (format === 'markdown') {
+            // For markdown, also support file path
+            let textContent = content;
+            if (filePath) {
+                textContent = fs.readFileSync(filePath, 'utf-8');
+            }
+            if (chunking === 'paragraphs') {
+                chunks = textContent.split(/\n\s*\n/).filter((c) => c.trim().length > 0);
+            }
+            else {
+                chunks = [textContent];
+            }
         }
         else if (format === 'json') {
+            let textContent = content;
+            if (filePath) {
+                textContent = fs.readFileSync(filePath, 'utf-8');
+            }
             try {
-                const data = JSON.parse(content);
+                const data = JSON.parse(textContent);
                 chunks = [JSON.stringify(data, null, 2)];
             }
             catch (e) {
-                chunks = [content];
+                chunks = [textContent];
             }
         }
         else {