@levalicious/server-memory 0.0.13 → 0.0.15

package/dist/server.js

@@ -2,11 +2,14 @@
 import { Server } from "@modelcontextprotocol/sdk/server/index.js";
 import { CallToolRequestSchema, ListToolsRequestSchema, } from "@modelcontextprotocol/sdk/types.js";
 import { randomBytes } from 'crypto';
+import fs from 'fs';
 import path from 'path';
 import { fileURLToPath } from 'url';
 import { GraphFile, DIR_FORWARD, DIR_BACKWARD } from './src/graphfile.js';
 import { StringTable } from './src/stringtable.js';
 import { structuralSample } from './src/pagerank.js';
+import { computeMerwPsi } from './src/merw.js';
+import { validateExtension, loadDocument } from './src/kb_load.js';
 // Define memory file path using environment variable with fallback
 const defaultMemoryPath = path.join(path.dirname(fileURLToPath(import.meta.url)), 'memory.json');
 // If MEMORY_FILE_PATH is just a filename, put it in the same directory as the script
@@ -104,7 +107,7 @@ function paginateItems(items, cursor = 0, maxChars = MAX_CHARS) {
     let i = cursor;
     // Calculate overhead for wrapper: {"items":[],"nextCursor":null,"totalCount":123}
     const wrapperTemplate = { items: [], nextCursor: null, totalCount: items.length };
-    let overhead = JSON.stringify(wrapperTemplate).length;
+    const overhead = JSON.stringify(wrapperTemplate).length;
     let charCount = overhead;
     while (i < items.length) {
         const itemJson = JSON.stringify(items[i]);
@@ -125,54 +128,13 @@ function paginateItems(items, cursor = 0, maxChars = MAX_CHARS) {
     };
 }
 function paginateGraph(graph, entityCursor = 0, relationCursor = 0) {
-    // Build incrementally, measuring actual serialized size
-    const entityCount = graph.entities.length;
-    const relationCount = graph.relations.length;
-    // Start with empty result to measure base overhead
-    const emptyResult = {
-        entities: { items: [], nextCursor: null, totalCount: entityCount },
-        relations: { items: [], nextCursor: null, totalCount: relationCount }
-    };
-    let currentSize = JSON.stringify(emptyResult).length;
-    const resultEntities = [];
-    const resultRelations = [];
-    let entityIdx = entityCursor;
-    let relationIdx = relationCursor;
-    // Add entities until we hit the limit
-    while (entityIdx < graph.entities.length) {
-        const entity = graph.entities[entityIdx];
-        const entityJson = JSON.stringify(entity);
-        const addedChars = entityJson.length + (resultEntities.length > 0 ? 1 : 0);
-        if (currentSize + addedChars > MAX_CHARS) {
-            break;
-        }
-        resultEntities.push(entity);
-        currentSize += addedChars;
-        entityIdx++;
-    }
-    // Add relations with remaining space
-    while (relationIdx < graph.relations.length) {
-        const relation = graph.relations[relationIdx];
-        const relationJson = JSON.stringify(relation);
-        const addedChars = relationJson.length + (resultRelations.length > 0 ? 1 : 0);
-        if (currentSize + addedChars > MAX_CHARS) {
-            break;
-        }
-        resultRelations.push(relation);
-        currentSize += addedChars;
-        relationIdx++;
-    }
+    // Entities and relations have independent cursors, so paginate them
+    // independently — each gets the full budget.  The caller already has
+    // previously-returned pages and only needs the next page of whichever
+    // section it is advancing.
     return {
-        entities: {
-            items: resultEntities,
-            nextCursor: entityIdx < graph.entities.length ? entityIdx : null,
-            totalCount: entityCount
-        },
-        relations: {
-            items: resultRelations,
-            nextCursor: relationIdx < graph.relations.length ? relationIdx : null,
-            totalCount: relationCount
-        }
+        entities: paginateItems(graph.entities, entityCursor),
+        relations: paginateItems(graph.relations, relationCursor),
     };
 }
 // The KnowledgeGraphManager class contains all operations to interact with the knowledge graph
@@ -193,12 +155,20 @@ export class KnowledgeGraphManager {
         this.gf = new GraphFile(graphPath, this.st);
         this.nameIndex = new Map();
         this.rebuildNameIndex();
-        // Run initial structural sampling if graph is non-empty
+        // Run initial structural sampling and MERW if graph is non-empty
         if (this.nameIndex.size > 0) {
             structuralSample(this.gf, 1, 0.85);
+            computeMerwPsi(this.gf);
             this.gf.sync();
         }
     }
+    /**
+     * Run the loadDocument pipeline under a shared (read) lock,
+     * so the StringTable is consistent during IDF derivation.
+     */
+    prepareDocumentLoad(text, title, topK) {
+        return this.withReadLock(() => loadDocument(text, title, this.st, topK));
+    }
     // --- Locking helpers ---
     /**
      * Acquire a shared (read) lock, refresh mappings (in case another process
@@ -288,11 +258,12 @@ export class KnowledgeGraphManager {
             }
         });
     }
-    /** Re-run structural sampling (call after graph mutations) */
+    /** Re-run structural sampling and MERW eigenvector computation (call after graph mutations) */
     resample() {
         this.withWriteLock(() => {
             if (this.nameIndex.size > 0) {
                 structuralSample(this.gf, 1, 0.85);
+                computeMerwPsi(this.gf);
             }
         });
     }
@@ -528,7 +499,7 @@ export class KnowledgeGraphManager {
         try {
             regex = new RegExp(query, 'i');
         }
-        catch (e) {
+        catch {
             throw new Error(`Invalid regex pattern: ${query}`);
         }
         return this.withReadLock(() => {
@@ -818,7 +789,7 @@ export class KnowledgeGraphManager {
                 if (!offset)
                     break;
                 const edges = this.gf.getEdges(offset);
-                const validNeighbors = new Set();
+                const candidates = [];
                 for (const edge of edges) {
                     if (direction === 'forward' && edge.direction !== DIR_FORWARD)
                         continue;
@@ -826,14 +797,46 @@ export class KnowledgeGraphManager {
                         continue;
                     const targetRec = this.gf.readEntity(edge.targetOffset);
                     const neighborName = this.st.get(BigInt(targetRec.nameId));
-                    if (neighborName !== current)
-                        validNeighbors.add(neighborName);
+                    if (neighborName !== current && this.nameIndex.has(neighborName)) {
+                        candidates.push({ name: neighborName, psi: targetRec.psi });
+                    }
+                }
+                // Deduplicate: keep max psi per name (multiple edge types to same target)
+                const byName = new Map();
+                for (const c of candidates) {
+                    const existing = byName.get(c.name);
+                    if (existing === undefined || c.psi > existing) {
+                        byName.set(c.name, c.psi);
+                    }
                 }
-                const neighborArr = Array.from(validNeighbors).filter(n => this.nameIndex.has(n));
-                if (neighborArr.length === 0)
+                if (byName.size === 0)
                     break;
-                const idx = Math.floor(random() * neighborArr.length);
-                current = neighborArr[idx];
+                const neighborArr = Array.from(byName.entries());
+                // MERW-weighted sampling: probability proportional to ψ_j
+                // (The ψ_i denominator is constant for all neighbors and cancels in normalization)
+                let totalPsi = 0;
+                for (const [, psi] of neighborArr)
+                    totalPsi += psi;
+                let chosen;
+                if (totalPsi > 0) {
+                    // Weighted sampling by psi
+                    const r = random() * totalPsi;
+                    let cumulative = 0;
+                    chosen = neighborArr[neighborArr.length - 1][0]; // fallback
+                    for (const [name, psi] of neighborArr) {
+                        cumulative += psi;
+                        if (r <= cumulative) {
+                            chosen = name;
+                            break;
+                        }
+                    }
+                }
+                else {
+                    // psi not yet computed (all zero) — fall back to uniform
+                    const idx = Math.floor(random() * neighborArr.length);
+                    chosen = neighborArr[idx][0];
+                }
+                current = chosen;
                 pathNames.push(current);
             }
             return { entity: current, path: pathNames };
@@ -950,7 +953,7 @@ export function createServer(memoryFilePath) {
                 sizes: ["any"]
             }
         ],
-        version: "0.0.13",
+        version: "0.0.15",
     }, {
         capabilities: {
             tools: {},
@@ -1271,6 +1274,30 @@ Use this to build chains of reasoning that persist in the graph. Each thought ca
                         required: ["observations"],
                     },
                 },
+                {
+                    name: "kb_load",
+                    description: `Load a plaintext document into the knowledge graph. Chunks the text into entities connected by a doubly-linked chain, runs sentence TextRank to identify the most important sentences, and creates an index entity that links directly to the chunks containing those sentences.
+
+The file MUST be plaintext (.txt, .tex, .md, source code, etc.). For PDFs, use pdftotext first. For other binary formats, convert to text before calling this tool.`,
+                    inputSchema: {
+                        type: "object",
+                        properties: {
+                            filePath: {
+                                type: "string",
+                                description: "Absolute path to the plaintext file to load. Must have a plaintext extension (.txt, .tex, .md, .py, .ts, etc.).",
+                            },
+                            title: {
+                                type: "string",
+                                description: "Optional title for the document entity. Defaults to the filename without extension.",
+                            },
+                            topK: {
+                                type: "number",
+                                description: "Number of top-ranked sentences to highlight in the index (default: 15).",
+                            },
+                        },
+                        required: ["filePath"],
+                    },
+                },
             ],
         };
     });
@@ -1351,6 +1378,47 @@ Use this to build chains of reasoning that persist in the graph. Each thought ca
                 const result = await knowledgeGraphManager.addThought(args.observations, args.previousCtxId);
                 return { content: [{ type: "text", text: JSON.stringify(result) }] };
             }
+            case "kb_load": {
+                const filePath = args.filePath;
+                // Validate extension
+                validateExtension(filePath);
+                // Read file
+                let text;
+                try {
+                    text = fs.readFileSync(filePath, 'utf-8');
+                }
+                catch (err) {
+                    throw new Error(`Failed to read file: ${err instanceof Error ? err.message : String(err)}`);
+                }
+                // Derive title
+                const title = args.title ?? path.basename(filePath, path.extname(filePath));
+                const topK = args.topK ?? 15;
+                // Run the pipeline (reads string table under read lock)
+                const loadResult = knowledgeGraphManager.prepareDocumentLoad(text, title, topK);
+                // Insert into KB
+                const entities = await knowledgeGraphManager.createEntities(loadResult.entities.map(e => ({
+                    name: e.name,
+                    entityType: e.entityType,
+                    observations: e.observations,
+                })));
+                const relations = await knowledgeGraphManager.createRelations(loadResult.relations.map(r => ({
+                    from: r.from,
+                    to: r.to,
+                    relationType: r.relationType,
+                })));
+                knowledgeGraphManager.resample();
+                return {
+                    content: [{
+                            type: "text",
+                            text: JSON.stringify({
+                                document: title,
+                                stats: loadResult.stats,
+                                entitiesCreated: entities.length,
+                                relationsCreated: relations.length,
+                            }, null, 2),
+                        }],
+                };
+            }
             default:
                 throw new Error(`Unknown tool: ${name}`);
         }

package/dist/src/graphfile.js

@@ -4,6 +4,11 @@
  * All records live in a MemoryFile (graph.mem). Variable-length strings are
  * stored in a separate StringTable (strings.mem) and referenced by u32 ID.
  *
+ * Versioning:
+ *   MEMFILE_VERSION 1  = original 64-byte entity records (no psi field)
+ *   MEMFILE_VERSION 2  = 72-byte entity records with f64 psi for MERW
+ *   On open, version 1 files are migrated to version 2 automatically.
+ *
  * Graph file layout:
  *   [memfile header: 32 bytes]
  *   [graph header block: first allocation]
@@ -12,7 +17,7 @@
  *     u64 walker_total         total walker visits (global counter)
  *   [entity records, adj blocks, node log ...]
  *
- * EntityRecord: 64 bytes fixed
+ * EntityRecord: 72 bytes fixed (v2)
  *   u32  name_id         string table ID
  *   u32  type_id         string table ID
  *   u64  adj_offset      offset to AdjBlock (0 = no edges)
@@ -24,6 +29,7 @@
  *   u32  obs1_id         string table ID (0 = empty)
  *   u64  structural_visits  structural PageRank visit count
  *   u64  walker_visits      walker PageRank visit count
+ *   f64  psi             MERW dominant eigenvector component
  *
  * AdjBlock:
  *   u32  count
@@ -44,15 +50,21 @@
  *   u32  capacity
  *   u64  offsets[capacity]
  */
+import * as fs from 'fs';
 import { MemoryFile } from './memoryfile.js';
 // --- Constants ---
-export const ENTITY_RECORD_SIZE = 64;
+export const ENTITY_RECORD_SIZE = 72;
+const OLD_ENTITY_RECORD_SIZE = 64; // v1 layout without psi
 export const ADJ_ENTRY_SIZE = 24; // 8 + 4 + 4 + 8, naturally aligned
 const ADJ_HEADER_SIZE = 8; // count:u32 + capacity:u32
 const NODE_LOG_HEADER_SIZE = 8; // count:u32 + capacity:u32
 const GRAPH_HEADER_SIZE = 24; // node_log_offset:u64 + structural_total:u64 + walker_total:u64
 const INITIAL_ADJ_CAPACITY = 4;
 const INITIAL_LOG_CAPACITY = 256;
+// Graph-layer version stored in the memfile header version field
+const GRAPH_VERSION_V1 = 1; // original 64-byte entity records
+const GRAPH_VERSION_V2 = 2; // 72-byte entity records with f64 psi
+export const CURRENT_GRAPH_VERSION = GRAPH_VERSION_V2;
 // Direction flags
 export const DIR_FORWARD = 0n;
 export const DIR_BACKWARD = 1n;
@@ -72,7 +84,8 @@ const E_OBS1_ID = 40;
 // 4 bytes pad at 44
 const E_STRUCTURAL_VISITS = 48; // u64: 48..55, 8-aligned
 const E_WALKER_VISITS = 56; // u64: 56..63, 8-aligned
-// total = 64
+const E_PSI = 64; // f64: 64..71, 8-aligned (MERW eigenvector component)
+// total = 72
 // AdjEntry field offsets (within each entry)
 const AE_TARGET_DIR = 0;
 const AE_RELTYPE_ID = 8;
@@ -107,6 +120,7 @@ export function readEntityRecord(mf, offset) {
         obs1Id: buf.readUInt32LE(E_OBS1_ID),
         structuralVisits: buf.readBigUInt64LE(E_STRUCTURAL_VISITS),
         walkerVisits: buf.readBigUInt64LE(E_WALKER_VISITS),
+        psi: buf.readDoubleLE(E_PSI),
     };
 }
 export function writeEntityRecord(mf, rec) {
@@ -121,6 +135,7 @@ export function writeEntityRecord(mf, rec) {
     buf.writeUInt32LE(rec.obs1Id, E_OBS1_ID);
     buf.writeBigUInt64LE(rec.structuralVisits, E_STRUCTURAL_VISITS);
     buf.writeBigUInt64LE(rec.walkerVisits, E_WALKER_VISITS);
+    buf.writeDoubleLE(rec.psi, E_PSI);
     mf.write(rec.offset, buf);
 }
 export function readAdjBlock(mf, adjOffset) {
@@ -174,12 +189,98 @@ export class GraphFile {
         this.st = stringTable;
         const stats = this.mf.stats();
         if (stats.allocated <= 32n) {
+            // Fresh DB — initialize with current version
             this.graphHeaderOffset = this.initGraphHeader();
+            this.mf.setVersion(CURRENT_GRAPH_VERSION);
         }
         else {
-            // First allocation is graph header, at offset 40 (32 memfile header + 8 alloc_t header)
+            // Existing DB — check version and migrate if needed
             this.graphHeaderOffset = 40n;
+            const version = this.mf.getVersion();
+            if (version === GRAPH_VERSION_V1) {
+                this.migrateV1toV2(graphPath);
+            }
+            else if (version !== CURRENT_GRAPH_VERSION) {
+                throw new Error(`GraphFile: unknown version ${version} (expected ${GRAPH_VERSION_V1} or ${CURRENT_GRAPH_VERSION})`);
+            }
+        }
+    }
+    /**
+     * Migrate a v1 graph file (64-byte entity records) to v2 (72-byte with psi).
+     *
+     * Strategy: read all entities and edges from the current (v1) file into memory,
+     * close it, rename it to .v1, create a fresh file, write everything back with
+     * the new layout.
+     */
+    migrateV1toV2(graphPath) {
+        // 1. Read all entities and adjacency data from v1 layout
+        const offsets = this.getAllEntityOffsets();
+        const oldEntities = [];
+        for (const offset of offsets) {
+            // Read v1 entity (64 bytes) — the first 64 bytes match our struct minus psi
+            const buf = this.mf.read(offset, BigInt(OLD_ENTITY_RECORD_SIZE));
+            const rec = {
+                offset,
+                nameId: buf.readUInt32LE(E_NAME_ID),
+                typeId: buf.readUInt32LE(E_TYPE_ID),
+                adjOffset: buf.readBigUInt64LE(E_ADJ_OFFSET),
+                mtime: buf.readBigUInt64LE(E_MTIME),
+                obsMtime: buf.readBigUInt64LE(E_OBS_MTIME),
+                obsCount: buf.readUInt8(E_OBS_COUNT),
+                obs0Id: buf.readUInt32LE(E_OBS0_ID),
+                obs1Id: buf.readUInt32LE(E_OBS1_ID),
+                structuralVisits: buf.readBigUInt64LE(E_STRUCTURAL_VISITS),
+                walkerVisits: buf.readBigUInt64LE(E_WALKER_VISITS),
+                psi: 0,
+            };
+            const edges = rec.adjOffset !== 0n ? readAdjBlock(this.mf, rec.adjOffset).entries : [];
+            oldEntities.push({ rec, edges });
+        }
+        // 2. Read global counters
+        const structuralTotal = this.getStructuralTotal();
+        const walkerTotal = this.getWalkerTotal();
+        // 3. Close old file, rename to .v1 backup
+        this.mf.sync();
+        this.mf.close();
+        const backupPath = graphPath + '.v1';
+        fs.renameSync(graphPath, backupPath);
+        // 4. Create fresh v2 file
+        this.mf = new MemoryFile(graphPath, 65536);
+        this.mf.setVersion(CURRENT_GRAPH_VERSION);
+        this.graphHeaderOffset = this.initGraphHeader();
+        // 5. Write global counters
+        const ghBuf = Buffer.alloc(8);
+        ghBuf.writeBigUInt64LE(structuralTotal, 0);
+        this.mf.write(this.graphHeaderOffset + BigInt(GH_STRUCTURAL_TOTAL), ghBuf);
+        ghBuf.writeBigUInt64LE(walkerTotal, 0);
+        this.mf.write(this.graphHeaderOffset + BigInt(GH_WALKER_TOTAL), ghBuf);
+        // 6. Write all entities with new 72-byte layout, building offset remap
+        const offsetMap = new Map(); // old offset → new offset
+        for (const { rec } of oldEntities) {
+            const newOffset = this.mf.alloc(BigInt(ENTITY_RECORD_SIZE));
+            if (newOffset === 0n)
+                throw new Error('GraphFile migration: entity alloc failed');
+            offsetMap.set(rec.offset, newOffset);
+            const newRec = { ...rec, offset: newOffset, adjOffset: 0n, psi: 0 };
+            writeEntityRecord(this.mf, newRec);
+            this.nodeLogAppend(newOffset);
+        }
+        // 7. Rebuild adjacency blocks with remapped target offsets
+        for (const { rec, edges } of oldEntities) {
+            const newOffset = offsetMap.get(rec.offset);
+            for (const edge of edges) {
+                const newTarget = offsetMap.get(edge.targetOffset);
+                if (newTarget === undefined)
+                    continue; // skip dangling refs
+                this.addEdge(newOffset, {
+                    targetOffset: newTarget,
+                    direction: edge.direction,
+                    relTypeId: edge.relTypeId,
+                    mtime: edge.mtime,
+                });
+            }
         }
+        this.mf.sync();
     }
     initGraphHeader() {
         // Allocate graph header block
@@ -233,6 +334,7 @@ export class GraphFile {
             obs1Id: 0,
             structuralVisits: 0n,
             walkerVisits: 0n,
+            psi: 0,
         };
         writeEntityRecord(this.mf, rec);
         this.nodeLogAppend(offset);
@@ -534,6 +636,18 @@ export class GraphFile {
         const rec = this.readEntity(entityOffset);
         return Number(rec.walkerVisits) / Number(total);
     }
+    // --- MERW eigenvector ---
+    /** Write the psi (MERW eigenvector component) for an entity. */
+    setPsi(entityOffset, psi) {
+        const buf = Buffer.alloc(8);
+        buf.writeDoubleLE(psi, 0);
+        this.mf.write(entityOffset + BigInt(E_PSI), buf);
+    }
+    /** Read the psi (MERW eigenvector component) for an entity. */
+    getPsi(entityOffset) {
+        const buf = this.mf.read(entityOffset + BigInt(E_PSI), 8n);
+        return buf.readDoubleLE(0);
+    }
     // --- Lifecycle & Concurrency ---
     /** Acquire a shared (read) lock on the graph file. */
     lockShared() {