@levalicious/server-memory 0.0.10 → 0.0.12

package/dist/server.js

@@ -1,11 +1,12 @@
 #!/usr/bin/env node
 import { Server } from "@modelcontextprotocol/sdk/server/index.js";
 import { CallToolRequestSchema, ListToolsRequestSchema, } from "@modelcontextprotocol/sdk/types.js";
-import { promises as fs } from 'fs';
 import { randomBytes } from 'crypto';
 import path from 'path';
 import { fileURLToPath } from 'url';
-import lockfile from 'proper-lockfile';
+import { GraphFile, DIR_FORWARD, DIR_BACKWARD } from './src/graphfile.js';
+import { StringTable } from './src/stringtable.js';
+import { structuralSample } from './src/pagerank.js';
 // Define memory file path using environment variable with fallback
 const defaultMemoryPath = path.join(path.dirname(fileURLToPath(import.meta.url)), 'memory.json');
 // If MEMORY_FILE_PATH is just a filename, put it in the same directory as the script
@@ -17,17 +18,43 @@ const DEFAULT_MEMORY_FILE_PATH = process.env.MEMORY_FILE_PATH
 /**
  * Sort entities by the specified field and direction.
  * Returns a new array (does not mutate input).
- * If sortBy is undefined, returns the original array (no sorting - preserves insertion order).
+ * Defaults to 'llmrank' when sortBy is undefined.
+ *
+ * For 'pagerank' sort: uses structural rank (desc by default).
+ * For 'llmrank' sort: uses walker rank, falls back to structural rank on tie, then random.
+ * Both rank sorts require rankMaps parameter.
  */
-function sortEntities(entities, sortBy, sortDir) {
-    if (!sortBy)
-        return entities; // No sorting - preserve current behavior
+function sortEntities(entities, sortBy = "llmrank", sortDir, rankMaps) {
     const dir = sortDir ?? (sortBy === "name" ? "asc" : "desc");
     const mult = dir === "asc" ? 1 : -1;
     return [...entities].sort((a, b) => {
         if (sortBy === "name") {
             return mult * a.name.localeCompare(b.name);
         }
+        if (sortBy === "pagerank") {
+            const aRank = rankMaps?.structural.get(a.name) ?? 0;
+            const bRank = rankMaps?.structural.get(b.name) ?? 0;
+            const diff = aRank - bRank;
+            if (diff !== 0)
+                return mult * diff;
+            return Math.random() - 0.5; // random tiebreak
+        }
+        if (sortBy === "llmrank") {
+            // Primary: walker rank
+            const aWalker = rankMaps?.walker.get(a.name) ?? 0;
+            const bWalker = rankMaps?.walker.get(b.name) ?? 0;
+            const walkerDiff = aWalker - bWalker;
+            if (walkerDiff !== 0)
+                return mult * walkerDiff;
+            // Fallback: structural rank
+            const aStruct = rankMaps?.structural.get(a.name) ?? 0;
+            const bStruct = rankMaps?.structural.get(b.name) ?? 0;
+            const structDiff = aStruct - bStruct;
+            if (structDiff !== 0)
+                return mult * structDiff;
+            // Final: random tiebreak
+            return Math.random() - 0.5;
+        }
         // For timestamps, treat undefined as 0 (oldest)
         const aVal = a[sortBy] ?? 0;
         const bVal = b[sortBy] ?? 0;
@@ -36,17 +63,36 @@ function sortEntities(entities, sortBy, sortDir) {
 }
 /**
  * Sort neighbors by the specified field and direction.
- * If sortBy is undefined, returns the original array (no sorting).
+ * Defaults to 'llmrank' when sortBy is undefined.
  */
-function sortNeighbors(neighbors, sortBy, sortDir) {
-    if (!sortBy)
-        return neighbors;
+function sortNeighbors(neighbors, sortBy = "llmrank", sortDir, rankMaps) {
     const dir = sortDir ?? (sortBy === "name" ? "asc" : "desc");
     const mult = dir === "asc" ? 1 : -1;
     return [...neighbors].sort((a, b) => {
         if (sortBy === "name") {
             return mult * a.name.localeCompare(b.name);
         }
+        if (sortBy === "pagerank") {
+            const aRank = rankMaps?.structural.get(a.name) ?? 0;
+            const bRank = rankMaps?.structural.get(b.name) ?? 0;
+            const diff = aRank - bRank;
+            if (diff !== 0)
+                return mult * diff;
+            return Math.random() - 0.5;
+        }
+        if (sortBy === "llmrank") {
+            const aWalker = rankMaps?.walker.get(a.name) ?? 0;
+            const bWalker = rankMaps?.walker.get(b.name) ?? 0;
+            const walkerDiff = aWalker - bWalker;
+            if (walkerDiff !== 0)
+                return mult * walkerDiff;
+            const aStruct = rankMaps?.structural.get(a.name) ?? 0;
+            const bStruct = rankMaps?.structural.get(b.name) ?? 0;
+            const structDiff = aStruct - bStruct;
+            if (structDiff !== 0)
+                return mult * structDiff;
+            return Math.random() - 0.5;
+        }
         const aVal = a[sortBy] ?? 0;
         const bVal = b[sortBy] ?? 0;
         return mult * (aVal - bVal);
@@ -131,574 +177,764 @@ function paginateGraph(graph, entityCursor = 0, relationCursor = 0) {
 }
 // The KnowledgeGraphManager class contains all operations to interact with the knowledge graph
 export class KnowledgeGraphManager {
-    bclCtr = 0;
-    bclTerm = "";
-    memoryFilePath;
+    gf;
+    st;
+    /** In-memory name→offset index, rebuilt from node log on open */
+    nameIndex;
+    /** Generation counter from last rebuildNameIndex — avoids redundant rebuilds */
+    cachedEntityCount = -1;
     constructor(memoryFilePath = DEFAULT_MEMORY_FILE_PATH) {
-        this.memoryFilePath = memoryFilePath;
+        // Derive binary file paths from the base path
+        const dir = path.dirname(memoryFilePath);
+        const base = path.basename(memoryFilePath, path.extname(memoryFilePath));
+        const graphPath = path.join(dir, `${base}.graph`);
+        const strPath = path.join(dir, `${base}.strings`);
+        this.st = new StringTable(strPath);
+        this.gf = new GraphFile(graphPath, this.st);
+        this.nameIndex = new Map();
+        this.rebuildNameIndex();
+        // Run initial structural sampling if graph is non-empty
+        if (this.nameIndex.size > 0) {
+            structuralSample(this.gf, 1, 0.85);
+            this.gf.sync();
+        }
     }
-    async withLock(fn) {
-        // Ensure file exists for locking
+    // --- Locking helpers ---
+    /**
+     * Acquire a shared (read) lock, refresh mappings (in case another process
+     * grew the files), rebuild the name index if the entity count changed,
+     * run the callback, then release the lock.
+     */
+    withReadLock(fn) {
+        this.gf.lockShared();
         try {
-            await fs.access(this.memoryFilePath);
+            this.gf.refresh();
+            this.st.refresh();
+            this.maybeRebuildNameIndex();
+            return fn();
         }
-        catch {
-            await fs.writeFile(this.memoryFilePath, "");
+        finally {
+            this.gf.unlock();
         }
-        const release = await lockfile.lock(this.memoryFilePath, { retries: { retries: 5, minTimeout: 100 } });
+    }
+    /**
+     * Acquire an exclusive (write) lock, refresh mappings, rebuild name index,
+     * run the callback, sync both files, then release the lock.
+     */
+    withWriteLock(fn) {
+        this.gf.lockExclusive();
         try {
-            return await fn();
+            this.gf.refresh();
+            this.st.refresh();
+            this.maybeRebuildNameIndex();
+            const result = fn();
+            this.gf.sync();
+            this.st.sync();
+            // Update cached count after write
+            this.cachedEntityCount = this.gf.getEntityCount();
+            return result;
         }
         finally {
-            await release();
+            this.gf.unlock();
         }
     }
-    async loadGraph() {
-        try {
-            const data = await fs.readFile(this.memoryFilePath, "utf-8");
-            const lines = data.split("\n").filter(line => line.trim() !== "");
-            return lines.reduce((graph, line) => {
-                const item = JSON.parse(line);
-                if (item.type === "entity")
-                    graph.entities.push(item);
-                if (item.type === "relation")
-                    graph.relations.push(item);
-                return graph;
-            }, { entities: [], relations: [] });
+    /** Rebuild the in-memory name→offset index from the node log */
+    rebuildNameIndex() {
+        this.nameIndex.clear();
+        const offsets = this.gf.getAllEntityOffsets();
+        for (const offset of offsets) {
+            const rec = this.gf.readEntity(offset);
+            const name = this.st.get(BigInt(rec.nameId));
+            this.nameIndex.set(name, offset);
+        }
+        this.cachedEntityCount = this.gf.getEntityCount();
+    }
+    /**
+     * Rebuild nameIndex only if the entity count changed (another process wrote).
+     * This is cheap to check (single u32 read) and avoids a full scan on every lock.
+     */
+    maybeRebuildNameIndex() {
+        const currentCount = this.gf.getEntityCount();
+        if (currentCount !== this.cachedEntityCount) {
+            this.rebuildNameIndex();
+        }
+    }
+    /** Build rank maps from the binary store for pagerank/llmrank sorting.
+     *  NOTE: Must be called inside a lock (read or write). */
+    getRankMapsUnlocked() {
+        const structural = new Map();
+        const walker = new Map();
+        const structTotal = this.gf.getStructuralTotal();
+        const walkerTotal = this.gf.getWalkerTotal();
+        for (const [name, offset] of this.nameIndex) {
+            const rec = this.gf.readEntity(offset);
+            structural.set(name, structTotal > 0n ? Number(rec.structuralVisits) / Number(structTotal) : 0);
+            walker.set(name, walkerTotal > 0n ? Number(rec.walkerVisits) / Number(walkerTotal) : 0);
         }
-        catch (error) {
-            if (error?.code === "ENOENT") {
-                return { entities: [], relations: [] };
+        return { structural, walker };
+    }
+    /** Build rank maps (acquires read lock). */
+    getRankMaps() {
+        return this.withReadLock(() => this.getRankMapsUnlocked());
+    }
+    /** Increment walker visit count for a list of entity names */
+    recordWalkerVisits(names) {
+        this.withWriteLock(() => {
+            for (const name of names) {
+                const offset = this.nameIndex.get(name);
+                if (offset !== undefined) {
+                    this.gf.incrementWalkerVisit(offset);
+                }
+            }
+        });
+    }
+    /** Re-run structural sampling (call after graph mutations) */
+    resample() {
+        this.withWriteLock(() => {
+            if (this.nameIndex.size > 0) {
+                structuralSample(this.gf, 1, 0.85);
+            }
+        });
+    }
+    /** Convert an EntityRecord to the public Entity interface */
+    recordToEntity(rec) {
+        const name = this.st.get(BigInt(rec.nameId));
+        const entityType = this.st.get(BigInt(rec.typeId));
+        const observations = [];
+        if (rec.obs0Id !== 0)
+            observations.push(this.st.get(BigInt(rec.obs0Id)));
+        if (rec.obs1Id !== 0)
+            observations.push(this.st.get(BigInt(rec.obs1Id)));
+        const entity = { name, entityType, observations };
+        const mtime = Number(rec.mtime);
+        const obsMtime = Number(rec.obsMtime);
+        if (mtime > 0)
+            entity.mtime = mtime;
+        if (obsMtime > 0)
+            entity.obsMtime = obsMtime;
+        return entity;
+    }
+    /** Get all entities as Entity objects (preserves node log order = insertion order) */
+    getAllEntities() {
+        const offsets = this.gf.getAllEntityOffsets();
+        return offsets.map(o => this.recordToEntity(this.gf.readEntity(o)));
+    }
+    /** Get all relations by scanning adjacency lists (forward edges only to avoid duplication) */
+    getAllRelations() {
+        const relations = [];
+        const offsets = this.gf.getAllEntityOffsets();
+        for (const offset of offsets) {
+            const rec = this.gf.readEntity(offset);
+            const fromName = this.st.get(BigInt(rec.nameId));
+            const edges = this.gf.getEdges(offset);
+            for (const edge of edges) {
+                if (edge.direction !== DIR_FORWARD)
+                    continue;
+                const targetRec = this.gf.readEntity(edge.targetOffset);
+                const toName = this.st.get(BigInt(targetRec.nameId));
+                const relationType = this.st.get(BigInt(edge.relTypeId));
+                const r = { from: fromName, to: toName, relationType };
+                const mtime = Number(edge.mtime);
+                if (mtime > 0)
+                    r.mtime = mtime;
+                relations.push(r);
             }
-            throw error;
         }
+        return relations;
     }
-    async saveGraph(graph) {
-        const lines = [
-            ...graph.entities.map(e => JSON.stringify({ type: "entity", ...e })),
-            ...graph.relations.map(r => JSON.stringify({ type: "relation", ...r })),
-        ];
-        const content = lines.join("\n") + (lines.length > 0 ? "\n" : "");
-        await fs.writeFile(this.memoryFilePath, content);
+    /** Load the full graph (entities + relations) */
+    loadGraph() {
+        return {
+            entities: this.getAllEntities(),
+            relations: this.getAllRelations(),
+        };
     }
     async createEntities(entities) {
-        return this.withLock(async () => {
-            const graph = await this.loadGraph();
-            // Validate observation limits
-            for (const entity of entities) {
-                if (entity.observations.length > 2) {
-                    throw new Error(`Entity "${entity.name}" has ${entity.observations.length} observations. Maximum allowed is 2.`);
+        // Validate observation limits (can do outside lock)
+        for (const entity of entities) {
+            if (entity.observations.length > 2) {
+                throw new Error(`Entity "${entity.name}" has ${entity.observations.length} observations. Maximum allowed is 2.`);
+            }
+            for (const obs of entity.observations) {
+                if (obs.length > 140) {
+                    throw new Error(`Observation in entity "${entity.name}" exceeds 140 characters (${obs.length} chars): "${obs.substring(0, 50)}..."`);
                 }
-                for (const obs of entity.observations) {
-                    if (obs.length > 140) {
-                        throw new Error(`Observation in entity "${entity.name}" exceeds 140 characters (${obs.length} chars): "${obs.substring(0, 50)}..."`);
-                    }
+            }
+        }
+        return this.withWriteLock(() => {
+            const now = BigInt(Date.now());
+            const newEntities = [];
+            for (const e of entities) {
+                const existingOffset = this.nameIndex.get(e.name);
+                if (existingOffset !== undefined) {
+                    const existing = this.recordToEntity(this.gf.readEntity(existingOffset));
+                    const sameType = existing.entityType === e.entityType;
+                    const sameObs = existing.observations.length === e.observations.length &&
+                        existing.observations.every((o, i) => o === e.observations[i]);
+                    if (sameType && sameObs)
+                        continue;
+                    throw new Error(`Entity "${e.name}" already exists with different data (type: "${existing.entityType}" vs "${e.entityType}", observations: ${existing.observations.length} vs ${e.observations.length})`);
+                }
+                const obsMtime = e.observations.length > 0 ? now : 0n;
+                const rec = this.gf.createEntity(e.name, e.entityType, now, obsMtime);
+                for (const obs of e.observations) {
+                    this.gf.addObservation(rec.offset, obs, now);
+                }
+                if (e.observations.length > 0) {
+                    const updated = this.gf.readEntity(rec.offset);
+                    updated.mtime = now;
+                    updated.obsMtime = now;
+                    this.gf.updateEntity(updated);
                 }
+                this.nameIndex.set(e.name, rec.offset);
+                const newEntity = {
+                    ...e,
+                    mtime: Number(now),
+                    obsMtime: e.observations.length > 0 ? Number(now) : undefined,
+                };
+                newEntities.push(newEntity);
             }
-            const now = Date.now();
-            const newEntities = entities
-                .filter(e => !graph.entities.some(existingEntity => existingEntity.name === e.name))
-                .map(e => ({ ...e, mtime: now, obsMtime: e.observations.length > 0 ? now : undefined }));
-            graph.entities.push(...newEntities);
-            await this.saveGraph(graph);
             return newEntities;
         });
     }
     async createRelations(relations) {
-        return this.withLock(async () => {
-            const graph = await this.loadGraph();
-            const now = Date.now();
-            // Update mtime on 'from' entities when relations are added
-            const fromEntityNames = new Set(relations.map(r => r.from));
-            graph.entities.forEach(e => {
-                if (fromEntityNames.has(e.name)) {
-                    e.mtime = now;
-                }
-            });
-            const newRelations = relations
-                .filter(r => !graph.relations.some(existingRelation => existingRelation.from === r.from &&
-                existingRelation.to === r.to &&
-                existingRelation.relationType === r.relationType))
-                .map(r => ({ ...r, mtime: now }));
-            graph.relations.push(...newRelations);
-            await this.saveGraph(graph);
+        return this.withWriteLock(() => {
+            const now = BigInt(Date.now());
+            const newRelations = [];
+            const fromOffsets = new Set();
+            for (const r of relations) {
+                const fromOffset = this.nameIndex.get(r.from);
+                const toOffset = this.nameIndex.get(r.to);
+                if (fromOffset === undefined || toOffset === undefined)
+                    continue;
+                const existingEdges = this.gf.getEdges(fromOffset);
+                const relTypeId = Number(this.st.find(r.relationType) ?? -1n);
+                const isDuplicate = existingEdges.some(e => e.direction === DIR_FORWARD &&
+                    e.targetOffset === toOffset &&
+                    e.relTypeId === relTypeId);
+                if (isDuplicate)
+                    continue;
+                const rTypeId = Number(this.st.intern(r.relationType));
+                const forwardEntry = {
+                    targetOffset: toOffset,
+                    direction: DIR_FORWARD,
+                    relTypeId: rTypeId,
+                    mtime: now,
+                };
+                this.gf.addEdge(fromOffset, forwardEntry);
+                const rTypeId2 = Number(this.st.intern(r.relationType));
+                const backwardEntry = {
+                    targetOffset: fromOffset,
+                    direction: DIR_BACKWARD,
+                    relTypeId: rTypeId2,
+                    mtime: now,
+                };
+                this.gf.addEdge(toOffset, backwardEntry);
+                fromOffsets.add(fromOffset);
+                newRelations.push({ ...r, mtime: Number(now) });
+            }
+            for (const offset of fromOffsets) {
+                const rec = this.gf.readEntity(offset);
+                rec.mtime = now;
+                this.gf.updateEntity(rec);
+            }
             return newRelations;
         });
     }
     async addObservations(observations) {
-        return this.withLock(async () => {
-            const graph = await this.loadGraph();
-            const results = observations.map(o => {
-                const entity = graph.entities.find(e => e.name === o.entityName);
-                if (!entity) {
+        return this.withWriteLock(() => {
+            const results = [];
+            for (const o of observations) {
+                const offset = this.nameIndex.get(o.entityName);
+                if (offset === undefined) {
                     throw new Error(`Entity with name ${o.entityName} not found`);
                 }
-                // Validate observation character limits
                 for (const obs of o.contents) {
                     if (obs.length > 140) {
                         throw new Error(`Observation for "${o.entityName}" exceeds 140 characters (${obs.length} chars): "${obs.substring(0, 50)}..."`);
                     }
                 }
-                const newObservations = o.contents.filter(content => !entity.observations.includes(content));
-                // Validate total observation count
-                if (entity.observations.length + newObservations.length > 2) {
-                    throw new Error(`Adding ${newObservations.length} observations to "${o.entityName}" would exceed limit of 2 (currently has ${entity.observations.length}).`);
+                const rec = this.gf.readEntity(offset);
+                const existingObs = [];
+                if (rec.obs0Id !== 0)
+                    existingObs.push(this.st.get(BigInt(rec.obs0Id)));
+                if (rec.obs1Id !== 0)
+                    existingObs.push(this.st.get(BigInt(rec.obs1Id)));
+                const newObservations = o.contents.filter(content => !existingObs.includes(content));
+                if (existingObs.length + newObservations.length > 2) {
+                    throw new Error(`Adding ${newObservations.length} observations to "${o.entityName}" would exceed limit of 2 (currently has ${existingObs.length}).`);
                 }
-                entity.observations.push(...newObservations);
-                if (newObservations.length > 0) {
-                    const now = Date.now();
-                    entity.mtime = now;
-                    entity.obsMtime = now;
+                const now = BigInt(Date.now());
+                for (const obs of newObservations) {
+                    this.gf.addObservation(offset, obs, now);
                 }
-                return { entityName: o.entityName, addedObservations: newObservations };
-            });
-            await this.saveGraph(graph);
+                results.push({ entityName: o.entityName, addedObservations: newObservations });
+            }
             return results;
         });
     }
     async deleteEntities(entityNames) {
-        return this.withLock(async () => {
-            const graph = await this.loadGraph();
-            graph.entities = graph.entities.filter(e => !entityNames.includes(e.name));
-            graph.relations = graph.relations.filter(r => !entityNames.includes(r.from) && !entityNames.includes(r.to));
-            await this.saveGraph(graph);
+        this.withWriteLock(() => {
+            for (const name of entityNames) {
+                const offset = this.nameIndex.get(name);
+                if (offset === undefined)
+                    continue;
+                const edges = this.gf.getEdges(offset);
+                for (const edge of edges) {
+                    const reverseDir = edge.direction === DIR_FORWARD ? DIR_BACKWARD : DIR_FORWARD;
+                    this.gf.removeEdge(edge.targetOffset, offset, edge.relTypeId, reverseDir);
+                    this.st.release(BigInt(edge.relTypeId));
+                }
+                this.gf.deleteEntity(offset);
+                this.nameIndex.delete(name);
+            }
         });
     }
     async deleteObservations(deletions) {
-        return this.withLock(async () => {
-            const graph = await this.loadGraph();
-            const now = Date.now();
-            deletions.forEach(d => {
-                const entity = graph.entities.find(e => e.name === d.entityName);
-                if (entity) {
-                    const originalLen = entity.observations.length;
-                    entity.observations = entity.observations.filter(o => !d.observations.includes(o));
-                    if (entity.observations.length !== originalLen) {
-                        entity.mtime = now;
-                        entity.obsMtime = now;
-                    }
+        this.withWriteLock(() => {
+            const now = BigInt(Date.now());
+            for (const d of deletions) {
+                const offset = this.nameIndex.get(d.entityName);
+                if (offset === undefined)
+                    continue;
+                for (const obs of d.observations) {
+                    this.gf.removeObservation(offset, obs, now);
                 }
-            });
-            await this.saveGraph(graph);
+            }
         });
     }
     async deleteRelations(relations) {
-        return this.withLock(async () => {
-            const graph = await this.loadGraph();
-            graph.relations = graph.relations.filter(r => !relations.some(delRelation => r.from === delRelation.from &&
-                r.to === delRelation.to &&
-                r.relationType === delRelation.relationType));
-            await this.saveGraph(graph);
+        this.withWriteLock(() => {
+            for (const r of relations) {
+                const fromOffset = this.nameIndex.get(r.from);
+                const toOffset = this.nameIndex.get(r.to);
+                if (fromOffset === undefined || toOffset === undefined)
+                    continue;
+                const relTypeId = this.st.find(r.relationType);
+                if (relTypeId === null)
+                    continue;
+                const removedForward = this.gf.removeEdge(fromOffset, toOffset, Number(relTypeId), DIR_FORWARD);
+                if (removedForward)
+                    this.st.release(relTypeId);
+                const removedBackward = this.gf.removeEdge(toOffset, fromOffset, Number(relTypeId), DIR_BACKWARD);
+                if (removedBackward)
+                    this.st.release(relTypeId);
+            }
         });
     }
     // Regex-based search function
-    async searchNodes(query, sortBy, sortDir) {
-        const graph = await this.loadGraph();
+    async searchNodes(query, sortBy, sortDir, direction = 'forward') {
         let regex;
         try {
-            regex = new RegExp(query, 'i'); // case-insensitive
+            regex = new RegExp(query, 'i');
         }
         catch (e) {
             throw new Error(`Invalid regex pattern: ${query}`);
         }
-        // Filter entities
-        const filteredEntities = graph.entities.filter(e => regex.test(e.name) ||
-            regex.test(e.entityType) ||
-            e.observations.some(o => regex.test(o)));
-        // Create a Set of filtered entity names for quick lookup
-        const filteredEntityNames = new Set(filteredEntities.map(e => e.name));
-        // Filter relations to only include those between filtered entities
-        const filteredRelations = graph.relations.filter(r => filteredEntityNames.has(r.from) && filteredEntityNames.has(r.to));
-        const filteredGraph = {
-            entities: sortEntities(filteredEntities, sortBy, sortDir),
-            relations: filteredRelations,
-        };
-        return filteredGraph;
-    }
-    async openNodesFiltered(names) {
-        const graph = await this.loadGraph();
-        // Filter entities
-        const filteredEntities = graph.entities.filter(e => names.includes(e.name));
-        // Create a Set of filtered entity names for quick lookup
-        const filteredEntityNames = new Set(filteredEntities.map(e => e.name));
-        // Filter relations to only include those between filtered entities
-        const filteredRelations = graph.relations.filter(r => filteredEntityNames.has(r.from) && filteredEntityNames.has(r.to));
-        const filteredGraph = {
-            entities: filteredEntities,
-            relations: filteredRelations,
-        };
-        return filteredGraph;
-    }
-    async openNodes(names) {
-        const graph = await this.loadGraph();
-        // Filter entities
-        const filteredEntities = graph.entities.filter(e => names.includes(e.name));
-        // Create a Set of filtered entity names for quick lookup
-        const filteredEntityNames = new Set(filteredEntities.map(e => e.name));
-        // Filter relations to only include those between filtered entities
-        const filteredRelations = graph.relations.filter(r => filteredEntityNames.has(r.from));
-        const filteredGraph = {
-            entities: filteredEntities,
-            relations: filteredRelations,
-        };
-        return filteredGraph;
-    }
-    async getNeighbors(entityName, depth = 1, sortBy, sortDir) {
-        const graph = await this.loadGraph();
-        const visited = new Set();
-        const neighborNames = new Set();
-        const traverse = (currentName, currentDepth) => {
-            if (currentDepth > depth || visited.has(currentName))
-                return;
-            visited.add(currentName);
-            // Find all relations involving this entity
-            const connectedRelations = graph.relations.filter(r => r.from === currentName || r.to === currentName);
-            // Collect neighbor names
-            connectedRelations.forEach(r => {
-                const neighborName = r.from === currentName ? r.to : r.from;
-                neighborNames.add(neighborName);
+        return this.withReadLock(() => {
+            const allEntities = this.getAllEntities();
+            const filteredEntities = allEntities.filter(e => regex.test(e.name) ||
+                regex.test(e.entityType) ||
+                e.observations.some(o => regex.test(o)));
+            const filteredEntityNames = new Set(filteredEntities.map(e => e.name));
+            const allRelations = this.getAllRelations();
+            const filteredRelations = allRelations.filter(r => {
+                if (direction === 'forward')
+                    return filteredEntityNames.has(r.from);
+                if (direction === 'backward')
+                    return filteredEntityNames.has(r.to);
+                return filteredEntityNames.has(r.from) && filteredEntityNames.has(r.to);
             });
-            if (currentDepth < depth) {
-                // Traverse to connected entities
-                connectedRelations.forEach(r => {
-                    const nextEntity = r.from === currentName ? r.to : r.from;
-                    traverse(nextEntity, currentDepth + 1);
-                });
-            }
-        };
-        traverse(entityName, 0);
-        // Remove the starting entity from neighbors (it's not its own neighbor)
-        neighborNames.delete(entityName);
-        // Build neighbor objects with timestamps
-        const entityMap = new Map(graph.entities.map(e => [e.name, e]));
-        const neighbors = Array.from(neighborNames).map(name => {
-            const entity = entityMap.get(name);
+            const rankMaps = this.getRankMapsUnlocked();
             return {
-                name,
-                mtime: entity?.mtime,
-                obsMtime: entity?.obsMtime,
+                entities: sortEntities(filteredEntities, sortBy, sortDir, rankMaps),
+                relations: filteredRelations,
             };
         });
-        return sortNeighbors(neighbors, sortBy, sortDir);
     }
-    async findPath(fromEntity, toEntity, maxDepth = 5) {
-        const graph = await this.loadGraph();
-        const visited = new Set();
-        const dfs = (current, target, path, depth) => {
-            if (depth > maxDepth || visited.has(current))
-                return null;
-            if (current === target)
-                return path;
-            visited.add(current);
-            const outgoingRelations = graph.relations.filter(r => r.from === current);
-            for (const relation of outgoingRelations) {
-                const result = dfs(relation.to, target, [...path, relation], depth + 1);
-                if (result)
-                    return result;
+    async openNodes(names, direction = 'forward') {
+        return this.withReadLock(() => {
+            const filteredEntities = [];
+            for (const name of names) {
+                const offset = this.nameIndex.get(name);
+                if (offset === undefined)
+                    continue;
+                filteredEntities.push(this.recordToEntity(this.gf.readEntity(offset)));
             }
-            visited.delete(current);
-            return null;
-        };
-        return dfs(fromEntity, toEntity, [], 0) || [];
+            const filteredEntityNames = new Set(filteredEntities.map(e => e.name));
+            const filteredRelations = [];
+            for (const name of filteredEntityNames) {
+                const offset = this.nameIndex.get(name);
+                const edges = this.gf.getEdges(offset);
+                for (const edge of edges) {
+                    if (edge.direction !== DIR_FORWARD && edge.direction !== DIR_BACKWARD)
+                        continue;
+                    const targetRec = this.gf.readEntity(edge.targetOffset);
+                    const targetName = this.st.get(BigInt(targetRec.nameId));
+                    const relationType = this.st.get(BigInt(edge.relTypeId));
+                    const mtime = Number(edge.mtime);
+                    if (edge.direction === DIR_FORWARD) {
+                        if (direction === 'backward')
+                            continue;
+                        const r = { from: name, to: targetName, relationType };
+                        if (mtime > 0)
+                            r.mtime = mtime;
+                        filteredRelations.push(r);
+                    }
+                    else {
+                        if (direction === 'forward')
+                            continue;
+                        if (direction === 'any' && !filteredEntityNames.has(targetName))
+                            continue;
+                        const r = { from: targetName, to: name, relationType };
+                        if (mtime > 0)
+                            r.mtime = mtime;
+                        filteredRelations.push(r);
+                    }
+                }
+            }
+            return { entities: filteredEntities, relations: filteredRelations };
+        });
+    }
+    async getNeighbors(entityName, depth = 1, sortBy, sortDir, direction = 'forward') {
+        return this.withReadLock(() => {
+            const startOffset = this.nameIndex.get(entityName);
+            if (startOffset === undefined)
+                return [];
+            const visited = new Set();
+            const neighborNames = new Set();
+            const traverse = (currentName, currentDepth) => {
+                if (currentDepth > depth || visited.has(currentName))
+                    return;
+                visited.add(currentName);
+                const offset = this.nameIndex.get(currentName);
+                if (offset === undefined)
+                    return;
+                const edges = this.gf.getEdges(offset);
+                for (const edge of edges) {
+                    if (direction === 'forward' && edge.direction !== DIR_FORWARD)
+                        continue;
+                    if (direction === 'backward' && edge.direction !== DIR_BACKWARD)
+                        continue;
+                    const targetRec = this.gf.readEntity(edge.targetOffset);
+                    const neighborName = this.st.get(BigInt(targetRec.nameId));
+                    neighborNames.add(neighborName);
+                    if (currentDepth < depth) {
+                        traverse(neighborName, currentDepth + 1);
+                    }
+                }
+            };
+            traverse(entityName, 0);
+            neighborNames.delete(entityName);
+            const neighbors = Array.from(neighborNames).map(name => {
+                const offset = this.nameIndex.get(name);
+                if (!offset)
+                    return { name };
+                const rec = this.gf.readEntity(offset);
+                const mtime = Number(rec.mtime);
+                const obsMtime = Number(rec.obsMtime);
+                const n = { name };
+                if (mtime > 0)
+                    n.mtime = mtime;
+                if (obsMtime > 0)
+                    n.obsMtime = obsMtime;
+                return n;
+            });
+            const rankMaps = this.getRankMapsUnlocked();
+            return sortNeighbors(neighbors, sortBy, sortDir, rankMaps);
+        });
+    }
+    async findPath(fromEntity, toEntity, maxDepth = 5, direction = 'forward') {
+        return this.withReadLock(() => {
+            const visited = new Set();
+            const dfs = (current, target, pathSoFar, depth) => {
+                if (depth > maxDepth || visited.has(current))
+                    return null;
+                if (current === target)
+                    return pathSoFar;
+                visited.add(current);
+                const offset = this.nameIndex.get(current);
+                if (offset === undefined) {
+                    visited.delete(current);
+                    return null;
+                }
+                const edges = this.gf.getEdges(offset);
+                for (const edge of edges) {
+                    if (direction === 'forward' && edge.direction !== DIR_FORWARD)
+                        continue;
+                    if (direction === 'backward' && edge.direction !== DIR_BACKWARD)
+                        continue;
+                    const targetRec = this.gf.readEntity(edge.targetOffset);
+                    const nextName = this.st.get(BigInt(targetRec.nameId));
+                    const relationType = this.st.get(BigInt(edge.relTypeId));
+                    const mtime = Number(edge.mtime);
+                    let rel;
+                    if (edge.direction === DIR_FORWARD) {
+                        rel = { from: current, to: nextName, relationType };
+                    }
+                    else {
+                        rel = { from: nextName, to: current, relationType };
+                    }
+                    if (mtime > 0)
+                        rel.mtime = mtime;
+                    const result = dfs(nextName, target, [...pathSoFar, rel], depth + 1);
+                    if (result)
+                        return result;
+                }
+                visited.delete(current);
+                return null;
+            };
+            return dfs(fromEntity, toEntity, [], 0) || [];
+        });
     }
     async getEntitiesByType(entityType, sortBy, sortDir) {
-        const graph = await this.loadGraph();
-        const filtered = graph.entities.filter(e => e.entityType === entityType);
-        return sortEntities(filtered, sortBy, sortDir);
+        return this.withReadLock(() => {
+            const filtered = this.getAllEntities().filter(e => e.entityType === entityType);
+            const rankMaps = this.getRankMapsUnlocked();
+            return sortEntities(filtered, sortBy, sortDir, rankMaps);
+        });
     }
     async getEntityTypes() {
-        const graph = await this.loadGraph();
-        const types = new Set(graph.entities.map(e => e.entityType));
-        return Array.from(types).sort();
+        return this.withReadLock(() => {
+            const types = new Set(this.getAllEntities().map(e => e.entityType));
+            return Array.from(types).sort();
+        });
     }
     async getRelationTypes() {
-        const graph = await this.loadGraph();
-        const types = new Set(graph.relations.map(r => r.relationType));
-        return Array.from(types).sort();
+        return this.withReadLock(() => {
+            const types = new Set(this.getAllRelations().map(r => r.relationType));
+            return Array.from(types).sort();
+        });
     }
     async getStats() {
-        const graph = await this.loadGraph();
-        const entityTypes = new Set(graph.entities.map(e => e.entityType));
-        const relationTypes = new Set(graph.relations.map(r => r.relationType));
-        return {
-            entityCount: graph.entities.length,
-            relationCount: graph.relations.length,
-            entityTypes: entityTypes.size,
-            relationTypes: relationTypes.size
-        };
+        return this.withReadLock(() => {
+            const entities = this.getAllEntities();
+            const relations = this.getAllRelations();
+            const entityTypes = new Set(entities.map(e => e.entityType));
+            const relationTypes = new Set(relations.map(r => r.relationType));
+            return {
+                entityCount: entities.length,
+                relationCount: relations.length,
+                entityTypes: entityTypes.size,
+                relationTypes: relationTypes.size,
+            };
+        });
     }
     async getOrphanedEntities(strict = false, sortBy, sortDir) {
-        const graph = await this.loadGraph();
-        if (!strict) {
-            // Simple mode: entities with no relations at all
-            const connectedEntityNames = new Set();
-            graph.relations.forEach(r => {
-                connectedEntityNames.add(r.from);
-                connectedEntityNames.add(r.to);
+        return this.withReadLock(() => {
+            const entities = this.getAllEntities();
+            if (!strict) {
+                const connectedEntityNames = new Set();
+                const relations = this.getAllRelations();
+                relations.forEach(r => {
+                    connectedEntityNames.add(r.from);
+                    connectedEntityNames.add(r.to);
+                });
+                const orphans = entities.filter(e => !connectedEntityNames.has(e.name));
+                const rankMaps = this.getRankMapsUnlocked();
+                return sortEntities(orphans, sortBy, sortDir, rankMaps);
+            }
+            const neighbors = new Map();
+            entities.forEach(e => neighbors.set(e.name, new Set()));
+            const relations = this.getAllRelations();
+            relations.forEach(r => {
+                neighbors.get(r.from)?.add(r.to);
+                neighbors.get(r.to)?.add(r.from);
             });
-            const orphans = graph.entities.filter(e => !connectedEntityNames.has(e.name));
-            return sortEntities(orphans, sortBy, sortDir);
-        }
-        // Strict mode: entities not connected to "Self" (directly or indirectly)
-        // Build adjacency list (bidirectional)
-        const neighbors = new Map();
-        graph.entities.forEach(e => neighbors.set(e.name, new Set()));
-        graph.relations.forEach(r => {
-            neighbors.get(r.from)?.add(r.to);
-            neighbors.get(r.to)?.add(r.from);
-        });
-        // BFS from Self to find all connected entities
-        const connectedToSelf = new Set();
-        const queue = ['Self'];
-        while (queue.length > 0) {
-            const current = queue.shift();
-            if (connectedToSelf.has(current))
-                continue;
-            connectedToSelf.add(current);
-            const currentNeighbors = neighbors.get(current);
-            if (currentNeighbors) {
-                for (const neighbor of currentNeighbors) {
-                    if (!connectedToSelf.has(neighbor)) {
-                        queue.push(neighbor);
+            const connectedToSelf = new Set();
+            const queue = ['Self'];
+            while (queue.length > 0) {
+                const current = queue.shift();
+                if (connectedToSelf.has(current))
+                    continue;
+                connectedToSelf.add(current);
+                const currentNeighbors = neighbors.get(current);
+                if (currentNeighbors) {
+                    for (const neighbor of currentNeighbors) {
+                        if (!connectedToSelf.has(neighbor)) {
+                            queue.push(neighbor);
+                        }
                     }
                 }
             }
-        }
-        // Return entities not connected to Self (excluding Self itself if it exists)
-        const orphans = graph.entities.filter(e => !connectedToSelf.has(e.name));
-        return sortEntities(orphans, sortBy, sortDir);
+            const orphans = entities.filter(e => !connectedToSelf.has(e.name));
+            const rankMaps = this.getRankMapsUnlocked();
+            return sortEntities(orphans, sortBy, sortDir, rankMaps);
+        });
     }
     async validateGraph() {
-        const graph = await this.loadGraph();
-        const entityNames = new Set(graph.entities.map(e => e.name));
-        const missingEntities = new Set();
-        const observationViolations = [];
-        // Check for missing entities in relations
-        graph.relations.forEach(r => {
-            if (!entityNames.has(r.from)) {
-                missingEntities.add(r.from);
-            }
-            if (!entityNames.has(r.to)) {
-                missingEntities.add(r.to);
-            }
-        });
-        // Check for observation limit violations
-        graph.entities.forEach(e => {
-            const oversizedObservations = [];
-            e.observations.forEach((obs, idx) => {
-                if (obs.length > 140) {
-                    oversizedObservations.push(idx);
-                }
+        return this.withReadLock(() => {
+            const entities = this.getAllEntities();
+            const relations = this.getAllRelations();
+            const entityNames = new Set(entities.map(e => e.name));
+            const missingEntities = new Set();
+            const observationViolations = [];
+            relations.forEach(r => {
+                if (!entityNames.has(r.from))
+                    missingEntities.add(r.from);
+                if (!entityNames.has(r.to))
+                    missingEntities.add(r.to);
             });
-            if (e.observations.length > 2 || oversizedObservations.length > 0) {
-                observationViolations.push({
-                    entity: e.name,
-                    count: e.observations.length,
-                    oversizedObservations
+            entities.forEach(e => {
+                const oversizedObservations = [];
+                e.observations.forEach((obs, idx) => {
+                    if (obs.length > 140)
+                        oversizedObservations.push(idx);
                 });
-            }
+                if (e.observations.length > 2 || oversizedObservations.length > 0) {
+                    observationViolations.push({
+                        entity: e.name,
+                        count: e.observations.length,
+                        oversizedObservations,
+                    });
+                }
+            });
+            return { missingEntities: Array.from(missingEntities), observationViolations };
         });
-        return {
-            missingEntities: Array.from(missingEntities),
-            observationViolations
-        };
     }
-    // BCL (Binary Combinatory Logic) evaluator
-    async evaluateBCL(program, maxSteps) {
-        let stepCount = 0;
-        let max_size = program.length;
-        let mode = 0;
-        let ctr = 1;
-        let t0 = program;
-        let t1 = '';
-        let t2 = '';
-        let t3 = '';
-        let t4 = '';
-        while (stepCount < maxSteps) {
-            if (t0.length == 0)
-                break;
-            let b = t0[0];
-            t0 = t0.slice(1);
-            if (mode === 0) {
-                t1 += b;
-                let size = t1.length + t0.length;
-                if (size > max_size)
-                    max_size = size;
-                if (t1.slice(-4) === '1100') {
-                    mode = 1;
-                    t1 = t1.slice(0, -4);
-                }
-                else if (t1.slice(-5) === '11101') {
-                    mode = 3;
-                    t1 = t1.slice(0, -5);
-                }
+    async randomWalk(start, depth = 3, seed, direction = 'forward') {
+        return this.withReadLock(() => {
+            const startOffset = this.nameIndex.get(start);
+            if (!startOffset) {
+                throw new Error(`Start entity not found: ${start}`);
             }
-            else if (mode === 1) {
-                t2 += b;
-                if (b == '1') {
-                    ctr += 1;
+            // Create seeded RNG if seed provided
+            let rngState = seed ? this.hashSeed(seed) : null;
+            const random = () => {
+                if (rngState !== null) {
+                    rngState ^= rngState << 13;
+                    rngState ^= rngState >>> 17;
+                    rngState ^= rngState << 5;
+                    return (rngState >>> 0) / 0xFFFFFFFF;
                 }
-                else if (b == '0') {
-                    ctr -= 1;
-                    t2 += t0[0];
-                    t0 = t0.slice(1);
+                else {
+                    return randomBytes(4).readUInt32BE() / 0xFFFFFFFF;
                 }
-                if (ctr === 0) {
-                    mode = 2;
-                    ctr = 1;
+            };
+            const pathNames = [start];
+            let current = start;
+            for (let i = 0; i < depth; i++) {
+                const offset = this.nameIndex.get(current);
+                if (!offset)
+                    break;
+                const edges = this.gf.getEdges(offset);
+                const validNeighbors = new Set();
+                for (const edge of edges) {
+                    if (direction === 'forward' && edge.direction !== DIR_FORWARD)
+                        continue;
+                    if (direction === 'backward' && edge.direction !== DIR_BACKWARD)
+                        continue;
+                    const targetRec = this.gf.readEntity(edge.targetOffset);
+                    const neighborName = this.st.get(BigInt(targetRec.nameId));
+                    if (neighborName !== current)
+                        validNeighbors.add(neighborName);
                 }
+                const neighborArr = Array.from(validNeighbors).filter(n => this.nameIndex.has(n));
+                if (neighborArr.length === 0)
+                    break;
+                const idx = Math.floor(random() * neighborArr.length);
+                current = neighborArr[idx];
+                pathNames.push(current);
             }
-            else if (mode === 2) {
-                if (b == '1') {
-                    ctr += 1;
-                }
-                else if (b == '0') {
-                    ctr -= 1;
-                    t0 = t0.slice(1);
-                }
-                if (ctr === 0) {
-                    t0 = t2 + t0;
-                    t2 = '';
-                    mode = 0;
-                    ctr = 1;
-                    stepCount += 1;
-                }
+            return { entity: current, path: pathNames };
+        });
+    }
+    hashSeed(seed) {
+        let hash = 0;
+        for (let i = 0; i < seed.length; i++) {
+            const char = seed.charCodeAt(i);
+            hash = ((hash << 5) - hash) + char;
+            hash = hash & hash;
+        }
+        return hash || 1;
+    }
+    decodeTimestamp(timestamp, relative = false) {
+        const ts = timestamp ?? Date.now();
+        const date = new Date(ts);
+        const result = {
+            timestamp: ts,
+            iso8601: date.toISOString(),
+            formatted: date.toUTCString(),
+        };
+        if (relative) {
+            const now = Date.now();
+            const diffMs = now - ts;
+            const diffSec = Math.abs(diffMs) / 1000;
+            const diffMin = diffSec / 60;
+            const diffHour = diffMin / 60;
+            const diffDay = diffHour / 24;
+            let relStr;
+            if (diffSec < 60) {
+                relStr = `${Math.floor(diffSec)} seconds`;
             }
-            else if (mode === 3) {
-                t2 += b;
-                if (b == '1') {
-                    ctr += 1;
-                }
-                else if (b == '0') {
-                    ctr -= 1;
-                    t2 += t0[0];
-                    t0 = t0.slice(1);
-                }
-                if (ctr === 0) {
-                    mode = 4;
-                    ctr = 1;
-                }
+            else if (diffMin < 60) {
+                relStr = `${Math.floor(diffMin)} minutes`;
             }
-            else if (mode === 4) {
-                t3 += b;
-                if (b == '1') {
-                    ctr += 1;
-                }
-                else if (b == '0') {
-                    ctr -= 1;
-                    t3 += t0[0];
-                    t0 = t0.slice(1);
-                }
-                if (ctr === 0) {
-                    mode = 5;
-                    ctr = 1;
-                }
+            else if (diffHour < 24) {
+                relStr = `${Math.floor(diffHour)} hours`;
             }
-            else if (mode === 5) {
-                t4 += b;
-                if (b == '1') {
-                    ctr += 1;
-                }
-                else if (b == '0') {
-                    ctr -= 1;
-                    t4 += t0[0];
-                    t0 = t0.slice(1);
-                }
-                if (ctr === 0) {
-                    t0 = '11' + t2 + t4 + '1' + t3 + t4 + t0;
-                    t2 = '';
-                    t3 = '';
-                    t4 = '';
-                    mode = 0;
-                    ctr = 1;
-                    stepCount += 1;
-                }
+            else if (diffDay < 30) {
+                relStr = `${Math.floor(diffDay)} days`;
             }
+            else if (diffDay < 365) {
+                relStr = `${Math.floor(diffDay / 30)} months`;
+            }
+            else {
+                relStr = `${Math.floor(diffDay / 365)} years`;
+            }
+            result.relative = diffMs >= 0 ? `${relStr} ago` : `in ${relStr}`;
         }
-        const halted = stepCount < maxSteps;
-        return {
-            result: t1,
-            info: `${stepCount} steps, max size ${max_size}`,
-            halted,
-            errored: halted && mode != 0,
-        };
-    }
-    async addBCLTerm(term) {
-        const termset = ["1", "00", "01"];
-        if (!term || term.trim() === "") {
-            throw new Error("BCL term cannot be empty");
-        }
-        // Term can be 1, 00, 01, or K, S, App (application)
-        const validTerms = ["1", "App", "00", "K", "01", "S"];
-        if (!validTerms.includes(term)) {
-            throw new Error(`Invalid BCL term: ${term}\nExpected one of: ${validTerms.join(", ")}`);
-        }
-        let processedTerm = 0;
-        if (term === "00" || term === "K")
-            processedTerm = 1;
-        else if (term === "01" || term === "S")
-            processedTerm = 2;
-        this.bclTerm += termset[processedTerm];
-        if (processedTerm === 0) {
-            if (this.bclCtr === 0)
-                this.bclCtr += 1;
-            this.bclCtr += 1;
-        }
-        else {
-            this.bclCtr -= 1;
-        }
-        if (this.bclCtr <= 0) {
-            const constructedProgram = this.bclTerm;
-            this.bclCtr = 0;
-            this.bclTerm = "";
-            return `Constructed Program: ${constructedProgram}`;
-        }
-        else {
-            return `Need ${this.bclCtr} more term(s) to complete the program.`;
-        }
-    }
-    async clearBCLTerm() {
-        this.bclCtr = 0;
-        this.bclTerm = "";
+        return result;
     }
     async addThought(observations, previousCtxId) {
-        return this.withLock(async () => {
-            const graph = await this.loadGraph();
-            // Validate observations
-            if (observations.length > 2) {
-                throw new Error(`Thought has ${observations.length} observations. Maximum allowed is 2.`);
+        // Validate observations (can do outside lock)
+        if (observations.length > 2) {
+            throw new Error(`Thought has ${observations.length} observations. Maximum allowed is 2.`);
+        }
+        for (const obs of observations) {
+            if (obs.length > 140) {
+                throw new Error(`Observation exceeds 140 characters (${obs.length} chars): "${obs.substring(0, 50)}..."`);
             }
+        }
+        return this.withWriteLock(() => {
+            const now = BigInt(Date.now());
+            const ctxId = randomBytes(12).toString('hex');
+            const obsMtime = observations.length > 0 ? now : 0n;
+            const rec = this.gf.createEntity(ctxId, 'Thought', now, obsMtime);
             for (const obs of observations) {
-                if (obs.length > 140) {
-                    throw new Error(`Observation exceeds 140 characters (${obs.length} chars): "${obs.substring(0, 50)}..."`);
-                }
+                this.gf.addObservation(rec.offset, obs, now);
             }
-            // Generate new context ID (24-char hex)
-            const now = Date.now();
-            const ctxId = randomBytes(12).toString('hex');
-            // Create thought entity
-            const thoughtEntity = {
-                name: ctxId,
-                entityType: "Thought",
-                observations,
-                mtime: now,
-                obsMtime: observations.length > 0 ? now : undefined,
-            };
-            graph.entities.push(thoughtEntity);
-            // Link to previous thought if it exists
+            if (observations.length > 0) {
+                const updated = this.gf.readEntity(rec.offset);
+                updated.mtime = now;
+                updated.obsMtime = now;
+                this.gf.updateEntity(updated);
+            }
+            this.nameIndex.set(ctxId, rec.offset);
             if (previousCtxId) {
-                const prevEntity = graph.entities.find(e => e.name === previousCtxId);
-                if (prevEntity) {
-                    // Update mtime on previous entity since we're adding a relation from it
-                    prevEntity.mtime = now;
-                    // Bidirectional chain: previous -> new (follows) and new -> previous (preceded_by)
-                    graph.relations.push({ from: previousCtxId, to: ctxId, relationType: "follows", mtime: now }, { from: ctxId, to: previousCtxId, relationType: "preceded_by", mtime: now });
+                const prevOffset = this.nameIndex.get(previousCtxId);
+                if (prevOffset !== undefined) {
+                    const prevRec = this.gf.readEntity(prevOffset);
+                    prevRec.mtime = now;
+                    this.gf.updateEntity(prevRec);
+                    const followsTypeId = Number(this.st.intern('follows'));
+                    this.gf.addEdge(prevOffset, { targetOffset: rec.offset, direction: DIR_FORWARD, relTypeId: followsTypeId, mtime: now });
+                    const followsTypeId2 = Number(this.st.intern('follows'));
+                    this.gf.addEdge(rec.offset, { targetOffset: prevOffset, direction: DIR_BACKWARD, relTypeId: followsTypeId2, mtime: now });
+                    const precededByTypeId = Number(this.st.intern('preceded_by'));
+                    this.gf.addEdge(rec.offset, { targetOffset: prevOffset, direction: DIR_FORWARD, relTypeId: precededByTypeId, mtime: now });
+                    const precededByTypeId2 = Number(this.st.intern('preceded_by'));
+                    this.gf.addEdge(prevOffset, { targetOffset: rec.offset, direction: DIR_BACKWARD, relTypeId: precededByTypeId2, mtime: now });
                 }
             }
-            await this.saveGraph(graph);
             return { ctxId };
         });
     }
+    /** Close the underlying binary store files */
+    close() {
+        this.gf.close();
+        this.st.close();
+    }
 }
 /**
  * Creates a configured MCP server instance with all tools registered.
@@ -714,12 +950,16 @@ export function createServer(memoryFilePath) {
                 sizes: ["any"]
             }
         ],
-        version: "0.0.10",
+        version: "0.0.12",
     }, {
         capabilities: {
             tools: {},
         },
     });
+    // Close binary store on server close
+    server.onclose = () => {
+        knowledgeGraphManager.close();
+    };
     server.setRequestHandler(ListToolsRequestSchema, async () => {
         return {
             tools: [
@@ -867,7 +1107,8 @@ export function createServer(memoryFilePath) {
                         type: "object",
                         properties: {
                             query: { type: "string", description: "Regex pattern to match against entity names, types, and observations." },
-                            sortBy: { type: "string", enum: ["mtime", "obsMtime", "name"], description: "Sort field for entities. Omit for insertion order." },
+                            direction: { type: "string", enum: ["forward", "backward", "any"], description: "Edge direction filter for returned relations. Default: forward" },
+                            sortBy: { type: "string", enum: ["mtime", "obsMtime", "name", "pagerank", "llmrank"], description: "Sort field for entities. Omit for insertion order." },
                             sortDir: { type: "string", enum: ["asc", "desc"], description: "Sort direction. Default: desc for timestamps, asc for name." },
                             entityCursor: { type: "number", description: "Cursor for entity pagination (from previous response's nextCursor)" },
                             relationCursor: { type: "number", description: "Cursor for relation pagination" },
@@ -875,23 +1116,6 @@ export function createServer(memoryFilePath) {
                         required: ["query"],
                     },
                 },
-                {
-                    name: "open_nodes_filtered",
-                    description: "Open specific nodes in the knowledge graph by their names, filtering relations to only those between the opened nodes. Results are paginated (max 512 chars).",
-                    inputSchema: {
-                        type: "object",
-                        properties: {
-                            names: {
-                                type: "array",
-                                items: { type: "string" },
-                                description: "An array of entity names to retrieve",
-                            },
-                            entityCursor: { type: "number", description: "Cursor for entity pagination" },
-                            relationCursor: { type: "number", description: "Cursor for relation pagination" },
-                        },
-                        required: ["names"],
-                    },
-                },
                 {
                     name: "open_nodes",
                     description: "Open specific nodes in the knowledge graph by their names. Results are paginated (max 512 chars).",
@@ -903,6 +1127,7 @@ export function createServer(memoryFilePath) {
                                 items: { type: "string" },
                                 description: "An array of entity names to retrieve",
                             },
+                            direction: { type: "string", enum: ["forward", "backward", "any"], description: "Edge direction filter for returned relations. Default: forward" },
                             entityCursor: { type: "number", description: "Cursor for entity pagination" },
                             relationCursor: { type: "number", description: "Cursor for relation pagination" },
                         },
@@ -917,7 +1142,8 @@ export function createServer(memoryFilePath) {
                         properties: {
                             entityName: { type: "string", description: "The name of the entity to find neighbors for" },
                             depth: { type: "number", description: "Maximum depth to traverse (default: 1)", default: 1 },
-                            sortBy: { type: "string", enum: ["mtime", "obsMtime", "name"], description: "Sort field for neighbors. Omit for arbitrary order." },
+                            direction: { type: "string", enum: ["forward", "backward", "any"], description: "Edge direction to follow. Default: forward" },
+                            sortBy: { type: "string", enum: ["mtime", "obsMtime", "name", "pagerank", "llmrank"], description: "Sort field for neighbors. Omit for arbitrary order." },
                             sortDir: { type: "string", enum: ["asc", "desc"], description: "Sort direction. Default: desc for timestamps, asc for name." },
                             cursor: { type: "number", description: "Cursor for pagination" },
                         },
@@ -933,6 +1159,7 @@ export function createServer(memoryFilePath) {
                             fromEntity: { type: "string", description: "The name of the starting entity" },
                             toEntity: { type: "string", description: "The name of the target entity" },
                             maxDepth: { type: "number", description: "Maximum depth to search (default: 5)", default: 5 },
+                            direction: { type: "string", enum: ["forward", "backward", "any"], description: "Edge direction to follow. Default: forward" },
                             cursor: { type: "number", description: "Cursor for pagination" },
                         },
                         required: ["fromEntity", "toEntity"],
@@ -945,7 +1172,7 @@ export function createServer(memoryFilePath) {
                         type: "object",
                         properties: {
                             entityType: { type: "string", description: "The type of entities to retrieve" },
-                            sortBy: { type: "string", enum: ["mtime", "obsMtime", "name"], description: "Sort field for entities. Omit for insertion order." },
+                            sortBy: { type: "string", enum: ["mtime", "obsMtime", "name", "pagerank", "llmrank"], description: "Sort field for entities. Omit for insertion order." },
                             sortDir: { type: "string", enum: ["asc", "desc"], description: "Sort direction. Default: desc for timestamps, asc for name." },
                             cursor: { type: "number", description: "Cursor for pagination" },
                         },
@@ -983,7 +1210,7 @@ export function createServer(memoryFilePath) {
                         type: "object",
                         properties: {
                             strict: { type: "boolean", description: "If true, returns entities not connected to 'Self' (directly or indirectly). Default: false" },
-                            sortBy: { type: "string", enum: ["mtime", "obsMtime", "name"], description: "Sort field for entities. Omit for insertion order." },
+                            sortBy: { type: "string", enum: ["mtime", "obsMtime", "name", "pagerank", "llmrank"], description: "Sort field for entities. Omit for insertion order." },
                             sortDir: { type: "string", enum: ["asc", "desc"], description: "Sort direction. Default: desc for timestamps, asc for name." },
                             cursor: { type: "number", description: "Cursor for pagination" },
                         },
@@ -998,37 +1225,28 @@ export function createServer(memoryFilePath) {
                     },
                 },
                 {
-                    name: "evaluate_bcl",
-                    description: "Evaluate a Binary Combinatory Logic (BCL) program",
+                    name: "decode_timestamp",
+                    description: "Decode a millisecond timestamp to human-readable UTC format. If no timestamp provided, returns the current time. Use this to interpret mtime/obsMtime values from entities.",
                     inputSchema: {
                         type: "object",
                         properties: {
-                            program: { type: "string", description: "The BCL program as a binary string (syntax: T:=00|01|1TT) 00=K, 01=S, 1=application." },
-                            maxSteps: { type: "number", description: "Maximum number of reduction steps to perform (default: 1000000)", default: 1000000 },
+                            timestamp: { type: "number", description: "Millisecond timestamp to decode. If omitted, returns current time." },
+                            relative: { type: "boolean", description: "If true, include relative time (e.g., '3 days ago'). Default: false" },
                         },
-                        required: ["program"],
                     },
                 },
                 {
-                    name: "add_bcl_term",
-                    description: "Add a BCL term to the constructor, maintaining valid syntax. Returns completion status.",
+                    name: "random_walk",
+                    description: "Perform a random walk from a starting entity, following random relations. Returns the terminal entity name and the path taken. Useful for serendipitous exploration of the knowledge graph.",
                     inputSchema: {
                         type: "object",
                         properties: {
-                            term: {
-                                type: "string",
-                                description: "BCL term to add. Valid values: '1' or 'App' (application), '00' or 'K' (K combinator), '01' or 'S' (S combinator)"
-                            },
+                            start: { type: "string", description: "Name of the entity to start the walk from." },
+                            depth: { type: "number", description: "Number of steps to take. Default: 3" },
+                            seed: { type: "string", description: "Optional seed for reproducible walks." },
+                            direction: { type: "string", enum: ["forward", "backward", "any"], description: "Edge direction to follow. Default: forward" },
                         },
-                        required: ["term"],
-                    },
-                },
-                {
-                    name: "clear_bcl_term",
-                    description: "Clear the current BCL term being constructed and reset the constructor state",
-                    inputSchema: {
-                        type: "object",
-                        properties: {},
+                        required: ["start"],
                     },
                 },
                 {
@@ -1062,39 +1280,49 @@ Use this to build chains of reasoning that persist in the graph. Each thought ca
             throw new Error(`No arguments provided for tool: ${name}`);
         }
         switch (name) {
-            case "create_entities":
-                return { content: [{ type: "text", text: JSON.stringify(await knowledgeGraphManager.createEntities(args.entities), null, 2) }] };
-            case "create_relations":
-                return { content: [{ type: "text", text: JSON.stringify(await knowledgeGraphManager.createRelations(args.relations), null, 2) }] };
+            case "create_entities": {
+                const result = await knowledgeGraphManager.createEntities(args.entities);
+                knowledgeGraphManager.resample(); // Re-run structural sampling after graph mutation
+                return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
+            }
+            case "create_relations": {
+                const result = await knowledgeGraphManager.createRelations(args.relations);
+                knowledgeGraphManager.resample(); // Re-run structural sampling after graph mutation
+                return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
+            }
             case "add_observations":
                 return { content: [{ type: "text", text: JSON.stringify(await knowledgeGraphManager.addObservations(args.observations), null, 2) }] };
             case "delete_entities":
                 await knowledgeGraphManager.deleteEntities(args.entityNames);
+                knowledgeGraphManager.resample(); // Re-run structural sampling after graph mutation
                 return { content: [{ type: "text", text: "Entities deleted successfully" }] };
             case "delete_observations":
                 await knowledgeGraphManager.deleteObservations(args.deletions);
                 return { content: [{ type: "text", text: "Observations deleted successfully" }] };
             case "delete_relations":
                 await knowledgeGraphManager.deleteRelations(args.relations);
+                knowledgeGraphManager.resample(); // Re-run structural sampling after graph mutation
                 return { content: [{ type: "text", text: "Relations deleted successfully" }] };
             case "search_nodes": {
-                const graph = await knowledgeGraphManager.searchNodes(args.query, args.sortBy, args.sortDir);
-                return { content: [{ type: "text", text: JSON.stringify(paginateGraph(graph, args.entityCursor ?? 0, args.relationCursor ?? 0)) }] };
-            }
-            case "open_nodes_filtered": {
-                const graph = await knowledgeGraphManager.openNodesFiltered(args.names);
+                const graph = await knowledgeGraphManager.searchNodes(args.query, args.sortBy, args.sortDir, args.direction ?? 'forward');
+                // Record walker visits for entities that will be returned to the LLM
+                knowledgeGraphManager.recordWalkerVisits(graph.entities.map(e => e.name));
                 return { content: [{ type: "text", text: JSON.stringify(paginateGraph(graph, args.entityCursor ?? 0, args.relationCursor ?? 0)) }] };
             }
             case "open_nodes": {
-                const graph = await knowledgeGraphManager.openNodes(args.names);
+                const graph = await knowledgeGraphManager.openNodes(args.names, args.direction ?? 'forward');
+                // Record walker visits for opened nodes
+                knowledgeGraphManager.recordWalkerVisits(graph.entities.map(e => e.name));
                 return { content: [{ type: "text", text: JSON.stringify(paginateGraph(graph, args.entityCursor ?? 0, args.relationCursor ?? 0)) }] };
             }
             case "get_neighbors": {
-                const neighbors = await knowledgeGraphManager.getNeighbors(args.entityName, args.depth ?? 1, args.sortBy, args.sortDir);
+                const neighbors = await knowledgeGraphManager.getNeighbors(args.entityName, args.depth ?? 1, args.sortBy, args.sortDir, args.direction ?? 'forward');
+                // Record walker visits for returned neighbors
+                knowledgeGraphManager.recordWalkerVisits(neighbors.map(n => n.name));
                 return { content: [{ type: "text", text: JSON.stringify(paginateItems(neighbors, args.cursor ?? 0)) }] };
             }
             case "find_path": {
-                const path = await knowledgeGraphManager.findPath(args.fromEntity, args.toEntity, args.maxDepth);
+                const path = await knowledgeGraphManager.findPath(args.fromEntity, args.toEntity, args.maxDepth, args.direction ?? 'forward');
                 return { content: [{ type: "text", text: JSON.stringify(paginateItems(path, args.cursor ?? 0)) }] };
             }
             case "get_entities_by_type": {
@@ -1113,13 +1341,12 @@ Use this to build chains of reasoning that persist in the graph. Each thought ca
             }
             case "validate_graph":
                 return { content: [{ type: "text", text: JSON.stringify(await knowledgeGraphManager.validateGraph(), null, 2) }] };
-            case "evaluate_bcl":
-                return { content: [{ type: "text", text: JSON.stringify(await knowledgeGraphManager.evaluateBCL(args.program, args.maxSteps), null, 2) }] };
-            case "add_bcl_term":
-                return { content: [{ type: "text", text: await knowledgeGraphManager.addBCLTerm(args.term) }] };
-            case "clear_bcl_term":
-                await knowledgeGraphManager.clearBCLTerm();
-                return { content: [{ type: "text", text: "BCL term constructor cleared successfully" }] };
+            case "decode_timestamp":
+                return { content: [{ type: "text", text: JSON.stringify(knowledgeGraphManager.decodeTimestamp(args.timestamp, args.relative ?? false)) }] };
+            case "random_walk": {
+                const result = await knowledgeGraphManager.randomWalk(args.start, args.depth ?? 3, args.seed, args.direction ?? 'forward');
+                return { content: [{ type: "text", text: JSON.stringify(result) }] };
+            }
             case "sequentialthinking": {
                 const result = await knowledgeGraphManager.addThought(args.observations, args.previousCtxId);
                 return { content: [{ type: "text", text: JSON.stringify(result) }] };