causantic 0.2.0

package/dist/storage/pruner.js

@@ -0,0 +1,280 @@
+/**
+ * Lazy pruning system for dead edges.
+ * Edges with weight <= 0 at query time are queued for async deletion.
+ * Chunks that lose all edges are marked for TTL-based cleanup (not deleted).
+ *
+ * Two pruning modes:
+ * 1. Lazy: Queued during traversal, flushed after debounce
+ * 2. Full: Background scan on startup, removes all dead edges
+ *
+ * Chunks remain in the database after losing edges so they can still be
+ * found via vector and keyword search. Their vectors are marked orphaned
+ * to start the TTL countdown; the cleanup-vectors task handles eventual
+ * deletion of both vector and chunk after TTL expires.
+ */
+import { deleteEdge, hasAnyEdges, getAllEdges } from './edge-store.js';
+import { vectorStore } from './vector-store.js';
+import { getReferenceClock } from './clock-store.js';
+import { calculateDirectionalDecayWeight } from './decay.js';
+import { deserialize } from '../temporal/vector-clock.js';
+import { createLogger } from '../utils/logger.js';
+const log = createLogger('pruner');
+/**
+ * Pruner class for managing lazy deletion of dead edges.
+ * Chunks that lose all edges are marked for TTL cleanup, not deleted.
+ */
+class Pruner {
+    pendingEdges = new Set();
+    isRunning = false;
+    flushTimeout = null;
+    flushDelayMs = 1000;
+    /**
+     * Queue an edge for pruning.
+     * Called during edge traversal when weight <= 0.
+     */
+    queueEdgePrune(edgeId) {
+        this.pendingEdges.add(edgeId);
+        this.scheduleFlush();
+    }
+    /**
+     * Queue multiple edges for pruning.
+     */
+    queueEdgePruneBatch(edgeIds) {
+        for (const id of edgeIds) {
+            this.pendingEdges.add(id);
+        }
+        this.scheduleFlush();
+    }
+    /**
+     * Get count of pending edges.
+     */
+    getPendingCount() {
+        return this.pendingEdges.size;
+    }
+    /**
+     * Force immediate flush (for testing or shutdown).
+     */
+    async flushNow() {
+        if (this.flushTimeout) {
+            clearTimeout(this.flushTimeout);
+            this.flushTimeout = null;
+        }
+        return this.flush();
+    }
+    /**
+     * Schedule a debounced flush.
+     */
+    scheduleFlush() {
+        if (this.flushTimeout) {
+            return; // Already scheduled
+        }
+        this.flushTimeout = setTimeout(() => {
+            this.flushTimeout = null;
+            this.flush().catch((err) => {
+                log.error('Pruner flush error', { error: err instanceof Error ? err.message : String(err) });
+            });
+        }, this.flushDelayMs);
+    }
+    /**
+     * Execute the pruning operation.
+     */
+    async flush() {
+        if (this.isRunning || this.pendingEdges.size === 0) {
+            return { edgesDeleted: 0, chunksOrphaned: 0 };
+        }
+        this.isRunning = true;
+        const result = { edgesDeleted: 0, chunksOrphaned: 0 };
+        try {
+            const edgeIds = [...this.pendingEdges];
+            this.pendingEdges.clear();
+            // Track chunks that might lose all edges
+            const chunkIdsToCheck = new Set();
+            for (const edgeId of edgeIds) {
+                // Get edge info before deleting (we need source/target)
+                const { getEdgeById } = await import('./edge-store.js');
+                const edge = getEdgeById(edgeId);
+                if (!edge) {
+                    continue; // Already deleted
+                }
+                // Delete the edge
+                const deleted = deleteEdge(edgeId);
+                if (deleted) {
+                    result.edgesDeleted++;
+                    chunkIdsToCheck.add(edge.sourceChunkId);
+                    chunkIdsToCheck.add(edge.targetChunkId);
+                }
+            }
+            // Mark chunks that lost all edges for TTL cleanup
+            for (const chunkId of chunkIdsToCheck) {
+                if (!hasAnyEdges(chunkId)) {
+                    await vectorStore.markOrphaned(chunkId);
+                    result.chunksOrphaned++;
+                }
+            }
+        }
+        finally {
+            this.isRunning = false;
+        }
+        return result;
+    }
+    /**
+     * Set flush delay (for testing).
+     */
+    setFlushDelay(ms) {
+        this.flushDelayMs = ms;
+    }
+    /**
+     * Check if pruner is currently running.
+     */
+    isActive() {
+        return this.isRunning;
+    }
+    /**
+     * Clear pending queue without flushing (for testing).
+     */
+    clearPending() {
+        this.pendingEdges.clear();
+        if (this.flushTimeout) {
+            clearTimeout(this.flushTimeout);
+            this.flushTimeout = null;
+        }
+    }
+    // ─── Full Background Prune ────────────────────────────────────────
+    fullPruneRunning = false;
+    fullPruneProgress = null;
+    /**
+     * Start a full background prune (non-blocking).
+     * Scans all edges, removes dead ones, marks orphaned chunks for TTL.
+     * Idempotent - if already running, returns existing progress.
+     */
+    startBackgroundPrune() {
+        if (this.fullPruneProgress && this.fullPruneRunning) {
+            return this.fullPruneProgress;
+        }
+        this.fullPruneProgress = {
+            status: 'running',
+            edgesScanned: 0,
+            edgesDeleted: 0,
+            chunksScanned: 0,
+            chunksOrphaned: 0,
+            startedAt: Date.now(),
+            completedAt: null,
+            error: null,
+        };
+        // Run in background - don't await
+        this.runFullPrune().catch((err) => {
+            if (this.fullPruneProgress) {
+                this.fullPruneProgress.status = 'error';
+                this.fullPruneProgress.error = err.message;
+                this.fullPruneProgress.completedAt = Date.now();
+            }
+            log.error('Full prune error', { error: err instanceof Error ? err.message : String(err) });
+        });
+        return this.fullPruneProgress;
+    }
+    /**
+     * Get current full prune progress (null if never started).
+     */
+    getFullPruneProgress() {
+        return this.fullPruneProgress;
+    }
+    /**
+     * Check if full prune is currently running.
+     */
+    isFullPruneRunning() {
+        return this.fullPruneRunning;
+    }
+    /**
+     * Run the full prune operation.
+     * Scans all edges, calculates weights, removes dead edges, and marks
+     * chunks that lose all edges for TTL cleanup.
+     */
+    async runFullPrune() {
+        if (this.fullPruneRunning) {
+            return;
+        }
+        this.fullPruneRunning = true;
+        const progress = this.fullPruneProgress;
+        try {
+            // Build reference clocks for all projects
+            const refClocks = new Map();
+            // Get all edges
+            const edges = getAllEdges();
+            const deadEdgeIds = [];
+            const chunkIdsToCheck = new Set();
+            // Scan edges and identify dead ones
+            for (const edge of edges) {
+                progress.edgesScanned++;
+                // Get reference clock for this edge's project (lazy load)
+                let refClock = refClocks.get(edge.sourceChunkId);
+                if (!refClock) {
+                    // Try to get from chunk's session
+                    const { getChunkById } = await import('./chunk-store.js');
+                    const chunk = getChunkById(edge.sourceChunkId);
+                    if (chunk) {
+                        refClock = getReferenceClock(chunk.sessionSlug);
+                        refClocks.set(edge.sourceChunkId, refClock);
+                    }
+                }
+                // Calculate weight
+                let weight = edge.initialWeight;
+                if (refClock && edge.vectorClock) {
+                    const edgeClock = deserialize(edge.vectorClock);
+                    const direction = edge.edgeType === 'forward' ? 'forward' : 'backward';
+                    weight = edge.initialWeight * calculateDirectionalDecayWeight(edgeClock, refClock, direction);
+                }
+                // Queue dead edges
+                if (weight <= 0) {
+                    deadEdgeIds.push(edge.id);
+                    chunkIdsToCheck.add(edge.sourceChunkId);
+                    chunkIdsToCheck.add(edge.targetChunkId);
+                }
+                // Yield periodically to avoid blocking
+                if (progress.edgesScanned % 500 === 0) {
+                    await new Promise((resolve) => setImmediate(resolve));
+                }
+            }
+            // Delete dead edges in batches
+            const batchSize = 100;
+            for (let i = 0; i < deadEdgeIds.length; i += batchSize) {
+                const batch = deadEdgeIds.slice(i, i + batchSize);
+                for (const edgeId of batch) {
+                    if (deleteEdge(edgeId)) {
+                        progress.edgesDeleted++;
+                    }
+                }
+                // Yield after each batch
+                await new Promise((resolve) => setImmediate(resolve));
+            }
+            // Mark chunks that lost all edges for TTL cleanup
+            for (const chunkId of chunkIdsToCheck) {
+                progress.chunksScanned++;
+                if (!hasAnyEdges(chunkId)) {
+                    await vectorStore.markOrphaned(chunkId);
+                    progress.chunksOrphaned++;
+                }
+                // Yield periodically
+                if (progress.chunksScanned % 100 === 0) {
+                    await new Promise((resolve) => setImmediate(resolve));
+                }
+            }
+            progress.status = 'completed';
+            progress.completedAt = Date.now();
+        }
+        finally {
+            this.fullPruneRunning = false;
+        }
+    }
+}
+// Export class for testing (allows creating fresh instances)
+export { Pruner };
+// Singleton instance for production use
+export const pruner = new Pruner();
+/**
+ * Start background pruning on module load if environment variable is set.
+ * This allows the MCP server or CLI to trigger startup pruning.
+ */
+export function initStartupPrune() {
+    return pruner.startBackgroundPrune();
+}
+//# sourceMappingURL=pruner.js.map

package/dist/storage/pruner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pruner.js","sourceRoot":"","sources":["../../src/storage/pruner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AACvE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,+BAA+B,EAAsB,MAAM,YAAY,CAAC;AACjF,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;AAEnC;;;GAGG;AACH,MAAM,MAAM;IACF,YAAY,GAAgB,IAAI,GAAG,EAAE,CAAC;IACtC,SAAS,GAAG,KAAK,CAAC;IAClB,YAAY,GAAyC,IAAI,CAAC;IAC1D,YAAY,GAAG,IAAI,CAAC;IAE5B;;;OAGG;IACH,cAAc,CAAC,MAAc;QAC3B,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC9B,IAAI,CAAC,aAAa,EAAE,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,mBAAmB,CAAC,OAAiB;QACnC,KAAK,MAAM,EAAE,IAAI,OAAO,EAAE,CAAC;YACzB,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5B,CAAC;QACD,IAAI,CAAC,aAAa,EAAE,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,eAAe;QACb,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ;QACZ,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAChC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC3B,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,EAAE,CAAC;IACtB,CAAC;IAED;;OAEG;IACK,aAAa;QACnB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,OAAO,CAAC,oBAAoB;QAC9B,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,EAAE;YAClC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;YACzB,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACzB,GAAG,CAAC,KAAK,CAAC,oBAAoB,EAAE,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC/F,CAAC,CAAC,CAAC;QACL,CAAC,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACxB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,KAAK;QACjB,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YACnD,OAAO,EAAE,YAAY,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC;QAChD,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,MAAM,MAAM,GAAgB,EAAE,YAAY,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC;QAEnE,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC;YACvC,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;YAE1B,yCAAyC;YACzC,MAAM,eAAe,GAAG,IAAI,GAAG,EAAU,CAAC;YAE1C,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,wDAAwD;gBACxD,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,CAAC;gBACxD,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;gBAEjC,IAAI,CAAC,IAAI,EAAE,CAAC;oBACV,SAAS,CAAC,kBAAkB;gBAC9B,CAAC;gBAED,kBAAkB;gBAClB,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;gBACnC,IAAI,OAAO,EAAE,CAAC;oBACZ,MAAM,CAAC,YAAY,EAAE,CAAC;oBACtB,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;oBACxC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBAC1C,CAAC;YACH,CAAC;YAED,kDAAkD;YAClD,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;gBACtC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC1B,MAAM,WAAW,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;oBACxC,MAAM,CAAC,cAAc,EAAE,CAAC;gBAC1B,CAAC;YACH,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACzB,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,EAAU;QACtB,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED;;OAEG;IACH,YAAY;QACV,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAChC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,qEAAqE;IAE7D,gBAAgB,GAAG,KAAK,CAAC;IACzB,iBAAiB,GAA6B,IAAI,CAAC;IAE3D;;;;OAIG;IACH,oBAAoB;QAClB,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACpD,OAAO,IAAI,CAAC,iBAAiB,CAAC;QAChC,CAAC;QAED,IAAI,CAAC,iBAAiB,GAAG;YACvB,MAAM,EAAE,SAAS;YACjB,YAAY,EAAE,CAAC;YACf,YAAY,EAAE,CAAC;YACf,aAAa,EAAE,CAAC;YAChB,cAAc,EAAE,CAAC;YACjB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,WAAW,EAAE,IAAI;YACjB,KAAK,EAAE,IAAI;SACZ,CAAC;QAEF,kCAAkC;QAClC,IAAI,CAAC,YAAY,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YAChC,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBAC3B,IAAI,CAAC,iBAAiB,CAAC,MAAM,GAAG,OAAO,CAAC;gBACxC,IAAI,CAAC,iBAAiB,CAAC,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC;gBAC3C,IAAI,CAAC,iBAAiB,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAClD,CAAC;YACD,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC7F,CAAC,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,oBAAoB;QAClB,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,kBAAkB;QAChB,OAAO,IAAI,CAAC,gBAAgB,CAAC;IAC/B,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,YAAY;QACxB,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,OAAO;QACT,CAAC;QAED,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,iBAAkB,CAAC;QAEzC,IAAI,CAAC;YACH,0CAA0C;YAC1C,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkC,CAAC;YAE5D,gBAAgB;YAChB,MAAM,KAAK,GAAG,WAAW,EAAE,CAAC;YAC5B,MAAM,WAAW,GAAa,EAAE,CAAC;YACjC,MAAM,eAAe,GAAG,IAAI,GAAG,EAAU,CAAC;YAE1C,oCAAoC;YACpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,QAAQ,CAAC,YAAY,EAAE,CAAC;gBAExB,0DAA0D;gBAC1D,IAAI,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBACjD,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,kCAAkC;oBAClC,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;oBAC1D,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;oBAC/C,IAAI,KAAK,EAAE,CAAC;wBACV,QAAQ,GAAG,iBAAiB,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;wBAChD,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;oBAC9C,CAAC;gBACH,CAAC;gBAED,mBAAmB;gBACnB,IAAI,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC;gBAChC,IAAI,QAAQ,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;oBACjC,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;oBAChD,MAAM,SAAS,GAAkB,IAAI,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC;oBACtF,MAAM,GAAG,IAAI,CAAC,aAAa,GAAG,+BAA+B,CAAC,SAAS,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;gBAChG,CAAC;gBAED,mBAAmB;gBACnB,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;oBAChB,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;oBAC1B,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;oBACxC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBAC1C,CAAC;gBAED,uCAAuC;gBACvC,IAAI,QAAQ,CAAC,YAAY,GAAG,GAAG,KAAK,CAAC,EAAE,CAAC;oBACtC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;gBACxD,CAAC;YACH,CAAC;YAED,+BAA+B;YAC/B,MAAM,SAAS,GAAG,GAAG,CAAC;YACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;gBACvD,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;gBAClD,KAAK,MAAM,MAAM,IAAI,KAAK,EAAE,CAAC;oBAC3B,IAAI,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;wBACvB,QAAQ,CAAC,YAAY,EAAE,CAAC;oBAC1B,CAAC;gBACH,CAAC;gBACD,yBAAyB;gBACzB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;YACxD,CAAC;YAED,kDAAkD;YAClD,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;gBACtC,QAAQ,CAAC,aAAa,EAAE,CAAC;gBAEzB,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC1B,MAAM,WAAW,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;oBACxC,QAAQ,CAAC,cAAc,EAAE,CAAC;gBAC5B,CAAC;gBAED,qBAAqB;gBACrB,IAAI,QAAQ,CAAC,aAAa,GAAG,GAAG,KAAK,CAAC,EAAE,CAAC;oBACvC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;gBACxD,CAAC;YACH,CAAC;YAED,QAAQ,CAAC,MAAM,GAAG,WAAW,CAAC;YAC9B,QAAQ,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACpC,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;QAChC,CAAC;IACH,CAAC;CACF;AAwBD,6DAA6D;AAC7D,OAAO,EAAE,MAAM,EAAE,CAAC;AAElB,wCAAwC;AACxC,MAAM,CAAC,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;AAEnC;;;GAGG;AACH,MAAM,UAAU,gBAAgB;IAC9B,OAAO,MAAM,CAAC,oBAAoB,EAAE,CAAC;AACvC,CAAC"}

package/dist/storage/schema-loader.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Schema SQL loading and statement splitting.
+ *
+ * Extracted from db.ts for clarity. Handles reading schema.sql and splitting
+ * it into individual statements, respecting BEGIN...END blocks (triggers).
+ */
+/**
+ * Load and parse the schema SQL file into individual statements.
+ *
+ * Splits on semicolons at line ends, but keeps BEGIN...END blocks (triggers)
+ * intact as single statements.
+ */
+export declare function loadSchemaStatements(): string[];
+/**
+ * Split SQL text into individual statements, respecting BEGIN...END blocks.
+ */
+export declare function splitStatements(sql: string): string[];
+//# sourceMappingURL=schema-loader.d.ts.map

package/dist/storage/schema-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-loader.d.ts","sourceRoot":"","sources":["../../src/storage/schema-loader.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAQH;;;;;GAKG;AACH,wBAAgB,oBAAoB,IAAI,MAAM,EAAE,CAI/C;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAyCrD"}

package/dist/storage/schema-loader.js

@@ -0,0 +1,62 @@
+/**
+ * Schema SQL loading and statement splitting.
+ *
+ * Extracted from db.ts for clarity. Handles reading schema.sql and splitting
+ * it into individual statements, respecting BEGIN...END blocks (triggers).
+ */
+import { readFileSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+const __dirname = dirname(fileURLToPath(import.meta.url));
+/**
+ * Load and parse the schema SQL file into individual statements.
+ *
+ * Splits on semicolons at line ends, but keeps BEGIN...END blocks (triggers)
+ * intact as single statements.
+ */
+export function loadSchemaStatements() {
+    const schemaPath = join(__dirname, 'schema.sql');
+    const schema = readFileSync(schemaPath, 'utf-8');
+    return splitStatements(schema);
+}
+/**
+ * Split SQL text into individual statements, respecting BEGIN...END blocks.
+ */
+export function splitStatements(sql) {
+    const statements = [];
+    let current = '';
+    let inTrigger = false;
+    for (const line of sql.split('\n')) {
+        const trimmed = line.trim();
+        // Skip pure comment lines when starting a new statement
+        if (!current && trimmed.startsWith('--'))
+            continue;
+        current += (current ? '\n' : '') + line;
+        // Detect entering a trigger body
+        if (/\bBEGIN\s*$/i.test(trimmed)) {
+            inTrigger = true;
+        }
+        // Detect end of trigger (END; on its own line)
+        if (inTrigger && /^END\s*;/i.test(trimmed)) {
+            inTrigger = false;
+            statements.push(current.trim());
+            current = '';
+            continue;
+        }
+        // Outside triggers, a semicolon at end of line ends the statement
+        if (!inTrigger && trimmed.endsWith(';')) {
+            const stmt = current.trim().replace(/;$/, '').trim();
+            if (stmt)
+                statements.push(stmt);
+            current = '';
+        }
+    }
+    // Handle any trailing statement
+    if (current.trim()) {
+        const stmt = current.trim().replace(/;$/, '').trim();
+        if (stmt)
+            statements.push(stmt);
+    }
+    return statements;
+}
+//# sourceMappingURL=schema-loader.js.map

package/dist/storage/schema-loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-loader.js","sourceRoot":"","sources":["../../src/storage/schema-loader.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AAEpC,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE1D;;;;;GAKG;AACH,MAAM,UAAU,oBAAoB;IAClC,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACjD,OAAO,eAAe,CAAC,MAAM,CAAC,CAAC;AACjC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,SAAS,GAAG,KAAK,CAAC;IAEtB,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACnC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAE5B,wDAAwD;QACxD,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,SAAS;QAEnD,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC;QAExC,iCAAiC;QACjC,IAAI,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,SAAS,GAAG,IAAI,CAAC;QACnB,CAAC;QAED,+CAA+C;QAC/C,IAAI,SAAS,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3C,SAAS,GAAG,KAAK,CAAC;YAClB,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;YAChC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS;QACX,CAAC;QAED,kEAAkE;QAClE,IAAI,CAAC,SAAS,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YACrD,IAAI,IAAI;gBAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAChC,OAAO,GAAG,EAAE,CAAC;QACf,CAAC;IACH,CAAC;IAED,gCAAgC;IAChC,IAAI,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;QACnB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACrD,IAAI,IAAI;YAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC"}

package/dist/storage/schema.sql

@@ -0,0 +1,133 @@
+-- Causantic Memory System Schema
+-- SQLite database for storing chunks, edges, and clusters
+
+-- Core entities: conversation chunks
+CREATE TABLE IF NOT EXISTS chunks (
+  id TEXT PRIMARY KEY,
+  session_id TEXT NOT NULL,
+  session_slug TEXT NOT NULL,
+  turn_indices TEXT NOT NULL,  -- JSON array of turn indices
+  start_time TEXT NOT NULL,    -- ISO timestamp
+  end_time TEXT NOT NULL,      -- ISO timestamp
+  content TEXT NOT NULL,       -- Full chunk text
+  code_block_count INTEGER DEFAULT 0,
+  tool_use_count INTEGER DEFAULT 0,
+  approx_tokens INTEGER DEFAULT 0,
+  created_at TEXT DEFAULT CURRENT_TIMESTAMP,
+  -- v2: Vector clock support
+  agent_id TEXT,               -- Agent that created this chunk (null = main UI agent)
+  vector_clock TEXT,           -- JSON: {"agentId": tick, ...}
+  spawn_depth INTEGER DEFAULT 0, -- Nesting level: 0=main, 1=sub-agent, 2=sub-sub-agent
+  project_path TEXT              -- Full cwd path for disambiguation
+);
+
+-- Clusters for topic grouping
+CREATE TABLE IF NOT EXISTS clusters (
+  id TEXT PRIMARY KEY,
+  name TEXT,                    -- Short cluster name
+  description TEXT,             -- LLM-generated description
+  centroid BLOB,                -- Float32Array serialized
+  exemplar_ids TEXT,            -- JSON array of exemplar chunk IDs
+  membership_hash TEXT,         -- For staleness detection
+  created_at TEXT DEFAULT CURRENT_TIMESTAMP,
+  refreshed_at TEXT             -- When description was last updated
+);
+
+-- Edges with decay
+CREATE TABLE IF NOT EXISTS edges (
+  id TEXT PRIMARY KEY,
+  source_chunk_id TEXT NOT NULL,
+  target_chunk_id TEXT NOT NULL,
+  edge_type TEXT NOT NULL,      -- 'backward' or 'forward'
+  reference_type TEXT,          -- 'file-path', 'code-entity', 'brief', 'debrief', etc.
+  initial_weight REAL NOT NULL DEFAULT 1.0,
+  created_at TEXT NOT NULL,
+  -- v2: Vector clock support
+  vector_clock TEXT,            -- JSON: {"agentId": tick, ...} for hop-based decay
+  link_count INTEGER DEFAULT 1, -- Number of times this edge was created (for boosting)
+  FOREIGN KEY (source_chunk_id) REFERENCES chunks(id) ON DELETE CASCADE,
+  FOREIGN KEY (target_chunk_id) REFERENCES chunks(id) ON DELETE CASCADE
+);
+
+-- Chunk-cluster membership (many-to-many)
+CREATE TABLE IF NOT EXISTS chunk_clusters (
+  chunk_id TEXT NOT NULL,
+  cluster_id TEXT NOT NULL,
+  distance REAL NOT NULL,       -- Angular distance to centroid
+  PRIMARY KEY (chunk_id, cluster_id),
+  FOREIGN KEY (chunk_id) REFERENCES chunks(id) ON DELETE CASCADE,
+  FOREIGN KEY (cluster_id) REFERENCES clusters(id) ON DELETE CASCADE
+);
+
+-- Indices for efficient querying
+CREATE INDEX IF NOT EXISTS idx_chunks_session ON chunks(session_id);
+CREATE INDEX IF NOT EXISTS idx_chunks_session_slug ON chunks(session_slug);
+CREATE INDEX IF NOT EXISTS idx_chunks_start_time ON chunks(start_time);
+CREATE INDEX IF NOT EXISTS idx_chunks_slug_start_time ON chunks(session_slug, start_time);
+CREATE INDEX IF NOT EXISTS idx_edges_source ON edges(source_chunk_id);
+CREATE INDEX IF NOT EXISTS idx_edges_target ON edges(target_chunk_id);
+CREATE INDEX IF NOT EXISTS idx_edges_type ON edges(edge_type);
+CREATE INDEX IF NOT EXISTS idx_chunk_clusters_cluster ON chunk_clusters(cluster_id);
+
+-- Vector clocks for logical time tracking
+CREATE TABLE IF NOT EXISTS vector_clocks (
+  id TEXT PRIMARY KEY,           -- "project:<slug>" or "agent:<slug>:<agentId>"
+  project_slug TEXT NOT NULL,
+  clock_data TEXT NOT NULL,      -- JSON: {"agentId": tick, ...}
+  updated_at TEXT DEFAULT CURRENT_TIMESTAMP
+);
+CREATE INDEX IF NOT EXISTS idx_vector_clocks_project ON vector_clocks(project_slug);
+
+-- Ingestion checkpoints for incremental ingestion
+CREATE TABLE IF NOT EXISTS ingestion_checkpoints (
+  session_id TEXT PRIMARY KEY,
+  project_slug TEXT NOT NULL,
+  last_turn_index INTEGER NOT NULL,
+  last_chunk_id TEXT,
+  vector_clock TEXT,
+  file_mtime TEXT,              -- ISO timestamp of file mtime at last ingest
+  updated_at TEXT DEFAULT CURRENT_TIMESTAMP
+);
+CREATE INDEX IF NOT EXISTS idx_ingestion_checkpoints_project ON ingestion_checkpoints(project_slug);
+
+-- Embedding cache for content-hash based caching
+CREATE TABLE IF NOT EXISTS embedding_cache (
+  content_hash TEXT NOT NULL,
+  model_id TEXT NOT NULL,
+  embedding BLOB NOT NULL,
+  created_at TEXT DEFAULT CURRENT_TIMESTAMP,
+  hit_count INTEGER DEFAULT 0,
+  PRIMARY KEY (content_hash, model_id)
+);
+CREATE INDEX IF NOT EXISTS idx_embedding_cache_model ON embedding_cache(model_id);
+
+-- Schema version for migrations
+CREATE TABLE IF NOT EXISTS schema_version (
+  version INTEGER PRIMARY KEY,
+  applied_at TEXT DEFAULT CURRENT_TIMESTAMP
+);
+
+-- Full-text search index (content-sync mode, porter stemming)
+CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
+  content,
+  content='chunks',
+  content_rowid='rowid',
+  tokenize='porter unicode61'
+);
+
+-- Keep FTS in sync with chunks table
+CREATE TRIGGER IF NOT EXISTS chunks_fts_insert AFTER INSERT ON chunks BEGIN
+  INSERT INTO chunks_fts(rowid, content) VALUES (new.rowid, new.content);
+END;
+
+CREATE TRIGGER IF NOT EXISTS chunks_fts_delete AFTER DELETE ON chunks BEGIN
+  INSERT INTO chunks_fts(chunks_fts, rowid, content) VALUES ('delete', old.rowid, old.content);
+END;
+
+CREATE TRIGGER IF NOT EXISTS chunks_fts_update AFTER UPDATE OF content ON chunks BEGIN
+  INSERT INTO chunks_fts(chunks_fts, rowid, content) VALUES ('delete', old.rowid, old.content);
+  INSERT INTO chunks_fts(rowid, content) VALUES (new.rowid, new.content);
+END;
+
+-- Insert initial version if not exists (v6 adds composite index for session reconstruction)
+INSERT OR IGNORE INTO schema_version (version) VALUES (6);