@mem-weave/server 0.2.0

package/dist/workers/consolidator.js

@@ -0,0 +1,201 @@
+import { transaction } from '../db/database.js';
+import { logger } from '../server/logger.js';
+/**
+ * Process-wide mutex: only one consolidation may run at a time.
+ * Without this, a manual `POST /api/v1/consolidate` triggered while
+ * the scheduler is also running would double-evict / double-promote
+ * the same memories and produce two `consolidation_runs` rows that
+ * share work. The scheduler AGENTS.md says "NEVER run two
+ * consolidations concurrently" — this enforces it.
+ *
+ * Not perfect (a second node wouldn't see this flag), but for the
+ * v1 single-node deployment the project targets, it's sufficient.
+ */
+let consolidationInFlight = false;
+export function isConsolidationRunning() {
+    return consolidationInFlight;
+}
+export function runConsolidation(db, tenantId, options = {}) {
+    if (consolidationInFlight) {
+        return {
+            promoted: 0,
+            evicted: 0,
+            merged: 0,
+            promotedIds: [],
+            evictedIds: [],
+            mergedPairs: [],
+            summary: 'Skipped: another consolidation is already running.'
+        };
+    }
+    consolidationInFlight = true;
+    try {
+        return runConsolidationInner(db, tenantId, options);
+    }
+    finally {
+        consolidationInFlight = false;
+    }
+}
+function runConsolidationInner(db, tenantId, options) {
+    const result = {
+        promoted: 0,
+        evicted: 0,
+        merged: 0,
+        promotedIds: [],
+        evictedIds: [],
+        mergedPairs: [],
+        summary: ''
+    };
+    const DAY = 24 * 60 * 60 * 1000;
+    try {
+        const now = Date.now();
+        // 1. Evict short-term: strength < 0.1 AND age > 7 days AND 0 access
+        const toEvict = db.prepare(`
+      SELECT id FROM memories
+      WHERE tenant_id = ? AND tier = 'short' AND deleted_at IS NULL
+        AND strength < 0.1 AND access_count = 0
+        AND (? - created_at) > ?
+    `).all(tenantId, now, 7 * DAY);
+        if (!options.dryRun && toEvict.length > 0) {
+            transaction(db, () => {
+                const stmt = db.prepare('UPDATE memories SET deleted_at = ?, eviction_reason = ? WHERE id = ?');
+                for (const row of toEvict) {
+                    stmt.run(now, 'low_strength_old_age', row.id);
+                }
+            });
+        }
+        result.evicted = toEvict.length;
+        result.evictedIds = toEvict.map((r) => r.id);
+        // 2. Promote short→medium: accessed >= 3 times recently OR importance >= 7
+        const toPromote = db.prepare(`
+      SELECT id FROM memories
+      WHERE tenant_id = ? AND tier = 'short' AND deleted_at IS NULL
+        AND ((access_count >= 3 AND (? - last_accessed_at) < ?) OR importance >= 7)
+    `).all(tenantId, now, 7 * DAY);
+        if (!options.dryRun && toPromote.length > 0) {
+            transaction(db, () => {
+                const stmt = db.prepare('UPDATE memories SET tier = ? WHERE id = ?');
+                for (const row of toPromote) {
+                    stmt.run('medium', row.id);
+                }
+            });
+        }
+        result.promoted = toPromote.length;
+        result.promotedIds = toPromote.map((r) => r.id);
+        // 3. Merge near-duplicates. Reuses the Jaccard-on-concepts logic
+        //    that `MemoryRepo.create` uses, but operating on existing rows.
+        //    For each pair (a, b) where a.id < b.id AND Jaccard(concepts) >= 0.8
+        //    AND same type AND same tenant, keep the older (a) and absorb b.
+        //
+        //    The same threshold is used as the live dedup gate, so a memory
+        //    that slipped through during a high-throughput write (e.g.
+        //    Jaccard = 0.75 because the new save only had partial concepts)
+        //    will still be merged here if the operator later enriches the
+        //    older memory's concepts.
+        if (!options.dryRun) {
+            const merged = mergeNearDuplicates(db, tenantId, now);
+            result.merged = merged.length;
+            result.mergedPairs = merged;
+        }
+        result.summary =
+            `Evicted ${result.evicted}, promoted ${result.promoted}, merged ${result.merged} pairs`;
+    }
+    catch (err) {
+        logger.error({ err }, 'consolidation failed');
+    }
+    return result;
+}
+/**
+ * Find pairs of near-duplicate memories (Jaccard on concepts >= 0.8,
+ * same type) and absorb the newer one into the older. Returns the list
+ * of [survivorId, absorbedId] pairs.
+ *
+ * This is the *background* counterpart to the live dedup gate in
+ * `MemoryRepo.create`. They use the same threshold and the same Jaccard
+ * formula, so behavior is consistent.
+ */
+function mergeNearDuplicates(db, tenantId, now) {
+    // Load all live memories for this tenant. In a real production system
+    // we'd page this; for v1 with O(thousands) memories per tenant, an
+    // in-memory O(n^2) pass is acceptable (a few seconds at most).
+    const allMemories = db.prepare(`
+    SELECT id, type, content, summary, importance, confidence,
+           concepts_json, files_json, created_at
+    FROM memories
+    WHERE tenant_id = ? AND deleted_at IS NULL
+    ORDER BY created_at ASC
+  `).all(tenantId);
+    const JACCARD_THRESHOLD = 0.8;
+    const SURVIVOR_BENEFIT = 0.05; // bump survivor's strength on absorb
+    const merged = [];
+    const absorbed = new Set();
+    for (let i = 0; i < allMemories.length; i++) {
+        const a = allMemories[i];
+        if (absorbed.has(a.id))
+            continue;
+        const aConcepts = new Set(JSON.parse(a.concepts_json).map((c) => c.toLowerCase()));
+        if (aConcepts.size === 0)
+            continue;
+        for (let j = i + 1; j < allMemories.length; j++) {
+            const b = allMemories[j];
+            if (absorbed.has(b.id))
+                continue;
+            if (a.type !== b.type)
+                continue;
+            const bConcepts = new Set(JSON.parse(b.concepts_json).map((c) => c.toLowerCase()));
+            const jaccard = jaccardSimilarity(aConcepts, bConcepts);
+            if (jaccard < JACCARD_THRESHOLD)
+                continue;
+            // a is older (ORDER BY created_at ASC) — keep a, absorb b.
+            // b's concepts/files are unioned into a; b is soft-deleted.
+            const mergedConcepts = Array.from(new Set([
+                ...aConcepts,
+                ...bConcepts
+            ]));
+            const mergedFiles = Array.from(new Set([
+                ...JSON.parse(a.files_json),
+                ...JSON.parse(b.files_json)
+            ]));
+            transaction(db, () => {
+                // Boost the survivor.
+                db.prepare(`
+          UPDATE memories
+          SET concepts_json = ?,
+              concepts_text = ?,
+              files_json = ?,
+              importance = MAX(importance, ?),
+              access_count = access_count + 1,
+              last_reinforced_at = ?,
+              reinforcement_score = min(1, reinforcement_score + ?),
+              strength = min(1, strength + ?),
+              updated_at = ?
+          WHERE id = ?
+        `).run(JSON.stringify(mergedConcepts), mergedConcepts.join(' '), JSON.stringify(mergedFiles), b.importance, now, SURVIVOR_BENEFIT, SURVIVOR_BENEFIT, now, a.id);
+                // Soft-delete the absorbed.
+                db.prepare(`
+          UPDATE memories
+          SET deleted_at = ?, eviction_reason = ?
+          WHERE id = ?
+        `).run(now, 'merged_into_consolidation', b.id);
+                // Audit log.
+                db.prepare(`
+          INSERT INTO consolidation_runs (id, tenant_id, started_at, ended_at, summary, dry_run)
+          SELECT ?, ?, ?, ?, ?, 0
+          WHERE NOT EXISTS (SELECT 1 FROM consolidation_runs WHERE id = ?)
+        `).run(`merge-${a.id}-${b.id}-${now}`, tenantId, now, now, `merged ${b.id} -> ${a.id}`, `merge-${a.id}-${b.id}-${now}`);
+            });
+            absorbed.add(b.id);
+            merged.push([a.id, b.id]);
+        }
+    }
+    return merged;
+}
+function jaccardSimilarity(a, b) {
+    if (a.size === 0 && b.size === 0)
+        return 0;
+    let intersection = 0;
+    for (const x of a)
+        if (b.has(x))
+            intersection++;
+    const union = a.size + b.size - intersection;
+    return union === 0 ? 0 : intersection / union;
+}

package/dist/workers/embedder.js

@@ -0,0 +1,102 @@
+import { openDatabase } from '../db/database.js';
+import { VectorRepo } from '../db/repositories/vector-repo.js';
+import { logger } from '../server/logger.js';
+/**
+ * Background embedder worker.
+ *
+ * Polls the `memories` table for rows that don't yet have a corresponding
+ * entry in the vec0 table, generates embeddings via the configured provider,
+ * and writes them to the vec table.
+ *
+ * When the embedding provider is `noop` (the default), this is a no-op for
+ * actual embeddings but the loop still runs so the structure is exercised.
+ */
+export function startEmbedderWorker(options) {
+    const tenantId = options.tenantId ?? 'tenant_default';
+    const batchSize = options.batchSize ?? 16;
+    const interval = options.intervalMs ?? 30_000;
+    let stopped = false;
+    let timer = null;
+    async function runOnce() {
+        const db = openDatabase(options.dbPath, { vectorDimensions: options.dimensions });
+        const result = { embedded: 0, skipped: 0, failed: 0, timestamp: Date.now() };
+        try {
+            const vecRepo = new VectorRepo(db, options.dimensions);
+            // Find candidate memories that lack a vector entry
+            const vecTable = `memory_vectors_${options.dimensions}`;
+            const candidates = db.prepare(`
+        SELECT m.id, m.tenant_id, m.title, m.content, m.summary, m.concepts_text
+        FROM memories m
+        LEFT JOIN ${vecTable} v ON v.memory_id = m.id
+        WHERE m.tenant_id = ? AND m.deleted_at IS NULL AND v.memory_id IS NULL
+        ORDER BY m.created_at ASC
+        LIMIT ?
+      `).all(tenantId, batchSize);
+            if (candidates.length > 0) {
+                const texts = candidates.map((m) => [m.title, m.summary, m.content, m.concepts_text ?? ''].filter(Boolean).join('\n'));
+                try {
+                    const vectors = await options.provider.embedBatch(texts);
+                    for (let i = 0; i < candidates.length; i++) {
+                        const m = candidates[i];
+                        const v = vectors[i];
+                        if (!v) {
+                            result.failed++;
+                            continue;
+                        }
+                        try {
+                            vecRepo.upsert(m.id, m.tenant_id, v);
+                            result.embedded++;
+                        }
+                        catch {
+                            result.failed++;
+                        }
+                    }
+                }
+                catch (err) {
+                    logger.error({ err: err.message }, 'batch embedding failed');
+                    result.failed = candidates.length;
+                }
+            }
+        }
+        finally {
+            db.close();
+        }
+        options.onRun?.(result);
+        return result;
+    }
+    function schedule() {
+        if (stopped)
+            return;
+        timer = setTimeout(async () => {
+            try {
+                await runOnce();
+            }
+            catch (err) {
+                logger.error({ err }, 'run failed');
+            }
+            schedule();
+        }, interval);
+    }
+    if (options.signal) {
+        if (options.signal.aborted)
+            stopped = true;
+        else
+            options.signal.addEventListener('abort', () => {
+                stopped = true;
+                if (timer)
+                    clearTimeout(timer);
+            });
+    }
+    if (options.runOnStart) {
+        void runOnce();
+    }
+    schedule();
+    return {
+        stop() {
+            stopped = true;
+            if (timer)
+                clearTimeout(timer);
+        },
+        runNow: runOnce
+    };
+}

package/dist/workers/graph-worker.js

@@ -0,0 +1,166 @@
+import { openDatabase } from '../db/database.js';
+import { EdgeRepo } from '../db/repositories/edge-repo.js';
+import { MemoryRepo } from '../db/repositories/memory-repo.js';
+import { logger } from '../server/logger.js';
+/**
+ * Background graph worker (edge discovery).
+ *
+ * Periodically:
+ *   1. Pull a batch of recent memories that have no outgoing edges yet.
+ *   2. For each, ask the LLM (via the edge-extract prompt) what edges it
+ *      should form with OTHER existing memories.
+ *   3. Persist the resulting edges.
+ *
+ * In v1, we use a simple heuristic + LLM call to find candidates:
+ *   - Use the noop or real LLM to extract edge candidates from the new
+ *     memory's title + content.
+ *   - Resolve `targetHint` to an existing memory by title or concept match.
+ *
+ * The worker never overwrites existing edges (skips when an edge of the same
+ * `(from, to, type)` already exists).
+ */
+export function startGraphWorker(options) {
+    const tenantId = options.tenantId ?? 'tenant_default';
+    const batchSize = options.batchSize ?? 32;
+    const interval = options.intervalMs ?? 60 * 60 * 1000;
+    let stopped = false;
+    let timer = null;
+    async function runOnce() {
+        const result = { scanned: 0, edgesCreated: 0, timestamp: Date.now() };
+        const db = openDatabase(options.dbPath);
+        try {
+            const memRepo = new MemoryRepo(db);
+            const edgeRepo = new EdgeRepo(db);
+            // Find recent memories that have no outgoing edges yet
+            const candidates = db.prepare(`
+        SELECT m.id, m.tenant_id, m.type, m.title, m.content, m.summary, m.concepts_json
+        FROM memories m
+        LEFT JOIN edges e ON e.from_memory_id = m.id
+        WHERE m.tenant_id = ? AND m.deleted_at IS NULL
+        GROUP BY m.id
+        HAVING COUNT(e.id) = 0
+        ORDER BY m.created_at DESC
+        LIMIT ?
+      `).all(tenantId, batchSize);
+            result.scanned = candidates.length;
+            // For each candidate, look up potential targets by simple keyword overlap
+            // (title contains a concept or vice versa). This is a simple heuristic to
+            // avoid burning LLM tokens on every pair; the LLM is only called to judge
+            // whether the candidate pair actually has a typed relationship.
+            for (const mem of candidates) {
+                const targets = findCandidateTargets(db, tenantId, mem);
+                for (const target of targets) {
+                    if (target.id === mem.id)
+                        continue;
+                    const llmCandidates = await extractEdgesViaLlm(options.llm, mem, target);
+                    for (const cand of llmCandidates) {
+                        if (cand.confidence < 0.6)
+                            continue;
+                        try {
+                            edgeRepo.create({
+                                tenantId: mem.tenant_id,
+                                fromMemoryId: mem.id,
+                                toMemoryId: target.id,
+                                type: cand.type,
+                                strength: cand.confidence,
+                                reason: cand.reason
+                            });
+                            result.edgesCreated++;
+                        }
+                        catch {
+                            // ignore duplicate / FK errors silently
+                        }
+                    }
+                }
+            }
+        }
+        finally {
+            db.close();
+        }
+        options.onRun?.(result);
+        return result;
+    }
+    function schedule() {
+        if (stopped)
+            return;
+        timer = setTimeout(async () => {
+            try {
+                await runOnce();
+            }
+            catch (err) {
+                logger.error({ err }, 'graph-worker run failed');
+            }
+            schedule();
+        }, interval);
+    }
+    if (options.signal) {
+        if (options.signal.aborted)
+            stopped = true;
+        else
+            options.signal.addEventListener('abort', () => {
+                stopped = true;
+                if (timer)
+                    clearTimeout(timer);
+            });
+    }
+    if (options.runOnStart) {
+        void runOnce();
+    }
+    schedule();
+    return {
+        stop() {
+            stopped = true;
+            if (timer)
+                clearTimeout(timer);
+        },
+        runNow: runOnce
+    };
+}
+function findCandidateTargets(db, tenantId, mem) {
+    // Cheap heuristic: pull up to 10 most recent memories of the same tenant
+    // whose title shares at least one word (>=4 chars) with the new memory.
+    // This is intentionally simple — the LLM call is the source of truth for
+    // whether a relationship actually exists.
+    const allRows = db.prepare(`
+    SELECT id, title, summary, concepts_json
+    FROM memories
+    WHERE tenant_id = ? AND id != ? AND deleted_at IS NULL
+    ORDER BY created_at DESC
+    LIMIT 50
+  `).all(tenantId, mem.id);
+    const memWords = tokenize(`${mem.title} ${mem.summary}`);
+    return allRows.filter((r) => {
+        const rWords = tokenize(`${r.title} ${r.summary}`);
+        return memWords.some((w) => rWords.includes(w));
+    }).slice(0, 10);
+}
+function tokenize(text) {
+    return text.toLowerCase().split(/[^a-z0-9\u4e00-\u9fff]+/i)
+        .filter((w) => w.length >= 4);
+}
+async function extractEdgesViaLlm(llm, newMemory, target) {
+    const newConcepts = JSON.parse(newMemory.concepts_json);
+    const targetConcepts = JSON.parse(target.concepts_json);
+    const prompt = `New memory:\nTitle: ${newMemory.title}\nContent: ${newMemory.content}\nConcepts: ${newConcepts.join(', ')}\n\nExisting memory:\nTitle: ${target.title}\nSummary: ${target.summary}\nConcepts: ${targetConcepts.join(', ')}`;
+    const raw = await llm.call('You are a relationship extraction engine. Given a new memory and an existing memory, return a JSON array of relationships (each with targetMemoryId="__target__", type, reason, confidence). Output [] if no relationship exists.', prompt);
+    if (!raw.trim())
+        return [];
+    try {
+        const parsed = JSON.parse(raw);
+        if (!Array.isArray(parsed))
+            return [];
+        const validTypes = new Set([
+            'causes', 'enables', 'contradicts', 'supersedes', 'references',
+            'related_to', 'before', 'after', 'duplicates', 'refines'
+        ]);
+        return parsed
+            .filter((e) => typeof e.type === 'string' && typeof e.reason === 'string' && typeof e.confidence === 'number')
+            .filter((e) => validTypes.has(e.type))
+            .map((e) => ({ type: e.type, reason: e.reason, confidence: e.confidence }));
+    }
+    catch {
+        return [];
+    }
+}
+// Helper to satisfy unused-warning
+void null;

package/dist/workers/value-gate.js

@@ -0,0 +1,38 @@
+const REMEMBER_PATTERNS = [
+    /记住/i, /记住这个/i, /以后遇到/i, /记住.*偏好/i,
+    /这个是我的偏好/i, /这个方案确定了/i, /以后记住/i
+];
+const DECISION_PATTERNS = [
+    /我们就用/i, /决定.*用/i, /选择.*而不是/i, /不用.*了/i,
+    /采用/i, /使用.*方案/i, /确定.*架构/i
+];
+const FAILURE_KEYWORDS = ['error', 'fail', 'crash', 'exception', 'build failed', 'test failed'];
+export function evaluateObservation(input) {
+    const combined = [
+        input.userPrompt || '',
+        input.toolOutput || '',
+        input.error || ''
+    ].join('\n').toLowerCase();
+    // Check for explicit "remember" requests
+    for (const pattern of REMEMBER_PATTERNS) {
+        if (pattern.test(input.userPrompt || '')) {
+            return { shouldCreateMemory: true, reason: 'User explicitly asked to remember', suggestedTypes: ['fact', 'preference'], priority: 'high' };
+        }
+    }
+    // Check for decisions
+    for (const pattern of DECISION_PATTERNS) {
+        if (pattern.test(input.userPrompt || '')) {
+            return { shouldCreateMemory: true, reason: 'Architectural decision detected', suggestedTypes: ['decision'], priority: 'high' };
+        }
+    }
+    // Check for tool failures
+    if (input.hookType === 'post_tool_use' && input.toolName === 'Bash' && FAILURE_KEYWORDS.some(k => combined.includes(k))) {
+        return { shouldCreateMemory: true, reason: 'Tool failure detected', suggestedTypes: ['bug'], priority: 'high' };
+    }
+    // Check for prompt_submit with substantive content
+    if (input.hookType === 'prompt_submit' && input.userPrompt && input.userPrompt.length > 50) {
+        return { shouldCreateMemory: true, reason: 'Substantive user prompt', suggestedTypes: ['event'], priority: 'medium' };
+    }
+    // Default: reject routine operations
+    return { shouldCreateMemory: false, reason: 'Routine operation, no memory value', suggestedTypes: [], priority: 'low' };
+}

package/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "@mem-weave/server",
+  "version": "0.2.0",
+  "description": "MemWeave local-first memory infrastructure for AI agents: structured memory, 4-layer retrieval (BM25 + vector + graph + causal), token-budgeted injection, server-side write deduplication, and background consolidation. Server process: Fastify REST API + CLI.",
+  "type": "module",
+  "main": "./dist/cli-entry.js",
+  "types": "./dist/cli-entry.d.ts",
+  "bin": {
+    "memweave": "dist/cli-entry.js"
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "typecheck": "tsc -p tsconfig.json --noEmit",
+    "clean": "rm -rf dist"
+  },
+  "engines": {
+    "node": ">=20.0.0"
+  },
+  "dependencies": {
+    "@fastify/cors": "11.2.0",
+    "@fastify/static": "^9.1.3",
+    "better-sqlite3": "12.10.0",
+    "fastify": "5.8.5",
+    "jsonc-parser": "3.3.1",
+    "pino": "10.3.1",
+    "sqlite-vec": "^0.1.9",
+    "zod": "4.4.3"
+  },
+  "optionalDependencies": {
+    "@xenova/transformers": "*"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "license": "MIT"
+}