mcp-memory-bucket 0.1.0

package/dist/src/store/sync.js

@@ -0,0 +1,144 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import chokidar, {} from 'chokidar';
+import { readMarkdownFile } from './markdown-file.js';
+import { flattenTags } from './db.js';
+const skillColumns = ['id', 'description', 'owner', 'status', 'tags', 'trigger_phrases', 'extends'];
+const memoryColumns = ['id', 'key', 'key_type', 'description', 'doc_type', 'tags', 'status', 'related_to'];
+export function skillSyncSpec(sources) {
+    return {
+        table: 'skills',
+        sources,
+        matchesFile: (filePath) => path.basename(filePath) === 'SKILL.md',
+        columns: skillColumns,
+        getId: (fm) => fm.name,
+        toRow: (fm) => ({
+            id: fm.name,
+            description: fm.description,
+            owner: fm.metadata?.owner ?? null,
+            status: fm.metadata?.status ?? 'unreviewed',
+            tags: JSON.stringify(fm.tags ?? []),
+            trigger_phrases: JSON.stringify(fm.trigger_phrases ?? []),
+            extends: fm.metadata?.extends ?? null,
+        }),
+    };
+}
+export function memorySyncSpec(sources) {
+    return {
+        table: 'memory_docs',
+        sources,
+        matchesFile: (filePath) => filePath.endsWith('.md'),
+        columns: memoryColumns,
+        getId: (fm) => fm.id,
+        toRow: (fm) => ({
+            id: fm.id,
+            key: fm.key,
+            key_type: fm.key_type ?? 'freeform',
+            description: fm.description,
+            doc_type: fm.doc_type ?? 'other',
+            tags: JSON.stringify(fm.tags ?? []),
+            status: fm.status ?? 'active',
+            related_to: fm.related_to ?? null,
+        }),
+    };
+}
+/**
+ * Upserts one file's frontmatter/body into its cache table, keyed by mtime
+ * so an unchanged file is skipped. Exported so repositories can index
+ * synchronously right after their own writes — the watcher's own event for
+ * that same write becomes a harmless no-op re-check once mtime matches.
+ */
+export function upsertFile(db, spec, filePath) {
+    const existing = db
+        .prepare(`SELECT mtime_ms FROM ${spec.table} WHERE source_path = ?`)
+        .get(filePath);
+    const parsed = readMarkdownFile(filePath);
+    if (existing && existing.mtime_ms === parsed.mtimeMs)
+        return; // unchanged, skip reprocessing
+    const id = spec.getId(parsed.frontmatter);
+    if (!id) {
+        console.error(`[memory-bucket] skipping ${filePath}: missing required id field in frontmatter`);
+        return;
+    }
+    const row = spec.toRow(parsed.frontmatter, filePath);
+    const cols = [...spec.columns, 'source_path', 'body', 'mtime_ms'];
+    const values = [...spec.columns.map((c) => row[c]), filePath, parsed.body, parsed.mtimeMs];
+    const placeholders = cols.map(() => '?').join(', ');
+    const updateClause = cols
+        .filter((c) => c !== 'id')
+        .map((c) => `${c} = excluded.${c}`)
+        .join(', ');
+    db.prepare(`INSERT INTO ${spec.table} (${cols.join(', ')}) VALUES (${placeholders})
+     ON CONFLICT(id) DO UPDATE SET ${updateClause}`).run(...values);
+    db.prepare(`DELETE FROM search_index WHERE ref_table = ? AND ref_id = ?`).run(spec.table, id);
+    db.prepare(`INSERT INTO search_index (ref_table, ref_id, description, body, tags) VALUES (?, ?, ?, ?, ?)`).run(spec.table, id, String(row.description ?? ''), parsed.body, flattenTags(String(row.tags ?? '[]')));
+}
+export function removeFile(db, table, filePath) {
+    const existing = db.prepare(`SELECT id FROM ${table} WHERE source_path = ?`).get(filePath);
+    db.prepare(`DELETE FROM ${table} WHERE source_path = ?`).run(filePath);
+    if (existing) {
+        db.prepare(`DELETE FROM search_index WHERE ref_table = ? AND ref_id = ?`).run(table, existing.id);
+    }
+}
+/** Full scan of all configured source dirs — used once at startup before the watcher takes over. */
+export function initialScan(db, spec) {
+    for (const dir of spec.sources) {
+        if (!fs.existsSync(dir))
+            continue;
+        for (const file of walkMarkdownFiles(dir)) {
+            if (!spec.matchesFile(file))
+                continue;
+            try {
+                upsertFile(db, spec, file);
+            }
+            catch (err) {
+                console.error(`[memory-bucket] failed to index ${file}:`, err);
+            }
+        }
+    }
+}
+function* walkMarkdownFiles(dir) {
+    for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+        const full = path.join(dir, entry.name);
+        if (entry.isDirectory()) {
+            yield* walkMarkdownFiles(full);
+        }
+        else if (entry.isFile() && entry.name.endsWith('.md')) {
+            yield full;
+        }
+    }
+}
+export function watchSources(db, spec) {
+    const watcher = chokidar.watch(spec.sources, {
+        ignoreInitial: true,
+        persistent: true,
+        depth: 10,
+    });
+    watcher
+        .on('add', (filePath) => {
+        if (!spec.matchesFile(filePath))
+            return;
+        try {
+            upsertFile(db, spec, filePath);
+        }
+        catch (err) {
+            console.error(`[memory-bucket] failed to index ${filePath}:`, err);
+        }
+    })
+        .on('change', (filePath) => {
+        if (!spec.matchesFile(filePath))
+            return;
+        try {
+            upsertFile(db, spec, filePath);
+        }
+        catch (err) {
+            console.error(`[memory-bucket] failed to reindex ${filePath}:`, err);
+        }
+    })
+        .on('unlink', (filePath) => {
+        if (!spec.matchesFile(filePath))
+            return;
+        removeFile(db, spec.table, filePath);
+    });
+    return watcher;
+}

package/dist/src/types.js

@@ -0,0 +1,4 @@
+/** Normalizes a lookup key the same way for authoring and querying: uppercase, hyphenated. */
+export function normalizeKey(key) {
+    return key.trim().toUpperCase().replace(/\s+/g, '-');
+}

package/dist/src/web/routes.js

@@ -0,0 +1,204 @@
+import express from 'express';
+function asArray(v) {
+    if (v === undefined)
+        return [];
+    return Array.isArray(v) ? v.map(String) : [String(v)];
+}
+function tagWhereClause(tags) {
+    if (tags.length === 0)
+        return { clause: '', params: [] };
+    const clauses = tags.map(() => `EXISTS (SELECT 1 FROM json_each(tags) WHERE value = ?)`);
+    return { clause: ` AND ${clauses.join(' AND ')}`, params: tags };
+}
+function queryEntries(db, req) {
+    const type = req.query.type ?? 'all';
+    const tags = asArray(req.query.tag);
+    const statuses = asArray(req.query.status);
+    const owners = asArray(req.query.owner);
+    const docTypes = asArray(req.query.doc_type);
+    const keyTypes = asArray(req.query.key_type);
+    const q = req.query.q?.trim();
+    const matchedIds = q
+        ? matchSearch(db, q)
+        : null;
+    if (q && matchedIds && matchedIds.skills.size === 0 && matchedIds.memory_docs.size === 0) {
+        return [];
+    }
+    const results = [];
+    if (type === 'skill' || type === 'all') {
+        results.push(...queryTable(db, 'skills', { tags, statuses, owners, docTypes: [], keyTypes: [] }, matchedIds?.skills));
+    }
+    if (type === 'memory' || type === 'all') {
+        results.push(...queryTable(db, 'memory_docs', { tags, statuses, owners: [], docTypes, keyTypes }, matchedIds?.memory_docs));
+    }
+    const sort = req.query.sort ?? 'mtime_desc';
+    results.sort((a, b) => {
+        if (sort === 'mtime_asc')
+            return a.mtime_ms - b.mtime_ms;
+        if (sort === 'name_asc')
+            return a.name.localeCompare(b.name);
+        return b.mtime_ms - a.mtime_ms; // mtime_desc, default
+    });
+    return results;
+}
+function queryTable(db, table, filters, restrictToIds) {
+    if (restrictToIds && restrictToIds.size === 0)
+        return [];
+    const params = [];
+    let where = '1 = 1';
+    const { clause: tagClause, params: tagParams } = tagWhereClause(filters.tags);
+    where += tagClause;
+    params.push(...tagParams);
+    if (filters.statuses.length > 0) {
+        where += ` AND status IN (${filters.statuses.map(() => '?').join(', ')})`;
+        params.push(...filters.statuses);
+    }
+    if (table === 'skills' && filters.owners.length > 0) {
+        where += ` AND owner IN (${filters.owners.map(() => '?').join(', ')})`;
+        params.push(...filters.owners);
+    }
+    if (table === 'memory_docs' && filters.docTypes.length > 0) {
+        where += ` AND doc_type IN (${filters.docTypes.map(() => '?').join(', ')})`;
+        params.push(...filters.docTypes);
+    }
+    if (table === 'memory_docs' && filters.keyTypes.length > 0) {
+        where += ` AND key_type IN (${filters.keyTypes.map(() => '?').join(', ')})`;
+        params.push(...filters.keyTypes);
+    }
+    if (restrictToIds) {
+        where += ` AND id IN (${[...restrictToIds].map(() => '?').join(', ')})`;
+        params.push(...restrictToIds);
+    }
+    if (table === 'skills') {
+        const rows = db
+            .prepare(`SELECT id, description, owner, status, tags, mtime_ms FROM skills WHERE ${where}`)
+            .all(...params);
+        return rows.map((r) => ({
+            _table: 'skills',
+            id: r.id,
+            name: r.id,
+            description: r.description,
+            tags: JSON.parse(r.tags),
+            status: r.status,
+            owner: r.owner,
+            doc_type: null,
+            key_type: null,
+            mtime_ms: r.mtime_ms,
+        }));
+    }
+    const rows = db
+        .prepare(`SELECT id, key, description, doc_type, key_type, status, tags, mtime_ms FROM memory_docs WHERE ${where}`)
+        .all(...params);
+    return rows.map((r) => ({
+        _table: 'memory_docs',
+        id: r.id,
+        name: r.key,
+        description: r.description,
+        tags: JSON.parse(r.tags),
+        status: r.status,
+        owner: null,
+        doc_type: r.doc_type,
+        key_type: r.key_type,
+        mtime_ms: r.mtime_ms,
+    }));
+}
+/** Runs the FTS5 query once and buckets matching ids by source table. */
+function matchSearch(db, q) {
+    const skills = new Set();
+    const memory_docs = new Set();
+    let rows;
+    try {
+        rows = db
+            .prepare(`SELECT ref_table, ref_id FROM search_index WHERE search_index MATCH ? ORDER BY rank`)
+            .all(q);
+    }
+    catch {
+        // Bad FTS5 query syntax (e.g. a bare quote) — treat as no matches rather than 500ing.
+        return { skills, memory_docs };
+    }
+    for (const row of rows) {
+        (row.ref_table === 'skills' ? skills : memory_docs).add(row.ref_id);
+    }
+    return { skills, memory_docs };
+}
+function buildFacets(db, type) {
+    const tags = new Set();
+    const statuses = new Set();
+    const owners = new Set();
+    const docTypes = new Set();
+    const keyTypes = new Set();
+    if (type === 'skill' || type === 'all') {
+        const rows = db.prepare(`SELECT tags, status, owner FROM skills`).all();
+        for (const r of rows) {
+            JSON.parse(r.tags).forEach((t) => tags.add(t));
+            statuses.add(r.status);
+            if (r.owner)
+                owners.add(r.owner);
+        }
+    }
+    if (type === 'memory' || type === 'all') {
+        const rows = db.prepare(`SELECT tags, status, doc_type, key_type FROM memory_docs`).all();
+        for (const r of rows) {
+            JSON.parse(r.tags).forEach((t) => tags.add(t));
+            statuses.add(r.status);
+            docTypes.add(r.doc_type);
+            keyTypes.add(r.key_type);
+        }
+    }
+    return {
+        tags: [...tags].sort(),
+        statuses: [...statuses].sort(),
+        owners: [...owners].sort(),
+        doc_types: [...docTypes].sort(),
+        key_types: [...keyTypes].sort(),
+    };
+}
+function buildHealth(db) {
+    const skillIds = new Set(db.prepare(`SELECT id FROM skills`).all().map((r) => r.id));
+    const memoryIds = new Set(db.prepare(`SELECT id FROM memory_docs`).all().map((r) => r.id));
+    const skills = db.prepare(`SELECT id, extends, trigger_phrases, mtime_ms FROM skills`).all();
+    const danglingExtends = skills
+        .filter((s) => s.extends && !skillIds.has(s.extends))
+        .map((s) => ({ id: s.id, extends: s.extends }));
+    const emptyTriggerPhrases = skills
+        .filter((s) => JSON.parse(s.trigger_phrases).length === 0)
+        .map((s) => s.id);
+    const memoryDocs = db.prepare(`SELECT id, related_to, status, mtime_ms FROM memory_docs`).all();
+    const danglingRelatedTo = memoryDocs
+        .filter((m) => m.related_to && !memoryIds.has(m.related_to) && !skillIds.has(m.related_to))
+        .map((m) => ({ id: m.id, related_to: m.related_to }));
+    const staleCutoff = Date.now() - 30 * 24 * 60 * 60 * 1000;
+    const staleActiveMemoryDocs = memoryDocs
+        .filter((m) => m.status === 'active' && m.mtime_ms < staleCutoff)
+        .map((m) => m.id);
+    return { danglingExtends, danglingRelatedTo, emptyTriggerPhrases, staleActiveMemoryDocs };
+}
+export function buildWebRouter(db) {
+    const router = express.Router();
+    router.get('/api/entries', (req, res) => {
+        res.json(queryEntries(db, req));
+    });
+    router.get('/api/entries/:table/:id', (req, res) => {
+        const { table, id } = req.params;
+        if (table !== 'skills' && table !== 'memory_docs') {
+            res.status(400).json({ error: 'table must be "skills" or "memory_docs"' });
+            return;
+        }
+        const row = db.prepare(`SELECT * FROM ${table} WHERE id = ?`).get(id);
+        if (!row) {
+            res.status(404).json({ error: 'not found' });
+            return;
+        }
+        const tags = JSON.parse(row.tags);
+        const trigger_phrases = row.trigger_phrases ? JSON.parse(row.trigger_phrases) : undefined;
+        res.json({ ...row, tags, trigger_phrases });
+    });
+    router.get('/api/facets', (req, res) => {
+        const type = req.query.type ?? 'all';
+        res.json(buildFacets(db, type));
+    });
+    router.get('/api/health', (_req, res) => {
+        res.json(buildHealth(db));
+    });
+    return router;
+}

package/dist/src/web/ui-tool.js

@@ -0,0 +1,6 @@
+export function registerUiTool(mcp, port) {
+    mcp.tool('bucket_open_ui', 'Returns the URL for the mem-bucket web viewer — a read-only browser UI for searching/filtering skills and memory docs by tag, status, owner, and fulltext. Not for editing; use the skill_*/memory_* tools for that.', {}, async () => {
+        const url = `http://localhost:${port}/`;
+        return { content: [{ type: 'text', text: url }] };
+    });
+}

package/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "mcp-memory-bucket",
+  "version": "0.1.0",
+  "type": "module",
+  "description": "MCP server exposing skill_* (reusable coding patterns) and memory_* (point-in-time working context) tools over a markdown+frontmatter source, cached into SQLite at runtime.",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/anatolipr/avo-mcp-tools.git",
+    "directory": "packages/mcp-memory-bucket"
+  },
+  "license": "MIT",
+  "bin": {
+    "mcp-memory-bucket": "./bin/mcp-memory-bucket.js"
+  },
+  "main": "dist/src/server.js",
+  "files": [
+    "bin",
+    "dist",
+    "README.md"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "dev": "npm-run-all --parallel dev:client dev:server",
+    "dev:client": "vite build --watch",
+    "dev:server": "tsx watch src/server.ts",
+    "build": "tsc -p tsconfig.server.json && vite build && npm run copy:builtin-skills",
+    "copy:builtin-skills": "mkdir -p dist/src/skills/builtin && cp -r src/skills/builtin/. dist/src/skills/builtin/",
+    "start": "tsx src/server.ts",
+    "start_in_folder": "tsx src/server.ts --memory-dir",
+    "typecheck": "tsc --noEmit -p tsconfig.server.json && tsc --noEmit -p tsconfig.client.json",
+    "test": "node --import tsx --test test/**/*.test.ts"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.12.0",
+    "avosignals": "^1.0.16",
+    "better-sqlite3": "^11.8.0",
+    "chokidar": "^4.0.0",
+    "express": "^4.21.0",
+    "gray-matter": "^4.0.3",
+    "lit": "^3.3.3",
+    "zod": "^3.23.8"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.0",
+    "@types/express": "^4.17.0",
+    "@types/node": "^24.0.0",
+    "npm-run-all": "^4.1.5",
+    "tsx": "^4.19.0",
+    "typescript": "^5.7.0",
+    "vite": "^6.0.0"
+  }
+}