clay-server 5.0.2-beta.1 → 5.1.0-beta.2

package/lib/knowledge-file-store.js

@@ -0,0 +1,199 @@
+var crypto = require("crypto");
+var fs = require("fs");
+var path = require("path");
+
+var MAX_CONTENT_BYTES = 2 * 1024 * 1024;
+var RESERVED = { "session-digests.jsonl": true, "sticky-notes.md": true, "memory-summary.md": true };
+
+function revision(content) { return crypto.createHash("sha256").update(content, "utf8").digest("hex"); }
+function cleanName(value) {
+  if (typeof value !== "string") return null;
+  var name = value.replace(/\\/g, "/").replace(/^\.\//, "");
+  if (!name || name.length > 500 || name[0] === "/" || /[\x00-\x1f\x7f]/.test(name)) return null;
+  var parts = name.split("/");
+  if (parts.length > 20 || parts.some(function (part) { return !part || part === "." || part === ".." || part.length > 120; })) return null;
+  if (!/\.(md|jsonl)$/i.test(name) || RESERVED[name]) return null;
+  return parts.join("/");
+}
+function validName(value) { return cleanName(value) !== null; }
+function itemType(name) { return /\.jsonl$/i.test(name) ? "db" : "text"; }
+function atomicJson(target, value) {
+  var temporary = target + ".tmp-" + process.pid + "-" + crypto.randomBytes(6).toString("hex");
+  try { fs.writeFileSync(temporary, JSON.stringify(value, null, 2), { flag: "wx", mode: 0o600 }); fs.renameSync(temporary, target); }
+  finally { try { fs.unlinkSync(temporary); } catch (error) {} }
+}
+
+function createKnowledgeFileStore(cwd, options) {
+  var directory = path.join(cwd, "knowledge");
+  var settings = options || {};
+  var metaRoot = settings.metaDir || path.join(cwd, ".clay", "knowledge-documents");
+  var registryPath = path.join(metaRoot, "documents.json");
+  var historyRoot = path.join(metaRoot, "history");
+  var fileCache = {};
+  function safeDirectory(target, create) {
+    var stat;
+    try { stat = fs.lstatSync(target); }
+    catch (error) { if (!create || error.code !== "ENOENT") throw error; fs.mkdirSync(target, { recursive: true, mode: 0o700 }); stat = fs.lstatSync(target); }
+    if (!stat.isDirectory() || stat.isSymbolicLink()) throw Object.assign(new Error("Knowledge metadata path is not a safe directory."), { code: "UNSAFE_PATH" });
+  }
+  function ensureMetadata(createHistory) {
+    safeDirectory(metaRoot, true);
+    var registryStat;
+    try { registryStat = fs.lstatSync(registryPath); } catch (error) { if (error.code !== "ENOENT") throw error; }
+    if (registryStat && (!registryStat.isFile() || registryStat.isSymbolicLink())) throw Object.assign(new Error("Knowledge metadata registry is not a safe file."), { code: "UNSAFE_PATH" });
+    if (createHistory) safeDirectory(historyRoot, true);
+    else { try { safeDirectory(historyRoot, false); } catch (error) { if (error.code !== "ENOENT") throw error; } }
+  }
+  function ensureDirectory(create) {
+    var stat;
+    try { stat = fs.lstatSync(directory); }
+    catch (error) { if (!create || error.code !== "ENOENT") throw error; fs.mkdirSync(directory, { recursive: true }); stat = fs.lstatSync(directory); }
+    if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error("Knowledge directory is not a safe directory.");
+  }
+  function targetPath(name, createParents) {
+    var clean = cleanName(name); if (!clean) throw new Error("Invalid knowledge file name."); ensureDirectory(!!createParents);
+    var parts = clean.split("/"); var cursor = directory;
+    for (var i = 0; i < parts.length - 1; i++) {
+      cursor = path.join(cursor, parts[i]); var stat;
+      try { stat = fs.lstatSync(cursor); }
+      catch (error) { if (!createParents || error.code !== "ENOENT") throw error; fs.mkdirSync(cursor, { mode: 0o700 }); stat = fs.lstatSync(cursor); }
+      if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error("Knowledge path contains an unsafe directory.");
+    }
+    return path.join(directory, clean);
+  }
+  function registry() {
+    ensureMetadata(false);
+    try { var parsed = JSON.parse(fs.readFileSync(registryPath, "utf8")); return parsed && parsed.version === 1 && parsed.documents ? parsed : { version: 1, documents: {} }; }
+    catch (error) { if (error.code === "ENOENT" || error instanceof SyntaxError) return { version: 1, documents: {} }; throw error; }
+  }
+  function writeRegistry(value) { ensureMetadata(false); atomicJson(registryPath, value); }
+  function existing(name) {
+    var clean = cleanName(name); if (!clean) throw new Error("Invalid knowledge file name."); var target; var stat;
+    try { target = targetPath(clean, false); } catch (error) { if (error.code === "ENOENT") return null; throw error; }
+    try { stat = fs.lstatSync(target); } catch (error) { if (error.code === "ENOENT") return null; throw error; }
+    if (!stat.isFile() || stat.isSymbolicLink()) throw new Error("Knowledge file is not a safe regular file.");
+    if (stat.size > MAX_CONTENT_BYTES) throw Object.assign(new Error("Knowledge document exceeds the safe read limit."), { code: "TOO_LARGE" });
+    var cached = fileCache[clean]; var content;
+    if (cached && cached.size === stat.size && cached.mtime === stat.mtimeMs && cached.ctime === stat.ctimeMs && cached.ino === stat.ino) content = cached.content;
+    else { content = fs.readFileSync(target, "utf8"); fileCache[clean] = { content: content, revision: revision(content), size: stat.size, mtime: stat.mtimeMs, ctime: stat.ctimeMs, ino: stat.ino }; }
+    cached = fileCache[clean]; return { name: clean, content: content, revision: cached.revision, size: stat.size, mtime: stat.mtimeMs, itemType: itemType(clean) };
+  }
+  function walk(relative, out) {
+    var base = relative ? path.join(directory, relative) : directory; var names = fs.readdirSync(base);
+    for (var i = 0; i < names.length; i++) {
+      var rel = relative ? relative + "/" + names[i] : names[i]; var stat;
+      try { stat = fs.lstatSync(path.join(directory, rel)); } catch (error) { continue; }
+      if (stat.isSymbolicLink()) continue;
+      if (stat.isDirectory()) { walk(rel, out); continue; }
+      if (!stat.isFile() || !validName(rel)) continue;
+      var item; try { item = existing(rel); } catch (error) {
+        if (error.code === "TOO_LARGE") item = { name: rel, content: null, revision: null, size: stat.size, mtime: stat.mtimeMs, itemType: itemType(rel), readError: error.message };
+        else continue;
+      }
+      if (item) out.push(item);
+    }
+  }
+  function reconcileIdentity(items) {
+    var data = registry(); var changed = false; var used = {}; var currentNames = {};
+    items.forEach(function (item) { currentNames[item.name] = true; });
+    Object.keys(data.documents).forEach(function (id) { if (!data.documents[id].deleted) used[data.documents[id].name] = id; });
+    for (var i = 0; i < items.length; i++) {
+      var id = used[items[i].name];
+      if (!id) {
+        var candidates = items[i].revision ? Object.keys(data.documents).filter(function (key) { var doc = data.documents[key]; return !currentNames[doc.name] && doc.lastRevision === items[i].revision; }) : [];
+        id = candidates.length === 1 ? candidates[0] : crypto.randomUUID(); changed = true;
+      }
+      if (!data.documents[id] || data.documents[id].name !== items[i].name || data.documents[id].lastRevision !== items[i].revision || data.documents[id].deleted) changed = true;
+      data.documents[id] = Object.assign({}, data.documents[id] || {}, { id: id, name: items[i].name,
+        lastRevision: items[i].revision || data.documents[id] && data.documents[id].lastRevision || null, deleted: false });
+      items[i].id = id; used[items[i].name] = id;
+    }
+    Object.keys(data.documents).forEach(function (id) {
+      var found = items.some(function (item) { return item.id === id; });
+      if (!found && !data.documents[id].deleted) { data.documents[id].deleted = true; changed = true; }
+    });
+    if (changed) writeRegistry(data); return data;
+  }
+  function list() {
+    try { ensureDirectory(false); } catch (error) { if (error.code === "ENOENT") return []; throw error; }
+    var items = []; walk("", items); reconcileIdentity(items);
+    return items.map(function (item) { return { id: item.id, name: item.name, path: item.name, folder: path.posix.dirname(item.name) === "." ? "" : path.posix.dirname(item.name),
+      itemType: item.itemType, size: item.size, mtime: item.mtime, revision: item.revision, writable: item.itemType === "text" }; })
+      .sort(function (a, b) { return a.name.localeCompare(b.name); });
+  }
+  function read(name) {
+    ensureDirectory(false); var item = existing(name); if (!item) throw Object.assign(new Error("Knowledge file not found."), { code: "ENOENT" });
+    var data = registry(); var id = Object.keys(data.documents).filter(function (key) { return !data.documents[key].deleted && data.documents[key].name === item.name; })[0];
+    if (!id) { list(); data = registry(); id = Object.keys(data.documents).filter(function (key) { return !data.documents[key].deleted && data.documents[key].name === item.name; })[0]; }
+    return Object.assign(item, { id: id || null });
+  }
+  function assertExpected(current, expected) {
+    if (!current && expected) throw Object.assign(new Error("This document was removed since it was opened."), { code: "STALE", currentRevision: null });
+    if (current && (!expected || expected !== current.revision)) throw Object.assign(new Error("This document changed since it was opened."), { code: "STALE", currentRevision: current.revision });
+  }
+  function snapshot(item, reason) {
+    if (!item) return;
+    var listed = list(); var identity = listed.filter(function (entry) { return entry.name === item.name; })[0]; if (!identity) return; ensureMetadata(true);
+    var folder = path.join(historyRoot, identity.id); safeDirectory(folder, true);
+    var target = path.join(folder, Date.now() + "-" + item.revision + ".md");
+    fs.writeFileSync(target, item.content, { flag: "wx", mode: 0o600 });
+  }
+  function save(name, content, expectedRevision) {
+    var clean = cleanName(name); if (!clean || itemType(clean) !== "text") throw new Error("Only valid Markdown documents can be edited.");
+    if (typeof content !== "string") throw new Error("Knowledge content is required.");
+    if (Buffer.byteLength(content, "utf8") > MAX_CONTENT_BYTES) throw new Error("Knowledge document is too large.");
+    ensureDirectory(true); var current = existing(clean); assertExpected(current, expectedRevision); if (current) snapshot(current, "before-save");
+    var target = targetPath(clean, true); var temporary = path.join(path.dirname(target), ".clay-knowledge-" + process.pid + "-" + crypto.randomBytes(8).toString("hex"));
+    try { fs.writeFileSync(temporary, content, { flag: "wx", mode: 0o600 }); fs.renameSync(temporary, target); }
+    finally { try { fs.unlinkSync(temporary); } catch (error) {} }
+    delete fileCache[clean];
+    return read(clean);
+  }
+  function rename(from, to, expectedRevision) {
+    var cleanFrom = cleanName(from); var cleanTo = cleanName(to);
+    if (!cleanTo || itemType(cleanTo) !== "text") throw new Error("Only valid Markdown documents can be renamed or moved.");
+    if (!cleanFrom || itemType(cleanFrom) !== "text") throw Object.assign(new Error("This Knowledge source is read only."), { code: "READ_ONLY" });
+    ensureDirectory(false); var current = existing(cleanFrom); if (!current) throw Object.assign(new Error("Knowledge file not found."), { code: "ENOENT" });
+    assertExpected(current, expectedRevision); if (existing(cleanTo)) throw Object.assign(new Error("A document with that path already exists."), { code: "EXISTS" }); snapshot(current, "before-move");
+    fs.renameSync(targetPath(cleanFrom, false), targetPath(cleanTo, true));
+    delete fileCache[cleanFrom];
+    var data = registry(); Object.keys(data.documents).forEach(function (id) { if (data.documents[id].name === cleanFrom && !data.documents[id].deleted) data.documents[id].name = cleanTo; }); writeRegistry(data);
+    return read(cleanTo);
+  }
+  function remove(name, expectedRevision) {
+    var clean = cleanName(name); if (!clean || itemType(clean) !== "text") throw Object.assign(new Error("This Knowledge source is read only."), { code: "READ_ONLY" });
+    ensureDirectory(false); var current = existing(clean); if (!current) throw Object.assign(new Error("Knowledge file not found."), { code: "ENOENT" });
+    assertExpected(current, expectedRevision); snapshot(current, "before-delete"); fs.unlinkSync(targetPath(clean, false)); delete fileCache[clean]; list(); return current;
+  }
+  function history(id) {
+    if (!/^[a-f0-9-]{20,50}$/i.test(String(id || ""))) throw new Error("Invalid document identity."); ensureMetadata(false); var folder = path.join(historyRoot, id); var names;
+    try { safeDirectory(folder, false); } catch (error) { if (error.code === "ENOENT") return []; throw error; }
+    try { names = fs.readdirSync(folder); } catch (error) { if (error.code === "ENOENT") return []; throw error; }
+    return names.filter(function (name) { return /^\d+-[a-f0-9]{64}\.md$/.test(name); }).sort().reverse().slice(0, 50).map(function (name) {
+      var target = path.join(folder, name); var stat = fs.lstatSync(target);
+      if (!stat.isFile() || stat.isSymbolicLink() || stat.size > MAX_CONTENT_BYTES) throw Object.assign(new Error("Knowledge history contains an unsafe version."), { code: "UNSAFE_PATH" });
+      return { version: name, size: stat.size, createdAt: Number(name.split("-")[0]) };
+    });
+  }
+  function readHistory(id, versionName) {
+    var versions = history(id); if (!versions.some(function (entry) { return entry.version === versionName; })) throw Object.assign(new Error("Knowledge history version not found."), { code: "ENOENT" });
+    return { version: versionName, content: fs.readFileSync(path.join(historyRoot, id, versionName), "utf8") };
+  }
+  function restore(id, versionName, name, expectedRevision) {
+    if (!/^[a-f0-9-]{20,50}$/i.test(String(id || "")) || !/^\d+-[a-f0-9]{64}\.md$/.test(String(versionName || ""))) throw new Error("Invalid history version.");
+    var selected = readHistory(id, versionName);
+    var data = registry(); var record = data.documents[id]; if (!record) throw Object.assign(new Error("Knowledge document identity not found."), { code: "ENOENT" });
+    var current = existing(name); var restored = save(name, selected.content, current ? expectedRevision || null : null);
+    data = registry(); Object.keys(data.documents).forEach(function (key) { if (key !== id && data.documents[key].name === restored.name && !data.documents[key].deleted) delete data.documents[key]; });
+    data.documents[id] = Object.assign({}, record, { id: id, name: restored.name, lastRevision: restored.revision, deleted: false }); writeRegistry(data);
+    return Object.assign(restored, { id: id });
+  }
+  function deleted() {
+    var data = registry(); return Object.keys(data.documents).filter(function (id) { return data.documents[id].deleted; }).map(function (id) {
+      return { id: id, name: data.documents[id].name, revision: data.documents[id].lastRevision || null, deleted: true, historyCount: history(id).length };
+    }).filter(function (item) { return item.historyCount > 0; }).sort(function (a, b) { return a.name.localeCompare(b.name); });
+  }
+  return { list: list, read: read, save: save, rename: rename, remove: remove, history: history, readHistory: readHistory, restore: restore, deleted: deleted };
+}
+
+module.exports = { createKnowledgeFileStore: createKnowledgeFileStore, revision: revision, validName: validName, cleanName: cleanName, itemType: itemType, MAX_CONTENT_BYTES: MAX_CONTENT_BYTES };

package/lib/knowledge-markdown.js

@@ -0,0 +1,128 @@
+var path = require("path");
+
+function slug(value) {
+  return String(value || "").trim().toLowerCase().replace(/<[^>]*>/g, "").replace(/[^\p{L}\p{N}\s-]/gu, "").replace(/\s+/g, "-").replace(/-+/g, "-");
+}
+
+function frontmatter(content) {
+  var text = String(content || "");
+  if (text.indexOf("---\n") !== 0) return { properties: {}, tags: [], end: 0, errors: [] };
+  var close = text.indexOf("\n---\n", 4);
+  if (close === -1 || close > 8192) return { properties: {}, tags: [], end: 0, errors: ["Frontmatter is missing its closing --- line."] };
+  var properties = {}; var errors = []; var lines = text.slice(4, close).split("\n");
+  for (var i = 0; i < lines.length; i++) {
+    if (!lines[i].trim() || /^\s*#/.test(lines[i])) continue;
+    var match = /^([A-Za-z][A-Za-z0-9_-]{0,63}):\s*(.*)$/.exec(lines[i]);
+    if (!match) { errors.push("Invalid property on line " + (i + 2) + "."); continue; }
+    var value = match[2].trim();
+    if (value.length > 500) errors.push("Property " + match[1] + " is too long.");
+    else properties[match[1]] = value;
+  }
+  var tags = [];
+  if (properties.tags) {
+    var raw = properties.tags.replace(/^\[/, "").replace(/\]$/, "").split(",");
+    for (var t = 0; t < raw.length; t++) {
+      var tag = raw[t].trim().replace(/^#/, "");
+      if (/^[\p{L}\p{N}_/-]{1,64}$/u.test(tag) && tags.indexOf(tag) === -1) tags.push(tag);
+      else if (tag) errors.push("Invalid tag: " + tag);
+    }
+  }
+  return { properties: properties, tags: tags, end: close + 5, errors: errors };
+}
+
+function ignoredRanges(text) {
+  var ranges = []; var lines = text.split("\n"); var offset = 0; var fence = null;
+  for (var i = 0; i < lines.length; i++) {
+    var line = lines[i]; var marker = /^( {0,3})(`{3,}|~{3,})(.*)$/.exec(line);
+    if (marker && (!fence || marker[2][0] === fence.marker && marker[2].length >= fence.length && /^\s*$/.test(marker[3]))) {
+      if (!fence) fence = { marker: marker[2][0], length: marker[2].length, start: offset };
+      else { ranges.push([fence.start, offset + line.length]); fence = null; }
+    }
+    if (!fence) {
+      var cursor = 0;
+      while (cursor < line.length) {
+        var opener = /`+/.exec(line.slice(cursor)); if (!opener) break;
+        var start = cursor + opener.index; var run = opener[0]; var close = line.indexOf(run, start + run.length);
+        if (close === -1) { cursor = start + run.length; continue; }
+        ranges.push([offset + start, offset + close + run.length]); cursor = close + run.length;
+      }
+    }
+    offset += line.length + 1;
+  }
+  if (fence) ranges.push([fence.start, text.length]);
+  return ranges;
+}
+
+function escaped(text, start) {
+  var slashes = 0;
+  for (var i = start - 1; i >= 0 && text[i] === "\\"; i--) slashes++;
+  return slashes % 2 === 1;
+}
+
+function inside(ranges, start) {
+  for (var i = 0; i < ranges.length; i++) if (start >= ranges[i][0] && start < ranges[i][1]) return true;
+  return false;
+}
+
+function parse(content) {
+  var text = String(content || ""); var ignored = ignoredRanges(text); var links = []; var headings = []; var match;
+  var headingPattern = /^(#{1,6})\s+(.+?)\s*#*\s*$/gm;
+  while ((match = headingPattern.exec(text))) {
+    if (!inside(ignored, match.index)) headings.push({ depth: match[1].length, text: match[2].trim(), slug: slug(match[2]), offset: match.index });
+  }
+  var wiki = /(!)?\[\[([^\]\n]+)\]\]/g;
+  while ((match = wiki.exec(text))) {
+    if (inside(ignored, match.index) || escaped(text, match.index)) continue;
+    var parts = match[2].split("|"); var target = parts[0].trim(); var alias = parts.length > 1 ? parts.slice(1).join("|").trim() : "";
+    var hash = target.indexOf("#");
+    var wikiTarget = hash === -1 ? target : target.slice(0, hash);
+    links.push({ kind: match[1] ? "embed" : "wiki", target: wikiTarget, heading: hash === -1 ? "" : target.slice(hash + 1), alias: alias,
+      start: match.index, end: match.index + match[0].length, raw: match[0], external: /^[a-z][a-z0-9+.-]*:/i.test(wikiTarget) || wikiTarget.indexOf("//") === 0 });
+  }
+  var markdown = /(!)?\[([^\]\n]*)\]\(([^)\n]+)\)/g;
+  while ((match = markdown.exec(text))) {
+    if (inside(ignored, match.index) || escaped(text, match.index)) continue;
+    var destination = match[3].trim().replace(/^<|>$/g, ""); var hashAt = destination.indexOf("#");
+    links.push({ kind: match[1] ? "image" : "markdown", target: hashAt === -1 ? destination : destination.slice(0, hashAt), heading: hashAt === -1 ? "" : destination.slice(hashAt + 1), alias: match[2],
+      start: match.index, end: match.index + match[0].length, raw: match[0], external: /^[a-z][a-z0-9+.-]*:/i.test(destination) || destination.indexOf("//") === 0 });
+  }
+  links.sort(function (a, b) { return a.start - b.start; });
+  var meta = frontmatter(text); var tags = meta.tags.slice(); var tagPattern = /(^|\s)#([\p{L}\p{N}_/-]{1,64})/gu;
+  while ((match = tagPattern.exec(text))) if (!inside(ignored, match.index) && tags.indexOf(match[2]) === -1) tags.push(match[2]);
+  return { links: links, headings: headings, properties: meta.properties, tags: tags, propertyErrors: meta.errors };
+}
+
+function normalizeTarget(sourceName, target) {
+  var raw = String(target || "").trim();
+  if (!raw || /^[a-z][a-z0-9+.-]*:/i.test(raw) || raw.indexOf("//") === 0) return null;
+  try { raw = decodeURIComponent(raw); } catch (error) {}
+  raw = raw.replace(/\\/g, "/");
+  if (!/\.md$/i.test(raw)) raw += ".md";
+  return path.posix.normalize(path.posix.join(path.posix.dirname(sourceName), raw)).replace(/^\.\//, "");
+}
+
+function resolveLink(sourceName, link, documents, sourceId) {
+  if (link.external) return null;
+  if (!link.target && link.heading) {
+    var self = documents.filter(function (item) { return sourceId ? item.id === sourceId : item.name === sourceName; });
+    return self.length === 1 ? self[0] : null;
+  }
+  var target = normalizeTarget(sourceName, link.target);
+  if (!target) return null;
+  var exact = documents.filter(function (item) { return item.name.toLowerCase() === target.toLowerCase(); });
+  if (exact.length === 1) return exact[0];
+  if (link.target.indexOf("/") === -1 && link.target.indexOf("\\") === -1) {
+    var base = path.posix.basename(target).toLowerCase();
+    var named = documents.filter(function (item) { return path.posix.basename(item.name).toLowerCase() === base; });
+    if (named.length === 1) return named[0];
+  }
+  return null;
+}
+
+function snippet(content, offset, needle) {
+  var raw = String(content || ""); var at = typeof offset === "number" ? offset : raw.toLowerCase().indexOf(String(needle || "").toLowerCase());
+  if (at < 0) at = 0; var start = Math.max(0, at - 70); var end = Math.min(raw.length, at + 150); var text = raw.slice(start, end).replace(/\s+/g, " ").trim();
+  return (start ? "…" : "") + text + (end < raw.length ? "…" : "");
+}
+
+module.exports = { parse: parse, slug: slug, normalizeTarget: normalizeTarget, resolveLink: resolveLink, snippet: snippet };

package/lib/mate-knowledge-mcp-server.js

@@ -46,14 +46,37 @@ function handler(bound, method) {
     if (!bound || typeof bound[method] !== "function") {
       return errorResult(new Error("Knowledge tools require an exact session-bound Mate."));
     }
-    try {
-      return textResult(bound[method](args || {}));
-    } catch (e) {
-      return errorResult(e);
-    }
+    try { return Promise.resolve(bound[method](args || {})).then(textResult, errorResult); } catch (e) { return errorResult(e); }
   };
 }
 
+var ID = { type: "string", description: "Stable opaque id returned by a Knowledge database tool." };
+var REVISION = { type: "number", description: "Revision returned by the most recent read or mutation; stale revisions are rejected." };
+var OPERATION = { type: "string", description: "Stable retry id formatted label_<current epoch milliseconds>_<unique suffix>. Reuse the exact id and payload after an uncertain response; receipts are retained for the documented 24-hour retry window." };
+var FIELD = { type: "object", description: "Field with stable id, name, type, required, and type-specific options or reference target." };
+var FILTER = { type: "object", description: "Validated filter: fieldId, op, and value. Supported ops are eq, neq, contains, gt, gte, lt, lte, empty, not-empty." };
+var SORT = { type: "object", description: "Validated sort with fieldId and asc or desc direction." };
+
+function databaseTools(bound) {
+  return [
+    { name: "list_databases", description: "List canonical Knowledge databases, including archived databases when requested, so the Mate can discover and recover them without relying on summary projection.", inputSchema: buildShape({ archived: { type: "boolean" } }, []), handler: handler(bound, "listDatabases") },
+    { name: "create_database", description: "Create a first-class database in this Mate's Knowledge. Defaults to one required title field and a Table view.", inputSchema: buildShape({ name: { type: "string" }, schema: { type: "array", items: FIELD }, operationId: OPERATION }, ["name", "operationId"]), handler: handler(bound, "createDatabase") },
+    { name: "read_database", description: "Read one Knowledge database's current schema, views, revision, and record count.", inputSchema: buildShape({ databaseId: ID, includeArchived: { type: "boolean" } }, ["databaseId"]), handler: handler(bound, "readDatabase") },
+    { name: "rename_database", description: "Rename a Knowledge database with optimistic revision checking.", inputSchema: buildShape({ databaseId: ID, name: { type: "string" }, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "name", "expectedRevision", "operationId"]), handler: handler(bound, "renameDatabase") },
+    { name: "archive_database", description: "Recoverably archive a Knowledge database.", inputSchema: buildShape({ databaseId: ID, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "expectedRevision", "operationId"]), handler: handler(bound, "archiveDatabase") },
+    { name: "recover_database", description: "Recover an archived Knowledge database.", inputSchema: buildShape({ databaseId: ID, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "expectedRevision", "operationId"]), handler: handler(bound, "recoverDatabase") },
+    { name: "update_database_schema", description: "Replace the schema using stable field ids. Exactly one active title field is required; populated fields cannot be removed or change type and should be archived instead.", inputSchema: buildShape({ databaseId: ID, schema: { type: "array", items: FIELD }, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "schema", "expectedRevision", "operationId"]), handler: handler(bound, "updateDatabaseSchema") },
+    { name: "save_table_view", description: "Create or revise a persisted Table view with selected columns, validated filters, and deterministic sorts.", inputSchema: buildShape({ databaseId: ID, view: { type: "object", description: "View with stable id, name, columns, filters, and sorts." }, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "view", "expectedRevision", "operationId"]), handler: handler(bound, "saveTableView") },
+    { name: "query_database", description: "Query a bounded deterministic page of records using a saved view or explicit validated filters and sorts. Returns count and numeric sums.", inputSchema: buildShape({ databaseId: ID, viewId: ID, filters: { type: "array", items: FILTER }, sorts: { type: "array", items: SORT }, cursor: { type: "string" }, limit: { type: "number" }, archived: { type: "boolean" } }, ["databaseId"]), handler: handler(bound, "queryDatabase") },
+    { name: "create_database_record", description: "Create a typed record. Values are keyed by stable field id; unknown fields and unsafe keys are rejected.", inputSchema: buildShape({ databaseId: ID, values: { type: "object" }, expectedDatabaseRevision: REVISION, operationId: OPERATION }, ["databaseId", "values", "expectedDatabaseRevision", "operationId"]), handler: handler(bound, "createDatabaseRecord") },
+    { name: "read_database_record", description: "Read one record with its values, revision, attribution, archive state, and bounded history.", inputSchema: buildShape({ databaseId: ID, recordId: ID }, ["databaseId", "recordId"]), handler: handler(bound, "readDatabaseRecord") },
+    { name: "update_database_record", description: "Update selected typed values using the record revision. Edits to other fields made since a prior save are not overwritten.", inputSchema: buildShape({ databaseId: ID, recordId: ID, values: { type: "object" }, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "recordId", "values", "expectedRevision", "operationId"]), handler: handler(bound, "updateDatabaseRecord") },
+    { name: "archive_database_record", description: "Recoverably archive one record using its current revision.", inputSchema: buildShape({ databaseId: ID, recordId: ID, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "recordId", "expectedRevision", "operationId"]), handler: handler(bound, "archiveDatabaseRecord") },
+    { name: "recover_database_record", description: "Recover one archived record using its current revision.", inputSchema: buildShape({ databaseId: ID, recordId: ID, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "recordId", "expectedRevision", "operationId"]), handler: handler(bound, "recoverDatabaseRecord") },
+    { name: "retry_database_sync", description: "Retry the shared append-only Knowledge summary projection after a canonical database mutation reported projectionPending.", inputSchema: buildShape({ databaseId: ID }, ["databaseId"]), handler: handler(bound, "retryDatabaseSync") },
+  ].map(function (tool) { tool.description = MATE_CONTRACT + " " + tool.description; return tool; });
+}
+
 function mateTools(bound) {
   return [
     {
@@ -87,7 +110,7 @@ function mateTools(bound) {
       }, ["ref"]),
       handler: handler(bound, "readKnowledge"),
     },
-  ];
+  ].concat(databaseTools(bound));
 }
 
 function clayTools(bound) {

package/lib/mate-knowledge-service.js

@@ -393,7 +393,9 @@ function attachMateKnowledgeService(ctx) {
 
     var def = mate.builtinKey ? builtinMates.getBuiltinByKey(mate.builtinKey) : null;
     var isClay = !!(mate.builtinKey === "clay" && def && def.hostAgent === true);
-    return { own: own, isClay: isClay, ownerId: source.projectOwnerId || null };
+    return { own: own, isClay: isClay, ownerId: source.projectOwnerId || null,
+      databaseService: typeof project.getKnowledgeDbService === "function" ? project.getKnowledgeDbService() : null,
+      actor: { type: "user", userId: source.session && source.session.ownerId || source.projectOwnerId || null, displayName: null } };
   }
 
   // A tool handler captured earlier stays valid only while its exact source
@@ -408,7 +410,7 @@ function attachMateKnowledgeService(ctx) {
     function revalidate() {
       var current = resolve(source);
       if (!current) throw new Error("This Knowledge binding is no longer valid for the current session.");
-      if (current.isClay !== initial.isClay || current.own.scopeId !== initial.own.scopeId) {
+      if (current.isClay !== initial.isClay || current.own.scopeId !== initial.own.scopeId || current.databaseService !== initial.databaseService) {
         throw new Error("This Knowledge binding is no longer valid for the current session.");
       }
       return current;
@@ -423,6 +425,10 @@ function attachMateKnowledgeService(ctx) {
     function ownScope() {
       return [revalidate().own];
     }
+    function db(method, args) {
+      var current = revalidate(); if (!current.databaseService || typeof current.databaseService[method] !== "function") throw new Error("Knowledge databases are unavailable for this Mate.");
+      return current.databaseService[method](Object.assign({}, args || {}, { actor: current.actor }));
+    }
     return {
       isClay: false,
       mateId: initial.own.mateId,
@@ -430,6 +436,21 @@ function attachMateKnowledgeService(ctx) {
       listKnowledge: function (args) { return listFor(ownScope(), args || {}, false); },
       searchKnowledge: function (args) { return searchFor(ownScope(), args || {}, false); },
       readKnowledge: function (args) { return readFor(ownScope(), args || {}, false); },
+      listDatabases: function (args) { return db("list", args); },
+      createDatabase: function (args) { return db("create", args); },
+      readDatabase: function (args) { return db("read", args); },
+      renameDatabase: function (args) { return db("rename", args); },
+      archiveDatabase: function (args) { return db("archive", args); },
+      recoverDatabase: function (args) { return db("recover", args); },
+      updateDatabaseSchema: function (args) { return db("updateSchema", args); },
+      saveTableView: function (args) { return db("saveView", args); },
+      queryDatabase: function (args) { return db("query", args); },
+      createDatabaseRecord: function (args) { return db("createRecord", args); },
+      readDatabaseRecord: function (args) { return db("readRecord", args); },
+      updateDatabaseRecord: function (args) { return db("updateRecord", args); },
+      archiveDatabaseRecord: function (args) { return db("archiveRecord", args); },
+      recoverDatabaseRecord: function (args) { return db("recoverRecord", args); },
+      retryDatabaseSync: function (args) { return db("retryProjection", args); },
     };
   }
 

package/lib/mate-knowledge-sync.js

@@ -23,7 +23,7 @@ var config = require("./config");
 var knowledgeImport = require("./knowledge-import");
 
 var SOURCE_NAMESPACE = "mate-knowledge";
-var SOURCE_VERSION = 2;
+var SOURCE_VERSION = 3;
 var MAX_SOURCE_BYTES = 64 * 1024 * 1024;
 
 // Identity and template artifacts. Never mirrored, at any version.
@@ -84,6 +84,13 @@ function sourceKindFor(name) {
   return null;
 }
 
+function safeSourceName(value) {
+  var name = String(value || "").replace(/\\/g, "/").replace(/^\.\//, ""); var parts = name.split("/");
+  if (!name || name[0] === "/" || name.length > 500 || parts.length > 20) return null;
+  if (parts.some(function (part) { return !part || part === "." || part === ".." || part.length > 120; })) return null;
+  return parts.join("/");
+}
+
 function isJournal(name) {
   return !!JOURNAL_KINDS[name] || /\.jsonl$/.test(name);
 }
@@ -133,9 +140,8 @@ function isMateSourceEntry(entry) {
   if (typeof source.sourceVersion !== "number" || source.sourceVersion > SOURCE_VERSION) return false;
   var name = source.fileName;
   if (!name || typeof name !== "string") return false;
-  // The file name and the relative path must agree, so neither field alone is
-  // trusted, and the name must be a plain basename.
-  if (path.basename(name) !== name) return false;
+  // The source name and relative path must agree, so neither field alone is trusted.
+  if (safeSourceName(name) !== name) return false;
   return source.relPath === relativeSource(name);
 }
 
@@ -180,6 +186,18 @@ function readStat(filePath) {
   }
 }
 
+function safeKnowledgeFileStat(root, name) {
+  var parts = name.split("/"); var cursor = root;
+  for (var i = 0; i < parts.length; i++) {
+    cursor = path.join(cursor, parts[i]); var stat;
+    try { stat = fs.lstatSync(cursor); } catch (error) { if (error.code === "ENOENT") return null; throw error; }
+    if (stat.isSymbolicLink()) throw new Error("Knowledge source contains a symbolic link.");
+    if (i < parts.length - 1 && !stat.isDirectory()) throw new Error("Knowledge source path is not a directory.");
+    if (i === parts.length - 1 && !stat.isFile()) return null;
+  }
+  return fs.statSync(cursor);
+}
+
 // --- File-addressed sources -------------------------------------------
 
 function syncFile(importer, scope, name, kind, filePath, stat, actor, summary) {
@@ -361,8 +379,8 @@ function syncMateSource(opts) {
     var scope = options.scope || scopeForMateDir(options.mateDir);
     if (!scope) return summary;
     if (options.mateName) scope.mateName = options.mateName;
-    var name = path.basename(String(options.fileName || ""));
-    if (!name) return summary;
+    var rawName = String(options.fileName || ""); var name = safeSourceName(rawName);
+    if (!name) { if (rawName) summary.unsupported++; return summary; }
     if (IDENTITY_FILES.indexOf(name) !== -1) { summary.identitySkipped++; return summary; }
     if (EXCLUDED_FILES.indexOf(name) !== -1) { summary.excluded++; return summary; }
     var kind = sourceKindFor(name);
@@ -370,8 +388,8 @@ function syncMateSource(opts) {
 
     var actor = normalizeActor(options.actor);
     var importer = options.importer || importerFor(scope, options.baseDir);
-    var filePath = path.join(scope.mateDir, "knowledge", name);
-    var stat = readStat(filePath);
+    var knowledgeRoot = path.join(scope.mateDir, "knowledge"); var filePath = path.join(knowledgeRoot, name);
+    var stat = safeKnowledgeFileStat(knowledgeRoot, name);
 
     if (isJournal(name)) syncJournal(importer, scope, name, kind, filePath, stat, actor, summary);
     else syncFile(importer, scope, name, kind, filePath, stat, actor, summary);
@@ -382,19 +400,11 @@ function syncMateSource(opts) {
   return summary;
 }
 
-// Alias that reads better at a call site reacting to a removal or a rewrite.
 function reconcileSource(opts) {
   return syncMateSource(opts);
 }
 
 // Reconcile every durable source in one Mate directory, in both directions.
-//
-// Enumerating the directory alone is not enough. A knowledge file or a whole
-// journal can be deleted while the daemon is stopped, or by direct filesystem
-// tooling, and nothing would ever call syncMateSource for it again. So the
-// scope's own live records are also consulted: any source this module
-// previously imported that is no longer on disk is reconciled, which tombstones
-// a missing file record or every line of a missing journal.
 function reconcileMate(opts) {
   var options = opts || {};
   var summary = options.summary || newSummary();
@@ -404,11 +414,17 @@ function reconcileMate(opts) {
   var knowledgeDir = path.join(scope.mateDir, "knowledge");
   var present = {};
   var entries = [];
-  try {
-    entries = fs.readdirSync(knowledgeDir, { withFileTypes: true });
-  } catch (e) {
-    entries = [];
+  function discover(relative) {
+    var folder = relative ? path.join(knowledgeDir, relative) : knowledgeDir; var children;
+    try { children = fs.readdirSync(folder, { withFileTypes: true }); } catch (error) { return; }
+    for (var d = 0; d < children.length; d++) {
+      var name = relative ? relative + "/" + children[d].name : children[d].name; var stat;
+      try { stat = fs.lstatSync(path.join(knowledgeDir, name)); } catch (error) { continue; }
+      if (stat.isSymbolicLink()) continue;
+      if (stat.isDirectory()) discover(name); else if (stat.isFile()) entries.push({ name: name });
+    }
   }
+  discover("");
   var importer = options.importer || importerFor(scope, options.baseDir);
 
   function reconcileOne(fileName) {
@@ -423,7 +439,6 @@ function reconcileMate(opts) {
   }
 
   for (var i = 0; i < entries.length; i++) {
-    if (!entries[i].isFile()) continue;
     // Recorded as present whatever its kind, so an excluded or unsupported file
     // that still exists is never mistaken for a deleted source.
     present[entries[i].name] = true;

package/lib/mate-workspace-color.js

@@ -0,0 +1,4 @@
+// Only curated pigments may affect a Mate's workspace surface.
+var COLORS = ['mint', 'rose', 'plum', 'coral', 'peach', 'gold', 'sky', 'indigo', 'stone'];
+function valid(value) { return typeof value === 'string' && COLORS.indexOf(value) !== -1; }
+module.exports = { colors: COLORS, valid: valid };

package/lib/mates.js

@@ -2,6 +2,7 @@ var fs = require("fs");
 var path = require("path");
 var crypto = require("crypto");
 var config = require("./config");
+var workspaceColor = require("./mate-workspace-color");
 
 var crisisSafety = require("./crisis-safety");
 var matesPrompts = require("./mates-prompts");
@@ -192,9 +193,17 @@ function getMate(ctx, id) {
 }
 
 function updateMate(ctx, id, updates) {
+  updates = Object.assign({}, updates);
+  delete updates.id;
+  delete updates.createdBy;
+  delete updates.builtinKey;
   var data = loadMates(ctx);
   for (var i = 0; i < data.mates.length; i++) {
     if (data.mates[i].id === id) {
+      if (Object.prototype.hasOwnProperty.call(updates, "workspaceColor")) {
+        if (data.mates[i].primary || data.mates[i].builtinKey === "clay") delete updates.workspaceColor;
+        else if (!workspaceColor.valid(updates.workspaceColor)) return { error: "Choose a supported workspace color." };
+      }
       // Primary mates: protect name and core identity fields
       if (data.mates[i].primary) {
         delete updates.name;
@@ -206,6 +215,15 @@ function updateMate(ctx, id, updates) {
           delete updates.profile.displayName;
         }
       }
+      // Locked built-in artwork is server-managed. Preserve it even when a
+      // stale or crafted client submits a replacement profile object.
+      if (data.mates[i].profile && data.mates[i].profile.avatarLocked && updates.profile) {
+        updates.profile.avatarStyle = data.mates[i].profile.avatarStyle;
+        updates.profile.avatarSeed = data.mates[i].profile.avatarSeed;
+        updates.profile.avatarColor = data.mates[i].profile.avatarColor;
+        updates.profile.avatarCustom = data.mates[i].profile.avatarCustom;
+        updates.profile.avatarLocked = true;
+      }
       var keys = Object.keys(updates);
       for (var j = 0; j < keys.length; j++) {
         data.mates[i][keys[j]] = updates[keys[j]];