clay-server 5.1.0-beta.1 → 6.0.0-beta.1

package/lib/knowledge-draft-store.js

@@ -0,0 +1,56 @@
+var crypto = require("crypto");
+var fs = require("fs");
+var path = require("path");
+
+var MAX_DRAFT_BYTES = 2 * 1024 * 1024;
+
+function safePart(value) { return crypto.createHash("sha256").update(String(value || "unknown")).digest("hex").slice(0, 24); }
+function createKnowledgeDraftStore(baseDir, scopeKey) {
+  var root = path.join(baseDir, "knowledge-document-drafts", safePart(scopeKey));
+  function safeDirectory(directory, create) {
+    var stat;
+    try { stat = fs.lstatSync(directory); } catch (error) { if (!create || error.code !== "ENOENT") throw error; fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); stat = fs.lstatSync(directory); }
+    if (!stat.isDirectory() || stat.isSymbolicLink()) throw Object.assign(new Error("Knowledge draft path is not a safe directory."), { code: "UNSAFE_PATH" });
+  }
+  function userRoot(userId) { return path.join(root, safePart(userId)); }
+  function target(userId, documentId) { return path.join(userRoot(userId), safePart(documentId) + ".json"); }
+  function read(userId, documentId) {
+    try { safeDirectory(root, false); safeDirectory(userRoot(userId), false); var file = target(userId, documentId); var stat = fs.lstatSync(file);
+      if (!stat.isFile() || stat.isSymbolicLink() || stat.size > MAX_DRAFT_BYTES + 4096) throw Object.assign(new Error("Knowledge draft is not a safe file."), { code: "UNSAFE_PATH" });
+      var value = JSON.parse(fs.readFileSync(file, "utf8")); return value && value.version === 1 ? value : null; }
+    catch (error) { if (error.code === "ENOENT" || error instanceof SyntaxError) return null; throw error; }
+  }
+  function save(userId, documentId, draft) {
+    var content = draft && draft.content;
+    if (typeof content !== "string" || Buffer.byteLength(content, "utf8") > MAX_DRAFT_BYTES) throw new Error("Knowledge draft is invalid or too large.");
+    var current = read(userId, documentId); var clientVersion = Math.max(0, Number(draft.clientVersion) || 0);
+    if (current && Number(current.clientVersion) > clientVersion) return current;
+    var value = { version: 1, documentId: String(documentId), name: String(draft.name || "").slice(0, 500), content: content,
+      baseRevision: draft.baseRevision || null, cursor: draft.cursor && typeof draft.cursor === "object" ? { line: Number(draft.cursor.line) || 0, ch: Number(draft.cursor.ch) || 0 } : null,
+      clientVersion: clientVersion, newDocument: draft.newDocument === true, savedAt: Date.now() };
+    safeDirectory(root, true); safeDirectory(userRoot(userId), true); var file = target(userId, documentId);
+    var temporary = file + ".tmp-" + process.pid + "-" + crypto.randomBytes(5).toString("hex");
+    try { fs.writeFileSync(temporary, JSON.stringify(value), { flag: "wx", mode: 0o600 }); fs.renameSync(temporary, file); }
+    finally { try { fs.unlinkSync(temporary); } catch (error) {} }
+    return value;
+  }
+  function remove(userId, documentId, throughVersion) {
+    var current = read(userId, documentId); if (!current) return false;
+    if (throughVersion != null && Number(current.clientVersion) > Number(throughVersion)) return false;
+    try { fs.unlinkSync(target(userId, documentId)); return true; } catch (error) { if (error.code === "ENOENT") return false; throw error; }
+  }
+  function list(userId) {
+    var folder = userRoot(userId); var names; try { safeDirectory(root, false); safeDirectory(folder, false); } catch (error) { if (error.code === "ENOENT") return []; throw error; }
+    try { names = fs.readdirSync(folder); } catch (error) { if (error.code === "ENOENT") return []; throw error; }
+    var values = [];
+    for (var i = 0; i < names.length; i++) {
+      if (!/^[a-f0-9]{24}\.json$/.test(names[i])) continue;
+      try { var file = path.join(folder, names[i]); var stat = fs.lstatSync(file); if (!stat.isFile() || stat.isSymbolicLink() || stat.size > MAX_DRAFT_BYTES + 4096) continue;
+        var value = JSON.parse(fs.readFileSync(file, "utf8")); if (value && value.version === 1) values.push(value); } catch (error) {}
+    }
+    return values.sort(function (a, b) { return b.savedAt - a.savedAt; });
+  }
+  return { read: read, save: save, remove: remove, list: list };
+}
+
+module.exports = { createKnowledgeDraftStore: createKnowledgeDraftStore };

package/lib/knowledge-file-store.js

@@ -0,0 +1,199 @@
+var crypto = require("crypto");
+var fs = require("fs");
+var path = require("path");
+
+var MAX_CONTENT_BYTES = 2 * 1024 * 1024;
+var RESERVED = { "session-digests.jsonl": true, "sticky-notes.md": true, "memory-summary.md": true };
+
+function revision(content) { return crypto.createHash("sha256").update(content, "utf8").digest("hex"); }
+function cleanName(value) {
+  if (typeof value !== "string") return null;
+  var name = value.replace(/\\/g, "/").replace(/^\.\//, "");
+  if (!name || name.length > 500 || name[0] === "/" || /[\x00-\x1f\x7f]/.test(name)) return null;
+  var parts = name.split("/");
+  if (parts.length > 20 || parts.some(function (part) { return !part || part === "." || part === ".." || part.length > 120; })) return null;
+  if (!/\.(md|jsonl)$/i.test(name) || RESERVED[name]) return null;
+  return parts.join("/");
+}
+function validName(value) { return cleanName(value) !== null; }
+function itemType(name) { return /\.jsonl$/i.test(name) ? "db" : "text"; }
+function atomicJson(target, value) {
+  var temporary = target + ".tmp-" + process.pid + "-" + crypto.randomBytes(6).toString("hex");
+  try { fs.writeFileSync(temporary, JSON.stringify(value, null, 2), { flag: "wx", mode: 0o600 }); fs.renameSync(temporary, target); }
+  finally { try { fs.unlinkSync(temporary); } catch (error) {} }
+}
+
+function createKnowledgeFileStore(cwd, options) {
+  var directory = path.join(cwd, "knowledge");
+  var settings = options || {};
+  var metaRoot = settings.metaDir || path.join(cwd, ".clay", "knowledge-documents");
+  var registryPath = path.join(metaRoot, "documents.json");
+  var historyRoot = path.join(metaRoot, "history");
+  var fileCache = {};
+  function safeDirectory(target, create) {
+    var stat;
+    try { stat = fs.lstatSync(target); }
+    catch (error) { if (!create || error.code !== "ENOENT") throw error; fs.mkdirSync(target, { recursive: true, mode: 0o700 }); stat = fs.lstatSync(target); }
+    if (!stat.isDirectory() || stat.isSymbolicLink()) throw Object.assign(new Error("Knowledge metadata path is not a safe directory."), { code: "UNSAFE_PATH" });
+  }
+  function ensureMetadata(createHistory) {
+    safeDirectory(metaRoot, true);
+    var registryStat;
+    try { registryStat = fs.lstatSync(registryPath); } catch (error) { if (error.code !== "ENOENT") throw error; }
+    if (registryStat && (!registryStat.isFile() || registryStat.isSymbolicLink())) throw Object.assign(new Error("Knowledge metadata registry is not a safe file."), { code: "UNSAFE_PATH" });
+    if (createHistory) safeDirectory(historyRoot, true);
+    else { try { safeDirectory(historyRoot, false); } catch (error) { if (error.code !== "ENOENT") throw error; } }
+  }
+  function ensureDirectory(create) {
+    var stat;
+    try { stat = fs.lstatSync(directory); }
+    catch (error) { if (!create || error.code !== "ENOENT") throw error; fs.mkdirSync(directory, { recursive: true }); stat = fs.lstatSync(directory); }
+    if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error("Knowledge directory is not a safe directory.");
+  }
+  function targetPath(name, createParents) {
+    var clean = cleanName(name); if (!clean) throw new Error("Invalid knowledge file name."); ensureDirectory(!!createParents);
+    var parts = clean.split("/"); var cursor = directory;
+    for (var i = 0; i < parts.length - 1; i++) {
+      cursor = path.join(cursor, parts[i]); var stat;
+      try { stat = fs.lstatSync(cursor); }
+      catch (error) { if (!createParents || error.code !== "ENOENT") throw error; fs.mkdirSync(cursor, { mode: 0o700 }); stat = fs.lstatSync(cursor); }
+      if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error("Knowledge path contains an unsafe directory.");
+    }
+    return path.join(directory, clean);
+  }
+  function registry() {
+    ensureMetadata(false);
+    try { var parsed = JSON.parse(fs.readFileSync(registryPath, "utf8")); return parsed && parsed.version === 1 && parsed.documents ? parsed : { version: 1, documents: {} }; }
+    catch (error) { if (error.code === "ENOENT" || error instanceof SyntaxError) return { version: 1, documents: {} }; throw error; }
+  }
+  function writeRegistry(value) { ensureMetadata(false); atomicJson(registryPath, value); }
+  function existing(name) {
+    var clean = cleanName(name); if (!clean) throw new Error("Invalid knowledge file name."); var target; var stat;
+    try { target = targetPath(clean, false); } catch (error) { if (error.code === "ENOENT") return null; throw error; }
+    try { stat = fs.lstatSync(target); } catch (error) { if (error.code === "ENOENT") return null; throw error; }
+    if (!stat.isFile() || stat.isSymbolicLink()) throw new Error("Knowledge file is not a safe regular file.");
+    if (stat.size > MAX_CONTENT_BYTES) throw Object.assign(new Error("Knowledge document exceeds the safe read limit."), { code: "TOO_LARGE" });
+    var cached = fileCache[clean]; var content;
+    if (cached && cached.size === stat.size && cached.mtime === stat.mtimeMs && cached.ctime === stat.ctimeMs && cached.ino === stat.ino) content = cached.content;
+    else { content = fs.readFileSync(target, "utf8"); fileCache[clean] = { content: content, revision: revision(content), size: stat.size, mtime: stat.mtimeMs, ctime: stat.ctimeMs, ino: stat.ino }; }
+    cached = fileCache[clean]; return { name: clean, content: content, revision: cached.revision, size: stat.size, mtime: stat.mtimeMs, itemType: itemType(clean) };
+  }
+  function walk(relative, out) {
+    var base = relative ? path.join(directory, relative) : directory; var names = fs.readdirSync(base);
+    for (var i = 0; i < names.length; i++) {
+      var rel = relative ? relative + "/" + names[i] : names[i]; var stat;
+      try { stat = fs.lstatSync(path.join(directory, rel)); } catch (error) { continue; }
+      if (stat.isSymbolicLink()) continue;
+      if (stat.isDirectory()) { walk(rel, out); continue; }
+      if (!stat.isFile() || !validName(rel)) continue;
+      var item; try { item = existing(rel); } catch (error) {
+        if (error.code === "TOO_LARGE") item = { name: rel, content: null, revision: null, size: stat.size, mtime: stat.mtimeMs, itemType: itemType(rel), readError: error.message };
+        else continue;
+      }
+      if (item) out.push(item);
+    }
+  }
+  function reconcileIdentity(items) {
+    var data = registry(); var changed = false; var used = {}; var currentNames = {};
+    items.forEach(function (item) { currentNames[item.name] = true; });
+    Object.keys(data.documents).forEach(function (id) { if (!data.documents[id].deleted) used[data.documents[id].name] = id; });
+    for (var i = 0; i < items.length; i++) {
+      var id = used[items[i].name];
+      if (!id) {
+        var candidates = items[i].revision ? Object.keys(data.documents).filter(function (key) { var doc = data.documents[key]; return !currentNames[doc.name] && doc.lastRevision === items[i].revision; }) : [];
+        id = candidates.length === 1 ? candidates[0] : crypto.randomUUID(); changed = true;
+      }
+      if (!data.documents[id] || data.documents[id].name !== items[i].name || data.documents[id].lastRevision !== items[i].revision || data.documents[id].deleted) changed = true;
+      data.documents[id] = Object.assign({}, data.documents[id] || {}, { id: id, name: items[i].name,
+        lastRevision: items[i].revision || data.documents[id] && data.documents[id].lastRevision || null, deleted: false });
+      items[i].id = id; used[items[i].name] = id;
+    }
+    Object.keys(data.documents).forEach(function (id) {
+      var found = items.some(function (item) { return item.id === id; });
+      if (!found && !data.documents[id].deleted) { data.documents[id].deleted = true; changed = true; }
+    });
+    if (changed) writeRegistry(data); return data;
+  }
+  function list() {
+    try { ensureDirectory(false); } catch (error) { if (error.code === "ENOENT") return []; throw error; }
+    var items = []; walk("", items); reconcileIdentity(items);
+    return items.map(function (item) { return { id: item.id, name: item.name, path: item.name, folder: path.posix.dirname(item.name) === "." ? "" : path.posix.dirname(item.name),
+      itemType: item.itemType, size: item.size, mtime: item.mtime, revision: item.revision, writable: item.itemType === "text" }; })
+      .sort(function (a, b) { return a.name.localeCompare(b.name); });
+  }
+  function read(name) {
+    ensureDirectory(false); var item = existing(name); if (!item) throw Object.assign(new Error("Knowledge file not found."), { code: "ENOENT" });
+    var data = registry(); var id = Object.keys(data.documents).filter(function (key) { return !data.documents[key].deleted && data.documents[key].name === item.name; })[0];
+    if (!id) { list(); data = registry(); id = Object.keys(data.documents).filter(function (key) { return !data.documents[key].deleted && data.documents[key].name === item.name; })[0]; }
+    return Object.assign(item, { id: id || null });
+  }
+  function assertExpected(current, expected) {
+    if (!current && expected) throw Object.assign(new Error("This document was removed since it was opened."), { code: "STALE", currentRevision: null });
+    if (current && (!expected || expected !== current.revision)) throw Object.assign(new Error("This document changed since it was opened."), { code: "STALE", currentRevision: current.revision });
+  }
+  function snapshot(item, reason) {
+    if (!item) return;
+    var listed = list(); var identity = listed.filter(function (entry) { return entry.name === item.name; })[0]; if (!identity) return; ensureMetadata(true);
+    var folder = path.join(historyRoot, identity.id); safeDirectory(folder, true);
+    var target = path.join(folder, Date.now() + "-" + item.revision + ".md");
+    fs.writeFileSync(target, item.content, { flag: "wx", mode: 0o600 });
+  }
+  function save(name, content, expectedRevision) {
+    var clean = cleanName(name); if (!clean || itemType(clean) !== "text") throw new Error("Only valid Markdown documents can be edited.");
+    if (typeof content !== "string") throw new Error("Knowledge content is required.");
+    if (Buffer.byteLength(content, "utf8") > MAX_CONTENT_BYTES) throw new Error("Knowledge document is too large.");
+    ensureDirectory(true); var current = existing(clean); assertExpected(current, expectedRevision); if (current) snapshot(current, "before-save");
+    var target = targetPath(clean, true); var temporary = path.join(path.dirname(target), ".clay-knowledge-" + process.pid + "-" + crypto.randomBytes(8).toString("hex"));
+    try { fs.writeFileSync(temporary, content, { flag: "wx", mode: 0o600 }); fs.renameSync(temporary, target); }
+    finally { try { fs.unlinkSync(temporary); } catch (error) {} }
+    delete fileCache[clean];
+    return read(clean);
+  }
+  function rename(from, to, expectedRevision) {
+    var cleanFrom = cleanName(from); var cleanTo = cleanName(to);
+    if (!cleanTo || itemType(cleanTo) !== "text") throw new Error("Only valid Markdown documents can be renamed or moved.");
+    if (!cleanFrom || itemType(cleanFrom) !== "text") throw Object.assign(new Error("This Knowledge source is read only."), { code: "READ_ONLY" });
+    ensureDirectory(false); var current = existing(cleanFrom); if (!current) throw Object.assign(new Error("Knowledge file not found."), { code: "ENOENT" });
+    assertExpected(current, expectedRevision); if (existing(cleanTo)) throw Object.assign(new Error("A document with that path already exists."), { code: "EXISTS" }); snapshot(current, "before-move");
+    fs.renameSync(targetPath(cleanFrom, false), targetPath(cleanTo, true));
+    delete fileCache[cleanFrom];
+    var data = registry(); Object.keys(data.documents).forEach(function (id) { if (data.documents[id].name === cleanFrom && !data.documents[id].deleted) data.documents[id].name = cleanTo; }); writeRegistry(data);
+    return read(cleanTo);
+  }
+  function remove(name, expectedRevision) {
+    var clean = cleanName(name); if (!clean || itemType(clean) !== "text") throw Object.assign(new Error("This Knowledge source is read only."), { code: "READ_ONLY" });
+    ensureDirectory(false); var current = existing(clean); if (!current) throw Object.assign(new Error("Knowledge file not found."), { code: "ENOENT" });
+    assertExpected(current, expectedRevision); snapshot(current, "before-delete"); fs.unlinkSync(targetPath(clean, false)); delete fileCache[clean]; list(); return current;
+  }
+  function history(id) {
+    if (!/^[a-f0-9-]{20,50}$/i.test(String(id || ""))) throw new Error("Invalid document identity."); ensureMetadata(false); var folder = path.join(historyRoot, id); var names;
+    try { safeDirectory(folder, false); } catch (error) { if (error.code === "ENOENT") return []; throw error; }
+    try { names = fs.readdirSync(folder); } catch (error) { if (error.code === "ENOENT") return []; throw error; }
+    return names.filter(function (name) { return /^\d+-[a-f0-9]{64}\.md$/.test(name); }).sort().reverse().slice(0, 50).map(function (name) {
+      var target = path.join(folder, name); var stat = fs.lstatSync(target);
+      if (!stat.isFile() || stat.isSymbolicLink() || stat.size > MAX_CONTENT_BYTES) throw Object.assign(new Error("Knowledge history contains an unsafe version."), { code: "UNSAFE_PATH" });
+      return { version: name, size: stat.size, createdAt: Number(name.split("-")[0]) };
+    });
+  }
+  function readHistory(id, versionName) {
+    var versions = history(id); if (!versions.some(function (entry) { return entry.version === versionName; })) throw Object.assign(new Error("Knowledge history version not found."), { code: "ENOENT" });
+    return { version: versionName, content: fs.readFileSync(path.join(historyRoot, id, versionName), "utf8") };
+  }
+  function restore(id, versionName, name, expectedRevision) {
+    if (!/^[a-f0-9-]{20,50}$/i.test(String(id || "")) || !/^\d+-[a-f0-9]{64}\.md$/.test(String(versionName || ""))) throw new Error("Invalid history version.");
+    var selected = readHistory(id, versionName);
+    var data = registry(); var record = data.documents[id]; if (!record) throw Object.assign(new Error("Knowledge document identity not found."), { code: "ENOENT" });
+    var current = existing(name); var restored = save(name, selected.content, current ? expectedRevision || null : null);
+    data = registry(); Object.keys(data.documents).forEach(function (key) { if (key !== id && data.documents[key].name === restored.name && !data.documents[key].deleted) delete data.documents[key]; });
+    data.documents[id] = Object.assign({}, record, { id: id, name: restored.name, lastRevision: restored.revision, deleted: false }); writeRegistry(data);
+    return Object.assign(restored, { id: id });
+  }
+  function deleted() {
+    var data = registry(); return Object.keys(data.documents).filter(function (id) { return data.documents[id].deleted; }).map(function (id) {
+      return { id: id, name: data.documents[id].name, revision: data.documents[id].lastRevision || null, deleted: true, historyCount: history(id).length };
+    }).filter(function (item) { return item.historyCount > 0; }).sort(function (a, b) { return a.name.localeCompare(b.name); });
+  }
+  return { list: list, read: read, save: save, rename: rename, remove: remove, history: history, readHistory: readHistory, restore: restore, deleted: deleted };
+}
+
+module.exports = { createKnowledgeFileStore: createKnowledgeFileStore, revision: revision, validName: validName, cleanName: cleanName, itemType: itemType, MAX_CONTENT_BYTES: MAX_CONTENT_BYTES };

package/lib/knowledge-markdown.js

@@ -0,0 +1,128 @@
+var path = require("path");
+
+function slug(value) {
+  return String(value || "").trim().toLowerCase().replace(/<[^>]*>/g, "").replace(/[^\p{L}\p{N}\s-]/gu, "").replace(/\s+/g, "-").replace(/-+/g, "-");
+}
+
+function frontmatter(content) {
+  var text = String(content || "");
+  if (text.indexOf("---\n") !== 0) return { properties: {}, tags: [], end: 0, errors: [] };
+  var close = text.indexOf("\n---\n", 4);
+  if (close === -1 || close > 8192) return { properties: {}, tags: [], end: 0, errors: ["Frontmatter is missing its closing --- line."] };
+  var properties = {}; var errors = []; var lines = text.slice(4, close).split("\n");
+  for (var i = 0; i < lines.length; i++) {
+    if (!lines[i].trim() || /^\s*#/.test(lines[i])) continue;
+    var match = /^([A-Za-z][A-Za-z0-9_-]{0,63}):\s*(.*)$/.exec(lines[i]);
+    if (!match) { errors.push("Invalid property on line " + (i + 2) + "."); continue; }
+    var value = match[2].trim();
+    if (value.length > 500) errors.push("Property " + match[1] + " is too long.");
+    else properties[match[1]] = value;
+  }
+  var tags = [];
+  if (properties.tags) {
+    var raw = properties.tags.replace(/^\[/, "").replace(/\]$/, "").split(",");
+    for (var t = 0; t < raw.length; t++) {
+      var tag = raw[t].trim().replace(/^#/, "");
+      if (/^[\p{L}\p{N}_/-]{1,64}$/u.test(tag) && tags.indexOf(tag) === -1) tags.push(tag);
+      else if (tag) errors.push("Invalid tag: " + tag);
+    }
+  }
+  return { properties: properties, tags: tags, end: close + 5, errors: errors };
+}
+
+function ignoredRanges(text) {
+  var ranges = []; var lines = text.split("\n"); var offset = 0; var fence = null;
+  for (var i = 0; i < lines.length; i++) {
+    var line = lines[i]; var marker = /^( {0,3})(`{3,}|~{3,})(.*)$/.exec(line);
+    if (marker && (!fence || marker[2][0] === fence.marker && marker[2].length >= fence.length && /^\s*$/.test(marker[3]))) {
+      if (!fence) fence = { marker: marker[2][0], length: marker[2].length, start: offset };
+      else { ranges.push([fence.start, offset + line.length]); fence = null; }
+    }
+    if (!fence) {
+      var cursor = 0;
+      while (cursor < line.length) {
+        var opener = /`+/.exec(line.slice(cursor)); if (!opener) break;
+        var start = cursor + opener.index; var run = opener[0]; var close = line.indexOf(run, start + run.length);
+        if (close === -1) { cursor = start + run.length; continue; }
+        ranges.push([offset + start, offset + close + run.length]); cursor = close + run.length;
+      }
+    }
+    offset += line.length + 1;
+  }
+  if (fence) ranges.push([fence.start, text.length]);
+  return ranges;
+}
+
+function escaped(text, start) {
+  var slashes = 0;
+  for (var i = start - 1; i >= 0 && text[i] === "\\"; i--) slashes++;
+  return slashes % 2 === 1;
+}
+
+function inside(ranges, start) {
+  for (var i = 0; i < ranges.length; i++) if (start >= ranges[i][0] && start < ranges[i][1]) return true;
+  return false;
+}
+
+function parse(content) {
+  var text = String(content || ""); var ignored = ignoredRanges(text); var links = []; var headings = []; var match;
+  var headingPattern = /^(#{1,6})\s+(.+?)\s*#*\s*$/gm;
+  while ((match = headingPattern.exec(text))) {
+    if (!inside(ignored, match.index)) headings.push({ depth: match[1].length, text: match[2].trim(), slug: slug(match[2]), offset: match.index });
+  }
+  var wiki = /(!)?\[\[([^\]\n]+)\]\]/g;
+  while ((match = wiki.exec(text))) {
+    if (inside(ignored, match.index) || escaped(text, match.index)) continue;
+    var parts = match[2].split("|"); var target = parts[0].trim(); var alias = parts.length > 1 ? parts.slice(1).join("|").trim() : "";
+    var hash = target.indexOf("#");
+    var wikiTarget = hash === -1 ? target : target.slice(0, hash);
+    links.push({ kind: match[1] ? "embed" : "wiki", target: wikiTarget, heading: hash === -1 ? "" : target.slice(hash + 1), alias: alias,
+      start: match.index, end: match.index + match[0].length, raw: match[0], external: /^[a-z][a-z0-9+.-]*:/i.test(wikiTarget) || wikiTarget.indexOf("//") === 0 });
+  }
+  var markdown = /(!)?\[([^\]\n]*)\]\(([^)\n]+)\)/g;
+  while ((match = markdown.exec(text))) {
+    if (inside(ignored, match.index) || escaped(text, match.index)) continue;
+    var destination = match[3].trim().replace(/^<|>$/g, ""); var hashAt = destination.indexOf("#");
+    links.push({ kind: match[1] ? "image" : "markdown", target: hashAt === -1 ? destination : destination.slice(0, hashAt), heading: hashAt === -1 ? "" : destination.slice(hashAt + 1), alias: match[2],
+      start: match.index, end: match.index + match[0].length, raw: match[0], external: /^[a-z][a-z0-9+.-]*:/i.test(destination) || destination.indexOf("//") === 0 });
+  }
+  links.sort(function (a, b) { return a.start - b.start; });
+  var meta = frontmatter(text); var tags = meta.tags.slice(); var tagPattern = /(^|\s)#([\p{L}\p{N}_/-]{1,64})/gu;
+  while ((match = tagPattern.exec(text))) if (!inside(ignored, match.index) && tags.indexOf(match[2]) === -1) tags.push(match[2]);
+  return { links: links, headings: headings, properties: meta.properties, tags: tags, propertyErrors: meta.errors };
+}
+
+function normalizeTarget(sourceName, target) {
+  var raw = String(target || "").trim();
+  if (!raw || /^[a-z][a-z0-9+.-]*:/i.test(raw) || raw.indexOf("//") === 0) return null;
+  try { raw = decodeURIComponent(raw); } catch (error) {}
+  raw = raw.replace(/\\/g, "/");
+  if (!/\.md$/i.test(raw)) raw += ".md";
+  return path.posix.normalize(path.posix.join(path.posix.dirname(sourceName), raw)).replace(/^\.\//, "");
+}
+
+function resolveLink(sourceName, link, documents, sourceId) {
+  if (link.external) return null;
+  if (!link.target && link.heading) {
+    var self = documents.filter(function (item) { return sourceId ? item.id === sourceId : item.name === sourceName; });
+    return self.length === 1 ? self[0] : null;
+  }
+  var target = normalizeTarget(sourceName, link.target);
+  if (!target) return null;
+  var exact = documents.filter(function (item) { return item.name.toLowerCase() === target.toLowerCase(); });
+  if (exact.length === 1) return exact[0];
+  if (link.target.indexOf("/") === -1 && link.target.indexOf("\\") === -1) {
+    var base = path.posix.basename(target).toLowerCase();
+    var named = documents.filter(function (item) { return path.posix.basename(item.name).toLowerCase() === base; });
+    if (named.length === 1) return named[0];
+  }
+  return null;
+}
+
+function snippet(content, offset, needle) {
+  var raw = String(content || ""); var at = typeof offset === "number" ? offset : raw.toLowerCase().indexOf(String(needle || "").toLowerCase());
+  if (at < 0) at = 0; var start = Math.max(0, at - 70); var end = Math.min(raw.length, at + 150); var text = raw.slice(start, end).replace(/\s+/g, " ").trim();
+  return (start ? "…" : "") + text + (end < raw.length ? "…" : "");
+}
+
+module.exports = { parse: parse, slug: slug, normalizeTarget: normalizeTarget, resolveLink: resolveLink, snippet: snippet };

package/lib/linear-store.js

@@ -19,34 +19,61 @@ function attachLinearStore(root) {
     catch (error) { if (error.code !== 'EEXIST') throw error; }
     return fs.readFileSync(file);
   }
-  function key(owner) {
+  function record(owner, id) {
     var data = read(owner);
+    return !id || id === "default" ? data : Object.hasOwn(data.connections || {}, id) ? data.connections[id] : {};
+  }
+  function key(owner, id) {
+    var data = record(owner, id);
     if (!data.secret) return null;
     var parts = data.secret.split('.').map(function (part) { return Buffer.from(part, 'base64'); });
     var cipher = crypto.createDecipheriv('aes-256-gcm', masterKey(), parts[0]);
-    cipher.setAAD(Buffer.from(owner)); cipher.setAuthTag(parts[1]);
+    cipher.setAAD(Buffer.from(id && id !== 'default' ? owner + ':' + id : owner)); cipher.setAuthTag(parts[1]);
     return Buffer.concat([cipher.update(parts[2]), cipher.final()]).toString('utf8');
   }
-  function view(owner) {
-    var data = read(owner);
+  function view(owner, id) {
+    var data = record(owner, id);
     return { connected: !!data.secret, workspace: data.workspace || null, revision: data.revision || null };
   }
-  function save(owner, secret, workspace) {
+  function save(owner, secret, workspace, id, name) {
+    if (id && (!/^[a-zA-Z0-9_-]{1,64}$/.test(id) || ['__proto__', 'constructor', 'prototype'].indexOf(id) >= 0)) throw new Error("Invalid connection.");
+    var previous = read(owner);
     var file = filename(owner), data = { revision: crypto.randomBytes(16).toString('hex') };
     if (secret !== null) {
       if (typeof secret !== 'string' || !secret.trim() || secret.length > 4096 || /[\r\n]/.test(secret)) throw new Error('Enter a valid Linear API key.');
       var iv = crypto.randomBytes(12), cipher = crypto.createCipheriv('aes-256-gcm', masterKey(), iv);
-      cipher.setAAD(Buffer.from(owner));
+      cipher.setAAD(Buffer.from(id && id !== 'default' ? owner + ':' + id : owner));
       var bytes = Buffer.concat([cipher.update(secret.trim(), 'utf8'), cipher.final()]);
       data.secret = [iv, cipher.getAuthTag(), bytes].map(function (part) { return part.toString('base64'); }).join('.');
       data.workspace = workspace;
     }
+    if (name !== undefined) {
+      if (typeof name !== 'string' || !name.trim() || name.trim().length > 80) throw new Error('Enter a connection name of up to 80 characters.');
+      data.name = name.trim();
+    } else data.name = record(owner, id).name;
+    if (id && id !== 'default') {
+      previous.connections = previous.connections || {};
+      if (secret === null) delete previous.connections[id];
+      else {
+        if (!previous.connections[id] && Object.keys(previous.connections).length >= 30) throw new Error('Connection limit reached.');
+        previous.connections[id] = data;
+      }
+      data = previous;
+    } else data.connections = previous.connections;
     fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
     var temp = file + '.' + crypto.randomBytes(8).toString('hex') + '.tmp';
     try { fs.writeFileSync(temp, JSON.stringify(data), { mode: 0o600 }); fs.renameSync(temp, file); }
     finally { try { fs.unlinkSync(temp); } catch (error) { if (error.code !== 'ENOENT') throw error; } }
-    return view(owner);
+    return view(owner, id);
+  }
+  function list(owner) {
+    var data = read(owner), entries = [];
+    if (data.secret) entries.push(Object.assign({ id: 'default', name: data.name || 'Existing connection', legacy: true }, view(owner)));
+    Object.keys(data.connections || {}).forEach(function (id) {
+      entries.push(Object.assign({ id: id, name: data.connections[id].name || 'Linear', legacy: false }, view(owner, id)));
+    });
+    return entries;
   }
-  return { key: key, view: view, save: save };
+  return { key: key, view: view, save: save, list: list };
 }
 module.exports = { attachLinearStore: attachLinearStore };

package/lib/mate-knowledge-mcp-server.js

@@ -46,14 +46,37 @@ function handler(bound, method) {
     if (!bound || typeof bound[method] !== "function") {
       return errorResult(new Error("Knowledge tools require an exact session-bound Mate."));
     }
-    try {
-      return textResult(bound[method](args || {}));
-    } catch (e) {
-      return errorResult(e);
-    }
+    try { return Promise.resolve(bound[method](args || {})).then(textResult, errorResult); } catch (e) { return errorResult(e); }
   };
 }
 
+var ID = { type: "string", description: "Stable opaque id returned by a Knowledge database tool." };
+var REVISION = { type: "number", description: "Revision returned by the most recent read or mutation; stale revisions are rejected." };
+var OPERATION = { type: "string", description: "Stable retry id formatted label_<current epoch milliseconds>_<unique suffix>. Reuse the exact id and payload after an uncertain response; receipts are retained for the documented 24-hour retry window." };
+var FIELD = { type: "object", description: "Field with stable id, name, type, required, and type-specific options or reference target." };
+var FILTER = { type: "object", description: "Validated filter: fieldId, op, and value. Supported ops are eq, neq, contains, gt, gte, lt, lte, empty, not-empty." };
+var SORT = { type: "object", description: "Validated sort with fieldId and asc or desc direction." };
+
+function databaseTools(bound) {
+  return [
+    { name: "list_databases", description: "List canonical Knowledge databases, including archived databases when requested, so the Mate can discover and recover them without relying on summary projection.", inputSchema: buildShape({ archived: { type: "boolean" } }, []), handler: handler(bound, "listDatabases") },
+    { name: "create_database", description: "Create a first-class database in this Mate's Knowledge. Defaults to one required title field and a Table view.", inputSchema: buildShape({ name: { type: "string" }, schema: { type: "array", items: FIELD }, operationId: OPERATION }, ["name", "operationId"]), handler: handler(bound, "createDatabase") },
+    { name: "read_database", description: "Read one Knowledge database's current schema, views, revision, and record count.", inputSchema: buildShape({ databaseId: ID, includeArchived: { type: "boolean" } }, ["databaseId"]), handler: handler(bound, "readDatabase") },
+    { name: "rename_database", description: "Rename a Knowledge database with optimistic revision checking.", inputSchema: buildShape({ databaseId: ID, name: { type: "string" }, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "name", "expectedRevision", "operationId"]), handler: handler(bound, "renameDatabase") },
+    { name: "archive_database", description: "Recoverably archive a Knowledge database.", inputSchema: buildShape({ databaseId: ID, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "expectedRevision", "operationId"]), handler: handler(bound, "archiveDatabase") },
+    { name: "recover_database", description: "Recover an archived Knowledge database.", inputSchema: buildShape({ databaseId: ID, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "expectedRevision", "operationId"]), handler: handler(bound, "recoverDatabase") },
+    { name: "update_database_schema", description: "Replace the schema using stable field ids. Exactly one active title field is required; populated fields cannot be removed or change type and should be archived instead.", inputSchema: buildShape({ databaseId: ID, schema: { type: "array", items: FIELD }, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "schema", "expectedRevision", "operationId"]), handler: handler(bound, "updateDatabaseSchema") },
+    { name: "save_table_view", description: "Create or revise a persisted Table view with selected columns, validated filters, and deterministic sorts.", inputSchema: buildShape({ databaseId: ID, view: { type: "object", description: "View with stable id, name, columns, filters, and sorts." }, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "view", "expectedRevision", "operationId"]), handler: handler(bound, "saveTableView") },
+    { name: "query_database", description: "Query a bounded deterministic page of records using a saved view or explicit validated filters and sorts. Returns count and numeric sums.", inputSchema: buildShape({ databaseId: ID, viewId: ID, filters: { type: "array", items: FILTER }, sorts: { type: "array", items: SORT }, cursor: { type: "string" }, limit: { type: "number" }, archived: { type: "boolean" } }, ["databaseId"]), handler: handler(bound, "queryDatabase") },
+    { name: "create_database_record", description: "Create a typed record. Values are keyed by stable field id; unknown fields and unsafe keys are rejected.", inputSchema: buildShape({ databaseId: ID, values: { type: "object" }, expectedDatabaseRevision: REVISION, operationId: OPERATION }, ["databaseId", "values", "expectedDatabaseRevision", "operationId"]), handler: handler(bound, "createDatabaseRecord") },
+    { name: "read_database_record", description: "Read one record with its values, revision, attribution, archive state, and bounded history.", inputSchema: buildShape({ databaseId: ID, recordId: ID }, ["databaseId", "recordId"]), handler: handler(bound, "readDatabaseRecord") },
+    { name: "update_database_record", description: "Update selected typed values using the record revision. Edits to other fields made since a prior save are not overwritten.", inputSchema: buildShape({ databaseId: ID, recordId: ID, values: { type: "object" }, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "recordId", "values", "expectedRevision", "operationId"]), handler: handler(bound, "updateDatabaseRecord") },
+    { name: "archive_database_record", description: "Recoverably archive one record using its current revision.", inputSchema: buildShape({ databaseId: ID, recordId: ID, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "recordId", "expectedRevision", "operationId"]), handler: handler(bound, "archiveDatabaseRecord") },
+    { name: "recover_database_record", description: "Recover one archived record using its current revision.", inputSchema: buildShape({ databaseId: ID, recordId: ID, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "recordId", "expectedRevision", "operationId"]), handler: handler(bound, "recoverDatabaseRecord") },
+    { name: "retry_database_sync", description: "Retry the shared append-only Knowledge summary projection after a canonical database mutation reported projectionPending.", inputSchema: buildShape({ databaseId: ID }, ["databaseId"]), handler: handler(bound, "retryDatabaseSync") },
+  ].map(function (tool) { tool.description = MATE_CONTRACT + " " + tool.description; return tool; });
+}
+
 function mateTools(bound) {
   return [
     {
@@ -87,7 +110,7 @@ function mateTools(bound) {
       }, ["ref"]),
       handler: handler(bound, "readKnowledge"),
     },
-  ];
+  ].concat(databaseTools(bound));
 }
 
 function clayTools(bound) {

package/lib/mate-knowledge-service.js

@@ -393,7 +393,9 @@ function attachMateKnowledgeService(ctx) {
 
     var def = mate.builtinKey ? builtinMates.getBuiltinByKey(mate.builtinKey) : null;
     var isClay = !!(mate.builtinKey === "clay" && def && def.hostAgent === true);
-    return { own: own, isClay: isClay, ownerId: source.projectOwnerId || null };
+    return { own: own, isClay: isClay, ownerId: source.projectOwnerId || null,
+      databaseService: typeof project.getKnowledgeDbService === "function" ? project.getKnowledgeDbService() : null,
+      actor: { type: "user", userId: source.session && source.session.ownerId || source.projectOwnerId || null, displayName: null } };
   }
 
   // A tool handler captured earlier stays valid only while its exact source
@@ -408,7 +410,7 @@ function attachMateKnowledgeService(ctx) {
     function revalidate() {
       var current = resolve(source);
       if (!current) throw new Error("This Knowledge binding is no longer valid for the current session.");
-      if (current.isClay !== initial.isClay || current.own.scopeId !== initial.own.scopeId) {
+      if (current.isClay !== initial.isClay || current.own.scopeId !== initial.own.scopeId || current.databaseService !== initial.databaseService) {
         throw new Error("This Knowledge binding is no longer valid for the current session.");
       }
       return current;
@@ -423,6 +425,10 @@ function attachMateKnowledgeService(ctx) {
     function ownScope() {
       return [revalidate().own];
     }
+    function db(method, args) {
+      var current = revalidate(); if (!current.databaseService || typeof current.databaseService[method] !== "function") throw new Error("Knowledge databases are unavailable for this Mate.");
+      return current.databaseService[method](Object.assign({}, args || {}, { actor: current.actor }));
+    }
     return {
       isClay: false,
       mateId: initial.own.mateId,
@@ -430,6 +436,21 @@ function attachMateKnowledgeService(ctx) {
       listKnowledge: function (args) { return listFor(ownScope(), args || {}, false); },
       searchKnowledge: function (args) { return searchFor(ownScope(), args || {}, false); },
       readKnowledge: function (args) { return readFor(ownScope(), args || {}, false); },
+      listDatabases: function (args) { return db("list", args); },
+      createDatabase: function (args) { return db("create", args); },
+      readDatabase: function (args) { return db("read", args); },
+      renameDatabase: function (args) { return db("rename", args); },
+      archiveDatabase: function (args) { return db("archive", args); },
+      recoverDatabase: function (args) { return db("recover", args); },
+      updateDatabaseSchema: function (args) { return db("updateSchema", args); },
+      saveTableView: function (args) { return db("saveView", args); },
+      queryDatabase: function (args) { return db("query", args); },
+      createDatabaseRecord: function (args) { return db("createRecord", args); },
+      readDatabaseRecord: function (args) { return db("readRecord", args); },
+      updateDatabaseRecord: function (args) { return db("updateRecord", args); },
+      archiveDatabaseRecord: function (args) { return db("archiveRecord", args); },
+      recoverDatabaseRecord: function (args) { return db("recoverRecord", args); },
+      retryDatabaseSync: function (args) { return db("retryProjection", args); },
     };
   }