clay-server 5.2.0-beta.2 → 5.2.0-beta.3

package/lib/dm-digest-cursor.js

@@ -0,0 +1,120 @@
+var CONVERSATION_CAP = 6000;
+
+// SDK text is persisted as delta before onTurnDone; the response preview
+// is only a notification excerpt. Historical complete text entries remain readable.
+function isDigestEntry(entry) {
+  return entry && (entry.type === "user_message" || entry.type === "assistant_message" || entry.type === "delta" || entry.type === "text");
+}
+
+function copyCursor(cursor) {
+  return {
+    historyIndex: Math.max(0, cursor.historyIndex || 0),
+    textOffset: Math.max(0, cursor.textOffset || 0),
+  };
+}
+
+function readState(history) {
+  var state = { cursor: { historyIndex: 0, textOffset: 0 } };
+  for (var i = history.length - 1; i >= 0; i--) {
+    var entry = history[i];
+    if (!entry || entry.type !== "digest_checkpoint") continue;
+    if (entry.cursor && typeof entry.cursor.historyIndex === "number") {
+      state.cursor = copyCursor(entry.cursor);
+    } else if (typeof entry.digestedIndex === "number") {
+      // Legacy checkpoints pointed at the next history entry and had no
+      // within-entry progress. Treat them as fully consumed entries.
+      state.cursor = { historyIndex: Math.max(0, entry.digestedIndex), textOffset: 0 };
+    }
+    break;
+  }
+  return state;
+}
+
+function readCursor(history) {
+  return readState(history).cursor;
+}
+
+function sameCursor(left, right) {
+  return left.historyIndex === right.historyIndex && left.textOffset === right.textOffset;
+}
+
+function safeTextEnd(text, start, maximum) {
+  var end = Math.min(text.length, start + maximum);
+  if (end < text.length && end > start) {
+    var previous = text.charCodeAt(end - 1);
+    if (previous >= 0xD800 && previous <= 0xDBFF) end--;
+  }
+  // The normal cap leaves enough room for a code point. Keep this fallback
+  // defensive so a future smaller cap cannot make the cursor stall.
+  if (end === start && end < text.length) {
+    end = start + (text.charCodeAt(start) >= 0xD800 && text.charCodeAt(start) <= 0xDBFF ? 2 : 1);
+  }
+  return Math.min(end, text.length);
+}
+
+function collectChunk(history, startCursor) {
+  var cursor = copyCursor(startCursor);
+  var parts = [];
+  var totalLength = 0;
+  var hadHistoryProgress = false;
+  var partCount = 0;
+
+  function appendPart(value) {
+    var separatorLength = partCount > 0 ? 1 : 0;
+    if (totalLength + separatorLength + value.length > CONVERSATION_CAP) return false;
+    parts.push(value);
+    totalLength += separatorLength + value.length;
+    partCount++;
+    return true;
+  }
+
+  while (cursor.historyIndex < history.length) {
+    var entry = history[cursor.historyIndex];
+    if (!isDigestEntry(entry) || typeof entry.text !== "string" || !entry.text) {
+      cursor.historyIndex++;
+      cursor.textOffset = 0;
+      hadHistoryProgress = true;
+      continue;
+    }
+
+    var text = entry.text;
+    var offset = Math.min(cursor.textOffset, text.length);
+
+    var label = entry.type === "user_message" ? "User: " : "Mate: ";
+    var available = CONVERSATION_CAP - totalLength - (partCount > 0 ? 1 : 0) - label.length;
+    if (available <= 0 && partCount > 0) break;
+    if (available <= 0) available = CONVERSATION_CAP - label.length;
+
+    if (offset < text.length) {
+      var end = safeTextEnd(text, offset, available);
+      var value = label + text.substring(offset, end);
+      if (!appendPart(value)) break;
+      cursor.textOffset = end;
+      hadHistoryProgress = true;
+      if (end < text.length) break;
+    }
+
+    cursor.historyIndex++;
+    cursor.textOffset = 0;
+  }
+
+  return {
+    conversationContent: parts.join("\n"),
+    cursor: cursor,
+    hasHistoryProgress: hadHistoryProgress,
+  };
+}
+
+function hasPending(history, cursor) {
+  var probe = collectChunk(history, cursor, "");
+  return !!probe.conversationContent;
+}
+
+module.exports = {
+  CONVERSATION_CAP: CONVERSATION_CAP,
+  collectChunk: collectChunk,
+  hasPending: hasPending,
+  readCursor: readCursor,
+  readState: readState,
+  sameCursor: sameCursor,
+};

package/lib/knowledge-db-import-validation.js

@@ -0,0 +1,46 @@
+var crypto = require("crypto");
+var validation = require("./knowledge-db-validation");
+var MAX_BATCH_ROWS = 100;
+var MAX_BATCH_BYTES = 256 * 1024;
+
+function fail(message, code) { throw Object.assign(new Error(message), { code: code || "INVALID" }); }
+function json(value, depth) {
+  if (depth > 32) fail("Import JSON exceeds 32 nesting levels.");
+  if (value === null || typeof value === "string" || typeof value === "boolean") return;
+  if (typeof value === "number" && Number.isFinite(value)) return;
+  if (typeof value !== "object" || !value) fail("Import payload must contain finite JSON values.");
+  if (!Array.isArray(value)) validation.plainObject(value, "Import payload");
+  Object.keys(value).forEach(function (key) { json(value[key], depth + 1); });
+}
+function canonical(value) {
+  if (Array.isArray(value)) return value.map(canonical);
+  if (!value || typeof value !== "object") return value;
+  var out = {}; Object.keys(value).sort().forEach(function (key) { out[key] = canonical(value[key]); }); return out;
+}
+function identity(namespace, key) { return JSON.stringify([namespace, key]); }
+function sourceText(value, label, max) {
+  var clean = validation.text(value, label, max, false);
+  if (clean !== value) fail(label + " must not have surrounding whitespace.");
+  return clean;
+}
+function batch(args) {
+  validation.plainObject(args, "Import request");
+  var namespace = sourceText(args.sourceNamespace, "Source namespace", 120);
+  if (!Array.isArray(args.rows) || !args.rows.length || args.rows.length > MAX_BATCH_ROWS) fail("Import batch must contain between 1 and " + MAX_BATCH_ROWS + " rows.", "IMPORT_BATCH_LIMIT");
+  json(args.rows, 0);
+  if (Buffer.byteLength(JSON.stringify(args.rows), "utf8") > MAX_BATCH_BYTES) fail("Import rows exceed the " + MAX_BATCH_BYTES + " byte input limit.", "IMPORT_BYTES_LIMIT");
+  var seen = new Set();
+  var rows = args.rows.map(function (row) {
+    validation.plainObject(row, "Import row");
+    var key = sourceText(row.sourceKey, "Source row key", 200);
+    if (seen.has(key)) fail("Import batch contains duplicate source row keys.");
+    seen.add(key);
+    if (!Object.prototype.hasOwnProperty.call(row, "sourcePayload")) fail("Each import row needs an immutable sourcePayload.");
+    validation.plainObject(row.values, "Import values");
+    var fingerprint = crypto.createHash("sha256").update(JSON.stringify(canonical({ sourcePayload: row.sourcePayload, values: row.values }))).digest("hex");
+    return { sourceKey: key, fingerprint: fingerprint, values: row.values };
+  });
+  return { sourceNamespace: namespace, rows: rows };
+}
+
+module.exports = { batch: batch, identity: identity, sourceText: sourceText, MAX_BATCH_ROWS: MAX_BATCH_ROWS, MAX_BATCH_BYTES: MAX_BATCH_BYTES };

package/lib/knowledge-db-import.js

@@ -0,0 +1,98 @@
+var validation = require("./knowledge-db-validation");
+var input = require("./knowledge-db-import-validation");
+
+// importRecords({ databaseId, expectedDatabaseRevision, operationId, sourceNamespace,
+// rows: [{ sourceKey, sourcePayload, values }], actor }) creates only unseen source rows.
+// Both sourcePayload and supplied values are immutable import identity; corrections use CRUD.
+// Record references can use sourceNamespace/sourceKey instead of id, including forward
+// references within this batch. Cross-database sources must already be imported locally.
+// readImportProgress({ databaseId, sourceNamespace, offset?, limit? }) returns native ids.
+// Import mappings survive receipt expiry; replay with a fresh operation id and current revision.
+// Limits: 100 rows / 256 KiB of row JSON per batch; shared 20000-record / 16 MiB
+// aggregate bounds include archives, history, mappings and receipts (500 live / 24 hours).
+// Each fresh operation retains one receipt and uses one canonical replacement; source
+// replay preserves the revision. New rows also use the existing optional summary
+// projection write. No filesystem input is accepted.
+function attachKnowledgeDbImport(ctx) {
+  function mappings(database) { return database.importMappings || []; }
+  function index(database) {
+    var out = new Map(); mappings(database).forEach(function (mapping) { out.set(input.identity(mapping.sourceNamespace, mapping.sourceKey), mapping); }); return out;
+  }
+  function convert(values, schema, databases, next, planned) {
+    var fields = {}; schema.forEach(function (field) { fields[field.id] = field; });
+    var out = Object.assign({}, values);
+    Object.keys(out).forEach(function (fieldId) {
+      var field = fields[fieldId]; var value = out[fieldId];
+      if (!field || field.type !== "reference" || !value || !Object.prototype.hasOwnProperty.call(value, "sourceKey")) return;
+      validation.plainObject(value, "Import reference");
+      if (value.type !== "record" || value.id !== undefined) throw ctx.error("Source references require type record and no native id.", "INVALID");
+      var databaseId = validation.id(value.databaseId || field.targetDatabaseId || next._id, "reference database id");
+      var namespace = input.sourceText(value.sourceNamespace, "Reference source namespace", 120);
+      var key = input.sourceText(value.sourceKey, "Reference source key", 200);
+      var target = databases.find(function (database) { return database._id === databaseId; });
+      var mapping = databaseId === next._id ? planned.get(input.identity(namespace, key)) : target && index(target).get(input.identity(namespace, key));
+      if (!mapping) throw ctx.error("Reference source row is outside this Mate or has not been imported.", "FORBIDDEN_REFERENCE");
+      out[fieldId] = { type: "record", databaseId: databaseId, id: mapping.recordId };
+    });
+    return out;
+  }
+  function importRecords(args) {
+    // Snapshot caller-owned JSON before any asynchronous operation.
+    var batch = ctx.clone(input.batch(args)); args = ctx.clone(args);
+    var operationId = ctx.mutationId(args.operationId); var fingerprint = ctx.fingerprint("import", args);
+    return ctx.serialize(args.databaseId, function () { return ctx.load(args.databaseId).then(async function (current) {
+      var duplicate = ctx.receipt(current, operationId, fingerprint);
+      if (duplicate) return Object.assign({}, duplicate, { projectionPending: current.projectionPending === true, projectionError: current.projectionError || null });
+      if (current.archivedAt) throw ctx.error("Knowledge database is archived.", "ARCHIVED", current.revision);
+      ctx.ensureRevision(args.expectedDatabaseRevision, current.revision, "database");
+      var next = ctx.clone(current); var planned = index(current); var added = [];
+      var resultMappings = batch.rows.map(function (row) {
+        var identity = input.identity(batch.sourceNamespace, row.sourceKey); var existing = planned.get(identity);
+        if (existing && existing.fingerprint !== row.fingerprint) throw ctx.error("Source row was already imported with a different payload: " + row.sourceKey, "IMPORT_SOURCE_CONFLICT");
+        if (existing) return Object.assign({}, existing, { created: false });
+        var mapping = { sourceNamespace: batch.sourceNamespace, sourceKey: row.sourceKey, fingerprint: row.fingerprint, recordId: ctx.newId("rec") };
+        planned.set(identity, mapping); added.push({ row: row, mapping: mapping });
+        return Object.assign({}, mapping, { created: true });
+      });
+      if (next.records.length + added.length > validation.MAX_RECORDS) throw ctx.error("Database exceeds the " + validation.MAX_RECORDS + " record limit (including archived records).", "LIMIT");
+      var result = { databaseId: current._id, databaseRevision: current.revision, sourceNamespace: batch.sourceNamespace, createdCount: added.length, replayedCount: batch.rows.length - added.length, mappings: resultMappings };
+      // Remember fresh replay operations too, so reusing an operation id with another
+      // payload still conflicts. Preserve record values, archives and database revision.
+      if (!added.length) {
+        ctx.remember(next, operationId, fingerprint, result);
+        await ctx.replace(current, next);
+        return Object.assign(result, { projectionPending: current.projectionPending === true, projectionError: current.projectionError || null });
+      }
+      var databases = await ctx.all(); var at = Date.now(); var who = ctx.actor(args.actor);
+      added.forEach(function (item) {
+        next.records.push({ id: item.mapping.recordId, revision: 1, values: {}, archivedAt: null, createdAt: at, updatedAt: at, createdBy: who, updatedBy: who });
+        next.recordHistory[item.mapping.recordId] = [];
+      });
+      var localDatabases = databases.filter(function (database) { return database._id !== next._id; }).concat([next]);
+      var context = ctx.referenceContext(localDatabases, next);
+      added.forEach(function (item, position) {
+        var values = convert(item.row.values, current.schema, databases, next, planned);
+        next.records[current.records.length + position].values = validation.normalizeValues(values, current.schema, context, null);
+      });
+      next.importMappings = mappings(next).concat(added.map(function (item) { return item.mapping; }));
+      next.revision++; next.updatedAt = at; next.updatedBy = who; next.projectionPending = true; next.projectionError = null;
+      result.databaseRevision = next.revision;
+      ctx.remember(next, operationId, fingerprint, result);
+      await ctx.replace(current, next);
+      return ctx.finish(next, result, who);
+    }); });
+  }
+  function readImportProgress(args) {
+    var namespace = input.sourceText(args.sourceNamespace, "Source namespace", 120);
+    var offset = args.offset === undefined ? 0 : args.offset; var limit = args.limit === undefined ? 100 : args.limit;
+    if (!Number.isInteger(offset) || offset < 0 || !Number.isInteger(limit) || limit < 1 || limit > 100) throw ctx.error("Import progress requires a nonnegative integer offset and limit between 1 and 100.", "INVALID");
+    return ctx.load(args.databaseId).then(function (database) {
+      var rows = mappings(database).filter(function (mapping) { return mapping.sourceNamespace === namespace; });
+      rows.sort(function (a, b) { return a.sourceKey === b.sourceKey ? 0 : a.sourceKey < b.sourceKey ? -1 : 1; });
+      return { databaseId: database._id, databaseRevision: database.revision, sourceNamespace: namespace, total: rows.length, mappings: ctx.clone(rows.slice(offset, offset + limit)), nextOffset: offset + limit < rows.length ? offset + limit : null };
+    });
+  }
+  return { importRecords: importRecords, readImportProgress: readImportProgress };
+}
+
+module.exports = { attachKnowledgeDbImport: attachKnowledgeDbImport, MAX_BATCH_ROWS: input.MAX_BATCH_ROWS, MAX_BATCH_BYTES: input.MAX_BATCH_BYTES };

package/lib/knowledge-db-service.js

@@ -1,17 +1,17 @@
 var crypto = require("crypto");
-var fs = require("fs");
-var path = require("path");
-var Datastore = require("@seald-io/nedb");
 var validation = require("./knowledge-db-validation");
+var imports = require("./knowledge-db-import");
+var storageModule = require("./knowledge-db-storage");
 
 var MAX_HISTORY = 50;
 var MAX_RECEIPTS = 500;
 var MAX_PAGE = 100;
 var MAX_FILTERS = 20;
 var MAX_SORTS = 10;
-var MAX_AGGREGATE_BYTES = 8 * 1024 * 1024;
+// Synthetic imports use 8.36 MB for 8563 narrative rows and exceed 8 MiB at
+// 11400 short rows. Keep finite headroom for the 20000-row envelope and history.
+var MAX_AGGREGATE_BYTES = 16 * 1024 * 1024;
 var IDEMPOTENCY_WINDOW_MS = 24 * 60 * 60 * 1000;
-var STORE_REGISTRY = {};
 
 function clone(value) { return JSON.parse(JSON.stringify(value)); }
 function error(message, code, currentRevision) { return Object.assign(new Error(message), { code: code || "FAILED", currentRevision: currentRevision }); }
@@ -107,33 +107,20 @@ function queryRecords(database, args, databases, settings) {
 }
 
 function createKnowledgeDbService(options) {
-  var settings = options || {}; var projectDir = path.resolve(settings.projectDir); var projectReal = null; var knowledgeDir = path.join(projectDir, "knowledge"); var dataDir = path.join(knowledgeDir, ".clay-db"); var filename = path.join(dataDir, "databases.db");
+  var settings = options || {}; var storage = storageModule.createKnowledgeDbStorage(settings);
   var aggregateLimit = Number.isInteger(settings.maxAggregateBytes) && settings.maxAggregateBytes > 1024 ? Math.min(MAX_AGGREGATE_BYTES, settings.maxAggregateBytes) : MAX_AGGREGATE_BYTES;
-  var entry = STORE_REGISTRY[filename];
-  if (!entry) { entry = { queues: {}, database: null, loading: null, ready: false }; STORE_REGISTRY[filename] = entry; }
-  function safeDirectory(target, create) {
-    var stat; try { stat = fs.lstatSync(target); } catch (cause) { if (!create || cause.code !== "ENOENT") throw cause; fs.mkdirSync(target, { recursive: true, mode: 0o700 }); stat = fs.lstatSync(target); }
-    if (!stat.isDirectory() || stat.isSymbolicLink()) throw error("Knowledge database path is unsafe.", "UNSAFE_PATH");
-    var real = fs.realpathSync(target); if (real !== projectReal && !real.startsWith(projectReal + path.sep)) throw error("Knowledge database path escapes the Mate project.", "UNSAFE_PATH");
-  }
-  function open() {
-    if (entry.loading) return entry.loading;
-    entry.loading = Promise.resolve().then(function () { projectReal = fs.realpathSync(projectDir); safeDirectory(knowledgeDir, true); safeDirectory(dataDir, true); var stat; try { stat = fs.lstatSync(filename); } catch (cause) { if (cause.code !== "ENOENT") throw cause; }
-      if (stat && (!stat.isFile() || stat.isSymbolicLink())) throw error("Knowledge database file is unsafe.", "UNSAFE_PATH"); entry.database = new Datastore({ filename: filename }); return entry.database.loadDatabaseAsync(); }).then(function () { return entry.database.ensureIndexAsync({ fieldName: "_id", unique: true }); }).then(function () { entry.ready = true; return entry.database; }).catch(function (cause) { entry.database = null; entry.loading = null; entry.ready = false; throw cause; });
-    return entry.loading;
-  }
-  function serialize(key, work) { var previous = entry.queues[key] || Promise.resolve(); var next = previous.catch(function () {}).then(work); entry.queues[key] = next.catch(function () {}); return next; }
-  function load(id) { return open().then(function (store) { return store.findOneAsync({ _id: validation.id(id, "database id") }); }).then(function (found) { if (!found) throw error("Knowledge database not found.", "NOT_FOUND"); return found; }); }
-  function all() { return open().then(function (store) { return store.findAsync({}); }); }
-  function listCached(args) { var items = entry.database ? entry.database.getAllData() : []; return { databases: items.filter(function (item) { return args && args.archived === true ? !!item.archivedAt : !item.archivedAt; }).sort(function (a, b) { return b.updatedAt - a.updatedAt || (a._id === b._id ? 0 : a._id < b._id ? -1 : 1); }).map(function (item) { return publicDatabase(item, false); }) }; }
-  function assertAggregate(next) { if (Buffer.byteLength(JSON.stringify(next), "utf8") > aggregateLimit) throw error("Knowledge database reached its storage limit.", "LIMIT"); }
-  function replace(current, next) { assertAggregate(next); return entry.database.updateAsync({ _id: current._id, revision: current.revision }, next, {}).then(function (result) { if (result.numAffected !== 1) throw error("This database changed during the operation.", "STALE", current.revision); return next; }); }
+  function serialize(key, work) { return storage.serialize(key, work); }
+  function load(id) { return storage.get(validation.id(id, "database id")).then(function (found) { if (!found) throw error("Knowledge database not found.", "NOT_FOUND"); return found; }); }
+  function all() { return storage.list(); }
+  function listCached(args) { var items = storage.getCached(); return { databases: items.filter(function (item) { return args && args.archived === true ? !!item.archivedAt : !item.archivedAt; }).sort(function (a, b) { return b.updatedAt - a.updatedAt || (a._id === b._id ? 0 : a._id < b._id ? -1 : 1); }).map(function (item) { return publicDatabase(item, false); }) }; }
+  function assertAggregate(next) { if (Buffer.byteLength(JSON.stringify(next), "utf8") > aggregateLimit) throw error("Knowledge database exceeds the " + aggregateLimit + " byte aggregate storage limit (records, history, mappings and live receipts).", "LIMIT"); }
+  function replace(current, next) { assertAggregate(next); return storage.replaceIfRevision(current._id, current.revision, next).then(function (replaced) { if (typeof replaced !== "boolean") throw storageModule.invalid("replaceIfRevision must resolve to a boolean."); if (!replaced) throw error("This database changed during the operation.", "STALE", current.revision); return next; }); }
   function projectionContent(value) { return JSON.stringify({ schemaVersion: 1, namespace: "knowledge-database-summary", databaseId: value._id, name: value.name, archived: !!value.archivedAt, revision: value.revision, fields: value.schema.filter(function (field) { return !field.archived; }).map(function (field) { return { id: field.id, name: field.name, type: field.type }; }), views: (value.views || []).filter(function (view) { return !view.archived; }).map(function (view) { return { id: view.id, name: view.name, type: "table" }; }), recordCount: value.records.filter(function (record) { return !record.archivedAt; }).length }); }
   function project(value, actorValue) {
     if (typeof settings.projectSummary !== "function") return Promise.resolve({ pending: false });
     return Promise.resolve().then(function () { return settings.projectSummary({ databaseId: value._id, name: value.name, revision: value.revision, archived: !!value.archivedAt, content: projectionContent(value), actor: actorValue }); }).then(function () {
-      var clean = clone(value); clean.projectionPending = false; clean.projectionError = null; return entry.database.updateAsync({ _id: value._id, revision: value.revision }, clean, {}).then(function (result) { if (result.numAffected !== 1) return { pending: true, error: "A newer database revision still needs summary projection." }; return { pending: false }; });
-    }).catch(function (cause) { var message = cause.message || "Knowledge database summary projection failed."; var failed = clone(value); failed.projectionPending = true; failed.projectionError = message; return entry.database.updateAsync({ _id: value._id, revision: value.revision }, failed, {}).then(function () { return { pending: true, error: message }; }, function () { return { pending: true, error: message }; }); });
+      var clean = clone(value); clean.projectionPending = false; clean.projectionError = null; return replace(value, clean).then(function () { return { pending: false }; });
+    }).catch(function (cause) { var message = cause.message || "Knowledge database summary projection failed."; var failed = clone(value); failed.projectionPending = true; failed.projectionError = message; return replace(value, failed).then(function () { return { pending: true, error: message }; }, function () { return { pending: true, error: message }; }); });
   }
   function finish(value, result, actorValue) { return project(value, actorValue).then(function (projection) { return Object.assign({}, result, { projectionPending: projection.pending, projectionError: projection.error || null }); }); }
   function mutate(databaseId, args, label, fn) {
@@ -149,7 +136,7 @@ function createKnowledgeDbService(options) {
       var at = now(); var who = actor(input.actor); var titleId = newId("fld"); var schema = input.schema ? validation.normalizeSchema(input.schema) : validation.normalizeSchema([{ id: titleId, name: "Name", type: "title", required: true }]);
       var viewId = newId("view"); var item = { _id: newId("db"), name: validation.text(input.name, "Database name", 120, false), revision: 1, archivedAt: null, schema: schema,
         views: [{ id: viewId, name: "Table", type: "table", columns: schema.filter(function (field) { return !field.archived; }).map(function (field) { return field.id; }), filters: [], sorts: [], archived: false }], records: [], recordHistory: {}, databaseHistory: [], receipts: [], createdAt: at, updatedAt: at, createdBy: who, updatedBy: who, projectionPending: true, projectionError: null };
-      var result = { database: publicDatabase(item, false) }; remember(item, operationId, operationFingerprint, result); assertAggregate(item); return open().then(function (store) { return store.insertAsync(item); }).then(function () { return finish(item, result, who); });
+      var result = { database: publicDatabase(item, false) }; remember(item, operationId, operationFingerprint, result); assertAggregate(item); return storage.insert(item).then(function () { return finish(item, result, who); });
     }); });
   }
   function list(args) { return all().then(function (items) { return { databases: items.filter(function (item) { return args && args.archived === true ? !!item.archivedAt : !item.archivedAt; }).sort(function (a, b) { return b.updatedAt - a.updatedAt || (a._id === b._id ? 0 : a._id < b._id ? -1 : 1); }).map(function (item) { return publicDatabase(item, false); }) }; }); }
@@ -163,13 +150,14 @@ function createKnowledgeDbService(options) {
   function updateSchema(args) { return mutate(args.databaseId, args, "schema", function (next, current) { ensureRevision(args.expectedRevision, current.revision, "database"); var schema = validation.normalizeSchema(args.schema); validation.assertCompatibleSchema(current.schema, schema, current.records); next.databaseHistory = next.databaseHistory.concat([{ revision: current.revision, at: now(), actor: actor(args.actor), action: "schema", schema: current.schema }]).slice(-MAX_HISTORY); next.schema = schema; next.views = reconcileViews(next.views, schema); next.revision++; next.updatedAt = now(); next.updatedBy = actor(args.actor); return { database: publicDatabase(next, false) }; }); }
   function saveView(args) { return mutate(args.databaseId, args, "view", function (next, current) { ensureRevision(args.expectedRevision, current.revision, "database"); var view = validation.normalizeView(args.view, current.schema); var index = next.views.findIndex(function (entry) { return entry.id === view.id; }); if (index === -1 && next.views.length >= validation.MAX_VIEWS) throw error("Database has too many views.", "LIMIT"); if (index === -1) next.views.push(view); else next.views[index] = view; next.revision++; next.updatedAt = now(); next.updatedBy = actor(args.actor); return { database: publicDatabase(next, false), view: clone(view) }; }); }
   function referenceContext(databases, current) { return { authorizeReference: function (reference) { if (reference.type === "text") return typeof settings.authorizeTextReference === "function" && settings.authorizeTextReference(reference.id) === true; var target = databases.filter(function (item) { return item._id === reference.databaseId; })[0]; return !!(target && target.records.some(function (record) { return record.id === reference.id; })); } }; }
-  function createRecord(args) { return all().then(function (databases) { return mutate(args.databaseId, args, "record", function (next, current) { ensureRevision(args.expectedDatabaseRevision, current.revision, "database"); if (next.records.length >= validation.MAX_RECORDS) throw error("Database reached the record limit.", "LIMIT"); var at = now(); var who = actor(args.actor); var record = { id: newId("rec"), revision: 1, values: validation.normalizeValues(args.values || {}, current.schema, referenceContext(databases, current), null), archivedAt: null, createdAt: at, updatedAt: at, createdBy: who, updatedBy: who }; next.records.push(record); next.recordHistory[record.id] = []; next.revision++; next.updatedAt = at; next.updatedBy = who; return { databaseRevision: next.revision, record: publicRecord(record) }; }); }); }
+  function createRecord(args) { return all().then(function (databases) { return mutate(args.databaseId, args, "record", function (next, current) { ensureRevision(args.expectedDatabaseRevision, current.revision, "database"); if (next.records.length >= validation.MAX_RECORDS) throw error("Database exceeds the " + validation.MAX_RECORDS + " record limit (including archived records).", "LIMIT"); var at = now(); var who = actor(args.actor); var record = { id: newId("rec"), revision: 1, values: validation.normalizeValues(args.values || {}, current.schema, referenceContext(databases, current), null), archivedAt: null, createdAt: at, updatedAt: at, createdBy: who, updatedBy: who }; next.records.push(record); next.recordHistory[record.id] = []; next.revision++; next.updatedAt = at; next.updatedBy = who; return { databaseRevision: next.revision, record: publicRecord(record) }; }); }); }
   function updateRecord(args) { return all().then(function (databases) { return mutate(args.databaseId, args, "record", function (next, current) { var index = next.records.findIndex(function (record) { return record.id === validation.id(args.recordId, "record id"); }); if (index === -1) throw error("Knowledge database record not found.", "NOT_FOUND"); var before = next.records[index]; ensureRevision(args.expectedRevision, before.revision, "record"); if (before.archivedAt) throw error("Knowledge database record is archived.", "ARCHIVED", before.revision); next.recordHistory[before.id] = (next.recordHistory[before.id] || []).concat([{ revision: before.revision, at: now(), actor: actor(args.actor), action: "update", values: before.values }]).slice(-MAX_HISTORY); var record = clone(before); record.values = validation.normalizeValues(args.values || {}, current.schema, referenceContext(databases, current), record.values); record.revision++; record.updatedAt = now(); record.updatedBy = actor(args.actor); next.records[index] = record; next.revision++; next.updatedAt = record.updatedAt; next.updatedBy = record.updatedBy; return { databaseRevision: next.revision, record: publicRecord(record) }; }); }); }
   function archiveRecord(args) { return mutate(args.databaseId, args, "record", function (next) { var index = next.records.findIndex(function (record) { return record.id === validation.id(args.recordId, "record id"); }); if (index === -1) throw error("Knowledge database record not found.", "NOT_FOUND"); var before = next.records[index]; ensureRevision(args.expectedRevision, before.revision, "record"); next.recordHistory[before.id] = (next.recordHistory[before.id] || []).concat([{ revision: before.revision, at: now(), actor: actor(args.actor), action: args.recover === true ? "recover" : "archive", values: before.values, archivedAt: before.archivedAt }]).slice(-MAX_HISTORY); var record = clone(before); record.archivedAt = args.recover === true ? null : now(); record.revision++; record.updatedAt = now(); record.updatedBy = actor(args.actor); next.records[index] = record; next.revision++; next.updatedAt = record.updatedAt; next.updatedBy = record.updatedBy; return { databaseRevision: next.revision, record: publicRecord(record) }; }); }
   function readRecord(args) { return all().then(function (databases) { var item = databases.filter(function (entry) { return entry._id === validation.id(args.databaseId, "database id"); })[0]; if (!item) throw error("Knowledge database not found.", "NOT_FOUND"); var record = item.records.filter(function (entry) { return entry.id === validation.id(args.recordId, "record id"); })[0]; if (!record) throw error("Knowledge database record not found.", "NOT_FOUND"); return { database: publicDatabase(item, false), record: publicRecord(record, referenceStates(item, record, databases, settings)), title: titleValue(item, record), history: clone(item.recordHistory[record.id] || []) }; }); }
   function query(args) { return all().then(function (databases) { var item = databases.filter(function (entry) { return entry._id === validation.id(args.databaseId, "database id"); })[0]; if (!item) throw error("Knowledge database not found.", "NOT_FOUND"); if (item.archivedAt) throw error("Knowledge database is archived.", "ARCHIVED", item.revision); return queryRecords(item, args, databases, settings); }); }
   function retryProjection(args) { return serialize(args.databaseId, function () { return load(args.databaseId).then(function (item) { return finish(item, { database: publicDatabase(item, false) }, actor(args.actor)); }); }); }
-  return { filename: filename, hasStore: function () { try { return fs.lstatSync(filename).isFile(); } catch (cause) { return false; } }, isReady: function () { return entry.ready === true; }, initialize: open, listCached: listCached, create: create, list: list, read: read, rename: rename, archive: archive, recover: function (args) { return archive(Object.assign({}, args, { recover: true })); }, updateSchema: updateSchema, saveView: saveView, createRecord: createRecord, updateRecord: updateRecord, archiveRecord: archiveRecord, recoverRecord: function (args) { return archiveRecord(Object.assign({}, args, { recover: true })); }, readRecord: readRecord, query: query, retryProjection: retryProjection };
+  var importer = imports.attachKnowledgeDbImport({ clone: clone, error: error, actor: actor, newId: newId, mutationId: mutationId, fingerprint: fingerprint, ensureRevision: ensureRevision, receipt: receipt, remember: remember, serialize: serialize, load: load, all: all, referenceContext: referenceContext, replace: replace, finish: finish });
+  return { readStorageStatus: function () { return typeof storage.diagnostics === "function" ? storage.diagnostics() : Promise.resolve(null); }, importRecords: importer.importRecords, readImportProgress: importer.readImportProgress, hasStore: function () { return storage.hasStore(); }, isReady: function () { return storage.isReady(); }, initialize: function () { return Promise.resolve(storage.initialize()).then(function () {}); }, listCached: listCached, create: create, list: list, read: read, rename: rename, archive: archive, recover: function (args) { return archive(Object.assign({}, args, { recover: true })); }, updateSchema: updateSchema, saveView: saveView, createRecord: createRecord, updateRecord: updateRecord, archiveRecord: archiveRecord, recoverRecord: function (args) { return archiveRecord(Object.assign({}, args, { recover: true })); }, readRecord: readRecord, query: query, retryProjection: retryProjection };
 }
 
 module.exports = { createKnowledgeDbService: createKnowledgeDbService, MAX_HISTORY: MAX_HISTORY, MAX_PAGE: MAX_PAGE, MAX_AGGREGATE_BYTES: MAX_AGGREGATE_BYTES, IDEMPOTENCY_WINDOW_MS: IDEMPOTENCY_WINDOW_MS };

package/lib/knowledge-db-storage-nedb-maintenance.js

@@ -0,0 +1,99 @@
+var fs = require("fs");
+
+var MIN_COMPACTION_BYTES = 1024 * 1024;
+var MAX_WRITE_AMPLIFICATION = 3;
+var MAX_WRITES_WITHOUT_COMPACTION = 64;
+var FAILURE_HEADROOM_BYTES = 4 * 1024 * 1024;
+var FAILURE_AMPLIFICATION = 6;
+
+// Shared by every service instance for this filename. No interval or background work:
+// maintenance runs inside the same I/O gate as writes, reads and failed-write reloads.
+function attachKnowledgeDbNedbMaintenance(ctx) {
+  var entry = ctx.entry;
+  if (!entry.storage) entry.storage = {
+    queue: Promise.resolve(), store: null, cached: [], sizes: new Map(), liveBytes: 0,
+    journalBytes: 0, peakJournalBytes: 0, journalUnknown: false, successfulWrites: 0,
+    writesSinceCompaction: 0, compactions: 0, failedCompactions: 0, recoveries: 0,
+    maintenancePending: false, maintenanceError: null
+  };
+  var state = entry.storage;
+  function enqueue(work) {
+    var next = state.queue.catch(function () {}).then(work);
+    state.queue = next.catch(function () {}); return next;
+  }
+  function size(value) { return Buffer.byteLength(JSON.stringify(value), "utf8") + 1; }
+  function refresh(store) {
+    state.store = store; state.cached = store.getAllData(); state.sizes = new Map(); state.liveBytes = 0;
+    state.cached.forEach(function (value) { var bytes = size(value); state.sizes.set(value._id, bytes); state.liveBytes += bytes; });
+  }
+  function issue(cause) { state.maintenancePending = true; state.maintenanceError = cause.message || String(cause); }
+  async function measure() {
+    try {
+      state.journalBytes = (await fs.promises.stat(ctx.filename)).size; state.journalUnknown = false;
+      state.peakJournalBytes = Math.max(state.peakJournalBytes, state.journalBytes);
+      return true;
+    } catch (cause) { state.journalUnknown = true; issue(cause); return false; }
+  }
+  async function ensure() {
+    var store = await ctx.open();
+    if (state.store !== store) { refresh(store); await measure(); }
+    return store;
+  }
+  async function compact(store) {
+    try {
+      // Supported datastore API also participates in NeDB's own executor.
+      await store.compactDatafileAsync();
+      state.compactions++; state.writesSinceCompaction = 0;
+      if (await measure()) { state.maintenancePending = false; state.maintenanceError = null; }
+    } catch (cause) { state.failedCompactions++; issue(cause); await measure(); }
+  }
+  function due() {
+    return state.journalBytes > Math.max(MIN_COMPACTION_BYTES, MAX_WRITE_AMPLIFICATION * state.liveBytes) ||
+      state.writesSinceCompaction >= MAX_WRITES_WITHOUT_COMPACTION && state.journalBytes - state.liveBytes >= MIN_COMPACTION_BYTES;
+  }
+  function blocked() {
+    return state.maintenancePending && (state.journalUnknown || state.journalBytes > Math.max(FAILURE_HEADROOM_BYTES, FAILURE_AMPLIFICATION * state.liveBytes));
+  }
+  function read(work) { return enqueue(async function () { return work(await ensure()); }); }
+  function write(work, value) {
+    return enqueue(async function () {
+      var store = await ensure(); var attempted = false;
+      if (state.maintenancePending) { attempted = true; await compact(store); }
+      // Repeated maintenance failure cannot allow unbounded journal growth. This
+      // rejection happens before the next canonical write, never after its success.
+      if (blocked()) throw Object.assign(new Error("Knowledge database maintenance must recover before more writes: " + state.maintenanceError), { code: "STORAGE_MAINTENANCE_BLOCKED" });
+      var result;
+      try { result = await work(store); }
+      catch (cause) {
+        // NeDB changes indexes before appending. Hold the shared gate through reload
+        // so another database cannot compact or persist an unsaved in-memory state.
+        try {
+          await store.loadDatabaseAsync(); refresh(store); state.recoveries++; state.writesSinceCompaction = 0;
+          if (await measure()) { state.maintenancePending = false; state.maintenanceError = null; }
+        } catch (recoveryCause) { state.store = null; state.cached = []; issue(recoveryCause); ctx.invalidate(); }
+        throw cause;
+      }
+      if (result && result.numAffected === 0) return result;
+      var bytes = size(value); state.liveBytes += bytes - (state.sizes.get(value._id) || 0); state.sizes.set(value._id, bytes);
+      state.cached = store.getAllData(); state.successfulWrites++; state.writesSinceCompaction++;
+      if (await measure() && !attempted && due()) await compact(store);
+      // A maintenance error never changes a successful canonical write into a failure.
+      return result;
+    });
+  }
+  function status() {
+    return read(async function () {
+      await measure();
+      return { maintenancePending: state.maintenancePending, maintenanceError: state.maintenanceError,
+        liveBytes: state.liveBytes, journalBytes: state.journalBytes, peakJournalBytes: state.peakJournalBytes,
+        successfulWrites: state.successfulWrites, writesSinceCompaction: state.writesSinceCompaction,
+        compactions: state.compactions, failedCompactions: state.failedCompactions, recoveries: state.recoveries };
+    });
+  }
+  return { read: read, write: write, cached: function () { return state.cached; }, readStorageStatus: status };
+}
+
+module.exports = { attachKnowledgeDbNedbMaintenance: attachKnowledgeDbNedbMaintenance,
+  MIN_COMPACTION_BYTES: MIN_COMPACTION_BYTES, MAX_WRITE_AMPLIFICATION: MAX_WRITE_AMPLIFICATION,
+  MAX_WRITES_WITHOUT_COMPACTION: MAX_WRITES_WITHOUT_COMPACTION,
+  FAILURE_HEADROOM_BYTES: FAILURE_HEADROOM_BYTES, FAILURE_AMPLIFICATION: FAILURE_AMPLIFICATION };

package/lib/knowledge-db-storage-nedb.js

@@ -0,0 +1,66 @@
+var fs = require("fs");
+var path = require("path");
+var Datastore = require("@seald-io/nedb");
+var maintenance = require("./knowledge-db-storage-nedb-maintenance");
+var STORE_REGISTRY = new Map();
+
+function clone(value) { return JSON.parse(JSON.stringify(value)); }
+function error(message, code) { return Object.assign(new Error(message), { code: code }); }
+function createKnowledgeDbNedbStorage(options) {
+  var projectDir = path.resolve(options.projectDir); var projectReal = null;
+  var knowledgeDir = path.join(projectDir, "knowledge"); var dataDir = path.join(knowledgeDir, ".clay-db");
+  var filename = path.join(dataDir, "databases.db"); var entry = STORE_REGISTRY.get(filename);
+  if (!entry) { entry = { queues: new Map(), database: null, loading: null, ready: false }; STORE_REGISTRY.set(filename, entry); }
+  function safeDirectory(target, create) {
+    var stat;
+    try { stat = fs.lstatSync(target); }
+    catch (cause) { if (!create || cause.code !== "ENOENT") throw cause; fs.mkdirSync(target, { recursive: true, mode: 0o700 }); stat = fs.lstatSync(target); }
+    if (!stat.isDirectory() || stat.isSymbolicLink()) throw error("Knowledge database path is unsafe.", "UNSAFE_PATH");
+    var real = fs.realpathSync(target);
+    if (real !== projectReal && !real.startsWith(projectReal + path.sep)) throw error("Knowledge database path escapes the Mate project.", "UNSAFE_PATH");
+  }
+  function open() {
+    if (entry.loading) return entry.loading;
+    entry.loading = Promise.resolve().then(function () {
+      projectReal = fs.realpathSync(projectDir); safeDirectory(knowledgeDir, true); safeDirectory(dataDir, true);
+      var stat; try { stat = fs.lstatSync(filename); } catch (cause) { if (cause.code !== "ENOENT") throw cause; }
+      if (stat && (!stat.isFile() || stat.isSymbolicLink())) throw error("Knowledge database file is unsafe.", "UNSAFE_PATH");
+      entry.database = new Datastore({ filename: filename }); return entry.database.loadDatabaseAsync();
+    }).then(function () { return entry.database.ensureIndexAsync({ fieldName: "_id", unique: true }); })
+      .then(function () { entry.ready = true; return entry.database; })
+      .catch(function (cause) { invalidate(); throw cause; });
+    return entry.loading;
+  }
+  function invalidate() { entry.database = null; entry.loading = null; entry.ready = false; }
+  var io = maintenance.attachKnowledgeDbNedbMaintenance({ entry: entry, filename: filename, open: open, invalidate: invalidate });
+  function serialize(scope, work) {
+    var previous = entry.queues.get(scope) || Promise.resolve(); var next = previous.catch(function () {}).then(function () { return work(); });
+    entry.queues.set(scope, next.catch(function () {})); return next;
+  }
+  function insert(snapshot) {
+    var value = clone(snapshot);
+    return io.write(function (store) { return store.insertAsync(value); }, value).then(function () {}, function (cause) {
+      if (cause.errorType === "uniqueViolated") throw error("Knowledge storage identity already exists.", "ALREADY_EXISTS");
+      throw cause;
+    });
+  }
+  function replaceIfRevision(id, revision, snapshot) {
+    var value = clone(snapshot);
+    if (value._id !== id) return Promise.reject(error("Knowledge storage replacement identity must match.", "INVALID_STORAGE_ID"));
+    return io.write(function (store) { return store.updateAsync({ _id: id, revision: revision }, value, {}); }, value)
+      .then(function (result) { return result.numAffected === 1; });
+  }
+  return {
+    contractVersion: 1,
+    initialize: function () { return io.read(function () {}); },
+    hasStore: function () { try { return fs.lstatSync(filename).isFile(); } catch (cause) { return false; } },
+    isReady: function () { return entry.ready === true; },
+    get: function (id) { return io.read(function (store) { return store.findOneAsync({ _id: id }); }); },
+    list: function () { return io.read(function (store) { return store.findAsync({}); }); },
+    getCached: function () { return clone(io.cached()); },
+    insert: insert, replaceIfRevision: replaceIfRevision, serialize: serialize,
+    diagnostics: function () { return io.readStorageStatus().then(function (status) { return Object.assign({ engine: "nedb", automaticCompaction: false }, status); }); }
+  };
+}
+
+module.exports = { createKnowledgeDbNedbStorage: createKnowledgeDbNedbStorage };

package/lib/knowledge-db-storage.js

@@ -0,0 +1,72 @@
+// Version 1 persists opaque JSON aggregate snapshots with a stable _id and revision.
+// _id is the existing Knowledge identity, not a query expression. Engines may map it
+// to their own primary key; aggregate format/limits remain application responsibilities.
+// Each adapter is bound to one server-authorized Mate-project store; get/list cannot
+// broaden that scope. Facades for the same store must share locks and committed cache.
+//
+// initialize() -> Promise<void>; hasStore()/isReady() -> boolean.
+// get(id) -> Promise<snapshot|null>; list() -> Promise<snapshot[]>;
+// getCached() -> snapshot[] (committed, detached, no initialization side effect).
+// insert(snapshot) -> Promise<void> (reject ALREADY_EXISTS; never overwrite).
+// replaceIfRevision(id, revision, snapshot) -> Promise<boolean>: atomic compare and
+// full replacement, false on missing/mismatch without writing. Same-revision writes
+// are allowed for receipts/projection status. Identity must remain unchanged.
+// serialize(scope, work) -> Promise<result>: same-scope callbacks exclude each other
+// across all facades of one physical store. Callback receives no engine objects and
+// may await adapter operations; this is not a multi-aggregate transaction or I/O lock.
+// Engines separately serialize reads/writes/maintenance/recovery as needed. A failed
+// write may have committed durably; restore readable committed state before releasing
+// the I/O gate, or invalidate it. Never turn committed success into maintenance failure.
+// Domain operation receipts resolve ambiguous retries; the adapter does not own them.
+// diagnostics() -> Promise<JSON> is optional, engine-specific and never a native handle.
+
+var METHODS = ["initialize", "hasStore", "isReady", "get", "list", "getCached", "insert", "replaceIfRevision", "serialize"];
+function invalid(message) { return Object.assign(new Error("Invalid Knowledge storage adapter: " + message), { code: "INVALID_STORAGE_ADAPTER" }); }
+function validateAdapter(adapter) {
+  if (!adapter || typeof adapter !== "object" || adapter.contractVersion !== 1) throw invalid("contractVersion must be 1.");
+  METHODS.forEach(function (name) { if (typeof adapter[name] !== "function") throw invalid(name + " must be a function."); });
+  if (adapter.diagnostics !== undefined && typeof adapter.diagnostics !== "function") throw invalid("diagnostics must be a function when supplied.");
+  return adapter;
+}
+function bindAdapter(adapter) {
+  function snapshot(value) {
+    if (!value || Object.getPrototypeOf(value) !== Object.prototype || typeof value._id !== "string" || !Number.isInteger(value.revision)) throw invalid("snapshot must be a plain aggregate with _id and integer revision.");
+    return value;
+  }
+  function snapshots(values) { if (!Array.isArray(values)) throw invalid("list/getCached must return an array."); values.forEach(snapshot); return values; }
+  function call(name, args, check) {
+    var result;
+    try { result = adapter[name].apply(adapter, args); } catch (cause) { return Promise.reject(cause); }
+    if (!result || typeof result.then !== "function") return Promise.reject(invalid(name + " must return a Promise."));
+    return result.then(check);
+  }
+  function flag(name) { var value = adapter[name](); if (typeof value !== "boolean") throw invalid(name + " must return a boolean."); return value; }
+  var bound = {
+    contractVersion: 1,
+    initialize: function () { return call("initialize", [], function () {}); },
+    hasStore: function () { return flag("hasStore"); }, isReady: function () { return flag("isReady"); },
+    get: function (id) { return call("get", [id], function (value) { if (value === null) return null; snapshot(value); if (value._id !== id) throw invalid("get returned a different identity."); return value; }); },
+    list: function () { return call("list", [], snapshots); }, getCached: function () { return snapshots(adapter.getCached()); },
+    insert: function (value) { return call("insert", [value], function (result) { if (result !== undefined) throw invalid("insert must resolve without a native result."); }); },
+    replaceIfRevision: function (id, revision, value) { return call("replaceIfRevision", [id, revision, value], function (result) { if (typeof result !== "boolean") throw invalid("replaceIfRevision must resolve to a boolean."); return result; }); },
+    serialize: function (scope, work) { return call("serialize", [scope, function () { return work(); }], function (result) { return result; }); }
+  };
+  if (typeof adapter.diagnostics === "function") bound.diagnostics = function () {
+    return call("diagnostics", [], function (result) { try { return JSON.parse(JSON.stringify(result)); } catch (cause) { throw invalid("diagnostics must resolve to JSON data."); } });
+  };
+  return bound;
+}
+function createDefaultKnowledgeDbStorage(options) {
+  // Lazy engine loading: injected adapters can run domain code without NeDB installed.
+  return require("./knowledge-db-storage-nedb").createKnowledgeDbNedbStorage(options);
+}
+function createKnowledgeDbStorage(options) {
+  var settings = options || {}; var adapter;
+  if (settings.storageAdapter !== undefined) adapter = settings.storageAdapter;
+  else if (settings.storageFactory !== undefined) {
+    if (typeof settings.storageFactory !== "function") throw invalid("storageFactory must be a synchronous function.");
+    adapter = settings.storageFactory({ projectDir: settings.projectDir });
+  } else adapter = module.exports.createDefaultKnowledgeDbStorage({ projectDir: settings.projectDir });
+  return bindAdapter(validateAdapter(adapter));
+}
+module.exports = { createKnowledgeDbStorage: createKnowledgeDbStorage, createDefaultKnowledgeDbStorage: createDefaultKnowledgeDbStorage, validateAdapter: validateAdapter, invalid: invalid };

package/lib/mate-knowledge-mate-bound.js

@@ -0,0 +1,53 @@
+// Ordinary Mate Knowledge bindings. The service supplies live validation and
+// the already-authorized scope; this module only exposes the Mate operations.
+
+function createMateBound(options) {
+  var settings = options || {};
+  var revalidate = settings.revalidate;
+  var initial = settings.initial;
+
+  function ownScope() {
+    return [revalidate().own];
+  }
+  function db(method, args) {
+    var current = revalidate();
+    if (!current.databaseService || typeof current.databaseService[method] !== "function") throw new Error("Knowledge databases are unavailable for this Mate.");
+    return current.databaseService[method](Object.assign({}, args || {}, { actor: current.actor }));
+  }
+  function text(method, args) {
+    var current = revalidate();
+    return settings.textTools[method](current.own, args || {}, current.actor);
+  }
+  return {
+    isClay: false,
+    mateId: initial.own.mateId,
+    scopeId: initial.own.scopeId,
+    listKnowledge: function (args) { return settings.listFor(ownScope(), args || {}, false); },
+    searchKnowledge: function (args) { return settings.searchFor(ownScope(), args || {}, false); },
+    readKnowledge: function (args) { return settings.readFor(ownScope(), args || {}, false); },
+    readKnowledgeDocument: function (args) { return text("read", args); },
+    createKnowledgeDocument: function (args) { return text("create", args); },
+    updateKnowledgeDocument: function (args) { return text("update", args); },
+    listKnowledgeDocumentHistory: function (args) { return text("history", args); },
+    restoreKnowledgeDocument: function (args) { return text("restore", args); },
+    listDatabases: function (args) { return db("list", args); },
+    createDatabase: function (args) { return db("create", args); },
+    readDatabase: function (args) { return db("read", args); },
+    renameDatabase: function (args) { return db("rename", args); },
+    archiveDatabase: function (args) { return db("archive", args); },
+    recoverDatabase: function (args) { return db("recover", args); },
+    updateDatabaseSchema: function (args) { return db("updateSchema", args); },
+    saveTableView: function (args) { return db("saveView", args); },
+    queryDatabase: function (args) { return db("query", args); },
+    createDatabaseRecord: function (args) { return db("createRecord", args); },
+    readDatabaseRecord: function (args) { return db("readRecord", args); },
+    updateDatabaseRecord: function (args) { return db("updateRecord", args); },
+    archiveDatabaseRecord: function (args) { return db("archiveRecord", args); },
+    recoverDatabaseRecord: function (args) { return db("recoverRecord", args); },
+    retryDatabaseSync: function (args) { return db("retryProjection", args); },
+    importDatabaseRecords: function (args) { return db("importRecords", args); },
+    readDatabaseImportProgress: function (args) { return db("readImportProgress", args); },
+  };
+}
+
+module.exports = { createMateBound: createMateBound };

package/lib/mate-knowledge-mcp-server.js

@@ -9,11 +9,17 @@
 
 var buildShape = require("./session-spawn-mcp-server").buildShape;
 var service = require("./mate-knowledge-service");
+var zod;
+try { zod = require("zod"); } catch (error) { zod = null; }
 
 var MATE_CONTRACT =
   "Your Knowledge is the durable personal and expertise context you own: what you have learned, the material you keep, and your accumulated observations. " +
   "It is yours alone. There are no other Knowledge collections available to you here, and project Logs are a separate, unrelated surface. " +
+  "Use text documents for narrative or multiline Markdown content, and databases for structured records, relationships, filters, and numeric totals. " +
   "Search it when recalling something specific would genuinely change your answer, and read only the records you actually need. " +
+  "Database list and search results are summaries for database discovery, not database row content: discover a database, read its schema, then query_database with stable field ids, typed filters, explicit sorts, bounded pagination, and totals; read individual rows by stable record id. Text search and text-document reads retain their own content and revision semantics. " +
+  "For text edits, use the returned document id and revision, reread after a conflict, and retry with the current revision. " +
+  "Treat errors as failures. A successful revision/id with indexSyncPending or projectionPending means canonical content was saved but indexing is pending: report that exact state, do not claim it is indexed, and do not blindly retry a mutation that could create a duplicate. " +
   "Do not enumerate or restate your Knowledge to the user unasked, and never reproduce whole records as filler.";
 
 var CLAY_CONTRACT =
@@ -56,24 +62,35 @@ var OPERATION = { type: "string", description: "Stable retry id formatted label_
 var FIELD = { type: "object", description: "Field with stable id, name, type, required, and type-specific options or reference target." };
 var FILTER = { type: "object", description: "Validated filter: fieldId, op, and value. Supported ops are eq, neq, contains, gt, gte, lt, lte, empty, not-empty." };
 var SORT = { type: "object", description: "Validated sort with fieldId and asc or desc direction." };
+var FIELD_SCHEMA = zod ? zod.object({ id: zod.string(), name: zod.string(), type: zod.string(), required: zod.boolean().optional(), options: zod.any().optional(), target: zod.string().optional(), targetDatabaseId: zod.string().optional(), archived: zod.boolean().optional() }) : null;
+var FILTER_SCHEMA = zod ? zod.object({ fieldId: zod.string(), op: zod.string(), value: zod.any().optional() }) : null;
+var SORT_SCHEMA = zod ? zod.object({ fieldId: zod.string(), direction: zod.enum(["asc", "desc"]) }) : null;
+var VALUES_SCHEMA = zod ? zod.record(zod.string(), zod.any()) : null;
+var VIEW_SCHEMA = zod ? zod.object({ id: zod.string(), name: zod.string(), columns: zod.array(zod.string()), filters: zod.array(FILTER_SCHEMA || zod.any()).optional(), sorts: zod.array(SORT_SCHEMA || zod.any()).optional(), archived: zod.boolean().optional() }) : null;
+var IMPORT_ROW_SCHEMA = zod ? zod.object({ sourceKey: zod.string(), sourcePayload: zod.any(), values: zod.record(zod.string(), zod.any()) }) : null;
+var IMPORT_ROW = IMPORT_ROW_SCHEMA ? { schema: IMPORT_ROW_SCHEMA, description: "Source row with sourceKey, immutable JSON sourcePayload, and typed values keyed by stable schema field id. Record references use sourceNamespace/sourceKey, including forward references within this batch." } : { type: "object", description: "Source row with sourceKey, immutable JSON sourcePayload, and typed values keyed by stable schema field id. Record references use sourceNamespace/sourceKey, including forward references within this batch." };
+var DOCUMENT_ID = { type: "string", description: "Stable document id returned by a text-document tool." };
+var DOCUMENT_REVISION = { type: "string", description: "Revision returned by the most recent text-document read or mutation; required for updates and restore." };
 
 function databaseTools(bound) {
   return [
-    { name: "list_databases", description: "List canonical Knowledge databases, including archived databases when requested, so the Mate can discover and recover them without relying on summary projection.", inputSchema: buildShape({ archived: { type: "boolean" } }, []), handler: handler(bound, "listDatabases") },
-    { name: "create_database", description: "Create a first-class database in this Mate's Knowledge. Defaults to one required title field and a Table view.", inputSchema: buildShape({ name: { type: "string" }, schema: { type: "array", items: FIELD }, operationId: OPERATION }, ["name", "operationId"]), handler: handler(bound, "createDatabase") },
-    { name: "read_database", description: "Read one Knowledge database's current schema, views, revision, and record count.", inputSchema: buildShape({ databaseId: ID, includeArchived: { type: "boolean" } }, ["databaseId"]), handler: handler(bound, "readDatabase") },
+    { name: "list_databases", description: "Discover canonical Knowledge databases, including archived databases when requested. Results are database summaries only, not row content; read a chosen database before querying it.", inputSchema: buildShape({ archived: { type: "boolean" } }, []), handler: handler(bound, "listDatabases") },
+    { name: "create_database", description: "Create a first-class structured database in this Mate's Knowledge. Define typed fields for values, relationships, and numeric totals; use text documents for narrative or multiline material.", inputSchema: buildShape({ name: { type: "string" }, schema: FIELD_SCHEMA ? { schema: zod.array(FIELD_SCHEMA), description: "Typed field descriptors." } : { type: "array", items: FIELD }, operationId: OPERATION }, ["name", "operationId"]), handler: handler(bound, "createDatabase") },
+    { name: "read_database", description: "Read one discovered Knowledge database by stable database id to obtain its current schema, views, revision, and record count before querying rows.", inputSchema: buildShape({ databaseId: ID, includeArchived: { type: "boolean" } }, ["databaseId"]), handler: handler(bound, "readDatabase") },
     { name: "rename_database", description: "Rename a Knowledge database with optimistic revision checking.", inputSchema: buildShape({ databaseId: ID, name: { type: "string" }, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "name", "expectedRevision", "operationId"]), handler: handler(bound, "renameDatabase") },
     { name: "archive_database", description: "Recoverably archive a Knowledge database.", inputSchema: buildShape({ databaseId: ID, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "expectedRevision", "operationId"]), handler: handler(bound, "archiveDatabase") },
     { name: "recover_database", description: "Recover an archived Knowledge database.", inputSchema: buildShape({ databaseId: ID, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "expectedRevision", "operationId"]), handler: handler(bound, "recoverDatabase") },
-    { name: "update_database_schema", description: "Replace the schema using stable field ids. Exactly one active title field is required; populated fields cannot be removed or change type and should be archived instead.", inputSchema: buildShape({ databaseId: ID, schema: { type: "array", items: FIELD }, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "schema", "expectedRevision", "operationId"]), handler: handler(bound, "updateDatabaseSchema") },
-    { name: "save_table_view", description: "Create or revise a persisted Table view with selected columns, validated filters, and deterministic sorts.", inputSchema: buildShape({ databaseId: ID, view: { type: "object", description: "View with stable id, name, columns, filters, and sorts." }, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "view", "expectedRevision", "operationId"]), handler: handler(bound, "saveTableView") },
-    { name: "query_database", description: "Query a bounded deterministic page of records using a saved view or explicit validated filters and sorts. Returns count and numeric sums.", inputSchema: buildShape({ databaseId: ID, viewId: ID, filters: { type: "array", items: FILTER }, sorts: { type: "array", items: SORT }, cursor: { type: "string" }, limit: { type: "number" }, archived: { type: "boolean" } }, ["databaseId"]), handler: handler(bound, "queryDatabase") },
-    { name: "create_database_record", description: "Create a typed record. Values are keyed by stable field id; unknown fields and unsafe keys are rejected.", inputSchema: buildShape({ databaseId: ID, values: { type: "object" }, expectedDatabaseRevision: REVISION, operationId: OPERATION }, ["databaseId", "values", "expectedDatabaseRevision", "operationId"]), handler: handler(bound, "createDatabaseRecord") },
+    { name: "update_database_schema", description: "Replace the schema using stable field ids. Exactly one active title field is required; populated fields cannot be removed or change type and should be archived instead.", inputSchema: buildShape({ databaseId: ID, schema: FIELD_SCHEMA ? { schema: zod.array(FIELD_SCHEMA), description: "Complete typed field descriptor array." } : { type: "array", items: FIELD }, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "schema", "expectedRevision", "operationId"]), handler: handler(bound, "updateDatabaseSchema") },
+    { name: "save_table_view", description: "Create or revise a persisted Table view with selected columns, validated filters, and deterministic sorts.", inputSchema: buildShape({ databaseId: ID, view: VIEW_SCHEMA ? { schema: VIEW_SCHEMA, description: "View with stable id, name, columns, filters, and sorts." } : { type: "object", description: "View with stable id, name, columns, filters, and sorts." }, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "view", "expectedRevision", "operationId"]), handler: handler(bound, "saveTableView") },
+    { name: "query_database", description: "Query a bounded deterministic page using stable field ids, typed validated filters, explicit sorts, and a cursor. Returns only the requested rows plus count and numeric sums for the same filter scope; do not infer totals from one page.", inputSchema: buildShape({ databaseId: ID, viewId: ID, filters: FILTER_SCHEMA ? { schema: zod.array(FILTER_SCHEMA) } : { type: "array", items: FILTER }, sorts: SORT_SCHEMA ? { schema: zod.array(SORT_SCHEMA) } : { type: "array", items: SORT }, cursor: { type: "string" }, limit: { type: "number" }, archived: { type: "boolean" } }, ["databaseId"]), handler: handler(bound, "queryDatabase") },
+    { name: "create_database_record", description: "Create a typed record. Values are keyed by stable field id; unknown fields and unsafe keys are rejected.", inputSchema: buildShape({ databaseId: ID, values: VALUES_SCHEMA ? { schema: VALUES_SCHEMA } : { type: "object" }, expectedDatabaseRevision: REVISION, operationId: OPERATION }, ["databaseId", "values", "expectedDatabaseRevision", "operationId"]), handler: handler(bound, "createDatabaseRecord") },
     { name: "read_database_record", description: "Read one record with its values, revision, attribution, archive state, and bounded history.", inputSchema: buildShape({ databaseId: ID, recordId: ID }, ["databaseId", "recordId"]), handler: handler(bound, "readDatabaseRecord") },
-    { name: "update_database_record", description: "Update selected typed values using the record revision. Edits to other fields made since a prior save are not overwritten.", inputSchema: buildShape({ databaseId: ID, recordId: ID, values: { type: "object" }, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "recordId", "values", "expectedRevision", "operationId"]), handler: handler(bound, "updateDatabaseRecord") },
+    { name: "update_database_record", description: "Update selected typed values using the record revision. Edits to other fields made since a prior save are not overwritten.", inputSchema: buildShape({ databaseId: ID, recordId: ID, values: VALUES_SCHEMA ? { schema: VALUES_SCHEMA } : { type: "object" }, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "recordId", "values", "expectedRevision", "operationId"]), handler: handler(bound, "updateDatabaseRecord") },
     { name: "archive_database_record", description: "Recoverably archive one record using its current revision.", inputSchema: buildShape({ databaseId: ID, recordId: ID, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "recordId", "expectedRevision", "operationId"]), handler: handler(bound, "archiveDatabaseRecord") },
     { name: "recover_database_record", description: "Recover one archived record using its current revision.", inputSchema: buildShape({ databaseId: ID, recordId: ID, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "recordId", "expectedRevision", "operationId"]), handler: handler(bound, "recoverDatabaseRecord") },
     { name: "retry_database_sync", description: "Retry the shared append-only Knowledge summary projection after a canonical database mutation reported projectionPending.", inputSchema: buildShape({ databaseId: ID }, ["databaseId"]), handler: handler(bound, "retryDatabaseSync") },
+    { name: "import_database_records", description: "Import up to 100 source rows (256 KiB per batch) into a discovered database using immutable source identities and typed values. Use sourceNamespace/sourceKey references for same-batch or already-imported cross-database links; replay the exact operation and payload after uncertainty, and reread the current database revision before a stale retry.", inputSchema: buildShape({ databaseId: ID, expectedDatabaseRevision: REVISION, operationId: OPERATION, sourceNamespace: { type: "string", description: "Stable source namespace for this import mapping." }, rows: IMPORT_ROW_SCHEMA ? { schema: zod.array(IMPORT_ROW_SCHEMA), description: "At most 100 source rows and 256 KiB of serialized row JSON." } : { type: "array", description: "At most 100 source rows and 256 KiB of serialized row JSON.", items: IMPORT_ROW } }, ["databaseId", "expectedDatabaseRevision", "operationId", "sourceNamespace", "rows"]), handler: handler(bound, "importDatabaseRecords") },
+    { name: "read_database_import_progress", description: "Read bounded source-to-native record mappings by database id and source namespace. Use returned stable record ids for later corrections; this does not read arbitrary filesystem paths or caller-selected owners.", inputSchema: buildShape({ databaseId: ID, sourceNamespace: { type: "string" }, offset: { type: "number" }, limit: { type: "number", description: "Mapping page size from 1 to 100." } }, ["databaseId", "sourceNamespace"]), handler: handler(bound, "readDatabaseImportProgress") },
   ].map(function (tool) { tool.description = MATE_CONTRACT + " " + tool.description; return tool; });
 }
 
@@ -110,6 +127,36 @@ function mateTools(bound) {
       }, ["ref"]),
       handler: handler(bound, "readKnowledge"),
     },
+    {
+      name: "read_knowledge_document",
+      description: MATE_CONTRACT + " Read the complete current Markdown document by its relative path or stable document id, including its current revision.",
+      inputSchema: buildShape({ name: { type: "string", description: "Relative Markdown path under this Mate's Knowledge directory." }, documentId: DOCUMENT_ID }, []),
+      handler: handler(bound, "readKnowledgeDocument"),
+    },
+    {
+      name: "create_knowledge_document",
+      description: MATE_CONTRACT + " Create a new Markdown document at a safe relative path. Use this for narrative or multiline content; it is synchronized into Knowledge search after the canonical save.",
+      inputSchema: buildShape({ name: { type: "string", description: "Relative Markdown path under this Mate's Knowledge directory." }, content: { type: "string", description: "Complete Markdown document content." } }, ["name", "content"]),
+      handler: handler(bound, "createKnowledgeDocument"),
+    },
+    {
+      name: "update_knowledge_document",
+      description: MATE_CONTRACT + " Update an existing Markdown document by relative path or stable document id. The current revision is mandatory, and stale writes are rejected.",
+      inputSchema: buildShape({ name: { type: "string", description: "Relative Markdown path under this Mate's Knowledge directory." }, documentId: DOCUMENT_ID, content: { type: "string", description: "Complete replacement Markdown document content." }, expectedRevision: DOCUMENT_REVISION }, ["content", "expectedRevision"]),
+      handler: handler(bound, "updateKnowledgeDocument"),
+    },
+    {
+      name: "list_knowledge_document_history",
+      description: MATE_CONTRACT + " List recoverable revisions for one Markdown document by relative path or stable document id.",
+      inputSchema: buildShape({ name: { type: "string", description: "Relative Markdown path under this Mate's Knowledge directory." }, documentId: DOCUMENT_ID }, []),
+      handler: handler(bound, "listKnowledgeDocumentHistory"),
+    },
+    {
+      name: "restore_knowledge_document",
+      description: MATE_CONTRACT + " Restore a selected historical Markdown revision. The current revision is mandatory so a concurrent edit cannot be overwritten.",
+      inputSchema: buildShape({ name: { type: "string", description: "Relative Markdown path under this Mate's Knowledge directory." }, documentId: DOCUMENT_ID, version: { type: "string", description: "Version returned by list_knowledge_document_history." }, expectedRevision: DOCUMENT_REVISION }, ["version", "expectedRevision"]),
+      handler: handler(bound, "restoreKnowledgeDocument"),
+    },
   ].concat(databaseTools(bound));
 }
 

package/lib/mate-knowledge-service.js

@@ -15,6 +15,8 @@ var builtinMates = require("./builtin-mates");
 var knowledgeImport = require("./knowledge-import");
 var knowledgeSearch = require("./knowledge-search");
 var mateSync = require("./mate-knowledge-sync");
+var textToolsModule = require("./mate-knowledge-text-tools");
+var mateBoundModule = require("./mate-knowledge-mate-bound");
 
 var MAX_PAGE = 50;
 var DEFAULT_PAGE = 20;
@@ -97,6 +99,7 @@ function attachMateKnowledgeService(ctx) {
   var listMates = context.listMates || function () { return []; };
   var baseDir = context.baseDir;
   var importerCache = new Map();
+  var textTools = textToolsModule.createMateKnowledgeTextTools({ baseDir: baseDir });
 
   function importerForScope(scopeId) {
     if (context.openImporter) return context.openImporter(scopeId);
@@ -379,6 +382,8 @@ function attachMateKnowledgeService(ctx) {
     if (source.session) {
       var manager = project.getSessionManager ? project.getSessionManager() : project.sm;
       if (!manager || !manager.sessions || manager.sessions.get(source.session.localId) !== source.session) return null;
+      if (source.projectOwnerId && source.session.ownerId && source.session.ownerId !== source.projectOwnerId) return null;
+      if (isMultiUser() && (!source.projectOwnerId || source.session.ownerId !== source.projectOwnerId)) return null;
     }
     if (typeof resolveMate !== "function") return null;
     var mate = resolveMate(source.projectOwnerId || null, source.mateId);
@@ -422,36 +427,10 @@ function attachMateKnowledgeService(ctx) {
   // An ordinary Mate's own scope, and nothing else. There is no argument on
   // any tool that could widen it.
   function createMateBound(revalidate, initial) {
-    function ownScope() {
-      return [revalidate().own];
-    }
-    function db(method, args) {
-      var current = revalidate(); if (!current.databaseService || typeof current.databaseService[method] !== "function") throw new Error("Knowledge databases are unavailable for this Mate.");
-      return current.databaseService[method](Object.assign({}, args || {}, { actor: current.actor }));
-    }
-    return {
-      isClay: false,
-      mateId: initial.own.mateId,
-      scopeId: initial.own.scopeId,
-      listKnowledge: function (args) { return listFor(ownScope(), args || {}, false); },
-      searchKnowledge: function (args) { return searchFor(ownScope(), args || {}, false); },
-      readKnowledge: function (args) { return readFor(ownScope(), args || {}, false); },
-      listDatabases: function (args) { return db("list", args); },
-      createDatabase: function (args) { return db("create", args); },
-      readDatabase: function (args) { return db("read", args); },
-      renameDatabase: function (args) { return db("rename", args); },
-      archiveDatabase: function (args) { return db("archive", args); },
-      recoverDatabase: function (args) { return db("recover", args); },
-      updateDatabaseSchema: function (args) { return db("updateSchema", args); },
-      saveTableView: function (args) { return db("saveView", args); },
-      queryDatabase: function (args) { return db("query", args); },
-      createDatabaseRecord: function (args) { return db("createRecord", args); },
-      readDatabaseRecord: function (args) { return db("readRecord", args); },
-      updateDatabaseRecord: function (args) { return db("updateRecord", args); },
-      archiveDatabaseRecord: function (args) { return db("archiveRecord", args); },
-      recoverDatabaseRecord: function (args) { return db("recoverRecord", args); },
-      retryDatabaseSync: function (args) { return db("retryProjection", args); },
-    };
+    return mateBoundModule.createMateBound({
+      revalidate: revalidate, initial: initial, textTools: textTools,
+      listFor: listFor, searchFor: searchFor, readFor: readFor,
+    });
   }
 
   // Every Mate belonging to the same user, resolved fresh on each call so a

package/lib/mate-knowledge-text-tools.js

@@ -0,0 +1,106 @@
+// Session-bound text Knowledge mutations for one authorized Mate.
+//
+// This module deliberately uses the existing portable file store and
+// write-through synchronizer. It never accepts a Mate, owner, or filesystem
+// root from a tool caller; the caller supplies only a relative document name
+// or the stable identity returned by the store.
+
+var crypto = require("crypto");
+var path = require("path");
+var knowledgeFiles = require("./knowledge-file-store");
+var mateSync = require("./mate-knowledge-sync");
+
+function metadataRoot(baseDir, mateDir) {
+  var key = crypto.createHash("sha256").update(path.resolve(mateDir)).digest("hex").slice(0, 24);
+  return path.join(baseDir || path.join(mateDir, ".clay"), "knowledge-documents", key);
+}
+
+function failedSync(summary) {
+  if (!summary) return { indexSyncPending: true, syncError: "Knowledge index synchronization returned no result." };
+  if (summary.failed === 0) return { indexSyncPending: false, syncError: null };
+  var error = summary.errors && summary.errors[0];
+  return { indexSyncPending: true, syncError: error && error.message || "Knowledge index synchronization failed." };
+}
+
+function createMateKnowledgeTextTools(options) {
+  var settings = options || {};
+  var stores = new Map();
+
+  function storeFor(mate) {
+    if (!mate || typeof mate.mateDir !== "string") throw new Error("Knowledge text tools require an authorized Mate.");
+    var directory = path.resolve(mate.mateDir);
+    if (!stores.has(directory)) stores.set(directory, knowledgeFiles.createKnowledgeFileStore(directory, { metaDir: metadataRoot(settings.baseDir, directory) }));
+    return stores.get(directory);
+  }
+
+  function find(store, input) {
+    var args = input || {};
+    var hasName = typeof args.name === "string" && args.name.length > 0;
+    var hasId = typeof args.documentId === "string" && args.documentId.length > 0;
+    if (!hasName && !hasId) throw new Error("A document name or stable document id is required.");
+    var item = null;
+    if (hasId) item = store.list().filter(function (entry) { return entry.id === args.documentId; })[0] || null;
+    if (hasId && !item) throw Object.assign(new Error("Knowledge document not found."), { code: "ENOENT" });
+    if (hasName) {
+      var byName = store.read(args.name);
+      if (item && item.id !== byName.id) throw new Error("The document name and id identify different documents.");
+      item = byName;
+    }
+    if (!item) throw Object.assign(new Error("Knowledge document not found."), { code: "ENOENT" });
+    if (hasId && !hasName) item = store.read(item.name);
+    if (item.itemType !== "text") throw Object.assign(new Error("Only Markdown Knowledge documents are writable through these tools."), { code: "READ_ONLY" });
+    return item;
+  }
+
+  function sync(mate, name, actor) {
+    try {
+      var summary = mateSync.syncMateSource({ mateDir: mate.mateDir, fileName: name, baseDir: settings.baseDir, actor: actor });
+      return failedSync(summary);
+    } catch (error) {
+      return { indexSyncPending: true, syncError: error.message || "Knowledge index synchronization failed." };
+    }
+  }
+
+  function read(mate, args) {
+    var item = find(storeFor(mate), args);
+    return { documentId: item.id, name: item.name, content: item.content, revision: item.revision };
+  }
+
+  function create(mate, args, actor) {
+    var input = args || {};
+    if (typeof input.name !== "string" || !input.name) throw new Error("A document name is required to create a Knowledge document.");
+    if (typeof input.content !== "string") throw new Error("Document content is required.");
+    var store = storeFor(mate);
+    var saved = store.save(input.name, input.content, null);
+    return Object.assign({ documentId: saved.id, name: saved.name, revision: saved.revision }, sync(mate, saved.name, actor));
+  }
+
+  function update(mate, args, actor) {
+    var input = args || {};
+    if (typeof input.expectedRevision !== "string" || !input.expectedRevision) throw Object.assign(new Error("A current document revision is required for updates."), { code: "STALE" });
+    if (typeof input.content !== "string") throw new Error("Document content is required.");
+    var store = storeFor(mate);
+    var current = find(store, input);
+    var saved = store.save(current.name, input.content, input.expectedRevision);
+    return Object.assign({ documentId: saved.id, name: saved.name, revision: saved.revision }, sync(mate, saved.name, actor));
+  }
+
+  function history(mate, args) {
+    var current = find(storeFor(mate), args);
+    return { documentId: current.id, name: current.name, versions: storeFor(mate).history(current.id) };
+  }
+
+  function restore(mate, args, actor) {
+    var input = args || {};
+    if (typeof input.expectedRevision !== "string" || !input.expectedRevision) throw Object.assign(new Error("A current document revision is required for restore."), { code: "STALE" });
+    if (typeof input.version !== "string" || !input.version) throw new Error("A history version is required.");
+    var store = storeFor(mate);
+    var current = find(store, input);
+    var restored = store.restore(current.id, input.version, current.name, input.expectedRevision);
+    return Object.assign({ documentId: restored.id, name: restored.name, revision: restored.revision }, sync(mate, restored.name, actor));
+  }
+
+  return { read: read, create: create, update: update, history: history, restore: restore };
+}
+
+module.exports = { createMateKnowledgeTextTools: createMateKnowledgeTextTools };

package/lib/project-mate-interaction.js

@@ -3,6 +3,7 @@ var fs = require("fs");
 var path = require("path");
 var crypto = require("crypto");
 var mateSync = require("./mate-knowledge-sync");
+var dmDigestCursor = require("./dm-digest-cursor");
 
 // Mirror a Mate-owned journal into the Clay-wide Knowledge backend after the
 // legacy append has already succeeded. Reconciling the whole journal keeps the
@@ -294,6 +295,9 @@ function attachMateInteraction(ctx) {
         mirrorMateJournal(mateDir, "session-digests.jsonl", job);
       } catch (e) {
         console.error("[digest-worker] Write failed for " + job.mateId + ":", e.message);
+        if (job.onError) job.onError(e);
+        processDigestQueue();
+        return;
       }
 
       // Write user observations if present
@@ -422,63 +426,19 @@ function attachMateInteraction(ctx) {
   // Delta-based: only collects new turns since the last successful digest.
   // Concurrency debounce: turns that arrive while a digest is in-flight
   // are naturally batched into the next flush.
-  var _dmDigestPending = false;
-  function digestDmTurn(session, responsePreview) {
-    if (!isMate || _dmDigestPending) return;
+  var _dmDigestPending = new WeakSet();
+  function digestDmTurn(session) {
+    if (!isMate || _dmDigestPending.has(session)) return;
     var mateId = path.basename(cwd);
     var mateCtx = matesModule.buildMateCtx(projectOwnerId);
     if (!matesModule.isMate(mateCtx, mateId)) return;
 
-    // Track digest index per session so switching sessions doesn't misalign.
-    // On resumed sessions (after restart), recover index from the last
-    // digest_checkpoint entry in history so undigested turns aren't lost.
-    if (typeof session._dmLastDigestedIndex !== "number") {
-      session._dmLastDigestedIndex = 0;
-      for (var ci = session.history.length - 1; ci >= 0; ci--) {
-        if (session.history[ci].type === "digest_checkpoint") {
-          session._dmLastDigestedIndex = session.history[ci].digestedIndex;
-          break;
-        }
-      }
-    }
-
-    // Collect only new turns since the last successful digest
-    var conversationParts = [];
-    var totalLen = 0;
-    var CONV_CAP = 6000;
-    var startIndex = session._dmLastDigestedIndex;
-    for (var hi = startIndex; hi < session.history.length; hi++) {
-      if (totalLen >= CONV_CAP) break;
-      var entry = session.history[hi];
-      if (entry.type === "user_message" && entry.text) {
-        var uText = entry.text;
-        if (totalLen + uText.length > CONV_CAP) {
-          uText = uText.substring(0, Math.max(200, CONV_CAP - totalLen)) + "...";
-        }
-        conversationParts.push("User: " + uText);
-        totalLen += uText.length;
-      } else if (entry.type === "assistant_message" && entry.text) {
-        var aText = entry.text;
-        if (totalLen + aText.length > CONV_CAP) {
-          aText = aText.substring(0, Math.max(200, CONV_CAP - totalLen)) + "...";
-        }
-        conversationParts.push("Mate: " + aText);
-        totalLen += aText.length;
-      }
-    }
-    // If the latest response hasn't landed in history yet, append it
-    var lastResponseText = responsePreview || "";
-    if (lastResponseText && conversationParts.length > 0) {
-      var lastPart = conversationParts[conversationParts.length - 1];
-      if (lastPart.indexOf("Mate:") !== 0 || lastPart.indexOf(lastResponseText.substring(0, 50)) === -1) {
-        var rText = lastResponseText;
-        if (totalLen + rText.length > CONV_CAP) {
-          rText = rText.substring(0, Math.max(200, CONV_CAP - totalLen)) + "...";
-        }
-        conversationParts.push("Mate: " + rText);
-      }
-    }
-    if (conversationParts.length === 0) return;
+    // Recover the exact next unread position. Legacy index-only checkpoints
+    // remain readable and are treated as fully consumed entries.
+    var digestState = dmDigestCursor.readState(session.history);
+    var startCursor = digestState.cursor;
+    var chunk = dmDigestCursor.collectChunk(session.history, startCursor);
+    if (!chunk.conversationContent) return;
 
     var mateDir = matesModule.getMateDir(mateCtx, mateId);
     var knowledgeDir = path.join(mateDir, "knowledge");
@@ -500,8 +460,8 @@ function attachMateInteraction(ctx) {
       }
     } catch (e) {}
 
-    _dmDigestPending = true;
-    var snapshotIndex = session.history.length;
+    _dmDigestPending.add(session);
+    var checkpointCursor = chunk.cursor;
 
     enqueueDigest({
       mateCtx: mateCtx,
@@ -509,17 +469,27 @@ function attachMateInteraction(ctx) {
       mateId: mateId,
       type: "dm",
       priorSummary: priorSummary || "",
-      conversationContent: conversationParts.join("\n"),
+      conversationContent: chunk.conversationContent,
       onDone: function () {
-        session._dmLastDigestedIndex = snapshotIndex;
-        // Persist checkpoint so resumed sessions know where to continue
-        var checkpoint = { type: "digest_checkpoint", digestedIndex: snapshotIndex };
+        // Journal and checkpoint are separate appends: if this append fails,
+        // retry may duplicate the saved digest, but must never skip its input.
+        var checkpoint = { type: "digest_checkpoint", cursor: checkpointCursor };
+        var saved = false;
+        try { saved = sm.appendToSessionFile(session, checkpoint) !== false; } catch (e) {
+          console.error("[dm-digest] Checkpoint append failed:", e.message);
+        }
+        if (!saved) {
+          _dmDigestPending.delete(session);
+          return;
+        }
         session.history.push(checkpoint);
-        sm.appendToSessionFile(session, checkpoint);
-        _dmDigestPending = false;
+        _dmDigestPending.delete(session);
+        if (dmDigestCursor.hasPending(session.history, checkpointCursor)) {
+          setImmediate(function () { digestDmTurn(session, ""); });
+        }
       },
       onError: function () {
-        _dmDigestPending = false;
+        _dmDigestPending.delete(session);
       },
     });
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clay-server",
-  "version": "5.2.0-beta.2",
+  "version": "5.2.0-beta.3",
   "description": "Self-hosted team workspace for Claude Code and Codex. Multi-user, browser-based, with persistent AI mates.",
   "bin": {
     "clay-server": "./bin/cli.js"