clay-server 4.0.0-beta.8 → 4.0.0

package/lib/mate-knowledge-sync.js

@@ -0,0 +1,482 @@
+// Compatibility write-through bridge for Mate durable memory and knowledge.
+//
+// The startup importer alone goes stale the moment a Mate writes a digest or a
+// user edits a knowledge file. This module keeps the Knowledge scope current
+// after every known mutation, without changing how the legacy files are
+// written. Legacy files stay authoritative and rollback-compatible; the new
+// backend is a continuously reconciled mirror.
+//
+// Reconciliation is source-level, not event-level. A caller says "this source
+// changed" and this module re-reads that one file and makes the scope match it:
+// creating, revising, reviving, or tombstoning as required. That makes every
+// call idempotent and makes deletions and rewrites work without the call site
+// having to describe what changed.
+//
+// Identity is never mirrored. CLAUDE.md, base-template.md, identity-backup.md,
+// and identity-history.jsonl define who a Mate is and remain canonical files.
+// Sticky notes are a separate attention layer and are excluded too.
+
+var fs = require("fs");
+var path = require("path");
+var crypto = require("crypto");
+var config = require("./config");
+var knowledgeImport = require("./knowledge-import");
+
+var SOURCE_NAMESPACE = "mate-knowledge";
+var SOURCE_VERSION = 2;
+var MAX_SOURCE_BYTES = 64 * 1024 * 1024;
+
+// Identity and template artifacts. Never mirrored, at any version.
+var IDENTITY_FILES = ["base-template.md", "identity-backup.md", "identity-history.jsonl"];
+// Sticky notes remain the separate attention layer, with their own store.
+var EXCLUDED_FILES = ["sticky-notes.md"];
+
+// Line-addressed journals get one record per line, keyed by line content.
+var JOURNAL_KINDS = {
+  "session-digests.jsonl": "session-digest",
+  "user-observations.jsonl": "user-observation",
+};
+
+function matesHome() {
+  return path.resolve(path.join(config.CONFIG_DIR, "mates"));
+}
+
+// Derived from the Mate storage root path alone, so the startup importer and
+// the live bridge always agree on a scope without needing the same context.
+function ownerKeyForRoot(matesRoot) {
+  var resolved = path.resolve(matesRoot);
+  var parent = path.dirname(resolved);
+  var name = path.basename(resolved);
+  if (parent === matesHome() && name.indexOf("mate_") !== 0) return "u-" + name;
+  return "h-" + crypto.createHash("sha256").update(resolved).digest("hex").substring(0, 16);
+}
+
+function scopeIdFor(ownerKey, mateId) {
+  return "mate/" + ownerKey + "/" + mateId;
+}
+
+// A Mate project's cwd is its Mate directory, so every call site can pass what
+// it already has.
+function scopeForMateDir(mateDir) {
+  var resolved = path.resolve(mateDir);
+  var mateId = path.basename(resolved);
+  if (mateId.indexOf("mate_") !== 0) return null;
+  var matesRoot = path.dirname(resolved);
+  var ownerKey = ownerKeyForRoot(matesRoot);
+  var parent = path.dirname(path.resolve(matesRoot));
+  return {
+    mateId: mateId,
+    mateDir: resolved,
+    matesRoot: matesRoot,
+    ownerKey: ownerKey,
+    userId: parent === matesHome() ? path.basename(matesRoot) : null,
+    scopeId: scopeIdFor(ownerKey, mateId),
+  };
+}
+
+function sourceKindFor(name) {
+  if (IDENTITY_FILES.indexOf(name) !== -1) return null;
+  if (EXCLUDED_FILES.indexOf(name) !== -1) return null;
+  if (name === "memory-summary.md") return "memory-summary";
+  if (JOURNAL_KINDS[name]) return JOURNAL_KINDS[name];
+  if (/\.md$/.test(name)) return "knowledge-file";
+  if (/\.jsonl$/.test(name)) return "knowledge-journal";
+  return null;
+}
+
+function isJournal(name) {
+  return !!JOURNAL_KINDS[name] || /\.jsonl$/.test(name);
+}
+
+function normalizeActor(actor) {
+  var source = actor || {};
+  var type = ["user", "agent", "system", "migration"].indexOf(source.type) !== -1 ? source.type : "system";
+  return {
+    type: type,
+    userId: source.userId ? String(source.userId).substring(0, 120) : null,
+    displayName: source.displayName ? String(source.displayName).substring(0, 120) : null,
+    sessionKey: source.sessionKey ? String(source.sessionKey).substring(0, 200) : null,
+    vendor: source.vendor ? String(source.vendor).substring(0, 40) : null,
+  };
+}
+
+// Relative to the Mate directory. The scope already identifies the owner and
+// the Mate, so an absolute filesystem path would add nothing but exposure.
+function relativeSource(name) {
+  return "knowledge/" + name;
+}
+
+function provenance(scope, name, extra) {
+  var base = {
+    namespace: SOURCE_NAMESPACE,
+    sourceVersion: SOURCE_VERSION,
+    relPath: relativeSource(name),
+    fileName: name,
+    ownerKey: scope.ownerKey,
+    userId: scope.userId,
+    mateId: scope.mateId,
+    mateName: scope.mateName || null,
+  };
+  if (!extra) return base;
+  return Object.assign(base, extra);
+}
+
+// Identify a live record that this module wrote, by provenance rather than by
+// parsing an import key. A record from another namespace, or from a source
+// version this build does not understand, is left strictly alone: reconciling
+// it could tombstone something whose meaning we do not know.
+function isMateSourceEntry(entry) {
+  if (!entry || entry.deleted === true) return false;
+  if (entry.kind === "chunk") return false;
+  var source = entry.source;
+  if (!source || source.namespace !== SOURCE_NAMESPACE) return false;
+  if (typeof source.sourceVersion !== "number" || source.sourceVersion > SOURCE_VERSION) return false;
+  var name = source.fileName;
+  if (!name || typeof name !== "string") return false;
+  // The file name and the relative path must agree, so neither field alone is
+  // trusted, and the name must be a plain basename.
+  if (path.basename(name) !== name) return false;
+  return source.relPath === relativeSource(name);
+}
+
+function fileKey(name) {
+  return "file:" + name;
+}
+
+function journalKey(name, hash) {
+  return "journal:" + name + ":" + hash.substring(0, 24);
+}
+
+function newSummary() {
+  return {
+    created: 0,
+    revised: 0,
+    revived: 0,
+    unchanged: 0,
+    deleted: 0,
+    absent: 0,
+    "already-deleted": 0,
+    chunked: 0,
+    malformedLines: 0,
+    identitySkipped: 0,
+    excluded: 0,
+    unsupported: 0,
+    empty: 0,
+    failed: 0,
+    errors: [],
+  };
+}
+
+function record(summary, action) {
+  if (summary[action] === undefined) summary[action] = 0;
+  summary[action]++;
+}
+
+function readStat(filePath) {
+  try {
+    return fs.statSync(filePath);
+  } catch (e) {
+    return null;
+  }
+}
+
+// --- File-addressed sources -------------------------------------------
+
+function syncFile(importer, scope, name, kind, filePath, stat, actor, summary) {
+  if (!stat) {
+    // The legacy file is gone. Tombstone rather than leaving stale content live.
+    var removed = importer.removeRecord(fileKey(name), {
+      actor: actor,
+      source: provenance(scope, name, { type: kind, reason: "source-removed" }),
+    });
+    record(summary, removed.action);
+    return;
+  }
+  if (stat.size > MAX_SOURCE_BYTES) {
+    summary.failed++;
+    summary.errors.push({ scopeId: scope.scopeId, file: name, message: "Source exceeds the maximum representable size." });
+    return;
+  }
+  var content;
+  try {
+    content = fs.readFileSync(filePath, "utf8");
+  } catch (e) {
+    summary.failed++;
+    summary.errors.push({ scopeId: scope.scopeId, file: name, message: "Unreadable source: " + e.message });
+    return;
+  }
+  if (!content.trim()) {
+    // An emptied file is a logical removal, not a blank record.
+    var emptied = importer.removeRecord(fileKey(name), {
+      actor: actor,
+      source: provenance(scope, name, { type: kind, reason: "source-emptied" }),
+    });
+    summary.empty++;
+    if (emptied.action === "deleted") summary.deleted++;
+    return;
+  }
+  var result = importer.importRecord({
+    importKey: fileKey(name),
+    sourceHash: knowledgeImport.contentHash(content),
+    kind: kind,
+    name: name,
+    content: content,
+    at: Math.round(stat.mtimeMs),
+    actor: actor,
+    source: provenance(scope, name, {
+      type: kind,
+      sourceBytes: stat.size,
+      sourceModifiedAt: stat.mtimeMs,
+    }),
+  });
+  record(summary, result.action);
+  if (content.length > knowledgeImport.MAX_CONTENT_CHARS) summary.chunked++;
+  // Verified whenever something was actually written. An unchanged source was
+  // already verified when it landed, so the steady-state path stays cheap.
+  if (result.action !== "unchanged") verifyRoundTrip(importer, fileKey(name), scope, name, summary);
+}
+
+// --- Line-addressed journals ------------------------------------------
+
+// Reconciles the whole journal: every present line is imported, and every line
+// previously imported from this journal that is no longer present is
+// tombstoned. A digest deleted through the memory UI therefore stops being a
+// live memory instead of lingering in the new backend.
+function syncJournal(importer, scope, name, kind, filePath, stat, actor, summary) {
+  var raw = "";
+  if (stat) {
+    if (stat.size > MAX_SOURCE_BYTES) {
+      summary.failed++;
+      summary.errors.push({ scopeId: scope.scopeId, file: name, message: "Source exceeds the maximum representable size." });
+      return;
+    }
+    try {
+      raw = fs.readFileSync(filePath, "utf8");
+    } catch (e) {
+      summary.failed++;
+      summary.errors.push({ scopeId: scope.scopeId, file: name, message: "Unreadable source: " + e.message });
+      return;
+    }
+  }
+
+  var present = {};
+  var lines = raw.split("\n");
+  for (var i = 0; i < lines.length; i++) {
+    var line = lines[i].trim();
+    if (!line) continue;
+    var parsed = null;
+    try {
+      parsed = JSON.parse(line);
+    } catch (e) {
+      summary.malformedLines++;
+      continue;
+    }
+    // A journal entry is a plain object. An array or a bare scalar parses
+    // cleanly but is not a record, so it is skipped rather than imported.
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+      summary.malformedLines++;
+      continue;
+    }
+    var hash = knowledgeImport.contentHash(line);
+    var key = journalKey(name, hash);
+    present[key] = true;
+    var result = importer.importRecord({
+      importKey: key,
+      sourceHash: hash,
+      kind: kind,
+      name: name,
+      content: line,
+      actor: actor,
+      source: provenance(scope, name, {
+        type: kind,
+        lineIndex: i,
+        sourceDate: typeof parsed.date === "string" ? parsed.date : null,
+        sourceBytes: Buffer.byteLength(line, "utf8"),
+        sourceModifiedAt: stat ? stat.mtimeMs : null,
+      }),
+    });
+    record(summary, result.action);
+    if (line.length > knowledgeImport.MAX_CONTENT_CHARS) summary.chunked++;
+    if (result.action === "created" || result.action === "revived") {
+      verifyRoundTrip(importer, key, scope, name, summary);
+    }
+  }
+
+  // Every live line this module imported from this journal that the legacy
+  // file no longer contains. Selected by provenance, so a record from another
+  // namespace sharing a file name is never touched.
+  var stale = [];
+  var live = importer.entries();
+  for (var e = 0; e < live.length; e++) {
+    if (!isMateSourceEntry(live[e])) continue;
+    if (live[e].source.fileName !== name) continue;
+    if (present[live[e].importKey]) continue;
+    stale.push(live[e].importKey);
+  }
+  for (var s = 0; s < stale.length; s++) {
+    var removed = importer.removeRecord(stale[s], {
+      actor: actor,
+      source: provenance(scope, name, { type: kind, reason: "line-removed" }),
+    });
+    record(summary, removed.action);
+  }
+}
+
+// No source may be reported as successfully mirrored while its content cannot
+// be read back exactly.
+function verifyRoundTrip(importer, importKey, scope, name, summary) {
+  var entries = importer.entries();
+  var entry = null;
+  for (var i = 0; i < entries.length; i++) {
+    if (entries[i].importKey === importKey) { entry = entries[i]; break; }
+  }
+  if (!entry) {
+    summary.failed++;
+    summary.errors.push({ scopeId: scope.scopeId, file: name, message: "Imported record is not readable: " + importKey });
+    return;
+  }
+  var read = importer.readContent(entry);
+  if (!read.complete) {
+    summary.failed++;
+    summary.errors.push({
+      scopeId: scope.scopeId,
+      file: name,
+      message: (entry.chunked ? "Chunked source did not reassemble: " : "Stored source did not verify: ") + (read.reason || "unknown"),
+    });
+  }
+}
+
+// --- Public API --------------------------------------------------------
+
+function importerFor(scope, baseDir) {
+  return knowledgeImport.createKnowledgeImporter({ scopeId: scope.scopeId, baseDir: baseDir });
+}
+
+// Reconcile exactly one Mate knowledge source against the new backend. Safe to
+// call after any legacy write, including a delete. Never throws.
+function syncMateSource(opts) {
+  var options = opts || {};
+  var summary = options.summary || newSummary();
+  try {
+    var scope = options.scope || scopeForMateDir(options.mateDir);
+    if (!scope) return summary;
+    if (options.mateName) scope.mateName = options.mateName;
+    var name = path.basename(String(options.fileName || ""));
+    if (!name) return summary;
+    if (IDENTITY_FILES.indexOf(name) !== -1) { summary.identitySkipped++; return summary; }
+    if (EXCLUDED_FILES.indexOf(name) !== -1) { summary.excluded++; return summary; }
+    var kind = sourceKindFor(name);
+    if (!kind) { summary.unsupported++; return summary; }
+
+    var actor = normalizeActor(options.actor);
+    var importer = options.importer || importerFor(scope, options.baseDir);
+    var filePath = path.join(scope.mateDir, "knowledge", name);
+    var stat = readStat(filePath);
+
+    if (isJournal(name)) syncJournal(importer, scope, name, kind, filePath, stat, actor, summary);
+    else syncFile(importer, scope, name, kind, filePath, stat, actor, summary);
+  } catch (e) {
+    summary.failed++;
+    summary.errors.push({ message: e.message });
+  }
+  return summary;
+}
+
+// Alias that reads better at a call site reacting to a removal or a rewrite.
+function reconcileSource(opts) {
+  return syncMateSource(opts);
+}
+
+// Reconcile every durable source in one Mate directory, in both directions.
+//
+// Enumerating the directory alone is not enough. A knowledge file or a whole
+// journal can be deleted while the daemon is stopped, or by direct filesystem
+// tooling, and nothing would ever call syncMateSource for it again. So the
+// scope's own live records are also consulted: any source this module
+// previously imported that is no longer on disk is reconciled, which tombstones
+// a missing file record or every line of a missing journal.
+function reconcileMate(opts) {
+  var options = opts || {};
+  var summary = options.summary || newSummary();
+  var scope = options.scope || scopeForMateDir(options.mateDir);
+  if (!scope) return summary;
+  if (options.mateName) scope.mateName = options.mateName;
+  var knowledgeDir = path.join(scope.mateDir, "knowledge");
+  var present = {};
+  var entries = [];
+  try {
+    entries = fs.readdirSync(knowledgeDir, { withFileTypes: true });
+  } catch (e) {
+    entries = [];
+  }
+  var importer = options.importer || importerFor(scope, options.baseDir);
+
+  function reconcileOne(fileName) {
+    syncMateSource({
+      scope: scope,
+      fileName: fileName,
+      actor: options.actor,
+      importer: importer,
+      baseDir: options.baseDir,
+      summary: summary,
+    });
+  }
+
+  for (var i = 0; i < entries.length; i++) {
+    if (!entries[i].isFile()) continue;
+    // Recorded as present whatever its kind, so an excluded or unsupported file
+    // that still exists is never mistaken for a deleted source.
+    present[entries[i].name] = true;
+    reconcileOne(entries[i].name);
+  }
+
+  var absent = {};
+  var live = importer.entries();
+  for (var e = 0; e < live.length; e++) {
+    if (!isMateSourceEntry(live[e])) continue;
+    var name = live[e].source.fileName;
+    if (present[name] || absent[name]) continue;
+    absent[name] = true;
+  }
+  var absentNames = Object.keys(absent);
+  for (var a = 0; a < absentNames.length; a++) reconcileOne(absentNames[a]);
+  return summary;
+}
+
+// Fire-and-forget wrapper for legacy write paths. A mirroring failure must
+// never break the legacy write that already succeeded.
+function syncQuietly(opts) {
+  try {
+    var summary = syncMateSource(opts);
+    if (summary.failed > 0 && summary.errors.length > 0) {
+      console.error("[knowledge-sync] Mate knowledge mirror failed:", summary.errors[0].message);
+    }
+    return summary;
+  } catch (e) {
+    console.error("[knowledge-sync] Mate knowledge mirror failed:", e.message);
+    return null;
+  }
+}
+
+module.exports = {
+  SOURCE_NAMESPACE: SOURCE_NAMESPACE,
+  SOURCE_VERSION: SOURCE_VERSION,
+  MAX_SOURCE_BYTES: MAX_SOURCE_BYTES,
+  IDENTITY_FILES: IDENTITY_FILES,
+  EXCLUDED_FILES: EXCLUDED_FILES,
+  JOURNAL_KINDS: JOURNAL_KINDS,
+  ownerKeyForRoot: ownerKeyForRoot,
+  scopeIdFor: scopeIdFor,
+  scopeForMateDir: scopeForMateDir,
+  sourceKindFor: sourceKindFor,
+  isMateSourceEntry: isMateSourceEntry,
+  normalizeActor: normalizeActor,
+  newSummary: newSummary,
+  fileKey: fileKey,
+  journalKey: journalKey,
+  importerFor: importerFor,
+  syncMateSource: syncMateSource,
+  reconcileSource: reconcileSource,
+  reconcileMate: reconcileMate,
+  syncQuietly: syncQuietly,
+};

package/lib/migrate-single-user.js

@@ -23,6 +23,7 @@ var SETTING_KEYS = [
   "chatLayout",
   "autoContinueOnRateLimit",
   "matesEnabled",
+  "capsulesEnabled",
   "terminalFont",
   "deletedBuiltinKeys",
   "mateOnboardingShown",
@@ -189,6 +190,7 @@ function migrateSingleUserToMulti() {
       if (cfg.chatLayout !== undefined && users.setChatLayout) users.setChatLayout(userId, cfg.chatLayout);
       if (cfg.autoContinueOnRateLimit !== undefined && users.setAutoContinue) users.setAutoContinue(userId, cfg.autoContinueOnRateLimit);
       if (cfg.matesEnabled !== undefined && users.setMatesEnabled) users.setMatesEnabled(userId, cfg.matesEnabled);
+      if (cfg.capsulesEnabled !== undefined && users.setCapsulesEnabled) users.setCapsulesEnabled(userId, cfg.capsulesEnabled);
       if (cfg.terminalFont && users.setTerminalFont) {
         users.setTerminalFont(userId, cfg.terminalFont.family, cfg.terminalFont.size);
       }

package/lib/notes-lifecycle.js

@@ -0,0 +1,149 @@
+// Sticky Note lifecycle.
+//
+// A note is an attention item, so "done" must never mean "gone". Completing a
+// note closes it: the record stays, stays queryable, and stays reversible. Only
+// its place on the active canvas changes.
+//
+// Legacy notes carry no state field. They project as open, with one exception:
+// the older `hidden: true` flag already meant "taken off the canvas and hidden
+// from agents", which is exactly what closed means, so those project as closed.
+// The original `hidden` value is preserved rather than rewritten, and no
+// closedAt is invented for a note that never recorded one.
+//
+// Projection is applied when notes are loaded, so nothing on disk is rewritten
+// just by reading. The normalized fields reach the file the next time that note
+// is written for some other reason.
+
+var STATE_OPEN = "open";
+var STATE_CLOSED = "closed";
+var STATES = [STATE_OPEN, STATE_CLOSED];
+
+function isState(value) {
+  return STATES.indexOf(value) !== -1;
+}
+
+// The state a note is in, derived rather than trusted: an unknown or missing
+// value falls back to the legacy flag and then to open.
+function stateOf(note) {
+  if (!note) return STATE_OPEN;
+  if (isState(note.state)) return note.state;
+  return note.hidden === true ? STATE_CLOSED : STATE_OPEN;
+}
+
+function isOpen(note) {
+  return stateOf(note) === STATE_OPEN;
+}
+
+function isClosed(note) {
+  return stateOf(note) === STATE_CLOSED;
+}
+
+// Normalize one note in place. Returns true when anything changed, so a caller
+// can decide whether the migration is worth persisting.
+function normalize(note) {
+  if (!note || typeof note !== "object") return false;
+  var changed = false;
+  var state = stateOf(note);
+  if (note.state !== state) {
+    note.state = state;
+    changed = true;
+  }
+  if (note.closedAt === undefined) {
+    // A legacy hidden note has no recorded close time and none is invented.
+    note.closedAt = null;
+    changed = true;
+  }
+  if (note.closedBy === undefined) {
+    note.closedBy = null;
+    changed = true;
+  }
+  // Keep the legacy flag consistent with the state so an old client that only
+  // understands `hidden` still takes a closed note off its canvas.
+  var hidden = state === STATE_CLOSED;
+  if (note.hidden !== hidden) {
+    note.hidden = hidden;
+    changed = true;
+  }
+  return changed;
+}
+
+function normalizeAll(notes) {
+  var list = Array.isArray(notes) ? notes : [];
+  var changed = false;
+  for (var i = 0; i < list.length; i++) {
+    if (normalize(list[i])) changed = true;
+  }
+  return { notes: list, changed: changed };
+}
+
+function openNotes(notes) {
+  return (notes || []).filter(function (note) { return note && isOpen(note); });
+}
+
+function closedNotes(notes) {
+  return (notes || []).filter(function (note) { return note && isClosed(note); });
+}
+
+// Actors are always built from server-bound context. Nothing here reads a
+// caller-supplied identity, so a payload cannot claim to be someone else.
+function sessionActor(session) {
+  if (!session) return null;
+  return {
+    type: "session",
+    sessionId: session.localId !== undefined && session.localId !== null ? session.localId : null,
+    vendor: session.vendor || null,
+  };
+}
+
+function userActor(user) {
+  return {
+    type: "user",
+    userId: (user && user.id) || null,
+    displayName: (user && (user.displayName || user.username)) || null,
+  };
+}
+
+// Apply the close transition. Idempotent: closing an already-closed note keeps
+// the original closedAt and actor rather than restamping it, so the record of
+// when it was actually completed survives repeated calls.
+function applyClose(note, actor, now) {
+  if (!note) return false;
+  normalize(note);
+  if (isClosed(note)) return false;
+  note.state = STATE_CLOSED;
+  note.hidden = true;
+  note.closedAt = typeof now === "number" ? now : Date.now();
+  note.closedBy = actor || null;
+  return true;
+}
+
+// Apply the reopen transition. Idempotent in the same way, and it clears the
+// close provenance because the note is live again.
+function applyReopen(note) {
+  if (!note) return false;
+  normalize(note);
+  if (isOpen(note)) return false;
+  note.state = STATE_OPEN;
+  note.hidden = false;
+  note.closedAt = null;
+  note.closedBy = null;
+  return true;
+}
+
+module.exports = {
+  STATE_OPEN: STATE_OPEN,
+  STATE_CLOSED: STATE_CLOSED,
+  STATES: STATES,
+  isState: isState,
+  stateOf: stateOf,
+  isOpen: isOpen,
+  isClosed: isClosed,
+  normalize: normalize,
+  normalizeAll: normalizeAll,
+  openNotes: openNotes,
+  closedNotes: closedNotes,
+  sessionActor: sessionActor,
+  userActor: userActor,
+  applyClose: applyClose,
+  applyReopen: applyReopen,
+};