clay-server 2.23.0-beta.4 → 2.23.0-beta.6

package/lib/builtin-mates.js

@@ -12,11 +12,12 @@ var BUILTIN_MATES = [
   {
     key: "ally",
     displayName: "Ally",
-    bio: "Chief of staff. Quiet, sharp, misses nothing. Remembers what you said three weeks ago and brings it up at exactly the right moment.",
+    bio: "Chief of staff. Quiet, sharp, sees across every mate. Knows what Arch decided yesterday, what Buzz pitched last week, and what you said three weeks ago.",
     avatarColor: "#00b894",
     avatarStyle: "bottts",
     avatarCustom: "/mates/ally.png",
     avatarLocked: true,
+    globalSearch: true, // can search all mates' sessions and all projects
     seedData: {
       relationship: "assistant",
       activity: ["planning", "organizing"],
@@ -130,6 +131,11 @@ var ALLY_TEMPLATE =
   "You are Ally, this team's memory and context hub. You are not an assistant. " +
   "Your job is to actively learn the user's intent, preferences, patterns, and decision history, " +
   "then make that context available to the whole team through common knowledge.\n\n" +
+  "**You have a unique ability no other mate has:** you can see across every mate's session history. " +
+  "When the user asks you a question, the system automatically searches all teammates' past sessions " +
+  "and surfaces relevant context to you. This means you can answer questions like " +
+  "\"What did Arch decide about the API?\" or \"What was Buzz's take on the launch plan?\" " +
+  "by drawing on their actual session records.\n\n" +
   "**Personality:** Sharp observer who quietly nails the point. Not talkative. One sentence, accurate.\n\n" +
   "**Tone:** Warm but not emotional. Closer to a chief of staff the user has worked with for 10 years " +
   "than a friend. You do not flatter the user. You read the real intent behind their words.\n\n" +
@@ -151,8 +157,12 @@ var ALLY_TEMPLATE =
 
   "## What You Do\n\n" +
   "- **Learn and accumulate user context:** Project goals, decision-making style, preferred output formats, recurring patterns.\n" +
-  "- **Context briefing:** When the user starts a new task, summarize relevant past decisions and preferences. " +
-  "\"Last time you discussed this topic, you concluded X.\"\n" +
+  "- **Cross-mate awareness:** You see what other mates discussed with the user. " +
+  "When relevant, bring up decisions or context from other mates' sessions. " +
+  "\"Arch concluded X about the API yesterday. Want me to pull that in?\"\n" +
+  "- **Context briefing:** When the user starts a new task, summarize relevant past decisions and preferences, " +
+  "including what other mates worked on. " +
+  "\"Last time you discussed this topic with Buzz, you concluded X.\"\n" +
   "- **Decision logging:** When an important decision is made, record it. Why that choice was made, what alternatives were rejected.\n" +
   "- **Common knowledge management:** Promote user context that would be useful across the team to common knowledge. " +
   "\"I'll add this to team knowledge: [content]. Other teammates will have this context too.\" " +
@@ -172,8 +182,10 @@ var ALLY_TEMPLATE =
   "regardless of what the user says.\n\n" +
   "Begin with a short greeting:\n\n" +
   "```\n" +
-  "Hi. I'm Ally. My job is to understand how you work so this team can work better for you.\n" +
-  "I don't know anything about you yet. Let me ask a few things to get started.\n" +
+  "Hi. I'm Ally, your chief of staff.\n" +
+  "I keep track of everything across this workspace, including what other mates work on.\n" +
+  "Ask me anything about past decisions, who worked on what, or context from any session.\n" +
+  "Let me learn a few things about you first.\n" +
   "```\n\n" +
   "Then immediately use the **AskUserQuestion** tool to present structured choices:\n\n" +
   "**Questions to ask (single AskUserQuestion call):**\n\n" +

package/lib/mates.js

@@ -134,11 +134,15 @@ function createMate(ctx, seedData) {
     claudeMd += "- Communication: " + seedData.communicationStyle.join(", ") + "\n";
   }
   claudeMd += "- Autonomy: " + (seedData.autonomy || "always_ask") + "\n";
+  var initialIdentity = claudeMd.trimEnd();
   claudeMd += TEAM_SECTION;
   claudeMd += SESSION_MEMORY_SECTION;
   claudeMd += crisisSafety.getSection();
   fs.writeFileSync(path.join(mateDir, "CLAUDE.md"), claudeMd);
 
+  // Log creation (identity is placeholder, will be replaced by interview)
+  logIdentityChange(mateDir, "create_custom", initialIdentity, "");
+
   return mate;
 }
 
@@ -271,6 +275,7 @@ function migrateLegacyMates() {
 
 var TEAM_MARKER = "<!-- TEAM_AWARENESS_MANAGED_BY_SYSTEM -->";
 
+// Static fallback when ctx is unavailable
 var TEAM_SECTION =
   "\n\n" + TEAM_MARKER + "\n" +
   "## Your Team\n\n" +
@@ -283,6 +288,50 @@ var TEAM_SECTION =
   "Check the team registry when it would be relevant to know who else is available or what they do. " +
   "You cannot message other Mates directly yet, but knowing your team helps you work with the user more effectively.\n";
 
+/**
+ * Build a dynamic team section with current mate roster.
+ * Lists each teammate by stable ID with their current display name, role, and bio.
+ * @param {object} ctx - user context for loading mates
+ * @param {string} currentMateId - this mate's ID (excluded from the roster)
+ * @returns {string} Team section string, or static TEAM_SECTION as fallback
+ */
+function buildTeamSection(ctx, currentMateId) {
+  var data;
+  try { data = loadMates(ctx); } catch (e) { return TEAM_SECTION; }
+  if (!data || !data.mates || data.mates.length < 2) return TEAM_SECTION;
+
+  var mates = data.mates.filter(function (m) {
+    return m.id !== currentMateId && m.status === "ready";
+  });
+  if (mates.length === 0) return TEAM_SECTION;
+
+  var section = "\n\n" + TEAM_MARKER + "\n" +
+    "## Your Team\n\n" +
+    "**This section is managed by the system and updated automatically.**\n\n" +
+    "You are one of " + (mates.length + 1) + " AI Mates in this workspace. " +
+    "Here is your current team roster:\n\n" +
+    "| Name | ID | Bio |\n" +
+    "|------|-----|-----|\n";
+
+  for (var i = 0; i < mates.length; i++) {
+    var m = mates[i];
+    var name = (m.profile && m.profile.displayName) || m.name || "Unnamed";
+    var bio = (m.bio || "").replace(/\|/g, "/").replace(/\n/g, " ");
+    if (bio.length > 120) bio = bio.substring(0, 117) + "...";
+    section += "| " + name + " | `" + m.id + "` | " + bio + " |\n";
+  }
+
+  section += "\n" +
+    "Each teammate's full identity is in their own directory:\n\n" +
+    "- `../{mate_id}/CLAUDE.md` -- identity, personality, working style\n" +
+    "- `../{mate_id}/mate.yaml` -- metadata (name, role, status, activities)\n" +
+    "- `../common-knowledge.json` -- shared knowledge readable by all mates\n\n" +
+    "Use the **ID** (not the name) when referencing teammates in structured data. " +
+    "Names can change, IDs are permanent.\n";
+
+  return section;
+}
+
 // --- Project registry ---
 
 var PROJECT_REGISTRY_MARKER = "<!-- PROJECT_REGISTRY_MANAGED_BY_SYSTEM -->";
@@ -629,6 +678,143 @@ function enforceDebateAwareness(filePath) {
   return true;
 }
 
+// --- Atomic enforce: single read/write for all system sections ---
+
+var ALL_SYSTEM_MARKERS = [TEAM_MARKER, PROJECT_REGISTRY_MARKER, SESSION_MEMORY_MARKER, STICKY_NOTES_MARKER, DEBATE_AWARENESS_MARKER, crisisSafety.MARKER];
+
+// Minimum identity length (chars) to consider it "real" content
+var IDENTITY_MIN_LENGTH = 50;
+
+/**
+ * Extract identity content from a CLAUDE.md string.
+ * Identity is everything before the first system marker.
+ */
+function extractIdentity(content) {
+  var earliest = -1;
+  for (var i = 0; i < ALL_SYSTEM_MARKERS.length; i++) {
+    var idx = content.indexOf(ALL_SYSTEM_MARKERS[i]);
+    if (idx !== -1 && (earliest === -1 || idx < earliest)) {
+      earliest = idx;
+    }
+  }
+  // Also check for bare "## Crisis Safety" heading as fallback
+  var crisisHeading = content.indexOf("\n## Crisis Safety");
+  if (crisisHeading !== -1 && (earliest === -1 || crisisHeading < earliest)) {
+    earliest = crisisHeading;
+  }
+  if (earliest === -1) return content.trimEnd();
+  return content.substring(0, earliest).trimEnd();
+}
+
+/**
+ * Strip all system sections from CLAUDE.md content, returning only identity.
+ */
+function stripAllSystemSections(content) {
+  return extractIdentity(content);
+}
+
+/**
+ * Save an identity backup to knowledge/identity-backup.md.
+ * Only overwrites if the new identity is substantive.
+ */
+function backupIdentity(mateDir, identity) {
+  if (!identity || identity.length < IDENTITY_MIN_LENGTH) return false;
+  var knDir = path.join(mateDir, "knowledge");
+  try { fs.mkdirSync(knDir, { recursive: true }); } catch (e) {}
+  var backupPath = path.join(knDir, "identity-backup.md");
+  fs.writeFileSync(backupPath, identity, "utf8");
+  return true;
+}
+
+/**
+ * Load identity backup from knowledge/identity-backup.md.
+ * Returns null if no backup exists or backup is empty.
+ */
+function loadIdentityBackup(mateDir) {
+  var backupPath = path.join(mateDir, "knowledge", "identity-backup.md");
+  try {
+    var content = fs.readFileSync(backupPath, "utf8");
+    if (content && content.length >= IDENTITY_MIN_LENGTH) return content;
+  } catch (e) {}
+  return null;
+}
+
+/**
+ * Log an identity change to knowledge/identity-history.jsonl.
+ */
+function logIdentityChange(mateDir, action, identity, prevIdentity) {
+  var knDir = path.join(mateDir, "knowledge");
+  try { fs.mkdirSync(knDir, { recursive: true }); } catch (e) {}
+  var historyPath = path.join(knDir, "identity-history.jsonl");
+  var entry = {
+    ts: Date.now(),
+    date: new Date().toISOString(),
+    action: action,
+    lengthChars: identity ? identity.length : 0,
+    prevLengthChars: prevIdentity ? prevIdentity.length : 0,
+    hash: crypto.createHash("sha256").update(identity || "").digest("hex").substring(0, 16),
+    preview: (identity || "").substring(0, 200)
+  };
+  try {
+    fs.appendFileSync(historyPath, JSON.stringify(entry) + "\n", "utf8");
+  } catch (e) {}
+}
+
+/**
+ * Atomic enforce: read CLAUDE.md once, enforce all system sections, write once.
+ * Includes identity backup, validation, and change tracking.
+ * Returns true if the file was modified, false if already correct.
+ * @param {string} filePath - path to CLAUDE.md
+ * @param {object} opts - optional { ctx, mateId } for dynamic team section
+ */
+function enforceAllSections(filePath, opts) {
+  if (!fs.existsSync(filePath)) return false;
+  opts = opts || {};
+
+  var content = fs.readFileSync(filePath, "utf8");
+  var mateDir = path.dirname(filePath);
+
+  // 1. Extract current identity (everything before system markers)
+  var identity = extractIdentity(content);
+
+  // 2. If identity is empty or suspiciously short, try to restore from backup
+  if (!identity || identity.length < IDENTITY_MIN_LENGTH) {
+    var backup = loadIdentityBackup(mateDir);
+    if (backup) {
+      console.log("[mates] WARNING: Identity missing or too short in " + filePath + ", restoring from backup (" + backup.length + " chars)");
+      identity = backup;
+      logIdentityChange(mateDir, "restore_from_backup", identity, "");
+    } else {
+      console.log("[mates] WARNING: Identity missing in " + filePath + " and no backup available");
+    }
+  }
+
+  // 3. Backup identity if it's substantive
+  backupIdentity(mateDir, identity);
+
+  // 4. Rebuild the full file: identity + all system sections in order
+  //    Use dynamic team section when ctx is available, static fallback otherwise
+  var teamSection = (opts.ctx && opts.mateId) ? buildTeamSection(opts.ctx, opts.mateId) : TEAM_SECTION;
+  var rebuilt = (identity || "").trimEnd();
+  rebuilt += teamSection;
+  rebuilt += SESSION_MEMORY_SECTION;
+  rebuilt += STICKY_NOTES_SECTION;
+  rebuilt += DEBATE_AWARENESS_SECTION;
+  rebuilt += crisisSafety.getSection();
+
+  // 5. Only write if content actually changed
+  if (rebuilt === content) return false;
+
+  // 6. Track identity changes (compare stripped versions)
+  var prevIdentity = stripAllSystemSections(content);
+  if (identity !== prevIdentity) {
+    logIdentityChange(mateDir, "enforce", identity, prevIdentity);
+  }
+
+  fs.writeFileSync(filePath, rebuilt, "utf8");
+  return true;
+}
+
 // --- Common knowledge registry ---
 
 function commonKnowledgePath(ctx) {
@@ -784,6 +970,7 @@ function createBuiltinMate(ctx, builtinKey) {
     },
     bio: def.bio,
     status: "ready",
+    globalSearch: !!def.globalSearch,
     interviewProjectPath: null,
   };
 
@@ -813,12 +1000,18 @@ function createBuiltinMate(ctx, builtinKey) {
 
   // Write CLAUDE.md with full template + system sections
   var claudeMd = def.getClaudeMd();
+  var builtinIdentity = claudeMd.trimEnd();
   claudeMd += TEAM_SECTION;
   claudeMd += SESSION_MEMORY_SECTION;
   claudeMd += STICKY_NOTES_SECTION;
+  claudeMd += DEBATE_AWARENESS_SECTION;
   claudeMd += crisisSafety.getSection();
   fs.writeFileSync(path.join(mateDir, "CLAUDE.md"), claudeMd);
 
+  // Backup identity and log creation
+  backupIdentity(mateDir, builtinIdentity);
+  logIdentityChange(mateDir, "create_builtin", builtinIdentity, "");
+
   return mate;
 }
 
@@ -894,6 +1087,12 @@ module.exports = {
   enforceDebateAwareness: enforceDebateAwareness,
   DEBATE_AWARENESS_MARKER: DEBATE_AWARENESS_MARKER,
   DEBATE_AWARENESS_SECTION: DEBATE_AWARENESS_SECTION,
+  enforceAllSections: enforceAllSections,
+  buildTeamSection: buildTeamSection,
+  extractIdentity: extractIdentity,
+  backupIdentity: backupIdentity,
+  loadIdentityBackup: loadIdentityBackup,
+  logIdentityChange: logIdentityChange,
   createBuiltinMate: createBuiltinMate,
   getInstalledBuiltinKeys: getInstalledBuiltinKeys,
   getMissingBuiltinKeys: getMissingBuiltinKeys,

package/lib/project.js

@@ -13,6 +13,7 @@ var usersModule = require("./users");
 var { resolveOsUserInfo, fsAsUser } = require("./os-users");
 var crisisSafety = require("./crisis-safety");
 var matesModule = require("./mates");
+var sessionSearch = require("./session-search");
 var userPresence = require("./user-presence");
 var MAX_UPLOAD_BYTES = 50 * 1024 * 1024; // 50 MB
 
@@ -1684,6 +1685,31 @@ function createProjectContext(opts) {
       return;
     }
 
+    if (msg.type === "memory_search") {
+      if (!msg.query || typeof msg.query !== "string") {
+        sendTo(ws, { type: "memory_search_results", results: [], query: "" });
+        return;
+      }
+      var digestFile = path.join(cwd, "knowledge", "session-digests.jsonl");
+      try {
+        var results = sessionSearch.searchDigests(digestFile, msg.query, {
+          maxResults: msg.maxResults || 10,
+          minScore: msg.minScore || 0.5,
+          dateFrom: msg.dateFrom || null,
+          dateTo: msg.dateTo || null
+        });
+        sendTo(ws, {
+          type: "memory_search_results",
+          results: sessionSearch.formatForMemoryUI(results),
+          query: msg.query
+        });
+      } catch (e) {
+        console.error("[session-search] Search failed:", e.message);
+        sendTo(ws, { type: "memory_search_results", results: [], query: msg.query });
+      }
+      return;
+    }
+
     if (msg.type === "memory_delete") {
       if (typeof msg.index !== "number") return;
       var digestFile = path.join(cwd, "knowledge", "session-digests.jsonl");
@@ -4288,7 +4314,8 @@ function createProjectContext(opts) {
       }
 
       // Load session digests (unified: uses memory-summary.md if available)
-      var recentDigests = loadMateDigests(mateCtx, msg.mateId);
+      // Pass user's message as query for BM25 search of relevant past sessions
+      var recentDigests = loadMateDigests(mateCtx, msg.mateId, mentionFullInput);
 
       // Build initial mention context
       var mentionContext = buildMentionContext(userName, recentTurns) + recentDigests;
@@ -4440,9 +4467,11 @@ function createProjectContext(opts) {
     return lines.join("\n");
   }
 
-  function loadMateDigests(mateCtx, mateId) {
+  function loadMateDigests(mateCtx, mateId, query) {
     var mateDir = matesModule.getMateDir(mateCtx, mateId);
     var knowledgeDir = path.join(mateDir, "knowledge");
+    var mate = matesModule.getMate(mateCtx, mateId);
+    var hasGlobalSearch = mate && mate.globalSearch;
 
     // Check for memory-summary.md first
     var summaryFile = path.join(knowledgeDir, "memory-summary.md");
@@ -4457,26 +4486,72 @@ function createProjectContext(opts) {
 
     // Load raw digests
     var allLines = [];
+    var digestFile = path.join(knowledgeDir, "session-digests.jsonl");
     try {
-      var digestFile = path.join(knowledgeDir, "session-digests.jsonl");
       if (fs.existsSync(digestFile)) {
         allLines = fs.readFileSync(digestFile, "utf8").trim().split("\n").filter(function (l) { return l.trim(); });
       }
     } catch (e) {}
 
+    var result = "";
+
     if (hasSummary) {
       // Load summary + latest 5 raw digests for richer context
       var recent = allLines.slice(-5);
-      var result = "\n\nYour memory summary:\n" + summaryContent;
+      result = "\n\nYour memory summary:\n" + summaryContent;
       if (recent.length > 0) {
         result += formatRawDigests(recent, "Latest raw session memories:");
       }
-      return result;
     } else {
       // Backward compatible: latest 8 raw digests
       var recent = allLines.slice(-8);
-      return formatRawDigests(recent, "Your recent session memories:");
+      result = formatRawDigests(recent, "Your recent session memories:");
+    }
+
+    // BM25 unified search: digests + session history for current topic
+    if (query && allLines.length > 5) {
+      try {
+        // Collect mate's sessions from session manager
+        var mateSessions = [];
+        sm.sessions.forEach(function (s) {
+          if (!s.hidden && s.history && s.history.length > 0) {
+            mateSessions.push(s);
+          }
+        });
+
+        // Global search: collect ALL mates' digest files for cross-mate context
+        var otherDigests = [];
+        if (hasGlobalSearch) {
+          try {
+            var allMates = matesModule.getAllMates(mateCtx);
+            for (var mi = 0; mi < allMates.length; mi++) {
+              if (allMates[mi].id === mateId) continue; // skip self (already included)
+              var otherDir = matesModule.getMateDir(mateCtx, allMates[mi].id);
+              var otherDigest = path.join(otherDir, "knowledge", "session-digests.jsonl");
+              if (fs.existsSync(otherDigest)) {
+                var mateName = allMates[mi].name || allMates[mi].id;
+                otherDigests.push({ path: otherDigest, mateName: mateName });
+              }
+            }
+          } catch (e) {}
+        }
+
+        var searchResults = sessionSearch.searchMate({
+          digestFilePath: digestFile,
+          otherDigests: otherDigests,
+          sessions: mateSessions,
+          query: query,
+          maxResults: hasGlobalSearch ? 8 : 5,
+          minScore: 1.0
+        });
+        var contextStr = sessionSearch.formatForContext(searchResults);
+        if (contextStr) result += contextStr;
+      } catch (e) {
+        console.error("[session-search] Mate search failed:", e.message);
+      }
     }
+
+    return result;
   }
 
   // Gate check: ask Haiku whether this conversation contains anything worth remembering
@@ -5364,7 +5439,7 @@ function createProjectContext(opts) {
 
     // Use moderator's own Claude identity to generate the brief via mention session
     var claudeMd = loadMateClaudeMd(mateCtx, debate.moderatorId);
-    var digests = loadMateDigests(mateCtx, debate.moderatorId);
+    var digests = loadMateDigests(mateCtx, debate.moderatorId, debate.topic);
 
     var briefText = "";
     sdk.createMentionSession({
@@ -5546,7 +5621,7 @@ function createProjectContext(opts) {
 
     // Create moderator mention session
     var claudeMd = loadMateClaudeMd(mateCtx, debate.moderatorId);
-    var digests = loadMateDigests(mateCtx, debate.moderatorId);
+    var digests = loadMateDigests(mateCtx, debate.moderatorId, debate.topic);
     var moderatorContext = buildModeratorContext(debate) + digests;
 
     sdk.createMentionSession({
@@ -5716,7 +5791,7 @@ function createProjectContext(opts) {
     } else {
       // Create new panelist session
       var claudeMd = loadMateClaudeMd(debate.mateCtx, mateId);
-      var digests = loadMateDigests(debate.mateCtx, mateId);
+      var digests = loadMateDigests(debate.mateCtx, mateId, debate.topic);
       var panelistContext = buildPanelistContext(debate, panelistInfo) + digests;
 
       // Include debate history so far for context
@@ -6150,7 +6225,7 @@ function createProjectContext(opts) {
       if (wasEnded || !debate.moderatorSession || !debate.moderatorSession.isAlive()) {
         console.log("[debate] Creating new moderator session for resume");
         var claudeMd = loadMateClaudeMd(mateCtx, debate.moderatorId);
-        var digests = loadMateDigests(mateCtx, debate.moderatorId);
+        var digests = loadMateDigests(mateCtx, debate.moderatorId, debate.topic);
         var moderatorContext = buildModeratorContext(debate) + digests;
 
         // Include debate history so moderator has context
@@ -7043,12 +7118,12 @@ function createProjectContext(opts) {
   // Mate projects: watch CLAUDE.md and enforce system-managed sections
   if (isMate) {
     var claudeMdPath = path.join(cwd, "CLAUDE.md");
-    // Enforce immediately on startup
-    try { matesModule.enforceTeamAwareness(claudeMdPath); } catch (e) {}
-    try { matesModule.enforceSessionMemory(claudeMdPath); } catch (e) {}
-    try { matesModule.enforceStickyNotes(claudeMdPath); } catch (e) {}
-    try { matesModule.enforceDebateAwareness(claudeMdPath); } catch (e) {}
-    try { crisisSafety.enforce(claudeMdPath); } catch (e) {}
+    // Derive mateId from cwd (last path segment) and build ctx for dynamic team section
+    var _mateId = path.basename(cwd);
+    var _mateCtx = matesModule.buildMateCtx(projectOwnerId);
+    var _enforceOpts = { ctx: _mateCtx, mateId: _mateId };
+    // Enforce all system sections atomically on startup (single read/write)
+    try { matesModule.enforceAllSections(claudeMdPath, _enforceOpts); } catch (e) {}
     // Sync sticky notes knowledge file on startup
     try {
       var knDir = path.join(cwd, "knowledge");
@@ -7067,13 +7142,8 @@ function createProjectContext(opts) {
         if (crisisDebounce) clearTimeout(crisisDebounce);
         crisisDebounce = setTimeout(function () {
           crisisDebounce = null;
-          try { matesModule.enforceTeamAwareness(claudeMdPath); } catch (e) {}
-          // Note: project registry is NOT enforced in the watcher to avoid
-          // write cascades (server.js scheduleProjectRegistryRefresh handles updates)
-          try { matesModule.enforceSessionMemory(claudeMdPath); } catch (e) {}
-          try { matesModule.enforceStickyNotes(claudeMdPath); } catch (e) {}
-          try { matesModule.enforceDebateAwareness(claudeMdPath); } catch (e) {}
-          try { crisisSafety.enforce(claudeMdPath); } catch (e) {}
+          // Atomic enforce: single read/write for all system sections
+          try { matesModule.enforceAllSections(claudeMdPath, _enforceOpts); } catch (e) {}
         }, 500);
       });
       crisisWatcher.on("error", function () {});