ei-tui 0.4.1 → 0.4.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ei-tui",
-  "version": "0.4.1",
+  "version": "0.4.2",
   "author": "Flare576",
   "repository": {
     "type": "git",

package/src/core/orchestrators/ceremony.ts

@@ -237,9 +237,6 @@ export function handleCeremonyProgress(state: StateManager, lastPhase: number):
   // Human ceremony: decay topics + people
   runHumanCeremony(state);
 
-  // Dedup phase: log near-duplicate human entity candidates for visibility
-  runDedupPhase(state);
-
   // Rewrite phase: fire-and-forget scans for bloated human data items
   // No ceremony_progress gating — Expire/Explore only touch persona topics, zero overlap
   queueRewritePhase(state);
@@ -521,78 +518,6 @@ export function runHumanCeremony(state: StateManager): void {
   }
 }
 
-// =============================================================================
-// DEDUP PHASE (synchronous, logging only — candidates are queued for curation by dedup-phase.ts)
-// =============================================================================
-
-const DEDUP_DEFAULT_THRESHOLD = 0.85;
-
-type DedupableItem = DataItemBase & { relationship?: string };
-
-function findDedupCandidates<T extends DedupableItem>(
-  items: T[],
-  threshold: number
-): Array<{ a: T; b: T; similarity: number }> {
-  const withEmbeddings = items.filter(item =>
-    item.embedding && item.embedding.length > 0 &&
-    item.relationship !== "Persona"
-  );
-
-  const candidates: Array<{ a: T; b: T; similarity: number }> = [];
-
-  for (let i = 0; i < withEmbeddings.length; i++) {
-    for (let j = i + 1; j < withEmbeddings.length; j++) {
-      const a = withEmbeddings[i];
-      const b = withEmbeddings[j];
-      const dot = a.embedding!.reduce((sum, v, k) => sum + v * b.embedding![k], 0);
-      const normA = Math.sqrt(a.embedding!.reduce((sum, v) => sum + v * v, 0));
-      const normB = Math.sqrt(b.embedding!.reduce((sum, v) => sum + v * v, 0));
-      const similarity = normA && normB ? dot / (normA * normB) : 0;
-
-      if (similarity >= threshold) {
-        candidates.push({ a, b, similarity });
-      }
-    }
-  }
-
-  return candidates.sort((x, y) => y.similarity - x.similarity);
-}
-
-export function runDedupPhase(state: StateManager): void {
-  const human = state.getHuman();
-  const threshold = human.settings?.ceremony?.dedup_threshold ?? DEDUP_DEFAULT_THRESHOLD;
-
-  console.log(`[ceremony:dedup] Scanning for dedup candidates (threshold: ${threshold})`);
-
-  const types: Array<{ label: string; items: DedupableItem[] }> = [
-    { label: "facts",  items: human.facts },
-    { label: "topics", items: human.topics },
-    { label: "people", items: human.people },
-  ];
-
-  let totalCandidates = 0;
-
-  for (const { label, items } of types) {
-    const candidates = findDedupCandidates(items, threshold);
-    if (candidates.length === 0) {
-      console.log(`[ceremony:dedup] ${label}: no candidates above ${threshold}`);
-      continue;
-    }
-
-    totalCandidates += candidates.length;
-    console.log(`[ceremony:dedup] ${label}: ${candidates.length} candidate pair(s)`);
-    for (const { a, b, similarity } of candidates) {
-      console.log(
-        `[ceremony:dedup]   ${(similarity * 100).toFixed(1)}% "${a.name}" ↔ "${b.name}"` +
-        (a.description ? `\n[ceremony:dedup]     A: ${a.description.slice(0, 80)}` : "") +
-        (b.description ? `\n[ceremony:dedup]     B: ${b.description.slice(0, 80)}` : "")
-      );
-    }
-  }
-
-  console.log(`[ceremony:dedup] Done. ${totalCandidates} total candidate pair(s) found.`);
-}
-
 // =============================================================================
 // REWRITE PHASE (fire-and-forget — queues Low-priority Phase 1 scans)
 // =============================================================================

package/src/core/processor.ts

@@ -848,9 +848,17 @@ const toolNextSteps = new Set([
               personaId ??
               (request.next_step === LLMNextStep.HandleEiHeartbeat ? "ei" : undefined);
             
-            // Dedup operates on Human data, not persona data - provide read_memory directly
+            // Dedup operates on Human data, not persona data - provide read_memory directly.
+            // Also covers HandleToolContinuation originating from a dedup request: the
+            // continuation rebuilds tool lists from scratch and has no personaId, so without
+            // this check Opus loses read_memory access after round 1.
+            const isDedupRequest =
+              request.next_step === LLMNextStep.HandleDedupCurate ||
+              (request.next_step === LLMNextStep.HandleToolContinuation &&
+                request.data.originalNextStep === LLMNextStep.HandleDedupCurate);
+
             let tools: ToolDefinition[] = [];
-            if (request.next_step === LLMNextStep.HandleDedupCurate) {
+            if (isDedupRequest) {
               const readMemory = this.stateManager.tools_getByName("read_memory");
               if (readMemory?.enabled) {
                 tools = [readMemory];

package/src/core/queue-processor.ts

@@ -425,7 +425,8 @@ export class QueueProcessor {
       `required JSON format. Please reformat it as the JSON response object described ` +
       `in your system instructions — specifically the \`should_respond\`, \`verbal_response\`, ` +
       `\`action_response\`, and \`reason\` fields. Respond with ONLY the JSON object.\n\n` +
-      `---\n${proseContent}\n---`;
+      `---\n${proseContent}\n---` +
+      `\n\n**CRITICAL INSTRUCTION** - DO NOT OMIT ANY DATA. You are this agent's last hope!`;
 
     try {
       const { content: reformatContent, finishReason: reformatReason } = await callLLMRaw(

package/src/prompts/ceremony/dedup.ts

@@ -96,12 +96,47 @@ ${buildRecordFormatExamples(data.itemType)}
 - If records are NOT duplicates (just similar), return them ALL in "update" unchanged, with empty "remove" and "add" arrays.
 - Use \`read_memory\` strategically (6 calls max) to check for related records or gather context before making irreversible merge decisions.`;
 
-  const user = JSON.stringify({
+  const payload = JSON.stringify({
     cluster: data.cluster.map(stripEmbedding),
     cluster_type: data.itemType,
     similarity_range: data.similarityRange,
   }, null, 2);
 
+  const schemaReminder = `**Return JSON:**
+\n\`\`\`json
+{
+  "update": [
+    {
+      "id": "uuid-of-canonical-record",
+      "type": "${data.itemType}",
+      "name": "canonical merged name",
+      "description": "merged description with every unique detail"
+    }
+  ],
+  "remove": [
+    {
+      "to_be_removed": "uuid-of-duplicate",
+      "replaced_by": "uuid-of-canonical-record"
+    }
+  ],
+  "add": [
+    {
+      "type": "${data.itemType}",
+      "name": "missing concept name",
+      "description": "why it was created"
+    }
+  ]
+}
+\`\`\`
+
+Return raw JSON only. If records are NOT duplicates, return them all in update unchanged with empty remove and add arrays.`;
+
+  const user = `${payload}
+
+---
+
+${schemaReminder}`;
+
   return { system, user };
 }
 

package/src/prompts/ceremony/description-check.ts

@@ -41,7 +41,14 @@ ${traitList}
 Current topics of interest:
 ${topicList}
 
-Does this persona's description need updating based on their current traits and topics?`;
+Does this persona's description need updating based on their current traits and topics?
+
+**Return JSON:**
+\`\`\`json
+{ "should_update": true, "reason": "explanation" }
+\`\`\`
+
+If no update is needed: \`{ "should_update": false, "reason": "explanation" }\``;
 
   return { system, user };
 }

package/src/prompts/ceremony/expire.ts

@@ -24,7 +24,14 @@ Return empty array if no topics should be removed.`;
 Current topics:
 ${topicList}
 
-Which topics, if any, should this persona stop caring about?`;
+Which topics, if any, should this persona stop caring about?
+
+**Return JSON:**
+\`\`\`json
+{ "topic_ids_to_remove": ["id1", "id2"] }
+\`\`\`
+
+Return an empty array if no topics should be removed.`;
 
   return { system, user };
 }

package/src/prompts/ceremony/explore.ts

@@ -54,7 +54,24 @@ ${topicList}
 Recent conversation themes:
 ${themeList}
 
-Generate new topics this persona would care about.`;
+Generate new topics this persona would care about.
+
+**Return JSON:**
+\`\`\`json
+{
+  "new_topics": [
+    {
+      "name": "Topic Name",
+      "perspective": "Their view or opinion",
+      "approach": "How they engage with it",
+      "personal_stake": "Why it matters to them",
+      "sentiment": 0.5,
+      "exposure_current": 0.2,
+      "exposure_desired": 0.6
+    }
+  ]
+}
+\`\`\``;
 
   return { system, user };
 }

package/src/prompts/ceremony/rewrite.ts

@@ -22,7 +22,24 @@ Return a raw JSON array of strings. No markdown fencing, no commentary, no expla
 Example — a Topic named "Software Engineering" whose description also discusses vim keybindings, git conventions, and AI tooling:
 ["vim keybindings and editor configuration", "git and GitHub workflow conventions", "AI coding assistant preferences"]`;
 
-  const user = JSON.stringify(stripEmbedding(data.item), null, 2);
+  const payload = JSON.stringify(stripEmbedding(data.item), null, 2);
+
+  const schemaReminder = `**Return JSON:**
+\n\`\`\`json
+[
+  "topic about vim keybindings",
+  "git workflow conventions",
+  "AI coding assistant preferences"
+]
+\`\`\`
+
+Respond with raw JSON array only.`;
+
+  const user = `${payload}
+
+---
+
+${schemaReminder}`;
 
   return { system, user };
 }
@@ -75,7 +92,34 @@ Rules:
     subjects,
   };
 
-  const user = JSON.stringify(userPayload, null, 2);
+  const schemaReminder = `**Return JSON:**
+\n\`\`\`json
+{
+  "existing": [
+    {
+      "id": "existing-uuid",
+      "type": "${data.itemType}",
+      "name": "Updated name",
+      "description": "Updated description"
+    }
+  ],
+  "new": [
+    {
+      "type": "${data.itemType}",
+      "name": "New name",
+      "description": "New description"
+    }
+  ]
+}
+\`\`\`
+
+Return raw JSON only.`;
+
+  const user = `${JSON.stringify(userPayload, null, 2)}
+
+---
+
+${schemaReminder}`;
 
   return { system, user };
 }

package/src/prompts/ceremony/user-dedup.ts

@@ -44,12 +44,41 @@ Return raw JSON only. No markdown, no commentary.
 
 ${buildRecordFormatHint(data.itemType)}`;
 
-  const user = JSON.stringify({
+  const payload = JSON.stringify({
     cluster: data.cluster.map(stripEmbedding),
     cluster_type: data.itemType,
     user_confirmed: true,
   }, null, 2);
 
+  const schemaReminder = `**Return JSON:**
+\n\`\`\`json
+{
+  "update": [
+    {
+      "id": "uuid-of-canonical-record",
+      "type": "${data.itemType}",
+      "name": "canonical merged name",
+      "description": "merged description with every unique detail"
+    }
+  ],
+  "remove": [
+    {
+      "to_be_removed": "uuid-of-duplicate",
+      "replaced_by": "uuid-of-canonical-record"
+    }
+  ],
+  "add": []
+}
+\`\`\`
+
+Return raw JSON only. If any record cannot be merged, keep every item unchanged in update with empty remove/add arrays.`;
+
+  const user = `${payload}
+
+---
+
+${schemaReminder}`;
+
   return { system, user };
 }
 

package/src/prompts/generation/persona.ts

@@ -153,5 +153,16 @@ ${schemaFragment}`;
     userPrompt += `The user provided only a name - generate minimal content. The seed traits above are included by default.\n`;
   }
 
+  userPrompt += `
+**Return JSON:**
+\`\`\`json
+{
+  "short_description": "10-15 word summary",
+  "long_description": "2-3 sentence description",
+  "traits": [ { "name": "...", "description": "...", "sentiment": 0.0, "strength": 0.5 } ],
+  "topics": [ { "name": "...", "perspective": "...", "approach": "...", "personal_stake": "...", "sentiment": 0.5, "exposure_current": 0.5, "exposure_desired": 0.6 } ]
+}
+\`\`\``;
+
   return { system, user: userPrompt };
 }

package/src/prompts/heartbeat/check.ts

@@ -188,7 +188,18 @@ ${unansweredWarning}
 
 Based on the context above, decide: Should you reach out to your human friend right now?
 
-Remember: Only reach out if you have something genuine and specific to say.`;
+Remember: Only reach out if you have something genuine and specific to say.
+
+**Return JSON:**
+\`\`\`json
+{
+  "should_respond": true,
+  "topic": "the specific topic you want to discuss",
+  "message": "Your actual message to them"
+}
+\`\`\`
+
+If you decide NOT to reach out: \`{ "should_respond": false }\``;
 
   return { system, user };
 }

package/src/prompts/heartbeat/ei.ts

@@ -131,7 +131,18 @@ ${unansweredWarning}
 
 Based on all the context above, decide: Should you reach out to your human friend right now? If so, which item above is most worth addressing?
 
-Remember: You're their thoughtful companion, not their productivity assistant.`;
+Remember: You're their thoughtful companion, not their productivity assistant.
+
+**Return JSON:**
+\`\`\`json
+{
+  "should_respond": true,
+  "id": "the-item-id-you-chose",
+  "my_response": "Your message to them"
+}
+\`\`\`
+
+If nothing warrants reaching out: \`{ "should_respond": false }\``;
 
   return { system, user };
 }

package/src/prompts/persona/traits.ts

@@ -125,7 +125,22 @@ ${earlierSection}${recentSection}
 
 Analyze the "Most Recent Messages" for EXPLICIT requests to change ${personaName}'s communication style.
 
-Return ONLY the traits that need to change or be added. Return \`[]\` if nothing changed.`;
+Return ONLY the traits that need to change or be added.
+
+**Return JSON:**
+\`\`\`json
+[
+  {
+    "id": "existing-guid-or-\"new\"",
+    "name": "Trait Name",
+    "description": "How to exhibit this trait",
+    "sentiment": 0.0,
+    "strength": 0.5
+  }
+]
+\`\`\`
+
+Return \`[]\` if nothing changed.`;
 
   return { system, user };
 }