ei-tui 0.1.7 → 0.1.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ei-tui",
-  "version": "0.1.7",
+  "version": "0.1.8",
   "author": "Flare576",
   "repository": {
     "type": "git",

package/src/core/orchestrators/ceremony.ts

@@ -1,4 +1,4 @@
-import { LLMRequestType, LLMPriority, LLMNextStep, MESSAGE_MIN_COUNT, MESSAGE_MAX_AGE_DAYS, type CeremonyConfig, type PersonaTopic, type Topic, type Message } from "../types.js";
+import { LLMRequestType, LLMPriority, LLMNextStep, MESSAGE_MIN_COUNT, MESSAGE_MAX_AGE_DAYS, type CeremonyConfig, type PersonaTopic, type Topic, type Message, type DataItemBase } from "../types.js";
 import type { StateManager } from "../state-manager.js";
 import { applyDecayToValue } from "../utils/index.js";
 import {
@@ -224,6 +224,9 @@ export function handleCeremonyProgress(state: StateManager): void {
   
   // Human ceremony: decay topics + people
   runHumanCeremony(state);
+
+  // Dedup phase: log near-duplicate human entities (dry-run only, no mutations)
+  runDedupPhase(state);
   
   // Expire phase: queue LLM calls for each active persona
   // handlePersonaExpire already chains to Explore → DescriptionCheck
@@ -498,3 +501,76 @@ export function runHumanCeremony(state: StateManager): void {
     console.log(`[ceremony:human] Low exposure items: ${lowExposureTopics.length} topics, ${lowExposurePeople.length} people`);
   }
 }
+
+// =============================================================================
+// DEDUP PHASE (synchronous, dry-run — logs candidates, no mutations)
+// =============================================================================
+
+const DEDUP_DEFAULT_THRESHOLD = 0.85;
+
+type DedupableItem = DataItemBase & { relationship?: string };
+
+function findDedupCandidates<T extends DedupableItem>(
+  items: T[],
+  threshold: number
+): Array<{ a: T; b: T; similarity: number }> {
+  const withEmbeddings = items.filter(item =>
+    item.embedding && item.embedding.length > 0 &&
+    item.relationship !== "Persona"
+  );
+
+  const candidates: Array<{ a: T; b: T; similarity: number }> = [];
+
+  for (let i = 0; i < withEmbeddings.length; i++) {
+    for (let j = i + 1; j < withEmbeddings.length; j++) {
+      const a = withEmbeddings[i];
+      const b = withEmbeddings[j];
+      const dot = a.embedding!.reduce((sum, v, k) => sum + v * b.embedding![k], 0);
+      const normA = Math.sqrt(a.embedding!.reduce((sum, v) => sum + v * v, 0));
+      const normB = Math.sqrt(b.embedding!.reduce((sum, v) => sum + v * v, 0));
+      const similarity = normA && normB ? dot / (normA * normB) : 0;
+
+      if (similarity >= threshold) {
+        candidates.push({ a, b, similarity });
+      }
+    }
+  }
+
+  return candidates.sort((x, y) => y.similarity - x.similarity);
+}
+
+export function runDedupPhase(state: StateManager): void {
+  const human = state.getHuman();
+  const threshold = human.settings?.ceremony?.dedup_threshold ?? DEDUP_DEFAULT_THRESHOLD;
+
+  console.log(`[ceremony:dedup] Running dry-run dedup (threshold: ${threshold})`);
+
+  const types: Array<{ label: string; items: DedupableItem[] }> = [
+    { label: "facts",  items: human.facts },
+    { label: "traits", items: human.traits },
+    { label: "topics", items: human.topics },
+    { label: "people", items: human.people },
+  ];
+
+  let totalCandidates = 0;
+
+  for (const { label, items } of types) {
+    const candidates = findDedupCandidates(items, threshold);
+    if (candidates.length === 0) {
+      console.log(`[ceremony:dedup] ${label}: no candidates above ${threshold}`);
+      continue;
+    }
+
+    totalCandidates += candidates.length;
+    console.log(`[ceremony:dedup] ${label}: ${candidates.length} candidate pair(s)`);
+    for (const { a, b, similarity } of candidates) {
+      console.log(
+        `[ceremony:dedup]   ${(similarity * 100).toFixed(1)}% "${a.name}" ↔ "${b.name}"` +
+        (a.description ? `\n[ceremony:dedup]     A: ${a.description.slice(0, 80)}` : "") +
+        (b.description ? `\n[ceremony:dedup]     B: ${b.description.slice(0, 80)}` : "")
+      );
+    }
+  }
+
+  console.log(`[ceremony:dedup] Done. ${totalCandidates} total candidate pair(s) found.`);
+}

package/src/core/processor.ts

@@ -1142,8 +1142,14 @@ export class Processor {
         }
       }
 
-      // Fallback: return all items (caller may apply its own limit)
-      return items;
+      // Fallback: return top items by recency — never return unbounded list
+      return [...items]
+        .sort((a, b) => {
+          const aTime = (a as { last_updated?: string }).last_updated ?? "";
+          const bTime = (b as { last_updated?: string }).last_updated ?? "";
+          return bTime.localeCompare(aTime);
+        })
+        .slice(0, limit);
     };
     const selectRelevantQuotes = async (quotes: Quote[]): Promise<Quote[]> => {
       if (quotes.length === 0) return [];

package/src/core/types.ts

@@ -188,6 +188,7 @@ export interface CeremonyConfig {
   last_ceremony?: string;  // ISO timestamp
   decay_rate?: number;  // Default: 0.1
   explore_threshold?: number;  // Default: 3
+  dedup_threshold?: number;  // Cosine similarity threshold for dedup candidates. Default: 0.85
 }
 
 export interface HumanSettings {

package/src/prompts/human/item-update.ts

@@ -2,6 +2,8 @@ import type { ItemUpdatePromptData, PromptOutput } from "./types.js";
 import type { DataItemBase } from "../../core/types.js";
 import { formatMessagesAsPlaceholders } from "../message-utils.js";
 
+const DESCRIPTION_MAX_CHARS = 500;
+
 function formatExistingItem(item: DataItemBase): string {
   return JSON.stringify({
     name: item.name,
@@ -97,9 +99,28 @@ Examples: "Name Unknown" -> "Robert Jordan", "User was married in the Summer" ->
   `;
 
   const defaultDescriptionSection = `
-This free-text field should be used to capture interesting details or references that the Human or Persona use while discussing this data point. Personas should be able to show topical recall, make references to the topic or event, or in other ways "Remember" details about it.
+A concise, evergreen summary of what is currently known about this ${typeLabel}. Personas use this to recall context and make meaningful references.
+
+## CRITICAL: Synthesize, don't accumulate
+
+Every update must **rewrite** the description as a current-state summary. Never append to it.
+
+**Good description**: "Active project to improve test coverage. Settled on Vitest + E2E harness. Currently focused on pipeline integration and extraction logic coverage."
+**Bad description**: "User asked Sisyphus to create a ticket... Later: pruned overengineered framework... Most recent session: added PR checks..."
+
+The description should:
+- Capture what is true NOW — the current state, decisions made, where things stand
+- Include details a persona would use to show genuine recall ("Oh right, you were working on the pipeline tests")
+- Be useful to a persona meeting this human for the first time
+- Read as a brief summary paragraph, not a session log
+
+The description should NOT:
+- Append "Most recent:", "Latest:", "Current session:", or any temporal marker
+- Accumulate a running history of every conversation that touched this ${typeLabel}
+- Reference specific ticket numbers, commit hashes, or PR numbers unless essential to meaning
+- Exceed 3-4 sentences under any circumstances
 
-**ABSOLUTELY VITAL INSTRUCTION**: Do **NOT** embelish these details - each Persona will use their own voice during interactions with the User - we need to capture EXACTLY what was said and how, or referring back to it won't have meaning.
+**ABSOLUTELY VITAL**: Do **NOT** embellish — personas use their own voice. Capture what is true, not a log of how you got here.
   `;
 
   const descriptionSection =
@@ -332,3 +353,12 @@ If no changes are needed, respond with: \`{}\``;
 
   return { system, user };
 }
+
+/**
+ * Truncate a description to DESCRIPTION_MAX_CHARS for use in prompts.
+ * The stored value is unchanged — this only affects what goes into the LLM context.
+ */
+export function truncateDescription(description: string): string {
+  if (description.length <= DESCRIPTION_MAX_CHARS) return description;
+  return description.slice(0, DESCRIPTION_MAX_CHARS) + "…";
+}

package/src/prompts/response/sections.ts

@@ -5,6 +5,7 @@
 
 import type { Trait, Quote, PersonaTopic } from "../../core/types.js";
 import type { ResponsePromptData } from "./types.js";
+import { truncateDescription } from "../human/item-update.js";
 
 // =============================================================================
 // IDENTITY SECTION
@@ -64,10 +65,10 @@ export function buildGuidelinesSection(personaName: string): string {
 export function buildTraitsSection(traits: Trait[], header: string): string {
   if (traits.length === 0) return "";
   
-  const sorted = [...traits].sort((a, b) => (b.strength ?? 0.5) - (a.strength ?? 0.5));
+  const sorted = [...traits].sort((a, b) => (b.strength ?? 0.5) - (a.strength ?? 0.5)).slice(0, 15);
   const formatted = sorted.map(t => {
     const strength = t.strength !== undefined ? ` (${Math.round(t.strength * 100)}%)` : "";
-    return `- **${t.name}**${strength}: ${t.description}`;
+    return `- **${t.name}**${strength}: ${truncateDescription(t.description)}`;
   }).join("\n");
   
   return `## ${header}
@@ -92,6 +93,8 @@ export function buildTopicsSection(topics: PersonaTopic[], header: string): stri
   const sorted = [...topics]
     .map(t => ({ topic: t, delta: t.exposure_desired - t.exposure_current }))
     .sort((a, b) => b.delta - a.delta)
+    .slice(0, 15)
+    .sort((a, b) => b.delta - a.delta)
     .map(x => x.topic);
   
   const formatted = sorted.map(t => {
@@ -137,7 +140,8 @@ export function buildHumanSection(human: ResponsePromptData["human"]): string {
   // Facts
   if (human.facts.length > 0) {
     const facts = human.facts
-      .map(f => `- ${f.name}: ${f.description}`)
+      .slice(0, 15)
+      .map(f => `- ${f.name}: ${truncateDescription(f.description)}`)
       .join("\n");
     if (facts) sections.push(`### Key Facts\n${facts}`);
   }
@@ -145,7 +149,8 @@ export function buildHumanSection(human: ResponsePromptData["human"]): string {
   // Traits
   if (human.traits.length > 0) {
     const traits = human.traits
-      .map(t => `- **${t.name}**: ${t.description}`)
+      .slice(0, 15)
+      .map(t => `- **${t.name}**: ${truncateDescription(t.description)}`)
       .join("\n");
     sections.push(`### Personality\n${traits}`);
   }
@@ -155,10 +160,10 @@ export function buildHumanSection(human: ResponsePromptData["human"]): string {
   if (activeTopics.length > 0) {
     const topics = activeTopics
       .sort((a, b) => b.exposure_current - a.exposure_current)
-      .slice(0, 10)
+      .slice(0, 15)
       .map(t => {
         const sentiment = t.sentiment > 0.3 ? "(enjoys)" : t.sentiment < -0.3 ? "(dislikes)" : "";
-        return `- **${t.name}** ${sentiment}: ${t.description}`;
+        return `- **${t.name}** ${sentiment}: ${truncateDescription(t.description)}`;
       })
       .join("\n");
     sections.push(`### Current Interests\n${topics}`);
@@ -168,8 +173,8 @@ export function buildHumanSection(human: ResponsePromptData["human"]): string {
   if (human.people.length > 0) {
     const people = human.people
       .sort((a, b) => b.exposure_current - a.exposure_current)
-      .slice(0, 10)
-      .map(p => `- **${p.name}** (${p.relationship}): ${p.description}`)
+      .slice(0, 15)
+      .map(p => `- **${p.name}** (${p.relationship}): ${truncateDescription(p.description)}`)
       .join("\n");
     sections.push(`### People in Their Life\n${people}`);
   }

package/src/storage/compress.ts

@@ -0,0 +1,82 @@
+/**
+ * Gzip compression utilities for storage.
+ *
+ * Uses the native CompressionStream/DecompressionStream API, available in
+ * both modern browsers and Bun (no external dependencies).
+ *
+ * Compressed output is base64-encoded so it can be stored as a plain string
+ * (LocalStorage, remote API body, etc.).
+ *
+ * FileStorage deliberately does NOT use these — uncompressed JSON on disk
+ * stays human-readable and debuggable.
+ */
+
+export async function compress(json: string): Promise<string> {
+  const encoder = new TextEncoder();
+  const input = encoder.encode(json);
+
+  const cs = new CompressionStream("gzip");
+  const writer = cs.writable.getWriter();
+  writer.write(input);
+  writer.close();
+
+  const chunks: Uint8Array[] = [];
+  const reader = cs.readable.getReader();
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    chunks.push(value);
+  }
+
+  const totalLength = chunks.reduce((sum, c) => sum + c.length, 0);
+  const merged = new Uint8Array(totalLength);
+  let offset = 0;
+  for (const chunk of chunks) {
+    merged.set(chunk, offset);
+    offset += chunk.length;
+  }
+
+  // btoa needs a binary string — convert byte-by-byte
+  let binary = "";
+  for (let i = 0; i < merged.length; i++) {
+    binary += String.fromCharCode(merged[i]);
+  }
+  return btoa(binary);
+}
+
+export async function decompress(b64: string): Promise<string> {
+  const binary = atob(b64);
+  const bytes = new Uint8Array(binary.length);
+  for (let i = 0; i < binary.length; i++) {
+    bytes[i] = binary.charCodeAt(i);
+  }
+
+  const ds = new DecompressionStream("gzip");
+  const writer = ds.writable.getWriter();
+  writer.write(bytes);
+  writer.close();
+
+  const chunks: Uint8Array[] = [];
+  const reader = ds.readable.getReader();
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    chunks.push(value);
+  }
+
+  const totalLength = chunks.reduce((sum, c) => sum + c.length, 0);
+  const merged = new Uint8Array(totalLength);
+  let offset = 0;
+  for (const chunk of chunks) {
+    merged.set(chunk, offset);
+    offset += chunk.length;
+  }
+
+  return new TextDecoder().decode(merged);
+}
+
+/** Returns true if the string looks like a base64-encoded gzip payload. */
+export function isCompressed(value: string): boolean {
+  // gzip magic bytes are 0x1f 0x8b — base64-encoded that starts with "H4sI"
+  return value.startsWith("H4sI");
+}

package/src/storage/index.ts

@@ -3,3 +3,4 @@ export { LocalStorage } from "./local.js";
 export { remoteSync, RemoteSync, type RemoteSyncCredentials, type RemoteTimestamp, type SyncResult, type FetchResult } from "./remote.js";
 export { encrypt, decrypt, generateUserId, type CryptoCredentials, type EncryptedPayload } from "./crypto.js";
 export { yoloMerge } from "./merge.js";
+export { compress, decompress, isCompressed } from "./compress.js";

package/src/storage/local.ts

@@ -1,5 +1,6 @@
 import type { StorageState } from "../core/types.js";
 import type { Storage } from "./interface.js";
+import { compress, decompress, isCompressed } from "./compress.js";
 
 const STATE_KEY = "ei_state";
 const BACKUP_KEY = "ei_state_backup";
@@ -19,7 +20,9 @@ export class LocalStorage implements Storage {
   async save(state: StorageState): Promise<void> {
     state.timestamp = new Date().toISOString();
     try {
-      globalThis.localStorage.setItem(STATE_KEY, JSON.stringify(state));
+      const json = JSON.stringify(state);
+      const payload = await compress(json);
+      globalThis.localStorage.setItem(STATE_KEY, payload);
     } catch (e) {
       if (this.isQuotaError(e)) {
         throw new Error("STORAGE_SAVE_FAILED: localStorage quota exceeded");
@@ -32,7 +35,8 @@ export class LocalStorage implements Storage {
     const current = globalThis.localStorage?.getItem(STATE_KEY);
     if (current) {
       try {
-        return JSON.parse(current) as StorageState;
+        const json = isCompressed(current) ? await decompress(current) : current;
+        return JSON.parse(json) as StorageState;
       } catch {
         return null;
       }
@@ -62,7 +66,8 @@ export class LocalStorage implements Storage {
     const backup = globalThis.localStorage?.getItem(BACKUP_KEY);
     if (backup) {
       try {
-        return JSON.parse(backup) as StorageState;
+        const json = isCompressed(backup) ? await decompress(backup) : backup;
+        return JSON.parse(json) as StorageState;
       } catch {
         return null;
       }

package/src/storage/remote.ts

@@ -1,5 +1,6 @@
 import type { StorageState } from "../core/types.js";
 import { encrypt, decrypt, generateUserId, type CryptoCredentials, type EncryptedPayload } from "./crypto.js";
+import { compress, decompress, isCompressed } from "./compress.js";
 
 const API_BASE = "https://flare576.com/ei/api";
 
@@ -71,7 +72,8 @@ export class RemoteSync {
 
     try {
       const stateJson = JSON.stringify(state);
-      const encrypted = await encrypt(stateJson, this.credentials);
+      const compressed = await compress(stateJson);
+      const encrypted = await encrypt(compressed, this.credentials);
       const encryptedJson = JSON.stringify(encrypted);
 
       const headers: Record<string, string> = { "Content-Type": "application/json" };
@@ -124,7 +126,9 @@ export class RemoteSync {
 
       const body = await response.json();
       const encrypted: EncryptedPayload = JSON.parse(body.data);
-      const decryptedJson = await decrypt(encrypted, this.credentials);
+      const decryptedPayload = await decrypt(encrypted, this.credentials);
+      // Support both compressed (new) and uncompressed (legacy) remote state
+      const decryptedJson = isCompressed(decryptedPayload) ? await decompress(decryptedPayload) : decryptedPayload;
       const state = JSON.parse(decryptedJson) as StorageState;
       // Capture etag for concurrency protection
       this.lastEtag = response.headers.get("ETag");