ei-tui 0.1.3

package/src/cli/retrieval.ts

@@ -0,0 +1,269 @@
+import type { StorageState, Quote, Fact, Trait, Person, Topic } from "../core/types";
+import { join } from "path";
+import { readFile } from "fs/promises";
+import { getEmbeddingService, findTopK } from "../core/embedding-service";
+
+const STATE_FILE = "state.json";
+const BACKUP_FILE = "state.backup.json";
+const EMBEDDING_MIN_SIMILARITY = 0.3;
+
+export function getDataPath(): string {
+  if (process.env.EI_DATA_PATH) {
+    return process.env.EI_DATA_PATH;
+  }
+  const xdgData = process.env.XDG_DATA_HOME || join(process.env.HOME || "~", ".local", "share");
+  return join(xdgData, "ei");
+}
+
+export async function loadLatestState(): Promise<StorageState | null> {
+  const dataPath = getDataPath();
+  for (const file of [STATE_FILE, BACKUP_FILE]) {
+    try {
+      const text = await readFile(join(dataPath, file), "utf-8");
+      if (text) return JSON.parse(text) as StorageState;
+    } catch {
+      continue;
+    }
+  }
+    return null;
+}
+
+export async function retrieve<T extends { id: string; embedding?: number[] }>(
+  items: T[],
+  query: string,
+  limit: number = 10
+): Promise<T[]> {
+  if (items.length === 0 || !query) {
+    return [];
+  }
+
+  const embeddingService = getEmbeddingService();
+  const queryVector = await embeddingService.embed(query);
+
+  const results = findTopK(queryVector, items, limit);
+
+  return results
+    .filter(({ similarity }) => similarity >= EMBEDDING_MIN_SIMILARITY)
+    .map(({ item }) => item);
+}
+
+export interface LinkedItem {
+  id: string;
+  name: string;
+  type: string;
+}
+export interface QuoteResult {
+  id: string;
+  text: string;
+  speaker: string;
+  timestamp: string;
+  linked_items: LinkedItem[];
+}
+
+export interface FactResult {
+  id: string;
+  name: string;
+  description: string;
+  sentiment: number;
+  validated: string;
+}
+
+export interface TraitResult {
+  id: string;
+  name: string;
+  description: string;
+  strength: number;
+  sentiment: number;
+}
+
+export interface PersonResult {
+  id: string;
+  name: string;
+  description: string;
+  relationship: string;
+  sentiment: number;
+}
+
+export interface TopicResult {
+  id: string;
+  name: string;
+  description: string;
+  category?: string;
+  sentiment: number;
+}
+
+export type BalancedResult =
+  | ({ type: "quote" } & QuoteResult)
+  | ({ type: "fact" } & FactResult)
+  | ({ type: "trait" } & TraitResult)
+  | ({ type: "person" } & PersonResult)
+  | ({ type: "topic" } & TopicResult);
+
+const DATA_TYPES = ["quote", "fact", "trait", "person", "topic"] as const;
+type DataType = typeof DATA_TYPES[number];
+
+interface ScoredEntry {
+  type: DataType;
+  similarity: number;
+  mapped: QuoteResult | FactResult | TraitResult | PersonResult | TopicResult;
+  itemId: string;
+}
+
+export function resolveLinkedItems(dataItemIds: string[], state: StorageState): LinkedItem[] {
+  const items: LinkedItem[] = [];
+  const collections: Array<{ type: string; source: Array<{ id: string; name: string }> }> = [
+    { type: "topic", source: state.human.topics },
+    { type: "person", source: state.human.people },
+    { type: "fact", source: state.human.facts },
+    { type: "trait", source: state.human.traits },
+  ];
+  for (const { type, source } of collections) {
+    for (const entity of source) {
+      if (dataItemIds.includes(entity.id)) {
+        items.push({ id: entity.id, name: entity.name, type });
+      }
+    }
+  }
+  return items;
+}
+export function mapQuote(quote: Quote, state: StorageState): QuoteResult {
+  return {
+    id: quote.id,
+    text: quote.text,
+    speaker: quote.speaker,
+    timestamp: quote.timestamp,
+    linked_items: resolveLinkedItems(quote.data_item_ids, state),
+  };
+}
+
+function mapFact(fact: Fact): FactResult {
+  return {
+    id: fact.id,
+    name: fact.name,
+    description: fact.description,
+    sentiment: fact.sentiment,
+    validated: fact.validated,
+  };
+}
+
+function mapTrait(trait: Trait): TraitResult {
+  return {
+    id: trait.id,
+    name: trait.name,
+    description: trait.description,
+    strength: trait.strength ?? 0.5,
+    sentiment: trait.sentiment,
+  };
+}
+
+function mapPerson(person: Person): PersonResult {
+  return {
+    id: person.id,
+    name: person.name,
+    description: person.description,
+    relationship: person.relationship,
+    sentiment: person.sentiment,
+  };
+}
+
+function mapTopic(topic: Topic): TopicResult {
+  return {
+    id: topic.id,
+    name: topic.name,
+    description: topic.description,
+    category: topic.category,
+    sentiment: topic.sentiment,
+  };
+}
+
+export async function retrieveBalanced(
+  query: string,
+  limit: number = 10
+): Promise<BalancedResult[]> {
+  const state = await loadLatestState();
+  if (!state) {
+    console.error("No saved state found. Is EI_DATA_PATH set correctly?");
+    return [];
+  }
+
+  const embeddingService = getEmbeddingService();
+  const queryVector = await embeddingService.embed(query);
+
+  const allScored: ScoredEntry[] = [];
+
+  const typeConfigs: Array<{
+    type: DataType;
+    items: Array<{ id: string; embedding?: number[] }>;
+    mapper: (item: any) => any;
+  }> = [
+    { type: "quote", items: state.human.quotes, mapper: (q: Quote) => mapQuote(q, state) },
+    { type: "fact", items: state.human.facts, mapper: mapFact },
+    { type: "trait", items: state.human.traits, mapper: mapTrait },
+    { type: "person", items: state.human.people, mapper: mapPerson },
+    { type: "topic", items: state.human.topics, mapper: mapTopic },
+  ];
+
+  for (const { type, items, mapper } of typeConfigs) {
+    const scored = findTopK(queryVector, items, items.length);
+    for (const { item, similarity } of scored) {
+      if (similarity >= EMBEDDING_MIN_SIMILARITY) {
+        allScored.push({ type, similarity, mapped: mapper(item), itemId: item.id });
+      }
+    }
+  }
+
+  const result: ScoredEntry[] = [];
+  const used = new Set<string>();
+
+  // Floor: at least 1 result per type (if available and meets threshold)
+  for (const type of DATA_TYPES) {
+    if (result.length >= limit) break;
+    const best = allScored
+      .filter(r => r.type === type && !used.has(r.itemId))
+      .sort((a, b) => b.similarity - a.similarity)[0];
+    if (best) {
+      result.push(best);
+      used.add(best.itemId);
+    }
+  }
+
+  // Fill remaining slots with highest-similarity results across all types
+  const remaining = allScored
+    .filter(r => !used.has(r.itemId))
+    .sort((a, b) => b.similarity - a.similarity);
+
+  for (const entry of remaining) {
+    if (result.length >= limit) break;
+    result.push(entry);
+    used.add(entry.itemId);
+  }
+
+  result.sort((a, b) => b.similarity - a.similarity);
+
+  return result.map(({ type, mapped }) => ({ type, ...mapped }) as BalancedResult);
+}
+
+export async function lookupById(id: string): Promise<({ type: string } & Record<string, unknown>) | null> {
+  const state = await loadLatestState();
+  if (!state) {
+    return null;
+  }
+
+  const collections: Array<{ type: string; source: Array<{ id: string; [k: string]: unknown }> }> = [
+    { type: "fact", source: state.human.facts },
+    { type: "trait", source: state.human.traits },
+    { type: "person", source: state.human.people },
+    { type: "topic", source: state.human.topics },
+    { type: "quote", source: state.human.quotes },
+  ];
+
+  for (const { type, source } of collections) {
+    const entity = source.find(item => item.id === id);
+    if (entity) {
+      const { embedding, ...rest } = entity;
+      return { type, ...rest };
+    }
+  }
+
+  return null;
+}

package/src/cli.ts

@@ -0,0 +1,176 @@
+#!/usr/bin/env bun
+/**
+ * EI CLI - Memory retrieval interface for OpenCode integration
+ *
+ * Usage:
+ *   ei "search text"               Search all data types
+ *   ei -n 5 "search text"          Limit results
+ *   ei quote "search text"         Search specific type
+ *   ei quote -n 5 "search text"    Type-specific with limit
+ *   ei --id <id>                   Look up entity by ID
+ *   echo <id> | ei --id             Look up entity by ID from stdin
+ */
+
+import { parseArgs } from "util";
+import { retrieveBalanced, lookupById } from "./cli/retrieval";
+
+const TYPE_ALIASES: Record<string, string> = {
+  quote: "quotes",
+  quotes: "quotes",
+  fact: "facts",
+  facts: "facts",
+  trait: "traits",
+  traits: "traits",
+  person: "people",
+  people: "people",
+  topic: "topics",
+  topics: "topics",
+};
+
+function printHelp(): void {
+  console.log(`
+Ei
+
+Usage:
+  ei                            Launch the TUI chat interface
+  ei "search text"              Search all data types (top 10)
+  ei -n 5 "search text"         Limit results
+  ei <type> "search text"       Search a specific data type
+  ei <type> -n 5 "search text"  Type-specific with limit
+  ei --id <id>                  Look up a specific entity by ID
+  echo <id> | ei --id           Look up entity by ID from stdin
+
+Types:
+  quote / quotes    Quotes from conversation history
+  fact / facts      Facts about the user
+  trait / traits    Personality traits
+  person / people   People from the user's life
+  topic / topics    Topics of interest
+
+Options:
+  --number, -n     Maximum number of results (default: 10)
+  --id             Look up entity by ID (accepts value or stdin)
+  --help, -h       Show this help message
+
+Examples:
+  ei "debugging"                         # Search everything
+  ei -n 5 "API design"                   # Top 5 across all types
+  ei quote "you guessed it"              # Search quotes only
+  ei trait -n 3 "problem solving"        # Top 3 matching traits
+  ei --id abc-123                          # Look up entity by ID
+  ei "memory leak" | jq .[0].id | ei --id  # Pipe ID from search
+`);
+}
+
+async function main(): Promise<void> {
+  const args = process.argv.slice(2);
+
+  if (args.length === 0) {
+    const tuiDir = new URL("../tui", import.meta.url).pathname;
+    const tuiEntry = new URL("../tui/src/index.tsx", import.meta.url).pathname;
+    const proc = Bun.spawn(["bun", "--conditions=browser", "run", tuiEntry], {
+      stdio: ["inherit", "inherit", "inherit"],
+      env: { ...process.env },
+      cwd: tuiDir,
+    });
+    await proc.exited;
+    process.exit(proc.exitCode ?? 0);
+  }
+
+  if (args[0] === "--help" || args[0] === "-h") {
+    printHelp();
+    process.exit(0);
+  }
+
+
+  // Handle --id flag: look up entity by ID
+  const idFlagIndex = args.indexOf("--id");
+  if (idFlagIndex !== -1) {
+    let id = args[idFlagIndex + 1]?.trim();
+
+    // If no value after --id, try reading from stdin
+    if (!id && !process.stdin.isTTY) {
+      const chunks: Buffer[] = [];
+      for await (const chunk of process.stdin) {
+        chunks.push(chunk as Buffer);
+      }
+      id = Buffer.concat(chunks).toString("utf-8").trim();
+    }
+
+    if (!id) {
+      console.error("--id requires a value. Usage: ei --id <id> or echo <id> | ei --id");
+      process.exit(1);
+    }
+
+    // Strip surrounding quotes (from jq output or shell quoting)
+    id = id.replace(/^["']|["']$/g, "");
+
+    const entity = await lookupById(id);
+    if (!entity) {
+      console.error(`No entity found with ID: ${id}`);
+      process.exit(1);
+    }
+    console.log(JSON.stringify(entity, null, 2));
+    process.exit(0);
+  }
+  let targetType: string | null = null;
+  let parseableArgs = args;
+
+  if (TYPE_ALIASES[args[0]]) {
+    targetType = TYPE_ALIASES[args[0]];
+    parseableArgs = args.slice(1);
+  }
+
+  let parsed;
+  try {
+    parsed = parseArgs({
+      args: parseableArgs,
+      options: {
+        number: { type: "string", short: "n" },
+        help: { type: "boolean", short: "h" },
+      },
+      allowPositionals: true,
+      strict: true,
+    });
+  } catch (e) {
+    console.error(`Error parsing arguments: ${(e as Error).message}`);
+    process.exit(1);
+  }
+
+  if (parsed.values.help) {
+    printHelp();
+    process.exit(0);
+  }
+
+  const query = parsed.positionals.join(" ").trim();
+  const limit = parsed.values.number ? parseInt(parsed.values.number, 10) : 10;
+
+  if (!query) {
+    if (targetType) {
+      console.error(`Search text required. Usage: ei ${targetType} "search text"`);
+    } else {
+      console.error(`Search text required. Usage: ei "search text"`);
+    }
+    process.exit(1);
+  }
+
+  if (isNaN(limit) || limit < 1) {
+    console.error("--number must be a positive integer");
+    process.exit(1);
+  }
+
+  let result;
+  if (targetType) {
+    const module = await import(`./cli/commands/${targetType}.js`);
+    result = await module.execute(query, limit);
+  } else {
+    result = await retrieveBalanced(query, limit);
+  }
+
+  console.log(JSON.stringify(result, null, 2));
+}
+
+main().catch((e) => {
+  console.error(`Fatal error: ${e.message}`);
+  process.exit(1);
+});

package/src/core/AGENTS.md

@@ -0,0 +1,104 @@
+# Core Module
+
+The brain of Ei. Handles state, queue, LLM communication, and orchestration.
+
+## Architecture
+
+```
+core/
+├── processor.ts        # Main orchestrator (1100+ lines)
+├── state-manager.ts    # In-memory state + persistence
+├── queue-processor.ts  # LLM request queue with priorities
+├── llm-client.ts       # Multi-provider LLM abstraction
+├── types.ts            # All core types (source: CONTRACTS.md)
+├── handlers/           # LLM response handlers
+├── orchestrators/      # Multi-step workflows
+├── personas/           # Persona loading logic
+└── state/              # State slices (human, persona, messages)
+```
+
+## Key Files
+
+### processor.ts (The Hub)
+
+Everything flows through Processor:
+- **Main loop**: 100ms tick checking queue, auto-save, heartbeat
+- **Message flow**: User input → queue response request → handle result → update state
+- **Background work**: Extraction, ceremony, heartbeat (all async, queued)
+
+```typescript
+// Entry points
+processor.start()                    // Begin main loop
+processor.sendMessage(persona, text) // User sends message
+processor.stop()                     // Graceful shutdown
+```
+
+### state-manager.ts
+
+In-memory state with dirty tracking:
+- `loadState()` / `saveState()` for persistence
+- Slices: human, personas, messages, config
+- Auto-save every 60s when dirty
+
+### queue-processor.ts
+
+Priority queue for LLM requests:
+- High: User-facing responses
+- Normal: Extraction, analysis
+- Low: Background maintenance
+
+**Async model**: Handlers queue work, don't await results inline.
+
+### handlers/index.ts (1000+ lines)
+
+All `LLMNextStep` handlers in one file. Each handler:
+1. Parses LLM response (JSON or text)
+2. Updates state via StateManager
+3. May queue follow-up requests
+
+### orchestrators/
+
+Multi-step workflows:
+- `persona-generation.ts`: Create new persona (multi-LLM-call process)
+- `extraction.ts`: Scan messages for facts/topics/people
+- `ceremony.ts`: Periodic exposure decay + persona enrichment
+
+## Patterns
+
+### Time-Based Triggers
+
+```typescript
+// ✅ CORRECT: Update timestamp BEFORE async work
+if (timeSinceLastX >= delay) {
+  lastX = Date.now();         // Prevent duplicate queueing
+  await doAsyncWork();
+}
+
+// ❌ WRONG: Other loop iterations queue duplicates
+if (timeSinceLastX >= delay) {
+  await doAsyncWork();
+  lastX = Date.now();
+}
+```
+
+### Adding New Handlers
+
+1. Add enum to `LLMNextStep` in types.ts
+2. Add handler function in handlers/index.ts
+3. Register in `handlers` map at bottom of file
+4. Queue from Processor or orchestrator
+
+### State Updates
+
+Always use StateManager methods, never mutate directly:
+```typescript
+// ✅ Correct
+stateManager.updateHuman(h => ({ ...h, last_interaction: now }))
+
+// ❌ Wrong - bypasses dirty tracking
+state.human.last_interaction = now
+```
+
+## Testing
+
+Unit tests in `tests/unit/core/`. Mock LLM responses for deterministic tests.