ei-tui 0.1.14 → 0.1.17

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ei-tui",
-  "version": "0.1.14",
+  "version": "0.1.17",
   "author": "Flare576",
   "repository": {
     "type": "git",

package/src/cli/retrieval.ts

@@ -1,4 +1,5 @@
 import type { StorageState, Quote, Fact, Trait, Person, Topic } from "../core/types";
+import { decodeAllEmbeddings } from "../storage/embeddings";
 import { crossFind } from "../core/utils/index.ts";
 import { join } from "path";
 import { readFile } from "fs/promises";
@@ -21,7 +22,7 @@ export async function loadLatestState(): Promise<StorageState | null> {
   for (const file of [STATE_FILE, BACKUP_FILE]) {
     try {
       const text = await readFile(join(dataPath, file), "utf-8");
-      if (text) return JSON.parse(text) as StorageState;
+      if (text) return decodeAllEmbeddings(JSON.parse(text) as StorageState);
     } catch {
       continue;
     }

package/src/core/processor.ts

@@ -292,11 +292,14 @@ export class Processor {
       const result = await remoteSync.sync(state);
       
       if (!result.success) {
-        // Push failed — likely 412 etag mismatch or network error
-        // Do NOT moveToBackup — leave state.json intact
-        // Next boot will detect primary + remote → conflict resolution
+        // Sync failed (e.g. 429, network error, 412 etag mismatch).
+        // Do NOT stop() — leave the processor loop running so the user can
+        // keep using the TUI and retry /quit later.
+        // Do NOT moveToBackup — leave state.json intact so next boot can
+        // detect primary + remote → conflict resolution if needed.
         console.log(`[Processor ${this.instanceId}] Remote sync failed: ${result.error}`);
-        await this.stop();
+        // Reset the import abort controller so imports can resume normally.
+        this.importAbortController = new AbortController();
         this.interface.onSaveAndExitFinish?.();
         return { success: false, error: result.error };
       }

package/src/storage/embeddings.ts

@@ -0,0 +1,103 @@
+/**
+ * Embedding serialization utilities.
+ *
+ * At runtime, embeddings are `number[]` (384-dim float vectors).
+ * In storage, they are base64-encoded Float32Array binary blobs — identical data,
+ * ~75% smaller than JSON float arrays, and still compressible by gzip for sync/LocalStorage.
+ *
+ * Format on disk/in LocalStorage:
+ *   "embedding": "AAAAAAAA..."   // btoa(Float32Array.buffer)
+ *
+ * Format in memory (unchanged — nothing outside storage layer sees strings):
+ *   embedding: [0.0234567, -0.0891234, ...]
+ *
+ * Backward compatibility: if a stored embedding is already a number[] (old format),
+ * decodeEmbedding returns it as-is. Mixed old/new files are handled transparently.
+ *
+ * IMPORTANT: encodeAllEmbeddings does NOT mutate the input state. It returns a new
+ * StorageState where human item arrays are shallow-copied with encoded embedding fields.
+ * This prevents the live in-memory state from being corrupted with base64 strings.
+ */
+
+import type { StorageState } from "../core/types.js";
+
+// ---------------------------------------------------------------------------
+// Encode: number[] → base64 string
+// ---------------------------------------------------------------------------
+
+function encodeEmbedding(embedding: number[]): string {
+  const buffer = new Float32Array(embedding).buffer;
+  const bytes = new Uint8Array(buffer);
+  let binary = "";
+  for (let i = 0; i < bytes.length; i++) {
+    binary += String.fromCharCode(bytes[i]);
+  }
+  return btoa(binary);
+}
+
+// ---------------------------------------------------------------------------
+// Decode: base64 string → number[]  (no-op if already number[])
+// ---------------------------------------------------------------------------
+
+function decodeEmbedding(value: unknown): number[] | undefined {
+  if (value == null) return undefined;
+  if (Array.isArray(value)) return value as number[]; // backward compat
+  if (typeof value !== "string") return undefined;
+
+  const binary = atob(value);
+  const bytes = new Uint8Array(binary.length);
+  for (let i = 0; i < binary.length; i++) {
+    bytes[i] = binary.charCodeAt(i);
+  }
+  return Array.from(new Float32Array(bytes.buffer));
+}
+
+// ---------------------------------------------------------------------------
+// Walk the entire StorageState and encode/decode all embedding fields
+// ---------------------------------------------------------------------------
+
+const HUMAN_ITEM_KEYS = ["facts", "traits", "topics", "people", "quotes"] as const;
+
+/**
+ * Returns a new StorageState with embeddings encoded as base64 strings.
+ * Does NOT mutate the input — human item arrays are shallow-copied.
+ */
+export function encodeAllEmbeddings(state: StorageState): StorageState {
+  const human = (state as unknown as Record<string, unknown>)["human"] as Record<string, unknown> | undefined;
+  if (!human) return state;
+
+  const encodedHuman: Record<string, unknown> = { ...human };
+  for (const key of HUMAN_ITEM_KEYS) {
+    const items = human[key];
+    if (Array.isArray(items)) {
+      encodedHuman[key] = items.map((item: Record<string, unknown>) => {
+        if (!Array.isArray(item.embedding) || item.embedding.length === 0) return item;
+        return { ...item, embedding: encodeEmbedding(item.embedding as number[]) };
+      });
+    }
+  }
+
+  return { ...state, human: encodedHuman as unknown as StorageState["human"] };
+}
+
+/**
+ * Returns a new StorageState with embeddings decoded from base64 to number[].
+ * Does NOT mutate the input — human item arrays are shallow-copied.
+ */
+export function decodeAllEmbeddings(state: StorageState): StorageState {
+  const human = (state as unknown as Record<string, unknown>)["human"] as Record<string, unknown> | undefined;
+  if (!human) return state;
+
+  const decodedHuman: Record<string, unknown> = { ...human };
+  for (const key of HUMAN_ITEM_KEYS) {
+    const items = human[key];
+    if (Array.isArray(items)) {
+      decodedHuman[key] = items.map((item: Record<string, unknown>) => {
+        if (item.embedding === undefined || Array.isArray(item.embedding)) return item;
+        return { ...item, embedding: decodeEmbedding(item.embedding) };
+      });
+    }
+  }
+
+  return { ...state, human: decodedHuman as unknown as StorageState["human"] };
+}

package/src/storage/local.ts

@@ -1,6 +1,7 @@
 import type { StorageState } from "../core/types.js";
 import type { Storage } from "./interface.js";
 import { compress, decompress, isCompressed } from "./compress.js";
+import { encodeAllEmbeddings, decodeAllEmbeddings } from "./embeddings.js";
 
 const STATE_KEY = "ei_state";
 const BACKUP_KEY = "ei_state_backup";
@@ -20,7 +21,7 @@ export class LocalStorage implements Storage {
   async save(state: StorageState): Promise<void> {
     state.timestamp = new Date().toISOString();
     try {
-      const json = JSON.stringify(state);
+      const json = JSON.stringify(encodeAllEmbeddings(state));
       const payload = await compress(json);
       globalThis.localStorage.setItem(STATE_KEY, payload);
     } catch (e) {
@@ -36,7 +37,7 @@ export class LocalStorage implements Storage {
     if (current) {
       try {
         const json = isCompressed(current) ? await decompress(current) : current;
-        return JSON.parse(json) as StorageState;
+        return decodeAllEmbeddings(JSON.parse(json) as StorageState);
       } catch {
         return null;
       }
@@ -52,8 +53,9 @@ export class LocalStorage implements Storage {
   async moveToBackup(): Promise<void> {
     const current = globalThis.localStorage?.getItem(STATE_KEY);
     if (current) {
-      globalThis.localStorage.setItem(BACKUP_KEY, current);
+      // Remove primary first so backup write doesn't double-count against quota.
       globalThis.localStorage.removeItem(STATE_KEY);
+      globalThis.localStorage.setItem(BACKUP_KEY, current);
     }
   }
 
@@ -67,7 +69,7 @@ export class LocalStorage implements Storage {
     if (backup) {
       try {
         const json = isCompressed(backup) ? await decompress(backup) : backup;
-        return JSON.parse(json) as StorageState;
+        return decodeAllEmbeddings(JSON.parse(json) as StorageState);
       } catch {
         return null;
       }

package/tui/src/context/keyboard.tsx

@@ -56,6 +56,7 @@ export const KeyboardProvider: ParentComponent = (props) => {
   const toggleSidebar = () => setSidebarVisible(!sidebarVisible());
 
   const exitApp = async () => {
+    showNotification("Saving and syncing...", "info");
     const result = await saveAndExit();
     if (!result.success) {
       showNotification(`Sync failed: ${result.error}. Use /quit force to exit anyway.`, "error");

package/tui/src/storage/file.ts

@@ -1,5 +1,6 @@
 import type { StorageState } from "../../../src/core/types";
 import type { Storage } from "../../../src/storage/interface";
+import { encodeAllEmbeddings, decodeAllEmbeddings } from "../../../src/storage/embeddings";
 import { join } from "path";
 import { mkdir, rename, unlink, readdir } from "fs/promises";
 
@@ -45,7 +46,7 @@ export class FileStorage implements Storage {
 
     await this.withLock(filePath, async () => {
       try {
-        await this.atomicWrite(filePath, JSON.stringify(state, null, 2));
+        await this.atomicWrite(filePath, JSON.stringify(encodeAllEmbeddings(state), null, 2));
       } catch (e) {
         if (this.isQuotaError(e)) {
           throw new Error("STORAGE_SAVE_FAILED: Disk quota exceeded");
@@ -63,7 +64,7 @@ export class FileStorage implements Storage {
       try {
         const text = await file.text();
         if (text) {
-          return JSON.parse(text) as StorageState;
+          return decodeAllEmbeddings(JSON.parse(text) as StorageState);
         }
       } catch {
         return null;
@@ -96,7 +97,7 @@ export class FileStorage implements Storage {
       try {
         const text = await backupFile.text();
         if (text) {
-          return JSON.parse(text) as StorageState;
+          return decodeAllEmbeddings(JSON.parse(text) as StorageState);
         }
       } catch {
         return null;
@@ -122,7 +123,7 @@ export class FileStorage implements Storage {
     ].join("") + ".json";
 
     const destPath = join(backupsPath, name);
-    await this.atomicWrite(destPath, JSON.stringify(state, null, 2));
+    await this.atomicWrite(destPath, JSON.stringify(encodeAllEmbeddings(state), null, 2));
 
     // Prune: keep only the newest maxBackups files
     const entries = await readdir(backupsPath);

package/tui/src/util/yaml-serializers.ts

@@ -451,10 +451,13 @@ interface EditableSettingsData {
   opencode?: {
     integration?: boolean | null;
     polling_interval_ms?: number | null;
+    last_sync?: string | null;
+    extraction_point?: string | null;
   };
   claudeCode?: {
     integration?: boolean | null;
     polling_interval_ms?: number | null;
+    last_sync?: string | null;
   };
   backup?: {
     enabled?: boolean | null;
@@ -478,10 +481,13 @@ export function settingsToYAML(settings: HumanSettings | undefined): string {
     opencode: {
       integration: settings?.opencode?.integration ?? false,
       polling_interval_ms: settings?.opencode?.polling_interval_ms ?? 1800000,
+      last_sync: settings?.opencode?.last_sync ?? null,
+      extraction_point: settings?.opencode?.extraction_point ?? null,
     },
     claudeCode: {
       integration: settings?.claudeCode?.integration ?? false,
       polling_interval_ms: settings?.claudeCode?.polling_interval_ms ?? 1800000,
+      last_sync: settings?.claudeCode?.last_sync ?? null,
     },
     backup: {
       enabled: settings?.backup?.enabled ?? false,
@@ -492,9 +498,10 @@ export function settingsToYAML(settings: HumanSettings | undefined): string {
   
   return YAML.stringify(data, {
     lineWidth: 0,
-  });
+  })
+  .replace(/^(\s+)(last_sync: .+)$/mg, '$1# [read-only] $2')
+  .replace(/^(\s+)(extraction_point: .+)$/mg, '$1# [read-only] $2');
 }
-
 export function settingsFromYAML(yamlContent: string, original: HumanSettings | undefined): HumanSettings {
   const data = YAML.parse(yamlContent) as EditableSettingsData;