ei-tui 0.1.14 → 0.1.16

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ei-tui",
-  "version": "0.1.14",
+  "version": "0.1.16",
   "author": "Flare576",
   "repository": {
     "type": "git",

package/src/storage/embeddings.ts

@@ -0,0 +1,103 @@
+/**
+ * Embedding serialization utilities.
+ *
+ * At runtime, embeddings are `number[]` (384-dim float vectors).
+ * In storage, they are base64-encoded Float32Array binary blobs — identical data,
+ * ~75% smaller than JSON float arrays, and still compressible by gzip for sync/LocalStorage.
+ *
+ * Format on disk/in LocalStorage:
+ *   "embedding": "AAAAAAAA..."   // btoa(Float32Array.buffer)
+ *
+ * Format in memory (unchanged — nothing outside storage layer sees strings):
+ *   embedding: [0.0234567, -0.0891234, ...]
+ *
+ * Backward compatibility: if a stored embedding is already a number[] (old format),
+ * decodeEmbedding returns it as-is. Mixed old/new files are handled transparently.
+ *
+ * IMPORTANT: encodeAllEmbeddings does NOT mutate the input state. It returns a new
+ * StorageState where human item arrays are shallow-copied with encoded embedding fields.
+ * This prevents the live in-memory state from being corrupted with base64 strings.
+ */
+
+import type { StorageState } from "../core/types.js";
+
+// ---------------------------------------------------------------------------
+// Encode: number[] → base64 string
+// ---------------------------------------------------------------------------
+
+function encodeEmbedding(embedding: number[]): string {
+  const buffer = new Float32Array(embedding).buffer;
+  const bytes = new Uint8Array(buffer);
+  let binary = "";
+  for (let i = 0; i < bytes.length; i++) {
+    binary += String.fromCharCode(bytes[i]);
+  }
+  return btoa(binary);
+}
+
+// ---------------------------------------------------------------------------
+// Decode: base64 string → number[]  (no-op if already number[])
+// ---------------------------------------------------------------------------
+
+function decodeEmbedding(value: unknown): number[] | undefined {
+  if (value == null) return undefined;
+  if (Array.isArray(value)) return value as number[]; // backward compat
+  if (typeof value !== "string") return undefined;
+
+  const binary = atob(value);
+  const bytes = new Uint8Array(binary.length);
+  for (let i = 0; i < binary.length; i++) {
+    bytes[i] = binary.charCodeAt(i);
+  }
+  return Array.from(new Float32Array(bytes.buffer));
+}
+
+// ---------------------------------------------------------------------------
+// Walk the entire StorageState and encode/decode all embedding fields
+// ---------------------------------------------------------------------------
+
+const HUMAN_ITEM_KEYS = ["facts", "traits", "topics", "people", "quotes"] as const;
+
+/**
+ * Returns a new StorageState with embeddings encoded as base64 strings.
+ * Does NOT mutate the input — human item arrays are shallow-copied.
+ */
+export function encodeAllEmbeddings(state: StorageState): StorageState {
+  const human = (state as unknown as Record<string, unknown>)["human"] as Record<string, unknown> | undefined;
+  if (!human) return state;
+
+  const encodedHuman: Record<string, unknown> = { ...human };
+  for (const key of HUMAN_ITEM_KEYS) {
+    const items = human[key];
+    if (Array.isArray(items)) {
+      encodedHuman[key] = items.map((item: Record<string, unknown>) => {
+        if (!Array.isArray(item.embedding) || item.embedding.length === 0) return item;
+        return { ...item, embedding: encodeEmbedding(item.embedding as number[]) };
+      });
+    }
+  }
+
+  return { ...state, human: encodedHuman as unknown as StorageState["human"] };
+}
+
+/**
+ * Returns a new StorageState with embeddings decoded from base64 to number[].
+ * Does NOT mutate the input — human item arrays are shallow-copied.
+ */
+export function decodeAllEmbeddings(state: StorageState): StorageState {
+  const human = (state as unknown as Record<string, unknown>)["human"] as Record<string, unknown> | undefined;
+  if (!human) return state;
+
+  const decodedHuman: Record<string, unknown> = { ...human };
+  for (const key of HUMAN_ITEM_KEYS) {
+    const items = human[key];
+    if (Array.isArray(items)) {
+      decodedHuman[key] = items.map((item: Record<string, unknown>) => {
+        if (item.embedding === undefined || Array.isArray(item.embedding)) return item;
+        return { ...item, embedding: decodeEmbedding(item.embedding) };
+      });
+    }
+  }
+
+  return { ...state, human: decodedHuman as unknown as StorageState["human"] };
+}

package/src/storage/local.ts

@@ -1,6 +1,7 @@
 import type { StorageState } from "../core/types.js";
 import type { Storage } from "./interface.js";
 import { compress, decompress, isCompressed } from "./compress.js";
+import { encodeAllEmbeddings, decodeAllEmbeddings } from "./embeddings.js";
 
 const STATE_KEY = "ei_state";
 const BACKUP_KEY = "ei_state_backup";
@@ -20,7 +21,7 @@ export class LocalStorage implements Storage {
   async save(state: StorageState): Promise<void> {
     state.timestamp = new Date().toISOString();
     try {
-      const json = JSON.stringify(state);
+      const json = JSON.stringify(encodeAllEmbeddings(state));
       const payload = await compress(json);
       globalThis.localStorage.setItem(STATE_KEY, payload);
     } catch (e) {
@@ -36,7 +37,7 @@ export class LocalStorage implements Storage {
     if (current) {
       try {
         const json = isCompressed(current) ? await decompress(current) : current;
-        return JSON.parse(json) as StorageState;
+        return decodeAllEmbeddings(JSON.parse(json) as StorageState);
       } catch {
         return null;
       }
@@ -52,8 +53,9 @@ export class LocalStorage implements Storage {
   async moveToBackup(): Promise<void> {
     const current = globalThis.localStorage?.getItem(STATE_KEY);
     if (current) {
-      globalThis.localStorage.setItem(BACKUP_KEY, current);
+      // Remove primary first so backup write doesn't double-count against quota.
       globalThis.localStorage.removeItem(STATE_KEY);
+      globalThis.localStorage.setItem(BACKUP_KEY, current);
     }
   }
 
@@ -67,7 +69,7 @@ export class LocalStorage implements Storage {
     if (backup) {
       try {
         const json = isCompressed(backup) ? await decompress(backup) : backup;
-        return JSON.parse(json) as StorageState;
+        return decodeAllEmbeddings(JSON.parse(json) as StorageState);
       } catch {
         return null;
       }

package/tui/src/context/keyboard.tsx

@@ -56,6 +56,7 @@ export const KeyboardProvider: ParentComponent = (props) => {
   const toggleSidebar = () => setSidebarVisible(!sidebarVisible());
 
   const exitApp = async () => {
+    showNotification("Saving and syncing...", "info");
     const result = await saveAndExit();
     if (!result.success) {
       showNotification(`Sync failed: ${result.error}. Use /quit force to exit anyway.`, "error");

package/tui/src/storage/file.ts

@@ -1,5 +1,6 @@
 import type { StorageState } from "../../../src/core/types";
 import type { Storage } from "../../../src/storage/interface";
+import { encodeAllEmbeddings, decodeAllEmbeddings } from "../../../src/storage/embeddings";
 import { join } from "path";
 import { mkdir, rename, unlink, readdir } from "fs/promises";
 
@@ -45,7 +46,7 @@ export class FileStorage implements Storage {
 
     await this.withLock(filePath, async () => {
       try {
-        await this.atomicWrite(filePath, JSON.stringify(state, null, 2));
+        await this.atomicWrite(filePath, JSON.stringify(encodeAllEmbeddings(state), null, 2));
       } catch (e) {
         if (this.isQuotaError(e)) {
           throw new Error("STORAGE_SAVE_FAILED: Disk quota exceeded");
@@ -63,7 +64,7 @@ export class FileStorage implements Storage {
       try {
         const text = await file.text();
         if (text) {
-          return JSON.parse(text) as StorageState;
+          return decodeAllEmbeddings(JSON.parse(text) as StorageState);
         }
       } catch {
         return null;
@@ -96,7 +97,7 @@ export class FileStorage implements Storage {
       try {
         const text = await backupFile.text();
         if (text) {
-          return JSON.parse(text) as StorageState;
+          return decodeAllEmbeddings(JSON.parse(text) as StorageState);
         }
       } catch {
         return null;
@@ -122,7 +123,7 @@ export class FileStorage implements Storage {
     ].join("") + ".json";
 
     const destPath = join(backupsPath, name);
-    await this.atomicWrite(destPath, JSON.stringify(state, null, 2));
+    await this.atomicWrite(destPath, JSON.stringify(encodeAllEmbeddings(state), null, 2));
 
     // Prune: keep only the newest maxBackups files
     const entries = await readdir(backupsPath);