holosphere 2.0.0-alpha2 → 2.0.0-alpha4

package/src/storage/gun-wrapper.js

@@ -1,10 +1,15 @@
 /**
  * GunDB Storage Wrapper with radisk persistence
  * Handles path construction and CRUD operations
+ * Note: GunDB doesn't handle nested objects well, so we store data as JSON strings
  */
 
 import { gunPromise, gunPut, gunCollect } from './gun-async.js';
 
+// ============================================================================
+// PATH BUILDERS
+// ============================================================================
+
 /**
  * Build Gun path from components
  * @param {string} appname - Application namespace
@@ -25,6 +30,22 @@ export function buildPath(appname, holon, lens, key = null) {
   return `${appname}/${encodedHolon}/${encodedLens}`;
 }
 
+/**
+ * Build Gun path for global tables (app-wide data not tied to holons)
+ * @param {string} appname - Application namespace
+ * @param {string} tableName - Global table name (e.g., 'schemas', 'federation')
+ * @param {string} key - Data key (optional)
+ * @returns {string} Gun path
+ */
+export function buildGlobalPath(appname, tableName, key = null) {
+  const encodedTable = encodePathComponent(tableName);
+  if (key) {
+    const encodedKey = encodePathComponent(key);
+    return `${appname}/${encodedTable}/${encodedKey}`;
+  }
+  return `${appname}/${encodedTable}`;
+}
+
 /**
  * Encode path component to handle special characters
  * @private
@@ -33,6 +54,94 @@ function encodePathComponent(component) {
   return encodeURIComponent(component).replace(/%2F/g, '/');
 }
 
+/**
+ * Serialize data for GunDB storage
+ * Wraps data in { _json: string } format for Gun compatibility
+ * Gun requires an object at graph roots, so we can't store raw JSON strings
+ * The deserialization handles both this format and raw JSON strings for reading old data
+ * @private
+ */
+function serializeForGun(data) {
+  return { _json: JSON.stringify(data) };
+}
+
+/**
+ * Deserialize data from GunDB storage
+ * Handles multiple formats:
+ * - Direct JSON string (holosphere original - now default)
+ * - _json wrapped format (holosphere2 legacy)
+ * - Gun internal references (_["#"])
+ * - Gun node data (_[">"])
+ * - Plain objects
+ * @private
+ */
+function deserializeFromGun(data) {
+  if (!data) {
+    return null;
+  }
+
+  try {
+    // Format 1: String data (holosphere original stores JSON as string directly)
+    if (typeof data === 'string') {
+      try {
+        return JSON.parse(data);
+      } catch (e) {
+        // Not JSON, return as-is
+        return data;
+      }
+    }
+
+    // Format 2: _json wrapped format (holosphere2 default)
+    if (data._json && typeof data._json === 'string') {
+      try {
+        return JSON.parse(data._json);
+      } catch (e) {
+        console.warn('Failed to parse _json field:', e);
+        return null;
+      }
+    }
+
+    // Format 3: Gun internal reference (_["#"]) - this indicates a Gun reference
+    // The actual data retrieval should happen via resolveReference, we just identify it here
+    if (data._ && data._['#']) {
+      // This is a Gun reference - return as-is for reference resolution
+      // The caller should check isReference() and handle appropriately
+      return data;
+    }
+
+    // Format 4: Gun node data with timestamps (_[">"])
+    // Gun stores metadata in _.> - find the actual value
+    if (data._ && data._['>']) {
+      const nodeValue = Object.entries(data).find(([k, v]) => k !== '_' && typeof v === 'string');
+      if (nodeValue) {
+        try {
+          return JSON.parse(nodeValue[1]);
+        } catch (e) {
+          return nodeValue[1];
+        }
+      }
+    }
+
+    // Format 5: Plain object - clean Gun metadata and return
+    if (typeof data === 'object' && data !== null) {
+      const cleaned = { ...data };
+      delete cleaned['_'];
+
+      // Check if any remaining keys - if empty after removing _, return null
+      if (Object.keys(cleaned).length === 0) {
+        return null;
+      }
+
+      return cleaned;
+    }
+
+    return data;
+  } catch (error) {
+    console.warn('Error deserializing Gun data:', error);
+    return data; // Return raw data as fallback
+  }
+}
+
 /**
  * Write data to Gun with radisk persistence
  * @param {Object} gun - Gun instance
@@ -42,7 +151,8 @@ function encodePathComponent(component) {
  */
 export async function write(gun, path, data) {
   try {
-    await gunPut(gun.get(path), data, 2000);
+    const serialized = serializeForGun(data);
+    await gunPut(gun.get(path), serialized, 2000);
     // Delay to allow Gun to propagate the write (50ms for better reliability)
     await new Promise(resolve => setTimeout(resolve, 50));
     return true;
@@ -58,7 +168,13 @@ export async function write(gun, path, data) {
  * @returns {Promise<Object|null>} Data or null if not found
  */
 export async function read(gun, path) {
-  const data = await gunPromise(gun.get(path), 2000);
+  const rawData = await gunPromise(gun.get(path), 2000);
+
+  if (!rawData) {
+    return null;
+  }
+
+  const data = deserializeFromGun(rawData);
 
   // Return null if deleted or not found
   if (!data || data._deleted) {
@@ -70,22 +186,114 @@ export async function read(gun, path) {
 
 /**
  * Read all data under a path (lens query)
+ * Uses Gun's .open() which waits for nested data to load
+ * Falls back to .once() with iteration
  * @param {Object} gun - Gun instance
  * @param {string} path - Gun path
  * @returns {Promise<Object[]>} Array of data objects
  */
 export async function readAll(gun, path) {
-  const results = await gunCollect(gun.get(path), 1000);
+  return new Promise((resolve) => {
+    const output = new Map();
+    let settled = false;
 
-  // Filter out deleted items and Gun metadata
-  return results
-    .filter(({ data, key }) => {
-      if (!data || typeof data !== 'object') return false;
-      if (key.startsWith('_')) return false;
-      if (data._deleted) return false;
-      return true;
-    })
-    .map(({ data }) => data);
+    const ref = gun.get(path);
+
+    // Use .once() on parent to get all keys, then process
+    ref.once((parentData) => {
+      if (settled) return;
+
+      if (!parentData) {
+        settled = true;
+        resolve([]);
+        return;
+      }
+
+      // Get all keys except Gun metadata
+      const keys = Object.keys(parentData).filter(k => k !== '_');
+
+      if (keys.length === 0) {
+        settled = true;
+        resolve([]);
+        return;
+      }
+
+      // Process inline data directly from parentData
+      // Gun often includes the data inline as references or direct values
+      for (const key of keys) {
+        const rawItem = parentData[key];
+        if (!rawItem) continue;
+
+        let item = null;
+
+        // Check if it's a Gun reference (soul)
+        if (typeof rawItem === 'object' && rawItem['#']) {
+          // Skip references for now, map().once will handle them
+          continue;
+        }
+
+        // Try to parse as _json wrapper
+        if (rawItem._json && typeof rawItem._json === 'string') {
+          try {
+            item = JSON.parse(rawItem._json);
+          } catch (e) {}
+        }
+        // Try as direct JSON string
+        else if (typeof rawItem === 'string') {
+          try {
+            item = JSON.parse(rawItem);
+          } catch (e) {}
+        }
+
+        if (item && item.id && !item._deleted) {
+          output.set(item.id, item);
+        }
+      }
+
+      // If we got all data inline, resolve now
+      if (output.size > 0) {
+        settled = true;
+        resolve(Array.from(output.values()));
+        return;
+      }
+
+      // Otherwise use map().once() to resolve references
+      ref.map().once((data, key) => {
+        if (settled || !data || key === '_') return;
+
+        let item = null;
+        if (data._json && typeof data._json === 'string') {
+          try {
+            item = JSON.parse(data._json);
+          } catch (e) {}
+        } else if (typeof data === 'string') {
+          try {
+            item = JSON.parse(data);
+          } catch (e) {}
+        }
+
+        if (item && item.id && !item._deleted) {
+          output.set(item.id, item);
+        }
+      });
+
+      // Give map().once() time to complete
+      setTimeout(() => {
+        if (!settled) {
+          settled = true;
+          resolve(Array.from(output.values()));
+        }
+      }, 500);
+    });
+
+    // Overall timeout
+    setTimeout(() => {
+      if (!settled) {
+        settled = true;
+        resolve(Array.from(output.values()));
+      }
+    }, 2000);
+  });
 }
 
 /**
@@ -96,21 +304,24 @@ export async function readAll(gun, path) {
  * @returns {Promise<boolean>} Success indicator
  */
 export async function update(gun, path, updates) {
-  const existing = await gunPromise(gun.get(path));
+  const rawData = await gunPromise(gun.get(path));
 
-  if (!existing || !existing.id || existing._deleted) {
-    return false; // Not found or deleted
+  if (!rawData) {
+    return false; // Not found
   }
 
-  // Remove Gun metadata before merging
-  const cleanExisting = { ...existing };
-  delete cleanExisting['_'];
+  // Deserialize existing data
+  const existing = deserializeFromGun(rawData);
+  if (!existing || !existing.id || existing._deleted) {
+    return false;
+  }
 
   // Merge updates
-  const merged = { ...cleanExisting, ...updates };
+  const merged = { ...existing, ...updates };
 
   try {
-    await gunPut(gun.get(path), merged);
+    const serialized = serializeForGun(merged);
+    await gunPut(gun.get(path), serialized);
     return true;
   } catch (error) {
     throw error;
@@ -125,21 +336,22 @@ export async function update(gun, path, updates) {
  */
 export async function deleteData(gun, path) {
   try {
-    // Gun requires tombstone to be an object with _deleted flag
-    // First read existing data to preserve metadata
-    const existing = await gunPromise(gun.get(path));
-    if (!existing) {
+    // First read existing data to get the id
+    const rawData = await gunPromise(gun.get(path));
+    if (!rawData) {
       return true; // Already deleted/doesn't exist
     }
 
-    // Create tombstone object
+    const existing = deserializeFromGun(rawData);
+
+    // Create tombstone object and serialize it
     const tombstone = {
-      id: existing.id,
+      id: existing?.id,
       _deleted: true,
       _deletedAt: Date.now()
     };
 
-    await gunPut(gun.get(path), tombstone);
+    await gunPut(gun.get(path), serializeForGun(tombstone));
     return true;
   } catch (error) {
     throw error;
@@ -186,8 +398,11 @@ export function subscribe(gun, path, callback, options = {}) {
     const ref = gun.get(path);
 
     ref.map().on((data, key) => {
-      if (data && !key.startsWith('_') && !data._deleted) {
-        callback(data, key);
+      if (data && !key.startsWith('_')) {
+        const deserialized = deserializeFromGun(data);
+        if (deserialized && !deserialized._deleted) {
+          callback(deserialized, key);
+        }
       }
     });
 
@@ -203,8 +418,11 @@ export function subscribe(gun, path, callback, options = {}) {
   } else {
     // Subscribe to single item
     const listener = gun.get(path).on((data, key) => {
-      if (data && !data._deleted) {
-        callback(data, key);
+      if (data) {
+        const deserialized = deserializeFromGun(data);
+        if (deserialized && !deserialized._deleted) {
+          callback(deserialized, key);
+        }
       }
     });
 
@@ -219,3 +437,101 @@ export function subscribe(gun, path, callback, options = {}) {
     };
   }
 }
+
+// ============================================================================
+// GLOBAL TABLE OPERATIONS
+// ============================================================================
+
+/**
+ * Write data to a global table
+ * Global tables are app-wide data not tied to specific holons (e.g., schemas, federation)
+ * @param {Object} gun - Gun instance
+ * @param {string} appname - Application namespace
+ * @param {string} tableName - Global table name
+ * @param {Object} data - Data to write (must have 'id' field)
+ * @returns {Promise<boolean>} Success indicator
+ */
+export async function writeGlobal(gun, appname, tableName, data) {
+  if (!data || !data.id) {
+    throw new Error('writeGlobal: data must have an id field');
+  }
+  const path = buildGlobalPath(appname, tableName, data.id);
+  return write(gun, path, data);
+}
+
+/**
+ * Read data from a global table
+ * @param {Object} gun - Gun instance
+ * @param {string} appname - Application namespace
+ * @param {string} tableName - Global table name
+ * @param {string} key - Data key
+ * @returns {Promise<Object|null>} Data or null if not found
+ */
+export async function readGlobal(gun, appname, tableName, key) {
+  const path = buildGlobalPath(appname, tableName, key);
+  return read(gun, path);
+}
+
+/**
+ * Read all data from a global table
+ * Uses same approach as readAll
+ * @param {Object} gun - Gun instance
+ * @param {string} appname - Application namespace
+ * @param {string} tableName - Global table name
+ * @param {number} timeout - Timeout in ms (default: 2000)
+ * @returns {Promise<Object[]>} Array of data objects
+ */
+export async function readAllGlobal(gun, appname, tableName, timeout = 2000) {
+  const path = buildGlobalPath(appname, tableName);
+  return readAll(gun, path);
+}
+
+/**
+ * Delete data from a global table
+ * @param {Object} gun - Gun instance
+ * @param {string} appname - Application namespace
+ * @param {string} tableName - Global table name
+ * @param {string} key - Data key
+ * @returns {Promise<boolean>} Success indicator
+ */
+export async function deleteGlobal(gun, appname, tableName, key) {
+  const path = buildGlobalPath(appname, tableName, key);
+  return deleteData(gun, path);
+}
+
+/**
+ * Delete all data from a global table
+ * @param {Object} gun - Gun instance
+ * @param {string} appname - Application namespace
+ * @param {string} tableName - Global table name
+ * @returns {Promise<Object>} Deletion results { success: boolean, count: number }
+ */
+export async function deleteAllGlobal(gun, appname, tableName) {
+  const path = buildGlobalPath(appname, tableName);
+  return deleteAll(gun, path);
+}
+
+// ============================================================================
+// DATA PARSING UTILITIES
+// ============================================================================
+
+/**
+ * Parse data from Gun storage, handling various formats
+ * - JSON string in _json field
+ * - Legacy object format
+ * - Gun references
+ * @param {*} rawData - Raw data from Gun
+ * @returns {Object|null} Parsed data or null
+ */
+export function parse(rawData) {
+  return deserializeFromGun(rawData);
+}
+
+/**
+ * Serialize data for Gun storage
+ * @param {Object} data - Data to serialize
+ * @returns {string} JSON string
+ */
+export function serialize(data) {
+  return serializeForGun(data);
+}

package/src/storage/indexeddb-storage.js

@@ -8,8 +8,10 @@ import { PersistentStorage } from './persistent-storage.js';
 export class IndexedDBStorage extends PersistentStorage {
   constructor() {
     super();
+    /** @type {IDBDatabase|null} */
     this.db = null;
-    this.dbName = null;
+    /** @type {string} */
+    this.dbName = '';
     this.storeName = 'events';
   }
 
@@ -22,11 +24,12 @@ export class IndexedDBStorage extends PersistentStorage {
       request.onerror = () => reject(request.error);
       request.onsuccess = () => {
         this.db = request.result;
-        resolve();
+        resolve(undefined);
       };
 
       request.onupgradeneeded = (event) => {
-        const db = event.target.result;
+        const target = /** @type {IDBOpenDBRequest} */ (event.target);
+        const db = target.result;
 
         // Create object store if it doesn't exist
         if (!db.objectStoreNames.contains(this.storeName)) {
@@ -40,18 +43,26 @@ export class IndexedDBStorage extends PersistentStorage {
 
   async put(key, event) {
     return new Promise((resolve, reject) => {
+      if (!this.db) {
+        reject(new Error('Database not initialized'));
+        return;
+      }
       const transaction = this.db.transaction([this.storeName], 'readwrite');
       const objectStore = transaction.objectStore(this.storeName);
 
       const request = objectStore.put({ key, event, timestamp: Date.now() });
 
-      request.onsuccess = () => resolve();
+      request.onsuccess = () => resolve(undefined);
       request.onerror = () => reject(request.error);
     });
   }
 
   async get(key) {
     return new Promise((resolve, reject) => {
+      if (!this.db) {
+        reject(new Error('Database not initialized'));
+        return;
+      }
       const transaction = this.db.transaction([this.storeName], 'readonly');
       const objectStore = transaction.objectStore(this.storeName);
 
@@ -65,20 +76,39 @@ export class IndexedDBStorage extends PersistentStorage {
     });
   }
 
+  /**
+   * @param {string} prefix
+   * @returns {Promise<any[]>}
+   */
   async getAll(prefix) {
     return new Promise((resolve, reject) => {
+      if (!this.db) {
+        reject(new Error('Database not initialized'));
+        return;
+      }
       const transaction = this.db.transaction([this.storeName], 'readonly');
       const objectStore = transaction.objectStore(this.storeName);
 
+      /** @type {any[]} */
       const results = [];
-      const request = objectStore.openCursor();
+
+      // Use IDBKeyRange for efficient prefix query instead of full table scan
+      // This creates a range from "prefix" to "prefix\uffff" (highest unicode char)
+      // which efficiently uses the B-tree index
+      let request;
+      if (prefix) {
+        const range = IDBKeyRange.bound(prefix, prefix + '\uffff', false, false);
+        request = objectStore.openCursor(range);
+      } else {
+        // No prefix = get all
+        request = objectStore.openCursor();
+      }
 
       request.onsuccess = (event) => {
-        const cursor = event.target.result;
+        const target = /** @type {IDBRequest} */ (event.target);
+        const cursor = target.result;
         if (cursor) {
-          if (cursor.value.key.startsWith(prefix)) {
-            results.push(cursor.value.event);
-          }
+          results.push(cursor.value.event);
           cursor.continue();
         } else {
           resolve(results);
@@ -91,24 +121,32 @@ export class IndexedDBStorage extends PersistentStorage {
 
   async delete(key) {
     return new Promise((resolve, reject) => {
+      if (!this.db) {
+        reject(new Error('Database not initialized'));
+        return;
+      }
       const transaction = this.db.transaction([this.storeName], 'readwrite');
       const objectStore = transaction.objectStore(this.storeName);
 
       const request = objectStore.delete(key);
 
-      request.onsuccess = () => resolve();
+      request.onsuccess = () => resolve(undefined);
       request.onerror = () => reject(request.error);
     });
   }
 
   async clear() {
     return new Promise((resolve, reject) => {
+      if (!this.db) {
+        reject(new Error('Database not initialized'));
+        return;
+      }
       const transaction = this.db.transaction([this.storeName], 'readwrite');
       const objectStore = transaction.objectStore(this.storeName);
 
       const request = objectStore.clear();
 
-      request.onsuccess = () => resolve();
+      request.onsuccess = () => resolve(undefined);
       request.onerror = () => reject(request.error);
     });
   }

package/src/storage/memory-storage.js

@@ -32,7 +32,12 @@ export class MemoryStorage extends PersistentStorage {
     return data ? JSON.parse(JSON.stringify(data)) : null; // Deep clone
   }
 
+  /**
+   * @param {string} prefix
+   * @returns {Promise<any[]>}
+   */
   async getAll(prefix) {
+    /** @type {any[]} */
     const results = [];
     for (const [key, value] of this.data.entries()) {
       if (key.startsWith(prefix)) {