holosphere 2.0.0-alpha1 → 2.0.0-alpha10

package/src/storage/gun-wrapper.js

@@ -1,17 +1,30 @@
 /**
- * GunDB Storage Wrapper with radisk persistence
- * Handles path construction and CRUD operations
+ * @fileoverview GunDB Storage Wrapper with radisk persistence.
+ *
+ * Handles path construction and CRUD operations for GunDB storage backend.
+ * Note: GunDB doesn't handle nested objects well, so we store data as JSON strings.
+ * Provides both holon-specific paths and global table operations.
+ *
+ * @module storage/gun-wrapper
  */
 
 import { gunPromise, gunPut, gunCollect } from './gun-async.js';
 
+// ============================================================================
+// PATH BUILDERS
+// ============================================================================
+
 /**
- * Build Gun path from components
+ * Build Gun path from components.
+ *
  * @param {string} appname - Application namespace
  * @param {string} holon - Holon ID (H3 or URI)
  * @param {string} lens - Lens name
- * @param {string} key - Data key (optional)
+ * @param {string} [key=null] - Data key (optional)
  * @returns {string} Gun path
+ * @example
+ * const path = buildPath('myapp', 'holon123', 'items', 'item1');
+ * // Returns: 'myapp/holon123/items/item1'
  */
 export function buildPath(appname, holon, lens, key = null) {
   // Encode components to handle special characters
@@ -26,39 +39,207 @@ export function buildPath(appname, holon, lens, key = null) {
 }
 
 /**
- * Encode path component to handle special characters
+ * Build Gun path for global tables (app-wide data not tied to holons).
+ *
+ * @param {string} appname - Application namespace
+ * @param {string} tableName - Global table name (e.g., 'schemas', 'federation')
+ * @param {string} [key=null] - Data key (optional)
+ * @returns {string} Gun path
+ * @example
+ * const path = buildGlobalPath('myapp', 'schemas', 'user');
+ * // Returns: 'myapp/schemas/user'
+ */
+export function buildGlobalPath(appname, tableName, key = null) {
+  const encodedTable = encodePathComponent(tableName);
+  if (key) {
+    const encodedKey = encodePathComponent(key);
+    return `${appname}/${encodedTable}/${encodedKey}`;
+  }
+  return `${appname}/${encodedTable}`;
+}
+
+/**
+ * Encode path component to handle special characters.
+ *
  * @private
+ * @param {string} component - Component to encode
+ * @returns {string} Encoded component
  */
 function encodePathComponent(component) {
   return encodeURIComponent(component).replace(/%2F/g, '/');
 }
 
 /**
- * Write data to Gun with radisk persistence
+ * Navigate to a Gun path using chained .get() calls.
+ *
+ * Gun treats 'a/b/c' as a literal key, not a path.
+ * This function splits the path and chains .get() calls properly.
+ *
+ * @private
+ * @param {Object} gun - Gun instance
+ * @param {string} path - Path string like "appname/holon/lens/key"
+ * @returns {Object} Gun chain reference at the path
+ */
+function getGunPath(gun, path) {
+  const parts = path.split('/').filter(p => p.length > 0);
+  let ref = gun;
+  for (const part of parts) {
+    ref = ref.get(part);
+  }
+  return ref;
+}
+
+/**
+ * Serialize data for GunDB storage.
+ *
+ * Stores data as raw JSON string for compatibility with holosphere original.
+ * This matches the format used in holosphere v1 for better interoperability.
+ *
+ * @private
+ * @param {Object} data - Data to serialize
+ * @returns {string} JSON string
+ */
+function serializeForGun(data) {
+  return JSON.stringify(data);
+}
+
+/**
+ * Deserialize data from GunDB storage.
+ *
+ * Handles multiple formats:
+ * - Direct JSON string (holosphere original - now default)
+ * - _json wrapped format (holosphere2 legacy)
+ * - Gun internal references (_["#"])
+ * - Gun node data (_[">"])
+ * - Plain objects
+ *
+ * @private
+ * @param {*} data - Raw data from Gun
+ * @returns {Object|null} Parsed data or null
+ */
+function deserializeFromGun(data) {
+  if (!data) {
+    return null;
+  }
+
+  try {
+    // Format 1: String data (holosphere original stores JSON as string directly)
+    if (typeof data === 'string') {
+      try {
+        return JSON.parse(data);
+      } catch (e) {
+        // Not JSON, return as-is
+        return data;
+      }
+    }
+
+    // Format 2: _json wrapped format (holosphere2 default)
+    if (data._json && typeof data._json === 'string') {
+      try {
+        return JSON.parse(data._json);
+      } catch (e) {
+        console.warn('Failed to parse _json field:', e);
+        return null;
+      }
+    }
+
+    // Format 3: Gun internal reference (_["#"]) - this indicates a Gun reference
+    // The actual data retrieval should happen via resolveReference, we just identify it here
+    if (data._ && data._['#']) {
+      // This is a Gun reference - return as-is for reference resolution
+      // The caller should check isReference() and handle appropriately
+      return data;
+    }
+
+    // Format 4: Gun node data with timestamps (_[">"])
+    // Gun stores metadata in _.> - find the actual value
+    if (data._ && data._['>']) {
+      const nodeValue = Object.entries(data).find(([k, v]) => k !== '_' && typeof v === 'string');
+      if (nodeValue) {
+        try {
+          return JSON.parse(nodeValue[1]);
+        } catch (e) {
+          return nodeValue[1];
+        }
+      }
+    }
+
+    // Format 5: Plain object - clean Gun metadata and return
+    if (typeof data === 'object' && data !== null) {
+      const cleaned = { ...data };
+      delete cleaned['_'];
+
+      // Check if any remaining keys - if empty after removing _, return null
+      if (Object.keys(cleaned).length === 0) {
+        return null;
+      }
+
+      return cleaned;
+    }
+
+    return data;
+  } catch (error) {
+    console.warn('Error deserializing Gun data:', error);
+    return data; // Return raw data as fallback
+  }
+}
+
+/**
+ * Write data to Gun with radisk persistence.
+ *
  * @param {Object} gun - Gun instance
  * @param {string} path - Gun path
  * @param {Object} data - Data to write
- * @returns {Promise<boolean>} Success indicator
+ * @returns {Promise<Object>} Success object with ok and timeout flags
+ * @example
+ * const result = await write(gun, 'myapp/holon1/items/item1', { name: 'Test' });
+ * if (result.timeout) console.warn('Write may not be persisted');
  */
 export async function write(gun, path, data) {
   try {
-    await gunPut(gun.get(path), data, 2000);
-    // Delay to allow Gun to propagate the write (50ms for better reliability)
-    await new Promise(resolve => setTimeout(resolve, 50));
-    return true;
+    const serialized = serializeForGun(data);
+    const parts = path.split('/').filter(p => p.length > 0);
+    console.log('[gun-wrapper] write:', { path, parts, dataId: data?.id });
+    const ref = getGunPath(gun, path);
+    console.log('[gun-wrapper] write ref soul:', ref?._.get);
+    const ack = await gunPut(ref, serialized, 5000); // Increased timeout from 2s to 5s
+    console.log('[gun-wrapper] write ack:', { ok: ack.ok, timeout: ack.timeout });
+    if (ack.timeout) {
+      console.warn('[gun-wrapper] write timed out (data may not be persisted):', path);
+    }
+    console.log('[gun-wrapper] write complete:', path);
+
+    // Return ack info so caller can handle timeouts
+    return { ok: true, timeout: ack.timeout || false };
   } catch (error) {
+    console.error('[gun-wrapper] write error:', error);
     throw error;
   }
 }
 
 /**
- * Read data from Gun
+ * Read data from Gun.
+ *
  * @param {Object} gun - Gun instance
  * @param {string} path - Gun path
- * @returns {Promise<Object|null>} Data or null if not found
+ * @returns {Promise<Object|null>} Data or null if not found or deleted
+ * @example
+ * const data = await read(gun, 'myapp/holon1/items/item1');
+ * if (data) console.log(data.name);
  */
 export async function read(gun, path) {
-  const data = await gunPromise(gun.get(path), 2000);
+  const parts = path.split('/').filter(p => p.length > 0);
+  console.log('[gun-wrapper] read:', { path, parts });
+  const ref = getGunPath(gun, path);
+  console.log('[gun-wrapper] read ref soul:', ref?._.get);
+  const rawData = await gunPromise(ref, 2000);
+  console.log('[gun-wrapper] read rawData:', rawData ? (typeof rawData === 'string' ? rawData.substring(0, 100) : 'object') : 'null');
+
+  if (!rawData) {
+    return null;
+  }
+
+  const data = deserializeFromGun(rawData);
 
   // Return null if deleted or not found
   if (!data || data._deleted) {
@@ -69,48 +250,158 @@ export async function read(gun, path) {
 }
 
 /**
- * Read all data under a path (lens query)
+ * Read all data under a path (lens query).
+ *
+ * First gets the count of expected items, then collects until count is reached.
+ *
  * @param {Object} gun - Gun instance
  * @param {string} path - Gun path
+ * @param {number} [timeout=5000] - Maximum timeout in ms
  * @returns {Promise<Object[]>} Array of data objects
+ * @example
+ * const items = await readAll(gun, 'myapp/holon1/items');
+ * console.log(`Found ${items.length} items`);
  */
-export async function readAll(gun, path) {
-  const results = await gunCollect(gun.get(path), 1000);
+export async function readAll(gun, path, timeout = 5000) {
+  const parts = path.split('/').filter(p => p.length > 0);
+  console.log('[gun-wrapper] readAll:', { path, parts });
+
+  return new Promise((resolve) => {
+    const output = new Map();
+    let settled = false;
+    let expectedCount = 0;
+    let receivedCount = 0;
+
+    const ref = getGunPath(gun, path);
+    console.log('[gun-wrapper] readAll ref soul:', ref?._.get);
+
+    const tryResolve = () => {
+      if (settled) return;
+      if (expectedCount > 0 && receivedCount >= expectedCount) {
+        settled = true;
+        console.log('[gun-wrapper] readAll resolved with', output.size, 'items');
+        resolve(Array.from(output.values()));
+      }
+    };
+
+    const parseItem = (data) => {
+      return deserializeFromGun(data);
+    };
+
+    // Step 1: Get the parent data to count expected items
+    ref.once((parentData) => {
+      if (settled) return;
+      console.log('[gun-wrapper] readAll parentData:', parentData);
+      console.log('[gun-wrapper] readAll parentData keys:', parentData ? Object.keys(parentData).filter(k => k !== '_') : 'null');
+      console.log('[gun-wrapper] readAll parentData type:', typeof parentData);
+
+      if (!parentData) {
+        settled = true;
+        console.log('[gun-wrapper] readAll: no parent data, returning empty');
+        resolve([]);
+        return;
+      }
+
+      // Get all keys except Gun metadata
+      const keys = Object.keys(parentData).filter(k => k !== '_');
+      console.log('[gun-wrapper] readAll keys:', keys);
+
+      if (keys.length === 0) {
+        settled = true;
+        console.log('[gun-wrapper] readAll: no keys, returning empty');
+        resolve([]);
+        return;
+      }
+
+      // Count expected items and check for inline data
+      const referenceKeys = [];
+
+      for (const key of keys) {
+        const rawItem = parentData[key];
+        if (!rawItem) continue;
+
+        // Check if it's a Gun reference (soul) - need to fetch separately
+        if (typeof rawItem === 'object' && rawItem['#']) {
+          referenceKeys.push(key);
+          continue;
+        }
+
+        // Try to parse inline data (don't count yet - will count in map().once() phase)
+        const item = parseItem(rawItem);
+        if (item && item.id && !item._deleted) {
+          output.set(item.id, item);
+        }
+      }
+
+      // Set expected count: ALL keys that could have data (references + inline)
+      // We use total keys because map().once() will fire for all of them
+      expectedCount = keys.length;
+
+      // If no keys, we're done (shouldn't happen but be safe)
+      if (expectedCount === 0) {
+        settled = true;
+        resolve(Array.from(output.values()));
+        return;
+      }
+
+      // Step 2: Use map().once() to resolve all items, counting as we go
+      ref.map().once((data, key) => {
+        if (settled || !data || key === '_') return;
+
+        const item = parseItem(data);
+        if (item && item.id && !item._deleted) {
+          // Add to output if not already there (inline items already added)
+          if (!output.has(item.id)) {
+            output.set(item.id, item);
+          }
+        }
+        // Count every item received (inline or reference)
+        receivedCount++;
+        tryResolve();
+      });
+    });
 
-  // Filter out deleted items and Gun metadata
-  return results
-    .filter(({ data, key }) => {
-      if (!data || typeof data !== 'object') return false;
-      if (key.startsWith('_')) return false;
-      if (data._deleted) return false;
-      return true;
-    })
-    .map(({ data }) => data);
+    // Fallback timeout in case count-based resolution fails
+    setTimeout(() => {
+      if (!settled) {
+        settled = true;
+        resolve(Array.from(output.values()));
+      }
+    }, timeout);
+  });
 }
 
 /**
- * Update data (merge fields)
+ * Update data (merge fields).
+ *
  * @param {Object} gun - Gun instance
  * @param {string} path - Gun path
  * @param {Object} updates - Fields to update
  * @returns {Promise<boolean>} Success indicator
+ * @example
+ * const success = await update(gun, 'myapp/holon1/items/item1', { status: 'active' });
  */
 export async function update(gun, path, updates) {
-  const existing = await gunPromise(gun.get(path));
+  const rawData = await gunPromise(getGunPath(gun, path));
 
-  if (!existing || !existing.id || existing._deleted) {
-    return false; // Not found or deleted
+  if (!rawData) {
+    return false; // Not found
   }
 
-  // Remove Gun metadata before merging
-  const cleanExisting = { ...existing };
-  delete cleanExisting['_'];
+  // Deserialize existing data
+  const existing = deserializeFromGun(rawData);
+  if (!existing || !existing.id || existing._deleted) {
+    return false;
+  }
 
   // Merge updates
-  const merged = { ...cleanExisting, ...updates };
+  const merged = { ...existing, ...updates };
 
   try {
-    await gunPut(gun.get(path), merged);
+    const serialized = serializeForGun(merged);
+    await gunPut(getGunPath(gun, path), serialized, 2000);
+    // Add delay for propagation
+    await new Promise(resolve => setTimeout(resolve, 200));
     return true;
   } catch (error) {
     throw error;
@@ -118,28 +409,34 @@ export async function update(gun, path, updates) {
 }
 
 /**
- * Delete data (tombstone)
+ * Delete data (tombstone).
+ *
  * @param {Object} gun - Gun instance
  * @param {string} path - Gun path
  * @returns {Promise<boolean>} Success indicator
+ * @example
+ * const deleted = await deleteData(gun, 'myapp/holon1/items/item1');
  */
 export async function deleteData(gun, path) {
   try {
-    // Gun requires tombstone to be an object with _deleted flag
-    // First read existing data to preserve metadata
-    const existing = await gunPromise(gun.get(path));
-    if (!existing) {
+    // First read existing data to get the id
+    const rawData = await gunPromise(getGunPath(gun, path));
+    if (!rawData) {
       return true; // Already deleted/doesn't exist
     }
 
-    // Create tombstone object
+    const existing = deserializeFromGun(rawData);
+
+    // Create tombstone object and serialize it
     const tombstone = {
-      id: existing.id,
+      id: existing?.id,
       _deleted: true,
       _deletedAt: Date.now()
     };
 
-    await gunPut(gun.get(path), tombstone);
+    await gunPut(getGunPath(gun, path), serializeForGun(tombstone), 2000);
+    // Add delay for propagation
+    await new Promise(resolve => setTimeout(resolve, 200));
     return true;
   } catch (error) {
     throw error;
@@ -147,7 +444,8 @@ export async function deleteData(gun, path) {
 }
 
 /**
- * Delete all data under path prefix (tombstone)
+ * Delete all data under path prefix (tombstone).
+ *
  * @param {Object} gun - Gun instance
  * @param {string} path - Gun path prefix
  * @returns {Promise<Object>} Deletion results { success: boolean, count: number }
@@ -168,13 +466,19 @@ export async function deleteAll(gun, path) {
 }
 
 /**
- * Subscribe to data changes
+ * Subscribe to data changes.
+ *
  * @param {Object} gun - Gun instance
  * @param {string} path - Gun path
- * @param {Function} callback - Called on data changes
- * @param {Object} options - Subscription options
- * @param {boolean} options.prefix - Subscribe to all items under path (default: auto-detect)
+ * @param {Function} callback - Called on data changes (data, key) => void
+ * @param {Object} [options={}] - Subscription options
+ * @param {boolean} [options.prefix] - Subscribe to all items under path (default: auto-detect)
  * @returns {Object} Subscription object with unsubscribe method
+ * @example
+ * const sub = subscribe(gun, 'myapp/holon1/items', (data, key) => {
+ *   console.log('Item changed:', key, data);
+ * });
+ * // Later: sub.unsubscribe();
  */
 export function subscribe(gun, path, callback, options = {}) {
   // Detect if this is a prefix subscription
@@ -183,11 +487,14 @@ export function subscribe(gun, path, callback, options = {}) {
 
   if (isPrefix) {
     // Subscribe to all items under this prefix
-    const ref = gun.get(path);
+    const ref = getGunPath(gun, path);
 
     ref.map().on((data, key) => {
-      if (data && !key.startsWith('_') && !data._deleted) {
-        callback(data, key);
+      if (data && !key.startsWith('_')) {
+        const deserialized = deserializeFromGun(data);
+        if (deserialized && !deserialized._deleted) {
+          callback(deserialized, key);
+        }
       }
     });
 
@@ -202,9 +509,12 @@ export function subscribe(gun, path, callback, options = {}) {
     };
   } else {
     // Subscribe to single item
-    const listener = gun.get(path).on((data, key) => {
-      if (data && !data._deleted) {
-        callback(data, key);
+    const listener = getGunPath(gun, path).on((data, key) => {
+      if (data) {
+        const deserialized = deserializeFromGun(data);
+        if (deserialized && !deserialized._deleted) {
+          callback(deserialized, key);
+        }
       }
     });
 
@@ -219,3 +529,114 @@ export function subscribe(gun, path, callback, options = {}) {
     };
   }
 }
+
+// ============================================================================
+// GLOBAL TABLE OPERATIONS
+// ============================================================================
+
+/**
+ * Write data to a global table.
+ *
+ * Global tables are app-wide data not tied to specific holons (e.g., schemas, federation).
+ *
+ * @param {Object} gun - Gun instance
+ * @param {string} appname - Application namespace
+ * @param {string} tableName - Global table name
+ * @param {Object} data - Data to write (must have 'id' field)
+ * @returns {Promise<Object>} Success object with ok and timeout flags
+ * @throws {Error} If data doesn't have an id field
+ * @example
+ * await writeGlobal(gun, 'myapp', 'schemas', { id: 'user', type: 'object' });
+ */
+export async function writeGlobal(gun, appname, tableName, data) {
+  if (!data || !data.id) {
+    throw new Error('writeGlobal: data must have an id field');
+  }
+  const path = buildGlobalPath(appname, tableName, data.id);
+  // Use write function which includes the propagation delay
+  return write(gun, path, data);
+}
+
+/**
+ * Read data from a global table.
+ *
+ * @param {Object} gun - Gun instance
+ * @param {string} appname - Application namespace
+ * @param {string} tableName - Global table name
+ * @param {string} key - Data key
+ * @returns {Promise<Object|null>} Data or null if not found
+ */
+export async function readGlobal(gun, appname, tableName, key) {
+  const path = buildGlobalPath(appname, tableName, key);
+  return read(gun, path);
+}
+
+/**
+ * Read all data from a global table.
+ *
+ * Uses same approach as readAll.
+ *
+ * @param {Object} gun - Gun instance
+ * @param {string} appname - Application namespace
+ * @param {string} tableName - Global table name
+ * @param {number} [timeout=2000] - Timeout in ms
+ * @returns {Promise<Object[]>} Array of data objects
+ */
+export async function readAllGlobal(gun, appname, tableName, timeout = 2000) {
+  const path = buildGlobalPath(appname, tableName);
+  return readAll(gun, path);
+}
+
+/**
+ * Delete data from a global table
+ * @param {Object} gun - Gun instance
+ * @param {string} appname - Application namespace
+ * @param {string} tableName - Global table name
+ * @param {string} key - Data key
+ * @returns {Promise<boolean>} Success indicator
+ */
+export async function deleteGlobal(gun, appname, tableName, key) {
+  const path = buildGlobalPath(appname, tableName, key);
+  return deleteData(gun, path);
+}
+
+/**
+ * Delete all data from a global table
+ * @param {Object} gun - Gun instance
+ * @param {string} appname - Application namespace
+ * @param {string} tableName - Global table name
+ * @returns {Promise<Object>} Deletion results { success: boolean, count: number }
+ */
+export async function deleteAllGlobal(gun, appname, tableName) {
+  const path = buildGlobalPath(appname, tableName);
+  return deleteAll(gun, path);
+}
+
+// ============================================================================
+// DATA PARSING UTILITIES
+// ============================================================================
+
+/**
+ * Parse data from Gun storage, handling various formats.
+ *
+ * Handles:
+ * - JSON string in _json field
+ * - Legacy object format
+ * - Gun references
+ *
+ * @param {*} rawData - Raw data from Gun
+ * @returns {Object|null} Parsed data or null
+ */
+export function parse(rawData) {
+  return deserializeFromGun(rawData);
+}
+
+/**
+ * Serialize data for Gun storage.
+ *
+ * @param {Object} data - Data to serialize
+ * @returns {string} JSON string
+ */
+export function serialize(data) {
+  return serializeForGun(data);
+}