holosphere 2.0.0-alpha7 → 2.0.0-alpha9

package/src/storage/backend-interface.js

@@ -1,8 +1,27 @@
 /**
- * Abstract Storage Backend Interface
- * All storage backends must implement this interface
+ * @fileoverview Abstract Storage Backend Interface.
+ *
+ * Defines the contract that all storage backends must implement.
+ * Provides a consistent API for CRUD operations, subscriptions,
+ * and data migration across different storage technologies.
+ *
+ * @module storage/backend-interface
+ */
+
+/**
+ * Abstract base class for storage backends.
+ * All concrete backends (Nostr, GunDB, ActivityPub) must extend this class.
+ *
+ * @abstract
+ * @class StorageBackend
  */
 export class StorageBackend {
+  /**
+   * Creates a new StorageBackend instance.
+   *
+   * @param {Object} config - Backend-specific configuration
+   * @throws {Error} If instantiated directly (abstract class)
+   */
   constructor(config) {
     if (new.target === StorageBackend) {
       throw new Error('StorageBackend is abstract and cannot be instantiated directly');

package/src/storage/backends/activitypub/server.js

@@ -1,19 +1,41 @@
 /**
- * Self-hosted ActivityPub server for HoloSphere
- * Provides federation with Mastodon, Pleroma, and other ActivityPub servers
+ * @fileoverview Self-hosted ActivityPub server for HoloSphere federation.
+ * Provides complete ActivityPub protocol support for federation with Mastodon, Pleroma,
+ * and other fediverse servers. Includes WebFinger, NodeInfo, actor endpoints, and SSE subscriptions.
+ * @module storage/backends/activitypub/server
  */
 
 import express from 'express';
 import crypto from 'crypto';
 import { createPersistentStorage } from '../../persistent-storage.js';
 
+/** @constant {string} ActivityStreams JSON-LD context */
 const ACTIVITY_STREAMS_CONTEXT = 'https://www.w3.org/ns/activitystreams';
+/** @constant {string} Security vocabulary JSON-LD context */
 const SECURITY_CONTEXT = 'https://w3id.org/security/v1';
 
 /**
- * ActivityPub Server for HoloSphere
+ * Self-hosted ActivityPub server for HoloSphere data federation.
+ * Implements WebFinger discovery, actor endpoints, inbox/outbox, and HoloSphere data API.
+ *
+ * @class ActivityPubServer
+ * @example
+ * const server = new ActivityPubServer({
+ *   port: 3000,
+ *   domain: 'holon.example.com',
+ *   dataDir: './data'
+ * });
+ * await server.start();
  */
 export class ActivityPubServer {
+  /**
+   * Create a new ActivityPub server instance.
+   * @param {Object} [config={}] - Server configuration
+   * @param {number} [config.port=3000] - HTTP port to listen on
+   * @param {string} [config.domain='localhost'] - Server domain for URLs
+   * @param {string} [config.protocol='http'] - Protocol (http or https)
+   * @param {string} [config.dataDir] - Data directory for persistent storage
+   */
   constructor(config = {}) {
     this.config = {
       port: config.port || 3000,

package/src/storage/backends/activitypub-backend.js

@@ -1,11 +1,34 @@
 /**
- * ActivityPub Storage Backend
- * Client adapter for connecting to an ActivityPub server
+ * @fileoverview ActivityPub storage backend client adapter.
+ * Provides HTTP-based connectivity to an ActivityPub server for federated data storage
+ * and real-time subscriptions via Server-Sent Events.
+ * @module storage/backends/activitypub-backend
  */
 
 import { StorageBackend } from '../backend-interface.js';
 
+/**
+ * ActivityPub storage backend implementation.
+ * Client adapter for connecting to an ActivityPub server via HTTP API
+ * with SSE-based real-time subscriptions.
+ *
+ * @class ActivityPubBackend
+ * @extends StorageBackend
+ * @example
+ * const backend = await BackendFactory.create('activitypub', {
+ *   serverUrl: 'https://ap.example.com',
+ *   appName: 'myapp',
+ *   apiKey: 'secret-key'
+ * });
+ */
 export class ActivityPubBackend extends StorageBackend {
+  /**
+   * Create a new ActivityPub backend instance.
+   * @param {Object} config - Backend configuration
+   * @param {string} config.serverUrl - ActivityPub server URL
+   * @param {string} [config.appName='holosphere'] - Application/actor name
+   * @param {string} [config.apiKey] - Optional API key for authentication
+   */
   constructor(config) {
     super(config);
     this.serverUrl = config.serverUrl;

package/src/storage/backends/gundb-backend.js

@@ -1,6 +1,7 @@
 /**
- * GunDB Storage Backend
- * Full-featured backend with authentication, schema validation, references, and global tables
+ * @fileoverview GunDB storage backend with authentication, schema validation, references, and global tables.
+ * Provides full-featured P2P data storage using GunDB with write caching for immediate consistency.
+ * @module storage/backends/gundb-backend
  */
 
 import { StorageBackend } from '../backend-interface.js';
@@ -10,7 +11,33 @@ import { GunReferenceHandler } from '../gun-references.js';
 import { GunAuth } from '../gun-auth.js';
 import { GunSchemaValidator } from '../gun-schema.js';
 
+/**
+ * GunDB storage backend implementation with advanced features.
+ * Supports authentication, schema validation, references, global tables,
+ * and write caching for immediate consistency.
+ *
+ * @class GunDBBackend
+ * @extends StorageBackend
+ * @example
+ * const backend = await BackendFactory.create('gundb', {
+ *   appName: 'myapp',
+ *   peers: ['wss://gun-server.example.com/gun'],
+ *   radisk: true
+ * });
+ */
 export class GunDBBackend extends StorageBackend {
+  /**
+   * Create a new GunDB backend instance.
+   * @param {Object} config - Backend configuration
+   * @param {string} [config.appName='holosphere'] - Application namespace
+   * @param {string[]} [config.peers=[]] - Array of peer URLs
+   * @param {boolean} [config.radisk=true] - Enable radisk persistence
+   * @param {boolean} [config.localStorage=true] - Enable localStorage
+   * @param {string} [config.dataDir='radata'] - Data directory for Node.js
+   * @param {string} [config.privateKey] - Optional existing private key
+   * @param {string} [config.publicKey] - Optional existing public key
+   * @param {boolean} [config.strict=false] - Enable strict schema validation
+   */
   constructor(config) {
     super(config);
     this.gun = null;
@@ -25,28 +52,61 @@ export class GunDBBackend extends StorageBackend {
     // Subscription tracking
     this.subscriptions = new Map();
     this.subscriptionCounter = 0;
+
+    // Write cache for immediate consistency
+    // Gun's readAll/map().once() doesn't immediately see new writes - this cache bridges that gap
+    this.writeCache = new Map(); // path -> Map(id -> {data, timestamp})
+    this.writeCacheTTL = 300000; // Cache entries expire after 5 minutes (increased from 30s)
+
+    // Pending writes queue for retry
+    this.pendingWrites = new Map(); // path -> {data, retries, lastAttempt}
+    this.maxWriteRetries = 5;
+    this.writeRetryInterval = 10000; // 10 seconds between retries
   }
 
   async init() {
     // Dynamically import Gun to avoid issues if not installed
     let Gun;
     try {
+      console.log('[gundb-backend] Importing Gun...');
       const gunModule = await import('gun');
       Gun = gunModule.default || gunModule;
+      console.log('[gundb-backend] Gun imported:', typeof Gun);
     } catch (error) {
       throw new Error(
         'GunDB backend requires the "gun" package. Install it with: npm install gun'
       );
     }
 
+    // In Node.js, Gun needs a file path for radisk persistence
+    // Default to 'radata' if not specified
+    const dataDir = this.config.dataDir || 'radata';
+
     const gunConfig = {
       peers: this.config.peers || [],
       radisk: this.config.radisk !== false,
       localStorage: this.config.localStorage !== false,
-      file: this.config.dataDir,
+      file: dataDir,
     };
 
+    console.log('[gundb-backend] Gun config:', JSON.stringify(gunConfig));
+
     this.gun = Gun(gunConfig);
+    console.log('[gundb-backend] Gun instance created:', !!this.gun);
+
+    // Test basic Gun functionality
+    try {
+      const testKey = `_test_${Date.now()}`;
+      this.gun.get(testKey).put({ test: true }, (ack) => {
+        console.log('[gundb-backend] Gun test write ack:', ack);
+      });
+      console.log('[gundb-backend] Gun test write initiated');
+    } catch (e) {
+      console.error('[gundb-backend] Gun test write failed:', e.message);
+    }
+
+    // Add a small delay to let Gun initialize
+    await new Promise(resolve => setTimeout(resolve, 500));
 
     // Generate or use provided key pair using Gun's SEA
     try {
@@ -82,6 +142,111 @@ export class GunDBBackend extends StorageBackend {
     await this.schemaValidator.init();
   }
 
+  // ============================================================================
+  // WRITE CACHE (for immediate consistency)
+  // ============================================================================
+
+  /**
+   * Add item to the local write cache for immediate consistency
+   * @private
+   */
+  _addToWriteCache(path, data) {
+    if (!data || !data.id) return;
+
+    // Extract the parent path (without the data id)
+    const parentPath = path.substring(0, path.lastIndexOf('/'));
+    if (!parentPath) return;
+
+    if (!this.writeCache.has(parentPath)) {
+      this.writeCache.set(parentPath, new Map());
+    }
+
+    const pathCache = this.writeCache.get(parentPath);
+    pathCache.set(data.id.toString(), {
+      data: data,
+      timestamp: Date.now()
+    });
+  }
+
+  /**
+   * Remove item from write cache (e.g., on delete)
+   * @private
+   */
+  _removeFromWriteCache(path, id) {
+    // Extract the parent path
+    const parentPath = path.substring(0, path.lastIndexOf('/'));
+    const pathCache = this.writeCache.get(parentPath);
+    if (pathCache) {
+      pathCache.delete(id.toString());
+    }
+  }
+
+  /**
+   * Get cached writes for a path, filtering out expired entries
+   * @private
+   */
+  _getWriteCacheEntries(path) {
+    const pathCache = this.writeCache.get(path);
+    if (!pathCache) return [];
+
+    const now = Date.now();
+    const validEntries = [];
+
+    for (const [id, entry] of pathCache) {
+      if (now - entry.timestamp < this.writeCacheTTL) {
+        validEntries.push(entry.data);
+      } else {
+        // Clean up expired entries
+        pathCache.delete(id);
+      }
+    }
+
+    return validEntries;
+  }
+
+  /**
+   * Merge cached writes with results from Gun
+   * @private
+   */
+  _mergeWithWriteCache(path, gunResults) {
+    const cachedWrites = this._getWriteCacheEntries(path);
+    if (cachedWrites.length === 0) return gunResults;
+
+    // Create a map by ID for deduplication
+    const resultMap = new Map();
+
+    // Add Gun results first
+    for (const item of gunResults) {
+      if (item && item.id) {
+        resultMap.set(item.id.toString(), item);
+      }
+    }
+
+    // Overlay cached writes (they're more recent)
+    for (const item of cachedWrites) {
+      if (item && item.id && !item._deleted) {
+        resultMap.set(item.id.toString(), item);
+      } else if (item && item.id && item._deleted) {
+        // Remove deleted items
+        resultMap.delete(item.id.toString());
+      }
+    }
+
+    return Array.from(resultMap.values());
+  }
+
+  /**
+   * Clear write cache for a specific path or all paths
+   * @param {string} [path] - Optional path to clear, clears all if not provided
+   */
+  clearWriteCache(path = null) {
+    if (path) {
+      this.writeCache.delete(path);
+    } else {
+      this.writeCache.clear();
+    }
+  }
+
   // ============================================================================
   // PATH BUILDING
   // ============================================================================
@@ -116,7 +281,59 @@ export class GunDBBackend extends StorageBackend {
       }
     }
 
-    return wrapper.write(this.gun, path, data);
+    // Add to write cache FIRST for immediate consistency
+    // This ensures readAll returns the new data even if Gun write is slow/times out
+    this._addToWriteCache(path, data);
+
+    // Try to write to Gun with retry on timeout
+    const result = await this._writeWithRetry(path, data);
+
+    return result;
+  }
+
+  /**
+   * Write to Gun with automatic retry on timeout
+   * @private
+   */
+  async _writeWithRetry(path, data, attempt = 0) {
+    const result = await wrapper.write(this.gun, path, data);
+
+    if (result.timeout && attempt < this.maxWriteRetries) {
+      // Queue for background retry
+      this.pendingWrites.set(path, {
+        data,
+        retries: attempt + 1,
+        lastAttempt: Date.now()
+      });
+
+      // Schedule retry
+      this._scheduleRetry(path);
+    } else if (!result.timeout) {
+      // Success - remove from pending
+      this.pendingWrites.delete(path);
+    }
+
+    return result;
+  }
+
+  /**
+   * Schedule a background retry for a pending write
+   * @private
+   */
+  _scheduleRetry(path) {
+    setTimeout(async () => {
+      const pending = this.pendingWrites.get(path);
+      if (!pending) return;
+
+      if (pending.retries >= this.maxWriteRetries) {
+        console.warn(`[gundb-backend] Max retries reached for: ${path}`);
+        this.pendingWrites.delete(path);
+        return;
+      }
+
+      console.log(`[gundb-backend] Retrying write (attempt ${pending.retries + 1}): ${path}`);
+      await this._writeWithRetry(path, pending.data, pending.retries);
+    }, this.writeRetryInterval);
   }
 
   async read(path, options = {}) {
@@ -131,7 +348,10 @@ export class GunDBBackend extends StorageBackend {
   }
 
   async readAll(path, options = {}) {
-    const items = await wrapper.readAll(this.gun, path);
+    const gunItems = await wrapper.readAll(this.gun, path);
+
+    // Merge with write cache for immediate consistency
+    const items = this._mergeWithWriteCache(path, gunItems);
 
     // Resolve references if requested
     if (options.resolveReferences) {
@@ -157,7 +377,24 @@ export class GunDBBackend extends StorageBackend {
   }
 
   async delete(path) {
-    return wrapper.deleteData(this.gun, path);
+    const result = await wrapper.deleteData(this.gun, path);
+
+    // Extract id from path and mark as deleted in cache
+    const pathParts = path.split('/');
+    const id = pathParts[pathParts.length - 1];
+    if (id) {
+      // Add tombstone to cache so readAll doesn't return this item
+      const parentPath = path.substring(0, path.lastIndexOf('/'));
+      if (!this.writeCache.has(parentPath)) {
+        this.writeCache.set(parentPath, new Map());
+      }
+      this.writeCache.get(parentPath).set(id.toString(), {
+        data: { id, _deleted: true },
+        timestamp: Date.now()
+      });
+    }
+
+    return result;
   }
 
   async deleteAll(path) {
@@ -180,7 +417,62 @@ export class GunDBBackend extends StorageBackend {
   // ============================================================================
 
   async writeGlobal(tableName, data) {
-    return wrapper.writeGlobal(this.gun, this.appName, tableName, data);
+    // Add to write cache FIRST for immediate consistency
+    // This ensures readAllGlobal returns the new data even if Gun write is slow/times out
+    const path = this.buildGlobalPath(tableName, data.id);
+    this._addToWriteCache(path, data);
+
+    // Try to write to Gun with retry on timeout
+    const result = await this._writeGlobalWithRetry(tableName, data);
+
+    return result;
+  }
+
+  /**
+   * Write to global table with automatic retry on timeout
+   * @private
+   */
+  async _writeGlobalWithRetry(tableName, data, attempt = 0) {
+    const path = this.buildGlobalPath(tableName, data.id);
+    const result = await wrapper.writeGlobal(this.gun, this.appName, tableName, data);
+
+    if (result.timeout && attempt < this.maxWriteRetries) {
+      // Queue for background retry
+      this.pendingWrites.set(path, {
+        data: { tableName, data },
+        isGlobal: true,
+        retries: attempt + 1,
+        lastAttempt: Date.now()
+      });
+
+      // Schedule retry
+      this._scheduleGlobalRetry(path, tableName);
+    } else if (!result.timeout) {
+      // Success - remove from pending
+      this.pendingWrites.delete(path);
+    }
+
+    return result;
+  }
+
+  /**
+   * Schedule a background retry for a pending global write
+   * @private
+   */
+  _scheduleGlobalRetry(path, tableName) {
+    setTimeout(async () => {
+      const pending = this.pendingWrites.get(path);
+      if (!pending || !pending.isGlobal) return;
+
+      if (pending.retries >= this.maxWriteRetries) {
+        console.warn(`[gundb-backend] Max retries reached for global: ${path}`);
+        this.pendingWrites.delete(path);
+        return;
+      }
+
+      console.log(`[gundb-backend] Retrying global write (attempt ${pending.retries + 1}): ${path}`);
+      await this._writeGlobalWithRetry(tableName, pending.data.data, pending.retries);
+    }, this.writeRetryInterval);
   }
 
   async readGlobal(tableName, key) {
@@ -188,11 +480,27 @@ export class GunDBBackend extends StorageBackend {
   }
 
   async readAllGlobal(tableName, timeout = 5000) {
-    return wrapper.readAllGlobal(this.gun, this.appName, tableName, timeout);
+    const gunItems = await wrapper.readAllGlobal(this.gun, this.appName, tableName, timeout);
+
+    // Merge with write cache for immediate consistency
+    const path = this.buildGlobalPath(tableName);
+    return this._mergeWithWriteCache(path, gunItems);
   }
 
   async deleteGlobal(tableName, key) {
-    return wrapper.deleteGlobal(this.gun, this.appName, tableName, key);
+    const result = await wrapper.deleteGlobal(this.gun, this.appName, tableName, key);
+
+    // Mark as deleted in cache
+    const path = this.buildGlobalPath(tableName);
+    if (!this.writeCache.has(path)) {
+      this.writeCache.set(path, new Map());
+    }
+    this.writeCache.get(path).set(key.toString(), {
+      data: { id: key, _deleted: true },
+      timestamp: Date.now()
+    });
+
+    return result;
   }
 
   async deleteAllGlobal(tableName) {
@@ -517,12 +825,15 @@ export class GunDBBackend extends StorageBackend {
       this.schemaValidator.clearCache();
     }
 
-    // 3. Logout auth
+    // 3. Clear write cache
+    this.writeCache.clear();
+
+    // 4. Logout auth
     if (this.auth) {
       this.auth.logout();
     }
 
-    // 4. Clean up Gun connections
+    // 5. Clean up Gun connections
     if (this.gun) {
       try {
         // Clean up mesh connections