holosphere 1.1.20 → 2.0.0-alpha1

package/src/storage/nostr-wrapper.js

@@ -0,0 +1,211 @@
+/**
+ * Nostr Storage Wrapper
+ * Drop-in replacement for gun-wrapper.js with same API
+ * Handles path construction and CRUD operations using Nostr relays
+ */
+
+import {
+  nostrPut,
+  nostrGet,
+  nostrGetAll,
+  nostrGetAllHybrid,
+  nostrUpdate as nostrAsyncUpdate,
+  nostrDelete,
+  nostrDeleteAll,
+  nostrSubscribe,
+  nostrSubscribeMany,
+} from './nostr-async.js';
+
+/**
+ * Build path from components
+ * @param {string} appname - Application namespace
+ * @param {string} holon - Holon ID (H3 or URI)
+ * @param {string} lens - Lens name
+ * @param {string} key - Data key (optional)
+ * @returns {string} Path string
+ */
+export function buildPath(appname, holon, lens, key = null) {
+  // Encode components to handle special characters
+  const encodedHolon = encodePathComponent(holon);
+  const encodedLens = encodePathComponent(lens);
+
+  if (key) {
+    const encodedKey = encodePathComponent(key);
+    return `${appname}/${encodedHolon}/${encodedLens}/${encodedKey}`;
+  }
+  return `${appname}/${encodedHolon}/${encodedLens}`;
+}
+
+/**
+ * Encode path component to handle special characters
+ * @private
+ */
+function encodePathComponent(component) {
+  return encodeURIComponent(component).replace(/%2F/g, '/');
+}
+
+/**
+ * Write data to Nostr
+ * @param {Object} client - NostrClient instance
+ * @param {string} path - Path
+ * @param {Object} data - Data to write
+ * @returns {Promise<boolean>} Success indicator
+ */
+export async function write(client, path, data) {
+  try {
+    const result = await nostrPut(client, path, data);
+    // Check if at least one relay accepted the event
+    // If no relays (testing mode), consider it successful if event was created
+    const success = result.results.length === 0 ? true : result.results.some(r => r.status === 'fulfilled');
+    return success;
+  } catch (error) {
+    console.error('Nostr write error:', error);
+    throw error;
+  }
+}
+
+/**
+ * Read data from Nostr
+ * @param {Object} client - NostrClient instance
+ * @param {string} path - Path
+ * @param {Object} options - Query options
+ * @param {string[]} options.authors - Array of public keys to query (for cross-holosphere reads)
+ * @param {boolean} options.includeAuthor - If true, adds _author field to returned data
+ * @returns {Promise<Object|null>} Data or null if not found
+ */
+export async function read(client, path, options = {}) {
+  const data = await nostrGet(client, path, 30000, options);
+
+  // Return null if deleted or not found
+  if (!data || data._deleted) {
+    return null;
+  }
+
+  return data;
+}
+
+/**
+ * Read all data under a path (lens query)
+ * @param {Object} client - NostrClient instance
+ * @param {string} path - Path prefix
+ * @param {Object} options - Query options
+ * @param {boolean} options.hybrid - Use hybrid mode (local + relay) - default: false
+ * @param {string[]} options.authors - Array of public keys to query (for cross-holosphere reads)
+ * @param {boolean} options.includeAuthor - If true, adds _author field to returned data
+ * @returns {Promise<Object[]>} Array of data objects
+ */
+export async function readAll(client, path, options = {}) {
+  // Use hybrid mode if enabled and client supports it
+  const useHybrid = options.hybrid && typeof client.queryHybrid === 'function';
+  const queryFn = useHybrid ? nostrGetAllHybrid : nostrGetAll;
+
+  const results = await queryFn(client, path, 30000, options);
+
+  // Filter out deleted items
+  return results.filter(data => {
+    if (!data || typeof data !== 'object') return false;
+    if (data._deleted) return false;
+    return true;
+  });
+}
+
+/**
+ * Update data (merge fields)
+ * @param {Object} client - NostrClient instance
+ * @param {string} path - Path
+ * @param {Object} updates - Fields to update
+ * @returns {Promise<boolean>} Success indicator
+ */
+export async function update(client, path, updates) {
+  try {
+    const existing = await nostrGet(client, path);
+
+    if (!existing || !existing.id || existing._deleted) {
+      return false; // Not found or deleted
+    }
+
+    // Merge updates
+    const merged = { ...existing, ...updates };
+
+    // Update timestamp
+    if (!merged._meta) merged._meta = {};
+    merged._meta.timestamp = Date.now();
+
+    // Write merged data
+    const result = await nostrPut(client, path, merged);
+    // If no relays (testing mode), consider it successful if event was created
+    const success = result.results.length === 0 ? true : result.results.some(r => r.status === 'fulfilled');
+    return success;
+  } catch (error) {
+    console.error('Nostr update error:', error);
+    throw error;
+  }
+}
+
+/**
+ * Delete data (tombstone)
+ * @param {Object} client - NostrClient instance
+ * @param {string} path - Path
+ * @returns {Promise<boolean>} Success indicator
+ */
+export async function deleteData(client, path) {
+  try {
+    const result = await nostrDelete(client, path);
+
+    if (result.reason === 'not_found') {
+      return false; // Item doesn't exist
+    }
+
+    // If no relays (testing mode), consider it successful if event was created
+    const success = result.results.length === 0 ? true : result.results.some(r => r.status === 'fulfilled');
+    return success;
+  } catch (error) {
+    console.error('Nostr delete error:', error);
+    throw error;
+  }
+}
+
+/**
+ * Delete all data under path prefix (tombstone)
+ * @param {Object} client - NostrClient instance
+ * @param {string} path - Path prefix
+ * @returns {Promise<Object>} Deletion results
+ */
+export async function deleteAll(client, path) {
+  try {
+    const result = await nostrDeleteAll(client, path);
+    return result;
+  } catch (error) {
+    console.error('Nostr deleteAll error:', error);
+    throw error;
+  }
+}
+
+/**
+ * Subscribe to data changes
+ * @param {Object} client - NostrClient instance
+ * @param {string} path - Path
+ * @param {Function} callback - Called on data changes
+ * @param {Object} options - Subscription options
+ * @param {boolean} options.realtimeOnly - Only receive new events (default: true)
+ * @returns {Promise<Object>} Subscription object with unsubscribe method
+ */
+export async function subscribe(client, path, callback, options = {}) {
+  // Determine if this is a prefix subscription (no key) or single item
+  const pathParts = path.split('/');
+  const isPrefix = pathParts.length <= 3; // appname/holon/lens
+
+  if (isPrefix) {
+    // Subscribe to all items under this prefix
+    return await nostrSubscribeMany(client, path + '/', (data, fullPath) => {
+      // Extract key from full path
+      const key = fullPath.split('/').pop();
+      callback(data, key);
+    }, options);
+  } else {
+    // Subscribe to single item
+    return nostrSubscribe(client, path, (data) => {
+      callback(data, path.split('/').pop());
+    }, options);
+  }
+}

package/src/storage/outbox-queue.js

@@ -0,0 +1,208 @@
+/**
+ * OutboxQueue - Persistent queue for reliable relay delivery
+ *
+ * Implements guaranteed delivery with exponential backoff retry.
+ * Events are persisted to local storage and retried until delivered
+ * or max retries exceeded.
+ */
+
+export class OutboxQueue {
+  /**
+   * @param {Object} persistentStorage - Storage adapter (IndexedDB/FileSystem)
+   * @param {Object} options - Configuration options
+   * @param {number} options.maxRetries - Max retry attempts (default: 5)
+   * @param {number} options.baseDelay - Base delay in ms (default: 1000)
+   * @param {number} options.maxDelay - Max delay cap in ms (default: 60000)
+   * @param {number} options.failedTTL - TTL for failed events in ms (default: 24 hours)
+   */
+  constructor(persistentStorage, options = {}) {
+    this.storage = persistentStorage;
+    this.queuePrefix = '_outbox/';
+    this.maxRetries = options.maxRetries || 5;
+    this.baseDelay = options.baseDelay || 1000; // 1 second
+    this.maxDelay = options.maxDelay || 60000;  // 1 minute
+    this.failedTTL = options.failedTTL || 24 * 60 * 60 * 1000; // 24 hours
+  }
+
+  /**
+   * Add an event to the outbox queue
+   * @param {Object} event - Signed Nostr event
+   * @param {string[]} relays - Target relay URLs
+   * @returns {Promise<Object>} Queue entry
+   */
+  async enqueue(event, relays) {
+    const entry = {
+      id: event.id,
+      event,
+      relays,
+      status: 'pending',
+      retries: 0,
+      createdAt: Date.now(),
+      nextRetryAt: Date.now(),
+      failedRelays: [],
+    };
+
+    const key = `${this.queuePrefix}${event.id}`;
+    await this.storage.put(key, entry);
+    return entry;
+  }
+
+  /**
+   * Mark an event as sent (removes from queue if successful)
+   * @param {string} eventId - Event ID
+   * @param {string[]} successfulRelays - Relays that accepted the event
+   */
+  async markSent(eventId, successfulRelays) {
+    const key = `${this.queuePrefix}${eventId}`;
+    const entry = await this.storage.get(key);
+    if (!entry) return;
+
+    if (successfulRelays.length > 0) {
+      // At least one relay accepted - remove from queue
+      await this.storage.delete(key);
+    } else {
+      // All relays failed - schedule retry
+      await this._scheduleRetry(key, entry);
+    }
+  }
+
+  /**
+   * Schedule a retry with exponential backoff
+   * @private
+   */
+  async _scheduleRetry(key, entry) {
+    entry.retries++;
+
+    if (entry.retries >= this.maxRetries) {
+      entry.status = 'failed';
+      entry.failedAt = Date.now();
+    } else {
+      entry.status = 'pending';
+      // Exponential backoff with jitter
+      const delay = Math.min(
+        this.baseDelay * Math.pow(2, entry.retries) + Math.random() * 1000,
+        this.maxDelay
+      );
+      entry.nextRetryAt = Date.now() + delay;
+    }
+
+    await this.storage.put(key, entry);
+  }
+
+  /**
+   * Get all pending events ready for retry
+   * @returns {Promise<Object[]>} Pending queue entries sorted by creation time
+   */
+  async getPendingEvents() {
+    const allEntries = await this.storage.getAll(this.queuePrefix);
+    const now = Date.now();
+
+    return allEntries
+      .filter(e => e && e.status === 'pending' && e.nextRetryAt <= now)
+      .sort((a, b) => a.createdAt - b.createdAt);
+  }
+
+  /**
+   * Get all failed events (exceeded max retries)
+   * @returns {Promise<Object[]>} Failed queue entries
+   */
+  async getFailedEvents() {
+    const allEntries = await this.storage.getAll(this.queuePrefix);
+    return allEntries.filter(e => e && e.status === 'failed');
+  }
+
+  /**
+   * Get queue statistics
+   * @returns {Promise<Object>} Stats object
+   */
+  async getStats() {
+    const allEntries = await this.storage.getAll(this.queuePrefix);
+
+    const stats = {
+      total: allEntries.length,
+      pending: 0,
+      failed: 0,
+      oldestPending: null,
+      oldestFailed: null,
+    };
+
+    for (const entry of allEntries) {
+      if (!entry) continue;
+
+      if (entry.status === 'pending') {
+        stats.pending++;
+        if (!stats.oldestPending || entry.createdAt < stats.oldestPending) {
+          stats.oldestPending = entry.createdAt;
+        }
+      } else if (entry.status === 'failed') {
+        stats.failed++;
+        if (!stats.oldestFailed || entry.failedAt < stats.oldestFailed) {
+          stats.oldestFailed = entry.failedAt;
+        }
+      }
+    }
+
+    return stats;
+  }
+
+  /**
+   * Purge failed events older than maxAge
+   * @param {number} maxAge - Max age in ms (default: 24 hours)
+   * @returns {Promise<number>} Number of entries purged
+   */
+  async purgeOldFailed(maxAge = this.failedTTL) {
+    const allEntries = await this.storage.getAll(this.queuePrefix);
+    const now = Date.now();
+    let purged = 0;
+
+    for (const entry of allEntries) {
+      if (entry && entry.status === 'failed' && (now - entry.failedAt) > maxAge) {
+        await this.storage.delete(`${this.queuePrefix}${entry.id}`);
+        purged++;
+      }
+    }
+
+    return purged;
+  }
+
+  /**
+   * Clear all entries from the queue
+   * @returns {Promise<number>} Number of entries cleared
+   */
+  async clear() {
+    const allEntries = await this.storage.getAll(this.queuePrefix);
+    let cleared = 0;
+
+    for (const entry of allEntries) {
+      if (entry && entry.id) {
+        await this.storage.delete(`${this.queuePrefix}${entry.id}`);
+        cleared++;
+      }
+    }
+
+    return cleared;
+  }
+
+  /**
+   * Manually retry a failed event
+   * @param {string} eventId - Event ID to retry
+   * @returns {Promise<Object|null>} Updated entry or null if not found
+   */
+  async retryFailed(eventId) {
+    const key = `${this.queuePrefix}${eventId}`;
+    const entry = await this.storage.get(key);
+
+    if (!entry || entry.status !== 'failed') {
+      return null;
+    }
+
+    // Reset for retry
+    entry.status = 'pending';
+    entry.retries = 0;
+    entry.nextRetryAt = Date.now();
+    delete entry.failedAt;
+
+    await this.storage.put(key, entry);
+    return entry;
+  }
+}

package/src/storage/persistent-storage.js

@@ -0,0 +1,109 @@
+/**
+ * Persistent Storage Interface
+ * Provides a unified API for local persistence across browser and Node.js
+ */
+
+/**
+ * Base class for persistent storage adapters
+ */
+class PersistentStorage {
+  /**
+   * Initialize storage
+   * @param {string} namespace - Storage namespace (appName)
+   */
+  async init(namespace) {
+    throw new Error('init() must be implemented by subclass');
+  }
+
+  /**
+   * Store event
+   * @param {string} key - Event ID or d-tag
+   * @param {Object} event - Nostr event
+   */
+  async put(key, event) {
+    throw new Error('put() must be implemented by subclass');
+  }
+
+  /**
+   * Retrieve event
+   * @param {string} key - Event ID or d-tag
+   * @returns {Promise<Object|null>} Event or null
+   */
+  async get(key) {
+    throw new Error('get() must be implemented by subclass');
+  }
+
+  /**
+   * Get all events matching prefix
+   * @param {string} prefix - Key prefix
+   * @returns {Promise<Object[]>} Array of events
+   */
+  async getAll(prefix) {
+    throw new Error('getAll() must be implemented by subclass');
+  }
+
+  /**
+   * Delete event
+   * @param {string} key - Event ID or d-tag
+   */
+  async delete(key) {
+    throw new Error('delete() must be implemented by subclass');
+  }
+
+  /**
+   * Clear all data in namespace
+   */
+  async clear() {
+    throw new Error('clear() must be implemented by subclass');
+  }
+
+  /**
+   * Close storage
+   */
+  async close() {
+    throw new Error('close() must be implemented by subclass');
+  }
+}
+
+/**
+ * Detect environment and return appropriate storage adapter
+ * @param {string} namespace - Storage namespace
+ * @param {Object} options - Storage options
+ * @returns {Promise<PersistentStorage>} Storage instance
+ */
+async function createPersistentStorage(namespace, options = {}) {
+  // Detect environment
+  const isBrowser = typeof window !== 'undefined' && typeof window.indexedDB !== 'undefined';
+  const isNode = typeof process !== 'undefined' && process.versions && process.versions.node;
+
+  if (isBrowser) {
+    // Use IndexedDB in browser
+    const { IndexedDBStorage } = await import('./indexeddb-storage.js');
+    const storage = new IndexedDBStorage();
+    await storage.init(namespace);
+    return storage;
+  } else if (isNode) {
+    // Use file system in Node.js
+    try {
+      const { FileSystemStorage } = await import('./filesystem-storage.js');
+      const storage = new FileSystemStorage(options.dataDir);
+      await storage.init(namespace);
+      return storage;
+    } catch (error) {
+      // Fall back to memory storage if filesystem module fails to load
+      console.warn('FileSystemStorage not available, using MemoryStorage instead');
+      const { MemoryStorage } = await import('./memory-storage.js');
+      const storage = new MemoryStorage();
+      await storage.init(namespace);
+      return storage;
+    }
+  } else {
+    // Fallback to memory-only storage
+    const { MemoryStorage } = await import('./memory-storage.js');
+    const storage = new MemoryStorage();
+    await storage.init(namespace);
+    return storage;
+  }
+}
+
+export { PersistentStorage, createPersistentStorage };

package/src/storage/sync-service.js

@@ -0,0 +1,164 @@
+/**
+ * SyncService - Background service for reliable sync
+ *
+ * Periodically processes the outbox queue to retry failed deliveries
+ * and purge old failed events.
+ */
+
+export class SyncService {
+  /**
+   * @param {Object} client - NostrClient instance
+   * @param {Object} options - Configuration options
+   * @param {number} options.interval - Sync interval in ms (default: 10000)
+   * @param {boolean} options.autoStart - Start automatically (default: false)
+   */
+  constructor(client, options = {}) {
+    this.client = client;
+    this.interval = options.interval || 10000; // 10 seconds
+    this.running = false;
+    this.timer = null;
+    this._processing = false;
+
+    if (options.autoStart) {
+      this.start();
+    }
+  }
+
+  /**
+   * Start the background sync service
+   */
+  start() {
+    if (this.running) return;
+    this.running = true;
+    this._scheduleNextRun();
+  }
+
+  /**
+   * Stop the background sync service
+   */
+  stop() {
+    this.running = false;
+    if (this.timer) {
+      clearTimeout(this.timer);
+      this.timer = null;
+    }
+  }
+
+  /**
+   * Check if the service is running
+   * @returns {boolean}
+   */
+  isRunning() {
+    return this.running;
+  }
+
+  /**
+   * Force an immediate sync (useful for testing or manual trigger)
+   * @returns {Promise<Object>} Sync result
+   */
+  async syncNow() {
+    return this._processOutbox();
+  }
+
+  /**
+   * Schedule the next run
+   * @private
+   */
+  _scheduleNextRun() {
+    if (!this.running) return;
+
+    this.timer = setTimeout(() => {
+      this._runLoop();
+    }, this.interval);
+  }
+
+  /**
+   * Main loop iteration
+   * @private
+   */
+  async _runLoop() {
+    if (!this.running) return;
+
+    try {
+      await this._processOutbox();
+    } catch (error) {
+      console.warn('[sync] Outbox processing failed:', error.message);
+    }
+
+    // Schedule next run
+    this._scheduleNextRun();
+  }
+
+  /**
+   * Process pending events in the outbox
+   * @private
+   * @returns {Promise<Object>} Processing result
+   */
+  async _processOutbox() {
+    // Prevent concurrent processing
+    if (this._processing) {
+      return { skipped: true, reason: 'already processing' };
+    }
+
+    this._processing = true;
+
+    const result = {
+      processed: 0,
+      succeeded: 0,
+      failed: 0,
+      purged: 0,
+    };
+
+    try {
+      if (!this.client.outboxQueue) {
+        return { ...result, skipped: true, reason: 'no outbox queue' };
+      }
+
+      // Get pending events ready for retry
+      const pending = await this.client.outboxQueue.getPendingEvents();
+      result.processed = pending.length;
+
+      // Process each pending event
+      for (const entry of pending) {
+        try {
+          const deliveryResult = await this.client._attemptDelivery(entry.event, entry.relays);
+
+          if (deliveryResult.successful.length > 0) {
+            result.succeeded++;
+          } else {
+            result.failed++;
+          }
+        } catch (error) {
+          console.warn(`[sync] Retry failed for ${entry.id}:`, error.message);
+          result.failed++;
+        }
+      }
+
+      // Purge old failed events (24 hour TTL)
+      result.purged = await this.client.outboxQueue.purgeOldFailed();
+
+    } finally {
+      this._processing = false;
+    }
+
+    return result;
+  }
+
+  /**
+   * Get sync service statistics
+   * @returns {Promise<Object>} Stats including queue status
+   */
+  async getStats() {
+    const stats = {
+      running: this.running,
+      interval: this.interval,
+      queue: null,
+    };
+
+    if (this.client.outboxQueue) {
+      stats.queue = await this.client.outboxQueue.getStats();
+    }
+
+    return stats;
+  }
+}