holosphere 2.0.0-alpha0 → 2.0.0-alpha10

package/src/storage/gun-federation.js

@@ -0,0 +1,785 @@
+/**
+ * @fileoverview GunDB Federation Module.
+ *
+ * Provides methods for creating, managing, and using federated spaces.
+ * Supports data propagation with references for lightweight federation,
+ * aggregation across federated sources, and subscription to federated changes.
+ *
+ * Ported from holosphere/federation.js to work with the GunDBBackend.
+ *
+ * @module storage/gun-federation
+ */
+
+/**
+ * Creates a federation relationship between two spaces.
+ *
+ * Federation is bidirectional by default, and data propagation uses soul references by default.
+ *
+ * @param {Object} backend - The GunDBBackend instance
+ * @param {string} spaceId1 - The first space ID
+ * @param {string} spaceId2 - The second space ID
+ * @param {boolean} [bidirectional=true] - Whether to set up bidirectional notifications
+ * @returns {Promise<boolean>} True if federation was created successfully
+ * @throws {Error} If space IDs are missing or identical
+ * @example
+ * await federate(backend, 'space1', 'space2');
+ */
+export async function federate(backend, spaceId1, spaceId2, bidirectional = true) {
+  if (!spaceId1 || !spaceId2) {
+    throw new Error('federate: Missing required space IDs');
+  }
+
+  if (spaceId1 === spaceId2) {
+    throw new Error('Cannot federate a space with itself');
+  }
+
+  try {
+    // Get or create federation info for first space (A)
+    let fedInfo1 = await backend.readGlobal('federation', spaceId1);
+
+    if (!fedInfo1) {
+      fedInfo1 = {
+        id: spaceId1,
+        name: spaceId1,
+        federation: [],
+        notify: [],
+        timestamp: Date.now()
+      };
+    }
+
+    // Ensure arrays exist
+    if (!fedInfo1.federation) fedInfo1.federation = [];
+    if (!fedInfo1.notify) fedInfo1.notify = [];
+
+    // Add space2 to space1's federation list if not already present
+    if (!fedInfo1.federation.includes(spaceId2)) {
+      fedInfo1.federation.push(spaceId2);
+    }
+
+    fedInfo1.timestamp = Date.now();
+    await backend.writeGlobal('federation', fedInfo1);
+
+    // Handle space2 (B) - add to notify list
+    let fedInfo2 = await backend.readGlobal('federation', spaceId2);
+
+    if (!fedInfo2) {
+      fedInfo2 = {
+        id: spaceId2,
+        name: spaceId2,
+        federation: [],
+        notify: [],
+        timestamp: Date.now()
+      };
+    }
+
+    if (!fedInfo2.notify) fedInfo2.notify = [];
+
+    // Add space1 to space2's notify list
+    if (!fedInfo2.notify.includes(spaceId1)) {
+      fedInfo2.notify.push(spaceId1);
+    }
+
+    fedInfo2.timestamp = Date.now();
+    await backend.writeGlobal('federation', fedInfo2);
+
+    // Create federation metadata record
+    const federationMeta = {
+      id: `${spaceId1}_${spaceId2}`,
+      space1: spaceId1,
+      space2: spaceId2,
+      created: Date.now(),
+      status: 'active',
+      bidirectional: bidirectional
+    };
+
+    await backend.writeGlobal('federationMeta', federationMeta);
+
+    return true;
+  } catch (error) {
+    console.error(`Federation creation failed: ${error.message}`);
+    throw error;
+  }
+}
+
+/**
+ * Gets federation info for a space
+ * @param {Object} backend - The GunDBBackend instance
+ * @param {string} spaceId - The space ID
+ * @returns {Promise<Object|null>} Federation info or null if not found
+ */
+export async function getFederation(backend, spaceId) {
+  if (!spaceId) {
+    throw new Error('getFederation: Missing space ID');
+  }
+  return backend.readGlobal('federation', spaceId);
+}
+
+/**
+ * Removes a federation relationship between spaces
+ * @param {Object} backend - The GunDBBackend instance
+ * @param {string} spaceId1 - The first space ID
+ * @param {string} spaceId2 - The second space ID
+ * @returns {Promise<boolean>} True if federation was removed successfully
+ */
+export async function unfederate(backend, spaceId1, spaceId2) {
+  if (!spaceId1 || !spaceId2) {
+    throw new Error('unfederate: Missing required space IDs');
+  }
+
+  try {
+    // Update first space federation info
+    let fedInfo1 = await backend.readGlobal('federation', spaceId1);
+
+    if (fedInfo1 && fedInfo1.federation) {
+      fedInfo1.federation = fedInfo1.federation.filter(id => id !== spaceId2);
+      fedInfo1.timestamp = Date.now();
+      await backend.writeGlobal('federation', fedInfo1);
+    }
+
+    // Update second space federation info
+    let fedInfo2 = await backend.readGlobal('federation', spaceId2);
+
+    if (fedInfo2 && fedInfo2.notify) {
+      fedInfo2.notify = fedInfo2.notify.filter(id => id !== spaceId1);
+      fedInfo2.timestamp = Date.now();
+      await backend.writeGlobal('federation', fedInfo2);
+    }
+
+    // Update federation metadata
+    const metaId = `${spaceId1}_${spaceId2}`;
+    const altMetaId = `${spaceId2}_${spaceId1}`;
+
+    let meta = await backend.readGlobal('federationMeta', metaId);
+    if (!meta) {
+      meta = await backend.readGlobal('federationMeta', altMetaId);
+    }
+
+    if (meta) {
+      meta.status = 'inactive';
+      meta.endedAt = Date.now();
+      await backend.writeGlobal('federationMeta', meta);
+    }
+
+    return true;
+  } catch (error) {
+    console.error(`Federation removal failed: ${error.message}`);
+    throw error;
+  }
+}
+
+/**
+ * Removes a notification relationship between two spaces
+ * @param {Object} backend - The GunDBBackend instance
+ * @param {string} spaceId1 - The space to modify
+ * @param {string} spaceId2 - The space to be removed from notifications
+ * @returns {Promise<boolean>} True if notification was removed successfully
+ */
+export async function removeNotify(backend, spaceId1, spaceId2) {
+  if (!spaceId1 || !spaceId2) {
+    throw new Error('removeNotify: Missing required space IDs');
+  }
+
+  try {
+    let fedInfo = await backend.readGlobal('federation', spaceId1);
+
+    if (!fedInfo) {
+      throw new Error(`No federation info found for ${spaceId1}`);
+    }
+
+    if (!fedInfo.notify) fedInfo.notify = [];
+
+    if (fedInfo.notify.includes(spaceId2)) {
+      fedInfo.notify = fedInfo.notify.filter(id => id !== spaceId2);
+      fedInfo.timestamp = Date.now();
+      await backend.writeGlobal('federation', fedInfo);
+      return true;
+    }
+
+    return false;
+  } catch (error) {
+    console.error(`Remove notification failed: ${error.message}`);
+    throw error;
+  }
+}
+
+/**
+ * Resets all federation relationships for a space
+ * @param {Object} backend - The GunDBBackend instance
+ * @param {string} spaceId - The ID of the space to reset
+ * @param {Object} options - Reset options
+ * @param {boolean} options.notifyPartners - Whether to notify federation partners (default: true)
+ * @param {string} options.spaceName - Optional name for the space
+ * @returns {Promise<Object>} Result object with success/error info
+ */
+export async function resetFederation(backend, spaceId, options = {}) {
+  if (!spaceId) {
+    throw new Error('resetFederation: Missing required space ID');
+  }
+
+  const {
+    notifyPartners = true,
+    spaceName = null
+  } = options;
+
+  const result = {
+    success: false,
+    federatedCount: 0,
+    notifyCount: 0,
+    partnersNotified: 0,
+    errors: []
+  };
+
+  try {
+    const fedInfo = await getFederation(backend, spaceId);
+
+    if (!fedInfo) {
+      return {
+        ...result,
+        success: true,
+        message: 'No federation configuration found for this space'
+      };
+    }
+
+    result.federatedCount = fedInfo.federation?.length || 0;
+    result.notifyCount = fedInfo.notify?.length || 0;
+
+    // Create empty federation record
+    const emptyFedInfo = {
+      id: spaceId,
+      name: spaceName || spaceId,
+      federation: [],
+      notify: [],
+      timestamp: Date.now()
+    };
+
+    await backend.writeGlobal('federation', emptyFedInfo);
+
+    // Notify federation partners if requested
+    if (notifyPartners && fedInfo.federation && fedInfo.federation.length > 0) {
+      for (const partnerSpace of fedInfo.federation) {
+        try {
+          const partnerFedInfo = await getFederation(backend, partnerSpace);
+
+          if (partnerFedInfo) {
+            if (partnerFedInfo.federation) {
+              partnerFedInfo.federation = partnerFedInfo.federation.filter(
+                id => id !== spaceId.toString()
+              );
+            }
+
+            if (partnerFedInfo.notify) {
+              partnerFedInfo.notify = partnerFedInfo.notify.filter(
+                id => id !== spaceId.toString()
+              );
+            }
+
+            partnerFedInfo.timestamp = Date.now();
+            await backend.writeGlobal('federation', partnerFedInfo);
+            result.partnersNotified++;
+          }
+        } catch (error) {
+          result.errors.push({
+            partner: partnerSpace,
+            error: error.message
+          });
+        }
+      }
+    }
+
+    // Update federation metadata records
+    if (fedInfo.federation && fedInfo.federation.length > 0) {
+      for (const partnerSpace of fedInfo.federation) {
+        try {
+          const metaId = `${spaceId}_${partnerSpace}`;
+          const altMetaId = `${partnerSpace}_${spaceId}`;
+
+          let meta = await backend.readGlobal('federationMeta', metaId);
+          if (!meta) {
+            meta = await backend.readGlobal('federationMeta', altMetaId);
+          }
+
+          if (meta) {
+            meta.status = 'inactive';
+            meta.endedAt = Date.now();
+            await backend.writeGlobal('federationMeta', meta);
+          }
+        } catch (error) {
+          // Ignore metadata errors
+        }
+      }
+    }
+
+    result.success = true;
+    return result;
+  } catch (error) {
+    console.error(`Federation reset failed: ${error.message}`);
+    return {
+      ...result,
+      success: false,
+      error: error.message
+    };
+  }
+}
+
+/**
+ * Propagates data to federated spaces
+ * @param {Object} backend - The GunDBBackend instance
+ * @param {string} holon - The holon identifier
+ * @param {string} lens - The lens identifier
+ * @param {Object} data - The data to propagate
+ * @param {Object} options - Propagation options
+ * @param {boolean} options.useReferences - Whether to use references instead of duplicating data (default: true)
+ * @param {string[]} options.targetSpaces - Specific target spaces to propagate to
+ * @returns {Promise<Object>} Result with success count and errors
+ */
+export async function propagate(backend, holon, lens, data, options = {}) {
+  if (!backend || !holon || !lens || !data) {
+    throw new Error('propagate: Missing required parameters');
+  }
+
+  const {
+    useReferences = true,
+    targetSpaces = null
+  } = options;
+
+  const result = {
+    success: 0,
+    errors: 0,
+    errorDetails: [],
+    propagated: false,
+    referencesUsed: useReferences
+  };
+
+  try {
+    const fedInfo = await getFederation(backend, holon);
+
+    if (!fedInfo || !fedInfo.federation || fedInfo.federation.length === 0) {
+      return {
+        ...result,
+        message: `No federation found for ${holon}`
+      };
+    }
+
+    if (!fedInfo.notify || fedInfo.notify.length === 0) {
+      return {
+        ...result,
+        message: `No notification targets found for ${holon}`
+      };
+    }
+
+    // Filter spaces to those in notify list
+    let spaces = fedInfo.notify;
+
+    if (targetSpaces && Array.isArray(targetSpaces) && targetSpaces.length > 0) {
+      spaces = spaces.filter(space => targetSpaces.includes(space));
+    }
+
+    if (spaces.length === 0) {
+      return {
+        ...result,
+        message: 'No valid target spaces found after filtering'
+      };
+    }
+
+    const isAlreadyReference = backend.isReference(data);
+
+    // Propagate to each target space
+    for (const targetSpace of spaces) {
+      try {
+        if (useReferences && !isAlreadyReference) {
+          // Create a reference
+          const reference = backend.createReference(holon, lens, data);
+
+          reference._federation = {
+            origin: holon,
+            lens: lens,
+            timestamp: Date.now()
+          };
+
+          // Build path and write reference
+          const path = backend.buildPath(backend.appName, targetSpace, lens, reference.id);
+          await backend.write(path, reference);
+
+          result.success++;
+        } else if (isAlreadyReference) {
+          // Propagate existing reference
+          const referenceToStore = {
+            ...data,
+            _federation: data._federation || {
+              origin: holon,
+              lens: lens,
+              timestamp: Date.now()
+            }
+          };
+
+          const path = backend.buildPath(backend.appName, targetSpace, lens, data.id);
+          await backend.write(path, referenceToStore);
+          result.success++;
+        } else {
+          // Store full copy
+          const dataToStore = {
+            ...data,
+            _federation: {
+              origin: holon,
+              lens: lens,
+              timestamp: Date.now()
+            }
+          };
+
+          const path = backend.buildPath(backend.appName, targetSpace, lens, data.id);
+          await backend.write(path, dataToStore);
+          result.success++;
+        }
+      } catch (error) {
+        result.errors++;
+        result.errorDetails.push({
+          space: targetSpace,
+          error: error.message
+        });
+      }
+    }
+
+    result.propagated = result.success > 0;
+    return result;
+  } catch (error) {
+    console.error('Error in propagate:', error);
+    return {
+      ...result,
+      error: error.message
+    };
+  }
+}
+
+/**
+ * Get and combine data from local and federated sources
+ * @param {Object} backend - The GunDBBackend instance
+ * @param {string} holon - The local holon name
+ * @param {string} lens - The lens to query
+ * @param {Object} options - Options for data retrieval and aggregation
+ * @returns {Promise<Array>} Combined array of local and federated data
+ */
+export async function getFederated(backend, holon, lens, options = {}) {
+  const {
+    aggregate = false,
+    idField = 'id',
+    sumFields = [],
+    concatArrays = [],
+    removeDuplicates = true,
+    mergeStrategy = null,
+    includeLocal = true,
+    includeFederated = true,
+    resolveReferences = true,
+    maxFederatedSpaces = -1,
+    timeout = 10000
+  } = options;
+
+  if (!backend || !holon || !lens) {
+    throw new Error('Missing required parameters: backend, holon, and lens are required');
+  }
+
+  const fedInfo = await getFederation(backend, holon);
+
+  const result = [];
+  const processedIds = new Set();
+
+  // Process federated spaces first
+  if (includeFederated && fedInfo && fedInfo.federation && fedInfo.federation.length > 0) {
+    const federatedSpaces = maxFederatedSpaces === -1
+      ? fedInfo.federation
+      : fedInfo.federation.slice(0, maxFederatedSpaces);
+
+    for (const federatedSpace of federatedSpaces) {
+      try {
+        const path = backend.buildPath(backend.appName, federatedSpace, lens);
+        const federatedItems = await backend.readAll(path);
+
+        for (const item of federatedItems) {
+          if (!item || !item[idField]) continue;
+
+          result.push(item);
+          processedIds.add(item[idField]);
+        }
+      } catch (error) {
+        console.warn(`Error processing federated space ${federatedSpace}: ${error.message}`);
+      }
+    }
+  }
+
+  // Get local data
+  if (includeLocal) {
+    const localPath = backend.buildPath(backend.appName, holon, lens);
+    const localData = await backend.readAll(localPath);
+
+    for (const item of localData) {
+      if (item && item[idField] && !processedIds.has(item[idField])) {
+        result.push(item);
+        processedIds.add(item[idField]);
+      }
+    }
+  }
+
+  // Resolve references
+  if (resolveReferences) {
+    for (let i = 0; i < result.length; i++) {
+      const item = result[i];
+
+      if (item.soul && item.id) {
+        // Simple reference format
+        try {
+          const resolved = await backend.resolveReference(item);
+          if (resolved) {
+            result[i] = resolved;
+          }
+        } catch (error) {
+          result[i] = {
+            id: item.id,
+            _federation: {
+              isReference: true,
+              resolved: false,
+              soul: item.soul,
+              error: error.message,
+              timestamp: Date.now()
+            }
+          };
+        }
+      } else if (item._federation && item._federation.isReference) {
+        // Legacy reference format
+        try {
+          const resolved = await backend.resolveReference(item);
+          if (resolved) {
+            result[i] = resolved;
+          }
+        } catch (error) {
+          console.warn(`Error resolving legacy reference: ${error.message}`);
+        }
+      }
+    }
+  }
+
+  // Apply aggregation if requested
+  if (aggregate && result.length > 0) {
+    const groupedById = result.reduce((acc, item) => {
+      const id = item[idField];
+      if (!acc[id]) {
+        acc[id] = [];
+      }
+      acc[id].push(item);
+      return acc;
+    }, {});
+
+    const aggregatedData = Object.values(groupedById).map(group => {
+      if (group.length === 1) return group[0];
+
+      if (mergeStrategy && typeof mergeStrategy === 'function') {
+        return mergeStrategy(group);
+      }
+
+      const base = { ...group[0] };
+
+      for (const field of sumFields) {
+        if (typeof base[field] === 'number') {
+          base[field] = group.reduce((sum, item) => sum + (Number(item[field]) || 0), 0);
+        }
+      }
+
+      for (const field of concatArrays) {
+        if (Array.isArray(base[field])) {
+          const allValues = group.reduce((all, item) => {
+            return Array.isArray(item[field]) ? [...all, ...item[field]] : all;
+          }, []);
+
+          base[field] = removeDuplicates ? Array.from(new Set(allValues)) : allValues;
+        }
+      }
+
+      base._aggregated = {
+        count: group.length,
+        timestamp: Date.now()
+      };
+
+      return base;
+    });
+
+    return aggregatedData;
+  }
+
+  return result;
+}
+
+/**
+ * Subscribes to federation notifications for a space
+ * @param {Object} backend - The GunDBBackend instance
+ * @param {string} spaceId - The space ID to subscribe to
+ * @param {Function} callback - The callback to execute on notifications
+ * @param {Object} options - Subscription options
+ * @returns {Promise<Object>} Subscription object with unsubscribe() method
+ */
+export async function subscribeFederation(backend, spaceId, callback, options = {}) {
+  if (!spaceId || !callback) {
+    throw new Error('subscribeFederation: Missing required parameters');
+  }
+
+  const { lenses = ['*'], throttle = 0, resolveReferences = true } = options;
+
+  const fedInfo = await getFederation(backend, spaceId);
+  if (!fedInfo) {
+    throw new Error('No federation info found for space');
+  }
+
+  const subscriptions = [];
+  let lastNotificationTime = {};
+
+  if (fedInfo.federation && fedInfo.federation.length > 0) {
+    for (const federatedSpace of fedInfo.federation) {
+      for (const lens of lenses) {
+        try {
+          const path = backend.buildPath(backend.appName, federatedSpace, lens);
+
+          const sub = await backend.subscribe(path, async (data, key) => {
+            try {
+              if (!data || !data.id) return;
+
+              // Apply throttling
+              const now = Date.now();
+              const throttleKey = `${federatedSpace}_${lens}_${data.id}`;
+
+              if (throttle > 0) {
+                if (lastNotificationTime[throttleKey] &&
+                    (now - lastNotificationTime[throttleKey]) < throttle) {
+                  return;
+                }
+                lastNotificationTime[throttleKey] = now;
+              }
+
+              // Resolve reference if needed
+              let resolvedData = data;
+              if (resolveReferences && backend.isReference(data)) {
+                resolvedData = await backend.resolveReference(data);
+              }
+
+              if (!resolvedData._federation) {
+                resolvedData._federation = {
+                  origin: federatedSpace,
+                  timestamp: now
+                };
+              }
+
+              await callback(resolvedData, federatedSpace, lens);
+            } catch (error) {
+              console.warn('Federation notification error:', error);
+            }
+          }, { resolveReferences: false }); // Don't resolve in subscribe, we handle it
+
+          if (sub && typeof sub.unsubscribe === 'function') {
+            subscriptions.push(sub);
+          }
+        } catch (error) {
+          console.warn(`Error creating subscription for ${federatedSpace}/${lens}:`, error);
+        }
+      }
+    }
+  }
+
+  return {
+    unsubscribe: () => {
+      subscriptions.forEach(sub => {
+        try {
+          if (sub && typeof sub.unsubscribe === 'function') {
+            sub.unsubscribe();
+          }
+        } catch (error) {
+          console.warn('Error unsubscribing:', error);
+        }
+      });
+      subscriptions.length = 0;
+      lastNotificationTime = {};
+    },
+    getSubscriptionCount: () => subscriptions.length
+  };
+}
+
+/**
+ * Tracks a federated message across different chats
+ * @param {Object} backend - The GunDBBackend instance
+ * @param {string} originalChatId - The ID of the original chat
+ * @param {string} messageId - The ID of the original message
+ * @param {string} federatedChatId - The ID of the federated chat
+ * @param {string} federatedMessageId - The ID of the message in the federated chat
+ * @param {string} type - The type of message
+ * @returns {Promise<void>}
+ */
+export async function federateMessage(backend, originalChatId, messageId, federatedChatId, federatedMessageId, type = 'generic') {
+  const trackingKey = `${originalChatId}_${messageId}_fedmsgs`;
+  let tracking = await backend.readGlobal('federation_messages', trackingKey);
+
+  if (!tracking) {
+    tracking = {
+      id: trackingKey,
+      originalChatId,
+      originalMessageId: messageId,
+      type,
+      messages: []
+    };
+  }
+
+  const existingMsg = tracking.messages.find(m => m.chatId === federatedChatId);
+  if (existingMsg) {
+    existingMsg.messageId = federatedMessageId;
+    existingMsg.timestamp = Date.now();
+  } else {
+    tracking.messages.push({
+      chatId: federatedChatId,
+      messageId: federatedMessageId,
+      timestamp: Date.now()
+    });
+  }
+
+  await backend.writeGlobal('federation_messages', tracking);
+}
+
+/**
+ * Gets all federated messages for a given original message
+ * @param {Object} backend - The GunDBBackend instance
+ * @param {string} originalChatId - The ID of the original chat
+ * @param {string} messageId - The ID of the original message
+ * @returns {Promise<Object|null>} The tracking information for the message
+ */
+export async function getFederatedMessages(backend, originalChatId, messageId) {
+  const trackingKey = `${originalChatId}_${messageId}_fedmsgs`;
+  return backend.readGlobal('federation_messages', trackingKey);
+}
+
+/**
+ * Updates a federated message across all federated chats
+ * @param {Object} backend - The GunDBBackend instance
+ * @param {string} originalChatId - The ID of the original chat
+ * @param {string} messageId - The ID of the original message
+ * @param {Function} updateCallback - Function to update the message in each chat
+ * @returns {Promise<void>}
+ */
+export async function updateFederatedMessages(backend, originalChatId, messageId, updateCallback) {
+  const tracking = await getFederatedMessages(backend, originalChatId, messageId);
+  if (!tracking?.messages) return;
+
+  for (const msg of tracking.messages) {
+    try {
+      await updateCallback(msg.chatId, msg.messageId);
+    } catch (error) {
+      console.warn(`Failed to update federated message in chat ${msg.chatId}:`, error);
+    }
+  }
+}
+
+export default {
+  federate,
+  unfederate,
+  getFederation,
+  resetFederation,
+  removeNotify,
+  propagate,
+  getFederated,
+  subscribeFederation,
+  federateMessage,
+  getFederatedMessages,
+  updateFederatedMessages
+};