holosphere 1.1.8 → 1.1.10

package/holosphere.js

@@ -1,11 +1,24 @@
 import * as h3 from 'h3-js';
 import OpenAI from 'openai';
 import Gun from 'gun'
-import SEA from 'gun/sea.js'
 import Ajv2019 from 'ajv/dist/2019.js'
 import * as Federation from './federation.js';
 
-export { federateMessage, getFederatedMessages, updateFederatedMessages, removeNotify } from './federation.js';
+/**
+ * HoloSphere is a cutting-edge holonic communication infrastructure designed to facilitate
+ * the creation, validation, and sharing of information.. By leveraging advanced technologies
+ * such as H3 for geospatial indexing, OpenAI for natural language processing, and Gun for decentralized
+ * data storage, HoloSphere provides a robust platform for developers to build innovative stygmergic applications.
+ * 
+ * Key Features:
+ * - Geospatial indexing using H3
+ * - Natural language processing with OpenAI
+ * - Decentralized data storage with Gun
+ * - JSON schema validation with Ajv
+ * - Federation capabilities for distributed systems
+ * 
+ *
+ */
 
 class HoloSphere {
     /**
@@ -15,8 +28,8 @@ class HoloSphere {
      * @param {string|null} openaikey - The OpenAI API key.
      * @param {Gun|null} gunInstance - The Gun instance to use.
      */
-    constructor(appname, strict = false, openaikey = null, gunInstance = null) {
-        console.log('HoloSphere v1.1.7'); 
+    constructor(appname, strict = false, openaikey = null) {
+        console.log('HoloSphere v1.1.10'); 
         this.appname = appname
         this.strict = strict;
         this.validator = new Ajv2019({
@@ -25,20 +38,13 @@ class HoloSphere {
             validateSchema: true // Always validate schemas
         });
 
-        // Handle different ways of providing Gun instance or options
-        if (gunInstance && gunInstance.opt) {
-            // If an object with 'opt' property is passed, create a new Gun instance with those options
-            this.gun = Gun(gunInstance.opt);
-        } else {
-            // Use provided Gun instance or create new one with default options
-            this.gun = gunInstance || Gun({
-                peers: ['https://gun.holons.io/gun'],
-                axe: false,
-            });
-        }
-
-        // Initialize SEA
-        this.sea = SEA;
+       
+        // Use provided Gun instance or create new one with default options
+        this.gun = Gun({
+            peers: ['https://gun.holons.io/gun'],
+            axe: false,
+        });
+    
 
         if (openaikey != null) {
             this.openai = new OpenAI({
@@ -48,6 +54,9 @@ class HoloSphere {
 
         // Initialize subscriptions
         this.subscriptions = {};
+        
+        // Initialize schema cache
+        this.schemaCache = new Map();
     }
 
     // ================================ SCHEMA FUNCTIONS ================================
@@ -109,6 +118,12 @@ class HoloSphere {
             schema: schema,
             timestamp: Date.now()
         });
+        
+        // Update the cache with the new schema
+        this.schemaCache.set(lens, {
+            schema,
+            timestamp: Date.now()
+        });
 
         return true;
     }
@@ -116,21 +131,61 @@ class HoloSphere {
     /**
      * Retrieves the JSON schema for a specific lens.
      * @param {string} lens - The lens identifier.
+     * @param {object} [options] - Additional options
+     * @param {boolean} [options.useCache=true] - Whether to use the schema cache
+     * @param {number} [options.maxCacheAge=3600000] - Maximum cache age in milliseconds (default: 1 hour)
      * @returns {Promise<object|null>} - The retrieved schema or null if not found.
      */
-    async getSchema(lens) {
+    async getSchema(lens, options = {}) {
         if (!lens) {
             throw new Error('getSchema: Missing lens parameter');
         }
-
+        
+        const { useCache = true, maxCacheAge = 3600000 } = options;
+        
+        // Check cache first if enabled
+        if (useCache && this.schemaCache.has(lens)) {
+            const cached = this.schemaCache.get(lens);
+            const cacheAge = Date.now() - cached.timestamp;
+            
+            // Use cache if it's fresh enough
+            if (cacheAge < maxCacheAge) {
+                return cached.schema;
+            }
+        }
+        
+        // Cache miss or expired, fetch from storage
         const schemaData = await this.getGlobal('schemas', lens);
+        
         if (!schemaData || !schemaData.schema) {
             return null;
         }
-
+        
+        // Update cache with fetched schema
+        this.schemaCache.set(lens, {
+            schema: schemaData.schema,
+            timestamp: Date.now()
+        });
+        
         return schemaData.schema;
     }
 
+    /**
+     * Clears the schema cache or a specific schema from the cache.
+     * @param {string} [lens] - Optional lens to clear from cache. If not provided, clears entire cache.
+     * @returns {boolean} - Returns true if successful
+     */
+    clearSchemaCache(lens = null) {
+        if (lens) {
+            // Clear specific schema
+            return this.schemaCache.delete(lens);
+        } else {
+            // Clear entire cache
+            this.schemaCache.clear();
+            return true;
+        }
+    }
+
     // ================================ CONTENT FUNCTIONS ================================
 
     /**
@@ -154,8 +209,11 @@ class HoloSphere {
             data.id = this.generateId();
         }
 
-        // Get and validate schema only in strict mode
-        if (this.strict) {
+        // Check if this is a reference we're storing
+        const isRef = this.isReference(data);
+
+        // Get and validate schema only in strict mode for non-references
+        if (this.strict && !isRef) {
             const schema = await this.getSchema(lens);
             if (!schema) {
                 throw new Error('Schema required in strict mode');
@@ -170,72 +228,15 @@ class HoloSphere {
         }
 
         try {
-            const user = this.gun.user();
-            
+            let user = null;
             if (password) {
-                try {
-                    await new Promise((resolve, reject) => {
-                        user.auth(this.userName(holon), password, (ack) => {
-                            if (ack.err) reject(new Error(ack.err));
-                            else resolve();
-                        });
+                user = this.gun.user();
+                await new Promise((resolve, reject) => {
+                    user.auth(this.userName(holon), password, (ack) => {
+                        if (ack.err) reject(new Error(ack.err));
+                        else resolve();
                     });
-                } catch (loginError) {
-                    // If authentication fails, try to create user and then authenticate
-                    try {
-                        await new Promise((resolve, reject) => {
-                            user.create(this.userName(holon), password, (ack) => {
-                                if (ack.err) {
-                                    // Don't reject if the user is already being created or already exists
-                                    if (ack.err.includes('already being created') || 
-                                        ack.err.includes('already created')) {
-                                        console.warn(`User creation note: ${ack.err}, continuing...`);
-                                        // Try to authenticate again
-                                        user.auth(this.userName(holon), password, (authAck) => {
-                                            if (authAck.err) {
-                                                if (authAck.err.includes('already being created') || 
-                                                    authAck.err.includes('already created')) {
-                                                    console.warn(`Auth note: ${authAck.err}, continuing...`);
-                                                    resolve(); // Continue anyway
-                                                } else {
-                                                    reject(new Error(authAck.err));
-                                                }
-                                            } else {
-                                                resolve();
-                                            }
-                                        });
-                                    } else {
-                                        reject(new Error(ack.err));
-                                    }
-                                } else {
-                                    user.auth(this.userName(holon), password, (authAck) => {
-                                        if (authAck.err) reject(new Error(authAck.err));
-                                        else resolve();
-                                    });
-                                }
-                            });
-                        });
-                    } catch (createError) {
-                        // Try one last authentication
-                        try {
-                            await new Promise((resolve, reject) => {
-                                setTimeout(() => {
-                                    user.auth(this.userName(holon), password, (ack) => {
-                                        if (ack.err) {
-                                            // Continue even if auth fails at this point
-                                            console.warn(`Final auth attempt note: ${ack.err}, continuing with limited functionality`);
-                                            resolve();
-                                        } else {
-                                            resolve();
-                                        }
-                                    });
-                                }, 100); // Short delay before retry
-                            });
-                        } catch (finalAuthError) {
-                            console.warn('All authentication attempts failed, continuing with limited functionality');
-                        }
-                    }
-                }
+                });
             }
 
             return new Promise((resolve, reject) => {
@@ -246,14 +247,17 @@ class HoloSphere {
                         if (ack.err) {
                             reject(new Error(ack.err));
                         } else {
-                            this.notifySubscribers({
-                                holon,
-                                lens,
-                                ...data
-                            });
+                            // Only notify subscribers for actual data, not references
+                            if (!isRef) {
+                                this.notifySubscribers({
+                                    holon,
+                                    lens,
+                                    ...data
+                                });
+                            }
                             
-                            // Auto-propagate to federation by default
-                            const shouldPropagate = options.autoPropagate !== false;
+                            // Auto-propagate to federation by default (if not a reference)
+                            const shouldPropagate = options.autoPropagate !== false && !isRef;
                             let propagationResult = null;
                             
                             if (shouldPropagate) {
@@ -282,18 +286,17 @@ class HoloSphere {
                             
                             resolve({
                                 success: true,
+                                isReference: isRef,
                                 propagationResult
                             });
                         }
                     };
                     
-                    if (password) {
-                        // For private data, use the authenticated user's holon
-                        user.get('private').get(lens).get(data.id).put(payload, putCallback);
-                    } else {
-                        // For public data, use the regular path
-                        this.gun.get(this.appname).get(holon).get(lens).get(data.id).put(payload, putCallback);
-                    }
+                    const dataPath = password ? 
+                        user.get('private').get(lens).get(data.id) :
+                        this.gun.get(this.appname).get(holon).get(lens).get(data.id);
+
+                    dataPath.put(payload, putCallback);
                 } catch (error) {
                     reject(error);
                 }
@@ -332,30 +335,15 @@ class HoloSphere {
         }
 
         try {
-            const user = this.gun.user();
-            
+            let user = null;
             if (password) {
-                try {
-                    await new Promise((resolve, reject) => {
-                        user.auth(this.userName(holon), password, (ack) => {
-                            if (ack.err) reject(new Error(ack.err));
-                            else resolve();
-                        });
-                    });
-                } catch (loginError) {
-                    // If authentication fails, try to create user and then authenticate
-                    await new Promise((resolve, reject) => {
-                        user.create(this.userName(holon), password, (ack) => {
-                            if (ack.err) reject(new Error(ack.err));
-                            else {
-                                user.auth(this.userName(holon), password, (authAck) => {
-                                    if (authAck.err) reject(new Error(authAck.err));
-                                    else resolve();
-                                });
-                            }
-                        });
+                user = this.gun.user();
+                await new Promise((resolve, reject) => {
+                    user.auth(this.userName(holon), password, (ack) => {
+                        if (ack.err) reject(new Error(ack.err));
+                        else resolve();
                     });
-                }
+                });
             }
 
             return new Promise((resolve) => {
@@ -374,84 +362,23 @@ class HoloSphere {
                         }
 
                         // Check if this is a reference that needs to be resolved
-                        if (resolveReferences !== false && parsed) {
-                            // Check if this is a simple reference (id + soul)
-                            if (parsed.soul) {
-                                console.log(`Resolving simple reference with soul: ${parsed.soul}`);
-                                try {
-                                    // For direct soul resolution, we need to parse the soul to get the right path
-                                    const soulParts = parsed.soul.split('/');
-                                    if (soulParts.length >= 4) {  // Expected format: appname/holon/lens/key
-                                        const originHolon = soulParts[1];
-                                        const originLens = soulParts[2];
-                                        const originKey = soulParts[3];
-                                        
-                                        console.log(`Extracting from soul - holon: ${originHolon}, lens: ${originLens}, key: ${originKey}`);
-                                        
-                                        // Get original data using the extracted path components
-                                        const originalData = await this.get(
-                                            originHolon,
-                                            originLens,
-                                            originKey,
-                                            null,
-                                            { resolveReferences: false } // Prevent infinite recursion
-                                        );
-                                        
-                                        if (originalData) {
-                                            console.log(`Original data found through soul path resolution:`, originalData);
-                                            resolve({
-                                                ...originalData,
-                                                _federation: {
-                                                    isReference: true,
-                                                    resolved: true,
-                                                    soul: parsed.soul,
-                                                    timestamp: Date.now()
-                                                }
-                                            });
-                                            return;
-                                        } else {
-                                            console.warn(`Could not resolve reference: original data not found at extracted path`);
-                                        }
-                                    } else {
-                                        console.warn(`Soul doesn't match expected format: ${parsed.soul}`);
-                                    }
-                                } catch (error) {
-                                    console.warn(`Error resolving reference by soul: ${error.message}`);
-                                }
-                            }
-                            // Legacy federation reference
-                            else if (parsed._federation && parsed._federation.isReference) {
-                                console.log(`Resolving legacy federation reference from ${parsed._federation.origin}`);
-                                try {
-                                    const reference = parsed._federation;
-                                    const originalData = await this.get(
-                                        reference.origin,
-                                        reference.lens,
-                                        key,
-                                        null,
-                                        { resolveReferences: false } // Prevent infinite recursion
-                                    );
-                                    
-                                    if (originalData) {
-                                        return {
-                                            ...originalData,
-                                            _federation: {
-                                                ...reference,
-                                                resolved: true,
-                                                timestamp: Date.now()
-                                            }
-                                        };
-                                    } else {
-                                        console.warn(`Could not resolve legacy reference: original data not found`);
-                                        return parsed; // Return the reference if we can't resolve it
-                                    }
-                                } catch (error) {
-                                    console.warn(`Error resolving legacy reference: ${error.message}`);
-                                    return parsed;
-                                }
+                        if (resolveReferences && this.isReference(parsed)) {
+                            const resolved = await this.resolveReference(parsed, {
+                                followReferences: true // Always follow nested references when resolving
+                            });
+                            
+                            if (schema && resolved._federation) {
+                                // Skip schema validation for resolved references
+                                resolve(resolved);
+                                return;
+                            } else if (resolved !== parsed) {
+                                // Reference was resolved successfully
+                                resolve(resolved);
+                                return;
                             }
                         }
 
+                        // Perform schema validation if needed
                         if (schema) {
                             const valid = this.validator.validate(schema, parsed);
                             if (!valid) {
@@ -470,13 +397,11 @@ class HoloSphere {
                     }
                 };
 
-                if (password) {
-                    // For private data, use the authenticated user's holon
-                    user.get('private').get(lens).get(key).once(handleData);
-                } else {
-                    // For public data, use the regular path
-                    this.gun.get(this.appname).get(holon).get(lens).get(key).once(handleData);
-                }
+                const dataPath = password ? 
+                    user.get('private').get(lens).get(key) :
+                    this.gun.get(this.appname).get(holon).get(lens).get(key);
+
+                dataPath.once(handleData);
             });
         } catch (error) {
             console.error('Error in get:', error);
@@ -496,7 +421,7 @@ class HoloSphere {
 
         console.log(`getNodeBySoul: Accessing soul ${soul}`);
 
-        return new Promise((resolve) => {
+        return new Promise((resolve, reject) => {
             try {
                 const ref = this.getNodeRef(soul);
                 ref.once((data) => {
@@ -509,7 +434,7 @@ class HoloSphere {
                 });
             } catch (error) {
                 console.error(`getNodeBySoul error:`, error);
-                resolve(null);
+                reject(error);
             }
         });
     }
@@ -544,7 +469,16 @@ class HoloSphere {
         }
 
         try {
-            const user = this.gun.user();
+            let user = null;
+            if (password) {
+                user = this.gun.user();
+                await new Promise((resolve, reject) => {
+                    user.auth(this.userName(holon), password, (ack) => {
+                        if (ack.err) reject(new Error(ack.err));
+                        else resolve();
+                    });
+                });
+            }
 
             return new Promise((resolve) => {
                 const output = new Map();
@@ -556,6 +490,26 @@ class HoloSphere {
                         const parsed = await this.parse(data);
                         if (!parsed || !parsed.id) return;
 
+                        // Check if this is a reference that needs to be resolved
+                        if (this.isReference(parsed)) {
+                            const resolved = await this.resolveReference(parsed, {
+                                followReferences: true // Always follow references
+                            });
+                            
+                            if (resolved !== parsed) {
+                                // Reference was resolved successfully
+                                if (schema) {
+                                    const valid = this.validator.validate(schema, resolved);
+                                    if (valid || !this.strict) {
+                                        output.set(resolved.id, resolved);
+                                    }
+                                } else {
+                                    output.set(resolved.id, resolved);
+                                }
+                                return;
+                            }
+                        }
+
                         if (schema) {
                             const valid = this.validator.validate(schema, parsed);
                             if (valid || !this.strict) {
@@ -591,13 +545,11 @@ class HoloSphere {
                     }
                 };
 
-                if (password) {
-                    // For private data, use the authenticated user's holon
-                    user.get('private').get(lens).once(handleData);
-                } else {
-                    // For public data, use the regular path
-                    this.gun.get(this.appname).get(holon).get(lens).once(handleData);
-                }
+                const dataPath = password ? 
+                    user.get('private').get(lens) :
+                    this.gun.get(this.appname).get(holon).get(lens);
+
+                dataPath.once(handleData);
             });
         } catch (error) {
             console.error('Error in getAll:', error);
@@ -677,30 +629,29 @@ class HoloSphere {
         }
 
         try {
-            // Get the appropriate holon
-            const user = this.gun.user();
+            let user = null;
+            if (password) {
+                user = this.gun.user();
+                await new Promise((resolve, reject) => {
+                    user.auth(this.userName(holon), password, (ack) => {
+                        if (ack.err) reject(new Error(ack.err));
+                        else resolve();
+                    });
+                });
+            }
 
-            // Delete data from holon
             return new Promise((resolve, reject) => {
-                if (password) {
-                    // For private data, use the authenticated user's holon
-                    user.get('private').get(lens).get(key).put(null, ack => {
-                        if (ack.err) {
-                            reject(new Error(ack.err));
-                        } else {
-                            resolve(true);
-                        }
-                    });
-                } else {
-                    // For public data, use the regular path
-                    this.gun.get(this.appname).get(holon).get(lens).get(key).put(null, ack => {
-                        if (ack.err) {
-                            reject(new Error(ack.err));
-                        } else {
-                            resolve(true);
-                        }
-                    });
-                }
+                const dataPath = password ? 
+                    user.get('private').get(lens).get(key) :
+                    this.gun.get(this.appname).get(holon).get(lens).get(key);
+
+                dataPath.put(null, ack => {
+                    if (ack.err) {
+                        reject(new Error(ack.err));
+                    } else {
+                        resolve(true);
+                    }
+                });
             });
         } catch (error) {
             console.error('Error in delete:', error);
@@ -722,8 +673,16 @@ class HoloSphere {
         }
 
         try {
-            // Get the appropriate holon
-            const user = this.gun.user();
+            let user = null;
+            if (password) {
+                user = this.gun.user();
+                await new Promise((resolve, reject) => {
+                    user.auth(this.userName(holon), password, (ack) => {
+                        if (ack.err) reject(new Error(ack.err));
+                        else resolve();
+                    });
+                });
+            }
 
             return new Promise((resolve) => {
                 let deletionPromises = [];
@@ -820,18 +779,22 @@ class HoloSphere {
             throw new Error('getNode: Missing required parameters');
         }
 
-        return new Promise((resolve) => {
-            this.gun.get(this.appname)
-                .get(holon)
-                .get(lens)
-                .get(key)
-                .once((data) => {
-                    if (!data) {
-                        resolve(null);
-                        return;
-                    }
-                    resolve(data);  // Return the data directly
-                });
+        return new Promise((resolve, reject) => {
+            try {
+                this.gun.get(this.appname)
+                    .get(holon)
+                    .get(lens)
+                    .get(key)
+                    .once((data) => {
+                        if (!data) {
+                            resolve(null);
+                            return;
+                        }
+                        resolve(data);  // Return the data directly
+                    });
+            } catch (error) {
+                reject(error);
+            }
         });
     }
 
@@ -898,96 +861,29 @@ class HoloSphere {
                 throw new Error('Table name and data are required');
             }
 
-            const user = this.gun.user();
-            
+            let user = null;
             if (password) {
-                try {
-                    // Try to authenticate first
-                    await new Promise((resolve, reject) => {
-                        user.auth(this.userName(tableName), password, (ack) => {
-                            if (ack.err) {
-                                // Handle wrong username/password gracefully
-                                if (ack.err.includes('Wrong user or password') || 
-                                    ack.err.includes('No user')) {
-                                    console.warn(`Authentication failed for ${tableName}: ${ack.err}`);
-                                    // Will try to create user next
-                                    reject(new Error(ack.err));
-                                } else {
-                                    reject(new Error(ack.err));
-                                }
-                            } else {
-                                resolve();
-                            }
-                        });
+                user = this.gun.user();
+                await new Promise((resolve, reject) => {
+                    user.auth(this.userName(tableName), password, (ack) => {
+                        if (ack.err) reject(new Error(ack.err));
+                        else resolve();
                     });
-                } catch (authError) {
-                    // If authentication fails, try to create user
-                    try {
-                        await new Promise((resolve, reject) => {
-                            user.create(this.userName(tableName), password, (ack) => {
-                                // Handle "User already created!" error gracefully
-                                if (ack.err && !ack.err.includes('already created')) {
-                                    reject(new Error(ack.err));
-                                } else {
-                                    // Whether user was created or already existed, try to authenticate
-                                    user.auth(this.userName(tableName), password, (authAck) => {
-                                        if (authAck.err) {
-                                            console.warn(`Authentication failed after creation for ${tableName}: ${authAck.err}`);
-                                            reject(new Error(authAck.err));
-                                        } else {
-                                            resolve();
-                                        }
-                                    });
-                                }
-                            });
-                        });
-                    } catch (createError) {
-                        // If both auth and create fail, try one last auth attempt
-                        await new Promise((resolve, reject) => {
-                            user.auth(this.userName(tableName), password, (ack) => {
-                                if (ack.err) {
-                                    console.warn(`Final authentication attempt failed for ${tableName}: ${ack.err}`);
-                                    // Continue with operation even if auth fails
-                                    resolve();
-                                } else {
-                                    resolve();
-                                }
-                            });
-                        });
-                    }
-                }
+                });
             }
 
             return new Promise((resolve, reject) => {
-                const payload = JSON.stringify(data);
-                
-                if (password) {
-                    // For private data, use the authenticated user's holon
-                    const path = user.get('private').get(tableName);
+                try {
+                    const payload = JSON.stringify(data);
                     
-                    if (data.id) {
-                        path.get(data.id).put(payload, ack => {
-                            if (ack.err) {
-                                reject(new Error(ack.err));
-                            } else {
-                                resolve();
-                            }
-                        });
-                    } else {
-                        path.put(payload, ack => {
-                            if (ack.err) {
-                                reject(new Error(ack.err));
-                            } else {
-                                resolve();
-                            }
-                        });
-                    }
-                } else {
-                    // For public data, use the regular path
-                    const path = this.gun.get(this.appname).get(tableName);
+                    const dataPath = password ? 
+                        user.get('private').get(tableName) :
+                        this.gun.get(this.appname).get(tableName);
                     
                     if (data.id) {
-                        path.get(data.id).put(payload, ack => {
+                        // Store at the specific key path
+                        dataPath.get(data.id).put(payload, ack => {
+
                             if (ack.err) {
                                 reject(new Error(ack.err));
                             } else {
@@ -995,7 +891,7 @@ class HoloSphere {
                             }
                         });
                     } else {
-                        path.put(payload, ack => {
+                        dataPath.put(payload, ack => {
                             if (ack.err) {
                                 reject(new Error(ack.err));
                             } else {
@@ -1003,6 +899,8 @@ class HoloSphere {
                             }
                         });
                     }
+                } catch (error) {
+                    reject(error);
                 }
             });
         } catch (error) {
@@ -1019,72 +917,59 @@ class HoloSphere {
      * @returns {Promise<object|null>} - The parsed data for the key or null if not found.
      */
     async getGlobal(tableName, key, password = null) {
-        try {
-            const user = this.gun.user();
-            
+        try { 
+            let user = null;
             if (password) {
-                try {
-                    await new Promise((resolve, reject) => {
-                        user.auth(this.userName(tableName), password, (ack) => {
-                            if (ack.err) {
-                                // Handle wrong username/password gracefully
-                                if (ack.err.includes('Wrong user or password') || 
-                                    ack.err.includes('No user')) {
-                                    console.warn(`Authentication failed for ${tableName}: ${ack.err}`);
-                                    // Will try to create user next
-                                    reject(new Error(ack.err));
-                                } else {
-                                    reject(new Error(ack.err));
-                                }
-                            } else {
-                                resolve();
-                            }
-                        });
-                    });
-                } catch (loginError) {
-                    // If authentication fails, try to create user and then authenticate
-                    await new Promise((resolve, reject) => {
-                        user.create(this.userName(tableName), password, (ack) => {
-                            // Handle "User already created!" error gracefully
-                            if (ack.err && !ack.err.includes('already created')) {
-                                reject(new Error(ack.err));
-                            } else {
-                                user.auth(this.userName(tableName), password, (authAck) => {
-                                    if (authAck.err) {
-                                        console.warn(`Authentication failed after creation for ${tableName}: ${authAck.err}`);
-                                        // Continue with operation even if auth fails
-                                        resolve();
-                                    } else {
-                                        resolve();
-                                    }
-                                });
-                            }
-                        });
+                user = this.gun.user();
+                await new Promise((resolve, reject) => {
+                    user.auth(this.userName(tableName), password, (ack) => {
+                        if (ack.err) reject(new Error(ack.err));
+                        else resolve();
                     });
-                }
+                });
             }
 
             return new Promise((resolve) => {
-                const handleData = (data) => {
+                const handleData = async (data) => {
                     if (!data) {
                         resolve(null);
                         return;
                     }
+                    
                     try {
-                        const parsed = this.parse(data);
+                        // The data should be a stringified JSON from putGlobal
+                        const parsed = await this.parse(data);
+                        
+                        if (!parsed) {
+                            resolve(null);
+                            return;
+                        }
+                        
+                        // Check if this is a reference that needs to be resolved
+                        if (this.isReference(parsed)) {
+                            const resolved = await this.resolveReference(parsed, {
+                                followReferences: true // Always follow references
+                            });
+                            
+                            if (resolved !== parsed) {
+                                // Reference was resolved successfully
+                                resolve(resolved);
+                                return;
+                            }
+                        }
+                        
                         resolve(parsed);
                     } catch (e) {
+                        console.error('Error parsing data in getGlobal:', e);
                         resolve(null);
                     }
                 };
                 
-                if (password) {
-                    // For private data, use the authenticated user's holon
-                    user.get('private').get(tableName).get(key).once(handleData);
-                } else {
-                    // For public data, use the regular path
-                    this.gun.get(this.appname).get(tableName).get(key).once(handleData);
-                }
+                const dataPath = password ? 
+                    user.get('private').get(tableName) :
+                    this.gun.get(this.appname).get(tableName);
+
+                dataPath.get(key).once(handleData);
             });
         } catch (error) {
             console.error('Error in getGlobal:', error);
@@ -1104,8 +989,16 @@ class HoloSphere {
         }
 
         try {
-            // Get the appropriate holon
-            const user = this.gun.user();
+            let user = null;
+            if (password) {
+                user = this.gun.user();
+                await new Promise((resolve, reject) => {
+                    user.auth(this.userName(tableName), password, (ack) => {
+                        if (ack.err) reject(new Error(ack.err));
+                        else resolve();
+                    });
+                });
+            }
 
             return new Promise((resolve) => {
                 let output = [];
@@ -1139,7 +1032,23 @@ class HoloSphere {
                             if (itemData) {
                                 try {
                                     const parsed = await this.parse(itemData);
-                                    if (parsed) output.push(parsed);
+                                    if (parsed) {
+                                        // Check if this is a reference that needs to be resolved
+                                        if (this.isReference(parsed)) {
+                                            const resolved = await this.resolveReference(parsed, {
+                                                followReferences: true // Always follow references
+                                            });
+                                            
+                                            if (resolved !== parsed) {
+                                                // Reference was resolved successfully
+                                                output.push(resolved);
+                                            } else {
+                                                output.push(parsed);
+                                            }
+                                        } else {
+                                            output.push(parsed);
+                                        }
+                                    }
                                 } catch (error) {
                                     console.error('Error parsing data:', error);
                                 }
@@ -1156,13 +1065,11 @@ class HoloSphere {
                     }
                 };
 
-                if (password) {
-                    // For private data, use the authenticated user's holon
-                    user.get('private').get(tableName).once(handleData);
-                } else {
-                    // For public data, use the regular path
-                    this.gun.get(this.appname).get(tableName).once(handleData);
-                }
+                const dataPath = password ? 
+                    user.get('private').get(tableName) :
+                    this.gun.get(this.appname).get(tableName);
+
+                dataPath.once(handleData);
             });
         } catch (error) {
             console.error('Error in getAllGlobal:', error);
@@ -1183,29 +1090,45 @@ class HoloSphere {
         }
 
         try {
-            // Get the appropriate holon
-            const user = this.gun.user();
+            console.log('deleteGlobal - Starting deletion:', { tableName, key, hasPassword: !!password });
+            
+            let user = null;
+            if (password) {
+                user = this.gun.user();
+                await new Promise((resolve, reject) => {
+                    user.auth(this.userName(tableName), password, (ack) => {
+                        if (ack.err) reject(new Error(ack.err));
+                        else resolve();
+                    });
+                });
+            }
 
             return new Promise((resolve, reject) => {
-                if (password) {
-                    // For private data, use the authenticated user's holon
-                    user.get('private').get(tableName).get(key).put(null, ack => {
-                        if (ack.err) {
-                            reject(new Error(ack.err));
-                        } else {
-                            resolve(true);
-                        }
-                    });
-                } else {
-                    // For public data, use the regular path
-                    this.gun.get(this.appname).get(tableName).get(key).put(null, ack => {
+                const dataPath = password ? 
+                    user.get('private').get(tableName) :
+                    this.gun.get(this.appname).get(tableName);
+
+                console.log('deleteGlobal - Constructed base path:', dataPath._.back);
+
+                // First verify the data exists
+                dataPath.get(key).once((data) => {
+                    console.log('deleteGlobal - Data before deletion:', data);
+                    
+                    // Now perform the deletion
+                    dataPath.get(key).put(null, ack => {
+                        console.log('deleteGlobal - Deletion acknowledgment:', ack);
                         if (ack.err) {
+                            console.error('deleteGlobal - Deletion error:', ack.err);
                             reject(new Error(ack.err));
                         } else {
-                            resolve(true);
+                            // Verify deletion
+                            dataPath.get(key).once((deletedData) => {
+                                console.log('deleteGlobal - Data after deletion:', deletedData);
+                                resolve(true);
+                            });
                         }
                     });
-                }
+                });
             });
         } catch (error) {
             console.error('Error in deleteGlobal:', error);
@@ -1225,8 +1148,16 @@ class HoloSphere {
         }
 
         try {
-            // Get the appropriate holon
-            const user = this.gun.user();
+            let user = null;
+            if (password) {
+                user = this.gun.user();
+                await new Promise((resolve, reject) => {
+                    user.auth(this.userName(tableName), password, (ack) => {
+                        if (ack.err) reject(new Error(ack.err));
+                        else resolve();
+                    });
+                });
+            }
 
             return new Promise((resolve, reject) => {
                 try {
@@ -1250,7 +1181,7 @@ class HoloSphere {
 
                         const keys = Object.keys(data).filter(key => key !== '_');
                         const promises = keys.map(key =>
-                            new Promise((resolveDelete) => {
+                            new Promise((resolveDelete, rejectDelete) => {
                                 const deletePath = password ? 
                                     user.get('private').get(tableName).get(key) :
                                     this.gun.get(this.appname).get(tableName).get(key);
@@ -1258,8 +1189,10 @@ class HoloSphere {
                                 deletePath.put(null, ack => {
                                     if (ack.err) {
                                         console.error(`Failed to delete ${key}:`, ack.err);
+                                        rejectDelete(new Error(ack.err));
+                                    } else {
+                                        resolveDelete();
                                     }
-                                    resolveDelete();
                                 });
                             })
                         );
@@ -1284,6 +1217,159 @@ class HoloSphere {
         }
     }
 
+    // ================================ REFERENCE FUNCTIONS ================================
+
+    /**
+     * Creates a soul reference object for a data item
+     * @param {string} holon - The holon where the original data is stored
+     * @param {string} lens - The lens where the original data is stored
+     * @param {object} data - The data to create a reference for
+     * @returns {object} - A reference object with id and soul
+     */
+    createReference(holon, lens, data) {
+        if (!holon || !lens || !data || !data.id) {
+            throw new Error('createReference: Missing required parameters');
+        }
+        
+        const soul = `${this.appname}/${holon}/${lens}/${data.id}`;
+        return {
+            id: data.id,
+            soul: soul
+        };
+    }
+    
+    /**
+     * Parses a soul path into its components
+     * @param {string} soul - The soul path to parse
+     * @returns {object|null} - The parsed components or null if invalid format
+     */
+    parseSoulPath(soul) {
+        if (!soul || typeof soul !== 'string') {
+            return null;
+        }
+        
+        const soulParts = soul.split('/');
+        if (soulParts.length < 4) {
+            return null;
+        }
+        
+        return {
+            appname: soulParts[0],
+            holon: soulParts[1],
+            lens: soulParts[2],
+            key: soulParts[3]
+        };
+    }
+    
+    /**
+     * Checks if an object is a reference
+     * @param {object} data - The data to check
+     * @returns {boolean} - True if the object is a reference
+     */
+    isReference(data) {
+        if (!data || typeof data !== 'object') {
+            return false;
+        }
+        
+        // Check for direct soul reference
+        if (data.soul && typeof data.soul === 'string' && data.id) {
+            return true;
+        }
+        
+        // Check for legacy federation reference
+        if (data._federation && data._federation.isReference) {
+            return true;
+        }
+        
+        return false;
+    }
+    
+    /**
+     * Resolves a reference to its actual data
+     * @param {object} reference - The reference to resolve
+     * @param {object} [options] - Optional parameters
+     * @param {boolean} [options.followReferences=true] - Whether to follow nested references
+     * @returns {Promise<object|null>} - The resolved data or null if not found
+     */
+    async resolveReference(reference, options = {}) {
+        if (!this.isReference(reference)) {
+            return reference; // Not a reference, return as is
+        }
+        
+        const { followReferences = true } = options;
+        
+        try {
+            // Handle direct soul reference
+            if (reference.soul) {
+                const soulInfo = this.parseSoulPath(reference.soul);
+                if (!soulInfo) {
+                    console.warn(`Invalid soul format: ${reference.soul}`);
+                    return reference;
+                }
+                
+                console.log(`Resolving reference with soul: ${reference.soul}`);
+                
+                // Get original data using the extracted path components
+                const originalData = await this.get(
+                    soulInfo.holon,
+                    soulInfo.lens,
+                    soulInfo.key,
+                    null,
+                    { resolveReferences: followReferences } // Control recursion
+                );
+                
+                if (originalData) {
+                    console.log(`Original data found through soul path resolution`);
+                    return {
+                        ...originalData,
+                        _federation: {
+                            isReference: true,
+                            resolved: true,
+                            soul: reference.soul,
+                            timestamp: Date.now()
+                        }
+                    };
+                } else {
+                    console.warn(`Could not resolve reference: original data not found at extracted path`);
+                    return reference; // Return original reference if resolution fails
+                }
+            }
+            
+            // Handle legacy federation reference
+            else if (reference._federation && reference._federation.isReference) {
+                const fedRef = reference._federation;
+                console.log(`Resolving legacy federation reference from ${fedRef.origin}`);
+                
+                const originalData = await this.get(
+                    fedRef.origin,
+                    fedRef.lens,
+                    reference.id || fedRef.key,
+                    null,
+                    { resolveReferences: followReferences }
+                );
+                
+                if (originalData) {
+                    return {
+                        ...originalData,
+                        _federation: {
+                            ...fedRef,
+                            resolved: true,
+                            timestamp: Date.now()
+                        }
+                    };
+                } else {
+                    console.warn(`Could not resolve legacy reference: original data not found`);
+                    return reference;
+                }
+            }
+            
+            return reference;
+        } catch (error) {
+            console.error(`Error resolving reference: ${error.message}`, error);
+            return reference;
+        }
+    }
+
     // ================================ COMPUTE FUNCTIONS ================================
     /**
      * Computes operations across multiple layers up the hierarchy
@@ -1491,8 +1577,7 @@ class HoloSphere {
     }
 
     /**
-     * Upcasts content to parent holonagons recursively using federation and soul references.
-     * This is the modern implementation that uses federation references instead of duplicating data.
+     * Upcasts content to parent holonagons recursively using references.
      * @param {string} holon - The current holon identifier.
      * @param {string} lens - The lens under which to upcast.
      * @param {object} content - The content to upcast.
@@ -1513,18 +1598,10 @@ class HoloSphere {
         // Get the parent cell
         let parent = h3.cellToParent(holon, res - 1);
         
-        // Create federation relationship if it doesn't exist
-        await this.federate(holon, parent);
-        
-        // Create a soul reference to store in the parent
-        const soul = `${this.appname}/${holon}/${lens}/${content.id}`;
-        const reference = {
-            id: content.id,
-            soul: soul
-        };
+        // Create a reference to store in the parent
+        const reference = this.createReference(holon, lens, content);
         
         // Store the reference in the parent cell
-        // We use { autoPropagate: false } to prevent circular propagation
         await this.put(parent, lens, reference, null, { 
             autoPropagate: false 
         });
@@ -1607,8 +1684,12 @@ class HoloSphere {
      * @returns {Promise<object>} - Subscription object with unsubscribe method
      */
     async subscribe(holon, lens, callback) {
-        if (!holon || !lens || typeof callback !== 'function') {
-            throw new Error('subscribe: Missing required parameters');
+        if (!holon || !lens) {
+            throw new Error('subscribe: Missing holon or lens parameters:', holon, lens);
+        }
+        
+        if (!callback || typeof callback !== 'function') {
+            throw new Error('subscribe: Callback must be a function');
         }
 
         const subscriptionId = this.generateId();
@@ -1616,10 +1697,35 @@ class HoloSphere {
         try {
             // Create the subscription
             const gunSubscription = this.gun.get(this.appname).get(holon).get(lens).map().on(async (data, key) => {
+                // Check if subscription is still active before processing
+                if (!this.subscriptions[subscriptionId]?.active) {
+                    return;
+                }
+
                 if (data) {
                     try {
                         let parsed = await this.parse(data);
-                        callback(parsed, key);
+                        
+                        // Check if the parsed data is a reference that needs resolution
+                        if (parsed && this.isReference(parsed)) {
+                            const resolved = await this.resolveReference(parsed, {
+                                followReferences: true // Always follow references
+                            });
+                            
+                            if (resolved !== parsed) {
+                                // Reference was resolved successfully
+                                // Check again if subscription is still active
+                                if (this.subscriptions[subscriptionId]?.active) {
+                                    callback(resolved, key);
+                                }
+                                return;
+                            }
+                        }
+                        
+                        // Check again if subscription is still active before final callback
+                        if (this.subscriptions[subscriptionId]?.active) {
+                            callback(parsed, key);
+                        }
                     } catch (error) {
                         console.error('Error in subscribe:', error);
                     }
@@ -1632,6 +1738,7 @@ class HoloSphere {
                 holon,
                 lens,
                 active: true,
+                callback,
                 gunSubscription
             };
             
@@ -1639,14 +1746,18 @@ class HoloSphere {
             return {
                 unsubscribe: () => {
                     try {
-                        // Turn off the Gun subscription
-                        this.gun.get(this.appname).get(holon).get(lens).map().off();
-                        
-                        // Mark as inactive and remove from subscriptions
+                        // Mark as inactive first to prevent any new callbacks
                         if (this.subscriptions[subscriptionId]) {
                             this.subscriptions[subscriptionId].active = false;
-                            delete this.subscriptions[subscriptionId];
                         }
+                        
+                        // Turn off the Gun subscription using the stored reference
+                        if (this.subscriptions[subscriptionId]?.gunSubscription) {
+                            this.subscriptions[subscriptionId].gunSubscription.off();
+                        }
+                        
+                        // Remove from subscriptions
+                        delete this.subscriptions[subscriptionId];
                     } catch (error) {
                         console.error('Error in unsubscribe:', error);
                     }
@@ -1658,7 +1769,6 @@ class HoloSphere {
         }
     }
 
-
     /**
      * Notifies subscribers about data changes
      * @param {object} data - The data to notify about
@@ -1834,11 +1944,10 @@ class HoloSphere {
                     try {
                         const subscription = this.subscriptions[id];
                         if (subscription && subscription.active) {
-                            // Turn off the Gun subscription
-                            this.gun.get(this.appname)
-                                .get(subscription.holon)
-                                .get(subscription.lens)
-                                .map().off();
+                            // Turn off the Gun subscription using the stored reference
+                            if (subscription.gunSubscription) {
+                                subscription.gunSubscription.off();
+                            }
                             
                             // Mark as inactive
                             subscription.active = false;
@@ -1850,29 +1959,66 @@ class HoloSphere {
 
                 // Clear subscriptions
                 this.subscriptions = {};
+                
+                // Clear schema cache
+                this.clearSchemaCache();
 
                 // Close Gun connections
                 if (this.gun.back) {
                     try {
+                        // Clean up mesh connections
                         const mesh = this.gun.back('opt.mesh');
-                        if (mesh && mesh.hear) {
-                            try {
-                                // Safely clear mesh.hear without modifying function properties
-                                const hearKeys = Object.keys(mesh.hear);
-                                for (const key of hearKeys) {
-                                    // Check if it's an array before trying to clear it
-                                    if (Array.isArray(mesh.hear[key])) {
-                                        mesh.hear[key] = [];
+                        if (mesh) {
+                            // Clean up mesh.hear
+                            if (mesh.hear) {
+                                try {
+                                    // Safely clear mesh.hear without modifying function properties
+                                    const hearKeys = Object.keys(mesh.hear);
+                                    for (const key of hearKeys) {
+                                        // Check if it's an array before trying to clear it
+                                        if (Array.isArray(mesh.hear[key])) {
+                                            mesh.hear[key] = [];
+                                        }
+                                    }
+                                    
+                                    // Create a new empty object for mesh.hear
+                                    // Only if mesh.hear is not a function
+                                    if (typeof mesh.hear !== 'function') {
+                                        mesh.hear = {};
                                     }
+                                } catch (meshError) {
+                                    console.warn('Error cleaning up Gun mesh hear:', meshError);
                                 }
-                                
-                                // Create a new empty object for mesh.hear
-                                // Only if mesh.hear is not a function
-                                if (typeof mesh.hear !== 'function') {
-                                    mesh.hear = {};
+                            }
+                            
+                            // Close any open sockets in the mesh
+                            if (mesh.way) {
+                                try {
+                                    Object.values(mesh.way).forEach(connection => {
+                                        if (connection && connection.wire && connection.wire.close) {
+                                            connection.wire.close();
+                                        }
+                                    });
+                                } catch (sockError) {
+                                    console.warn('Error closing mesh sockets:', sockError);
+                                }
+                            }
+                            
+                            // Clear the peers list
+                            if (mesh.opt && mesh.opt.peers) {
+                                mesh.opt.peers = {};
+                            }
+                        }
+                        
+                        // Attempt to clean up any TCP connections
+                        if (this.gun.back('opt.web')) {
+                            try {
+                                const server = this.gun.back('opt.web');
+                                if (server && server.close) {
+                                    server.close();
                                 }
-                            } catch (meshError) {
-                                console.warn('Error cleaning up Gun mesh hear:', meshError);
+                            } catch (webError) {
+                                console.warn('Error closing web server:', webError);
                             }
                         }
                     } catch (error) {