holosphere 1.1.19 → 2.0.0-alpha0

package/scripts/import-holons.js

@@ -0,0 +1,578 @@
+#!/usr/bin/env node
+
+/**
+ * Import script for holosphere from GUN folder structure
+ *
+ * Folder structure expected:
+ * - 3-level: {appOrHolon}/{lens}/{id}.json
+ * - 4-level: {app}/{holon}/{lens}/{id}.json
+ *
+ * Usage:
+ *   node scripts/import-holons.js [options]
+ *
+ * Options:
+ *   --source <path>     Source directory (default: ../gun/Holons)
+ *   --subfolder <path>  Subfolder within source (e.g., "Appname/holonid")
+ *   --app <name>        App name for holosphere (default: use folder name or 'imported')
+ *   --dry-run           Show what would be imported without actually importing
+ *   --verbose           Show detailed progress
+ *   --levels <3|4>      Folder depth: 3 or 4 (default: auto-detect)
+ */
+
+import fs from 'fs/promises';
+import path from 'path';
+import { HoloSphere } from '../src/index.js';
+import dotenv from 'dotenv';
+
+// Load environment variables
+dotenv.config();
+
+// Parse command line arguments
+const args = process.argv.slice(2);
+const options = {
+  source: '../Holons',
+  subfolder: null,
+  app: 'Holons',
+  dryRun: false,
+  verbose: true,
+  levels: 'auto',
+  relays: [process.env.HOLOSPHERE_RELAY || 'wss://relay.holons.io'],
+  persistence: true,
+  // Retry and rate limiting options
+  maxRetries: 3,
+  retryDelay: 1000,
+  writeDelay: 100,
+  verifyAndResync: true
+};
+
+for (let i = 0; i < args.length; i++) {
+  switch (args[i]) {
+    case '--source':
+      options.source = args[++i];
+      break;
+    case '--subfolder':
+      options.subfolder = args[++i];
+      break;
+    case '--app':
+      options.app = args[++i];
+      break;
+    case '--dry-run':
+      options.dryRun = true;
+      break;
+    case '--verbose':
+      options.verbose = true;
+      break;
+    case '--relay':
+      options.relays = [args[++i]];
+      break;
+    case '--levels':
+      options.levels = args[++i];
+      break;
+    case '--help':
+      console.log(`
+Import holons from GUN folder structure into holosphere
+
+Usage: node scripts/import-holons.js [options]
+
+Options:
+  --source <path>     Source directory (default: ../Holons)
+  --subfolder <path>  Subfolder within source (e.g., "Appname/holonid")
+  --app <name>        App name for holosphere (default: use folder name or 'Holons')
+  --dry-run           Show what would be imported without actually importing
+  --verbose           Show detailed progress
+  --relay <url>       Relay URL (default: from HOLOSPHERE_RELAY env or wss://relay.holons.io)
+  --levels <3|4>      Folder depth: 3 or 4 (default: auto-detect)
+  --help              Show this help message
+
+Features:
+  - Automatic retry logic (up to 3 retries per item)
+  - Rate limiting to prevent overwhelming the relay (100ms between writes)
+  - Post-import verification and automatic resync of missing items
+  - All data is guaranteed to be on the relay when import completes
+
+Examples:
+  # Import from default location
+  node scripts/import-holons.js
+
+  # Import with specific app name
+  node scripts/import-holons.js --app myapp
+
+  # Dry run to see what would be imported
+  node scripts/import-holons.js --dry-run --verbose
+
+  # Import from custom location
+  node scripts/import-holons.js --source /path/to/holons
+
+  # Import from specific subfolder (recommended for single holon)
+  node scripts/import-holons.js --subfolder 235114395
+      `);
+      process.exit(0);
+  }
+}
+
+// Statistics
+const stats = {
+  apps: new Set(),
+  holons: new Set(),
+  lenses: new Set(),
+  items: 0,
+  errors: 0,
+  skipped: 0,
+  retries: 0,
+  resynced: 0
+};
+
+/**
+ * Sleep helper for delays
+ */
+async function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * Write with retry logic
+ */
+async function writeWithRetry(hs, holonId, lensName, data, retries = 0) {
+  try {
+    if (holonId === 'global') {
+      await hs.writeGlobal(lensName, data);
+    } else {
+      await hs.write(holonId, lensName, data);
+    }
+    return true;
+  } catch (error) {
+    if (retries < options.maxRetries) {
+      stats.retries++;
+      if (options.verbose) {
+        console.log(`  Retry ${retries + 1}/${options.maxRetries} for ${lensName}/${data.id}`);
+      }
+      await sleep(options.retryDelay);
+      return writeWithRetry(hs, holonId, lensName, data, retries + 1);
+    } else {
+      throw error;
+    }
+  }
+}
+
+/**
+ * Check if a string is a valid H3 cell ID
+ */
+function isH3Cell(str) {
+  return /^[0-9a-f]{15}$/i.test(str);
+}
+
+/**
+ * Recursively scan directory and import data
+ */
+async function scanAndImport(dir, hs, depth = 0, context = {}) {
+  try {
+    const entries = await fs.readdir(dir, { withFileTypes: true });
+
+    for (const entry of entries) {
+      const fullPath = path.join(dir, entry.name);
+
+      if (entry.isDirectory()) {
+        // Skip special directories
+        if (entry.name.startsWith('.') || entry.name === '_holograms') {
+          continue;
+        }
+
+        // Update context based on depth
+        const newContext = { ...context };
+
+        if (depth === 0) {
+          // First level: if we have a holon from subfolder, treat this as lens
+          if (context.holon) {
+            newContext.lens = entry.name;
+            stats.lenses.add(entry.name);
+          } else {
+            // Otherwise, treat as app name or holon ID
+            newContext.appOrHolon = entry.name;
+            stats.apps.add(entry.name);
+          }
+        } else if (depth === 1) {
+          // Second level: could be holon (if 4-level) or lens (if 3-level)
+          // If we already have a lens, this is just a container folder - keep the lens
+          if (context.lens) {
+            // Don't change lens, just keep recursing
+            // This handles cases like quests/1002/ where 1002 is just a container
+          } else if (isH3Cell(entry.name)) {
+            // Auto-detect: if it's an H3 cell, treat as holon
+            newContext.holon = entry.name;
+            stats.holons.add(entry.name);
+          } else {
+            // Otherwise treat as lens
+            newContext.lens = entry.name;
+            stats.lenses.add(entry.name);
+          }
+        } else if (depth === 2) {
+          // Third level or deeper: if we have a lens, keep it (container folders)
+          // Otherwise set as lens (4-level structure)
+          if (!context.lens) {
+            newContext.lens = entry.name;
+            stats.lenses.add(entry.name);
+          }
+        }
+
+        // Recurse into subdirectory
+        await scanAndImport(fullPath, hs, depth + 1, newContext);
+
+      } else if (entry.isFile() && entry.name.endsWith('.json')) {
+        // Import JSON file
+        await importFile(fullPath, hs, context, entry.name);
+      }
+    }
+  } catch (error) {
+    console.error(`Error scanning directory ${dir}:`, error.message);
+    stats.errors++;
+  }
+}
+
+/**
+ * Import a single JSON file into holosphere
+ */
+async function importFile(filePath, hs, context, filename) {
+  try {
+    // Read and parse JSON file
+    const content = await fs.readFile(filePath, 'utf-8');
+    let data = JSON.parse(content);
+
+    // Handle double-encoded JSON strings
+    if (typeof data === 'string') {
+      try {
+        data = JSON.parse(data);
+      } catch (e) {
+        // If it fails to parse again, keep it as a string value
+        // Wrap it in an object
+        data = { value: data };
+      }
+    }
+
+    // Handle non-object data
+    if (typeof data !== 'object' || data === null) {
+      data = { value: data };
+    }
+
+    // Extract ID from filename (remove .json extension)
+    const id = path.basename(filename, '.json');
+
+    // Ensure data has an id field
+    if (!data.id) {
+      data.id = id;
+    }
+
+    // Determine app, holon, and lens
+    let appName, holonId, lensName;
+
+    if (context.holon) {
+      // 4-level structure: app/holon/lens/id
+      appName = context.appOrHolon || options.app || 'imported';
+      holonId = context.holon;
+      lensName = context.lens;
+    } else if (context.lens) {
+      // 3-level structure: could be app/lens/id or holon/lens/id
+      // If the first level is an H3 cell, treat it as holon with default app
+      if (isH3Cell(context.appOrHolon)) {
+        appName = options.app || 'imported';
+        holonId = context.appOrHolon;
+      } else {
+        appName = context.appOrHolon || options.app || 'imported';
+        holonId = 'global'; // Use global for non-geographic data
+      }
+      lensName = context.lens;
+    } else if (context.appOrHolon) {
+      // 2-level structure: treat first level as lens, second as id
+      // This handles root-level files like /announcements/13.json
+      appName = options.app || 'imported';
+      holonId = 'global';
+      lensName = context.appOrHolon;
+    } else {
+      console.warn(`Skipping ${filePath}: unable to determine structure`);
+      stats.skipped++;
+      return;
+    }
+
+    if (!lensName) {
+      console.warn(`Skipping ${filePath}: no lens found`);
+      stats.skipped++;
+      return;
+    }
+
+    if (options.verbose) {
+      console.log(`Importing: app=${appName}, holon=${holonId}, lens=${lensName}, id=${id}`);
+    }
+
+    if (!options.dryRun) {
+      // Import into holosphere with retry logic
+      await writeWithRetry(hs, holonId, lensName, data);
+
+      // Rate limiting between writes
+      if (options.writeDelay > 0) {
+        await sleep(options.writeDelay);
+      }
+    }
+
+    stats.items++;
+
+    if (stats.items % 100 === 0) {
+      console.log(`Imported ${stats.items} items...`);
+    }
+
+  } catch (error) {
+    console.error(`Error importing ${filePath}:`, error.message);
+    stats.errors++;
+  }
+}
+
+/**
+ * Verify and resync data to relay
+ */
+async function verifyAndResync(hs, initialContext) {
+  console.log('\n======================');
+  console.log('Verifying Relay Sync');
+  console.log('======================\n');
+
+  // Get the holon ID from context
+  const holonId = initialContext.holon;
+  if (!holonId) {
+    console.log('No holon ID found, skipping verification');
+    return;
+  }
+
+  console.log(`Checking holon: ${holonId}`);
+  console.log('Waiting for relay sync...\n');
+  await sleep(3000); // Wait for initial sync
+
+  // Initialize a relay-only client to check what's actually on the relay
+  const hsRelay = new HoloSphere({
+    appName: options.app || 'imported',
+    relays: options.relays,
+    persistence: false, // Don't use local storage
+    logLevel: 'ERROR',
+    privateKey: process.env.HOLOSPHERE_PRIVATE_KEY
+  });
+
+  await sleep(2000);
+
+  const lensesToCheck = Array.from(stats.lenses);
+  let totalMissing = 0;
+  const missingByLens = {};
+
+  console.log('Comparing local vs relay data:\n');
+  console.log('Lens'.padEnd(20) + 'Local'.padEnd(10) + 'Relay'.padEnd(10) + 'Missing');
+  console.log('-'.repeat(50));
+
+  for (const lens of lensesToCheck) {
+    try {
+      // Read from local (hs has both local and relay)
+      const localData = await hs.read(holonId, lens);
+      // Read from relay only
+      const relayData = await hsRelay.read(holonId, lens);
+
+      const localMap = new Map();
+      const relayMap = new Map();
+
+      // Build local map
+      if (Array.isArray(localData)) {
+        localData.forEach(item => {
+          if (item && item.id) localMap.set(item.id.toString(), item);
+        });
+      } else if (localData && typeof localData === 'object') {
+        Object.keys(localData).forEach(key => {
+          const item = localData[key];
+          if (item && item.id) localMap.set(item.id.toString(), item);
+        });
+      }
+
+      // Build relay map
+      if (Array.isArray(relayData)) {
+        relayData.forEach(item => {
+          if (item && item.id) relayMap.set(item.id.toString(), item);
+        });
+      } else if (relayData && typeof relayData === 'object') {
+        Object.keys(relayData).forEach(key => {
+          const item = relayData[key];
+          if (item && item.id) relayMap.set(item.id.toString(), item);
+        });
+      }
+
+      const missing = [];
+      for (const [id, item] of localMap) {
+        if (!relayMap.has(id)) {
+          missing.push(item);
+        }
+      }
+
+      const missingCount = missing.length;
+      totalMissing += missingCount;
+
+      if (missingCount > 0) {
+        missingByLens[lens] = missing;
+      }
+
+      console.log(
+        lens.padEnd(20) +
+        localMap.size.toString().padEnd(10) +
+        relayMap.size.toString().padEnd(10) +
+        missingCount.toString()
+      );
+    } catch (error) {
+      console.log(lens.padEnd(20) + 'Error: ' + error.message);
+    }
+  }
+
+  console.log('-'.repeat(50));
+  console.log(`Total missing from relay: ${totalMissing}`);
+
+  // Resync missing items
+  if (totalMissing > 0) {
+    console.log('\n======================');
+    console.log('Re-syncing Missing Items');
+    console.log('======================\n');
+
+    for (const [lens, missing] of Object.entries(missingByLens)) {
+      console.log(`\nSyncing ${missing.length} items for lens: ${lens}`);
+
+      for (const item of missing) {
+        try {
+          await writeWithRetry(hsRelay, holonId, lens, item);
+          stats.resynced++;
+          if (options.verbose) {
+            console.log(`  ✓ ${lens}/${item.id}`);
+          }
+          await sleep(options.writeDelay);
+        } catch (error) {
+          console.log(`  ✗ ${lens}/${item.id}: ${error.message}`);
+        }
+      }
+    }
+
+    console.log(`\n✓ Re-synced ${stats.resynced} items to relay`);
+  } else {
+    console.log('\n✓ All data is already on the relay!');
+  }
+}
+
+/**
+ * Main import function
+ */
+async function main() {
+  console.log('Holosphere Import Tool');
+  console.log('======================\n');
+
+  // Resolve source path
+  let sourcePath = path.resolve(options.source);
+  let initialContext = {};
+  let startDepth = 0;
+
+  // Handle subfolder as holon ID
+  if (options.subfolder) {
+    sourcePath = path.join(sourcePath, options.subfolder);
+
+    // Extract holon ID from subfolder (last component of path)
+    const holonId = path.basename(options.subfolder);
+
+    // Pre-populate context with holon ID
+    // When holon is set, folders will be treated as lenses even at depth 0
+    initialContext = {
+      holon: holonId
+    };
+
+    stats.holons.add(holonId);
+
+    console.log(`Using subfolder as holon ID: ${holonId}`);
+  }
+
+  try {
+    await fs.access(sourcePath);
+  } catch (error) {
+    console.error(`Error: Source directory not found: ${sourcePath}`);
+    process.exit(1);
+  }
+
+  console.log(`Source: ${sourcePath}`);
+  if (options.subfolder) {
+    console.log(`Subfolder: ${options.subfolder}`);
+    console.log(`Holon ID: ${initialContext.holon}`);
+  }
+  console.log(`App: ${options.app || 'auto-detect'}`);
+  console.log(`Dry run: ${options.dryRun ? 'Yes' : 'No'}`);
+  console.log('');
+
+  // Initialize holosphere
+  let hs = null;
+  if (!options.dryRun) {
+    console.log('Initializing holosphere...');
+    hs = new HoloSphere({
+      appName: options.app || 'imported',
+      relays: options.relays,
+      persistence: options.persistence,
+      logLevel: options.verbose ? 'DEBUG' : 'INFO',
+      privateKey: process.env.HOLOSPHERE_PRIVATE_KEY
+    });
+    console.log('Holosphere initialized.\n');
+  }
+
+  // Start import
+  const startTime = Date.now();
+  console.log('Starting import...\n');
+
+  await scanAndImport(sourcePath, hs, startDepth, initialContext);
+
+  const duration = ((Date.now() - startTime) / 1000).toFixed(2);
+
+  // Print statistics
+  console.log('\n======================');
+  console.log('Import Complete');
+  console.log('======================');
+  console.log(`Duration: ${duration}s`);
+  console.log(`Apps: ${stats.apps.size}`);
+  console.log(`Holons: ${stats.holons.size}`);
+  console.log(`Lenses: ${stats.lenses.size}`);
+  console.log(`Items imported: ${stats.items}`);
+  console.log(`Errors: ${stats.errors}`);
+  console.log(`Skipped: ${stats.skipped}`);
+  if (stats.retries > 0) {
+    console.log(`Retries: ${stats.retries}`);
+  }
+
+  if (options.verbose) {
+    console.log('\nApps:', Array.from(stats.apps).join(', '));
+    console.log('Holons:', Array.from(stats.holons).join(', '));
+    console.log('Lenses:', Array.from(stats.lenses).join(', '));
+  }
+
+  if (!options.dryRun && hs) {
+    console.log('\nHolosphere metrics:', hs.metrics());
+
+    // Verify and resync if enabled
+    if (options.verifyAndResync && options.relays.length > 0) {
+      await verifyAndResync(hs, initialContext);
+    }
+  }
+
+  // Cleanup: Close connections and exit
+  if (!options.dryRun && hs) {
+    console.log('\nClosing connections...');
+    if (hs.client && hs.client.pool && hs.client.pool.close) {
+      try {
+        hs.client.pool.close(hs.client.relays);
+      } catch (e) {
+        // Ignore errors during cleanup
+      }
+    }
+  }
+
+  if (stats.resynced > 0) {
+    console.log(`\n✓ Import complete - ${stats.items} items imported, ${stats.resynced} re-synced to relay\n`);
+  } else {
+    console.log('\n✓ Import complete\n');
+  }
+  process.exit(0);
+}
+
+// Run main function
+main().catch(error => {
+  console.error('Fatal error:', error);
+  process.exit(1);
+});

package/scripts/publish-to-relay.js

@@ -0,0 +1,101 @@
+#!/usr/bin/env node
+
+/**
+ * Publish a sample of imported data to relay.holons.io
+ */
+
+import { HoloSphere } from '../src/index.js';
+import dotenv from 'dotenv';
+
+// Load environment variables
+dotenv.config();
+
+async function main() {
+  console.log('Publishing Sample Data to relay.holons.io');
+  console.log('==========================================\n');
+
+  // First, read from local cache (no relays)
+  console.log('Step 1: Reading sample data from local cache...');
+  const hsLocal = new HoloSphere({
+    appName: 'imported',
+    relays: [],
+    persistence: true,
+    logLevel: 'INFO',
+    privateKey: process.env.HOLOSPHERE_PRIVATE_KEY
+  });
+
+  await new Promise(resolve => setTimeout(resolve, 1000));
+
+  const quests = await hsLocal.readGlobal('quests');
+  const questArray = Object.values(quests || {});
+
+  console.log(`✓ Found ${questArray.length} quests locally\n`);
+
+  if (questArray.length === 0) {
+    console.log('No quests found to publish.');
+    return;
+  }
+
+  // Take first 5 quests
+  const sampleQuests = questArray.slice(0, 5);
+  console.log(`Selected ${sampleQuests.length} quests to publish:\n`);
+
+  sampleQuests.forEach((q, idx) => {
+    console.log(`${idx + 1}. ${q.title || q.id} (${q.status || 'unknown'})`);
+  });
+
+  console.log('\n');
+
+  // Now publish to relay.holons.io
+  console.log('Step 2: Publishing to relay.holons.io...');
+  const hsRelay = new HoloSphere({
+    appName: 'imported',
+    relays: ['wss://relay.holons.io'],
+    persistence: true,
+    logLevel: 'INFO',
+    privateKey: process.env.HOLOSPHERE_PRIVATE_KEY
+  });
+
+  let successCount = 0;
+  for (const quest of sampleQuests) {
+    try {
+      const success = await hsRelay.writeGlobal('quests', quest);
+      if (success) {
+        console.log(`✓ Published: ${quest.title || quest.id}`);
+        successCount++;
+      } else {
+        console.log(`✗ Failed: ${quest.title || quest.id}`);
+      }
+    } catch (error) {
+      console.log(`✗ Error: ${quest.title || quest.id} - ${error.message}`);
+    }
+  }
+
+  console.log(`\n✓ Published ${successCount}/${sampleQuests.length} quests\n`);
+
+  // Wait for propagation
+  console.log('Step 3: Waiting for relay propagation...');
+  await new Promise(resolve => setTimeout(resolve, 3000));
+  console.log('✓ Wait complete\n');
+
+  // Read back from relay
+  console.log('Step 4: Reading back from relay.holons.io...');
+  const retrievedQuests = await hsRelay.readGlobal('quests');
+  const retrievedArray = Object.values(retrievedQuests || {});
+
+  console.log(`✓ Found ${retrievedArray.length} quests on relay\n`);
+
+  if (retrievedArray.length > 0) {
+    console.log('Retrieved quests:');
+    retrievedArray.forEach((q, idx) => {
+      console.log(`${idx + 1}. ${q.title || q.id}`);
+      console.log(`   Status: ${q.status || 'unknown'}`);
+      console.log(`   ID: ${q.id}`);
+      console.log('');
+    });
+  }
+
+  console.log('✓ Complete!');
+}
+
+main().catch(console.error);