@pioneer-platform/pioneer-discovery 8.11.11 → 8.11.15

package/scripts/download-shapeshift-icons.js

@@ -0,0 +1,179 @@
+/**
+ * Download icons from ShapeShift icon mapping
+ */
+
+const fs = require('fs');
+const path = require('path');
+const https = require('https');
+const http = require('http');
+
+const TAG = ' | download-shapeshift-icons | ';
+
+// Paths
+const MAPPING_PATH = path.join(__dirname, 'shapeshift-icon-mapping.json');
+const DOWNLOAD_DIR = path.join(__dirname, '../../../../../services/pioneer-server/public/coins');
+const PROGRESS_FILE = path.join(__dirname, '.shapeshift-download-progress.json');
+const FAILED_FILE = path.join(__dirname, '.shapeshift-download-failed.json');
+
+// Helper: Encode CAIP ID to base64 (matches upload-to-s3.js pattern)
+function encodeAssetId(assetId) {
+  return Buffer.from(assetId).toString('base64');
+}
+
+// Configuration
+const BATCH_SIZE = 10; // Download 10 icons at a time
+const DELAY_BETWEEN_BATCHES = 1000; // 1 second delay between batches
+
+// Ensure download directory exists
+if (!fs.existsSync(DOWNLOAD_DIR)) {
+  fs.mkdirSync(DOWNLOAD_DIR, { recursive: true });
+}
+
+// Load mapping
+console.log(TAG, 'Loading ShapeShift icon mapping...');
+const mapping = JSON.parse(fs.readFileSync(MAPPING_PATH, 'utf8'));
+const { matches, stats } = mapping;
+
+console.log(TAG, `Total matches: ${matches.length}`);
+console.log(TAG, `NEW icons to download: ${stats.newIconsAvailable}`);
+
+// Load progress if exists
+let progress = { processed: new Set(), successful: 0, failed: [] };
+if (fs.existsSync(PROGRESS_FILE)) {
+  const saved = JSON.parse(fs.readFileSync(PROGRESS_FILE, 'utf8'));
+  progress.processed = new Set(saved.processed || []);
+  progress.successful = saved.successful || 0;
+  progress.failed = saved.failed || [];
+  console.log(TAG, `Resuming from previous run: ${progress.processed.size} already processed`);
+}
+
+// Download function
+function downloadIcon(url, filePath) {
+  return new Promise((resolve, reject) => {
+    const protocol = url.startsWith('https') ? https : http;
+
+    const request = protocol.get(url, (response) => {
+      if (response.statusCode === 200) {
+        const fileStream = fs.createWriteStream(filePath);
+        response.pipe(fileStream);
+
+        fileStream.on('finish', () => {
+          fileStream.close();
+          resolve(true);
+        });
+
+        fileStream.on('error', (err) => {
+          fs.unlink(filePath, () => {}); // Delete partial file
+          reject(err);
+        });
+      } else if (response.statusCode === 301 || response.statusCode === 302) {
+        // Follow redirect
+        const redirectUrl = response.headers.location;
+        resolve(downloadIcon(redirectUrl, filePath));
+      } else {
+        reject(new Error(`HTTP ${response.statusCode}: ${url}`));
+      }
+    });
+
+    request.on('error', reject);
+    request.setTimeout(10000, () => {
+      request.destroy();
+      reject(new Error('Timeout'));
+    });
+  });
+}
+
+// Save progress
+function saveProgress() {
+  fs.writeFileSync(PROGRESS_FILE, JSON.stringify({
+    processed: Array.from(progress.processed),
+    successful: progress.successful,
+    failed: progress.failed,
+    timestamp: Date.now()
+  }, null, 2));
+
+  fs.writeFileSync(FAILED_FILE, JSON.stringify(progress.failed, null, 2));
+}
+
+// Process matches in batches
+async function processMatches() {
+  const toProcess = matches.filter(m => !progress.processed.has(m.caipId));
+
+  console.log(TAG, `\nProcessing ${toProcess.length} icons in batches of ${BATCH_SIZE}...\n`);
+
+  for (let i = 0; i < toProcess.length; i += BATCH_SIZE) {
+    const batch = toProcess.slice(i, i + BATCH_SIZE);
+    const batchNum = Math.floor(i / BATCH_SIZE) + 1;
+    const totalBatches = Math.ceil(toProcess.length / BATCH_SIZE);
+
+    console.log(`\n📦 Batch ${batchNum}/${totalBatches} (${i + 1}-${Math.min(i + BATCH_SIZE, toProcess.length)}/${toProcess.length})`);
+
+    const downloads = batch.map(async (match) => {
+      const filename = match.encodedFilename || `${encodeAssetId(match.caipId)}.png`;
+      const filePath = path.join(DOWNLOAD_DIR, filename);
+
+      // Skip if already exists
+      if (fs.existsSync(filePath)) {
+        console.log(`  ⏭️  ${match.pioneerSymbol.padEnd(10)} Already exists`);
+        progress.processed.add(match.caipId);
+        return { success: true, skipped: true };
+      }
+
+      try {
+        await downloadIcon(match.iconUrl, filePath);
+        console.log(`  ✅ ${match.pioneerSymbol.padEnd(10)} ${match.pioneerName.slice(0, 30)}`);
+        progress.processed.add(match.caipId);
+        progress.successful++;
+        return { success: true };
+      } catch (error) {
+        console.log(`  ❌ ${match.pioneerSymbol.padEnd(10)} ${error.message}`);
+        progress.processed.add(match.caipId);
+        progress.failed.push({
+          caipId: match.caipId,
+          symbol: match.pioneerSymbol,
+          name: match.pioneerName,
+          iconUrl: match.iconUrl,
+          error: error.message
+        });
+        return { success: false };
+      }
+    });
+
+    await Promise.all(downloads);
+
+    // Save progress after each batch
+    saveProgress();
+
+    // Delay between batches
+    if (i + BATCH_SIZE < toProcess.length) {
+      await new Promise(resolve => setTimeout(resolve, DELAY_BETWEEN_BATCHES));
+    }
+  }
+
+  // Final summary
+  console.log('\n========================================');
+  console.log('📊 DOWNLOAD COMPLETE');
+  console.log('========================================');
+  console.log(`Total processed:       ${progress.processed.size}`);
+  console.log(`Successfully downloaded: ${progress.successful}`);
+  console.log(`Failed:                ${progress.failed.length}`);
+  console.log(`Success rate:          ${((progress.successful / progress.processed.size) * 100).toFixed(1)}%`);
+  console.log('========================================\n');
+
+  if (progress.failed.length > 0) {
+    console.log(`⚠️  Failed downloads saved to: ${FAILED_FILE}`);
+  }
+  console.log(`✅ Progress saved to: ${PROGRESS_FILE}`);
+}
+
+// Run
+processMatches()
+  .then(() => {
+    console.log(TAG, '✅ All done!');
+    process.exit(0);
+  })
+  .catch((error) => {
+    console.error(TAG, '❌ Error:', error);
+    saveProgress();
+    process.exit(1);
+  });

package/scripts/extract-shapeshift-icons.js

@@ -0,0 +1,152 @@
+/**
+ * Extract icon URLs from ShapeShift's asset data and map to Pioneer assets
+ *
+ * ShapeShift structure:
+ * - assetIdPrefixes: array of CAIP prefixes
+ * - encodedAssetIds: array of compressed IDs like "6:0xdac17..."
+ * - encodedAssets: array of [assetIdx, name, precision, color, icon[], symbol, relatedAssetKey, isPool]
+ *
+ * FIELDS order: ['assetIdx', 'name', 'precision', 'color', 'icon', 'symbol', 'relatedAssetKey', 'isPool']
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const TAG = ' | extract-shapeshift-icons | ';
+
+// Paths
+const SHAPESHIFT_DATA_PATH = '/Users/highlander/WebstormProjects/web/src/lib/asset-service/service/encodedAssetData.json';
+const PIONEER_ASSETS_PATH = path.join(__dirname, '../src/generatedAssetData.json');
+const OUTPUT_MAPPING = path.join(__dirname, 'shapeshift-icon-mapping.json');
+const DOWNLOAD_DIR = path.join(__dirname, '../../../../../services/pioneer-server/public/coins');
+
+console.log(TAG, 'Loading ShapeShift asset data...');
+const shapeshiftData = JSON.parse(fs.readFileSync(SHAPESHIFT_DATA_PATH, 'utf8'));
+const { assetIdPrefixes, encodedAssetIds, encodedAssets } = shapeshiftData;
+
+console.log(TAG, `Loaded ${encodedAssetIds.length} ShapeShift assets`);
+
+// Decode ShapeShift asset IDs
+function decodeAssetId(encodedId, prefixes) {
+  const colonIndex = encodedId.lastIndexOf(':');
+  const prefixIdx = Number(encodedId.substring(0, colonIndex));
+  const assetReference = encodedId.substring(colonIndex + 1);
+  return prefixes[prefixIdx] + ':' + assetReference;
+}
+
+// Field indices from FIELDS constant
+const FIELD_ICON = 4;
+const FIELD_SYMBOL = 5;
+const FIELD_NAME = 1;
+
+// Build ShapeShift asset map: CAIP ID -> icon URL(s)
+const shapeshiftIconMap = new Map();
+let totalWithIcons = 0;
+
+encodedAssetIds.forEach((encodedId, idx) => {
+  const assetId = decodeAssetId(encodedId, assetIdPrefixes);
+  const encodedAsset = encodedAssets[idx];
+  const iconArray = encodedAsset[FIELD_ICON];
+  const symbol = encodedAsset[FIELD_SYMBOL];
+  const name = encodedAsset[FIELD_NAME];
+
+  if (iconArray && iconArray.length > 0) {
+    shapeshiftIconMap.set(assetId, {
+      assetId,
+      symbol,
+      name,
+      icons: iconArray,
+      primaryIcon: iconArray[0]
+    });
+    totalWithIcons++;
+  }
+});
+
+console.log(TAG, `ShapeShift assets with icons: ${totalWithIcons} / ${encodedAssetIds.length}`);
+
+// Load Pioneer assets
+console.log(TAG, 'Loading Pioneer assets...');
+const pioneerAssets = JSON.parse(fs.readFileSync(PIONEER_ASSETS_PATH, 'utf8'));
+console.log(TAG, `Loaded ${Object.keys(pioneerAssets).length} Pioneer assets`);
+
+// Helper: Encode CAIP ID to base64 (matches upload-to-s3.js pattern)
+function encodeAssetId(assetId) {
+  return Buffer.from(assetId).toString('base64');
+}
+
+// Match Pioneer assets to ShapeShift icons
+const matches = [];
+const missing = [];
+
+Object.entries(pioneerAssets).forEach(([caipId, asset]) => {
+  const shapeshiftAsset = shapeshiftIconMap.get(caipId);
+
+  if (shapeshiftAsset) {
+    const encodedFilename = `${encodeAssetId(caipId)}.png`;
+    matches.push({
+      caipId,
+      encodedFilename,
+      pioneerSymbol: asset.symbol,
+      pioneerName: asset.name,
+      shapeshiftSymbol: shapeshiftAsset.symbol,
+      shapeshiftName: shapeshiftAsset.name,
+      iconUrl: shapeshiftAsset.primaryIcon,
+      allIcons: shapeshiftAsset.icons,
+      hasLocalIcon: fs.existsSync(path.join(DOWNLOAD_DIR, encodedFilename))
+    });
+  } else {
+    missing.push({
+      caipId,
+      symbol: asset.symbol,
+      name: asset.name
+    });
+  }
+});
+
+console.log(TAG, '\n========================================');
+console.log(TAG, 'MATCHING RESULTS');
+console.log(TAG, '========================================');
+console.log(TAG, `Total Pioneer assets:     ${Object.keys(pioneerAssets).length}`);
+console.log(TAG, `ShapeShift matches found: ${matches.length}`);
+console.log(TAG, `Missing from ShapeShift:  ${missing.length}`);
+console.log(TAG, `Match rate:               ${((matches.length / Object.keys(pioneerAssets).length) * 100).toFixed(1)}%`);
+
+// Count how many we already have locally
+const alreadyHaveLocal = matches.filter(m => m.hasLocalIcon).length;
+const newIconsAvailable = matches.filter(m => !m.hasLocalIcon).length;
+
+console.log(TAG, '\n========================================');
+console.log(TAG, 'ICON AVAILABILITY');
+console.log(TAG, '========================================');
+console.log(TAG, `Already have locally:     ${alreadyHaveLocal}`);
+console.log(TAG, `NEW icons available:      ${newIconsAvailable} 🎉`);
+
+// Save mapping file
+const mapping = {
+  generatedAt: new Date().toISOString(),
+  stats: {
+    totalPioneerAssets: Object.keys(pioneerAssets).length,
+    shapeshiftMatches: matches.length,
+    alreadyHaveLocal,
+    newIconsAvailable,
+    missingFromShapeshift: missing.length
+  },
+  matches,
+  missing: missing.slice(0, 100) // Sample of missing
+};
+
+fs.writeFileSync(OUTPUT_MAPPING, JSON.stringify(mapping, null, 2));
+console.log(TAG, `\n✅ Mapping saved to: ${OUTPUT_MAPPING}`);
+
+// Show sample of new icons available
+console.log(TAG, '\n========================================');
+console.log(TAG, 'SAMPLE NEW ICONS (first 20)');
+console.log(TAG, '========================================');
+matches
+  .filter(m => !m.hasLocalIcon)
+  .slice(0, 20)
+  .forEach(m => {
+    console.log(`${m.pioneerSymbol.padEnd(10)} ${m.pioneerName.slice(0, 30).padEnd(32)} ${m.iconUrl}`);
+  });
+
+console.log(TAG, '\n✅ Done! Run download-shapeshift-icons.js to download the new icons');

package/scripts/fix-s3-permissions.js

@@ -0,0 +1,315 @@
+#!/usr/bin/env node
+
+/*
+ * Fix S3 Permissions for Existing Files
+ *
+ * This script:
+ * 1. Lists all files in the S3 bucket
+ * 2. Sets ACL to 'public-read' for each file
+ * 3. Verifies accessibility after setting ACL
+ * 4. Reports any files that still have issues
+ *
+ * Use this to fix existing files that return 403 Forbidden
+ */
+
+const { execSync } = require('child_process');
+const https = require('https');
+const fs = require('fs');
+const path = require('path');
+
+// Configuration
+const S3_ENDPOINT = 'https://sfo3.digitaloceanspaces.com';
+const S3_BUCKET = 'keepkey';
+const S3_REGION = 'sfo3';
+const CDN_URL = 'https://keepkey.sfo3.cdn.digitaloceanspaces.com';
+
+// AWS credentials
+const AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID || 'DO00FXP8KK64LCXYAEZP';
+const AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY || 'Uyw/cq63rrQmFV9yy1HbovTSMNhLkEwImqPa88N/E/s';
+
+const PROGRESS_FILE = path.join(__dirname, '.fix-permissions-progress.json');
+const FAILED_FILE = path.join(__dirname, '.fix-permissions-failed.json');
+
+/**
+ * Check if URL is accessible (returns 200)
+ */
+async function urlExists(url, timeout = 5000) {
+  return new Promise((resolve) => {
+    const req = https.request(url, { method: 'HEAD', timeout }, (res) => {
+      resolve(res.statusCode === 200);
+    });
+
+    req.on('error', () => resolve(false));
+    req.on('timeout', () => {
+      req.destroy();
+      resolve(false);
+    });
+
+    req.end();
+  });
+}
+
+/**
+ * Set ACL to public-read for a file
+ */
+function setPublicAcl(s3Key) {
+  try {
+    const cmd = `AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} aws s3api put-object-acl --bucket ${S3_BUCKET} --key "${s3Key}" --acl public-read --endpoint-url=${S3_ENDPOINT}`;
+
+    execSync(cmd, { stdio: 'pipe' });
+    return true;
+  } catch (error) {
+    console.error(`   ❌ ACL set error: ${error.message}`);
+    return false;
+  }
+}
+
+/**
+ * Wait for specified milliseconds
+ */
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * Get list of all files in S3 bucket
+ */
+function listS3Files() {
+  // Use cached file list to avoid ENOBUFS error
+  const cachedFile = path.join(__dirname, '..', '..', '..', '..', 's3-files.txt');
+
+  if (fs.existsSync(cachedFile)) {
+    console.log('📋 Using cached S3 file list...');
+    const output = fs.readFileSync(cachedFile, 'utf8');
+
+    const files = output
+      .split('\n')
+      .filter(line => line.trim())
+      .map(line => {
+        const parts = line.trim().split(/\s+/);
+        return parts[3]; // filename
+      })
+      .filter(filename => filename && filename.endsWith('.png'));
+
+    return files;
+  }
+
+  // Fallback to AWS CLI if no cached file
+  try {
+    console.log('📋 Fetching S3 file list (this may take a while)...');
+    const cmd = `AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} aws s3 ls s3://${S3_BUCKET}/coins/ --endpoint-url=${S3_ENDPOINT} --recursive`;
+
+    const output = execSync(cmd, { encoding: 'utf8', maxBuffer: 50 * 1024 * 1024 });
+
+    const files = output
+      .split('\n')
+      .filter(line => line.trim())
+      .map(line => {
+        const parts = line.trim().split(/\s+/);
+        return parts[3]; // filename
+      })
+      .filter(filename => filename && filename.endsWith('.png'));
+
+    return files;
+  } catch (error) {
+    console.error('❌ Failed to list S3 files:', error.message);
+    return [];
+  }
+}
+
+/**
+ * Load progress
+ */
+function loadProgress() {
+  if (fs.existsSync(PROGRESS_FILE)) {
+    return JSON.parse(fs.readFileSync(PROGRESS_FILE, 'utf8'));
+  }
+  return { processed: [], fixed: 0, failed: 0 };
+}
+
+/**
+ * Save progress
+ */
+function saveProgress(progress) {
+  fs.writeFileSync(PROGRESS_FILE, JSON.stringify(progress, null, 2));
+}
+
+/**
+ * Load failed
+ */
+function loadFailed() {
+  if (fs.existsSync(FAILED_FILE)) {
+    return JSON.parse(fs.readFileSync(FAILED_FILE, 'utf8'));
+  }
+  return { files: [] };
+}
+
+/**
+ * Save failed
+ */
+function saveFailed(failed) {
+  fs.writeFileSync(FAILED_FILE, JSON.stringify(failed, null, 2));
+}
+
+/**
+ * Process a single file
+ */
+async function processFile(s3Key, progress, failed, stats) {
+  stats.total++;
+
+  // Skip if already processed
+  if (progress.processed.includes(s3Key)) {
+    console.log(`⏭️  [${stats.total}] Skipping (already processed): ${s3Key.substring(0, 80)}`);
+    stats.skipped++;
+    return;
+  }
+
+  const cdnUrl = `${CDN_URL}/${s3Key}`;
+
+  console.log(`\n🔄 [${stats.total}/${stats.totalFiles}] Processing: ${s3Key.substring(0, 80)}...`);
+
+  // Check current accessibility
+  console.log(`   🔍 Checking current accessibility...`);
+  const accessible = await urlExists(cdnUrl);
+
+  if (accessible) {
+    console.log(`   ✅ Already accessible - skipping`);
+    progress.processed.push(s3Key);
+    stats.alreadyAccessible++;
+    return;
+  }
+
+  console.log(`   ⚠️  Currently returns 403 - setting public-read ACL...`);
+
+  // Set ACL
+  const aclSet = setPublicAcl(s3Key);
+
+  if (!aclSet) {
+    console.log(`   ❌ Failed to set ACL`);
+    failed.files.push({
+      s3Key,
+      cdnUrl,
+      reason: 'acl_set_failed',
+    });
+    progress.processed.push(s3Key);
+    progress.failed++;
+    stats.aclFailed++;
+    return;
+  }
+
+  console.log(`   ✅ ACL set - waiting for propagation...`);
+  await sleep(1000);
+
+  // Verify accessibility
+  console.log(`   🔍 Verifying accessibility...`);
+  const nowAccessible = await urlExists(cdnUrl);
+
+  if (nowAccessible) {
+    console.log(`   ✅ Now accessible!`);
+    progress.fixed++;
+    stats.fixed++;
+  } else {
+    console.log(`   ❌ Still not accessible`);
+    failed.files.push({
+      s3Key,
+      cdnUrl,
+      reason: 'still_not_accessible',
+    });
+    progress.failed++;
+    stats.stillFailed++;
+  }
+
+  progress.processed.push(s3Key);
+
+  // Save progress every 10 files
+  if (stats.total % 10 === 0) {
+    saveProgress(progress);
+    saveFailed(failed);
+    console.log(`\n📊 Progress saved - Total: ${stats.total}/${stats.totalFiles}, Fixed: ${stats.fixed}, Failed: ${stats.aclFailed + stats.stillFailed}`);
+  }
+}
+
+/**
+ * Main execution
+ */
+async function main() {
+  console.log('🚀 Starting S3 permissions fix process...\n');
+  console.log(`☁️  S3 bucket: ${S3_BUCKET}`);
+  console.log(`🌐 CDN URL: ${CDN_URL}\n`);
+
+  // Get list of all files
+  console.log('📋 Getting list of all files in S3...');
+  const files = listS3Files();
+
+  if (files.length === 0) {
+    console.error('❌ No files found in S3');
+    process.exit(1);
+  }
+
+  console.log(`✅ Found ${files.length} files\n`);
+
+  // Load progress
+  const progress = loadProgress();
+  const failed = loadFailed();
+
+  const stats = {
+    total: 0,
+    totalFiles: files.length,
+    skipped: 0,
+    alreadyAccessible: 0,
+    fixed: 0,
+    aclFailed: 0,
+    stillFailed: 0,
+  };
+
+  console.log(`📊 Already processed: ${progress.processed.length}`);
+  console.log(`📊 Remaining: ${files.length - progress.processed.length}\n`);
+
+  // Process each file
+  for (const s3Key of files) {
+    await processFile(s3Key, progress, failed, stats);
+
+    // Small delay to be nice to the API
+    await sleep(200);
+  }
+
+  // Final save
+  saveProgress(progress);
+  saveFailed(failed);
+
+  // Cleanup progress file if complete
+  if (progress.processed.length === files.length && failed.files.length === 0) {
+    fs.unlinkSync(PROGRESS_FILE);
+    console.log('\n✅ All files processed successfully - progress file cleaned up');
+  }
+
+  // Print summary
+  console.log('\n' + '='.repeat(80));
+  console.log('📊 FINAL SUMMARY');
+  console.log('='.repeat(80));
+  console.log(`Total files:                 ${files.length}`);
+  console.log(`Already processed (skipped): ${stats.skipped}`);
+  console.log(`Already accessible:          ${stats.alreadyAccessible}`);
+  console.log(`Fixed successfully:          ${stats.fixed}`);
+  console.log(`ACL set failed:              ${stats.aclFailed}`);
+  console.log(`Still not accessible:        ${stats.stillFailed}`);
+  console.log(`Total failed:                ${stats.aclFailed + stats.stillFailed}`);
+  console.log('='.repeat(80));
+
+  if (failed.files.length > 0) {
+    console.log(`\n⚠️  ${failed.files.length} files failed - see ${FAILED_FILE}`);
+
+    // Sample of failed files
+    console.log('\nSample of failed files:');
+    failed.files.slice(0, 10).forEach(file => {
+      console.log(`   ${file.s3Key} - ${file.reason}`);
+    });
+  }
+
+  console.log('\n✅ Permission fix process complete!\n');
+}
+
+main().catch(err => {
+  console.error('❌ Fatal error:', err);
+  process.exit(1);
+});