@pioneer-platform/pioneer-discovery 0.8.3 → 4.21.15

package/scripts/validate-and-download-icons.js

@@ -0,0 +1,326 @@
+#!/usr/bin/env node
+/*
+ * Validate and Download Icon URLs
+ *
+ * Priority order:
+ * 1. Check if api.keepkey.com/coins/{base64_caip}.png exists
+ * 2. Check if local file exists in pioneer-server/public/coins/
+ * 3. Try old URL (CoinGecko, TrustWallet, etc.)
+ * 4. Try known fallbacks (CoinGecko API search, symbol-based)
+ * 5. Log missing assets for manual resolution
+ */
+
+const fs = require('fs');
+const path = require('path');
+const https = require('https');
+const http = require('http');
+
+const DATA_FILE = path.join(__dirname, '..', 'src', 'generatedAssetData.json');
+const LOCAL_COINS_DIR = path.join(__dirname, '..', '..', '..', '..', '..', 'services', 'pioneer-server', 'public', 'coins');
+const MISSING_ASSETS_FILE = path.join(__dirname, 'missing-assets.json');
+
+// Ensure local coins directory exists
+if (!fs.existsSync(LOCAL_COINS_DIR)) {
+  console.log(`📁 Creating directory: ${LOCAL_COINS_DIR}`);
+  fs.mkdirSync(LOCAL_COINS_DIR, { recursive: true });
+}
+
+/**
+ * Convert assetId to base64-encoded format
+ */
+function encodeAssetId(assetId) {
+  return Buffer.from(assetId).toString('base64');
+}
+
+/**
+ * Check if URL exists (HEAD request)
+ */
+async function urlExists(url, timeout = 5000) {
+  return new Promise((resolve) => {
+    const urlObj = new URL(url);
+    const client = urlObj.protocol === 'https:' ? https : http;
+
+    const req = client.request(url, { method: 'HEAD', timeout }, (res) => {
+      resolve(res.statusCode >= 200 && res.statusCode < 400);
+    });
+
+    req.on('error', () => resolve(false));
+    req.on('timeout', () => {
+      req.destroy();
+      resolve(false);
+    });
+
+    req.end();
+  });
+}
+
+/**
+ * Download file from URL
+ */
+async function downloadFile(url, outputPath) {
+  return new Promise((resolve, reject) => {
+    const urlObj = new URL(url);
+    const client = urlObj.protocol === 'https:' ? https : http;
+
+    const req = client.get(url, { timeout: 10000 }, (res) => {
+      if (res.statusCode >= 200 && res.statusCode < 300) {
+        const fileStream = fs.createWriteStream(outputPath);
+        res.pipe(fileStream);
+        fileStream.on('finish', () => {
+          fileStream.close();
+          resolve(true);
+        });
+      } else if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
+        // Follow redirect
+        downloadFile(res.headers.location, outputPath).then(resolve).catch(reject);
+      } else {
+        resolve(false);
+      }
+    });
+
+    req.on('error', () => resolve(false));
+    req.on('timeout', () => {
+      req.destroy();
+      resolve(false);
+    });
+  });
+}
+
+/**
+ * Extract symbol/contract for fallback searches
+ */
+function extractSymbol(assetData) {
+  return assetData.symbol?.toLowerCase() || '';
+}
+
+/**
+ * Try various CoinGecko fallback URLs
+ */
+function getCoinGeckoFallbacks(assetData) {
+  const symbol = extractSymbol(assetData);
+  const fallbacks = [];
+
+  // Common CoinGecko patterns
+  if (symbol) {
+    fallbacks.push(`https://assets.coingecko.com/coins/images/1/small/${symbol}.png`);
+    fallbacks.push(`https://assets.coingecko.com/coins/images/1/large/${symbol}.png`);
+    fallbacks.push(`https://assets.coingecko.com/coins/images/1/thumb/${symbol}.png`);
+  }
+
+  // Common coins with known CoinGecko IDs
+  const knownIds = {
+    'btc': 'https://assets.coingecko.com/coins/images/1/small/bitcoin.png',
+    'eth': 'https://assets.coingecko.com/coins/images/279/large/ethereum.png',
+    'usdc': 'https://assets.coingecko.com/coins/images/6319/large/usdc.png',
+    'usdt': 'https://assets.coingecko.com/coins/images/325/large/tether.png',
+    'dai': 'https://assets.coingecko.com/coins/images/9956/large/dai.png',
+    'wbtc': 'https://assets.coingecko.com/coins/images/7598/large/wrapped_bitcoin.png',
+    'link': 'https://assets.coingecko.com/coins/images/877/large/chainlink.png',
+    'uni': 'https://assets.coingecko.com/coins/images/12504/large/uniswap.png',
+    'atom': 'https://assets.coingecko.com/coins/images/1481/large/cosmos_hub.png',
+    'osmo': 'https://assets.coingecko.com/coins/images/16724/large/osmosis.png',
+    'rune': 'https://assets.coingecko.com/coins/images/6595/large/thorchain.png',
+    'cacao': 'https://assets.coingecko.com/coins/images/27929/large/cacao.png',
+  };
+
+  if (knownIds[symbol]) {
+    fallbacks.push(knownIds[symbol]);
+  }
+
+  return fallbacks;
+}
+
+/**
+ * Try TrustWallet fallbacks
+ */
+function getTrustWalletFallbacks(assetData) {
+  const fallbacks = [];
+
+  // Extract contract address for ERC20 tokens
+  if (assetData.assetId?.includes('erc20:')) {
+    const match = assetData.assetId.match(/erc20:(0x[a-fA-F0-9]+)/);
+    if (match) {
+      const contract = match[1];
+      fallbacks.push(`https://raw.githubusercontent.com/trustwallet/assets/master/blockchains/ethereum/assets/${contract}/logo.png`);
+      fallbacks.push(`https://rawcdn.githack.com/trustwallet/assets/master/blockchains/ethereum/assets/${contract}/logo.png`);
+    }
+  }
+
+  return fallbacks;
+}
+
+/**
+ * Try all fallback sources
+ */
+async function tryFallbacks(assetData) {
+  const fallbacks = [
+    ...getCoinGeckoFallbacks(assetData),
+    ...getTrustWalletFallbacks(assetData),
+  ];
+
+  for (const url of fallbacks) {
+    const exists = await urlExists(url);
+    if (exists) {
+      return url;
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Process a single asset
+ */
+async function processAsset(assetId, assetData, stats) {
+  const encoded = encodeAssetId(assetId);
+  const targetUrl = `https://api.keepkey.com/coins/${encoded}.png`;
+  const localPath = path.join(LOCAL_COINS_DIR, `${encoded}.png`);
+
+  stats.total++;
+
+  // Step 1: Check if api.keepkey.com already has it
+  console.log(`\n🔍 [${stats.total}] Checking ${assetData.symbol} (${assetId.substring(0, 40)}...)`);
+
+  const remoteExists = await urlExists(targetUrl);
+  if (remoteExists) {
+    console.log(`   ✅ Already on api.keepkey.com`);
+    stats.alreadyOnRemote++;
+    assetData.icon = targetUrl;
+    return true;
+  }
+
+  // Step 2: Check if local file exists
+  if (fs.existsSync(localPath)) {
+    console.log(`   ✅ Already in local storage (not deployed yet)`);
+    stats.alreadyLocal++;
+    assetData.icon = targetUrl;
+    return true;
+  }
+
+  // Step 3: Try old URL if it exists
+  if (assetData.icon && !assetData.icon.includes('api.keepkey.com')) {
+    const oldUrl = assetData.icon;
+    console.log(`   🔄 Trying old URL: ${oldUrl.substring(0, 60)}...`);
+
+    const oldExists = await urlExists(oldUrl);
+    if (oldExists) {
+      console.log(`   📥 Downloading from old URL...`);
+      const downloaded = await downloadFile(oldUrl, localPath);
+      if (downloaded) {
+        console.log(`   ✅ Downloaded and saved locally`);
+        stats.downloaded++;
+        assetData.icon = targetUrl;
+        return true;
+      } else {
+        console.log(`   ❌ Failed to download from old URL`);
+      }
+    } else {
+      console.log(`   ❌ Old URL not accessible`);
+    }
+  }
+
+  // Step 4: Try known fallbacks
+  console.log(`   🔄 Trying fallback sources...`);
+  const fallbackUrl = await tryFallbacks(assetData);
+  if (fallbackUrl) {
+    console.log(`   📥 Found fallback: ${fallbackUrl.substring(0, 60)}...`);
+    const downloaded = await downloadFile(fallbackUrl, localPath);
+    if (downloaded) {
+      console.log(`   ✅ Downloaded from fallback and saved locally`);
+      stats.downloadedFallback++;
+      assetData.icon = targetUrl;
+      return true;
+    } else {
+      console.log(`   ❌ Failed to download from fallback`);
+    }
+  }
+
+  // Step 5: Mark as missing
+  console.log(`   ❌ No icon found - marked as missing`);
+  stats.missing.push({
+    assetId,
+    symbol: assetData.symbol,
+    name: assetData.name,
+    chainId: assetData.chainId,
+    oldIcon: assetData.icon || null,
+  });
+  stats.missingCount++;
+
+  // Keep the target URL even if file doesn't exist yet
+  assetData.icon = targetUrl;
+  return false;
+}
+
+/**
+ * Main execution
+ */
+async function main() {
+  console.log('🚀 Starting icon validation and download process...\n');
+  console.log(`📂 Local coins directory: ${LOCAL_COINS_DIR}\n`);
+
+  const data = JSON.parse(fs.readFileSync(DATA_FILE, 'utf8'));
+  const entries = Object.entries(data);
+
+  const stats = {
+    total: 0,
+    alreadyOnRemote: 0,
+    alreadyLocal: 0,
+    downloaded: 0,
+    downloadedFallback: 0,
+    missingCount: 0,
+    missing: [],
+  };
+
+  // Process assets with concurrency limit
+  const concurrency = 5;
+  for (let i = 0; i < entries.length; i += concurrency) {
+    const batch = entries.slice(i, i + concurrency);
+    await Promise.all(
+      batch.map(([assetId, assetData]) => processAsset(assetId, assetData, stats))
+    );
+  }
+
+  // Save updated data
+  console.log('\n\n💾 Saving updated asset data...');
+  fs.writeFileSync(DATA_FILE, JSON.stringify(data, null, 2), 'utf8');
+
+  // Save missing assets report
+  if (stats.missing.length > 0) {
+    console.log(`📝 Saving missing assets report to ${MISSING_ASSETS_FILE}...`);
+    fs.writeFileSync(MISSING_ASSETS_FILE, JSON.stringify(stats.missing, null, 2), 'utf8');
+  }
+
+  // Print summary
+  console.log('\n' + '='.repeat(80));
+  console.log('📊 SUMMARY REPORT');
+  console.log('='.repeat(80));
+  console.log(`Total assets processed:          ${stats.total}`);
+  console.log(`Already on api.keepkey.com:      ${stats.alreadyOnRemote} (${(stats.alreadyOnRemote/stats.total*100).toFixed(1)}%)`);
+  console.log(`Already in local storage:        ${stats.alreadyLocal} (${(stats.alreadyLocal/stats.total*100).toFixed(1)}%)`);
+  console.log(`Downloaded from old URL:         ${stats.downloaded}`);
+  console.log(`Downloaded from fallback:        ${stats.downloadedFallback}`);
+  console.log(`Missing (needs manual work):     ${stats.missingCount} (${(stats.missingCount/stats.total*100).toFixed(1)}%)`);
+  console.log('='.repeat(80));
+
+  if (stats.missing.length > 0) {
+    console.log('\n⚠️  Missing assets by chain:');
+    const byChain = {};
+    stats.missing.forEach(asset => {
+      const chain = asset.chainId || 'unknown';
+      byChain[chain] = (byChain[chain] || 0) + 1;
+    });
+    Object.entries(byChain)
+      .sort((a, b) => b[1] - a[1])
+      .forEach(([chain, count]) => {
+        console.log(`   ${chain}: ${count} missing`);
+      });
+    console.log(`\n   Full report saved to: ${MISSING_ASSETS_FILE}`);
+  }
+
+  console.log('\n✅ Process complete!\n');
+}
+
+main().catch(err => {
+  console.error('❌ Fatal error:', err);
+  process.exit(1);
+});