@pioneer-platform/pioneer-discovery 0.8.3 → 4.21.15

package/scripts/update-icon-urls.js

@@ -0,0 +1,86 @@
+#!/usr/bin/env node
+/*
+ * Update Icon URLs to use api.keepkey.info
+ * Converts all assets.coingecko.com URLs to base64-encoded CAIP format
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const DATA_FILE = path.join(__dirname, '..', 'src', 'generatedAssetData.json');
+
+/**
+ * Convert assetId to base64-encoded format for icon URL
+ * @param {string} assetId - CAIP format assetId (e.g., "eip155:8453/erc20:0x833...")
+ * @returns {string} - Base64 encoded assetId
+ */
+function encodeAssetId(assetId) {
+  return Buffer.from(assetId).toString('base64');
+}
+
+/**
+ * Generate icon URL from assetId
+ * @param {string} assetId - CAIP format assetId
+ * @returns {string} - Full icon URL
+ */
+function generateIconUrl(assetId) {
+  const encoded = encodeAssetId(assetId);
+  return `https://api.keepkey.info/coins/${encoded}.png`;
+}
+
+async function updateIconUrls() {
+  console.log('Reading generatedAssetData.json...');
+
+  const data = JSON.parse(fs.readFileSync(DATA_FILE, 'utf8'));
+
+  let updatedCount = 0;
+  let alreadyCorrect = 0;
+  let skippedCount = 0;
+
+  for (const [assetId, assetData] of Object.entries(data)) {
+    if (!assetData.icon) {
+      console.log(`⚠️  Missing icon for ${assetId}`);
+      skippedCount++;
+      continue;
+    }
+
+    // Check if already using api.keepkey.info
+    if (assetData.icon.includes('api.keepkey.info')) {
+      alreadyCorrect++;
+      continue;
+    }
+
+    // Update to use api.keepkey.info with base64-encoded CAIP
+    const newIconUrl = generateIconUrl(assetId);
+    const oldIconUrl = assetData.icon;
+
+    assetData.icon = newIconUrl;
+    updatedCount++;
+
+    if (updatedCount <= 10) {
+      console.log(`\n✅ Updated ${assetId}:`);
+      console.log(`   Old: ${oldIconUrl}`);
+      console.log(`   New: ${newIconUrl}`);
+    }
+  }
+
+  console.log('\n📊 Summary:');
+  console.log(`   Total entries: ${Object.keys(data).length}`);
+  console.log(`   Already correct: ${alreadyCorrect}`);
+  console.log(`   Updated: ${updatedCount}`);
+  console.log(`   Skipped (no icon): ${skippedCount}`);
+
+  if (updatedCount > 0) {
+    console.log('\n💾 Writing updated data...');
+    fs.writeFileSync(DATA_FILE, JSON.stringify(data, null, 2), 'utf8');
+    console.log('✅ Successfully updated generatedAssetData.json');
+  } else {
+    console.log('\n✨ No updates needed - all icons already correct!');
+  }
+}
+
+// Run the update
+updateIconUrls().catch(err => {
+  console.error('❌ Error:', err);
+  process.exit(1);
+});

package/scripts/validate-and-download-icons-fast.js

@@ -0,0 +1,304 @@
+#!/usr/bin/env node
+/*
+ * Fast Icon Validation and Download
+ *
+ * Optimizations:
+ * - Faster timeouts (2s instead of 5s)
+ * - Progress saving (can resume)
+ * - Better concurrency handling
+ * - Skip already processed assets
+ */
+
+const fs = require('fs');
+const path = require('path');
+const https = require('https');
+const http = require('http');
+
+const DATA_FILE = path.join(__dirname, '..', 'src', 'generatedAssetData.json');
+const LOCAL_COINS_DIR = path.join(__dirname, '..', '..', '..', '..', '..', 'services', 'pioneer-server', 'public', 'coins');
+const MISSING_ASSETS_FILE = path.join(__dirname, 'missing-assets.json');
+const PROGRESS_FILE = path.join(__dirname, '.download-progress.json');
+
+// Ensure local coins directory exists
+if (!fs.existsSync(LOCAL_COINS_DIR)) {
+  fs.mkdirSync(LOCAL_COINS_DIR, { recursive: true });
+}
+
+/**
+ * Convert assetId to base64-encoded format
+ */
+function encodeAssetId(assetId) {
+  return Buffer.from(assetId).toString('base64');
+}
+
+/**
+ * Check if URL exists (HEAD request) - Fast version
+ */
+async function urlExists(url, timeout = 2000) {
+  return new Promise((resolve) => {
+    try {
+      const urlObj = new URL(url);
+      const client = urlObj.protocol === 'https:' ? https : http;
+
+      const req = client.request(url, { method: 'HEAD', timeout }, (res) => {
+        resolve(res.statusCode >= 200 && res.statusCode < 400);
+      });
+
+      req.on('error', () => resolve(false));
+      req.on('timeout', () => {
+        req.destroy();
+        resolve(false);
+      });
+
+      req.end();
+    } catch (e) {
+      resolve(false);
+    }
+  });
+}
+
+/**
+ * Download file from URL - Fast version
+ */
+async function downloadFile(url, outputPath, timeout = 5000) {
+  return new Promise((resolve) => {
+    try {
+      const urlObj = new URL(url);
+      const client = urlObj.protocol === 'https:' ? https : http;
+
+      const req = client.get(url, { timeout }, (res) => {
+        if (res.statusCode >= 200 && res.statusCode < 300) {
+          const fileStream = fs.createWriteStream(outputPath);
+          res.pipe(fileStream);
+          fileStream.on('finish', () => {
+            fileStream.close();
+            resolve(true);
+          });
+          fileStream.on('error', () => resolve(false));
+        } else if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
+          // Follow redirect
+          downloadFile(res.headers.location, outputPath, timeout).then(resolve);
+        } else {
+          resolve(false);
+        }
+      });
+
+      req.on('error', () => resolve(false));
+      req.on('timeout', () => {
+        req.destroy();
+        resolve(false);
+      });
+    } catch (e) {
+      resolve(false);
+    }
+  });
+}
+
+/**
+ * Get CoinGecko fallbacks
+ */
+function getCoinGeckoFallbacks(assetData) {
+  const symbol = assetData.symbol?.toLowerCase() || '';
+
+  const knownIds = {
+    'btc': '1', 'eth': '279', 'usdc': '6319', 'usdt': '325',
+    'dai': '9956', 'wbtc': '7598', 'link': '877', 'uni': '12504',
+    'atom': '1481', 'osmo': '16724', 'rune': '6595', 'cacao': '27929',
+    'bnb': '825', 'bch': '780', 'ltc': '2', 'doge': '5', 'ada': '2010',
+    'dot': '6636', 'matic': '4713', 'avax': '12559', 'sol': '4128',
+  };
+
+  const fallbacks = [];
+  if (knownIds[symbol]) {
+    fallbacks.push(`https://assets.coingecko.com/coins/images/${knownIds[symbol]}/large/${symbol}.png`);
+    fallbacks.push(`https://assets.coingecko.com/coins/images/${knownIds[symbol]}/small/${symbol}.png`);
+  }
+
+  return fallbacks;
+}
+
+/**
+ * Get TrustWallet fallbacks
+ */
+function getTrustWalletFallbacks(assetData) {
+  const fallbacks = [];
+
+  if (assetData.assetId?.includes('erc20:')) {
+    const match = assetData.assetId.match(/erc20:(0x[a-fA-F0-9]+)/);
+    if (match) {
+      const contract = match[1];
+      // Checksum the address properly
+      const checksummed = contract.substring(0, 2) + contract.substring(2).split('')
+        .map((char, i) => i % 2 === 0 && parseInt(char, 16) >= 8 ? char.toUpperCase() : char)
+        .join('');
+
+      fallbacks.push(`https://raw.githubusercontent.com/trustwallet/assets/master/blockchains/ethereum/assets/${checksummed}/logo.png`);
+    }
+  }
+
+  return fallbacks;
+}
+
+/**
+ * Process a single asset - Fast version
+ */
+async function processAsset(assetId, assetData, stats, progress) {
+  const encoded = encodeAssetId(assetId);
+  const localPath = path.join(LOCAL_COINS_DIR, `${encoded}.png`);
+
+  stats.total++;
+
+  // Skip if already processed
+  if (progress.processed[assetId]) {
+    stats.skipped++;
+    return true;
+  }
+
+  // Check local file first (fastest)
+  if (fs.existsSync(localPath)) {
+    const fileSize = fs.statSync(localPath).size;
+    if (fileSize > 0) {
+      stats.alreadyLocal++;
+      progress.processed[assetId] = 'local';
+      assetData.icon = `https://api.keepkey.com/coins/${encoded}.png`;
+      return true;
+    }
+  }
+
+  console.log(`[${stats.total}/${stats.totalAssets}] ${assetData.symbol} - ${assetId.substring(0, 50)}...`);
+
+  // Try old URL if it exists
+  if (assetData.icon && !assetData.icon.includes('api.keepkey.com') && !assetData.icon.includes('api.keepkey.info')) {
+    const oldUrl = assetData.icon;
+    const downloaded = await downloadFile(oldUrl, localPath);
+    if (downloaded) {
+      console.log(`  ✅ Downloaded from old URL`);
+      stats.downloaded++;
+      progress.processed[assetId] = 'old_url';
+      assetData.icon = `https://api.keepkey.com/coins/${encoded}.png`;
+      return true;
+    }
+  }
+
+  // Try fallbacks
+  const fallbacks = [
+    ...getCoinGeckoFallbacks(assetData),
+    ...getTrustWalletFallbacks(assetData),
+  ];
+
+  for (const url of fallbacks) {
+    const downloaded = await downloadFile(url, localPath);
+    if (downloaded) {
+      console.log(`  ✅ Downloaded from fallback: ${url.substring(0, 60)}...`);
+      stats.downloadedFallback++;
+      progress.processed[assetId] = 'fallback';
+      assetData.icon = `https://api.keepkey.com/coins/${encoded}.png`;
+      return true;
+    }
+  }
+
+  // Mark as missing
+  console.log(`  ❌ No icon found`);
+  stats.missing.push({
+    assetId,
+    symbol: assetData.symbol,
+    name: assetData.name,
+    chainId: assetData.chainId,
+    oldIcon: assetData.icon || null,
+  });
+  stats.missingCount++;
+  progress.processed[assetId] = 'missing';
+
+  // Still set the target URL
+  assetData.icon = `https://api.keepkey.com/coins/${encoded}.png`;
+  return false;
+}
+
+/**
+ * Save progress
+ */
+function saveProgress(progress, data, stats) {
+  fs.writeFileSync(PROGRESS_FILE, JSON.stringify(progress, null, 2));
+  fs.writeFileSync(DATA_FILE, JSON.stringify(data, null, 2));
+  if (stats.missing.length > 0) {
+    fs.writeFileSync(MISSING_ASSETS_FILE, JSON.stringify(stats.missing, null, 2));
+  }
+}
+
+/**
+ * Main execution
+ */
+async function main() {
+  console.log('🚀 Fast icon validation and download\n');
+
+  const data = JSON.parse(fs.readFileSync(DATA_FILE, 'utf8'));
+  const entries = Object.entries(data);
+
+  // Load progress if exists
+  let progress = { processed: {}, timestamp: Date.now() };
+  if (fs.existsSync(PROGRESS_FILE)) {
+    progress = JSON.parse(fs.readFileSync(PROGRESS_FILE, 'utf8'));
+    console.log(`📂 Resuming from previous run (${Object.keys(progress.processed).length} already processed)\n`);
+  }
+
+  const stats = {
+    total: 0,
+    totalAssets: entries.length,
+    skipped: 0,
+    alreadyLocal: 0,
+    downloaded: 0,
+    downloadedFallback: 0,
+    missingCount: 0,
+    missing: [],
+  };
+
+  // Process in batches with concurrency
+  const batchSize = 10;
+  const saveInterval = 50; // Save progress every 50 assets
+
+  for (let i = 0; i < entries.length; i += batchSize) {
+    const batch = entries.slice(i, i + batchSize);
+    await Promise.all(
+      batch.map(([assetId, assetData]) => processAsset(assetId, assetData, stats, progress))
+    );
+
+    // Save progress periodically
+    if (i % saveInterval === 0) {
+      saveProgress(progress, data, stats);
+    }
+  }
+
+  // Final save
+  saveProgress(progress, data, stats);
+
+  // Print summary
+  console.log('\n' + '='.repeat(80));
+  console.log('📊 SUMMARY');
+  console.log('='.repeat(80));
+  console.log(`Total assets:                    ${stats.totalAssets}`);
+  console.log(`Skipped (already processed):     ${stats.skipped}`);
+  console.log(`Already in local storage:        ${stats.alreadyLocal}`);
+  console.log(`Downloaded from old URL:         ${stats.downloaded}`);
+  console.log(`Downloaded from fallback:        ${stats.downloadedFallback}`);
+  console.log(`Missing (needs manual work):     ${stats.missingCount}`);
+  console.log('='.repeat(80));
+
+  const localFiles = fs.readdirSync(LOCAL_COINS_DIR).filter(f => f.endsWith('.png')).length;
+  console.log(`\n📁 Local coins directory: ${localFiles} PNG files\n`);
+
+  if (stats.missing.length > 0) {
+    console.log(`⚠️  ${stats.missing.length} assets missing icons - see ${MISSING_ASSETS_FILE}\n`);
+  }
+
+  // Clean up progress file on completion
+  if (fs.existsSync(PROGRESS_FILE)) {
+    fs.unlinkSync(PROGRESS_FILE);
+  }
+
+  console.log('✅ Complete!\n');
+}
+
+main().catch(err => {
+  console.error('❌ Error:', err);
+  process.exit(1);
+});