@pioneer-platform/pioneer-discovery 4.21.15 → 4.21.16

package/scripts/test-coingecko-download.js

@@ -0,0 +1,317 @@
+#!/usr/bin/env node
+/*
+ * TEST VERSION - Enhanced Icon Download with CoinGecko API
+ * Uses test-sample.json (5 assets) for quick testing
+ */
+
+const fs = require('fs');
+const path = require('path');
+const https = require('https');
+const http = require('http');
+
+const LOCAL_COINS_DIR = path.join(__dirname, '..', '..', '..', '..', '..', 'services', 'pioneer-server', 'public', 'coins');
+const MISSING_ASSETS_FILE = path.join(__dirname, 'test-sample.json');
+
+// API rate limiting
+const API_DELAY_MS = 1500; // 1.5 seconds between API calls
+const MAX_RETRIES = 3;
+
+// Ensure local coins directory exists
+if (!fs.existsSync(LOCAL_COINS_DIR)) {
+  fs.mkdirSync(LOCAL_COINS_DIR, { recursive: true });
+}
+
+/**
+ * Map CAIP chainId to CoinGecko platform identifier
+ */
+const CHAIN_ID_TO_COINGECKO_PLATFORM = {
+  'eip155:1': 'ethereum',
+  'eip155:10': 'optimism',
+  'eip155:56': 'binance-smart-chain',
+  'eip155:100': 'xdai',
+  'eip155:137': 'polygon-pos',
+  'eip155:250': 'fantom',
+  'eip155:8453': 'base',
+  'eip155:42161': 'arbitrum-one',
+  'eip155:42220': 'celo',
+  'eip155:43114': 'avalanche',
+  'eip155:534352': 'scroll',
+  'eip155:59144': 'linea',
+  'eip155:324': 'zksync',
+  'eip155:1101': 'polygon-zkevm',
+  'eip155:5000': 'mantle',
+  'eip155:81457': 'blast',
+};
+
+/**
+ * Convert assetId to base64-encoded format
+ */
+function encodeAssetId(assetId) {
+  return Buffer.from(assetId).toString('base64');
+}
+
+/**
+ * Extract contract address from assetId
+ */
+function extractContractAddress(assetId) {
+  const match = assetId.match(/erc20:(0x[a-fA-F0-9]{40})/);
+  return match ? match[1].toLowerCase() : null;
+}
+
+/**
+ * Sleep for specified milliseconds
+ */
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * Make HTTP/HTTPS request with timeout
+ */
+async function makeRequest(url, timeout = 10000) {
+  return new Promise((resolve, reject) => {
+    try {
+      const urlObj = new URL(url);
+      const client = urlObj.protocol === 'https:' ? https : http;
+
+      const req = client.get(url, { timeout }, (res) => {
+        let data = '';
+
+        res.on('data', chunk => {
+          data += chunk;
+        });
+
+        res.on('end', () => {
+          if (res.statusCode >= 200 && res.statusCode < 300) {
+            try {
+              const parsed = JSON.parse(data);
+              resolve({ success: true, data: parsed, statusCode: res.statusCode });
+            } catch (e) {
+              resolve({ success: false, error: 'Invalid JSON', statusCode: res.statusCode });
+            }
+          } else if (res.statusCode === 429) {
+            resolve({ success: false, error: 'Rate limited', statusCode: 429 });
+          } else if (res.statusCode === 404) {
+            resolve({ success: false, error: 'Not found', statusCode: 404 });
+          } else {
+            resolve({ success: false, error: `HTTP ${res.statusCode}`, statusCode: res.statusCode });
+          }
+        });
+      });
+
+      req.on('error', (err) => {
+        resolve({ success: false, error: err.message });
+      });
+
+      req.on('timeout', () => {
+        req.destroy();
+        resolve({ success: false, error: 'Timeout' });
+      });
+    } catch (e) {
+      resolve({ success: false, error: e.message });
+    }
+  });
+}
+
+/**
+ * Download file from URL
+ */
+async function downloadFile(url, outputPath, timeout = 10000) {
+  return new Promise((resolve) => {
+    try {
+      const urlObj = new URL(url);
+      const client = urlObj.protocol === 'https:' ? https : http;
+
+      const req = client.get(url, { timeout }, (res) => {
+        if (res.statusCode >= 200 && res.statusCode < 300) {
+          const fileStream = fs.createWriteStream(outputPath);
+          res.pipe(fileStream);
+          fileStream.on('finish', () => {
+            fileStream.close();
+            resolve(true);
+          });
+          fileStream.on('error', () => resolve(false));
+        } else if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
+          // Follow redirect
+          downloadFile(res.headers.location, outputPath, timeout).then(resolve);
+        } else {
+          resolve(false);
+        }
+      });
+
+      req.on('error', () => resolve(false));
+      req.on('timeout', () => {
+        req.destroy();
+        resolve(false);
+      });
+    } catch (e) {
+      resolve(false);
+    }
+  });
+}
+
+/**
+ * Query CoinGecko API for token info by contract address
+ */
+async function getCoinGeckoTokenInfo(chainId, contractAddress, retries = 0) {
+  const platform = CHAIN_ID_TO_COINGECKO_PLATFORM[chainId];
+  if (!platform) {
+    return { success: false, error: 'Unsupported chain' };
+  }
+
+  const url = `https://api.coingecko.com/api/v3/coins/${platform}/contract/${contractAddress}`;
+  console.log(`  🔗 API URL: ${url}`);
+
+  const result = await makeRequest(url);
+
+  // Handle rate limiting with exponential backoff
+  if (result.statusCode === 429 && retries < MAX_RETRIES) {
+    const backoffMs = API_DELAY_MS * Math.pow(2, retries);
+    console.log(`  ⏳ Rate limited, waiting ${backoffMs}ms...`);
+    await sleep(backoffMs);
+    return getCoinGeckoTokenInfo(chainId, contractAddress, retries + 1);
+  }
+
+  return result;
+}
+
+/**
+ * Get TrustWallet fallback URLs
+ */
+function getTrustWalletFallbacks(contractAddress) {
+  // Simple checksum approach
+  const checksummed = contractAddress.substring(0, 2) + contractAddress.substring(2).split('')
+    .map((char, i) => i % 2 === 0 && parseInt(char, 16) >= 8 ? char.toUpperCase() : char)
+    .join('');
+
+  return [`https://raw.githubusercontent.com/trustwallet/assets/master/blockchains/ethereum/assets/${checksummed}/logo.png`];
+}
+
+/**
+ * Process a single asset with CoinGecko API
+ */
+async function processAsset(asset, index, total, stats) {
+  const { assetId, symbol, name, chainId } = asset;
+  const encoded = encodeAssetId(assetId);
+  const localPath = path.join(LOCAL_COINS_DIR, `${encoded}.png`);
+
+  console.log(`\n[${ index + 1}/${total}] ${symbol} - ${name}`);
+  console.log(`  Asset: ${assetId}`);
+
+  // Check local file first
+  if (fs.existsSync(localPath)) {
+    const fileSize = fs.statSync(localPath).size;
+    if (fileSize > 0) {
+      console.log(`  ✅ Already exists locally (${fileSize} bytes)`);
+      stats.alreadyLocal++;
+      return true;
+    }
+  }
+
+  // Extract contract address
+  const contractAddress = extractContractAddress(assetId);
+  if (!contractAddress) {
+    console.log(`  ⚠️  Not an ERC20 token`);
+    stats.notErc20++;
+    return false;
+  }
+
+  console.log(`  📍 Contract: ${contractAddress}`);
+
+  // Query CoinGecko API
+  console.log(`  🔍 Querying CoinGecko API...`);
+  const result = await getCoinGeckoTokenInfo(chainId, contractAddress);
+
+  // Respect rate limits
+  await sleep(API_DELAY_MS);
+
+  if (result.success && result.data?.image) {
+    const imageUrl = result.data.image.large || result.data.image.small || result.data.image.thumb;
+
+    if (imageUrl) {
+      console.log(`  📥 Downloading: ${imageUrl}`);
+      const downloaded = await downloadFile(imageUrl, localPath);
+
+      if (downloaded) {
+        const fileSize = fs.statSync(localPath).size;
+        console.log(`  ✅ Successfully downloaded (${fileSize} bytes)`);
+        stats.downloadedFromCoinGecko++;
+        return true;
+      } else {
+        console.log(`  ❌ Download failed`);
+      }
+    }
+  } else {
+    console.log(`  ℹ️  CoinGecko: ${result.error || 'Not found'} (HTTP ${result.statusCode || 'N/A'})`);
+  }
+
+  // Try TrustWallet fallback
+  const trustWalletUrls = getTrustWalletFallbacks(contractAddress);
+  for (const url of trustWalletUrls) {
+    console.log(`  🔄 Trying TrustWallet: ${url.substring(0, 80)}...`);
+    const downloaded = await downloadFile(url, localPath);
+    if (downloaded) {
+      const fileSize = fs.statSync(localPath).size;
+      console.log(`  ✅ Downloaded from TrustWallet (${fileSize} bytes)`);
+      stats.downloadedFromTrustWallet++;
+      return true;
+    }
+  }
+
+  console.log(`  ❌ No icon found`);
+  stats.failed++;
+  return false;
+}
+
+/**
+ * Main execution
+ */
+async function main() {
+  console.log('🧪 TEST MODE - Enhanced icon download with CoinGecko API\n');
+  console.log('Testing with 5 sample assets from test-sample.json\n');
+
+  // Load test sample
+  if (!fs.existsSync(MISSING_ASSETS_FILE)) {
+    console.log('❌ test-sample.json not found');
+    process.exit(1);
+  }
+
+  const assets = JSON.parse(fs.readFileSync(MISSING_ASSETS_FILE, 'utf8'));
+  console.log(`📂 Loaded ${assets.length} test assets\n`);
+
+  const stats = {
+    alreadyLocal: 0,
+    downloadedFromCoinGecko: 0,
+    downloadedFromTrustWallet: 0,
+    notErc20: 0,
+    failed: 0,
+  };
+
+  // Process each asset
+  for (let i = 0; i < assets.length; i++) {
+    await processAsset(assets[i], i, assets.length, stats);
+  }
+
+  // Print summary
+  console.log('\n' + '='.repeat(80));
+  console.log('📊 TEST SUMMARY');
+  console.log('='.repeat(80));
+  console.log(`Total assets:                    ${assets.length}`);
+  console.log(`Already local:                   ${stats.alreadyLocal}`);
+  console.log(`Downloaded from CoinGecko:       ${stats.downloadedFromCoinGecko}`);
+  console.log(`Downloaded from TrustWallet:     ${stats.downloadedFromTrustWallet}`);
+  console.log(`Not ERC20 (skipped):             ${stats.notErc20}`);
+  console.log(`Failed:                          ${stats.failed}`);
+  console.log('='.repeat(80));
+
+  const successRate = ((stats.downloadedFromCoinGecko + stats.downloadedFromTrustWallet) / assets.length * 100).toFixed(1);
+  console.log(`\n✅ Success rate: ${successRate}%\n`);
+
+  const localFiles = fs.readdirSync(LOCAL_COINS_DIR).filter(f => f.endsWith('.png')).length;
+  console.log(`📁 Local coins directory: ${localFiles} PNG files\n`);
+}
+
+main().catch(err => {
+  console.error('❌ Error:', err);
+  process.exit(1);
+});

package/scripts/test-sample.json

@@ -0,0 +1,37 @@
+[
+  {
+    "assetId": "eip155:1/erc20:0x0173661769325565d4f011b2e5cda688689cc87c",
+    "symbol": "QLT",
+    "name": "Quantland",
+    "chainId": "eip155:1",
+    "oldIcon": "https://api.keepkey.info/coins/ZWlwMTU1OjEvZXJjMjA6MHgwMTczNjYxNzY5MzI1NTY1ZDRmMDExYjJlNWNkYTY4ODY4OWNjODdj.png"
+  },
+  {
+    "assetId": "eip155:1/erc20:0x01ba67aac7f75f647d94220cc98fb30fcc5105bf",
+    "symbol": "LYRA",
+    "name": "Lyra Finance on Ethereum",
+    "chainId": "eip155:1",
+    "oldIcon": "https://api.keepkey.info/coins/ZWlwMTU1OjEvZXJjMjA6MHgwMWJhNjdhYWM3Zjc1ZjY0N2Q5NDIyMGNjOThmYjMwZmNjNTEwNWJm.png"
+  },
+  {
+    "assetId": "eip155:1/erc20:0x0198f46f520f33cd4329bd4be380a25a90536cd5",
+    "symbol": "PLA",
+    "name": "PlayChip",
+    "chainId": "eip155:1",
+    "oldIcon": "https://api.keepkey.info/coins/ZWlwMTU1OjEvZXJjMjA6MHgwMTk4ZjQ2ZjUyMGYzM2NkNDMyOWJkNGJlMzgwYTI1YTkwNTM2Y2Q1.png"
+  },
+  {
+    "assetId": "eip155:1/erc20:0x01792e1548dc317bde6123fe92da1fe6d7311c3c",
+    "symbol": "SPIRAL",
+    "name": "Spiral",
+    "chainId": "eip155:1",
+    "oldIcon": "https://api.keepkey.info/coins/ZWlwMTU1OjEvZXJjMjA6MHgwMTc5MmUxNTQ4ZGMzMTdiZGU2MTIzZmU5MmRhMWZlNmQ3MzExYzNj.png"
+  },
+  {
+    "assetId": "eip155:1/erc20:0x0176b898e92e814c06cc379e508ceb571f70bd40",
+    "symbol": "TIP",
+    "name": "Tipcoin",
+    "chainId": "eip155:1",
+    "oldIcon": "https://api.keepkey.info/coins/ZWlwMTU1OjEvZXJjMjA6MHgwMTc2Yjg5OGU5MmU4MTRjMDZjYzM3OWU1MDhjZWI1NzFmNzBiZDQw.png"
+  }
+]