@pioneer-platform/pioneer-discovery 0.8.4 → 4.21.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@pioneer-platform/pioneer-discovery",
3
- "version": "0.8.4",
3
+ "version": "4.21.16",
4
4
  "main": "./lib/index.js",
5
5
  "types": "./lib/main.d.ts",
6
6
  "_moduleAliases": {
@@ -0,0 +1,46 @@
1
+ {
2
+ "assets": {
3
+ "eip155:1/erc20:0x0173661769325565d4f011b2e5cda688689cc87c": {
4
+ "symbol": "QLT",
5
+ "name": "Quantland",
6
+ "chainId": "eip155:1",
7
+ "error": "Not found",
8
+ "attempts": 1
9
+ },
10
+ "eip155:1/erc20:0x0198f46f520f33cd4329bd4be380a25a90536cd5": {
11
+ "symbol": "PLA",
12
+ "name": "PlayChip",
13
+ "chainId": "eip155:1",
14
+ "error": "Not found",
15
+ "attempts": 1
16
+ },
17
+ "eip155:1/erc20:0x01792e1548dc317bde6123fe92da1fe6d7311c3c": {
18
+ "symbol": "SPIRAL",
19
+ "name": "Spiral",
20
+ "chainId": "eip155:1",
21
+ "error": "Not found",
22
+ "attempts": 1
23
+ },
24
+ "eip155:1/erc20:0x017e9db34fc69af0dc7c7b4b33511226971cddc7": {
25
+ "symbol": "OCD",
26
+ "name": "On Chain Dynamics",
27
+ "chainId": "eip155:1",
28
+ "error": "Not found",
29
+ "attempts": 1
30
+ },
31
+ "eip155:1/erc20:0x018fb5af9d015af25592a014c4266a84143de7a0": {
32
+ "symbol": "MP3",
33
+ "name": "MP3",
34
+ "chainId": "eip155:1",
35
+ "error": "Not found",
36
+ "attempts": 1
37
+ },
38
+ "eip155:1/erc20:0x01b23286ff60a543ec29366ae8d6b6274ca20541": {
39
+ "symbol": "BMP",
40
+ "name": "Brother Music Platform",
41
+ "chainId": "eip155:1",
42
+ "error": "Not found",
43
+ "attempts": 1
44
+ }
45
+ }
46
+ }
@@ -0,0 +1,9 @@
1
+ {
2
+ "processed": {
3
+ "eip155:1/erc20:0x01ba67aac7f75f647d94220cc98fb30fcc5105bf": "local",
4
+ "eip155:1/erc20:0x0176b898e92e814c06cc379e508ceb571f70bd40": "local",
5
+ "eip155:1/erc20:0x01824357d7d7eaf4677bc17786abd26cbdec9ad7": "coingecko",
6
+ "eip155:1/erc20:0x018008bfb33d285247a21d44e50697654f754e63": "coingecko"
7
+ },
8
+ "timestamp": 1761709065992
9
+ }
@@ -0,0 +1,220 @@
1
+ # Icon Download Scripts
2
+
3
+ This directory contains scripts to download missing asset icons using CoinGecko API and TrustWallet fallbacks.
4
+
5
+ ## Overview
6
+
7
+ We have **14,157 missing assets** that need icons downloaded. The enhanced download script uses:
8
+ 1. **CoinGecko API** - Query tokens by contract address to get official icon URLs
9
+ 2. **TrustWallet Assets** - Fallback to community-maintained token icons
10
+ 3. **Rate limiting** - Respects API limits (1.5s between calls = ~40 calls/minute)
11
+
12
+ ## Scripts
13
+
14
+ ### `test-coingecko-download.js` (Test Mode)
15
+ Tests the download process on 5 sample assets from `test-sample.json`.
16
+
17
+ **Usage:**
18
+ ```bash
19
+ cd scripts
20
+ node test-coingecko-download.js
21
+ ```
22
+
23
+ **Expected output:**
24
+ - Downloads 1-2 icons successfully (20-40% success rate typical for obscure tokens)
25
+ - Shows detailed logging for each asset
26
+ - Saves to: `../../../../../../services/pioneer-server/public/coins/`
27
+
28
+ ### `download-with-coingecko-api.js` (Production Mode)
29
+ Full production script that processes all 14,157 missing assets from `missing-assets.json`.
30
+
31
+ **Features:**
32
+ - Progress saving (resume if interrupted)
33
+ - Failed asset tracking
34
+ - Periodic stats reporting
35
+ - Respects API rate limits
36
+
37
+ **Usage:**
38
+ ```bash
39
+ cd scripts
40
+ node download-with-coingecko-api.js
41
+ ```
42
+
43
+ **Estimated time:** 6-7 hours for 14,157 assets (due to API rate limits)
44
+
45
+ **Output files:**
46
+ - `.coingecko-progress.json` - Progress tracking (auto-deleted on completion)
47
+ - `.coingecko-failed.json` - Assets that couldn't be downloaded
48
+ - `generatedAssetData.json` - Updated with new icon URLs
49
+
50
+ ## CoinGecko API Details
51
+
52
+ ### Chain Mappings
53
+ The script supports multiple chains via CoinGecko platform IDs:
54
+
55
+ | CAIP Chain ID | CoinGecko Platform |
56
+ |---------------|-------------------|
57
+ | eip155:1 | ethereum |
58
+ | eip155:10 | optimism |
59
+ | eip155:56 | binance-smart-chain |
60
+ | eip155:100 | xdai |
61
+ | eip155:137 | polygon-pos |
62
+ | eip155:250 | fantom |
63
+ | eip155:8453 | base |
64
+ | eip155:42161 | arbitrum-one |
65
+ | eip155:42220 | celo |
66
+ | eip155:43114 | avalanche |
67
+ | And more... |
68
+
69
+ ### API Endpoint Format
70
+ ```
71
+ GET https://api.coingecko.com/api/v3/coins/{platform}/contract/{contract_address}
72
+ ```
73
+
74
+ **Example:**
75
+ ```bash
76
+ curl "https://api.coingecko.com/api/v3/coins/ethereum/contract/0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48"
77
+ ```
78
+
79
+ **Response includes:**
80
+ ```json
81
+ {
82
+ "image": {
83
+ "thumb": "https://coin-images.coingecko.com/coins/images/6319/thumb/usdc.png",
84
+ "small": "https://coin-images.coingecko.com/coins/images/6319/small/usdc.png",
85
+ "large": "https://coin-images.coingecko.com/coins/images/6319/large/usdc.png"
86
+ }
87
+ }
88
+ ```
89
+
90
+ ### Rate Limits
91
+ - **Free tier**: 10-50 calls/minute
92
+ - **Script delay**: 1.5 seconds between calls (~40 calls/minute)
93
+ - **Retry logic**: Exponential backoff on rate limit (429) responses
94
+
95
+ ## Expected Results
96
+
97
+ Based on the nature of the missing assets:
98
+
99
+ - **Well-known tokens** (USDC, UNI, LINK, AAVE): ~90% success rate
100
+ - **Popular DeFi tokens**: ~60-70% success rate
101
+ - **Obscure/defunct tokens**: ~20-40% success rate
102
+ - **Overall estimated success**: ~50-60% (7,000-8,500 icons)
103
+
104
+ Many of these are:
105
+ - Deprecated tokens
106
+ - Low-liquidity tokens not listed on CoinGecko
107
+ - Tokens on newly supported chains
108
+ - Tokens with contract address changes
109
+
110
+ ## Fallback Strategy
111
+
112
+ For each asset, the script tries in order:
113
+
114
+ 1. ✅ Check if file already exists locally (skip if found)
115
+ 2. 🔍 Query CoinGecko API by contract address
116
+ 3. 📥 Download from CoinGecko if found
117
+ 4. 🔄 Try TrustWallet GitHub repository as fallback
118
+ 5. ❌ Mark as failed if all attempts fail
119
+
120
+ ## Running the Full Download
121
+
122
+ ### Prerequisites
123
+ ```bash
124
+ # Ensure directory exists
125
+ mkdir -p ../../../../../../services/pioneer-server/public/coins
126
+
127
+ # Verify missing-assets.json exists
128
+ ls -lh missing-assets.json
129
+ ```
130
+
131
+ ### Step 1: Test on Sample
132
+ ```bash
133
+ # Create test sample if needed
134
+ jq '.[0:10]' missing-assets.json > test-sample.json
135
+
136
+ # Run test
137
+ node test-coingecko-download.js
138
+ ```
139
+
140
+ ### Step 2: Run Full Download
141
+ ```bash
142
+ # Start the full download (6-7 hours)
143
+ node download-with-coingecko-api.js
144
+
145
+ # The script saves progress every 10 assets, so you can safely Ctrl+C and resume
146
+ ```
147
+
148
+ ### Step 3: Monitor Progress
149
+ The script outputs:
150
+ - Real-time status for each asset
151
+ - Periodic stats every 10 assets
152
+ - Final summary with success/fail breakdown
153
+
154
+ ### Step 4: Handle Failures
155
+ After completion, check `.coingecko-failed.json` for assets that couldn't be downloaded:
156
+ ```bash
157
+ jq '. | length' .coingecko-failed.json
158
+ jq '.assets | to_entries | .[0:10]' .coingecko-failed.json
159
+ ```
160
+
161
+ ## Resuming After Interruption
162
+
163
+ If the download is interrupted (Ctrl+C, network issue, etc.):
164
+
165
+ ```bash
166
+ # Simply run again - it will resume from progress file
167
+ node download-with-coingecko-api.js
168
+ ```
169
+
170
+ The script automatically:
171
+ - Loads `.coingecko-progress.json`
172
+ - Skips already processed assets
173
+ - Continues from where it left off
174
+
175
+ ## Troubleshooting
176
+
177
+ ### Rate Limiting
178
+ If you see frequent "Rate limited" messages:
179
+ - Increase `API_DELAY_MS` in the script (try 2000ms or 2500ms)
180
+ - Wait 1 hour and resume (API limits reset)
181
+
182
+ ### Timeouts
183
+ If you see many timeout errors:
184
+ - Check your internet connection
185
+ - Increase `timeout` parameter in `makeRequest()` and `downloadFile()`
186
+
187
+ ### 404 Not Found
188
+ This is normal for:
189
+ - Obscure tokens not in CoinGecko database
190
+ - Newly launched tokens
191
+ - Tokens on less popular chains
192
+
193
+ ### Invalid Chain
194
+ Some assets may use chain IDs not yet mapped. Add them to `CHAIN_ID_TO_COINGECKO_PLATFORM` if needed.
195
+
196
+ ## Next Steps After Download
197
+
198
+ 1. **Verify downloads:**
199
+ ```bash
200
+ ls -lh ../../../../../../services/pioneer-server/public/coins/*.png | wc -l
201
+ ```
202
+
203
+ 2. **Update generatedAssetData.json:**
204
+ The script automatically updates icon URLs to point to the new local files
205
+
206
+ 3. **Deploy to production:**
207
+ Upload the coins directory to your CDN/server
208
+
209
+ 4. **Manual fixes:**
210
+ Review `.coingecko-failed.json` and manually source icons for high-priority tokens
211
+
212
+ ## Stats from Test Run
213
+
214
+ Sample of 5 assets:
215
+ - Already local: 0 → 1 (after first run)
216
+ - Downloaded from CoinGecko: 2 (40%)
217
+ - Downloaded from TrustWallet: 0
218
+ - Failed: 3 (60%)
219
+
220
+ Note: Success rate will vary significantly based on token popularity.
@@ -0,0 +1,414 @@
1
+ #!/usr/bin/env node
2
+ /*
3
+ * Enhanced Icon Download with CoinGecko API
4
+ *
5
+ * Features:
6
+ * - Uses CoinGecko API to lookup tokens by contract address
7
+ * - Supports multiple chains (Ethereum, Optimism, Arbitrum, Base, BSC, Polygon, etc.)
8
+ * - Respects API rate limits (10-50 calls/minute on free tier)
9
+ * - Downloads high-quality images
10
+ * - Progress saving and resumption
11
+ */
12
+
13
+ const fs = require('fs');
14
+ const path = require('path');
15
+ const https = require('https');
16
+ const http = require('http');
17
+
18
+ const DATA_FILE = path.join(__dirname, '..', 'src', 'generatedAssetData.json');
19
+ const LOCAL_COINS_DIR = path.join(__dirname, '..', '..', '..', '..', '..', 'services', 'pioneer-server', 'public', 'coins');
20
+ const MISSING_ASSETS_FILE = path.join(__dirname, 'missing-assets.json');
21
+ const PROGRESS_FILE = path.join(__dirname, '.coingecko-progress.json');
22
+ const FAILED_FILE = path.join(__dirname, '.coingecko-failed.json');
23
+
24
+ // API rate limiting
25
+ const API_DELAY_MS = 1500; // 1.5 seconds between API calls (40 calls/minute)
26
+ const MAX_RETRIES = 3;
27
+
28
+ // Ensure local coins directory exists
29
+ if (!fs.existsSync(LOCAL_COINS_DIR)) {
30
+ fs.mkdirSync(LOCAL_COINS_DIR, { recursive: true });
31
+ }
32
+
33
+ /**
34
+ * Map CAIP chainId to CoinGecko platform identifier
35
+ */
36
+ const CHAIN_ID_TO_COINGECKO_PLATFORM = {
37
+ 'eip155:1': 'ethereum',
38
+ 'eip155:10': 'optimism',
39
+ 'eip155:56': 'binance-smart-chain',
40
+ 'eip155:100': 'xdai',
41
+ 'eip155:137': 'polygon-pos',
42
+ 'eip155:250': 'fantom',
43
+ 'eip155:8453': 'base',
44
+ 'eip155:42161': 'arbitrum-one',
45
+ 'eip155:42220': 'celo',
46
+ 'eip155:43114': 'avalanche',
47
+ 'eip155:534352': 'scroll',
48
+ 'eip155:59144': 'linea',
49
+ 'eip155:324': 'zksync',
50
+ 'eip155:1101': 'polygon-zkevm',
51
+ 'eip155:5000': 'mantle',
52
+ 'eip155:81457': 'blast',
53
+ };
54
+
55
+ /**
56
+ * Convert assetId to base64-encoded format
57
+ */
58
+ function encodeAssetId(assetId) {
59
+ return Buffer.from(assetId).toString('base64');
60
+ }
61
+
62
+ /**
63
+ * Extract contract address from assetId
64
+ */
65
+ function extractContractAddress(assetId) {
66
+ const match = assetId.match(/erc20:(0x[a-fA-F0-9]{40})/);
67
+ return match ? match[1].toLowerCase() : null;
68
+ }
69
+
70
+ /**
71
+ * Sleep for specified milliseconds
72
+ */
73
+ function sleep(ms) {
74
+ return new Promise(resolve => setTimeout(resolve, ms));
75
+ }
76
+
77
+ /**
78
+ * Make HTTP/HTTPS request with timeout
79
+ */
80
+ async function makeRequest(url, timeout = 10000) {
81
+ return new Promise((resolve, reject) => {
82
+ try {
83
+ const urlObj = new URL(url);
84
+ const client = urlObj.protocol === 'https:' ? https : http;
85
+
86
+ const req = client.get(url, { timeout }, (res) => {
87
+ let data = '';
88
+
89
+ res.on('data', chunk => {
90
+ data += chunk;
91
+ });
92
+
93
+ res.on('end', () => {
94
+ if (res.statusCode >= 200 && res.statusCode < 300) {
95
+ try {
96
+ const parsed = JSON.parse(data);
97
+ resolve({ success: true, data: parsed, statusCode: res.statusCode });
98
+ } catch (e) {
99
+ resolve({ success: false, error: 'Invalid JSON', statusCode: res.statusCode });
100
+ }
101
+ } else if (res.statusCode === 429) {
102
+ resolve({ success: false, error: 'Rate limited', statusCode: 429 });
103
+ } else if (res.statusCode === 404) {
104
+ resolve({ success: false, error: 'Not found', statusCode: 404 });
105
+ } else {
106
+ resolve({ success: false, error: `HTTP ${res.statusCode}`, statusCode: res.statusCode });
107
+ }
108
+ });
109
+ });
110
+
111
+ req.on('error', (err) => {
112
+ resolve({ success: false, error: err.message });
113
+ });
114
+
115
+ req.on('timeout', () => {
116
+ req.destroy();
117
+ resolve({ success: false, error: 'Timeout' });
118
+ });
119
+ } catch (e) {
120
+ resolve({ success: false, error: e.message });
121
+ }
122
+ });
123
+ }
124
+
125
+ /**
126
+ * Download file from URL
127
+ */
128
+ async function downloadFile(url, outputPath, timeout = 10000) {
129
+ return new Promise((resolve) => {
130
+ try {
131
+ const urlObj = new URL(url);
132
+ const client = urlObj.protocol === 'https:' ? https : http;
133
+
134
+ const req = client.get(url, { timeout }, (res) => {
135
+ if (res.statusCode >= 200 && res.statusCode < 300) {
136
+ const fileStream = fs.createWriteStream(outputPath);
137
+ res.pipe(fileStream);
138
+ fileStream.on('finish', () => {
139
+ fileStream.close();
140
+ resolve(true);
141
+ });
142
+ fileStream.on('error', () => resolve(false));
143
+ } else if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
144
+ // Follow redirect
145
+ downloadFile(res.headers.location, outputPath, timeout).then(resolve);
146
+ } else {
147
+ resolve(false);
148
+ }
149
+ });
150
+
151
+ req.on('error', () => resolve(false));
152
+ req.on('timeout', () => {
153
+ req.destroy();
154
+ resolve(false);
155
+ });
156
+ } catch (e) {
157
+ resolve(false);
158
+ }
159
+ });
160
+ }
161
+
162
+ /**
163
+ * Query CoinGecko API for token info by contract address
164
+ */
165
+ async function getCoinGeckoTokenInfo(chainId, contractAddress, retries = 0) {
166
+ const platform = CHAIN_ID_TO_COINGECKO_PLATFORM[chainId];
167
+ if (!platform) {
168
+ return { success: false, error: 'Unsupported chain' };
169
+ }
170
+
171
+ const url = `https://api.coingecko.com/api/v3/coins/${platform}/contract/${contractAddress}`;
172
+
173
+ const result = await makeRequest(url);
174
+
175
+ // Handle rate limiting with exponential backoff
176
+ if (result.statusCode === 429 && retries < MAX_RETRIES) {
177
+ const backoffMs = API_DELAY_MS * Math.pow(2, retries);
178
+ console.log(` ⏳ Rate limited, waiting ${backoffMs}ms...`);
179
+ await sleep(backoffMs);
180
+ return getCoinGeckoTokenInfo(chainId, contractAddress, retries + 1);
181
+ }
182
+
183
+ return result;
184
+ }
185
+
186
+ /**
187
+ * Get TrustWallet fallback URLs
188
+ */
189
+ function getTrustWalletFallbacks(assetData) {
190
+ const fallbacks = [];
191
+
192
+ if (assetData.assetId?.includes('erc20:')) {
193
+ const match = assetData.assetId.match(/erc20:(0x[a-fA-F0-9]+)/);
194
+ if (match) {
195
+ const contract = match[1];
196
+ // Simple checksum approach
197
+ const checksummed = contract.substring(0, 2) + contract.substring(2).split('')
198
+ .map((char, i) => i % 2 === 0 && parseInt(char, 16) >= 8 ? char.toUpperCase() : char)
199
+ .join('');
200
+
201
+ fallbacks.push(`https://raw.githubusercontent.com/trustwallet/assets/master/blockchains/ethereum/assets/${checksummed}/logo.png`);
202
+ }
203
+ }
204
+
205
+ return fallbacks;
206
+ }
207
+
208
+ /**
209
+ * Process a single asset with CoinGecko API
210
+ */
211
+ async function processAsset(assetId, assetData, stats, progress, failed) {
212
+ const encoded = encodeAssetId(assetId);
213
+ const localPath = path.join(LOCAL_COINS_DIR, `${encoded}.png`);
214
+
215
+ stats.total++;
216
+
217
+ // Skip if already processed successfully
218
+ if (progress.processed[assetId]) {
219
+ stats.skipped++;
220
+ return true;
221
+ }
222
+
223
+ // Check local file first
224
+ if (fs.existsSync(localPath)) {
225
+ const fileSize = fs.statSync(localPath).size;
226
+ if (fileSize > 0) {
227
+ stats.alreadyLocal++;
228
+ progress.processed[assetId] = 'local';
229
+ assetData.icon = `https://api.keepkey.com/coins/${encoded}.png`;
230
+ return true;
231
+ }
232
+ }
233
+
234
+ console.log(`\n[${stats.total}/${stats.totalAssets}] ${assetData.symbol} - ${assetData.name}`);
235
+ console.log(` Asset: ${assetId.substring(0, 70)}...`);
236
+
237
+ // Extract contract address
238
+ const contractAddress = extractContractAddress(assetId);
239
+ if (!contractAddress) {
240
+ console.log(` ⚠️ Not an ERC20 token - skipping CoinGecko lookup`);
241
+ stats.notErc20++;
242
+ progress.processed[assetId] = 'not_erc20';
243
+ return false;
244
+ }
245
+
246
+ // Query CoinGecko API
247
+ console.log(` 🔍 Querying CoinGecko API...`);
248
+ const result = await getCoinGeckoTokenInfo(assetData.chainId, contractAddress);
249
+
250
+ // Respect rate limits
251
+ await sleep(API_DELAY_MS);
252
+
253
+ if (result.success && result.data?.image) {
254
+ const imageUrl = result.data.image.large || result.data.image.small || result.data.image.thumb;
255
+
256
+ if (imageUrl) {
257
+ console.log(` 📥 Downloading from CoinGecko: ${imageUrl.substring(0, 70)}...`);
258
+ const downloaded = await downloadFile(imageUrl, localPath);
259
+
260
+ if (downloaded) {
261
+ console.log(` ✅ Successfully downloaded from CoinGecko`);
262
+ stats.downloadedFromCoinGecko++;
263
+ progress.processed[assetId] = 'coingecko';
264
+ assetData.icon = `https://api.keepkey.com/coins/${encoded}.png`;
265
+ return true;
266
+ } else {
267
+ console.log(` ❌ Download failed`);
268
+ }
269
+ }
270
+ } else {
271
+ console.log(` ℹ️ CoinGecko: ${result.error || 'Not found'}`);
272
+ }
273
+
274
+ // Try TrustWallet fallback
275
+ const trustWalletUrls = getTrustWalletFallbacks(assetData);
276
+ for (const url of trustWalletUrls) {
277
+ console.log(` 🔄 Trying TrustWallet: ${url.substring(0, 70)}...`);
278
+ const downloaded = await downloadFile(url, localPath);
279
+ if (downloaded) {
280
+ console.log(` ✅ Downloaded from TrustWallet`);
281
+ stats.downloadedFromTrustWallet++;
282
+ progress.processed[assetId] = 'trustwallet';
283
+ assetData.icon = `https://api.keepkey.com/coins/${encoded}.png`;
284
+ return true;
285
+ }
286
+ }
287
+
288
+ // Mark as failed
289
+ console.log(` ❌ No icon found`);
290
+ stats.failed++;
291
+ failed.assets[assetId] = {
292
+ symbol: assetData.symbol,
293
+ name: assetData.name,
294
+ chainId: assetData.chainId,
295
+ error: result.error || 'No icon available',
296
+ attempts: (failed.assets[assetId]?.attempts || 0) + 1,
297
+ };
298
+
299
+ // Still set the target URL
300
+ assetData.icon = `https://api.keepkey.com/coins/${encoded}.png`;
301
+ return false;
302
+ }
303
+
304
+ /**
305
+ * Save progress
306
+ */
307
+ function saveProgress(progress, data, stats, failed) {
308
+ fs.writeFileSync(PROGRESS_FILE, JSON.stringify(progress, null, 2));
309
+ fs.writeFileSync(DATA_FILE, JSON.stringify(data, null, 2));
310
+ fs.writeFileSync(FAILED_FILE, JSON.stringify(failed, null, 2));
311
+ }
312
+
313
+ /**
314
+ * Main execution
315
+ */
316
+ async function main() {
317
+ console.log('🚀 Enhanced icon download with CoinGecko API\n');
318
+ console.log('⚠️ Rate limit: ~40 API calls per minute');
319
+ console.log('⏱️ This will take approximately 6-7 hours for 14,000 assets\n');
320
+
321
+ // Load missing assets
322
+ let assetsToProcess = {};
323
+ if (fs.existsSync(MISSING_ASSETS_FILE)) {
324
+ const missingAssets = JSON.parse(fs.readFileSync(MISSING_ASSETS_FILE, 'utf8'));
325
+ console.log(`📂 Loading ${missingAssets.length} assets from missing-assets.json\n`);
326
+
327
+ // Convert array to object keyed by assetId
328
+ for (const asset of missingAssets) {
329
+ assetsToProcess[asset.assetId] = {
330
+ symbol: asset.symbol,
331
+ name: asset.name,
332
+ chainId: asset.chainId,
333
+ icon: asset.oldIcon,
334
+ assetId: asset.assetId,
335
+ };
336
+ }
337
+ } else {
338
+ console.log('❌ missing-assets.json not found');
339
+ process.exit(1);
340
+ }
341
+
342
+ const entries = Object.entries(assetsToProcess);
343
+
344
+ // Load progress
345
+ let progress = { processed: {}, timestamp: Date.now() };
346
+ if (fs.existsSync(PROGRESS_FILE)) {
347
+ progress = JSON.parse(fs.readFileSync(PROGRESS_FILE, 'utf8'));
348
+ console.log(`📂 Resuming from previous run (${Object.keys(progress.processed).length} already processed)\n`);
349
+ }
350
+
351
+ // Load failed attempts
352
+ let failed = { assets: {} };
353
+ if (fs.existsSync(FAILED_FILE)) {
354
+ failed = JSON.parse(fs.readFileSync(FAILED_FILE, 'utf8'));
355
+ }
356
+
357
+ const stats = {
358
+ total: 0,
359
+ totalAssets: entries.length,
360
+ skipped: 0,
361
+ alreadyLocal: 0,
362
+ downloadedFromCoinGecko: 0,
363
+ downloadedFromTrustWallet: 0,
364
+ notErc20: 0,
365
+ failed: 0,
366
+ };
367
+
368
+ const saveInterval = 10; // Save every 10 assets
369
+
370
+ // Process sequentially to respect API rate limits
371
+ for (let i = 0; i < entries.length; i++) {
372
+ const [assetId, assetData] = entries[i];
373
+ await processAsset(assetId, assetData, stats, progress, failed);
374
+
375
+ // Save progress periodically
376
+ if ((i + 1) % saveInterval === 0) {
377
+ saveProgress(progress, assetsToProcess, stats, failed);
378
+ console.log(`\n💾 Progress saved (${i + 1}/${entries.length})`);
379
+
380
+ // Print interim stats
381
+ const percentComplete = ((i + 1) / entries.length * 100).toFixed(1);
382
+ console.log(`📊 ${percentComplete}% complete | CoinGecko: ${stats.downloadedFromCoinGecko} | TrustWallet: ${stats.downloadedFromTrustWallet} | Failed: ${stats.failed}\n`);
383
+ }
384
+ }
385
+
386
+ // Final save
387
+ saveProgress(progress, assetsToProcess, stats, failed);
388
+
389
+ // Print summary
390
+ console.log('\n' + '='.repeat(80));
391
+ console.log('📊 FINAL SUMMARY');
392
+ console.log('='.repeat(80));
393
+ console.log(`Total assets processed: ${stats.totalAssets}`);
394
+ console.log(`Skipped (already done): ${stats.skipped}`);
395
+ console.log(`Already in local storage: ${stats.alreadyLocal}`);
396
+ console.log(`Downloaded from CoinGecko: ${stats.downloadedFromCoinGecko}`);
397
+ console.log(`Downloaded from TrustWallet: ${stats.downloadedFromTrustWallet}`);
398
+ console.log(`Not ERC20 (skipped): ${stats.notErc20}`);
399
+ console.log(`Failed (no icon found): ${stats.failed}`);
400
+ console.log('='.repeat(80));
401
+
402
+ const successRate = ((stats.downloadedFromCoinGecko + stats.downloadedFromTrustWallet) / stats.totalAssets * 100).toFixed(1);
403
+ console.log(`\n✅ Success rate: ${successRate}%\n`);
404
+
405
+ const localFiles = fs.readdirSync(LOCAL_COINS_DIR).filter(f => f.endsWith('.png')).length;
406
+ console.log(`📁 Local coins directory: ${localFiles} PNG files\n`);
407
+
408
+ console.log(`📝 Failed assets saved to: ${FAILED_FILE}\n`);
409
+ }
410
+
411
+ main().catch(err => {
412
+ console.error('❌ Error:', err);
413
+ process.exit(1);
414
+ });