@pioneer-platform/pioneer-discovery 0.8.4 → 4.21.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/settings.local.json +12 -0
- package/CHANGELOG.md +12 -0
- package/lib/generatedAssetData.json +55497 -141801
- package/package.json +1 -1
- package/scripts/.coingecko-failed.json +46 -0
- package/scripts/.coingecko-progress.json +9 -0
- package/scripts/README-ICON-DOWNLOAD.md +220 -0
- package/scripts/download-with-coingecko-api.js +414 -0
- package/scripts/missing-assets.json +99101 -0
- package/scripts/test-coingecko-download.js +317 -0
- package/scripts/test-sample.json +37 -0
- package/scripts/update-icon-urls.js +86 -0
- package/scripts/validate-and-download-icons-fast.js +304 -0
- package/scripts/validate-and-download-icons.js +326 -0
package/package.json
CHANGED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
{
|
|
2
|
+
"assets": {
|
|
3
|
+
"eip155:1/erc20:0x0173661769325565d4f011b2e5cda688689cc87c": {
|
|
4
|
+
"symbol": "QLT",
|
|
5
|
+
"name": "Quantland",
|
|
6
|
+
"chainId": "eip155:1",
|
|
7
|
+
"error": "Not found",
|
|
8
|
+
"attempts": 1
|
|
9
|
+
},
|
|
10
|
+
"eip155:1/erc20:0x0198f46f520f33cd4329bd4be380a25a90536cd5": {
|
|
11
|
+
"symbol": "PLA",
|
|
12
|
+
"name": "PlayChip",
|
|
13
|
+
"chainId": "eip155:1",
|
|
14
|
+
"error": "Not found",
|
|
15
|
+
"attempts": 1
|
|
16
|
+
},
|
|
17
|
+
"eip155:1/erc20:0x01792e1548dc317bde6123fe92da1fe6d7311c3c": {
|
|
18
|
+
"symbol": "SPIRAL",
|
|
19
|
+
"name": "Spiral",
|
|
20
|
+
"chainId": "eip155:1",
|
|
21
|
+
"error": "Not found",
|
|
22
|
+
"attempts": 1
|
|
23
|
+
},
|
|
24
|
+
"eip155:1/erc20:0x017e9db34fc69af0dc7c7b4b33511226971cddc7": {
|
|
25
|
+
"symbol": "OCD",
|
|
26
|
+
"name": "On Chain Dynamics",
|
|
27
|
+
"chainId": "eip155:1",
|
|
28
|
+
"error": "Not found",
|
|
29
|
+
"attempts": 1
|
|
30
|
+
},
|
|
31
|
+
"eip155:1/erc20:0x018fb5af9d015af25592a014c4266a84143de7a0": {
|
|
32
|
+
"symbol": "MP3",
|
|
33
|
+
"name": "MP3",
|
|
34
|
+
"chainId": "eip155:1",
|
|
35
|
+
"error": "Not found",
|
|
36
|
+
"attempts": 1
|
|
37
|
+
},
|
|
38
|
+
"eip155:1/erc20:0x01b23286ff60a543ec29366ae8d6b6274ca20541": {
|
|
39
|
+
"symbol": "BMP",
|
|
40
|
+
"name": "Brother Music Platform",
|
|
41
|
+
"chainId": "eip155:1",
|
|
42
|
+
"error": "Not found",
|
|
43
|
+
"attempts": 1
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
{
|
|
2
|
+
"processed": {
|
|
3
|
+
"eip155:1/erc20:0x01ba67aac7f75f647d94220cc98fb30fcc5105bf": "local",
|
|
4
|
+
"eip155:1/erc20:0x0176b898e92e814c06cc379e508ceb571f70bd40": "local",
|
|
5
|
+
"eip155:1/erc20:0x01824357d7d7eaf4677bc17786abd26cbdec9ad7": "coingecko",
|
|
6
|
+
"eip155:1/erc20:0x018008bfb33d285247a21d44e50697654f754e63": "coingecko"
|
|
7
|
+
},
|
|
8
|
+
"timestamp": 1761709065992
|
|
9
|
+
}
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
# Icon Download Scripts
|
|
2
|
+
|
|
3
|
+
This directory contains scripts to download missing asset icons using CoinGecko API and TrustWallet fallbacks.
|
|
4
|
+
|
|
5
|
+
## Overview
|
|
6
|
+
|
|
7
|
+
We have **14,157 missing assets** that need icons downloaded. The enhanced download script uses:
|
|
8
|
+
1. **CoinGecko API** - Query tokens by contract address to get official icon URLs
|
|
9
|
+
2. **TrustWallet Assets** - Fallback to community-maintained token icons
|
|
10
|
+
3. **Rate limiting** - Respects API limits (1.5s between calls = ~40 calls/minute)
|
|
11
|
+
|
|
12
|
+
## Scripts
|
|
13
|
+
|
|
14
|
+
### `test-coingecko-download.js` (Test Mode)
|
|
15
|
+
Tests the download process on 5 sample assets from `test-sample.json`.
|
|
16
|
+
|
|
17
|
+
**Usage:**
|
|
18
|
+
```bash
|
|
19
|
+
cd scripts
|
|
20
|
+
node test-coingecko-download.js
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
**Expected output:**
|
|
24
|
+
- Downloads 1-2 icons successfully (20-40% success rate typical for obscure tokens)
|
|
25
|
+
- Shows detailed logging for each asset
|
|
26
|
+
- Saves to: `../../../../../../services/pioneer-server/public/coins/`
|
|
27
|
+
|
|
28
|
+
### `download-with-coingecko-api.js` (Production Mode)
|
|
29
|
+
Full production script that processes all 14,157 missing assets from `missing-assets.json`.
|
|
30
|
+
|
|
31
|
+
**Features:**
|
|
32
|
+
- Progress saving (resume if interrupted)
|
|
33
|
+
- Failed asset tracking
|
|
34
|
+
- Periodic stats reporting
|
|
35
|
+
- Respects API rate limits
|
|
36
|
+
|
|
37
|
+
**Usage:**
|
|
38
|
+
```bash
|
|
39
|
+
cd scripts
|
|
40
|
+
node download-with-coingecko-api.js
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
**Estimated time:** 6-7 hours for 14,157 assets (due to API rate limits)
|
|
44
|
+
|
|
45
|
+
**Output files:**
|
|
46
|
+
- `.coingecko-progress.json` - Progress tracking (auto-deleted on completion)
|
|
47
|
+
- `.coingecko-failed.json` - Assets that couldn't be downloaded
|
|
48
|
+
- `generatedAssetData.json` - Updated with new icon URLs
|
|
49
|
+
|
|
50
|
+
## CoinGecko API Details
|
|
51
|
+
|
|
52
|
+
### Chain Mappings
|
|
53
|
+
The script supports multiple chains via CoinGecko platform IDs:
|
|
54
|
+
|
|
55
|
+
| CAIP Chain ID | CoinGecko Platform |
|
|
56
|
+
|---------------|-------------------|
|
|
57
|
+
| eip155:1 | ethereum |
|
|
58
|
+
| eip155:10 | optimism |
|
|
59
|
+
| eip155:56 | binance-smart-chain |
|
|
60
|
+
| eip155:100 | xdai |
|
|
61
|
+
| eip155:137 | polygon-pos |
|
|
62
|
+
| eip155:250 | fantom |
|
|
63
|
+
| eip155:8453 | base |
|
|
64
|
+
| eip155:42161 | arbitrum-one |
|
|
65
|
+
| eip155:42220 | celo |
|
|
66
|
+
| eip155:43114 | avalanche |
|
|
67
|
+
| And more... |
|
|
68
|
+
|
|
69
|
+
### API Endpoint Format
|
|
70
|
+
```
|
|
71
|
+
GET https://api.coingecko.com/api/v3/coins/{platform}/contract/{contract_address}
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
**Example:**
|
|
75
|
+
```bash
|
|
76
|
+
curl "https://api.coingecko.com/api/v3/coins/ethereum/contract/0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48"
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
**Response includes:**
|
|
80
|
+
```json
|
|
81
|
+
{
|
|
82
|
+
"image": {
|
|
83
|
+
"thumb": "https://coin-images.coingecko.com/coins/images/6319/thumb/usdc.png",
|
|
84
|
+
"small": "https://coin-images.coingecko.com/coins/images/6319/small/usdc.png",
|
|
85
|
+
"large": "https://coin-images.coingecko.com/coins/images/6319/large/usdc.png"
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
### Rate Limits
|
|
91
|
+
- **Free tier**: 10-50 calls/minute
|
|
92
|
+
- **Script delay**: 1.5 seconds between calls (~40 calls/minute)
|
|
93
|
+
- **Retry logic**: Exponential backoff on rate limit (429) responses
|
|
94
|
+
|
|
95
|
+
## Expected Results
|
|
96
|
+
|
|
97
|
+
Based on the nature of the missing assets:
|
|
98
|
+
|
|
99
|
+
- **Well-known tokens** (USDC, UNI, LINK, AAVE): ~90% success rate
|
|
100
|
+
- **Popular DeFi tokens**: ~60-70% success rate
|
|
101
|
+
- **Obscure/defunct tokens**: ~20-40% success rate
|
|
102
|
+
- **Overall estimated success**: ~50-60% (7,000-8,500 icons)
|
|
103
|
+
|
|
104
|
+
Many of these are:
|
|
105
|
+
- Deprecated tokens
|
|
106
|
+
- Low-liquidity tokens not listed on CoinGecko
|
|
107
|
+
- Tokens on newly supported chains
|
|
108
|
+
- Tokens with contract address changes
|
|
109
|
+
|
|
110
|
+
## Fallback Strategy
|
|
111
|
+
|
|
112
|
+
For each asset, the script tries in order:
|
|
113
|
+
|
|
114
|
+
1. ✅ Check if file already exists locally (skip if found)
|
|
115
|
+
2. 🔍 Query CoinGecko API by contract address
|
|
116
|
+
3. 📥 Download from CoinGecko if found
|
|
117
|
+
4. 🔄 Try TrustWallet GitHub repository as fallback
|
|
118
|
+
5. ❌ Mark as failed if all attempts fail
|
|
119
|
+
|
|
120
|
+
## Running the Full Download
|
|
121
|
+
|
|
122
|
+
### Prerequisites
|
|
123
|
+
```bash
|
|
124
|
+
# Ensure directory exists
|
|
125
|
+
mkdir -p ../../../../../../services/pioneer-server/public/coins
|
|
126
|
+
|
|
127
|
+
# Verify missing-assets.json exists
|
|
128
|
+
ls -lh missing-assets.json
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
### Step 1: Test on Sample
|
|
132
|
+
```bash
|
|
133
|
+
# Create test sample if needed
|
|
134
|
+
jq '.[0:10]' missing-assets.json > test-sample.json
|
|
135
|
+
|
|
136
|
+
# Run test
|
|
137
|
+
node test-coingecko-download.js
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
### Step 2: Run Full Download
|
|
141
|
+
```bash
|
|
142
|
+
# Start the full download (6-7 hours)
|
|
143
|
+
node download-with-coingecko-api.js
|
|
144
|
+
|
|
145
|
+
# The script saves progress every 10 assets, so you can safely Ctrl+C and resume
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
### Step 3: Monitor Progress
|
|
149
|
+
The script outputs:
|
|
150
|
+
- Real-time status for each asset
|
|
151
|
+
- Periodic stats every 10 assets
|
|
152
|
+
- Final summary with success/fail breakdown
|
|
153
|
+
|
|
154
|
+
### Step 4: Handle Failures
|
|
155
|
+
After completion, check `.coingecko-failed.json` for assets that couldn't be downloaded:
|
|
156
|
+
```bash
|
|
157
|
+
jq '. | length' .coingecko-failed.json
|
|
158
|
+
jq '.assets | to_entries | .[0:10]' .coingecko-failed.json
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
## Resuming After Interruption
|
|
162
|
+
|
|
163
|
+
If the download is interrupted (Ctrl+C, network issue, etc.):
|
|
164
|
+
|
|
165
|
+
```bash
|
|
166
|
+
# Simply run again - it will resume from progress file
|
|
167
|
+
node download-with-coingecko-api.js
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
The script automatically:
|
|
171
|
+
- Loads `.coingecko-progress.json`
|
|
172
|
+
- Skips already processed assets
|
|
173
|
+
- Continues from where it left off
|
|
174
|
+
|
|
175
|
+
## Troubleshooting
|
|
176
|
+
|
|
177
|
+
### Rate Limiting
|
|
178
|
+
If you see frequent "Rate limited" messages:
|
|
179
|
+
- Increase `API_DELAY_MS` in the script (try 2000ms or 2500ms)
|
|
180
|
+
- Wait 1 hour and resume (API limits reset)
|
|
181
|
+
|
|
182
|
+
### Timeouts
|
|
183
|
+
If you see many timeout errors:
|
|
184
|
+
- Check your internet connection
|
|
185
|
+
- Increase `timeout` parameter in `makeRequest()` and `downloadFile()`
|
|
186
|
+
|
|
187
|
+
### 404 Not Found
|
|
188
|
+
This is normal for:
|
|
189
|
+
- Obscure tokens not in CoinGecko database
|
|
190
|
+
- Newly launched tokens
|
|
191
|
+
- Tokens on less popular chains
|
|
192
|
+
|
|
193
|
+
### Invalid Chain
|
|
194
|
+
Some assets may use chain IDs not yet mapped. Add them to `CHAIN_ID_TO_COINGECKO_PLATFORM` if needed.
|
|
195
|
+
|
|
196
|
+
## Next Steps After Download
|
|
197
|
+
|
|
198
|
+
1. **Verify downloads:**
|
|
199
|
+
```bash
|
|
200
|
+
ls -lh ../../../../../../services/pioneer-server/public/coins/*.png | wc -l
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
2. **Update generatedAssetData.json:**
|
|
204
|
+
The script automatically updates icon URLs to point to the new local files
|
|
205
|
+
|
|
206
|
+
3. **Deploy to production:**
|
|
207
|
+
Upload the coins directory to your CDN/server
|
|
208
|
+
|
|
209
|
+
4. **Manual fixes:**
|
|
210
|
+
Review `.coingecko-failed.json` and manually source icons for high-priority tokens
|
|
211
|
+
|
|
212
|
+
## Stats from Test Run
|
|
213
|
+
|
|
214
|
+
Sample of 5 assets:
|
|
215
|
+
- Already local: 0 → 1 (after first run)
|
|
216
|
+
- Downloaded from CoinGecko: 2 (40%)
|
|
217
|
+
- Downloaded from TrustWallet: 0
|
|
218
|
+
- Failed: 3 (60%)
|
|
219
|
+
|
|
220
|
+
Note: Success rate will vary significantly based on token popularity.
|
|
@@ -0,0 +1,414 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/*
|
|
3
|
+
* Enhanced Icon Download with CoinGecko API
|
|
4
|
+
*
|
|
5
|
+
* Features:
|
|
6
|
+
* - Uses CoinGecko API to lookup tokens by contract address
|
|
7
|
+
* - Supports multiple chains (Ethereum, Optimism, Arbitrum, Base, BSC, Polygon, etc.)
|
|
8
|
+
* - Respects API rate limits (10-50 calls/minute on free tier)
|
|
9
|
+
* - Downloads high-quality images
|
|
10
|
+
* - Progress saving and resumption
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
const fs = require('fs');
|
|
14
|
+
const path = require('path');
|
|
15
|
+
const https = require('https');
|
|
16
|
+
const http = require('http');
|
|
17
|
+
|
|
18
|
+
const DATA_FILE = path.join(__dirname, '..', 'src', 'generatedAssetData.json');
|
|
19
|
+
const LOCAL_COINS_DIR = path.join(__dirname, '..', '..', '..', '..', '..', 'services', 'pioneer-server', 'public', 'coins');
|
|
20
|
+
const MISSING_ASSETS_FILE = path.join(__dirname, 'missing-assets.json');
|
|
21
|
+
const PROGRESS_FILE = path.join(__dirname, '.coingecko-progress.json');
|
|
22
|
+
const FAILED_FILE = path.join(__dirname, '.coingecko-failed.json');
|
|
23
|
+
|
|
24
|
+
// API rate limiting
|
|
25
|
+
const API_DELAY_MS = 1500; // 1.5 seconds between API calls (40 calls/minute)
|
|
26
|
+
const MAX_RETRIES = 3;
|
|
27
|
+
|
|
28
|
+
// Ensure local coins directory exists
|
|
29
|
+
if (!fs.existsSync(LOCAL_COINS_DIR)) {
|
|
30
|
+
fs.mkdirSync(LOCAL_COINS_DIR, { recursive: true });
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Map CAIP chainId to CoinGecko platform identifier
|
|
35
|
+
*/
|
|
36
|
+
const CHAIN_ID_TO_COINGECKO_PLATFORM = {
|
|
37
|
+
'eip155:1': 'ethereum',
|
|
38
|
+
'eip155:10': 'optimism',
|
|
39
|
+
'eip155:56': 'binance-smart-chain',
|
|
40
|
+
'eip155:100': 'xdai',
|
|
41
|
+
'eip155:137': 'polygon-pos',
|
|
42
|
+
'eip155:250': 'fantom',
|
|
43
|
+
'eip155:8453': 'base',
|
|
44
|
+
'eip155:42161': 'arbitrum-one',
|
|
45
|
+
'eip155:42220': 'celo',
|
|
46
|
+
'eip155:43114': 'avalanche',
|
|
47
|
+
'eip155:534352': 'scroll',
|
|
48
|
+
'eip155:59144': 'linea',
|
|
49
|
+
'eip155:324': 'zksync',
|
|
50
|
+
'eip155:1101': 'polygon-zkevm',
|
|
51
|
+
'eip155:5000': 'mantle',
|
|
52
|
+
'eip155:81457': 'blast',
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Convert assetId to base64-encoded format
|
|
57
|
+
*/
|
|
58
|
+
function encodeAssetId(assetId) {
|
|
59
|
+
return Buffer.from(assetId).toString('base64');
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Extract contract address from assetId
|
|
64
|
+
*/
|
|
65
|
+
function extractContractAddress(assetId) {
|
|
66
|
+
const match = assetId.match(/erc20:(0x[a-fA-F0-9]{40})/);
|
|
67
|
+
return match ? match[1].toLowerCase() : null;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Sleep for specified milliseconds
|
|
72
|
+
*/
|
|
73
|
+
function sleep(ms) {
|
|
74
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Make HTTP/HTTPS request with timeout
|
|
79
|
+
*/
|
|
80
|
+
async function makeRequest(url, timeout = 10000) {
|
|
81
|
+
return new Promise((resolve, reject) => {
|
|
82
|
+
try {
|
|
83
|
+
const urlObj = new URL(url);
|
|
84
|
+
const client = urlObj.protocol === 'https:' ? https : http;
|
|
85
|
+
|
|
86
|
+
const req = client.get(url, { timeout }, (res) => {
|
|
87
|
+
let data = '';
|
|
88
|
+
|
|
89
|
+
res.on('data', chunk => {
|
|
90
|
+
data += chunk;
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
res.on('end', () => {
|
|
94
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
95
|
+
try {
|
|
96
|
+
const parsed = JSON.parse(data);
|
|
97
|
+
resolve({ success: true, data: parsed, statusCode: res.statusCode });
|
|
98
|
+
} catch (e) {
|
|
99
|
+
resolve({ success: false, error: 'Invalid JSON', statusCode: res.statusCode });
|
|
100
|
+
}
|
|
101
|
+
} else if (res.statusCode === 429) {
|
|
102
|
+
resolve({ success: false, error: 'Rate limited', statusCode: 429 });
|
|
103
|
+
} else if (res.statusCode === 404) {
|
|
104
|
+
resolve({ success: false, error: 'Not found', statusCode: 404 });
|
|
105
|
+
} else {
|
|
106
|
+
resolve({ success: false, error: `HTTP ${res.statusCode}`, statusCode: res.statusCode });
|
|
107
|
+
}
|
|
108
|
+
});
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
req.on('error', (err) => {
|
|
112
|
+
resolve({ success: false, error: err.message });
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
req.on('timeout', () => {
|
|
116
|
+
req.destroy();
|
|
117
|
+
resolve({ success: false, error: 'Timeout' });
|
|
118
|
+
});
|
|
119
|
+
} catch (e) {
|
|
120
|
+
resolve({ success: false, error: e.message });
|
|
121
|
+
}
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Download file from URL
|
|
127
|
+
*/
|
|
128
|
+
async function downloadFile(url, outputPath, timeout = 10000) {
|
|
129
|
+
return new Promise((resolve) => {
|
|
130
|
+
try {
|
|
131
|
+
const urlObj = new URL(url);
|
|
132
|
+
const client = urlObj.protocol === 'https:' ? https : http;
|
|
133
|
+
|
|
134
|
+
const req = client.get(url, { timeout }, (res) => {
|
|
135
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
136
|
+
const fileStream = fs.createWriteStream(outputPath);
|
|
137
|
+
res.pipe(fileStream);
|
|
138
|
+
fileStream.on('finish', () => {
|
|
139
|
+
fileStream.close();
|
|
140
|
+
resolve(true);
|
|
141
|
+
});
|
|
142
|
+
fileStream.on('error', () => resolve(false));
|
|
143
|
+
} else if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
|
|
144
|
+
// Follow redirect
|
|
145
|
+
downloadFile(res.headers.location, outputPath, timeout).then(resolve);
|
|
146
|
+
} else {
|
|
147
|
+
resolve(false);
|
|
148
|
+
}
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
req.on('error', () => resolve(false));
|
|
152
|
+
req.on('timeout', () => {
|
|
153
|
+
req.destroy();
|
|
154
|
+
resolve(false);
|
|
155
|
+
});
|
|
156
|
+
} catch (e) {
|
|
157
|
+
resolve(false);
|
|
158
|
+
}
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* Query CoinGecko API for token info by contract address
|
|
164
|
+
*/
|
|
165
|
+
async function getCoinGeckoTokenInfo(chainId, contractAddress, retries = 0) {
|
|
166
|
+
const platform = CHAIN_ID_TO_COINGECKO_PLATFORM[chainId];
|
|
167
|
+
if (!platform) {
|
|
168
|
+
return { success: false, error: 'Unsupported chain' };
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
const url = `https://api.coingecko.com/api/v3/coins/${platform}/contract/${contractAddress}`;
|
|
172
|
+
|
|
173
|
+
const result = await makeRequest(url);
|
|
174
|
+
|
|
175
|
+
// Handle rate limiting with exponential backoff
|
|
176
|
+
if (result.statusCode === 429 && retries < MAX_RETRIES) {
|
|
177
|
+
const backoffMs = API_DELAY_MS * Math.pow(2, retries);
|
|
178
|
+
console.log(` ⏳ Rate limited, waiting ${backoffMs}ms...`);
|
|
179
|
+
await sleep(backoffMs);
|
|
180
|
+
return getCoinGeckoTokenInfo(chainId, contractAddress, retries + 1);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
return result;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Get TrustWallet fallback URLs
|
|
188
|
+
*/
|
|
189
|
+
function getTrustWalletFallbacks(assetData) {
|
|
190
|
+
const fallbacks = [];
|
|
191
|
+
|
|
192
|
+
if (assetData.assetId?.includes('erc20:')) {
|
|
193
|
+
const match = assetData.assetId.match(/erc20:(0x[a-fA-F0-9]+)/);
|
|
194
|
+
if (match) {
|
|
195
|
+
const contract = match[1];
|
|
196
|
+
// Simple checksum approach
|
|
197
|
+
const checksummed = contract.substring(0, 2) + contract.substring(2).split('')
|
|
198
|
+
.map((char, i) => i % 2 === 0 && parseInt(char, 16) >= 8 ? char.toUpperCase() : char)
|
|
199
|
+
.join('');
|
|
200
|
+
|
|
201
|
+
fallbacks.push(`https://raw.githubusercontent.com/trustwallet/assets/master/blockchains/ethereum/assets/${checksummed}/logo.png`);
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
return fallbacks;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* Process a single asset with CoinGecko API
|
|
210
|
+
*/
|
|
211
|
+
async function processAsset(assetId, assetData, stats, progress, failed) {
|
|
212
|
+
const encoded = encodeAssetId(assetId);
|
|
213
|
+
const localPath = path.join(LOCAL_COINS_DIR, `${encoded}.png`);
|
|
214
|
+
|
|
215
|
+
stats.total++;
|
|
216
|
+
|
|
217
|
+
// Skip if already processed successfully
|
|
218
|
+
if (progress.processed[assetId]) {
|
|
219
|
+
stats.skipped++;
|
|
220
|
+
return true;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// Check local file first
|
|
224
|
+
if (fs.existsSync(localPath)) {
|
|
225
|
+
const fileSize = fs.statSync(localPath).size;
|
|
226
|
+
if (fileSize > 0) {
|
|
227
|
+
stats.alreadyLocal++;
|
|
228
|
+
progress.processed[assetId] = 'local';
|
|
229
|
+
assetData.icon = `https://api.keepkey.com/coins/${encoded}.png`;
|
|
230
|
+
return true;
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
console.log(`\n[${stats.total}/${stats.totalAssets}] ${assetData.symbol} - ${assetData.name}`);
|
|
235
|
+
console.log(` Asset: ${assetId.substring(0, 70)}...`);
|
|
236
|
+
|
|
237
|
+
// Extract contract address
|
|
238
|
+
const contractAddress = extractContractAddress(assetId);
|
|
239
|
+
if (!contractAddress) {
|
|
240
|
+
console.log(` ⚠️ Not an ERC20 token - skipping CoinGecko lookup`);
|
|
241
|
+
stats.notErc20++;
|
|
242
|
+
progress.processed[assetId] = 'not_erc20';
|
|
243
|
+
return false;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// Query CoinGecko API
|
|
247
|
+
console.log(` 🔍 Querying CoinGecko API...`);
|
|
248
|
+
const result = await getCoinGeckoTokenInfo(assetData.chainId, contractAddress);
|
|
249
|
+
|
|
250
|
+
// Respect rate limits
|
|
251
|
+
await sleep(API_DELAY_MS);
|
|
252
|
+
|
|
253
|
+
if (result.success && result.data?.image) {
|
|
254
|
+
const imageUrl = result.data.image.large || result.data.image.small || result.data.image.thumb;
|
|
255
|
+
|
|
256
|
+
if (imageUrl) {
|
|
257
|
+
console.log(` 📥 Downloading from CoinGecko: ${imageUrl.substring(0, 70)}...`);
|
|
258
|
+
const downloaded = await downloadFile(imageUrl, localPath);
|
|
259
|
+
|
|
260
|
+
if (downloaded) {
|
|
261
|
+
console.log(` ✅ Successfully downloaded from CoinGecko`);
|
|
262
|
+
stats.downloadedFromCoinGecko++;
|
|
263
|
+
progress.processed[assetId] = 'coingecko';
|
|
264
|
+
assetData.icon = `https://api.keepkey.com/coins/${encoded}.png`;
|
|
265
|
+
return true;
|
|
266
|
+
} else {
|
|
267
|
+
console.log(` ❌ Download failed`);
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
} else {
|
|
271
|
+
console.log(` ℹ️ CoinGecko: ${result.error || 'Not found'}`);
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
// Try TrustWallet fallback
|
|
275
|
+
const trustWalletUrls = getTrustWalletFallbacks(assetData);
|
|
276
|
+
for (const url of trustWalletUrls) {
|
|
277
|
+
console.log(` 🔄 Trying TrustWallet: ${url.substring(0, 70)}...`);
|
|
278
|
+
const downloaded = await downloadFile(url, localPath);
|
|
279
|
+
if (downloaded) {
|
|
280
|
+
console.log(` ✅ Downloaded from TrustWallet`);
|
|
281
|
+
stats.downloadedFromTrustWallet++;
|
|
282
|
+
progress.processed[assetId] = 'trustwallet';
|
|
283
|
+
assetData.icon = `https://api.keepkey.com/coins/${encoded}.png`;
|
|
284
|
+
return true;
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
// Mark as failed
|
|
289
|
+
console.log(` ❌ No icon found`);
|
|
290
|
+
stats.failed++;
|
|
291
|
+
failed.assets[assetId] = {
|
|
292
|
+
symbol: assetData.symbol,
|
|
293
|
+
name: assetData.name,
|
|
294
|
+
chainId: assetData.chainId,
|
|
295
|
+
error: result.error || 'No icon available',
|
|
296
|
+
attempts: (failed.assets[assetId]?.attempts || 0) + 1,
|
|
297
|
+
};
|
|
298
|
+
|
|
299
|
+
// Still set the target URL
|
|
300
|
+
assetData.icon = `https://api.keepkey.com/coins/${encoded}.png`;
|
|
301
|
+
return false;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* Save progress
|
|
306
|
+
*/
|
|
307
|
+
function saveProgress(progress, data, stats, failed) {
|
|
308
|
+
fs.writeFileSync(PROGRESS_FILE, JSON.stringify(progress, null, 2));
|
|
309
|
+
fs.writeFileSync(DATA_FILE, JSON.stringify(data, null, 2));
|
|
310
|
+
fs.writeFileSync(FAILED_FILE, JSON.stringify(failed, null, 2));
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
/**
|
|
314
|
+
* Main execution
|
|
315
|
+
*/
|
|
316
|
+
async function main() {
|
|
317
|
+
console.log('🚀 Enhanced icon download with CoinGecko API\n');
|
|
318
|
+
console.log('⚠️ Rate limit: ~40 API calls per minute');
|
|
319
|
+
console.log('⏱️ This will take approximately 6-7 hours for 14,000 assets\n');
|
|
320
|
+
|
|
321
|
+
// Load missing assets
|
|
322
|
+
let assetsToProcess = {};
|
|
323
|
+
if (fs.existsSync(MISSING_ASSETS_FILE)) {
|
|
324
|
+
const missingAssets = JSON.parse(fs.readFileSync(MISSING_ASSETS_FILE, 'utf8'));
|
|
325
|
+
console.log(`📂 Loading ${missingAssets.length} assets from missing-assets.json\n`);
|
|
326
|
+
|
|
327
|
+
// Convert array to object keyed by assetId
|
|
328
|
+
for (const asset of missingAssets) {
|
|
329
|
+
assetsToProcess[asset.assetId] = {
|
|
330
|
+
symbol: asset.symbol,
|
|
331
|
+
name: asset.name,
|
|
332
|
+
chainId: asset.chainId,
|
|
333
|
+
icon: asset.oldIcon,
|
|
334
|
+
assetId: asset.assetId,
|
|
335
|
+
};
|
|
336
|
+
}
|
|
337
|
+
} else {
|
|
338
|
+
console.log('❌ missing-assets.json not found');
|
|
339
|
+
process.exit(1);
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
const entries = Object.entries(assetsToProcess);
|
|
343
|
+
|
|
344
|
+
// Load progress
|
|
345
|
+
let progress = { processed: {}, timestamp: Date.now() };
|
|
346
|
+
if (fs.existsSync(PROGRESS_FILE)) {
|
|
347
|
+
progress = JSON.parse(fs.readFileSync(PROGRESS_FILE, 'utf8'));
|
|
348
|
+
console.log(`📂 Resuming from previous run (${Object.keys(progress.processed).length} already processed)\n`);
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
// Load failed attempts
|
|
352
|
+
let failed = { assets: {} };
|
|
353
|
+
if (fs.existsSync(FAILED_FILE)) {
|
|
354
|
+
failed = JSON.parse(fs.readFileSync(FAILED_FILE, 'utf8'));
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
const stats = {
|
|
358
|
+
total: 0,
|
|
359
|
+
totalAssets: entries.length,
|
|
360
|
+
skipped: 0,
|
|
361
|
+
alreadyLocal: 0,
|
|
362
|
+
downloadedFromCoinGecko: 0,
|
|
363
|
+
downloadedFromTrustWallet: 0,
|
|
364
|
+
notErc20: 0,
|
|
365
|
+
failed: 0,
|
|
366
|
+
};
|
|
367
|
+
|
|
368
|
+
const saveInterval = 10; // Save every 10 assets
|
|
369
|
+
|
|
370
|
+
// Process sequentially to respect API rate limits
|
|
371
|
+
for (let i = 0; i < entries.length; i++) {
|
|
372
|
+
const [assetId, assetData] = entries[i];
|
|
373
|
+
await processAsset(assetId, assetData, stats, progress, failed);
|
|
374
|
+
|
|
375
|
+
// Save progress periodically
|
|
376
|
+
if ((i + 1) % saveInterval === 0) {
|
|
377
|
+
saveProgress(progress, assetsToProcess, stats, failed);
|
|
378
|
+
console.log(`\n💾 Progress saved (${i + 1}/${entries.length})`);
|
|
379
|
+
|
|
380
|
+
// Print interim stats
|
|
381
|
+
const percentComplete = ((i + 1) / entries.length * 100).toFixed(1);
|
|
382
|
+
console.log(`📊 ${percentComplete}% complete | CoinGecko: ${stats.downloadedFromCoinGecko} | TrustWallet: ${stats.downloadedFromTrustWallet} | Failed: ${stats.failed}\n`);
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
// Final save
|
|
387
|
+
saveProgress(progress, assetsToProcess, stats, failed);
|
|
388
|
+
|
|
389
|
+
// Print summary
|
|
390
|
+
console.log('\n' + '='.repeat(80));
|
|
391
|
+
console.log('📊 FINAL SUMMARY');
|
|
392
|
+
console.log('='.repeat(80));
|
|
393
|
+
console.log(`Total assets processed: ${stats.totalAssets}`);
|
|
394
|
+
console.log(`Skipped (already done): ${stats.skipped}`);
|
|
395
|
+
console.log(`Already in local storage: ${stats.alreadyLocal}`);
|
|
396
|
+
console.log(`Downloaded from CoinGecko: ${stats.downloadedFromCoinGecko}`);
|
|
397
|
+
console.log(`Downloaded from TrustWallet: ${stats.downloadedFromTrustWallet}`);
|
|
398
|
+
console.log(`Not ERC20 (skipped): ${stats.notErc20}`);
|
|
399
|
+
console.log(`Failed (no icon found): ${stats.failed}`);
|
|
400
|
+
console.log('='.repeat(80));
|
|
401
|
+
|
|
402
|
+
const successRate = ((stats.downloadedFromCoinGecko + stats.downloadedFromTrustWallet) / stats.totalAssets * 100).toFixed(1);
|
|
403
|
+
console.log(`\n✅ Success rate: ${successRate}%\n`);
|
|
404
|
+
|
|
405
|
+
const localFiles = fs.readdirSync(LOCAL_COINS_DIR).filter(f => f.endsWith('.png')).length;
|
|
406
|
+
console.log(`📁 Local coins directory: ${localFiles} PNG files\n`);
|
|
407
|
+
|
|
408
|
+
console.log(`📝 Failed assets saved to: ${FAILED_FILE}\n`);
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
main().catch(err => {
|
|
412
|
+
console.error('❌ Error:', err);
|
|
413
|
+
process.exit(1);
|
|
414
|
+
});
|