@icgio/icg-exchanges-wrapper 1.14.255 → 1.14.257
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -4,7 +4,111 @@ const DEFAULT_ORDERBOOK_LEVEL = 50
|
|
|
4
4
|
|
|
5
5
|
let first_time = {},
|
|
6
6
|
exchange_pair_rates = {},
|
|
7
|
-
pair_exchange_rates = {}
|
|
7
|
+
pair_exchange_rates = {},
|
|
8
|
+
// Track active WebSocket connections per exchange to avoid duplicates
|
|
9
|
+
ws_connection_promises = {},
|
|
10
|
+
ws_connection_pairs = {},
|
|
11
|
+
// Track wrapped promises with fallback handlers to avoid duplicate handlers
|
|
12
|
+
ws_connection_promises_with_fallback = {},
|
|
13
|
+
// Track persistent WebSocket failures per pair to avoid repeated retries
|
|
14
|
+
ws_failed_pairs = {}, // { "ExchangeName": { "PAIR": { count: number, last_failed: timestamp } } }
|
|
15
|
+
WS_FAILURE_THRESHOLD = 3, // Skip WebSocket after 3 consecutive failures
|
|
16
|
+
WS_RETRY_INTERVAL_MS = 5 * 60 * 1000 // Retry WebSocket after 5 minutes
|
|
17
|
+
|
|
18
|
+
// ============ HELPER FUNCTIONS ============
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Group exchanges by name and merge all pairs
|
|
22
|
+
* Returns: { exchange_instances: {}, exchange_all_pairs: {} }
|
|
23
|
+
*/
|
|
24
|
+
function group_exchanges_by_name(Exchanges, exchange_pair_dict) {
|
|
25
|
+
let exchange_instances = {}
|
|
26
|
+
let exchange_all_pairs = {}
|
|
27
|
+
for (let Exchange of Exchanges) {
|
|
28
|
+
let Exchange_name = _.upperFirst(Exchange.name)
|
|
29
|
+
if (!exchange_pair_dict[Exchange_name]) {
|
|
30
|
+
continue
|
|
31
|
+
}
|
|
32
|
+
if (!exchange_instances[Exchange_name]) {
|
|
33
|
+
exchange_instances[Exchange_name] = Exchange
|
|
34
|
+
exchange_all_pairs[Exchange_name] = []
|
|
35
|
+
}
|
|
36
|
+
exchange_all_pairs[Exchange_name] = _.union(exchange_all_pairs[Exchange_name], exchange_pair_dict[Exchange_name])
|
|
37
|
+
}
|
|
38
|
+
return { exchange_instances, exchange_all_pairs }
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Set data in both exchange_pair and pair_exchange structures
|
|
43
|
+
*/
|
|
44
|
+
function set_pair_data(exchange_pair_dict, pair_exchange_dict, exchange_name, pair, data) {
|
|
45
|
+
_.set(exchange_pair_dict, [exchange_name, pair], data)
|
|
46
|
+
_.set(pair_exchange_dict, [pair, exchange_name], data)
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Delete data from both exchange_pair and pair_exchange structures
|
|
51
|
+
*/
|
|
52
|
+
function delete_pair_data(exchange_pair_dict, pair_exchange_dict, exchange_name, pair) {
|
|
53
|
+
if (exchange_pair_dict[exchange_name]) {
|
|
54
|
+
delete exchange_pair_dict[exchange_name][pair]
|
|
55
|
+
}
|
|
56
|
+
if (pair_exchange_dict[pair]) {
|
|
57
|
+
delete pair_exchange_dict[pair][exchange_name]
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Track WebSocket failure for a pair
|
|
63
|
+
*/
|
|
64
|
+
function track_ws_failure(failed_pairs_dict, exchange_name, pair) {
|
|
65
|
+
if (!failed_pairs_dict[exchange_name]) {
|
|
66
|
+
failed_pairs_dict[exchange_name] = {}
|
|
67
|
+
}
|
|
68
|
+
if (!failed_pairs_dict[exchange_name][pair]) {
|
|
69
|
+
failed_pairs_dict[exchange_name][pair] = { count: 0, last_failed: 0 }
|
|
70
|
+
}
|
|
71
|
+
failed_pairs_dict[exchange_name][pair].count++
|
|
72
|
+
failed_pairs_dict[exchange_name][pair].last_failed = Date.now()
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Reset WebSocket failure count for a pair (on success)
|
|
77
|
+
*/
|
|
78
|
+
function reset_ws_failure(failed_pairs_dict, exchange_name, pair) {
|
|
79
|
+
if (failed_pairs_dict[exchange_name] && failed_pairs_dict[exchange_name][pair]) {
|
|
80
|
+
delete failed_pairs_dict[exchange_name][pair]
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Check if pair should skip WebSocket due to persistent failures
|
|
86
|
+
*/
|
|
87
|
+
function should_skip_websocket(failed_pairs_dict, exchange_name, pair) {
|
|
88
|
+
const failure_info = failed_pairs_dict[exchange_name]?.[pair]
|
|
89
|
+
return (
|
|
90
|
+
failure_info &&
|
|
91
|
+
failure_info.count >= WS_FAILURE_THRESHOLD &&
|
|
92
|
+
(Date.now() - failure_info.last_failed) < WS_RETRY_INTERVAL_MS
|
|
93
|
+
)
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Create WebSocket timeout checker
|
|
98
|
+
*/
|
|
99
|
+
function create_ws_timeout_checker(Exchange_name, timeout_ms, max_checks, on_timeout) {
|
|
100
|
+
const start_time = Date.now()
|
|
101
|
+
let check_count = 0
|
|
102
|
+
return () => {
|
|
103
|
+
check_count++
|
|
104
|
+
const elapsed = Date.now() - start_time
|
|
105
|
+
if (elapsed > timeout_ms || check_count > max_checks) {
|
|
106
|
+
on_timeout(elapsed)
|
|
107
|
+
return true
|
|
108
|
+
}
|
|
109
|
+
return false
|
|
110
|
+
}
|
|
111
|
+
}
|
|
8
112
|
|
|
9
113
|
const get_exchanges_rates = function (Exchanges, exchange_pair_dict, cb, ws_enabled = true) {
|
|
10
114
|
let checked_rates = {}
|
|
@@ -14,79 +118,247 @@ const get_exchanges_rates = function (Exchanges, exchange_pair_dict, cb, ws_enab
|
|
|
14
118
|
console.error('Exchanges should be an array')
|
|
15
119
|
return
|
|
16
120
|
} else {
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
121
|
+
// First pass: group pairs by exchange name and find the Exchange instance
|
|
122
|
+
// This ensures ALL pairs for each exchange are collected BEFORE creating any WebSocket
|
|
123
|
+
const { exchange_instances, exchange_all_pairs } = group_exchanges_by_name(Exchanges, exchange_pair_dict)
|
|
124
|
+
|
|
125
|
+
// Second pass: process each unique exchange once
|
|
126
|
+
// Now we have all pairs collected, create ONE WebSocket per exchange with ALL pairs
|
|
127
|
+
for (let Exchange_name in exchange_instances) {
|
|
128
|
+
let Exchange = exchange_instances[Exchange_name]
|
|
129
|
+
let all_pairs = exchange_all_pairs[Exchange_name]
|
|
130
|
+
|
|
131
|
+
// Log how many pairs will be used for this exchange
|
|
132
|
+
if (all_pairs.length > 0 && Exchange.ws_market && ws_enabled) {
|
|
133
|
+
console.log(`[WebSocket Rates] ${Exchange_name}: Creating connection with ${all_pairs.length} pair(s): ${all_pairs.join(', ')}`)
|
|
25
134
|
}
|
|
135
|
+
|
|
26
136
|
if (Exchange.ws_market && ws_enabled) {
|
|
27
|
-
|
|
28
|
-
|
|
137
|
+
// Check if WebSocket is already opening/opened for this exchange
|
|
138
|
+
const ws_key = Exchange_name
|
|
139
|
+
const ws_status = Exchange.ws?.market?.status
|
|
140
|
+
const is_opening_or_opened = ws_status === 'opening' || ws_status === 'opened'
|
|
141
|
+
|
|
142
|
+
let ws_promise
|
|
143
|
+
let ws_promise_with_fallback
|
|
144
|
+
if (is_opening_or_opened && ws_connection_promises[ws_key]) {
|
|
145
|
+
// Reuse existing WebSocket connection promise
|
|
146
|
+
ws_promise = ws_connection_promises[ws_key]
|
|
147
|
+
// Merge pairs if needed
|
|
148
|
+
const existing_pairs = ws_connection_pairs[ws_key] || []
|
|
149
|
+
const new_pairs = _.difference(all_pairs, existing_pairs)
|
|
150
|
+
if (new_pairs.length > 0) {
|
|
151
|
+
ws_connection_pairs[ws_key] = _.union(existing_pairs, all_pairs)
|
|
152
|
+
// Add new pairs to existing connection if exchange supports it
|
|
153
|
+
// Note: This depends on exchange implementation - some may need to restart
|
|
154
|
+
// For now, we'll wait for existing connection and check if new pairs are already included
|
|
155
|
+
}
|
|
156
|
+
// Reuse wrapped promise with fallback if it exists (avoid duplicate catch handlers)
|
|
157
|
+
if (ws_connection_promises_with_fallback[ws_key]) {
|
|
158
|
+
ws_promise_with_fallback = ws_connection_promises_with_fallback[ws_key]
|
|
159
|
+
}
|
|
160
|
+
} else {
|
|
161
|
+
// Create new WebSocket connection
|
|
162
|
+
ws_connection_pairs[ws_key] = all_pairs
|
|
163
|
+
ws_promise = new Promise((resolve, reject) => {
|
|
164
|
+
if (!Exchange.ws || !Exchange.ws.market) {
|
|
165
|
+
console.error(`${Exchange_name} WebSocket not properly initialized`)
|
|
166
|
+
reject(new Error(`WebSocket not initialized for ${Exchange_name}`))
|
|
167
|
+
return
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
// Store promise before starting connection
|
|
171
|
+
ws_connection_promises[ws_key] = ws_promise
|
|
172
|
+
|
|
173
|
+
// Create WebSocket with ALL pairs collected in first pass
|
|
174
|
+
// This ensures only ONE WebSocket per exchange, even with multiple clients/pairs
|
|
175
|
+
Exchange.ws_market(all_pairs)
|
|
176
|
+
|
|
177
|
+
const WS_TIMEOUT_MS = 30000 // 30 seconds timeout
|
|
178
|
+
const MAX_CHECKS = 300 // 30 seconds / 100ms
|
|
179
|
+
const check_timeout = create_ws_timeout_checker(
|
|
180
|
+
Exchange_name,
|
|
181
|
+
WS_TIMEOUT_MS,
|
|
182
|
+
MAX_CHECKS,
|
|
183
|
+
(elapsed) => {
|
|
184
|
+
console.error(`WebSocket timeout for ${Exchange_name} after ${elapsed}ms`)
|
|
185
|
+
reject(new Error(`WebSocket timeout for ${Exchange_name}`))
|
|
186
|
+
}
|
|
187
|
+
)
|
|
29
188
|
|
|
30
189
|
const check_status = () => {
|
|
31
|
-
|
|
32
|
-
return
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
|
|
190
|
+
if (check_timeout()) {
|
|
191
|
+
return
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// Safety check for WebSocket object
|
|
195
|
+
if (!Exchange.ws || !Exchange.ws.market) {
|
|
196
|
+
console.error(`${Exchange_name} WebSocket object missing during check`)
|
|
197
|
+
reject(new Error(`WebSocket object missing for ${Exchange_name}`))
|
|
198
|
+
return
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
// Check only the pairs we need for this call (not all pairs in connection)
|
|
202
|
+
let pairs_to_check = exchange_pair_dict[Exchange_name] || all_pairs
|
|
203
|
+
|
|
204
|
+
// Check if WebSocket connection is at least opening/opened (not closed/stopped)
|
|
205
|
+
const ws_connection_ready = Exchange.ws.market.status === 'opening' || Exchange.ws.market.status === 'opened'
|
|
206
|
+
|
|
207
|
+
if (ws_connection_ready) {
|
|
208
|
+
// Try each pair individually - don't wait for all pairs to be ready
|
|
209
|
+
// This allows us to detect individual pair failures
|
|
210
|
+
let rate_promises = pairs_to_check.map((pair) => {
|
|
36
211
|
return new Promise((rate_resolve) => {
|
|
212
|
+
// Check if this pair should skip WebSocket due to persistent failures
|
|
213
|
+
if (should_skip_websocket(ws_failed_pairs, Exchange_name, pair)) {
|
|
214
|
+
const failure_info = ws_failed_pairs[Exchange_name][pair]
|
|
215
|
+
console.log(`[WebSocket Skip] ${Exchange_name} ${pair}: Skipping WebSocket (${failure_info.count} consecutive failures, last failed ${Math.round((Date.now() - failure_info.last_failed) / 1000)}s ago)`)
|
|
216
|
+
rate_resolve({ pair, failed: true, skip_ws: true })
|
|
217
|
+
return
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// Check if this specific pair is ready before calling rate_ws
|
|
221
|
+
const pair_ready = (
|
|
222
|
+
Exchange.ws.market.pair_status &&
|
|
223
|
+
Exchange.ws.market.pair_status[pair] === 'opened' &&
|
|
224
|
+
Exchange.ws.market.asks_dict &&
|
|
225
|
+
Exchange.ws.market.asks_dict[pair] &&
|
|
226
|
+
Exchange.ws.market.asks_dict[pair].length > 0 &&
|
|
227
|
+
Exchange.ws.market.bids_dict &&
|
|
228
|
+
Exchange.ws.market.bids_dict[pair] &&
|
|
229
|
+
Exchange.ws.market.bids_dict[pair].length > 0
|
|
230
|
+
)
|
|
231
|
+
|
|
232
|
+
if (!pair_ready) {
|
|
233
|
+
// Pair not ready yet - mark as failed, will retry later or fall back to REST
|
|
234
|
+
console.log(`[WebSocket] ${Exchange_name} ${pair}: Not ready yet (status: ${Exchange.ws.market.pair_status?.[pair] || 'unknown'})`)
|
|
235
|
+
rate_resolve({ pair, failed: true })
|
|
236
|
+
return
|
|
237
|
+
}
|
|
238
|
+
|
|
37
239
|
Exchange.rate_ws(pair, function (res) {
|
|
38
240
|
if (res.success) {
|
|
39
|
-
|
|
40
|
-
|
|
241
|
+
// Success - reset failure count
|
|
242
|
+
reset_ws_failure(ws_failed_pairs, Exchange_name, pair)
|
|
243
|
+
set_pair_data(exchange_pair_rates, pair_exchange_rates, Exchange_name, pair, res.body)
|
|
244
|
+
rate_resolve({ pair, failed: false })
|
|
41
245
|
} else {
|
|
42
246
|
console.error(`getting rate ws failed for ${Exchange_name} ${pair}:`, res)
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
}
|
|
46
|
-
if (pair_exchange_rates[pair]) {
|
|
47
|
-
delete pair_exchange_rates[pair][Exchange_name]
|
|
48
|
-
}
|
|
247
|
+
delete_pair_data(exchange_pair_rates, pair_exchange_rates, Exchange_name, pair)
|
|
248
|
+
track_ws_failure(ws_failed_pairs, Exchange_name, pair)
|
|
249
|
+
rate_resolve({ pair, failed: true })
|
|
49
250
|
}
|
|
50
|
-
rate_resolve()
|
|
51
251
|
})
|
|
52
252
|
})
|
|
53
253
|
})
|
|
54
254
|
Promise.all(rate_promises)
|
|
55
|
-
.then(() => {
|
|
56
|
-
//
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
255
|
+
.then((results) => {
|
|
256
|
+
// Check which pairs failed and fall back to REST for those pairs only
|
|
257
|
+
const failed_pairs = results.filter((r) => r && r.failed).map((r) => r.pair)
|
|
258
|
+
if (failed_pairs.length > 0 && Exchange.rate) {
|
|
259
|
+
console.log(`[WebSocket Fallback] ${Exchange_name}: Falling back to REST for ${failed_pairs.length} pair(s): ${failed_pairs.join(', ')}`)
|
|
260
|
+
const rest_promises = failed_pairs.map((pair) => {
|
|
261
|
+
return new Promise((rest_resolve) => {
|
|
262
|
+
Exchange.rate(pair, DEFAULT_ORDERBOOK_LEVEL, function (res) {
|
|
263
|
+
if (res.success) {
|
|
264
|
+
set_pair_data(exchange_pair_rates, pair_exchange_rates, Exchange_name, pair, res.body)
|
|
265
|
+
console.log(`[REST Fallback] ${Exchange_name} ${pair}: Successfully fetched via REST`)
|
|
266
|
+
} else {
|
|
267
|
+
console.error(`[REST Fallback] getting rate failed for ${Exchange_name} ${pair}:`, res)
|
|
268
|
+
delete_pair_data(exchange_pair_rates, pair_exchange_rates, Exchange_name, pair)
|
|
269
|
+
}
|
|
270
|
+
rest_resolve()
|
|
271
|
+
})
|
|
272
|
+
})
|
|
273
|
+
})
|
|
274
|
+
// Wait for REST fallback to complete before resolving
|
|
275
|
+
return Promise.all(rest_promises)
|
|
276
|
+
.catch((err) => {
|
|
277
|
+
console.error(`[REST Fallback] Error fetching rates for ${Exchange_name}:`, err)
|
|
278
|
+
})
|
|
279
|
+
.then(() => {
|
|
280
|
+
// Resolve after REST fallback completes
|
|
281
|
+
resolve()
|
|
282
|
+
})
|
|
283
|
+
} else {
|
|
284
|
+
// if (Exchange.ws.market.end_base) {
|
|
285
|
+
// Exchange.ws.market.end_base(); // Close the connection and prevent reconnection
|
|
286
|
+
// console.log(`${Exchange_name} WebSocket connection closed.`);
|
|
287
|
+
// }
|
|
288
|
+
resolve()
|
|
289
|
+
}
|
|
61
290
|
})
|
|
62
291
|
.catch((err) => {
|
|
63
292
|
console.error(`Error in rate_ws for ${Exchange_name}:`, err)
|
|
64
293
|
resolve()
|
|
65
294
|
})
|
|
66
295
|
} else {
|
|
296
|
+
// WebSocket connection not ready yet, wait and retry
|
|
67
297
|
setTimeout(check_status, 100)
|
|
68
298
|
}
|
|
69
299
|
}
|
|
70
300
|
|
|
71
301
|
check_status()
|
|
72
302
|
})
|
|
303
|
+
// Clean up promise reference when done (but keep connection open for reuse)
|
|
304
|
+
ws_promise.finally(() => {
|
|
305
|
+
// Don't delete immediately - keep for a short time to allow reuse
|
|
306
|
+
setTimeout(() => {
|
|
307
|
+
if (ws_connection_promises[ws_key] === ws_promise) {
|
|
308
|
+
delete ws_connection_promises[ws_key]
|
|
309
|
+
// Also clean up wrapped promise if it exists
|
|
310
|
+
if (ws_connection_promises_with_fallback[ws_key]) {
|
|
311
|
+
delete ws_connection_promises_with_fallback[ws_key]
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
}, 1000)
|
|
315
|
+
})
|
|
316
|
+
}
|
|
73
317
|
|
|
74
|
-
|
|
318
|
+
// Wrap WebSocket promise to handle rejection and fall back to REST
|
|
319
|
+
// Only create wrapper if it doesn't exist (avoid duplicate catch handlers on reused promises)
|
|
320
|
+
if (!ws_promise_with_fallback) {
|
|
321
|
+
ws_promise_with_fallback = ws_promise.catch((err) => {
|
|
322
|
+
console.error(`[WebSocket Failed] ${Exchange_name}: Falling back to REST for all pairs. Error:`, err.message)
|
|
323
|
+
// Fall back to REST for all pairs
|
|
324
|
+
if (Exchange.rate) {
|
|
325
|
+
return Promise.all(
|
|
326
|
+
all_pairs.map((pair) => {
|
|
327
|
+
return new Promise((rest_resolve) => {
|
|
328
|
+
Exchange.rate(pair, DEFAULT_ORDERBOOK_LEVEL, function (res) {
|
|
329
|
+
if (res.success) {
|
|
330
|
+
set_pair_data(exchange_pair_rates, pair_exchange_rates, Exchange_name, pair, res.body)
|
|
331
|
+
console.log(`[REST Fallback] ${Exchange_name} ${pair}: Successfully fetched via REST`)
|
|
332
|
+
} else {
|
|
333
|
+
console.error(`[REST Fallback] getting rate failed for ${Exchange_name} ${pair}:`, res)
|
|
334
|
+
delete_pair_data(exchange_pair_rates, pair_exchange_rates, Exchange_name, pair)
|
|
335
|
+
}
|
|
336
|
+
rest_resolve()
|
|
337
|
+
})
|
|
338
|
+
})
|
|
339
|
+
})
|
|
340
|
+
).then(() => {
|
|
341
|
+
// Resolve after REST fallback completes
|
|
342
|
+
return Promise.resolve()
|
|
343
|
+
})
|
|
344
|
+
}
|
|
345
|
+
// If no REST fallback available, still resolve to prevent Promise.all from failing
|
|
346
|
+
return Promise.resolve()
|
|
347
|
+
})
|
|
348
|
+
// Store wrapped promise for reuse
|
|
349
|
+
ws_connection_promises_with_fallback[ws_key] = ws_promise_with_fallback
|
|
350
|
+
}
|
|
351
|
+
promises.push(ws_promise_with_fallback)
|
|
75
352
|
} else if (Exchange.rate) {
|
|
76
|
-
let
|
|
353
|
+
let pairs_to_fetch = exchange_pair_dict[Exchange_name] || all_pairs
|
|
354
|
+
let rate_promises = pairs_to_fetch.map((pair) => {
|
|
77
355
|
return new Promise((resolve, reject) => {
|
|
78
356
|
Exchange.rate(pair, DEFAULT_ORDERBOOK_LEVEL, function (res) {
|
|
79
357
|
if (res.success) {
|
|
80
|
-
|
|
81
|
-
_.set(pair_exchange_rates, [pair, Exchange_name], res.body)
|
|
358
|
+
set_pair_data(exchange_pair_rates, pair_exchange_rates, Exchange_name, pair, res.body)
|
|
82
359
|
} else {
|
|
83
360
|
console.error(`getting rate failed for ${Exchange_name} ${pair}:`, res)
|
|
84
|
-
|
|
85
|
-
delete exchange_pair_rates[Exchange_name][pair]
|
|
86
|
-
}
|
|
87
|
-
if (pair_exchange_rates[pair]) {
|
|
88
|
-
delete pair_exchange_rates[pair][Exchange_name]
|
|
89
|
-
}
|
|
361
|
+
delete_pair_data(exchange_pair_rates, pair_exchange_rates, Exchange_name, pair)
|
|
90
362
|
}
|
|
91
363
|
resolve()
|
|
92
364
|
})
|
|
@@ -95,23 +367,18 @@ const get_exchanges_rates = function (Exchanges, exchange_pair_dict, cb, ws_enab
|
|
|
95
367
|
|
|
96
368
|
promises = promises.concat(rate_promises) // Add all rate promises
|
|
97
369
|
} else if (Exchange.rate_with_routes) {
|
|
98
|
-
let
|
|
370
|
+
let pairs_to_fetch = exchange_pair_dict[Exchange_name] || all_pairs
|
|
371
|
+
let rate_with_routes_promises = pairs_to_fetch.map((pair) => {
|
|
99
372
|
return new Promise((resolve, reject) => {
|
|
100
373
|
Exchange.rate_with_routes(pair, function (res) {
|
|
101
374
|
if (res.success) {
|
|
102
375
|
let { asks, bids } = res.body
|
|
103
376
|
asks = asks.map((ask) => ask.slice(0, 2))
|
|
104
377
|
bids = bids.map((bid) => bid.slice(0, 2))
|
|
105
|
-
|
|
106
|
-
_.set(pair_exchange_rates, [pair, Exchange_name], { asks, bids })
|
|
378
|
+
set_pair_data(exchange_pair_rates, pair_exchange_rates, Exchange_name, pair, { asks, bids })
|
|
107
379
|
} else {
|
|
108
380
|
console.error(`getting rate with routes failed for ${Exchange_name} ${pair}:`, res)
|
|
109
|
-
|
|
110
|
-
delete exchange_pair_rates[Exchange_name][pair]
|
|
111
|
-
}
|
|
112
|
-
if (pair_exchange_rates[pair]) {
|
|
113
|
-
delete pair_exchange_rates[pair][Exchange_name]
|
|
114
|
-
}
|
|
381
|
+
delete_pair_data(exchange_pair_rates, pair_exchange_rates, Exchange_name, pair)
|
|
115
382
|
}
|
|
116
383
|
resolve()
|
|
117
384
|
})
|
|
@@ -129,12 +396,20 @@ const get_exchanges_rates = function (Exchanges, exchange_pair_dict, cb, ws_enab
|
|
|
129
396
|
})
|
|
130
397
|
.catch((error) => {
|
|
131
398
|
console.error('Error in fetching exchange rates:', error)
|
|
399
|
+
// Still call callback with available data even if some promises failed
|
|
132
400
|
cb({ exchange_pair_rates, pair_exchange_rates })
|
|
133
401
|
})
|
|
134
402
|
}
|
|
135
403
|
|
|
136
404
|
let exchange_pair_market_history = {},
|
|
137
|
-
pair_exchange_market_history = {}
|
|
405
|
+
pair_exchange_market_history = {},
|
|
406
|
+
// Track active WebSocket connections for market history
|
|
407
|
+
ws_market_history_promises = {},
|
|
408
|
+
ws_market_history_pairs = {},
|
|
409
|
+
// Track wrapped promises with fallback handlers to avoid duplicate handlers
|
|
410
|
+
ws_market_history_promises_with_fallback = {},
|
|
411
|
+
// Track persistent WebSocket failures for market history
|
|
412
|
+
ws_market_history_failed_pairs = {} // { "ExchangeName": { "PAIR": { count: number, last_failed: timestamp } } }
|
|
138
413
|
const get_exchanges_market_history = function (Exchanges, exchange_pair_dict, timeframe_in_ms, cb, ws_enabled = true) {
|
|
139
414
|
let checked_market_history = {}
|
|
140
415
|
let promises = []
|
|
@@ -143,47 +418,164 @@ const get_exchanges_market_history = function (Exchanges, exchange_pair_dict, ti
|
|
|
143
418
|
console.error('Exchanges should be an array')
|
|
144
419
|
return
|
|
145
420
|
} else {
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
}
|
|
421
|
+
// First pass: group pairs by exchange name and find the Exchange instance
|
|
422
|
+
const { exchange_instances, exchange_all_pairs } = group_exchanges_by_name(Exchanges, exchange_pair_dict)
|
|
423
|
+
|
|
424
|
+
// Second pass: process each unique exchange once
|
|
425
|
+
for (let Exchange_name in exchange_instances) {
|
|
426
|
+
let Exchange = exchange_instances[Exchange_name]
|
|
427
|
+
let all_pairs = exchange_all_pairs[Exchange_name]
|
|
428
|
+
|
|
155
429
|
if (Exchange.ws_market && ws_enabled) {
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
430
|
+
// Check if WebSocket is already opening/opened for this exchange
|
|
431
|
+
const ws_key = `market_history_${Exchange_name}`
|
|
432
|
+
const ws_status = Exchange.ws?.market?.status
|
|
433
|
+
const is_opening_or_opened = ws_status === 'opening' || ws_status === 'opened'
|
|
434
|
+
|
|
435
|
+
let ws_promise
|
|
436
|
+
let ws_promise_with_fallback
|
|
437
|
+
if (is_opening_or_opened && ws_market_history_promises[ws_key]) {
|
|
438
|
+
// Reuse existing WebSocket connection promise
|
|
439
|
+
ws_promise = ws_market_history_promises[ws_key]
|
|
440
|
+
// Merge pairs if needed
|
|
441
|
+
const existing_pairs = ws_market_history_pairs[ws_key] || []
|
|
442
|
+
const new_pairs = _.difference(all_pairs, existing_pairs)
|
|
443
|
+
if (new_pairs.length > 0) {
|
|
444
|
+
ws_market_history_pairs[ws_key] = _.union(existing_pairs, all_pairs)
|
|
160
445
|
}
|
|
446
|
+
// Reuse wrapped promise with fallback if it exists (avoid duplicate catch handlers)
|
|
447
|
+
if (ws_market_history_promises_with_fallback[ws_key]) {
|
|
448
|
+
ws_promise_with_fallback = ws_market_history_promises_with_fallback[ws_key]
|
|
449
|
+
}
|
|
450
|
+
} else {
|
|
451
|
+
// Create new WebSocket connection (or use existing one from rates)
|
|
452
|
+
ws_market_history_pairs[ws_key] = all_pairs
|
|
453
|
+
ws_promise = new Promise((resolve, reject) => {
|
|
454
|
+
if (!Exchange.ws || !Exchange.ws.market) {
|
|
455
|
+
console.error(`${Exchange_name} WebSocket not properly initialized`)
|
|
456
|
+
reject(new Error(`WebSocket not initialized for ${Exchange_name}`))
|
|
457
|
+
return
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
// Store promise before starting connection
|
|
461
|
+
ws_market_history_promises[ws_key] = ws_promise
|
|
462
|
+
|
|
463
|
+
// Only start if not already started (reuse connection from rates if available)
|
|
464
|
+
// If starting new, use ALL pairs collected in first pass
|
|
465
|
+
if (!first_time[Exchange_name]) {
|
|
466
|
+
// Create WebSocket with ALL pairs collected in first pass
|
|
467
|
+
// This ensures only ONE WebSocket per exchange, even with multiple clients/pairs
|
|
468
|
+
Exchange.ws_market(all_pairs)
|
|
469
|
+
first_time[Exchange_name] = true
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
const WS_TIMEOUT_MS = 30000 // 30 seconds timeout
|
|
473
|
+
const MAX_CHECKS = 300 // 30 seconds / 100ms
|
|
474
|
+
const check_timeout = create_ws_timeout_checker(
|
|
475
|
+
Exchange_name,
|
|
476
|
+
WS_TIMEOUT_MS,
|
|
477
|
+
MAX_CHECKS,
|
|
478
|
+
(elapsed) => {
|
|
479
|
+
console.error(`WebSocket timeout for ${Exchange_name} market history after ${elapsed}ms`)
|
|
480
|
+
reject(new Error(`WebSocket timeout for ${Exchange_name} market history`))
|
|
481
|
+
}
|
|
482
|
+
)
|
|
483
|
+
|
|
161
484
|
const check_status = () => {
|
|
162
|
-
|
|
163
|
-
return
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
|
|
485
|
+
if (check_timeout()) {
|
|
486
|
+
return
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
// Safety check for WebSocket object
|
|
490
|
+
if (!Exchange.ws || !Exchange.ws.market) {
|
|
491
|
+
console.error(`${Exchange_name} WebSocket object missing during check`)
|
|
492
|
+
reject(new Error(`WebSocket object missing for ${Exchange_name}`))
|
|
493
|
+
return
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
// Check only the pairs we need for this call
|
|
497
|
+
let pairs_to_check = exchange_pair_dict[Exchange_name] || all_pairs
|
|
498
|
+
|
|
499
|
+
// Check if WebSocket connection is at least opening/opened (not closed/stopped)
|
|
500
|
+
const ws_connection_ready = Exchange.ws.market.status === 'opening' || Exchange.ws.market.status === 'opened'
|
|
501
|
+
|
|
502
|
+
if (ws_connection_ready) {
|
|
503
|
+
// Try each pair individually - don't wait for all pairs to be ready
|
|
504
|
+
// This allows us to detect individual pair failures
|
|
505
|
+
let market_history_promises = pairs_to_check.map((pair) => {
|
|
167
506
|
return new Promise((rate_resolve) => {
|
|
507
|
+
// Check if this pair should skip WebSocket due to persistent failures
|
|
508
|
+
if (should_skip_websocket(ws_market_history_failed_pairs, Exchange_name, pair)) {
|
|
509
|
+
const failure_info = ws_market_history_failed_pairs[Exchange_name][pair]
|
|
510
|
+
console.log(`[WebSocket Skip] ${Exchange_name} ${pair} market history: Skipping WebSocket (${failure_info.count} consecutive failures, last failed ${Math.round((Date.now() - failure_info.last_failed) / 1000)}s ago)`)
|
|
511
|
+
rate_resolve({ pair, failed: true, skip_ws: true })
|
|
512
|
+
return
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
// Check if this specific pair is ready before calling market_history_ws
|
|
516
|
+
const pair_ready = (
|
|
517
|
+
Exchange.ws.market.pair_status &&
|
|
518
|
+
Exchange.ws.market.pair_status[pair] === 'opened' &&
|
|
519
|
+
Exchange.ws.market.market_history_dict &&
|
|
520
|
+
Exchange.ws.market.market_history_dict[pair] &&
|
|
521
|
+
Exchange.ws.market.market_history_dict[pair].length >= 0
|
|
522
|
+
)
|
|
523
|
+
|
|
524
|
+
if (!pair_ready) {
|
|
525
|
+
// Pair not ready yet - mark as failed, will retry later or fall back to REST
|
|
526
|
+
console.log(`[WebSocket Market History] ${Exchange_name} ${pair}: Not ready yet (status: ${Exchange.ws.market.pair_status?.[pair] || 'unknown'})`)
|
|
527
|
+
rate_resolve({ pair, failed: true })
|
|
528
|
+
return
|
|
529
|
+
}
|
|
530
|
+
|
|
168
531
|
Exchange.market_history_ws(pair, timeframe_in_ms, function (res) {
|
|
169
532
|
if (res.success) {
|
|
170
|
-
|
|
171
|
-
|
|
533
|
+
// Success - reset failure count
|
|
534
|
+
reset_ws_failure(ws_market_history_failed_pairs, Exchange_name, pair)
|
|
535
|
+
set_pair_data(exchange_pair_market_history, pair_exchange_market_history, Exchange_name, pair, res.body)
|
|
536
|
+
rate_resolve({ pair, failed: false })
|
|
172
537
|
} else {
|
|
173
538
|
console.error(`getting market history ws failed for ${Exchange_name} ${pair}:`, res)
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
}
|
|
177
|
-
if (pair_exchange_market_history[pair]) {
|
|
178
|
-
delete pair_exchange_market_history[pair][Exchange_name]
|
|
179
|
-
}
|
|
539
|
+
delete_pair_data(exchange_pair_market_history, pair_exchange_market_history, Exchange_name, pair)
|
|
540
|
+
track_ws_failure(ws_market_history_failed_pairs, Exchange_name, pair)
|
|
541
|
+
rate_resolve({ pair, failed: true })
|
|
180
542
|
}
|
|
181
|
-
rate_resolve()
|
|
182
543
|
})
|
|
183
544
|
})
|
|
184
545
|
})
|
|
185
546
|
Promise.all(market_history_promises)
|
|
186
|
-
.then(() =>
|
|
547
|
+
.then((results) => {
|
|
548
|
+
// Check which pairs failed and fall back to REST for those pairs only
|
|
549
|
+
const failed_pairs = results.filter((r) => r && r.failed).map((r) => r.pair)
|
|
550
|
+
if (failed_pairs.length > 0 && Exchange.market_history) {
|
|
551
|
+
console.log(`[WebSocket Fallback] ${Exchange_name} market history: Falling back to REST for ${failed_pairs.length} pair(s): ${failed_pairs.join(', ')}`)
|
|
552
|
+
const rest_promises = failed_pairs.map((pair) => {
|
|
553
|
+
return new Promise((rest_resolve) => {
|
|
554
|
+
Exchange.market_history(pair, timeframe_in_ms, function (res) {
|
|
555
|
+
if (res.success) {
|
|
556
|
+
set_pair_data(exchange_pair_market_history, pair_exchange_market_history, Exchange_name, pair, res.body)
|
|
557
|
+
console.log(`[REST Fallback] ${Exchange_name} ${pair} market history: Successfully fetched via REST`)
|
|
558
|
+
} else {
|
|
559
|
+
console.error(`[REST Fallback] getting market history failed for ${Exchange_name} ${pair}:`, res)
|
|
560
|
+
delete_pair_data(exchange_pair_market_history, pair_exchange_market_history, Exchange_name, pair)
|
|
561
|
+
}
|
|
562
|
+
rest_resolve()
|
|
563
|
+
})
|
|
564
|
+
})
|
|
565
|
+
})
|
|
566
|
+
// Wait for REST fallback to complete before resolving
|
|
567
|
+
return Promise.all(rest_promises)
|
|
568
|
+
.catch((err) => {
|
|
569
|
+
console.error(`[REST Fallback] Error fetching market history for ${Exchange_name}:`, err)
|
|
570
|
+
})
|
|
571
|
+
.then(() => {
|
|
572
|
+
// Resolve after REST fallback completes
|
|
573
|
+
resolve()
|
|
574
|
+
})
|
|
575
|
+
} else {
|
|
576
|
+
resolve()
|
|
577
|
+
}
|
|
578
|
+
})
|
|
187
579
|
.catch((err) => {
|
|
188
580
|
console.error(`Error in market_history_ws for ${Exchange_name}:`, err)
|
|
189
581
|
resolve()
|
|
@@ -194,22 +586,64 @@ const get_exchanges_market_history = function (Exchanges, exchange_pair_dict, ti
|
|
|
194
586
|
}
|
|
195
587
|
check_status()
|
|
196
588
|
})
|
|
197
|
-
|
|
589
|
+
// Clean up promise reference when done (but keep connection open for reuse)
|
|
590
|
+
ws_promise.finally(() => {
|
|
591
|
+
setTimeout(() => {
|
|
592
|
+
if (ws_market_history_promises[ws_key] === ws_promise) {
|
|
593
|
+
delete ws_market_history_promises[ws_key]
|
|
594
|
+
// Also clean up wrapped promise if it exists
|
|
595
|
+
if (ws_market_history_promises_with_fallback[ws_key]) {
|
|
596
|
+
delete ws_market_history_promises_with_fallback[ws_key]
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
}, 1000)
|
|
600
|
+
})
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
// If WebSocket promise rejects (timeout/error), fall back to REST for all pairs
|
|
604
|
+
// Only create wrapper if it doesn't exist (avoid duplicate catch handlers on reused promises)
|
|
605
|
+
if (!ws_promise_with_fallback) {
|
|
606
|
+
ws_promise_with_fallback = ws_promise.catch((err) => {
|
|
607
|
+
console.error(`[WebSocket Failed] ${Exchange_name} market history: Falling back to REST for all pairs. Error:`, err.message)
|
|
608
|
+
// Fall back to REST for all pairs
|
|
609
|
+
if (Exchange.market_history) {
|
|
610
|
+
return Promise.all(
|
|
611
|
+
all_pairs.map((pair) => {
|
|
612
|
+
return new Promise((rest_resolve) => {
|
|
613
|
+
Exchange.market_history(pair, timeframe_in_ms, function (res) {
|
|
614
|
+
if (res.success) {
|
|
615
|
+
set_pair_data(exchange_pair_market_history, pair_exchange_market_history, Exchange_name, pair, res.body)
|
|
616
|
+
console.log(`[REST Fallback] ${Exchange_name} ${pair} market history: Successfully fetched via REST`)
|
|
617
|
+
} else {
|
|
618
|
+
console.error(`[REST Fallback] getting market history failed for ${Exchange_name} ${pair}:`, res)
|
|
619
|
+
delete_pair_data(exchange_pair_market_history, pair_exchange_market_history, Exchange_name, pair)
|
|
620
|
+
}
|
|
621
|
+
rest_resolve()
|
|
622
|
+
})
|
|
623
|
+
})
|
|
624
|
+
})
|
|
625
|
+
).then(() => {
|
|
626
|
+
// Resolve after REST fallback completes
|
|
627
|
+
return Promise.resolve()
|
|
628
|
+
})
|
|
629
|
+
}
|
|
630
|
+
// If no REST fallback available, still resolve to prevent Promise.all from failing
|
|
631
|
+
return Promise.resolve()
|
|
632
|
+
})
|
|
633
|
+
// Store wrapped promise for reuse
|
|
634
|
+
ws_market_history_promises_with_fallback[ws_key] = ws_promise_with_fallback
|
|
635
|
+
}
|
|
636
|
+
promises.push(ws_promise_with_fallback)
|
|
198
637
|
} else if (Exchange.market_history) {
|
|
199
|
-
let
|
|
638
|
+
let pairs_to_fetch = exchange_pair_dict[Exchange_name] || all_pairs
|
|
639
|
+
let market_history_promises = pairs_to_fetch.map((pair) => {
|
|
200
640
|
return new Promise((resolve) => {
|
|
201
641
|
Exchange.market_history(pair, timeframe_in_ms, function (res) {
|
|
202
642
|
if (res.success) {
|
|
203
|
-
|
|
204
|
-
_.set(pair_exchange_market_history, [pair, Exchange_name], res.body)
|
|
643
|
+
set_pair_data(exchange_pair_market_history, pair_exchange_market_history, Exchange_name, pair, res.body)
|
|
205
644
|
} else {
|
|
206
645
|
console.error(`getting market history failed for ${Exchange_name} ${pair}:`, res)
|
|
207
|
-
|
|
208
|
-
delete exchange_pair_market_history[Exchange_name][pair]
|
|
209
|
-
}
|
|
210
|
-
if (pair_exchange_market_history[pair]) {
|
|
211
|
-
delete pair_exchange_market_history[pair][Exchange_name]
|
|
212
|
-
}
|
|
646
|
+
delete_pair_data(exchange_pair_market_history, pair_exchange_market_history, Exchange_name, pair)
|
|
213
647
|
}
|
|
214
648
|
resolve()
|
|
215
649
|
})
|
|
@@ -237,6 +671,8 @@ let checked_ohlcv_exchange = {},
|
|
|
237
671
|
pair_exchange_ohlcv = {}
|
|
238
672
|
const get_exchanges_ohlcv = function (Exchanges, exchange_pair_dict, from_in_ms, to_in_ms, cb) {
|
|
239
673
|
checked_ohlcv_exchange = {}
|
|
674
|
+
ohlcv_status = {}
|
|
675
|
+
ohlcv_status._callback_fired = false // Reset callback flag
|
|
240
676
|
if (!Array.isArray(Exchanges)) {
|
|
241
677
|
console.error('Exchanges should be an array')
|
|
242
678
|
return
|
|
@@ -255,25 +691,23 @@ const get_exchanges_ohlcv = function (Exchanges, exchange_pair_dict, from_in_ms,
|
|
|
255
691
|
if (Exchange.ohlcv) {
|
|
256
692
|
Exchange.ohlcv(pair, from_in_ms, to_in_ms, function (res) {
|
|
257
693
|
if (res.success) {
|
|
258
|
-
|
|
259
|
-
_.set(pair_exchange_ohlcv, [pair, Exchange_name], res.body)
|
|
694
|
+
set_pair_data(exchange_pair_ohlcv, pair_exchange_ohlcv, Exchange_name, pair, res.body)
|
|
260
695
|
} else {
|
|
261
696
|
console.error(`getting ohlcv failed for ${Exchange_name} ${pair}:`, res)
|
|
262
|
-
|
|
263
|
-
delete exchange_pair_ohlcv[Exchange_name][pair]
|
|
264
|
-
}
|
|
265
|
-
if (pair_exchange_ohlcv[pair]) {
|
|
266
|
-
delete pair_exchange_ohlcv[pair][Exchange_name]
|
|
267
|
-
}
|
|
697
|
+
delete_pair_data(exchange_pair_ohlcv, pair_exchange_ohlcv, Exchange_name, pair)
|
|
268
698
|
}
|
|
269
699
|
_.set(ohlcv_status, [Exchange_name + pair], true)
|
|
270
|
-
|
|
700
|
+
// Use a flag to prevent multiple callback calls
|
|
701
|
+
if (!_.includes(_.values(ohlcv_status), false) && !ohlcv_status._callback_fired) {
|
|
702
|
+
ohlcv_status._callback_fired = true
|
|
271
703
|
cb({ exchange_pair_ohlcv, pair_exchange_ohlcv })
|
|
272
704
|
}
|
|
273
705
|
})
|
|
274
706
|
} else {
|
|
275
707
|
_.set(ohlcv_status, [Exchange_name + pair], true)
|
|
276
|
-
|
|
708
|
+
// Use a flag to prevent multiple callback calls
|
|
709
|
+
if (!_.includes(_.values(ohlcv_status), false) && !ohlcv_status._callback_fired) {
|
|
710
|
+
ohlcv_status._callback_fired = true
|
|
277
711
|
cb({ exchange_pair_ohlcv, pair_exchange_ohlcv })
|
|
278
712
|
}
|
|
279
713
|
}
|