@icgio/icg-exchanges-wrapper 1.14.256 → 1.14.257
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -4,7 +4,111 @@ const DEFAULT_ORDERBOOK_LEVEL = 50
|
|
|
4
4
|
|
|
5
5
|
let first_time = {},
|
|
6
6
|
exchange_pair_rates = {},
|
|
7
|
-
pair_exchange_rates = {}
|
|
7
|
+
pair_exchange_rates = {},
|
|
8
|
+
// Track active WebSocket connections per exchange to avoid duplicates
|
|
9
|
+
ws_connection_promises = {},
|
|
10
|
+
ws_connection_pairs = {},
|
|
11
|
+
// Track wrapped promises with fallback handlers to avoid duplicate handlers
|
|
12
|
+
ws_connection_promises_with_fallback = {},
|
|
13
|
+
// Track persistent WebSocket failures per pair to avoid repeated retries
|
|
14
|
+
ws_failed_pairs = {}, // { "ExchangeName": { "PAIR": { count: number, last_failed: timestamp } } }
|
|
15
|
+
WS_FAILURE_THRESHOLD = 3, // Skip WebSocket after 3 consecutive failures
|
|
16
|
+
WS_RETRY_INTERVAL_MS = 5 * 60 * 1000 // Retry WebSocket after 5 minutes
|
|
17
|
+
|
|
18
|
+
// ============ HELPER FUNCTIONS ============
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Group exchanges by name and merge all pairs
|
|
22
|
+
* Returns: { exchange_instances: {}, exchange_all_pairs: {} }
|
|
23
|
+
*/
|
|
24
|
+
function group_exchanges_by_name(Exchanges, exchange_pair_dict) {
|
|
25
|
+
let exchange_instances = {}
|
|
26
|
+
let exchange_all_pairs = {}
|
|
27
|
+
for (let Exchange of Exchanges) {
|
|
28
|
+
let Exchange_name = _.upperFirst(Exchange.name)
|
|
29
|
+
if (!exchange_pair_dict[Exchange_name]) {
|
|
30
|
+
continue
|
|
31
|
+
}
|
|
32
|
+
if (!exchange_instances[Exchange_name]) {
|
|
33
|
+
exchange_instances[Exchange_name] = Exchange
|
|
34
|
+
exchange_all_pairs[Exchange_name] = []
|
|
35
|
+
}
|
|
36
|
+
exchange_all_pairs[Exchange_name] = _.union(exchange_all_pairs[Exchange_name], exchange_pair_dict[Exchange_name])
|
|
37
|
+
}
|
|
38
|
+
return { exchange_instances, exchange_all_pairs }
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Set data in both exchange_pair and pair_exchange structures
|
|
43
|
+
*/
|
|
44
|
+
function set_pair_data(exchange_pair_dict, pair_exchange_dict, exchange_name, pair, data) {
|
|
45
|
+
_.set(exchange_pair_dict, [exchange_name, pair], data)
|
|
46
|
+
_.set(pair_exchange_dict, [pair, exchange_name], data)
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Delete data from both exchange_pair and pair_exchange structures
|
|
51
|
+
*/
|
|
52
|
+
function delete_pair_data(exchange_pair_dict, pair_exchange_dict, exchange_name, pair) {
|
|
53
|
+
if (exchange_pair_dict[exchange_name]) {
|
|
54
|
+
delete exchange_pair_dict[exchange_name][pair]
|
|
55
|
+
}
|
|
56
|
+
if (pair_exchange_dict[pair]) {
|
|
57
|
+
delete pair_exchange_dict[pair][exchange_name]
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Track WebSocket failure for a pair
|
|
63
|
+
*/
|
|
64
|
+
function track_ws_failure(failed_pairs_dict, exchange_name, pair) {
|
|
65
|
+
if (!failed_pairs_dict[exchange_name]) {
|
|
66
|
+
failed_pairs_dict[exchange_name] = {}
|
|
67
|
+
}
|
|
68
|
+
if (!failed_pairs_dict[exchange_name][pair]) {
|
|
69
|
+
failed_pairs_dict[exchange_name][pair] = { count: 0, last_failed: 0 }
|
|
70
|
+
}
|
|
71
|
+
failed_pairs_dict[exchange_name][pair].count++
|
|
72
|
+
failed_pairs_dict[exchange_name][pair].last_failed = Date.now()
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Reset WebSocket failure count for a pair (on success)
|
|
77
|
+
*/
|
|
78
|
+
function reset_ws_failure(failed_pairs_dict, exchange_name, pair) {
|
|
79
|
+
if (failed_pairs_dict[exchange_name] && failed_pairs_dict[exchange_name][pair]) {
|
|
80
|
+
delete failed_pairs_dict[exchange_name][pair]
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Check if pair should skip WebSocket due to persistent failures
|
|
86
|
+
*/
|
|
87
|
+
function should_skip_websocket(failed_pairs_dict, exchange_name, pair) {
|
|
88
|
+
const failure_info = failed_pairs_dict[exchange_name]?.[pair]
|
|
89
|
+
return (
|
|
90
|
+
failure_info &&
|
|
91
|
+
failure_info.count >= WS_FAILURE_THRESHOLD &&
|
|
92
|
+
(Date.now() - failure_info.last_failed) < WS_RETRY_INTERVAL_MS
|
|
93
|
+
)
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Create WebSocket timeout checker
|
|
98
|
+
*/
|
|
99
|
+
function create_ws_timeout_checker(Exchange_name, timeout_ms, max_checks, on_timeout) {
|
|
100
|
+
const start_time = Date.now()
|
|
101
|
+
let check_count = 0
|
|
102
|
+
return () => {
|
|
103
|
+
check_count++
|
|
104
|
+
const elapsed = Date.now() - start_time
|
|
105
|
+
if (elapsed > timeout_ms || check_count > max_checks) {
|
|
106
|
+
on_timeout(elapsed)
|
|
107
|
+
return true
|
|
108
|
+
}
|
|
109
|
+
return false
|
|
110
|
+
}
|
|
111
|
+
}
|
|
8
112
|
|
|
9
113
|
const get_exchanges_rates = function (Exchanges, exchange_pair_dict, cb, ws_enabled = true) {
|
|
10
114
|
let checked_rates = {}
|
|
@@ -14,38 +118,76 @@ const get_exchanges_rates = function (Exchanges, exchange_pair_dict, cb, ws_enab
|
|
|
14
118
|
console.error('Exchanges should be an array')
|
|
15
119
|
return
|
|
16
120
|
} else {
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
121
|
+
// First pass: group pairs by exchange name and find the Exchange instance
|
|
122
|
+
// This ensures ALL pairs for each exchange are collected BEFORE creating any WebSocket
|
|
123
|
+
const { exchange_instances, exchange_all_pairs } = group_exchanges_by_name(Exchanges, exchange_pair_dict)
|
|
124
|
+
|
|
125
|
+
// Second pass: process each unique exchange once
|
|
126
|
+
// Now we have all pairs collected, create ONE WebSocket per exchange with ALL pairs
|
|
127
|
+
for (let Exchange_name in exchange_instances) {
|
|
128
|
+
let Exchange = exchange_instances[Exchange_name]
|
|
129
|
+
let all_pairs = exchange_all_pairs[Exchange_name]
|
|
130
|
+
|
|
131
|
+
// Log how many pairs will be used for this exchange
|
|
132
|
+
if (all_pairs.length > 0 && Exchange.ws_market && ws_enabled) {
|
|
133
|
+
console.log(`[WebSocket Rates] ${Exchange_name}: Creating connection with ${all_pairs.length} pair(s): ${all_pairs.join(', ')}`)
|
|
25
134
|
}
|
|
135
|
+
|
|
26
136
|
if (Exchange.ws_market && ws_enabled) {
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
137
|
+
// Check if WebSocket is already opening/opened for this exchange
|
|
138
|
+
const ws_key = Exchange_name
|
|
139
|
+
const ws_status = Exchange.ws?.market?.status
|
|
140
|
+
const is_opening_or_opened = ws_status === 'opening' || ws_status === 'opened'
|
|
141
|
+
|
|
142
|
+
let ws_promise
|
|
143
|
+
let ws_promise_with_fallback
|
|
144
|
+
if (is_opening_or_opened && ws_connection_promises[ws_key]) {
|
|
145
|
+
// Reuse existing WebSocket connection promise
|
|
146
|
+
ws_promise = ws_connection_promises[ws_key]
|
|
147
|
+
// Merge pairs if needed
|
|
148
|
+
const existing_pairs = ws_connection_pairs[ws_key] || []
|
|
149
|
+
const new_pairs = _.difference(all_pairs, existing_pairs)
|
|
150
|
+
if (new_pairs.length > 0) {
|
|
151
|
+
ws_connection_pairs[ws_key] = _.union(existing_pairs, all_pairs)
|
|
152
|
+
// Add new pairs to existing connection if exchange supports it
|
|
153
|
+
// Note: This depends on exchange implementation - some may need to restart
|
|
154
|
+
// For now, we'll wait for existing connection and check if new pairs are already included
|
|
155
|
+
}
|
|
156
|
+
// Reuse wrapped promise with fallback if it exists (avoid duplicate catch handlers)
|
|
157
|
+
if (ws_connection_promises_with_fallback[ws_key]) {
|
|
158
|
+
ws_promise_with_fallback = ws_connection_promises_with_fallback[ws_key]
|
|
32
159
|
}
|
|
160
|
+
} else {
|
|
161
|
+
// Create new WebSocket connection
|
|
162
|
+
ws_connection_pairs[ws_key] = all_pairs
|
|
163
|
+
ws_promise = new Promise((resolve, reject) => {
|
|
164
|
+
if (!Exchange.ws || !Exchange.ws.market) {
|
|
165
|
+
console.error(`${Exchange_name} WebSocket not properly initialized`)
|
|
166
|
+
reject(new Error(`WebSocket not initialized for ${Exchange_name}`))
|
|
167
|
+
return
|
|
168
|
+
}
|
|
33
169
|
|
|
34
|
-
|
|
170
|
+
// Store promise before starting connection
|
|
171
|
+
ws_connection_promises[ws_key] = ws_promise
|
|
172
|
+
|
|
173
|
+
// Create WebSocket with ALL pairs collected in first pass
|
|
174
|
+
// This ensures only ONE WebSocket per exchange, even with multiple clients/pairs
|
|
175
|
+
Exchange.ws_market(all_pairs)
|
|
35
176
|
|
|
36
177
|
const WS_TIMEOUT_MS = 30000 // 30 seconds timeout
|
|
37
|
-
const start_time = Date.now()
|
|
38
|
-
let check_count = 0
|
|
39
178
|
const MAX_CHECKS = 300 // 30 seconds / 100ms
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
// Timeout check
|
|
46
|
-
if (elapsed > WS_TIMEOUT_MS || check_count > MAX_CHECKS) {
|
|
179
|
+
const check_timeout = create_ws_timeout_checker(
|
|
180
|
+
Exchange_name,
|
|
181
|
+
WS_TIMEOUT_MS,
|
|
182
|
+
MAX_CHECKS,
|
|
183
|
+
(elapsed) => {
|
|
47
184
|
console.error(`WebSocket timeout for ${Exchange_name} after ${elapsed}ms`)
|
|
48
185
|
reject(new Error(`WebSocket timeout for ${Exchange_name}`))
|
|
186
|
+
}
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
const check_status = () => {
|
|
190
|
+
if (check_timeout()) {
|
|
49
191
|
return
|
|
50
192
|
}
|
|
51
193
|
|
|
@@ -56,74 +198,167 @@ const get_exchanges_rates = function (Exchanges, exchange_pair_dict, cb, ws_enab
|
|
|
56
198
|
return
|
|
57
199
|
}
|
|
58
200
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
)
|
|
70
|
-
})
|
|
71
|
-
if (all_pairs_opened) {
|
|
72
|
-
let rate_promises = exchange_pair_dict[Exchange_name].map((pair) => {
|
|
201
|
+
// Check only the pairs we need for this call (not all pairs in connection)
|
|
202
|
+
let pairs_to_check = exchange_pair_dict[Exchange_name] || all_pairs
|
|
203
|
+
|
|
204
|
+
// Check if WebSocket connection is at least opening/opened (not closed/stopped)
|
|
205
|
+
const ws_connection_ready = Exchange.ws.market.status === 'opening' || Exchange.ws.market.status === 'opened'
|
|
206
|
+
|
|
207
|
+
if (ws_connection_ready) {
|
|
208
|
+
// Try each pair individually - don't wait for all pairs to be ready
|
|
209
|
+
// This allows us to detect individual pair failures
|
|
210
|
+
let rate_promises = pairs_to_check.map((pair) => {
|
|
73
211
|
return new Promise((rate_resolve) => {
|
|
212
|
+
// Check if this pair should skip WebSocket due to persistent failures
|
|
213
|
+
if (should_skip_websocket(ws_failed_pairs, Exchange_name, pair)) {
|
|
214
|
+
const failure_info = ws_failed_pairs[Exchange_name][pair]
|
|
215
|
+
console.log(`[WebSocket Skip] ${Exchange_name} ${pair}: Skipping WebSocket (${failure_info.count} consecutive failures, last failed ${Math.round((Date.now() - failure_info.last_failed) / 1000)}s ago)`)
|
|
216
|
+
rate_resolve({ pair, failed: true, skip_ws: true })
|
|
217
|
+
return
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// Check if this specific pair is ready before calling rate_ws
|
|
221
|
+
const pair_ready = (
|
|
222
|
+
Exchange.ws.market.pair_status &&
|
|
223
|
+
Exchange.ws.market.pair_status[pair] === 'opened' &&
|
|
224
|
+
Exchange.ws.market.asks_dict &&
|
|
225
|
+
Exchange.ws.market.asks_dict[pair] &&
|
|
226
|
+
Exchange.ws.market.asks_dict[pair].length > 0 &&
|
|
227
|
+
Exchange.ws.market.bids_dict &&
|
|
228
|
+
Exchange.ws.market.bids_dict[pair] &&
|
|
229
|
+
Exchange.ws.market.bids_dict[pair].length > 0
|
|
230
|
+
)
|
|
231
|
+
|
|
232
|
+
if (!pair_ready) {
|
|
233
|
+
// Pair not ready yet - mark as failed, will retry later or fall back to REST
|
|
234
|
+
console.log(`[WebSocket] ${Exchange_name} ${pair}: Not ready yet (status: ${Exchange.ws.market.pair_status?.[pair] || 'unknown'})`)
|
|
235
|
+
rate_resolve({ pair, failed: true })
|
|
236
|
+
return
|
|
237
|
+
}
|
|
238
|
+
|
|
74
239
|
Exchange.rate_ws(pair, function (res) {
|
|
75
240
|
if (res.success) {
|
|
76
|
-
|
|
77
|
-
|
|
241
|
+
// Success - reset failure count
|
|
242
|
+
reset_ws_failure(ws_failed_pairs, Exchange_name, pair)
|
|
243
|
+
set_pair_data(exchange_pair_rates, pair_exchange_rates, Exchange_name, pair, res.body)
|
|
244
|
+
rate_resolve({ pair, failed: false })
|
|
78
245
|
} else {
|
|
79
246
|
console.error(`getting rate ws failed for ${Exchange_name} ${pair}:`, res)
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
}
|
|
83
|
-
if (pair_exchange_rates[pair]) {
|
|
84
|
-
delete pair_exchange_rates[pair][Exchange_name]
|
|
85
|
-
}
|
|
247
|
+
delete_pair_data(exchange_pair_rates, pair_exchange_rates, Exchange_name, pair)
|
|
248
|
+
track_ws_failure(ws_failed_pairs, Exchange_name, pair)
|
|
249
|
+
rate_resolve({ pair, failed: true })
|
|
86
250
|
}
|
|
87
|
-
rate_resolve()
|
|
88
251
|
})
|
|
89
252
|
})
|
|
90
253
|
})
|
|
91
254
|
Promise.all(rate_promises)
|
|
92
|
-
.then(() => {
|
|
93
|
-
//
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
255
|
+
.then((results) => {
|
|
256
|
+
// Check which pairs failed and fall back to REST for those pairs only
|
|
257
|
+
const failed_pairs = results.filter((r) => r && r.failed).map((r) => r.pair)
|
|
258
|
+
if (failed_pairs.length > 0 && Exchange.rate) {
|
|
259
|
+
console.log(`[WebSocket Fallback] ${Exchange_name}: Falling back to REST for ${failed_pairs.length} pair(s): ${failed_pairs.join(', ')}`)
|
|
260
|
+
const rest_promises = failed_pairs.map((pair) => {
|
|
261
|
+
return new Promise((rest_resolve) => {
|
|
262
|
+
Exchange.rate(pair, DEFAULT_ORDERBOOK_LEVEL, function (res) {
|
|
263
|
+
if (res.success) {
|
|
264
|
+
set_pair_data(exchange_pair_rates, pair_exchange_rates, Exchange_name, pair, res.body)
|
|
265
|
+
console.log(`[REST Fallback] ${Exchange_name} ${pair}: Successfully fetched via REST`)
|
|
266
|
+
} else {
|
|
267
|
+
console.error(`[REST Fallback] getting rate failed for ${Exchange_name} ${pair}:`, res)
|
|
268
|
+
delete_pair_data(exchange_pair_rates, pair_exchange_rates, Exchange_name, pair)
|
|
269
|
+
}
|
|
270
|
+
rest_resolve()
|
|
271
|
+
})
|
|
272
|
+
})
|
|
273
|
+
})
|
|
274
|
+
// Wait for REST fallback to complete before resolving
|
|
275
|
+
return Promise.all(rest_promises)
|
|
276
|
+
.catch((err) => {
|
|
277
|
+
console.error(`[REST Fallback] Error fetching rates for ${Exchange_name}:`, err)
|
|
278
|
+
})
|
|
279
|
+
.then(() => {
|
|
280
|
+
// Resolve after REST fallback completes
|
|
281
|
+
resolve()
|
|
282
|
+
})
|
|
283
|
+
} else {
|
|
284
|
+
// if (Exchange.ws.market.end_base) {
|
|
285
|
+
// Exchange.ws.market.end_base(); // Close the connection and prevent reconnection
|
|
286
|
+
// console.log(`${Exchange_name} WebSocket connection closed.`);
|
|
287
|
+
// }
|
|
288
|
+
resolve()
|
|
289
|
+
}
|
|
98
290
|
})
|
|
99
291
|
.catch((err) => {
|
|
100
292
|
console.error(`Error in rate_ws for ${Exchange_name}:`, err)
|
|
101
293
|
resolve()
|
|
102
294
|
})
|
|
103
295
|
} else {
|
|
296
|
+
// WebSocket connection not ready yet, wait and retry
|
|
104
297
|
setTimeout(check_status, 100)
|
|
105
298
|
}
|
|
106
299
|
}
|
|
107
300
|
|
|
108
301
|
check_status()
|
|
109
302
|
})
|
|
303
|
+
// Clean up promise reference when done (but keep connection open for reuse)
|
|
304
|
+
ws_promise.finally(() => {
|
|
305
|
+
// Don't delete immediately - keep for a short time to allow reuse
|
|
306
|
+
setTimeout(() => {
|
|
307
|
+
if (ws_connection_promises[ws_key] === ws_promise) {
|
|
308
|
+
delete ws_connection_promises[ws_key]
|
|
309
|
+
// Also clean up wrapped promise if it exists
|
|
310
|
+
if (ws_connection_promises_with_fallback[ws_key]) {
|
|
311
|
+
delete ws_connection_promises_with_fallback[ws_key]
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
}, 1000)
|
|
315
|
+
})
|
|
316
|
+
}
|
|
110
317
|
|
|
111
|
-
|
|
318
|
+
// Wrap WebSocket promise to handle rejection and fall back to REST
|
|
319
|
+
// Only create wrapper if it doesn't exist (avoid duplicate catch handlers on reused promises)
|
|
320
|
+
if (!ws_promise_with_fallback) {
|
|
321
|
+
ws_promise_with_fallback = ws_promise.catch((err) => {
|
|
322
|
+
console.error(`[WebSocket Failed] ${Exchange_name}: Falling back to REST for all pairs. Error:`, err.message)
|
|
323
|
+
// Fall back to REST for all pairs
|
|
324
|
+
if (Exchange.rate) {
|
|
325
|
+
return Promise.all(
|
|
326
|
+
all_pairs.map((pair) => {
|
|
327
|
+
return new Promise((rest_resolve) => {
|
|
328
|
+
Exchange.rate(pair, DEFAULT_ORDERBOOK_LEVEL, function (res) {
|
|
329
|
+
if (res.success) {
|
|
330
|
+
set_pair_data(exchange_pair_rates, pair_exchange_rates, Exchange_name, pair, res.body)
|
|
331
|
+
console.log(`[REST Fallback] ${Exchange_name} ${pair}: Successfully fetched via REST`)
|
|
332
|
+
} else {
|
|
333
|
+
console.error(`[REST Fallback] getting rate failed for ${Exchange_name} ${pair}:`, res)
|
|
334
|
+
delete_pair_data(exchange_pair_rates, pair_exchange_rates, Exchange_name, pair)
|
|
335
|
+
}
|
|
336
|
+
rest_resolve()
|
|
337
|
+
})
|
|
338
|
+
})
|
|
339
|
+
})
|
|
340
|
+
).then(() => {
|
|
341
|
+
// Resolve after REST fallback completes
|
|
342
|
+
return Promise.resolve()
|
|
343
|
+
})
|
|
344
|
+
}
|
|
345
|
+
// If no REST fallback available, still resolve to prevent Promise.all from failing
|
|
346
|
+
return Promise.resolve()
|
|
347
|
+
})
|
|
348
|
+
// Store wrapped promise for reuse
|
|
349
|
+
ws_connection_promises_with_fallback[ws_key] = ws_promise_with_fallback
|
|
350
|
+
}
|
|
351
|
+
promises.push(ws_promise_with_fallback)
|
|
112
352
|
} else if (Exchange.rate) {
|
|
113
|
-
let
|
|
353
|
+
let pairs_to_fetch = exchange_pair_dict[Exchange_name] || all_pairs
|
|
354
|
+
let rate_promises = pairs_to_fetch.map((pair) => {
|
|
114
355
|
return new Promise((resolve, reject) => {
|
|
115
356
|
Exchange.rate(pair, DEFAULT_ORDERBOOK_LEVEL, function (res) {
|
|
116
357
|
if (res.success) {
|
|
117
|
-
|
|
118
|
-
_.set(pair_exchange_rates, [pair, Exchange_name], res.body)
|
|
358
|
+
set_pair_data(exchange_pair_rates, pair_exchange_rates, Exchange_name, pair, res.body)
|
|
119
359
|
} else {
|
|
120
360
|
console.error(`getting rate failed for ${Exchange_name} ${pair}:`, res)
|
|
121
|
-
|
|
122
|
-
delete exchange_pair_rates[Exchange_name][pair]
|
|
123
|
-
}
|
|
124
|
-
if (pair_exchange_rates[pair]) {
|
|
125
|
-
delete pair_exchange_rates[pair][Exchange_name]
|
|
126
|
-
}
|
|
361
|
+
delete_pair_data(exchange_pair_rates, pair_exchange_rates, Exchange_name, pair)
|
|
127
362
|
}
|
|
128
363
|
resolve()
|
|
129
364
|
})
|
|
@@ -132,23 +367,18 @@ const get_exchanges_rates = function (Exchanges, exchange_pair_dict, cb, ws_enab
|
|
|
132
367
|
|
|
133
368
|
promises = promises.concat(rate_promises) // Add all rate promises
|
|
134
369
|
} else if (Exchange.rate_with_routes) {
|
|
135
|
-
let
|
|
370
|
+
let pairs_to_fetch = exchange_pair_dict[Exchange_name] || all_pairs
|
|
371
|
+
let rate_with_routes_promises = pairs_to_fetch.map((pair) => {
|
|
136
372
|
return new Promise((resolve, reject) => {
|
|
137
373
|
Exchange.rate_with_routes(pair, function (res) {
|
|
138
374
|
if (res.success) {
|
|
139
375
|
let { asks, bids } = res.body
|
|
140
376
|
asks = asks.map((ask) => ask.slice(0, 2))
|
|
141
377
|
bids = bids.map((bid) => bid.slice(0, 2))
|
|
142
|
-
|
|
143
|
-
_.set(pair_exchange_rates, [pair, Exchange_name], { asks, bids })
|
|
378
|
+
set_pair_data(exchange_pair_rates, pair_exchange_rates, Exchange_name, pair, { asks, bids })
|
|
144
379
|
} else {
|
|
145
380
|
console.error(`getting rate with routes failed for ${Exchange_name} ${pair}:`, res)
|
|
146
|
-
|
|
147
|
-
delete exchange_pair_rates[Exchange_name][pair]
|
|
148
|
-
}
|
|
149
|
-
if (pair_exchange_rates[pair]) {
|
|
150
|
-
delete pair_exchange_rates[pair][Exchange_name]
|
|
151
|
-
}
|
|
381
|
+
delete_pair_data(exchange_pair_rates, pair_exchange_rates, Exchange_name, pair)
|
|
152
382
|
}
|
|
153
383
|
resolve()
|
|
154
384
|
})
|
|
@@ -172,7 +402,14 @@ const get_exchanges_rates = function (Exchanges, exchange_pair_dict, cb, ws_enab
|
|
|
172
402
|
}
|
|
173
403
|
|
|
174
404
|
let exchange_pair_market_history = {},
|
|
175
|
-
pair_exchange_market_history = {}
|
|
405
|
+
pair_exchange_market_history = {},
|
|
406
|
+
// Track active WebSocket connections for market history
|
|
407
|
+
ws_market_history_promises = {},
|
|
408
|
+
ws_market_history_pairs = {},
|
|
409
|
+
// Track wrapped promises with fallback handlers to avoid duplicate handlers
|
|
410
|
+
ws_market_history_promises_with_fallback = {},
|
|
411
|
+
// Track persistent WebSocket failures for market history
|
|
412
|
+
ws_market_history_failed_pairs = {} // { "ExchangeName": { "PAIR": { count: number, last_failed: timestamp } } }
|
|
176
413
|
const get_exchanges_market_history = function (Exchanges, exchange_pair_dict, timeframe_in_ms, cb, ws_enabled = true) {
|
|
177
414
|
let checked_market_history = {}
|
|
178
415
|
let promises = []
|
|
@@ -181,41 +418,71 @@ const get_exchanges_market_history = function (Exchanges, exchange_pair_dict, ti
|
|
|
181
418
|
console.error('Exchanges should be an array')
|
|
182
419
|
return
|
|
183
420
|
} else {
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
}
|
|
421
|
+
// First pass: group pairs by exchange name and find the Exchange instance
|
|
422
|
+
const { exchange_instances, exchange_all_pairs } = group_exchanges_by_name(Exchanges, exchange_pair_dict)
|
|
423
|
+
|
|
424
|
+
// Second pass: process each unique exchange once
|
|
425
|
+
for (let Exchange_name in exchange_instances) {
|
|
426
|
+
let Exchange = exchange_instances[Exchange_name]
|
|
427
|
+
let all_pairs = exchange_all_pairs[Exchange_name]
|
|
428
|
+
|
|
193
429
|
if (Exchange.ws_market && ws_enabled) {
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
return
|
|
199
|
-
}
|
|
430
|
+
// Check if WebSocket is already opening/opened for this exchange
|
|
431
|
+
const ws_key = `market_history_${Exchange_name}`
|
|
432
|
+
const ws_status = Exchange.ws?.market?.status
|
|
433
|
+
const is_opening_or_opened = ws_status === 'opening' || ws_status === 'opened'
|
|
200
434
|
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
435
|
+
let ws_promise
|
|
436
|
+
let ws_promise_with_fallback
|
|
437
|
+
if (is_opening_or_opened && ws_market_history_promises[ws_key]) {
|
|
438
|
+
// Reuse existing WebSocket connection promise
|
|
439
|
+
ws_promise = ws_market_history_promises[ws_key]
|
|
440
|
+
// Merge pairs if needed
|
|
441
|
+
const existing_pairs = ws_market_history_pairs[ws_key] || []
|
|
442
|
+
const new_pairs = _.difference(all_pairs, existing_pairs)
|
|
443
|
+
if (new_pairs.length > 0) {
|
|
444
|
+
ws_market_history_pairs[ws_key] = _.union(existing_pairs, all_pairs)
|
|
445
|
+
}
|
|
446
|
+
// Reuse wrapped promise with fallback if it exists (avoid duplicate catch handlers)
|
|
447
|
+
if (ws_market_history_promises_with_fallback[ws_key]) {
|
|
448
|
+
ws_promise_with_fallback = ws_market_history_promises_with_fallback[ws_key]
|
|
204
449
|
}
|
|
450
|
+
} else {
|
|
451
|
+
// Create new WebSocket connection (or use existing one from rates)
|
|
452
|
+
ws_market_history_pairs[ws_key] = all_pairs
|
|
453
|
+
ws_promise = new Promise((resolve, reject) => {
|
|
454
|
+
if (!Exchange.ws || !Exchange.ws.market) {
|
|
455
|
+
console.error(`${Exchange_name} WebSocket not properly initialized`)
|
|
456
|
+
reject(new Error(`WebSocket not initialized for ${Exchange_name}`))
|
|
457
|
+
return
|
|
458
|
+
}
|
|
205
459
|
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
let check_count = 0
|
|
209
|
-
const MAX_CHECKS = 300 // 30 seconds / 100ms
|
|
460
|
+
// Store promise before starting connection
|
|
461
|
+
ws_market_history_promises[ws_key] = ws_promise
|
|
210
462
|
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
463
|
+
// Only start if not already started (reuse connection from rates if available)
|
|
464
|
+
// If starting new, use ALL pairs collected in first pass
|
|
465
|
+
if (!first_time[Exchange_name]) {
|
|
466
|
+
// Create WebSocket with ALL pairs collected in first pass
|
|
467
|
+
// This ensures only ONE WebSocket per exchange, even with multiple clients/pairs
|
|
468
|
+
Exchange.ws_market(all_pairs)
|
|
469
|
+
first_time[Exchange_name] = true
|
|
470
|
+
}
|
|
214
471
|
|
|
215
|
-
|
|
216
|
-
|
|
472
|
+
const WS_TIMEOUT_MS = 30000 // 30 seconds timeout
|
|
473
|
+
const MAX_CHECKS = 300 // 30 seconds / 100ms
|
|
474
|
+
const check_timeout = create_ws_timeout_checker(
|
|
475
|
+
Exchange_name,
|
|
476
|
+
WS_TIMEOUT_MS,
|
|
477
|
+
MAX_CHECKS,
|
|
478
|
+
(elapsed) => {
|
|
217
479
|
console.error(`WebSocket timeout for ${Exchange_name} market history after ${elapsed}ms`)
|
|
218
480
|
reject(new Error(`WebSocket timeout for ${Exchange_name} market history`))
|
|
481
|
+
}
|
|
482
|
+
)
|
|
483
|
+
|
|
484
|
+
const check_status = () => {
|
|
485
|
+
if (check_timeout()) {
|
|
219
486
|
return
|
|
220
487
|
}
|
|
221
488
|
|
|
@@ -226,37 +493,89 @@ const get_exchanges_market_history = function (Exchanges, exchange_pair_dict, ti
|
|
|
226
493
|
return
|
|
227
494
|
}
|
|
228
495
|
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
let market_history_promises = exchange_pair_dict[Exchange_name].map((pair) => {
|
|
496
|
+
// Check only the pairs we need for this call
|
|
497
|
+
let pairs_to_check = exchange_pair_dict[Exchange_name] || all_pairs
|
|
498
|
+
|
|
499
|
+
// Check if WebSocket connection is at least opening/opened (not closed/stopped)
|
|
500
|
+
const ws_connection_ready = Exchange.ws.market.status === 'opening' || Exchange.ws.market.status === 'opened'
|
|
501
|
+
|
|
502
|
+
if (ws_connection_ready) {
|
|
503
|
+
// Try each pair individually - don't wait for all pairs to be ready
|
|
504
|
+
// This allows us to detect individual pair failures
|
|
505
|
+
let market_history_promises = pairs_to_check.map((pair) => {
|
|
240
506
|
return new Promise((rate_resolve) => {
|
|
507
|
+
// Check if this pair should skip WebSocket due to persistent failures
|
|
508
|
+
if (should_skip_websocket(ws_market_history_failed_pairs, Exchange_name, pair)) {
|
|
509
|
+
const failure_info = ws_market_history_failed_pairs[Exchange_name][pair]
|
|
510
|
+
console.log(`[WebSocket Skip] ${Exchange_name} ${pair} market history: Skipping WebSocket (${failure_info.count} consecutive failures, last failed ${Math.round((Date.now() - failure_info.last_failed) / 1000)}s ago)`)
|
|
511
|
+
rate_resolve({ pair, failed: true, skip_ws: true })
|
|
512
|
+
return
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
// Check if this specific pair is ready before calling market_history_ws
|
|
516
|
+
const pair_ready = (
|
|
517
|
+
Exchange.ws.market.pair_status &&
|
|
518
|
+
Exchange.ws.market.pair_status[pair] === 'opened' &&
|
|
519
|
+
Exchange.ws.market.market_history_dict &&
|
|
520
|
+
Exchange.ws.market.market_history_dict[pair] &&
|
|
521
|
+
Exchange.ws.market.market_history_dict[pair].length >= 0
|
|
522
|
+
)
|
|
523
|
+
|
|
524
|
+
if (!pair_ready) {
|
|
525
|
+
// Pair not ready yet - mark as failed, will retry later or fall back to REST
|
|
526
|
+
console.log(`[WebSocket Market History] ${Exchange_name} ${pair}: Not ready yet (status: ${Exchange.ws.market.pair_status?.[pair] || 'unknown'})`)
|
|
527
|
+
rate_resolve({ pair, failed: true })
|
|
528
|
+
return
|
|
529
|
+
}
|
|
530
|
+
|
|
241
531
|
Exchange.market_history_ws(pair, timeframe_in_ms, function (res) {
|
|
242
532
|
if (res.success) {
|
|
243
|
-
|
|
244
|
-
|
|
533
|
+
// Success - reset failure count
|
|
534
|
+
reset_ws_failure(ws_market_history_failed_pairs, Exchange_name, pair)
|
|
535
|
+
set_pair_data(exchange_pair_market_history, pair_exchange_market_history, Exchange_name, pair, res.body)
|
|
536
|
+
rate_resolve({ pair, failed: false })
|
|
245
537
|
} else {
|
|
246
538
|
console.error(`getting market history ws failed for ${Exchange_name} ${pair}:`, res)
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
}
|
|
250
|
-
if (pair_exchange_market_history[pair]) {
|
|
251
|
-
delete pair_exchange_market_history[pair][Exchange_name]
|
|
252
|
-
}
|
|
539
|
+
delete_pair_data(exchange_pair_market_history, pair_exchange_market_history, Exchange_name, pair)
|
|
540
|
+
track_ws_failure(ws_market_history_failed_pairs, Exchange_name, pair)
|
|
541
|
+
rate_resolve({ pair, failed: true })
|
|
253
542
|
}
|
|
254
|
-
rate_resolve()
|
|
255
543
|
})
|
|
256
544
|
})
|
|
257
545
|
})
|
|
258
546
|
Promise.all(market_history_promises)
|
|
259
|
-
.then(() =>
|
|
547
|
+
.then((results) => {
|
|
548
|
+
// Check which pairs failed and fall back to REST for those pairs only
|
|
549
|
+
const failed_pairs = results.filter((r) => r && r.failed).map((r) => r.pair)
|
|
550
|
+
if (failed_pairs.length > 0 && Exchange.market_history) {
|
|
551
|
+
console.log(`[WebSocket Fallback] ${Exchange_name} market history: Falling back to REST for ${failed_pairs.length} pair(s): ${failed_pairs.join(', ')}`)
|
|
552
|
+
const rest_promises = failed_pairs.map((pair) => {
|
|
553
|
+
return new Promise((rest_resolve) => {
|
|
554
|
+
Exchange.market_history(pair, timeframe_in_ms, function (res) {
|
|
555
|
+
if (res.success) {
|
|
556
|
+
set_pair_data(exchange_pair_market_history, pair_exchange_market_history, Exchange_name, pair, res.body)
|
|
557
|
+
console.log(`[REST Fallback] ${Exchange_name} ${pair} market history: Successfully fetched via REST`)
|
|
558
|
+
} else {
|
|
559
|
+
console.error(`[REST Fallback] getting market history failed for ${Exchange_name} ${pair}:`, res)
|
|
560
|
+
delete_pair_data(exchange_pair_market_history, pair_exchange_market_history, Exchange_name, pair)
|
|
561
|
+
}
|
|
562
|
+
rest_resolve()
|
|
563
|
+
})
|
|
564
|
+
})
|
|
565
|
+
})
|
|
566
|
+
// Wait for REST fallback to complete before resolving
|
|
567
|
+
return Promise.all(rest_promises)
|
|
568
|
+
.catch((err) => {
|
|
569
|
+
console.error(`[REST Fallback] Error fetching market history for ${Exchange_name}:`, err)
|
|
570
|
+
})
|
|
571
|
+
.then(() => {
|
|
572
|
+
// Resolve after REST fallback completes
|
|
573
|
+
resolve()
|
|
574
|
+
})
|
|
575
|
+
} else {
|
|
576
|
+
resolve()
|
|
577
|
+
}
|
|
578
|
+
})
|
|
260
579
|
.catch((err) => {
|
|
261
580
|
console.error(`Error in market_history_ws for ${Exchange_name}:`, err)
|
|
262
581
|
resolve()
|
|
@@ -267,22 +586,64 @@ const get_exchanges_market_history = function (Exchanges, exchange_pair_dict, ti
|
|
|
267
586
|
}
|
|
268
587
|
check_status()
|
|
269
588
|
})
|
|
270
|
-
|
|
589
|
+
// Clean up promise reference when done (but keep connection open for reuse)
|
|
590
|
+
ws_promise.finally(() => {
|
|
591
|
+
setTimeout(() => {
|
|
592
|
+
if (ws_market_history_promises[ws_key] === ws_promise) {
|
|
593
|
+
delete ws_market_history_promises[ws_key]
|
|
594
|
+
// Also clean up wrapped promise if it exists
|
|
595
|
+
if (ws_market_history_promises_with_fallback[ws_key]) {
|
|
596
|
+
delete ws_market_history_promises_with_fallback[ws_key]
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
}, 1000)
|
|
600
|
+
})
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
// If WebSocket promise rejects (timeout/error), fall back to REST for all pairs
|
|
604
|
+
// Only create wrapper if it doesn't exist (avoid duplicate catch handlers on reused promises)
|
|
605
|
+
if (!ws_promise_with_fallback) {
|
|
606
|
+
ws_promise_with_fallback = ws_promise.catch((err) => {
|
|
607
|
+
console.error(`[WebSocket Failed] ${Exchange_name} market history: Falling back to REST for all pairs. Error:`, err.message)
|
|
608
|
+
// Fall back to REST for all pairs
|
|
609
|
+
if (Exchange.market_history) {
|
|
610
|
+
return Promise.all(
|
|
611
|
+
all_pairs.map((pair) => {
|
|
612
|
+
return new Promise((rest_resolve) => {
|
|
613
|
+
Exchange.market_history(pair, timeframe_in_ms, function (res) {
|
|
614
|
+
if (res.success) {
|
|
615
|
+
set_pair_data(exchange_pair_market_history, pair_exchange_market_history, Exchange_name, pair, res.body)
|
|
616
|
+
console.log(`[REST Fallback] ${Exchange_name} ${pair} market history: Successfully fetched via REST`)
|
|
617
|
+
} else {
|
|
618
|
+
console.error(`[REST Fallback] getting market history failed for ${Exchange_name} ${pair}:`, res)
|
|
619
|
+
delete_pair_data(exchange_pair_market_history, pair_exchange_market_history, Exchange_name, pair)
|
|
620
|
+
}
|
|
621
|
+
rest_resolve()
|
|
622
|
+
})
|
|
623
|
+
})
|
|
624
|
+
})
|
|
625
|
+
).then(() => {
|
|
626
|
+
// Resolve after REST fallback completes
|
|
627
|
+
return Promise.resolve()
|
|
628
|
+
})
|
|
629
|
+
}
|
|
630
|
+
// If no REST fallback available, still resolve to prevent Promise.all from failing
|
|
631
|
+
return Promise.resolve()
|
|
632
|
+
})
|
|
633
|
+
// Store wrapped promise for reuse
|
|
634
|
+
ws_market_history_promises_with_fallback[ws_key] = ws_promise_with_fallback
|
|
635
|
+
}
|
|
636
|
+
promises.push(ws_promise_with_fallback)
|
|
271
637
|
} else if (Exchange.market_history) {
|
|
272
|
-
let
|
|
638
|
+
let pairs_to_fetch = exchange_pair_dict[Exchange_name] || all_pairs
|
|
639
|
+
let market_history_promises = pairs_to_fetch.map((pair) => {
|
|
273
640
|
return new Promise((resolve) => {
|
|
274
641
|
Exchange.market_history(pair, timeframe_in_ms, function (res) {
|
|
275
642
|
if (res.success) {
|
|
276
|
-
|
|
277
|
-
_.set(pair_exchange_market_history, [pair, Exchange_name], res.body)
|
|
643
|
+
set_pair_data(exchange_pair_market_history, pair_exchange_market_history, Exchange_name, pair, res.body)
|
|
278
644
|
} else {
|
|
279
645
|
console.error(`getting market history failed for ${Exchange_name} ${pair}:`, res)
|
|
280
|
-
|
|
281
|
-
delete exchange_pair_market_history[Exchange_name][pair]
|
|
282
|
-
}
|
|
283
|
-
if (pair_exchange_market_history[pair]) {
|
|
284
|
-
delete pair_exchange_market_history[pair][Exchange_name]
|
|
285
|
-
}
|
|
646
|
+
delete_pair_data(exchange_pair_market_history, pair_exchange_market_history, Exchange_name, pair)
|
|
286
647
|
}
|
|
287
648
|
resolve()
|
|
288
649
|
})
|
|
@@ -330,16 +691,10 @@ const get_exchanges_ohlcv = function (Exchanges, exchange_pair_dict, from_in_ms,
|
|
|
330
691
|
if (Exchange.ohlcv) {
|
|
331
692
|
Exchange.ohlcv(pair, from_in_ms, to_in_ms, function (res) {
|
|
332
693
|
if (res.success) {
|
|
333
|
-
|
|
334
|
-
_.set(pair_exchange_ohlcv, [pair, Exchange_name], res.body)
|
|
694
|
+
set_pair_data(exchange_pair_ohlcv, pair_exchange_ohlcv, Exchange_name, pair, res.body)
|
|
335
695
|
} else {
|
|
336
696
|
console.error(`getting ohlcv failed for ${Exchange_name} ${pair}:`, res)
|
|
337
|
-
|
|
338
|
-
delete exchange_pair_ohlcv[Exchange_name][pair]
|
|
339
|
-
}
|
|
340
|
-
if (pair_exchange_ohlcv[pair]) {
|
|
341
|
-
delete pair_exchange_ohlcv[pair][Exchange_name]
|
|
342
|
-
}
|
|
697
|
+
delete_pair_data(exchange_pair_ohlcv, pair_exchange_ohlcv, Exchange_name, pair)
|
|
343
698
|
}
|
|
344
699
|
_.set(ohlcv_status, [Exchange_name + pair], true)
|
|
345
700
|
// Use a flag to prevent multiple callback calls
|