@icgio/icg-exchanges-wrapper 1.21.41 → 1.21.43

package/index.js

@@ -1,19 +1,9 @@
 const { get_exchanges_rates, get_exchanges_market_history, get_exchanges_ohlcv } = require('./functions/get-exchanges-rates-market-history-ohlcv.js')
-const get_exchanges_balance = require('./functions/get-exchanges-balance.js')
-const get_exchanges_trades = require('./functions/get-exchanges-trades.js')
-const get_exchanges_open_orders = require('./functions/get-exchanges-open-orders.js')
-const get_exchanges_deposits = require('./functions/get-exchanges-deposits.js')
-const get_exchanges_withdrawals = require('./functions/get-exchanges-withdrawals.js')
 
 module.exports.functions = {
   get_exchanges_rates,
   get_exchanges_market_history,
   get_exchanges_ohlcv,
-  get_exchanges_balance,
-  get_exchanges_trades,
-  get_exchanges_open_orders,
-  get_exchanges_deposits,
-  get_exchanges_withdrawals,
 }
 
 module.exports.orders = require('./middleware/orders.js')

package/middleware/orders.js

@@ -174,6 +174,7 @@ module.exports = class Orders {
     this.f_filled_threshold = 0.999
     this.previous_orders_canceled = false
     this.previous_orders_canceled_exchange_pair = {}
+    this.previous_orders_cleanup = {}
     this.account_init = false
     this.cmc_curs = settings.cmc_curs || []
     this.accounting_benchmark = settings.accounting_benchmark
@@ -1211,46 +1212,117 @@ module.exports = class Orders {
   }
   check_open_orders() {
     const CANCEL_PREVIOUS_ORDERS_DELAY_MS = 60 * 1000
-    let previous_orders_processer = (exchange, pair, orders) => {
-      // no cancel previous orders
-      if (_.includes(this.no_cancel_previous_exchanges, exchange) && !this.previous_orders_canceled) {
-        _.set(this.previous_orders_canceled_exchange_pair, [exchange, pair], true)
+    let update_previous_orders_canceled = () => {
+      let all_canceled = true
+      outer: for (const exch of Object.keys(this.exchange_pairs)) {
+        const pairs = this.exchange_pairs[exch]
+        const exch_marks = this.previous_orders_canceled_exchange_pair[exch]
+        for (let i = 0; i < pairs.length; i++) {
+          if (!exch_marks || !exch_marks[pairs[i]]) {
+            all_canceled = false
+            break outer
+          }
+        }
       }
-      // cancel previous orders
-      else if (this.cancel_previous_orders && !this.previous_orders_canceled) {
-        if (this.cancel_previous_orders_with_delay) {
-          setTimeout(() => {
-            orders.map((order) => {
-              console.log('cancel original order with delay: %s %s %j', exchange, pair, order)
-              this.Exchanges[exchange].cancel_order_by_order(order, (res) => {})
-            })
-          }, CANCEL_PREVIOUS_ORDERS_DELAY_MS)
-          _.set(this.previous_orders_canceled_exchange_pair, [exchange, pair], true)
-        } else {
-          orders.map((order) => {
-            console.log('cancel original order: %s %s %j', exchange, pair, order)
-            this.Exchanges[exchange].cancel_order_by_order(order, (res) => {})
+      if (all_canceled && !this.previous_orders_canceled) {
+        console.log('original orders cleanup verified')
+      }
+      this.previous_orders_canceled = all_canceled
+    }
+    let complete_previous_orders_cleanup = (exchange, pair, state) => {
+      if (state.complete) return
+      state.complete = true
+      _.set(this.previous_orders_canceled_exchange_pair, [exchange, pair], true)
+      console.log('original orders cleanup verified: %s %s', exchange, pair)
+      update_previous_orders_canceled()
+    }
+    let cancel_previous_orders_for_pair = (exchange, pair, state) => {
+      if (!state.cancel_started || state.complete) return
+      const order_ids = Array.from(state.remaining_ids).filter((order_id) => !state.in_flight_ids.has(order_id))
+      if (order_ids.length === 0) return
+      console.log('canceling original orders: %s %s count=%s remaining=%s', exchange, pair, order_ids.length, state.remaining_ids.size)
+      order_ids.forEach((order_id) => {
+        state.in_flight_ids.add(order_id)
+        this.Exchanges[exchange].cancel_order_by_order(state.orders_by_id.get(order_id), (res) => {
+          state.in_flight_ids.delete(order_id)
+          if (!res || (!res.success && res.error !== 'order-not-open')) {
+            console.error('cancel original order failed: %s %s order_id=%s %j', exchange, pair, order_id, res)
+          }
+        })
+      })
+    }
+    let previous_orders_processer = (exchange, pair, orders) => {
+      if (!this.previous_orders_canceled) {
+        let state = _.get(this.previous_orders_cleanup, [exchange, pair])
+        if (!state && _.includes(this.no_cancel_previous_exchanges, exchange)) {
+          state = { complete: false }
+          _.set(this.previous_orders_cleanup, [exchange, pair], state)
+          complete_previous_orders_cleanup(exchange, pair, state)
+        } else if (!state && this.cancel_previous_orders) {
+          const orders_by_id = new Map()
+          let has_malformed_orders = false
+          orders.forEach((order) => {
+            if (!order || order.order_id == null || order.order_id === '') {
+              has_malformed_orders = true
+              console.error('original order missing order_id: %s %s %j', exchange, pair, order)
+              return
+            }
+            const order_id = String(order.order_id)
+            if (!orders_by_id.has(order_id)) orders_by_id.set(order_id, _.cloneDeep(order))
           })
-          if (orders.length === 0) {
-            _.set(this.previous_orders_canceled_exchange_pair, [exchange, pair], true)
+          state = {
+            orders_by_id,
+            remaining_ids: new Set(orders_by_id.keys()),
+            in_flight_ids: new Set(),
+            unexpected_ids: new Set(),
+            cancel_started: false,
+            has_malformed_orders,
+            complete: false,
           }
-        }
-        if (!this.previous_orders_canceled) {
-          let all_canceled = true
-          outer: for (const exch of Object.keys(this.exchange_pairs)) {
-            const pairs = this.exchange_pairs[exch]
-            const exch_marks = this.previous_orders_canceled_exchange_pair[exch]
-            for (let i = 0; i < pairs.length; i++) {
-              if (!exch_marks || !exch_marks[pairs[i]]) {
-                all_canceled = false
-                break outer
-              }
+          _.set(this.previous_orders_cleanup, [exchange, pair], state)
+          _.set(this.previous_orders_canceled_exchange_pair, [exchange, pair], false)
+          const delay_ms = this.cancel_previous_orders_with_delay ? CANCEL_PREVIOUS_ORDERS_DELAY_MS : 0
+          console.log('original orders captured: %s %s count=%s delay_ms=%s', exchange, pair, state.remaining_ids.size, delay_ms)
+          if (state.remaining_ids.size === 0 && !state.has_malformed_orders) {
+            complete_previous_orders_cleanup(exchange, pair, state)
+          } else {
+            const start_cancel = () => {
+              if (state.complete || state.cancel_started) return
+              state.cancel_started = true
+              cancel_previous_orders_for_pair(exchange, pair, state)
             }
+            if (delay_ms > 0) setTimeout(start_cancel, delay_ms)
+            else start_cancel()
+          }
+        } else if (state && !state.complete && state.remaining_ids) {
+          const live_ids = new Set()
+          state.has_malformed_orders = false
+          orders.forEach((order) => {
+            if (!order || order.order_id == null || order.order_id === '') state.has_malformed_orders = true
+            else live_ids.add(String(order.order_id))
+          })
+          const unexpected_ids = new Set(Array.from(live_ids).filter((order_id) => !state.orders_by_id.has(order_id)))
+          const unexpected_ids_changed =
+            unexpected_ids.size !== state.unexpected_ids.size || Array.from(unexpected_ids).some((order_id) => !state.unexpected_ids.has(order_id))
+          if (unexpected_ids.size > 0 && unexpected_ids_changed) {
+            console.error('unexpected orders during original cleanup: %s %s count=%s order_ids=%j', exchange, pair, unexpected_ids.size, Array.from(unexpected_ids).slice(0, 10))
+          }
+          state.unexpected_ids = unexpected_ids
+          const previous_remaining = state.remaining_ids.size
+          state.remaining_ids.forEach((order_id) => {
+            if (!live_ids.has(order_id)) {
+              state.remaining_ids.delete(order_id)
+              state.in_flight_ids.delete(order_id)
+            }
+          })
+          if (state.remaining_ids.size !== previous_remaining) {
+            console.log('original orders cleanup progress: %s %s remaining=%s', exchange, pair, state.remaining_ids.size)
           }
-          if (all_canceled) {
-            console.log('original order canceled')
+          if (state.remaining_ids.size === 0 && !state.has_malformed_orders && state.unexpected_ids.size === 0) {
+            complete_previous_orders_cleanup(exchange, pair, state)
+          } else if (state.cancel_started) {
+            cancel_previous_orders_for_pair(exchange, pair, state)
           }
-          this.previous_orders_canceled = all_canceled
         }
       }
       let rate_major = _.get(this.rates_combined_major, [pair])

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@icgio/icg-exchanges-wrapper",
-  "version": "1.21.41",
+  "version": "1.21.43",
   "description": "",
   "main": "index.js",
   "scripts": {
@@ -17,7 +17,7 @@
   },
   "homepage": "https://github.com/icgio/icg-exchanges-wrapper#readme",
   "dependencies": {
-    "@icgio/icg-exchanges": "^1.45.28",
+    "@icgio/icg-exchanges": "^1.45.30",
     "@icgio/icg-utils": "^1.9.64",
     "lodash": "^4.17.23",
     "pg": "^8.17.2"

package/functions/get-exchanges-balance.js

@@ -1,52 +0,0 @@
-const _ = require('lodash')
-let balances = {}
-const get_exchanges_balance = function (exchanges, cb) {
-  if (!Array.isArray(exchanges)) {
-    console.error('exchanges should be an array')
-    return
-  }
-
-  const promises = exchanges.map((exchange) => {
-    return new Promise((resolve, reject) => {
-      if (exchange.balance_ws) {
-        if (exchange.ws_status().trading === 'n-opened') exchange.ws_account()
-        exchange.balance_ws(function (res) {
-          if (res.success) {
-            _.set(balances, exchange.settings.id, res.body)
-            resolve()
-          } else {
-            console.error(`getting balance ws failed for ${exchange.name}:`, res)
-            delete balances[exchange.name]
-            resolve()
-          }
-        })
-      } else if (exchange.balance) {
-        exchange.balance(function (res) {
-          if (res.success) {
-            _.set(balances, exchange.settings.id, res.body)
-            resolve()
-          } else {
-            console.error(`getting balance failed for ${exchange.name}:`, res)
-            delete balances[exchange.name]
-            resolve()
-          }
-        })
-      } else {
-        resolve()
-      }
-    })
-  })
-
-  Promise.all(promises)
-    .then(() => {
-      cb(balances)
-    })
-    .catch((error) => {
-      console.error('Error fetching balances:', error)
-      cb({})
-    })
-}
-
-module.exports = {
-  get_exchanges_balance,
-}

package/functions/get-exchanges-deposits.js

@@ -1,85 +0,0 @@
-const _ = require('lodash')
-const TIMEFRAME_IN_THE_PAST = 90 * 24 * 60 * 60 * 1000
-const INVALID_EXCHANGES_LIST = ['Bitforex', 'Thinkbit', 'Kkcoin']
-
-let exchange_pair_deposits = {}
-let deposits_status = {}
-
-function get_exchanges_deposits(exchanges, client_currency, cb) {
-  if (!Array.isArray(exchanges)) {
-    console.error('exchanges should be an array')
-    return
-  }
-
-  const promises = exchanges.map((exchange) => {
-    return new Promise((resolve, reject) => {
-      const exchange_name = exchange.Name
-      const exchange_id = exchange.settings.id
-
-      if (_.includes(INVALID_EXCHANGES_LIST, exchange_name)) {
-        resolve()
-        return
-      }
-
-      let deposits_fetch_promises = []
-
-      if (exchange.get_all_deposits) {
-        deposits_fetch_promises.push(
-          new Promise((resolve_pair, reject_pair) => {
-            exchange.get_all_deposits(TIMEFRAME_IN_THE_PAST, function (res) {
-              if (res.success) {
-                const deposits = JSON.parse(JSON.stringify(res.body))
-                _.set(exchange_pair_deposits, [exchange_id], deposits)
-              } else {
-                console.error(`getting all deposits failed for ${exchange_name}:`, res)
-                delete exchange_pair_deposits[exchange_id]
-              }
-              _.set(deposits_status, [exchange_name], true)
-              resolve_pair()
-            })
-          })
-        )
-      } else if (exchange.get_deposits) {
-        client_currency.forEach((pair) => {
-          deposits_fetch_promises.push(
-            new Promise((resolve_pair, reject_pair) => {
-              exchange.get_deposits(pair, TIMEFRAME_IN_THE_PAST, function (res) {
-                if (res.success) {
-                  const deposits = JSON.parse(JSON.stringify(res.body))
-                  _.set(exchange_pair_deposits, [exchange_id, pair], deposits)
-                } else {
-                  console.error(`getting deposits failed for ${exchange_name} ${pair}:`, res)
-                  delete exchange_pair_deposits[exchange_id][pair]
-                }
-                _.set(deposits_status, [exchange_name + pair], true)
-                resolve_pair()
-              })
-            })
-          )
-        })
-      }
-
-      Promise.all(deposits_fetch_promises)
-        .then(() => {
-          resolve()
-        })
-        .catch((error) => {
-          console.error('Error fetching deposits:', error)
-          resolve()
-        })
-    })
-  })
-
-  Promise.all(promises)
-    .then(() => {
-      cb(exchange_pair_deposits)
-    })
-    .catch((error) => {
-      console.error('Error fetching deposits:', error)
-      cb({})
-    })
-}
-
-module.exports = {
-  get_exchanges_deposits,
-}

package/functions/get-exchanges-open-orders.js

@@ -1,98 +0,0 @@
-const _ = require('lodash')
-
-const INVALID_EXCHANGES_LIST = ['Bitforex', 'Thinkbit', 'Kkcoin']
-
-let exchange_pair_open_orders = {}
-let open_orders_status = {}
-
-const get_exchanges_open_orders = function (exchanges, exchange_pair_dict, cb, ws_enabled = true) {
-  if (!Array.isArray(exchanges)) {
-    console.error('exchanges should be an array')
-    return
-  }
-
-  const promises = exchanges.map((exchange) => {
-    return new Promise((resolve, reject) => {
-      const exchange_name = exchange.Name
-      const exchange_id = exchange.settings.id
-
-      if (!exchange_pair_dict[exchange_name]) {
-        resolve()
-        return
-      }
-
-      const pair_list = exchange_pair_dict[exchange_name]
-
-      for (let pair of pair_list) {
-        _.set(open_orders_status, [exchange_name + pair], false)
-      }
-
-      if (_.includes(INVALID_EXCHANGES_LIST, exchange_name)) {
-        resolve()
-        return
-      }
-
-      let open_orders_fetch_promises = []
-
-      if (exchange.get_all_open_orders) {
-        open_orders_fetch_promises.push(
-          new Promise((resolvePair, rejectPair) => {
-            exchange.get_all_open_orders(function (res) {
-              if (res.success) {
-                const open_orders = res.body
-                _.set(exchange_pair_open_orders, [exchange_id], open_orders)
-              } else {
-                console.error(`getting open orders failed for ${exchange_name}:`, res)
-                delete exchange_pair_open_orders[exchange_id]
-              }
-              _.set(open_orders_status, [exchange_name], true)
-              resolvePair()
-            })
-          })
-        )
-      } else if (exchange.get_open_orders) {
-        pair_list.forEach((pair) => {
-          open_orders_fetch_promises.push(
-            new Promise((resolvePair, rejectPair) => {
-              exchange.get_open_orders(pair, function (res) {
-                if (res.success) {
-                  const open_orders = res.body
-                  _.set(exchange_pair_open_orders, [exchange_id, pair], open_orders)
-                } else {
-                  console.error(`getting open orders failed for ${exchange_name} ${pair}:`, res)
-                  if (exchange_pair_open_orders[exchange_id]) {
-                    delete exchange_pair_open_orders[exchange_id][pair]
-                  }
-                }
-                _.set(open_orders_status, [exchange_name + pair], true)
-                resolvePair()
-              })
-            })
-          )
-        })
-      }
-
-      Promise.all(open_orders_fetch_promises)
-        .then(() => {
-          resolve()
-        })
-        .catch((error) => {
-          console.error('Error fetching open orders:', error)
-          resolve()
-        })
-    })
-  })
-
-  Promise.all(promises)
-    .then(() => {
-      cb(exchange_pair_open_orders)
-    })
-    .catch((error) => {
-      console.error('Error fetching open orders:', error)
-      cb({})
-    })
-}
-
-module.exports = {
-  get_exchanges_open_orders,
-}

package/functions/get-exchanges-trades.js

@@ -1,101 +0,0 @@
-const _ = require('lodash')
-
-const INVALID_EXCHANGES_LIST = ['Bitforex', 'Thinkbit', 'Kkcoin']
-const TRADES_TIMEFRAME = 1000000 * 60 * 1000
-
-let exchange_pair_trades = {}
-let trades_status = {}
-
-const get_exchanges_trades = function (exchanges, exchange_pair_dict, cb, ws_enabled = true) {
-  if (!Array.isArray(exchanges)) {
-    console.error('exchanges should be an array')
-    return
-  }
-
-  const promises = exchanges.map((exchange) => {
-    return new Promise((resolve, reject) => {
-      const exchange_name = exchange.Name
-      const exchange_id = exchange.settings.id
-
-      if (!exchange_pair_dict[exchange_name]) {
-        resolve()
-        return
-      }
-
-      const pair_list = exchange_pair_dict[exchange_name]
-
-      for (let pair of pair_list) {
-        _.set(trades_status, [exchange_name + pair], false)
-      }
-
-      if (_.includes(INVALID_EXCHANGES_LIST, exchange_name)) {
-        resolve()
-        return
-      }
-
-      let trades_fetch_promises = []
-
-      if (exchange.get_all_trades) {
-        trades_fetch_promises.push(
-          new Promise((resolve_pair, reject_pair) => {
-            exchange.get_all_trades(pair, TRADES_TIMEFRAME, function (res) {
-              if (res.success) {
-                const trades = res.body
-                _.set(exchange_pair_trades, [exchange_id, pair], trades)
-              } else {
-                console.error(`getting all trades failed for ${exchange_name} ${pair}:`, res)
-                if (exchange_pair_trades[exchange_id]) {
-                  delete exchange_pair_trades[exchange_id][pair]
-                }
-              }
-              _.set(trades_status, [exchange_name + pair], true)
-              resolve_pair()
-            })
-          })
-        )
-      } else if (exchange.get_trades) {
-        pair_list.forEach((pair) => {
-          trades_fetch_promises.push(
-            new Promise((resolve_pair, reject_pair) => {
-              exchange.get_trades(pair, TRADES_TIMEFRAME, function (res) {
-                if (res.success) {
-                  const trades = res.body
-                  _.set(exchange_pair_trades, [exchange_id, pair], trades)
-                } else {
-                  console.error(`getting trades failed for ${exchange_name} ${pair}:`, res)
-                  if (exchange_pair_trades[exchange_id]) {
-                    delete exchange_pair_trades[exchange_id][pair]
-                  }
-                }
-                _.set(trades_status, [exchange_name + pair], true)
-                resolve_pair()
-              })
-            })
-          )
-        })
-      }
-
-      Promise.all(trades_fetch_promises)
-        .then(() => {
-          resolve()
-        })
-        .catch((error) => {
-          console.error('Error fetching trades:', error)
-          resolve()
-        })
-    })
-  })
-
-  Promise.all(promises)
-    .then(() => {
-      cb(exchange_pair_trades)
-    })
-    .catch((error) => {
-      console.error('Error fetching trades:', error)
-      cb({})
-    })
-}
-
-module.exports = {
-  get_exchanges_trades,
-}

package/functions/get-exchanges-withdrawals.js

@@ -1,85 +0,0 @@
-const _ = require('lodash')
-const TIMEFRAME_IN_THE_PAST = 90 * 24 * 60 * 60 * 1000
-const INVALID_EXCHANGES_LIST = ['Bitforex', 'Thinkbit', 'Kkcoin']
-
-let exchange_pair_withdrawals = {}
-let withdrawals_status = {}
-
-const get_exchanges_withdrawals = function (exchanges, client_currency, cb) {
-  if (!Array.isArray(exchanges)) {
-    console.error('exchanges should be an array')
-    return
-  }
-
-  const promises = exchanges.map((exchange) => {
-    return new Promise((resolve, reject) => {
-      const exchange_name = exchange.Name
-      const exchange_id = exchange.settings.id
-
-      if (_.includes(INVALID_EXCHANGES_LIST, exchange_name)) {
-        resolve()
-        return
-      }
-
-      let withdrawals_fetch_promises = []
-
-      if (exchange.get_all_withdrawals) {
-        withdrawals_fetch_promises.push(
-          new Promise((resolve_pair, reject_pair) => {
-            exchange.get_all_withdrawals(TIMEFRAME_IN_THE_PAST, function (res) {
-              if (res.success) {
-                const withdrawals = JSON.parse(JSON.stringify(res.body))
-                _.set(exchange_pair_withdrawals, [exchange_id], withdrawals)
-              } else {
-                console.error(`getting all withdrawals failed for ${exchange_name}:`, res)
-                delete exchange_pair_withdrawals[exchange_id]
-              }
-              _.set(withdrawals_status, [exchange_name], true)
-              resolve_pair()
-            })
-          })
-        )
-      } else if (exchange.get_withdrawals) {
-        client_currency.forEach((pair) => {
-          withdrawals_fetch_promises.push(
-            new Promise((resolve_pair, reject_pair) => {
-              exchange.get_withdrawals(pair, TIMEFRAME_IN_THE_PAST, function (res) {
-                if (res.success) {
-                  const withdrawals = JSON.parse(JSON.stringify(res.body))
-                  _.set(exchange_pair_withdrawals, [exchange_id, pair], withdrawals)
-                } else {
-                  console.error(`getting withdrawals failed for ${exchange_name} ${pair}:`, res)
-                  delete exchange_pair_withdrawals[exchange_id][pair]
-                }
-                _.set(withdrawals_status, [exchange_name + pair], true)
-                resolve_pair()
-              })
-            })
-          )
-        })
-      }
-
-      Promise.all(withdrawals_fetch_promises)
-        .then(() => {
-          resolve()
-        })
-        .catch((error) => {
-          console.error('Error fetching withdrawals:', error)
-          resolve()
-        })
-    })
-  })
-
-  Promise.all(promises)
-    .then(() => {
-      cb(exchange_pair_withdrawals)
-    })
-    .catch((error) => {
-      console.error('Error fetching withdrawals:', error)
-      cb({})
-    })
-}
-
-module.exports = {
-  get_exchanges_withdrawals,
-}