@icgio/icg-exchanges 1.40.41 → 1.40.43

package/lib/exchanges/mexc.js

@@ -7,6 +7,7 @@ const needle = require('needle')
 
 const ExchangeBase = require('./exchange-base.js')
 const ExchangeWs = require('./exchange-ws.js')
+const { fetch_split_windows, sort_history_rows } = ExchangeBase.history_utils
 
 const { utils, rate_limiter: Limiter } = require('@icgio/icg-utils')
 
@@ -130,7 +131,7 @@ module.exports = class Mexc extends ExchangeBase {
           return false
         }
       }
-      this.ws.market.on_message((data) => {
+      this.ws.market.on_message((data, t_recv) => {
         if (is_json(data)) {
           try {
             const json = JSON.parse(data.toString())
@@ -151,6 +152,7 @@ module.exports = class Mexc extends ExchangeBase {
         try {
           const message = PushDataV3ApiWrapper.decode(data)
           const obj = PushDataV3ApiWrapper.toObject(message, { enums: String, longs: String })
+          const t_parsed = process.hrtime()
           const pair = post_process_pair(obj.symbol)
           const channel = obj.channel
           if (channel.startsWith('spot@public.limit.depth.v3.api.pb@')) {
@@ -158,7 +160,7 @@ module.exports = class Mexc extends ExchangeBase {
             this.ws.market.pair_status[pair] = 'opened'
             this.ws.market.asks_dict[pair] = asks.map((ask) => [parseFloat(ask.price), parseFloat(ask.quantity)])
             this.ws.market.bids_dict[pair] = bids.map((bid) => [parseFloat(bid.price), parseFloat(bid.quantity)])
-            this.ws.market.ts_dict[pair] = { received_ts: Date.now(), event_ts: parseInt(obj.sendTime) || undefined }
+            this.ws.market.ts_dict[pair] = { received_ts: Date.now(), event_ts: parseInt(obj.sendTime) || undefined, t_recv, t_parsed }
             // Derive BBO from depth data
             if (options.bbo !== false && this.ws.market.asks_dict[pair].length > 0 && this.ws.market.bids_dict[pair].length > 0) {
               this.ws.market.tickers[pair] = {
@@ -515,6 +517,63 @@ module.exports = class Mexc extends ExchangeBase {
     }
     this.private_limiter.process(get_trades_base, pair, timeframe_in_ms, cb)
   }
+  get_history_trades(pair, start_time_in_ms, end_time_in_ms, cb) {
+    const get_history_trades_base = (pair, start_time_in_ms, end_time_in_ms, cb) => {
+      let [api_key, secret_key] = this.api_secret_key
+      const options = get_options(api_key)
+      ;(async () => {
+        const rows = await fetch_split_windows({
+          start_ms: start_time_in_ms,
+          stop_ms: end_time_in_ms,
+          initial_window_ms: Math.max(end_time_in_ms - start_time_in_ms, 60 * 1000),
+          page_limit: 1000,
+          fetch_window: async (window_start_ms, window_stop_ms) =>
+            await new Promise((resolve, reject) => {
+              let params = {
+                symbol: pre_process_pair(pair),
+                startTime: window_start_ms,
+                endTime: window_stop_ms - 1,
+                limit: 1000,
+                timestamp: parseInt(Date.now()),
+              }
+              let signature = get_signature_mexc(secret_key, params)
+              let url = 'https://api.mexc.com/api/v3/myTrades?' + qs.stringify(params) + '&signature=' + signature
+              needle.get(url, options, (err, res, body) => {
+                if (body && Array.isArray(body)) {
+                  resolve(
+                    body.map((trade) => {
+                      let result = {
+                        order_id: trade.orderId != null ? trade.orderId.toString() : '',
+                        pair,
+                        type: trade.isBuyer ? 'buy' : 'sell',
+                        price: parseFloat(trade.price),
+                        amount: parseFloat(trade.qty),
+                        total: parseFloat(trade.quoteQty),
+                        close_time: new Date(parseInt(trade.time)),
+                        fees: {
+                          [post_process_cur(trade.commissionAsset)]: parseFloat(trade.commission),
+                        },
+                        role: trade.isMaker ? 'maker' : 'taker',
+                      }
+                      if (trade.id != null) result.trade_id = trade.id.toString()
+                      return result
+                    }),
+                  )
+                } else if (body) {
+                  reject({ error: error_codes.other, body })
+                } else {
+                  reject({ error: error_codes.timeout })
+                }
+              })
+            }),
+        })
+        cb({ success: true, body: sort_history_rows(rows) })
+      })().catch((error) => {
+        cb({ success: false, error: error?.error || error_codes.other, body: error?.body })
+      })
+    }
+    this.private_limiter.process(get_history_trades_base, pair, start_time_in_ms, end_time_in_ms, cb)
+  }
   get_all_deposits(timeframe_in_ms, cb) {
     let get_all_deposits_base = (timeframe_in_ms, cb) => {
       let [api_key, secret_key] = this.api_secret_key

package/lib/exchanges/okx.js

@@ -217,13 +217,14 @@ module.exports = class Okx extends ExchangeBase {
         this.ws.market.ping('ping')
         start_market_history_socket()
       })
-      this.ws.market.on_message((data) => {
+      this.ws.market.on_message((data, t_recv) => {
         if (data.toString() === 'pong') {
           this.ws.market.last_pong = new Date()
         }
         if (data.toString() !== 'pong') {
           try {
             data = JSON.parse(data)
+            const t_parsed = process.hrtime()
             if (data && data.arg && data.arg.channel === 'books' && data.data && data.data[0]) {
               const action = data.action
               const pair = post_process_pair(data.arg.instId, true)
@@ -262,7 +263,7 @@ module.exports = class Okx extends ExchangeBase {
                 } else {
                 }
               }
-              this.ws.market.ts_dict[pair] = { received_ts: Date.now(), event_ts: parseInt(data.data[0].ts) }
+              this.ws.market.ts_dict[pair] = { received_ts: Date.now(), event_ts: parseInt(data.data[0].ts), t_recv, t_parsed }
             } else if (data && data.data && data.arg && (data.arg.channel === 'trades' || data.arg.channel === 'trades-all')) {
               for (let datum of data.data) {
                 let pair = post_process_pair(datum.instId)
@@ -299,7 +300,7 @@ module.exports = class Okx extends ExchangeBase {
               if (options.depth === false) {
                 this.ws.market.asks_dict[pair] = [[askPrice, askSize]]
                 this.ws.market.bids_dict[pair] = [[bidPrice, bidSize]]
-                this.ws.market.ts_dict[pair] = { received_ts: Date.now(), event_ts: parseInt(bbo.ts) }
+                this.ws.market.ts_dict[pair] = { received_ts: Date.now(), event_ts: parseInt(bbo.ts), t_recv, t_parsed }
                 this.ws.market.pair_status[pair] = 'opened'
               }
               this.ws.bbo_observable.notify()

package/lib/exchanges/phemex.js

@@ -773,8 +773,9 @@ module.exports = class Phemex extends ExchangeBase {
           }),
         )
       })
-      this.ws.market.on_message((data) => {
+      this.ws.market.on_message((data, t_recv) => {
         data = JSON.parse(data)
+        const t_parsed = process.hrtime()
         if (data && data.book && data.type === 'snapshot') {
           let pair = post_process_pair(data.symbol)
           this.ws.market.pair_status[pair] = 'opened'
@@ -784,7 +785,7 @@ module.exports = class Phemex extends ExchangeBase {
           for (let bid of data.book.bids) {
             this.ws.market.bids_dict[pair].insert(bid[0], bid[1])
           }
-          this.ws.market.ts_dict[pair] = { received_ts: Date.now(), event_ts: data.timestamp ? Math.floor(data.timestamp / 1e6) : undefined }
+          this.ws.market.ts_dict[pair] = { received_ts: Date.now(), event_ts: data.timestamp ? Math.floor(data.timestamp / 1e6) : undefined, t_recv, t_parsed }
           {
             const _asks = this.ws.market.asks_dict[pair].entries()
             const _bids = this.ws.market.bids_dict[pair].entries()
@@ -812,7 +813,7 @@ module.exports = class Phemex extends ExchangeBase {
               this.ws.market.bids_dict[pair].insert(parseFloat(bid[0]), parseFloat(bid[1]))
             }
           }
-          this.ws.market.ts_dict[pair] = { received_ts: Date.now(), event_ts: data.timestamp ? Math.floor(data.timestamp / 1e6) : undefined }
+          this.ws.market.ts_dict[pair] = { received_ts: Date.now(), event_ts: data.timestamp ? Math.floor(data.timestamp / 1e6) : undefined, t_recv, t_parsed }
           {
             const _asks = this.ws.market.asks_dict[pair].entries()
             const _bids = this.ws.market.bids_dict[pair].entries()

package/lib/exchanges/swft.js

@@ -81,7 +81,7 @@ module.exports = class Swft extends ExchangeBase {
   // }
   rate(pair, orderbook_depth, cb) {
     let [api_key, secret_key] = this.public_api_secret_key
-    let timestamp = new Date().addHours(8).getTime().toString()
+    let timestamp = (Date.now() + 8 * 3_600_000).toString()
     let params = {
       channelId: api_key,
       depth: '1',
@@ -110,7 +110,7 @@ module.exports = class Swft extends ExchangeBase {
   }
   market_history(pair, timeframe_in_ms, cb) {
     let [api_key, secret_key] = this.public_api_secret_key
-    let timestamp = new Date().addHours(8).getTime().toString()
+    let timestamp = (Date.now() + 8 * 3_600_000).toString()
     let params = {
       channelId: api_key,
       symbol: pre_process_pair(pair),
@@ -166,7 +166,7 @@ module.exports = class Swft extends ExchangeBase {
       return filled_data
     }
     let [api_key, secret_key] = this.public_api_secret_key
-    let timestamp = new Date().addHours(8).getTime().toString()
+    let timestamp = (Date.now() + 8 * 3_600_000).toString()
     let params = {
       channelId: api_key,
       period: '60min',
@@ -206,7 +206,7 @@ module.exports = class Swft extends ExchangeBase {
       return
     }
     let [api_key, secret_key] = this.api_secret_key
-    let timestamp = new Date().addHours(8).getTime().toString()
+    let timestamp = (Date.now() + 8 * 3_600_000).toString()
     let params = {
       amount,
       channelId: api_key,
@@ -241,7 +241,7 @@ module.exports = class Swft extends ExchangeBase {
   }
   balance(cb) {
     let [api_key, secret_key] = this.api_secret_key
-    let timestamp = new Date().addHours(8).getTime().toString()
+    let timestamp = (Date.now() + 8 * 3_600_000).toString()
     let params = {
       channelId: api_key,
       timestamp,
@@ -272,7 +272,7 @@ module.exports = class Swft extends ExchangeBase {
   get_open_orders(pair, cb) {
     let get_open_orders_base = (pair, cb) => {
       let [api_key, secret_key] = this.api_secret_key
-      let timestamp = new Date().addHours(8).getTime().toString()
+      let timestamp = (Date.now() + 8 * 3_600_000).toString()
       let params = {
         channelId: api_key,
         timestamp,
@@ -304,7 +304,7 @@ module.exports = class Swft extends ExchangeBase {
   }
   cancel_order(pair, id, cb) {
     let [api_key, secret_key] = this.api_secret_key
-    let timestamp = new Date().addHours(8).getTime().toString()
+    let timestamp = (Date.now() + 8 * 3_600_000).toString()
     let params = {
       channelId: api_key,
       orderId: id,
@@ -329,7 +329,7 @@ module.exports = class Swft extends ExchangeBase {
   }
   query_order_base(pair, order_id, cb) {
     let [api_key, secret_key] = this.api_secret_key
-    let timestamp = new Date().addHours(8).getTime().toString()
+    let timestamp = (Date.now() + 8 * 3_600_000).toString()
     let params = {
       channelId: api_key,
       orderId: order_id,
@@ -376,7 +376,7 @@ module.exports = class Swft extends ExchangeBase {
   get_trades(pair, timeframe_in_ms, cb) {
     let get_trades_base = (pair, timeframe_in_ms, cb) => {
       let [api_key, secret_key] = this.api_secret_key
-      let timestamp = new Date().addHours(8).getTime().toString()
+      let timestamp = (Date.now() + 8 * 3_600_000).toString()
       let params = {
         channelId: api_key,
         timestamp,
@@ -416,6 +416,9 @@ module.exports = class Swft extends ExchangeBase {
     }
     this.private_limiter.process(get_trades_base, pair, timeframe_in_ms, cb)
   }
+  get_history_trades(pair, start_time_in_ms, end_time_in_ms, cb) {
+    this._get_history_trades_via_timeframe(pair, start_time_in_ms, end_time_in_ms, cb)
+  }
   // get_all_deposits(timeframe_in_ms, cb) {
   //
   // }

package/lib/exchanges/upbit.js

@@ -97,7 +97,7 @@ module.exports = class Upbit extends ExchangeBase {
         let message = JSON.stringify([JSON.stringify(sub_channels).toString()])
         this.ws.market.send(message)
       })
-      this.ws.market.on_message((data) => {
+      this.ws.market.on_message((data, t_recv) => {
         if (data.startsWith('a')) {
           data = _.trim(data, 'a')
           data = JSON.parse(JSON.parse(data)[0])
@@ -107,7 +107,8 @@ module.exports = class Upbit extends ExchangeBase {
           this.ws.market.pair_status[pair] = 'opened'
           this.ws.market.asks_dict[pair] = data.orderbookUnits.map((o) => [parseFloat(o.askPrice), parseFloat(o.askSize)])
           this.ws.market.bids_dict[pair] = data.orderbookUnits.map((o) => [parseFloat(o.bidPrice), parseFloat(o.bidSize)])
-          this.ws.market.ts_dict[pair] = { received_ts: Date.now(), event_ts: data.timestamp }
+          const t_parsed = process.hrtime()
+          this.ws.market.ts_dict[pair] = { received_ts: Date.now(), event_ts: data.timestamp, t_recv, t_parsed }
           const asks = this.ws.market.asks_dict[pair]
           const bids = this.ws.market.bids_dict[pair]
           if (asks && asks.length > 0 && bids && bids.length > 0) {
@@ -374,6 +375,63 @@ module.exports = class Upbit extends ExchangeBase {
     }
     this.private_limiter.process(get_trades_base, pair, timeframe_in_ms, cb)
   }
+  get_history_trades(pair, start_time_in_ms, end_time_in_ms, cb) {
+    let [api_key, secret_key] = this.api_secret_key
+    let { filter_history_in_range, sort_history_rows, split_windows, sleep } = ExchangeBase.history_utils
+    let fetch_page = (params) =>
+      new Promise((resolve, reject) => {
+        let options = get_options(api_key, secret_key, params)
+        let url = 'https://api.upbit.com/v1/orders/closed?' + qs.stringify(params)
+        this.private_limiter.process(needle.get, url, options, (err, res, body) => {
+          if (body && Array.isArray(body)) {
+            resolve(body)
+          } else if (body) {
+            reject({ success: false, error: error_codes.other, body })
+          } else {
+            reject({ success: false, error: error_codes.timeout })
+          }
+        })
+      })
+    ;(async () => {
+      let trades = []
+      let windows = split_windows(start_time_in_ms, end_time_in_ms, 7 * 24 * 60 * 60 * 1000)
+      for (let [window_start_ms, window_stop_ms] of windows) {
+        let page = 1
+        while (true) {
+          let params = {
+            market: pre_process_pair(pair),
+            state: 'done',
+            page,
+            limit: 100,
+            order_by: 'asc',
+            start_time: window_start_ms,
+            end_time: window_stop_ms - 1,
+          }
+          let rows = await fetch_page(params)
+          trades.push(
+            ...rows.map((order) => {
+              let price = parseFloat(order.avg_price) > 0 ? parseFloat(order.avg_price) : parseFloat(order.price)
+              let amount = parseFloat(order.executed_volume)
+              return {
+                order_id: order.uuid,
+                pair,
+                type: order.side === 'bid' ? 'buy' : 'sell',
+                price,
+                amount,
+                total: Number.isFinite(parseFloat(order.executed_funds)) ? parseFloat(order.executed_funds) : price * amount,
+                close_time: new Date(order.done_at || order.created_at),
+              }
+            })
+          )
+          if (rows.length < 100) break
+          page++
+          await sleep(250)
+        }
+      }
+      trades = sort_history_rows(filter_history_in_range(trades, start_time_in_ms, end_time_in_ms))
+      cb({ success: true, body: trades })
+    })().catch((error) => cb(error))
+  }
   get_trades_id(pair, order_id, timeframe_in_ms, cb) {
     let get_trades_id_base = (pair, order_id, timeframe_in_ms, cb) => {
       let params = {

package/lib/exchanges/weex.js

@@ -393,6 +393,9 @@ module.exports = class Weex extends ExchangeBase {
     }
     this.private_limiter.process(get_trades_base, pair, timeframe_in_ms, cb)
   }
+  get_history_trades(pair, start_time_in_ms, end_time_in_ms, cb) {
+    this._get_history_trades_via_timeframe(pair, start_time_in_ms, end_time_in_ms, cb)
+  }
   get_all_deposits(timeframe_in_ms, cb) {
     let get_all_deposits_base = (cb) => {
       let [api_key, secret_key] = this.api_secret_key

package/lib/exchanges/xt.js

@@ -7,6 +7,7 @@ const websocket = require('ws')
 
 const ExchangeBase = require('./exchange-base.js')
 const ExchangeWs = require('./exchange-ws.js')
+const { fetch_split_windows, sort_query_params, sort_history_rows } = ExchangeBase.history_utils
 
 const { error_codes } = require('../error_codes.json')
 const { utils, rate_limiter: Limiter } = require('@icgio/icg-utils')
@@ -121,7 +122,7 @@ module.exports = class Xt extends ExchangeBase {
         })
         this.ws.market.ping('ping')
       })
-      this.ws.market.on_message((data) => {
+      this.ws.market.on_message((data, t_recv) => {
         if (data.toString() === 'pong') {
           this.ws.market.last_pong = new Date()
         } else {
@@ -130,13 +131,14 @@ module.exports = class Xt extends ExchangeBase {
           } catch (err) {
             console.error('xt websocket', data.toString(), err)
           }
+          const t_parsed = process.hrtime()
           let book = data.data
           if (data && data.topic === 'depth' && book) {
             let pair = post_process_pair(data.data.s)
             this.ws.market.pair_status[pair] = 'opened'
             this.ws.market.asks_dict[pair] = data.data.a.map((ask) => [parseFloat(ask[0]), parseFloat(ask[1])])
             this.ws.market.bids_dict[pair] = data.data.b.map((bid) => [parseFloat(bid[0]), parseFloat(bid[1])])
-            this.ws.market.ts_dict[pair] = { received_ts: Date.now() }
+            this.ws.market.ts_dict[pair] = { received_ts: Date.now(), t_recv, t_parsed }
             const asks = this.ws.market.asks_dict[pair]
             const bids = this.ws.market.bids_dict[pair]
             if (asks && asks.length > 0 && bids && bids.length > 0) {
@@ -516,6 +518,66 @@ module.exports = class Xt extends ExchangeBase {
     }
     this.private_limiter.process(get_trades_base, pair, timeframe_in_ms, cb)
   }
+  get_history_trades(pair, start_time_in_ms, end_time_in_ms, cb) {
+    const get_history_trades_base = (pair, start_time_in_ms, end_time_in_ms, cb) => {
+      let [api_key, secret_key] = this.api_secret_key
+      ;(async () => {
+        const rows = await fetch_split_windows({
+          start_ms: start_time_in_ms,
+          stop_ms: end_time_in_ms,
+          initial_window_ms: 4 * 60 * 60 * 1000,
+          page_limit: 100,
+          fetch_window: async (window_start_ms, window_stop_ms) =>
+            await new Promise((resolve, reject) => {
+              const path = '/v4/trade'
+              const params = sort_query_params({
+                bizType: 'SPOT',
+                endTime: window_stop_ms - 1,
+                limit: 100,
+                startTime: window_start_ms,
+                symbol: pre_process_pair(pair),
+              })
+              const url = 'https://sapi.xt.com' + path + '?' + qs.stringify(params)
+              const timestamp = new Date().getTime()
+              const sign = get_signature_xt(api_key, secret_key, 'GET', path, params, '', timestamp)
+              const options = get_options(api_key, timestamp, sign)
+              needle.get(url, options, (err, res, body) => {
+                body = parse_body(body)
+                if (body && body.mc === 'SUCCESS' && body.result && Array.isArray(body.result.items)) {
+                  resolve(
+                    body.result.items.map((item) => {
+                      let trade = {
+                        order_id: item.orderId != null ? item.orderId.toString() : '',
+                        pair,
+                        type: String(item.orderSide || '').toLowerCase(),
+                        price: parseFloat(item.price),
+                        amount: parseFloat(item.quantity),
+                        total: parseFloat(item.quoteQty),
+                        close_time: new Date(item.time),
+                        fees: {
+                          [post_process_cur(item.feeCurrency)]: parseFloat(item.fee),
+                        },
+                        role: String(item.takerMaker || '').toLowerCase(),
+                      }
+                      if (item.tradeId != null) trade.trade_id = item.tradeId.toString()
+                      return trade
+                    }),
+                  )
+                } else if (body) {
+                  reject({ error: error_codes.other, body })
+                } else {
+                  reject({ error: error_codes.timeout })
+                }
+              })
+            }),
+        })
+        cb({ success: true, body: sort_history_rows(rows) })
+      })().catch((error) => {
+        cb({ success: false, error: error?.error || error_codes.other, body: error?.body })
+      })
+    }
+    this.private_limiter.process(get_history_trades_base, pair, start_time_in_ms, end_time_in_ms, cb)
+  }
   get_deposits(cur, timeframe_in_ms, cb) {
     const chains = ['Tron', 'BNB Smart Chain', 'Ethereum']
     let get_deposits_base = (timeframe_in_ms, cb) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@icgio/icg-exchanges",
-  "version": "1.40.41",
+  "version": "1.40.43",
   "description": "icgio exchanges package",
   "main": "./exchanges.js",
   "scripts": {
@@ -18,7 +18,7 @@
   },
   "homepage": "https://github.com/icgio/icg-exchanges#readme",
   "dependencies": {
-    "@icgio/icg-utils": "^1.9.58",
+    "@icgio/icg-utils": "^1.9.60",
     "json-bigint": "^1.0.0",
     "jsonwebtoken": "^9.0.3",
     "lodash": "^4.17.20",