@icgio/icg-exchanges-wrapper 1.17.21 → 1.17.23

package/middleware/orders.js

@@ -73,6 +73,7 @@ const MARKET_HISTORY_DEFAULT_MS = 10 * 60 * 1000
 const TRADES_DEFAULT_MS = 7 * 24 * 60 * 60 * 1000
 const NO_CALLBACK_TIMEOUT_MS = 2 * 60 * 1000
 const PROCESS_RATES_THROTTLE_MS = 75
+const MAX_ORDER_TRACE_ENTRIES = 20
 const ORDER_TYPE_PRIORITY = {
   self: 2,
   maker_1: 3,
@@ -89,6 +90,36 @@ const getOrderPriority = (order_type, is_cancel = false) => {
   return is_cancel ? priority - 0.5 : priority
 }
 
+const hrtime_diff = (start, end) => {
+  if (!Array.isArray(start) || !Array.isArray(end)) {
+    return undefined
+  }
+  return (end[0] - start[0]) * 1e9 + (end[1] - start[1])
+}
+
+const is_finite_number = (value) => value != null && Number.isFinite(Number(value))
+
+const normalize_market_ts = (raw_market_ts) => {
+  if (!raw_market_ts || typeof raw_market_ts !== 'object') {
+    return {}
+  }
+  return {
+    ts_ms_venue_event: is_finite_number(raw_market_ts.ts_ms_venue_event) ? Number(raw_market_ts.ts_ms_venue_event) : null,
+    ts_ms_venue_send: is_finite_number(raw_market_ts.ts_ms_venue_send) ? Number(raw_market_ts.ts_ms_venue_send) : null,
+    ts_ms_local_received: is_finite_number(raw_market_ts.ts_ms_local_received) ? Number(raw_market_ts.ts_ms_local_received) : null,
+  }
+}
+
+const public_decision_context = (decision_context) => {
+  if (!decision_context || typeof decision_context !== 'object') {
+    return null
+  }
+  const public_context = _.cloneDeep(decision_context)
+  delete public_context.hr_rate_root
+  delete public_context.hr_decision
+  return public_context
+}
+
 module.exports = class Orders {
   constructor(exchange_cur_quote_cur_dict, cd, settings) {
     let { exchange_cur_quote_cur, exchange_cur_quote_cur_rates_only, pair_map_rates_only } = exchange_cur_quote_cur_dict
@@ -272,60 +303,124 @@ module.exports = class Orders {
       ws_client.set_order_info_ref(this.order_info)
     }
   }
+  build_trace_entry(order, note) {
+    return {
+      ts: Date.now(),
+      status: order.status,
+      note,
+      f_amount: order.f_amount,
+    }
+  }
+  append_trace(order, note) {
+    if (!note || !order) return
+    if (!Array.isArray(order.trace)) {
+      order.trace = []
+    }
+    order.trace.push(this.build_trace_entry(order, note))
+    if (order.trace.length > MAX_ORDER_TRACE_ENTRIES) {
+      order.trace.splice(0, order.trace.length - MAX_ORDER_TRACE_ENTRIES)
+    }
+  }
+  move_order_info_key(exchange, pair, type, from_order_id, to_order_id) {
+    if (!from_order_id || !to_order_id || from_order_id === to_order_id) {
+      return
+    }
+    const from_bucket = this.order_info?.[exchange]?.[pair]?.[type]
+    if (!from_bucket || !from_bucket[from_order_id]) {
+      return
+    }
+    const order = from_bucket[from_order_id]
+    from_bucket[to_order_id] = order
+    delete from_bucket[from_order_id]
+    if (order.order_type) {
+      const by_type = this.order_info_by_type?.[order.order_type]?.[exchange]?.[pair]?.[type]
+      if (by_type) {
+        by_type[to_order_id] = order
+        delete by_type[from_order_id]
+      }
+    }
+  }
   init_account(balances, benchmarks) {
     this.account = new Account(balances, benchmarks)
   }
   update_non_initial_from_balances(balances) {
     this.account?.update_non_initial_from_balances(balances)
   }
-  limit_trade(exchange, pair, method, price, amount, id = uuid(), order_type = '', limiter_on = true, api_key_index, order_options = {}) {
+  limit_trade(exchange, pair, method, price, amount, id = uuid(), order_type = '', limiter_on = true, api_key_index, order_options = {}, decision_context = null) {
     price = round_price(exchange, pair, price)
     amount = round_amount(exchange, pair, amount)
-    // console.log('%s %s %s %s %s %s at %s', limiter_on, order_type, exchange, pair, method, amount, price)
-    this.update_order_info(
-      exchange,
-      pair,
-      method,
-      id,
-      {
-        order_type,
-        status: 'INIT',
-        id,
-        order_id: id,
-        type: method,
-        price,
-        amount,
-        f_amount: 0,
-        f_percentage: 0,
-        t_amount: amount,
-        open_time: new Date(),
-        close_time: null,
-      },
-      order_type + ' init',
-    )
-    let no_callback_timeout = setTimeout(() => {
-      if (_.get(this.order_info, [exchange, pair, method, id, 'status']) === 'INIT') {
-        console.log('no callback: %s %s %s %s %s', exchange, pair, method, id, order_type)
-        this.update_order_info(
-          exchange,
-          pair,
-          method,
-          id,
-          {
-            status: 'N-OPENED-TIMEOUT',
-          },
-          order_type + ' n-opened-timeout-init',
-        )
+    const internal_id = id
+    let no_callback_timeout = null
+    let init_emitted = false
+    const start_no_callback_timeout = () => {
+      if (no_callback_timeout) {
+        clearTimeout(no_callback_timeout)
+      }
+      no_callback_timeout = setTimeout(() => {
+        if (_.get(this.order_info, [exchange, pair, method, internal_id, 'status']) === 'INIT') {
+          console.log('no callback: %s %s %s %s %s', exchange, pair, method, internal_id, order_type)
+          this.update_order_info(
+            exchange,
+            pair,
+            method,
+            internal_id,
+            {
+              status: 'N-OPENED-TIMEOUT',
+            },
+            order_type + ' n-opened-timeout-init',
+          )
+        }
+      }, NO_CALLBACK_TIMEOUT_MS)
+    }
+    const emit_init = ({ t_order_sent } = {}) => {
+      if (init_emitted) {
+        return
       }
-    }, NO_CALLBACK_TIMEOUT_MS)
+      init_emitted = true
+      const init_decision_context = public_decision_context(decision_context)
+      const du_ns_decision_to_send = hrtime_diff(decision_context?.hr_decision, t_order_sent)
+      const du_ns_rate_to_send = hrtime_diff(decision_context?.hr_rate_root, t_order_sent)
+      this.update_order_info(
+        exchange,
+        pair,
+        method,
+        internal_id,
+        {
+          order_type,
+          status: 'INIT',
+          id: internal_id,
+          internal_id,
+          order_id: null,
+          type: method,
+          price,
+          amount,
+          f_amount: 0,
+          f_percentage: 0,
+          t_amount: amount,
+          open_time: Date.now(),
+          close_time: null,
+          decision_context: init_decision_context,
+          du_ns_decision_to_send,
+          du_ns_rate_to_send,
+          event_hrtime: t_order_sent,
+        },
+        order_type + ' init',
+      )
+      start_no_callback_timeout()
+    }
     let limit_trade = (cb) => {
+      const order_options_with_meta = Object.assign({}, order_options, {
+        on_sent: ({ t_order_sent }) => {
+          emit_init({ t_order_sent })
+        },
+      })
       if (limiter_on) {
         if (this.Exchanges[exchange].private_limiter) {
           this.Exchanges[exchange].private_limiter.priority = getOrderPriority(order_type)
         }
-        this.Exchanges[exchange].limit_trade(pair, method, price, amount, order_options, cb)
+        this.Exchanges[exchange].limit_trade(pair, method, price, amount, order_options_with_meta, cb)
       } else {
-        this.Exchanges[exchange].limit_trade_base(pair, method, price, amount, order_options, cb)
+        this.Exchanges[exchange].limit_trade_base(pair, method, price, amount, order_options_with_meta, cb)
       }
     }
     if (api_key_index) {
@@ -334,7 +429,8 @@ module.exports = class Orders {
     limit_trade((res) => {
       if (res.success) {
         let order_id = res.body
-        this.delete_order_info(exchange, pair, method, id, order_type + ' n-filled')
+        emit_init(res.meta || {})
+        this.move_order_info_key(exchange, pair, method, internal_id, order_id)
         this.update_order_info(
           exchange,
           pair,
@@ -343,7 +439,8 @@ module.exports = class Orders {
           {
             order_type,
             status: 'N-FILLED',
-            id,
+            id: internal_id,
+            internal_id,
             order_id,
             type: method,
             price,
@@ -351,33 +448,38 @@ module.exports = class Orders {
             f_amount: 0,
             f_percentage: 0,
             t_amount: amount,
-            open_time: new Date(),
             close_time: null,
+            du_ns_rate_to_ack: hrtime_diff(decision_context?.hr_rate_root, _.get(res, ['meta', 't_ack_recv'])),
+            du_ns_order_rtt: hrtime_diff(_.get(res, ['meta', 't_order_sent']), _.get(res, ['meta', 't_ack_recv'])),
+            event_hrtime: _.get(res, ['meta', 't_ack_recv']),
           },
           order_type + ' n-filled',
         )
       } else if (res.error === 'amount-too-small' || res.error === 'insufficient-funds') {
-        this.delete_order_info(exchange, pair, method, id, order_type + ' limit-trade-' + res.error)
+        this.delete_order_info(exchange, pair, method, internal_id, order_type + ' limit-trade-' + res.error)
       } else if (res.error === 'connection-error') {
         setTimeout(() => {
-          this.delete_order_info(exchange, pair, method, id, order_type + ' limit-trade-' + res.error)
+          this.delete_order_info(exchange, pair, method, internal_id, order_type + ' limit-trade-' + res.error)
         }, this.delete_connection_error_orders_timeout)
       } else {
         if (order_type.startsWith('self') || order_type.startsWith('maker_2')) {
-          this.delete_order_info(exchange, pair, method, id, order_type + ' limit-trade-timeout')
+          this.delete_order_info(exchange, pair, method, internal_id, order_type + ' limit-trade-timeout')
         }
         // else if (_.get(this.order_info, [exchange, pair, method, id, 'status']) === 'INIT') { // make sure no_callback_timeout is NOT running zero
         //   this.update_order_info(exchange, pair, method, id, {
         //     status: 'N-OPENED-TIMEOUT'
         //   }, order_type + ' n-opened-timeout-callback')
         // }
-        console.error('limit trade error: %s %s %s %s %s %s %s %j', order_type, id, exchange, pair, method, price, amount, res)
+        console.error('limit trade error: %s %s %s %s %s %s %s %j', order_type, internal_id, exchange, pair, method, price, amount, res)
+      }
+      if (no_callback_timeout) {
+        clearTimeout(no_callback_timeout)
       }
-      clearTimeout(no_callback_timeout)
     })
   }
-  cancel_order(exchange, pair, order) {
+  cancel_order(exchange, pair, order, decision_context = null) {
     let original_status = order.status
+    const cancel_decision_context = public_decision_context(decision_context) || _.cloneDeep(order.decision_context) || null
     this.update_order_info(
       exchange,
       pair,
@@ -385,6 +487,7 @@ module.exports = class Orders {
       order.order_id,
       {
         status: 'CANCELING',
+        decision_context: cancel_decision_context,
       },
       order.order_type + ' canceling',
     )
@@ -401,7 +504,11 @@ module.exports = class Orders {
           order.order_id,
           {
             status: 'CANCELED',
-            close_time: new Date(),
+            close_time: Date.now(),
+            decision_context: cancel_decision_context,
+            du_ns_cancel_latency: hrtime_diff(_.get(res, ['meta', 't_cancel_sent']), _.get(res, ['meta', 't_cancel_ack'])),
+            du_ns_rate_to_cancel_ack: hrtime_diff(decision_context?.hr_rate_root, _.get(res, ['meta', 't_cancel_ack'])),
+            event_hrtime: _.get(res, ['meta', 't_cancel_ack']),
           },
           order.order_type + ' canceled',
         )
@@ -420,7 +527,7 @@ module.exports = class Orders {
             amount: 0,
             f_amount: order.amount,
             f_percentage: 1,
-            close_time: new Date(),
+            close_time: Date.now(),
           },
           order.order_type + ' cannot-be-canceled',
         )
@@ -552,8 +659,9 @@ module.exports = class Orders {
                 amount: 0,
                 f_amount: trade_amount,
                 f_percentage: trade_amount / order_info.t_amount,
-                close_time: trade.close_time || new Date(),
+                close_time: trade.close_time ? new Date(trade.close_time).getTime() : Date.now(),
                 fees: trade.fees,
+                event_hrtime: process.hrtime(),
               },
               order_info.order_type + ' canceled-order-filled(-more)',
             )
@@ -592,8 +700,9 @@ module.exports = class Orders {
                 amount: order_info.t_amount - trade_amount,
                 f_amount: trade.amount,
                 f_percentage: trade_amount / order_info.t_amount,
-                close_time: trade.close_time || new Date(),
+                close_time: trade.close_time ? new Date(trade.close_time).getTime() : Date.now(),
                 fees: trade.fees,
+                event_hrtime: process.hrtime(),
               },
               order_info.order_type + ' order-f-filled-1',
             )
@@ -678,7 +787,8 @@ module.exports = class Orders {
                     amount: 0,
                     f_amount: order_info_order.t_amount,
                     f_percentage: 1,
-                    close_time: new Date(),
+                    close_time: Date.now(),
+                    event_hrtime: process.hrtime(),
                   },
                   order_info_order.order_type + ' order-f-filled-2',
                 )
@@ -713,7 +823,8 @@ module.exports = class Orders {
                   amount: order.amount,
                   f_amount: order_f_amount,
                   f_percentage: order_f_amount / order_info.t_amount,
-                  close_time: order.close_time || new Date(),
+                  close_time: order.close_time ? new Date(order.close_time).getTime() : Date.now(),
+                  event_hrtime: process.hrtime(),
                 },
                 order_info.order_type + ' order-f-filled-3',
               )
@@ -744,8 +855,8 @@ module.exports = class Orders {
             this.get_order_info_status_all(exchange, pair, 'both', 'INIT').map((init_order) => {
               let init_order_price = round_price(exchange, pair, init_order.price)
               let order_price = round_price(exchange, pair, order.price)
-              let init_order_time = init_order.open_time.getTime()
-              let order_time = order.open_time.getTime()
+              let init_order_time = typeof init_order.open_time === 'number' ? init_order.open_time : new Date(init_order.open_time).getTime()
+              let order_time = typeof order.open_time === 'number' ? order.open_time : new Date(order.open_time).getTime()
               if (
                 init_order.type === order.type &&
                 init_order_price === order_price &&
@@ -753,7 +864,7 @@ module.exports = class Orders {
                 order_time - init_order_time < this.check_init_order_opened_timeout
               ) {
                 // console.log('ORDERS: %s %s %s %s INIT ORDER OPENED', init_order.order_id, order.order_id, exchange, pair)
-                this.delete_order_info(exchange, pair, init_order.type, init_order.order_id, init_order.order_type + ' init-order-opened')
+                this.move_order_info_key(exchange, pair, init_order.type, init_order.internal_id || init_order.id, order.order_id)
                 this.update_order_info(
                   exchange,
                   pair,
@@ -767,8 +878,9 @@ module.exports = class Orders {
                     amount: order.amount,
                     f_amount: 0,
                     t_amount: order.amount,
-                    open_time: order.open_time || new Date(),
+                    open_time: order.open_time ? new Date(order.open_time).getTime() : Date.now(),
                     close_time: null,
+                    event_hrtime: process.hrtime(),
                   },
                   init_order.order_type + ' init-order-opened',
                 )
@@ -788,7 +900,8 @@ module.exports = class Orders {
               order_info.order_id,
               {
                 status: 'CANCELED',
-                close_time: new Date(),
+                close_time: Date.now(),
+                event_hrtime: process.hrtime(),
               },
               (order_info.order_type || '') + ' canceling-canceled-from-open-orders',
             )
@@ -809,6 +922,8 @@ module.exports = class Orders {
     const existing = this.order_info[exchange]?.[pair]?.[type]?.[order_id]
     const old_status = existing && existing.status
     const old_order_type = existing && existing.order_type
+    const event_hrtime = dict.event_hrtime
+    delete dict.event_hrtime
 
     for (let key in dict) {
       if (key === 'f_percentage') {
@@ -822,7 +937,15 @@ module.exports = class Orders {
     }
 
     const order = this.order_info[exchange][pair][type][order_id]
+    if (order.internal_id == null && order.id != null) {
+      order.internal_id = order.id
+    }
+    if (order.trace == null) {
+      order.trace = []
+    }
+    order.lifecycle_seq = existing && Number.isFinite(existing.lifecycle_seq) ? existing.lifecycle_seq + 1 : dict.lifecycle_seq || 1
     order.last_update_ts = Date.now()
+    this.append_trace(order, note)
 
     const new_status = order.status
     if (old_status !== new_status) {
@@ -879,14 +1002,16 @@ module.exports = class Orders {
 
     // Send order update via WebSocket to dashboard backend
     const order_data = _.get(this.order_info, [exchange, pair, type, order_id], {})
-    // Only send N-FILLED and later statuses (skip INIT which has temporary order_id)
-    if (dict.status !== 'INIT' && order_data.status !== 'INIT' && ws_client) {
+    if (ws_client) {
       ws_client.send_order({
         action: 'update',
         exchange,
         pair,
         side: type,
         order_id,
+        internal_id: order_data.internal_id,
+        lifecycle_seq: order_data.lifecycle_seq,
+        event_hrtime,
         data: order_data,
         note,
       })
@@ -909,14 +1034,16 @@ module.exports = class Orders {
     }
     console.log('DELETE %s: %s %s %s %s', note, exchange, pair, type, order_id)
 
-    // Send delete event via WebSocket (only for non-INIT orders)
-    if (order_data.status !== 'INIT' && ws_client) {
+    if (ws_client && order_data.internal_id) {
       ws_client.send_order({
         action: 'delete',
         exchange,
         pair,
         side: type,
         order_id,
+        internal_id: order_data.internal_id,
+        lifecycle_seq: (order_data.lifecycle_seq || 0) + 1,
+        event_hrtime: process.hrtime(),
         data: order_data,
         note,
       })
@@ -1331,7 +1458,7 @@ module.exports = class Orders {
           for (let order_id in order_info[type]) {
             let { open_time } = order_info[type][order_id]
             // console.log(order_id, open_time, open_time < new Date() - 30 * 60 * 1000)
-            if (open_time < new Date() - QUERY_ORDER_TIME_IN_MS) {
+            if (open_time < Date.now() - QUERY_ORDER_TIME_IN_MS) {
               // console.log('query_order_maker_1: %s %s %s', exchange, pair, order_id)
               Exchanges[exchange].query_order(pair, order_id, (res) => {
                 count++
@@ -1406,10 +1533,29 @@ module.exports = class Orders {
     const throttled_process_rates = _.throttle(process_rates, PROCESS_RATES_THROTTLE_MS, { leading: true, trailing: true })
     let rates_handler = (exchange, pair, res) => {
       if (res.success) {
-        let { asks, bids } = res.body
+        const hr_rate_root = process.hrtime()
+        const ts_ms_local_rate_root = Date.now()
+        let { asks, bids, market_ts } = res.body
+        const raw_market_ts = market_ts && typeof market_ts === 'object' ? market_ts : null
+        const normalized_market_ts = normalize_market_ts(raw_market_ts)
+        const prev_local_received = _.get(this.rates, [exchange, pair, 'market_ts', 'ts_ms_local_received'])
+        const du_ms_interarrival_local_received =
+          is_finite_number(prev_local_received) && is_finite_number(normalized_market_ts.ts_ms_local_received)
+            ? Number(normalized_market_ts.ts_ms_local_received) - Number(prev_local_received)
+            : null
         asks = asks.slice(0, RATES_LENGTH_MAX)
         bids = bids.slice(0, RATES_LENGTH_MAX)
-        _.set(this.rates, [exchange, pair], { asks, bids, update_time: new Date() })
+        _.set(this.rates, [exchange, pair], {
+          asks,
+          bids,
+          market_ts: normalized_market_ts,
+          hr_rate_root,
+          hr_recv: Array.isArray(raw_market_ts?.hr_recv) ? raw_market_ts.hr_recv : null,
+          hr_parsed: Array.isArray(raw_market_ts?.hr_parsed) ? raw_market_ts.hr_parsed : null,
+          ts_ms_local_rate_root,
+          du_ms_interarrival_local_received,
+          update_time: new Date(),
+        })
         /*
          * throttle to 20ms
          */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@icgio/icg-exchanges-wrapper",
-  "version": "1.17.21",
+  "version": "1.17.23",
   "description": "",
   "main": "index.js",
   "scripts": {
@@ -17,7 +17,7 @@
   },
   "homepage": "https://github.com/icgio/icg-exchanges-wrapper#readme",
   "dependencies": {
-    "@icgio/icg-exchanges": "^1.40.44",
+    "@icgio/icg-exchanges": "^1.40.46",
     "@icgio/icg-utils": "^1.9.61",
     "lodash": "^4.17.23",
     "pg": "^8.17.2"