hedgequantx 2.6.161 → 2.6.163

package/src/services/rithmic/latency-tracker.js

@@ -0,0 +1,182 @@
+/**
+ * Rithmic Latency Tracking
+ * @module services/rithmic/latency-tracker
+ * 
+ * High-precision order-to-fill latency tracking.
+ * Uses circular buffer and high-resolution timing.
+ */
+
+/**
+ * Get high-resolution timestamp in nanoseconds
+ * @returns {bigint}
+ */
+const hrNow = () => process.hrtime.bigint();
+
+/**
+ * Convert nanoseconds to milliseconds with precision
+ * @param {bigint} ns 
+ * @returns {number}
+ */
+const nsToMs = (ns) => Number(ns) / 1_000_000;
+
+/**
+ * Latency tracker with circular buffer for O(1) operations
+ */
+const LatencyTracker = {
+  _pending: new Map(), // orderTag -> entryTime (bigint nanoseconds)
+  _samples: null,      // Pre-allocated Float64Array circular buffer
+  _maxSamples: 100,
+  _head: 0,            // Next write position
+  _count: 0,           // Number of valid samples
+  _initialized: false,
+  
+  /** Initialize circular buffer (lazy init) */
+  _init() {
+    if (this._initialized) return;
+    this._samples = new Float64Array(this._maxSamples);
+    this._initialized = true;
+  },
+  
+  /** Record order sent time with high-resolution timestamp */
+  recordEntry(orderTag, entryTimeMs) {
+    this._pending.set(orderTag, hrNow());
+  },
+  
+  /** Record fill received, calculate latency with sub-ms precision */
+  recordFill(orderTag) {
+    const entryTime = this._pending.get(orderTag);
+    if (!entryTime) return null;
+    
+    this._pending.delete(orderTag);
+    const latencyNs = hrNow() - entryTime;
+    const latencyMs = nsToMs(latencyNs);
+    
+    // Store in circular buffer (no shift, O(1))
+    this._init();
+    this._samples[this._head] = latencyMs;
+    this._head = (this._head + 1) % this._maxSamples;
+    if (this._count < this._maxSamples) this._count++;
+    
+    return latencyMs;
+  },
+  
+  /** Get average latency */
+  getAverage() {
+    if (this._count === 0) return null;
+    let sum = 0;
+    for (let i = 0; i < this._count; i++) {
+      sum += this._samples[i];
+    }
+    return sum / this._count;
+  },
+  
+  /** Get min/max/avg stats with high precision */
+  getStats() {
+    if (this._count === 0) {
+      return { min: null, max: null, avg: null, p50: null, p99: null, samples: 0 };
+    }
+    
+    const valid = [];
+    for (let i = 0; i < this._count; i++) {
+      valid.push(this._samples[i]);
+    }
+    valid.sort((a, b) => a - b);
+    
+    const sum = valid.reduce((a, b) => a + b, 0);
+    
+    return {
+      min: valid[0],
+      max: valid[valid.length - 1],
+      avg: sum / valid.length,
+      p50: valid[Math.floor(valid.length * 0.5)],
+      p99: valid[Math.floor(valid.length * 0.99)] || valid[valid.length - 1],
+      samples: this._count,
+    };
+  },
+  
+  /** Get last N latency samples */
+  getRecent(n = 10) {
+    if (this._count === 0) return [];
+    const result = [];
+    const start = this._count < this._maxSamples ? 0 : this._head;
+    for (let i = 0; i < Math.min(n, this._count); i++) {
+      const idx = (start + this._count - 1 - i + this._maxSamples) % this._maxSamples;
+      result.push(this._samples[idx]);
+    }
+    return result;
+  },
+  
+  /** Clear all tracking data */
+  clear() {
+    this._pending.clear();
+    this._head = 0;
+    this._count = 0;
+    if (this._samples) {
+      this._samples.fill(0);
+    }
+  }
+};
+
+/**
+ * Pre-allocated fill info pool for zero-allocation in hot path
+ */
+const FillInfoPool = {
+  _template: {
+    orderTag: null,
+    basketId: null,
+    orderId: null,
+    status: null,
+    symbol: null,
+    exchange: null,
+    accountId: null,
+    fillQuantity: 0,
+    totalFillQuantity: 0,
+    remainingQuantity: 0,
+    avgFillPrice: 0,
+    lastFillPrice: 0,
+    transactionType: 0,
+    orderType: 0,
+    quantity: 0,
+    ssboe: 0,
+    usecs: 0,
+    localTimestamp: 0,
+    roundTripLatencyMs: null,
+  },
+  
+  /** Fill template with notification data */
+  fill(notif, receiveTime, latency) {
+    const o = this._template;
+    o.orderTag = notif.userTag || null;
+    o.basketId = notif.basketId;
+    o.orderId = notif.exchangeOrderId || notif.orderId;
+    o.status = notif.status;
+    o.symbol = notif.symbol;
+    o.exchange = notif.exchange;
+    o.accountId = notif.accountId;
+    o.fillQuantity = notif.totalFillSize || notif.fillQuantity || 0;
+    o.totalFillQuantity = notif.totalFillSize || notif.totalFillQuantity || 0;
+    o.remainingQuantity = notif.totalUnfilledSize || notif.remainingQuantity || 0;
+    o.avgFillPrice = parseFloat(notif.avgFillPrice || 0);
+    o.lastFillPrice = parseFloat(notif.price || notif.fillPrice || 0);
+    o.transactionType = notif.transactionType;
+    o.orderType = notif.priceType || notif.orderType;
+    o.quantity = notif.quantity;
+    o.ssboe = notif.ssboe;
+    o.usecs = notif.usecs;
+    o.localTimestamp = receiveTime;
+    o.roundTripLatencyMs = latency;
+    return o;
+  },
+  
+  /** Create a copy for async operations */
+  clone(fillInfo) {
+    return { ...fillInfo };
+  }
+};
+
+module.exports = {
+  hrNow,
+  nsToMs,
+  LatencyTracker,
+  FillInfoPool,
+};

package/src/services/rithmic/market-data-decoders.js

@@ -0,0 +1,229 @@
+/**
+ * Rithmic Market Data Decoders
+ * @module services/rithmic/market-data-decoders
+ * 
+ * Manual decoders for LastTrade and BestBidOffer messages.
+ * Required because protobufjs can't handle field IDs > 100000
+ */
+
+const { readVarint, readLengthDelimited, skipField } = require('./proto-decoders');
+
+// Rithmic field IDs for LastTrade (from protobuf)
+const LAST_TRADE_FIELDS = {
+  TEMPLATE_ID: 154467,
+  SYMBOL: 110100,
+  EXCHANGE: 110101,
+  TRADE_PRICE: 100006,
+  TRADE_SIZE: 100178,
+  AGGRESSOR: 112003,  // 1=BUY, 2=SELL
+  SSBOE: 150100,
+  USECS: 150101,
+};
+
+// Rithmic field IDs for BestBidOffer (from protobuf)
+const BBO_FIELDS = {
+  TEMPLATE_ID: 154467,
+  SYMBOL: 110100,
+  EXCHANGE: 110101,
+  BID_PRICE: 100022,
+  BID_SIZE: 100030,
+  ASK_PRICE: 100025,
+  ASK_SIZE: 100031,
+  SSBOE: 150100,
+  USECS: 150101,
+};
+
+/**
+ * Manually decode LastTrade message from Rithmic
+ * @param {Buffer} buffer 
+ * @returns {Object}
+ */
+function decodeLastTrade(buffer) {
+  const result = {};
+  let offset = 0;
+
+  while (offset < buffer.length) {
+    try {
+      const [tag, newOffset] = readVarint(buffer, offset);
+      const fieldNumber = tag >>> 3;
+      const wireType = tag & 0x7;
+      offset = newOffset;
+
+      switch (fieldNumber) {
+        case LAST_TRADE_FIELDS.SYMBOL:
+          if (wireType === 2) {
+            const [val, next] = readLengthDelimited(buffer, offset);
+            result.symbol = val;
+            offset = next;
+          } else {
+            offset = skipField(buffer, offset, wireType);
+          }
+          break;
+        case LAST_TRADE_FIELDS.EXCHANGE:
+          if (wireType === 2) {
+            const [val, next] = readLengthDelimited(buffer, offset);
+            result.exchange = val;
+            offset = next;
+          } else {
+            offset = skipField(buffer, offset, wireType);
+          }
+          break;
+        case LAST_TRADE_FIELDS.TRADE_PRICE:
+          if (wireType === 1) {
+            result.tradePrice = buffer.readDoubleLE(offset);
+            offset += 8;
+          } else {
+            offset = skipField(buffer, offset, wireType);
+          }
+          break;
+        case LAST_TRADE_FIELDS.TRADE_SIZE:
+          if (wireType === 0) {
+            const [val, next] = readVarint(buffer, offset);
+            result.tradeSize = val;
+            offset = next;
+          } else {
+            offset = skipField(buffer, offset, wireType);
+          }
+          break;
+        case LAST_TRADE_FIELDS.AGGRESSOR:
+          if (wireType === 0) {
+            const [val, next] = readVarint(buffer, offset);
+            result.aggressor = val;
+            offset = next;
+          } else {
+            offset = skipField(buffer, offset, wireType);
+          }
+          break;
+        case LAST_TRADE_FIELDS.SSBOE:
+          if (wireType === 0) {
+            const [val, next] = readVarint(buffer, offset);
+            result.ssboe = val;
+            offset = next;
+          } else {
+            offset = skipField(buffer, offset, wireType);
+          }
+          break;
+        case LAST_TRADE_FIELDS.USECS:
+          if (wireType === 0) {
+            const [val, next] = readVarint(buffer, offset);
+            result.usecs = val;
+            offset = next;
+          } else {
+            offset = skipField(buffer, offset, wireType);
+          }
+          break;
+        default:
+          offset = skipField(buffer, offset, wireType);
+      }
+    } catch {
+      break;
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Manually decode BestBidOffer message from Rithmic
+ * @param {Buffer} buffer 
+ * @returns {Object}
+ */
+function decodeBestBidOffer(buffer) {
+  const result = {};
+  let offset = 0;
+
+  while (offset < buffer.length) {
+    try {
+      const [tag, newOffset] = readVarint(buffer, offset);
+      const fieldNumber = tag >>> 3;
+      const wireType = tag & 0x7;
+      offset = newOffset;
+
+      switch (fieldNumber) {
+        case BBO_FIELDS.SYMBOL:
+          if (wireType === 2) {
+            const [val, next] = readLengthDelimited(buffer, offset);
+            result.symbol = val;
+            offset = next;
+          } else {
+            offset = skipField(buffer, offset, wireType);
+          }
+          break;
+        case BBO_FIELDS.EXCHANGE:
+          if (wireType === 2) {
+            const [val, next] = readLengthDelimited(buffer, offset);
+            result.exchange = val;
+            offset = next;
+          } else {
+            offset = skipField(buffer, offset, wireType);
+          }
+          break;
+        case BBO_FIELDS.BID_PRICE:
+          if (wireType === 1) {
+            result.bidPrice = buffer.readDoubleLE(offset);
+            offset += 8;
+          } else {
+            offset = skipField(buffer, offset, wireType);
+          }
+          break;
+        case BBO_FIELDS.BID_SIZE:
+          if (wireType === 0) {
+            const [val, next] = readVarint(buffer, offset);
+            result.bidSize = val;
+            offset = next;
+          } else {
+            offset = skipField(buffer, offset, wireType);
+          }
+          break;
+        case BBO_FIELDS.ASK_PRICE:
+          if (wireType === 1) {
+            result.askPrice = buffer.readDoubleLE(offset);
+            offset += 8;
+          } else {
+            offset = skipField(buffer, offset, wireType);
+          }
+          break;
+        case BBO_FIELDS.ASK_SIZE:
+          if (wireType === 0) {
+            const [val, next] = readVarint(buffer, offset);
+            result.askSize = val;
+            offset = next;
+          } else {
+            offset = skipField(buffer, offset, wireType);
+          }
+          break;
+        case BBO_FIELDS.SSBOE:
+          if (wireType === 0) {
+            const [val, next] = readVarint(buffer, offset);
+            result.ssboe = val;
+            offset = next;
+          } else {
+            offset = skipField(buffer, offset, wireType);
+          }
+          break;
+        case BBO_FIELDS.USECS:
+          if (wireType === 0) {
+            const [val, next] = readVarint(buffer, offset);
+            result.usecs = val;
+            offset = next;
+          } else {
+            offset = skipField(buffer, offset, wireType);
+          }
+          break;
+        default:
+          offset = skipField(buffer, offset, wireType);
+      }
+    } catch {
+      break;
+    }
+  }
+
+  return result;
+}
+
+module.exports = {
+  LAST_TRADE_FIELDS,
+  BBO_FIELDS,
+  decodeLastTrade,
+  decodeBestBidOffer,
+};

package/src/services/rithmic/market-data.js

@@ -13,6 +13,7 @@
 
 const EventEmitter = require('events');
 const { logger } = require('../../utils/logger');
+const { decodeLastTrade, decodeBestBidOffer } = require('./market-data-decoders');
 
 const log = logger.scope('RithmicMD');
 
@@ -25,284 +26,6 @@ const TEMPLATE_IDS = {
   BEST_BID_OFFER: 151,
 };
 
-// Rithmic field IDs for LastTrade (from protobuf)
-const LAST_TRADE_FIELDS = {
-  TEMPLATE_ID: 154467,
-  SYMBOL: 110100,
-  EXCHANGE: 110101,
-  TRADE_PRICE: 100006,
-  TRADE_SIZE: 100178,
-  AGGRESSOR: 112003,  // 1=BUY, 2=SELL
-  SSBOE: 150100,
-  USECS: 150101,
-};
-
-// Rithmic field IDs for BestBidOffer (from protobuf)
-const BBO_FIELDS = {
-  TEMPLATE_ID: 154467,
-  SYMBOL: 110100,
-  EXCHANGE: 110101,
-  BID_PRICE: 100022,
-  BID_SIZE: 100030,
-  ASK_PRICE: 100025,
-  ASK_SIZE: 100031,
-  SSBOE: 150100,
-  USECS: 150101,
-};
-
-/**
- * Read a varint from buffer starting at offset
- * Uses BigInt internally to handle large field IDs correctly
- * @param {Buffer} buffer 
- * @param {number} offset 
- * @returns {[number, number]} [value, newOffset]
- */
-function readVarint(buffer, offset) {
-  let result = BigInt(0);
-  let shift = BigInt(0);
-  let pos = offset;
-
-  while (pos < buffer.length) {
-    const byte = buffer[pos++];
-    result |= BigInt(byte & 0x7f) << shift;
-    if ((byte & 0x80) === 0) {
-      return [Number(result), pos];
-    }
-    shift += BigInt(7);
-    if (shift > BigInt(63)) {
-      throw new Error('Varint too large');
-    }
-  }
-
-  throw new Error('Incomplete varint');
-}
-
-/**
- * Read a length-delimited field (string/bytes)
- * @param {Buffer} buffer 
- * @param {number} offset 
- * @returns {[string, number]} [value, newOffset]
- */
-function readLengthDelimited(buffer, offset) {
-  const [length, newOffset] = readVarint(buffer, offset);
-  const value = buffer.slice(newOffset, newOffset + length).toString('utf8');
-  return [value, newOffset + length];
-}
-
-/**
- * Skip a field based on wire type
- * @param {Buffer} buffer 
- * @param {number} offset 
- * @param {number} wireType 
- * @returns {number} newOffset
- */
-function skipField(buffer, offset, wireType) {
-  switch (wireType) {
-    case 0: // Varint
-      const [, newOffset] = readVarint(buffer, offset);
-      return newOffset;
-    case 1: // 64-bit
-      return offset + 8;
-    case 2: // Length-delimited
-      const [length, lenOffset] = readVarint(buffer, offset);
-      return lenOffset + length;
-    case 5: // 32-bit
-      return offset + 4;
-    default:
-      throw new Error(`Unknown wire type: ${wireType}`);
-  }
-}
-
-/**
- * Manually decode LastTrade message from Rithmic
- * Required because protobufjs can't handle field IDs > 100000
- * @param {Buffer} buffer 
- * @returns {Object}
- */
-function decodeLastTrade(buffer) {
-  const result = {};
-  let offset = 0;
-
-  while (offset < buffer.length) {
-    try {
-      const [tag, newOffset] = readVarint(buffer, offset);
-      const fieldNumber = tag >>> 3;
-      const wireType = tag & 0x7;
-      offset = newOffset;
-
-      switch (fieldNumber) {
-        case LAST_TRADE_FIELDS.SYMBOL:
-          if (wireType === 2) {
-            const [val, next] = readLengthDelimited(buffer, offset);
-            result.symbol = val;
-            offset = next;
-          } else {
-            offset = skipField(buffer, offset, wireType);
-          }
-          break;
-        case LAST_TRADE_FIELDS.EXCHANGE:
-          if (wireType === 2) {
-            const [val, next] = readLengthDelimited(buffer, offset);
-            result.exchange = val;
-            offset = next;
-          } else {
-            offset = skipField(buffer, offset, wireType);
-          }
-          break;
-        case LAST_TRADE_FIELDS.TRADE_PRICE:
-          if (wireType === 1) {
-            result.tradePrice = buffer.readDoubleLE(offset);
-            offset += 8;
-          } else {
-            offset = skipField(buffer, offset, wireType);
-          }
-          break;
-        case LAST_TRADE_FIELDS.TRADE_SIZE:
-          if (wireType === 0) {
-            const [val, next] = readVarint(buffer, offset);
-            result.tradeSize = val;
-            offset = next;
-          } else {
-            offset = skipField(buffer, offset, wireType);
-          }
-          break;
-        case LAST_TRADE_FIELDS.AGGRESSOR:
-          if (wireType === 0) {
-            const [val, next] = readVarint(buffer, offset);
-            result.aggressor = val;
-            offset = next;
-          } else {
-            offset = skipField(buffer, offset, wireType);
-          }
-          break;
-        case LAST_TRADE_FIELDS.SSBOE:
-          if (wireType === 0) {
-            const [val, next] = readVarint(buffer, offset);
-            result.ssboe = val;
-            offset = next;
-          } else {
-            offset = skipField(buffer, offset, wireType);
-          }
-          break;
-        case LAST_TRADE_FIELDS.USECS:
-          if (wireType === 0) {
-            const [val, next] = readVarint(buffer, offset);
-            result.usecs = val;
-            offset = next;
-          } else {
-            offset = skipField(buffer, offset, wireType);
-          }
-          break;
-        default:
-          offset = skipField(buffer, offset, wireType);
-      }
-    } catch {
-      break;
-    }
-  }
-
-  return result;
-}
-
-/**
- * Manually decode BestBidOffer message from Rithmic
- * Required because protobufjs can't handle field IDs > 100000
- * @param {Buffer} buffer 
- * @returns {Object}
- */
-function decodeBestBidOffer(buffer) {
-  const result = {};
-  let offset = 0;
-
-  while (offset < buffer.length) {
-    try {
-      const [tag, newOffset] = readVarint(buffer, offset);
-      const fieldNumber = tag >>> 3;
-      const wireType = tag & 0x7;
-      offset = newOffset;
-
-      switch (fieldNumber) {
-        case BBO_FIELDS.SYMBOL:
-          if (wireType === 2) {
-            const [val, next] = readLengthDelimited(buffer, offset);
-            result.symbol = val;
-            offset = next;
-          } else {
-            offset = skipField(buffer, offset, wireType);
-          }
-          break;
-        case BBO_FIELDS.EXCHANGE:
-          if (wireType === 2) {
-            const [val, next] = readLengthDelimited(buffer, offset);
-            result.exchange = val;
-            offset = next;
-          } else {
-            offset = skipField(buffer, offset, wireType);
-          }
-          break;
-        case BBO_FIELDS.BID_PRICE:
-          if (wireType === 1) {
-            result.bidPrice = buffer.readDoubleLE(offset);
-            offset += 8;
-          } else {
-            offset = skipField(buffer, offset, wireType);
-          }
-          break;
-        case BBO_FIELDS.BID_SIZE:
-          if (wireType === 0) {
-            const [val, next] = readVarint(buffer, offset);
-            result.bidSize = val;
-            offset = next;
-          } else {
-            offset = skipField(buffer, offset, wireType);
-          }
-          break;
-        case BBO_FIELDS.ASK_PRICE:
-          if (wireType === 1) {
-            result.askPrice = buffer.readDoubleLE(offset);
-            offset += 8;
-          } else {
-            offset = skipField(buffer, offset, wireType);
-          }
-          break;
-        case BBO_FIELDS.ASK_SIZE:
-          if (wireType === 0) {
-            const [val, next] = readVarint(buffer, offset);
-            result.askSize = val;
-            offset = next;
-          } else {
-            offset = skipField(buffer, offset, wireType);
-          }
-          break;
-        case BBO_FIELDS.SSBOE:
-          if (wireType === 0) {
-            const [val, next] = readVarint(buffer, offset);
-            result.ssboe = val;
-            offset = next;
-          } else {
-            offset = skipField(buffer, offset, wireType);
-          }
-          break;
-        case BBO_FIELDS.USECS:
-          if (wireType === 0) {
-            const [val, next] = readVarint(buffer, offset);
-            result.usecs = val;
-            offset = next;
-          } else {
-            offset = skipField(buffer, offset, wireType);
-          }
-          break;
-        default:
-          offset = skipField(buffer, offset, wireType);
-      }
-    } catch {
-      break;
-    }
-  }
-
-  return result;
-}
-
 /**
  * Rithmic Market Data Feed
  * Provides real-time market data via Rithmic WebSocket connection