hedgequantx 2.6.163 → 2.7.0

package/src/services/rithmic/protobuf.js

@@ -6,79 +6,313 @@
 const protobuf = require('protobufjs');
 const path = require('path');
 const { PROTO_FILES } = require('./constants');
-const { 
-  readVarint, readLengthDelimited, skipField,
-  decodeAccountPnL, decodeInstrumentPnL, decodeProductCodes, decodeFrontMonthContract 
-} = require('./proto-decoders');
-
-/** High-performance buffer pool for zero-allocation encoding */
-class BufferPool {
-  constructor(poolSize = 16, bufferSize = 512) {
-    this._pool = new Array(poolSize);
-    this._available = new Array(poolSize);
-    this._size = poolSize;
-    this._bufferSize = bufferSize;
-    this._head = 0;
-    this._tail = 0;
-    this._count = poolSize;
-    for (let i = 0; i < poolSize; i++) {
-      this._pool[i] = Buffer.allocUnsafe(bufferSize);
-      this._available[i] = i;
+
+// PnL field IDs (Rithmic uses very large field IDs)
+const PNL_FIELDS = {
+  TEMPLATE_ID: 154467,
+  IS_SNAPSHOT: 110121,
+  FCM_ID: 154013,
+  IB_ID: 154014,
+  ACCOUNT_ID: 154008,
+  ACCOUNT_BALANCE: 156970,
+  CASH_ON_HAND: 156971,
+  MARGIN_BALANCE: 156977,
+  MIN_ACCOUNT_BALANCE: 156968,
+  OPEN_POSITION_PNL: 156961,
+  CLOSED_POSITION_PNL: 156963,
+  DAY_PNL: 157956,
+  DAY_OPEN_PNL: 157954,
+  DAY_CLOSED_PNL: 157955,
+  AVAILABLE_BUYING_POWER: 157015,
+  SSBOE: 150100,
+  USECS: 150101,
+};
+
+// Symbol/Contract field IDs (ResponseProductCodes, ResponseFrontMonthContract)
+const SYMBOL_FIELDS = {
+  TEMPLATE_ID: 154467,
+  RP_CODE: 132766,
+  EXCHANGE: 110101,
+  PRODUCT_CODE: 110102,   // Base symbol (ES, NQ, MNQ)
+  PRODUCT_NAME: 110103,   // Product name
+  SYMBOL: 110100,         // Full contract symbol (ESH26)
+  TRADING_SYMBOL: 157095, // Trading symbol
+  DESCRIPTION: 110114,    // Contract description
+  USER_MSG: 132760,
+};
+
+// Instrument PnL Position Update field IDs
+const INSTRUMENT_PNL_FIELDS = {
+  TEMPLATE_ID: 154467,
+  IS_SNAPSHOT: 110121,
+  FCM_ID: 154013,
+  IB_ID: 154014,
+  ACCOUNT_ID: 154008,
+  SYMBOL: 110100,
+  EXCHANGE: 110101,
+  PRODUCT_CODE: 100749,
+  INSTRUMENT_TYPE: 110116,
+  FILL_BUY_QTY: 154041,
+  FILL_SELL_QTY: 154042,
+  ORDER_BUY_QTY: 154037,
+  ORDER_SELL_QTY: 154038,
+  BUY_QTY: 154260,
+  SELL_QTY: 154261,
+  AVG_OPEN_FILL_PRICE: 154434,
+  DAY_OPEN_PNL: 157954,
+  DAY_CLOSED_PNL: 157955,
+  DAY_PNL: 157956,
+  OPEN_POSITION_PNL: 156961,
+  OPEN_POSITION_QUANTITY: 156962,
+  CLOSED_POSITION_PNL: 156963,
+  CLOSED_POSITION_QUANTITY: 156964,
+  NET_QUANTITY: 156967,
+  SSBOE: 150100,
+  USECS: 150101,
+};
+
+/**
+ * Read a varint from buffer
+ */
+function readVarint(buffer, offset) {
+  let result = BigInt(0);
+  let shift = BigInt(0);
+  let pos = offset;
+
+  while (pos < buffer.length) {
+    const byte = buffer[pos++];
+    result |= BigInt(byte & 0x7f) << shift;
+    if ((byte & 0x80) === 0) {
+      return [Number(result), pos];
+    }
+    shift += BigInt(7);
+    if (shift > BigInt(63)) {
+      throw new Error('Varint too large');
     }
   }
-  
-  acquire() {
-    if (this._count === 0) return Buffer.allocUnsafe(this._bufferSize);
-    const idx = this._available[this._head];
-    this._head = (this._head + 1) % this._size;
-    this._count--;
-    return this._pool[idx];
+  throw new Error('Incomplete varint');
+}
+
+/**
+ * Read a length-delimited field (string/bytes)
+ */
+function readLengthDelimited(buffer, offset) {
+  const [length, newOffset] = readVarint(buffer, offset);
+  const value = buffer.slice(newOffset, newOffset + length).toString('utf8');
+  return [value, newOffset + length];
+}
+
+/**
+ * Skip a field based on wire type
+ */
+function skipField(buffer, offset, wireType) {
+  switch (wireType) {
+    case 0: // Varint
+      const [, newOffset] = readVarint(buffer, offset);
+      return newOffset;
+    case 1: // 64-bit
+      return offset + 8;
+    case 2: // Length-delimited
+      const [length, lenOffset] = readVarint(buffer, offset);
+      return lenOffset + length;
+    case 5: // 32-bit
+      return offset + 4;
+    default:
+      throw new Error(`Unknown wire type: ${wireType}`);
   }
-  
-  release(buffer) {
-    const idx = this._pool.indexOf(buffer);
-    if (idx !== -1 && this._count < this._size) {
-      this._available[this._tail] = idx;
-      this._tail = (this._tail + 1) % this._size;
-      this._count++;
+}
+
+/**
+ * Manually decode AccountPnL from raw bytes
+ */
+function decodeAccountPnL(buffer) {
+  const result = {};
+  let offset = 0;
+
+  while (offset < buffer.length) {
+    try {
+      const [tag, tagOffset] = readVarint(buffer, offset);
+      const wireType = tag & 0x7;
+      const fieldNumber = tag >>> 3;
+      offset = tagOffset;
+
+      switch (fieldNumber) {
+        case PNL_FIELDS.TEMPLATE_ID:
+          [result.templateId, offset] = readVarint(buffer, offset);
+          break;
+        case PNL_FIELDS.IS_SNAPSHOT:
+          const [isSnap, snapOffset] = readVarint(buffer, offset);
+          result.isSnapshot = isSnap !== 0;
+          offset = snapOffset;
+          break;
+        case PNL_FIELDS.FCM_ID:
+          [result.fcmId, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case PNL_FIELDS.IB_ID:
+          [result.ibId, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case PNL_FIELDS.ACCOUNT_ID:
+          [result.accountId, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case PNL_FIELDS.ACCOUNT_BALANCE:
+          [result.accountBalance, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case PNL_FIELDS.CASH_ON_HAND:
+          [result.cashOnHand, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case PNL_FIELDS.MARGIN_BALANCE:
+          [result.marginBalance, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case PNL_FIELDS.MIN_ACCOUNT_BALANCE:
+          [result.minAccountBalance, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case PNL_FIELDS.OPEN_POSITION_PNL:
+          [result.openPositionPnl, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case PNL_FIELDS.CLOSED_POSITION_PNL:
+          [result.closedPositionPnl, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case PNL_FIELDS.DAY_PNL:
+          [result.dayPnl, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case PNL_FIELDS.DAY_OPEN_PNL:
+          [result.dayOpenPnl, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case PNL_FIELDS.DAY_CLOSED_PNL:
+          [result.dayClosedPnl, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case PNL_FIELDS.AVAILABLE_BUYING_POWER:
+          [result.availableBuyingPower, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case PNL_FIELDS.SSBOE:
+          [result.ssboe, offset] = readVarint(buffer, offset);
+          break;
+        case PNL_FIELDS.USECS:
+          [result.usecs, offset] = readVarint(buffer, offset);
+          break;
+        default:
+          offset = skipField(buffer, offset, wireType);
+      }
+    } catch (error) {
+      break;
     }
   }
-  
-  getStats() {
-    return { size: this._size, available: this._count, bufferSize: this._bufferSize };
+
+  return result;
+}
+
+/**
+ * Manually decode InstrumentPnLPositionUpdate from raw bytes
+ */
+function decodeInstrumentPnL(buffer) {
+  const result = {};
+  let offset = 0;
+
+  while (offset < buffer.length) {
+    try {
+      const [tag, tagOffset] = readVarint(buffer, offset);
+      const wireType = tag & 0x7;
+      const fieldNumber = tag >>> 3;
+      offset = tagOffset;
+
+      switch (fieldNumber) {
+        case INSTRUMENT_PNL_FIELDS.TEMPLATE_ID:
+          [result.templateId, offset] = readVarint(buffer, offset);
+          break;
+        case INSTRUMENT_PNL_FIELDS.IS_SNAPSHOT:
+          const [isSnap, snapOffset] = readVarint(buffer, offset);
+          result.isSnapshot = isSnap !== 0;
+          offset = snapOffset;
+          break;
+        case INSTRUMENT_PNL_FIELDS.FCM_ID:
+          [result.fcmId, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case INSTRUMENT_PNL_FIELDS.IB_ID:
+          [result.ibId, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case INSTRUMENT_PNL_FIELDS.ACCOUNT_ID:
+          [result.accountId, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case INSTRUMENT_PNL_FIELDS.SYMBOL:
+          [result.symbol, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case INSTRUMENT_PNL_FIELDS.EXCHANGE:
+          [result.exchange, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case INSTRUMENT_PNL_FIELDS.PRODUCT_CODE:
+          [result.productCode, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case INSTRUMENT_PNL_FIELDS.BUY_QTY:
+          [result.buyQty, offset] = readVarint(buffer, offset);
+          break;
+        case INSTRUMENT_PNL_FIELDS.SELL_QTY:
+          [result.sellQty, offset] = readVarint(buffer, offset);
+          break;
+        case INSTRUMENT_PNL_FIELDS.FILL_BUY_QTY:
+          [result.fillBuyQty, offset] = readVarint(buffer, offset);
+          break;
+        case INSTRUMENT_PNL_FIELDS.FILL_SELL_QTY:
+          [result.fillSellQty, offset] = readVarint(buffer, offset);
+          break;
+        case INSTRUMENT_PNL_FIELDS.NET_QUANTITY:
+          [result.netQuantity, offset] = readVarint(buffer, offset);
+          break;
+        case INSTRUMENT_PNL_FIELDS.OPEN_POSITION_QUANTITY:
+          [result.openPositionQuantity, offset] = readVarint(buffer, offset);
+          break;
+        case INSTRUMENT_PNL_FIELDS.AVG_OPEN_FILL_PRICE:
+          // Double is 64-bit fixed
+          if (wireType === 1) {
+            result.avgOpenFillPrice = buffer.readDoubleLE(offset);
+            offset += 8;
+          } else {
+            offset = skipField(buffer, offset, wireType);
+          }
+          break;
+        case INSTRUMENT_PNL_FIELDS.OPEN_POSITION_PNL:
+          [result.openPositionPnl, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case INSTRUMENT_PNL_FIELDS.CLOSED_POSITION_PNL:
+          [result.closedPositionPnl, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case INSTRUMENT_PNL_FIELDS.DAY_PNL:
+          [result.dayPnl, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case INSTRUMENT_PNL_FIELDS.DAY_OPEN_PNL:
+          [result.dayOpenPnl, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case INSTRUMENT_PNL_FIELDS.DAY_CLOSED_PNL:
+          [result.dayClosedPnl, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case INSTRUMENT_PNL_FIELDS.SSBOE:
+          [result.ssboe, offset] = readVarint(buffer, offset);
+          break;
+        case INSTRUMENT_PNL_FIELDS.USECS:
+          [result.usecs, offset] = readVarint(buffer, offset);
+          break;
+        default:
+          offset = skipField(buffer, offset, wireType);
+      }
+    } catch (error) {
+      break;
+    }
   }
+
+  return result;
 }
 
 /**
  * Protobuf Handler class
- * OPTIMIZED: Pre-compile types, cache encoders, buffer pooling
- * 
- * ULTRA-LOW LATENCY FEATURES:
- * - Pre-allocated buffer pool (zero allocation in hot path)
- * - Reusable protobuf Writer
- * - Cached compiled message types
- * - Direct buffer encoding without intermediate objects
  */
 class ProtobufHandler {
   constructor() {
     this.root = null;
     this.loaded = false;
     this.protoPath = path.join(__dirname, 'proto');
-    
-    // OPTIMIZATION: Cache compiled types for hot path
-    this._typeCache = new Map();
-    this._encoderCache = new Map();
-    
-    // OPTIMIZATION: Pre-allocated buffer pool for zero-allocation encoding
-    this._bufferPool = new BufferPool(16, 512); // 16 buffers of 512 bytes each
-    
-    // OPTIMIZATION: Reusable protobuf Writer instance
-    this._writer = null;
   }
 
   /**
    * Load all proto files
-   * Call once at startup, not per-connection
    */
   async load() {
     if (this.loaded) return;
@@ -94,109 +328,26 @@ class ProtobufHandler {
     }
 
     this.loaded = true;
-    
-    // Pre-compile frequently used types
-    this._precompileTypes();
-  }
-
-  /**
-   * Pre-compile frequently used message types
-   * @private
-   */
-  _precompileTypes() {
-    const hotTypes = [
-      'RequestNewOrder',
-      'RequestCancelOrder', 
-      'RequestHeartbeat',
-      'RequestLogin',
-      'ResponseLogin',
-      'RithmicOrderNotification',
-      'ExchangeOrderNotification',
-    ];
-    
-    for (const typeName of hotTypes) {
-      try {
-        const Type = this.root.lookupType(typeName);
-        this._typeCache.set(typeName, Type);
-      } catch (e) {
-        // Type may not exist in all proto files
-      }
-    }
-  }
-
-  /**
-   * Get cached type or lookup
-   * @private
-   */
-  _getType(typeName) {
-    let Type = this._typeCache.get(typeName);
-    if (!Type) {
-      Type = this.root.lookupType(typeName);
-      this._typeCache.set(typeName, Type);
-    }
-    return Type;
   }
 
   /**
    * Encode a message to Buffer
-   * OPTIMIZED: Uses cached type lookup
    */
   encode(typeName, data) {
     if (!this.root) throw new Error('Proto not loaded');
 
-    const Type = this._getType(typeName);
+    const Type = this.root.lookupType(typeName);
     const msg = Type.create(data);
     return Buffer.from(Type.encode(msg).finish());
   }
 
-  /**
-   * Fast encode for hot path - uses buffer pool and reuses writer
-   * ULTRA-LOW LATENCY: Zero allocation in typical case
-   * 
-   * @param {string} typeName 
-   * @param {Object} data 
-   * @returns {Buffer}
-   */
-  fastEncode(typeName, data) {
-    const Type = this._typeCache.get(typeName);
-    if (!Type) return this.encode(typeName, data);
-    
-    // OPTIMIZATION: Create message without validation (faster)
-    const msg = Type.fromObject(data);
-    
-    // OPTIMIZATION: Get length first to check if pool buffer fits
-    const len = Type.encode(msg).len;
-    
-    if (len <= 512) {
-      // Use pooled buffer for small messages (typical orders are ~100-200 bytes)
-      const poolBuf = this._bufferPool.acquire();
-      const writer = protobuf.Writer.create();
-      Type.encode(msg, writer);
-      const encoded = writer.finish();
-      
-      // Copy to pooled buffer and return a slice
-      encoded.copy(poolBuf, 0, 0, len);
-      
-      // Return a NEW buffer (copy) because pooled buffer will be reused
-      // This is still faster than allocating fresh each time due to copy being optimized
-      const result = Buffer.allocUnsafe(len);
-      poolBuf.copy(result, 0, 0, len);
-      this._bufferPool.release(poolBuf);
-      return result;
-    }
-    
-    // Large message - use standard path
-    return Buffer.from(Type.encode(msg).finish());
-  }
-
   /**
    * Decode a Buffer to object
-   * OPTIMIZED: Uses cached type lookup
    */
   decode(typeName, buffer) {
     if (!this.root) throw new Error('Proto not loaded');
 
-    const Type = this._getType(typeName);
+    const Type = this.root.lookupType(typeName);
     return Type.decode(buffer);
   }
 
@@ -239,6 +390,101 @@ class ProtobufHandler {
   }
 }
 
+/**
+ * Decode ResponseProductCodes (template 112) - list of available symbols
+ */
+function decodeProductCodes(buffer) {
+  const result = { rpCode: [] };
+  let offset = 0;
+
+  while (offset < buffer.length) {
+    try {
+      const [tag, tagOffset] = readVarint(buffer, offset);
+      const wireType = tag & 0x7;
+      const fieldNumber = tag >>> 3;
+      offset = tagOffset;
+
+      switch (fieldNumber) {
+        case SYMBOL_FIELDS.TEMPLATE_ID:
+          [result.templateId, offset] = readVarint(buffer, offset);
+          break;
+        case SYMBOL_FIELDS.RP_CODE:
+          let rpCode;
+          [rpCode, offset] = readLengthDelimited(buffer, offset);
+          result.rpCode.push(rpCode);
+          break;
+        case SYMBOL_FIELDS.EXCHANGE:
+          [result.exchange, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case SYMBOL_FIELDS.PRODUCT_CODE:
+          [result.productCode, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case SYMBOL_FIELDS.PRODUCT_NAME:
+          [result.productName, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case SYMBOL_FIELDS.USER_MSG:
+          [result.userMsg, offset] = readLengthDelimited(buffer, offset);
+          break;
+        default:
+          offset = skipField(buffer, offset, wireType);
+      }
+    } catch (error) {
+      break;
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Decode ResponseFrontMonthContract (template 114) - current tradeable contract
+ */
+function decodeFrontMonthContract(buffer) {
+  const result = { rpCode: [] };
+  let offset = 0;
+
+  while (offset < buffer.length) {
+    try {
+      const [tag, tagOffset] = readVarint(buffer, offset);
+      const wireType = tag & 0x7;
+      const fieldNumber = tag >>> 3;
+      offset = tagOffset;
+
+      switch (fieldNumber) {
+        case SYMBOL_FIELDS.TEMPLATE_ID:
+          [result.templateId, offset] = readVarint(buffer, offset);
+          break;
+        case SYMBOL_FIELDS.RP_CODE:
+          let rpCode;
+          [rpCode, offset] = readLengthDelimited(buffer, offset);
+          result.rpCode.push(rpCode);
+          break;
+        case SYMBOL_FIELDS.SYMBOL:
+          [result.symbol, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case SYMBOL_FIELDS.EXCHANGE:
+          [result.exchange, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case SYMBOL_FIELDS.TRADING_SYMBOL:
+          [result.tradingSymbol, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case SYMBOL_FIELDS.DESCRIPTION:
+          [result.description, offset] = readLengthDelimited(buffer, offset);
+          break;
+        case SYMBOL_FIELDS.USER_MSG:
+          [result.userMsg, offset] = readLengthDelimited(buffer, offset);
+          break;
+        default:
+          offset = skipField(buffer, offset, wireType);
+      }
+    } catch (error) {
+      break;
+    }
+  }
+
+  return result;
+}
+
 // Singleton
 const proto = new ProtobufHandler();