@icgio/icg-exchanges-wrapper 1.14.252 → 1.14.254

package/index.js

@@ -4,6 +4,7 @@ const get_exchanges_trades = require('./functions/get-exchanges-trades.js')
 const get_exchanges_open_orders = require('./functions/get-exchanges-open-orders.js')
 const get_exchanges_deposits = require('./functions/get-exchanges-deposits.js')
 const get_exchanges_withdrawals = require('./functions/get-exchanges-withdrawals.js')
+const { ws_data_client } = require('./lib/ws-data-client.js')
 
 module.exports.functions = {
   get_exchanges_rates,
@@ -17,3 +18,7 @@ module.exports.functions = {
 }
 
 module.exports.orders = require('./middleware/orders.js')
+
+// WebSocket data client for log streaming
+module.exports.ws_data_client = ws_data_client
+module.exports.send_log = (level, message) => ws_data_client.send_log(level, message)

package/lib/ws-data-client.js

@@ -0,0 +1,248 @@
+const WebSocket = require('ws')
+const { randomUUID: uuid } = require('crypto')
+
+// Hardcoded backend URL for production
+const WS_BACKEND_URL = 'wss://api-prod.icg.io/ws/data/ingest'
+
+/**
+ * WebSocket client for streaming logs and order status to dashboard backend.
+ * Singleton instance shared across the application.
+ */
+class WSDataClient {
+  constructor() {
+    this.backend_url = WS_BACKEND_URL
+    this.secret = null
+    this.client_id = null
+    this.ws = null
+    this.queue = []
+    this.max_queue = 100
+    this.reconnect_interval = 5000
+    this.enabled = false
+    this.connected = false
+    this.reconnect_timer = null
+    this.sync_interval = null
+    this.sync_interval_ms = 30 * 1000 // 30 seconds
+    this.order_info_ref = null // Reference to Orders.order_info for periodic sync
+  }
+
+  /**
+   * Initialize the WS client with connection details.
+   * @param {string} secret - Shared secret for authentication
+   * @param {string} client_id - Client identifier (e.g., "FREEZONE")
+   */
+  init(secret, client_id) {
+    if (!secret || !client_id) {
+      console.log('[WSDataClient] Missing config (secret or client_id), disabled')
+      return
+    }
+
+    this.secret = secret
+    this.client_id = client_id
+    this.enabled = true
+
+    console.log('[WSDataClient] Initializing for client:', client_id, 'URL:', this.backend_url)
+    this.connect()
+  }
+
+  /**
+   * Set reference to order_info for periodic sync.
+   * @param {object} order_info - Reference to Orders.order_info object
+   */
+  set_order_info_ref(order_info) {
+    this.order_info_ref = order_info
+
+    // Start periodic sync if not already started
+    if (!this.sync_interval && this.enabled) {
+      this.start_periodic_sync()
+    }
+  }
+
+  /**
+   * Start periodic full sync of order_info (every 30 seconds).
+   */
+  start_periodic_sync() {
+    if (this.sync_interval) return
+
+    // Send initial reset on startup
+    this.send_reset('orders')
+
+    this.sync_interval = setInterval(() => {
+      if (this.order_info_ref && this.connected) {
+        this.send_order_sync(this.order_info_ref)
+      }
+    }, this.sync_interval_ms)
+
+    console.log('[WSDataClient] Periodic sync started (every %ds)', this.sync_interval_ms / 1000)
+  }
+
+  /**
+   * Connect to the dashboard backend WebSocket.
+   */
+  connect() {
+    if (!this.enabled) return
+
+    try {
+      this.ws = new WebSocket(this.backend_url, {
+        headers: {
+          'x-client-secret': this.secret,
+          'x-client-id': this.client_id,
+        },
+      })
+
+      this.ws.on('open', () => {
+        console.log('[WSDataClient] Connected to backend')
+        this.connected = true
+        this.flush_queue()
+
+        // Send reset on reconnect to clear stale state
+        this.send_reset('orders')
+
+        // Send full sync immediately after connect
+        if (this.order_info_ref) {
+          this.send_order_sync(this.order_info_ref)
+        }
+      })
+
+      this.ws.on('close', (code, reason) => {
+        console.log('[WSDataClient] Disconnected:', code, reason?.toString())
+        this.connected = false
+        this.schedule_reconnect()
+      })
+
+      this.ws.on('error', (err) => {
+        console.error('[WSDataClient] Error:', err.message)
+        this.connected = false
+      })
+
+      this.ws.on('message', (data) => {
+        // Handle any messages from backend (e.g., acknowledgments)
+        try {
+          const msg = JSON.parse(data.toString())
+          if (msg.type === 'error') {
+            console.error('[WSDataClient] Backend error:', msg.message)
+          }
+        } catch (e) {
+          // Ignore parse errors
+        }
+      })
+    } catch (err) {
+      console.error('[WSDataClient] Connection error:', err.message)
+      this.schedule_reconnect()
+    }
+  }
+
+  /**
+   * Schedule reconnection attempt.
+   */
+  schedule_reconnect() {
+    if (this.reconnect_timer) return
+
+    this.reconnect_timer = setTimeout(() => {
+      this.reconnect_timer = null
+      console.log('[WSDataClient] Attempting reconnect...')
+      this.connect()
+    }, this.reconnect_interval)
+  }
+
+  /**
+   * Send a message to the backend.
+   * @param {string} type - Message type
+   * @param {object} data - Message data
+   */
+  send(type, data) {
+    if (!this.enabled) return
+
+    const msg = JSON.stringify({
+      type,
+      client: this.client_id,
+      ts: Date.now(),
+      ...data,
+    })
+
+    if (this.connected && this.ws?.readyState === WebSocket.OPEN) {
+      this.ws.send(msg)
+    } else {
+      // Queue message for later
+      this.queue.push(msg)
+      if (this.queue.length > this.max_queue) {
+        this.queue.shift() // Drop oldest
+      }
+    }
+  }
+
+  /**
+   * Flush queued messages after reconnection.
+   */
+  flush_queue() {
+    while (this.queue.length > 0 && this.connected && this.ws?.readyState === WebSocket.OPEN) {
+      const msg = this.queue.shift()
+      this.ws.send(msg)
+    }
+  }
+
+  /**
+   * Send reset event to clear state on backend.
+   * @param {string} scope - What to reset ('orders', 'logs', 'all')
+   */
+  send_reset(scope = 'orders') {
+    this.send('reset', { scope })
+  }
+
+  /**
+   * Send full order_info sync to backend.
+   * @param {object} order_info - Complete order_info object
+   */
+  send_order_sync(order_info) {
+    this.send('order_sync', { data: order_info })
+  }
+
+  /**
+   * Send order update event to backend.
+   * @param {object} event - Order event data
+   */
+  send_order(event) {
+    this.send('order', event)
+  }
+
+  /**
+   * Send log entry to backend.
+   * @param {string} level - Log level ('info', 'warn', 'error')
+   * @param {string} message - Log message
+   */
+  send_log(level, message) {
+    this.send('log', {
+      id: uuid(),
+      level,
+      msg: message,
+    })
+  }
+
+  /**
+   * Close the WebSocket connection.
+   */
+  close() {
+    this.enabled = false
+
+    if (this.sync_interval) {
+      clearInterval(this.sync_interval)
+      this.sync_interval = null
+    }
+
+    if (this.reconnect_timer) {
+      clearTimeout(this.reconnect_timer)
+      this.reconnect_timer = null
+    }
+
+    if (this.ws) {
+      this.ws.close()
+      this.ws = null
+    }
+
+    this.connected = false
+  }
+}
+
+// Singleton instance
+const ws_data_client = new WSDataClient()
+
+module.exports = { WSDataClient, ws_data_client }

package/middleware/orders.js

@@ -7,7 +7,7 @@ const utils = require('@icgio/icg-utils').utils
 const { coinmarketcap: get_coinmarketcap_rates, observable: Observable } = require('@icgio/icg-utils')
 
 const Account = require('./account.js')
-const AllOrdersStore = require('../lib/order-status-store.js')
+const { ws_data_client } = require('../lib/ws-data-client.js')
 
 // Default precision functions - will be overridden if provided in settings
 let round_price = (exchange, pair, price) => price
@@ -244,9 +244,10 @@ module.exports = class Orders {
         this.process_query_order(this.query_order)
       }, PROCESS_OPEN_ORDERS_TRADERS_INTERVAL_MS)
     }
-    if (this.function_enabled.all_orders_store) {
-      // Initialize all orders store for PostgreSQL persistence (async, non-blocking)
-      this.all_orders_store = new AllOrdersStore(cd.all_orders_store_connection_string)
+    // Initialize WS data client for streaming to dashboard backend
+    if (settings.ws_backend_secret && settings.client_id) {
+      ws_data_client.init(settings.ws_backend_secret, settings.client_id)
+      ws_data_client.set_order_info_ref(this.order_info)
     }
   }
   init_account(balances, benchmarks) {
@@ -500,10 +501,12 @@ module.exports = class Orders {
           trade.amount = round_amount(exchange, pair, trade.amount)
           let { type, order_id } = trade
           let order_info = _.get(this.order_info, [exchange, pair, type, order_id])
+          if (!order_info) {
+            return
+          }
           const trade_amount = round_amount(exchange, pair, trade.amount)
           const order_info_f_amount = round_amount(exchange, pair, order_info.f_amount)
-          if (!order_info) {
-          } else if (_.includes(['CANCELED', 'CANCELING'], order_info.status) && trade_amount > order_info_f_amount) {
+          if (_.includes(['CANCELED', 'CANCELING'], order_info.status) && trade_amount > order_info_f_amount) {
             // console.log('TRADES: %s ORDER FILLED', order_info.status, trade, order_info)
             this.update_order_info(
               exchange,
@@ -660,6 +663,9 @@ module.exports = class Orders {
         open_orders[exchange][pair].open_orders.map((order) => {
           if (_.get(this.order_info, [exchange, pair, order.type, order.order_id])) {
             let order_info = this.order_info[exchange][pair][order.type][order.order_id]
+            if (!order_info) {
+              return
+            }
             const order_f_amount = round_amount(exchange, pair, order.f_amount)
             const order_info_f_amount = round_amount(exchange, pair, order_info.f_amount)
             if (_.includes(['N-FILLED', 'P-FILLED'], order_info.status) && order_f_amount > order_info_f_amount && order_f_amount / order_info.t_amount > this.f_filled_threshold) {
@@ -776,6 +782,17 @@ module.exports = class Orders {
       }
       _.setWith(this.order_info, `${exchange}.${pair}.${type}.${order_id}.${key}`, dict[key], Object)
     }
+
+    // Add trace entry for order history
+    const trace_entry = {
+      ts: Date.now(),
+      status: dict.status || _.get(this.order_info, [exchange, pair, type, order_id, 'status']),
+      note,
+      f_amount: dict.f_amount,
+    }
+    const existing_trace = _.get(this.order_info, [exchange, pair, type, order_id, 'trace'], [])
+    _.setWith(this.order_info, `${exchange}.${pair}.${type}.${order_id}.trace`, [...existing_trace, trace_entry], Object)
+
     console.log('UPDATE %s: %s %s %s %s %j', note, exchange, pair, type, order_id, dict)
     // start: log every f-filled trade for future analysis
     if (dict.status === 'F-FILLED') {
@@ -784,21 +801,38 @@ module.exports = class Orders {
       console.log(`f_filled_orders_info:${exchange},${pair},${type},${order_id},${price},${amount},${f_amount},${t_amount},${fees},${close_time},${note}`)
     }
     // end: log every f-filled trade for future analysis
-    // Store order status to PostgreSQL asynchronously (non-blocking)
+
+    // Send order update via WebSocket to dashboard backend
     const order_data = _.get(this.order_info, [exchange, pair, type, order_id], {})
-    if (this.function_enabled.all_orders_store) {
-      this.all_orders_store.store(exchange, pair, type, order_id, order_data, note)
+    // Only send N-FILLED and later statuses (skip INIT which has temporary order_id)
+    if (dict.status !== 'INIT' && order_data.status !== 'INIT') {
+      ws_data_client.send_order({
+        action: 'update',
+        exchange,
+        pair,
+        side: type,
+        order_id,
+        data: order_data,
+        note,
+      })
     }
   }
   delete_order_info(exchange, pair, type, order_id, note) {
     const order_data = _.get(this.order_info, [exchange, pair, type, order_id], {})
     this.order_info[exchange][pair][type] = _.omit(this.order_info[exchange][pair][type], [order_id])
     console.log('DELETE %s: %s %s %s %s', note, exchange, pair, type, order_id)
-    // Store deletion to PostgreSQL (non-blocking)
-    // Keep final F-FILLED / P-FILLED states in DB (do NOT mark them as DELETED)
-    if (order_data.status === 'F-FILLED' || order_data.status === 'P-FILLED') {
-    } else if (this.function_enabled.all_orders_store) {
-      this.all_orders_store.store(exchange, pair, type, order_id, { ...order_data, status: 'DELETED' }, note)
+
+    // Send delete event via WebSocket (only for non-INIT orders)
+    if (order_data.status !== 'INIT') {
+      ws_data_client.send_order({
+        action: 'delete',
+        exchange,
+        pair,
+        side: type,
+        order_id,
+        data: order_data,
+        note,
+      })
     }
   }
   check_balances() {
@@ -920,10 +954,6 @@ module.exports = class Orders {
       }
       // cancel previous orders
       else if (this.cancel_previous_orders && !this.previous_orders_canceled) {
-        // remove all previous orders for this exchange/pair from all_orders (fire-and-forget)
-        if (this.function_enabled.all_orders_store) {
-          this.all_orders_store.clear_orders(exchange, pair).catch((err) => console.error('clear_all_orders error', exchange, pair, err.message))
-        }
         if (this.cancel_previous_orders_with_delay) {
           setTimeout(() => {
             orders.map((order) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@icgio/icg-exchanges-wrapper",
-  "version": "1.14.252",
+  "version": "1.14.254",
   "description": "",
   "main": "index.js",
   "scripts": {

package/lib/order-status-store.js

@@ -1,255 +0,0 @@
-const { Pool } = require('pg')
-
-/**
- * Async PostgreSQL store for all orders
- * Non-blocking: all writes are fire-and-forget to avoid blocking the main process
- */
-class AllOrdersStore {
-  constructor(connection_string) {
-    this.enabled = true
-    this.pool = null
-    this.table_ready = false
-    this.queue = []
-    this.max_queue_size = 2000
-    this.flush_interval = null
-    this.flush_interval_ms = 5000
-
-    if (connection_string) {
-      this.init(connection_string)
-    }
-  }
-
-  async init(connection_string) {
-    try {
-      this.pool = new Pool({
-        connectionString: connection_string,
-        max: 5,
-        idleTimeoutMillis: 30000,
-        connectionTimeoutMillis: 5000,
-        ssl: { rejectUnauthorized: false },
-      })
-
-      this.pool.on('error', (err) => {
-        console.error('AllOrdersStore pool error:', err.message)
-      })
-
-      // Set timezone to UTC for all connections
-      this.pool.on('connect', async (client) => {
-        await client.query("SET timezone = 'UTC'")
-      })
-
-      await this.ensure_table()
-      this.enabled = true
-      this.start_flush_interval()
-      console.log('AllOrdersStore initialized successfully')
-    } catch (err) {
-      console.error('AllOrdersStore init failed:', err.message)
-      this.enabled = false
-    }
-  }
-
-  async ensure_table() {
-    const create_table_query = `
-      -- All orders (persistent mapping of id/order_id to order_type & status)
-      CREATE TABLE IF NOT EXISTS all_orders (
-        id VARCHAR(128),
-        exchange VARCHAR(64) NOT NULL,
-        pair VARCHAR(32) NOT NULL,
-        order_type VARCHAR(64),
-        type VARCHAR(16) NOT NULL,
-        order_id VARCHAR(128) NOT NULL,
-        status VARCHAR(32) NOT NULL,
-        price DECIMAL(32, 16),
-        amount DECIMAL(32, 16),
-        f_amount DECIMAL(32, 16),
-        f_percentage DECIMAL(8, 4),
-        t_amount DECIMAL(32, 16),
-        fees JSONB,
-        open_time TIMESTAMPTZ,
-        close_time TIMESTAMPTZ,
-        note VARCHAR(256),
-        created_at TIMESTAMPTZ DEFAULT (NOW() AT TIME ZONE 'UTC'),
-        updated_at TIMESTAMPTZ DEFAULT (NOW() AT TIME ZONE 'UTC'),
-        PRIMARY KEY (exchange, pair, type, order_id)
-      );
-      CREATE INDEX IF NOT EXISTS idx_all_orders_exchange_pair_order ON all_orders(exchange, pair, order_id);
-    `
-
-    const client = await this.pool.connect()
-    try {
-      await client.query(create_table_query)
-      this.table_ready = true
-    } finally {
-      client.release()
-    }
-  }
-
-  /**
-   * Remove all rows for a specific exchange/pair (fresh start).
-   * @param {string} exchange
-   * @param {string} pair
-   */
-  async clear_orders(exchange, pair) {
-    if (!this.enabled || !this.table_ready) return
-    const client = await this.pool.connect()
-    try {
-      await client.query('DELETE FROM all_orders WHERE exchange = $1 AND pair = $2', [exchange, pair])
-    } finally {
-      client.release()
-    }
-  }
-
-  start_flush_interval() {
-    if (this.flush_interval) return
-
-    this.flush_interval = setInterval(() => {
-      this.flush().catch((err) => {
-        console.error('AllOrdersStore flush error:', err.message)
-      })
-    }, this.flush_interval_ms)
-  }
-
-  stop_flush_interval() {
-    if (this.flush_interval) {
-      clearInterval(this.flush_interval)
-      this.flush_interval = null
-    }
-  }
-
-  /**
-   * Queue an order status update (non-blocking)
-   * @param {string} exchange
-   * @param {string} pair
-   * @param {string} type - 'buy' or 'sell'
-   * @param {string} order_id
-   * @param {object} data - order data
-   * @param {string} note - update note
-   */
-  store(exchange, pair, type, order_id, data, note) {
-    if (!this.enabled) return
-
-    // Convert Date objects to UTC ISO strings for consistent timezone handling
-    const to_utc_iso = (date) => {
-      if (!date) return null
-      if (date instanceof Date) {
-        return date.toISOString()
-      }
-      return date
-    }
-
-    const record = {
-      id: data.id,
-      exchange,
-      pair,
-      type,
-      order_id: order_id,
-      order_type: data.order_type || null,
-      status: data.status || null,
-      price: data.price || null,
-      amount: data.amount || null,
-      f_amount: data.f_amount || null,
-      f_percentage: data.f_percentage || null,
-      t_amount: data.t_amount || null,
-      fees: data.fees ? JSON.stringify(data.fees) : null,
-      open_time: to_utc_iso(data.open_time),
-      close_time: to_utc_iso(data.close_time),
-      note: note || null,
-    }
-
-    this.queue.push(record)
-
-    // Prevent memory overflow
-    if (this.queue.length > this.max_queue_size) {
-      this.queue.shift()
-    }
-  }
-
-  /**
-   * Flush queued records to database (async, non-blocking to caller)
-   */
-  async flush() {
-    if (!this.enabled || !this.table_ready || this.queue.length === 0) return
-
-    const batch = this.queue.splice(0, 100) // Process up to 100 at a time
-
-    const upsert_all_orders_query = `
-      INSERT INTO all_orders (
-        id, exchange, pair, type, order_id, order_type, status,
-        price, amount, f_amount, f_percentage, t_amount,
-        fees, open_time, close_time, note, created_at, updated_at
-      ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, (NOW() AT TIME ZONE 'UTC'), (NOW() AT TIME ZONE 'UTC'))
-      ON CONFLICT (exchange, pair, type, order_id) DO UPDATE SET
-        id = EXCLUDED.id,
-        order_type = EXCLUDED.order_type,
-        status = EXCLUDED.status,
-        price = EXCLUDED.price,
-        amount = EXCLUDED.amount,
-        f_amount = EXCLUDED.f_amount,
-        f_percentage = EXCLUDED.f_percentage,
-        t_amount = EXCLUDED.t_amount,
-        fees = EXCLUDED.fees,
-        open_time = EXCLUDED.open_time,
-        close_time = EXCLUDED.close_time,
-        note = EXCLUDED.note,
-        updated_at = (NOW() AT TIME ZONE 'UTC')
-    `
-
-    const delete_all_orders_query = `
-      DELETE FROM all_orders 
-      WHERE exchange = $1 AND pair = $2 AND type = $3 AND order_id = $4
-    `
-
-    const client = await this.pool.connect()
-    try {
-      for (const record of batch) {
-        if (record.status === 'DELETED') {
-          // Physically delete rows for deleted orders to limit DB size
-          await client.query(delete_all_orders_query, [record.exchange, record.pair, record.type, record.order_id])
-        } else {
-          // Upsert active / updated orders
-          await client.query(upsert_all_orders_query, [
-            record.id,
-            record.exchange,
-            record.pair,
-            record.type,
-            record.order_id,
-            record.order_type,
-            record.status,
-            record.price,
-            record.amount,
-            record.f_amount,
-            record.f_percentage,
-            record.t_amount,
-            record.fees,
-            record.open_time,
-            record.close_time,
-            record.note,
-          ])
-        }
-      }
-    } catch (err) {
-      // Re-queue failed records
-      this.queue.unshift(...batch)
-      throw err
-    } finally {
-      client.release()
-    }
-  }
-
-  async close() {
-    this.stop_flush_interval()
-    if (this.queue.length > 0) {
-      try {
-        await this.flush()
-      } catch (err) {
-        console.error('AllOrdersStore final flush error:', err.message)
-      }
-    }
-    if (this.pool) {
-      await this.pool.end()
-    }
-    this.enabled = false
-  }
-}
-
-module.exports = AllOrdersStore