@usherlabs/cex-broker 0.2.24 → 0.2.25

package/dist/index.js

@@ -274461,6 +274461,9 @@ function compactUndefined(record) {
   return Object.fromEntries(Object.entries(record).filter(([, value]) => value !== undefined));
 }
 
+// src/helpers/broker-execution-archive/rows.ts
+import { createHash as createHash2 } from "crypto";
+
 // src/helpers/broker-execution-archive/redact.ts
 import { createHash } from "crypto";
 var SECRET_KEY_PATTERN = /\b(api[_-]?key|api[_-]?secret|secret|signature|passphrase|password|token|credential)\b/i;
@@ -274533,6 +274536,8 @@ function hashMarketMetadata(payload) {
 var BROKER_WRITE_SOURCE = "broker_write";
 var ARCHIVE_SCHEMA_VERSION = "1";
 var FILL_EVENT_KIND = "trade_history_fill";
+var ACCOUNT_BALANCE_SCOPE = "spot";
+var ACCOUNT_BALANCE_PRECISION_BASIS = "ccxt_normalized_number";
 
 // src/helpers/broker-execution-archive/rows.ts
 function firstString2(...values2) {
@@ -274566,6 +274571,151 @@ function normalizeTimestamp2(value) {
   const date = new Date(ms);
   return Number.isNaN(date.getTime()) ? undefined : date.toISOString();
 }
+function decimalString(value) {
+  if (typeof value !== "number" || !Number.isFinite(value)) {
+    return;
+  }
+  if (Object.is(value, -0)) {
+    return "0";
+  }
+  const rendered = String(value).toLowerCase();
+  if (!rendered.includes("e")) {
+    return rendered;
+  }
+  const [coefficient = "", rawExponent = "0"] = rendered.split("e");
+  const exponent = Number.parseInt(rawExponent, 10);
+  const negative = coefficient.startsWith("-");
+  const unsigned = negative ? coefficient.slice(1) : coefficient;
+  const [integer = "0", fraction = ""] = unsigned.split(".");
+  const digits = `${integer}${fraction}`;
+  const decimalIndex = integer.length + exponent;
+  let expanded;
+  if (decimalIndex <= 0) {
+    expanded = `0.${"0".repeat(-decimalIndex)}${digits}`;
+  } else if (decimalIndex >= digits.length) {
+    expanded = `${digits}${"0".repeat(decimalIndex - digits.length)}`;
+  } else {
+    expanded = `${digits.slice(0, decimalIndex)}.${digits.slice(decimalIndex)}`;
+  }
+  return negative ? `-${expanded}` : expanded;
+}
+function normalizedBalanceMap(value) {
+  const record = asRecord(value);
+  if (!record) {
+    return {};
+  }
+  const entries = [];
+  for (const [rawAsset, quantity] of Object.entries(record)) {
+    const asset = rawAsset.trim();
+    const normalized = decimalString(quantity);
+    if (asset && normalized !== undefined) {
+      entries.push([asset, normalized]);
+    }
+  }
+  return Object.fromEntries(entries.sort(([left], [right]) => left.localeCompare(right)));
+}
+function sortedBalanceMap(map) {
+  return Object.fromEntries(Object.entries(map).sort(([left], [right]) => left.localeCompare(right)));
+}
+var BALANCE_METADATA_KEYS = new Set([
+  "info",
+  "timestamp",
+  "datetime",
+  "free",
+  "used",
+  "total",
+  "debt"
+]);
+function normalizeCcxtBalanceForArchive(balance) {
+  const record = asRecord(balance) ?? {};
+  const aggregateRecords = {
+    free: asRecord(record.free),
+    used: asRecord(record.used),
+    total: asRecord(record.total)
+  };
+  const maps = {
+    free: normalizedBalanceMap(record.free),
+    used: normalizedBalanceMap(record.used),
+    total: normalizedBalanceMap(record.total)
+  };
+  const assetEntryAssets = new Set;
+  for (const [rawAsset, value] of Object.entries(record)) {
+    if (BALANCE_METADATA_KEYS.has(rawAsset)) {
+      continue;
+    }
+    const asset = rawAsset.trim();
+    const assetEntry = asRecord(value);
+    if (!asset || !assetEntry || !(("free" in assetEntry) || ("used" in assetEntry) || ("total" in assetEntry))) {
+      continue;
+    }
+    assetEntryAssets.add(asset);
+    for (const field of ["free", "used", "total"]) {
+      const normalized = decimalString(assetEntry[field]);
+      if (normalized !== undefined && maps[field][asset] === undefined) {
+        maps[field][asset] = normalized;
+      }
+    }
+  }
+  const reportedAssets = new Set(assetEntryAssets);
+  for (const aggregateRecord of Object.values(aggregateRecords)) {
+    for (const rawAsset of Object.keys(aggregateRecord ?? {})) {
+      const asset = rawAsset.trim();
+      if (asset) {
+        reportedAssets.add(asset);
+      }
+    }
+  }
+  for (const map of Object.values(maps)) {
+    for (const asset of Object.keys(map)) {
+      reportedAssets.add(asset);
+    }
+  }
+  return {
+    exchangeTimestamp: normalizeTimestamp2(record.timestamp ?? record.datetime),
+    reportedAssets: [...reportedAssets].sort(),
+    assetEntryAssets: [...assetEntryAssets].sort(),
+    freeBalances: sortedBalanceMap(maps.free),
+    usedBalances: sortedBalanceMap(maps.used),
+    totalBalances: sortedBalanceMap(maps.total),
+    freeMapPresent: aggregateRecords.free !== undefined,
+    usedMapPresent: aggregateRecords.used !== undefined,
+    totalMapPresent: aggregateRecords.total !== undefined
+  };
+}
+function buildAccountBalanceSnapshotRow(input) {
+  const { tags, balance } = input;
+  const observationId = createHash2("sha256").update(JSON.stringify({
+    deployment_id: tags.deployment_id,
+    exchange: tags.exchange,
+    account_selector: tags.account_selector,
+    balance_scope: ACCOUNT_BALANCE_SCOPE,
+    broker_observed_timestamp: tags.broker_observed_timestamp,
+    balance
+  })).digest("hex");
+  return {
+    table: "broker_account.balance_snapshots",
+    row: compactUndefined2({
+      broker_observed_timestamp: tags.broker_observed_timestamp,
+      exchange_timestamp: balance.exchangeTimestamp,
+      source: tags.source,
+      deployment_id: tags.deployment_id,
+      schema_version: ARCHIVE_SCHEMA_VERSION,
+      exchange: tags.exchange,
+      account_selector: tags.account_selector,
+      balance_scope: ACCOUNT_BALANCE_SCOPE,
+      observation_id: observationId,
+      reported_assets: balance.reportedAssets,
+      asset_entry_assets: balance.assetEntryAssets,
+      free_balances: balance.freeBalances,
+      used_balances: balance.usedBalances,
+      total_balances: balance.totalBalances,
+      aggregate_free_map_present: Number(balance.freeMapPresent),
+      aggregate_used_map_present: Number(balance.usedMapPresent),
+      aggregate_total_map_present: Number(balance.totalMapPresent),
+      precision_basis: ACCOUNT_BALANCE_PRECISION_BASIS
+    })
+  };
+}
 function compactUndefined2(record) {
   return Object.fromEntries(Object.entries(record).filter(([, value]) => value !== undefined));
 }
@@ -275146,6 +275296,15 @@ function isMarketArchiveTable(table) {
 }
 
 // src/helpers/broker-execution-archive/writer.ts
+var BROKER_EXECUTION_ARCHIVE_TABLES = new Set([
+  "broker_execution.order_events",
+  "broker_execution.market_metadata_snapshots",
+  "broker_execution.transfer_events",
+  "broker_execution.fill_events"
+]);
+function isBrokerExecutionArchiveTable(table) {
+  return BROKER_EXECUTION_ARCHIVE_TABLES.has(table);
+}
 var DEFAULT_MAX_QUEUE_SIZE = 1e4;
 var DEFAULT_BATCH_SIZE = 10;
 var DEFAULT_FLUSH_INTERVAL_MS = 1000;
@@ -275230,6 +275389,9 @@ class BrokerExecutionArchiver {
   canPersistMarketMetadataSnapshot() {
     return this.isEnabled() && (Boolean(this.otelLogs?.isOtelEnabled()) || Boolean(this.forwarderUrl));
   }
+  canPersistAccountBalanceSnapshots() {
+    return this.isEnabled() && Boolean(this.forwarderUrl);
+  }
   enqueue(row) {
     if (!this.enabled || this.closed || !this.otelLogs && !this.forwarderUrl) {
       return;
@@ -275241,6 +275403,9 @@ class BrokerExecutionArchiver {
       }
       return;
     }
+    if (row.table === "broker_account.balance_snapshots" && !this.forwarderUrl) {
+      return;
+    }
     if (this.queue.length >= this.maxQueueSize) {
       this.queue.shift();
       this.stats.shed += 1;
@@ -275319,7 +275484,7 @@ class BrokerExecutionArchiver {
       return true;
     }
     for (const entry of batch) {
-      if (!isMarketArchiveTable(entry.table)) {
+      if (isBrokerExecutionArchiveTable(entry.table)) {
         this.emitOtelLog(entry);
       }
     }
@@ -275455,8 +275620,138 @@ function parsePositiveInt(value, fallback) {
   const parsed = Number.parseInt(value, 10);
   return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
 }
-// src/helpers/fill-archive-poller.ts
+// src/helpers/account-balance-archive-poller.ts
 var DEFAULT_CONFIG = {
+  pollIntervalMs: 60000
+};
+var metricLabels = (target) => ({
+  exchange: target.exchangeId,
+  account_selector: target.account.label,
+  balance_scope: ACCOUNT_BALANCE_SCOPE
+});
+
+class AccountBalanceArchivePoller {
+  params;
+  #timer = null;
+  #stopped = false;
+  #running = null;
+  #lastSuccessMs = new Map;
+  #config;
+  constructor(params) {
+    this.params = params;
+    this.#config = { ...DEFAULT_CONFIG, ...params.config };
+  }
+  start() {
+    if (this.#timer || this.#stopped || !this.params.archiver.canPersistAccountBalanceSnapshots()) {
+      return;
+    }
+    log.info("\uD83D\uDCB0 Account balance archive poller started", {
+      balanceScope: ACCOUNT_BALANCE_SCOPE
+    });
+    this.#schedule(0);
+  }
+  async stop() {
+    this.#stopped = true;
+    if (this.#timer) {
+      clearTimeout(this.#timer);
+      this.#timer = null;
+    }
+    await this.#running;
+  }
+  async pollAllOnce() {
+    if (this.#stopped || this.#running || !this.params.archiver.canPersistAccountBalanceSnapshots()) {
+      return false;
+    }
+    this.#running = this.#pollAllSequentially();
+    try {
+      return await this.#running;
+    } finally {
+      this.#running = null;
+    }
+  }
+  #targets() {
+    const targets = [];
+    for (const [exchangeId, pool] of Object.entries(this.params.brokers)) {
+      for (const account of [pool.primary, ...pool.secondaryBrokers]) {
+        targets.push({ exchangeId, account });
+      }
+    }
+    return targets;
+  }
+  async#pollAllSequentially() {
+    for (const target of this.#targets()) {
+      if (this.#stopped) {
+        break;
+      }
+      await this.#pollOne(target);
+    }
+    return true;
+  }
+  async#pollOne(target) {
+    const labels = metricLabels(target);
+    this.params.metrics?.recordCounter("cex_account_balance_poll_attempts_total", 1, labels);
+    try {
+      const exchange = target.account.exchange;
+      const balance = await exchange.fetchBalance({
+        type: ACCOUNT_BALANCE_SCOPE
+      });
+      const observedAt = new Date;
+      const normalized = normalizeCcxtBalanceForArchive(balance);
+      this.params.archiver.enqueue(buildAccountBalanceSnapshotRow({
+        tags: buildCommonArchiveTags({
+          deploymentId: this.params.archiver.getDeploymentId(),
+          accountSelector: target.account.label,
+          exchange: target.exchangeId,
+          brokerObservedTimestamp: observedAt.toISOString()
+        }),
+        balance: normalized
+      }));
+      const successMs = Date.now();
+      this.#lastSuccessMs.set(this.#targetKey(target), successMs);
+      this.params.metrics?.recordCounter("cex_account_balance_poll_successes_total", 1, labels);
+      this.params.metrics?.recordGauge("cex_account_balance_poll_last_success_timestamp_seconds", Math.floor(successMs / 1000), labels);
+      this.#recordFreshness(labels, successMs, successMs);
+    } catch (error) {
+      this.params.metrics?.recordCounter("cex_account_balance_poll_failures_total", 1, labels);
+      const now3 = Date.now();
+      const lastSuccess = this.#lastSuccessMs.get(this.#targetKey(target));
+      if (lastSuccess !== undefined) {
+        this.#recordFreshness(labels, lastSuccess, now3);
+      }
+      log.warn("Account balance archive poll failed", {
+        exchange: target.exchangeId,
+        account: target.account.label,
+        balanceScope: ACCOUNT_BALANCE_SCOPE,
+        errorType: error instanceof Error ? error.name : "unknown"
+      });
+    }
+  }
+  #recordFreshness(labels, lastSuccessMs, nowMs) {
+    this.params.metrics?.recordGauge("cex_account_balance_poll_freshness_seconds", Math.max(0, (nowMs - lastSuccessMs) / 1000), labels);
+  }
+  #targetKey(target) {
+    return `${target.exchangeId}|${target.account.label}|${ACCOUNT_BALANCE_SCOPE}`;
+  }
+  #schedule(delayMs) {
+    this.#timer = setTimeout(() => void this.#tick(), delayMs);
+    this.#timer.unref?.();
+  }
+  async#tick() {
+    this.#timer = null;
+    try {
+      await this.pollAllOnce();
+    } catch (error) {
+      log.error("Account balance archive poller tick failed", error);
+    } finally {
+      if (!this.#stopped) {
+        this.#schedule(this.#config.pollIntervalMs);
+      }
+    }
+  }
+}
+
+// src/helpers/fill-archive-poller.ts
+var DEFAULT_CONFIG2 = {
   pollIntervalMs: 60000,
   lookbackMs: 24 * 60 * 60 * 1000,
   tradesLimit: 500
@@ -275481,7 +275776,7 @@ class FillArchivePoller {
   #config;
   constructor(params) {
     this.params = params;
-    this.#config = { ...DEFAULT_CONFIG, ...params.config };
+    this.#config = { ...DEFAULT_CONFIG2, ...params.config };
   }
   start() {
     if (this.#timer || this.#stopped) {
@@ -295087,6 +295382,7 @@ import WebSocket2 from "ws";
 var BINANCE_SPOT_WS_API_URL = "wss://ws-api.binance.com:443/ws-api/v3";
 var DEFAULT_BINANCE_USER_DATA_MAX_BUFFERED_EVENTS = 16;
 var createWebSocket = (url3) => new WebSocket2(url3);
+var userDataRequestCounter = 0;
 function getExchangeString(exchange, key) {
   const value = exchange[key];
   if (typeof value !== "string" || value.length === 0) {
@@ -295193,7 +295489,7 @@ class BinanceSpotUserDataStream {
   exchange;
   ws;
   secretValues;
-  requestId = `user-data-${Date.now()}-${Math.random()}`;
+  requestId = `user-data-${Date.now()}-${userDataRequestCounter++}`;
   maxBufferedEvents;
   queue = [];
   waiters = [];
@@ -295271,6 +295567,12 @@ class BinanceSpotUserDataStream {
       this.subscriptionId = message.result?.subscriptionId ?? null;
       return;
     }
+    if ("status" in message && typeof message.status === "number" && message.status !== 200) {
+      const errorMessage = message.error?.msg ?? message.error?.message ?? `Binance user-data request failed with status ${message.status}`;
+      const errorCode = message.error?.code;
+      this.fail(new Error(typeof errorCode === "number" ? `${errorMessage} (code ${errorCode})` : errorMessage));
+      return;
+    }
     if (!("event" in message) || !message.event) {
       return;
     }
@@ -296801,6 +297103,7 @@ class CEXBroker {
   orderActivityTracker = new OrderActivityTracker;
   withdrawalObservationTracker = new WithdrawalObservationTracker;
   fillArchivePoller;
+  accountBalanceArchivePoller;
   loadEnvConfig() {
     log.info("\uD83D\uDD27 Loading CEX_BROKER_ environment variables:");
     const configMap = {};
@@ -296945,6 +297248,10 @@ class CEXBroker {
       this.fillArchivePoller.stop();
       this.fillArchivePoller = undefined;
     }
+    if (this.accountBalanceArchivePoller) {
+      await this.accountBalanceArchivePoller.stop();
+      this.accountBalanceArchivePoller = undefined;
+    }
     if (this.server) {
       await this.server.forceShutdown();
     }
@@ -296970,6 +297277,10 @@ class CEXBroker {
       this.fillArchivePoller.stop();
       this.fillArchivePoller = undefined;
     }
+    if (this.accountBalanceArchivePoller) {
+      await this.accountBalanceArchivePoller.stop();
+      this.accountBalanceArchivePoller = undefined;
+    }
     log.info(`Running CEXBroker at ${new Date().toISOString()}`);
     if (this.otelMetrics?.isOtelEnabled()) {
       await this.otelMetrics.initialize();
@@ -296998,6 +297309,14 @@ class CEXBroker {
       });
       this.fillArchivePoller.start();
     }
+    if (this.brokerArchiver?.canPersistAccountBalanceSnapshots()) {
+      this.accountBalanceArchivePoller = new AccountBalanceArchivePoller({
+        brokers: this.brokers,
+        archiver: this.brokerArchiver,
+        metrics: this.otelMetrics
+      });
+      this.accountBalanceArchivePoller.start();
+    }
     return this;
   }
 }
@@ -297005,4 +297324,4 @@ export {
   CEXBroker as default
 };
 
-//# debugId=61BAB4069A0158B064756E2164756E21
+//# debugId=186957481DC5AB2064756E2164756E21