@usherlabs/cex-broker 0.2.34 → 0.2.36

package/dist/commands/cli.js

@@ -316534,8 +316534,256 @@ class AccountBalanceArchivePoller {
   }
 }
 
-// src/helpers/fill-archive-poller.ts
+// src/helpers/deposit-archive-poller.ts
 var DEFAULT_CONFIG2 = {
+  pollIntervalMs: 60000,
+  lookbackMs: 24 * 60 * 60 * 1000,
+  depositsLimit: 50
+};
+var ALL_CURRENCIES_CODE = "*";
+function depositTimestamp(record) {
+  const observedAt = depositField(record, [
+    "timestamp",
+    "creditedAt",
+    "credited_at",
+    "updated",
+    "updatedAt",
+    "datetime"
+  ]);
+  const timestamp = typeof observedAt === "number" ? observedAt : typeof observedAt === "string" ? Date.parse(observedAt) : Number.NaN;
+  return Number.isFinite(timestamp) ? timestamp : undefined;
+}
+function archivedDepositStatus(exchangeId, record) {
+  const rawStatus = asRecord(record.info)?.status;
+  if (exchangeId?.toLowerCase() === "binance" && String(rawStatus ?? "").trim() === "6") {
+    return "credited_not_withdrawable";
+  }
+  return normalizeCcxtTransactionForArchive(record).status;
+}
+function nextDepositCursor(deposits, currentSince, depositsLimit, exchangeId) {
+  if (deposits.length >= depositsLimit) {
+    return currentSince;
+  }
+  let next = currentSince;
+  let oldestPending = Number.POSITIVE_INFINITY;
+  for (const deposit of deposits) {
+    const record = asRecord(deposit);
+    if (!record) {
+      return currentSince;
+    }
+    const timestamp = depositTimestamp(record);
+    const archiveStatus = archivedDepositStatus(exchangeId, record);
+    const status = normalizeDepositStatus(depositField(record, ["status", "state"]));
+    const isPending = archiveStatus === "credited_not_withdrawable" || status === "pending";
+    if (timestamp === undefined) {
+      if (isPending) {
+        return currentSince;
+      }
+      continue;
+    }
+    if (isPending) {
+      oldestPending = Math.min(oldestPending, timestamp);
+    } else {
+      next = Math.max(next, timestamp + 1);
+    }
+  }
+  return Number.isFinite(oldestPending) ? Math.max(currentSince, oldestPending) : next;
+}
+
+class DepositArchivePoller {
+  params;
+  #timer = null;
+  #stopped = false;
+  #running = null;
+  #cursors = new Map;
+  #lastArchivedByTarget = new Map;
+  #unsupportedLogged = new Set;
+  #config;
+  constructor(params) {
+    this.params = params;
+    this.#config = { ...DEFAULT_CONFIG2, ...params.config };
+  }
+  start() {
+    if (this.#timer || this.#stopped || !this.params.archiver.isEnabled()) {
+      return;
+    }
+    log.info("\uD83D\uDCE5 Deposit archive poller started");
+    this.#schedule(0);
+  }
+  async stop() {
+    this.#stopped = true;
+    if (this.#timer) {
+      clearTimeout(this.#timer);
+      this.#timer = null;
+    }
+    await this.#running;
+  }
+  async pollAllOnce() {
+    if (this.#stopped || this.#running || !this.params.archiver.isEnabled()) {
+      return false;
+    }
+    this.#running = this.#pollAllSequentially();
+    try {
+      return await this.#running;
+    } finally {
+      this.#running = null;
+    }
+  }
+  #targets() {
+    const targets = [];
+    for (const [exchangeId, pool] of Object.entries(this.params.brokers)) {
+      for (const account of [pool.primary, ...pool.secondaryBrokers]) {
+        targets.push({
+          exchangeId,
+          account,
+          code: ALL_CURRENCIES_CODE
+        });
+      }
+    }
+    return targets;
+  }
+  async#pollAllSequentially() {
+    for (const target of this.#targets()) {
+      if (this.#stopped) {
+        break;
+      }
+      await this.#pollOne(target);
+    }
+    return true;
+  }
+  async#pollOne(target) {
+    const exchange = target.account.exchange;
+    const key = this.#targetKey(target);
+    if (typeof exchange.fetchDeposits !== "function" || exchange.has?.fetchDeposits === false) {
+      if (!this.#unsupportedLogged.has(key)) {
+        this.#unsupportedLogged.add(key);
+        log.info("Deposit archive poll skipped: fetchDeposits unsupported", {
+          exchange: target.exchangeId,
+          account: target.account.label
+        });
+      }
+      return;
+    }
+    const since = this.#cursors.get(key) ?? Date.now() - this.#config.lookbackMs;
+    let deposits;
+    try {
+      deposits = await exchange.fetchDeposits(undefined, since, this.#config.depositsLimit);
+    } catch (error) {
+      this.params.metrics?.recordCounter("cex_deposit_poller_errors_total", 1, { exchange: target.exchangeId });
+      log.warn("Deposit archive poll failed", {
+        exchange: target.exchangeId,
+        account: target.account.label,
+        error
+      });
+      return;
+    }
+    if (!Array.isArray(deposits) || deposits.length === 0) {
+      return;
+    }
+    let archived = 0;
+    for (const deposit of deposits) {
+      const record = asRecord(deposit);
+      if (!record) {
+        continue;
+      }
+      const assetSymbol = depositField(record, ["currency", "code", "asset"]);
+      const amount = depositField(record, ["amount"]);
+      const address = depositField(record, [
+        "address",
+        "recipientAddress",
+        "to",
+        "destination"
+      ]);
+      const txid = depositField(record, ["txid", "txId", "tx_hash", "txHash"]);
+      const network = depositField(record, ["network", "chain"]);
+      const archiveStatus = archivedDepositStatus(target.exchangeId, record);
+      const creditedAt = depositField(record, [
+        "creditedAt",
+        "credited_at",
+        "updated",
+        "updatedAt",
+        "timestamp",
+        "datetime"
+      ]);
+      const depositTxid = txid === undefined ? undefined : String(txid);
+      const lastArchived = depositTxid === undefined ? undefined : this.#lastArchivedByTarget.get(key)?.get(depositTxid);
+      if (lastArchived && lastArchived.status === archiveStatus) {
+        continue;
+      }
+      this.params.archiver.enqueue(buildTransferEventArchiveRow({
+        tags: buildCommonArchiveTags({
+          deploymentId: this.params.archiver.getDeploymentId(),
+          accountSelector: target.account.label,
+          exchange: target.exchangeId,
+          symbol: assetSymbol === undefined ? undefined : String(assetSymbol)
+        }),
+        transfer: {
+          eventKind: "deposit",
+          lifecycleAction: "observe_deposit",
+          status: archiveStatus,
+          amount: amount === undefined ? undefined : String(amount),
+          address: address === undefined ? undefined : String(address),
+          network: network === undefined ? undefined : String(network),
+          externalId: depositTxid,
+          txid: depositTxid,
+          exchangeTimestamp: typeof creditedAt === "string" ? creditedAt : undefined,
+          payload: record
+        }
+      }));
+      if (depositTxid !== undefined) {
+        let targetDeposits2 = this.#lastArchivedByTarget.get(key);
+        if (!targetDeposits2) {
+          targetDeposits2 = new Map;
+          this.#lastArchivedByTarget.set(key, targetDeposits2);
+        }
+        targetDeposits2.set(depositTxid, {
+          status: archiveStatus,
+          timestamp: depositTimestamp(record)
+        });
+      }
+      archived += 1;
+    }
+    if (archived > 0) {
+      this.params.metrics?.recordCounter("cex_deposit_poller_deposits_archived_total", archived, { exchange: target.exchangeId });
+    }
+    const nextCursor = nextDepositCursor(deposits, since, this.#config.depositsLimit, target.exchangeId);
+    this.#cursors.set(key, nextCursor);
+    const targetDeposits = this.#lastArchivedByTarget.get(key);
+    if (targetDeposits) {
+      for (const [externalId, lastArchived] of targetDeposits) {
+        if (lastArchived.timestamp !== undefined && lastArchived.timestamp < nextCursor) {
+          targetDeposits.delete(externalId);
+        }
+      }
+      if (targetDeposits.size === 0) {
+        this.#lastArchivedByTarget.delete(key);
+      }
+    }
+  }
+  #targetKey(target) {
+    return `${target.exchangeId}|${target.account.label}|${target.code}`;
+  }
+  #schedule(delayMs) {
+    this.#timer = setTimeout(() => void this.#tick(), delayMs);
+    this.#timer.unref?.();
+  }
+  async#tick() {
+    this.#timer = null;
+    try {
+      await this.pollAllOnce();
+    } catch (error) {
+      rethrowArchiveDurabilityError(error);
+      log.error("Deposit archive poller tick failed", error);
+    } finally {
+      if (!this.#stopped) {
+        this.#schedule(this.#config.pollIntervalMs);
+      }
+    }
+  }
+}
+
+// src/helpers/fill-archive-poller.ts
+var DEFAULT_CONFIG3 = {
   pollIntervalMs: 60000,
   lookbackMs: 24 * 60 * 60 * 1000,
   tradesLimit: 500
@@ -316560,7 +316808,7 @@ class FillArchivePoller {
   #config;
   constructor(params) {
     this.params = params;
-    this.#config = { ...DEFAULT_CONFIG2, ...params.config };
+    this.#config = { ...DEFAULT_CONFIG3, ...params.config };
   }
   start() {
     if (this.#timer || this.#stopped) {
@@ -316764,6 +317012,7 @@ function toAttributes(labels, service) {
 class OtelMetrics extends BaseOtelSignal {
   counters = new Map;
   histograms = new Map;
+  observableGauges = new Map;
   constructor(config) {
     super(config, "metrics");
   }
@@ -316849,6 +317098,32 @@ class OtelMetrics extends BaseOtelSignal {
       log.error("Failed to record gauge:", error);
     }
   }
+  async setObservableGauge(metricName, value, labels, service = this.getServiceName()) {
+    const provider = this.getProvider();
+    if (!this.isOtelEnabled() || !provider)
+      return;
+    try {
+      let state = this.observableGauges.get(metricName);
+      if (!state) {
+        const observations = new Map;
+        const instrument = provider.getMeter("cex-broker-metrics", "1.0.0").createObservableGauge(metricName, { description: metricName });
+        instrument.addCallback((result) => {
+          for (const observation of observations.values()) {
+            result.observe(observation.value, observation.attributes);
+          }
+        });
+        state = { instrument, observations };
+        this.observableGauges.set(metricName, state);
+      }
+      const attributes = toAttributes(labels, service);
+      state.observations.set(stableAttributeKey(attributes), {
+        value,
+        attributes
+      });
+    } catch (error) {
+      log.error("Failed to set observable gauge:", error);
+    }
+  }
   async recordHistogram(metricName, value, labels, service = this.getServiceName()) {
     const provider = this.getProvider();
     if (!this.isOtelEnabled() || !provider)
@@ -316953,27 +317228,33 @@ function getOtelProtocolFromEnv() {
   const protocol = process.env.CEX_BROKER_OTEL_PROTOCOL ?? process.env.CEX_BROKER_CLICKHOUSE_PROTOCOL;
   return protocol || "http";
 }
-function createOtelMetricsFromEnv() {
+function createOtelMetricsFromEnv(options = {}) {
   const otlpEndpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
-  const host = getOtelHostFromEnv();
+  const serviceName = process.env.OTEL_SERVICE_NAME || options.defaultServiceName || DEFAULT_SERVICE;
   if (otlpEndpoint) {
     return new OtelMetrics({
-      otlpEndpoint: otlpEndpoint.replace(/\/v1\/metrics\/?$/, ""),
-      serviceName: process.env.OTEL_SERVICE_NAME || DEFAULT_SERVICE
+      otlpEndpoint,
+      serviceName
     });
   }
-  if (!host) {
-    return new OtelMetrics;
+  if (options.allowLegacyBrokerConfig === false) {
+    return new OtelMetrics({ serviceName });
   }
+  const host = getOtelHostFromEnv();
+  if (!host)
+    return new OtelMetrics({ serviceName });
   const port = getOtelPortFromEnv();
   const config = {
     host,
     port: port ?? DEFAULT_OTLP_PORT,
     protocol: getOtelProtocolFromEnv(),
-    serviceName: process.env.OTEL_SERVICE_NAME || DEFAULT_SERVICE
+    serviceName
   };
   return new OtelMetrics(config);
 }
+function stableAttributeKey(attributes) {
+  return JSON.stringify(Object.entries(attributes).sort(([left], [right]) => left.localeCompare(right)));
+}
 function createOtelLogsFromEnv() {
   const logsEndpoint = process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT;
   const genericEndpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
@@ -330786,6 +331067,12 @@ function parseActionPayload(schema, rawPayload, callback) {
 // src/helpers/grpc/status.ts
 var grpc2 = __toESM(require_src3(), 1);
 function stableGrpcErrorCode(message) {
+  if (message.startsWith("AuthenticationError:")) {
+    return grpc2.status.UNAUTHENTICATED;
+  }
+  if (message.startsWith("InsufficientFunds:")) {
+    return grpc2.status.FAILED_PRECONDITION;
+  }
   if (message.startsWith("venue_discovery_unavailable:")) {
     return grpc2.status.UNIMPLEMENTED;
   }
@@ -330936,7 +331223,8 @@ async function handleDeposit(ctx) {
           message: `deposit_address_mismatch: expected address ${value.recipientAddress}, observed ${String(observedAddress)}`
         }, null);
       }
-      const status4 = normalizeDepositStatus(depositField(deposit, ["status", "state"]));
+      const responseStatus = normalizeDepositStatus(depositField(deposit, ["status", "state"]));
+      const archiveStatus = normalizeCcxtTransactionForArchive(deposit).status;
       const depositTxid = String(depositField(deposit, ["txid", "txId", "tx_hash", "txHash"]) ?? value.transactionHash);
       const creditedAt = depositField(deposit, [
         "creditedAt",
@@ -330953,7 +331241,7 @@ async function handleDeposit(ctx) {
         transfer: {
           eventKind: "deposit",
           lifecycleAction: "observe_deposit",
-          status: status4,
+          status: archiveStatus,
           amount: observedAmount !== undefined ? String(observedAmount) : undefined,
           address: String(observedAddress ?? value.recipientAddress),
           network: depositNetwork?.exchangeNetworkId,
@@ -330967,7 +331255,7 @@ async function handleDeposit(ctx) {
       return ctx.wrappedCallback(null, {
         proof: ctx.verity.proof,
         result: JSON.stringify({
-          status: status4,
+          status: responseStatus,
           exchange: normalizedCex,
           accountSelector: selectedBrokerAccount?.label,
           asset: symbol2,
@@ -331486,6 +331774,12 @@ function identifiesUnsupported(message) {
   return /post[\s-]?only\b.*\b(?:not supported|unsupported|does not support)\b/.test(normalized) || /\b(?:not supported|unsupported|does not support)\b.*\bpost[\s-]?only\b/.test(normalized);
 }
 function classifyPassiveOrderError(error48) {
+  if (error48 instanceof ccxt_default.InsufficientFunds) {
+    return "InsufficientFunds";
+  }
+  if (error48 instanceof ccxt_default.AuthenticationError) {
+    return "AuthenticationError";
+  }
   const message = getErrorMessage(error48);
   if (error48 instanceof ccxt_default.OrderImmediatelyFillable || identifiesWouldCross(message)) {
     return PASSIVE_ORDER_ERROR_CODES.wouldCross;
@@ -331614,9 +331908,9 @@ async function handleCreateOrder(ctx) {
     emitOrderExecutionTelemetryInBackground(otelMetrics, failedCreateContext, undefined, error48);
     archiveOrderExecutionInBackground(brokerArchiver, failedCreateContext, undefined, error48, { marketMetadataHash });
     if (isPassiveOrder && submission === "in_flight") {
-      const passiveErrorCode = classifyPassiveOrderError(error48);
+      const stableErrorCode = classifyPassiveOrderError(error48);
       return rejectWithGrpcError(ctx, error48, {
-        message: `${passiveErrorCode}: ${sanitizeErrorDetail(error48)}`,
+        message: `${stableErrorCode}: ${sanitizeErrorDetail(error48)}`,
         preferStableMessageOnly: true
       });
     }
@@ -334516,6 +334810,7 @@ class CEXBroker {
   orderActivityTracker = new OrderActivityTracker;
   withdrawalObservationTracker = new WithdrawalObservationTracker;
   fillArchivePoller;
+  depositArchivePoller;
   accountBalanceArchivePoller;
   loadEnvConfig() {
     log.info("\uD83D\uDD27 Loading CEX_BROKER_ environment variables:");
@@ -334661,6 +334956,10 @@ class CEXBroker {
       this.fillArchivePoller.stop();
       this.fillArchivePoller = undefined;
     }
+    if (this.depositArchivePoller) {
+      await this.depositArchivePoller.stop();
+      this.depositArchivePoller = undefined;
+    }
     if (this.accountBalanceArchivePoller) {
       await this.accountBalanceArchivePoller.stop();
       this.accountBalanceArchivePoller = undefined;
@@ -334690,6 +334989,10 @@ class CEXBroker {
       this.fillArchivePoller.stop();
       this.fillArchivePoller = undefined;
     }
+    if (this.depositArchivePoller) {
+      await this.depositArchivePoller.stop();
+      this.depositArchivePoller = undefined;
+    }
     if (this.accountBalanceArchivePoller) {
       await this.accountBalanceArchivePoller.stop();
       this.accountBalanceArchivePoller = undefined;
@@ -334721,6 +335024,12 @@ class CEXBroker {
         metrics: this.otelMetrics
       });
       this.fillArchivePoller.start();
+      this.depositArchivePoller = new DepositArchivePoller({
+        brokers: this.brokers,
+        archiver: this.brokerArchiver,
+        metrics: this.otelMetrics
+      });
+      this.depositArchivePoller.start();
     }
     if (this.brokerArchiver?.canPersistAccountBalanceSnapshots()) {
       this.accountBalanceArchivePoller = new AccountBalanceArchivePoller({

package/dist/helpers/deposit-archive-poller.d.ts

@@ -0,0 +1,32 @@
+import type { BrokerPoolEntry } from "./broker";
+import { type BrokerExecutionArchiver } from "./broker-execution-archive";
+import type { OtelMetrics } from "./otel";
+export type DepositArchivePollerConfig = {
+    pollIntervalMs: number;
+    lookbackMs: number;
+    depositsLimit: number;
+};
+/**
+ * Advances the inclusive since-watermark only across a fully observed,
+ * terminal prefix of deposit history.
+ */
+export declare function nextDepositCursor(deposits: unknown[], currentSince: number, depositsLimit: number, exchangeId?: string): number;
+/**
+ * Broker-internal periodic capture of venue deposit history. It polls every
+ * configured account sequentially, using ccxt's unfiltered deposit-history
+ * surface so newly funded currencies do not depend on balance or market
+ * discovery.
+ */
+export declare class DepositArchivePoller {
+    #private;
+    private readonly params;
+    constructor(params: {
+        brokers: Record<string, BrokerPoolEntry>;
+        archiver: BrokerExecutionArchiver;
+        metrics?: OtelMetrics;
+        config?: Partial<DepositArchivePollerConfig>;
+    });
+    start(): void;
+    stop(): Promise<void>;
+    pollAllOnce(): Promise<boolean>;
+}

package/dist/helpers/otel.d.ts

@@ -27,6 +27,12 @@ export interface MetricData {
     labels: string;
     service: string;
 }
+export interface OtelMetricsEnvOptions {
+    /** Service name used when OTEL_SERVICE_NAME is not configured. */
+    defaultServiceName?: string;
+    /** Whether broker-specific legacy collector host variables are accepted. */
+    allowLegacyBrokerConfig?: boolean;
+}
 declare abstract class BaseOtelSignal<TProvider> {
     private readonly signal;
     private provider;
@@ -45,6 +51,7 @@ declare abstract class BaseOtelSignal<TProvider> {
 export declare class OtelMetrics extends BaseOtelSignal<MeterProviderType> {
     private readonly counters;
     private readonly histograms;
+    private readonly observableGauges;
     constructor(config?: OtelConfig);
     protected createProvider(endpoint: string, serviceName: string, appendSignalPath: boolean): MeterProviderType;
     protected onProviderCreated(provider: MeterProviderType): void;
@@ -54,6 +61,11 @@ export declare class OtelMetrics extends BaseOtelSignal<MeterProviderType> {
     insertMetrics(metricsList: MetricData[]): Promise<void>;
     recordCounter(metricName: string, value: number, labels: Record<string, string | number>, service?: string): Promise<void>;
     recordGauge(metricName: string, value: number, labels: Record<string, string | number>, service?: string): Promise<void>;
+    /**
+     * Set a gauge value that is observed on every export. This is suitable for
+     * staleness signals where a quiet live process must keep exporting its last value.
+     */
+    setObservableGauge(metricName: string, value: number, labels: Record<string, string | number>, service?: string): Promise<void>;
     recordHistogram(metricName: string, value: number, labels: Record<string, string | number>, service?: string): Promise<void>;
 }
 export declare class OtelLogs extends BaseOtelSignal<LoggerProvider> {
@@ -65,6 +77,6 @@ export declare class OtelLogs extends BaseOtelSignal<LoggerProvider> {
     protected onProviderClosed(): void;
     emit(record: LogRecord): void;
 }
-export declare function createOtelMetricsFromEnv(): OtelMetrics;
+export declare function createOtelMetricsFromEnv(options?: OtelMetricsEnvOptions): OtelMetrics;
 export declare function createOtelLogsFromEnv(): OtelLogs;
 export {};

package/dist/helpers/passive-order.d.ts

@@ -4,4 +4,5 @@ export declare const PASSIVE_ORDER_ERROR_CODES: {
     readonly wouldCross: "passive_order_would_cross";
 };
 export type PassiveOrderErrorCode = (typeof PASSIVE_ORDER_ERROR_CODES)[keyof typeof PASSIVE_ORDER_ERROR_CODES];
-export declare function classifyPassiveOrderError(error: unknown): PassiveOrderErrorCode;
+export type PassiveOrderSubmissionErrorCode = PassiveOrderErrorCode | "AuthenticationError" | "InsufficientFunds";
+export declare function classifyPassiveOrderError(error: unknown): PassiveOrderSubmissionErrorCode;

package/dist/index.d.ts

@@ -16,6 +16,7 @@ export default class CEXBroker {
     private readonly orderActivityTracker;
     private readonly withdrawalObservationTracker;
     private fillArchivePoller?;
+    private depositArchivePoller?;
     private accountBalanceArchivePoller?;
     /**
      * Loads environment variables prefixed with CEX_BROKER_