@usherlabs/cex-broker 0.2.35 → 0.2.36

package/dist/commands/cli.js

@@ -316541,7 +316541,26 @@ var DEFAULT_CONFIG2 = {
   depositsLimit: 50
 };
 var ALL_CURRENCIES_CODE = "*";
-function nextDepositCursor(deposits, currentSince, depositsLimit) {
+function depositTimestamp(record) {
+  const observedAt = depositField(record, [
+    "timestamp",
+    "creditedAt",
+    "credited_at",
+    "updated",
+    "updatedAt",
+    "datetime"
+  ]);
+  const timestamp = typeof observedAt === "number" ? observedAt : typeof observedAt === "string" ? Date.parse(observedAt) : Number.NaN;
+  return Number.isFinite(timestamp) ? timestamp : undefined;
+}
+function archivedDepositStatus(exchangeId, record) {
+  const rawStatus = asRecord(record.info)?.status;
+  if (exchangeId?.toLowerCase() === "binance" && String(rawStatus ?? "").trim() === "6") {
+    return "credited_not_withdrawable";
+  }
+  return normalizeCcxtTransactionForArchive(record).status;
+}
+function nextDepositCursor(deposits, currentSince, depositsLimit, exchangeId) {
   if (deposits.length >= depositsLimit) {
     return currentSince;
   }
@@ -316552,23 +316571,17 @@ function nextDepositCursor(deposits, currentSince, depositsLimit) {
     if (!record) {
       return currentSince;
     }
-    const observedAt = depositField(record, [
-      "timestamp",
-      "creditedAt",
-      "credited_at",
-      "updated",
-      "updatedAt",
-      "datetime"
-    ]);
-    const timestamp = typeof observedAt === "number" ? observedAt : typeof observedAt === "string" ? Date.parse(observedAt) : Number.NaN;
+    const timestamp = depositTimestamp(record);
+    const archiveStatus = archivedDepositStatus(exchangeId, record);
     const status = normalizeDepositStatus(depositField(record, ["status", "state"]));
-    if (!Number.isFinite(timestamp)) {
-      if (status === "pending") {
+    const isPending = archiveStatus === "credited_not_withdrawable" || status === "pending";
+    if (timestamp === undefined) {
+      if (isPending) {
         return currentSince;
       }
       continue;
     }
-    if (status === "pending") {
+    if (isPending) {
       oldestPending = Math.min(oldestPending, timestamp);
     } else {
       next = Math.max(next, timestamp + 1);
@@ -316583,6 +316596,7 @@ class DepositArchivePoller {
   #stopped = false;
   #running = null;
   #cursors = new Map;
+  #lastArchivedByTarget = new Map;
   #unsupportedLogged = new Set;
   #config;
   constructor(params) {
@@ -316682,7 +316696,7 @@ class DepositArchivePoller {
       ]);
       const txid = depositField(record, ["txid", "txId", "tx_hash", "txHash"]);
       const network = depositField(record, ["network", "chain"]);
-      const archiveStatus = normalizeCcxtTransactionForArchive(record).status;
+      const archiveStatus = archivedDepositStatus(target.exchangeId, record);
       const creditedAt = depositField(record, [
         "creditedAt",
         "credited_at",
@@ -316692,6 +316706,10 @@ class DepositArchivePoller {
         "datetime"
       ]);
       const depositTxid = txid === undefined ? undefined : String(txid);
+      const lastArchived = depositTxid === undefined ? undefined : this.#lastArchivedByTarget.get(key)?.get(depositTxid);
+      if (lastArchived && lastArchived.status === archiveStatus) {
+        continue;
+      }
       this.params.archiver.enqueue(buildTransferEventArchiveRow({
         tags: buildCommonArchiveTags({
           deploymentId: this.params.archiver.getDeploymentId(),
@@ -316712,12 +316730,35 @@ class DepositArchivePoller {
           payload: record
         }
       }));
+      if (depositTxid !== undefined) {
+        let targetDeposits2 = this.#lastArchivedByTarget.get(key);
+        if (!targetDeposits2) {
+          targetDeposits2 = new Map;
+          this.#lastArchivedByTarget.set(key, targetDeposits2);
+        }
+        targetDeposits2.set(depositTxid, {
+          status: archiveStatus,
+          timestamp: depositTimestamp(record)
+        });
+      }
       archived += 1;
     }
     if (archived > 0) {
       this.params.metrics?.recordCounter("cex_deposit_poller_deposits_archived_total", archived, { exchange: target.exchangeId });
     }
-    this.#cursors.set(key, nextDepositCursor(deposits, since, this.#config.depositsLimit));
+    const nextCursor = nextDepositCursor(deposits, since, this.#config.depositsLimit, target.exchangeId);
+    this.#cursors.set(key, nextCursor);
+    const targetDeposits = this.#lastArchivedByTarget.get(key);
+    if (targetDeposits) {
+      for (const [externalId, lastArchived] of targetDeposits) {
+        if (lastArchived.timestamp !== undefined && lastArchived.timestamp < nextCursor) {
+          targetDeposits.delete(externalId);
+        }
+      }
+      if (targetDeposits.size === 0) {
+        this.#lastArchivedByTarget.delete(key);
+      }
+    }
   }
   #targetKey(target) {
     return `${target.exchangeId}|${target.account.label}|${target.code}`;
@@ -316971,6 +317012,7 @@ function toAttributes(labels, service) {
 class OtelMetrics extends BaseOtelSignal {
   counters = new Map;
   histograms = new Map;
+  observableGauges = new Map;
   constructor(config) {
     super(config, "metrics");
   }
@@ -317056,6 +317098,32 @@ class OtelMetrics extends BaseOtelSignal {
       log.error("Failed to record gauge:", error);
     }
   }
+  async setObservableGauge(metricName, value, labels, service = this.getServiceName()) {
+    const provider = this.getProvider();
+    if (!this.isOtelEnabled() || !provider)
+      return;
+    try {
+      let state = this.observableGauges.get(metricName);
+      if (!state) {
+        const observations = new Map;
+        const instrument = provider.getMeter("cex-broker-metrics", "1.0.0").createObservableGauge(metricName, { description: metricName });
+        instrument.addCallback((result) => {
+          for (const observation of observations.values()) {
+            result.observe(observation.value, observation.attributes);
+          }
+        });
+        state = { instrument, observations };
+        this.observableGauges.set(metricName, state);
+      }
+      const attributes = toAttributes(labels, service);
+      state.observations.set(stableAttributeKey(attributes), {
+        value,
+        attributes
+      });
+    } catch (error) {
+      log.error("Failed to set observable gauge:", error);
+    }
+  }
   async recordHistogram(metricName, value, labels, service = this.getServiceName()) {
     const provider = this.getProvider();
     if (!this.isOtelEnabled() || !provider)
@@ -317160,27 +317228,33 @@ function getOtelProtocolFromEnv() {
   const protocol = process.env.CEX_BROKER_OTEL_PROTOCOL ?? process.env.CEX_BROKER_CLICKHOUSE_PROTOCOL;
   return protocol || "http";
 }
-function createOtelMetricsFromEnv() {
+function createOtelMetricsFromEnv(options = {}) {
   const otlpEndpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
-  const host = getOtelHostFromEnv();
+  const serviceName = process.env.OTEL_SERVICE_NAME || options.defaultServiceName || DEFAULT_SERVICE;
   if (otlpEndpoint) {
     return new OtelMetrics({
-      otlpEndpoint: otlpEndpoint.replace(/\/v1\/metrics\/?$/, ""),
-      serviceName: process.env.OTEL_SERVICE_NAME || DEFAULT_SERVICE
+      otlpEndpoint,
+      serviceName
     });
   }
-  if (!host) {
-    return new OtelMetrics;
+  if (options.allowLegacyBrokerConfig === false) {
+    return new OtelMetrics({ serviceName });
   }
+  const host = getOtelHostFromEnv();
+  if (!host)
+    return new OtelMetrics({ serviceName });
   const port = getOtelPortFromEnv();
   const config = {
     host,
     port: port ?? DEFAULT_OTLP_PORT,
     protocol: getOtelProtocolFromEnv(),
-    serviceName: process.env.OTEL_SERVICE_NAME || DEFAULT_SERVICE
+    serviceName
   };
   return new OtelMetrics(config);
 }
+function stableAttributeKey(attributes) {
+  return JSON.stringify(Object.entries(attributes).sort(([left], [right]) => left.localeCompare(right)));
+}
 function createOtelLogsFromEnv() {
   const logsEndpoint = process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT;
   const genericEndpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;

package/dist/helpers/deposit-archive-poller.d.ts

@@ -10,7 +10,7 @@ export type DepositArchivePollerConfig = {
  * Advances the inclusive since-watermark only across a fully observed,
  * terminal prefix of deposit history.
  */
-export declare function nextDepositCursor(deposits: unknown[], currentSince: number, depositsLimit: number): number;
+export declare function nextDepositCursor(deposits: unknown[], currentSince: number, depositsLimit: number, exchangeId?: string): number;
 /**
  * Broker-internal periodic capture of venue deposit history. It polls every
  * configured account sequentially, using ccxt's unfiltered deposit-history

package/dist/helpers/otel.d.ts

@@ -27,6 +27,12 @@ export interface MetricData {
     labels: string;
     service: string;
 }
+export interface OtelMetricsEnvOptions {
+    /** Service name used when OTEL_SERVICE_NAME is not configured. */
+    defaultServiceName?: string;
+    /** Whether broker-specific legacy collector host variables are accepted. */
+    allowLegacyBrokerConfig?: boolean;
+}
 declare abstract class BaseOtelSignal<TProvider> {
     private readonly signal;
     private provider;
@@ -45,6 +51,7 @@ declare abstract class BaseOtelSignal<TProvider> {
 export declare class OtelMetrics extends BaseOtelSignal<MeterProviderType> {
     private readonly counters;
     private readonly histograms;
+    private readonly observableGauges;
     constructor(config?: OtelConfig);
     protected createProvider(endpoint: string, serviceName: string, appendSignalPath: boolean): MeterProviderType;
     protected onProviderCreated(provider: MeterProviderType): void;
@@ -54,6 +61,11 @@ export declare class OtelMetrics extends BaseOtelSignal<MeterProviderType> {
     insertMetrics(metricsList: MetricData[]): Promise<void>;
     recordCounter(metricName: string, value: number, labels: Record<string, string | number>, service?: string): Promise<void>;
     recordGauge(metricName: string, value: number, labels: Record<string, string | number>, service?: string): Promise<void>;
+    /**
+     * Set a gauge value that is observed on every export. This is suitable for
+     * staleness signals where a quiet live process must keep exporting its last value.
+     */
+    setObservableGauge(metricName: string, value: number, labels: Record<string, string | number>, service?: string): Promise<void>;
     recordHistogram(metricName: string, value: number, labels: Record<string, string | number>, service?: string): Promise<void>;
 }
 export declare class OtelLogs extends BaseOtelSignal<LoggerProvider> {
@@ -65,6 +77,6 @@ export declare class OtelLogs extends BaseOtelSignal<LoggerProvider> {
     protected onProviderClosed(): void;
     emit(record: LogRecord): void;
 }
-export declare function createOtelMetricsFromEnv(): OtelMetrics;
+export declare function createOtelMetricsFromEnv(options?: OtelMetricsEnvOptions): OtelMetrics;
 export declare function createOtelLogsFromEnv(): OtelLogs;
 export {};

package/dist/index.js

@@ -292055,7 +292055,26 @@ var DEFAULT_CONFIG2 = {
   depositsLimit: 50
 };
 var ALL_CURRENCIES_CODE = "*";
-function nextDepositCursor(deposits, currentSince, depositsLimit) {
+function depositTimestamp(record) {
+  const observedAt = depositField(record, [
+    "timestamp",
+    "creditedAt",
+    "credited_at",
+    "updated",
+    "updatedAt",
+    "datetime"
+  ]);
+  const timestamp = typeof observedAt === "number" ? observedAt : typeof observedAt === "string" ? Date.parse(observedAt) : Number.NaN;
+  return Number.isFinite(timestamp) ? timestamp : undefined;
+}
+function archivedDepositStatus(exchangeId, record) {
+  const rawStatus = asRecord(record.info)?.status;
+  if (exchangeId?.toLowerCase() === "binance" && String(rawStatus ?? "").trim() === "6") {
+    return "credited_not_withdrawable";
+  }
+  return normalizeCcxtTransactionForArchive(record).status;
+}
+function nextDepositCursor(deposits, currentSince, depositsLimit, exchangeId) {
   if (deposits.length >= depositsLimit) {
     return currentSince;
   }
@@ -292066,23 +292085,17 @@ function nextDepositCursor(deposits, currentSince, depositsLimit) {
     if (!record) {
       return currentSince;
     }
-    const observedAt = depositField(record, [
-      "timestamp",
-      "creditedAt",
-      "credited_at",
-      "updated",
-      "updatedAt",
-      "datetime"
-    ]);
-    const timestamp = typeof observedAt === "number" ? observedAt : typeof observedAt === "string" ? Date.parse(observedAt) : Number.NaN;
+    const timestamp = depositTimestamp(record);
+    const archiveStatus = archivedDepositStatus(exchangeId, record);
     const status = normalizeDepositStatus(depositField(record, ["status", "state"]));
-    if (!Number.isFinite(timestamp)) {
-      if (status === "pending") {
+    const isPending = archiveStatus === "credited_not_withdrawable" || status === "pending";
+    if (timestamp === undefined) {
+      if (isPending) {
         return currentSince;
       }
       continue;
     }
-    if (status === "pending") {
+    if (isPending) {
       oldestPending = Math.min(oldestPending, timestamp);
     } else {
       next = Math.max(next, timestamp + 1);
@@ -292097,6 +292110,7 @@ class DepositArchivePoller {
   #stopped = false;
   #running = null;
   #cursors = new Map;
+  #lastArchivedByTarget = new Map;
   #unsupportedLogged = new Set;
   #config;
   constructor(params) {
@@ -292196,7 +292210,7 @@ class DepositArchivePoller {
       ]);
       const txid = depositField(record, ["txid", "txId", "tx_hash", "txHash"]);
       const network = depositField(record, ["network", "chain"]);
-      const archiveStatus = normalizeCcxtTransactionForArchive(record).status;
+      const archiveStatus = archivedDepositStatus(target.exchangeId, record);
       const creditedAt = depositField(record, [
         "creditedAt",
         "credited_at",
@@ -292206,6 +292220,10 @@ class DepositArchivePoller {
         "datetime"
       ]);
       const depositTxid = txid === undefined ? undefined : String(txid);
+      const lastArchived = depositTxid === undefined ? undefined : this.#lastArchivedByTarget.get(key)?.get(depositTxid);
+      if (lastArchived && lastArchived.status === archiveStatus) {
+        continue;
+      }
       this.params.archiver.enqueue(buildTransferEventArchiveRow({
         tags: buildCommonArchiveTags({
           deploymentId: this.params.archiver.getDeploymentId(),
@@ -292226,12 +292244,35 @@ class DepositArchivePoller {
           payload: record
         }
       }));
+      if (depositTxid !== undefined) {
+        let targetDeposits2 = this.#lastArchivedByTarget.get(key);
+        if (!targetDeposits2) {
+          targetDeposits2 = new Map;
+          this.#lastArchivedByTarget.set(key, targetDeposits2);
+        }
+        targetDeposits2.set(depositTxid, {
+          status: archiveStatus,
+          timestamp: depositTimestamp(record)
+        });
+      }
       archived += 1;
     }
     if (archived > 0) {
       this.params.metrics?.recordCounter("cex_deposit_poller_deposits_archived_total", archived, { exchange: target.exchangeId });
     }
-    this.#cursors.set(key, nextDepositCursor(deposits, since, this.#config.depositsLimit));
+    const nextCursor = nextDepositCursor(deposits, since, this.#config.depositsLimit, target.exchangeId);
+    this.#cursors.set(key, nextCursor);
+    const targetDeposits = this.#lastArchivedByTarget.get(key);
+    if (targetDeposits) {
+      for (const [externalId, lastArchived] of targetDeposits) {
+        if (lastArchived.timestamp !== undefined && lastArchived.timestamp < nextCursor) {
+          targetDeposits.delete(externalId);
+        }
+      }
+      if (targetDeposits.size === 0) {
+        this.#lastArchivedByTarget.delete(key);
+      }
+    }
   }
   #targetKey(target) {
     return `${target.exchangeId}|${target.account.label}|${target.code}`;
@@ -292485,6 +292526,7 @@ function toAttributes(labels, service) {
 class OtelMetrics extends BaseOtelSignal {
   counters = new Map;
   histograms = new Map;
+  observableGauges = new Map;
   constructor(config) {
     super(config, "metrics");
   }
@@ -292570,6 +292612,32 @@ class OtelMetrics extends BaseOtelSignal {
       log.error("Failed to record gauge:", error);
     }
   }
+  async setObservableGauge(metricName, value, labels, service = this.getServiceName()) {
+    const provider = this.getProvider();
+    if (!this.isOtelEnabled() || !provider)
+      return;
+    try {
+      let state = this.observableGauges.get(metricName);
+      if (!state) {
+        const observations = new Map;
+        const instrument = provider.getMeter("cex-broker-metrics", "1.0.0").createObservableGauge(metricName, { description: metricName });
+        instrument.addCallback((result) => {
+          for (const observation of observations.values()) {
+            result.observe(observation.value, observation.attributes);
+          }
+        });
+        state = { instrument, observations };
+        this.observableGauges.set(metricName, state);
+      }
+      const attributes = toAttributes(labels, service);
+      state.observations.set(stableAttributeKey(attributes), {
+        value,
+        attributes
+      });
+    } catch (error) {
+      log.error("Failed to set observable gauge:", error);
+    }
+  }
   async recordHistogram(metricName, value, labels, service = this.getServiceName()) {
     const provider = this.getProvider();
     if (!this.isOtelEnabled() || !provider)
@@ -292674,27 +292742,33 @@ function getOtelProtocolFromEnv() {
   const protocol = process.env.CEX_BROKER_OTEL_PROTOCOL ?? process.env.CEX_BROKER_CLICKHOUSE_PROTOCOL;
   return protocol || "http";
 }
-function createOtelMetricsFromEnv() {
+function createOtelMetricsFromEnv(options = {}) {
   const otlpEndpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
-  const host = getOtelHostFromEnv();
+  const serviceName = process.env.OTEL_SERVICE_NAME || options.defaultServiceName || DEFAULT_SERVICE;
   if (otlpEndpoint) {
     return new OtelMetrics({
-      otlpEndpoint: otlpEndpoint.replace(/\/v1\/metrics\/?$/, ""),
-      serviceName: process.env.OTEL_SERVICE_NAME || DEFAULT_SERVICE
+      otlpEndpoint,
+      serviceName
     });
   }
-  if (!host) {
-    return new OtelMetrics;
+  if (options.allowLegacyBrokerConfig === false) {
+    return new OtelMetrics({ serviceName });
   }
+  const host = getOtelHostFromEnv();
+  if (!host)
+    return new OtelMetrics({ serviceName });
   const port = getOtelPortFromEnv();
   const config = {
     host,
     port: port ?? DEFAULT_OTLP_PORT,
     protocol: getOtelProtocolFromEnv(),
-    serviceName: process.env.OTEL_SERVICE_NAME || DEFAULT_SERVICE
+    serviceName
   };
   return new OtelMetrics(config);
 }
+function stableAttributeKey(attributes) {
+  return JSON.stringify(Object.entries(attributes).sort(([left], [right]) => left.localeCompare(right)));
+}
 function createOtelLogsFromEnv() {
   const logsEndpoint = process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT;
   const genericEndpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
@@ -310486,4 +310560,4 @@ export {
   CEXBroker as default
 };
 
-//# debugId=A02D63306D4E3E3864756E2164756E21
+//# debugId=10F44147004B63EC64756E2164756E21