@usherlabs/cex-broker 0.2.25 → 0.2.27

package/dist/commands/cli.js

@@ -315929,6 +315929,7 @@ var DEFAULT_MAX_QUEUE_SIZE = 1e4;
 var DEFAULT_BATCH_SIZE = 10;
 var DEFAULT_FLUSH_INTERVAL_MS = 1000;
 var DEFAULT_FORWARDER_TIMEOUT_MS = 3000;
+var SHED_WARN_INTERVAL_MS = 60000;
 var DEFAULT_ARCHIVE_FORWARDER_PATH = "/archive";
 var DEFAULT_ARCHIVE_FORWARDER_PORT = 8090;
 function isArchiveOtelLogsEnabled() {
@@ -315968,6 +315969,7 @@ class BrokerExecutionArchiver {
   };
   flushTimer = null;
   flushInFlight = null;
+  lastShedWarnAtMs = 0;
   closed = false;
   loggedMissingMarketForwarder = false;
   enabled;
@@ -316032,6 +316034,15 @@ class BrokerExecutionArchiver {
       this.recordArchiveMetric("cex_archive_rows_shed_total", {
         table: row.table
       });
+      const now3 = Date.now();
+      if (now3 - this.lastShedWarnAtMs >= SHED_WARN_INTERVAL_MS) {
+        log.warn("Archive queue full: shedding oldest rows", {
+          shed_total: this.stats.shed,
+          queue_max: this.maxQueueSize,
+          table: row.table
+        });
+        this.lastShedWarnAtMs = now3;
+      }
     }
     this.queue.push(row);
     this.stats.enqueued += 1;
@@ -332420,6 +332431,59 @@ function createExecuteActionHandler(deps) {
     }
   };
 }
+// src/handlers/subscribe/broker-lifecycle.ts
+class SubscribeBrokerLifecycle {
+  #brokers = new Map;
+  #closing = new Map;
+  #shuttingDown = false;
+  register(broker, context2) {
+    this.#brokers.set(broker, context2);
+    if (this.#shuttingDown) {
+      this.close(broker);
+    }
+  }
+  close(broker) {
+    const existing = this.#closing.get(broker);
+    if (existing) {
+      return existing;
+    }
+    const context2 = this.#brokers.get(broker) ?? {
+      cex: "unknown",
+      symbol: "unknown"
+    };
+    this.#brokers.delete(broker);
+    const closing = (async () => {
+      try {
+        await broker.close();
+        log.debug("Request-scoped Subscribe broker closed", context2);
+        return "closed";
+      } catch (error48) {
+        log.warn("Failed to close request-scoped Subscribe broker", {
+          ...context2,
+          error: error48
+        });
+        return "failed";
+      } finally {
+        this.#closing.delete(broker);
+      }
+    })();
+    this.#closing.set(broker, closing);
+    return closing;
+  }
+  async closeAll() {
+    this.#shuttingDown = true;
+    let failed = 0;
+    while (this.#brokers.size > 0 || this.#closing.size > 0) {
+      const inFlight = [...this.#closing.values()];
+      const fresh = [...this.#brokers.keys()].map((broker) => this.close(broker));
+      const outcomes = await Promise.all([...fresh, ...inFlight]);
+      failed += outcomes.filter((outcome) => outcome === "failed").length;
+    }
+    if (failed > 0) {
+      throw new Error(`${failed} request-scoped Subscribe broker(s) failed to close`);
+    }
+  }
+}
 // src/handlers/subscribe/handler.ts
 var grpc13 = __toESM(require_src3(), 1);
 
@@ -332769,19 +332833,26 @@ class OhlcvBarTracker {
     if (!firstBar || !lastBar) {
       return [];
     }
-    if (this.lastOpenTimeMs !== null && lastBar.openTimeMs < this.lastOpenTimeMs) {
+    const lastOpenTimeMs = this.lastOpenTimeMs;
+    if (lastOpenTimeMs !== null && lastBar.openTimeMs < lastOpenTimeMs) {
+      return [];
+    }
+    const barsToProcess = lastOpenTimeMs === null ? bars : bars.filter((bar) => bar.openTimeMs >= lastOpenTimeMs);
+    const firstBarToProcess = barsToProcess[0];
+    const lastBarToProcess = barsToProcess[barsToProcess.length - 1];
+    if (!firstBarToProcess || !lastBarToProcess) {
       return [];
     }
     const candidates = [];
-    if (this.lastBar !== null && this.lastOpenTimeMs !== null && !bars.some((bar) => bar.openTimeMs === this.lastOpenTimeMs) && this.lastOpenTimeMs < firstBar.openTimeMs) {
+    if (this.lastBar !== null && lastOpenTimeMs !== null && lastOpenTimeMs < firstBarToProcess.openTimeMs) {
       candidates.push({
         bar: this.lastBar,
         isClosed: true,
         brokerVersion
       });
     }
-    for (let index2 = 0;index2 < bars.length - 1; index2 += 1) {
-      const bar = bars[index2];
+    for (let index2 = 0;index2 < barsToProcess.length - 1; index2 += 1) {
+      const bar = barsToProcess[index2];
       if (bar) {
         candidates.push({
           bar,
@@ -332791,12 +332862,12 @@ class OhlcvBarTracker {
       }
     }
     candidates.push({
-      bar: lastBar,
+      bar: lastBarToProcess,
       isClosed: false,
       brokerVersion
     });
-    this.lastOpenTimeMs = lastBar.openTimeMs;
-    this.lastBar = lastBar;
+    this.lastOpenTimeMs = lastBarToProcess.openTimeMs;
+    this.lastBar = lastBarToProcess;
     return candidates;
   }
 }
@@ -333496,15 +333567,34 @@ async function runCcxtSubscribeLoop(call, isClosed, symbol2, subscriptionType, w
 }
 function createSubscribeHandler(deps) {
   const { brokers, whitelistIps, otelMetrics, brokerArchiver } = deps;
+  const brokerLifecycle = deps.brokerLifecycle ?? new SubscribeBrokerLifecycle;
   return async (call) => {
     const subscribeStartTime = Date.now();
     let streamClosed = false;
+    let ownedBroker = null;
+    let ownedBrokerClosePromise;
     const markStreamClosed = () => {
       streamClosed = true;
     };
-    const isStreamClosed = () => streamClosed || call.destroyed;
-    call.once("close", markStreamClosed);
+    const isStreamClosed = () => streamClosed || call.cancelled || call.writableEnded;
+    const closeOwnedBroker = () => {
+      if (ownedBrokerClosePromise) {
+        return ownedBrokerClosePromise;
+      }
+      if (!ownedBroker) {
+        return Promise.resolve();
+      }
+      const broker = ownedBroker;
+      ownedBroker = null;
+      ownedBrokerClosePromise = brokerLifecycle.close(broker);
+      return ownedBrokerClosePromise;
+    };
+    const closeOwnedBrokerOnCallEnd = () => {
+      closeOwnedBroker();
+    };
     call.once("cancelled", markStreamClosed);
+    call.once("cancelled", closeOwnedBrokerOnCallEnd);
+    call.once("error", closeOwnedBrokerOnCallEnd);
     call.once("end", () => {
       markStreamClosed();
       log.info("Subscribe stream ended");
@@ -333576,6 +333666,17 @@ function createSubscribeHandler(deps) {
         });
         return;
       }
+      if (!selectedBrokerAccount) {
+        ownedBroker = broker;
+        brokerLifecycle.register(broker, {
+          cex: normalizedCex,
+          symbol: symbol2
+        });
+        if (isStreamClosed()) {
+          await closeOwnedBroker();
+          return;
+        }
+      }
       const resolvedSymbol = await resolveSubscriptionSymbol(broker, symbol2, options?.marketType);
       const assetType = parseMarketType(options?.marketType);
       const deploymentId = brokerArchiver?.getDeploymentId() ?? "unknown";
@@ -333751,7 +333852,7 @@ function createSubscribeHandler(deps) {
               }
             }
             while (ohlcvStreamActive && !isStreamClosed()) {
-              const data = await broker.fetchOHLCVWs(resolvedSymbol, timeframe);
+              const data = await broker.watchOHLCV(resolvedSymbol, timeframe);
               const receivedTimestamp = Date.now();
               if (!await writeSubscribeFrame(call, isStreamClosed, {
                 data: JSON.stringify(data),
@@ -333840,6 +333941,10 @@ function createSubscribeHandler(deps) {
         symbol: "",
         type: subscriptionType
       });
+    } finally {
+      call.off("cancelled", closeOwnedBrokerOnCallEnd);
+      call.off("error", closeOwnedBrokerOnCallEnd);
+      await closeOwnedBroker();
     }
   };
 }
@@ -333990,7 +334095,7 @@ var CEX_BROKER_PACKAGE_DEFINITION = protoLoader.fromJSON(node_descriptor_default
 // src/server.ts
 var grpcObj = grpc14.loadPackageDefinition(CEX_BROKER_PACKAGE_DEFINITION);
 var cexNode = grpcObj.cex_broker;
-function getServer(policy, brokers, whitelistIps, useVerity, verityProverUrl, otelMetrics, brokerArchiver, orderActivityTracker, withdrawalObservationTracker) {
+function getServer(policy, brokers, whitelistIps, useVerity, verityProverUrl, otelMetrics, brokerArchiver, orderActivityTracker, withdrawalObservationTracker, subscribeBrokerLifecycle) {
   const server = new grpc14.Server;
   server.addService(cexNode.cex_service.service, {
     ExecuteAction: createExecuteActionHandler({
@@ -334008,7 +334113,8 @@ function getServer(policy, brokers, whitelistIps, useVerity, verityProverUrl, ot
       brokers,
       whitelistIps,
       otelMetrics,
-      brokerArchiver
+      brokerArchiver,
+      brokerLifecycle: subscribeBrokerLifecycle
     })
   });
   return server;

package/dist/handlers/subscribe/broker-lifecycle.d.ts

@@ -0,0 +1,13 @@
+import type { Exchange } from "@usherlabs/ccxt";
+type BrokerContext = {
+    cex: string;
+    symbol: string;
+};
+export type BrokerCloseOutcome = "closed" | "failed";
+export declare class SubscribeBrokerLifecycle {
+    #private;
+    register(broker: Exchange, context: BrokerContext): void;
+    close(broker: Exchange): Promise<BrokerCloseOutcome>;
+    closeAll(): Promise<void>;
+}
+export {};

package/dist/handlers/subscribe/handler.d.ts

@@ -3,11 +3,13 @@ import { type BrokerPoolEntry } from "../../helpers/broker";
 import type { BrokerExecutionArchiver } from "../../helpers/broker-execution-archive";
 import type { OtelMetrics } from "../../helpers/otel";
 import type { SubscribeRequest, SubscribeResponse } from "../types";
+import { SubscribeBrokerLifecycle } from "./broker-lifecycle";
 export type SubscribeDeps = {
     brokers: Record<string, BrokerPoolEntry>;
     whitelistIps: string[];
     otelMetrics?: OtelMetrics;
     brokerArchiver?: BrokerExecutionArchiver;
+    brokerLifecycle?: SubscribeBrokerLifecycle;
 };
 type SubscribeCall = grpc.ServerWritableStream<SubscribeRequest, SubscribeResponse>;
 export declare function createSubscribeHandler(deps: SubscribeDeps): (call: SubscribeCall) => Promise<void>;

package/dist/handlers/subscribe/index.d.ts

@@ -1 +1,2 @@
+export { SubscribeBrokerLifecycle } from "./broker-lifecycle";
 export { createSubscribeHandler, type SubscribeDeps } from "./handler";

package/dist/helpers/broker-execution-archive/writer.d.ts

@@ -32,6 +32,7 @@ export declare class BrokerExecutionArchiver {
     private readonly stats;
     private flushTimer;
     private flushInFlight;
+    private lastShedWarnAtMs;
     private closed;
     private loggedMissingMarketForwarder;
     private readonly enabled;

package/dist/index.js

@@ -275309,6 +275309,7 @@ var DEFAULT_MAX_QUEUE_SIZE = 1e4;
 var DEFAULT_BATCH_SIZE = 10;
 var DEFAULT_FLUSH_INTERVAL_MS = 1000;
 var DEFAULT_FORWARDER_TIMEOUT_MS = 3000;
+var SHED_WARN_INTERVAL_MS = 60000;
 var DEFAULT_ARCHIVE_FORWARDER_PATH = "/archive";
 var DEFAULT_ARCHIVE_FORWARDER_PORT = 8090;
 function isArchiveOtelLogsEnabled() {
@@ -275348,6 +275349,7 @@ class BrokerExecutionArchiver {
   };
   flushTimer = null;
   flushInFlight = null;
+  lastShedWarnAtMs = 0;
   closed = false;
   loggedMissingMarketForwarder = false;
   enabled;
@@ -275412,6 +275414,15 @@ class BrokerExecutionArchiver {
       this.recordArchiveMetric("cex_archive_rows_shed_total", {
         table: row.table
       });
+      const now3 = Date.now();
+      if (now3 - this.lastShedWarnAtMs >= SHED_WARN_INTERVAL_MS) {
+        log.warn("Archive queue full: shedding oldest rows", {
+          shed_total: this.stats.shed,
+          queue_max: this.maxQueueSize,
+          table: row.table
+        });
+        this.lastShedWarnAtMs = now3;
+      }
     }
     this.queue.push(row);
     this.stats.enqueued += 1;
@@ -295372,6 +295383,59 @@ function createExecuteActionHandler(deps) {
     }
   };
 }
+// src/handlers/subscribe/broker-lifecycle.ts
+class SubscribeBrokerLifecycle {
+  #brokers = new Map;
+  #closing = new Map;
+  #shuttingDown = false;
+  register(broker, context3) {
+    this.#brokers.set(broker, context3);
+    if (this.#shuttingDown) {
+      this.close(broker);
+    }
+  }
+  close(broker) {
+    const existing = this.#closing.get(broker);
+    if (existing) {
+      return existing;
+    }
+    const context3 = this.#brokers.get(broker) ?? {
+      cex: "unknown",
+      symbol: "unknown"
+    };
+    this.#brokers.delete(broker);
+    const closing = (async () => {
+      try {
+        await broker.close();
+        log.debug("Request-scoped Subscribe broker closed", context3);
+        return "closed";
+      } catch (error48) {
+        log.warn("Failed to close request-scoped Subscribe broker", {
+          ...context3,
+          error: error48
+        });
+        return "failed";
+      } finally {
+        this.#closing.delete(broker);
+      }
+    })();
+    this.#closing.set(broker, closing);
+    return closing;
+  }
+  async closeAll() {
+    this.#shuttingDown = true;
+    let failed = 0;
+    while (this.#brokers.size > 0 || this.#closing.size > 0) {
+      const inFlight = [...this.#closing.values()];
+      const fresh = [...this.#brokers.keys()].map((broker) => this.close(broker));
+      const outcomes = await Promise.all([...fresh, ...inFlight]);
+      failed += outcomes.filter((outcome) => outcome === "failed").length;
+    }
+    if (failed > 0) {
+      throw new Error(`${failed} request-scoped Subscribe broker(s) failed to close`);
+    }
+  }
+}
 // src/handlers/subscribe/handler.ts
 import * as grpc13 from "@grpc/grpc-js";
 
@@ -295722,19 +295786,26 @@ class OhlcvBarTracker {
     if (!firstBar || !lastBar) {
       return [];
     }
-    if (this.lastOpenTimeMs !== null && lastBar.openTimeMs < this.lastOpenTimeMs) {
+    const lastOpenTimeMs = this.lastOpenTimeMs;
+    if (lastOpenTimeMs !== null && lastBar.openTimeMs < lastOpenTimeMs) {
+      return [];
+    }
+    const barsToProcess = lastOpenTimeMs === null ? bars : bars.filter((bar) => bar.openTimeMs >= lastOpenTimeMs);
+    const firstBarToProcess = barsToProcess[0];
+    const lastBarToProcess = barsToProcess[barsToProcess.length - 1];
+    if (!firstBarToProcess || !lastBarToProcess) {
       return [];
     }
     const candidates = [];
-    if (this.lastBar !== null && this.lastOpenTimeMs !== null && !bars.some((bar) => bar.openTimeMs === this.lastOpenTimeMs) && this.lastOpenTimeMs < firstBar.openTimeMs) {
+    if (this.lastBar !== null && lastOpenTimeMs !== null && lastOpenTimeMs < firstBarToProcess.openTimeMs) {
       candidates.push({
         bar: this.lastBar,
         isClosed: true,
         brokerVersion
       });
     }
-    for (let index2 = 0;index2 < bars.length - 1; index2 += 1) {
-      const bar = bars[index2];
+    for (let index2 = 0;index2 < barsToProcess.length - 1; index2 += 1) {
+      const bar = barsToProcess[index2];
       if (bar) {
         candidates.push({
           bar,
@@ -295744,12 +295815,12 @@ class OhlcvBarTracker {
       }
     }
     candidates.push({
-      bar: lastBar,
+      bar: lastBarToProcess,
       isClosed: false,
       brokerVersion
     });
-    this.lastOpenTimeMs = lastBar.openTimeMs;
-    this.lastBar = lastBar;
+    this.lastOpenTimeMs = lastBarToProcess.openTimeMs;
+    this.lastBar = lastBarToProcess;
     return candidates;
   }
 }
@@ -296449,15 +296520,34 @@ async function runCcxtSubscribeLoop(call, isClosed, symbol2, subscriptionType, w
 }
 function createSubscribeHandler(deps) {
   const { brokers, whitelistIps, otelMetrics, brokerArchiver } = deps;
+  const brokerLifecycle = deps.brokerLifecycle ?? new SubscribeBrokerLifecycle;
   return async (call) => {
     const subscribeStartTime = Date.now();
     let streamClosed = false;
+    let ownedBroker = null;
+    let ownedBrokerClosePromise;
     const markStreamClosed = () => {
       streamClosed = true;
     };
-    const isStreamClosed = () => streamClosed || call.destroyed;
-    call.once("close", markStreamClosed);
+    const isStreamClosed = () => streamClosed || call.cancelled || call.writableEnded;
+    const closeOwnedBroker = () => {
+      if (ownedBrokerClosePromise) {
+        return ownedBrokerClosePromise;
+      }
+      if (!ownedBroker) {
+        return Promise.resolve();
+      }
+      const broker = ownedBroker;
+      ownedBroker = null;
+      ownedBrokerClosePromise = brokerLifecycle.close(broker);
+      return ownedBrokerClosePromise;
+    };
+    const closeOwnedBrokerOnCallEnd = () => {
+      closeOwnedBroker();
+    };
     call.once("cancelled", markStreamClosed);
+    call.once("cancelled", closeOwnedBrokerOnCallEnd);
+    call.once("error", closeOwnedBrokerOnCallEnd);
     call.once("end", () => {
       markStreamClosed();
       log.info("Subscribe stream ended");
@@ -296529,6 +296619,17 @@ function createSubscribeHandler(deps) {
         });
         return;
       }
+      if (!selectedBrokerAccount) {
+        ownedBroker = broker;
+        brokerLifecycle.register(broker, {
+          cex: normalizedCex,
+          symbol: symbol2
+        });
+        if (isStreamClosed()) {
+          await closeOwnedBroker();
+          return;
+        }
+      }
       const resolvedSymbol = await resolveSubscriptionSymbol(broker, symbol2, options?.marketType);
       const assetType = parseMarketType(options?.marketType);
       const deploymentId = brokerArchiver?.getDeploymentId() ?? "unknown";
@@ -296704,7 +296805,7 @@ function createSubscribeHandler(deps) {
               }
             }
             while (ohlcvStreamActive && !isStreamClosed()) {
-              const data = await broker.fetchOHLCVWs(resolvedSymbol, timeframe);
+              const data = await broker.watchOHLCV(resolvedSymbol, timeframe);
               const receivedTimestamp = Date.now();
               if (!await writeSubscribeFrame(call, isStreamClosed, {
                 data: JSON.stringify(data),
@@ -296793,6 +296894,10 @@ function createSubscribeHandler(deps) {
         symbol: "",
         type: subscriptionType
       });
+    } finally {
+      call.off("cancelled", closeOwnedBrokerOnCallEnd);
+      call.off("error", closeOwnedBrokerOnCallEnd);
+      await closeOwnedBroker();
     }
   };
 }
@@ -296943,7 +297048,7 @@ var CEX_BROKER_PACKAGE_DEFINITION = protoLoader.fromJSON(node_descriptor_default
 // src/server.ts
 var grpcObj = grpc14.loadPackageDefinition(CEX_BROKER_PACKAGE_DEFINITION);
 var cexNode = grpcObj.cex_broker;
-function getServer(policy, brokers, whitelistIps, useVerity, verityProverUrl, otelMetrics, brokerArchiver, orderActivityTracker, withdrawalObservationTracker) {
+function getServer(policy, brokers, whitelistIps, useVerity, verityProverUrl, otelMetrics, brokerArchiver, orderActivityTracker, withdrawalObservationTracker, subscribeBrokerLifecycle) {
   const server = new grpc14.Server;
   server.addService(cexNode.cex_service.service, {
     ExecuteAction: createExecuteActionHandler({
@@ -296961,7 +297066,8 @@ function getServer(policy, brokers, whitelistIps, useVerity, verityProverUrl, ot
       brokers,
       whitelistIps,
       otelMetrics,
-      brokerArchiver
+      brokerArchiver,
+      brokerLifecycle: subscribeBrokerLifecycle
     })
   });
   return server;
@@ -297324,4 +297430,4 @@ export {
   CEXBroker as default
 };
 
-//# debugId=186957481DC5AB2064756E2164756E21
+//# debugId=1B2709360791D68C64756E2164756E21