@usherlabs/cex-broker 0.2.48 → 0.2.50

package/dist/commands/cli.js

@@ -313534,16 +313534,20 @@ var tslogLogger = new Logger({
   minLevel: process.env.LOG_LEVEL === "debug" ? 0 : 3
 });
 var baseLogger = new LogLayer({
-  transport: [
-    new TsLogTransport({ id: "tslog", logger: tslogLogger }),
-    new OpenTelemetryTransport({
-      id: "otel",
-      enabled: true
-    })
-  ],
+  transport: [new TsLogTransport({ id: "tslog", logger: tslogLogger })],
   plugins: [openTelemetryPlugin()],
   errorSerializer: serializeError
 });
+var OTEL_TRANSPORT_ID = "otel";
+function attachOtelLogTransport() {
+  baseLogger.addTransport(new OpenTelemetryTransport({
+    id: OTEL_TRANSPORT_ID,
+    enabled: true
+  }));
+}
+function detachOtelLogTransport() {
+  baseLogger.removeTransport(OTEL_TRANSPORT_ID);
+}
 if (process.env.LOG_LEVEL !== "debug") {
   baseLogger.setLevel("info");
 }
@@ -317970,11 +317974,14 @@ var import_sdk_metrics = __toESM(require_src12(), 1);
 var DEFAULT_SERVICE = "cex-broker";
 var DEFAULT_OTLP_PORT = 4318;
 var EXPORT_INTERVAL_MS = 5000;
+var EXPORT_TIMEOUT_MS = 2000;
+var SHUTDOWN_TIMEOUT_MS = 3000;
 
 class BaseOtelSignal {
   signal;
   provider = null;
   isEnabled = false;
+  closePromise = null;
   serviceName;
   constructor(config, signal) {
     this.signal = signal;
@@ -318006,19 +318013,35 @@ class BaseOtelSignal {
   isOtelEnabled() {
     return this.isEnabled && this.provider !== null;
   }
-  async close() {
-    if (!this.provider) {
-      return;
-    }
-    try {
-      await this.shutdownProvider(this.provider);
-      log.info(`OTel ${this.signal} provider shut down`);
-    } catch (error) {
-      log.error(`Error shutting down OTel ${this.signal} provider:`, error);
-    }
+  close() {
+    if (this.closePromise)
+      return this.closePromise;
+    const provider = this.provider;
     this.provider = null;
     this.isEnabled = false;
     this.onProviderClosed();
+    if (!provider) {
+      this.closePromise = Promise.resolve();
+      return this.closePromise;
+    }
+    const shutdown = Promise.resolve().then(() => this.shutdownProvider(provider)).then(() => {
+      log.info(`OTel ${this.signal} provider shut down`);
+    }, (error) => {
+      log.error(`Error shutting down OTel ${this.signal} provider:`, error);
+    });
+    let deadlineTimer;
+    const deadline = new Promise((resolve) => {
+      deadlineTimer = setTimeout(() => {
+        deadlineTimer = undefined;
+        log.warn(`OTel ${this.signal} provider shutdown exceeded ${SHUTDOWN_TIMEOUT_MS}ms; continuing shutdown`);
+        resolve();
+      }, SHUTDOWN_TIMEOUT_MS);
+    });
+    this.closePromise = Promise.race([shutdown, deadline]).finally(() => {
+      if (deadlineTimer !== undefined)
+        clearTimeout(deadlineTimer);
+    });
+    return this.closePromise;
   }
 }
 function toAttributes(labels, service) {
@@ -318041,11 +318064,13 @@ class OtelMetrics extends BaseOtelSignal {
   }
   createProvider(endpoint, serviceName, appendSignalPath) {
     const exporter = new import_exporter_metrics_otlp_http.OTLPMetricExporter({
-      url: appendOtlpPath(endpoint, "metrics", appendSignalPath)
+      url: appendOtlpPath(endpoint, "metrics", appendSignalPath),
+      timeoutMillis: EXPORT_TIMEOUT_MS
     });
     const reader = new import_sdk_metrics.PeriodicExportingMetricReader({
       exporter,
-      exportIntervalMillis: EXPORT_INTERVAL_MS
+      exportIntervalMillis: EXPORT_INTERVAL_MS,
+      exportTimeoutMillis: EXPORT_TIMEOUT_MS
     });
     const resource = import_resources.resourceFromAttributes({
       "service.name": serviceName
@@ -318166,15 +318191,23 @@ class OtelMetrics extends BaseOtelSignal {
 }
 
 class OtelLogs extends BaseOtelSignal {
-  logger = null;
+  logger;
   constructor(config) {
     super(config, "logs");
+    const provider = this.getProvider();
+    this.logger = provider?.getLogger("cex-broker-logs", "1.0.0") ?? null;
+    if (provider) {
+      attachOtelLogTransport();
+    }
   }
   createProvider(endpoint, serviceName, appendSignalPath) {
     const exporter = new import_exporter_logs_otlp_http.OTLPLogExporter({
-      url: appendOtlpPath(endpoint, "logs", appendSignalPath)
+      url: appendOtlpPath(endpoint, "logs", appendSignalPath),
+      timeoutMillis: EXPORT_TIMEOUT_MS
+    });
+    const processor = new import_sdk_logs.BatchLogRecordProcessor(exporter, {
+      exportTimeoutMillis: EXPORT_TIMEOUT_MS
     });
-    const processor = new import_sdk_logs.BatchLogRecordProcessor(exporter);
     const resource = import_resources.resourceFromAttributes({
       "service.name": serviceName
     });
@@ -318185,12 +318218,12 @@ class OtelLogs extends BaseOtelSignal {
   }
   onProviderCreated(provider) {
     import_api_logs3.logs.setGlobalLoggerProvider(provider);
-    this.logger = provider.getLogger("cex-broker-logs", "1.0.0");
   }
   shutdownProvider(provider) {
-    return provider.forceFlush().then(() => provider.shutdown());
+    return provider.shutdown();
   }
   onProviderClosed() {
+    detachOtelLogTransport();
     this.logger = null;
   }
   emit(record) {
@@ -335121,6 +335154,24 @@ function selectBrokerAccountForCex(normalizedCex, brokers, metadata) {
   return selectBrokerAccount(brokers[normalizedCex], metadata) ?? undefined;
 }
 
+// src/helpers/trace-context.ts
+var TRACE_METADATA_KEY = "x-trace-id";
+var MAX_TRACE_METADATA_LENGTH = 128;
+var OTEL_TRACE_ID_PATTERN = /^[0-9a-f]{32}$/;
+var UUID_V4_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
+var ZERO_OTEL_TRACE_ID = "00000000000000000000000000000000";
+function extractTraceId(metadata) {
+  const raw = metadata.get(TRACE_METADATA_KEY)[0];
+  if (typeof raw !== "string" || raw.length > MAX_TRACE_METADATA_LENGTH) {
+    return;
+  }
+  const traceId = raw.trim();
+  if (OTEL_TRACE_ID_PATTERN.test(traceId) && traceId !== ZERO_OTEL_TRACE_ID || UUID_V4_PATTERN.test(traceId)) {
+    return traceId;
+  }
+  return;
+}
+
 // src/handlers/execute-action/order-book-call.ts
 var grpc4 = __toESM(require_src3(), 1);
 async function handleOrderBookCall(ctx) {
@@ -336424,6 +336475,15 @@ function isPublicMarketDataAction(action, payload) {
     return false;
   return isOrderBookCallMethod(payload?.method ?? payload?.functionName);
 }
+function grpcStatusName(error48) {
+  if (!error48) {
+    return "OK";
+  }
+  if (typeof error48.code !== "number") {
+    return "UNKNOWN";
+  }
+  return grpc12.status[error48.code] ?? "UNKNOWN";
+}
 function createExecuteActionHandler(deps) {
   const {
     policy,
@@ -336440,12 +336500,28 @@ function createExecuteActionHandler(deps) {
     const startTime = Date.now();
     const { action: rawAction, cex: cex3, symbol: symbol2 } = call.request;
     const action = resolveAction(rawAction);
+    const actionName = getActionName(action);
+    const operationalCex = cex3?.trim().toLowerCase() || "unknown";
+    const traceId = extractTraceId(call.metadata);
+    const traceFields = traceId === undefined ? {} : { trace_id: traceId };
     let actionCompleted = false;
     const wrappedCallback = (error48, value) => {
       if (!actionCompleted) {
         actionCompleted = true;
         const latency = Date.now() - startTime;
-        const actionName = getActionName(action);
+        const terminalFields = {
+          action: actionName,
+          cex: operationalCex,
+          latency_ms: latency,
+          outcome: error48 ? "error" : "success",
+          grpc_status: grpcStatusName(error48),
+          ...traceFields
+        };
+        if (error48) {
+          log.withMetadata(terminalFields).error("ExecuteAction failed");
+        } else {
+          log.withMetadata(terminalFields).info("ExecuteAction completed");
+        }
         otelMetrics?.recordHistogram("execute_action_duration_ms", latency, {
           action: actionName,
           cex: cex3 || "unknown"
@@ -336466,9 +336542,13 @@ function createExecuteActionHandler(deps) {
       callback(error48, value);
     };
     try {
-      log.info(`Request - ExecuteAction:`, { action, cex: cex3, symbol: symbol2 });
+      log.withMetadata({
+        action: actionName,
+        cex: operationalCex,
+        ...traceFields
+      }).info("ExecuteAction started");
       otelMetrics?.recordCounter("execute_action_requests_total", 1, {
-        action: getActionName(action),
+        action: actionName,
         cex: cex3 || "unknown"
       });
       if (!authenticateRequest(call, whitelistIps)) {
@@ -336706,6 +336786,9 @@ async function writeSubscribeError(call, isClosed, frame) {
     call.end();
   }
 }
+function grpcStatusName2(status14) {
+  return typeof status14 === "number" ? grpc13.status[status14] ?? "UNKNOWN" : "UNKNOWN";
+}
 function getBinanceEventMarketId(event) {
   const value = event.s;
   return typeof value === "string" ? value : null;
@@ -336838,6 +336921,24 @@ function createSubscribeHandler(deps) {
   });
   return async (call) => {
     const subscribeStartTime = Date.now();
+    const request = call.request;
+    const { cex: cex3, symbol: symbol2, type: type2 } = request;
+    const metadata = call.metadata;
+    const traceId = extractTraceId(metadata);
+    const traceFields = traceId === undefined ? {} : { trace_id: traceId };
+    const operationalCex = cex3?.trim().toLowerCase() || "unknown";
+    const operationalSymbol = symbol2?.trim() || "unknown";
+    const subscriptionType2 = resolveSubscriptionType(type2);
+    const subscriptionTypeName = getSubscriptionTypeName(subscriptionType2);
+    const operationalFields = {
+      cex: operationalCex,
+      symbol: operationalSymbol,
+      subscription_type: subscriptionTypeName,
+      ...traceFields
+    };
+    let terminalLogged = false;
+    let terminalOutcome = "completed";
+    let terminalGrpcStatus = "OK";
     let streamClosed = false;
     let ownedBroker = null;
     let ownedBrokerClosePromise;
@@ -336860,12 +336961,41 @@ function createSubscribeHandler(deps) {
     const closeOwnedBrokerOnCallEnd = () => {
       closeOwnedBroker();
     };
-    call.once("cancelled", markStreamClosed);
+    const logTerminalOutcome = (outcome) => {
+      if (terminalLogged) {
+        return;
+      }
+      terminalLogged = true;
+      const fields = {
+        ...operationalFields,
+        duration_ms: Date.now() - subscribeStartTime,
+        outcome,
+        grpc_status: outcome === "cancelled" ? "CANCELLED" : terminalGrpcStatus
+      };
+      if (outcome === "error") {
+        log.withMetadata(fields).error("Subscribe failed");
+      } else if (outcome === "cancelled") {
+        log.withMetadata(fields).info("Subscribe cancelled");
+      } else {
+        log.withMetadata(fields).info("Subscribe ended");
+      }
+    };
+    const writeTerminalError = async (frame, status14 = grpc13.status.UNKNOWN) => {
+      terminalOutcome = "error";
+      terminalGrpcStatus = grpcStatusName2(status14);
+      await writeSubscribeError(call, isStreamClosed, frame);
+    };
+    log.withMetadata(operationalFields).info("Subscribe started");
+    call.once("cancelled", () => {
+      terminalGrpcStatus = "CANCELLED";
+      markStreamClosed();
+      logTerminalOutcome("cancelled");
+    });
     call.once("cancelled", closeOwnedBrokerOnCallEnd);
     call.once("error", closeOwnedBrokerOnCallEnd);
     call.once("end", () => {
       markStreamClosed();
-      log.info("Subscribe stream ended");
+      logTerminalOutcome(terminalOutcome);
       const duration3 = Date.now() - subscribeStartTime;
       otelMetrics?.recordHistogram("subscribe_duration_ms", duration3, {
         cex: call.request?.cex || "unknown",
@@ -336874,12 +337004,16 @@ function createSubscribeHandler(deps) {
     });
     call.once("error", (error48) => {
       markStreamClosed();
-      log.error("Subscribe stream error:", error48);
+      terminalOutcome = "error";
+      terminalGrpcStatus = grpcStatusName2(typeof error48 === "object" && error48 !== null && "code" in error48 ? error48.code : undefined);
+      logTerminalOutcome(call.cancelled ? "cancelled" : "error");
       otelMetrics?.recordCounter("subscribe_errors_total", 1, {
         error_type: error48 instanceof Error ? error48.message : "unknown"
       });
     });
     if (!authenticateRequest(call, whitelistIps)) {
+      terminalOutcome = "error";
+      terminalGrpcStatus = "PERMISSION_DENIED";
       otelMetrics?.recordCounter("subscribe_errors_total", 1, {
         error_type: "permission_denied"
       });
@@ -336890,32 +337024,22 @@ function createSubscribeHandler(deps) {
       call.destroy(new Error("Access denied: Unauthorized IP"));
       return;
     }
-    const metadata = call.metadata;
-    let subscriptionType2 = SubscriptionType.ORDERBOOK;
     try {
-      const request = call.request;
-      const { cex: cex3, symbol: symbol2, type: type2, options } = request;
-      subscriptionType2 = resolveSubscriptionType(type2);
-      log.info(`Request - Subscribe:`, {
-        cex: request.cex,
-        symbol: request.symbol,
-        type: subscriptionType2
-      });
-      const subscriptionTypeName = getSubscriptionTypeName(subscriptionType2);
+      const { options } = request;
       otelMetrics?.recordCounter("subscribe_requests_total", 1, {
         cex: cex3 || "unknown",
         symbol: symbol2 || "unknown",
         type: subscriptionTypeName
       });
       if (!cex3 || !symbol2) {
-        await writeSubscribeError(call, isStreamClosed, {
+        await writeTerminalError({
           data: JSON.stringify({
             error: "cex, symbol, and type are required"
           }),
           timestamp: Date.now(),
           symbol: symbol2 || "",
           type: subscriptionType2
-        });
+        }, grpc13.status.INVALID_ARGUMENT);
         return;
       }
       if (isPublicMarketDataSubscription(subscriptionType2)) {
@@ -336944,7 +337068,7 @@ function createSubscribeHandler(deps) {
         } catch (error48) {
           const message = getErrorMessage(error48);
           if (!isStreamClosed()) {
-            await writeSubscribeError(call, isStreamClosed, {
+            await writeTerminalError({
               data: JSON.stringify({ error: message }),
               timestamp: Date.now(),
               symbol: symbol2,
@@ -336965,14 +337089,14 @@ function createSubscribeHandler(deps) {
       const selectedBroker = selectedBrokerAccount?.exchange ?? createBroker(normalizedCex, metadata);
       const broker = selectedBroker ?? createPublicBroker(normalizedCex);
       if (!broker) {
-        await writeSubscribeError(call, isStreamClosed, {
+        await writeTerminalError({
           data: JSON.stringify({
             error: "Exchange not registered and no API metadata found"
           }),
           timestamp: Date.now(),
           symbol: symbol2,
           type: subscriptionType2
-        });
+        }, grpc13.status.NOT_FOUND);
         return;
       }
       if (!selectedBrokerAccount) {
@@ -337000,28 +337124,28 @@ function createSubscribeHandler(deps) {
       if (isBinanceSpotAccountSubscription(normalizedCex, subscriptionType2, options?.marketType)) {
         const accountBroker = selectedBrokerAccount?.exchange ?? selectedBroker;
         if (!accountBroker) {
-          await writeSubscribeError(call, isStreamClosed, {
+          await writeTerminalError({
             data: JSON.stringify({
               error: "Binance account subscriptions require API credentials"
             }),
             timestamp: Date.now(),
             symbol: resolvedSymbol,
             type: subscriptionType2
-          });
+          }, grpc13.status.FAILED_PRECONDITION);
           return;
         }
         const marketId = subscriptionType2 === SubscriptionType.ORDERS ? await getBinanceMarketId(accountBroker, resolvedSymbol) : undefined;
         let userDataSource;
         if (selectedBrokerAccount) {
           if (!userDataStreamSupervisor) {
-            await writeSubscribeError(call, isStreamClosed, {
+            await writeTerminalError({
               data: JSON.stringify({
                 error: "Configured account user-data supervisor is unavailable"
               }),
               timestamp: Date.now(),
               symbol: resolvedSymbol,
               type: subscriptionType2
-            });
+            }, grpc13.status.FAILED_PRECONDITION);
             return;
           }
           userDataSource = userDataStreamSupervisor.subscribe({
@@ -337044,7 +337168,7 @@ function createSubscribeHandler(deps) {
           } catch (error48) {
             const message = getErrorMessage(error48);
             log.error(`Error fetching balance for ${cex3}:`, error48);
-            await writeSubscribeError(call, isStreamClosed, {
+            await writeTerminalError({
               data: JSON.stringify({
                 error: `Failed to fetch balance: ${message}`
               }),
@@ -337063,7 +337187,7 @@ function createSubscribeHandler(deps) {
           } catch (error48) {
             log.error(`Error fetching orders for ${resolvedSymbol} on ${cex3}:`, error48);
             const message = getErrorMessage(error48);
-            await writeSubscribeError(call, isStreamClosed, {
+            await writeTerminalError({
               data: JSON.stringify({
                 error: `Failed to fetch orders: ${message}`
               }),
@@ -337074,26 +337198,27 @@ function createSubscribeHandler(deps) {
           }
           break;
         default:
-          await writeSubscribeError(call, isStreamClosed, {
+          await writeTerminalError({
             data: JSON.stringify({ error: "Invalid subscription type" }),
             timestamp: Date.now(),
             symbol: symbol2,
             type: subscriptionType2
-          });
+          }, grpc13.status.INVALID_ARGUMENT);
       }
     } catch (error48) {
       log.error("Error in Subscribe stream:", error48);
       const message = getErrorMessage(error48);
-      await writeSubscribeError(call, isStreamClosed, {
+      await writeTerminalError({
         data: JSON.stringify({ error: `Internal server error: ${message}` }),
         timestamp: Date.now(),
         symbol: "",
         type: subscriptionType2
-      });
+      }, grpc13.status.INTERNAL);
     } finally {
       call.off("cancelled", closeOwnedBrokerOnCallEnd);
       call.off("error", closeOwnedBrokerOnCallEnd);
       await closeOwnedBroker();
+      logTerminalOutcome(call.cancelled ? "cancelled" : terminalOutcome);
     }
   };
 }
@@ -337587,12 +337712,7 @@ class CEXBroker {
     if (this.brokerArchiver) {
       await this.brokerArchiver.close();
     }
-    if (this.otelMetrics) {
-      await this.otelMetrics.close();
-    }
-    if (this.otelLogs) {
-      await this.otelLogs.close();
-    }
+    await Promise.all([this.otelMetrics?.close(), this.otelLogs?.close()]);
   }
   async run() {
     const marketArchiveState = resolveMarketCaptureArchiveState({
@@ -337717,7 +337837,24 @@ async function startBrokerCommand(policyPath, port, whitelistIps, verityProverUr
   });
   broker.loadEnvConfig();
   await broker.run();
+  return broker;
+}
+
+// src/helpers/logger.ts
+var tslogLogger2 = new Logger({
+  type: "pretty",
+  stylePrettyLogs: true,
+  minLevel: process.env.LOG_LEVEL === "debug" ? 0 : 3
+});
+var baseLogger2 = new LogLayer({
+  transport: [new TsLogTransport({ id: "tslog", logger: tslogLogger2 })],
+  plugins: [openTelemetryPlugin()],
+  errorSerializer: serializeError
+});
+if (process.env.LOG_LEVEL !== "debug") {
+  baseLogger2.setLevel("info");
 }
+var log2 = baseLogger2;
 
 // src/cli.ts
 var program2 = new Command;
@@ -337728,15 +337865,35 @@ program2.name("cex-broker").description("CLI to start the CEXBroker service").re
       const isValidIPv4 = (ip) => /^(\d{1,3}\.){3}\d{1,3}$/.test(ip) && ip.split(".").every((part) => Number(part) >= 0 && Number(part) <= 255);
       for (const ip of whitelist) {
         if (!isValidIPv4(ip)) {
-          console.error(`\u274C Invalid IPv4 address: ${ip}`);
-          process.exit(1);
+          throw new Error(`Invalid IPv4 address: ${ip}`);
         }
       }
     }
-    await startBrokerCommand(options.policy, parseInt(options.port, 10), whitelist, options.verityProverUrl);
+    const broker = await startBrokerCommand(options.policy, parseInt(options.port, 10), whitelist, options.verityProverUrl);
+    let shutdownPromise;
+    const onSignal = (signal) => {
+      if (shutdownPromise)
+        return;
+      log2.withMetadata({ signal }).info("CEXBroker graceful shutdown requested");
+      shutdownPromise = (async () => {
+        try {
+          await broker.stop();
+          log2.withMetadata({ signal }).info("CEXBroker graceful shutdown complete");
+          process.exitCode = 0;
+        } catch (error48) {
+          log2.withMetadata({ signal, error: error48 }).error("CEXBroker graceful shutdown failed");
+          process.exitCode = 1;
+        } finally {
+          process.off("SIGTERM", onSignal);
+          process.off("SIGINT", onSignal);
+        }
+      })();
+    };
+    process.on("SIGTERM", onSignal);
+    process.on("SIGINT", onSignal);
   } catch (err2) {
     console.error("\u274C Failed to start broker:", err2);
-    process.exit(1);
+    process.exitCode = 1;
   }
 });
-program2.parse(process.argv);
+await program2.parseAsync(process.argv);