@usherlabs/cex-broker 0.2.40 → 0.2.41

package/dist/commands/cli.js

@@ -6,25 +6,43 @@ var __getProtoOf = Object.getPrototypeOf;
 var __defProp = Object.defineProperty;
 var __getOwnPropNames = Object.getOwnPropertyNames;
 var __hasOwnProp = Object.prototype.hasOwnProperty;
+function __accessProp(key) {
+  return this[key];
+}
+var __toESMCache_node;
+var __toESMCache_esm;
 var __toESM = (mod, isNodeMode, target) => {
+  var canCache = mod != null && typeof mod === "object";
+  if (canCache) {
+    var cache = isNodeMode ? __toESMCache_node ??= new WeakMap : __toESMCache_esm ??= new WeakMap;
+    var cached = cache.get(mod);
+    if (cached)
+      return cached;
+  }
   target = mod != null ? __create(__getProtoOf(mod)) : {};
   const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
   for (let key of __getOwnPropNames(mod))
     if (!__hasOwnProp.call(to, key))
       __defProp(to, key, {
-        get: () => mod[key],
+        get: __accessProp.bind(mod, key),
         enumerable: true
       });
+  if (canCache)
+    cache.set(mod, to);
   return to;
 };
 var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
+var __returnValue = (v) => v;
+function __exportSetter(name, newValue) {
+  this[name] = __returnValue.bind(null, newValue);
+}
 var __export = (target, all) => {
   for (var name in all)
     __defProp(target, name, {
       get: all[name],
       enumerable: true,
       configurable: true,
-      set: (newValue) => all[name] = () => newValue
+      set: __exportSetter.bind(all, name)
     });
 };
 var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
@@ -997,7 +1015,7 @@ Expecting one of '${allowedValues.join("', '")}'`);
         this._exitCallback = (err) => {
           if (err.code !== "commander.executeSubCommandAsync") {
             throw err;
-          } else {}
+          }
         };
       }
       return this;
@@ -50030,7 +50048,7 @@ var require_tracestate_impl = __commonJS((exports) => {
           const value = listMember.slice(i2 + 1, part.length);
           if ((0, tracestate_validators_1.validateKey)(key) && (0, tracestate_validators_1.validateValue)(value)) {
             agg.set(key, value);
-          } else {}
+          }
         }
         return agg;
       }, new Map);
@@ -59163,7 +59181,7 @@ var require_TraceState = __commonJS((exports) => {
           const value = listMember.slice(i2 + 1, part.length);
           if ((0, validators_1.validateKey)(key) && (0, validators_1.validateValue)(value)) {
             agg.set(key, value);
-          } else {}
+          }
         }
         return agg;
       }, new Map);
@@ -315706,7 +315724,9 @@ function buildMarketMetadataSnapshotRow(input) {
 // src/helpers/broker-execution-archive/writer.ts
 var import_api_logs2 = __toESM(require_src7(), 1);
 import { closeSync, fsyncSync, openSync, writeSync } from "node:fs";
-import { request as httpRequest2 } from "node:http";
+import {
+  request as httpRequest2
+} from "node:http";
 import { request as httpsRequest2 } from "node:https";
 var BROKER_EXECUTION_ARCHIVE_TABLES = new Set([
   "broker_execution.order_events",
@@ -315779,6 +315799,7 @@ class BrokerExecutionArchiver {
   flushIntervalMs;
   forwarderTimeoutMs;
   queue = [];
+  enqueueTimes = new WeakMap;
   stats = {
     enqueued: 0,
     shed: 0,
@@ -315787,6 +315808,16 @@ class BrokerExecutionArchiver {
   };
   flushTimer = null;
   flushInFlight = null;
+  inFlightBatch = null;
+  inFlightStartedAtMs = null;
+  lastFailureAtMs = null;
+  lastShedAtMs = null;
+  lastSuccessAtMs = null;
+  lastFailureEventSequence = null;
+  lastShedEventSequence = null;
+  lastSuccessEventSequence = null;
+  lastSinkLatencyMs = null;
+  healthEventSequence = 0;
   lastShedWarnAtMs = 0;
   closing = false;
   closed = false;
@@ -315820,6 +315851,7 @@ class BrokerExecutionArchiver {
     }
     try {
       this.flushTimer = setInterval(() => {
+        this.emitArchiveHealthMetrics();
         this.flush();
       }, this.flushIntervalMs);
       this.flushTimer.unref?.();
@@ -315863,32 +315895,34 @@ class BrokerExecutionArchiver {
       row: { ...row.row, source: this.source }
     };
     if (this.queue.length >= this.maxQueueSize) {
-      const shedRow = this.queue[0];
-      if (shedRow) {
-        this.appendLossRecords([shedRow], "queue_shed");
+      const shedEntry = this.queue[0];
+      if (shedEntry) {
+        this.appendLossRecords([shedEntry], "queue_shed");
         this.queue.shift();
       }
       this.stats.shed += 1;
+      this.recordHealthEvent("shed");
       this.recordArchiveMetric("cex_archive_rows_shed_total", {
-        table: shedRow?.table ?? "unknown",
+        table: shedEntry?.table ?? "unknown",
         source: this.source,
-        feed: shedRow ? archiveFeed(shedRow) : "NON_MARKET"
+        feed: shedEntry ? archiveFeed(shedEntry) : "NON_MARKET"
       });
       this.recordArchiveMetric("cex_archive_queue_saturated_rows_total", {
-        table: shedRow?.table ?? "unknown",
+        table: shedEntry?.table ?? "unknown",
         source: this.source,
-        feed: shedRow ? archiveFeed(shedRow) : "NON_MARKET"
+        feed: shedEntry ? archiveFeed(shedEntry) : "NON_MARKET"
       });
       const now3 = Date.now();
       if (now3 - this.lastShedWarnAtMs >= SHED_WARN_INTERVAL_MS) {
         log.warn("Archive queue full: shedding oldest rows", {
           shed_total: this.stats.shed,
           queue_max: this.maxQueueSize,
-          table: shedRow?.table ?? "unknown"
+          table: shedEntry?.table ?? "unknown"
         });
         this.lastShedWarnAtMs = now3;
       }
     }
+    this.enqueueTimes.set(archiveRow, Date.now());
     this.queue.push(archiveRow);
     this.stats.enqueued += 1;
     this.recordArchiveMetric("cex_archive_rows_enqueued_total", {
@@ -315896,6 +315930,7 @@ class BrokerExecutionArchiver {
       source: this.source,
       feed: archiveFeed(archiveRow)
     });
+    this.emitArchiveHealthMetrics();
     if (this.queue.length >= this.batchSize) {
       this.flush();
     }
@@ -315910,14 +315945,19 @@ class BrokerExecutionArchiver {
     if (this.flushInFlight) {
       return this.flushInFlight;
     }
+    const inFlightState = {};
     const inFlight = this.flushBatch().then(() => {
       return;
     }).finally(() => {
+      if (this.flushInFlight !== inFlightState.promise) {
+        return;
+      }
       this.flushInFlight = null;
       if (!this.closed && !this.closing && this.enabled && this.queue.length >= this.batchSize) {
         queueMicrotask(() => void this.flush());
       }
     });
+    inFlightState.promise = inFlight;
     this.flushInFlight = inFlight;
     return inFlight;
   }
@@ -315964,6 +316004,79 @@ class BrokerExecutionArchiver {
   getQueueDepth() {
     return this.queue.length;
   }
+  getHealthSnapshot() {
+    const now3 = Date.now();
+    const oldestPendingAtMs = this.oldestPendingEnqueueAtMs();
+    const oldestPendingAgeMs = oldestPendingAtMs === null ? 0 : Math.max(0, now3 - oldestPendingAtMs);
+    const inFlightAgeMs = this.inFlightStartedAtMs === null ? 0 : Math.max(0, now3 - this.inFlightStartedAtMs);
+    const lastUnhealthyEventSequence = Math.max(this.lastFailureEventSequence ?? Number.NEGATIVE_INFINITY, this.lastShedEventSequence ?? Number.NEGATIVE_INFINITY);
+    const recoveredFromEvents = lastUnhealthyEventSequence === Number.NEGATIVE_INFINITY || this.lastSuccessEventSequence !== null && this.lastSuccessEventSequence > lastUnhealthyEventSequence;
+    const healthy = this.enabled && !this.closing && !this.closed && this.queue.length < this.maxQueueSize && oldestPendingAgeMs <= this.forwarderTimeoutMs && inFlightAgeMs <= this.forwarderTimeoutMs && recoveredFromEvents;
+    return {
+      healthy,
+      queue_depth: this.queue.length,
+      oldest_pending_age_ms: oldestPendingAgeMs,
+      shed_total: this.stats.shed,
+      last_failure_at: this.lastFailureAtMs,
+      last_shed_at: this.lastShedAtMs,
+      last_success_at: this.lastSuccessAtMs,
+      sink_latency_ms: this.lastSinkLatencyMs,
+      sink_errors_total: this.stats.forwarderFailures,
+      in_flight: this.inFlightBatch !== null,
+      in_flight_age_ms: inFlightAgeMs
+    };
+  }
+  oldestPendingEnqueueAtMs() {
+    let oldest = null;
+    for (const entry of this.queue) {
+      const enqueuedAtMs = this.enqueueTimes.get(entry);
+      if (enqueuedAtMs === undefined) {
+        continue;
+      }
+      oldest = oldest === null ? enqueuedAtMs : Math.min(oldest, enqueuedAtMs);
+    }
+    for (const entry of this.inFlightBatch ?? []) {
+      const enqueuedAtMs = this.enqueueTimes.get(entry);
+      if (enqueuedAtMs === undefined) {
+        continue;
+      }
+      oldest = oldest === null ? enqueuedAtMs : Math.min(oldest, enqueuedAtMs);
+    }
+    return oldest;
+  }
+  recordHealthEvent(event) {
+    const eventSequence = ++this.healthEventSequence;
+    const eventTimestampMs = Date.now();
+    switch (event) {
+      case "failure":
+        this.lastFailureAtMs = eventTimestampMs;
+        this.lastFailureEventSequence = eventSequence;
+        return;
+      case "shed":
+        this.lastShedAtMs = eventTimestampMs;
+        this.lastShedEventSequence = eventSequence;
+        return;
+      case "success":
+        this.lastSuccessAtMs = eventTimestampMs;
+        this.lastSuccessEventSequence = eventSequence;
+        return;
+    }
+  }
+  emitArchiveHealthMetrics() {
+    const snapshot = this.getHealthSnapshot();
+    const labels = { source: this.source };
+    this.recordArchiveObservableGauge("cex_archive_health", snapshot.healthy ? 1 : 0, labels);
+    this.recordArchiveGauge("cex_archive_queue_depth", snapshot.queue_depth, labels);
+    this.recordArchiveGauge("cex_archive_oldest_pending_age_ms", snapshot.oldest_pending_age_ms, labels);
+    this.recordArchiveGauge("cex_archive_shed_total", snapshot.shed_total, labels);
+    this.recordArchiveGauge("cex_archive_last_failure_at", snapshot.last_failure_at ?? 0, labels);
+    this.recordArchiveGauge("cex_archive_last_shed_at", snapshot.last_shed_at ?? 0, labels);
+    this.recordArchiveGauge("cex_archive_last_success_at", snapshot.last_success_at ?? 0, labels);
+    this.recordArchiveGauge("cex_archive_sink_latency_ms", snapshot.sink_latency_ms ?? 0, labels);
+    this.recordArchiveGauge("cex_archive_sink_errors_total", snapshot.sink_errors_total, labels);
+    this.recordArchiveGauge("cex_archive_in_flight", snapshot.in_flight ? 1 : 0, labels);
+    this.recordArchiveGauge("cex_archive_in_flight_age_ms", snapshot.in_flight_age_ms, labels);
+  }
   closeLossJournal() {
     if (this.deadLetterFd === undefined) {
       return;
@@ -315986,8 +316099,9 @@ class BrokerExecutionArchiver {
       this.appendLossRecords([dropped], "queue_shed");
       this.queue.shift();
       this.stats.shed += 1;
+      this.recordHealthEvent("shed");
       this.recordArchiveMetric("cex_archive_rows_shed_total", {
-        table: dropped?.table ?? "unknown",
+        table: dropped.table,
         source: this.source,
         feed: archiveFeed(dropped)
       });
@@ -316046,6 +316160,8 @@ class BrokerExecutionArchiver {
     if (batch.length === 0) {
       return true;
     }
+    this.inFlightBatch = batch;
+    this.inFlightStartedAtMs = Date.now();
     for (const entry of batch) {
       if (isBrokerExecutionArchiveTable(entry.table)) {
         this.emitOtelLog(entry);
@@ -316056,8 +316172,15 @@ class BrokerExecutionArchiver {
         await this.postToForwarder(batch);
       } catch (error) {
         this.stats.forwarderFailures += 1;
-        this.queue.push(...batch);
-        this.enforceQueueBound();
+        this.recordHealthEvent("failure");
+        this.lastSinkLatencyMs = Math.max(0, Date.now() - (this.inFlightStartedAtMs ?? Date.now()));
+        this.queue.unshift(...batch);
+        try {
+          this.enforceQueueBound();
+        } finally {
+          this.clearInFlightBatch(batch);
+          this.emitArchiveHealthMetrics();
+        }
         this.recordArchiveMetric("cex_archive_forwarder_failures_total", {
           count: batch.length
         });
@@ -316066,14 +316189,26 @@ class BrokerExecutionArchiver {
       }
     }
     this.stats.flushed += batch.length;
+    this.recordHealthEvent("success");
+    this.lastSinkLatencyMs = Math.max(0, Date.now() - (this.inFlightStartedAtMs ?? Date.now()));
+    this.clearInFlightBatch(batch);
     this.recordFlushHealth(batch);
+    this.emitArchiveHealthMetrics();
     return true;
   }
+  clearInFlightBatch(batch) {
+    if (this.inFlightBatch !== batch) {
+      return;
+    }
+    this.inFlightBatch = null;
+    this.inFlightStartedAtMs = null;
+  }
   recordFlushHealth(batch) {
     const countByTable = new Map;
     for (const entry of batch) {
-      const key = `${entry.table}\x00${archiveFeed(entry)}`;
-      const grouped = countByTable.get(key) ?? { row: entry, count: 0 };
+      const row = entry;
+      const key = `${row.table}\x00${archiveFeed(row)}`;
+      const grouped = countByTable.get(key) ?? { row, count: 0 };
       grouped.count += 1;
       countByTable.set(key, grouped);
     }
@@ -316091,9 +316226,14 @@ class BrokerExecutionArchiver {
       await this.otelMetrics?.recordCounter(metricName, value, labels);
     } catch {}
   }
-  async recordArchiveGauge(metricName, value) {
+  async recordArchiveGauge(metricName, value, labels = {}) {
     try {
-      await this.otelMetrics?.recordGauge(metricName, value, {});
+      await this.otelMetrics?.recordGauge(metricName, value, labels);
+    } catch {}
+  }
+  async recordArchiveObservableGauge(metricName, value, labels) {
+    try {
+      await this.otelMetrics?.setObservableGauge(metricName, value, labels);
     } catch {}
   }
   emitOtelLog(entry) {
@@ -316130,27 +316270,136 @@ class BrokerExecutionArchiver {
       headers.authorization = `Bearer ${this.forwarderAuthToken}`;
     }
     return new Promise((resolve, reject) => {
-      const req = doRequest(url2, {
-        method: "POST",
-        headers,
-        timeout: this.forwarderTimeoutMs
-      }, (res) => {
-        res.on("data", () => {});
-        res.on("end", () => {
-          const status = res.statusCode ?? 0;
-          if (status < 200 || status >= 300) {
-            reject(new Error(`Archive forwarder returned ${status} ${res.statusMessage ?? ""}`));
-            return;
-          }
+      let req;
+      let response;
+      let settled = false;
+      let requestClosed = false;
+      let requestCloseFailureScheduled = false;
+      let responseClosed = false;
+      let responseEnded = false;
+      let outerTimer;
+      function cleanupRequestListeners() {
+        req?.removeListener("error", onRequestError);
+        req?.removeListener("timeout", onRequestTimeout);
+        req?.removeListener("close", onRequestClose);
+      }
+      function cleanupResponseListeners() {
+        response?.removeListener("data", onResponseData);
+        response?.removeListener("end", onResponseEnd);
+        response?.removeListener("aborted", onResponseAborted);
+        response?.removeListener("error", onResponseError);
+        response?.removeListener("close", onResponseClose);
+      }
+      function onRequestClose() {
+        requestClosed = true;
+        if (!settled && !response && !requestCloseFailureScheduled) {
+          requestCloseFailureScheduled = true;
+          queueMicrotask(() => {
+            if (!settled && !response) {
+              settle2(new Error("Archive forwarder request closed before response headers"), false, true);
+            }
+          });
+        }
+        if (settled) {
+          cleanupRequestListeners();
+        }
+      }
+      function onRequestError(error) {
+        if (!settled) {
+          settle2(error, true);
+        } else {
+          cleanupRequestListeners();
+        }
+      }
+      function onRequestTimeout() {
+        settle2(new Error("Archive forwarder socket timed out"), true);
+      }
+      function onResponseData() {}
+      function onResponseEnd() {
+        responseEnded = true;
+        if (!response?.complete) {
+          settle2(new Error("Archive forwarder response was incomplete"), true);
+          return;
+        }
+        const status = response.statusCode ?? 0;
+        if (status < 200 || status >= 300) {
+          settle2(new Error(`Archive forwarder returned ${status} ${response.statusMessage ?? ""}`), false);
+          return;
+        }
+        settle2(undefined, false);
+      }
+      function onResponseAborted() {
+        settle2(new Error("Archive forwarder response was aborted"), true);
+      }
+      function onResponseError(error) {
+        if (!settled) {
+          settle2(error, true);
+        }
+      }
+      function onResponseClose() {
+        responseClosed = true;
+        if (!settled && !responseEnded) {
+          settle2(new Error("Archive forwarder response closed before completion"), true);
+          return;
+        }
+        cleanupResponseListeners();
+      }
+      function settle2(error, destroyRequest, retainClosedRequestError = false) {
+        if (settled) {
+          return;
+        }
+        settled = true;
+        if (outerTimer) {
+          clearTimeout(outerTimer);
+          outerTimer = undefined;
+        }
+        response?.removeListener("data", onResponseData);
+        response?.removeListener("end", onResponseEnd);
+        response?.removeListener("aborted", onResponseAborted);
+        if (responseClosed) {
+          cleanupResponseListeners();
+        }
+        if (destroyRequest && req && !requestClosed) {
+          req.destroy(error);
+        } else if (retainClosedRequestError && req && requestClosed) {
+          req.removeListener("timeout", onRequestTimeout);
+          req.removeListener("close", onRequestClose);
+        } else if (!req || requestClosed) {
+          cleanupRequestListeners();
+        }
+        if (error) {
+          reject(error);
+        } else {
           resolve();
+        }
+      }
+      outerTimer = setTimeout(() => {
+        settle2(new Error("Archive forwarder request deadline exceeded"), true);
+      }, this.forwarderTimeoutMs);
+      try {
+        req = doRequest(url2, {
+          method: "POST",
+          headers,
+          timeout: this.forwarderTimeoutMs
+        }, (nextResponse) => {
+          response = nextResponse;
+          response.on("data", onResponseData);
+          response.on("end", onResponseEnd);
+          response.on("aborted", onResponseAborted);
+          response.on("error", onResponseError);
+          response.on("close", onResponseClose);
+          if (settled) {
+            response.resume();
+          }
         });
-      });
-      req.on("error", reject);
-      req.on("timeout", () => {
-        req.destroy(new Error("Archive forwarder request timed out"));
-      });
-      req.write(body);
-      req.end();
+        req.on("error", onRequestError);
+        req.on("timeout", onRequestTimeout);
+        req.on("close", onRequestClose);
+        req.write(body);
+        req.end();
+      } catch (error) {
+        settle2(error instanceof Error ? error : new Error(String(error)), true);
+      }
     });
   }
 }
@@ -329666,7 +329915,7 @@ function finalize(ctx, schema) {
     result.$schema = "http://json-schema.org/draft-07/schema#";
   } else if (ctx.target === "draft-04") {
     result.$schema = "http://json-schema.org/draft-04/schema#";
-  } else if (ctx.target === "openapi-3.0") {} else {}
+  } else if (ctx.target === "openapi-3.0") {}
   if (ctx.external?.uri) {
     const id2 = ctx.external.registry.get(schema)?.id;
     if (!id2)
@@ -329914,7 +330163,7 @@ var literalProcessor = (schema, ctx, json3, _params) => {
     if (val === undefined) {
       if (ctx.unrepresentable === "throw") {
         throw new Error("Literal `undefined` cannot be represented in JSON Schema");
-      } else {}
+      }
     } else if (typeof val === "bigint") {
       if (ctx.unrepresentable === "throw") {
         throw new Error("BigInt literals cannot be represented in JSON Schema");

package/dist/helpers/broker-execution-archive/writer.d.ts

@@ -23,6 +23,19 @@ type ArchiverStats = {
     flushed: number;
     forwarderFailures: number;
 };
+export type BrokerExecutionArchiveHealthSnapshot = {
+    healthy: boolean;
+    queue_depth: number;
+    oldest_pending_age_ms: number;
+    shed_total: number;
+    last_failure_at: number | null;
+    last_shed_at: number | null;
+    last_success_at: number | null;
+    sink_latency_ms: number | null;
+    sink_errors_total: number;
+    in_flight: boolean;
+    in_flight_age_ms: number;
+};
 export declare function isArchiveOtelLogsEnabled(): boolean;
 export declare function resolveArchiveForwarderUrlFromEnv(): string | undefined;
 export declare function resolveArchiveSourceFromEnv(value?: string | undefined): BrokerArchiveSource;
@@ -39,9 +52,20 @@ export declare class BrokerExecutionArchiver {
     private readonly flushIntervalMs;
     private readonly forwarderTimeoutMs;
     private readonly queue;
+    private readonly enqueueTimes;
     private readonly stats;
     private flushTimer;
     private flushInFlight;
+    private inFlightBatch;
+    private inFlightStartedAtMs;
+    private lastFailureAtMs;
+    private lastShedAtMs;
+    private lastSuccessAtMs;
+    private lastFailureEventSequence;
+    private lastShedEventSequence;
+    private lastSuccessEventSequence;
+    private lastSinkLatencyMs;
+    private healthEventSequence;
     private lastShedWarnAtMs;
     private closing;
     private closed;
@@ -61,13 +85,19 @@ export declare class BrokerExecutionArchiver {
     close(): Promise<void>;
     getStats(): Readonly<ArchiverStats>;
     getQueueDepth(): number;
+    getHealthSnapshot(): Readonly<BrokerExecutionArchiveHealthSnapshot>;
+    private oldestPendingEnqueueAtMs;
+    private recordHealthEvent;
+    private emitArchiveHealthMetrics;
     private closeLossJournal;
     private enforceQueueBound;
     private appendLossRecords;
     private flushBatch;
+    private clearInFlightBatch;
     private recordFlushHealth;
     private recordArchiveMetric;
     private recordArchiveGauge;
+    private recordArchiveObservableGauge;
     private emitOtelLog;
     private postToForwarder;
 }