@imbingox/acex 0.4.0-beta.15 → 0.4.0-beta.17

package/src/managers/market-manager.ts

@@ -19,10 +19,15 @@ import {
   buildAcexErrorDetails,
   formatAcexErrorMessage,
 } from "../errors.ts";
+import type {
+  AsyncEventBusOverflowInfo,
+  AsyncEventBusStreamOptions,
+} from "../internal/async-event-bus.ts";
 import { AsyncEventBus } from "../internal/async-event-bus.ts";
 import { toCanonical } from "../internal/decimal.ts";
 import { matchesMarketFilter } from "../internal/filters.ts";
 import type {
+  EventStreamOptions,
   FundingRateSnapshot,
   FundingRateUpdatedEvent,
   L1Book,
@@ -67,6 +72,14 @@ interface MarketRecord {
   status: MarketDataStatus;
   l1BookStream?: StreamHandle;
   fundingRateStream?: StreamHandle;
+  lastPublishedStatusKey?: MarketStatusPublicationKey;
+}
+
+interface MarketStatusPublicationKey {
+  activity: MarketDataStatus["activity"];
+  ready: MarketDataStatus["ready"];
+  freshness: MarketDataStatus["freshness"];
+  reason: MarketDataStatus["reason"];
 }
 
 interface CatalogFetchResult {
@@ -85,10 +98,38 @@ function marketKey(input: MarketKeyInput): string {
   return `${input.venue}:${input.symbol}`;
 }
 
+function marketEventConflateKey(event: MarketEvent): string {
+  return `${event.type}:${marketKey(event)}`;
+}
+
 function cloneMarketStatus(status: MarketDataStatus): MarketDataStatus {
   return { ...status };
 }
 
+function statusPublicationKey(
+  status: MarketDataStatus,
+): MarketStatusPublicationKey {
+  return {
+    activity: status.activity,
+    ready: status.ready,
+    freshness: status.freshness,
+    reason: status.reason,
+  };
+}
+
+function sameStatusPublicationKey(
+  current: MarketStatusPublicationKey,
+  previous: MarketStatusPublicationKey | undefined,
+): boolean {
+  return (
+    previous !== undefined &&
+    current.activity === previous.activity &&
+    current.ready === previous.ready &&
+    current.freshness === previous.freshness &&
+    current.reason === previous.reason
+  );
+}
+
 function cloneStreamStatus(
   status: MarketDataStreamStatus,
 ): MarketDataStreamStatus {
@@ -153,23 +194,49 @@ export class MarketManagerImpl
       options.l1ReconnectMaxDelayMs ?? DEFAULT_L1_RECONNECT_MAX_DELAY_MS;
 
     this.events = {
-      all: (filter) =>
-        this.marketBus.stream((event) => matchesMarketFilter(event, filter)),
-      fundingRateUpdates: (filter) =>
+      all: (filter, options) =>
+        this.marketBus.stream(
+          (event) => matchesMarketFilter(event, filter),
+          this.createStreamOptions(
+            "market.all",
+            options,
+            "buffer",
+            marketEventConflateKey,
+          ),
+        ),
+      fundingRateUpdates: (filter, options) =>
         this.marketBus.stream(
           (event): event is FundingRateUpdatedEvent =>
             event.type === "funding_rate.updated" &&
             matchesMarketFilter(event, filter),
+          this.createStreamOptions(
+            "market.fundingRateUpdates",
+            options,
+            "conflate",
+            marketKey,
+          ),
         ),
-      l1BookUpdates: (filter) =>
+      l1BookUpdates: (filter, options) =>
         this.marketBus.stream(
           (event): event is L1BookUpdatedEvent =>
             event.type === "l1_book.updated" &&
             matchesMarketFilter(event, filter),
+          this.createStreamOptions(
+            "market.l1BookUpdates",
+            options,
+            "conflate",
+            marketKey,
+          ),
         ),
-      status: (filter) =>
-        this.marketStatusBus.stream((event) =>
-          matchesMarketFilter(event, filter),
+      status: (filter, options) =>
+        this.marketStatusBus.stream(
+          (event) => matchesMarketFilter(event, filter),
+          this.createStreamOptions(
+            "market.status",
+            options,
+            "buffer",
+            marketKey,
+          ),
         ),
     };
   }
@@ -1064,6 +1131,14 @@ export class MarketManagerImpl
       record.status.lastReadyAt = this.resolveLastReadyAt(record);
     }
 
+    const publicationKey = statusPublicationKey(record.status);
+    if (
+      sameStatusPublicationKey(publicationKey, record.lastPublishedStatusKey)
+    ) {
+      return;
+    }
+
+    record.lastPublishedStatusKey = publicationKey;
     this.publishStatus(record);
   }
 
@@ -1184,6 +1259,35 @@ export class MarketManagerImpl
     this.context.publishHealthEvent(event);
   }
 
+  private createStreamOptions<U extends { venue: Venue; symbol: string }>(
+    stream: string,
+    options: EventStreamOptions | undefined,
+    defaultMode: "buffer" | "conflate",
+    conflateKey: (event: U) => string,
+  ): AsyncEventBusStreamOptions<U> {
+    return {
+      mode: options?.mode ?? defaultMode,
+      maxBuffer: options?.maxBuffer,
+      conflateKey,
+      onOverflow: this.createOverflowHandler(stream),
+    };
+  }
+
+  private createOverflowHandler(
+    stream: string,
+  ): (info: AsyncEventBusOverflowInfo) => void {
+    return ({ maxBuffer }) => {
+      const error = new AcexError(
+        "EVENT_BUFFER_OVERFLOW",
+        `Event stream buffer overflow: ${stream}`,
+      );
+      this.context.publishRuntimeError("market", error, {
+        stream,
+        maxBuffer,
+      });
+    };
+  }
+
   private async resumeStreams(): Promise<void> {
     for (const record of this.records.values()) {
       const market = record.market;

package/src/managers/order/data-status.ts

@@ -1,6 +1,7 @@
 import type { OrderDataStatus, Venue } from "../../types/index.ts";
 
 export const DEFAULT_MAX_CLOSED_ORDERS_PER_SYMBOL = 500;
+export const DEFAULT_MISSING_ORDER_EVICTION_THRESHOLD = 3;
 
 export function cloneOrderStatus(status: OrderDataStatus): OrderDataStatus {
   return { ...status };
@@ -28,6 +29,14 @@ export function normalizeMaxClosedOrdersPerSymbol(
     : DEFAULT_MAX_CLOSED_ORDERS_PER_SYMBOL;
 }
 
+export function normalizeMissingOrderEvictionThreshold(
+  value: number | undefined,
+): number {
+  return value !== undefined && Number.isInteger(value) && value > 0
+    ? value
+    : DEFAULT_MISSING_ORDER_EVICTION_THRESHOLD;
+}
+
 export function successfulStatus(
   status: OrderDataStatus,
   options: {

package/src/managers/order/model.ts

@@ -15,6 +15,7 @@ export interface OrderRecord {
   orderIdOnlyIndex: Map<string, Set<string>>;
   clientOrderIdIndex: Map<string, Set<string>>;
   pendingClientOrderIdIndex: Map<string, PendingOrderClaim>;
+  missingOrderConfirmations: Map<string, number>;
   status: OrderDataStatus;
 }
 
@@ -29,8 +30,10 @@ export interface OrderLocation {
 export interface PendingOrderClaim {
   localOrderId: string;
   symbol: string;
+  claimedAt: number;
 }
 
 export interface OrderManagerOptions {
   maxClosedOrdersPerSymbol?: number;
+  missingOrderEvictionThreshold?: number;
 }

package/src/managers/order/snapshot.ts

@@ -78,6 +78,7 @@ export function orderPriority(status: OrderSnapshot["status"]): number {
     case "expired":
       return 4;
     case "rejected":
+    case "unknown":
       return 3;
     case "partially_filled":
       return 2;

package/src/managers/order-manager.ts

@@ -5,6 +5,7 @@ import type {
 import type {
   AccountAwareManager,
   ClientContext,
+  ExpiredPendingOrderClaim,
   HealthReporter,
   ManagerLifecycle,
   PrivateOrderDataConsumer,
@@ -16,6 +17,7 @@ import {
   buildAcexErrorDetails,
   formatAcexErrorMessage,
 } from "../errors.ts";
+import type { AsyncEventBusOverflowInfo } from "../internal/async-event-bus.ts";
 import { AsyncEventBus } from "../internal/async-event-bus.ts";
 import { matchesOrderFilter } from "../internal/filters.ts";
 import { isTransportError } from "../internal/http-client.ts";
@@ -43,6 +45,7 @@ import {
   cloneOrderStatus,
   createOrderDataStatus,
   normalizeMaxClosedOrdersPerSymbol,
+  normalizeMissingOrderEvictionThreshold,
   successfulStatus,
 } from "./order/data-status.ts";
 import {
@@ -61,6 +64,7 @@ import { createSnapshot, isOpenOrder } from "./order/snapshot.ts";
 import {
   getAllSnapshots,
   getExistingSnapshot,
+  getExistingSnapshotLocation,
   getLocalOrderIdForVenueOrderId,
   getLocationByLocalOrderId,
   getOpenOrderSnapshots,
@@ -96,6 +100,7 @@ export class OrderManagerImpl
 
   private readonly context: ClientContext;
   private readonly maxClosedOrdersPerSymbol: number;
+  private readonly missingOrderEvictionThreshold: number;
   private readonly orderBus = new AsyncEventBus<OrderEvent>();
   private readonly orderStatusBus =
     new AsyncEventBus<OrderStatusChangedEvent>();
@@ -107,25 +112,38 @@ export class OrderManagerImpl
     this.maxClosedOrdersPerSymbol = normalizeMaxClosedOrdersPerSymbol(
       options.maxClosedOrdersPerSymbol,
     );
+    this.missingOrderEvictionThreshold = normalizeMissingOrderEvictionThreshold(
+      options.missingOrderEvictionThreshold,
+    );
 
     this.events = {
-      status: (filter) =>
-        this.orderStatusBus.stream((event) =>
-          matchesOrderFilter(
-            { accountId: event.accountId, venue: event.venue },
-            filter,
-          ),
+      status: (filter, options) =>
+        this.orderStatusBus.stream(
+          (event) =>
+            matchesOrderFilter(
+              { accountId: event.accountId, venue: event.venue },
+              filter,
+            ),
+          {
+            maxBuffer: options?.maxBuffer,
+            onOverflow: this.createOverflowHandler("order.status"),
+          },
         ),
-      updates: (filter) =>
-        this.orderBus.stream((event) =>
-          matchesOrderFilter(
-            {
-              accountId: event.accountId,
-              venue: event.venue,
-              symbol: "symbol" in event ? event.symbol : undefined,
-            },
-            filter,
-          ),
+      updates: (filter, options) =>
+        this.orderBus.stream(
+          (event) =>
+            matchesOrderFilter(
+              {
+                accountId: event.accountId,
+                venue: event.venue,
+                symbol: "symbol" in event ? event.symbol : undefined,
+              },
+              filter,
+            ),
+          {
+            maxBuffer: options?.maxBuffer,
+            onOverflow: this.createOverflowHandler("order.updates"),
+          },
         ),
     };
   }
@@ -494,6 +512,9 @@ export class OrderManagerImpl
         openSetKeys.add(lookupKey);
       }
       const current = getExistingSnapshot(record, update);
+      if (current) {
+        this.clearMissingOrderConfirmationsForUpdate(record, current);
+      }
       const nextSnapshot = this.applyUpdateToRecord(
         record,
         accountId,
@@ -565,6 +586,36 @@ export class OrderManagerImpl
     return this.getOpenOrders(accountId);
   }
 
+  getExpiredPrivateOrderClaims(
+    accountId: string,
+    now: number,
+    ttlMs: number,
+  ): ExpiredPendingOrderClaim[] {
+    const record = this.records.get(accountId);
+    if (!record || ttlMs <= 0) {
+      return [];
+    }
+
+    const expired: ExpiredPendingOrderClaim[] = [];
+    for (const [
+      venueClientOrderId,
+      claim,
+    ] of record.pendingClientOrderIdIndex) {
+      if (now - claim.claimedAt < ttlMs) {
+        continue;
+      }
+
+      expired.push({
+        venueClientOrderId,
+        localOrderId: claim.localOrderId,
+        symbol: claim.symbol,
+        claimedAt: claim.claimedAt,
+      });
+    }
+
+    return expired;
+  }
+
   onPrivateOrderUpdate(
     accountId: string,
     venue: Venue,
@@ -590,32 +641,129 @@ export class OrderManagerImpl
       return;
     }
 
-    const eventType =
-      snapshot.status === "filled"
-        ? "order.filled"
-        : snapshot.status === "rejected"
-          ? "order.rejected"
-          : snapshot.status === "canceled" || snapshot.status === "expired"
-            ? "order.canceled"
-            : "order.updated";
+    this.publishOrderEvent(accountId, venue, snapshot);
 
-    this.orderBus.publish({
-      type: eventType,
+    record.status = successfulStatus(record.status, {
+      preserveStatus: options.preserveStatus,
+      lastReceivedAt: snapshot.receivedAt,
+      lastReadyAt: snapshot.updatedAt,
+    });
+    this.publishStatus(record);
+  }
+
+  onPrivateOrderConfirmedMissing(
+    accountId: string,
+    venue: Venue,
+    order: OrderSnapshot,
+  ): void {
+    const record = this.getOrCreateRecord(accountId, venue);
+    if (!record.subscribed) {
+      return;
+    }
+
+    const location = getExistingSnapshotLocation(record, order);
+    if (!location || location.table !== "open") {
+      return;
+    }
+
+    const current = getSnapshotAtLocation(record, location);
+    if (!current || !isOpenOrder(current)) {
+      return;
+    }
+
+    const confirmations =
+      (record.missingOrderConfirmations.get(location.localOrderId) ?? 0) + 1;
+    if (confirmations < this.missingOrderEvictionThreshold) {
+      record.missingOrderConfirmations.set(
+        location.localOrderId,
+        confirmations,
+      );
+      return;
+    }
+
+    const receivedAt = this.context.now();
+    const snapshot = createSnapshot(
       accountId,
       venue,
-      symbol: snapshot.symbol,
-      snapshot,
-      ts: this.context.now(),
-    });
+      {
+        orderId: current.orderId,
+        clientOrderId: current.clientOrderId,
+        symbol: current.symbol,
+        side: current.side,
+        type: current.type,
+        status: "unknown",
+        price: current.price,
+        triggerPrice: current.triggerPrice,
+        amount: current.amount,
+        filled: current.filled,
+        remaining: current.remaining,
+        reduceOnly: current.reduceOnly,
+        positionSide: current.positionSide,
+        avgFillPrice: current.avgFillPrice,
+        receivedAt,
+      },
+      current,
+    );
 
+    if (
+      !this.writeSnapshot(record, location.localOrderId, snapshot, location)
+    ) {
+      return;
+    }
+
+    this.context.publishRuntimeError(
+      "order",
+      new Error(
+        `Evicted ${venue} open order after ${confirmations} confirmed missing checks`,
+      ),
+      {
+        accountId,
+        venue,
+        symbol: current.symbol,
+      },
+    );
+    this.publishOrderEvent(accountId, venue, snapshot);
     record.status = successfulStatus(record.status, {
-      preserveStatus: options.preserveStatus,
       lastReceivedAt: snapshot.receivedAt,
       lastReadyAt: snapshot.updatedAt,
     });
     this.publishStatus(record);
   }
 
+  onPrivateOrderClaimNotFound(
+    accountId: string,
+    venue: Venue,
+    claim: ExpiredPendingOrderClaim,
+  ): void {
+    const record = this.records.get(accountId);
+    if (!record) {
+      return;
+    }
+
+    const pending = record.pendingClientOrderIdIndex.get(
+      claim.venueClientOrderId,
+    );
+    if (
+      pending?.localOrderId !== claim.localOrderId ||
+      pending.symbol !== claim.symbol
+    ) {
+      return;
+    }
+
+    record.pendingClientOrderIdIndex.delete(claim.venueClientOrderId);
+    this.context.publishRuntimeError(
+      "order",
+      new Error(
+        `createOrder timed out and the order was not found on the venue: ${claim.venueClientOrderId}`,
+      ),
+      {
+        accountId,
+        venue,
+        symbol: claim.symbol,
+      },
+    );
+  }
+
   onPrivateOrderStreamState(
     accountId: string,
     venue: Venue,
@@ -670,6 +818,7 @@ export class OrderManagerImpl
       orderIdOnlyIndex: new Map(),
       clientOrderIdIndex: new Map(),
       pendingClientOrderIdIndex: new Map(),
+      missingOrderConfirmations: new Map(),
       status: createOrderDataStatus(accountId, venue, "inactive"),
     };
 
@@ -718,9 +867,51 @@ export class OrderManagerImpl
 
     this.warnSystemClientOrderIdOnlyClaim(record, snapshot);
     this.warnProvisionalTerminalOrder(record, snapshot);
+    this.clearMissingOrderConfirmations(record, localOrderId);
     return true;
   }
 
+  private clearMissingOrderConfirmations(
+    record: OrderRecord,
+    localOrderId: string,
+  ): void {
+    record.missingOrderConfirmations.delete(localOrderId);
+  }
+
+  private clearMissingOrderConfirmationsForUpdate(
+    record: OrderRecord,
+    update: { symbol: string; orderId?: string; clientOrderId?: string },
+  ): void {
+    const location = getExistingSnapshotLocation(record, update);
+    if (location) {
+      this.clearMissingOrderConfirmations(record, location.localOrderId);
+    }
+  }
+
+  private publishOrderEvent(
+    accountId: string,
+    venue: Venue,
+    snapshot: OrderSnapshot,
+  ): void {
+    const eventType =
+      snapshot.status === "filled"
+        ? "order.filled"
+        : snapshot.status === "rejected"
+          ? "order.rejected"
+          : isOpenOrder(snapshot)
+            ? "order.updated"
+            : "order.canceled";
+
+    this.orderBus.publish({
+      type: eventType,
+      accountId,
+      venue,
+      symbol: snapshot.symbol,
+      snapshot,
+      ts: this.context.now(),
+    });
+  }
+
   private warnDroppedUnkeyedTerminalOrder(
     record: OrderRecord,
     snapshot: OrderSnapshot,
@@ -800,6 +991,9 @@ export class OrderManagerImpl
     options: { requestStartedAt?: number; preserveStatus?: boolean } = {},
   ): OrderSnapshot | undefined {
     const resolution = this.resolveLocalOrderIdForUpdate(record, update);
+    if (resolution.localOrderId) {
+      this.clearMissingOrderConfirmations(record, resolution.localOrderId);
+    }
     const localOrderId = resolution.localOrderId ?? this.generateLocalOrderId();
     const previousLocation = getLocationByLocalOrderId(record, localOrderId);
     const previous = previousLocation
@@ -874,6 +1068,7 @@ export class OrderManagerImpl
     record.pendingClientOrderIdIndex.set(venueClientOrderId, {
       localOrderId,
       symbol,
+      claimedAt: this.context.now(),
     });
   }
 
@@ -955,6 +1150,9 @@ export class OrderManagerImpl
       update,
       options.localOrderId,
     );
+    if (resolution.localOrderId) {
+      this.clearMissingOrderConfirmations(record, resolution.localOrderId);
+    }
     const localOrderId = resolution.localOrderId ?? this.generateLocalOrderId();
     const previousLocation = getLocationByLocalOrderId(record, localOrderId);
     const previous = previousLocation
@@ -1083,6 +1281,21 @@ export class OrderManagerImpl
       },
     };
   }
+
+  private createOverflowHandler(
+    stream: string,
+  ): (info: AsyncEventBusOverflowInfo) => void {
+    return ({ maxBuffer }) => {
+      const error = new AcexError(
+        "EVENT_BUFFER_OVERFLOW",
+        `Event stream buffer overflow: ${stream}`,
+      );
+      this.context.publishRuntimeError("order", error, {
+        stream,
+        maxBuffer,
+      });
+    };
+  }
 }
 
 function isOrderErrorCode(

package/src/types/account.ts

@@ -1,4 +1,5 @@
 import type {
+  BufferedEventStreamOptions,
   PrivateRuntimeReason,
   PrivateRuntimeStatus,
   SubscriptionActivity,
@@ -158,8 +159,14 @@ export type AccountEvent =
   | AccountSnapshotReplacedEvent;
 
 export interface AccountEventStreams {
-  updates(filter?: AccountEventFilter): AsyncIterable<AccountEvent>;
-  status(filter?: AccountEventFilter): AsyncIterable<AccountStatusChangedEvent>;
+  updates(
+    filter?: AccountEventFilter,
+    options?: BufferedEventStreamOptions,
+  ): AsyncIterable<AccountEvent>;
+  status(
+    filter?: AccountEventFilter,
+    options?: BufferedEventStreamOptions,
+  ): AsyncIterable<AccountStatusChangedEvent>;
 }
 
 export interface AccountManager {

package/src/types/client.ts

@@ -18,6 +18,7 @@ import type {
 import type {
   AccountCredentials,
   AcexInternalError,
+  BufferedEventStreamOptions,
   ClientStatus,
   CreateClientOptions,
   RegisterAccountInput,
@@ -54,8 +55,13 @@ export interface HealthEventFilter {
 }
 
 export interface ClientEventStreams {
-  health(filter?: HealthEventFilter): AsyncIterable<HealthEvent>;
-  errors(): AsyncIterable<AcexInternalError>;
+  health(
+    filter?: HealthEventFilter,
+    options?: BufferedEventStreamOptions,
+  ): AsyncIterable<HealthEvent>;
+  errors(
+    options?: BufferedEventStreamOptions,
+  ): AsyncIterable<AcexInternalError>;
 }
 
 export type VenueRuntimeStatus = "available" | "type_only" | "reserved";

package/src/types/market.ts

@@ -1,6 +1,11 @@
 import type BigNumber from "bignumber.js";
 import type { AcexError } from "../errors.ts";
-import type { MarketFreshness, SubscriptionActivity, Venue } from "./shared.ts";
+import type {
+  EventStreamOptions,
+  MarketFreshness,
+  SubscriptionActivity,
+  Venue,
+} from "./shared.ts";
 
 export type MarketType = "spot" | "swap" | "future";
 
@@ -170,12 +175,22 @@ export type MarketEvent =
   | MarketStatusChangedEvent;
 
 export interface MarketEventStreams {
-  l1BookUpdates(filter?: MarketEventFilter): AsyncIterable<L1BookUpdatedEvent>;
+  l1BookUpdates(
+    filter?: MarketEventFilter,
+    options?: EventStreamOptions,
+  ): AsyncIterable<L1BookUpdatedEvent>;
   fundingRateUpdates(
     filter?: MarketEventFilter,
+    options?: EventStreamOptions,
   ): AsyncIterable<FundingRateUpdatedEvent>;
-  status(filter?: MarketEventFilter): AsyncIterable<MarketStatusChangedEvent>;
-  all(filter?: MarketEventFilter): AsyncIterable<MarketEvent>;
+  status(
+    filter?: MarketEventFilter,
+    options?: EventStreamOptions,
+  ): AsyncIterable<MarketStatusChangedEvent>;
+  all(
+    filter?: MarketEventFilter,
+    options?: EventStreamOptions,
+  ): AsyncIterable<MarketEvent>;
 }
 
 export interface MarketManager {