@imbingox/acex 0.4.0-beta.1 → 0.4.0-beta.10

package/src/internal/rate-limiter.ts

@@ -0,0 +1,181 @@
+import type {
+  RateLimiter,
+  RateLimitRequestContext,
+  RateLimitResponseContext,
+  RateLimitScope,
+  RateLimitSnapshot,
+  RateLimitTransportErrorContext,
+  RateLimitUsage,
+} from "../types/index.ts";
+
+interface ReactiveRateLimiterOptions {
+  readonly now?: () => number;
+  readonly sleep?: (ms: number) => Promise<void>;
+  readonly defaultRateLimitMs?: number;
+  readonly defaultBanMs?: number;
+}
+
+interface RateLimitState {
+  usage?: RateLimitUsage;
+  blockedUntil?: number;
+  retryAfterMs?: number;
+  state: RateLimitSnapshot["state"];
+  updatedAt?: number;
+}
+
+const DEFAULT_RATE_LIMIT_MS = 0;
+const DEFAULT_BAN_MS = 60_000;
+
+export class ReactiveRateLimiter implements RateLimiter {
+  private readonly now: () => number;
+  private readonly sleep: (ms: number) => Promise<void>;
+  private readonly defaultRateLimitMs: number;
+  private readonly defaultBanMs: number;
+  private readonly states = new Map<string, RateLimitState>();
+
+  constructor(options: ReactiveRateLimiterOptions = {}) {
+    this.now = options.now ?? Date.now;
+    this.sleep = options.sleep ?? defaultSleep;
+    this.defaultRateLimitMs =
+      options.defaultRateLimitMs ?? DEFAULT_RATE_LIMIT_MS;
+    this.defaultBanMs = options.defaultBanMs ?? DEFAULT_BAN_MS;
+  }
+
+  async beforeRequest(ctx: RateLimitRequestContext): Promise<void> {
+    const snapshot = this.getSnapshot(ctx.scope);
+    if (!snapshot?.blockedUntil || snapshot.blockedUntil <= this.now()) {
+      return;
+    }
+
+    await this.sleep(Math.max(0, snapshot.blockedUntil - this.now()));
+  }
+
+  afterResponse(
+    ctx: RateLimitRequestContext,
+    response: RateLimitResponseContext,
+  ): void {
+    if (response.usage) {
+      const existing = this.getState(ctx.scope);
+      const hasActiveBlock =
+        existing?.blockedUntil !== undefined &&
+        existing.blockedUntil > this.now();
+      this.updateState(ctx.scope, {
+        usage: cloneUsage(response.usage),
+        state: hasActiveBlock ? existing.state : "ok",
+      });
+    }
+  }
+
+  onTransportError(
+    ctx: RateLimitRequestContext,
+    error: RateLimitTransportErrorContext,
+  ): void {
+    if (error.usage) {
+      this.updateState(ctx.scope, {
+        usage: cloneUsage(error.usage),
+      });
+    }
+
+    if (error.status !== 429 && error.status !== 418) {
+      return;
+    }
+
+    const now = this.now();
+    const isBan = error.status === 418;
+    const retryAfterMs =
+      error.retryAfterMs ??
+      (isBan ? this.defaultBanMs : this.defaultRateLimitMs);
+    const blockedUntil =
+      retryAfterMs > 0
+        ? now + retryAfterMs
+        : this.getState(ctx.scope)?.blockedUntil;
+
+    this.updateState(ctx.scope, {
+      blockedUntil,
+      retryAfterMs,
+      state: isBan ? "banned" : "rate_limited",
+    });
+  }
+
+  getSnapshot(scope: RateLimitScope): RateLimitSnapshot | undefined {
+    const state = this.getState(scope);
+    if (!state) {
+      return undefined;
+    }
+
+    const now = this.now();
+    const blockedUntil =
+      state.blockedUntil !== undefined && state.blockedUntil > now
+        ? state.blockedUntil
+        : undefined;
+    const runtimeState =
+      blockedUntil === undefined && state.state !== "ok" ? "ok" : state.state;
+
+    return {
+      scope: { ...scope },
+      usage: state.usage ? cloneUsage(state.usage) : undefined,
+      blockedUntil,
+      retryAfterMs: blockedUntil ? state.retryAfterMs : undefined,
+      state: runtimeState,
+      updatedAt: state.updatedAt,
+    };
+  }
+
+  private getState(scope: RateLimitScope): RateLimitState | undefined {
+    return this.states.get(scopeKey(scope));
+  }
+
+  private updateState(
+    scope: RateLimitScope,
+    patch: Partial<RateLimitState>,
+  ): void {
+    const existing = this.getState(scope);
+    const nextBlockedUntil = maxOptional(
+      existing?.blockedUntil,
+      patch.blockedUntil,
+    );
+    const nextState =
+      patch.state ??
+      (nextBlockedUntil !== undefined
+        ? (existing?.state ?? "ok")
+        : existing?.state);
+
+    this.states.set(scopeKey(scope), {
+      usage: patch.usage ?? existing?.usage,
+      blockedUntil: nextBlockedUntil,
+      retryAfterMs: patch.retryAfterMs ?? existing?.retryAfterMs,
+      state: nextState ?? "ok",
+      updatedAt: this.now(),
+    });
+  }
+}
+
+function scopeKey(scope: RateLimitScope): string {
+  return [scope.venue, scope.accountId ?? "", scope.endpointKey].join("\0");
+}
+
+function cloneUsage(usage: RateLimitUsage): RateLimitUsage {
+  return {
+    weight: usage.weight ? { ...usage.weight } : undefined,
+    orderCount: usage.orderCount ? { ...usage.orderCount } : undefined,
+  };
+}
+
+function maxOptional(
+  left: number | undefined,
+  right: number | undefined,
+): number | undefined {
+  if (left === undefined) {
+    return right;
+  }
+  if (right === undefined) {
+    return left;
+  }
+  return Math.max(left, right);
+}
+
+function defaultSleep(ms: number): Promise<void> {
+  return new Promise((resolve) => {
+    setTimeout(resolve, ms);
+  });
+}

package/src/internal/watermark.ts

@@ -0,0 +1,83 @@
+export const CROSS_CLOCK_WATERMARK_GRACE_MS = 10_000;
+
+export interface WatermarkedRecord {
+  exchangeTs?: number;
+  receivedAt: number;
+}
+
+export interface WatermarkApplyOptions {
+  requestStartedAt?: number;
+  source?: "command" | "rest" | "stream";
+  graceMs?: number;
+}
+
+export interface SnapshotDeletionGuard {
+  requestStartedAt: number;
+  snapshotExchangeTs?: number;
+}
+
+export function shouldApplyWatermarkedUpdate(
+  current: WatermarkedRecord | undefined,
+  incoming: WatermarkedRecord,
+  options: WatermarkApplyOptions = {},
+): boolean {
+  if (!current) {
+    return true;
+  }
+
+  const graceMs = options.graceMs ?? CROSS_CLOCK_WATERMARK_GRACE_MS;
+  const requestStartedAt = options.requestStartedAt;
+
+  if (
+    options.source === "rest" &&
+    requestStartedAt !== undefined &&
+    current.receivedAt > requestStartedAt &&
+    (current.exchangeTs === undefined || incoming.exchangeTs === undefined)
+  ) {
+    return false;
+  }
+
+  if (current.exchangeTs !== undefined && incoming.exchangeTs !== undefined) {
+    if (incoming.exchangeTs < current.exchangeTs) {
+      return false;
+    }
+    if (incoming.exchangeTs > current.exchangeTs) {
+      return true;
+    }
+
+    return incoming.receivedAt >= current.receivedAt;
+  }
+
+  if (current.exchangeTs !== undefined) {
+    if (incoming.receivedAt < current.exchangeTs + graceMs) {
+      return false;
+    }
+
+    return incoming.receivedAt >= current.receivedAt;
+  }
+
+  if (incoming.exchangeTs !== undefined) {
+    if (incoming.exchangeTs < current.receivedAt - graceMs) {
+      return false;
+    }
+
+    return incoming.receivedAt >= current.receivedAt;
+  }
+
+  return incoming.receivedAt >= current.receivedAt;
+}
+
+export function canDeleteMissingFromSnapshot(
+  current: WatermarkedRecord,
+  guard: SnapshotDeletionGuard,
+): boolean {
+  if (current.receivedAt > guard.requestStartedAt) {
+    return false;
+  }
+
+  return !(
+    current.exchangeTs !== undefined &&
+    guard.snapshotExchangeTs !== undefined &&
+    current.exchangeTs > guard.snapshotExchangeTs
+  );
+}

package/src/managers/account-manager.ts

@@ -17,6 +17,10 @@ import type {
 import { AsyncEventBus } from "../internal/async-event-bus.ts";
 import { toCanonical } from "../internal/decimal.ts";
 import { matchesAccountFilter } from "../internal/filters.ts";
+import {
+  canDeleteMissingFromSnapshot,
+  shouldApplyWatermarkedUpdate,
+} from "../internal/watermark.ts";
 import type {
   AccountDataStatus,
   AccountEvent,
@@ -61,6 +65,46 @@ function isZeroDecimal(value: string): boolean {
   return new BigNumber(value).isZero();
 }
 
+function isZeroBalance(balance: BalanceSnapshot): boolean {
+  return (
+    isZeroDecimal(balance.free) &&
+    isZeroDecimal(balance.used) &&
+    isZeroDecimal(balance.total)
+  );
+}
+
+function successfulStatus(
+  status: AccountDataStatus,
+  options: {
+    ready?: boolean;
+    lastReceivedAt?: number;
+    lastReadyAt?: number;
+    preserveStatus?: boolean;
+  },
+): AccountDataStatus {
+  const preservesStreamState =
+    options.preserveStatus &&
+    (status.runtimeStatus === "reconnecting" ||
+      status.reason === "ws_disconnected" ||
+      status.reason === "heartbeat_timeout");
+  const ready = options.ready ?? true;
+
+  return {
+    ...status,
+    activity: "active",
+    ready,
+    runtimeStatus: preservesStreamState ? status.runtimeStatus : "healthy",
+    reason: preservesStreamState ? status.reason : undefined,
+    lastReceivedAt: options.lastReceivedAt ?? status.lastReceivedAt,
+    lastReadyAt: ready
+      ? (options.lastReadyAt ??
+        (options.preserveStatus ? status.lastReadyAt : undefined) ??
+        Date.now())
+      : status.lastReadyAt,
+    inactiveSince: undefined,
+  };
+}
+
 export class AccountManagerImpl
   implements
     AccountManager,
@@ -282,7 +326,7 @@ export class AccountManagerImpl
     accountId: string,
     venue: Venue,
     update: RawAccountUpdate,
-    options: { preserveStatus?: boolean } = {},
+    options: { preserveStatus?: boolean; requestStartedAt?: number } = {},
   ): void {
     const record = this.getOrCreateRecord(accountId, venue);
     if (!record.subscribed) {
@@ -300,7 +344,17 @@ export class AccountManagerImpl
     );
     let risk = previous.risk;
 
+    let latestAppliedAt = 0;
     for (const balance of update.balances ?? []) {
+      if (
+        !shouldApplyWatermarkedUpdate(balances[balance.asset], balance, {
+          requestStartedAt: options.requestStartedAt,
+          source: options.requestStartedAt === undefined ? "stream" : "rest",
+        })
+      ) {
+        continue;
+      }
+
       const nextBalance = this.createBalance(
         accountId,
         venue,
@@ -308,6 +362,7 @@ export class AccountManagerImpl
         balances[balance.asset],
       );
       balances[balance.asset] = nextBalance;
+      latestAppliedAt = Math.max(latestAppliedAt, nextBalance.receivedAt);
       this.accountBus.publish({
         type: "balance.updated",
         accountId,
@@ -320,6 +375,15 @@ export class AccountManagerImpl
 
     for (const position of update.positions ?? []) {
       const key = positionKey(position.symbol, position.side);
+      if (
+        !shouldApplyWatermarkedUpdate(positions.get(key), position, {
+          requestStartedAt: options.requestStartedAt,
+          source: options.requestStartedAt === undefined ? "stream" : "rest",
+        })
+      ) {
+        continue;
+      }
+
       const nextPosition = this.createPosition(
         accountId,
         venue,
@@ -333,6 +397,7 @@ export class AccountManagerImpl
         positions.set(key, nextPosition);
       }
 
+      latestAppliedAt = Math.max(latestAppliedAt, nextPosition.receivedAt);
       this.accountBus.publish({
         type: "position.updated",
         accountId,
@@ -343,8 +408,15 @@ export class AccountManagerImpl
       });
     }
 
-    if (update.risk) {
+    if (
+      update.risk &&
+      shouldApplyWatermarkedUpdate(previous.risk, update.risk, {
+        requestStartedAt: options.requestStartedAt,
+        source: options.requestStartedAt === undefined ? "stream" : "rest",
+      })
+    ) {
       risk = this.createRisk(accountId, venue, update.risk, previous.risk);
+      latestAppliedAt = Math.max(latestAppliedAt, risk.receivedAt);
       this.accountBus.publish({
         type: "risk.updated",
         accountId,
@@ -354,34 +426,166 @@ export class AccountManagerImpl
       });
     }
 
+    if (latestAppliedAt === 0) {
+      return;
+    }
+
     record.snapshot = {
       accountId,
       venue,
       balances,
       positions: [...positions.values()],
       risk,
-      exchangeTs: update.exchangeTs ?? previous.exchangeTs,
-      receivedAt: update.receivedAt,
-      updatedAt: update.receivedAt,
+      exchangeTs:
+        update.exchangeTs === undefined
+          ? previous.exchangeTs
+          : update.exchangeTs,
+      receivedAt: latestAppliedAt,
+      updatedAt: latestAppliedAt,
     };
-    record.status = options.preserveStatus
-      ? {
-          ...record.status,
-          activity: "active",
-          lastReceivedAt: update.receivedAt,
-          lastReadyAt: record.status.lastReadyAt ?? update.receivedAt,
-          inactiveSince: undefined,
-        }
-      : {
-          ...record.status,
-          activity: "active",
-          ready: true,
-          runtimeStatus: "healthy",
-          reason: undefined,
-          lastReceivedAt: update.receivedAt,
-          lastReadyAt: update.receivedAt,
-          inactiveSince: undefined,
-        };
+    record.status = successfulStatus(record.status, {
+      preserveStatus: options.preserveStatus,
+      lastReceivedAt: latestAppliedAt,
+      lastReadyAt: latestAppliedAt,
+    });
+    this.publishStatus(record);
+  }
+
+  onPrivateAccountReconcile(
+    accountId: string,
+    venue: Venue,
+    snapshot: RawAccountBootstrap,
+    options: { requestStartedAt: number; preserveStatus?: boolean },
+  ): void {
+    const record = this.getOrCreateRecord(accountId, venue);
+    if (!record.subscribed) {
+      return;
+    }
+
+    const previous =
+      record.snapshot ?? this.createEmptySnapshot(accountId, venue);
+    const balances = { ...previous.balances };
+    const positions = new Map(
+      previous.positions.map((position) => [
+        positionKey(position.symbol, position.side),
+        position,
+      ]),
+    );
+    let risk = previous.risk;
+
+    const incomingBalanceAssets = new Set<string>();
+    for (const balance of snapshot.balances) {
+      incomingBalanceAssets.add(balance.asset);
+      if (
+        !shouldApplyWatermarkedUpdate(balances[balance.asset], balance, {
+          requestStartedAt: options.requestStartedAt,
+          source: "rest",
+        })
+      ) {
+        continue;
+      }
+
+      const nextBalance = this.createBalance(
+        accountId,
+        venue,
+        balance,
+        balances[balance.asset],
+      );
+      if (isZeroBalance(nextBalance)) {
+        delete balances[balance.asset];
+      } else {
+        balances[balance.asset] = nextBalance;
+      }
+    }
+
+    for (const [asset, balance] of Object.entries(balances)) {
+      if (
+        (!incomingBalanceAssets.has(asset) || isZeroBalance(balance)) &&
+        canDeleteMissingFromSnapshot(balance, {
+          requestStartedAt: options.requestStartedAt,
+          snapshotExchangeTs: snapshot.exchangeTs,
+        })
+      ) {
+        delete balances[asset];
+      }
+    }
+
+    const incomingPositionKeys = new Set<string>();
+    for (const position of snapshot.positions) {
+      const key = positionKey(position.symbol, position.side);
+      incomingPositionKeys.add(key);
+      if (
+        !shouldApplyWatermarkedUpdate(positions.get(key), position, {
+          requestStartedAt: options.requestStartedAt,
+          source: "rest",
+        })
+      ) {
+        continue;
+      }
+
+      const nextPosition = this.createPosition(
+        accountId,
+        venue,
+        position,
+        positions.get(key),
+      );
+      if (isZeroDecimal(nextPosition.size)) {
+        positions.delete(key);
+      } else {
+        positions.set(key, nextPosition);
+      }
+    }
+
+    for (const [key, position] of positions.entries()) {
+      if (
+        !incomingPositionKeys.has(key) &&
+        canDeleteMissingFromSnapshot(position, {
+          requestStartedAt: options.requestStartedAt,
+          snapshotExchangeTs: snapshot.exchangeTs,
+        })
+      ) {
+        positions.delete(key);
+      }
+    }
+
+    if (
+      snapshot.risk &&
+      shouldApplyWatermarkedUpdate(previous.risk, snapshot.risk, {
+        requestStartedAt: options.requestStartedAt,
+        source: "rest",
+      })
+    ) {
+      risk = this.createRisk(accountId, venue, snapshot.risk, previous.risk);
+    }
+
+    record.snapshot = {
+      accountId,
+      venue,
+      balances,
+      positions: [...positions.values()],
+      risk,
+      exchangeTs:
+        snapshot.exchangeTs === undefined
+          ? previous.exchangeTs
+          : snapshot.exchangeTs,
+      receivedAt: snapshot.receivedAt,
+      updatedAt: snapshot.receivedAt,
+    };
+    record.status = successfulStatus(record.status, {
+      preserveStatus: options.preserveStatus,
+      lastReceivedAt: snapshot.receivedAt,
+      lastReadyAt: snapshot.receivedAt,
+    });
+
+    const event: AccountSnapshotReplacedEvent = {
+      type: "account.snapshot_replaced",
+      accountId,
+      venue,
+      snapshot: record.snapshot,
+      ts: this.context.now(),
+    };
+
+    this.accountBus.publish(event);
     this.publishStatus(record);
   }