@imbingox/acex 0.4.0-beta.1 → 0.4.0-beta.10

package/src/client/private-subscription-coordinator.ts

@@ -1,9 +1,21 @@
 import type {
+  FetchOrderRequest,
   PrivateUserDataAdapter,
+  RawOpenOrdersSnapshot,
   StreamHandle,
 } from "../adapters/types.ts";
-import { AcexError } from "../errors.ts";
-import type { AccountRuntimeOptions, Venue } from "../types/index.ts";
+import {
+  AcexError,
+  buildAcexErrorDetails,
+  formatAcexErrorMessage,
+} from "../errors.ts";
+import { isTransportError } from "../internal/http-client.ts";
+import type {
+  AccountRuntimeOptions,
+  OrderSnapshot,
+  PrivateRuntimeReason,
+  Venue,
+} from "../types/index.ts";
 import type {
   ClientContext,
   PrivateAccountDataConsumer,
@@ -22,6 +34,11 @@ interface PrivateSubscriptionRecord {
   accountRefreshTimer?: ReturnType<typeof setTimeout>;
   accountRefreshInFlight?: Promise<void>;
   accountRefreshGeneration: number;
+  accountSubscriptionGeneration: number;
+  orderSubscriptionGeneration: number;
+  privateReconcileTimer?: ReturnType<typeof setTimeout>;
+  privateReconcileInFlight?: Promise<void>;
+  privateReconcileGeneration: number;
   startPromise?: Promise<void>;
   reconcilePromise?: Promise<void>;
 }
@@ -31,6 +48,9 @@ const DEFAULT_STREAM_RECONNECT_DELAY_MS = 1_000;
 const DEFAULT_STREAM_RECONNECT_MAX_DELAY_MS = 10_000;
 const DEFAULT_LISTEN_KEY_KEEPALIVE_MS = 30 * 60 * 1_000;
 const DEFAULT_BINANCE_RISK_POLL_INTERVAL_MS = 5_000;
+const DEFAULT_BINANCE_PRIVATE_RECONCILE_INTERVAL_MS = 60_000;
+const MAX_ORDER_TERMINAL_BACKFILLS_PER_RECONCILE = 20;
+const MAX_ORDER_TERMINAL_BACKFILL_CONCURRENCY = 4;
 
 function normalizePositiveInterval(
   value: number | undefined,
@@ -41,6 +61,26 @@ function normalizePositiveInterval(
     : fallback;
 }
 
+function normalizeReconcileInterval(
+  value: number | undefined,
+  fallback: number,
+): number | undefined {
+  if (value === 0) {
+    return undefined;
+  }
+
+  return normalizePositiveInterval(value, fallback);
+}
+
+function transportReason(
+  error: unknown,
+  fallback: PrivateRuntimeReason,
+): PrivateRuntimeReason {
+  return isTransportError(error) && error.kind === "rate_limited"
+    ? "rate_limited"
+    : fallback;
+}
+
 export class PrivateSubscriptionCoordinator {
   private readonly context: ClientContext;
   private readonly adapters: Map<Venue, PrivateUserDataAdapter>;
@@ -51,7 +91,7 @@ export class PrivateSubscriptionCoordinator {
   private readonly streamReconnectMaxDelayMs: number;
   private readonly listenKeyKeepAliveMs: number;
   private readonly binanceRiskPollIntervalMs: number;
-  private readonly juplendPollIntervalMs?: number;
+  private readonly binancePrivateReconcileIntervalMs: number | undefined;
   private readonly records = new Map<string, PrivateSubscriptionRecord>();
 
   constructor(
@@ -80,7 +120,10 @@ export class PrivateSubscriptionCoordinator {
       options.binance?.riskPollIntervalMs,
       DEFAULT_BINANCE_RISK_POLL_INTERVAL_MS,
     );
-    this.juplendPollIntervalMs = options.juplend?.pollIntervalMs;
+    this.binancePrivateReconcileIntervalMs = normalizeReconcileInterval(
+      options.binance?.privateReconcileIntervalMs,
+      DEFAULT_BINANCE_PRIVATE_RECONCILE_INTERVAL_MS,
+    );
   }
 
   async subscribeAccountFeed(accountId: string): Promise<void> {
@@ -88,18 +131,69 @@ export class PrivateSubscriptionCoordinator {
     const record = this.getOrCreateRecord(account);
     const needsPending = !record.stream && !record.startPromise;
     record.accountSubscribed = true;
+    const generation = record.privateReconcileGeneration;
+    const accountGeneration = record.accountSubscriptionGeneration;
     if (needsPending) {
       this.accountConsumer.onPrivateAccountPending(accountId, record.venue);
     }
 
     try {
-      if (record.venue === "juplend") {
-        await this.bootstrapAccount(record, account);
+      const adapter = this.getAdapter(record.venue);
+      if (adapter.accountCapabilities.updates === "polling") {
+        await this.bootstrapAccount(
+          record,
+          account,
+          generation,
+          accountGeneration,
+        );
+        if (
+          !this.shouldContinueAccountBootstrap(
+            record,
+            generation,
+            accountGeneration,
+          )
+        ) {
+          return;
+        }
         await this.ensureStream(record, account);
+        if (
+          !this.shouldContinueAccountBootstrap(
+            record,
+            generation,
+            accountGeneration,
+          )
+        ) {
+          return;
+        }
+        this.ensurePrivateReconcilePolling(record);
       } else {
         await this.ensureStream(record, account);
-        await this.bootstrapAccount(record, account);
+        if (
+          !this.shouldContinueAccountBootstrap(
+            record,
+            generation,
+            accountGeneration,
+          )
+        ) {
+          return;
+        }
+        await this.bootstrapAccount(
+          record,
+          account,
+          generation,
+          accountGeneration,
+        );
+        if (
+          !this.shouldContinueAccountBootstrap(
+            record,
+            generation,
+            accountGeneration,
+          )
+        ) {
+          return;
+        }
         this.ensureAccountRefreshPolling(record);
+        this.ensurePrivateReconcilePolling(record);
       }
     } catch (error) {
       record.accountSubscribed = false;
@@ -115,7 +209,9 @@ export class PrivateSubscriptionCoordinator {
     }
 
     record.accountSubscribed = false;
+    record.accountSubscriptionGeneration += 1;
     this.stopAccountRefreshPolling(record);
+    this.restartPrivateReconcilePolling(record);
     this.closeIfUnused(record);
   }
 
@@ -124,13 +220,26 @@ export class PrivateSubscriptionCoordinator {
     const record = this.getOrCreateRecord(account);
     const needsPending = !record.stream && !record.startPromise;
     record.ordersSubscribed = true;
+    const generation = record.privateReconcileGeneration;
+    const orderGeneration = record.orderSubscriptionGeneration;
     if (needsPending) {
       this.orderConsumer.onPrivateOrderPending(accountId, record.venue);
     }
 
     try {
       await this.ensureStream(record, account);
-      await this.bootstrapOrders(record, account);
+      if (
+        !this.shouldContinueOrderBootstrap(record, generation, orderGeneration)
+      ) {
+        return;
+      }
+      await this.bootstrapOrders(record, account, generation, orderGeneration);
+      if (
+        !this.shouldContinueOrderBootstrap(record, generation, orderGeneration)
+      ) {
+        return;
+      }
+      this.ensurePrivateReconcilePolling(record);
     } catch (error) {
       record.ordersSubscribed = false;
       this.closeIfUnused(record);
@@ -145,6 +254,8 @@ export class PrivateSubscriptionCoordinator {
     }
 
     record.ordersSubscribed = false;
+    record.orderSubscriptionGeneration += 1;
+    this.restartPrivateReconcilePolling(record);
     this.closeIfUnused(record);
   }
 
@@ -174,6 +285,7 @@ export class PrivateSubscriptionCoordinator {
   onClientStopping(): void {
     for (const record of this.records.values()) {
       this.stopAccountRefreshPolling(record);
+      this.stopPrivateReconcilePolling(record);
       this.closeStream(record);
     }
   }
@@ -186,6 +298,7 @@ export class PrivateSubscriptionCoordinator {
 
     this.closeStream(record);
     this.stopAccountRefreshPolling(record);
+    this.stopPrivateReconcilePolling(record);
     this.records.delete(accountId);
   }
 
@@ -209,20 +322,82 @@ export class PrivateSubscriptionCoordinator {
     const account = this.getAccount(record.accountId);
     this.closeStream(record);
     this.stopAccountRefreshPolling(record);
+    this.stopPrivateReconcilePolling(record);
+    const generation = record.privateReconcileGeneration;
+    const accountGeneration = record.accountSubscriptionGeneration;
+    const orderGeneration = record.orderSubscriptionGeneration;
 
     try {
-      if (record.venue === "juplend" && record.accountSubscribed) {
-        await this.bootstrapAccount(record, account);
-        await this.ensureStream(record, account);
+      const adapter = this.getAdapter(record.venue);
+      if (
+        adapter.accountCapabilities.updates === "polling" &&
+        record.accountSubscribed
+      ) {
+        await this.bootstrapAccount(
+          record,
+          account,
+          generation,
+          accountGeneration,
+        );
+        if (
+          this.shouldContinueAccountBootstrap(
+            record,
+            generation,
+            accountGeneration,
+          )
+        ) {
+          await this.ensureStream(record, account);
+          if (
+            this.shouldContinueAccountBootstrap(
+              record,
+              generation,
+              accountGeneration,
+            )
+          ) {
+            this.ensurePrivateReconcilePolling(record);
+          }
+        }
       } else {
         await this.ensureStream(record, account);
-        if (record.accountSubscribed) {
-          await this.bootstrapAccount(record, account);
+        if (
+          this.shouldContinueAccountBootstrap(
+            record,
+            generation,
+            accountGeneration,
+          )
+        ) {
+          await this.bootstrapAccount(
+            record,
+            account,
+            generation,
+            accountGeneration,
+          );
+        }
+        if (
+          this.shouldContinueAccountBootstrap(
+            record,
+            generation,
+            accountGeneration,
+          )
+        ) {
           this.ensureAccountRefreshPolling(record);
+          this.ensurePrivateReconcilePolling(record);
         }
       }
-      if (record.ordersSubscribed) {
-        await this.bootstrapOrders(record, account);
+      if (
+        this.shouldContinueOrderBootstrap(record, generation, orderGeneration)
+      ) {
+        await this.bootstrapOrders(
+          record,
+          account,
+          generation,
+          orderGeneration,
+        );
+      }
+      if (
+        this.shouldContinueOrderBootstrap(record, generation, orderGeneration)
+      ) {
+        this.ensurePrivateReconcilePolling(record);
       }
     } catch {
       // Errors are already published to the runtime error bus.
@@ -235,6 +410,7 @@ export class PrivateSubscriptionCoordinator {
       throw new AcexError(
         "VENUE_NOT_SUPPORTED",
         `Venue is not supported yet: ${account.venue}`,
+        { details: buildAcexErrorDetails({ venue: account.venue }) },
       );
     }
 
@@ -247,6 +423,7 @@ export class PrivateSubscriptionCoordinator {
       throw new AcexError(
         "VENUE_NOT_SUPPORTED",
         `Venue is not supported yet: ${venue}`,
+        { details: buildAcexErrorDetails({ venue }) },
       );
     }
 
@@ -269,6 +446,9 @@ export class PrivateSubscriptionCoordinator {
       accountReady: false,
       orderReady: false,
       accountRefreshGeneration: 0,
+      accountSubscriptionGeneration: 0,
+      orderSubscriptionGeneration: 0,
+      privateReconcileGeneration: 0,
     };
 
     this.records.set(account.accountId, record);
@@ -279,12 +459,37 @@ export class PrivateSubscriptionCoordinator {
     return record.accountSubscribed || record.ordersSubscribed;
   }
 
+  private shouldContinueAccountBootstrap(
+    record: PrivateSubscriptionRecord,
+    generation: number,
+    accountGeneration: number,
+  ): boolean {
+    return (
+      record.accountSubscribed &&
+      generation === record.privateReconcileGeneration &&
+      accountGeneration === record.accountSubscriptionGeneration
+    );
+  }
+
+  private shouldContinueOrderBootstrap(
+    record: PrivateSubscriptionRecord,
+    generation: number,
+    orderGeneration: number,
+  ): boolean {
+    return (
+      record.ordersSubscribed &&
+      generation === record.privateReconcileGeneration &&
+      orderGeneration === record.orderSubscriptionGeneration
+    );
+  }
+
   private closeIfUnused(record: PrivateSubscriptionRecord): void {
     if (this.isActive(record)) {
       return;
     }
 
     this.stopAccountRefreshPolling(record);
+    this.stopPrivateReconcilePolling(record);
     this.closeStream(record);
     this.records.delete(record.accountId);
   }
@@ -296,7 +501,7 @@ export class PrivateSubscriptionCoordinator {
 
   private ensureAccountRefreshPolling(record: PrivateSubscriptionRecord): void {
     if (
-      record.venue !== "binance" ||
+      typeof this.getAdapter(record.venue).refreshAccount !== "function" ||
       !record.accountSubscribed ||
       record.accountRefreshTimer ||
       record.accountRefreshInFlight
@@ -317,7 +522,10 @@ export class PrivateSubscriptionCoordinator {
   }
 
   private scheduleAccountRefreshPoll(record: PrivateSubscriptionRecord): void {
-    if (record.venue !== "binance" || !record.accountSubscribed) {
+    if (
+      typeof this.getAdapter(record.venue).refreshAccount !== "function" ||
+      !record.accountSubscribed
+    ) {
       return;
     }
 
@@ -326,7 +534,7 @@ export class PrivateSubscriptionCoordinator {
       record.accountRefreshTimer = undefined;
       if (
         generation !== record.accountRefreshGeneration ||
-        record.venue !== "binance" ||
+        typeof this.getAdapter(record.venue).refreshAccount !== "function" ||
         !record.accountSubscribed
       ) {
         return;
@@ -352,13 +560,139 @@ export class PrivateSubscriptionCoordinator {
           }
 
           record.accountRefreshInFlight = undefined;
-          if (record.accountSubscribed && record.venue === "binance") {
+          if (
+            record.accountSubscribed &&
+            typeof this.getAdapter(record.venue).refreshAccount === "function"
+          ) {
             this.scheduleAccountRefreshPoll(record);
           }
         });
     }, this.binanceRiskPollIntervalMs);
   }
 
+  private hasPrivateReconcileCapability(
+    record: PrivateSubscriptionRecord,
+  ): boolean {
+    const adapter = this.getAdapter(record.venue);
+    return (
+      (record.accountSubscribed &&
+        (typeof adapter.reconcileAccount === "function" ||
+          typeof adapter.bootstrapAccount === "function")) ||
+      (record.ordersSubscribed && typeof adapter.fetchOpenOrders === "function")
+    );
+  }
+
+  private ensurePrivateReconcilePolling(
+    record: PrivateSubscriptionRecord,
+  ): void {
+    if (
+      this.binancePrivateReconcileIntervalMs === undefined ||
+      !this.isActive(record) ||
+      !this.hasPrivateReconcileCapability(record) ||
+      record.privateReconcileTimer ||
+      record.privateReconcileInFlight
+    ) {
+      return;
+    }
+
+    this.schedulePrivateReconcilePoll(record);
+  }
+
+  private restartPrivateReconcilePolling(
+    record: PrivateSubscriptionRecord,
+  ): void {
+    if (record.privateReconcileTimer) {
+      clearTimeout(record.privateReconcileTimer);
+      record.privateReconcileTimer = undefined;
+    }
+    this.ensurePrivateReconcilePolling(record);
+  }
+
+  private stopPrivateReconcilePolling(record: PrivateSubscriptionRecord): void {
+    record.privateReconcileGeneration += 1;
+    if (record.privateReconcileTimer) {
+      clearTimeout(record.privateReconcileTimer);
+      record.privateReconcileTimer = undefined;
+    }
+    record.privateReconcileInFlight = undefined;
+  }
+
+  private schedulePrivateReconcilePoll(
+    record: PrivateSubscriptionRecord,
+  ): void {
+    if (
+      this.binancePrivateReconcileIntervalMs === undefined ||
+      !this.isActive(record) ||
+      !this.hasPrivateReconcileCapability(record)
+    ) {
+      return;
+    }
+
+    const generation = record.privateReconcileGeneration;
+    record.privateReconcileTimer = setTimeout(() => {
+      record.privateReconcileTimer = undefined;
+      if (
+        generation !== record.privateReconcileGeneration ||
+        this.binancePrivateReconcileIntervalMs === undefined ||
+        !this.isActive(record) ||
+        !this.hasPrivateReconcileCapability(record)
+      ) {
+        return;
+      }
+
+      let latestAccount: RegisteredAccountRecord;
+      try {
+        latestAccount = this.getAccount(record.accountId);
+      } catch (error) {
+        this.handlePrivateReconcileLookupError(record, error);
+        return;
+      }
+
+      record.privateReconcileInFlight = this.reconcilePrivateData(
+        record,
+        latestAccount,
+        generation,
+        true,
+      )
+        .catch(() => {})
+        .finally(() => {
+          if (generation !== record.privateReconcileGeneration) {
+            return;
+          }
+
+          record.privateReconcileInFlight = undefined;
+          if (
+            this.binancePrivateReconcileIntervalMs !== undefined &&
+            this.isActive(record) &&
+            this.hasPrivateReconcileCapability(record)
+          ) {
+            this.schedulePrivateReconcilePoll(record);
+          }
+        });
+    }, this.binancePrivateReconcileIntervalMs);
+  }
+
+  private handlePrivateReconcileLookupError(
+    record: PrivateSubscriptionRecord,
+    error: unknown,
+  ): void {
+    this.stopPrivateReconcilePolling(record);
+    if (error instanceof AcexError && error.code === "ACCOUNT_NOT_FOUND") {
+      return;
+    }
+
+    this.context.publishRuntimeError(
+      "adapter",
+      error instanceof Error
+        ? error
+        : new Error(`Failed to load ${record.venue} account for reconcile`),
+      {
+        accountId: record.accountId,
+        venue: record.venue,
+      },
+    );
+  }
+
   private handleAccountRefreshLookupError(
     record: PrivateSubscriptionRecord,
     error: unknown,
@@ -390,6 +724,7 @@ export class PrivateSubscriptionCoordinator {
       return;
     }
 
+    const requestStartedAt = this.context.now();
     try {
       const update = await adapter.refreshAccount(account.credentials ?? {}, {
         ...account.options,
@@ -407,7 +742,7 @@ export class PrivateSubscriptionCoordinator {
         record.accountId,
         record.venue,
         update,
-        { preserveStatus: true },
+        { preserveStatus: true, requestStartedAt },
       );
     } catch (error) {
       if (
@@ -435,12 +770,337 @@ export class PrivateSubscriptionCoordinator {
         {
           runtimeStatus: "degraded",
           ready: record.accountReady,
-          reason: "http_failed",
+          reason: transportReason(error, "http_failed"),
+        },
+      );
+    }
+  }
+
+  private async reconcilePrivateData(
+    record: PrivateSubscriptionRecord,
+    account: RegisteredAccountRecord,
+    generation: number,
+    preserveStatus: boolean,
+  ): Promise<void> {
+    const accountGeneration = record.accountSubscriptionGeneration;
+    const orderGeneration = record.orderSubscriptionGeneration;
+
+    await Promise.all([
+      this.reconcileAccount(
+        record,
+        account,
+        generation,
+        accountGeneration,
+        preserveStatus,
+      ),
+      this.reconcileOrders(
+        record,
+        account,
+        generation,
+        orderGeneration,
+        preserveStatus,
+      ),
+    ]);
+  }
+
+  private async reconcileAccount(
+    record: PrivateSubscriptionRecord,
+    account: RegisteredAccountRecord,
+    generation: number,
+    accountGeneration: number,
+    preserveStatus: boolean,
+  ): Promise<void> {
+    const adapter = this.getAdapter(record.venue);
+    if (
+      !this.shouldContinueAccountBootstrap(
+        record,
+        generation,
+        accountGeneration,
+      )
+    ) {
+      return;
+    }
+
+    const requestStartedAt = this.context.now();
+    try {
+      const snapshot = adapter.reconcileAccount
+        ? await adapter.reconcileAccount(account.credentials ?? {}, {
+            ...account.options,
+            accountId: account.accountId,
+          })
+        : await adapter.bootstrapAccount(account.credentials ?? {}, {
+            ...account.options,
+            accountId: account.accountId,
+          });
+      if (
+        !this.shouldContinueAccountBootstrap(
+          record,
+          generation,
+          accountGeneration,
+        )
+      ) {
+        return;
+      }
+
+      record.accountReady = true;
+      this.accountConsumer.onPrivateAccountReconcile(
+        record.accountId,
+        record.venue,
+        snapshot,
+        {
+          requestStartedAt,
+          preserveStatus,
+        },
+      );
+    } catch (error) {
+      if (
+        !this.shouldContinueAccountBootstrap(
+          record,
+          generation,
+          accountGeneration,
+        )
+      ) {
+        return;
+      }
+
+      this.context.publishRuntimeError(
+        "adapter",
+        error instanceof Error
+          ? error
+          : new Error(
+              `Failed to reconcile ${record.venue} private account state`,
+            ),
+        {
+          accountId: record.accountId,
+          venue: record.venue,
+        },
+      );
+      this.accountConsumer.onPrivateAccountStreamState(
+        record.accountId,
+        record.venue,
+        {
+          runtimeStatus: "degraded",
+          ready: record.accountReady,
+          reason: transportReason(error, "http_failed"),
+        },
+      );
+    }
+  }
+
+  private async reconcileOrders(
+    record: PrivateSubscriptionRecord,
+    account: RegisteredAccountRecord,
+    generation: number,
+    orderGeneration: number,
+    preserveStatus: boolean,
+  ): Promise<void> {
+    const adapter = this.getAdapter(record.venue);
+    if (
+      !adapter.fetchOpenOrders ||
+      !this.shouldContinueOrderBootstrap(record, generation, orderGeneration)
+    ) {
+      return;
+    }
+
+    const requestStartedAt = this.context.now();
+    try {
+      const snapshot = await adapter.fetchOpenOrders(
+        account.credentials ?? {},
+        {
+          ...account.options,
+          accountId: account.accountId,
+        },
+      );
+      if (
+        !this.shouldContinueOrderBootstrap(record, generation, orderGeneration)
+      ) {
+        return;
+      }
+
+      record.orderReady = true;
+      const disappeared = this.orderConsumer.onPrivateOrderReconcile(
+        record.accountId,
+        record.venue,
+        snapshot,
+        {
+          requestStartedAt,
+          preserveStatus,
+        },
+      );
+      await this.backfillDisappearedOrders(
+        record,
+        account,
+        generation,
+        orderGeneration,
+        disappeared,
+      );
+    } catch (error) {
+      if (
+        !this.shouldContinueOrderBootstrap(record, generation, orderGeneration)
+      ) {
+        return;
+      }
+
+      this.handleOrderReconcileError(record, error);
+    }
+  }
+
+  private async backfillDisappearedOrders(
+    record: PrivateSubscriptionRecord,
+    account: RegisteredAccountRecord,
+    generation: number,
+    orderGeneration: number,
+    disappeared: OrderSnapshot[],
+  ): Promise<void> {
+    const adapter = this.getAdapter(record.venue);
+    if (!adapter.fetchOrder || disappeared.length === 0) {
+      return;
+    }
+
+    const pending = disappeared
+      .filter((order) => order.orderId || order.clientOrderId)
+      .slice(0, MAX_ORDER_TERMINAL_BACKFILLS_PER_RECONCILE);
+    if (pending.length === 0) {
+      return;
+    }
+
+    let cursor = 0;
+    const workers = Array.from(
+      {
+        length: Math.min(
+          MAX_ORDER_TERMINAL_BACKFILL_CONCURRENCY,
+          pending.length,
+        ),
+      },
+      async () => {
+        while (cursor < pending.length) {
+          if (
+            !this.shouldContinueOrderBootstrap(
+              record,
+              generation,
+              orderGeneration,
+            )
+          ) {
+            return;
+          }
+
+          const order = pending[cursor];
+          cursor += 1;
+          if (!order) {
+            continue;
+          }
+          await this.backfillDisappearedOrder(
+            record,
+            account,
+            generation,
+            orderGeneration,
+            order,
+          );
+        }
+      },
+    );
+
+    await Promise.all(workers);
+  }
+
+  private async backfillDisappearedOrder(
+    record: PrivateSubscriptionRecord,
+    account: RegisteredAccountRecord,
+    generation: number,
+    orderGeneration: number,
+    order: OrderSnapshot,
+  ): Promise<void> {
+    const adapter = this.getAdapter(record.venue);
+    if (
+      !adapter.fetchOrder ||
+      !this.shouldContinueOrderBootstrap(record, generation, orderGeneration)
+    ) {
+      return;
+    }
+
+    const requestStartedAt = this.context.now();
+    try {
+      let request: FetchOrderRequest;
+      if (order.orderId) {
+        request = {
+          symbol: order.symbol,
+          orderId: order.orderId,
+          clientOrderId: order.clientOrderId,
+        };
+      } else if (order.clientOrderId) {
+        request = {
+          symbol: order.symbol,
+          clientOrderId: order.clientOrderId,
+        };
+      } else {
+        return;
+      }
+
+      const update = await adapter.fetchOrder(
+        account.credentials ?? {},
+        request,
+        { ...account.options, accountId: account.accountId },
+      );
+      if (
+        !this.shouldContinueOrderBootstrap(record, generation, orderGeneration)
+      ) {
+        return;
+      }
+
+      if (!update) {
+        this.handleOrderReconcileError(
+          record,
+          new Error(
+            `Failed to backfill disappeared ${record.venue} order terminal state`,
+          ),
+        );
+        return;
+      }
+
+      this.orderConsumer.onPrivateOrderUpdate(
+        record.accountId,
+        record.venue,
+        update,
+        {
+          requestStartedAt,
         },
       );
+    } catch (error) {
+      if (
+        !this.shouldContinueOrderBootstrap(record, generation, orderGeneration)
+      ) {
+        return;
+      }
+
+      this.handleOrderReconcileError(record, error);
     }
   }
 
+  private handleOrderReconcileError(
+    record: PrivateSubscriptionRecord,
+    error: unknown,
+  ): void {
+    this.context.publishRuntimeError(
+      "adapter",
+      error instanceof Error
+        ? error
+        : new Error(`Failed to reconcile ${record.venue} private order state`),
+      {
+        accountId: record.accountId,
+        venue: record.venue,
+      },
+    );
+    this.orderConsumer.onPrivateOrderStreamState(
+      record.accountId,
+      record.venue,
+      {
+        runtimeStatus: "degraded",
+        ready: record.orderReady,
+        reason: transportReason(error, "http_failed"),
+      },
+    );
+  }
+
   private async ensureStream(
     record: PrivateSubscriptionRecord,
     account: RegisteredAccountRecord,
@@ -466,15 +1126,21 @@ export class PrivateSubscriptionCoordinator {
     record: PrivateSubscriptionRecord,
     account: RegisteredAccountRecord,
   ): Promise<void> {
+    const adapter = this.getAdapter(record.venue);
     const credentials = account.credentials;
-    if (!credentials && record.venue !== "juplend") {
+    if (adapter.accountCapabilities.credentialsRequired && !credentials) {
       throw new AcexError(
         "CREDENTIALS_MISSING",
         `Account credentials are required for private subscriptions: ${account.accountId}`,
+        {
+          details: buildAcexErrorDetails({
+            accountId: account.accountId,
+            venue: account.venue,
+          }),
+        },
       );
     }
 
-    const adapter = this.getAdapter(record.venue);
     const stream = adapter.createPrivateStream(
       credentials ?? {},
       {
@@ -559,7 +1225,7 @@ export class PrivateSubscriptionCoordinator {
               {
                 runtimeStatus: "degraded",
                 ready: record.accountReady,
-                reason: "http_failed",
+                reason: transportReason(error, "http_failed"),
               },
             );
           }
@@ -570,10 +1236,9 @@ export class PrivateSubscriptionCoordinator {
         reconnectDelayMs: this.streamReconnectDelayMs,
         reconnectMaxDelayMs: this.streamReconnectMaxDelayMs,
         listenKeyKeepAliveMs: this.listenKeyKeepAliveMs,
-        juplendPollIntervalMs: this.juplendPollIntervalMs,
         now: () => this.context.now(),
       },
-      account.options,
+      { ...account.options, accountId: account.accountId },
     );
 
     record.stream = stream;
@@ -622,31 +1287,65 @@ export class PrivateSubscriptionCoordinator {
     record: PrivateSubscriptionRecord,
   ): Promise<void> {
     const account = this.getAccount(record.accountId);
+    const generation = record.privateReconcileGeneration;
+    const accountGeneration = record.accountSubscriptionGeneration;
+    const orderGeneration = record.orderSubscriptionGeneration;
 
     if (record.accountSubscribed) {
       this.accountConsumer.onPrivateAccountPending(
         record.accountId,
         record.venue,
       );
-      await this.bootstrapAccount(record, account);
+      await this.reconcileAccount(
+        record,
+        account,
+        generation,
+        accountGeneration,
+        false,
+      );
     }
 
     if (record.ordersSubscribed) {
       this.orderConsumer.onPrivateOrderPending(record.accountId, record.venue);
-      await this.bootstrapOrders(record, account);
+      await this.reconcileOrders(
+        record,
+        account,
+        generation,
+        orderGeneration,
+        false,
+      );
     }
   }
 
   private async bootstrapAccount(
     record: PrivateSubscriptionRecord,
     account: RegisteredAccountRecord,
+    generation: number,
+    accountGeneration: number,
   ): Promise<void> {
+    if (
+      !this.shouldContinueAccountBootstrap(
+        record,
+        generation,
+        accountGeneration,
+      )
+    ) {
+      return;
+    }
+
     try {
-      const bootstrap = await this.getAdapter(record.venue).bootstrapAccount(
+      const adapter = this.getAdapter(record.venue);
+      const bootstrap = await adapter.bootstrapAccount(
         account.credentials ?? {},
         { ...account.options, accountId: account.accountId },
       );
-      if (!record.accountSubscribed) {
+      if (
+        !this.shouldContinueAccountBootstrap(
+          record,
+          generation,
+          accountGeneration,
+        )
+      ) {
         return;
       }
 
@@ -657,6 +1356,16 @@ export class PrivateSubscriptionCoordinator {
         bootstrap,
       );
     } catch (error) {
+      if (
+        !this.shouldContinueAccountBootstrap(
+          record,
+          generation,
+          accountGeneration,
+        )
+      ) {
+        return;
+      }
+
       record.accountReady = false;
       this.context.publishRuntimeError(
         "adapter",
@@ -676,14 +1385,32 @@ export class PrivateSubscriptionCoordinator {
         {
           runtimeStatus: "degraded",
           ready: false,
-          reason: record.venue === "juplend" ? "http_failed" : "auth_failed",
+          reason: transportReason(
+            error,
+            this.getAdapter(record.venue).accountCapabilities
+              .credentialsRequired
+              ? "auth_failed"
+              : "http_failed",
+          ),
         },
       );
-      const reason =
-        error instanceof Error && error.message ? ` (${error.message})` : "";
+      const details = buildAcexErrorDetails(
+        {
+          accountId: record.accountId,
+          venue: record.venue,
+        },
+        error,
+      );
       throw new AcexError(
         "ACCOUNT_BOOTSTRAP_FAILED",
-        `Failed to bootstrap account data: ${record.accountId}${reason}`,
+        formatAcexErrorMessage(
+          `Failed to bootstrap account data: ${record.accountId}`,
+          details,
+        ),
+        {
+          cause: error,
+          details,
+        },
       );
     }
   }
@@ -691,49 +1418,157 @@ export class PrivateSubscriptionCoordinator {
   private async bootstrapOrders(
     record: PrivateSubscriptionRecord,
     account: RegisteredAccountRecord,
+    generation: number,
+    orderGeneration: number,
   ): Promise<void> {
+    if (
+      !this.shouldContinueOrderBootstrap(record, generation, orderGeneration)
+    ) {
+      return;
+    }
+
+    const adapter = this.getAdapter(record.venue);
+    if (!adapter.fetchOpenOrders) {
+      try {
+        const requestStartedAt = this.context.now();
+        const orders = await adapter.bootstrapOpenOrders(
+          account.credentials ?? {},
+          { ...account.options, accountId: account.accountId },
+        );
+        const snapshot: RawOpenOrdersSnapshot = {
+          orders,
+          snapshotReceivedAt:
+            orders.reduce(
+              (latest, order) => Math.max(latest, order.receivedAt),
+              0,
+            ) || this.context.now(),
+        };
+        if (
+          !this.shouldContinueOrderBootstrap(
+            record,
+            generation,
+            orderGeneration,
+          )
+        ) {
+          return;
+        }
+
+        record.orderReady = true;
+        const disappeared = this.orderConsumer.onPrivateOrderBootstrap(
+          record.accountId,
+          record.venue,
+          snapshot,
+          {
+            requestStartedAt,
+          },
+        );
+        await this.backfillDisappearedOrders(
+          record,
+          account,
+          generation,
+          orderGeneration,
+          disappeared,
+        );
+        return;
+      } catch (error) {
+        if (
+          !this.shouldContinueOrderBootstrap(
+            record,
+            generation,
+            orderGeneration,
+          )
+        ) {
+          return;
+        }
+
+        this.handleBootstrapOrdersError(record, error);
+        return;
+      }
+    }
+
+    const requestStartedAt = this.context.now();
     try {
-      const snapshots = await this.getAdapter(record.venue).bootstrapOpenOrders(
+      const snapshot = await adapter.fetchOpenOrders(
         account.credentials ?? {},
-        account.options,
+        {
+          ...account.options,
+          accountId: account.accountId,
+        },
       );
-      if (!record.ordersSubscribed) {
+      if (
+        !this.shouldContinueOrderBootstrap(record, generation, orderGeneration)
+      ) {
         return;
       }
 
       record.orderReady = true;
-      this.orderConsumer.onPrivateOrderBootstrap(
+      const disappeared = this.orderConsumer.onPrivateOrderBootstrap(
         record.accountId,
         record.venue,
-        snapshots,
-      );
-    } catch (error) {
-      record.orderReady = false;
-      this.context.publishRuntimeError(
-        "adapter",
-        error instanceof Error
-          ? error
-          : new Error(
-              `Failed to bootstrap ${record.venue} private order state`,
-            ),
+        snapshot,
         {
-          accountId: record.accountId,
-          venue: record.venue,
+          requestStartedAt,
         },
       );
-      this.orderConsumer.onPrivateOrderStreamState(
-        record.accountId,
-        record.venue,
-        {
-          runtimeStatus: "degraded",
-          ready: false,
-          reason: "auth_failed",
-        },
-      );
-      throw new AcexError(
-        "ORDER_BOOTSTRAP_FAILED",
-        `Failed to bootstrap order data: ${record.accountId}`,
+      await this.backfillDisappearedOrders(
+        record,
+        account,
+        generation,
+        orderGeneration,
+        disappeared,
       );
+    } catch (error) {
+      if (
+        !this.shouldContinueOrderBootstrap(record, generation, orderGeneration)
+      ) {
+        return;
+      }
+
+      this.handleBootstrapOrdersError(record, error);
     }
   }
+
+  private handleBootstrapOrdersError(
+    record: PrivateSubscriptionRecord,
+    error: unknown,
+  ): never {
+    record.orderReady = false;
+    this.context.publishRuntimeError(
+      "adapter",
+      error instanceof Error
+        ? error
+        : new Error(`Failed to bootstrap ${record.venue} private order state`),
+      {
+        accountId: record.accountId,
+        venue: record.venue,
+      },
+    );
+    this.orderConsumer.onPrivateOrderStreamState(
+      record.accountId,
+      record.venue,
+      {
+        runtimeStatus: "degraded",
+        ready: false,
+        reason: transportReason(error, "auth_failed"),
+      },
+    );
+    const details = buildAcexErrorDetails(
+      {
+        accountId: record.accountId,
+        venue: record.venue,
+      },
+      error,
+    );
+    throw new AcexError(
+      "ORDER_BOOTSTRAP_FAILED",
+      formatAcexErrorMessage(
+        `Failed to bootstrap order data: ${record.accountId}`,
+        details,
+      ),
+      {
+        cause: error,
+        details,
+      },
+    );
+  }
 }