gloomberb 0.7.4 → 0.8.0

package/src/plugins/ibkr/flex/statement-client.ts

@@ -0,0 +1,181 @@
+import { httpFetch } from "../../../utils/http-transport";
+import type { FlexQueryConfig } from "../config";
+import { IBKR_STATEMENT_URL } from "../config";
+
+const IBKR_STATEMENT_GET_URL = "https://gdcdyn.interactivebrokers.com/Universal/servlet/FlexStatementService.GetStatement";
+const FLEX_STATEMENT_CACHE_MS = 15 * 60_000;
+const FLEX_STATEMENT_INITIAL_WAIT_MS = 3_000;
+const FLEX_STATEMENT_POLL_DELAY_MS = 5_000;
+const FLEX_STATEMENT_MAX_DOWNLOAD_ATTEMPTS = 60;
+const flexStatementCache = new Map<string, { createdAt: number; promise: Promise<string> }>();
+
+type FlexRequestPhase = "request" | "download";
+
+interface FlexErrorContext {
+  phase: FlexRequestPhase;
+  endpoint?: string;
+  queryId?: string;
+  token?: string;
+  referenceCode?: string;
+  httpStatus?: number;
+}
+
+function decodeXmlText(value: string): string {
+  return value
+    .replace(/&amp;/g, "&")
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, "\"")
+    .replace(/&apos;/g, "'");
+}
+
+function extractFlexErrorMessage(text: string): string | null {
+  const errorMatch = text.match(/<ErrorMessage>([^<]+)<\/ErrorMessage>/);
+  if (errorMatch?.[1]) return decodeXmlText(errorMatch[1].trim());
+  return null;
+}
+
+function endpointName(endpoint: string | undefined): string {
+  if (!endpoint) return "default";
+  try {
+    const url = new URL(endpoint);
+    return url.pathname.split("/").filter(Boolean).at(-1) || url.hostname;
+  } catch {
+    return endpoint.split("?")[0]?.split("/").filter(Boolean).at(-1) || endpoint;
+  }
+}
+
+function buildFlexErrorMessage(providerMessage: string, context: FlexErrorContext): string {
+  const phase = context.phase === "request"
+    ? "requesting the statement"
+    : "downloading the generated statement";
+  const details = [
+    `Endpoint ${endpointName(context.endpoint)}`,
+    context.queryId ? `query ID ${context.queryId}` : null,
+    context.referenceCode ? `reference code ${context.referenceCode}` : null,
+    context.token ? "token configured" : "token missing",
+    context.httpStatus && context.httpStatus >= 400 ? `HTTP ${context.httpStatus}` : null,
+  ].filter(Boolean).join(", ");
+  const advice = context.phase === "request"
+    ? "Check that Flex Web Service is enabled, the token is active, and the query ID belongs to that token."
+    : "IBKR accepted the request but did not return the statement; retry shortly or check that the Flex query can generate a statement.";
+
+  return `IBKR Flex request failed while ${phase}: ${providerMessage}. ${details}. ${advice}`;
+}
+
+function flexError(providerMessage: string, context: FlexErrorContext): Error {
+  return new Error(buildFlexErrorMessage(providerMessage, context));
+}
+
+export async function requestFlexStatement(config: FlexQueryConfig): Promise<string> {
+  const endpoint = config.endpoint || IBKR_STATEMENT_URL;
+  const url = `${endpoint}?t=${config.token}&q=${config.queryId}&v=3`;
+  let resp: Response;
+  let text: string;
+  try {
+    resp = await httpFetch(url, { signal: AbortSignal.timeout(30_000) });
+    text = await resp.text();
+  } catch (error) {
+    throw flexError(error instanceof Error ? error.message : "Network request failed", {
+      phase: "request",
+      endpoint,
+      queryId: config.queryId,
+      token: config.token,
+    });
+  }
+
+  const codeMatch = text.match(/<ReferenceCode>(\d+)<\/ReferenceCode>/);
+  if (!codeMatch) {
+    throw flexError(extractFlexErrorMessage(text) || "No reference code returned", {
+      phase: "request",
+      endpoint,
+      queryId: config.queryId,
+      token: config.token,
+      httpStatus: resp.status,
+    });
+  }
+
+  return codeMatch[1]!;
+}
+
+async function getFlexStatement(
+  token: string,
+  referenceCode: string,
+  context: Partial<Pick<FlexQueryConfig, "endpoint" | "queryId">> = {},
+): Promise<string> {
+  await new Promise((resolve) => setTimeout(resolve, FLEX_STATEMENT_INITIAL_WAIT_MS));
+
+  const url = `${IBKR_STATEMENT_GET_URL}?t=${token}&q=${referenceCode}&v=3`;
+  for (let attempt = 0; attempt < FLEX_STATEMENT_MAX_DOWNLOAD_ATTEMPTS; attempt += 1) {
+    let resp: Response;
+    let text: string;
+    try {
+      resp = await httpFetch(url, { signal: AbortSignal.timeout(30_000) });
+      text = await resp.text();
+    } catch (error) {
+      throw flexError(error instanceof Error ? error.message : "Network request failed", {
+        phase: "download",
+        endpoint: context.endpoint,
+        queryId: context.queryId,
+        token,
+        referenceCode,
+      });
+    }
+
+    if (text.includes("<FlexQueryResponse") || text.includes("<FlexStatements")) {
+      return text;
+    }
+
+    if (text.includes("Statement generation in progress")) {
+      await new Promise((resolve) => setTimeout(resolve, FLEX_STATEMENT_POLL_DELAY_MS));
+      continue;
+    }
+
+    const providerMessage = extractFlexErrorMessage(text);
+    if (providerMessage) {
+      throw flexError(providerMessage, {
+        phase: "download",
+        endpoint: context.endpoint,
+        queryId: context.queryId,
+        token,
+        referenceCode,
+        httpStatus: resp.status,
+      });
+    }
+  }
+
+  throw flexError("statement generation timed out", {
+    phase: "download",
+    endpoint: context.endpoint,
+    queryId: context.queryId,
+    token,
+    referenceCode,
+  });
+}
+
+export async function loadFlexStatement(config: FlexQueryConfig): Promise<string> {
+  const cacheKey = `${config.endpoint || IBKR_STATEMENT_URL}|${config.token}|${config.queryId}`;
+  const existing = flexStatementCache.get(cacheKey);
+  if (existing && Date.now() - existing.createdAt < FLEX_STATEMENT_CACHE_MS) {
+    return existing.promise;
+  }
+
+  const promise = (async () => {
+    const referenceCode = await requestFlexStatement(config);
+    return getFlexStatement(config.token, referenceCode, config);
+  })();
+  flexStatementCache.set(cacheKey, { createdAt: Date.now(), promise });
+  promise.catch(() => {
+    const cached = flexStatementCache.get(cacheKey);
+    if (cached?.promise === promise) {
+      flexStatementCache.delete(cacheKey);
+    }
+  });
+  setTimeout(() => {
+    const cached = flexStatementCache.get(cacheKey);
+    if (cached?.promise === promise) {
+      flexStatementCache.delete(cacheKey);
+    }
+  }, FLEX_STATEMENT_CACHE_MS);
+  return promise;
+}

package/src/plugins/ibkr/gateway/account-loaders.ts

@@ -0,0 +1,266 @@
+import {
+  EventName,
+  type Contract,
+  type IBApiNext,
+} from "@stoqey/ib";
+import { firstValueFrom, filter, take, timeout } from "rxjs";
+import type { BrokerPosition } from "../../../types/broker";
+import type {
+  BrokerAccount,
+  BrokerExecution,
+  BrokerOrder,
+} from "../../../types/trading";
+import { debugLog } from "../../../utils/debug-log";
+import {
+  finiteNumber,
+  summarizeBrokerAccount,
+  type AccountPnlSnapshot,
+  type AccountSummaryTags,
+} from "./account-summary";
+import { contractToBrokerRef } from "./contracts";
+import { openIbkrOrderToBrokerOrder, type OpenIbkrOrder } from "./orders";
+import { getIbkrPriceDivisor, normalizeIbkrPriceValue } from "./price-normalization";
+import {
+  IBKR_DATA_TIMEOUT,
+  IBKR_PNL_TIMEOUT,
+  withTimeout,
+} from "./timeouts";
+
+interface AccountPortfolioSnapshotPosition {
+  contract: Contract;
+  avgCost?: number;
+  marketPrice?: number;
+  marketValue?: number;
+  unrealizedPNL?: number;
+}
+
+interface GatewayAccountLoadOptions {
+  api: IBApiNext;
+  rawApi: any;
+  instanceId?: string;
+  cachedAccountIds: string[];
+  setCachedAccountIds(accountIds: string[]): void;
+}
+
+interface GatewayApiOptions {
+  api: IBApiNext;
+  instanceId?: string;
+}
+
+const gatewayLog = debugLog.createLogger("ibkr-gateway");
+
+function buildPortfolioPositionKey(accountId: string, contract: Contract): string {
+  return [
+    accountId.trim(),
+    contract.conId ?? "",
+    contract.localSymbol ?? "",
+    contract.symbol ?? "",
+    contract.secType ?? "",
+  ].join("|");
+}
+
+export async function loadIbkrAccounts({
+  api,
+  rawApi,
+  instanceId,
+  cachedAccountIds,
+  setCachedAccountIds,
+}: GatewayAccountLoadOptions): Promise<BrokerAccount[]> {
+  let managedAccounts: string[];
+  try {
+    managedAccounts = await withTimeout(api.getManagedAccounts(), IBKR_DATA_TIMEOUT, "getManagedAccounts");
+  } catch {
+    // getManagedAccounts() can hang after reconnects or when another client with the same
+    // clientId is connected. Fall back to requesting via the raw event API, or use cached IDs.
+    if (cachedAccountIds.length > 0) {
+      managedAccounts = cachedAccountIds;
+    } else {
+      managedAccounts = await withTimeout(
+        new Promise<string[]>((resolve) => {
+          const handler = (accountsList: string) => {
+            rawApi.off(EventName.managedAccounts, handler);
+            resolve(accountsList.split(",").map((s: string) => s.trim()).filter(Boolean));
+          };
+          rawApi.on(EventName.managedAccounts, handler);
+          rawApi.reqManagedAccts();
+        }),
+        IBKR_DATA_TIMEOUT,
+        "getManagedAccounts-fallback",
+      );
+      setCachedAccountIds(managedAccounts);
+    }
+  }
+  if (!managedAccounts.length) return [];
+  let summary: ReadonlyMap<string, AccountSummaryTags> | undefined;
+  try {
+    const result = await firstValueFrom(
+      api.getAccountSummary(
+        "All",
+        "NetLiquidation,TotalCashValue,SettledCash,AvailableFunds,BuyingPower,ExcessLiquidity,InitMarginReq,MaintMarginReq,$LEDGER:ALL",
+      )
+        .pipe(take(1), timeout(10_000)),
+    );
+    summary = result.all;
+  } catch {
+    // Account summary may fail; return accounts with basic info only.
+  }
+
+  const updatedAt = Date.now();
+  const aggregateTags = summary?.get("All");
+  const allowAggregateCashBalances = managedAccounts.length === 1;
+  const pnlByAccount = await loadAccountPnlForAccounts(api, instanceId, managedAccounts);
+  return managedAccounts.map((accountId) => summarizeBrokerAccount(
+    accountId,
+    summary?.get(accountId),
+    updatedAt,
+    aggregateTags,
+    allowAggregateCashBalances,
+    pnlByAccount.get(accountId),
+  ));
+}
+
+export async function loadIbkrPositions({
+  api,
+  instanceId,
+}: GatewayApiOptions): Promise<BrokerPosition[]> {
+  const update = await firstValueFrom(api.getPositions().pipe(take(1), timeout(10_000)));
+  const portfolioSnapshots = await loadPortfolioSnapshotsForAccounts(api, instanceId, [...update.all.keys()]);
+  const positions: BrokerPosition[] = [];
+  for (const [accountId, accountPositions] of update.all) {
+    for (const position of accountPositions) {
+      if (!position.contract.symbol) continue;
+      const portfolioSnapshot = portfolioSnapshots.get(buildPortfolioPositionKey(accountId, position.contract));
+      const priceDivisor = getIbkrPriceDivisor(position.contract);
+      positions.push({
+        ticker: position.contract.localSymbol || position.contract.symbol,
+        exchange: position.contract.primaryExch || position.contract.exchange || "",
+        shares: Math.abs(position.pos),
+        avgCost: normalizeIbkrPriceValue(portfolioSnapshot?.avgCost ?? position.avgCost, priceDivisor),
+        currency: position.contract.currency || "USD",
+        accountId,
+        name: position.contract.description || position.contract.localSymbol || position.contract.symbol,
+        assetCategory: position.contract.secType,
+        markPrice: normalizeIbkrPriceValue(portfolioSnapshot?.marketPrice ?? position.marketPrice, priceDivisor),
+        marketValue: portfolioSnapshot?.marketValue ?? position.marketValue,
+        unrealizedPnl: portfolioSnapshot?.unrealizedPNL ?? position.unrealizedPNL,
+        side: position.pos < 0 ? "short" : "long",
+        multiplier: position.contract.multiplier,
+        brokerContract: contractToBrokerRef(position.contract, instanceId),
+      });
+    }
+  }
+  return positions;
+}
+
+export async function loadIbkrOpenOrders({
+  api,
+  instanceId,
+}: GatewayApiOptions): Promise<BrokerOrder[]> {
+  const orders = await withTimeout(api.getAllOpenOrders(), IBKR_DATA_TIMEOUT, "getAllOpenOrders");
+  return orders.map((order) => openIbkrOrderToBrokerOrder(
+    order as OpenIbkrOrder,
+    instanceId,
+    (contract) => contractToBrokerRef(contract, instanceId),
+  ));
+}
+
+export async function loadIbkrExecutions({
+  api,
+  instanceId,
+}: GatewayApiOptions): Promise<BrokerExecution[]> {
+  const executions = await withTimeout(api.getExecutionDetails({}), IBKR_DATA_TIMEOUT, "getExecutionDetails");
+  return executions.map((detail) => ({
+    execId: detail.execution.execId || `${detail.execution.orderId ?? "exec"}-${detail.execution.time ?? Date.now()}`,
+    brokerInstanceId: instanceId,
+    orderId: detail.execution.orderId,
+    accountId: detail.execution.acctNumber,
+    side: detail.execution.side || "",
+    shares: detail.execution.shares ?? 0,
+    price: detail.execution.price ?? 0,
+    time: detail.execution.time ? Date.parse(detail.execution.time) : Date.now(),
+    exchange: detail.execution.exchange,
+    contract: contractToBrokerRef(detail.contract, instanceId),
+  }));
+}
+
+async function loadAccountPnlForAccounts(
+  api: IBApiNext,
+  instanceId: string | undefined,
+  accountIds: string[],
+): Promise<Map<string, AccountPnlSnapshot>> {
+  const entries = await Promise.all(accountIds.map(async (accountId) => {
+    const pnl = await loadAccountPnl(api, instanceId, accountId);
+    return pnl ? ([accountId, pnl] as const) : null;
+  }));
+  return new Map(entries.filter((entry): entry is readonly [string, AccountPnlSnapshot] => entry != null));
+}
+
+async function loadAccountPnl(
+  api: IBApiNext,
+  instanceId: string | undefined,
+  accountId: string,
+): Promise<AccountPnlSnapshot | null> {
+  if (typeof api.getPnL !== "function") return null;
+
+  try {
+    const pnl = await firstValueFrom(
+      api.getPnL(accountId).pipe(
+        take(1),
+        timeout(IBKR_PNL_TIMEOUT),
+      ),
+    );
+    return {
+      dailyPnl: finiteNumber(pnl.dailyPnL),
+      unrealizedPnl: finiteNumber(pnl.unrealizedPnL),
+      realizedPnl: finiteNumber(pnl.realizedPnL),
+    };
+  } catch (error: any) {
+    gatewayLog.warn("Failed to load IBKR account P&L", {
+      instanceId,
+      accountId,
+      error: error?.message || String(error || ""),
+    });
+    return null;
+  }
+}
+
+async function loadPortfolioSnapshotsForAccounts(
+  api: IBApiNext,
+  instanceId: string | undefined,
+  accountIds: string[],
+): Promise<Map<string, AccountPortfolioSnapshotPosition>> {
+  const snapshots = new Map<string, AccountPortfolioSnapshotPosition>();
+  for (const accountId of accountIds) {
+    const positions = await loadPortfolioSnapshotForAccount(api, instanceId, accountId);
+    for (const position of positions) {
+      snapshots.set(buildPortfolioPositionKey(accountId, position.contract), position);
+    }
+  }
+  return snapshots;
+}
+
+async function loadPortfolioSnapshotForAccount(
+  api: IBApiNext,
+  instanceId: string | undefined,
+  accountId: string,
+): Promise<AccountPortfolioSnapshotPosition[]> {
+  if (!accountId) return [];
+
+  try {
+    const update = await firstValueFrom(
+      api.getAccountUpdates(accountId).pipe(
+        filter((value) => value.changed == null && value.added == null && value.removed == null),
+        take(1),
+        timeout(IBKR_DATA_TIMEOUT),
+      ),
+    );
+    return (update.all.portfolio?.get(accountId) ?? []) as AccountPortfolioSnapshotPosition[];
+  } catch (error: any) {
+    gatewayLog.warn("Failed to load IBKR portfolio snapshot", {
+      instanceId,
+      accountId,
+      error: error?.message || String(error || ""),
+    });
+    return [];
+  }
+}

package/src/plugins/ibkr/gateway/account-summary.ts

@@ -0,0 +1,163 @@
+import type { BrokerAccount, BrokerCashBalance } from "../../../types/trading";
+
+type AccountSummaryValueMap = ReadonlyMap<string, { value: string }>;
+export type AccountSummaryTags = ReadonlyMap<string, AccountSummaryValueMap>;
+
+export interface AccountPnlSnapshot {
+  dailyPnl?: number;
+  unrealizedPnl?: number;
+  realizedPnl?: number;
+}
+
+export function finiteNumber(value: unknown): number | undefined {
+  return typeof value === "number" && Number.isFinite(value) ? value : undefined;
+}
+
+function getAccountSummaryEntry(
+  values: AccountSummaryValueMap | undefined,
+  preferredCurrency?: string,
+): { currency: string; value: number } | null {
+  if (!values) return null;
+
+  if (preferredCurrency) {
+    const preferred = values.get(preferredCurrency);
+    if (preferred?.value) {
+      const numeric = parseFloat(preferred.value);
+      if (Number.isFinite(numeric)) {
+        return { currency: preferredCurrency, value: numeric };
+      }
+    }
+  }
+
+  for (const [currency, entry] of values.entries()) {
+    const numeric = parseFloat(entry?.value ?? "");
+    if (Number.isFinite(numeric)) {
+      return { currency, value: numeric };
+    }
+  }
+  return null;
+}
+
+function getAccountSummaryNumber(
+  tags: AccountSummaryTags | undefined,
+  tagName: string,
+  preferredCurrency?: string,
+): number | undefined {
+  return getAccountSummaryEntry(tags?.get(tagName), preferredCurrency)?.value;
+}
+
+function getAccountSummaryNumberWithAggregateFallback(
+  tags: AccountSummaryTags | undefined,
+  aggregateTags: AccountSummaryTags | undefined,
+  tagName: string,
+  preferredCurrency: string | undefined,
+  allowAggregateFallback: boolean,
+): number | undefined {
+  const direct = getAccountSummaryNumber(tags, tagName, preferredCurrency);
+  if (direct != null) return direct;
+  if (!allowAggregateFallback) return undefined;
+  return getAccountSummaryNumber(aggregateTags, tagName, preferredCurrency);
+}
+
+function inferAccountCurrency(tags: AccountSummaryTags | undefined): string | undefined {
+  if (!tags) return undefined;
+  for (const tagName of ["NetLiquidation", "TotalCashValue", "SettledCash", "AvailableFunds"]) {
+    const entry = getAccountSummaryEntry(tags.get(tagName));
+    if (entry?.currency) return entry.currency;
+  }
+  return undefined;
+}
+
+function getSummaryMap(
+  tags: AccountSummaryTags | undefined,
+  tagName: string,
+): AccountSummaryValueMap | undefined {
+  const values = tags?.get(tagName);
+  return values && values.size > 0 ? values : undefined;
+}
+
+function buildCashBalancesFromSummary(
+  cashBalanceMap: AccountSummaryValueMap | undefined,
+  exchangeRateMap: AccountSummaryValueMap | undefined,
+  baseCurrency?: string,
+): BrokerCashBalance[] | undefined {
+  const balances: BrokerCashBalance[] = [];
+  for (const [currency, entry] of cashBalanceMap?.entries() ?? []) {
+    if (!currency || currency === "BASE") continue;
+    const numeric = parseFloat(entry?.value ?? "");
+    if (!Number.isFinite(numeric)) continue;
+
+    const exchangeRate = currency === baseCurrency
+      ? 1
+      : parseFloat(exchangeRateMap?.get(currency)?.value ?? "");
+    const baseValue = Number.isFinite(exchangeRate) ? numeric * exchangeRate : undefined;
+
+    balances.push({
+      currency,
+      quantity: numeric,
+      baseValue,
+      baseCurrency,
+    });
+  }
+
+  return balances.length > 0 ? balances : undefined;
+}
+
+function buildCashBalances(
+  tags: AccountSummaryTags | undefined,
+  aggregateTags: AccountSummaryTags | undefined,
+  baseCurrency: string | undefined,
+  allowAggregateFallback: boolean,
+): BrokerCashBalance[] | undefined {
+  const directLedger = buildCashBalancesFromSummary(
+    getSummaryMap(tags, "$LEDGER:ALL"),
+    undefined,
+    baseCurrency,
+  );
+  if (directLedger) return directLedger;
+
+  const directSummary = buildCashBalancesFromSummary(
+    getSummaryMap(tags, "CashBalance") ?? getSummaryMap(tags, "TotalCashBalance"),
+    getSummaryMap(tags, "ExchangeRate"),
+    baseCurrency,
+  );
+  if (directSummary) return directSummary;
+
+  if (!allowAggregateFallback) return undefined;
+
+  return buildCashBalancesFromSummary(
+    getSummaryMap(aggregateTags, "CashBalance") ?? getSummaryMap(aggregateTags, "TotalCashBalance"),
+    getSummaryMap(aggregateTags, "ExchangeRate"),
+    baseCurrency,
+  );
+}
+
+export function summarizeBrokerAccount(
+  accountId: string,
+  tags: AccountSummaryTags | undefined,
+  updatedAt: number,
+  aggregateTags?: AccountSummaryTags,
+  allowAggregateCashBalances = false,
+  pnl?: AccountPnlSnapshot,
+): BrokerAccount {
+  const currency = inferAccountCurrency(tags) ?? (allowAggregateCashBalances ? inferAccountCurrency(aggregateTags) : undefined);
+  return {
+    accountId,
+    name: accountId,
+    currency,
+    source: "gateway",
+    updatedAt,
+    netLiquidation: getAccountSummaryNumberWithAggregateFallback(tags, aggregateTags, "NetLiquidation", currency, allowAggregateCashBalances),
+    totalCashValue: getAccountSummaryNumberWithAggregateFallback(tags, aggregateTags, "TotalCashValue", currency, allowAggregateCashBalances),
+    settledCash: getAccountSummaryNumberWithAggregateFallback(tags, aggregateTags, "SettledCash", currency, allowAggregateCashBalances),
+    availableFunds: getAccountSummaryNumberWithAggregateFallback(tags, aggregateTags, "AvailableFunds", currency, allowAggregateCashBalances),
+    buyingPower: getAccountSummaryNumberWithAggregateFallback(tags, aggregateTags, "BuyingPower", currency, allowAggregateCashBalances),
+    excessLiquidity: getAccountSummaryNumberWithAggregateFallback(tags, aggregateTags, "ExcessLiquidity", currency, allowAggregateCashBalances),
+    initMarginReq: getAccountSummaryNumberWithAggregateFallback(tags, aggregateTags, "InitMarginReq", currency, allowAggregateCashBalances),
+    maintMarginReq: getAccountSummaryNumberWithAggregateFallback(tags, aggregateTags, "MaintMarginReq", currency, allowAggregateCashBalances),
+    dailyPnl: pnl?.dailyPnl,
+    unrealizedPnl: pnl?.unrealizedPnl,
+    realizedPnl: pnl?.realizedPnl,
+    cashBalances: buildCashBalances(tags, aggregateTags, currency, allowAggregateCashBalances),
+  };
+}