@reefclaw/connect 0.1.12 → 0.1.13

package/assets/plugin/ingest/rest-gap-filler.d.ts

@@ -0,0 +1,191 @@
+import type { BinancePrivateApi } from '../ccxt/binance-private.js';
+import type { UserDataStream } from '../live/user-data-stream.js';
+import type { LiveStateStore } from '../live/live-state-store.js';
+import type { TradeStoreClient } from './trade-store-client.js';
+import { TouchedSymbolsStore } from './touched-symbols-store.js';
+export interface RestGapFillerOptions {
+    api: BinancePrivateApi;
+    stream: UserDataStream;
+    store: LiveStateStore;
+    client: TradeStoreClient;
+    userId: string;
+    /** Defaults to 'binance_futures'. */
+    exchange?: string;
+    /** How far BEFORE the disconnect to start the backfill window. A 10s
+     *  buffer compensates for clock skew between the client and exchange
+     *  and for any in-flight frames we may have already processed.
+     *  Dedupe at the webapp prevents duplicate rows. */
+    safetyBufferMs?: number;
+    /** Hard floor on the backfill window. A long outage shouldn't trigger
+     *  a fetchMyTrades that walks back days. Binance returns up to 1000
+     *  trades per symbol per call; past that you need multiple paginated
+     *  calls. Default 1h. If a real outage exceeds this, operator runs the
+     *  Phase 2 continuous reconciler separately. */
+    maxWindowMs?: number;
+    /** Limit on trades per symbol per call. Binance cap is 1000. */
+    perSymbolLimit?: number;
+    /** Run a one-shot deep backfill on start() that recovers fills which landed
+     *  while the plugin was DOWN — the reconnect path (handleResyncRequired) only
+     *  covers gaps while the plugin is RUNNING, so a position's closing trade that
+     *  fills during downtime is never re-pushed by the WS and accumulates silently
+     *  across restarts. Defaults to OFF; activate via env `RC_GAPFILL_STARTUP=on`. */
+    startupDeepFill?: boolean;
+    /** How far back the startup deep-fill looks. Default 7 days (env
+     *  `RC_GAPFILL_STARTUP_LOOKBACK_MS`). Sized to cover realistic inter-restart
+     *  downtime; a longer outage is a one-off the operator backfills manually. */
+    startupLookbackMs?: number;
+    /** Delay between paced Binance calls in the deep-fill (per page + per symbol),
+     *  to stay clear of the per-IP weight ceiling. Default 250ms
+     *  (env `RC_GAPFILL_PACE_MS`). */
+    deepFillPaceMs?: number;
+    /** Safety cap on paginated pages per 7-day window in the deep-fill — bounds
+     *  the worst case if a symbol churned a huge number of fills. Default 25. */
+    maxPagesPerWindow?: number;
+    /** Persistent symbol store. Defaults to a file-backed implementation
+     *  under ~/.reefclaw/touched-symbols-<userId>.json. Tests inject a
+     *  store pointed at a tmpdir path or a stub. Pass `null` to disable
+     *  persistence entirely (in-memory only). */
+    touchedSymbolsStore?: TouchedSymbolsStore | null;
+    /** For unit tests — injected clock. */
+    now?: () => number;
+}
+export interface RestGapFillerHealth {
+    reconnectEvents: number;
+    gapFillRuns: number;
+    symbolsQueried: number;
+    tradesBackfilled: number;
+    errorCount: number;
+}
+export declare class RestGapFiller {
+    private readonly api;
+    private readonly stream;
+    private readonly store;
+    private readonly client;
+    private readonly userId;
+    private readonly exchange;
+    private readonly safetyBufferMs;
+    private readonly maxWindowMs;
+    private readonly perSymbolLimit;
+    private readonly startupDeepFill;
+    private readonly startupLookbackMs;
+    private readonly deepFillPaceMs;
+    private readonly maxPagesPerWindow;
+    private readonly now;
+    /** Every symbol we've seen via WS orderUpdate, a position snapshot, or
+     *  the persistent disk store. Never shrinks — a symbol's closing trade
+     *  may still be pending delivery when its position hits zero, and a
+     *  symbol traded in a previous session is still a candidate for fills
+     *  the WS missed during a plugin restart. */
+    private readonly knownSymbols;
+    /** Optional file-backed persistence so the symbol universe survives
+     *  plugin restarts. Null in tests that want pure in-memory behaviour. */
+    private readonly touchedStore;
+    /** Timestamp of the last observed disconnect. Bootstrapped to null so
+     *  the initial window (before the first disconnect) contributes no
+     *  backfill — no blind window existed before the stream was ever up. */
+    private lastDisconnectAt;
+    private started;
+    private readonly onResyncRequired;
+    private readonly onDisconnected;
+    private readonly onOrderUpdate;
+    private reconnectEvents;
+    private gapFillRuns;
+    private symbolsQueried;
+    private tradesBackfilled;
+    private errorCount;
+    constructor(opts: RestGapFillerOptions);
+    start(): void;
+    stop(): void;
+    getHealth(): RestGapFillerHealth;
+    /** Testing hook — lets unit tests trigger a gap-fill pass with a known
+     *  time window instead of replaying stream events. */
+    runGapFill(since: number): Promise<void>;
+    /** One-shot startup deep-fill — the recurrence fix.
+     *
+     *  The reconnect gap-fill (handleResyncRequired) only covers blind windows
+     *  while the plugin is RUNNING and caps its lookback at maxWindowMs (1h). A
+     *  fill that lands while the plugin is DOWN — most damagingly a position's
+     *  closing trade — is never re-pushed by the event-driven WS and was never
+     *  gap-filled, so it silently never reaches the `trades` ledger. Over many
+     *  restarts these missing closes accumulate and make the journal's net-PnL
+     *  (derived from the ledger) wrong. This pass queries userTrades for every
+     *  known symbol over a configurable lookback so downtime closes land on the
+     *  next start. Paginated + 7-day-window-chunked (Binance caps userTrades at
+     *  1000 rows / 7-day span per call) and paced to stay clear of the per-IP
+     *  weight ceiling. Idempotent at the webapp (ON CONFLICT DO NOTHING), so
+     *  re-querying already-recorded fills is a harmless no-op. */
+    runStartupDeepFill(sinceFloor: number): Promise<void>;
+    /** Deep-fill a single symbol over [sinceFloor, now], chunked into ≤7-day
+     *  windows (Binance's per-call span cap). Within a window the FIRST page is
+     *  time-bounded (startTime/endTime); FULL pages continue via Binance's
+     *  `fromId = lastTradeId + 1` (doc-verified: fromId pages ascending by
+     *  trade id and CANNOT be combined with startTime/endTime), stopping when
+     *  a page runs past the window end, comes back short, or hits the page
+     *  cap. Paging by id — not by `lastTime + 1` — is what keeps
+     *  same-millisecond trades on a full-page boundary from being skipped.
+     *
+     *  Every fetch goes through the ban-gated `api.fetchMyTradesOrNull`
+     *  wrapper — we deliberately do NOT use CCXT's built-in `{ paginate: true }`,
+     *  which re-enters CCXT directly and would bypass the process-wide IP-ban
+     *  gate. A `null` read (paced / banned / garbled) is a FAILED read, not an
+     *  empty window: we abort THIS symbol by throwing (caller logs + counts the
+     *  error and continues other symbols) instead of advancing the cursor —
+     *  error-as-empty silently truncated the backfill exactly when Binance was
+     *  shedding us. Returns the number of fills posted. */
+    private deepFillSymbol;
+    /** Epoch-ms timestamp of a CCXT trade — unified `timestamp` first, raw
+     *  Binance `info.time` as fallback. 0 when neither is present. */
+    private tradeTime;
+    /** Numeric Binance trade id of a CCXT trade — unified `id` first, raw
+     *  `info.id` fallback. Null when not a finite number (`fromId` is a LONG
+     *  on the Binance side, so a non-numeric id cannot seed pagination). */
+    private tradeId;
+    /** Cooperative delay used to pace deep-fill Binance calls. Resolves
+     *  immediately for a non-positive duration (e.g. tests pass 0). */
+    private sleep;
+    private handleResyncRequired;
+    private handleDisconnected;
+    private seedKnownSymbolsFromStore;
+    /** Merge the on-disk symbol history into the in-memory set. Best-effort —
+     *  load() returns an empty Set on any error so a corrupt cache file is a
+     *  silent regression to "session-only" behaviour, never a startup blocker. */
+    private hydrateKnownSymbolsFromDisk;
+    /** Discovery via /fapi/v1/income. Catches symbols traded during a blind
+     *  window (plugin downtime, mid-session WS outage) that aren't represented
+     *  in the persisted touched cache or the current open-position seed —
+     *  i.e. brand-new symbols that opened AND closed entirely during the gap.
+     *
+     *  `since` defaults to the maxWindow lookback (used by start()'s one-shot
+     *  pass). On every resync, handleResyncRequired passes the reconnect's
+     *  computed `since` so the discovery window precisely matches the
+     *  fetchMyTrades window — a single REST call recovers every symbol that
+     *  could have had activity in the gap.
+     *
+     *  Best-effort: any error logs and returns; gap-fill still runs against
+     *  whatever knownSymbols already contains. */
+    private discoverSymbolsFromIncome;
+    /** Convert raw CCXT trade objects to FillEvents and post each to the
+     *  webapp ingest endpoint. Returns the count successfully posted
+     *  (posted means handed to the HTTP client; upsert dedup happens in
+     *  the webapp). Silent on empty input. */
+    private postTrades;
+    /** Map a CCXT trade (Binance /fapi/v1/userTrades) to our FillEvent shape.
+     *  Returns null on malformed input — tradeId + orderId + price + qty are
+     *  all required columns on the trades table.
+     *
+     *  Binance raw field names (NOT CCXT-unified) per the docs — we read these
+     *  via `t.info.*` as the second-choice source after CCXT's unified fields:
+     *    id, orderId, symbol, side, positionSide, price, qty, quoteQty,
+     *    commission, commissionAsset, time, buyer, maker, realizedPnl.
+     *  Spec: https://developers.binance.com/docs/derivatives/usds-margined-futures/trade/rest-api/Account-Trade-List
+     *
+     *  Fields NOT returned by userTrades (document-verified):
+     *   - `reduceOnly` — lives on order endpoints, not trades. Gap-filled rows
+     *     get reduceOnly=undefined; the primary WS path (source='ws') always
+     *     has it. Since dedup preserves the first-writer row, trades delivered
+     *     by WS retain reduceOnly; only genuinely-missed-by-WS trades that the
+     *     gap-filler rescues will have this field blank. Acceptable gap.
+     *   - `clientOrderId` — same: lives on the order, not the trade. We infer
+     *     from the ledger by crosswalking `orderId` if consumers need it. */
+    private tradeToFillEvent;
+}