@reefclaw/connect 0.1.11 → 0.1.12

package/assets/plugin/ingest/rest-gap-filler.d.ts

@@ -1,191 +0,0 @@
-import type { BinancePrivateApi } from '../ccxt/binance-private.js';
-import type { UserDataStream } from '../live/user-data-stream.js';
-import type { LiveStateStore } from '../live/live-state-store.js';
-import type { TradeStoreClient } from './trade-store-client.js';
-import { TouchedSymbolsStore } from './touched-symbols-store.js';
-export interface RestGapFillerOptions {
-    api: BinancePrivateApi;
-    stream: UserDataStream;
-    store: LiveStateStore;
-    client: TradeStoreClient;
-    userId: string;
-    /** Defaults to 'binance_futures'. */
-    exchange?: string;
-    /** How far BEFORE the disconnect to start the backfill window. A 10s
-     *  buffer compensates for clock skew between the client and exchange
-     *  and for any in-flight frames we may have already processed.
-     *  Dedupe at the webapp prevents duplicate rows. */
-    safetyBufferMs?: number;
-    /** Hard floor on the backfill window. A long outage shouldn't trigger
-     *  a fetchMyTrades that walks back days. Binance returns up to 1000
-     *  trades per symbol per call; past that you need multiple paginated
-     *  calls. Default 1h. If a real outage exceeds this, operator runs the
-     *  Phase 2 continuous reconciler separately. */
-    maxWindowMs?: number;
-    /** Limit on trades per symbol per call. Binance cap is 1000. */
-    perSymbolLimit?: number;
-    /** Run a one-shot deep backfill on start() that recovers fills which landed
-     *  while the plugin was DOWN — the reconnect path (handleResyncRequired) only
-     *  covers gaps while the plugin is RUNNING, so a position's closing trade that
-     *  fills during downtime is never re-pushed by the WS and accumulates silently
-     *  across restarts. Defaults to OFF; activate via env `RC_GAPFILL_STARTUP=on`. */
-    startupDeepFill?: boolean;
-    /** How far back the startup deep-fill looks. Default 7 days (env
-     *  `RC_GAPFILL_STARTUP_LOOKBACK_MS`). Sized to cover realistic inter-restart
-     *  downtime; a longer outage is a one-off the operator backfills manually. */
-    startupLookbackMs?: number;
-    /** Delay between paced Binance calls in the deep-fill (per page + per symbol),
-     *  to stay clear of the per-IP weight ceiling. Default 250ms
-     *  (env `RC_GAPFILL_PACE_MS`). */
-    deepFillPaceMs?: number;
-    /** Safety cap on paginated pages per 7-day window in the deep-fill — bounds
-     *  the worst case if a symbol churned a huge number of fills. Default 25. */
-    maxPagesPerWindow?: number;
-    /** Persistent symbol store. Defaults to a file-backed implementation
-     *  under ~/.reefclaw/touched-symbols-<userId>.json. Tests inject a
-     *  store pointed at a tmpdir path or a stub. Pass `null` to disable
-     *  persistence entirely (in-memory only). */
-    touchedSymbolsStore?: TouchedSymbolsStore | null;
-    /** For unit tests — injected clock. */
-    now?: () => number;
-}
-export interface RestGapFillerHealth {
-    reconnectEvents: number;
-    gapFillRuns: number;
-    symbolsQueried: number;
-    tradesBackfilled: number;
-    errorCount: number;
-}
-export declare class RestGapFiller {
-    private readonly api;
-    private readonly stream;
-    private readonly store;
-    private readonly client;
-    private readonly userId;
-    private readonly exchange;
-    private readonly safetyBufferMs;
-    private readonly maxWindowMs;
-    private readonly perSymbolLimit;
-    private readonly startupDeepFill;
-    private readonly startupLookbackMs;
-    private readonly deepFillPaceMs;
-    private readonly maxPagesPerWindow;
-    private readonly now;
-    /** Every symbol we've seen via WS orderUpdate, a position snapshot, or
-     *  the persistent disk store. Never shrinks — a symbol's closing trade
-     *  may still be pending delivery when its position hits zero, and a
-     *  symbol traded in a previous session is still a candidate for fills
-     *  the WS missed during a plugin restart. */
-    private readonly knownSymbols;
-    /** Optional file-backed persistence so the symbol universe survives
-     *  plugin restarts. Null in tests that want pure in-memory behaviour. */
-    private readonly touchedStore;
-    /** Timestamp of the last observed disconnect. Bootstrapped to null so
-     *  the initial window (before the first disconnect) contributes no
-     *  backfill — no blind window existed before the stream was ever up. */
-    private lastDisconnectAt;
-    private started;
-    private readonly onResyncRequired;
-    private readonly onDisconnected;
-    private readonly onOrderUpdate;
-    private reconnectEvents;
-    private gapFillRuns;
-    private symbolsQueried;
-    private tradesBackfilled;
-    private errorCount;
-    constructor(opts: RestGapFillerOptions);
-    start(): void;
-    stop(): void;
-    getHealth(): RestGapFillerHealth;
-    /** Testing hook — lets unit tests trigger a gap-fill pass with a known
-     *  time window instead of replaying stream events. */
-    runGapFill(since: number): Promise<void>;
-    /** One-shot startup deep-fill — the recurrence fix.
-     *
-     *  The reconnect gap-fill (handleResyncRequired) only covers blind windows
-     *  while the plugin is RUNNING and caps its lookback at maxWindowMs (1h). A
-     *  fill that lands while the plugin is DOWN — most damagingly a position's
-     *  closing trade — is never re-pushed by the event-driven WS and was never
-     *  gap-filled, so it silently never reaches the `trades` ledger. Over many
-     *  restarts these missing closes accumulate and make the journal's net-PnL
-     *  (derived from the ledger) wrong. This pass queries userTrades for every
-     *  known symbol over a configurable lookback so downtime closes land on the
-     *  next start. Paginated + 7-day-window-chunked (Binance caps userTrades at
-     *  1000 rows / 7-day span per call) and paced to stay clear of the per-IP
-     *  weight ceiling. Idempotent at the webapp (ON CONFLICT DO NOTHING), so
-     *  re-querying already-recorded fills is a harmless no-op. */
-    runStartupDeepFill(sinceFloor: number): Promise<void>;
-    /** Deep-fill a single symbol over [sinceFloor, now], chunked into ≤7-day
-     *  windows (Binance's per-call span cap). Within a window the FIRST page is
-     *  time-bounded (startTime/endTime); FULL pages continue via Binance's
-     *  `fromId = lastTradeId + 1` (doc-verified: fromId pages ascending by
-     *  trade id and CANNOT be combined with startTime/endTime), stopping when
-     *  a page runs past the window end, comes back short, or hits the page
-     *  cap. Paging by id — not by `lastTime + 1` — is what keeps
-     *  same-millisecond trades on a full-page boundary from being skipped.
-     *
-     *  Every fetch goes through the ban-gated `api.fetchMyTradesOrNull`
-     *  wrapper — we deliberately do NOT use CCXT's built-in `{ paginate: true }`,
-     *  which re-enters CCXT directly and would bypass the process-wide IP-ban
-     *  gate. A `null` read (paced / banned / garbled) is a FAILED read, not an
-     *  empty window: we abort THIS symbol by throwing (caller logs + counts the
-     *  error and continues other symbols) instead of advancing the cursor —
-     *  error-as-empty silently truncated the backfill exactly when Binance was
-     *  shedding us. Returns the number of fills posted. */
-    private deepFillSymbol;
-    /** Epoch-ms timestamp of a CCXT trade — unified `timestamp` first, raw
-     *  Binance `info.time` as fallback. 0 when neither is present. */
-    private tradeTime;
-    /** Numeric Binance trade id of a CCXT trade — unified `id` first, raw
-     *  `info.id` fallback. Null when not a finite number (`fromId` is a LONG
-     *  on the Binance side, so a non-numeric id cannot seed pagination). */
-    private tradeId;
-    /** Cooperative delay used to pace deep-fill Binance calls. Resolves
-     *  immediately for a non-positive duration (e.g. tests pass 0). */
-    private sleep;
-    private handleResyncRequired;
-    private handleDisconnected;
-    private seedKnownSymbolsFromStore;
-    /** Merge the on-disk symbol history into the in-memory set. Best-effort —
-     *  load() returns an empty Set on any error so a corrupt cache file is a
-     *  silent regression to "session-only" behaviour, never a startup blocker. */
-    private hydrateKnownSymbolsFromDisk;
-    /** Discovery via /fapi/v1/income. Catches symbols traded during a blind
-     *  window (plugin downtime, mid-session WS outage) that aren't represented
-     *  in the persisted touched cache or the current open-position seed —
-     *  i.e. brand-new symbols that opened AND closed entirely during the gap.
-     *
-     *  `since` defaults to the maxWindow lookback (used by start()'s one-shot
-     *  pass). On every resync, handleResyncRequired passes the reconnect's
-     *  computed `since` so the discovery window precisely matches the
-     *  fetchMyTrades window — a single REST call recovers every symbol that
-     *  could have had activity in the gap.
-     *
-     *  Best-effort: any error logs and returns; gap-fill still runs against
-     *  whatever knownSymbols already contains. */
-    private discoverSymbolsFromIncome;
-    /** Convert raw CCXT trade objects to FillEvents and post each to the
-     *  webapp ingest endpoint. Returns the count successfully posted
-     *  (posted means handed to the HTTP client; upsert dedup happens in
-     *  the webapp). Silent on empty input. */
-    private postTrades;
-    /** Map a CCXT trade (Binance /fapi/v1/userTrades) to our FillEvent shape.
-     *  Returns null on malformed input — tradeId + orderId + price + qty are
-     *  all required columns on the trades table.
-     *
-     *  Binance raw field names (NOT CCXT-unified) per the docs — we read these
-     *  via `t.info.*` as the second-choice source after CCXT's unified fields:
-     *    id, orderId, symbol, side, positionSide, price, qty, quoteQty,
-     *    commission, commissionAsset, time, buyer, maker, realizedPnl.
-     *  Spec: https://developers.binance.com/docs/derivatives/usds-margined-futures/trade/rest-api/Account-Trade-List
-     *
-     *  Fields NOT returned by userTrades (document-verified):
-     *   - `reduceOnly` — lives on order endpoints, not trades. Gap-filled rows
-     *     get reduceOnly=undefined; the primary WS path (source='ws') always
-     *     has it. Since dedup preserves the first-writer row, trades delivered
-     *     by WS retain reduceOnly; only genuinely-missed-by-WS trades that the
-     *     gap-filler rescues will have this field blank. Acceptable gap.
-     *   - `clientOrderId` — same: lives on the order, not the trade. We infer
-     *     from the ledger by crosswalking `orderId` if consumers need it. */
-    private tradeToFillEvent;
-}