@reefclaw/openclaw-plugin 0.1.0

package/ingest/rest-gap-filler.js

@@ -0,0 +1,565 @@
+// REST gap-filler — Phase 2 of TRADE_AUDIT_TRAIL_PLAN.
+//
+// WHY THIS EXISTS
+// ---------------
+// The user-data WebSocket is event-driven: Binance only pushes frames when
+// the account moves. A quiet account sits silent for minutes at a time.
+// When a fill lands during a WS reconnect window (stale-watchdog or network
+// blip), the ORDER_TRADE_UPDATE event for that fill is lost forever — it's
+// a pushed stream, not a queryable log. The positions/balance snapshot
+// taken on reconnect reflects the POST-fill state, so truth-check returns
+// OK, but the individual fill row never lands in the `trades` table.
+//
+// In shadow mode this is benign (REST is authoritative for reads). In
+// observe/enforce mode, WS is the sole audit-trail source and missed fills
+// silently degrade correctness of query_trades, daily P&L reconciliation,
+// and downstream analytics. This class closes that gap.
+//
+// WHAT IT DOES
+// ------------
+// On every WS 'connected' event AFTER the initial connect, query
+// `/fapi/v1/userTrades` for every symbol this session has touched, using
+// `since = lastDisconnectTime - safetyBufferMs` so the window overlaps
+// what the WS may already have delivered. Convert each trade to a
+// FillEvent with source='rest_reconcile' and POST via TradeStoreClient.
+//
+// The webapp's ingest endpoint upserts ON CONFLICT (exchange,
+// exchange_trade_id) DO NOTHING — see webapp/src/app/api/internal/trades.
+// Any trade the WS already delivered via source='ws' will lose the race
+// and stay recorded as 'ws' (correct). New trades land with
+// source='rest_reconcile'.
+//
+// WHAT IT DOES NOT DO
+// -------------------
+// - It does NOT touch the LiveStateStore. Positions/balance seeding is
+//   the snapshot's job; this class is audit-trail only.
+// - It does NOT gap-fill on the initial connect. The plugin just started —
+//   there's no blind window to backfill. Initial snapshot seeds state from
+//   REST, which is already authoritative for any fills before startup.
+// - It does NOT retry failed fetches. If fetchMyTrades returns [] because
+//   of an error, we log and move on. The next reconnect will retry over
+//   a superset window. (Reconnect path only — the startup deep-fill uses
+//   fetchMyTradesOrNull and DISTINGUISHES a failed read (null) from a
+//   genuinely empty window, aborting the symbol instead of silently
+//   truncating the backfill.)
+//
+// SYMBOL TRACKING
+// ---------------
+// Binance requires a symbol per fetchMyTrades call; there's no
+// portfolio-wide trades endpoint. We track every symbol we've seen via
+// WS orderUpdate events + position snapshots. Set never shrinks during
+// the session — a closed position's symbol is still queried on reconnect
+// because late fills (closing trade) may have been missed.
+//
+// Spec:
+// https://developers.binance.com/docs/derivatives/usds-margined-futures/trade/rest-api/Account-Trade-List
+import { logger, formatError } from '../logger.js';
+import { TouchedSymbolsStore } from './touched-symbols-store.js';
+const TAG = 'rest-gap-filler';
+/** Binance caps a single userTrades window (startTime..endTime) at 7 days, so
+ *  the startup deep-fill chunks its lookback into windows no larger than this. */
+const DEEP_FILL_WINDOW_MS = 7 * 24 * 60 * 60_000;
+/** CCXT unified symbol 'BTC/USDT:USDT' → canonical 'BTC/USDT'. Matches the
+ *  WS-side normalizer so gap-fill rows land under the same symbol key
+ *  every downstream consumer uses. */
+function normSymbol(s) {
+    return (s ?? '').replace(/:[A-Z]+$/, '');
+}
+/** Parse a positive-integer env override, falling back to `def` on missing /
+ *  non-numeric / non-positive input. */
+function parseEnvInt(raw, def) {
+    const n = Number(raw);
+    return Number.isFinite(n) && n > 0 ? Math.floor(n) : def;
+}
+export class RestGapFiller {
+    api;
+    stream;
+    store;
+    client;
+    userId;
+    exchange;
+    safetyBufferMs;
+    maxWindowMs;
+    perSymbolLimit;
+    startupDeepFill;
+    startupLookbackMs;
+    deepFillPaceMs;
+    maxPagesPerWindow;
+    now;
+    /** Every symbol we've seen via WS orderUpdate, a position snapshot, or
+     *  the persistent disk store. Never shrinks — a symbol's closing trade
+     *  may still be pending delivery when its position hits zero, and a
+     *  symbol traded in a previous session is still a candidate for fills
+     *  the WS missed during a plugin restart. */
+    knownSymbols = new Set();
+    /** Optional file-backed persistence so the symbol universe survives
+     *  plugin restarts. Null in tests that want pure in-memory behaviour. */
+    touchedStore;
+    /** Timestamp of the last observed disconnect. Bootstrapped to null so
+     *  the initial window (before the first disconnect) contributes no
+     *  backfill — no blind window existed before the stream was ever up. */
+    lastDisconnectAt = null;
+    started = false;
+    onResyncRequired;
+    onDisconnected;
+    onOrderUpdate;
+    // Metrics — exposed via getHealth() for operator dashboards.
+    reconnectEvents = 0;
+    gapFillRuns = 0;
+    symbolsQueried = 0;
+    tradesBackfilled = 0;
+    errorCount = 0;
+    constructor(opts) {
+        this.api = opts.api;
+        this.stream = opts.stream;
+        this.store = opts.store;
+        this.client = opts.client;
+        this.userId = opts.userId;
+        this.exchange = opts.exchange ?? 'binance_futures';
+        this.safetyBufferMs = opts.safetyBufferMs ?? 10_000;
+        this.maxWindowMs = opts.maxWindowMs ?? 60 * 60_000;
+        this.perSymbolLimit = opts.perSymbolLimit ?? 500;
+        // Startup deep-fill: opt-in via env RC_GAPFILL_STARTUP=on (gated-flag
+        // convention — every restart makes paced Binance calls + idempotent webapp
+        // POSTs, so it's activated explicitly on deploy, not silently on by default).
+        this.startupDeepFill =
+            opts.startupDeepFill ?? (process.env.RC_GAPFILL_STARTUP === 'on');
+        this.startupLookbackMs =
+            opts.startupLookbackMs ??
+                parseEnvInt(process.env.RC_GAPFILL_STARTUP_LOOKBACK_MS, DEEP_FILL_WINDOW_MS);
+        this.deepFillPaceMs =
+            opts.deepFillPaceMs ?? parseEnvInt(process.env.RC_GAPFILL_PACE_MS, 250);
+        this.maxPagesPerWindow = opts.maxPagesPerWindow ?? 25;
+        this.now = opts.now ?? (() => Date.now());
+        // touchedSymbolsStore: explicit null disables persistence; undefined
+        // takes the production default (file-backed under ~/.reefclaw).
+        this.touchedStore =
+            opts.touchedSymbolsStore === null
+                ? null
+                : (opts.touchedSymbolsStore ?? new TouchedSymbolsStore({ userId: this.userId }));
+        this.onResyncRequired = (cause) => this.handleResyncRequired(cause);
+        this.onDisconnected = () => this.handleDisconnected();
+        this.onOrderUpdate = (ev) => {
+            if (!ev?.symbol)
+                return;
+            const norm = normSymbol(ev.symbol);
+            if (this.knownSymbols.has(norm))
+                return;
+            this.knownSymbols.add(norm);
+            // Persist on first sighting only — same-symbol updates are a no-op.
+            this.touchedStore?.save(this.knownSymbols);
+        };
+    }
+    start() {
+        if (this.started)
+            return;
+        // `resyncRequired` fires on every successful (re)connect AFTER the
+        // initial connect — exactly the hook we need. Initial connect has no
+        // blind window so we skip it by design (UserDataStream only emits
+        // resyncRequired on the 2nd, 3rd, ... connects).
+        this.stream.on('resyncRequired', this.onResyncRequired);
+        this.stream.on('disconnected', this.onDisconnected);
+        this.stream.on('orderUpdate', this.onOrderUpdate);
+        this.started = true;
+        // Hydrate first so any open-position seed merges in without erasing
+        // the persisted history. Then persist if the merge added anything.
+        this.hydrateKnownSymbolsFromDisk();
+        this.seedKnownSymbolsFromStore();
+        // Background income discovery — recovers symbols that were traded
+        // during plugin downtime even when they aren't in the persisted
+        // touched-symbols cache (e.g. a brand-new symbol that opened AND
+        // closed entirely while the plugin was offline). Discovery resolves
+        // BEFORE the startup deep-fill so a symbol traded entirely during the
+        // outage is in knownSymbols when its trades are queried. Fire-and-forget
+        // so start() never blocks the boot path.
+        void (async () => {
+            try {
+                const lookbackSince = this.now() - this.startupLookbackMs;
+                // Match the discovery window to the deep-fill lookback so a
+                // multi-day-old symbol is found; falls back to the 1h default
+                // window when the deep-fill is disabled.
+                await this.discoverSymbolsFromIncome(this.startupDeepFill ? lookbackSince : undefined);
+                if (this.startupDeepFill) {
+                    await this.runStartupDeepFill(lookbackSince);
+                }
+            }
+            catch (err) {
+                this.errorCount++;
+                logger.error(TAG, `startup discovery/deep-fill chain rejected: ${formatError(err)}`);
+            }
+        })();
+        logger.info(TAG, `Started — userId=${this.userId} safetyBuffer=${this.safetyBufferMs}ms maxWindow=${this.maxWindowMs}ms ` +
+            `startupDeepFill=${this.startupDeepFill} startupLookback=${this.startupLookbackMs}ms knownSymbols=${this.knownSymbols.size}`);
+    }
+    stop() {
+        if (!this.started)
+            return;
+        this.stream.off('resyncRequired', this.onResyncRequired);
+        this.stream.off('disconnected', this.onDisconnected);
+        this.stream.off('orderUpdate', this.onOrderUpdate);
+        this.started = false;
+        logger.info(TAG, 'Stopped');
+    }
+    getHealth() {
+        return {
+            reconnectEvents: this.reconnectEvents,
+            gapFillRuns: this.gapFillRuns,
+            symbolsQueried: this.symbolsQueried,
+            tradesBackfilled: this.tradesBackfilled,
+            errorCount: this.errorCount,
+        };
+    }
+    /** Testing hook — lets unit tests trigger a gap-fill pass with a known
+     *  time window instead of replaying stream events. */
+    async runGapFill(since) {
+        this.gapFillRuns++;
+        this.seedKnownSymbolsFromStore();
+        const symbols = Array.from(this.knownSymbols);
+        if (symbols.length === 0) {
+            logger.info(TAG, `gap-fill skipped: no known symbols (since=${new Date(since).toISOString()})`);
+            return;
+        }
+        logger.info(TAG, `gap-fill starting: ${symbols.length} symbol(s), since=${new Date(since).toISOString()}`);
+        let totalThisRun = 0;
+        for (const symbol of symbols) {
+            this.symbolsQueried++;
+            try {
+                // Binance userTrades takes the raw symbol (e.g. 'BTC/USDT:USDT' —
+                // CCXT's unified form). We stored normalized keys, add the suffix
+                // back for the API call. CCXT handles either form.
+                const apiSymbol = symbol.includes(':') ? symbol : `${symbol}:${symbol.split('/')[1]}`;
+                const trades = await this.api.fetchMyTrades(apiSymbol, since, this.perSymbolLimit);
+                const posted = this.postTrades(symbol, trades);
+                totalThisRun += posted;
+            }
+            catch (err) {
+                this.errorCount++;
+                logger.warn(TAG, `gap-fill ${symbol} threw: ${formatError(err)} — continuing other symbols`);
+            }
+        }
+        logger.info(TAG, `gap-fill complete: ${totalThisRun} fills backfilled across ${symbols.length} symbol(s)`);
+    }
+    /** One-shot startup deep-fill — the recurrence fix.
+     *
+     *  The reconnect gap-fill (handleResyncRequired) only covers blind windows
+     *  while the plugin is RUNNING and caps its lookback at maxWindowMs (1h). A
+     *  fill that lands while the plugin is DOWN — most damagingly a position's
+     *  closing trade — is never re-pushed by the event-driven WS and was never
+     *  gap-filled, so it silently never reaches the `trades` ledger. Over many
+     *  restarts these missing closes accumulate and make the journal's net-PnL
+     *  (derived from the ledger) wrong. This pass queries userTrades for every
+     *  known symbol over a configurable lookback so downtime closes land on the
+     *  next start. Paginated + 7-day-window-chunked (Binance caps userTrades at
+     *  1000 rows / 7-day span per call) and paced to stay clear of the per-IP
+     *  weight ceiling. Idempotent at the webapp (ON CONFLICT DO NOTHING), so
+     *  re-querying already-recorded fills is a harmless no-op. */
+    async runStartupDeepFill(sinceFloor) {
+        this.seedKnownSymbolsFromStore();
+        const symbols = Array.from(this.knownSymbols);
+        if (symbols.length === 0) {
+            logger.info(TAG, 'startup deep-fill skipped: no known symbols');
+            return;
+        }
+        this.gapFillRuns++;
+        logger.info(TAG, `startup deep-fill starting: ${symbols.length} symbol(s), since=${new Date(sinceFloor).toISOString()} ` +
+            `(lookback=${this.startupLookbackMs}ms, pace=${this.deepFillPaceMs}ms)`);
+        let total = 0;
+        for (const symbol of symbols) {
+            try {
+                total += await this.deepFillSymbol(symbol, sinceFloor);
+            }
+            catch (err) {
+                this.errorCount++;
+                logger.warn(TAG, `startup deep-fill ${symbol} threw: ${formatError(err)} — continuing other symbols`);
+            }
+            await this.sleep(this.deepFillPaceMs);
+        }
+        logger.info(TAG, `startup deep-fill complete: ${total} fill(s) backfilled across ${symbols.length} symbol(s)`);
+    }
+    /** Deep-fill a single symbol over [sinceFloor, now], chunked into ≤7-day
+     *  windows (Binance's per-call span cap). Within a window the FIRST page is
+     *  time-bounded (startTime/endTime); FULL pages continue via Binance's
+     *  `fromId = lastTradeId + 1` (doc-verified: fromId pages ascending by
+     *  trade id and CANNOT be combined with startTime/endTime), stopping when
+     *  a page runs past the window end, comes back short, or hits the page
+     *  cap. Paging by id — not by `lastTime + 1` — is what keeps
+     *  same-millisecond trades on a full-page boundary from being skipped.
+     *
+     *  Every fetch goes through the ban-gated `api.fetchMyTradesOrNull`
+     *  wrapper — we deliberately do NOT use CCXT's built-in `{ paginate: true }`,
+     *  which re-enters CCXT directly and would bypass the process-wide IP-ban
+     *  gate. A `null` read (paced / banned / garbled) is a FAILED read, not an
+     *  empty window: we abort THIS symbol by throwing (caller logs + counts the
+     *  error and continues other symbols) instead of advancing the cursor —
+     *  error-as-empty silently truncated the backfill exactly when Binance was
+     *  shedding us. Returns the number of fills posted. */
+    async deepFillSymbol(symbol, sinceFloor) {
+        this.symbolsQueried++;
+        // Binance userTrades wants CCXT's unified suffixed form ('BTC/USDT:USDT');
+        // knownSymbols holds normalized keys, so re-append the quote suffix.
+        const apiSymbol = symbol.includes(':') ? symbol : `${symbol}:${symbol.split('/')[1]}`;
+        const nowMs = this.now();
+        let windowStart = sinceFloor;
+        let posted = 0;
+        while (windowStart < nowMs) {
+            const windowEnd = Math.min(windowStart + DEEP_FILL_WINDOW_MS, nowMs);
+            let fromId = null;
+            for (let page = 0; page < this.maxPagesPerWindow; page++) {
+                const trades = fromId == null
+                    ? await this.api.fetchMyTradesOrNull(apiSymbol, windowStart, this.perSymbolLimit, { endTime: windowEnd })
+                    : await this.api.fetchMyTradesOrNull(apiSymbol, undefined, this.perSymbolLimit, { fromId });
+                if (trades === null) {
+                    throw new Error(`deep-fill ${symbol}: fetchMyTrades returned null (read failed — paced/banned/garbled). ` +
+                        'Aborting this symbol WITHOUT marking any window complete; the next startup deep-fill retries (posting is idempotent).');
+                }
+                if (trades.length === 0)
+                    break;
+                // A fromId continuation page walks forward in id order with NO upper
+                // time bound — drop anything past this window (the next window's
+                // time-bounded first page re-fetches it; posting is idempotent, the
+                // filter just keeps the metrics honest).
+                const inWindow = trades.filter((t) => this.tradeTime(t) <= windowEnd);
+                posted += this.postTrades(symbol, inWindow);
+                if (inWindow.length < trades.length)
+                    break; // paged past the window end
+                // A short page means this window is exhausted.
+                if (trades.length < this.perSymbolLimit)
+                    break;
+                const lastId = this.tradeId(trades[trades.length - 1]);
+                if (lastId == null) {
+                    // Cannot continue by id (malformed row) — stop rather than loop.
+                    logger.warn(TAG, `deep-fill ${symbol}: full page with no parseable trade id — stopping pagination for this window`);
+                    break;
+                }
+                fromId = lastId + 1; // fromId is inclusive — skip the last posted trade
+                await this.sleep(this.deepFillPaceMs);
+            }
+            windowStart = windowEnd;
+        }
+        return posted;
+    }
+    /** Epoch-ms timestamp of a CCXT trade — unified `timestamp` first, raw
+     *  Binance `info.time` as fallback. 0 when neither is present. */
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    tradeTime(t) {
+        const ts = t?.timestamp != null ? Number(t.timestamp) : Number(t?.info?.time);
+        return Number.isFinite(ts) ? ts : 0;
+    }
+    /** Numeric Binance trade id of a CCXT trade — unified `id` first, raw
+     *  `info.id` fallback. Null when not a finite number (`fromId` is a LONG
+     *  on the Binance side, so a non-numeric id cannot seed pagination). */
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    tradeId(t) {
+        const raw = t?.id ?? t?.info?.id;
+        const n = Number(raw);
+        return Number.isFinite(n) ? n : null;
+    }
+    /** Cooperative delay used to pace deep-fill Binance calls. Resolves
+     *  immediately for a non-positive duration (e.g. tests pass 0). */
+    sleep(ms) {
+        if (!(ms > 0))
+            return Promise.resolve();
+        return new Promise((resolve) => setTimeout(resolve, ms));
+    }
+    // ---- Internals ----
+    handleResyncRequired(cause) {
+        this.reconnectEvents++;
+        const nowMs = this.now();
+        // The gap-fill window must cover everything the WS may have missed. Two
+        // independent triggers can drive a reconnect:
+        //   1. Stale-watchdog / listenKey rotation — happens when WS goes briefly
+        //      silent but didn't necessarily miss a fill. lastDisconnectAt is
+        //      ~now and lastEventAt is also recent.
+        //   2. Active-probe-detected silent stream — REST sees a fill the WS
+        //      never delivered. lastDisconnectAt becomes ~now (probe forces
+        //      reconnect), but lastEventAt may be tens of minutes ago. Using
+        //      disconnectAt as `since` makes the gap-fill window start AFTER
+        //      the missed fill, so fetchMyTrades returns 0 every time.
+        // Fix: use the OLDER of (lastDisconnectAt, lastEventAt) so the window
+        // always covers the actual gap. maxWindow still caps the lookback so
+        // we don't backfill multi-hour outages in one shot.
+        const disconnectAt = this.lastDisconnectAt ?? nowMs;
+        const lastEventAt = this.stream.getHealth().lastEventAt ?? disconnectAt;
+        const baseSince = Math.min(disconnectAt, lastEventAt);
+        const rawSince = baseSince - this.safetyBufferMs;
+        const floorSince = nowMs - this.maxWindowMs;
+        const since = Math.max(rawSince, floorSince);
+        logger.info(TAG, `resync (cause=${cause}) — scheduling gap-fill since=${new Date(since).toISOString()} ` +
+            `(disconnectAt=${new Date(disconnectAt).toISOString()}, lastEventAt=${new Date(lastEventAt).toISOString()})`);
+        // Run async; don't block the stream handler. Discovery MUST resolve
+        // before runGapFill so a brand-new symbol traded entirely during this
+        // outage is in knownSymbols when fetchMyTrades runs. Discovery uses the
+        // exact same `since` window as the gap-fill so a single REST call
+        // covers everything that may have been missed.
+        void (async () => {
+            try {
+                await this.discoverSymbolsFromIncome(since);
+                await this.runGapFill(since);
+            }
+            catch (err) {
+                this.errorCount++;
+                logger.error(TAG, `resync gap-fill chain rejected: ${formatError(err)}`);
+            }
+        })();
+    }
+    handleDisconnected() {
+        this.lastDisconnectAt = this.now();
+    }
+    seedKnownSymbolsFromStore() {
+        const sizeBefore = this.knownSymbols.size;
+        try {
+            const positions = this.store.getPositions();
+            for (const p of positions) {
+                if (p?.symbol)
+                    this.knownSymbols.add(normSymbol(p.symbol));
+            }
+        }
+        catch {
+            // Store may not be initialised yet on the first start() call; no-op.
+        }
+        if (this.knownSymbols.size > sizeBefore) {
+            this.touchedStore?.save(this.knownSymbols);
+        }
+    }
+    /** Merge the on-disk symbol history into the in-memory set. Best-effort —
+     *  load() returns an empty Set on any error so a corrupt cache file is a
+     *  silent regression to "session-only" behaviour, never a startup blocker. */
+    hydrateKnownSymbolsFromDisk() {
+        if (!this.touchedStore)
+            return;
+        const persisted = this.touchedStore.load();
+        if (persisted.size === 0)
+            return;
+        for (const s of persisted)
+            this.knownSymbols.add(s);
+        logger.info(TAG, `hydrated ${persisted.size} touched symbol(s) from disk`);
+    }
+    /** Discovery via /fapi/v1/income. Catches symbols traded during a blind
+     *  window (plugin downtime, mid-session WS outage) that aren't represented
+     *  in the persisted touched cache or the current open-position seed —
+     *  i.e. brand-new symbols that opened AND closed entirely during the gap.
+     *
+     *  `since` defaults to the maxWindow lookback (used by start()'s one-shot
+     *  pass). On every resync, handleResyncRequired passes the reconnect's
+     *  computed `since` so the discovery window precisely matches the
+     *  fetchMyTrades window — a single REST call recovers every symbol that
+     *  could have had activity in the gap.
+     *
+     *  Best-effort: any error logs and returns; gap-fill still runs against
+     *  whatever knownSymbols already contains. */
+    async discoverSymbolsFromIncome(since) {
+        if (typeof this.api.fetchRecentTradedSymbols !== 'function') {
+            // Older API surfaces / mocks that don't implement the discovery path.
+            // Silent skip preserves existing behaviour for unit tests that don't
+            // care about cross-restart symbol recovery.
+            return;
+        }
+        const sinceMs = since ?? this.now() - this.maxWindowMs;
+        try {
+            const symbols = await this.api.fetchRecentTradedSymbols(sinceMs);
+            if (!Array.isArray(symbols) || symbols.length === 0)
+                return;
+            let added = 0;
+            for (const s of symbols) {
+                const norm = normSymbol(s);
+                if (!norm)
+                    continue;
+                if (this.knownSymbols.has(norm))
+                    continue;
+                this.knownSymbols.add(norm);
+                added++;
+            }
+            if (added > 0) {
+                this.touchedStore?.save(this.knownSymbols);
+                logger.info(TAG, `income discovery added ${added} symbol(s) (since=${new Date(sinceMs).toISOString()}, total=${this.knownSymbols.size})`);
+            }
+        }
+        catch (err) {
+            logger.warn(TAG, `income discovery failed: ${formatError(err)} — gap-fill will use existing knownSymbols`);
+        }
+    }
+    /** Convert raw CCXT trade objects to FillEvents and post each to the
+     *  webapp ingest endpoint. Returns the count successfully posted
+     *  (posted means handed to the HTTP client; upsert dedup happens in
+     *  the webapp). Silent on empty input. */
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    postTrades(symbol, raw) {
+        if (!Array.isArray(raw) || raw.length === 0)
+            return 0;
+        let posted = 0;
+        for (const t of raw) {
+            const fill = this.tradeToFillEvent(symbol, t);
+            if (!fill)
+                continue;
+            this.client.post(this.userId, fill);
+            this.tradesBackfilled++;
+            posted++;
+        }
+        return posted;
+    }
+    /** Map a CCXT trade (Binance /fapi/v1/userTrades) to our FillEvent shape.
+     *  Returns null on malformed input — tradeId + orderId + price + qty are
+     *  all required columns on the trades table.
+     *
+     *  Binance raw field names (NOT CCXT-unified) per the docs — we read these
+     *  via `t.info.*` as the second-choice source after CCXT's unified fields:
+     *    id, orderId, symbol, side, positionSide, price, qty, quoteQty,
+     *    commission, commissionAsset, time, buyer, maker, realizedPnl.
+     *  Spec: https://developers.binance.com/docs/derivatives/usds-margined-futures/trade/rest-api/Account-Trade-List
+     *
+     *  Fields NOT returned by userTrades (document-verified):
+     *   - `reduceOnly` — lives on order endpoints, not trades. Gap-filled rows
+     *     get reduceOnly=undefined; the primary WS path (source='ws') always
+     *     has it. Since dedup preserves the first-writer row, trades delivered
+     *     by WS retain reduceOnly; only genuinely-missed-by-WS trades that the
+     *     gap-filler rescues will have this field blank. Acceptable gap.
+     *   - `clientOrderId` — same: lives on the order, not the trade. We infer
+     *     from the ledger by crosswalking `orderId` if consumers need it. */
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    tradeToFillEvent(symbol, t) {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const info = (t?.info ?? {});
+        // Binance raw field is `id` (not `tradeId`). CCXT maps raw `id` onto the
+        // unified `id` field — primary path is `t.id`; fallback to `info.id` for
+        // raw responses that haven't been through CCXT's unifier.
+        const tradeId = t?.id != null ? String(t.id) : (info.id != null ? String(info.id) : null);
+        const orderId = t?.order != null ? String(t.order) : (info.orderId != null ? String(info.orderId) : null);
+        const priceRaw = t?.price != null ? Number(t.price) : Number(info.price);
+        const qtyRaw = t?.amount != null ? Number(t.amount) : Number(info.qty);
+        const timeRaw = t?.timestamp != null ? Number(t.timestamp) : Number(info.time);
+        if (!tradeId || !orderId || !(priceRaw > 0) || !(qtyRaw > 0) || !(timeRaw > 0)) {
+            return null;
+        }
+        const sideRaw = String(t?.side ?? info.side ?? 'buy').toLowerCase();
+        const side = sideRaw === 'sell' ? 'SELL' : 'BUY';
+        // CCXT's unified `takerOrMaker` is the authoritative source; raw
+        // Binance exposes `maker: boolean` under info as the second-choice.
+        const maker = typeof t?.takerOrMaker === 'string'
+            ? t.takerOrMaker === 'maker'
+            : (info.maker != null ? Boolean(info.maker) : undefined);
+        const commissionRaw = t?.fee?.cost != null ? Number(t.fee.cost) : Number(info.commission);
+        const commissionAsset = typeof t?.fee?.currency === 'string'
+            ? t.fee.currency
+            : (typeof info.commissionAsset === 'string' ? info.commissionAsset : undefined);
+        const realizedPnlRaw = info.realizedPnl != null ? Number(info.realizedPnl) : undefined;
+        return {
+            exchange: this.exchange,
+            exchangeTradeId: tradeId,
+            exchangeOrderId: orderId,
+            // clientOrderId + reduceOnly are NOT in the Binance userTrades schema.
+            // See docstring on this method for why we don't attempt to source them.
+            source: 'rest_reconcile',
+            userId: this.userId,
+            symbol: normSymbol(symbol),
+            side,
+            quantity: qtyRaw,
+            price: priceRaw,
+            maker,
+            commission: Number.isFinite(commissionRaw) ? commissionRaw : undefined,
+            commissionAsset,
+            realizedPnl: Number.isFinite(realizedPnlRaw) ? realizedPnlRaw : undefined,
+            exchangeTime: timeRaw,
+            rawPayload: t,
+        };
+    }
+}

package/ingest/touched-symbols-store.d.ts

@@ -0,0 +1,25 @@
+export interface TouchedSymbolsStorageOptions {
+    /** Override path — primarily for tests. Default is per-user under ~/.reefclaw. */
+    path?: string;
+    /** Identifies the file owner; written into the JSON for forensic clarity. */
+    userId: string;
+    /** Test-only injected clock for `updatedAt`. */
+    now?: () => number;
+}
+/** Default per-user path. The userId scoping prevents two parallel plugin
+ *  instances on the same machine (rare, but possible during ops) from
+ *  clobbering each other's history. */
+export declare function defaultTouchedSymbolsPath(userId: string): string;
+export declare class TouchedSymbolsStore {
+    private readonly path;
+    private readonly userId;
+    private readonly now;
+    constructor(opts: TouchedSymbolsStorageOptions);
+    /** Read the persisted set. Returns an empty Set on any error so a corrupt
+     *  file can't block ingest startup. The corrupt file isn't deleted —
+     *  the next save() naturally overwrites it via tmp+rename. */
+    load(): Set<string>;
+    /** Atomically persist the set to disk. Best-effort — failures are logged
+     *  but never thrown; persistence is a recovery helper, not on the hot path. */
+    save(symbols: Set<string>): void;
+}