@reefclaw/openclaw-plugin 0.1.0

package/ccxt/binance-ban-gate.js

@@ -0,0 +1,409 @@
+// Process-wide Binance rate-limit / IP-ban gate.
+//
+// Why this exists (2026-05-15 incident): the plugin runs ~6 independent
+// Binance pollers (LiveBalanceEnricher, position reconciler, bracket
+// reconciler, user-data active probe, deposit tracker, public ticker/OHLCV)
+// plus agent-driven tool calls. NONE of them shared any ban awareness. When
+// Binance returned a 418 (IP ban), every poller kept firing on its own
+// cadence, and since Binance EXTENDS an IP ban on every request received
+// DURING the ban, the system never recovered on its own — a
+// self-perpetuating lockout that zeroed the dashboard and blinded the agent
+// for the whole window (REST dead; only the user-data WS kept working,
+// which is exactly what Binance's error message tells you to use).
+//
+// This module is a SINGLETON (module-level state) deliberately: the ban is
+// per-IP, not per-adapter-instance, so the gate must be shared across every
+// BinancePrivateApi / BinancePublicApi instance in the process. The
+// per-instance BinanceRateLimiter (rate-limiter.ts) tracks weight windows
+// but has no ban model and is not consulted before every call — this gate
+// is the hard stop that complements it.
+//
+// Binance taxonomy (USD-M Futures, General Info docs):
+//   - 429 / -1003 "Too many requests": soft rate-limit violation, no ban yet.
+//   - 418: auto-ban for CONTINUING to send after a 429. Message carries an
+//     explicit `banned until <epoch-ms>` we must respect — guessing a
+//     shorter window just re-bans and extends it.
+//
+// Integration contract (see binance-private.ts / binance-public.ts):
+//   - `assertNotBanned(ctx)` is called as the FIRST line inside each
+//     fetch/order method's existing try-block. While banned it throws a
+//     `BinanceBannedError`, which the method's EXISTING catch absorbs into
+//     its existing contract (return null / [] / rethrow) — so callers see
+//     the same shape they'd see for any failed fetch, but ZERO network
+//     traffic is generated during the ban. That's what lets the ban expire.
+//   - `noteBinanceError(err)` is called in each catch to arm/extend the gate.
+//   - `noteSuccess()` is called after a clean call to relax the soft-429
+//     backoff ladder.
+import { logger } from '../logger.js';
+const TAG = 'binance-ban-gate';
+/** Soft-429 exponential backoff schedule (ms). Advance one step per
+ *  consecutive 429; a clean call resets to step -1 (→ step 0 on next hit).
+ *  Capped so we still probe ~every 2 min — enough to notice the limit
+ *  clearing without climbing back toward a 418. Mirrors the skill-side
+ *  poller circuit-breaker for behavioural consistency. */
+const RATE_LIMIT_BACKOFF_MS = [5_000, 15_000, 45_000, 90_000, 120_000];
+/** Slack added on top of Binance's stated `banned until` before we resume —
+ *  clock skew + margin so the first post-ban call doesn't land on the
+ *  boundary and immediately re-ban. */
+const BAN_RESUME_SLACK_MS = 8_000;
+// ---------------------------------------------------------------------------
+// Proactive weight-budget pacer (prevention half — 2026-05-15).
+//
+// The reactive 418/429 backoff above only engages AFTER Binance has already
+// rate-limited us. The recurring incident driver is the agent heartbeat
+// SCAN burst: scan_pairs across the configured symbol set + per-candidate
+// drill-downs fire dozens of fetchTicker/fetchOrderBook/fetchOHLCV calls within seconds,
+// spiking the 1-min IP weight from baseline to 2000-2400 (Binance cap 2400)
+// and tripping 418 at the window edge — every heartbeat.
+//
+// This is a per-IP, process-wide budget (every BinancePrivateApi /
+// BinancePublicApi call already funnels through assertNotBanned), so the
+// pacer lives here. It self-limits at a SOFT ceiling well below Binance's
+// hard 2400 cap: once the rolling 1-min estimate would cross the ceiling,
+// non-order reads fast-fail (same proven absorbed-by-existing-catch contract
+// as the ban error) until the 1-min window rolls. Result: the burst is
+// shed BEFORE it can reach 418 territory — the ban becomes structurally
+// unreachable in normal operation instead of merely recovered-from.
+//
+// Local estimation (incremented on every allowed call) leads the headers,
+// which lag by in-flight concurrency; noteResponseWeight() reconciles the
+// estimate up to Binance truth when a response header is available.
+// ---------------------------------------------------------------------------
+const WEIGHT_WINDOW_MS = 60_000;
+/** Binance USD-M Futures hard IP cap is 2400 weight / 1-min. We self-limit
+ *  under it with ~400 weight of headroom (absorbs in-flight concurrency +
+ *  header-vs-local lag; reactive 418/429 gate is the hard backstop).
+ *
+ *  2026-05-15 calibration (evidence-based, NOT a guess): after the measured
+ *  efficiency pass — audit no longer sweeps unscoped when flat, algoRefresh
+ *  15s→60s, skill ticker/ohlcv/market-structure cadence cuts, all verified
+ *  via the per-context + skill poll-window instrumentation — the residual
+ *  flat-state IP weight is ~1650-1750/min of IRREDUCIBLE legitimate live-
+ *  trading polling (positions, balance, open-orders, OHLCV, income,
+ *  transfers, the 60s safety reconcilers). 1600 was below that floor, so
+ *  the pacer shed legitimate heartbeat reads indefinitely and blinded the
+ *  agent. Measured Binance x-mbx-used-weight-1m peaks ~1750-1900 << 2400,
+ *  so 2000 stops the false shedding while keeping ~400 real headroom.
+ *  Env-overridable for emergency tuning without a code change. */
+const WEIGHT_SOFT_CEILING = (() => {
+    const raw = Number(process.env.RC_BINANCE_WEIGHT_CEILING);
+    return Number.isFinite(raw) && raw >= 200 && raw <= 2400 ? raw : 2000;
+})();
+// ---------------------------------------------------------------------------
+// Safety-reserve priority tiering (2026-05-16).
+//
+// The pacer above sheds ALL non-order reads uniformly at the soft ceiling.
+// That let the recurring real-money failure persist: a heartbeat SCAN burst
+// (fetchTicker/fetchOHLCV/fetchOrderBook across the symbol set) pins the 1-min IP
+// weight at the ceiling, so the bracket audit's getOpenOrders /
+// queryAlgoOrder / fetchPositions were shed every heartbeat and the agent
+// went blind on live-position protection for HOURS (2026-05-14 ATOM →
+// 2026-05-16 INJ 04:29→05:54 straight). Absolute weight cuts (Option A
+// income poll; 983cef0 reconciler/deposit) lower the floor; this makes the
+// starvation STRUCTURALLY impossible regardless of future polling creep.
+//
+// Mechanism: low-priority cosmetic/market-data contexts (SHED_FIRST) are
+// paced against COSMETIC_SOFT_CEILING; the reserve band
+// [COSMETIC_SOFT_CEILING, WEIGHT_SOFT_CEILING] is reachable ONLY by
+// safety-critical reads. This is strictly additive — nothing is made more
+// lenient than today; only SHED_FIRST contexts shed earlier, so a safety
+// read can never be newly starved by this change.
+// ---------------------------------------------------------------------------
+/** Weight reserved exclusively for safety-critical reads. Env-overridable;
+ *  clamp 0–1000 (0 disables tiering → exact pre-2026-05-16 behaviour). */
+const SAFETY_RESERVE = (() => {
+    const raw = Number(process.env.RC_BINANCE_SAFETY_RESERVE);
+    return Number.isFinite(raw) && raw >= 0 && raw <= 1000 ? raw : 400;
+})();
+/** Effective ceiling for SHED_FIRST contexts. Floored at 200 so a large
+ *  reserve can't drive it to zero and shed cosmetic reads forever. */
+const COSMETIC_SOFT_CEILING = Math.max(200, WEIGHT_SOFT_CEILING - SAFETY_RESERVE);
+/** Approx Binance request weight per call context (the string passed to
+ *  assertNotBanned). Conservative — over-estimating paces slightly early,
+ *  which is the safe direction. Unknown contexts default to 5. */
+const CONTEXT_WEIGHT = {
+    fetchTicker: 2,
+    fetchTickerRaw: 2,
+    fetchOrderBook: 5,
+    fetchOHLCV: 2,
+    fetchFundingRate: 1,
+    fetchOpenInterest: 1,
+    fetchBalance: 5,
+    fetchPositions: 5,
+    // BinancePrivateApi.fetchOpenOrders merges TWO REST calls (regular
+    // /fapi/v1/openOrders + algo /fapi/v1/openAlgoOrders), each 1 scoped /
+    // 40 unscoped (doc-verified developers.binance.com 2026-05-15). The call
+    // site passes the scoped/unscoped context so this is exact, not a guess.
+    fetchOpenOrders: 2, // 1 regular + 1 algo (symbol given)
+    fetchOpenOrdersAll: 80, // 40 regular + 40 algo (symbol omitted)
+    fetchMyTrades: 5,
+    fetchOrder: 1, // GET /fapi/v1/order (also used for by-clientId)
+    // GET /fapi/v1/algoOrder — single conditional-order status by clientAlgoId.
+    // Weight 1, doc-verified developers.binance.com/.../Query-Algo-Order
+    // (2026-05-15). The per-order bracket-spiral disambiguator: a successful
+    // empty /fapi/v1/openAlgoOrders is NOT proof a leg is gone; this asks
+    // about ONE exact order id. Same weight as the scoped snapshot, but it
+    // answers the actual question instead of inferring from absence.
+    queryAlgoOrder: 1,
+    fetchPositionMode: 30, // GET /fapi/v1/positionSide/dual (doc-verified)
+    fetchTransfers: 30,
+    fetchTodayIncomeBreakdown: 30,
+    // Same /fapi/v1/income call (weight 30) but the never-paced variant the
+    // boot + UTC-day-rollover KPI anchor uses. Still counted so the estimate
+    // stays honest; just not shed (see NEVER_PACE doc).
+    fetchTodayIncomeBreakdownForced: 30,
+    fetchRecentTradedSymbols: 30,
+    createListenKey: 1,
+    keepAliveListenKey: 1,
+    closeListenKey: 1,
+};
+/** Contexts that consume weight but must NEVER be blocked by the pacer:
+ *  order/cancel paths (must always reach the exchange — same invariant as
+ *  the ban gate), listenKey lifecycle (a paced-out keep-alive could let the
+ *  user-data WS key expire; weight is trivial = 1), and the FORCED KPI
+ *  income anchor. The latter (`fetchTodayIncomeBreakdownForced`) is the
+ *  /fapi/v1/income call made only at boot and at UTC-day rollover — it
+ *  establishes `sessionStartNav = wallet_at_midnight`, the single value the
+ *  ENTIRE Day-P&L / Realized KPI chain is derived from. It runs at most
+ *  ~twice a day, so pacing it buys ~0 weight relief but, if shed, the
+ *  failed fetch used to corrupt the anchor to wallet_now and zero Day P&L
+ *  (2026-05-16 KPI incident). Routine same-day income refreshes stay
+ *  SHED_FIRST/cosmetic (the anchor is invariant within a UTC day, so a
+ *  paced miss there is a harmless no-op). All NEVER_PACE contexts still
+ *  COUNT toward the window so the estimate stays honest. */
+const NEVER_PACE = new Set([
+    'createOrder', 'cancelOrder', 'cancelAllOrders', 'cancelOrderByClientId',
+    'createBracketOrder', 'createListenKey', 'keepAliveListenKey', 'closeListenKey',
+    'fetchTodayIncomeBreakdownForced',
+]);
+/** Low-priority contexts that shed FIRST under weight pressure (paced
+ *  against COSMETIC_SOFT_CEILING, not the full ceiling). Two safe-to-shed
+ *  groups: dashboard/KPI anchors that explicitly tolerate minute-scale
+ *  staleness (income / transfers / recent-symbols) and public market-data
+ *  scan reads that have a WS alternative and are re-fetched next heartbeat.
+ *
+ *  DELIBERATELY EXCLUDES the bracket/position safety reads —
+ *  fetchOpenOrders, fetchOpenOrdersAll, fetchPositions, queryAlgoOrder,
+ *  fetchOrder, fetchMyTrades, fetchBalance, fetchPositionMode — which keep
+ *  the full WEIGHT_SOFT_CEILING (today's behaviour, never made stricter
+ *  here). fetchBalance/fetchMyTrades stay protected because they feed
+ *  risk/sizing + fill reconciliation, not just display. */
+const SHED_FIRST = new Set([
+    'fetchTodayIncomeBreakdown', 'fetchTransfers', 'fetchRecentTradedSymbols',
+    'fetchTicker', 'fetchTickerRaw', 'fetchOHLCV', 'fetchOrderBook',
+    'fetchFundingRate', 'fetchOpenInterest',
+]);
+let weightWindowStart = 0;
+let weightUsed = 0;
+/** Throttle the "weight pacer engaged" log to once per window. */
+let pacerLogged = false;
+/** Per-context attribution for the current window (observability only —
+ *  NOT consulted by any decision). Lets prod prove WHICH callers drive the
+ *  weight so durable cadence cuts target the real consumer instead of a
+ *  guess. `allowed` = weight let through; `shed` = paced-out call count. */
+const weightByContext = new Map();
+const shedByContext = new Map();
+function rollWeightWindowIfDue(now) {
+    if (now - weightWindowStart >= WEIGHT_WINDOW_MS) {
+        if (weightUsed > 0 || shedByContext.size > 0) {
+            const fmt = (m, n) => [...m.entries()].sort((a, b) => b[1] - a[1]).slice(0, n)
+                .map(([k, v]) => `${k}=${v}`).join(' ');
+            // `weightUsed` (=total) is reconciled UP to Binance's IP-wide
+            // x-mbx-used-weight-1m header (noteResponseWeight); `weightByContext`
+            // only sees calls that passed through THIS gate (plugin reads). The
+            // difference is weight Binance charged this IP that the gate never
+            // saw — the skill process's own REST polls, ccxt-internal calls
+            // (loadMarkets/fetchTime), or anything else sharing the IP/key.
+            // Naming it explicitly is the whole point of the 2026-05-18
+            // "instrument, don't guess" pass: the per-context map summed to
+            // ~275 while total was ~1900 and the ~1600 gap was invisible,
+            // folded silently into `total`. Print ALL contexts (no top-N
+            // truncation) so no real consumer can hide either.
+            const gatedSum = [...weightByContext.values()].reduce((s, v) => s + v, 0);
+            const ungatedIp = Math.max(0, weightUsed - gatedSum);
+            logger.info(TAG, `weight window summary: total=${weightUsed}/${WEIGHT_SOFT_CEILING} ` +
+                `gated=${gatedSum} ungated_ip=${ungatedIp} ` +
+                `allowed[${fmt(weightByContext, 100)}] shed[${fmt(shedByContext, 100)}]`);
+        }
+        weightWindowStart = now;
+        weightUsed = 0;
+        pacerLogged = false;
+        weightByContext.clear();
+        shedByContext.clear();
+    }
+}
+function contextWeight(ctx) {
+    return CONTEXT_WEIGHT[ctx] ?? 5;
+}
+/** Thrown by `assertNotBanned` while the gate is closed. Carries the
+ *  remaining cooldown so callers/telemetry can reason about it. Methods
+ *  catch this in their existing catch block — it is NOT meant to escape
+ *  the API layer with special handling; it degrades to the method's normal
+ *  failure contract. */
+export class BinanceBannedError extends Error {
+    retryAfterMs;
+    constructor(retryAfterMs, context) {
+        super(`Binance rate-limit/IP-ban gate is OPEN (retry in ~${Math.ceil(retryAfterMs / 1000)}s) — skipping ${context} to let the ban expire instead of extending it`);
+        this.retryAfterMs = retryAfterMs;
+        this.name = 'BinanceBannedError';
+    }
+}
+/** Thrown by the proactive weight pacer. Subclass of BinanceBannedError so
+ *  every `instanceof BinanceBannedError` check and every method's existing
+ *  catch keep working unchanged — it degrades to the same null/[]/throw
+ *  contract (treated exactly like a transient fetch failure). The distinct
+ *  name lets journal/telemetry tell prevention apart from a real IP ban. */
+export class BinanceWeightPacedError extends BinanceBannedError {
+    constructor(retryAfterMs, context) {
+        super(retryAfterMs, context);
+        this.name = 'BinanceWeightPacedError';
+        this.message =
+            `Binance weight pacer engaged (1-min window reached the ~${WEIGHT_SOFT_CEILING}/2400 ` +
+                `soft ceiling, retry in ~${Math.ceil(retryAfterMs / 1000)}s) — shedding ${context} ` +
+                `to stay below the 418 threshold instead of bursting into an IP ban`;
+    }
+}
+let banUntilMs = 0;
+let rateLimitStep = -1;
+/** Throttles the "gate open, skipping" log to once per cooldown window so a
+ *  multi-minute ban doesn't emit hundreds of identical lines. */
+let cooldownLogged = false;
+/** Fast pre-flight check. Call as the first line inside a Binance call's
+ *  try-block. Throws `BinanceBannedError` (no network) while the gate is
+ *  open. */
+export function assertNotBanned(context) {
+    const now = Date.now();
+    if (now < banUntilMs) {
+        const remaining = banUntilMs - now;
+        if (!cooldownLogged) {
+            logger.warn(TAG, `gate OPEN — pausing ALL Binance calls for ~${Math.ceil(remaining / 1000)}s (418/429 backoff). First skipped: ${context}`);
+            cooldownLogged = true;
+        }
+        throw new BinanceBannedError(remaining, context);
+    }
+    if (cooldownLogged) {
+        logger.info(TAG, 'gate CLOSED — Binance calls resuming');
+        cooldownLogged = false;
+    }
+    // --- Proactive weight pacing (prevention; runs only when not banned) ---
+    rollWeightWindowIfDue(now);
+    const w = contextWeight(context);
+    if (!NEVER_PACE.has(context)) {
+        // Priority tiering: cosmetic/market-data reads (SHED_FIRST) pace against
+        // the lower COSMETIC_SOFT_CEILING so the reserve band up to the full
+        // ceiling stays exclusively available to safety-critical reads. Anything
+        // not in SHED_FIRST keeps the full ceiling — identical to pre-2026-05-16
+        // behaviour, never more lenient.
+        const isCosmetic = SHED_FIRST.has(context);
+        const effectiveCeiling = isCosmetic ? COSMETIC_SOFT_CEILING : WEIGHT_SOFT_CEILING;
+        if (weightUsed + w > effectiveCeiling) {
+            const retryAfterMs = Math.max(0, weightWindowStart + WEIGHT_WINDOW_MS - now);
+            if (!pacerLogged) {
+                logger.warn(TAG, `weight pacer ENGAGED — 1-min IP weight ~${weightUsed}/${WEIGHT_SOFT_CEILING} ` +
+                    `(cosmetic ceiling ${COSMETIC_SOFT_CEILING}, hard cap 2400); shedding ` +
+                    `${isCosmetic ? 'cosmetic' : 'non-order'} reads ~${Math.ceil(retryAfterMs / 1000)}s until the window rolls. First shed: ${context}`);
+                pacerLogged = true;
+            }
+            shedByContext.set(context, (shedByContext.get(context) ?? 0) + 1);
+            throw new BinanceWeightPacedError(retryAfterMs, context);
+        }
+    }
+    // Allowed (or NEVER_PACE) → the request will go out; count its weight
+    // immediately so the estimate leads (not lags) the in-flight burst.
+    weightUsed += w;
+    weightByContext.set(context, (weightByContext.get(context) ?? 0) + w);
+}
+/** Inspect a failed call's error and arm/extend the gate on a Binance
+ *  rate-limit signal. Non-rate-limit errors are ignored (normal transient
+ *  handling stays in the caller). Safe to call with any error shape. */
+export function noteBinanceError(err) {
+    // A BinanceBannedError means the gate was already open and we short-
+    // circuited — nothing to arm.
+    if (err instanceof BinanceBannedError)
+        return;
+    const msg = err instanceof Error ? err.message : typeof err === 'string' ? err : String(err);
+    // 418 — hard IP ban with an explicit expiry Binance gives us.
+    if (msg.includes('418') || /banned until/i.test(msg)) {
+        const m = /banned until (\d{10,})/i.exec(msg);
+        const stated = m ? Number(m[1]) : 0;
+        const base = Number.isFinite(stated) && stated > Date.now()
+            ? stated
+            : Date.now() + RATE_LIMIT_BACKOFF_MS[RATE_LIMIT_BACKOFF_MS.length - 1];
+        const target = base + BAN_RESUME_SLACK_MS;
+        if (target > banUntilMs) {
+            banUntilMs = target;
+            cooldownLogged = false; // force a fresh "gate OPEN ~Ns" line
+            logger.error(TAG, `Binance 418 IP BAN — gate engaged, ALL Binance calls paused ~${Math.ceil((target - Date.now()) / 1000)}s (Binance-stated expiry +slack)`);
+        }
+        return;
+    }
+    // 429 / -1003 — soft rate-limit, no ban yet. Exponential backoff so we
+    // stop climbing toward a 418.
+    if (msg.includes('429') ||
+        msg.includes('-1003') ||
+        /too many requests/i.test(msg)) {
+        rateLimitStep = Math.min(rateLimitStep + 1, RATE_LIMIT_BACKOFF_MS.length - 1);
+        const window = RATE_LIMIT_BACKOFF_MS[rateLimitStep];
+        const target = Date.now() + window;
+        if (target > banUntilMs) {
+            banUntilMs = target;
+            cooldownLogged = false;
+            logger.warn(TAG, `Binance 429 — gate backoff step ${rateLimitStep}, ALL Binance calls paused ${window / 1000}s`);
+        }
+    }
+}
+/** Relax the soft-429 ladder after a clean call. (A hard 418 window is NOT
+ *  shortened by this — only Binance's stated expiry clears it.) */
+export function noteSuccess() {
+    if (rateLimitStep !== -1)
+        rateLimitStep = -1;
+}
+/** Reconcile the local weight estimate against Binance truth from a response
+ *  header (`x-mbx-used-weight-1m`). Best-effort: callers pass whatever header
+ *  bag they have; absent/garbage values are ignored. The local estimate
+ *  leads (counts in-flight calls Binance hasn't tallied yet), so we take the
+ *  MAX within the window — never let a lagging header lower our guard. A
+ *  header far BELOW the estimate means Binance's minute rolled; re-anchor the
+ *  window to Binance's clock so we don't shed for a stale local total. */
+export function noteResponseWeight(headers) {
+    if (!headers)
+        return;
+    const raw = headers['x-mbx-used-weight-1m'] ?? headers['X-MBX-USED-WEIGHT-1M'];
+    if (raw == null)
+        return;
+    const actual = parseInt(String(raw), 10);
+    if (!Number.isFinite(actual) || actual < 0)
+        return;
+    const now = Date.now();
+    rollWeightWindowIfDue(now);
+    if (actual + 50 < weightUsed) {
+        // Binance reports much less than our local tally → its 1-min counter
+        // reset on the exchange side. Re-anchor to Binance truth.
+        weightWindowStart = now;
+        weightUsed = actual;
+        pacerLogged = false;
+        return;
+    }
+    if (actual > weightUsed)
+        weightUsed = actual;
+}
+/** Test/diagnostic helper — current gate state. */
+export function getBanGateState() {
+    return {
+        banUntilMs,
+        remainingMs: Math.max(0, banUntilMs - Date.now()),
+        rateLimitStep,
+        weightUsed,
+        weightCeiling: WEIGHT_SOFT_CEILING,
+        cosmeticCeiling: COSMETIC_SOFT_CEILING,
+    };
+}
+/** Test-only reset. Not used in production code paths. */
+export function __resetBanGateForTests() {
+    banUntilMs = 0;
+    rateLimitStep = -1;
+    cooldownLogged = false;
+    weightWindowStart = 0;
+    weightUsed = 0;
+    pacerLogged = false;
+}