@tradejs/cli 2.0.13 → 2.0.14

package/dist/scripts/backtest.js

@@ -4539,6 +4539,412 @@ var backfillCoinMarketCapContextForBacktest = backfillCoinMarketCapContext;
 var backfillCoinMarketCapContextForReplay = backfillCoinMarketCapContext;
 var backfillCoinMarketCapContextForSignals = (params) => backfillCoinMarketCapContext({ ...params, refreshStaleFearGreed: true });
 
+// src/lib/hyperliquidWhaleBackfill.ts
+var import_timescale4 = require("@tradejs/infra/timescale");
+var import_strategies2 = require("@tradejs/node/strategies");
+
+// src/lib/hyperliquidWhaleData.ts
+var HYPERLIQUID_WHALE_BUCKET_MS = 6e4;
+var finiteNumber = (value) => {
+  const numeric = typeof value === "number" ? value : Number(value);
+  return Number.isFinite(numeric) ? numeric : null;
+};
+var POSITION_EPSILON = 1e-12;
+var normalizePosition = (value) => Math.abs(value) <= POSITION_EPSILON ? 0 : value;
+var classifyHyperliquidPositionChange = (params) => {
+  const startValue = finiteNumber(params.startPosition);
+  const size = finiteNumber(params.size);
+  const side = String(params.side ?? "").toUpperCase();
+  if (startValue == null || size == null || size <= 0) return null;
+  if (side !== "B" && side !== "A") return null;
+  const startPosition = normalizePosition(startValue);
+  const endPosition = normalizePosition(
+    startPosition + (side === "B" ? size : -size)
+  );
+  const startLong = Math.max(startPosition, 0);
+  const endLong = Math.max(endPosition, 0);
+  const startShort = Math.max(-startPosition, 0);
+  const endShort = Math.max(-endPosition, 0);
+  const longEntrySize = Math.max(0, endLong - startLong);
+  const shortEntrySize = Math.max(0, endShort - startShort);
+  const longExitSize = Math.max(0, startLong - endLong);
+  const shortExitSize = Math.max(0, startShort - endShort);
+  const flipped = startPosition * endPosition < 0;
+  const opened = startPosition === 0;
+  const closed = endPosition === 0;
+  const increased = Math.sign(startPosition) === Math.sign(endPosition) && Math.abs(endPosition) > Math.abs(startPosition);
+  return {
+    startPosition,
+    endPosition,
+    action: flipped ? "flip" : opened ? "open" : closed ? "close" : increased ? "increase" : "reduce",
+    longEntrySize,
+    shortEntrySize,
+    longExitSize,
+    shortExitSize
+  };
+};
+var normalizeAddress = (value) => typeof value === "string" && /^0x[0-9a-fA-F]{40}$/.test(value) ? value.toLowerCase() : null;
+var normalizeTradeFields = (value) => {
+  const symbol = typeof value.coin === "string" ? value.coin.trim() : "";
+  const price = finiteNumber(value.px);
+  const size = finiteNumber(value.sz);
+  const timestamp = finiteNumber(value.time);
+  const tid = typeof value.tid === "string" || typeof value.tid === "number" ? String(value.tid) : "";
+  if (!symbol || price == null || size == null || timestamp == null || !tid || price <= 0 || size <= 0) {
+    return null;
+  }
+  return {
+    symbol,
+    price,
+    size,
+    timestamp,
+    tid,
+    notionalUsd: price * size
+  };
+};
+var normalizeHyperliquidUserFill = (params) => {
+  const normalized = normalizeTradeFields(params.fill);
+  if (!normalized || !params.trackedSymbols.has(normalized.symbol)) return null;
+  const address = normalizeAddress(params.address);
+  if (!address) return null;
+  const side = String(params.fill.side ?? "").toUpperCase();
+  if (side !== "B" && side !== "A") return null;
+  const positionChange = classifyHyperliquidPositionChange({
+    startPosition: params.fill.startPosition,
+    side,
+    size: normalized.size
+  });
+  const closedPnl = finiteNumber(params.fill.closedPnl);
+  const liquidation = params.fill.liquidation != null && typeof params.fill.liquidation === "object";
+  const positionFields = positionChange ? side === "B" ? {
+    buyerStartPosition: positionChange.startPosition,
+    buyerEndPosition: positionChange.endPosition,
+    buyerPositionAction: positionChange.action,
+    buyerClosedPnl: closedPnl,
+    buyerLiquidation: liquidation
+  } : {
+    sellerStartPosition: positionChange.startPosition,
+    sellerEndPosition: positionChange.endPosition,
+    sellerPositionAction: positionChange.action,
+    sellerClosedPnl: closedPnl,
+    sellerLiquidation: liquidation
+  } : {};
+  return {
+    symbol: normalized.symbol,
+    ts: new Date(normalized.timestamp),
+    tid: normalized.tid,
+    price: normalized.price,
+    size: normalized.size,
+    notionalUsd: normalized.notionalUsd,
+    buyerAddress: side === "B" ? address : null,
+    sellerAddress: side === "A" ? address : null,
+    buyerTracked: side === "B",
+    sellerTracked: side === "A",
+    ...positionFields,
+    ...params.identity,
+    source: "hyperliquid_user_fills"
+  };
+};
+
+// src/lib/hyperliquidRateLimiter.ts
+var DEFAULT_WEIGHT_PER_MINUTE = 1e3;
+var WINDOW_MS = 6e4;
+var sleep2 = (ms) => new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)));
+var HyperliquidInfoRateLimiter = class {
+  constructor(capacity = DEFAULT_WEIGHT_PER_MINUTE, now = Date.now, wait = sleep2) {
+    this.capacity = capacity;
+    this.now = now;
+    this.wait = wait;
+    this.reservations = [];
+    this.queue = Promise.resolve();
+  }
+  reserve(weight) {
+    const normalizedWeight = Math.max(1, Math.ceil(weight));
+    const pending = this.queue.then(
+      () => this.reserveInternal(normalizedWeight)
+    );
+    this.queue = pending.catch(() => void 0);
+    return pending;
+  }
+  async reserveInternal(weight) {
+    if (weight > this.capacity) {
+      throw new Error(
+        `Hyperliquid request weight ${weight} exceeds limiter capacity ${this.capacity}`
+      );
+    }
+    for (; ; ) {
+      const nowMs = this.now();
+      this.reservations = this.reservations.filter(
+        (reservation) => reservation.atMs > nowMs - WINDOW_MS
+      );
+      const used = this.reservations.reduce(
+        (sum, reservation) => sum + reservation.weight,
+        0
+      );
+      if (used + weight <= this.capacity) {
+        this.reservations.push({ atMs: nowMs, weight });
+        return;
+      }
+      const oldest = this.reservations[0];
+      await this.wait(Math.max(1, oldest.atMs + WINDOW_MS - nowMs));
+    }
+  }
+};
+var hyperliquidInfoResponseWeight = (rows) => Math.max(0, Math.ceil(Math.max(0, rows) / 20));
+
+// src/lib/hyperliquidWhaleBackfill.ts
+var DEFAULT_BASE_URL = "https://api.hyperliquid.xyz";
+var DEFAULT_CONCURRENCY = 2;
+var DEFAULT_RATE_LIMIT_WEIGHT = 1e3;
+var MAX_PAGE_SIZE = 2e3;
+var MAX_PAGES_PER_WHALE = 5;
+var REQUEST_WEIGHT = 20;
+var sleep3 = (ms) => new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)));
+var positiveInt = (value, fallback) => {
+  const parsed = Number.parseInt(String(value ?? ""), 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+};
+var minuteCeil = (value) => Math.ceil(value / HYPERLIQUID_WHALE_BUCKET_MS) * HYPERLIQUID_WHALE_BUCKET_MS;
+var nextMinute = (value) => (Math.floor(value / HYPERLIQUID_WHALE_BUCKET_MS) + 1) * HYPERLIQUID_WHALE_BUCKET_MS;
+var fetchHyperliquidUserFillsByTime = async (params) => {
+  const fetchImpl = params.fetchImpl ?? fetch;
+  const limiter = params.rateLimiter ?? new HyperliquidInfoRateLimiter(
+    positiveInt(
+      process.env.HYPERLIQUID_WHALE_RATE_LIMIT_WEIGHT,
+      DEFAULT_RATE_LIMIT_WEIGHT
+    ),
+    Date.now,
+    params.wait ?? sleep3
+  );
+  const fills = [];
+  let cursor = params.startTime;
+  for (let page = 0; page < (params.maxPages ?? MAX_PAGES_PER_WHALE) && cursor <= params.endTime; page += 1) {
+    await limiter.reserve(REQUEST_WEIGHT);
+    const response = await fetchImpl(
+      `${process.env.HYPERLIQUID_API_BASE_URL?.trim() || DEFAULT_BASE_URL}/info`,
+      {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: JSON.stringify({
+          type: "userFillsByTime",
+          user: params.address,
+          startTime: cursor,
+          endTime: params.endTime,
+          aggregateByTime: true
+        })
+      }
+    );
+    if (!response.ok) {
+      throw new Error(
+        `Hyperliquid userFillsByTime ${response.status}: ${await response.text()}`
+      );
+    }
+    const raw = await response.json();
+    const pageRows = Array.isArray(raw) ? raw : [];
+    const responseWeight = hyperliquidInfoResponseWeight(pageRows.length);
+    if (responseWeight > 0) await limiter.reserve(responseWeight);
+    fills.push(...pageRows);
+    if (pageRows.length < MAX_PAGE_SIZE) {
+      return { fills, truncated: false, coveredFromMs: params.startTime };
+    }
+    const lastTime = Math.max(
+      ...pageRows.map((row) => Number(row.time)).filter(Number.isFinite)
+    );
+    if (!Number.isFinite(lastTime) || lastTime < cursor) {
+      throw new Error(
+        `Hyperliquid fills pagination stalled for ${params.address}`
+      );
+    }
+    cursor = lastTime + 1;
+  }
+  if (cursor > params.endTime) {
+    return { fills, truncated: false, coveredFromMs: params.startTime };
+  }
+  const firstAvailableTime = Math.min(
+    ...fills.map((fill) => Number(fill.time)).filter(Number.isFinite)
+  );
+  return {
+    fills,
+    truncated: true,
+    coveredFromMs: Math.min(params.endTime + 1, nextMinute(firstAvailableTime))
+  };
+};
+var runConcurrent = async (params) => {
+  let nextIndex = 0;
+  await Promise.all(
+    Array.from(
+      { length: Math.min(params.concurrency, params.items.length) },
+      async () => {
+        for (; ; ) {
+          const index = nextIndex;
+          nextIndex += 1;
+          if (index >= params.items.length) return;
+          await params.worker(params.items[index], index);
+        }
+      }
+    )
+  );
+};
+var backfillHyperliquidWhaleContext = async (params) => {
+  const universe = (0, import_strategies2.getHyperliquidPerpUniverseSnapshot)();
+  const whales = (0, import_strategies2.getHyperliquidWhaleRegistrySnapshot)();
+  const trackedSymbols = new Set(universe.symbols);
+  const identity = {
+    universeFingerprint: universe.fingerprint,
+    whaleRegistryFingerprint: whales.fingerprint
+  };
+  const fromMs = Math.floor(params.startMs / HYPERLIQUID_WHALE_BUCKET_MS) * HYPERLIQUID_WHALE_BUCKET_MS;
+  const toMs = minuteCeil(params.endMs);
+  const cached = await (0, import_timescale4.hasHyperliquidWhaleBackfillCoverage)({
+    fromMs,
+    toMs,
+    ...identity
+  });
+  const emptyResult = () => ({
+    cached,
+    cacheOnly: params.cacheOnly,
+    fills: 0,
+    events: 0,
+    buckets: 0,
+    coverageBuckets: 0,
+    complete: cached,
+    walletsProcessed: 0,
+    walletsCached: 0,
+    completeWallets: 0,
+    truncatedWallets: 0,
+    failedWallets: 0,
+    fromMs,
+    toMs
+  });
+  if (cached || params.cacheOnly) return emptyResult();
+  const limiter = new HyperliquidInfoRateLimiter(
+    positiveInt(
+      process.env.HYPERLIQUID_WHALE_RATE_LIMIT_WEIGHT,
+      DEFAULT_RATE_LIMIT_WEIGHT
+    ),
+    Date.now,
+    params.wait ?? sleep3
+  );
+  const concurrency = Math.min(
+    4,
+    positiveInt(
+      params.concurrency ?? process.env.HYPERLIQUID_WHALE_CONCURRENCY,
+      DEFAULT_CONCURRENCY
+    )
+  );
+  let fills = 0;
+  let events = 0;
+  let walletsProcessed = 0;
+  let walletsCached = 0;
+  let completeWallets = 0;
+  let truncatedWallets = 0;
+  let failedWallets = 0;
+  const failures = [];
+  await runConcurrent({
+    items: whales.addresses,
+    concurrency,
+    worker: async (address) => {
+      const previous = await (0, import_timescale4.getHyperliquidWhaleWalletCoverage)({
+        address,
+        fromMs,
+        toMs,
+        ...identity
+      });
+      if (previous && previous.status !== "failed") {
+        walletsCached += 1;
+        if (previous.status === "complete") completeWallets += 1;
+        else truncatedWallets += 1;
+      } else {
+        try {
+          const result2 = await fetchHyperliquidUserFillsByTime({
+            address,
+            startTime: fromMs,
+            endTime: toMs - 1,
+            fetchImpl: params.fetchImpl,
+            rateLimiter: limiter
+          });
+          const addressEvents = result2.fills.map(
+            (fill) => normalizeHyperliquidUserFill({
+              fill,
+              address,
+              trackedSymbols,
+              identity
+            })
+          ).filter((row) => row != null);
+          await (0, import_timescale4.upsertHyperliquidWhaleTradeEvents)(addressEvents);
+          await (0, import_timescale4.upsertHyperliquidWhaleWalletCoverage)({
+            address,
+            fromMs,
+            toMs,
+            coveredFromMs: result2.coveredFromMs,
+            coveredToMs: toMs,
+            status: result2.truncated ? "truncated" : "complete",
+            fillsCount: result2.fills.length,
+            ...identity
+          });
+          fills += result2.fills.length;
+          events += addressEvents.length;
+          if (result2.truncated) truncatedWallets += 1;
+          else completeWallets += 1;
+        } catch (error) {
+          const normalizedError = error instanceof Error ? error : new Error(String(error));
+          failures.push(normalizedError);
+          failedWallets += 1;
+          await (0, import_timescale4.upsertHyperliquidWhaleWalletCoverage)({
+            address,
+            fromMs,
+            toMs,
+            coveredFromMs: null,
+            coveredToMs: null,
+            status: "failed",
+            fillsCount: 0,
+            error: normalizedError.message,
+            ...identity
+          });
+        }
+      }
+      walletsProcessed += 1;
+      params.log?.(
+        `Hyperliquid whales ${walletsProcessed}/${whales.addresses.length}: complete=${completeWallets}, truncated=${truncatedWallets}, failed=${failedWallets}, fills=${fills}, events=${events}`
+      );
+    }
+  });
+  const buckets = await (0, import_timescale4.rebuildHyperliquidWhaleFlowRows)({
+    fromMs,
+    toMs,
+    ...identity
+  });
+  const coverageBuckets = await (0, import_timescale4.rebuildHyperliquidWhaleCoverageRows)({
+    fromMs,
+    toMs,
+    expectedWhales: whales.addresses.length,
+    ...identity
+  });
+  const result = {
+    cached: false,
+    cacheOnly: false,
+    fills,
+    events,
+    buckets,
+    coverageBuckets,
+    complete: truncatedWallets === 0 && failedWallets === 0,
+    walletsProcessed,
+    walletsCached,
+    completeWallets,
+    truncatedWallets,
+    failedWallets,
+    fromMs,
+    toMs
+  };
+  if (params.strict !== false && failures.length) {
+    throw new AggregateError(
+      failures,
+      `Hyperliquid whale backfill failed for ${failures.length}/${whales.addresses.length} wallets`
+    );
+  }
+  return result;
+};
+
 // src/lib/marketContextPrepare.ts
 var shouldPrepareDerivativesContextForRun = (params) => {
   if (params.universe === "tradfi") return false;
@@ -4589,6 +4995,20 @@ var shouldPrepareCoinMarketCapContextForRun = (params) => {
     cacheOnly: params.cacheOnly
   });
 };
+var shouldPrepareHyperliquidWhaleContextForRun = (params) => {
+  const strategyRequiresContext = params.strategyNames?.includes(
+    "HyperliquidConsensus"
+  );
+  if (params.universe === "tradfi" || params.mode === "signals" || params.cacheOnly || !params.aiEnabled && !params.mlEnabled && !strategyRequiresContext) {
+    return false;
+  }
+  const configured = String(
+    process.env.HYPERLIQUID_WHALE_BACKFILL_ENABLED ?? ""
+  ).trim().toLowerCase();
+  if (["0", "false", "no", "off"].includes(configured)) return false;
+  if (strategyRequiresContext) return true;
+  return ["1", "true", "yes", "on"].includes(configured);
+};
 var resolveCoinMarketCapBackfillForMode = (mode) => mode === "backtest" ? backfillCoinMarketCapContextForBacktest : mode === "signals" ? backfillCoinMarketCapContextForSignals : backfillCoinMarketCapContextForReplay;
 var buildCoinMarketCapBackfillParams = (params) => ({
   userName: params.userName,
@@ -4625,6 +5045,18 @@ var prepareMarketContextForRun = async (params) => {
       )
     );
   }
+  if (shouldPrepareHyperliquidWhaleContextForRun(params)) {
+    await timeOperation3(
+      "hyperliquid whale context backfill",
+      () => backfillHyperliquidWhaleContext({
+        startMs: params.preloadStartMs ?? params.startMs,
+        endMs: params.endMs,
+        cacheOnly: params.cacheOnly,
+        strict: false,
+        log
+      })
+    );
+  }
   if (shouldPrepareBinanceMarketContextForRun(params)) {
     await timeOperation3("binance market context backfill", () => {
       const backfill = params.mode === "backtest" ? backfillBinanceMarketContextForBacktest : params.mode === "signals" ? backfillBinanceMarketContextForSignals : backfillBinanceMarketContextForReplay;
@@ -6209,6 +6641,9 @@ var buildPreparedTestSuite = async ({
     cacheOnly: Boolean(flags.cacheOnly),
     aiEnabled,
     mlEnabled,
+    strategyNames: Array.from(
+      new Set(preparedSuite.map((test) => test.strategyName))
+    ),
     log: (message) => console.log(import_chalk9.default.gray(message))
   });
   return preparedSuite;
@@ -6727,6 +7162,9 @@ var prepareContinuedBacktestMarketContext = async ({
     cacheOnly: manifest.flags.cacheOnly,
     aiEnabled: manifest.flags.ai,
     mlEnabled: manifest.flags.ml,
+    strategyNames: Array.from(
+      new Set(remainingSuite.map((test) => test.strategyName))
+    ),
     log: (message) => console.log(import_chalk10.default.gray(message))
   });
 };