@tradejs/cli 2.0.13 → 2.0.14

package/dist/scripts/hyperliquidWhaleBackfill.js

@@ -0,0 +1,476 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/scripts/hyperliquidWhaleBackfill.ts
+var hyperliquidWhaleBackfill_exports = {};
+__export(hyperliquidWhaleBackfill_exports, {
+  main: () => main
+});
+module.exports = __toCommonJS(hyperliquidWhaleBackfill_exports);
+var import_args = __toESM(require("args"));
+
+// src/lib/hyperliquidWhaleBackfill.ts
+var import_timescale = require("@tradejs/infra/timescale");
+var import_strategies = require("@tradejs/node/strategies");
+
+// src/lib/hyperliquidWhaleData.ts
+var HYPERLIQUID_WHALE_BUCKET_MS = 6e4;
+var finiteNumber = (value) => {
+  const numeric = typeof value === "number" ? value : Number(value);
+  return Number.isFinite(numeric) ? numeric : null;
+};
+var POSITION_EPSILON = 1e-12;
+var normalizePosition = (value) => Math.abs(value) <= POSITION_EPSILON ? 0 : value;
+var classifyHyperliquidPositionChange = (params) => {
+  const startValue = finiteNumber(params.startPosition);
+  const size = finiteNumber(params.size);
+  const side = String(params.side ?? "").toUpperCase();
+  if (startValue == null || size == null || size <= 0) return null;
+  if (side !== "B" && side !== "A") return null;
+  const startPosition = normalizePosition(startValue);
+  const endPosition = normalizePosition(
+    startPosition + (side === "B" ? size : -size)
+  );
+  const startLong = Math.max(startPosition, 0);
+  const endLong = Math.max(endPosition, 0);
+  const startShort = Math.max(-startPosition, 0);
+  const endShort = Math.max(-endPosition, 0);
+  const longEntrySize = Math.max(0, endLong - startLong);
+  const shortEntrySize = Math.max(0, endShort - startShort);
+  const longExitSize = Math.max(0, startLong - endLong);
+  const shortExitSize = Math.max(0, startShort - endShort);
+  const flipped = startPosition * endPosition < 0;
+  const opened = startPosition === 0;
+  const closed = endPosition === 0;
+  const increased = Math.sign(startPosition) === Math.sign(endPosition) && Math.abs(endPosition) > Math.abs(startPosition);
+  return {
+    startPosition,
+    endPosition,
+    action: flipped ? "flip" : opened ? "open" : closed ? "close" : increased ? "increase" : "reduce",
+    longEntrySize,
+    shortEntrySize,
+    longExitSize,
+    shortExitSize
+  };
+};
+var normalizeAddress = (value) => typeof value === "string" && /^0x[0-9a-fA-F]{40}$/.test(value) ? value.toLowerCase() : null;
+var normalizeTradeFields = (value) => {
+  const symbol = typeof value.coin === "string" ? value.coin.trim() : "";
+  const price = finiteNumber(value.px);
+  const size = finiteNumber(value.sz);
+  const timestamp = finiteNumber(value.time);
+  const tid = typeof value.tid === "string" || typeof value.tid === "number" ? String(value.tid) : "";
+  if (!symbol || price == null || size == null || timestamp == null || !tid || price <= 0 || size <= 0) {
+    return null;
+  }
+  return {
+    symbol,
+    price,
+    size,
+    timestamp,
+    tid,
+    notionalUsd: price * size
+  };
+};
+var normalizeHyperliquidUserFill = (params) => {
+  const normalized = normalizeTradeFields(params.fill);
+  if (!normalized || !params.trackedSymbols.has(normalized.symbol)) return null;
+  const address = normalizeAddress(params.address);
+  if (!address) return null;
+  const side = String(params.fill.side ?? "").toUpperCase();
+  if (side !== "B" && side !== "A") return null;
+  const positionChange = classifyHyperliquidPositionChange({
+    startPosition: params.fill.startPosition,
+    side,
+    size: normalized.size
+  });
+  const closedPnl = finiteNumber(params.fill.closedPnl);
+  const liquidation = params.fill.liquidation != null && typeof params.fill.liquidation === "object";
+  const positionFields = positionChange ? side === "B" ? {
+    buyerStartPosition: positionChange.startPosition,
+    buyerEndPosition: positionChange.endPosition,
+    buyerPositionAction: positionChange.action,
+    buyerClosedPnl: closedPnl,
+    buyerLiquidation: liquidation
+  } : {
+    sellerStartPosition: positionChange.startPosition,
+    sellerEndPosition: positionChange.endPosition,
+    sellerPositionAction: positionChange.action,
+    sellerClosedPnl: closedPnl,
+    sellerLiquidation: liquidation
+  } : {};
+  return {
+    symbol: normalized.symbol,
+    ts: new Date(normalized.timestamp),
+    tid: normalized.tid,
+    price: normalized.price,
+    size: normalized.size,
+    notionalUsd: normalized.notionalUsd,
+    buyerAddress: side === "B" ? address : null,
+    sellerAddress: side === "A" ? address : null,
+    buyerTracked: side === "B",
+    sellerTracked: side === "A",
+    ...positionFields,
+    ...params.identity,
+    source: "hyperliquid_user_fills"
+  };
+};
+
+// src/lib/hyperliquidRateLimiter.ts
+var DEFAULT_WEIGHT_PER_MINUTE = 1e3;
+var WINDOW_MS = 6e4;
+var sleep = (ms) => new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)));
+var HyperliquidInfoRateLimiter = class {
+  constructor(capacity = DEFAULT_WEIGHT_PER_MINUTE, now = Date.now, wait = sleep) {
+    this.capacity = capacity;
+    this.now = now;
+    this.wait = wait;
+    this.reservations = [];
+    this.queue = Promise.resolve();
+  }
+  reserve(weight) {
+    const normalizedWeight = Math.max(1, Math.ceil(weight));
+    const pending = this.queue.then(
+      () => this.reserveInternal(normalizedWeight)
+    );
+    this.queue = pending.catch(() => void 0);
+    return pending;
+  }
+  async reserveInternal(weight) {
+    if (weight > this.capacity) {
+      throw new Error(
+        `Hyperliquid request weight ${weight} exceeds limiter capacity ${this.capacity}`
+      );
+    }
+    for (; ; ) {
+      const nowMs = this.now();
+      this.reservations = this.reservations.filter(
+        (reservation) => reservation.atMs > nowMs - WINDOW_MS
+      );
+      const used = this.reservations.reduce(
+        (sum, reservation) => sum + reservation.weight,
+        0
+      );
+      if (used + weight <= this.capacity) {
+        this.reservations.push({ atMs: nowMs, weight });
+        return;
+      }
+      const oldest = this.reservations[0];
+      await this.wait(Math.max(1, oldest.atMs + WINDOW_MS - nowMs));
+    }
+  }
+};
+var hyperliquidInfoResponseWeight = (rows) => Math.max(0, Math.ceil(Math.max(0, rows) / 20));
+
+// src/lib/hyperliquidWhaleBackfill.ts
+var DEFAULT_BASE_URL = "https://api.hyperliquid.xyz";
+var DEFAULT_CONCURRENCY = 2;
+var DEFAULT_RATE_LIMIT_WEIGHT = 1e3;
+var MAX_PAGE_SIZE = 2e3;
+var MAX_PAGES_PER_WHALE = 5;
+var REQUEST_WEIGHT = 20;
+var sleep2 = (ms) => new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)));
+var positiveInt = (value, fallback) => {
+  const parsed = Number.parseInt(String(value ?? ""), 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+};
+var minuteCeil = (value) => Math.ceil(value / HYPERLIQUID_WHALE_BUCKET_MS) * HYPERLIQUID_WHALE_BUCKET_MS;
+var nextMinute = (value) => (Math.floor(value / HYPERLIQUID_WHALE_BUCKET_MS) + 1) * HYPERLIQUID_WHALE_BUCKET_MS;
+var fetchHyperliquidUserFillsByTime = async (params) => {
+  const fetchImpl = params.fetchImpl ?? fetch;
+  const limiter = params.rateLimiter ?? new HyperliquidInfoRateLimiter(
+    positiveInt(
+      process.env.HYPERLIQUID_WHALE_RATE_LIMIT_WEIGHT,
+      DEFAULT_RATE_LIMIT_WEIGHT
+    ),
+    Date.now,
+    params.wait ?? sleep2
+  );
+  const fills = [];
+  let cursor = params.startTime;
+  for (let page = 0; page < (params.maxPages ?? MAX_PAGES_PER_WHALE) && cursor <= params.endTime; page += 1) {
+    await limiter.reserve(REQUEST_WEIGHT);
+    const response = await fetchImpl(
+      `${process.env.HYPERLIQUID_API_BASE_URL?.trim() || DEFAULT_BASE_URL}/info`,
+      {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: JSON.stringify({
+          type: "userFillsByTime",
+          user: params.address,
+          startTime: cursor,
+          endTime: params.endTime,
+          aggregateByTime: true
+        })
+      }
+    );
+    if (!response.ok) {
+      throw new Error(
+        `Hyperliquid userFillsByTime ${response.status}: ${await response.text()}`
+      );
+    }
+    const raw = await response.json();
+    const pageRows = Array.isArray(raw) ? raw : [];
+    const responseWeight = hyperliquidInfoResponseWeight(pageRows.length);
+    if (responseWeight > 0) await limiter.reserve(responseWeight);
+    fills.push(...pageRows);
+    if (pageRows.length < MAX_PAGE_SIZE) {
+      return { fills, truncated: false, coveredFromMs: params.startTime };
+    }
+    const lastTime = Math.max(
+      ...pageRows.map((row) => Number(row.time)).filter(Number.isFinite)
+    );
+    if (!Number.isFinite(lastTime) || lastTime < cursor) {
+      throw new Error(
+        `Hyperliquid fills pagination stalled for ${params.address}`
+      );
+    }
+    cursor = lastTime + 1;
+  }
+  if (cursor > params.endTime) {
+    return { fills, truncated: false, coveredFromMs: params.startTime };
+  }
+  const firstAvailableTime = Math.min(
+    ...fills.map((fill) => Number(fill.time)).filter(Number.isFinite)
+  );
+  return {
+    fills,
+    truncated: true,
+    coveredFromMs: Math.min(params.endTime + 1, nextMinute(firstAvailableTime))
+  };
+};
+var runConcurrent = async (params) => {
+  let nextIndex = 0;
+  await Promise.all(
+    Array.from(
+      { length: Math.min(params.concurrency, params.items.length) },
+      async () => {
+        for (; ; ) {
+          const index = nextIndex;
+          nextIndex += 1;
+          if (index >= params.items.length) return;
+          await params.worker(params.items[index], index);
+        }
+      }
+    )
+  );
+};
+var backfillHyperliquidWhaleContext = async (params) => {
+  const universe = (0, import_strategies.getHyperliquidPerpUniverseSnapshot)();
+  const whales = (0, import_strategies.getHyperliquidWhaleRegistrySnapshot)();
+  const trackedSymbols = new Set(universe.symbols);
+  const identity = {
+    universeFingerprint: universe.fingerprint,
+    whaleRegistryFingerprint: whales.fingerprint
+  };
+  const fromMs = Math.floor(params.startMs / HYPERLIQUID_WHALE_BUCKET_MS) * HYPERLIQUID_WHALE_BUCKET_MS;
+  const toMs = minuteCeil(params.endMs);
+  const cached = await (0, import_timescale.hasHyperliquidWhaleBackfillCoverage)({
+    fromMs,
+    toMs,
+    ...identity
+  });
+  const emptyResult = () => ({
+    cached,
+    cacheOnly: params.cacheOnly,
+    fills: 0,
+    events: 0,
+    buckets: 0,
+    coverageBuckets: 0,
+    complete: cached,
+    walletsProcessed: 0,
+    walletsCached: 0,
+    completeWallets: 0,
+    truncatedWallets: 0,
+    failedWallets: 0,
+    fromMs,
+    toMs
+  });
+  if (cached || params.cacheOnly) return emptyResult();
+  const limiter = new HyperliquidInfoRateLimiter(
+    positiveInt(
+      process.env.HYPERLIQUID_WHALE_RATE_LIMIT_WEIGHT,
+      DEFAULT_RATE_LIMIT_WEIGHT
+    ),
+    Date.now,
+    params.wait ?? sleep2
+  );
+  const concurrency = Math.min(
+    4,
+    positiveInt(
+      params.concurrency ?? process.env.HYPERLIQUID_WHALE_CONCURRENCY,
+      DEFAULT_CONCURRENCY
+    )
+  );
+  let fills = 0;
+  let events = 0;
+  let walletsProcessed = 0;
+  let walletsCached = 0;
+  let completeWallets = 0;
+  let truncatedWallets = 0;
+  let failedWallets = 0;
+  const failures = [];
+  await runConcurrent({
+    items: whales.addresses,
+    concurrency,
+    worker: async (address) => {
+      const previous = await (0, import_timescale.getHyperliquidWhaleWalletCoverage)({
+        address,
+        fromMs,
+        toMs,
+        ...identity
+      });
+      if (previous && previous.status !== "failed") {
+        walletsCached += 1;
+        if (previous.status === "complete") completeWallets += 1;
+        else truncatedWallets += 1;
+      } else {
+        try {
+          const result2 = await fetchHyperliquidUserFillsByTime({
+            address,
+            startTime: fromMs,
+            endTime: toMs - 1,
+            fetchImpl: params.fetchImpl,
+            rateLimiter: limiter
+          });
+          const addressEvents = result2.fills.map(
+            (fill) => normalizeHyperliquidUserFill({
+              fill,
+              address,
+              trackedSymbols,
+              identity
+            })
+          ).filter((row) => row != null);
+          await (0, import_timescale.upsertHyperliquidWhaleTradeEvents)(addressEvents);
+          await (0, import_timescale.upsertHyperliquidWhaleWalletCoverage)({
+            address,
+            fromMs,
+            toMs,
+            coveredFromMs: result2.coveredFromMs,
+            coveredToMs: toMs,
+            status: result2.truncated ? "truncated" : "complete",
+            fillsCount: result2.fills.length,
+            ...identity
+          });
+          fills += result2.fills.length;
+          events += addressEvents.length;
+          if (result2.truncated) truncatedWallets += 1;
+          else completeWallets += 1;
+        } catch (error) {
+          const normalizedError = error instanceof Error ? error : new Error(String(error));
+          failures.push(normalizedError);
+          failedWallets += 1;
+          await (0, import_timescale.upsertHyperliquidWhaleWalletCoverage)({
+            address,
+            fromMs,
+            toMs,
+            coveredFromMs: null,
+            coveredToMs: null,
+            status: "failed",
+            fillsCount: 0,
+            error: normalizedError.message,
+            ...identity
+          });
+        }
+      }
+      walletsProcessed += 1;
+      params.log?.(
+        `Hyperliquid whales ${walletsProcessed}/${whales.addresses.length}: complete=${completeWallets}, truncated=${truncatedWallets}, failed=${failedWallets}, fills=${fills}, events=${events}`
+      );
+    }
+  });
+  const buckets = await (0, import_timescale.rebuildHyperliquidWhaleFlowRows)({
+    fromMs,
+    toMs,
+    ...identity
+  });
+  const coverageBuckets = await (0, import_timescale.rebuildHyperliquidWhaleCoverageRows)({
+    fromMs,
+    toMs,
+    expectedWhales: whales.addresses.length,
+    ...identity
+  });
+  const result = {
+    cached: false,
+    cacheOnly: false,
+    fills,
+    events,
+    buckets,
+    coverageBuckets,
+    complete: truncatedWallets === 0 && failedWallets === 0,
+    walletsProcessed,
+    walletsCached,
+    completeWallets,
+    truncatedWallets,
+    failedWallets,
+    fromMs,
+    toMs
+  };
+  if (params.strict !== false && failures.length) {
+    throw new AggregateError(
+      failures,
+      `Hyperliquid whale backfill failed for ${failures.length}/${whales.addresses.length} wallets`
+    );
+  }
+  return result;
+};
+
+// src/scripts/hyperliquidWhaleBackfill.ts
+import_args.default.option(["d", "days"], "Historical lookback in days", 1);
+import_args.default.option(["f", "from"], "Inclusive ISO timestamp");
+import_args.default.option(["t", "to"], "Exclusive ISO timestamp");
+import_args.default.option(
+  ["c", "cacheOnly"],
+  "Do not call Hyperliquid when coverage is missing"
+);
+var flags = import_args.default.parse(process.argv);
+var parseTime = (value, fallback) => {
+  if (value == null || value === "") return fallback;
+  const parsed = Date.parse(String(value));
+  if (!Number.isFinite(parsed)) throw new Error(`Invalid timestamp: ${value}`);
+  return parsed;
+};
+var main = async () => {
+  const toMs = parseTime(flags.to, Date.now());
+  const days = Number(flags.days);
+  const lookbackMs = (Number.isFinite(days) && days > 0 ? days : 1) * 864e5;
+  const startMs = parseTime(flags.from, toMs - lookbackMs);
+  if (startMs >= toMs) throw new Error("--from must be before --to");
+  const result = await backfillHyperliquidWhaleContext({
+    startMs,
+    endMs: toMs,
+    cacheOnly: Boolean(flags.cacheOnly),
+    log: console.log
+  });
+  console.log(JSON.stringify(result, null, 2));
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  main
+});