@tradejs/cli 1.0.0

package/dist/scripts/derivativesIngestCoinalyzeAll.js

@@ -0,0 +1,391 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+
+// src/scripts/derivativesIngestCoinalyzeAll.ts
+var import_config = require("dotenv/config");
+var import_args = __toESM(require("args"));
+var import_chalk = __toESM(require("chalk"));
+var import_progress = __toESM(require("progress"));
+var import_lodash = __toESM(require("lodash"));
+var import_connectors = require("@tradejs/connectors");
+var import_indicators = require("@tradejs/core/indicators");
+var import_async = require("@tradejs/core/async");
+var import_cli = require("@tradejs/node/cli");
+var import_timescale = require("@tradejs/infra/timescale");
+var coinalyzeIntervalMap = {
+  "15m": "15min",
+  "1h": "1hour"
+};
+import_args.default.example(
+  "yarn ts-node ./src/scripts/derivativesIngestCoinalyzeAll --days 120 --intervals 15m,1h",
+  "Fetch derivatives for all getTickers symbols matched to Coinalyze markets"
+);
+import_args.default.option(["U", "user"], "Bybit user config name from Redis", "root");
+import_args.default.option(["t", "tickers"], "Comma-separated include symbols");
+import_args.default.option(["e", "exclude"], "Comma-separated exclude symbols");
+import_args.default.option(["l", "tickersLimit"], "Tickers limit");
+import_args.default.option(["c", "chunk"], "Chunk selector, e.g. 1/4");
+import_args.default.option(["i", "intervals"], "Intervals: 15m,1h", "15m,1h");
+import_args.default.option(["d", "days"], "Lookback in days", 120);
+import_args.default.option(["b", "batchDays"], "Request chunk size in days", 120);
+import_args.default.option(
+  ["E", "exchangePriority"],
+  "Coinalyze exchange priority, comma-separated",
+  "A,6,0"
+);
+import_args.default.option(
+  ["S", "symbolBatchSize"],
+  "How many symbols to request in one Coinalyze call",
+  8
+);
+import_args.default.option(
+  ["w", "requestDelayMs"],
+  "Delay between API requests to smooth rate limit",
+  200
+);
+import_args.default.option(
+  ["T", "requestTimeoutMs"],
+  "HTTP request timeout in milliseconds",
+  45e3
+);
+import_args.default.option(["L", "showTickersList"], "Print matched symbols only", false);
+var flags = import_args.default.parse(process.argv);
+var asInt = (value, fallback) => {
+  const parsed = Number.parseInt(String(value ?? ""), 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+};
+var parseList = (value) => String(value ?? "").split(",").map((item) => item.trim().toUpperCase()).filter(Boolean);
+var apiKey = process.env.COINALYZE_API_KEY?.trim();
+if (!apiKey) {
+  throw new Error("Missing COINALYZE_API_KEY");
+}
+var coinalyzeBaseUrl = process.env.COINALYZE_BASE_URL?.trim() || "https://api.coinalyze.net/v1";
+var coinalyzeMaxRetries = asInt(process.env.COINALYZE_MAX_RETRIES, 4);
+var lastRequestTs = 0;
+var fetchJsonWithRateLimit = async (url) => {
+  const requestDelayMs = Math.max(100, asInt(flags.requestDelayMs, 200));
+  const requestTimeoutMs = Math.max(
+    5e3,
+    asInt(flags.requestTimeoutMs, 45e3)
+  );
+  for (let attempt = 0; attempt <= coinalyzeMaxRetries; attempt += 1) {
+    const waitMs = Math.max(0, lastRequestTs + requestDelayMs - Date.now());
+    if (waitMs > 0) {
+      await (0, import_async.delay)(waitMs);
+    }
+    lastRequestTs = Date.now();
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), requestTimeoutMs);
+    let response;
+    try {
+      response = await fetch(url, {
+        headers: {
+          api_key: apiKey,
+          "x-api-key": apiKey,
+          Authorization: `Bearer ${apiKey}`
+        },
+        signal: controller.signal
+      });
+    } catch (error) {
+      clearTimeout(timer);
+      const abortError = error instanceof Error && (error.name === "AbortError" || String(error.message).toLowerCase().includes("aborted"));
+      if (attempt < coinalyzeMaxRetries && abortError) {
+        const backoffMs = Math.min(12e3, 800 * 2 ** attempt);
+        await (0, import_async.delay)(backoffMs);
+        continue;
+      }
+      throw error;
+    } finally {
+      clearTimeout(timer);
+    }
+    if (response.ok) {
+      return response.json();
+    }
+    const text = await response.text();
+    const retryAfterRaw = Number(response.headers.get("retry-after") ?? "");
+    const retryAfterMs = Number.isFinite(retryAfterRaw) && retryAfterRaw > 0 ? retryAfterRaw * 1e3 : null;
+    const transient = response.status === 429 || response.status >= 500;
+    if (attempt < coinalyzeMaxRetries && transient) {
+      const backoffMs = Math.min(12e3, 800 * 2 ** attempt);
+      await (0, import_async.delay)(retryAfterMs ?? backoffMs);
+      continue;
+    }
+    throw new Error(`Coinalyze ${response.status}: ${text}`);
+  }
+  throw new Error("Coinalyze request failed after retries");
+};
+var fetchCoinalyzeMarkets = async () => {
+  const raw = await fetchJsonWithRateLimit(
+    `${coinalyzeBaseUrl}/future-markets`
+  );
+  return Array.isArray(raw) ? raw : [];
+};
+var selectBestMarket = (candidates, exchangePriority) => {
+  const scored = candidates.map((item) => {
+    let score = 0;
+    if (item.is_perpetual) score += 50;
+    if (String(item.quote_asset || "").toUpperCase() === "USDT") score += 25;
+    if (item.symbol.includes("_PERP")) score += 10;
+    const exchange = String(item.exchange || "").toUpperCase();
+    const idx = exchangePriority.indexOf(exchange);
+    if (idx >= 0) {
+      score += (exchangePriority.length - idx) * 10;
+    }
+    return { item, score };
+  });
+  scored.sort((a, b) => {
+    if (b.score !== a.score) return b.score - a.score;
+    return a.item.symbol.localeCompare(b.item.symbol);
+  });
+  return scored[0]?.item ?? null;
+};
+var buildMatches = (tickers, markets, exchangePriority) => {
+  const marketByTicker = /* @__PURE__ */ new Map();
+  for (const market of markets) {
+    const key = String(market.symbol_on_exchange || "").trim().toUpperCase();
+    if (!key) continue;
+    const list = marketByTicker.get(key) ?? [];
+    list.push(market);
+    marketByTicker.set(key, list);
+  }
+  const matches = [];
+  const unmatched = [];
+  for (const ticker of tickers) {
+    const candidates = marketByTicker.get(ticker) ?? [];
+    const selected = selectBestMarket(candidates, exchangePriority);
+    if (!selected) {
+      unmatched.push(ticker);
+      continue;
+    }
+    matches.push({
+      symbol: ticker,
+      marketSymbol: selected.symbol,
+      exchange: selected.exchange
+    });
+  }
+  return { matches, unmatched };
+};
+var normalizeMetricPoint = (metric, point) => {
+  if (metric === "oi") {
+    return {
+      ...point,
+      open_interest: point.open_interest ?? point.openInterest ?? point.oi ?? point.c
+    };
+  }
+  if (metric === "funding") {
+    return {
+      ...point,
+      funding_rate: point.funding_rate ?? point.fundingRate ?? point.rate ?? point.c
+    };
+  }
+  return {
+    ...point,
+    liq_long: point.liq_long ?? point.long_liq ?? point.long ?? point.l,
+    liq_short: point.liq_short ?? point.short_liq ?? point.short ?? point.s
+  };
+};
+var toSeriesMap = (raw, metric) => {
+  const out = /* @__PURE__ */ new Map();
+  if (!Array.isArray(raw)) return out;
+  for (const item of raw) {
+    const symbol = String(item.symbol || "").trim().toUpperCase();
+    const history = Array.isArray(item.history) ? item.history : [];
+    if (!symbol) continue;
+    out.set(
+      symbol,
+      history.filter((point) => point && typeof point === "object").map((point) => normalizeMetricPoint(metric, point))
+    );
+  }
+  return out;
+};
+var fetchMetricBatch = async (params) => {
+  const { endpoint, metric, marketSymbols, interval, fromMs, toMs } = params;
+  const url = new URL(`${coinalyzeBaseUrl}${endpoint}`);
+  url.searchParams.set("symbols", marketSymbols.join(","));
+  url.searchParams.set("interval", coinalyzeIntervalMap[interval] || interval);
+  url.searchParams.set("from", String(Math.floor(fromMs / 1e3)));
+  url.searchParams.set("to", String(Math.floor(toMs / 1e3)));
+  const raw = await fetchJsonWithRateLimit(url.toString());
+  return toSeriesMap(raw, metric);
+};
+var run = async () => {
+  const intervals = (0, import_indicators.normalizeDerivativesIntervals)(
+    flags.intervals
+  );
+  const days = asInt(flags.days, 120);
+  const batchDays = asInt(flags.batchDays, 120);
+  const symbolBatchSize = asInt(flags.symbolBatchSize, 25);
+  const tickersLimit = flags.tickersLimit !== void 0 ? asInt(flags.tickersLimit, 0) || void 0 : void 0;
+  const exchangePriority = parseList(flags.exchangePriority || "A,6,0");
+  if (!intervals.length) throw new Error("No intervals provided");
+  const connectorFactory = import_connectors.connectors[import_connectors.ConnectorNames.ByBit];
+  const bybit = await connectorFactory({
+    userName: String(flags.user || "root")
+  });
+  const tickers = await (0, import_cli.getTickers)(
+    bybit,
+    String(flags.tickers || ""),
+    String(flags.exclude || ""),
+    tickersLimit,
+    String(flags.chunk || "")
+  );
+  if (!tickers.length) {
+    throw new Error("No tickers loaded via getTickers");
+  }
+  const markets = await fetchCoinalyzeMarkets();
+  if (!markets.length) {
+    throw new Error("No markets returned by Coinalyze /future-markets");
+  }
+  const { matches, unmatched } = buildMatches(
+    tickers,
+    markets,
+    exchangePriority
+  );
+  if (!matches.length) {
+    throw new Error("No matched symbols between getTickers and Coinalyze");
+  }
+  console.log(
+    import_chalk.default.cyan(
+      `Tickers=${tickers.length}, matched=${matches.length}, unmatched=${unmatched.length}, intervals=${intervals.join(",")}, symbolBatch=${symbolBatchSize}`
+    )
+  );
+  if (unmatched.length) {
+    console.log(
+      import_chalk.default.yellow(
+        `Unmatched sample (${Math.min(20, unmatched.length)}): ${unmatched.slice(0, 20).join(", ")}`
+      )
+    );
+  }
+  if (flags.showTickersList) {
+    console.log(
+      JSON.stringify(
+        matches.map((item) => ({
+          symbol: item.symbol,
+          coinalyze: item.marketSymbol,
+          exchange: item.exchange
+        })),
+        null,
+        2
+      )
+    );
+    return;
+  }
+  await (0, import_timescale.waitForDbReady)();
+  const now = Date.now();
+  const fromMs = now - days * 24 * 60 * 60 * 1e3;
+  const symbolBatches = import_lodash.default.chunk(matches, symbolBatchSize);
+  const windowsPerPair = Math.ceil(days / batchDays);
+  const totalWindows = symbolBatches.length * intervals.length * windowsPerPair;
+  const bar = new import_progress.default(
+    ":current/:total [:bar][:percent] :eta(s) :batch rows=:rows fail=:fail",
+    {
+      total: Math.max(1, totalWindows),
+      width: 30
+    }
+  );
+  const oiPath = process.env.COINALYZE_OI_PATH?.trim() || "/open-interest-history";
+  const fundingPath = process.env.COINALYZE_FUNDING_PATH?.trim() || "/funding-rate-history";
+  const liqPath = process.env.COINALYZE_LIQ_PATH?.trim() || "/liquidation-history";
+  let totalRows = 0;
+  let failedWindows = 0;
+  for (const interval of intervals) {
+    for (let batchIdx = 0; batchIdx < symbolBatches.length; batchIdx += 1) {
+      const batch = symbolBatches[batchIdx];
+      const marketSymbols = batch.map((item) => item.marketSymbol);
+      let cursor = fromMs;
+      while (cursor < now) {
+        const toMs = Math.min(now, cursor + batchDays * 24 * 60 * 60 * 1e3);
+        try {
+          const oiMap = await fetchMetricBatch({
+            endpoint: oiPath,
+            metric: "oi",
+            marketSymbols,
+            interval,
+            fromMs: cursor,
+            toMs
+          });
+          const fundingMap = await fetchMetricBatch({
+            endpoint: fundingPath,
+            metric: "funding",
+            marketSymbols,
+            interval,
+            fromMs: cursor,
+            toMs
+          });
+          const liqMap = await fetchMetricBatch({
+            endpoint: liqPath,
+            metric: "liq",
+            marketSymbols,
+            interval,
+            fromMs: cursor,
+            toMs
+          });
+          const rows = batch.flatMap((item) => {
+            const marketSymbol = item.marketSymbol.toUpperCase();
+            const points = (0, import_indicators.mergeCoinalyzeMetrics)({
+              symbol: item.symbol,
+              oiRaw: oiMap.get(marketSymbol) ?? [],
+              fundingRaw: fundingMap.get(marketSymbol) ?? [],
+              liqRaw: liqMap.get(marketSymbol) ?? []
+            });
+            return (0, import_indicators.coinalyzePointsToRows)(points, interval, "coinalyze");
+          });
+          if (rows.length) {
+            await (0, import_timescale.upsertDerivatives)(rows);
+            totalRows += rows.length;
+          }
+        } catch (error) {
+          failedWindows += 1;
+          console.error(
+            import_chalk.default.red(
+              `batch window failed batch=${batchIdx + 1}/${symbolBatches.length} interval=${interval} ${new Date(cursor).toISOString()}..${new Date(toMs).toISOString()}: ${error}`
+            )
+          );
+        } finally {
+          bar.tick(1, {
+            batch: import_chalk.default.gray(
+              `${batchIdx + 1}/${symbolBatches.length} ${interval} (${batch.length} symbols)`
+            ),
+            rows: totalRows,
+            fail: failedWindows
+          });
+        }
+        cursor = toMs + 1;
+      }
+    }
+  }
+  console.log("");
+  console.log(
+    import_chalk.default.green(
+      `Done. symbols=${matches.length} intervals=${intervals.join(",")} rows=${totalRows} failed_windows=${failedWindows}`
+    )
+  );
+  if (failedWindows > 0) {
+    process.exit(1);
+  }
+};
+run().catch((error) => {
+  console.error(import_chalk.default.red(`derivativesIngestCoinalyzeAll failed: ${error}`));
+  process.exit(1);
+});