@tradejs/cli 2.0.0 → 2.0.2

package/dist/lib/derivativesContextBackfill.js

@@ -33,6 +33,7 @@ __export(derivativesContextBackfill_exports, {
   backfillDerivativesContextForBacktest: () => backfillDerivativesContextForBacktest,
   backfillDerivativesContextForSignals: () => backfillDerivativesContextForSignals,
   formatCoinalyzeRequestError: () => formatCoinalyzeRequestError,
+  getMissingClosedDerivativesSymbols: () => getMissingClosedDerivativesSymbols,
   groupDerivativesContextMissingFetchRanges: () => groupDerivativesContextMissingFetchRanges,
   hasDerivativesContextCoverageRange: () => hasDerivativesContextCoverageRange,
   isBacktestDerivativesContextEnabled: () => isBacktestDerivativesContextEnabled,
@@ -40,11 +41,13 @@ __export(derivativesContextBackfill_exports, {
   isSignalsDerivativesContextEnabled: () => isSignalsDerivativesContextEnabled,
   resolveDerivativesContextBackfillSymbols: () => resolveDerivativesContextBackfillSymbols,
   resolveDerivativesContextBackfillWindow: () => resolveDerivativesContextBackfillWindow,
+  resolveDerivativesContextFetchFromMs: () => resolveDerivativesContextFetchFromMs,
   resolveDerivativesContextIntervalBackfillWindow: () => resolveDerivativesContextIntervalBackfillWindow,
   resolveDerivativesContextIntervals: () => resolveDerivativesContextIntervals,
   resolveDerivativesContextLookbackMs: () => resolveDerivativesContextLookbackMs,
   resolveDerivativesContextMissingCoverageFetchFromMs: () => resolveDerivativesContextMissingCoverageFetchFromMs,
   resolveDerivativesContextMissingFetchFromMs: () => resolveDerivativesContextMissingFetchFromMs,
+  resolveDerivativesContextRequiredFetchFromMs: () => resolveDerivativesContextRequiredFetchFromMs,
   shouldBackfillDerivativesContextForBacktest: () => shouldBackfillDerivativesContextForBacktest,
   shouldBackfillDerivativesContextForSignals: () => shouldBackfillDerivativesContextForSignals
 });
@@ -59,7 +62,7 @@ var import_userSettings = require("@tradejs/infra/userSettings");
 var HOUR_MS = 60 * 60 * 1e3;
 var DAY_MS = 24 * HOUR_MS;
 var DEFAULT_LOOKBACK_HOURS = 48;
-var DEFAULT_INTERVALS = ["15m", "1h"];
+var SOURCE_INTERVALS = ["15m"];
 var coinalyzeIntervalMap = {
   "15m": "15min",
   "1h": "1hour"
@@ -120,10 +123,7 @@ var isDerivativesContextBackfillEnabled = (env) => {
 var shouldBackfillDerivativesContextForBacktest = (params) => !params.cacheOnly && (params.aiEnabled || params.mlEnabled) && isBacktestDerivativesContextEnabled();
 var shouldBackfillDerivativesContextForSignals = (params) => !params.cacheOnly && isSignalsDerivativesContextEnabled();
 var resolveDerivativesContextIntervals = () => {
-  const intervals = (0, import_indicators.normalizeDerivativesIntervals)(
-    process.env.DERIVATIVES_CONTEXT_INTERVALS
-  );
-  return intervals.length ? intervals : DEFAULT_INTERVALS;
+  return [...SOURCE_INTERVALS];
 };
 var resolveDerivativesContextLookbackMs = () => {
   const hours = Number(process.env.DERIVATIVES_CONTEXT_LOOKBACK_HOURS);
@@ -158,9 +158,10 @@ var resolveDerivativesContextBackfillWindow = (params) => {
 };
 var resolveDerivativesContextIntervalBackfillWindow = (params) => {
   const intervalMs = derivativesIntervalMs(params.interval);
+  const alignedToMs = params.closedOnly ? (0, import_indicators.getLastClosedDerivativesBarStartMs)(params.toMs, params.interval) : Math.floor(params.toMs / intervalMs) * intervalMs;
   return {
     fromMs: Math.floor(params.fromMs / intervalMs) * intervalMs,
-    toMs: Math.floor(params.toMs / intervalMs) * intervalMs,
+    toMs: alignedToMs,
     intervalMs
   };
 };
@@ -178,6 +179,39 @@ var resolveDerivativesContextMissingFetchFromMs = (params) => {
   }
   return null;
 };
+var resolveDerivativesContextFetchFromMs = (params) => resolveDerivativesContextMissingFetchFromMs(params) ?? (params.refreshClosedTail ? params.toMs : null);
+var resolveDerivativesContextRequiredFetchFromMs = (params) => {
+  const dataCoverageFromMs = params.dataCoverageKeyExists ? null : params.mode === "signals" ? params.fromMs : resolveDerivativesContextMissingCoverageFetchFromMs({
+    ranges: params.dataCoverageRanges,
+    fromMs: params.fromMs,
+    toMs: params.toMs,
+    intervalMs: params.intervalMs
+  });
+  const edgesFromMs = dataCoverageFromMs == null ? null : resolveDerivativesContextFetchFromMs({
+    edges: params.edges,
+    fromMs: params.fromMs,
+    toMs: params.toMs,
+    intervalMs: params.intervalMs,
+    refreshClosedTail: params.mode === "signals"
+  });
+  const dataFetchFromMs = dataCoverageFromMs != null && edgesFromMs != null ? Math.max(dataCoverageFromMs, edgesFromMs) : null;
+  const confirmedLiquidationWindow = (0, import_indicators.resolveCoinalyzeConfirmedIntradayCoverage)({
+    interval: params.interval,
+    fromMs: params.fromMs,
+    toMs: params.toMs,
+    nowMs: params.nowMs
+  });
+  const liquidationFetchFromMs = confirmedLiquidationWindow ? resolveDerivativesContextMissingCoverageFetchFromMs({
+    ranges: params.liquidationCoverageRanges,
+    fromMs: confirmedLiquidationWindow.fromMs,
+    toMs: confirmedLiquidationWindow.toMs,
+    intervalMs: params.intervalMs
+  }) : null;
+  const fetchStarts = [dataFetchFromMs, liquidationFetchFromMs].filter(
+    (value) => value != null
+  );
+  return fetchStarts.length ? Math.min(...fetchStarts) : null;
+};
 var countBackfillWindows = (params) => {
   let count = 0;
   let cursor = params.fromMs;
@@ -244,6 +278,14 @@ var groupDerivativesContextMissingFetchRanges = (ranges) => {
   }
   return [...groups.entries()].sort(([a], [b]) => a - b).map(([fromMs, items]) => ({ fromMs, items }));
 };
+var getMissingClosedDerivativesSymbols = (params) => {
+  const availableSymbols = new Set(
+    params.rows.filter((row) => row.ts.getTime() === params.expectedTimestamp).map((row) => row.symbol.trim().toUpperCase())
+  );
+  return params.symbols.filter(
+    (symbol) => !availableSymbols.has(symbol.trim().toUpperCase())
+  );
+};
 var coverageKey = (params) => [
   params.symbol.trim().toUpperCase(),
   params.interval,
@@ -508,7 +550,8 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
     mode,
     startMs,
     endMs,
-    preloadStartMs: params.preloadStartMs
+    preloadStartMs: params.preloadStartMs,
+    nowMs: Date.now()
   });
   if (safeEndMs <= fromMs) {
     return skippedBackfillResult();
@@ -523,7 +566,8 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
     ...resolveDerivativesContextIntervalBackfillWindow({
       fromMs,
       toMs: safeEndMs,
-      interval
+      interval,
+      closedOnly: mode === "signals"
     })
   })).filter((item) => item.toMs > item.fromMs);
   if (!intervalWindows.length) {
@@ -541,15 +585,33 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
   );
   const coverageKeysByInterval = /* @__PURE__ */ new Map();
   const coverageRangesByInterval = /* @__PURE__ */ new Map();
+  const liquidationCoverageRangesByInterval = /* @__PURE__ */ new Map();
+  const coverageNowMs = Date.now();
   await Promise.all(
     intervalWindows.map(async (window) => {
-      const coverageRows = await (0, import_timescale.getDerivativesBackfillCoverage)({
-        source: "coinalyze",
-        symbols,
+      const confirmedLiquidationWindow = (0, import_indicators.resolveCoinalyzeConfirmedIntradayCoverage)({
         interval: window.interval,
         fromMs: window.fromMs,
-        toMs: window.toMs
+        toMs: window.toMs,
+        nowMs: coverageNowMs
       });
+      const [coverageRows, liquidationCoverageRows] = await Promise.all([
+        (0, import_timescale.getDerivativesBackfillCoverage)({
+          source: "coinalyze",
+          symbols,
+          interval: window.interval,
+          fromMs: window.fromMs,
+          toMs: window.toMs
+        }),
+        confirmedLiquidationWindow ? (0, import_timescale.getDerivativesMetricCoverage)({
+          source: "coinalyze",
+          metric: "liquidation",
+          symbols,
+          interval: window.interval,
+          fromMs: confirmedLiquidationWindow.fromMs,
+          toMs: confirmedLiquidationWindow.toMs
+        }) : Promise.resolve([])
+      ]);
       coverageKeysByInterval.set(
         window.interval,
         new Set(
@@ -571,6 +633,17 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
         rangesBySymbol.set(symbol, ranges);
       }
       coverageRangesByInterval.set(window.interval, rangesBySymbol);
+      const liquidationRangesBySymbol = /* @__PURE__ */ new Map();
+      for (const row of liquidationCoverageRows) {
+        const symbol = row.symbol.toUpperCase();
+        const ranges = liquidationRangesBySymbol.get(symbol) ?? [];
+        ranges.push({ fromMs: row.fromMs, toMs: row.toMs });
+        liquidationRangesBySymbol.set(symbol, ranges);
+      }
+      liquidationCoverageRangesByInterval.set(
+        window.interval,
+        liquidationRangesBySymbol
+      );
     })
   );
   const cachedWindows = intervalWindows.reduce(
@@ -582,7 +655,7 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
     }),
     0
   );
-  const allBackfillWindowsCached = intervalWindows.every((window) => {
+  const allBackfillWindowsCached = mode !== "signals" && intervalWindows.every((window) => {
     const edgesBySymbol = edgesByInterval.get(window.interval);
     const coverageKeys = coverageKeysByInterval.get(window.interval);
     const backfillWindows = buildBackfillWindows({
@@ -595,7 +668,7 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
       (symbol) => backfillWindows.every((backfillWindow) => {
         const normalizedSymbol = symbol.toUpperCase();
         const coverageRanges = coverageRangesByInterval.get(window.interval)?.get(normalizedSymbol) ?? [];
-        return hasDerivativesWindowCoverage({
+        const dataCovered = hasDerivativesWindowCoverage({
           edges: edgesBySymbol?.get(normalizedSymbol),
           fromMs: backfillWindow.fromMs,
           toMs: backfillWindow.toMs
@@ -612,6 +685,19 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
             toMs: backfillWindow.toMs
           })
         );
+        const confirmedLiquidationWindow = (0, import_indicators.resolveCoinalyzeConfirmedIntradayCoverage)({
+          interval: window.interval,
+          fromMs: backfillWindow.fromMs,
+          toMs: backfillWindow.toMs,
+          nowMs: coverageNowMs
+        });
+        const liquidationCovered = confirmedLiquidationWindow == null || resolveDerivativesContextMissingCoverageFetchFromMs({
+          ranges: liquidationCoverageRangesByInterval.get(window.interval)?.get(normalizedSymbol) ?? [],
+          fromMs: confirmedLiquidationWindow.fromMs,
+          toMs: confirmedLiquidationWindow.toMs,
+          intervalMs: window.intervalMs
+        }) == null;
+        return dataCovered && liquidationCovered;
       })
     );
   });
@@ -696,6 +782,11 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
     coverageKeysByInterval.set(interval, coverageKeys);
     const coverageRangesBySymbol = coverageRangesByInterval.get(interval) ?? /* @__PURE__ */ new Map();
     coverageRangesByInterval.set(interval, coverageRangesBySymbol);
+    const liquidationCoverageRangesBySymbol = liquidationCoverageRangesByInterval.get(interval) ?? /* @__PURE__ */ new Map();
+    liquidationCoverageRangesByInterval.set(
+      interval,
+      liquidationCoverageRangesBySymbol
+    );
     for (let batchIdx = 0; batchIdx < symbolBatches.length; batchIdx += 1) {
       const batch = symbolBatches[batchIdx];
       let cursor = window.fromMs;
@@ -708,30 +799,26 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
             fromMs: cursor,
             toMs
           });
-          if (coverageKeys.has(key)) {
-            return null;
-          }
           const normalizedSymbol = item.symbol.toUpperCase();
-          const coverageFromMs = resolveDerivativesContextMissingCoverageFetchFromMs({
-            ranges: coverageRangesBySymbol.get(normalizedSymbol),
-            fromMs: cursor,
-            toMs,
-            intervalMs
-          });
-          if (coverageFromMs == null) {
-            return null;
-          }
-          const edges = edgesBySymbol.get(normalizedSymbol);
-          const edgesFromMs = resolveDerivativesContextMissingFetchFromMs({
-            edges,
-            fromMs: cursor,
-            toMs,
-            intervalMs
-          });
-          if (edgesFromMs == null) {
-            return null;
-          }
-          return { item, fromMs: Math.max(coverageFromMs, edgesFromMs) };
+          const requiredFromMs = resolveDerivativesContextRequiredFetchFromMs(
+            {
+              mode,
+              interval,
+              intervalMs,
+              fromMs: cursor,
+              toMs,
+              nowMs: coverageNowMs,
+              dataCoverageKeyExists: coverageKeys.has(key),
+              dataCoverageRanges: coverageRangesBySymbol.get(normalizedSymbol),
+              liquidationCoverageRanges: liquidationCoverageRangesBySymbol.get(normalizedSymbol),
+              edges: edgesBySymbol.get(normalizedSymbol)
+            }
+          );
+          if (requiredFromMs == null) return null;
+          return {
+            item,
+            fromMs: requiredFromMs
+          };
         }).filter(
           (item) => item != null
         );
@@ -745,85 +832,150 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
               const marketSymbols = missingBatch.map(
                 (item) => item.marketSymbol
               );
-              const oiMap = await fetchMetricBatch({
-                endpoint: oiPath,
-                metric: "oi",
-                marketSymbols,
-                apiKey,
-                interval,
-                fromMs: group.fromMs,
-                toMs
-              });
-              const fundingMap = await fetchMetricBatch({
-                endpoint: fundingPath,
-                metric: "funding",
-                marketSymbols,
-                apiKey,
-                interval,
-                fromMs: group.fromMs,
-                toMs
-              });
-              const liqMap = await fetchMetricBatch({
-                endpoint: liqPath,
-                metric: "liq",
-                marketSymbols,
-                apiKey,
-                interval,
-                fromMs: group.fromMs,
-                toMs
-              });
-              const rows = missingBatch.flatMap((item) => {
-                const marketSymbol = item.marketSymbol.toUpperCase();
-                const points = (0, import_indicators.mergeCoinalyzeMetrics)({
-                  symbol: item.symbol,
-                  oiRaw: oiMap.get(marketSymbol) ?? [],
-                  fundingRaw: fundingMap.get(marketSymbol) ?? [],
-                  liqRaw: liqMap.get(marketSymbol) ?? []
+              const maxClosedBarAttempts = mode === "signals" ? asInt(
+                process.env.DERIVATIVES_CONTEXT_CLOSED_BAR_MAX_ATTEMPTS,
+                3
+              ) : 1;
+              const closedBarRetryDelayMs = asInt(
+                process.env.DERIVATIVES_CONTEXT_CLOSED_BAR_RETRY_DELAY_MS,
+                2e3
+              );
+              let rows = [];
+              let missingClosedSymbols = [];
+              let liquidationRowsByMarket = /* @__PURE__ */ new Map();
+              for (let attempt = 1; attempt <= maxClosedBarAttempts; attempt += 1) {
+                const oiMap = await fetchMetricBatch({
+                  endpoint: oiPath,
+                  metric: "oi",
+                  marketSymbols,
+                  apiKey,
+                  interval,
+                  fromMs: group.fromMs,
+                  toMs
                 });
-                return (0, import_indicators.coinalyzePointsToRows)(points, interval, "coinalyze");
-              });
+                const fundingMap = await fetchMetricBatch({
+                  endpoint: fundingPath,
+                  metric: "funding",
+                  marketSymbols,
+                  apiKey,
+                  interval,
+                  fromMs: group.fromMs,
+                  toMs
+                });
+                liquidationRowsByMarket = await fetchMetricBatch({
+                  endpoint: liqPath,
+                  metric: "liq",
+                  marketSymbols,
+                  apiKey,
+                  interval,
+                  fromMs: group.fromMs,
+                  toMs
+                });
+                rows = missingBatch.flatMap((item) => {
+                  const marketSymbol = item.marketSymbol.toUpperCase();
+                  const points = (0, import_indicators.mergeCoinalyzeMetrics)({
+                    symbol: item.symbol,
+                    oiRaw: oiMap.get(marketSymbol) ?? [],
+                    fundingRaw: fundingMap.get(marketSymbol) ?? [],
+                    liqRaw: liquidationRowsByMarket.get(marketSymbol) ?? []
+                  });
+                  return (0, import_indicators.coinalyzePointsToRows)(points, interval, "coinalyze");
+                });
+                missingClosedSymbols = mode === "signals" ? getMissingClosedDerivativesSymbols({
+                  symbols: missingBatch.map((item) => item.symbol),
+                  rows,
+                  expectedTimestamp: toMs
+                }) : [];
+                if (!missingClosedSymbols.length) break;
+                if (attempt < maxClosedBarAttempts) {
+                  await (0, import_async.delay)(closedBarRetryDelayMs);
+                }
+              }
+              if (missingClosedSymbols.length) {
+                throw new Error(
+                  `Coinalyze closed ${interval} bar ${new Date(toMs).toISOString()} unavailable for ${missingClosedSymbols.join(",")}`
+                );
+              }
               if (rows.length) {
                 await (0, import_timescale.upsertDerivatives)(rows);
                 totalRows += rows.length;
               }
-              const rowsCountBySymbol = /* @__PURE__ */ new Map();
-              for (const row of rows) {
-                const symbol = row.symbol.toUpperCase();
-                rowsCountBySymbol.set(
-                  symbol,
-                  (rowsCountBySymbol.get(symbol) ?? 0) + 1
-                );
-              }
-              const coverageRows = missingBatch.map((item) => {
-                const normalizedSymbol = item.symbol.toUpperCase();
-                const rowsCount = rowsCountBySymbol.get(normalizedSymbol) ?? 0;
-                return {
-                  source: "coinalyze",
-                  symbol: item.symbol,
-                  interval,
-                  fromMs: cursor,
-                  toMs,
-                  rowsCount
-                };
+              const confirmedLiquidationWindow = (0, import_indicators.resolveCoinalyzeConfirmedIntradayCoverage)({
+                interval,
+                fromMs: group.fromMs,
+                toMs,
+                nowMs: coverageNowMs
               });
-              await (0, import_timescale.upsertDerivativesBackfillCoverage)(coverageRows);
-              for (const coverageRow of coverageRows) {
-                const symbol = coverageRow.symbol.toUpperCase();
-                edgesBySymbol.set(
-                  symbol,
-                  extendEdges(edgesBySymbol.get(symbol), cursor, toMs)
-                );
-                coverageKeys.add(
-                  coverageKey({
+              if (confirmedLiquidationWindow) {
+                const metricCoverageRows = missingBatch.map((item) => {
+                  const marketSymbol = item.marketSymbol.toUpperCase();
+                  const eventRowsCount = (liquidationRowsByMarket.get(marketSymbol) ?? []).filter((point) => {
+                    const timestamp = (0, import_indicators.toCoinalyzeTimestampMs)(
+                      point.t ?? point.ts ?? point.time ?? point.timestamp
+                    );
+                    return timestamp != null && timestamp >= confirmedLiquidationWindow.fromMs && timestamp <= confirmedLiquidationWindow.toMs;
+                  }).length;
+                  return {
+                    source: "coinalyze",
+                    metric: "liquidation",
+                    symbol: item.symbol,
+                    interval,
+                    fromMs: confirmedLiquidationWindow.fromMs,
+                    toMs: confirmedLiquidationWindow.toMs,
+                    eventRowsCount
+                  };
+                });
+                await (0, import_timescale.applyDerivativesMetricCoverage)(metricCoverageRows);
+                for (const coverageRow of metricCoverageRows) {
+                  const symbol = coverageRow.symbol.toUpperCase();
+                  const ranges = liquidationCoverageRangesBySymbol.get(symbol) ?? [];
+                  ranges.push({
+                    fromMs: coverageRow.fromMs,
+                    toMs: coverageRow.toMs
+                  });
+                  liquidationCoverageRangesBySymbol.set(symbol, ranges);
+                }
+              }
+              if (mode === "backtest") {
+                const rowsCountBySymbol = /* @__PURE__ */ new Map();
+                for (const row of rows) {
+                  const symbol = row.symbol.toUpperCase();
+                  rowsCountBySymbol.set(
                     symbol,
+                    (rowsCountBySymbol.get(symbol) ?? 0) + 1
+                  );
+                }
+                const coverageRows = missingBatch.map((item) => {
+                  const normalizedSymbol = item.symbol.toUpperCase();
+                  const rowsCount = rowsCountBySymbol.get(normalizedSymbol) ?? 0;
+                  return {
+                    source: "coinalyze",
+                    symbol: item.symbol,
                     interval,
                     fromMs: cursor,
-                    toMs
-                  })
-                );
-                const coverageRanges = coverageRangesBySymbol.get(symbol) ?? [];
-                coverageRanges.push({ fromMs: cursor, toMs });
-                coverageRangesBySymbol.set(symbol, coverageRanges);
+                    toMs,
+                    rowsCount
+                  };
+                });
+                await (0, import_timescale.upsertDerivativesBackfillCoverage)(coverageRows);
+                for (const coverageRow of coverageRows) {
+                  const symbol = coverageRow.symbol.toUpperCase();
+                  edgesBySymbol.set(
+                    symbol,
+                    extendEdges(edgesBySymbol.get(symbol), cursor, toMs)
+                  );
+                  coverageKeys.add(
+                    coverageKey({
+                      symbol,
+                      interval,
+                      fromMs: cursor,
+                      toMs
+                    })
+                  );
+                  const coverageRanges = coverageRangesBySymbol.get(symbol) ?? [];
+                  coverageRanges.push({ fromMs: cursor, toMs });
+                  coverageRangesBySymbol.set(symbol, coverageRanges);
+                }
               }
             }
           }
@@ -883,6 +1035,7 @@ var backfillDerivativesContextForSignals = async (params) => backfillDerivatives
   backfillDerivativesContextForBacktest,
   backfillDerivativesContextForSignals,
   formatCoinalyzeRequestError,
+  getMissingClosedDerivativesSymbols,
   groupDerivativesContextMissingFetchRanges,
   hasDerivativesContextCoverageRange,
   isBacktestDerivativesContextEnabled,
@@ -890,11 +1043,13 @@ var backfillDerivativesContextForSignals = async (params) => backfillDerivatives
   isSignalsDerivativesContextEnabled,
   resolveDerivativesContextBackfillSymbols,
   resolveDerivativesContextBackfillWindow,
+  resolveDerivativesContextFetchFromMs,
   resolveDerivativesContextIntervalBackfillWindow,
   resolveDerivativesContextIntervals,
   resolveDerivativesContextLookbackMs,
   resolveDerivativesContextMissingCoverageFetchFromMs,
   resolveDerivativesContextMissingFetchFromMs,
+  resolveDerivativesContextRequiredFetchFromMs,
   shouldBackfillDerivativesContextForBacktest,
   shouldBackfillDerivativesContextForSignals
 });