@tradejs/cli 2.0.0 → 2.0.1

package/dist/scripts/replayRunner.js

@@ -3109,7 +3109,7 @@ var import_userSettings = require("@tradejs/infra/userSettings");
 var HOUR_MS = 60 * 60 * 1e3;
 var DAY_MS2 = 24 * HOUR_MS;
 var DEFAULT_LOOKBACK_HOURS = 48;
-var DEFAULT_INTERVALS = ["15m", "1h"];
+var SOURCE_INTERVALS = ["15m"];
 var coinalyzeIntervalMap = {
   "15m": "15min",
   "1h": "1hour"
@@ -3170,10 +3170,7 @@ var isDerivativesContextBackfillEnabled = (env) => {
 var shouldBackfillDerivativesContextForBacktest = (params) => !params.cacheOnly && (params.aiEnabled || params.mlEnabled) && isBacktestDerivativesContextEnabled();
 var shouldBackfillDerivativesContextForSignals = (params) => !params.cacheOnly && isSignalsDerivativesContextEnabled();
 var resolveDerivativesContextIntervals = () => {
-  const intervals = (0, import_indicators.normalizeDerivativesIntervals)(
-    process.env.DERIVATIVES_CONTEXT_INTERVALS
-  );
-  return intervals.length ? intervals : DEFAULT_INTERVALS;
+  return [...SOURCE_INTERVALS];
 };
 var resolveDerivativesContextLookbackMs = () => {
   const hours = Number(process.env.DERIVATIVES_CONTEXT_LOOKBACK_HOURS);
@@ -3208,9 +3205,10 @@ var resolveDerivativesContextBackfillWindow = (params) => {
 };
 var resolveDerivativesContextIntervalBackfillWindow = (params) => {
   const intervalMs = derivativesIntervalMs(params.interval);
+  const alignedToMs = params.closedOnly ? (0, import_indicators.getLastClosedDerivativesBarStartMs)(params.toMs, params.interval) : Math.floor(params.toMs / intervalMs) * intervalMs;
   return {
     fromMs: Math.floor(params.fromMs / intervalMs) * intervalMs,
-    toMs: Math.floor(params.toMs / intervalMs) * intervalMs,
+    toMs: alignedToMs,
     intervalMs
   };
 };
@@ -3228,6 +3226,7 @@ var resolveDerivativesContextMissingFetchFromMs = (params) => {
   }
   return null;
 };
+var resolveDerivativesContextFetchFromMs = (params) => resolveDerivativesContextMissingFetchFromMs(params) ?? (params.refreshClosedTail ? params.toMs : null);
 var countBackfillWindows = (params) => {
   let count = 0;
   let cursor = params.fromMs;
@@ -3291,6 +3290,14 @@ var groupDerivativesContextMissingFetchRanges = (ranges) => {
   }
   return [...groups.entries()].sort(([a], [b]) => a - b).map(([fromMs, items]) => ({ fromMs, items }));
 };
+var getMissingClosedDerivativesSymbols = (params) => {
+  const availableSymbols = new Set(
+    params.rows.filter((row) => row.ts.getTime() === params.expectedTimestamp).map((row) => row.symbol.trim().toUpperCase())
+  );
+  return params.symbols.filter(
+    (symbol) => !availableSymbols.has(symbol.trim().toUpperCase())
+  );
+};
 var coverageKey = (params) => [
   params.symbol.trim().toUpperCase(),
   params.interval,
@@ -3570,7 +3577,8 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
     ...resolveDerivativesContextIntervalBackfillWindow({
       fromMs,
       toMs: safeEndMs,
-      interval
+      interval,
+      closedOnly: mode === "signals"
     })
   })).filter((item) => item.toMs > item.fromMs);
   if (!intervalWindows.length) {
@@ -3629,7 +3637,7 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
     }),
     0
   );
-  const allBackfillWindowsCached = intervalWindows.every((window2) => {
+  const allBackfillWindowsCached = mode !== "signals" && intervalWindows.every((window2) => {
     const edgesBySymbol = edgesByInterval.get(window2.interval);
     const coverageKeys = coverageKeysByInterval.get(window2.interval);
     const backfillWindows = buildBackfillWindows({
@@ -3759,7 +3767,7 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
             return null;
           }
           const normalizedSymbol = item.symbol.toUpperCase();
-          const coverageFromMs = resolveDerivativesContextMissingCoverageFetchFromMs({
+          const coverageFromMs = mode === "signals" ? cursor : resolveDerivativesContextMissingCoverageFetchFromMs({
             ranges: coverageRangesBySymbol.get(normalizedSymbol),
             fromMs: cursor,
             toMs,
@@ -3769,16 +3777,20 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
             return null;
           }
           const edges = edgesBySymbol.get(normalizedSymbol);
-          const edgesFromMs = resolveDerivativesContextMissingFetchFromMs({
+          const edgesFromMs = resolveDerivativesContextFetchFromMs({
             edges,
             fromMs: cursor,
             toMs,
-            intervalMs
+            intervalMs,
+            refreshClosedTail: mode === "signals"
           });
           if (edgesFromMs == null) {
             return null;
           }
-          return { item, fromMs: Math.max(coverageFromMs, edgesFromMs) };
+          return {
+            item,
+            fromMs: Math.max(coverageFromMs, edgesFromMs)
+          };
         }).filter(
           (item) => item != null
         );
@@ -3792,85 +3804,113 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
               const marketSymbols = missingBatch.map(
                 (item) => item.marketSymbol
               );
-              const oiMap = await fetchMetricBatch({
-                endpoint: oiPath,
-                metric: "oi",
-                marketSymbols,
-                apiKey,
-                interval,
-                fromMs: group.fromMs,
-                toMs
-              });
-              const fundingMap = await fetchMetricBatch({
-                endpoint: fundingPath,
-                metric: "funding",
-                marketSymbols,
-                apiKey,
-                interval,
-                fromMs: group.fromMs,
-                toMs
-              });
-              const liqMap = await fetchMetricBatch({
-                endpoint: liqPath,
-                metric: "liq",
-                marketSymbols,
-                apiKey,
-                interval,
-                fromMs: group.fromMs,
-                toMs
-              });
-              const rows = missingBatch.flatMap((item) => {
-                const marketSymbol = item.marketSymbol.toUpperCase();
-                const points = (0, import_indicators.mergeCoinalyzeMetrics)({
-                  symbol: item.symbol,
-                  oiRaw: oiMap.get(marketSymbol) ?? [],
-                  fundingRaw: fundingMap.get(marketSymbol) ?? [],
-                  liqRaw: liqMap.get(marketSymbol) ?? []
+              const maxClosedBarAttempts = mode === "signals" ? asInt2(
+                process.env.DERIVATIVES_CONTEXT_CLOSED_BAR_MAX_ATTEMPTS,
+                3
+              ) : 1;
+              const closedBarRetryDelayMs = asInt2(
+                process.env.DERIVATIVES_CONTEXT_CLOSED_BAR_RETRY_DELAY_MS,
+                2e3
+              );
+              let rows = [];
+              let missingClosedSymbols = [];
+              for (let attempt = 1; attempt <= maxClosedBarAttempts; attempt += 1) {
+                const oiMap = await fetchMetricBatch({
+                  endpoint: oiPath,
+                  metric: "oi",
+                  marketSymbols,
+                  apiKey,
+                  interval,
+                  fromMs: group.fromMs,
+                  toMs
+                });
+                const fundingMap = await fetchMetricBatch({
+                  endpoint: fundingPath,
+                  metric: "funding",
+                  marketSymbols,
+                  apiKey,
+                  interval,
+                  fromMs: group.fromMs,
+                  toMs
                 });
-                return (0, import_indicators.coinalyzePointsToRows)(points, interval, "coinalyze");
-              });
+                const liqMap = await fetchMetricBatch({
+                  endpoint: liqPath,
+                  metric: "liq",
+                  marketSymbols,
+                  apiKey,
+                  interval,
+                  fromMs: group.fromMs,
+                  toMs
+                });
+                rows = missingBatch.flatMap((item) => {
+                  const marketSymbol = item.marketSymbol.toUpperCase();
+                  const points = (0, import_indicators.mergeCoinalyzeMetrics)({
+                    symbol: item.symbol,
+                    oiRaw: oiMap.get(marketSymbol) ?? [],
+                    fundingRaw: fundingMap.get(marketSymbol) ?? [],
+                    liqRaw: liqMap.get(marketSymbol) ?? []
+                  });
+                  return (0, import_indicators.coinalyzePointsToRows)(points, interval, "coinalyze");
+                });
+                missingClosedSymbols = mode === "signals" ? getMissingClosedDerivativesSymbols({
+                  symbols: missingBatch.map((item) => item.symbol),
+                  rows,
+                  expectedTimestamp: toMs
+                }) : [];
+                if (!missingClosedSymbols.length) break;
+                if (attempt < maxClosedBarAttempts) {
+                  await (0, import_async.delay)(closedBarRetryDelayMs);
+                }
+              }
+              if (missingClosedSymbols.length) {
+                throw new Error(
+                  `Coinalyze closed ${interval} bar ${new Date(toMs).toISOString()} unavailable for ${missingClosedSymbols.join(",")}`
+                );
+              }
               if (rows.length) {
                 await (0, import_timescale2.upsertDerivatives)(rows);
                 totalRows += rows.length;
               }
-              const rowsCountBySymbol = /* @__PURE__ */ new Map();
-              for (const row of rows) {
-                const symbol = row.symbol.toUpperCase();
-                rowsCountBySymbol.set(
-                  symbol,
-                  (rowsCountBySymbol.get(symbol) ?? 0) + 1
-                );
-              }
-              const coverageRows = missingBatch.map((item) => {
-                const normalizedSymbol = item.symbol.toUpperCase();
-                const rowsCount = rowsCountBySymbol.get(normalizedSymbol) ?? 0;
-                return {
-                  source: "coinalyze",
-                  symbol: item.symbol,
-                  interval,
-                  fromMs: cursor,
-                  toMs,
-                  rowsCount
-                };
-              });
-              await (0, import_timescale2.upsertDerivativesBackfillCoverage)(coverageRows);
-              for (const coverageRow of coverageRows) {
-                const symbol = coverageRow.symbol.toUpperCase();
-                edgesBySymbol.set(
-                  symbol,
-                  extendEdges(edgesBySymbol.get(symbol), cursor, toMs)
-                );
-                coverageKeys.add(
-                  coverageKey({
+              if (mode === "backtest") {
+                const rowsCountBySymbol = /* @__PURE__ */ new Map();
+                for (const row of rows) {
+                  const symbol = row.symbol.toUpperCase();
+                  rowsCountBySymbol.set(
                     symbol,
+                    (rowsCountBySymbol.get(symbol) ?? 0) + 1
+                  );
+                }
+                const coverageRows = missingBatch.map((item) => {
+                  const normalizedSymbol = item.symbol.toUpperCase();
+                  const rowsCount = rowsCountBySymbol.get(normalizedSymbol) ?? 0;
+                  return {
+                    source: "coinalyze",
+                    symbol: item.symbol,
                     interval,
                     fromMs: cursor,
-                    toMs
-                  })
-                );
-                const coverageRanges = coverageRangesBySymbol.get(symbol) ?? [];
-                coverageRanges.push({ fromMs: cursor, toMs });
-                coverageRangesBySymbol.set(symbol, coverageRanges);
+                    toMs,
+                    rowsCount
+                  };
+                });
+                await (0, import_timescale2.upsertDerivativesBackfillCoverage)(coverageRows);
+                for (const coverageRow of coverageRows) {
+                  const symbol = coverageRow.symbol.toUpperCase();
+                  edgesBySymbol.set(
+                    symbol,
+                    extendEdges(edgesBySymbol.get(symbol), cursor, toMs)
+                  );
+                  coverageKeys.add(
+                    coverageKey({
+                      symbol,
+                      interval,
+                      fromMs: cursor,
+                      toMs
+                    })
+                  );
+                  const coverageRanges = coverageRangesBySymbol.get(symbol) ?? [];
+                  coverageRanges.push({ fromMs: cursor, toMs });
+                  coverageRangesBySymbol.set(symbol, coverageRanges);
+                }
               }
             }
           }

package/dist/scripts/runtimeParity.js

@@ -3114,7 +3114,7 @@ var import_userSettings = require("@tradejs/infra/userSettings");
 var HOUR_MS = 60 * 60 * 1e3;
 var DAY_MS2 = 24 * HOUR_MS;
 var DEFAULT_LOOKBACK_HOURS = 48;
-var DEFAULT_INTERVALS = ["15m", "1h"];
+var SOURCE_INTERVALS = ["15m"];
 var coinalyzeIntervalMap = {
   "15m": "15min",
   "1h": "1hour"
@@ -3175,10 +3175,7 @@ var isDerivativesContextBackfillEnabled = (env) => {
 var shouldBackfillDerivativesContextForBacktest = (params) => !params.cacheOnly && (params.aiEnabled || params.mlEnabled) && isBacktestDerivativesContextEnabled();
 var shouldBackfillDerivativesContextForSignals = (params) => !params.cacheOnly && isSignalsDerivativesContextEnabled();
 var resolveDerivativesContextIntervals = () => {
-  const intervals = (0, import_indicators.normalizeDerivativesIntervals)(
-    process.env.DERIVATIVES_CONTEXT_INTERVALS
-  );
-  return intervals.length ? intervals : DEFAULT_INTERVALS;
+  return [...SOURCE_INTERVALS];
 };
 var resolveDerivativesContextLookbackMs = () => {
   const hours = Number(process.env.DERIVATIVES_CONTEXT_LOOKBACK_HOURS);
@@ -3213,9 +3210,10 @@ var resolveDerivativesContextBackfillWindow = (params) => {
 };
 var resolveDerivativesContextIntervalBackfillWindow = (params) => {
   const intervalMs = derivativesIntervalMs(params.interval);
+  const alignedToMs = params.closedOnly ? (0, import_indicators.getLastClosedDerivativesBarStartMs)(params.toMs, params.interval) : Math.floor(params.toMs / intervalMs) * intervalMs;
   return {
     fromMs: Math.floor(params.fromMs / intervalMs) * intervalMs,
-    toMs: Math.floor(params.toMs / intervalMs) * intervalMs,
+    toMs: alignedToMs,
     intervalMs
   };
 };
@@ -3233,6 +3231,7 @@ var resolveDerivativesContextMissingFetchFromMs = (params) => {
   }
   return null;
 };
+var resolveDerivativesContextFetchFromMs = (params) => resolveDerivativesContextMissingFetchFromMs(params) ?? (params.refreshClosedTail ? params.toMs : null);
 var countBackfillWindows = (params) => {
   let count = 0;
   let cursor = params.fromMs;
@@ -3296,6 +3295,14 @@ var groupDerivativesContextMissingFetchRanges = (ranges) => {
   }
   return [...groups.entries()].sort(([a], [b]) => a - b).map(([fromMs, items]) => ({ fromMs, items }));
 };
+var getMissingClosedDerivativesSymbols = (params) => {
+  const availableSymbols = new Set(
+    params.rows.filter((row) => row.ts.getTime() === params.expectedTimestamp).map((row) => row.symbol.trim().toUpperCase())
+  );
+  return params.symbols.filter(
+    (symbol) => !availableSymbols.has(symbol.trim().toUpperCase())
+  );
+};
 var coverageKey = (params) => [
   params.symbol.trim().toUpperCase(),
   params.interval,
@@ -3575,7 +3582,8 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
     ...resolveDerivativesContextIntervalBackfillWindow({
       fromMs,
       toMs: safeEndMs,
-      interval: interval2
+      interval: interval2,
+      closedOnly: mode === "signals"
     })
   })).filter((item) => item.toMs > item.fromMs);
   if (!intervalWindows.length) {
@@ -3634,7 +3642,7 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
     }),
     0
   );
-  const allBackfillWindowsCached = intervalWindows.every((window2) => {
+  const allBackfillWindowsCached = mode !== "signals" && intervalWindows.every((window2) => {
     const edgesBySymbol = edgesByInterval.get(window2.interval);
     const coverageKeys = coverageKeysByInterval.get(window2.interval);
     const backfillWindows = buildBackfillWindows({
@@ -3764,7 +3772,7 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
             return null;
           }
           const normalizedSymbol = item.symbol.toUpperCase();
-          const coverageFromMs = resolveDerivativesContextMissingCoverageFetchFromMs({
+          const coverageFromMs = mode === "signals" ? cursor : resolveDerivativesContextMissingCoverageFetchFromMs({
             ranges: coverageRangesBySymbol.get(normalizedSymbol),
             fromMs: cursor,
             toMs,
@@ -3774,16 +3782,20 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
             return null;
           }
           const edges = edgesBySymbol.get(normalizedSymbol);
-          const edgesFromMs = resolveDerivativesContextMissingFetchFromMs({
+          const edgesFromMs = resolveDerivativesContextFetchFromMs({
             edges,
             fromMs: cursor,
             toMs,
-            intervalMs
+            intervalMs,
+            refreshClosedTail: mode === "signals"
           });
           if (edgesFromMs == null) {
             return null;
           }
-          return { item, fromMs: Math.max(coverageFromMs, edgesFromMs) };
+          return {
+            item,
+            fromMs: Math.max(coverageFromMs, edgesFromMs)
+          };
         }).filter(
           (item) => item != null
         );
@@ -3797,85 +3809,113 @@ var backfillDerivativesContext = async (params, enabled, mode) => {
               const marketSymbols = missingBatch.map(
                 (item) => item.marketSymbol
               );
-              const oiMap = await fetchMetricBatch({
-                endpoint: oiPath,
-                metric: "oi",
-                marketSymbols,
-                apiKey,
-                interval: interval2,
-                fromMs: group.fromMs,
-                toMs
-              });
-              const fundingMap = await fetchMetricBatch({
-                endpoint: fundingPath,
-                metric: "funding",
-                marketSymbols,
-                apiKey,
-                interval: interval2,
-                fromMs: group.fromMs,
-                toMs
-              });
-              const liqMap = await fetchMetricBatch({
-                endpoint: liqPath,
-                metric: "liq",
-                marketSymbols,
-                apiKey,
-                interval: interval2,
-                fromMs: group.fromMs,
-                toMs
-              });
-              const rows = missingBatch.flatMap((item) => {
-                const marketSymbol = item.marketSymbol.toUpperCase();
-                const points = (0, import_indicators.mergeCoinalyzeMetrics)({
-                  symbol: item.symbol,
-                  oiRaw: oiMap.get(marketSymbol) ?? [],
-                  fundingRaw: fundingMap.get(marketSymbol) ?? [],
-                  liqRaw: liqMap.get(marketSymbol) ?? []
+              const maxClosedBarAttempts = mode === "signals" ? asInt2(
+                process.env.DERIVATIVES_CONTEXT_CLOSED_BAR_MAX_ATTEMPTS,
+                3
+              ) : 1;
+              const closedBarRetryDelayMs = asInt2(
+                process.env.DERIVATIVES_CONTEXT_CLOSED_BAR_RETRY_DELAY_MS,
+                2e3
+              );
+              let rows = [];
+              let missingClosedSymbols = [];
+              for (let attempt = 1; attempt <= maxClosedBarAttempts; attempt += 1) {
+                const oiMap = await fetchMetricBatch({
+                  endpoint: oiPath,
+                  metric: "oi",
+                  marketSymbols,
+                  apiKey,
+                  interval: interval2,
+                  fromMs: group.fromMs,
+                  toMs
                 });
-                return (0, import_indicators.coinalyzePointsToRows)(points, interval2, "coinalyze");
-              });
+                const fundingMap = await fetchMetricBatch({
+                  endpoint: fundingPath,
+                  metric: "funding",
+                  marketSymbols,
+                  apiKey,
+                  interval: interval2,
+                  fromMs: group.fromMs,
+                  toMs
+                });
+                const liqMap = await fetchMetricBatch({
+                  endpoint: liqPath,
+                  metric: "liq",
+                  marketSymbols,
+                  apiKey,
+                  interval: interval2,
+                  fromMs: group.fromMs,
+                  toMs
+                });
+                rows = missingBatch.flatMap((item) => {
+                  const marketSymbol = item.marketSymbol.toUpperCase();
+                  const points = (0, import_indicators.mergeCoinalyzeMetrics)({
+                    symbol: item.symbol,
+                    oiRaw: oiMap.get(marketSymbol) ?? [],
+                    fundingRaw: fundingMap.get(marketSymbol) ?? [],
+                    liqRaw: liqMap.get(marketSymbol) ?? []
+                  });
+                  return (0, import_indicators.coinalyzePointsToRows)(points, interval2, "coinalyze");
+                });
+                missingClosedSymbols = mode === "signals" ? getMissingClosedDerivativesSymbols({
+                  symbols: missingBatch.map((item) => item.symbol),
+                  rows,
+                  expectedTimestamp: toMs
+                }) : [];
+                if (!missingClosedSymbols.length) break;
+                if (attempt < maxClosedBarAttempts) {
+                  await (0, import_async.delay)(closedBarRetryDelayMs);
+                }
+              }
+              if (missingClosedSymbols.length) {
+                throw new Error(
+                  `Coinalyze closed ${interval2} bar ${new Date(toMs).toISOString()} unavailable for ${missingClosedSymbols.join(",")}`
+                );
+              }
               if (rows.length) {
                 await (0, import_timescale2.upsertDerivatives)(rows);
                 totalRows += rows.length;
               }
-              const rowsCountBySymbol = /* @__PURE__ */ new Map();
-              for (const row of rows) {
-                const symbol = row.symbol.toUpperCase();
-                rowsCountBySymbol.set(
-                  symbol,
-                  (rowsCountBySymbol.get(symbol) ?? 0) + 1
-                );
-              }
-              const coverageRows = missingBatch.map((item) => {
-                const normalizedSymbol = item.symbol.toUpperCase();
-                const rowsCount = rowsCountBySymbol.get(normalizedSymbol) ?? 0;
-                return {
-                  source: "coinalyze",
-                  symbol: item.symbol,
-                  interval: interval2,
-                  fromMs: cursor,
-                  toMs,
-                  rowsCount
-                };
-              });
-              await (0, import_timescale2.upsertDerivativesBackfillCoverage)(coverageRows);
-              for (const coverageRow of coverageRows) {
-                const symbol = coverageRow.symbol.toUpperCase();
-                edgesBySymbol.set(
-                  symbol,
-                  extendEdges(edgesBySymbol.get(symbol), cursor, toMs)
-                );
-                coverageKeys.add(
-                  coverageKey({
+              if (mode === "backtest") {
+                const rowsCountBySymbol = /* @__PURE__ */ new Map();
+                for (const row of rows) {
+                  const symbol = row.symbol.toUpperCase();
+                  rowsCountBySymbol.set(
                     symbol,
+                    (rowsCountBySymbol.get(symbol) ?? 0) + 1
+                  );
+                }
+                const coverageRows = missingBatch.map((item) => {
+                  const normalizedSymbol = item.symbol.toUpperCase();
+                  const rowsCount = rowsCountBySymbol.get(normalizedSymbol) ?? 0;
+                  return {
+                    source: "coinalyze",
+                    symbol: item.symbol,
                     interval: interval2,
                     fromMs: cursor,
-                    toMs
-                  })
-                );
-                const coverageRanges = coverageRangesBySymbol.get(symbol) ?? [];
-                coverageRanges.push({ fromMs: cursor, toMs });
-                coverageRangesBySymbol.set(symbol, coverageRanges);
+                    toMs,
+                    rowsCount
+                  };
+                });
+                await (0, import_timescale2.upsertDerivativesBackfillCoverage)(coverageRows);
+                for (const coverageRow of coverageRows) {
+                  const symbol = coverageRow.symbol.toUpperCase();
+                  edgesBySymbol.set(
+                    symbol,
+                    extendEdges(edgesBySymbol.get(symbol), cursor, toMs)
+                  );
+                  coverageKeys.add(
+                    coverageKey({
+                      symbol,
+                      interval: interval2,
+                      fromMs: cursor,
+                      toMs
+                    })
+                  );
+                  const coverageRanges = coverageRangesBySymbol.get(symbol) ?? [];
+                  coverageRanges.push({ fromMs: cursor, toMs });
+                  coverageRangesBySymbol.set(symbol, coverageRanges);
+                }
               }
             }
           }