@tradejs/cli 2.0.0 → 2.0.1

package/dist/cli.js

@@ -6151,7 +6151,6 @@ var init_aiTrainResearch = __esm({
       "INTERVAL",
       "DERIVATIVES_CONTEXT_ENABLED",
       "DERIVATIVES_CONTEXT_TARGET_ENABLED",
-      "DERIVATIVES_CONTEXT_INTERVALS",
       "DERIVATIVES_CONTEXT_LOOKBACK_HOURS",
       "DERIVATIVES_CONTEXT_EXTRA_REFERENCE_SYMBOLS",
       "DERIVATIVES_CONTEXT_EXCHANGE_PRIORITY",
@@ -8591,7 +8590,7 @@ var init_binanceMarketContextBackfill = __esm({
 });
 
 // src/lib/derivativesContextBackfill.ts
-var import_chalk7, import_progress4, import_async, import_constants5, import_indicators, import_timescale2, import_userSettings2, HOUR_MS, DAY_MS6, DEFAULT_LOOKBACK_HOURS, DEFAULT_INTERVALS, coinalyzeIntervalMap, derivativesIntervalMs, lastRequestTs, asInt2, parseList, parseBooleanFlag, normalizeSymbols, isDerivativesTargetContextEnabled, resolveDerivativesContextBackfillSymbols, chunkArray, isBacktestDerivativesContextEnabled, isSignalsDerivativesContextEnabled, isDerivativesContextBackfillEnabled, shouldBackfillDerivativesContextForBacktest, shouldBackfillDerivativesContextForSignals, resolveDerivativesContextIntervals, resolveDerivativesContextLookbackMs, resolveDerivativesContextBackfillWindow, resolveDerivativesContextIntervalBackfillWindow, resolveDerivativesContextMissingFetchFromMs, countBackfillWindows, buildBackfillWindows, hasDerivativesWindowCoverage, resolveDerivativesContextMissingCoverageFetchFromMs, groupDerivativesContextMissingFetchRanges, coverageKey, extendEdges, getCoinalyzeApiKey, getCoinalyzeBaseUrl, getRequestDelayMs, getRequestTimeoutMs, networkErrorCodes, getNestedErrorValue, getCoinalyzeErrorCause, isRetryableCoinalyzeFetchError, formatCoinalyzeRequestError, fetchJsonWithRateLimit, fetchCoinalyzeMarkets, selectBestMarket, buildMatches, normalizeMetricPoint, toSeriesMap, fetchMetricBatch, skippedBackfillResult2, backfillDerivativesContext, backfillDerivativesContextForBacktest, backfillDerivativesContextForSignals;
+var import_chalk7, import_progress4, import_async, import_constants5, import_indicators, import_timescale2, import_userSettings2, HOUR_MS, DAY_MS6, DEFAULT_LOOKBACK_HOURS, SOURCE_INTERVALS, coinalyzeIntervalMap, derivativesIntervalMs, lastRequestTs, asInt2, parseList, parseBooleanFlag, normalizeSymbols, isDerivativesTargetContextEnabled, resolveDerivativesContextBackfillSymbols, chunkArray, isBacktestDerivativesContextEnabled, isSignalsDerivativesContextEnabled, isDerivativesContextBackfillEnabled, shouldBackfillDerivativesContextForBacktest, shouldBackfillDerivativesContextForSignals, resolveDerivativesContextIntervals, resolveDerivativesContextLookbackMs, resolveDerivativesContextBackfillWindow, resolveDerivativesContextIntervalBackfillWindow, resolveDerivativesContextMissingFetchFromMs, resolveDerivativesContextFetchFromMs, countBackfillWindows, buildBackfillWindows, hasDerivativesWindowCoverage, resolveDerivativesContextMissingCoverageFetchFromMs, groupDerivativesContextMissingFetchRanges, getMissingClosedDerivativesSymbols, coverageKey, extendEdges, getCoinalyzeApiKey, getCoinalyzeBaseUrl, getRequestDelayMs, getRequestTimeoutMs, networkErrorCodes, getNestedErrorValue, getCoinalyzeErrorCause, isRetryableCoinalyzeFetchError, formatCoinalyzeRequestError, fetchJsonWithRateLimit, fetchCoinalyzeMarkets, selectBestMarket, buildMatches, normalizeMetricPoint, toSeriesMap, fetchMetricBatch, skippedBackfillResult2, backfillDerivativesContext, backfillDerivativesContextForBacktest, backfillDerivativesContextForSignals;
 var init_derivativesContextBackfill = __esm({
   "src/lib/derivativesContextBackfill.ts"() {
     "use strict";
@@ -8605,7 +8604,7 @@ var init_derivativesContextBackfill = __esm({
     HOUR_MS = 60 * 60 * 1e3;
     DAY_MS6 = 24 * HOUR_MS;
     DEFAULT_LOOKBACK_HOURS = 48;
-    DEFAULT_INTERVALS = ["15m", "1h"];
+    SOURCE_INTERVALS = ["15m"];
     coinalyzeIntervalMap = {
       "15m": "15min",
       "1h": "1hour"
@@ -8666,10 +8665,7 @@ var init_derivativesContextBackfill = __esm({
     shouldBackfillDerivativesContextForBacktest = (params) => !params.cacheOnly && (params.aiEnabled || params.mlEnabled) && isBacktestDerivativesContextEnabled();
     shouldBackfillDerivativesContextForSignals = (params) => !params.cacheOnly && isSignalsDerivativesContextEnabled();
     resolveDerivativesContextIntervals = () => {
-      const intervals = (0, import_indicators.normalizeDerivativesIntervals)(
-        process.env.DERIVATIVES_CONTEXT_INTERVALS
-      );
-      return intervals.length ? intervals : DEFAULT_INTERVALS;
+      return [...SOURCE_INTERVALS];
     };
     resolveDerivativesContextLookbackMs = () => {
       const hours = Number(process.env.DERIVATIVES_CONTEXT_LOOKBACK_HOURS);
@@ -8704,9 +8700,10 @@ var init_derivativesContextBackfill = __esm({
     };
     resolveDerivativesContextIntervalBackfillWindow = (params) => {
       const intervalMs2 = derivativesIntervalMs(params.interval);
+      const alignedToMs = params.closedOnly ? (0, import_indicators.getLastClosedDerivativesBarStartMs)(params.toMs, params.interval) : Math.floor(params.toMs / intervalMs2) * intervalMs2;
       return {
         fromMs: Math.floor(params.fromMs / intervalMs2) * intervalMs2,
-        toMs: Math.floor(params.toMs / intervalMs2) * intervalMs2,
+        toMs: alignedToMs,
         intervalMs: intervalMs2
       };
     };
@@ -8724,6 +8721,7 @@ var init_derivativesContextBackfill = __esm({
       }
       return null;
     };
+    resolveDerivativesContextFetchFromMs = (params) => resolveDerivativesContextMissingFetchFromMs(params) ?? (params.refreshClosedTail ? params.toMs : null);
     countBackfillWindows = (params) => {
       let count = 0;
       let cursor = params.fromMs;
@@ -8787,6 +8785,14 @@ var init_derivativesContextBackfill = __esm({
       }
       return [...groups.entries()].sort(([a], [b]) => a - b).map(([fromMs, items]) => ({ fromMs, items }));
     };
+    getMissingClosedDerivativesSymbols = (params) => {
+      const availableSymbols = new Set(
+        params.rows.filter((row) => row.ts.getTime() === params.expectedTimestamp).map((row) => row.symbol.trim().toUpperCase())
+      );
+      return params.symbols.filter(
+        (symbol) => !availableSymbols.has(symbol.trim().toUpperCase())
+      );
+    };
     coverageKey = (params) => [
       params.symbol.trim().toUpperCase(),
       params.interval,
@@ -9066,7 +9072,8 @@ var init_derivativesContextBackfill = __esm({
         ...resolveDerivativesContextIntervalBackfillWindow({
           fromMs,
           toMs: safeEndMs,
-          interval: interval5
+          interval: interval5,
+          closedOnly: mode === "signals"
         })
       })).filter((item) => item.toMs > item.fromMs);
       if (!intervalWindows.length) {
@@ -9125,7 +9132,7 @@ var init_derivativesContextBackfill = __esm({
         }),
         0
       );
-      const allBackfillWindowsCached = intervalWindows.every((window2) => {
+      const allBackfillWindowsCached = mode !== "signals" && intervalWindows.every((window2) => {
         const edgesBySymbol = edgesByInterval.get(window2.interval);
         const coverageKeys = coverageKeysByInterval.get(window2.interval);
         const backfillWindows = buildBackfillWindows({
@@ -9255,7 +9262,7 @@ var init_derivativesContextBackfill = __esm({
                 return null;
               }
               const normalizedSymbol = item.symbol.toUpperCase();
-              const coverageFromMs = resolveDerivativesContextMissingCoverageFetchFromMs({
+              const coverageFromMs = mode === "signals" ? cursor : resolveDerivativesContextMissingCoverageFetchFromMs({
                 ranges: coverageRangesBySymbol.get(normalizedSymbol),
                 fromMs: cursor,
                 toMs,
@@ -9265,16 +9272,20 @@ var init_derivativesContextBackfill = __esm({
                 return null;
               }
               const edges = edgesBySymbol.get(normalizedSymbol);
-              const edgesFromMs = resolveDerivativesContextMissingFetchFromMs({
+              const edgesFromMs = resolveDerivativesContextFetchFromMs({
                 edges,
                 fromMs: cursor,
                 toMs,
-                intervalMs: intervalMs2
+                intervalMs: intervalMs2,
+                refreshClosedTail: mode === "signals"
               });
               if (edgesFromMs == null) {
                 return null;
               }
-              return { item, fromMs: Math.max(coverageFromMs, edgesFromMs) };
+              return {
+                item,
+                fromMs: Math.max(coverageFromMs, edgesFromMs)
+              };
             }).filter(
               (item) => item != null
             );
@@ -9288,85 +9299,113 @@ var init_derivativesContextBackfill = __esm({
                   const marketSymbols = missingBatch.map(
                     (item) => item.marketSymbol
                   );
-                  const oiMap = await fetchMetricBatch({
-                    endpoint: oiPath,
-                    metric: "oi",
-                    marketSymbols,
-                    apiKey,
-                    interval: interval5,
-                    fromMs: group.fromMs,
-                    toMs
-                  });
-                  const fundingMap = await fetchMetricBatch({
-                    endpoint: fundingPath,
-                    metric: "funding",
-                    marketSymbols,
-                    apiKey,
-                    interval: interval5,
-                    fromMs: group.fromMs,
-                    toMs
-                  });
-                  const liqMap = await fetchMetricBatch({
-                    endpoint: liqPath,
-                    metric: "liq",
-                    marketSymbols,
-                    apiKey,
-                    interval: interval5,
-                    fromMs: group.fromMs,
-                    toMs
-                  });
-                  const rows = missingBatch.flatMap((item) => {
-                    const marketSymbol = item.marketSymbol.toUpperCase();
-                    const points = (0, import_indicators.mergeCoinalyzeMetrics)({
-                      symbol: item.symbol,
-                      oiRaw: oiMap.get(marketSymbol) ?? [],
-                      fundingRaw: fundingMap.get(marketSymbol) ?? [],
-                      liqRaw: liqMap.get(marketSymbol) ?? []
+                  const maxClosedBarAttempts = mode === "signals" ? asInt2(
+                    process.env.DERIVATIVES_CONTEXT_CLOSED_BAR_MAX_ATTEMPTS,
+                    3
+                  ) : 1;
+                  const closedBarRetryDelayMs = asInt2(
+                    process.env.DERIVATIVES_CONTEXT_CLOSED_BAR_RETRY_DELAY_MS,
+                    2e3
+                  );
+                  let rows = [];
+                  let missingClosedSymbols = [];
+                  for (let attempt = 1; attempt <= maxClosedBarAttempts; attempt += 1) {
+                    const oiMap = await fetchMetricBatch({
+                      endpoint: oiPath,
+                      metric: "oi",
+                      marketSymbols,
+                      apiKey,
+                      interval: interval5,
+                      fromMs: group.fromMs,
+                      toMs
+                    });
+                    const fundingMap = await fetchMetricBatch({
+                      endpoint: fundingPath,
+                      metric: "funding",
+                      marketSymbols,
+                      apiKey,
+                      interval: interval5,
+                      fromMs: group.fromMs,
+                      toMs
+                    });
+                    const liqMap = await fetchMetricBatch({
+                      endpoint: liqPath,
+                      metric: "liq",
+                      marketSymbols,
+                      apiKey,
+                      interval: interval5,
+                      fromMs: group.fromMs,
+                      toMs
                     });
-                    return (0, import_indicators.coinalyzePointsToRows)(points, interval5, "coinalyze");
-                  });
+                    rows = missingBatch.flatMap((item) => {
+                      const marketSymbol = item.marketSymbol.toUpperCase();
+                      const points = (0, import_indicators.mergeCoinalyzeMetrics)({
+                        symbol: item.symbol,
+                        oiRaw: oiMap.get(marketSymbol) ?? [],
+                        fundingRaw: fundingMap.get(marketSymbol) ?? [],
+                        liqRaw: liqMap.get(marketSymbol) ?? []
+                      });
+                      return (0, import_indicators.coinalyzePointsToRows)(points, interval5, "coinalyze");
+                    });
+                    missingClosedSymbols = mode === "signals" ? getMissingClosedDerivativesSymbols({
+                      symbols: missingBatch.map((item) => item.symbol),
+                      rows,
+                      expectedTimestamp: toMs
+                    }) : [];
+                    if (!missingClosedSymbols.length) break;
+                    if (attempt < maxClosedBarAttempts) {
+                      await (0, import_async.delay)(closedBarRetryDelayMs);
+                    }
+                  }
+                  if (missingClosedSymbols.length) {
+                    throw new Error(
+                      `Coinalyze closed ${interval5} bar ${new Date(toMs).toISOString()} unavailable for ${missingClosedSymbols.join(",")}`
+                    );
+                  }
                   if (rows.length) {
                     await (0, import_timescale2.upsertDerivatives)(rows);
                     totalRows += rows.length;
                   }
-                  const rowsCountBySymbol = /* @__PURE__ */ new Map();
-                  for (const row of rows) {
-                    const symbol = row.symbol.toUpperCase();
-                    rowsCountBySymbol.set(
-                      symbol,
-                      (rowsCountBySymbol.get(symbol) ?? 0) + 1
-                    );
-                  }
-                  const coverageRows = missingBatch.map((item) => {
-                    const normalizedSymbol = item.symbol.toUpperCase();
-                    const rowsCount = rowsCountBySymbol.get(normalizedSymbol) ?? 0;
-                    return {
-                      source: "coinalyze",
-                      symbol: item.symbol,
-                      interval: interval5,
-                      fromMs: cursor,
-                      toMs,
-                      rowsCount
-                    };
-                  });
-                  await (0, import_timescale2.upsertDerivativesBackfillCoverage)(coverageRows);
-                  for (const coverageRow of coverageRows) {
-                    const symbol = coverageRow.symbol.toUpperCase();
-                    edgesBySymbol.set(
-                      symbol,
-                      extendEdges(edgesBySymbol.get(symbol), cursor, toMs)
-                    );
-                    coverageKeys.add(
-                      coverageKey({
+                  if (mode === "backtest") {
+                    const rowsCountBySymbol = /* @__PURE__ */ new Map();
+                    for (const row of rows) {
+                      const symbol = row.symbol.toUpperCase();
+                      rowsCountBySymbol.set(
                         symbol,
+                        (rowsCountBySymbol.get(symbol) ?? 0) + 1
+                      );
+                    }
+                    const coverageRows = missingBatch.map((item) => {
+                      const normalizedSymbol = item.symbol.toUpperCase();
+                      const rowsCount = rowsCountBySymbol.get(normalizedSymbol) ?? 0;
+                      return {
+                        source: "coinalyze",
+                        symbol: item.symbol,
                         interval: interval5,
                         fromMs: cursor,
-                        toMs
-                      })
-                    );
-                    const coverageRanges = coverageRangesBySymbol.get(symbol) ?? [];
-                    coverageRanges.push({ fromMs: cursor, toMs });
-                    coverageRangesBySymbol.set(symbol, coverageRanges);
+                        toMs,
+                        rowsCount
+                      };
+                    });
+                    await (0, import_timescale2.upsertDerivativesBackfillCoverage)(coverageRows);
+                    for (const coverageRow of coverageRows) {
+                      const symbol = coverageRow.symbol.toUpperCase();
+                      edgesBySymbol.set(
+                        symbol,
+                        extendEdges(edgesBySymbol.get(symbol), cursor, toMs)
+                      );
+                      coverageKeys.add(
+                        coverageKey({
+                          symbol,
+                          interval: interval5,
+                          fromMs: cursor,
+                          toMs
+                        })
+                      );
+                      const coverageRanges = coverageRangesBySymbol.get(symbol) ?? [];
+                      coverageRanges.push({ fromMs: cursor, toMs });
+                      coverageRangesBySymbol.set(symbol, coverageRanges);
+                    }
                   }
                 }
               }
@@ -18915,11 +18954,11 @@ var init_derivativesIngest = __esm({
     import_userSettings4 = require("@tradejs/infra/userSettings");
     import_connectors9 = require("@tradejs/connectors");
     import_args12.default.example(
-      "yarn ts-node ./src/scripts/derivativesIngest --provider coinalyze --symbols BTCUSDT,ETHUSDT --intervals 15m,1h --days 120",
+      "yarn ts-node ./src/scripts/derivativesIngest --provider coinalyze --symbols BTCUSDT,ETHUSDT --intervals 15m --days 120",
       "Ingest market features (derivatives/spread) into Timescale by provider"
     );
     import_args12.default.option(["s", "symbols"], "Comma-separated symbols", "BTCUSDT,ETHUSDT");
-    import_args12.default.option(["t", "intervals"], "Comma-separated intervals: 15m,1h", "15m,1h");
+    import_args12.default.option(["t", "intervals"], "Comma-separated intervals: 15m,1h", "15m");
     import_args12.default.option(["d", "days"], "Lookback in days", 120);
     import_args12.default.option(
       ["p", "provider"],
@@ -18956,9 +18995,16 @@ var init_derivativesIngest = __esm({
       let totalSpreadRows = 0;
       for (const symbol of symbols) {
         for (const interval5 of intervals) {
+          const lastClosedStartMs = (0, import_indicators2.getLastClosedDerivativesBarStartMs)(
+            now2,
+            interval5
+          );
           let cursor = fromMs;
-          while (cursor < now2) {
-            const toMs = Math.min(now2, cursor + batchDays * 24 * 60 * 60 * 1e3);
+          while (cursor < lastClosedStartMs) {
+            const toMs = Math.min(
+              lastClosedStartMs,
+              cursor + batchDays * 24 * 60 * 60 * 1e3
+            );
             process.stdout.write(
               `\r${import_chalk20.default.cyan(providerName)} ${import_chalk20.default.yellow(symbol)} ${interval5} ${new Date(cursor).toISOString()} .. ${new Date(toMs).toISOString()}   `
             );
@@ -19018,7 +19064,7 @@ var init_derivativesIngestCoinalyzeAll = __esm({
       "1h": "1hour"
     };
     import_args13.default.example(
-      "yarn ts-node ./src/scripts/derivativesIngestCoinalyzeAll --days 120 --intervals 15m,1h",
+      "yarn ts-node ./src/scripts/derivativesIngestCoinalyzeAll --days 120 --intervals 15m",
       "Fetch derivatives for all getTickers symbols matched to Coinalyze markets"
     );
     import_args13.default.option(["U", "user"], "User settings profile name from Redis", "root");
@@ -19026,7 +19072,7 @@ var init_derivativesIngestCoinalyzeAll = __esm({
     import_args13.default.option(["e", "exclude"], "Comma-separated exclude symbols");
     import_args13.default.option(["l", "tickersLimit"], "Tickers limit");
     import_args13.default.option(["c", "chunk"], "Chunk selector, e.g. 1/4");
-    import_args13.default.option(["i", "intervals"], "Intervals: 15m,1h", "15m,1h");
+    import_args13.default.option(["i", "intervals"], "Intervals: 15m,1h", "15m");
     import_args13.default.option(["d", "days"], "Lookback in days", 120);
     import_args13.default.option(["b", "batchDays"], "Request chunk size in days", 120);
     import_args13.default.option(
@@ -19295,12 +19341,16 @@ var init_derivativesIngestCoinalyzeAll = __esm({
       let totalRows = 0;
       let failedWindows = 0;
       for (const interval5 of intervals) {
+        const lastClosedStartMs = (0, import_indicators3.getLastClosedDerivativesBarStartMs)(now2, interval5);
         for (let batchIdx = 0; batchIdx < symbolBatches.length; batchIdx += 1) {
           const batch = symbolBatches[batchIdx];
           const marketSymbols = batch.map((item) => item.marketSymbol);
           let cursor = fromMs;
-          while (cursor < now2) {
-            const toMs = Math.min(now2, cursor + batchDays * 24 * 60 * 60 * 1e3);
+          while (cursor < lastClosedStartMs) {
+            const toMs = Math.min(
+              lastClosedStartMs,
+              cursor + batchDays * 24 * 60 * 60 * 1e3
+            );
             try {
               const oiMap = await fetchMetricBatch2({
                 endpoint: oiPath,

package/dist/lib/aiTrainResearch.js

@@ -273,7 +273,6 @@ var RESEARCH_CONTEXT_ENV_KEYS = [
   "INTERVAL",
   "DERIVATIVES_CONTEXT_ENABLED",
   "DERIVATIVES_CONTEXT_TARGET_ENABLED",
-  "DERIVATIVES_CONTEXT_INTERVALS",
   "DERIVATIVES_CONTEXT_LOOKBACK_HOURS",
   "DERIVATIVES_CONTEXT_EXTRA_REFERENCE_SYMBOLS",
   "DERIVATIVES_CONTEXT_EXCHANGE_PRIORITY",