@tradejs/cli 2.0.6 → 2.0.8

package/dist/lib/binanceMarketContextBackfill.js

@@ -46,6 +46,7 @@ var import_progress = __toESM(require("progress"));
 var import_chalk = __toESM(require("chalk"));
 var import_connectors = require("@tradejs/connectors");
 var import_connectors2 = require("@tradejs/node/connectors");
+var import_strategies = require("@tradejs/node/strategies");
 var import_timescale = require("@tradejs/infra/timescale");
 
 // src/lib/binanceMarketData.ts
@@ -55,14 +56,6 @@ var MARKET_FEATURE_INTERVAL_MS = {
   "15m": 9e5,
   "1h": 36e5
 };
-var STABLE_QUOTE_SYMBOLS = /* @__PURE__ */ new Set([
-  "USDCUSDT",
-  "FDUSDUSDT",
-  "TUSDUSDT",
-  "BUSDUSDT",
-  "USDPUSDT",
-  "DAIUSDT"
-]);
 var safeDivide = (num, den) => Number.isFinite(num) && Number.isFinite(den) && den !== 0 ? num / den : null;
 var mean = (values) => values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : null;
 var standardDeviation = (values) => {
@@ -106,9 +99,6 @@ var classifyBtcAltRegime = ({
   if (btcVsAltReturn24h < -5e-3) return "alt_lead";
   return "neutral";
 };
-var selectBreadthUniverseFromTickers = (tickers, limit) => tickers.filter(
-  (ticker) => ticker.symbol.endsWith("USDT") && ticker.symbol !== "BTCUSDT" && !STABLE_QUOTE_SYMBOLS.has(ticker.symbol) && Number.isFinite(ticker.turnover24h) && ticker.turnover24h > 0
-).sort((a, b) => b.turnover24h - a.turnover24h).slice(0, Math.max(0, limit)).map((ticker) => ticker.symbol);
 var aggregateAggTradesToRows = ({
   symbol,
   interval,
@@ -590,10 +580,6 @@ var backfillBinanceMarketContext = async (params, enabled) => {
     process.env.BINANCE_MARKET_CONTEXT_TRADE_FLOW_SYMBOL_LIMIT,
     2
   );
-  const breadthLimit = asInt(
-    process.env.BINANCE_MARKET_CONTEXT_BREADTH_LIMIT,
-    30
-  );
   const breadthChunkDays = asFloat(
     process.env.BINANCE_MARKET_CONTEXT_BREADTH_CHUNK_DAYS,
     30
@@ -716,88 +702,113 @@ var backfillBinanceMarketContext = async (params, enabled) => {
     }
   }
   if (includeBreadth) {
-    const tickers = await connector.getTickers();
-    const breadthSymbols = selectBreadthUniverseFromTickers(
-      tickers,
-      breadthLimit
-    );
-    const universe = `binance_top${breadthSymbols.length}_usdt`;
-    const coverage = await (0, import_timescale.getMarketBreadthCoverage)({
-      universe,
-      interval,
-      startMs: breadthStartMs,
-      endMs
-    });
-    const hasBtcAltMetrics = (coverage?.btcAltMetricsRows ?? 0) >= Math.floor(Math.max(1, coverage?.rows ?? 0) * 0.9);
-    if (!hasCoverage({
-      coverage,
+    const breadthUniverses = (0, import_strategies.getBinanceBreadthUniverses)();
+    const breadthSymbols = breadthUniverses.at(-1)?.symbols ?? [];
+    const connectorInterval = marketIntervalToConnectorInterval(interval);
+    const chunks = buildBreadthBackfillChunks({
       startMs: breadthStartMs,
       endMs,
-      intervalMs
-    }) || !hasBtcAltMetrics) {
-      const connectorInterval = marketIntervalToConnectorInterval(interval);
-      const chunks = buildBreadthBackfillChunks({
-        startMs: breadthStartMs,
-        endMs,
-        intervalMs,
-        chunkDays: breadthChunkDays
-      });
-      const missingChunks = filterMissingBreadthBackfillChunks({
-        chunks,
-        coverage,
-        intervalMs
-      });
-      const bar = new import_progress.default(
-        "breadth :current/:total [:bar][:percent] :etas(s) candles=:candles skip=:skip chunk=:chunk :symbol",
-        {
-          total: Math.max(1, missingChunks.length * breadthSymbols.length),
-          width: 24
-        }
-      );
-      let candlesRead = 0;
-      let skippedBreadthChunks = chunks.length - missingChunks.length;
-      if (!missingChunks.length) {
-        bar.tick(1, {
-          candles: 0,
-          skip: skippedBreadthChunks,
-          chunk: "cached",
-          symbol: universe
+      intervalMs,
+      chunkDays: breadthChunkDays
+    });
+    const plans = await Promise.all(
+      breadthUniverses.map(async (definition) => {
+        const coverage = await (0, import_timescale.getMarketBreadthCoverage)({
+          universe: definition.universe,
+          interval,
+          startMs: breadthStartMs,
+          endMs
         });
-      }
-      for (let chunkIndex = 0; chunkIndex < missingChunks.length; chunkIndex += 1) {
-        const chunk = missingChunks[chunkIndex];
-        const originalChunkIndex = chunks.findIndex(
-          (item) => item.startMs === chunk.startMs && item.endMs === chunk.endMs
-        );
-        const candlesBySymbol = {};
-        for (const symbol of breadthSymbols) {
-          const candles = await connector.kline({
-            symbol,
-            interval: connectorInterval,
-            start: chunk.fetchStartMs,
-            end: chunk.endMs,
-            silent: true
-          });
-          candlesBySymbol[symbol] = candles;
-          candlesRead += candles.length;
-          bar.tick(1, {
-            candles: candlesRead,
-            skip: skippedBreadthChunks,
-            chunk: `${originalChunkIndex + 1}/${chunks.length}`,
-            symbol
-          });
+        const hasBtcAltMetrics = (coverage?.btcAltMetricsRows ?? 0) >= Math.floor(Math.max(1, coverage?.rows ?? 0) * 0.9);
+        const missingChunks2 = !hasCoverage({
+          coverage,
+          startMs: breadthStartMs,
+          endMs,
+          intervalMs
+        }) || !hasBtcAltMetrics ? !hasBtcAltMetrics ? chunks : filterMissingBreadthBackfillChunks({
+          chunks,
+          coverage,
+          intervalMs
+        }) : [];
+        if (!missingChunks2.length) {
+          console.log(
+            import_chalk.default.gray(`breadth cached: universe=${definition.universe}`)
+          );
         }
-        const btcCandles = await connector.kline({
-          symbol: "BTCUSDT",
+        return {
+          definition,
+          missingChunkKeys: new Set(
+            missingChunks2.map((chunk) => `${chunk.startMs}:${chunk.endMs}`)
+          )
+        };
+      })
+    );
+    const missingChunks = chunks.filter((chunk) => {
+      const key = `${chunk.startMs}:${chunk.endMs}`;
+      return plans.some((plan) => plan.missingChunkKeys.has(key));
+    });
+    const bar = new import_progress.default(
+      "breadth :current/:total [:bar][:percent] :etas(s) candles=:candles skip=:skip chunk=:chunk :symbol",
+      {
+        total: Math.max(1, missingChunks.length * breadthSymbols.length),
+        width: 24
+      }
+    );
+    let candlesRead = 0;
+    const skippedBreadthChunks = chunks.length - missingChunks.length;
+    if (!missingChunks.length) {
+      bar.tick(1, {
+        candles: 0,
+        skip: skippedBreadthChunks,
+        chunk: "cached",
+        symbol: "top5/top10/top30/top50/top100"
+      });
+    }
+    for (let chunkIndex = 0; chunkIndex < missingChunks.length; chunkIndex += 1) {
+      const chunk = missingChunks[chunkIndex];
+      const originalChunkIndex = chunks.findIndex(
+        (item) => item.startMs === chunk.startMs && item.endMs === chunk.endMs
+      );
+      const candlesBySymbol = {};
+      for (const symbol of breadthSymbols) {
+        const candles = await connector.kline({
+          symbol,
           interval: connectorInterval,
           start: chunk.fetchStartMs,
           end: chunk.endMs,
           silent: true
         });
+        candlesBySymbol[symbol] = candles;
+        candlesRead += candles.length;
+        bar.tick(1, {
+          candles: candlesRead,
+          skip: skippedBreadthChunks,
+          chunk: `${originalChunkIndex + 1}/${chunks.length}`,
+          symbol
+        });
+      }
+      const btcCandles = await connector.kline({
+        symbol: "BTCUSDT",
+        interval: connectorInterval,
+        start: chunk.fetchStartMs,
+        end: chunk.endMs,
+        silent: true
+      });
+      const chunkKey = `${chunk.startMs}:${chunk.endMs}`;
+      for (const plan of plans) {
+        if (!plan.missingChunkKeys.has(chunkKey)) {
+          continue;
+        }
+        const symbolSet = new Set(plan.definition.symbols);
+        const universeCandles = Object.fromEntries(
+          Object.entries(candlesBySymbol).filter(
+            ([symbol]) => symbolSet.has(symbol)
+          )
+        );
         const rows = buildMarketBreadthRows({
-          universe,
+          universe: plan.definition.universe,
           interval,
-          candlesBySymbol,
+          candlesBySymbol: universeCandles,
           btcCandles
         }).filter((row) => {
           const ts = row.ts.getTime();
@@ -806,8 +817,6 @@ var backfillBinanceMarketContext = async (params, enabled) => {
         await (0, import_timescale.upsertMarketBreadthRows)(rows);
         breadthRows += rows.length;
       }
-    } else {
-      console.log(import_chalk.default.gray(`breadth cached: universe=${universe}`));
     }
   }
   console.log(

package/dist/lib/binanceMarketData.js

@@ -106,14 +106,17 @@ var estimateBinanceMarketDataVolume = ({
   interval,
   includeAggTrades,
   includeBreadth,
-  breadthLimit
+  breadthSizes
 }) => {
   const intervalMs = MARKET_FEATURE_INTERVAL_MS[interval];
   const bucketRowsPerSymbol = Math.ceil(days * 864e5 / intervalMs);
   const aggTradeBucketRows = includeAggTrades ? symbols.length * bucketRowsPerSymbol : 0;
-  const breadthSymbols = includeBreadth ? Math.max(0, breadthLimit) : 0;
+  const normalizedBreadthSizes = breadthSizes.filter(
+    (size) => Number.isFinite(size) && size > 0
+  );
+  const breadthSymbols = includeBreadth ? Math.max(0, ...normalizedBreadthSizes) : 0;
   const breadthCandleRows = breadthSymbols * bucketRowsPerSymbol;
-  const breadthRows = includeBreadth ? bucketRowsPerSymbol : 0;
+  const breadthRows = includeBreadth ? bucketRowsPerSymbol * normalizedBreadthSizes.length : 0;
   return {
     interval,
     days,

package/dist/lib/marketContextPrepare.js

@@ -1649,6 +1649,7 @@ var import_progress = __toESM(require("progress"));
 var import_chalk = __toESM(require("chalk"));
 var import_connectors = require("@tradejs/connectors");
 var import_connectors2 = require("@tradejs/node/connectors");
+var import_strategies = require("@tradejs/node/strategies");
 var import_timescale = require("@tradejs/infra/timescale");
 
 // src/lib/binanceMarketData.ts
@@ -1658,14 +1659,6 @@ var MARKET_FEATURE_INTERVAL_MS = {
   "15m": 9e5,
   "1h": 36e5
 };
-var STABLE_QUOTE_SYMBOLS = /* @__PURE__ */ new Set([
-  "USDCUSDT",
-  "FDUSDUSDT",
-  "TUSDUSDT",
-  "BUSDUSDT",
-  "USDPUSDT",
-  "DAIUSDT"
-]);
 var safeDivide = (num, den) => Number.isFinite(num) && Number.isFinite(den) && den !== 0 ? num / den : null;
 var mean = (values) => values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : null;
 var standardDeviation = (values) => {
@@ -1709,9 +1702,6 @@ var classifyBtcAltRegime = ({
   if (btcVsAltReturn24h < -5e-3) return "alt_lead";
   return "neutral";
 };
-var selectBreadthUniverseFromTickers = (tickers, limit) => tickers.filter(
-  (ticker) => ticker.symbol.endsWith("USDT") && ticker.symbol !== "BTCUSDT" && !STABLE_QUOTE_SYMBOLS.has(ticker.symbol) && Number.isFinite(ticker.turnover24h) && ticker.turnover24h > 0
-).sort((a, b) => b.turnover24h - a.turnover24h).slice(0, Math.max(0, limit)).map((ticker) => ticker.symbol);
 var aggregateAggTradesToRows = ({
   symbol,
   interval,
@@ -2193,10 +2183,6 @@ var backfillBinanceMarketContext = async (params, enabled) => {
     process.env.BINANCE_MARKET_CONTEXT_TRADE_FLOW_SYMBOL_LIMIT,
     2
   );
-  const breadthLimit = asInt(
-    process.env.BINANCE_MARKET_CONTEXT_BREADTH_LIMIT,
-    30
-  );
   const breadthChunkDays = asFloat(
     process.env.BINANCE_MARKET_CONTEXT_BREADTH_CHUNK_DAYS,
     30
@@ -2319,88 +2305,113 @@ var backfillBinanceMarketContext = async (params, enabled) => {
     }
   }
   if (includeBreadth) {
-    const tickers = await connector.getTickers();
-    const breadthSymbols = selectBreadthUniverseFromTickers(
-      tickers,
-      breadthLimit
-    );
-    const universe = `binance_top${breadthSymbols.length}_usdt`;
-    const coverage = await (0, import_timescale.getMarketBreadthCoverage)({
-      universe,
-      interval,
-      startMs: breadthStartMs,
-      endMs
-    });
-    const hasBtcAltMetrics = (coverage?.btcAltMetricsRows ?? 0) >= Math.floor(Math.max(1, coverage?.rows ?? 0) * 0.9);
-    if (!hasCoverage({
-      coverage,
+    const breadthUniverses = (0, import_strategies.getBinanceBreadthUniverses)();
+    const breadthSymbols = breadthUniverses.at(-1)?.symbols ?? [];
+    const connectorInterval = marketIntervalToConnectorInterval(interval);
+    const chunks = buildBreadthBackfillChunks({
       startMs: breadthStartMs,
       endMs,
-      intervalMs
-    }) || !hasBtcAltMetrics) {
-      const connectorInterval = marketIntervalToConnectorInterval(interval);
-      const chunks = buildBreadthBackfillChunks({
-        startMs: breadthStartMs,
-        endMs,
-        intervalMs,
-        chunkDays: breadthChunkDays
-      });
-      const missingChunks = filterMissingBreadthBackfillChunks({
-        chunks,
-        coverage,
-        intervalMs
-      });
-      const bar = new import_progress.default(
-        "breadth :current/:total [:bar][:percent] :etas(s) candles=:candles skip=:skip chunk=:chunk :symbol",
-        {
-          total: Math.max(1, missingChunks.length * breadthSymbols.length),
-          width: 24
-        }
-      );
-      let candlesRead = 0;
-      let skippedBreadthChunks = chunks.length - missingChunks.length;
-      if (!missingChunks.length) {
-        bar.tick(1, {
-          candles: 0,
-          skip: skippedBreadthChunks,
-          chunk: "cached",
-          symbol: universe
+      intervalMs,
+      chunkDays: breadthChunkDays
+    });
+    const plans = await Promise.all(
+      breadthUniverses.map(async (definition) => {
+        const coverage = await (0, import_timescale.getMarketBreadthCoverage)({
+          universe: definition.universe,
+          interval,
+          startMs: breadthStartMs,
+          endMs
         });
-      }
-      for (let chunkIndex = 0; chunkIndex < missingChunks.length; chunkIndex += 1) {
-        const chunk = missingChunks[chunkIndex];
-        const originalChunkIndex = chunks.findIndex(
-          (item) => item.startMs === chunk.startMs && item.endMs === chunk.endMs
-        );
-        const candlesBySymbol = {};
-        for (const symbol of breadthSymbols) {
-          const candles = await connector.kline({
-            symbol,
-            interval: connectorInterval,
-            start: chunk.fetchStartMs,
-            end: chunk.endMs,
-            silent: true
-          });
-          candlesBySymbol[symbol] = candles;
-          candlesRead += candles.length;
-          bar.tick(1, {
-            candles: candlesRead,
-            skip: skippedBreadthChunks,
-            chunk: `${originalChunkIndex + 1}/${chunks.length}`,
-            symbol
-          });
+        const hasBtcAltMetrics = (coverage?.btcAltMetricsRows ?? 0) >= Math.floor(Math.max(1, coverage?.rows ?? 0) * 0.9);
+        const missingChunks2 = !hasCoverage({
+          coverage,
+          startMs: breadthStartMs,
+          endMs,
+          intervalMs
+        }) || !hasBtcAltMetrics ? !hasBtcAltMetrics ? chunks : filterMissingBreadthBackfillChunks({
+          chunks,
+          coverage,
+          intervalMs
+        }) : [];
+        if (!missingChunks2.length) {
+          console.log(
+            import_chalk.default.gray(`breadth cached: universe=${definition.universe}`)
+          );
         }
-        const btcCandles = await connector.kline({
-          symbol: "BTCUSDT",
+        return {
+          definition,
+          missingChunkKeys: new Set(
+            missingChunks2.map((chunk) => `${chunk.startMs}:${chunk.endMs}`)
+          )
+        };
+      })
+    );
+    const missingChunks = chunks.filter((chunk) => {
+      const key = `${chunk.startMs}:${chunk.endMs}`;
+      return plans.some((plan) => plan.missingChunkKeys.has(key));
+    });
+    const bar = new import_progress.default(
+      "breadth :current/:total [:bar][:percent] :etas(s) candles=:candles skip=:skip chunk=:chunk :symbol",
+      {
+        total: Math.max(1, missingChunks.length * breadthSymbols.length),
+        width: 24
+      }
+    );
+    let candlesRead = 0;
+    const skippedBreadthChunks = chunks.length - missingChunks.length;
+    if (!missingChunks.length) {
+      bar.tick(1, {
+        candles: 0,
+        skip: skippedBreadthChunks,
+        chunk: "cached",
+        symbol: "top5/top10/top30/top50/top100"
+      });
+    }
+    for (let chunkIndex = 0; chunkIndex < missingChunks.length; chunkIndex += 1) {
+      const chunk = missingChunks[chunkIndex];
+      const originalChunkIndex = chunks.findIndex(
+        (item) => item.startMs === chunk.startMs && item.endMs === chunk.endMs
+      );
+      const candlesBySymbol = {};
+      for (const symbol of breadthSymbols) {
+        const candles = await connector.kline({
+          symbol,
           interval: connectorInterval,
           start: chunk.fetchStartMs,
           end: chunk.endMs,
           silent: true
         });
+        candlesBySymbol[symbol] = candles;
+        candlesRead += candles.length;
+        bar.tick(1, {
+          candles: candlesRead,
+          skip: skippedBreadthChunks,
+          chunk: `${originalChunkIndex + 1}/${chunks.length}`,
+          symbol
+        });
+      }
+      const btcCandles = await connector.kline({
+        symbol: "BTCUSDT",
+        interval: connectorInterval,
+        start: chunk.fetchStartMs,
+        end: chunk.endMs,
+        silent: true
+      });
+      const chunkKey = `${chunk.startMs}:${chunk.endMs}`;
+      for (const plan of plans) {
+        if (!plan.missingChunkKeys.has(chunkKey)) {
+          continue;
+        }
+        const symbolSet = new Set(plan.definition.symbols);
+        const universeCandles = Object.fromEntries(
+          Object.entries(candlesBySymbol).filter(
+            ([symbol]) => symbolSet.has(symbol)
+          )
+        );
         const rows = buildMarketBreadthRows({
-          universe,
+          universe: plan.definition.universe,
           interval,
-          candlesBySymbol,
+          candlesBySymbol: universeCandles,
           btcCandles
         }).filter((row) => {
           const ts = row.ts.getTime();
@@ -2409,8 +2420,6 @@ var backfillBinanceMarketContext = async (params, enabled) => {
         await (0, import_timescale.upsertMarketBreadthRows)(rows);
         breadthRows += rows.length;
       }
-    } else {
-      console.log(import_chalk.default.gray(`breadth cached: universe=${universe}`));
     }
   }
   console.log(

package/dist/lib/runEnvironment.js

@@ -1675,11 +1675,30 @@ var resolveTimeWindow = ({
   endTime,
   nowMs = Date.now(),
   defaultStartMs,
-  defaultEndMs
+  defaultEndMs,
+  closedIntervalMs
 }) => {
   const explicitStart = toEpochMs(startTime);
   const explicitEnd = toEpochMs(endTime);
   const resolvedDays = toPositiveNumber(days);
+  const alignToLastClosedInterval = (window2, preserveDuration) => {
+    if (typeof closedIntervalMs !== "number" || !Number.isFinite(closedIntervalMs) || closedIntervalMs <= 0) {
+      return window2;
+    }
+    const alignedEnd = Math.floor(window2.end / closedIntervalMs) * closedIntervalMs - 1;
+    const delta = window2.end - alignedEnd;
+    const alignedStart = preserveDuration ? window2.start - delta : window2.start;
+    if (alignedStart >= alignedEnd) {
+      throw new Error(
+        `Invalid closed-candle window: start (${alignedStart}) must be less than end (${alignedEnd})`
+      );
+    }
+    return {
+      ...window2,
+      start: Math.trunc(alignedStart),
+      end: Math.trunc(alignedEnd)
+    };
+  };
   if (explicitStart != null || explicitEnd != null) {
     const resolvedStart = explicitStart ?? (resolvedDays != null && explicitEnd != null ? explicitEnd - resolvedDays * ONE_DAY_MS : defaultStartMs);
     const resolvedEnd2 = explicitEnd ?? (resolvedDays != null && explicitStart != null ? explicitStart + resolvedDays * ONE_DAY_MS : defaultEndMs ?? nowMs);
@@ -1688,11 +1707,14 @@ var resolveTimeWindow = ({
         `Invalid time window: startTime (${resolvedStart}) must be less than endTime (${resolvedEnd2})`
       );
     }
-    return {
-      start: resolvedStart,
-      end: resolvedEnd2,
-      source: "explicit"
-    };
+    return alignToLastClosedInterval(
+      {
+        start: resolvedStart,
+        end: resolvedEnd2,
+        source: "explicit"
+      },
+      explicitStart == null && resolvedDays != null
+    );
   }
   if (resolvedDays != null) {
     const resolvedEnd2 = defaultEndMs ?? nowMs;
@@ -1702,11 +1724,14 @@ var resolveTimeWindow = ({
         `Invalid time window: computed start (${resolvedStart}) must be less than end (${resolvedEnd2})`
       );
     }
-    return {
-      start: Math.trunc(resolvedStart),
-      end: Math.trunc(resolvedEnd2),
-      source: "days"
-    };
+    return alignToLastClosedInterval(
+      {
+        start: Math.trunc(resolvedStart),
+        end: Math.trunc(resolvedEnd2),
+        source: "days"
+      },
+      true
+    );
   }
   const resolvedEnd = defaultEndMs ?? nowMs;
   if (defaultStartMs >= resolvedEnd) {
@@ -1714,11 +1739,14 @@ var resolveTimeWindow = ({
       `Invalid default time window: start (${defaultStartMs}) must be less than end (${resolvedEnd})`
     );
   }
-  return {
-    start: Math.trunc(defaultStartMs),
-    end: Math.trunc(resolvedEnd),
-    source: "default"
-  };
+  return alignToLastClosedInterval(
+    {
+      start: Math.trunc(defaultStartMs),
+      end: Math.trunc(resolvedEnd),
+      source: "default"
+    },
+    true
+  );
 };
 
 // src/lib/runtimeStrategyBacktest.ts
@@ -2135,7 +2163,8 @@ var prepareRunEnvironment = async ({
   accountId,
   deploymentId,
   assetClasses,
-  deployment
+  deployment,
+  closedIntervalMs
 }) => {
   const connectorName = await resolveRunConnectorName({
     value: connector,
@@ -2198,7 +2227,8 @@ var prepareRunEnvironment = async ({
     startTime,
     endTime,
     defaultStartMs: (0, import_time2.getTimestamp)(import_constants.BACKTEST_DEFAULT_DAYS),
-    defaultEndMs: (0, import_time2.getTimestamp)()
+    defaultEndMs: (0, import_time2.getTimestamp)(),
+    closedIntervalMs
   });
   const preloadStart = (0, import_time2.getBacktestPreloadStart)(
     window2.start,