@tradejs/cli 2.0.5 → 2.0.7

package/dist/lib/binanceMarketContextBackfill.js

@@ -46,6 +46,7 @@ var import_progress = __toESM(require("progress"));
 var import_chalk = __toESM(require("chalk"));
 var import_connectors = require("@tradejs/connectors");
 var import_connectors2 = require("@tradejs/node/connectors");
+var import_strategies = require("@tradejs/node/strategies");
 var import_timescale = require("@tradejs/infra/timescale");
 
 // src/lib/binanceMarketData.ts
@@ -55,14 +56,6 @@ var MARKET_FEATURE_INTERVAL_MS = {
   "15m": 9e5,
   "1h": 36e5
 };
-var STABLE_QUOTE_SYMBOLS = /* @__PURE__ */ new Set([
-  "USDCUSDT",
-  "FDUSDUSDT",
-  "TUSDUSDT",
-  "BUSDUSDT",
-  "USDPUSDT",
-  "DAIUSDT"
-]);
 var safeDivide = (num, den) => Number.isFinite(num) && Number.isFinite(den) && den !== 0 ? num / den : null;
 var mean = (values) => values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : null;
 var standardDeviation = (values) => {
@@ -106,9 +99,6 @@ var classifyBtcAltRegime = ({
   if (btcVsAltReturn24h < -5e-3) return "alt_lead";
   return "neutral";
 };
-var selectBreadthUniverseFromTickers = (tickers, limit) => tickers.filter(
-  (ticker) => ticker.symbol.endsWith("USDT") && ticker.symbol !== "BTCUSDT" && !STABLE_QUOTE_SYMBOLS.has(ticker.symbol) && Number.isFinite(ticker.turnover24h) && ticker.turnover24h > 0
-).sort((a, b) => b.turnover24h - a.turnover24h).slice(0, Math.max(0, limit)).map((ticker) => ticker.symbol);
 var aggregateAggTradesToRows = ({
   symbol,
   interval,
@@ -590,10 +580,6 @@ var backfillBinanceMarketContext = async (params, enabled) => {
     process.env.BINANCE_MARKET_CONTEXT_TRADE_FLOW_SYMBOL_LIMIT,
     2
   );
-  const breadthLimit = asInt(
-    process.env.BINANCE_MARKET_CONTEXT_BREADTH_LIMIT,
-    30
-  );
   const breadthChunkDays = asFloat(
     process.env.BINANCE_MARKET_CONTEXT_BREADTH_CHUNK_DAYS,
     30
@@ -716,88 +702,113 @@ var backfillBinanceMarketContext = async (params, enabled) => {
     }
   }
   if (includeBreadth) {
-    const tickers = await connector.getTickers();
-    const breadthSymbols = selectBreadthUniverseFromTickers(
-      tickers,
-      breadthLimit
-    );
-    const universe = `binance_top${breadthSymbols.length}_usdt`;
-    const coverage = await (0, import_timescale.getMarketBreadthCoverage)({
-      universe,
-      interval,
-      startMs: breadthStartMs,
-      endMs
-    });
-    const hasBtcAltMetrics = (coverage?.btcAltMetricsRows ?? 0) >= Math.floor(Math.max(1, coverage?.rows ?? 0) * 0.9);
-    if (!hasCoverage({
-      coverage,
+    const breadthUniverses = (0, import_strategies.getBinanceBreadthUniverses)();
+    const breadthSymbols = breadthUniverses.at(-1)?.symbols ?? [];
+    const connectorInterval = marketIntervalToConnectorInterval(interval);
+    const chunks = buildBreadthBackfillChunks({
       startMs: breadthStartMs,
       endMs,
-      intervalMs
-    }) || !hasBtcAltMetrics) {
-      const connectorInterval = marketIntervalToConnectorInterval(interval);
-      const chunks = buildBreadthBackfillChunks({
-        startMs: breadthStartMs,
-        endMs,
-        intervalMs,
-        chunkDays: breadthChunkDays
-      });
-      const missingChunks = filterMissingBreadthBackfillChunks({
-        chunks,
-        coverage,
-        intervalMs
-      });
-      const bar = new import_progress.default(
-        "breadth :current/:total [:bar][:percent] :etas(s) candles=:candles skip=:skip chunk=:chunk :symbol",
-        {
-          total: Math.max(1, missingChunks.length * breadthSymbols.length),
-          width: 24
-        }
-      );
-      let candlesRead = 0;
-      let skippedBreadthChunks = chunks.length - missingChunks.length;
-      if (!missingChunks.length) {
-        bar.tick(1, {
-          candles: 0,
-          skip: skippedBreadthChunks,
-          chunk: "cached",
-          symbol: universe
+      intervalMs,
+      chunkDays: breadthChunkDays
+    });
+    const plans = await Promise.all(
+      breadthUniverses.map(async (definition) => {
+        const coverage = await (0, import_timescale.getMarketBreadthCoverage)({
+          universe: definition.universe,
+          interval,
+          startMs: breadthStartMs,
+          endMs
         });
-      }
-      for (let chunkIndex = 0; chunkIndex < missingChunks.length; chunkIndex += 1) {
-        const chunk = missingChunks[chunkIndex];
-        const originalChunkIndex = chunks.findIndex(
-          (item) => item.startMs === chunk.startMs && item.endMs === chunk.endMs
-        );
-        const candlesBySymbol = {};
-        for (const symbol of breadthSymbols) {
-          const candles = await connector.kline({
-            symbol,
-            interval: connectorInterval,
-            start: chunk.fetchStartMs,
-            end: chunk.endMs,
-            silent: true
-          });
-          candlesBySymbol[symbol] = candles;
-          candlesRead += candles.length;
-          bar.tick(1, {
-            candles: candlesRead,
-            skip: skippedBreadthChunks,
-            chunk: `${originalChunkIndex + 1}/${chunks.length}`,
-            symbol
-          });
+        const hasBtcAltMetrics = (coverage?.btcAltMetricsRows ?? 0) >= Math.floor(Math.max(1, coverage?.rows ?? 0) * 0.9);
+        const missingChunks2 = !hasCoverage({
+          coverage,
+          startMs: breadthStartMs,
+          endMs,
+          intervalMs
+        }) || !hasBtcAltMetrics ? !hasBtcAltMetrics ? chunks : filterMissingBreadthBackfillChunks({
+          chunks,
+          coverage,
+          intervalMs
+        }) : [];
+        if (!missingChunks2.length) {
+          console.log(
+            import_chalk.default.gray(`breadth cached: universe=${definition.universe}`)
+          );
         }
-        const btcCandles = await connector.kline({
-          symbol: "BTCUSDT",
+        return {
+          definition,
+          missingChunkKeys: new Set(
+            missingChunks2.map((chunk) => `${chunk.startMs}:${chunk.endMs}`)
+          )
+        };
+      })
+    );
+    const missingChunks = chunks.filter((chunk) => {
+      const key = `${chunk.startMs}:${chunk.endMs}`;
+      return plans.some((plan) => plan.missingChunkKeys.has(key));
+    });
+    const bar = new import_progress.default(
+      "breadth :current/:total [:bar][:percent] :etas(s) candles=:candles skip=:skip chunk=:chunk :symbol",
+      {
+        total: Math.max(1, missingChunks.length * breadthSymbols.length),
+        width: 24
+      }
+    );
+    let candlesRead = 0;
+    const skippedBreadthChunks = chunks.length - missingChunks.length;
+    if (!missingChunks.length) {
+      bar.tick(1, {
+        candles: 0,
+        skip: skippedBreadthChunks,
+        chunk: "cached",
+        symbol: "top5/top10/top30/top50/top100"
+      });
+    }
+    for (let chunkIndex = 0; chunkIndex < missingChunks.length; chunkIndex += 1) {
+      const chunk = missingChunks[chunkIndex];
+      const originalChunkIndex = chunks.findIndex(
+        (item) => item.startMs === chunk.startMs && item.endMs === chunk.endMs
+      );
+      const candlesBySymbol = {};
+      for (const symbol of breadthSymbols) {
+        const candles = await connector.kline({
+          symbol,
           interval: connectorInterval,
           start: chunk.fetchStartMs,
           end: chunk.endMs,
           silent: true
         });
+        candlesBySymbol[symbol] = candles;
+        candlesRead += candles.length;
+        bar.tick(1, {
+          candles: candlesRead,
+          skip: skippedBreadthChunks,
+          chunk: `${originalChunkIndex + 1}/${chunks.length}`,
+          symbol
+        });
+      }
+      const btcCandles = await connector.kline({
+        symbol: "BTCUSDT",
+        interval: connectorInterval,
+        start: chunk.fetchStartMs,
+        end: chunk.endMs,
+        silent: true
+      });
+      const chunkKey = `${chunk.startMs}:${chunk.endMs}`;
+      for (const plan of plans) {
+        if (!plan.missingChunkKeys.has(chunkKey)) {
+          continue;
+        }
+        const symbolSet = new Set(plan.definition.symbols);
+        const universeCandles = Object.fromEntries(
+          Object.entries(candlesBySymbol).filter(
+            ([symbol]) => symbolSet.has(symbol)
+          )
+        );
         const rows = buildMarketBreadthRows({
-          universe,
+          universe: plan.definition.universe,
           interval,
-          candlesBySymbol,
+          candlesBySymbol: universeCandles,
           btcCandles
         }).filter((row) => {
           const ts = row.ts.getTime();
@@ -806,8 +817,6 @@ var backfillBinanceMarketContext = async (params, enabled) => {
         await (0, import_timescale.upsertMarketBreadthRows)(rows);
         breadthRows += rows.length;
       }
-    } else {
-      console.log(import_chalk.default.gray(`breadth cached: universe=${universe}`));
     }
   }
   console.log(

package/dist/lib/binanceMarketData.js

@@ -106,14 +106,17 @@ var estimateBinanceMarketDataVolume = ({
   interval,
   includeAggTrades,
   includeBreadth,
-  breadthLimit
+  breadthSizes
 }) => {
   const intervalMs = MARKET_FEATURE_INTERVAL_MS[interval];
   const bucketRowsPerSymbol = Math.ceil(days * 864e5 / intervalMs);
   const aggTradeBucketRows = includeAggTrades ? symbols.length * bucketRowsPerSymbol : 0;
-  const breadthSymbols = includeBreadth ? Math.max(0, breadthLimit) : 0;
+  const normalizedBreadthSizes = breadthSizes.filter(
+    (size) => Number.isFinite(size) && size > 0
+  );
+  const breadthSymbols = includeBreadth ? Math.max(0, ...normalizedBreadthSizes) : 0;
   const breadthCandleRows = breadthSymbols * bucketRowsPerSymbol;
-  const breadthRows = includeBreadth ? bucketRowsPerSymbol : 0;
+  const breadthRows = includeBreadth ? bucketRowsPerSymbol * normalizedBreadthSizes.length : 0;
   return {
     interval,
     days,

package/dist/lib/marketContextPrepare.js

@@ -34,9 +34,9 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 ));
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 
-// ../../node_modules/debug/node_modules/ms/index.js
+// ../../node_modules/ms/index.js
 var require_ms = __commonJS({
-  "../../node_modules/debug/node_modules/ms/index.js"(exports2, module2) {
+  "../../node_modules/ms/index.js"(exports2, module2) {
     "use strict";
     var s = 1e3;
     var m = s * 60;
@@ -151,9 +151,9 @@ var require_ms = __commonJS({
   }
 });
 
-// ../../node_modules/debug/src/common.js
+// ../../node_modules/list-it/node_modules/debug/src/common.js
 var require_common = __commonJS({
-  "../../node_modules/debug/src/common.js"(exports2, module2) {
+  "../../node_modules/list-it/node_modules/debug/src/common.js"(exports2, module2) {
     "use strict";
     function setup(env) {
       createDebug.debug = createDebug;
@@ -255,50 +255,64 @@ var require_common = __commonJS({
         createDebug.namespaces = namespaces;
         createDebug.names = [];
         createDebug.skips = [];
-        let i;
-        const split = (typeof namespaces === "string" ? namespaces : "").split(/[\s,]+/);
-        const len = split.length;
-        for (i = 0; i < len; i++) {
-          if (!split[i]) {
-            continue;
+        const split = (typeof namespaces === "string" ? namespaces : "").trim().replace(/\s+/g, ",").split(",").filter(Boolean);
+        for (const ns of split) {
+          if (ns[0] === "-") {
+            createDebug.skips.push(ns.slice(1));
+          } else {
+            createDebug.names.push(ns);
           }
-          namespaces = split[i].replace(/\*/g, ".*?");
-          if (namespaces[0] === "-") {
-            createDebug.skips.push(new RegExp("^" + namespaces.slice(1) + "$"));
+        }
+      }
+      function matchesTemplate(search, template) {
+        let searchIndex = 0;
+        let templateIndex = 0;
+        let starIndex = -1;
+        let matchIndex = 0;
+        while (searchIndex < search.length) {
+          if (templateIndex < template.length && (template[templateIndex] === search[searchIndex] || template[templateIndex] === "*")) {
+            if (template[templateIndex] === "*") {
+              starIndex = templateIndex;
+              matchIndex = searchIndex;
+              templateIndex++;
+            } else {
+              searchIndex++;
+              templateIndex++;
+            }
+          } else if (starIndex !== -1) {
+            templateIndex = starIndex + 1;
+            matchIndex++;
+            searchIndex = matchIndex;
           } else {
-            createDebug.names.push(new RegExp("^" + namespaces + "$"));
+            return false;
           }
         }
+        while (templateIndex < template.length && template[templateIndex] === "*") {
+          templateIndex++;
+        }
+        return templateIndex === template.length;
       }
       function disable() {
         const namespaces = [
-          ...createDebug.names.map(toNamespace),
-          ...createDebug.skips.map(toNamespace).map((namespace) => "-" + namespace)
+          ...createDebug.names,
+          ...createDebug.skips.map((namespace) => "-" + namespace)
         ].join(",");
         createDebug.enable("");
         return namespaces;
       }
       function enabled(name) {
-        if (name[name.length - 1] === "*") {
-          return true;
-        }
-        let i;
-        let len;
-        for (i = 0, len = createDebug.skips.length; i < len; i++) {
-          if (createDebug.skips[i].test(name)) {
+        for (const skip of createDebug.skips) {
+          if (matchesTemplate(name, skip)) {
             return false;
           }
         }
-        for (i = 0, len = createDebug.names.length; i < len; i++) {
-          if (createDebug.names[i].test(name)) {
+        for (const ns of createDebug.names) {
+          if (matchesTemplate(name, ns)) {
             return true;
           }
         }
         return false;
       }
-      function toNamespace(regexp) {
-        return regexp.toString().substring(2, regexp.toString().length - 2).replace(/\.\*\?$/, "*");
-      }
       function coerce(val) {
         if (val instanceof Error) {
           return val.stack || val.message;
@@ -315,9 +329,9 @@ var require_common = __commonJS({
   }
 });
 
-// ../../node_modules/debug/src/browser.js
+// ../../node_modules/list-it/node_modules/debug/src/browser.js
 var require_browser = __commonJS({
-  "../../node_modules/debug/src/browser.js"(exports2, module2) {
+  "../../node_modules/list-it/node_modules/debug/src/browser.js"(exports2, module2) {
     "use strict";
     exports2.formatArgs = formatArgs;
     exports2.save = save;
@@ -418,10 +432,11 @@ var require_browser = __commonJS({
       if (typeof navigator !== "undefined" && navigator.userAgent && navigator.userAgent.toLowerCase().match(/(edge|trident)\/(\d+)/)) {
         return false;
       }
+      let m;
       return typeof document !== "undefined" && document.documentElement && document.documentElement.style && document.documentElement.style.WebkitAppearance || // Is firebug? http://stackoverflow.com/a/398120/376773
       typeof window !== "undefined" && window.console && (window.console.firebug || window.console.exception && window.console.table) || // Is firefox >= v31?
       // https://developer.mozilla.org/en-US/docs/Tools/Web_Console#Styling_messages
-      typeof navigator !== "undefined" && navigator.userAgent && navigator.userAgent.toLowerCase().match(/firefox\/(\d+)/) && parseInt(RegExp.$1, 10) >= 31 || // Double check webkit in userAgent just in case we are in a worker
+      typeof navigator !== "undefined" && navigator.userAgent && (m = navigator.userAgent.toLowerCase().match(/firefox\/(\d+)/)) && parseInt(m[1], 10) >= 31 || // Double check webkit in userAgent just in case we are in a worker
       typeof navigator !== "undefined" && navigator.userAgent && navigator.userAgent.toLowerCase().match(/applewebkit\/(\d+)/);
     }
     function formatArgs(args) {
@@ -459,7 +474,7 @@ var require_browser = __commonJS({
     function load() {
       let r;
       try {
-        r = exports2.storage.getItem("debug");
+        r = exports2.storage.getItem("debug") || exports2.storage.getItem("DEBUG");
       } catch (error) {
       }
       if (!r && typeof process !== "undefined" && "env" in process) {
@@ -600,9 +615,9 @@ var require_supports_color = __commonJS({
   }
 });
 
-// ../../node_modules/debug/src/node.js
+// ../../node_modules/list-it/node_modules/debug/src/node.js
 var require_node = __commonJS({
-  "../../node_modules/debug/src/node.js"(exports2, module2) {
+  "../../node_modules/list-it/node_modules/debug/src/node.js"(exports2, module2) {
     "use strict";
     var tty = require("tty");
     var util = require("util");
@@ -743,7 +758,7 @@ var require_node = __commonJS({
       return (/* @__PURE__ */ new Date()).toISOString() + " ";
     }
     function log(...args) {
-      return process.stderr.write(util.format(...args) + "\n");
+      return process.stderr.write(util.formatWithOptions(exports2.inspectOpts, ...args) + "\n");
     }
     function save(namespaces) {
       if (namespaces) {
@@ -775,9 +790,9 @@ var require_node = __commonJS({
   }
 });
 
-// ../../node_modules/debug/src/index.js
+// ../../node_modules/list-it/node_modules/debug/src/index.js
 var require_src = __commonJS({
-  "../../node_modules/debug/src/index.js"(exports2, module2) {
+  "../../node_modules/list-it/node_modules/debug/src/index.js"(exports2, module2) {
     "use strict";
     if (typeof process === "undefined" || process.type === "renderer" || process.browser === true || process.__nwjs) {
       module2.exports = require_browser();
@@ -1634,6 +1649,7 @@ var import_progress = __toESM(require("progress"));
 var import_chalk = __toESM(require("chalk"));
 var import_connectors = require("@tradejs/connectors");
 var import_connectors2 = require("@tradejs/node/connectors");
+var import_strategies = require("@tradejs/node/strategies");
 var import_timescale = require("@tradejs/infra/timescale");
 
 // src/lib/binanceMarketData.ts
@@ -1643,14 +1659,6 @@ var MARKET_FEATURE_INTERVAL_MS = {
   "15m": 9e5,
   "1h": 36e5
 };
-var STABLE_QUOTE_SYMBOLS = /* @__PURE__ */ new Set([
-  "USDCUSDT",
-  "FDUSDUSDT",
-  "TUSDUSDT",
-  "BUSDUSDT",
-  "USDPUSDT",
-  "DAIUSDT"
-]);
 var safeDivide = (num, den) => Number.isFinite(num) && Number.isFinite(den) && den !== 0 ? num / den : null;
 var mean = (values) => values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : null;
 var standardDeviation = (values) => {
@@ -1694,9 +1702,6 @@ var classifyBtcAltRegime = ({
   if (btcVsAltReturn24h < -5e-3) return "alt_lead";
   return "neutral";
 };
-var selectBreadthUniverseFromTickers = (tickers, limit) => tickers.filter(
-  (ticker) => ticker.symbol.endsWith("USDT") && ticker.symbol !== "BTCUSDT" && !STABLE_QUOTE_SYMBOLS.has(ticker.symbol) && Number.isFinite(ticker.turnover24h) && ticker.turnover24h > 0
-).sort((a, b) => b.turnover24h - a.turnover24h).slice(0, Math.max(0, limit)).map((ticker) => ticker.symbol);
 var aggregateAggTradesToRows = ({
   symbol,
   interval,
@@ -2178,10 +2183,6 @@ var backfillBinanceMarketContext = async (params, enabled) => {
     process.env.BINANCE_MARKET_CONTEXT_TRADE_FLOW_SYMBOL_LIMIT,
     2
   );
-  const breadthLimit = asInt(
-    process.env.BINANCE_MARKET_CONTEXT_BREADTH_LIMIT,
-    30
-  );
   const breadthChunkDays = asFloat(
     process.env.BINANCE_MARKET_CONTEXT_BREADTH_CHUNK_DAYS,
     30
@@ -2304,88 +2305,113 @@ var backfillBinanceMarketContext = async (params, enabled) => {
     }
   }
   if (includeBreadth) {
-    const tickers = await connector.getTickers();
-    const breadthSymbols = selectBreadthUniverseFromTickers(
-      tickers,
-      breadthLimit
-    );
-    const universe = `binance_top${breadthSymbols.length}_usdt`;
-    const coverage = await (0, import_timescale.getMarketBreadthCoverage)({
-      universe,
-      interval,
-      startMs: breadthStartMs,
-      endMs
-    });
-    const hasBtcAltMetrics = (coverage?.btcAltMetricsRows ?? 0) >= Math.floor(Math.max(1, coverage?.rows ?? 0) * 0.9);
-    if (!hasCoverage({
-      coverage,
+    const breadthUniverses = (0, import_strategies.getBinanceBreadthUniverses)();
+    const breadthSymbols = breadthUniverses.at(-1)?.symbols ?? [];
+    const connectorInterval = marketIntervalToConnectorInterval(interval);
+    const chunks = buildBreadthBackfillChunks({
       startMs: breadthStartMs,
       endMs,
-      intervalMs
-    }) || !hasBtcAltMetrics) {
-      const connectorInterval = marketIntervalToConnectorInterval(interval);
-      const chunks = buildBreadthBackfillChunks({
-        startMs: breadthStartMs,
-        endMs,
-        intervalMs,
-        chunkDays: breadthChunkDays
-      });
-      const missingChunks = filterMissingBreadthBackfillChunks({
-        chunks,
-        coverage,
-        intervalMs
-      });
-      const bar = new import_progress.default(
-        "breadth :current/:total [:bar][:percent] :etas(s) candles=:candles skip=:skip chunk=:chunk :symbol",
-        {
-          total: Math.max(1, missingChunks.length * breadthSymbols.length),
-          width: 24
-        }
-      );
-      let candlesRead = 0;
-      let skippedBreadthChunks = chunks.length - missingChunks.length;
-      if (!missingChunks.length) {
-        bar.tick(1, {
-          candles: 0,
-          skip: skippedBreadthChunks,
-          chunk: "cached",
-          symbol: universe
+      intervalMs,
+      chunkDays: breadthChunkDays
+    });
+    const plans = await Promise.all(
+      breadthUniverses.map(async (definition) => {
+        const coverage = await (0, import_timescale.getMarketBreadthCoverage)({
+          universe: definition.universe,
+          interval,
+          startMs: breadthStartMs,
+          endMs
         });
-      }
-      for (let chunkIndex = 0; chunkIndex < missingChunks.length; chunkIndex += 1) {
-        const chunk = missingChunks[chunkIndex];
-        const originalChunkIndex = chunks.findIndex(
-          (item) => item.startMs === chunk.startMs && item.endMs === chunk.endMs
-        );
-        const candlesBySymbol = {};
-        for (const symbol of breadthSymbols) {
-          const candles = await connector.kline({
-            symbol,
-            interval: connectorInterval,
-            start: chunk.fetchStartMs,
-            end: chunk.endMs,
-            silent: true
-          });
-          candlesBySymbol[symbol] = candles;
-          candlesRead += candles.length;
-          bar.tick(1, {
-            candles: candlesRead,
-            skip: skippedBreadthChunks,
-            chunk: `${originalChunkIndex + 1}/${chunks.length}`,
-            symbol
-          });
+        const hasBtcAltMetrics = (coverage?.btcAltMetricsRows ?? 0) >= Math.floor(Math.max(1, coverage?.rows ?? 0) * 0.9);
+        const missingChunks2 = !hasCoverage({
+          coverage,
+          startMs: breadthStartMs,
+          endMs,
+          intervalMs
+        }) || !hasBtcAltMetrics ? !hasBtcAltMetrics ? chunks : filterMissingBreadthBackfillChunks({
+          chunks,
+          coverage,
+          intervalMs
+        }) : [];
+        if (!missingChunks2.length) {
+          console.log(
+            import_chalk.default.gray(`breadth cached: universe=${definition.universe}`)
+          );
         }
-        const btcCandles = await connector.kline({
-          symbol: "BTCUSDT",
+        return {
+          definition,
+          missingChunkKeys: new Set(
+            missingChunks2.map((chunk) => `${chunk.startMs}:${chunk.endMs}`)
+          )
+        };
+      })
+    );
+    const missingChunks = chunks.filter((chunk) => {
+      const key = `${chunk.startMs}:${chunk.endMs}`;
+      return plans.some((plan) => plan.missingChunkKeys.has(key));
+    });
+    const bar = new import_progress.default(
+      "breadth :current/:total [:bar][:percent] :etas(s) candles=:candles skip=:skip chunk=:chunk :symbol",
+      {
+        total: Math.max(1, missingChunks.length * breadthSymbols.length),
+        width: 24
+      }
+    );
+    let candlesRead = 0;
+    const skippedBreadthChunks = chunks.length - missingChunks.length;
+    if (!missingChunks.length) {
+      bar.tick(1, {
+        candles: 0,
+        skip: skippedBreadthChunks,
+        chunk: "cached",
+        symbol: "top5/top10/top30/top50/top100"
+      });
+    }
+    for (let chunkIndex = 0; chunkIndex < missingChunks.length; chunkIndex += 1) {
+      const chunk = missingChunks[chunkIndex];
+      const originalChunkIndex = chunks.findIndex(
+        (item) => item.startMs === chunk.startMs && item.endMs === chunk.endMs
+      );
+      const candlesBySymbol = {};
+      for (const symbol of breadthSymbols) {
+        const candles = await connector.kline({
+          symbol,
           interval: connectorInterval,
           start: chunk.fetchStartMs,
           end: chunk.endMs,
           silent: true
         });
+        candlesBySymbol[symbol] = candles;
+        candlesRead += candles.length;
+        bar.tick(1, {
+          candles: candlesRead,
+          skip: skippedBreadthChunks,
+          chunk: `${originalChunkIndex + 1}/${chunks.length}`,
+          symbol
+        });
+      }
+      const btcCandles = await connector.kline({
+        symbol: "BTCUSDT",
+        interval: connectorInterval,
+        start: chunk.fetchStartMs,
+        end: chunk.endMs,
+        silent: true
+      });
+      const chunkKey = `${chunk.startMs}:${chunk.endMs}`;
+      for (const plan of plans) {
+        if (!plan.missingChunkKeys.has(chunkKey)) {
+          continue;
+        }
+        const symbolSet = new Set(plan.definition.symbols);
+        const universeCandles = Object.fromEntries(
+          Object.entries(candlesBySymbol).filter(
+            ([symbol]) => symbolSet.has(symbol)
+          )
+        );
         const rows = buildMarketBreadthRows({
-          universe,
+          universe: plan.definition.universe,
           interval,
-          candlesBySymbol,
+          candlesBySymbol: universeCandles,
           btcCandles
         }).filter((row) => {
           const ts = row.ts.getTime();
@@ -2394,8 +2420,6 @@ var backfillBinanceMarketContext = async (params, enabled) => {
         await (0, import_timescale.upsertMarketBreadthRows)(rows);
         breadthRows += rows.length;
       }
-    } else {
-      console.log(import_chalk.default.gray(`breadth cached: universe=${universe}`));
     }
   }
   console.log(