@tradejs/cli 2.0.15 → 2.0.17

package/dist/scripts/replayRunner.js

@@ -2349,6 +2349,8 @@ var prepareRunEnvironment = async ({
 
 // src/lib/marketContextPrepare.ts
 var import_chalk7 = __toESM(require("chalk"));
+var import_timescale5 = require("@tradejs/infra/timescale");
+var import_registry = require("@tradejs/node/registry");
 
 // src/lib/binanceMarketContextBackfill.ts
 var import_progress = __toESM(require("progress"));
@@ -5184,6 +5186,7 @@ var backfillCoinMarketCapContextForReplay = backfillCoinMarketCapContext;
 var backfillCoinMarketCapContextForSignals = (params) => backfillCoinMarketCapContext({ ...params, refreshStaleFearGreed: true });
 
 // src/lib/hyperliquidWhaleBackfill.ts
+var import_progress3 = __toESM(require("progress"));
 var import_timescale4 = require("@tradejs/infra/timescale");
 var import_strategies2 = require("@tradejs/node/strategies");
 
@@ -5558,11 +5561,36 @@ var backfillHyperliquidWhaleContext = async (params) => {
     toMs,
     ...identity
   });
+  const totalCoverageBuckets = Math.ceil(
+    (toMs - fromMs) / HYPERLIQUID_WHALE_BUCKET_MS
+  );
+  const coverageBar = new import_progress3.default(
+    "Hyperliquid coverage :current/:total [:bar][:percent] :eta(s) rows=:rows chunk=:chunk",
+    {
+      total: Math.max(1, totalCoverageBuckets),
+      width: 24
+    }
+  );
+  let displayedCoverageBuckets = 0;
   const coverageBuckets = await (0, import_timescale4.rebuildHyperliquidWhaleCoverageRows)({
     fromMs,
     toMs,
     expectedWhales: whales.addresses.length,
-    ...identity
+    ...identity,
+    onProgress: (progress) => {
+      const next = Math.max(
+        displayedCoverageBuckets,
+        Math.min(progress.completedBuckets, coverageBar.total)
+      );
+      const delta = next - displayedCoverageBuckets;
+      if (delta > 0) {
+        coverageBar.tick(delta, {
+          rows: progress.rows,
+          chunk: `${progress.chunkIndex}/${progress.totalChunks}`
+        });
+        displayedCoverageBuckets = next;
+      }
+    }
   });
   const result = {
     cached: false,
@@ -5590,6 +5618,9 @@ var backfillHyperliquidWhaleContext = async (params) => {
 };
 
 // src/lib/marketContextPrepare.ts
+var uniqueReasons = (reasons) => [
+  ...new Set(reasons)
+];
 var shouldPrepareDerivativesContextForRun = (params) => {
   if (params.universe === "tradfi") return false;
   if (params.mode === "signals") {
@@ -5597,6 +5628,9 @@ var shouldPrepareDerivativesContextForRun = (params) => {
       cacheOnly: params.cacheOnly
     });
   }
+  if (params.mode === "backtest" && !params.aiEnabled && !params.mlEnabled) {
+    return false;
+  }
   return shouldBackfillDerivativesContextForBacktest({
     aiEnabled: Boolean(params.aiEnabled),
     cacheOnly: params.cacheOnly,
@@ -5606,6 +5640,7 @@ var shouldPrepareDerivativesContextForRun = (params) => {
 var shouldPrepareBinanceMarketContextForRun = (params) => {
   if (params.universe === "tradfi") return false;
   if (params.mode === "backtest") {
+    if (!params.aiEnabled && !params.mlEnabled) return false;
     return shouldBackfillBinanceMarketContextForBacktest({
       aiEnabled: Boolean(params.aiEnabled),
       cacheOnly: params.cacheOnly,
@@ -5624,6 +5659,7 @@ var shouldPrepareBinanceMarketContextForRun = (params) => {
 var shouldPrepareCoinMarketCapContextForRun = (params) => {
   if (params.universe === "tradfi") return false;
   if (params.mode === "backtest") {
+    if (!params.aiEnabled && !params.mlEnabled) return false;
     return shouldBackfillCoinMarketCapContextForBacktest({
       aiEnabled: Boolean(params.aiEnabled),
       cacheOnly: params.cacheOnly,
@@ -5639,10 +5675,8 @@ var shouldPrepareCoinMarketCapContextForRun = (params) => {
     cacheOnly: params.cacheOnly
   });
 };
-var shouldPrepareHyperliquidWhaleContextForRun = (params) => {
-  const strategyRequiresContext = params.strategyNames?.includes(
-    "HyperliquidConsensus"
-  );
+var shouldPrepareHyperliquidWhaleContextForRun = (params, coreContextSources = []) => {
+  const strategyRequiresContext = coreContextSources.includes("hyperliquidWhales");
   if (params.universe === "tradfi" || params.mode === "signals" || params.cacheOnly || !params.aiEnabled && !params.mlEnabled && !strategyRequiresContext) {
     return false;
   }
@@ -5653,6 +5687,64 @@ var shouldPrepareHyperliquidWhaleContextForRun = (params) => {
   if (strategyRequiresContext) return true;
   return ["1", "true", "yes", "on"].includes(configured);
 };
+var resolveCoreContextSources = async (params) => {
+  const strategyNames = [...new Set(params.strategyNames ?? [])];
+  if (!strategyNames.length) return [];
+  await (0, import_registry.ensureStrategyPluginsLoaded)(params.projectRoot);
+  return [
+    ...new Set(
+      strategyNames.flatMap(
+        (strategyName) => (0, import_registry.getStrategyManifest)(strategyName, params.projectRoot)?.contextRequirements?.core ?? []
+      )
+    )
+  ];
+};
+var resolveMarketContextRunRequirements = async (params) => {
+  const coreContextSources = await resolveCoreContextSources(params);
+  const paramsWithCoreDemand = (source) => coreContextSources.includes(source) ? { ...params, aiEnabled: true } : params;
+  const isCrypto = params.universe !== "tradfi";
+  const capturesPayload = Boolean(params.aiEnabled || params.mlEnabled);
+  const runtimeReadsContext = params.mode !== "backtest";
+  const reasonsFor = (source) => uniqueReasons([
+    ...coreContextSources.includes(source) ? ["core"] : [],
+    ...params.aiEnabled ? ["ai"] : [],
+    ...params.mlEnabled ? ["ml"] : [],
+    ...runtimeReadsContext ? ["runtime"] : []
+  ]);
+  const readsStandardContext = isCrypto && (capturesPayload || runtimeReadsContext);
+  const readsHyperliquid = isCrypto && (readsStandardContext || coreContextSources.includes("hyperliquidWhales"));
+  return {
+    binance: {
+      read: readsStandardContext || coreContextSources.includes("binance"),
+      backfill: shouldPrepareBinanceMarketContextForRun(
+        paramsWithCoreDemand("binance")
+      ),
+      requiredBy: reasonsFor("binance")
+    },
+    coinmarketcap: {
+      read: readsStandardContext || coreContextSources.includes("coinmarketcap"),
+      backfill: shouldPrepareCoinMarketCapContextForRun(
+        paramsWithCoreDemand("coinmarketcap")
+      ),
+      requiredBy: reasonsFor("coinmarketcap")
+    },
+    derivatives: {
+      read: readsStandardContext || coreContextSources.includes("derivatives"),
+      backfill: shouldPrepareDerivativesContextForRun(
+        paramsWithCoreDemand("derivatives")
+      ),
+      requiredBy: reasonsFor("derivatives")
+    },
+    hyperliquidWhales: {
+      read: readsHyperliquid,
+      backfill: shouldPrepareHyperliquidWhaleContextForRun(
+        params,
+        coreContextSources
+      ),
+      requiredBy: reasonsFor("hyperliquidWhales")
+    }
+  };
+};
 var resolveCoinMarketCapBackfillForMode = (mode) => mode === "backtest" ? backfillCoinMarketCapContextForBacktest : mode === "signals" ? backfillCoinMarketCapContextForSignals : backfillCoinMarketCapContextForReplay;
 var buildCoinMarketCapBackfillParams = (params) => ({
   userName: params.userName,
@@ -5681,7 +5773,15 @@ var prepareMarketContextForRun = async (params) => {
     console.log(import_chalk7.default.gray(message));
   });
   const timeOperation3 = (label, operation) => timeOperation(label, operation, log);
-  if (shouldPrepareDerivativesContextForRun(params)) {
+  const requirements = await resolveMarketContextRunRequirements(params);
+  const readableSources = Object.entries(requirements).filter(([, requirement]) => requirement.read).map(([source]) => source);
+  if (readableSources.length) {
+    await timeOperation3(
+      "market context schema ensure",
+      () => (0, import_timescale5.ensureMarketContextSchemas)(readableSources)
+    );
+  }
+  if (requirements.derivatives.backfill) {
     await timeOperation3(
       "derivatives context backfill",
       () => (params.mode === "signals" ? backfillDerivativesContextForSignals : backfillDerivativesContextForBacktest)(
@@ -5689,7 +5789,7 @@ var prepareMarketContextForRun = async (params) => {
       )
     );
   }
-  if (shouldPrepareHyperliquidWhaleContextForRun(params)) {
+  if (requirements.hyperliquidWhales.backfill) {
     await timeOperation3(
       "hyperliquid whale context backfill",
       () => backfillHyperliquidWhaleContext({
@@ -5701,13 +5801,13 @@ var prepareMarketContextForRun = async (params) => {
       })
     );
   }
-  if (shouldPrepareBinanceMarketContextForRun(params)) {
+  if (requirements.binance.backfill) {
     await timeOperation3("binance market context backfill", () => {
       const backfill = params.mode === "backtest" ? backfillBinanceMarketContextForBacktest : params.mode === "signals" ? backfillBinanceMarketContextForSignals : backfillBinanceMarketContextForReplay;
       return backfill(buildBinanceMarketBackfillParams(params));
     });
   }
-  if (shouldPrepareCoinMarketCapContextForRun(params)) {
+  if (requirements.coinmarketcap.backfill) {
     await timeOperation3(
       "coinmarketcap historical context backfill",
       () => resolveCoinMarketCapBackfillForMode(params.mode)(
@@ -5715,6 +5815,7 @@ var prepareMarketContextForRun = async (params) => {
       )
     );
   }
+  return requirements;
 };
 
 // src/lib/backtest/runState.ts
@@ -5987,7 +6088,7 @@ var compactReplaySignal = (signal) => {
 
 // src/lib/replay/historicalSignalsReplay.ts
 var import_chalk9 = __toESM(require("chalk"));
-var import_progress3 = __toESM(require("progress"));
+var import_progress4 = __toESM(require("progress"));
 var import_backtest = require("@tradejs/core/backtest");
 var import_strategies4 = require("@tradejs/core/strategies");
 var import_time3 = require("@tradejs/core/time");
@@ -6871,7 +6972,7 @@ var runHistoricalSignalsReplay = async ({
   const sharedReplayKeyPrefixes = [];
   let preparedSymbols = 0;
   let skippedSymbols = 0;
-  const prepareBar = new import_progress3.default(
+  const prepareBar = new import_progress4.default(
     "prepare :current/:total [:bar][:percent] skipped=:skipped :etas(s) :symbol",
     {
       total: preparedRun.tickers.length,
@@ -6993,7 +7094,7 @@ var runHistoricalSignalsReplay = async ({
     interval
   });
   let abortedCycles = 0;
-  const cycleBar = new import_progress3.default(
+  const cycleBar = new import_progress4.default(
     "cycles  :current/:total [:bar][:percent] sig=:signals abort=:aborted :etas(s) :ts",
     {
       total: orderedTimestamps.length,
@@ -7963,6 +8064,15 @@ var syncRuntimeTrades = async ({
         nextTrade,
         { expire: import_constants5.TTL_1M }
       ),
+      (0, import_redis5.setHashJsonField)(
+        import_redis5.redisKeys.runtimeClosedTradeBucket(
+          userName,
+          (0, import_time5.getRuntimeStorageDayKey)(nextTrade.exitTimestamp)
+        ),
+        trade.orderId,
+        nextTrade,
+        { expire: import_constants5.TTL_1M }
+      ),
       ...isCurrentActiveTrade ? [(0, import_redis5.delKey)(activeTradeKey)] : []
     ]);
     syncedTrades.push(nextTrade);

package/dist/scripts/replayRuntimeEvidence.js

@@ -135,6 +135,29 @@ var loadRuntimeTrades = async (userName, {
   const trades = await Promise.all(keys.map((key) => (0, import_redis.getData)(key, null)));
   return trades.filter(isRuntimeTradeRecord).filter(filterByWindow).sort((left, right) => left.entryTimestamp - right.entryTimestamp);
 };
+var loadRuntimeClosedTrades = async (userName, {
+  startTime,
+  endTime
+}) => {
+  const dayKeys = (0, import_time.getRuntimeStorageDayKeys)(startTime, endTime);
+  const trades = (await Promise.all(
+    dayKeys.map(
+      (dayKey) => (0, import_redis.getHashJsonValues)(
+        import_redis.redisKeys.runtimeClosedTradeBucket(userName, dayKey)
+      )
+    )
+  )).flat();
+  const deduped = /* @__PURE__ */ new Map();
+  for (const trade of trades) {
+    if (!isRuntimeTradeRecord(trade) || trade.status !== "closed" || !Number.isFinite(trade.exitTimestamp) || trade.exitTimestamp < startTime || trade.exitTimestamp >= endTime) {
+      continue;
+    }
+    deduped.set(trade.orderId, trade);
+  }
+  return [...deduped.values()].sort(
+    (left, right) => left.exitTimestamp - right.exitTimestamp
+  );
+};
 
 // src/lib/runtimeSignalsLoader.ts
 var import_redis2 = require("@tradejs/infra/redis");
@@ -299,11 +322,6 @@ var formatRuntimeDebugDateTime = (timestamp) => new Intl.DateTimeFormat("ru-RU",
   minute: "2-digit",
   hour12: false
 }).format(new Date(timestamp));
-var buildRuntimeDebugEvaluationId = ({
-  strategy,
-  symbol,
-  timestamp
-}) => `${strategy}:${symbol}:${timestamp}`;
 var filterByStrategies = (values, strategies) => {
   if (!strategies?.length) {
     return values;
@@ -319,13 +337,30 @@ var collectRuntimeDebugEvidence = async ({
 }) => {
   const dayKeys = (0, import_time2.getRuntimeStorageDayKeys)(startTime, endTime);
   const dayKeySet = new Set(dayKeys);
-  const [trades, signals, evaluations, evaluationStatsBuckets, configs] = await Promise.all([
+  const [
+    entryTrades,
+    closedTrades,
+    signals,
+    evaluations,
+    evaluationStatsBuckets,
+    configs
+  ] = await Promise.all([
     loadRuntimeTrades(userName, { startTime, endTime }),
+    loadRuntimeClosedTrades(userName, { startTime, endTime }),
     loadRuntimeSignals(userName, { startTime, endTime }),
     loadRuntimeSignalEvaluations(userName, { startTime, endTime }),
     loadRuntimeSignalEvaluationStatsBuckets(userName),
     loadRuntimeStrategyConfigs(userName)
   ]);
+  const tradesByOrderId = new Map(
+    entryTrades.map((trade) => [trade.orderId, trade])
+  );
+  for (const trade of closedTrades) {
+    tradesByOrderId.set(trade.orderId, trade);
+  }
+  const trades = [...tradesByOrderId.values()].sort(
+    (left, right) => left.entryTimestamp - right.entryTimestamp
+  );
   const filteredStats = evaluationStatsBuckets.filter(
     (entry) => dayKeySet.has(entry.dayKey) && (!strategies?.length || strategies.includes(entry.strategy))
   );
@@ -340,7 +375,7 @@ var collectRuntimeDebugEvidence = async ({
     )
   };
 };
-var buildRuntimeDebugReportPayload = async ({
+var buildRuntimeEvidenceReportPayload = ({
   userName,
   startTime,
   endTime,
@@ -349,159 +384,30 @@ var buildRuntimeDebugReportPayload = async ({
   trades,
   strategyConfigs,
   evaluationStatsBuckets
-}) => {
-  const signalById = new Map(
-    signals.map((signal) => [signal.signalId, signal])
-  );
-  const evaluationById = new Map(
-    evaluations.map((evaluation) => [evaluation.evaluationId, evaluation])
-  );
-  const evaluationBySignalShape = new Map(
-    evaluations.map((evaluation) => [
-      buildRuntimeDebugEvaluationId(evaluation),
-      evaluation
-    ])
-  );
-  const dayKeys = (0, import_time2.getRuntimeStorageDayKeys)(startTime, endTime);
-  const debugTrades = await Promise.all(
-    trades.map(async (trade) => {
-      const signal = trade.signalId != null ? signalById.get(trade.signalId) : void 0;
-      const evaluationId = buildRuntimeDebugEvaluationId({
-        strategy: trade.strategy,
-        symbol: trade.symbol,
-        timestamp: signal?.timestamp ?? trade.entryTimestamp
-      });
-      const knownEvaluation = evaluationById.get(evaluationId) ?? evaluationBySignalShape.get(evaluationId);
-      const tradeDayKey = (0, import_time2.getRuntimeStorageDayKey)(trade.entryTimestamp);
-      const signalDayKey = signal != null ? (0, import_time2.getRuntimeStorageDayKey)(signal.timestamp) : tradeDayKey;
-      const evaluationDayKey = knownEvaluation != null ? (0, import_time2.getRuntimeStorageDayKey)(knownEvaluation.timestamp) : signalDayKey;
-      const tradeKey = import_redis3.redisKeys.runtimeTrade(userName, trade.orderId);
-      const tradeBucketKey = import_redis3.redisKeys.runtimeTradeBucket(
-        userName,
-        tradeDayKey
-      );
-      const activeTradeKey = import_redis3.redisKeys.runtimeActiveTrade(
-        userName,
-        trade.symbol,
-        trade.deploymentId ?? trade.accountId
-      );
-      const signalKey = trade.signalId != null ? import_redis3.redisKeys.storeSignal(trade.symbol, trade.signalId) : null;
-      const evaluationBucketKey = import_redis3.redisKeys.runtimeSignalEvaluationBucket(
-        userName,
-        evaluationDayKey,
-        knownEvaluation?.strategy ?? trade.strategy
-      );
-      const evaluationField = knownEvaluation?.evaluationId ?? evaluationId;
-      const evaluationKey = import_redis3.redisKeys.runtimeSignalEvaluation(
-        userName,
-        evaluationField
-      );
-      const [
-        tradeValue,
-        activeTradeValue,
-        signalValue,
-        directEvaluationValue,
-        bucketEvaluationValue
-      ] = await Promise.all([
-        (0, import_redis3.getData)(tradeKey, null),
-        (0, import_redis3.getData)(activeTradeKey, null),
-        signalKey ? (0, import_redis3.getData)(signalKey, null) : Promise.resolve(null),
-        (0, import_redis3.getData)(evaluationKey, null),
-        knownEvaluation ? Promise.resolve(knownEvaluation) : (0, import_redis3.getHashJsonField)(
-          evaluationBucketKey,
-          evaluationField,
-          null
-        )
-      ]);
-      const evaluationValue = directEvaluationValue ?? bucketEvaluationValue ?? knownEvaluation ?? null;
-      return {
-        redisDebug: {
-          trade: tradeKey,
-          tradeBucket: {
-            key: tradeBucketKey,
-            field: trade.orderId
-          },
-          activeTrade: activeTradeKey,
-          signal: signalKey,
-          evaluation: {
-            key: evaluationBucketKey,
-            field: evaluationField,
-            directKey: evaluationKey
-          }
-        },
-        trade,
-        redisValues: {
-          trade: tradeValue,
-          tradeBucket: trade,
-          activeTrade: activeTradeValue,
-          signal: signalValue ?? signal ?? null,
-          evaluation: evaluationValue ?? knownEvaluation ?? null
-        }
-      };
-    })
-  );
-  const linkedEvaluations = /* @__PURE__ */ new Map();
-  for (const trade of debugTrades) {
-    const evaluation = trade.redisValues.evaluation;
-    if (evaluation && typeof evaluation === "object" && "evaluationId" in evaluation && typeof evaluation.evaluationId === "string") {
-      linkedEvaluations.set(
-        evaluation.evaluationId,
-        evaluation
-      );
-    }
-  }
-  const reportEvaluations = evaluations.length > 0 ? evaluations : [...linkedEvaluations.values()];
-  return {
-    reportType: "runtime-daily-debug",
-    generatedAt: Date.now(),
-    generatedAtMsk: formatRuntimeDebugDateTime(Date.now()),
-    userName,
-    window: {
-      startTime,
-      endTime,
-      startMsk: formatRuntimeDebugDateTime(startTime),
-      endMsk: formatRuntimeDebugDateTime(endTime),
-      dayKeys
-    },
-    redisPrefixes: {
-      runtimeTrades: import_redis3.redisKeys.runtimeTrades(userName),
-      runtimeTradeBuckets: import_redis3.redisKeys.runtimeTradeBuckets(userName),
-      runtimeActiveTrades: import_redis3.redisKeys.runtimeActiveTrades(userName),
-      runtimeSignals: import_redis3.redisKeys.runtimeSignalBuckets(userName),
-      runtimeSignalEvaluations: import_redis3.redisKeys.runtimeSignalEvaluationBuckets(userName),
-      runtimeSignalEvaluationStats: import_redis3.redisKeys.runtimeSignalEvaluationStatsBuckets(userName)
-    },
-    counts: {
-      trades: trades.length,
-      signals: signals.length,
-      evaluations: reportEvaluations.length,
-      evaluationStatsBuckets: evaluationStatsBuckets?.length ?? 0,
-      strategyConfigs: strategyConfigs?.length ?? 0
-    },
-    trades: debugTrades,
-    signals: signals.map((signal) => ({
-      redisKey: import_redis3.redisKeys.storeSignal(signal.symbol, signal.signalId),
-      signal
-    })),
-    evaluations: reportEvaluations.map((evaluation) => ({
-      redisDebug: {
-        key: import_redis3.redisKeys.runtimeSignalEvaluationBucket(
-          userName,
-          (0, import_time2.getRuntimeStorageDayKey)(evaluation.timestamp),
-          evaluation.strategy
-        ),
-        field: evaluation.evaluationId,
-        directKey: import_redis3.redisKeys.runtimeSignalEvaluation(
-          userName,
-          evaluation.evaluationId
-        )
-      },
-      evaluation
-    })),
-    evaluationStatsBuckets: evaluationStatsBuckets ?? [],
-    strategyConfigs: strategyConfigs ?? []
-  };
-};
+}) => ({
+  reportType: "runtime-daily-evidence",
+  generatedAt: Date.now(),
+  userName,
+  window: {
+    startTime,
+    endTime,
+    startMsk: formatRuntimeDebugDateTime(startTime),
+    endMsk: formatRuntimeDebugDateTime(endTime),
+    dayKeys: (0, import_time2.getRuntimeStorageDayKeys)(startTime, endTime)
+  },
+  counts: {
+    trades: trades.length,
+    signals: signals.length,
+    evaluations: evaluations.length,
+    evaluationStatsBuckets: evaluationStatsBuckets?.length ?? 0,
+    strategyConfigs: strategyConfigs?.length ?? 0
+  },
+  trades: trades.map((trade) => ({ trade })),
+  signals: signals.map((signal) => ({ signal })),
+  evaluations: evaluations.map((evaluation) => ({ evaluation })),
+  evaluationStatsBuckets: evaluationStatsBuckets ?? [],
+  strategyConfigs: strategyConfigs ?? []
+});
 
 // src/lib/replay/support.ts
 var import_chalk = __toESM(require("chalk"));
@@ -659,6 +565,7 @@ var summarizeReplayComparison = (runtimeComparison, strategies) => {
   ].sort((left, right) => left.localeCompare(right));
   return {
     mode: comparison.mode ?? null,
+    lineage: comparison.lineage ?? null,
     rows: asArray(comparison.rows).filter(
       (row) => !useFilter || strategySet.has(String(row.strategyName ?? ""))
     ),
@@ -786,7 +693,7 @@ var loadRuntimePayload = async ({
   return {
     source: "local-redis",
     path: null,
-    payload: await buildRuntimeDebugReportPayload({
+    payload: buildRuntimeEvidenceReportPayload({
       userName: flags.user,
       startTime,
       endTime,