@tradejs/cli 2.0.8 → 2.0.10

package/dist/scripts/signals.js

@@ -4709,7 +4709,7 @@ var buildCoinMarketCapBackfillParams = (params) => ({
   userName: params.userName,
   startMs: params.startMs,
   endMs: params.endMs,
-  preloadStartMs: params.preloadStartMs
+  preloadStartMs: params.mode === "signals" ? void 0 : params.preloadStartMs
 });
 var buildBinanceMarketBackfillParams = (params) => ({
   userName: params.userName,
@@ -4718,7 +4718,7 @@ var buildBinanceMarketBackfillParams = (params) => ({
   interval: params.interval,
   startMs: params.startMs,
   endMs: params.endMs,
-  preloadStartMs: params.preloadStartMs
+  preloadStartMs: params.mode === "signals" ? void 0 : params.preloadStartMs
 });
 var buildDerivativesBackfillParams = (params) => ({
   userName: params.userName,
@@ -5027,6 +5027,7 @@ var loadRuntimeStrategies = async ({
           universe: configUniverse,
           accountId: effectiveAccountId,
           strategyCreator,
+          sourceStrategyConfig: strategyConfig,
           strategyConfig: {
             ...strategyConfig,
             ...deploymentStrategy?.config,
@@ -5315,79 +5316,140 @@ var runtimeLineageKey = (lineage) => [
 // src/lib/signals/evaluations.ts
 var runtimeLineageScopeCache = /* @__PURE__ */ new Map();
 var runtimeLineageScopeCacheDayKey = "";
+var runtimeLineageScopeCacheLoaded = false;
 var buildRuntimeSignalEvaluationId = ({
   strategyName,
   symbol,
   timestamp,
   runtimeConfigId
 }) => runtimeConfigId && runtimeConfigId !== "config" ? [strategyName, runtimeConfigId, symbol, timestamp].join(":") : [strategyName, symbol, timestamp].join(":");
-var saveRuntimeSignalEvaluation = async (evaluation) => {
-  const dayKey = (0, import_time.getRuntimeStorageDayKey)(evaluation.timestamp);
-  if (runtimeLineageScopeCacheDayKey !== dayKey) {
-    runtimeLineageScopeCache.clear();
-    runtimeLineageScopeCacheDayKey = dayKey;
+var buildLineageField = (scope) => [
+  scope.strategy,
+  scope.runtimeConfigId ?? "config",
+  scope.symbol,
+  runtimeLineageKey(scope.lineage)
+].join(":");
+var flushRuntimeSignalEvaluations = async (evaluations) => {
+  if (evaluations.length === 0) return;
+  const evaluationsByDay = /* @__PURE__ */ new Map();
+  for (const evaluation of evaluations) {
+    const dayKey = (0, import_time.getRuntimeStorageDayKey)(evaluation.timestamp);
+    const dayEvaluations = evaluationsByDay.get(dayKey) ?? [];
+    dayEvaluations.push(evaluation);
+    evaluationsByDay.set(dayKey, dayEvaluations);
   }
   const runtimeSignalRetentionTtl = getRuntimeSignalRetentionTtlSeconds();
-  if (evaluation.runtimeLineage) {
-    const lineageField = [
-      evaluation.strategy,
-      evaluation.runtimeConfigId ?? "config",
-      evaluation.symbol,
-      runtimeLineageKey(evaluation.runtimeLineage)
-    ].join(":");
+  for (const [dayKey, dayEvaluations] of evaluationsByDay) {
+    if (runtimeLineageScopeCacheDayKey !== dayKey) {
+      runtimeLineageScopeCache.clear();
+      runtimeLineageScopeCacheDayKey = dayKey;
+      runtimeLineageScopeCacheLoaded = false;
+    }
     const lineageBucket = import_redis3.redisKeys.runtimeLineageScopeBucket(
-      evaluation.userName,
+      dayEvaluations[0].userName,
       dayKey
     );
-    const cacheKey = `${lineageBucket}:${lineageField}`;
-    const existing = runtimeLineageScopeCache.get(cacheKey) ?? await (0, import_redis3.getHashJsonField)(
-      lineageBucket,
-      lineageField
+    const lineageEvaluations = dayEvaluations.filter(
+      (evaluation) => evaluation.runtimeLineage != null
     );
-    const scope = {
-      strategy: evaluation.strategy,
-      symbol: evaluation.symbol,
-      runtimeConfigId: evaluation.runtimeConfigId,
-      lineage: evaluation.runtimeLineage,
-      firstTimestamp: Math.min(
-        existing?.firstTimestamp ?? evaluation.timestamp,
-        evaluation.timestamp
-      ),
-      lastTimestamp: Math.max(
-        existing?.lastTimestamp ?? evaluation.timestamp,
-        evaluation.timestamp
-      )
-    };
-    runtimeLineageScopeCache.set(cacheKey, scope);
-    await (0, import_redis3.setHashJsonField)(lineageBucket, lineageField, scope, {
-      expire: RUNTIME_LINEAGE_SCOPE_RETENTION_TTL_SECONDS
-    });
-  }
-  if (shouldStoreDetailedRuntimeSignalEvaluation(evaluation)) {
-    await (0, import_redis3.setHashJsonField)(
-      import_redis3.redisKeys.runtimeSignalEvaluationBucket(
+    if (lineageEvaluations.length > 0 && !runtimeLineageScopeCacheLoaded) {
+      const storedScopes = await (0, import_redis3.getHashJsonValues)(lineageBucket);
+      for (const scope of storedScopes) {
+        const lineageField = buildLineageField(scope);
+        runtimeLineageScopeCache.set(`${lineageBucket}:${lineageField}`, scope);
+      }
+      runtimeLineageScopeCacheLoaded = true;
+    }
+    const lineageWrites = /* @__PURE__ */ new Map();
+    for (const evaluation of lineageEvaluations) {
+      const lineageField = buildLineageField({
+        strategy: evaluation.strategy,
+        runtimeConfigId: evaluation.runtimeConfigId,
+        symbol: evaluation.symbol,
+        lineage: evaluation.runtimeLineage
+      });
+      const cacheKey = `${lineageBucket}:${lineageField}`;
+      const existing = runtimeLineageScopeCache.get(cacheKey);
+      const scope = {
+        strategy: evaluation.strategy,
+        symbol: evaluation.symbol,
+        runtimeConfigId: evaluation.runtimeConfigId,
+        lineage: evaluation.runtimeLineage,
+        firstTimestamp: Math.min(
+          existing?.firstTimestamp ?? evaluation.timestamp,
+          evaluation.timestamp
+        ),
+        lastTimestamp: Math.max(
+          existing?.lastTimestamp ?? evaluation.timestamp,
+          evaluation.timestamp
+        )
+      };
+      runtimeLineageScopeCache.set(cacheKey, scope);
+      lineageWrites.set(lineageField, scope);
+    }
+    if (lineageWrites.size > 0) {
+      await (0, import_redis3.setHashJsonFields)(
+        lineageBucket,
+        [...lineageWrites].map(([field, data]) => ({ field, data })),
+        {
+          expire: RUNTIME_LINEAGE_SCOPE_RETENTION_TTL_SECONDS
+        }
+      );
+    }
+    const detailedWrites = /* @__PURE__ */ new Map();
+    const statsIncrements = /* @__PURE__ */ new Map();
+    for (const evaluation of dayEvaluations) {
+      if (shouldStoreDetailedRuntimeSignalEvaluation(evaluation)) {
+        const bucket = import_redis3.redisKeys.runtimeSignalEvaluationBucket(
+          evaluation.userName,
+          dayKey,
+          evaluation.strategy
+        );
+        const entries = detailedWrites.get(bucket) ?? [];
+        entries.push({
+          field: evaluation.evaluationId,
+          data: evaluation
+        });
+        detailedWrites.set(bucket, entries);
+      }
+      const statsBucket = import_redis3.redisKeys.runtimeSignalEvaluationStatsBucket(
         evaluation.userName,
         dayKey,
         evaluation.strategy
-      ),
-      evaluation.evaluationId,
-      evaluation,
-      {
-        expire: runtimeSignalRetentionTtl
+      );
+      const increments = statsIncrements.get(statsBucket) ?? {};
+      for (const [field, increment] of Object.entries(
+        buildRuntimeSignalStatsIncrements(evaluation)
+      )) {
+        increments[field] = (increments[field] ?? 0) + increment;
       }
-    );
+      statsIncrements.set(statsBucket, increments);
+    }
+    await Promise.all([
+      ...[...detailedWrites].map(
+        ([bucket, entries]) => (0, import_redis3.setHashJsonFields)(bucket, entries, {
+          expire: runtimeSignalRetentionTtl
+        })
+      ),
+      ...[...statsIncrements].map(
+        ([bucket, increments]) => (0, import_redis3.incrHashFields)(bucket, increments, {
+          expire: runtimeSignalRetentionTtl
+        })
+      )
+    ]);
   }
-  await (0, import_redis3.incrHashFields)(
-    import_redis3.redisKeys.runtimeSignalEvaluationStatsBucket(
-      evaluation.userName,
-      dayKey,
-      evaluation.strategy
-    ),
-    buildRuntimeSignalStatsIncrements(evaluation),
-    {
-      expire: runtimeSignalRetentionTtl
+};
+var createRuntimeSignalEvaluationBuffer = () => {
+  const evaluations = [];
+  return {
+    save: async (evaluation) => {
+      evaluations.push(evaluation);
+    },
+    flush: async () => {
+      const pending = evaluations.splice(0);
+      await flushRuntimeSignalEvaluations(pending);
     }
-  );
+  };
 };
 
 // src/lib/signals/telegram.ts
@@ -5499,8 +5561,8 @@ var createSignalsStrategyLifecycle = ({
     }
     try {
       const strategy = await create({
-        btcBinanceData: [...btcBinanceData ?? []],
-        btcCoinbaseData: [...btcCoinbaseData ?? []],
+        btcBinanceData: btcBinanceData ?? [],
+        btcCoinbaseData: btcCoinbaseData ?? [],
         onRuntimeClose
       });
       strategy.__tradejsUpdateReferenceData?.({
@@ -5906,7 +5968,7 @@ var createSignalsSession = async (lifecycle, scope = {}) => {
     lifecycle
   };
 };
-var findSignals = async (symbol, connectorName, connector, btcBinanceData, btcCoinbaseData, runtimeStrategies, strategyStats, runtimeCloseNotifications, lifecycle, preloadStart, currentTimestamp, persistStrategyState, universe, interval2, intervalMs2, accountId, deploymentId, instrument, runtimeScopeUniverse) => {
+var findSignals = async (symbol, connectorName, connector, btcBinanceData, btcCoinbaseData, primaryBtcClosedData, primaryEthClosedData, primaryEthByTimestamp, runtimeStrategies, strategyStats, runtimeCloseNotifications, lifecycle, preloadStart, currentTimestamp, persistStrategyState, universe, interval2, intervalMs2, accountId, deploymentId, instrument, runtimeScopeUniverse, saveRuntimeSignalEvaluation) => {
   const strategySignals = [];
   const loadCached = (cachedSymbol) => connector.kline({
     symbol: cachedSymbol,
@@ -5915,36 +5977,21 @@ var findSignals = async (symbol, connectorName, connector, btcBinanceData, btcCo
     cacheOnly: true,
     interval: interval2
   });
-  const [cachedData, btcCachedData, ethCachedData] = universe === "crypto" ? await Promise.all([
-    loadCached(symbol),
-    loadCached("BTCUSDT"),
-    loadCached("ETHUSDT")
-  ]) : [await loadCached(symbol), [], []];
+  const cachedData = universe === "crypto" && symbol === "BTCUSDT" ? primaryBtcClosedData : universe === "crypto" && symbol === "ETHUSDT" ? primaryEthClosedData : await loadCached(symbol);
   const closedData = getClosedCandlesForInterval(
     cachedData,
     currentTimestamp,
     intervalMs2
   );
-  const closedBtcData = getClosedCandlesForInterval(
-    btcCachedData,
-    currentTimestamp,
-    intervalMs2
-  );
-  const closedEthData = getClosedCandlesForInterval(
-    ethCachedData,
-    currentTimestamp,
-    intervalMs2
-  );
+  const closedBtcData = primaryBtcClosedData;
+  const closedEthData = primaryEthClosedData;
   const { alignedCoinCandles, alignedBtcCandles } = universe === "crypto" ? alignSymbolWithBtcReference(closedData, closedBtcData) : { alignedCoinCandles: closedData, alignedBtcCandles: closedData };
-  const ethByTimestamp = new Map(
-    closedEthData.map((candle) => [candle.timestamp, candle])
-  );
-  const alignedEthCandles = alignedCoinCandles.map((candle) => ethByTimestamp.get(candle.timestamp)).filter(
+  const alignedEthCandles = alignedCoinCandles.map((candle) => primaryEthByTimestamp.get(candle.timestamp)).filter(
     (candle) => Boolean(candle)
   );
   const lastCandle = alignedCoinCandles.at(-1);
   const btcLastCandle = alignedBtcCandles.at(-1);
-  const ethLastCandle = lastCandle == null ? void 0 : ethByTimestamp.get(lastCandle.timestamp);
+  const ethLastCandle = lastCandle == null ? void 0 : primaryEthByTimestamp.get(lastCandle.timestamp);
   if (!lastCandle || !btcLastCandle) {
     return strategySignals;
   }
@@ -5958,6 +6005,7 @@ var findSignals = async (symbol, connectorName, connector, btcBinanceData, btcCo
       strategyName,
       configId,
       strategyCreator,
+      sourceStrategyConfig,
       strategyConfig,
       strategyResults
     } = runtimeStrategy;
@@ -5992,46 +6040,61 @@ var findSignals = async (symbol, connectorName, connector, btcBinanceData, btcCo
       configId
     });
     const stats = strategyStats.get(strategyName);
-    const evaluation = await lifecycle.evaluate({
-      key: lifecycleKey,
-      timestamp: lastCandle.timestamp,
-      config: {
-        runtimeConfig,
-        symbolResultConfig: strategyResults?.[symbol]?.config ?? null
-      },
-      btcBinanceData,
-      btcCoinbaseData,
-      onRuntimeClose: (event) => {
-        runtimeCloseNotifications.push(event);
-      },
-      create: async ({
-        btcBinanceData: lifecycleBtcBinanceData,
-        btcCoinbaseData: lifecycleBtcCoinbaseData,
-        onRuntimeClose
-      }) => strategyCreator({
-        userName: flags.user,
-        connectorName,
-        runtimeConfigId: configId,
-        connector,
-        symbol,
-        ...runtimeScopeUniverse ? {
-          universe,
-          assetClass: instrument?.assetClass,
-          instrument,
-          accountId,
-          deploymentId
-        } : {},
-        data: [...previousData],
-        btcData: universe === "crypto" ? [...previousBtcData] : [],
-        ethData: universe === "crypto" ? [...previousEthData] : [],
-        btcBinanceData: lifecycleBtcBinanceData,
-        btcCoinbaseData: lifecycleBtcCoinbaseData,
-        config: runtimeConfig,
-        ...persistStrategyState ? { sharedStrategyStateKey: lifecycleKey } : {},
-        onRuntimeClose
-      }),
-      run: (strategy) => strategy(lastCandle, btcLastCandle, ethLastCandle)
-    });
+    const initialDataLength = previousData.length;
+    const initialBtcDataLength = previousBtcData.length;
+    const initialEthDataLength = previousEthData.length;
+    let evaluation;
+    try {
+      evaluation = await lifecycle.evaluate({
+        key: lifecycleKey,
+        timestamp: lastCandle.timestamp,
+        config: {
+          runtimeConfig,
+          sourceStrategyConfig,
+          symbolResultConfig: strategyResults?.[symbol]?.config ?? null
+        },
+        btcBinanceData,
+        btcCoinbaseData,
+        onRuntimeClose: (event) => {
+          runtimeCloseNotifications.push(event);
+        },
+        create: async ({
+          btcBinanceData: lifecycleBtcBinanceData,
+          btcCoinbaseData: lifecycleBtcCoinbaseData,
+          onRuntimeClose
+        }) => strategyCreator({
+          userName: flags.user,
+          connectorName,
+          runtimeConfigId: configId,
+          runtimeConfigSnapshot: {
+            userConfig: sourceStrategyConfig,
+            symbolResultConfig: strategyResults?.[symbol]?.config ?? null
+          },
+          connector,
+          symbol,
+          ...runtimeScopeUniverse ? {
+            universe,
+            assetClass: instrument?.assetClass,
+            instrument,
+            accountId,
+            deploymentId
+          } : {},
+          data: previousData,
+          btcData: universe === "crypto" ? previousBtcData : [],
+          ethData: universe === "crypto" ? previousEthData : [],
+          btcBinanceData: lifecycleBtcBinanceData,
+          btcCoinbaseData: lifecycleBtcCoinbaseData,
+          config: runtimeConfig,
+          ...persistStrategyState ? { sharedStrategyStateKey: lifecycleKey } : {},
+          onRuntimeClose
+        }),
+        run: (strategy) => strategy(lastCandle, btcLastCandle, ethLastCandle)
+      });
+    } finally {
+      previousData.length = initialDataLength;
+      previousBtcData.length = initialBtcDataLength;
+      previousEthData.length = initialEthDataLength;
+    }
     if (evaluation.action === "duplicate" || evaluation.action === "stale") {
       continue;
     }
@@ -6052,7 +6115,7 @@ var findSignals = async (symbol, connectorName, connector, btcBinanceData, btcCo
       if (typeof signal === "string") {
         recordStrategyReason(strategyStats, strategyName, signal, "core");
       }
-      await saveRuntimeSignalEvaluation({
+      await saveRuntimeSignalEvaluation?.({
         evaluationId: buildRuntimeSignalEvaluationId({
           strategyName,
           symbol,
@@ -6118,7 +6181,7 @@ var findSignals = async (symbol, connectorName, connector, btcBinanceData, btcCo
         expire: runtimeSignalRetentionTtl
       }
     );
-    await saveRuntimeSignalEvaluation({
+    await saveRuntimeSignalEvaluation?.({
       evaluationId: buildRuntimeSignalEvaluationId({
         strategyName,
         symbol,
@@ -6160,6 +6223,7 @@ var signals = async (options = {}) => {
   let activeSession;
   let projectHooks;
   let afterSignalsHookContext = null;
+  const runtimeSignalEvaluations = createRuntimeSignalEvaluationBuffer();
   try {
     const session = options.session ?? await createSignalsSession(createDefaultSignalsLifecycle());
     activeSession = session;
@@ -6271,7 +6335,12 @@ var signals = async (options = {}) => {
     if (flags.updateOnly) {
       return;
     }
-    const [btcBinanceData, btcCoinbaseData] = universe === "crypto" ? await timeOperation2(
+    const [
+      btcBinanceData,
+      btcCoinbaseData,
+      primaryBtcCachedData,
+      primaryEthCachedData
+    ] = universe === "crypto" ? await timeOperation2(
       "reference candles load",
       () => Promise.all([
         btcReferences.binance.kline({
@@ -6287,9 +6356,36 @@ var signals = async (options = {}) => {
           end: currentTimestamp,
           cacheOnly: true,
           interval: interval2
+        }),
+        marketConnector.kline({
+          symbol: "BTCUSDT",
+          start: preloadStart,
+          end: currentTimestamp,
+          cacheOnly: true,
+          interval: interval2
+        }),
+        marketConnector.kline({
+          symbol: "ETHUSDT",
+          start: preloadStart,
+          end: currentTimestamp,
+          cacheOnly: true,
+          interval: interval2
         })
       ])
-    ) : [[], []];
+    ) : [[], [], [], []];
+    const primaryBtcClosedData = getClosedCandlesForInterval(
+      primaryBtcCachedData,
+      currentTimestamp,
+      intervalMs2
+    );
+    const primaryEthClosedData = getClosedCandlesForInterval(
+      primaryEthCachedData,
+      currentTimestamp,
+      intervalMs2
+    );
+    const primaryEthByTimestamp = new Map(
+      primaryEthClosedData.map((candle) => [candle.timestamp, candle])
+    );
     const runtimeStrategies = await loadRuntimeStrategies({
       userName: flags.user,
       projectRoot,
@@ -6373,6 +6469,9 @@ var signals = async (options = {}) => {
           marketConnector,
           btcBinanceData,
           btcCoinbaseData,
+          primaryBtcClosedData,
+          primaryEthClosedData,
+          primaryEthByTimestamp,
           runtimeStrategies,
           strategyStats,
           runtimeCloseNotifications,
@@ -6386,7 +6485,8 @@ var signals = async (options = {}) => {
           accountId,
           deployment?.id,
           instrumentsBySymbol.get(symbol),
-          sessionUniverse
+          sessionUniverse,
+          runtimeSignalEvaluations.save
         );
         if (strategySignals.length > 0) {
           signals2.push(...strategySignals);
@@ -6427,6 +6527,14 @@ var signals = async (options = {}) => {
     );
     throw error;
   } finally {
+    try {
+      await runtimeSignalEvaluations.flush();
+    } catch (error) {
+      import_logger4.logger.error(
+        "runtime signal evaluation flush failed: %s",
+        error?.message || String(error)
+      );
+    }
     const durationMs = Date.now() - startedAt;
     if (activeSession?.deployment) {
       await (0, import_tradingAccounts2.saveRuntimeDeploymentHeartbeat)(flags.user, {