@alphafox/cli 0.3.21 → 0.3.23

package/vendor/backtest-runner/lib/tape-loader-concurrency.mjs

@@ -0,0 +1,72 @@
+export const DEFAULT_TAPE_SERIES_CONCURRENCY = 8;
+export const MAX_TAPE_SERIES_CONCURRENCY = 8;
+
+export function resolveTapeSeriesConcurrency(value) {
+  if (typeof value === "number" && Number.isFinite(value) && value >= 1) {
+    return Math.min(
+      MAX_TAPE_SERIES_CONCURRENCY,
+      Math.max(1, Math.floor(value))
+    );
+  }
+  return DEFAULT_TAPE_SERIES_CONCURRENCY;
+}
+
+export async function mapWithConcurrency(items, concurrency, worker) {
+  const list = [...items];
+  if (list.length === 0) return [];
+
+  const limit = Math.min(
+    list.length,
+    Math.max(1, Math.floor(Number(concurrency)) || 1)
+  );
+  const results = new Array(list.length);
+  let nextIndex = 0;
+  async function runWorker() {
+    while (true) {
+      const index = nextIndex;
+      nextIndex += 1;
+      if (index >= list.length) return;
+      results[index] = await worker(list[index], index);
+    }
+  }
+  await Promise.all(Array.from({ length: limit }, () => runWorker()));
+  return results;
+}
+
+/** Shares one request budget across series workers and their time windows. */
+export function limitTapeOhlcvConcurrency(exchange, concurrency, signal) {
+  const limit = resolveTapeSeriesConcurrency(concurrency);
+  const queue = [];
+  let active = 0;
+
+  const drain = () => {
+    while (active < limit) {
+      const task = queue.shift();
+      if (!task) return;
+      active += 1;
+      void task().finally(() => {
+        active -= 1;
+        drain();
+      });
+    }
+  };
+  const schedule = (operation) =>
+    new Promise((resolve, reject) => {
+      queue.push(async () => {
+        try {
+          signal?.throwIfAborted();
+          resolve(await operation());
+        } catch (error) {
+          reject(error);
+        }
+      });
+      drain();
+    });
+
+  return {
+    fetchOHLCV: (symbol, timeframe, since, pageLimit, params) =>
+      schedule(() =>
+        exchange.fetchOHLCV(symbol, timeframe, since, pageLimit, params)
+      ),
+  };
+}

package/vendor/backtest-runner/lib/tape-loader-range-plan.mjs

@@ -0,0 +1,77 @@
+import { TapeDataUnavailableError } from "./coverage.mjs";
+import { MAX_TAPE_SERIES_CONCURRENCY } from "./tape-loader-concurrency.mjs";
+
+export function planOhlcvTimeSegments(request, stepMs, pageLimit) {
+  const candleCount = Math.max(
+    0,
+    Math.ceil((request.toMs - request.sinceMs) / stepMs)
+  );
+  const segmentCount = Math.min(
+    MAX_TAPE_SERIES_CONCURRENCY,
+    Math.max(1, Math.ceil(candleCount / pageLimit))
+  );
+  if (segmentCount === 1) {
+    return [{ sinceMs: request.sinceMs, untilMs: request.toMs }];
+  }
+
+  const candlesPerSegment = Math.ceil(candleCount / segmentCount);
+  return Array.from({ length: segmentCount }, (_, index) => {
+    const logicalStartMs = request.sinceMs + index * candlesPerSegment * stepMs;
+    return {
+      // Overlap preserves bars whose exchange open time is not split-aligned.
+      sinceMs:
+        index === 0
+          ? request.sinceMs
+          : Math.max(request.sinceMs, logicalStartMs - stepMs),
+      untilMs: Math.min(
+        request.toMs,
+        request.sinceMs + (index + 1) * candlesPerSegment * stepMs
+      ),
+    };
+  }).filter((segment) => segment.untilMs > segment.sinceMs);
+}
+
+export function createSegmentRequest(request, segment, onFraction) {
+  return {
+    ...request,
+    sinceMs: segment.sinceMs,
+    toMs: segment.untilMs,
+    fromMs: Math.min(
+      segment.untilMs,
+      Math.max(segment.sinceMs, request.fromMs)
+    ),
+    allowEmpty: true,
+    onFraction,
+  };
+}
+
+export function mergeOhlcvTimeSegments(request, segments) {
+  const rowsByTimestamp = new Map();
+  for (const rows of segments) {
+    for (const row of rows) {
+      if (row[0] < request.sinceMs) continue;
+      const existing = rowsByTimestamp.get(row[0]);
+      if (existing && !sameOhlcvRow(existing, row)) {
+        throw new TapeDataUnavailableError([
+          {
+            code: "invalid_ohlcv",
+            symbol: request.symbol,
+            timeframe: request.timeframe,
+            timestamp: row[0],
+            message: "concurrent OHLCV windows returned conflicting candles",
+          },
+        ]);
+      }
+      rowsByTimestamp.set(row[0], row);
+    }
+  }
+  return [...rowsByTimestamp.values()].sort((left, right) => left[0] - right[0]);
+}
+
+export function averageProgress(values) {
+  return values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+
+function sameOhlcvRow(left, right) {
+  return left.every((value, index) => value === right[index]);
+}

package/vendor/backtest-runner/lib/tape-loader-range.mjs

@@ -0,0 +1,222 @@
+import { abortable } from "./abortable.mjs";
+import { TapeDataUnavailableError } from "./coverage.mjs";
+import {
+  averageProgress,
+  createSegmentRequest,
+  mergeOhlcvTimeSegments,
+  planOhlcvTimeSegments,
+} from "./tape-loader-range-plan.mjs";
+import { TIMEFRAME_MS } from "./timeframes.mjs";
+
+const BITGET_OHLCV_MAX_REQUEST_SPAN_MS = 89 * 86_400_000;
+
+export function isClosedCandle(timestampMs, timeframe, toMs) {
+  return timestampMs + timeframeStepMs(timeframe) <= toMs;
+}
+
+export async function fetchClosedOhlcvRange(request) {
+  const stepMs = timeframeStepMs(request.timeframe);
+  const pageLimit = ohlcvPageLimitForTimeframe(
+    request.exchangeDefinition,
+    request.runtimeConfig,
+    request.timeframe
+  );
+  const segments = planOhlcvTimeSegments(request, stepMs, pageLimit);
+  if (segments.length === 1) {
+    return fetchClosedOhlcvTimeSegment(request, pageLimit);
+  }
+
+  const progress = new Array(segments.length).fill(0);
+  const segmentRows = await Promise.all(
+    segments.map((segment, index) =>
+      fetchClosedOhlcvTimeSegment(
+        createSegmentRequest(request, segment, (fraction) => {
+          progress[index] = fraction;
+          request.onFraction?.(averageProgress(progress));
+        }),
+        pageLimit
+      )
+    )
+  );
+  const rows = mergeOhlcvTimeSegments(request, segmentRows);
+  if (rows.length === 0 && !request.allowEmpty) {
+    throw missingOhlcvError(request);
+  }
+  request.onFraction?.(1);
+  return rows;
+}
+
+async function fetchClosedOhlcvTimeSegment(request, pageLimit) {
+  const stepMs = timeframeStepMs(request.timeframe);
+  const rows = [];
+  const totalSpan = Math.max(1, request.toMs - request.sinceMs);
+  let cursor = request.sinceMs;
+  let emptyPrefixLowerMs = null;
+  let nonEmptyPrefixUpperMs = null;
+  let prefixResolved = false;
+
+  while (cursor < request.toMs) {
+    request.signal?.throwIfAborted();
+    const fetchSinceMs = resolveFetchSinceMs(request, cursor);
+    const page = await fetchOhlcvPage(request, fetchSinceMs, pageLimit, stepMs);
+    request.signal?.throwIfAborted();
+
+    const prefix = resolveEmptyPrefix({
+      request,
+      page,
+      rows,
+      cursor,
+      stepMs,
+      emptyPrefixLowerMs,
+      nonEmptyPrefixUpperMs,
+      prefixResolved,
+    });
+    if (prefix) {
+      ({ cursor, emptyPrefixLowerMs, nonEmptyPrefixUpperMs, prefixResolved } =
+        prefix);
+      if (prefix.done) break;
+      continue;
+    }
+    if (page.length === 0) break;
+    prefixResolved = true;
+    appendPageRows(request, rows, page, stepMs);
+    const nextCursor = Number(page[page.length - 1][0]) + stepMs;
+    if (nextCursor <= cursor) break;
+    cursor = nextCursor;
+    request.onFraction?.(Math.min(1, (cursor - request.sinceMs) / totalSpan));
+  }
+  if (rows.length === 0 && !request.allowEmpty) {
+    throw missingOhlcvError(request);
+  }
+  return rows;
+}
+
+function resolveEmptyPrefix(input) {
+  if (input.prefixResolved || input.rows.length > 0) return null;
+  if (input.page.length === 0 && input.cursor >= input.request.fromMs) {
+    return { ...input, done: true };
+  }
+  if (input.page.length > 0 && input.emptyPrefixLowerMs === null) return null;
+
+  const emptyPrefixLowerMs =
+    input.page.length === 0 ? input.cursor : input.emptyPrefixLowerMs;
+  const nonEmptyPrefixUpperMs =
+    input.page.length > 0 ? input.cursor : input.nonEmptyPrefixUpperMs;
+  const upperMs = nonEmptyPrefixUpperMs ?? input.request.fromMs;
+  const prefixResolved =
+    nonEmptyPrefixUpperMs !== null &&
+    upperMs - (emptyPrefixLowerMs ?? upperMs) <= input.stepMs;
+  return {
+    cursor: prefixResolved
+      ? upperMs
+      : midpointCursorMs(
+          emptyPrefixLowerMs ?? input.cursor,
+          upperMs,
+          input.stepMs
+        ),
+    emptyPrefixLowerMs,
+    nonEmptyPrefixUpperMs,
+    prefixResolved,
+    done: false,
+  };
+}
+
+async function fetchOhlcvPage(request, fetchSinceMs, pageLimit, stepMs) {
+  const params = { ...request.runtimeConfig.requestParams };
+  if (request.exchangeDefinition.ccxtId === "hyperliquid") {
+    params.until = Math.min(
+      request.toMs,
+      fetchSinceMs + (pageLimit - 1) * stepMs
+    );
+  }
+  return abortable(
+    request.exchange.fetchOHLCV(
+      request.symbol,
+      request.timeframe,
+      fetchSinceMs,
+      pageLimit,
+      params
+    ),
+    request.signal
+  );
+}
+
+function appendPageRows(request, rows, page, stepMs) {
+  for (const raw of page) {
+    const timestamp = Number(raw[0]);
+    if (raw.length < 6 || !Number.isFinite(timestamp)) {
+      throw invalidOhlcvError(request);
+    }
+    if (!isClosedCandle(timestamp, request.timeframe, request.toMs)) continue;
+    if (rows.length > 0 && timestamp <= rows[rows.length - 1][0]) {
+      throw new TapeDataUnavailableError([
+        {
+          code: "non_monotonic",
+          symbol: request.symbol,
+          timeframe: request.timeframe,
+          expected: rows[rows.length - 1][0] + stepMs,
+          actual: timestamp,
+          timestamp,
+        },
+      ]);
+    }
+    rows.push([
+      timestamp,
+      Number(raw[1]),
+      Number(raw[2]),
+      Number(raw[3]),
+      Number(raw[4]),
+      Number(raw[5]),
+    ]);
+  }
+}
+
+function resolveFetchSinceMs(request, cursor) {
+  return request.exchangeDefinition.ccxtId === "bitget"
+    ? Math.max(0, cursor - 1)
+    : cursor;
+}
+
+function ohlcvPageLimitForTimeframe(exchange, runtimeConfig, timeframe) {
+  if (exchange.ccxtId !== "bitget") return runtimeConfig.ohlcvPageLimit;
+  return Math.max(
+    1,
+    Math.min(
+      runtimeConfig.ohlcvPageLimit,
+      Math.floor(BITGET_OHLCV_MAX_REQUEST_SPAN_MS / timeframeStepMs(timeframe))
+    )
+  );
+}
+
+function timeframeStepMs(timeframe) {
+  const stepMs = TIMEFRAME_MS[timeframe];
+  if (!stepMs) throw new Error(`不支持的 timeframe：${timeframe}`);
+  return stepMs;
+}
+
+function midpointCursorMs(lowerMs, upperMs, stepMs) {
+  if (upperMs - lowerMs <= stepMs) return upperMs;
+  const midpointMs = lowerMs + Math.floor((upperMs - lowerMs) / 2);
+  const alignedMs = Math.floor(midpointMs / stepMs) * stepMs;
+  return Math.min(upperMs, Math.max(lowerMs + 1, alignedMs));
+}
+
+function invalidOhlcvError(request) {
+  return new TapeDataUnavailableError([
+    {
+      code: "invalid_ohlcv",
+      symbol: request.symbol,
+      timeframe: request.timeframe,
+    },
+  ]);
+}
+
+function missingOhlcvError(request) {
+  return new TapeDataUnavailableError([
+    {
+      code: "ohlcv_missing",
+      symbol: request.symbol,
+      timeframe: request.timeframe,
+    },
+  ]);
+}

package/vendor/backtest-runner/lib/tape-loader.mjs

@@ -19,9 +19,15 @@ import {
   buildCcxtConstructorOptions,
   ccxtExchangeClientCacheKey,
 } from "./proxy.mjs";
-import { loadSeriesWithCache, TIMEFRAME_MS } from "./series.mjs";
+import { loadSeriesWithCache } from "./series.mjs";
+import {
+  limitTapeOhlcvConcurrency,
+  mapWithConcurrency,
+  resolveTapeSeriesConcurrency,
+} from "./tape-loader-concurrency.mjs";
 import {
   ENGINE_BACKTEST_BASE_TIMEFRAME,
+  TIMEFRAME_MS,
   baseTimeframeStepMs,
   resolvePlanBaseTimeframe,
 } from "./timeframes.mjs";
@@ -40,47 +46,8 @@ const FUNDING_INTERVALS = [
   { interval: "8h", spacingMs: 28_800_000 },
 ];
 
-/** Independent symbol×timeframe (and funding) fetches. Pagination stays serial. */
-export const DEFAULT_TAPE_SERIES_CONCURRENCY = 4;
-export const MAX_TAPE_SERIES_CONCURRENCY = 8;
-
 const exchangePromises = new Map();
 
-export function resolveTapeSeriesConcurrency(value) {
-  if (typeof value === "number" && Number.isFinite(value) && value >= 1) {
-    return Math.min(
-      MAX_TAPE_SERIES_CONCURRENCY,
-      Math.max(1, Math.floor(value))
-    );
-  }
-  return DEFAULT_TAPE_SERIES_CONCURRENCY;
-}
-
-export async function mapWithConcurrency(items, concurrency, worker) {
-  const list = [...items];
-  if (list.length === 0) {
-    return [];
-  }
-  const limit = Math.min(
-    list.length,
-    Math.max(1, Math.floor(Number(concurrency)) || 1)
-  );
-  const results = new Array(list.length);
-  let nextIndex = 0;
-  async function runWorker() {
-    while (true) {
-      const index = nextIndex;
-      nextIndex += 1;
-      if (index >= list.length) {
-        return;
-      }
-      results[index] = await worker(list[index], index);
-    }
-  }
-  await Promise.all(Array.from({ length: limit }, () => runWorker()));
-  return results;
-}
-
 export function effectiveTapeEndMs(
   requestedToMs,
   nowMs = Date.now(),
@@ -287,6 +254,11 @@ export async function loadTape(request, options = {}) {
   const dataIssues = [];
   const coverageWarnings = [];
   const coverageIssues = [];
+  const ohlcvExchange = limitTapeOhlcvConcurrency(
+    exchange,
+    seriesConcurrency,
+    request.signal
+  );
   const seriesFractions = new Array(totalSeries).fill(0);
   let lastOhlcvDetail = "";
   const reportOhlcv = (index, fraction, detail) => {
@@ -311,7 +283,7 @@ export async function loadTape(request, options = {}) {
       const detail = `${symbol} ${timeframe}`;
       try {
         const loaded = await loadSeriesWithCache(
-          exchange,
+          ohlcvExchange,
           exchangeDefinition,
           runtimeConfig,
           symbol,