@tradejs/infra 1.0.9 → 1.0.10

package/dist/ai.mjs

@@ -1,7 +1,7 @@
 import {
   mergeJsonlFiles,
   toFileToken
-} from "./chunk-KVEZORMS.mjs";
+} from "./chunk-RQP5VSTH.mjs";
 
 // src/aiDatasetFile.ts
 import { once } from "events";
@@ -10,16 +10,81 @@ import fs from "fs/promises";
 import path from "path";
 import { createReadStream } from "fs";
 import readline from "readline";
+
+// ../../node_modules/date-fns/toDate.mjs
+function toDate(argument) {
+  const argStr = Object.prototype.toString.call(argument);
+  if (argument instanceof Date || typeof argument === "object" && argStr === "[object Date]") {
+    return new argument.constructor(+argument);
+  } else if (typeof argument === "number" || argStr === "[object Number]" || typeof argument === "string" || argStr === "[object String]") {
+    return new Date(argument);
+  } else {
+    return /* @__PURE__ */ new Date(NaN);
+  }
+}
+
+// ../../node_modules/date-fns/constructFrom.mjs
+function constructFrom(date, value) {
+  if (date instanceof Date) {
+    return new date.constructor(value);
+  } else {
+    return new Date(value);
+  }
+}
+
+// ../../node_modules/date-fns/addMonths.mjs
+function addMonths(date, amount) {
+  const _date = toDate(date);
+  if (isNaN(amount)) return constructFrom(date, NaN);
+  if (!amount) {
+    return _date;
+  }
+  const dayOfMonth = _date.getDate();
+  const endOfDesiredMonth = constructFrom(date, _date.getTime());
+  endOfDesiredMonth.setMonth(_date.getMonth() + amount + 1, 0);
+  const daysInMonth = endOfDesiredMonth.getDate();
+  if (dayOfMonth >= daysInMonth) {
+    return endOfDesiredMonth;
+  } else {
+    _date.setFullYear(
+      endOfDesiredMonth.getFullYear(),
+      endOfDesiredMonth.getMonth(),
+      dayOfMonth
+    );
+    return _date;
+  }
+}
+
+// ../../node_modules/date-fns/startOfMonth.mjs
+function startOfMonth(date) {
+  const _date = toDate(date);
+  _date.setDate(1);
+  _date.setHours(0, 0, 0, 0);
+  return _date;
+}
+
+// src/aiDatasetFile.ts
 var DEFAULT_DIR = "data/ai/export";
 var AI_DATASET_WRITE_BATCH_SIZE = 100;
 var AI_MERGE_SORT_RUN_MAX_ROWS = 2e3;
 var AI_MERGE_SORT_RUN_MAX_BYTES = 16 * 1024 * 1024;
+var AI_MERGE_SORT_MAX_OPEN_RUNS = 16;
 var AI_CHUNK_FILE_RE = /^ai-dataset-(.+)-chunk-[^.]+\.jsonl$/;
+var BACKTEST_RUN_CHUNK_ID_RE = /^(\d{12}-[a-f0-9]{8})-/;
 var writerByPath = /* @__PURE__ */ new Map();
 var getAiChunkFilePath = (strategyName, chunkId, outDir = DEFAULT_DIR) => path.join(
   outDir,
   `ai-dataset-${toFileToken(strategyName)}-chunk-${toFileToken(chunkId)}.jsonl`
 );
+var getAiChunkFilePrefix = (strategyName, runId) => `ai-dataset-${toFileToken(strategyName)}-chunk-${runId ? `${toFileToken(runId)}-` : ""}`;
+var getRunIdFromAiChunkFileName = (strategyName, fileName) => {
+  const prefix = getAiChunkFilePrefix(strategyName);
+  if (!fileName.startsWith(prefix) || !fileName.endsWith(".jsonl")) {
+    return "";
+  }
+  const chunkToken = fileName.slice(prefix.length, -".jsonl".length);
+  return chunkToken.match(BACKTEST_RUN_CHUNK_ID_RE)?.[1] ?? "";
+};
 var appendAiDatasetRow = async (params) => {
   const { strategyName, chunkId, row, outDir = DEFAULT_DIR } = params;
   const filePath = getAiChunkFilePath(strategyName, chunkId, outDir);
@@ -95,8 +160,8 @@ var closeAllAiDatasetWriters = async () => {
   }
 };
 var listAiChunkFiles = async (params) => {
-  const { strategyName, outDir = DEFAULT_DIR } = params;
-  const prefix = `ai-dataset-${toFileToken(strategyName)}-chunk-`;
+  const { strategyName, outDir = DEFAULT_DIR, runId } = params;
+  const prefix = getAiChunkFilePrefix(strategyName, runId);
   let entries = [];
   try {
     entries = await fs.readdir(outDir);
@@ -105,6 +170,20 @@ var listAiChunkFiles = async (params) => {
   }
   return entries.filter((name) => name.startsWith(prefix) && name.endsWith(".jsonl")).map((name) => path.join(outDir, name)).sort();
 };
+var listAiChunkRunIds = async (params) => {
+  const { strategyName, outDir = DEFAULT_DIR } = params;
+  let entries = [];
+  try {
+    entries = await fs.readdir(outDir);
+  } catch {
+    return [];
+  }
+  return [
+    ...new Set(
+      entries.map((name) => getRunIdFromAiChunkFileName(strategyName, name)).filter(Boolean)
+    )
+  ].sort();
+};
 var listAiChunkStrategies = async (params) => {
   const outDir = params?.outDir ?? DEFAULT_DIR;
   let entries = [];
@@ -203,7 +282,7 @@ var readNextNonEmptyLine = async (iterator) => {
     }
   }
 };
-var mergeSortedRuns = async (params) => {
+var mergeSortedRunBatch = async (params) => {
   const { runPaths, outPath } = params;
   await fs.mkdir(path.dirname(outPath), { recursive: true });
   const output = createWriteStream(outPath, { encoding: "utf8" });
@@ -264,12 +343,58 @@ var mergeSortedRuns = async (params) => {
     await outputDone;
   }
 };
+var mergeSortedRuns = async (params) => {
+  const {
+    runPaths,
+    outPath,
+    tempDir,
+    maxOpenRuns = AI_MERGE_SORT_MAX_OPEN_RUNS
+  } = params;
+  const resolvedMaxOpenRuns = Math.max(2, Math.trunc(maxOpenRuns));
+  let currentRunPaths = [...runPaths];
+  let passIndex = 0;
+  while (currentRunPaths.length > resolvedMaxOpenRuns) {
+    const nextRunPaths = [];
+    for (let groupStart = 0, groupIndex = 0; groupStart < currentRunPaths.length; groupStart += resolvedMaxOpenRuns, groupIndex += 1) {
+      const groupRunPaths = currentRunPaths.slice(
+        groupStart,
+        groupStart + resolvedMaxOpenRuns
+      );
+      if (groupRunPaths.length === 1) {
+        nextRunPaths.push(groupRunPaths[0]);
+        continue;
+      }
+      const passPath = path.join(
+        tempDir,
+        `merge-pass-${String(passIndex).padStart(3, "0")}-${String(
+          groupIndex
+        ).padStart(6, "0")}.jsonl`
+      );
+      await mergeSortedRunBatch({
+        runPaths: groupRunPaths,
+        outPath: passPath
+      });
+      nextRunPaths.push(passPath);
+      await Promise.all(
+        groupRunPaths.map((runPath) => fs.rm(runPath, { force: true }))
+      );
+    }
+    currentRunPaths = nextRunPaths;
+    passIndex += 1;
+  }
+  await mergeSortedRunBatch({
+    runPaths: currentRunPaths,
+    outPath
+  });
+};
 var mergeAiJsonlFiles = async (params) => {
   const {
     filePaths,
     outPath,
     maxRowsInMemory = AI_MERGE_SORT_RUN_MAX_ROWS,
-    maxBytesInMemory = AI_MERGE_SORT_RUN_MAX_BYTES
+    maxBytesInMemory = AI_MERGE_SORT_RUN_MAX_BYTES,
+    maxOpenRuns = AI_MERGE_SORT_MAX_OPEN_RUNS,
+    shouldIncludeRow
   } = params;
   const tempDir = path.join(
     path.dirname(outPath),
@@ -308,9 +433,13 @@ var mergeAiJsonlFiles = async (params) => {
           if (!trimmed) {
             continue;
           }
+          const row = parseAiDatasetLine(trimmed, filePath);
+          if (shouldIncludeRow && !shouldIncludeRow(row)) {
+            continue;
+          }
           batch.push({
             line: trimmed,
-            sortKey: getAiDatasetSortKey(parseAiDatasetLine(trimmed, filePath)),
+            sortKey: getAiDatasetSortKey(row),
             sourceIndex
           });
           sourceIndex += 1;
@@ -331,56 +460,275 @@ var mergeAiJsonlFiles = async (params) => {
     }
     await mergeSortedRuns({
       runPaths,
-      outPath
+      outPath,
+      tempDir,
+      maxOpenRuns
     });
   } finally {
     await fs.rm(tempDir, { recursive: true, force: true });
   }
 };
+var getAiDatasetPartPath = (filePath, partIndex) => {
+  const parsed = path.parse(filePath);
+  return path.join(parsed.dir, `${parsed.name}-part${partIndex}${parsed.ext}`);
+};
+var getPartWindowEndExclusive = (timestamp, monthsPerPart) => {
+  const partStart = startOfMonth(new Date(timestamp));
+  return addMonths(partStart, Math.max(1, monthsPerPart)).getTime();
+};
+var splitAiMergedDatasetFile = async (params) => {
+  const { filePath, monthsPerPart = 2 } = params;
+  const resolvedMonthsPerPart = Math.max(1, Math.trunc(monthsPerPart));
+  const reader = readline.createInterface({
+    input: createReadStream(filePath, { encoding: "utf8" }),
+    crlfDelay: Infinity
+  });
+  const partPaths = [];
+  let currentPartIndex = 0;
+  let currentPartPath = "";
+  let currentPartEndExclusive = Number.NaN;
+  let currentWriter = null;
+  let currentWriterDone = null;
+  let rowCount = 0;
+  const openPart = async (timestamp) => {
+    currentPartIndex += 1;
+    currentPartPath = getAiDatasetPartPath(filePath, currentPartIndex);
+    currentPartEndExclusive = getPartWindowEndExclusive(
+      timestamp,
+      resolvedMonthsPerPart
+    );
+    currentWriter = createWriteStream(currentPartPath, { encoding: "utf8" });
+    currentWriterDone = Promise.all([
+      once(currentWriter, "finish"),
+      once(currentWriter, "close")
+    ]);
+    partPaths.push(currentPartPath);
+  };
+  const closeCurrentWriter = async () => {
+    if (!currentWriter || !currentWriterDone) {
+      return;
+    }
+    currentWriter.end();
+    await currentWriterDone;
+    currentWriter = null;
+    currentWriterDone = null;
+  };
+  const getCurrentWriter = () => {
+    if (!currentWriter) {
+      throw new Error(
+        `AI dataset part writer was not initialized for ${filePath}`
+      );
+    }
+    return currentWriter;
+  };
+  try {
+    for await (const line of reader) {
+      const trimmed = line.trim();
+      if (!trimmed) {
+        continue;
+      }
+      const row = parseAiDatasetLine(trimmed, filePath);
+      const timestamp = Number(row.timestamp);
+      const safeTimestamp = Number.isFinite(timestamp) ? timestamp : Date.now();
+      if (!currentWriter || !Number.isFinite(currentPartEndExclusive) || safeTimestamp >= currentPartEndExclusive) {
+        await closeCurrentWriter();
+        await openPart(safeTimestamp);
+      }
+      const writer = getCurrentWriter();
+      if (!writer.write(`${trimmed}
+`)) {
+        await once(writer, "drain");
+      }
+      rowCount += 1;
+    }
+  } finally {
+    reader.close();
+    await closeCurrentWriter();
+  }
+  if (partPaths.length <= 1) {
+    for (const partPath of partPaths) {
+      await fs.rm(partPath, { force: true });
+    }
+    return {
+      partPaths: [filePath],
+      partCount: partPaths.length || (rowCount > 0 ? 1 : 0),
+      rowCount,
+      splitApplied: false
+    };
+  }
+  await fs.rm(filePath, { force: true });
+  return {
+    partPaths,
+    partCount: partPaths.length,
+    rowCount,
+    splitApplied: true
+  };
+};
 var readAiDatasetRows = async (params) => {
-  const { filePath, limitFromEnd = 0, skipFromEnd = 0 } = params;
+  const { limitFromEnd = 0, skipFromEnd = 0 } = params;
+  const resolvedFilePaths = resolveAiDatasetFilePaths(params);
+  if (!resolvedFilePaths.length) {
+    return {
+      rows: [],
+      totalRows: 0
+    };
+  }
   const rows = [];
   const recentLines = [];
   let totalRows = 0;
   const recentWindowLimit = limitFromEnd > 0 ? limitFromEnd + Math.max(0, skipFromEnd) : 0;
+  for (const currentFilePath of resolvedFilePaths) {
+    const reader = readline.createInterface({
+      input: createReadStream(currentFilePath, { encoding: "utf8" }),
+      crlfDelay: Infinity
+    });
+    for await (const line of reader) {
+      const trimmed = line.trim();
+      if (!trimmed) {
+        continue;
+      }
+      totalRows += 1;
+      if (limitFromEnd > 0) {
+        if (recentLines.length === recentWindowLimit) {
+          recentLines.shift();
+        }
+        recentLines.push(trimmed);
+      } else {
+        const row = JSON.parse(trimmed);
+        rows.push(row);
+      }
+    }
+    reader.close();
+  }
+  const effectiveSkip = Math.max(0, skipFromEnd);
+  const selectedRecentLines = effectiveSkip > 0 ? recentLines.slice(0, Math.max(0, recentLines.length - effectiveSkip)) : recentLines;
+  const selectedRows = limitFromEnd > 0 ? selectedRecentLines.map((line) => JSON.parse(line)) : effectiveSkip > 0 ? rows.slice(0, Math.max(0, rows.length - effectiveSkip)) : rows;
+  return {
+    rows: selectedRows,
+    totalRows
+  };
+};
+var resolveAiDatasetFilePaths = (params) => (Array.isArray(params.filePaths) && params.filePaths.length ? params.filePaths : params.filePath ? [params.filePath] : []).map((item) => path.resolve(item));
+var countNonEmptyAiDatasetRows = async (filePath) => {
+  let totalRows = 0;
   const reader = readline.createInterface({
     input: createReadStream(filePath, { encoding: "utf8" }),
     crlfDelay: Infinity
   });
   for await (const line of reader) {
-    const trimmed = line.trim();
-    if (!trimmed) {
-      continue;
+    if (line.trim()) {
+      totalRows += 1;
+    }
+  }
+  reader.close();
+  return totalRows;
+};
+var countAiDatasetRows = async (params) => {
+  const { limitFromEnd = 0, skipFromEnd = 0 } = params;
+  const resolvedFilePaths = resolveAiDatasetFilePaths(params);
+  let totalRows = 0;
+  for (const filePath of resolvedFilePaths) {
+    totalRows += await countNonEmptyAiDatasetRows(filePath);
+  }
+  const effectiveSkip = Math.max(0, skipFromEnd);
+  const selectedRows = limitFromEnd > 0 ? Math.min(limitFromEnd, Math.max(0, totalRows - effectiveSkip)) : Math.max(0, totalRows - effectiveSkip);
+  return {
+    totalRows,
+    selectedRows
+  };
+};
+var streamAiDatasetRows = async (params) => {
+  const { limitFromEnd = 0, skipFromEnd = 0, onRow } = params;
+  const resolvedFilePaths = resolveAiDatasetFilePaths(params);
+  if (!resolvedFilePaths.length) {
+    return {
+      totalRows: 0,
+      selectedRows: 0
+    };
+  }
+  const effectiveSkip = Math.max(0, skipFromEnd);
+  let totalRows = 0;
+  let selectedRows = 0;
+  if (limitFromEnd > 0) {
+    const recentLines = [];
+    const recentWindowLimit = limitFromEnd + effectiveSkip;
+    for (const currentFilePath of resolvedFilePaths) {
+      const reader = readline.createInterface({
+        input: createReadStream(currentFilePath, { encoding: "utf8" }),
+        crlfDelay: Infinity
+      });
+      for await (const line of reader) {
+        const trimmed = line.trim();
+        if (!trimmed) {
+          continue;
+        }
+        totalRows += 1;
+        if (recentLines.length === recentWindowLimit) {
+          recentLines.shift();
+        }
+        recentLines.push(trimmed);
+      }
+      reader.close();
+    }
+    const selectedRecentLines = effectiveSkip > 0 ? recentLines.slice(0, Math.max(0, recentLines.length - effectiveSkip)) : recentLines;
+    for (const line of selectedRecentLines) {
+      await onRow(JSON.parse(line), selectedRows);
+      selectedRows += 1;
     }
-    totalRows += 1;
-    if (limitFromEnd > 0) {
-      if (recentLines.length === recentWindowLimit) {
-        recentLines.shift();
+    return {
+      totalRows,
+      selectedRows
+    };
+  }
+  const trailingSkipBuffer = [];
+  for (const currentFilePath of resolvedFilePaths) {
+    const reader = readline.createInterface({
+      input: createReadStream(currentFilePath, { encoding: "utf8" }),
+      crlfDelay: Infinity
+    });
+    for await (const line of reader) {
+      const trimmed = line.trim();
+      if (!trimmed) {
+        continue;
       }
-      recentLines.push(trimmed);
-    } else {
-      const row = JSON.parse(trimmed);
-      rows.push(row);
+      totalRows += 1;
+      if (effectiveSkip > 0) {
+        trailingSkipBuffer.push(trimmed);
+        if (trailingSkipBuffer.length <= effectiveSkip) {
+          continue;
+        }
+        const selectedLine = trailingSkipBuffer.shift();
+        if (!selectedLine) {
+          continue;
+        }
+        await onRow(JSON.parse(selectedLine), selectedRows);
+        selectedRows += 1;
+        continue;
+      }
+      await onRow(JSON.parse(trimmed), selectedRows);
+      selectedRows += 1;
     }
+    reader.close();
   }
-  const effectiveSkip = Math.max(0, skipFromEnd);
-  const selectedRecentLines = effectiveSkip > 0 ? recentLines.slice(0, Math.max(0, recentLines.length - effectiveSkip)) : recentLines;
-  const selectedRows = limitFromEnd > 0 ? selectedRecentLines.map((line) => JSON.parse(line)) : effectiveSkip > 0 ? rows.slice(0, Math.max(0, rows.length - effectiveSkip)) : rows;
   return {
-    rows: selectedRows,
-    totalRows
+    totalRows,
+    selectedRows
   };
 };
 export {
   appendAiDatasetRow,
   closeAiDatasetWriter,
   closeAllAiDatasetWriters,
+  countAiDatasetRows,
   flushAiDatasetWriter,
   getAiChunkFilePath,
   listAiChunkFiles,
+  listAiChunkRunIds,
   listAiChunkStrategies,
   mergeAiJsonlFiles,
   mergeJsonlFiles,
   readAiDatasetRows,
+  splitAiMergedDatasetFile,
+  streamAiDatasetRows,
   toFileToken
 };

package/dist/backtestArtifacts.d.mts

@@ -0,0 +1,57 @@
+import { OrderLogData, PositionLogData } from '@tradejs/types';
+
+interface BacktestArtifactRef {
+    kind: 'file';
+    version: 1;
+    path: string;
+}
+declare const writeCachedBacktestArtifacts: ({ orderLog, orderLogId, positionLog, projectRoot, userName, }: {
+    orderLog: OrderLogData;
+    orderLogId: string;
+    positionLog: PositionLogData;
+    projectRoot?: string;
+    userName: string;
+}) => Promise<{
+    orderLog: BacktestArtifactRef;
+    positionLog: BacktestArtifactRef;
+}>;
+declare const readCachedBacktestArtifacts: ({ orderLogId, projectRoot, userName, }: {
+    orderLogId: string;
+    projectRoot?: string;
+    userName: string;
+}) => Promise<{
+    orderLog: OrderLogData | null;
+    positionLog: PositionLogData | null;
+}>;
+declare const deleteCachedBacktestArtifacts: ({ orderLogId, projectRoot, userName, }: {
+    orderLogId: string;
+    projectRoot?: string;
+    userName: string;
+}) => Promise<boolean>;
+declare const writePersistedBacktestOrderLog: ({ orderLog, projectRoot, strategyName, testName, userName, }: {
+    orderLog: OrderLogData;
+    projectRoot?: string;
+    strategyName: string;
+    testName: string;
+    userName: string;
+}) => Promise<BacktestArtifactRef>;
+declare const readPersistedBacktestOrderLog: ({ projectRoot, ref, strategyName, testName, userName, }: {
+    projectRoot?: string;
+    ref?: BacktestArtifactRef | null;
+    strategyName: string;
+    testName: string;
+    userName: string;
+}) => Promise<OrderLogData | null>;
+declare const deletePersistedBacktestOrderLog: ({ projectRoot, ref, strategyName, testName, userName, }: {
+    projectRoot?: string;
+    ref?: BacktestArtifactRef | null;
+    strategyName: string;
+    testName: string;
+    userName: string;
+}) => Promise<boolean>;
+declare const parseBacktestArtifactRef: (value: unknown) => BacktestArtifactRef | null;
+declare const getBacktestArtifactsRootDir: (projectRoot?: string) => string;
+declare const getBacktestCacheArtifactsDirForUser: (userName: string, projectRoot?: string) => string;
+declare const getPersistedBacktestArtifactsDirForUser: (userName: string, projectRoot?: string) => string;
+
+export { type BacktestArtifactRef, deleteCachedBacktestArtifacts, deletePersistedBacktestOrderLog, getBacktestArtifactsRootDir, getBacktestCacheArtifactsDirForUser, getPersistedBacktestArtifactsDirForUser, parseBacktestArtifactRef, readCachedBacktestArtifacts, readPersistedBacktestOrderLog, writeCachedBacktestArtifacts, writePersistedBacktestOrderLog };

package/dist/backtestArtifacts.d.ts

@@ -0,0 +1,57 @@
+import { OrderLogData, PositionLogData } from '@tradejs/types';
+
+interface BacktestArtifactRef {
+    kind: 'file';
+    version: 1;
+    path: string;
+}
+declare const writeCachedBacktestArtifacts: ({ orderLog, orderLogId, positionLog, projectRoot, userName, }: {
+    orderLog: OrderLogData;
+    orderLogId: string;
+    positionLog: PositionLogData;
+    projectRoot?: string;
+    userName: string;
+}) => Promise<{
+    orderLog: BacktestArtifactRef;
+    positionLog: BacktestArtifactRef;
+}>;
+declare const readCachedBacktestArtifacts: ({ orderLogId, projectRoot, userName, }: {
+    orderLogId: string;
+    projectRoot?: string;
+    userName: string;
+}) => Promise<{
+    orderLog: OrderLogData | null;
+    positionLog: PositionLogData | null;
+}>;
+declare const deleteCachedBacktestArtifacts: ({ orderLogId, projectRoot, userName, }: {
+    orderLogId: string;
+    projectRoot?: string;
+    userName: string;
+}) => Promise<boolean>;
+declare const writePersistedBacktestOrderLog: ({ orderLog, projectRoot, strategyName, testName, userName, }: {
+    orderLog: OrderLogData;
+    projectRoot?: string;
+    strategyName: string;
+    testName: string;
+    userName: string;
+}) => Promise<BacktestArtifactRef>;
+declare const readPersistedBacktestOrderLog: ({ projectRoot, ref, strategyName, testName, userName, }: {
+    projectRoot?: string;
+    ref?: BacktestArtifactRef | null;
+    strategyName: string;
+    testName: string;
+    userName: string;
+}) => Promise<OrderLogData | null>;
+declare const deletePersistedBacktestOrderLog: ({ projectRoot, ref, strategyName, testName, userName, }: {
+    projectRoot?: string;
+    ref?: BacktestArtifactRef | null;
+    strategyName: string;
+    testName: string;
+    userName: string;
+}) => Promise<boolean>;
+declare const parseBacktestArtifactRef: (value: unknown) => BacktestArtifactRef | null;
+declare const getBacktestArtifactsRootDir: (projectRoot?: string) => string;
+declare const getBacktestCacheArtifactsDirForUser: (userName: string, projectRoot?: string) => string;
+declare const getPersistedBacktestArtifactsDirForUser: (userName: string, projectRoot?: string) => string;
+
+export { type BacktestArtifactRef, deleteCachedBacktestArtifacts, deletePersistedBacktestOrderLog, getBacktestArtifactsRootDir, getBacktestCacheArtifactsDirForUser, getPersistedBacktestArtifactsDirForUser, parseBacktestArtifactRef, readCachedBacktestArtifacts, readPersistedBacktestOrderLog, writeCachedBacktestArtifacts, writePersistedBacktestOrderLog };