@tradejs/infra 1.0.9 → 1.0.11

package/dist/ai.d.mts

@@ -1,5 +1,5 @@
 import { AiDatasetRow } from '@tradejs/types';
-export { m as mergeJsonlFiles, t as toFileToken } from './mlDatasetFile-sRGWgR_o.mjs';
+export { m as mergeJsonlFiles, t as toFileToken } from './mlDatasetFile-Czx__g9M.mjs';
 
 declare const getAiChunkFilePath: (strategyName: string, chunkId: string, outDir?: string) => string;
 declare const appendAiDatasetRow: (params: {
@@ -14,6 +14,11 @@ declare const closeAllAiDatasetWriters: () => Promise<void>;
 declare const listAiChunkFiles: (params: {
     strategyName: string;
     outDir?: string;
+    runId?: string;
+}) => Promise<string[]>;
+declare const listAiChunkRunIds: (params: {
+    strategyName: string;
+    outDir?: string;
 }) => Promise<string[]>;
 declare const listAiChunkStrategies: (params?: {
     outDir?: string;
@@ -23,14 +28,45 @@ declare const mergeAiJsonlFiles: (params: {
     outPath: string;
     maxRowsInMemory?: number;
     maxBytesInMemory?: number;
+    maxOpenRuns?: number;
+    shouldIncludeRow?: (row: AiDatasetRow) => boolean;
 }) => Promise<void>;
-declare const readAiDatasetRows: (params: {
+declare const splitAiMergedDatasetFile: (params: {
     filePath: string;
+    monthsPerPart?: number;
+}) => Promise<{
+    partPaths: string[];
+    partCount: number;
+    rowCount: number;
+    splitApplied: boolean;
+}>;
+declare const readAiDatasetRows: (params: {
+    filePath?: string;
+    filePaths?: string[];
     limitFromEnd?: number;
     skipFromEnd?: number;
 }) => Promise<{
     rows: AiDatasetRow[];
     totalRows: number;
 }>;
+declare const countAiDatasetRows: (params: {
+    filePath?: string;
+    filePaths?: string[];
+    limitFromEnd?: number;
+    skipFromEnd?: number;
+}) => Promise<{
+    totalRows: number;
+    selectedRows: number;
+}>;
+declare const streamAiDatasetRows: (params: {
+    filePath?: string;
+    filePaths?: string[];
+    limitFromEnd?: number;
+    skipFromEnd?: number;
+    onRow: (row: AiDatasetRow, index: number) => Promise<void> | void;
+}) => Promise<{
+    totalRows: number;
+    selectedRows: number;
+}>;
 
-export { appendAiDatasetRow, closeAiDatasetWriter, closeAllAiDatasetWriters, flushAiDatasetWriter, getAiChunkFilePath, listAiChunkFiles, listAiChunkStrategies, mergeAiJsonlFiles, readAiDatasetRows };
+export { appendAiDatasetRow, closeAiDatasetWriter, closeAllAiDatasetWriters, countAiDatasetRows, flushAiDatasetWriter, getAiChunkFilePath, listAiChunkFiles, listAiChunkRunIds, listAiChunkStrategies, mergeAiJsonlFiles, readAiDatasetRows, splitAiMergedDatasetFile, streamAiDatasetRows };

package/dist/ai.d.ts

@@ -1,5 +1,5 @@
 import { AiDatasetRow } from '@tradejs/types';
-export { m as mergeJsonlFiles, t as toFileToken } from './mlDatasetFile-sRGWgR_o.js';
+export { m as mergeJsonlFiles, t as toFileToken } from './mlDatasetFile-Czx__g9M.js';
 
 declare const getAiChunkFilePath: (strategyName: string, chunkId: string, outDir?: string) => string;
 declare const appendAiDatasetRow: (params: {
@@ -14,6 +14,11 @@ declare const closeAllAiDatasetWriters: () => Promise<void>;
 declare const listAiChunkFiles: (params: {
     strategyName: string;
     outDir?: string;
+    runId?: string;
+}) => Promise<string[]>;
+declare const listAiChunkRunIds: (params: {
+    strategyName: string;
+    outDir?: string;
 }) => Promise<string[]>;
 declare const listAiChunkStrategies: (params?: {
     outDir?: string;
@@ -23,14 +28,45 @@ declare const mergeAiJsonlFiles: (params: {
     outPath: string;
     maxRowsInMemory?: number;
     maxBytesInMemory?: number;
+    maxOpenRuns?: number;
+    shouldIncludeRow?: (row: AiDatasetRow) => boolean;
 }) => Promise<void>;
-declare const readAiDatasetRows: (params: {
+declare const splitAiMergedDatasetFile: (params: {
     filePath: string;
+    monthsPerPart?: number;
+}) => Promise<{
+    partPaths: string[];
+    partCount: number;
+    rowCount: number;
+    splitApplied: boolean;
+}>;
+declare const readAiDatasetRows: (params: {
+    filePath?: string;
+    filePaths?: string[];
     limitFromEnd?: number;
     skipFromEnd?: number;
 }) => Promise<{
     rows: AiDatasetRow[];
     totalRows: number;
 }>;
+declare const countAiDatasetRows: (params: {
+    filePath?: string;
+    filePaths?: string[];
+    limitFromEnd?: number;
+    skipFromEnd?: number;
+}) => Promise<{
+    totalRows: number;
+    selectedRows: number;
+}>;
+declare const streamAiDatasetRows: (params: {
+    filePath?: string;
+    filePaths?: string[];
+    limitFromEnd?: number;
+    skipFromEnd?: number;
+    onRow: (row: AiDatasetRow, index: number) => Promise<void> | void;
+}) => Promise<{
+    totalRows: number;
+    selectedRows: number;
+}>;
 
-export { appendAiDatasetRow, closeAiDatasetWriter, closeAllAiDatasetWriters, flushAiDatasetWriter, getAiChunkFilePath, listAiChunkFiles, listAiChunkStrategies, mergeAiJsonlFiles, readAiDatasetRows };
+export { appendAiDatasetRow, closeAiDatasetWriter, closeAllAiDatasetWriters, countAiDatasetRows, flushAiDatasetWriter, getAiChunkFilePath, listAiChunkFiles, listAiChunkRunIds, listAiChunkStrategies, mergeAiJsonlFiles, readAiDatasetRows, splitAiMergedDatasetFile, streamAiDatasetRows };

package/dist/ai.js

@@ -33,13 +33,17 @@ __export(ai_exports, {
   appendAiDatasetRow: () => appendAiDatasetRow,
   closeAiDatasetWriter: () => closeAiDatasetWriter,
   closeAllAiDatasetWriters: () => closeAllAiDatasetWriters,
+  countAiDatasetRows: () => countAiDatasetRows,
   flushAiDatasetWriter: () => flushAiDatasetWriter,
   getAiChunkFilePath: () => getAiChunkFilePath,
   listAiChunkFiles: () => listAiChunkFiles,
+  listAiChunkRunIds: () => listAiChunkRunIds,
   listAiChunkStrategies: () => listAiChunkStrategies,
   mergeAiJsonlFiles: () => mergeAiJsonlFiles,
   mergeJsonlFiles: () => mergeJsonlFiles,
   readAiDatasetRows: () => readAiDatasetRows,
+  splitAiMergedDatasetFile: () => splitAiMergedDatasetFile,
+  streamAiDatasetRows: () => streamAiDatasetRows,
   toFileToken: () => toFileToken
 });
 module.exports = __toCommonJS(ai_exports);
@@ -50,21 +54,99 @@ var import_fs2 = require("fs");
 var import_promises2 = __toESM(require("fs/promises"));
 var import_path2 = __toESM(require("path"));
 var import_node_fs = require("fs");
-var import_node_readline = __toESM(require("readline"));
+var import_node_readline2 = __toESM(require("readline"));
+
+// ../../node_modules/date-fns/toDate.mjs
+function toDate(argument) {
+  const argStr = Object.prototype.toString.call(argument);
+  if (argument instanceof Date || typeof argument === "object" && argStr === "[object Date]") {
+    return new argument.constructor(+argument);
+  } else if (typeof argument === "number" || argStr === "[object Number]" || typeof argument === "string" || argStr === "[object String]") {
+    return new Date(argument);
+  } else {
+    return /* @__PURE__ */ new Date(NaN);
+  }
+}
+
+// ../../node_modules/date-fns/constructFrom.mjs
+function constructFrom(date, value) {
+  if (date instanceof Date) {
+    return new date.constructor(value);
+  } else {
+    return new Date(value);
+  }
+}
+
+// ../../node_modules/date-fns/addMonths.mjs
+function addMonths(date, amount) {
+  const _date = toDate(date);
+  if (isNaN(amount)) return constructFrom(date, NaN);
+  if (!amount) {
+    return _date;
+  }
+  const dayOfMonth = _date.getDate();
+  const endOfDesiredMonth = constructFrom(date, _date.getTime());
+  endOfDesiredMonth.setMonth(_date.getMonth() + amount + 1, 0);
+  const daysInMonth = endOfDesiredMonth.getDate();
+  if (dayOfMonth >= daysInMonth) {
+    return endOfDesiredMonth;
+  } else {
+    _date.setFullYear(
+      endOfDesiredMonth.getFullYear(),
+      endOfDesiredMonth.getMonth(),
+      dayOfMonth
+    );
+    return _date;
+  }
+}
+
+// ../../node_modules/date-fns/startOfMonth.mjs
+function startOfMonth(date) {
+  const _date = toDate(date);
+  _date.setDate(1);
+  _date.setHours(0, 0, 0, 0);
+  return _date;
+}
 
 // src/mlDatasetFile.ts
 var import_events = require("events");
 var import_fs = require("fs");
 var import_promises = __toESM(require("fs/promises"));
 var import_path = __toESM(require("path"));
+var import_node_readline = __toESM(require("readline"));
 var toFileToken = (value) => value.trim().toLowerCase().replace(/[^a-z0-9_-]+/g, "_").replace(/^_+|_+$/g, "") || "any";
 var mergeJsonlFiles = async (params) => {
-  const { filePaths, outPath } = params;
+  const { filePaths, outPath, shouldIncludeRow } = params;
   await import_promises.default.mkdir(import_path.default.dirname(outPath), { recursive: true });
   const stream = (0, import_fs.createWriteStream)(outPath, { encoding: "utf8" });
   const done = Promise.all([(0, import_events.once)(stream, "finish"), (0, import_events.once)(stream, "close")]);
   try {
     for (const filePath of filePaths) {
+      if (shouldIncludeRow) {
+        const reader2 = import_node_readline.default.createInterface({
+          input: (0, import_fs.createReadStream)(filePath, { encoding: "utf8" }),
+          crlfDelay: Infinity
+        });
+        try {
+          for await (const line of reader2) {
+            const trimmed = line.trim();
+            if (!trimmed) {
+              continue;
+            }
+            const row = JSON.parse(trimmed);
+            if (!shouldIncludeRow(row)) {
+              continue;
+            }
+            if (!stream.write(`${trimmed}
+`)) {
+              await (0, import_events.once)(stream, "drain");
+            }
+          }
+        } finally {
+          reader2.close();
+        }
+        continue;
+      }
       const reader = (0, import_fs.createReadStream)(filePath, { encoding: "utf8" });
       for await (const chunk of reader) {
         if (!stream.write(chunk)) {
@@ -83,12 +165,23 @@ var DEFAULT_DIR = "data/ai/export";
 var AI_DATASET_WRITE_BATCH_SIZE = 100;
 var AI_MERGE_SORT_RUN_MAX_ROWS = 2e3;
 var AI_MERGE_SORT_RUN_MAX_BYTES = 16 * 1024 * 1024;
+var AI_MERGE_SORT_MAX_OPEN_RUNS = 16;
 var AI_CHUNK_FILE_RE = /^ai-dataset-(.+)-chunk-[^.]+\.jsonl$/;
+var BACKTEST_RUN_CHUNK_ID_RE = /^(\d{12}-[a-f0-9]{8})-/;
 var writerByPath = /* @__PURE__ */ new Map();
 var getAiChunkFilePath = (strategyName, chunkId, outDir = DEFAULT_DIR) => import_path2.default.join(
   outDir,
   `ai-dataset-${toFileToken(strategyName)}-chunk-${toFileToken(chunkId)}.jsonl`
 );
+var getAiChunkFilePrefix = (strategyName, runId) => `ai-dataset-${toFileToken(strategyName)}-chunk-${runId ? `${toFileToken(runId)}-` : ""}`;
+var getRunIdFromAiChunkFileName = (strategyName, fileName) => {
+  const prefix = getAiChunkFilePrefix(strategyName);
+  if (!fileName.startsWith(prefix) || !fileName.endsWith(".jsonl")) {
+    return "";
+  }
+  const chunkToken = fileName.slice(prefix.length, -".jsonl".length);
+  return chunkToken.match(BACKTEST_RUN_CHUNK_ID_RE)?.[1] ?? "";
+};
 var appendAiDatasetRow = async (params) => {
   const { strategyName, chunkId, row, outDir = DEFAULT_DIR } = params;
   const filePath = getAiChunkFilePath(strategyName, chunkId, outDir);
@@ -164,8 +257,8 @@ var closeAllAiDatasetWriters = async () => {
   }
 };
 var listAiChunkFiles = async (params) => {
-  const { strategyName, outDir = DEFAULT_DIR } = params;
-  const prefix = `ai-dataset-${toFileToken(strategyName)}-chunk-`;
+  const { strategyName, outDir = DEFAULT_DIR, runId } = params;
+  const prefix = getAiChunkFilePrefix(strategyName, runId);
   let entries = [];
   try {
     entries = await import_promises2.default.readdir(outDir);
@@ -174,6 +267,20 @@ var listAiChunkFiles = async (params) => {
   }
   return entries.filter((name) => name.startsWith(prefix) && name.endsWith(".jsonl")).map((name) => import_path2.default.join(outDir, name)).sort();
 };
+var listAiChunkRunIds = async (params) => {
+  const { strategyName, outDir = DEFAULT_DIR } = params;
+  let entries = [];
+  try {
+    entries = await import_promises2.default.readdir(outDir);
+  } catch {
+    return [];
+  }
+  return [
+    ...new Set(
+      entries.map((name) => getRunIdFromAiChunkFileName(strategyName, name)).filter(Boolean)
+    )
+  ].sort();
+};
 var listAiChunkStrategies = async (params) => {
   const outDir = params?.outDir ?? DEFAULT_DIR;
   let entries = [];
@@ -272,7 +379,7 @@ var readNextNonEmptyLine = async (iterator) => {
     }
   }
 };
-var mergeSortedRuns = async (params) => {
+var mergeSortedRunBatch = async (params) => {
   const { runPaths, outPath } = params;
   await import_promises2.default.mkdir(import_path2.default.dirname(outPath), { recursive: true });
   const output = (0, import_fs2.createWriteStream)(outPath, { encoding: "utf8" });
@@ -284,7 +391,7 @@ var mergeSortedRuns = async (params) => {
   try {
     for (let runIndex = 0; runIndex < runPaths.length; runIndex += 1) {
       const runPath = runPaths[runIndex];
-      const reader = import_node_readline.default.createInterface({
+      const reader = import_node_readline2.default.createInterface({
         input: (0, import_node_fs.createReadStream)(runPath, { encoding: "utf8" }),
         crlfDelay: Infinity
       });
@@ -333,12 +440,58 @@ var mergeSortedRuns = async (params) => {
     await outputDone;
   }
 };
+var mergeSortedRuns = async (params) => {
+  const {
+    runPaths,
+    outPath,
+    tempDir,
+    maxOpenRuns = AI_MERGE_SORT_MAX_OPEN_RUNS
+  } = params;
+  const resolvedMaxOpenRuns = Math.max(2, Math.trunc(maxOpenRuns));
+  let currentRunPaths = [...runPaths];
+  let passIndex = 0;
+  while (currentRunPaths.length > resolvedMaxOpenRuns) {
+    const nextRunPaths = [];
+    for (let groupStart = 0, groupIndex = 0; groupStart < currentRunPaths.length; groupStart += resolvedMaxOpenRuns, groupIndex += 1) {
+      const groupRunPaths = currentRunPaths.slice(
+        groupStart,
+        groupStart + resolvedMaxOpenRuns
+      );
+      if (groupRunPaths.length === 1) {
+        nextRunPaths.push(groupRunPaths[0]);
+        continue;
+      }
+      const passPath = import_path2.default.join(
+        tempDir,
+        `merge-pass-${String(passIndex).padStart(3, "0")}-${String(
+          groupIndex
+        ).padStart(6, "0")}.jsonl`
+      );
+      await mergeSortedRunBatch({
+        runPaths: groupRunPaths,
+        outPath: passPath
+      });
+      nextRunPaths.push(passPath);
+      await Promise.all(
+        groupRunPaths.map((runPath) => import_promises2.default.rm(runPath, { force: true }))
+      );
+    }
+    currentRunPaths = nextRunPaths;
+    passIndex += 1;
+  }
+  await mergeSortedRunBatch({
+    runPaths: currentRunPaths,
+    outPath
+  });
+};
 var mergeAiJsonlFiles = async (params) => {
   const {
     filePaths,
     outPath,
     maxRowsInMemory = AI_MERGE_SORT_RUN_MAX_ROWS,
-    maxBytesInMemory = AI_MERGE_SORT_RUN_MAX_BYTES
+    maxBytesInMemory = AI_MERGE_SORT_RUN_MAX_BYTES,
+    maxOpenRuns = AI_MERGE_SORT_MAX_OPEN_RUNS,
+    shouldIncludeRow
   } = params;
   const tempDir = import_path2.default.join(
     import_path2.default.dirname(outPath),
@@ -367,7 +520,7 @@ var mergeAiJsonlFiles = async (params) => {
   try {
     await import_promises2.default.mkdir(tempDir, { recursive: true });
     for (const filePath of filePaths) {
-      const reader = import_node_readline.default.createInterface({
+      const reader = import_node_readline2.default.createInterface({
         input: (0, import_node_fs.createReadStream)(filePath, { encoding: "utf8" }),
         crlfDelay: Infinity
       });
@@ -377,9 +530,13 @@ var mergeAiJsonlFiles = async (params) => {
           if (!trimmed) {
             continue;
           }
+          const row = parseAiDatasetLine(trimmed, filePath);
+          if (shouldIncludeRow && !shouldIncludeRow(row)) {
+            continue;
+          }
           batch.push({
             line: trimmed,
-            sortKey: getAiDatasetSortKey(parseAiDatasetLine(trimmed, filePath)),
+            sortKey: getAiDatasetSortKey(row),
             sourceIndex
           });
           sourceIndex += 1;
@@ -400,37 +557,145 @@ var mergeAiJsonlFiles = async (params) => {
     }
     await mergeSortedRuns({
       runPaths,
-      outPath
+      outPath,
+      tempDir,
+      maxOpenRuns
     });
   } finally {
     await import_promises2.default.rm(tempDir, { recursive: true, force: true });
   }
 };
+var getAiDatasetPartPath = (filePath, partIndex) => {
+  const parsed = import_path2.default.parse(filePath);
+  return import_path2.default.join(parsed.dir, `${parsed.name}-part${partIndex}${parsed.ext}`);
+};
+var getPartWindowEndExclusive = (timestamp, monthsPerPart) => {
+  const partStart = startOfMonth(new Date(timestamp));
+  return addMonths(partStart, Math.max(1, monthsPerPart)).getTime();
+};
+var splitAiMergedDatasetFile = async (params) => {
+  const { filePath, monthsPerPart = 2 } = params;
+  const resolvedMonthsPerPart = Math.max(1, Math.trunc(monthsPerPart));
+  const reader = import_node_readline2.default.createInterface({
+    input: (0, import_node_fs.createReadStream)(filePath, { encoding: "utf8" }),
+    crlfDelay: Infinity
+  });
+  const partPaths = [];
+  let currentPartIndex = 0;
+  let currentPartPath = "";
+  let currentPartEndExclusive = Number.NaN;
+  let currentWriter = null;
+  let currentWriterDone = null;
+  let rowCount = 0;
+  const openPart = async (timestamp) => {
+    currentPartIndex += 1;
+    currentPartPath = getAiDatasetPartPath(filePath, currentPartIndex);
+    currentPartEndExclusive = getPartWindowEndExclusive(
+      timestamp,
+      resolvedMonthsPerPart
+    );
+    currentWriter = (0, import_fs2.createWriteStream)(currentPartPath, { encoding: "utf8" });
+    currentWriterDone = Promise.all([
+      (0, import_events2.once)(currentWriter, "finish"),
+      (0, import_events2.once)(currentWriter, "close")
+    ]);
+    partPaths.push(currentPartPath);
+  };
+  const closeCurrentWriter = async () => {
+    if (!currentWriter || !currentWriterDone) {
+      return;
+    }
+    currentWriter.end();
+    await currentWriterDone;
+    currentWriter = null;
+    currentWriterDone = null;
+  };
+  const getCurrentWriter = () => {
+    if (!currentWriter) {
+      throw new Error(
+        `AI dataset part writer was not initialized for ${filePath}`
+      );
+    }
+    return currentWriter;
+  };
+  try {
+    for await (const line of reader) {
+      const trimmed = line.trim();
+      if (!trimmed) {
+        continue;
+      }
+      const row = parseAiDatasetLine(trimmed, filePath);
+      const timestamp = Number(row.timestamp);
+      const safeTimestamp = Number.isFinite(timestamp) ? timestamp : Date.now();
+      if (!currentWriter || !Number.isFinite(currentPartEndExclusive) || safeTimestamp >= currentPartEndExclusive) {
+        await closeCurrentWriter();
+        await openPart(safeTimestamp);
+      }
+      const writer = getCurrentWriter();
+      if (!writer.write(`${trimmed}
+`)) {
+        await (0, import_events2.once)(writer, "drain");
+      }
+      rowCount += 1;
+    }
+  } finally {
+    reader.close();
+    await closeCurrentWriter();
+  }
+  if (partPaths.length <= 1) {
+    for (const partPath of partPaths) {
+      await import_promises2.default.rm(partPath, { force: true });
+    }
+    return {
+      partPaths: [filePath],
+      partCount: partPaths.length || (rowCount > 0 ? 1 : 0),
+      rowCount,
+      splitApplied: false
+    };
+  }
+  await import_promises2.default.rm(filePath, { force: true });
+  return {
+    partPaths,
+    partCount: partPaths.length,
+    rowCount,
+    splitApplied: true
+  };
+};
 var readAiDatasetRows = async (params) => {
-  const { filePath, limitFromEnd = 0, skipFromEnd = 0 } = params;
+  const { limitFromEnd = 0, skipFromEnd = 0 } = params;
+  const resolvedFilePaths = resolveAiDatasetFilePaths(params);
+  if (!resolvedFilePaths.length) {
+    return {
+      rows: [],
+      totalRows: 0
+    };
+  }
   const rows = [];
   const recentLines = [];
   let totalRows = 0;
   const recentWindowLimit = limitFromEnd > 0 ? limitFromEnd + Math.max(0, skipFromEnd) : 0;
-  const reader = import_node_readline.default.createInterface({
-    input: (0, import_node_fs.createReadStream)(filePath, { encoding: "utf8" }),
-    crlfDelay: Infinity
-  });
-  for await (const line of reader) {
-    const trimmed = line.trim();
-    if (!trimmed) {
-      continue;
-    }
-    totalRows += 1;
-    if (limitFromEnd > 0) {
-      if (recentLines.length === recentWindowLimit) {
-        recentLines.shift();
+  for (const currentFilePath of resolvedFilePaths) {
+    const reader = import_node_readline2.default.createInterface({
+      input: (0, import_node_fs.createReadStream)(currentFilePath, { encoding: "utf8" }),
+      crlfDelay: Infinity
+    });
+    for await (const line of reader) {
+      const trimmed = line.trim();
+      if (!trimmed) {
+        continue;
+      }
+      totalRows += 1;
+      if (limitFromEnd > 0) {
+        if (recentLines.length === recentWindowLimit) {
+          recentLines.shift();
+        }
+        recentLines.push(trimmed);
+      } else {
+        const row = JSON.parse(trimmed);
+        rows.push(row);
       }
-      recentLines.push(trimmed);
-    } else {
-      const row = JSON.parse(trimmed);
-      rows.push(row);
     }
+    reader.close();
   }
   const effectiveSkip = Math.max(0, skipFromEnd);
   const selectedRecentLines = effectiveSkip > 0 ? recentLines.slice(0, Math.max(0, recentLines.length - effectiveSkip)) : recentLines;
@@ -440,17 +705,128 @@ var readAiDatasetRows = async (params) => {
     totalRows
   };
 };
+var resolveAiDatasetFilePaths = (params) => (Array.isArray(params.filePaths) && params.filePaths.length ? params.filePaths : params.filePath ? [params.filePath] : []).map((item) => import_path2.default.resolve(item));
+var countNonEmptyAiDatasetRows = async (filePath) => {
+  let totalRows = 0;
+  const reader = import_node_readline2.default.createInterface({
+    input: (0, import_node_fs.createReadStream)(filePath, { encoding: "utf8" }),
+    crlfDelay: Infinity
+  });
+  for await (const line of reader) {
+    if (line.trim()) {
+      totalRows += 1;
+    }
+  }
+  reader.close();
+  return totalRows;
+};
+var countAiDatasetRows = async (params) => {
+  const { limitFromEnd = 0, skipFromEnd = 0 } = params;
+  const resolvedFilePaths = resolveAiDatasetFilePaths(params);
+  let totalRows = 0;
+  for (const filePath of resolvedFilePaths) {
+    totalRows += await countNonEmptyAiDatasetRows(filePath);
+  }
+  const effectiveSkip = Math.max(0, skipFromEnd);
+  const selectedRows = limitFromEnd > 0 ? Math.min(limitFromEnd, Math.max(0, totalRows - effectiveSkip)) : Math.max(0, totalRows - effectiveSkip);
+  return {
+    totalRows,
+    selectedRows
+  };
+};
+var streamAiDatasetRows = async (params) => {
+  const { limitFromEnd = 0, skipFromEnd = 0, onRow } = params;
+  const resolvedFilePaths = resolveAiDatasetFilePaths(params);
+  if (!resolvedFilePaths.length) {
+    return {
+      totalRows: 0,
+      selectedRows: 0
+    };
+  }
+  const effectiveSkip = Math.max(0, skipFromEnd);
+  let totalRows = 0;
+  let selectedRows = 0;
+  if (limitFromEnd > 0) {
+    const recentLines = [];
+    const recentWindowLimit = limitFromEnd + effectiveSkip;
+    for (const currentFilePath of resolvedFilePaths) {
+      const reader = import_node_readline2.default.createInterface({
+        input: (0, import_node_fs.createReadStream)(currentFilePath, { encoding: "utf8" }),
+        crlfDelay: Infinity
+      });
+      for await (const line of reader) {
+        const trimmed = line.trim();
+        if (!trimmed) {
+          continue;
+        }
+        totalRows += 1;
+        if (recentLines.length === recentWindowLimit) {
+          recentLines.shift();
+        }
+        recentLines.push(trimmed);
+      }
+      reader.close();
+    }
+    const selectedRecentLines = effectiveSkip > 0 ? recentLines.slice(0, Math.max(0, recentLines.length - effectiveSkip)) : recentLines;
+    for (const line of selectedRecentLines) {
+      await onRow(JSON.parse(line), selectedRows);
+      selectedRows += 1;
+    }
+    return {
+      totalRows,
+      selectedRows
+    };
+  }
+  const trailingSkipBuffer = [];
+  for (const currentFilePath of resolvedFilePaths) {
+    const reader = import_node_readline2.default.createInterface({
+      input: (0, import_node_fs.createReadStream)(currentFilePath, { encoding: "utf8" }),
+      crlfDelay: Infinity
+    });
+    for await (const line of reader) {
+      const trimmed = line.trim();
+      if (!trimmed) {
+        continue;
+      }
+      totalRows += 1;
+      if (effectiveSkip > 0) {
+        trailingSkipBuffer.push(trimmed);
+        if (trailingSkipBuffer.length <= effectiveSkip) {
+          continue;
+        }
+        const selectedLine = trailingSkipBuffer.shift();
+        if (!selectedLine) {
+          continue;
+        }
+        await onRow(JSON.parse(selectedLine), selectedRows);
+        selectedRows += 1;
+        continue;
+      }
+      await onRow(JSON.parse(trimmed), selectedRows);
+      selectedRows += 1;
+    }
+    reader.close();
+  }
+  return {
+    totalRows,
+    selectedRows
+  };
+};
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   appendAiDatasetRow,
   closeAiDatasetWriter,
   closeAllAiDatasetWriters,
+  countAiDatasetRows,
   flushAiDatasetWriter,
   getAiChunkFilePath,
   listAiChunkFiles,
+  listAiChunkRunIds,
   listAiChunkStrategies,
   mergeAiJsonlFiles,
   mergeJsonlFiles,
   readAiDatasetRows,
+  splitAiMergedDatasetFile,
+  streamAiDatasetRows,
   toFileToken
 });