@tradejs/infra 1.0.4 → 1.0.6

package/README.md

@@ -1,6 +1,6 @@
 # @tradejs/infra
 
-Server-only infrastructure adapters for TradeJS.
+Server-only infrastructure adapters for the TradeJS open-source framework.
 
 - Homepage: https://tradejs.dev
 - Documentation: https://docs.tradejs.dev

package/dist/ai.d.mts

@@ -0,0 +1,36 @@
+import { AiDatasetRow } from '@tradejs/types';
+export { m as mergeJsonlFiles, t as toFileToken } from './mlDatasetFile-sRGWgR_o.mjs';
+
+declare const getAiChunkFilePath: (strategyName: string, chunkId: string, outDir?: string) => string;
+declare const appendAiDatasetRow: (params: {
+    strategyName: string;
+    chunkId: string;
+    row: AiDatasetRow;
+    outDir?: string;
+}) => Promise<string>;
+declare const flushAiDatasetWriter: (filePath: string) => Promise<void>;
+declare const closeAiDatasetWriter: (filePath: string) => Promise<void>;
+declare const closeAllAiDatasetWriters: () => Promise<void>;
+declare const listAiChunkFiles: (params: {
+    strategyName: string;
+    outDir?: string;
+}) => Promise<string[]>;
+declare const listAiChunkStrategies: (params?: {
+    outDir?: string;
+}) => Promise<string[]>;
+declare const mergeAiJsonlFiles: (params: {
+    filePaths: string[];
+    outPath: string;
+    maxRowsInMemory?: number;
+    maxBytesInMemory?: number;
+}) => Promise<void>;
+declare const readAiDatasetRows: (params: {
+    filePath: string;
+    limitFromEnd?: number;
+    skipFromEnd?: number;
+}) => Promise<{
+    rows: AiDatasetRow[];
+    totalRows: number;
+}>;
+
+export { appendAiDatasetRow, closeAiDatasetWriter, closeAllAiDatasetWriters, flushAiDatasetWriter, getAiChunkFilePath, listAiChunkFiles, listAiChunkStrategies, mergeAiJsonlFiles, readAiDatasetRows };

package/dist/ai.d.ts

@@ -0,0 +1,36 @@
+import { AiDatasetRow } from '@tradejs/types';
+export { m as mergeJsonlFiles, t as toFileToken } from './mlDatasetFile-sRGWgR_o.js';
+
+declare const getAiChunkFilePath: (strategyName: string, chunkId: string, outDir?: string) => string;
+declare const appendAiDatasetRow: (params: {
+    strategyName: string;
+    chunkId: string;
+    row: AiDatasetRow;
+    outDir?: string;
+}) => Promise<string>;
+declare const flushAiDatasetWriter: (filePath: string) => Promise<void>;
+declare const closeAiDatasetWriter: (filePath: string) => Promise<void>;
+declare const closeAllAiDatasetWriters: () => Promise<void>;
+declare const listAiChunkFiles: (params: {
+    strategyName: string;
+    outDir?: string;
+}) => Promise<string[]>;
+declare const listAiChunkStrategies: (params?: {
+    outDir?: string;
+}) => Promise<string[]>;
+declare const mergeAiJsonlFiles: (params: {
+    filePaths: string[];
+    outPath: string;
+    maxRowsInMemory?: number;
+    maxBytesInMemory?: number;
+}) => Promise<void>;
+declare const readAiDatasetRows: (params: {
+    filePath: string;
+    limitFromEnd?: number;
+    skipFromEnd?: number;
+}) => Promise<{
+    rows: AiDatasetRow[];
+    totalRows: number;
+}>;
+
+export { appendAiDatasetRow, closeAiDatasetWriter, closeAllAiDatasetWriters, flushAiDatasetWriter, getAiChunkFilePath, listAiChunkFiles, listAiChunkStrategies, mergeAiJsonlFiles, readAiDatasetRows };

package/dist/ai.js

@@ -0,0 +1,456 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/ai.ts
+var ai_exports = {};
+__export(ai_exports, {
+  appendAiDatasetRow: () => appendAiDatasetRow,
+  closeAiDatasetWriter: () => closeAiDatasetWriter,
+  closeAllAiDatasetWriters: () => closeAllAiDatasetWriters,
+  flushAiDatasetWriter: () => flushAiDatasetWriter,
+  getAiChunkFilePath: () => getAiChunkFilePath,
+  listAiChunkFiles: () => listAiChunkFiles,
+  listAiChunkStrategies: () => listAiChunkStrategies,
+  mergeAiJsonlFiles: () => mergeAiJsonlFiles,
+  mergeJsonlFiles: () => mergeJsonlFiles,
+  readAiDatasetRows: () => readAiDatasetRows,
+  toFileToken: () => toFileToken
+});
+module.exports = __toCommonJS(ai_exports);
+
+// src/aiDatasetFile.ts
+var import_events2 = require("events");
+var import_fs2 = require("fs");
+var import_promises2 = __toESM(require("fs/promises"));
+var import_path2 = __toESM(require("path"));
+var import_node_fs = require("fs");
+var import_node_readline = __toESM(require("readline"));
+
+// src/mlDatasetFile.ts
+var import_events = require("events");
+var import_fs = require("fs");
+var import_promises = __toESM(require("fs/promises"));
+var import_path = __toESM(require("path"));
+var toFileToken = (value) => value.trim().toLowerCase().replace(/[^a-z0-9_-]+/g, "_").replace(/^_+|_+$/g, "") || "any";
+var mergeJsonlFiles = async (params) => {
+  const { filePaths, outPath } = params;
+  await import_promises.default.mkdir(import_path.default.dirname(outPath), { recursive: true });
+  const stream = (0, import_fs.createWriteStream)(outPath, { encoding: "utf8" });
+  const done = Promise.all([(0, import_events.once)(stream, "finish"), (0, import_events.once)(stream, "close")]);
+  try {
+    for (const filePath of filePaths) {
+      const reader = (0, import_fs.createReadStream)(filePath, { encoding: "utf8" });
+      for await (const chunk of reader) {
+        if (!stream.write(chunk)) {
+          await (0, import_events.once)(stream, "drain");
+        }
+      }
+    }
+  } finally {
+    stream.end();
+    await done;
+  }
+};
+
+// src/aiDatasetFile.ts
+var DEFAULT_DIR = "data/ai/export";
+var AI_DATASET_WRITE_BATCH_SIZE = 100;
+var AI_MERGE_SORT_RUN_MAX_ROWS = 2e3;
+var AI_MERGE_SORT_RUN_MAX_BYTES = 16 * 1024 * 1024;
+var AI_CHUNK_FILE_RE = /^ai-dataset-(.+)-chunk-[^.]+\.jsonl$/;
+var writerByPath = /* @__PURE__ */ new Map();
+var getAiChunkFilePath = (strategyName, chunkId, outDir = DEFAULT_DIR) => import_path2.default.join(
+  outDir,
+  `ai-dataset-${toFileToken(strategyName)}-chunk-${toFileToken(chunkId)}.jsonl`
+);
+var appendAiDatasetRow = async (params) => {
+  const { strategyName, chunkId, row, outDir = DEFAULT_DIR } = params;
+  const filePath = getAiChunkFilePath(strategyName, chunkId, outDir);
+  let state = writerByPath.get(filePath);
+  if (!state) {
+    await import_promises2.default.mkdir(outDir, { recursive: true });
+    const stream = (0, import_fs2.createWriteStream)(filePath, {
+      encoding: "utf8",
+      flags: "a"
+    });
+    state = {
+      filePath,
+      stream,
+      buffer: [],
+      writeQueue: Promise.resolve(),
+      closed: false
+    };
+    writerByPath.set(filePath, state);
+  }
+  if (state.closed) {
+    throw new Error(`AI dataset writer is closed: ${filePath}`);
+  }
+  state.buffer.push(`${JSON.stringify(row)}
+`);
+  if (state.buffer.length >= AI_DATASET_WRITE_BATCH_SIZE) {
+    await flushAiDatasetWriter(filePath);
+  }
+  return filePath;
+};
+var flushState = async (state) => {
+  if (state.closed || state.buffer.length === 0) {
+    return;
+  }
+  const chunk = state.buffer.join("");
+  state.buffer = [];
+  if (!state.stream.write(chunk)) {
+    await (0, import_events2.once)(state.stream, "drain");
+  }
+};
+var flushAiDatasetWriter = async (filePath) => {
+  const state = writerByPath.get(filePath);
+  if (!state || state.closed) {
+    return;
+  }
+  state.writeQueue = state.writeQueue.then(() => flushState(state));
+  await state.writeQueue;
+};
+var closeState = async (state) => {
+  if (state.closed) {
+    return;
+  }
+  await flushState(state);
+  state.closed = true;
+  state.stream.end();
+  await Promise.all([
+    (0, import_events2.once)(state.stream, "finish"),
+    (0, import_events2.once)(state.stream, "close")
+  ]);
+};
+var closeAiDatasetWriter = async (filePath) => {
+  const state = writerByPath.get(filePath);
+  if (!state) {
+    return;
+  }
+  state.writeQueue = state.writeQueue.then(() => closeState(state));
+  await state.writeQueue;
+  writerByPath.delete(filePath);
+};
+var closeAllAiDatasetWriters = async () => {
+  const filePaths = [...writerByPath.keys()];
+  for (const filePath of filePaths) {
+    await closeAiDatasetWriter(filePath);
+  }
+};
+var listAiChunkFiles = async (params) => {
+  const { strategyName, outDir = DEFAULT_DIR } = params;
+  const prefix = `ai-dataset-${toFileToken(strategyName)}-chunk-`;
+  let entries = [];
+  try {
+    entries = await import_promises2.default.readdir(outDir);
+  } catch {
+    return [];
+  }
+  return entries.filter((name) => name.startsWith(prefix) && name.endsWith(".jsonl")).map((name) => import_path2.default.join(outDir, name)).sort();
+};
+var listAiChunkStrategies = async (params) => {
+  const outDir = params?.outDir ?? DEFAULT_DIR;
+  let entries = [];
+  try {
+    entries = await import_promises2.default.readdir(outDir);
+  } catch {
+    return [];
+  }
+  return [
+    ...new Set(
+      entries.map((name) => name.match(AI_CHUNK_FILE_RE)?.[1] || "").filter(Boolean)
+    )
+  ].sort();
+};
+var parseAiDatasetLine = (line, filePath) => {
+  try {
+    return JSON.parse(line);
+  } catch (error) {
+    const message = error?.message || String(error);
+    throw new Error(
+      `Failed to parse AI dataset row from ${filePath}: ${message}`
+    );
+  }
+};
+var getAiDatasetSortKey = (row) => {
+  const timestamp = Number(row.timestamp);
+  return {
+    timestamp: Number.isFinite(timestamp) ? timestamp : Number.MAX_SAFE_INTEGER,
+    symbol: String(row.symbol || ""),
+    signalId: String(row.signalId || "")
+  };
+};
+var compareAiDatasetSortKeys = (left, right) => {
+  if (left.timestamp !== right.timestamp) {
+    return left.timestamp - right.timestamp;
+  }
+  const symbolCompare = left.symbol.localeCompare(right.symbol);
+  if (symbolCompare !== 0) {
+    return symbolCompare;
+  }
+  return left.signalId.localeCompare(right.signalId);
+};
+var compareSortableAiDatasetLines = (left, right) => {
+  const keyCompare = compareAiDatasetSortKeys(left.sortKey, right.sortKey);
+  if (keyCompare !== 0) {
+    return keyCompare;
+  }
+  return left.sourceIndex - right.sourceIndex;
+};
+var compareSortedRunHeads = (left, right) => {
+  const keyCompare = compareAiDatasetSortKeys(left.sortKey, right.sortKey);
+  if (keyCompare !== 0) {
+    return keyCompare;
+  }
+  return left.runIndex - right.runIndex;
+};
+var writeJsonlLines = async (params) => {
+  const { filePath, lines } = params;
+  await import_promises2.default.mkdir(import_path2.default.dirname(filePath), { recursive: true });
+  const stream = (0, import_fs2.createWriteStream)(filePath, { encoding: "utf8" });
+  const done = Promise.all([(0, import_events2.once)(stream, "finish"), (0, import_events2.once)(stream, "close")]);
+  try {
+    for (const line of lines) {
+      if (!stream.write(`${line}
+`)) {
+        await (0, import_events2.once)(stream, "drain");
+      }
+    }
+  } finally {
+    stream.end();
+    await done;
+  }
+};
+var flushSortedRun = async (params) => {
+  const { tempDir, runIndex, entries } = params;
+  entries.sort(compareSortableAiDatasetLines);
+  const filePath = import_path2.default.join(
+    tempDir,
+    `run-${String(runIndex).padStart(6, "0")}.jsonl`
+  );
+  await writeJsonlLines({
+    filePath,
+    lines: entries.map(({ line }) => line)
+  });
+  return filePath;
+};
+var readNextNonEmptyLine = async (iterator) => {
+  while (true) {
+    const next = await iterator.next();
+    if (next.done) {
+      return null;
+    }
+    const trimmed = String(next.value || "").trim();
+    if (trimmed) {
+      return trimmed;
+    }
+  }
+};
+var mergeSortedRuns = async (params) => {
+  const { runPaths, outPath } = params;
+  await import_promises2.default.mkdir(import_path2.default.dirname(outPath), { recursive: true });
+  const output = (0, import_fs2.createWriteStream)(outPath, { encoding: "utf8" });
+  const outputDone = Promise.all([
+    (0, import_events2.once)(output, "finish"),
+    (0, import_events2.once)(output, "close")
+  ]);
+  const heads = [];
+  try {
+    for (let runIndex = 0; runIndex < runPaths.length; runIndex += 1) {
+      const runPath = runPaths[runIndex];
+      const reader = import_node_readline.default.createInterface({
+        input: (0, import_node_fs.createReadStream)(runPath, { encoding: "utf8" }),
+        crlfDelay: Infinity
+      });
+      const iterator = reader[Symbol.asyncIterator]();
+      const line = await readNextNonEmptyLine(iterator);
+      if (!line) {
+        reader.close();
+        continue;
+      }
+      heads.push({
+        runIndex,
+        line,
+        sortKey: getAiDatasetSortKey(parseAiDatasetLine(line, runPath)),
+        iterator,
+        reader
+      });
+    }
+    while (heads.length > 0) {
+      let bestIndex = 0;
+      for (let index = 1; index < heads.length; index += 1) {
+        if (compareSortedRunHeads(heads[index], heads[bestIndex]) < 0) {
+          bestIndex = index;
+        }
+      }
+      const best = heads[bestIndex];
+      if (!output.write(`${best.line}
+`)) {
+        await (0, import_events2.once)(output, "drain");
+      }
+      const nextLine = await readNextNonEmptyLine(best.iterator);
+      if (!nextLine) {
+        best.reader.close();
+        heads.splice(bestIndex, 1);
+        continue;
+      }
+      best.line = nextLine;
+      best.sortKey = getAiDatasetSortKey(
+        parseAiDatasetLine(nextLine, runPaths[best.runIndex])
+      );
+    }
+  } finally {
+    for (const head of heads) {
+      head.reader.close();
+    }
+    output.end();
+    await outputDone;
+  }
+};
+var mergeAiJsonlFiles = async (params) => {
+  const {
+    filePaths,
+    outPath,
+    maxRowsInMemory = AI_MERGE_SORT_RUN_MAX_ROWS,
+    maxBytesInMemory = AI_MERGE_SORT_RUN_MAX_BYTES
+  } = params;
+  const tempDir = import_path2.default.join(
+    import_path2.default.dirname(outPath),
+    `.ai-merge-${import_path2.default.basename(outPath)}-${Date.now()}-${process.pid}`
+  );
+  let batch = [];
+  let batchBytes = 0;
+  let sourceIndex = 0;
+  let runIndex = 0;
+  const runPaths = [];
+  const flushBatch = async () => {
+    if (!batch.length) {
+      return;
+    }
+    runPaths.push(
+      await flushSortedRun({
+        tempDir,
+        runIndex,
+        entries: batch
+      })
+    );
+    runIndex += 1;
+    batch = [];
+    batchBytes = 0;
+  };
+  try {
+    await import_promises2.default.mkdir(tempDir, { recursive: true });
+    for (const filePath of filePaths) {
+      const reader = import_node_readline.default.createInterface({
+        input: (0, import_node_fs.createReadStream)(filePath, { encoding: "utf8" }),
+        crlfDelay: Infinity
+      });
+      try {
+        for await (const line of reader) {
+          const trimmed = line.trim();
+          if (!trimmed) {
+            continue;
+          }
+          batch.push({
+            line: trimmed,
+            sortKey: getAiDatasetSortKey(parseAiDatasetLine(trimmed, filePath)),
+            sourceIndex
+          });
+          sourceIndex += 1;
+          batchBytes += Buffer.byteLength(trimmed, "utf8") + 1;
+          if (batch.length >= maxRowsInMemory || batchBytes >= maxBytesInMemory) {
+            await flushBatch();
+          }
+        }
+      } finally {
+        reader.close();
+      }
+    }
+    await flushBatch();
+    if (!runPaths.length) {
+      await import_promises2.default.mkdir(import_path2.default.dirname(outPath), { recursive: true });
+      await import_promises2.default.writeFile(outPath, "", "utf8");
+      return;
+    }
+    await mergeSortedRuns({
+      runPaths,
+      outPath
+    });
+  } finally {
+    await import_promises2.default.rm(tempDir, { recursive: true, force: true });
+  }
+};
+var readAiDatasetRows = async (params) => {
+  const { filePath, limitFromEnd = 0, skipFromEnd = 0 } = params;
+  const rows = [];
+  const recentLines = [];
+  let totalRows = 0;
+  const recentWindowLimit = limitFromEnd > 0 ? limitFromEnd + Math.max(0, skipFromEnd) : 0;
+  const reader = import_node_readline.default.createInterface({
+    input: (0, import_node_fs.createReadStream)(filePath, { encoding: "utf8" }),
+    crlfDelay: Infinity
+  });
+  for await (const line of reader) {
+    const trimmed = line.trim();
+    if (!trimmed) {
+      continue;
+    }
+    totalRows += 1;
+    if (limitFromEnd > 0) {
+      if (recentLines.length === recentWindowLimit) {
+        recentLines.shift();
+      }
+      recentLines.push(trimmed);
+    } else {
+      const row = JSON.parse(trimmed);
+      rows.push(row);
+    }
+  }
+  const effectiveSkip = Math.max(0, skipFromEnd);
+  const selectedRecentLines = effectiveSkip > 0 ? recentLines.slice(0, Math.max(0, recentLines.length - effectiveSkip)) : recentLines;
+  const selectedRows = limitFromEnd > 0 ? selectedRecentLines.map((line) => JSON.parse(line)) : effectiveSkip > 0 ? rows.slice(0, Math.max(0, rows.length - effectiveSkip)) : rows;
+  return {
+    rows: selectedRows,
+    totalRows
+  };
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  appendAiDatasetRow,
+  closeAiDatasetWriter,
+  closeAllAiDatasetWriters,
+  flushAiDatasetWriter,
+  getAiChunkFilePath,
+  listAiChunkFiles,
+  listAiChunkStrategies,
+  mergeAiJsonlFiles,
+  mergeJsonlFiles,
+  readAiDatasetRows,
+  toFileToken
+});