@tradejs/infra 1.0.4 → 1.0.6

package/dist/ai.mjs

@@ -0,0 +1,386 @@
+import {
+  mergeJsonlFiles,
+  toFileToken
+} from "./chunk-KVEZORMS.mjs";
+
+// src/aiDatasetFile.ts
+import { once } from "events";
+import { createWriteStream } from "fs";
+import fs from "fs/promises";
+import path from "path";
+import { createReadStream } from "fs";
+import readline from "readline";
+var DEFAULT_DIR = "data/ai/export";
+var AI_DATASET_WRITE_BATCH_SIZE = 100;
+var AI_MERGE_SORT_RUN_MAX_ROWS = 2e3;
+var AI_MERGE_SORT_RUN_MAX_BYTES = 16 * 1024 * 1024;
+var AI_CHUNK_FILE_RE = /^ai-dataset-(.+)-chunk-[^.]+\.jsonl$/;
+var writerByPath = /* @__PURE__ */ new Map();
+var getAiChunkFilePath = (strategyName, chunkId, outDir = DEFAULT_DIR) => path.join(
+  outDir,
+  `ai-dataset-${toFileToken(strategyName)}-chunk-${toFileToken(chunkId)}.jsonl`
+);
+var appendAiDatasetRow = async (params) => {
+  const { strategyName, chunkId, row, outDir = DEFAULT_DIR } = params;
+  const filePath = getAiChunkFilePath(strategyName, chunkId, outDir);
+  let state = writerByPath.get(filePath);
+  if (!state) {
+    await fs.mkdir(outDir, { recursive: true });
+    const stream = createWriteStream(filePath, {
+      encoding: "utf8",
+      flags: "a"
+    });
+    state = {
+      filePath,
+      stream,
+      buffer: [],
+      writeQueue: Promise.resolve(),
+      closed: false
+    };
+    writerByPath.set(filePath, state);
+  }
+  if (state.closed) {
+    throw new Error(`AI dataset writer is closed: ${filePath}`);
+  }
+  state.buffer.push(`${JSON.stringify(row)}
+`);
+  if (state.buffer.length >= AI_DATASET_WRITE_BATCH_SIZE) {
+    await flushAiDatasetWriter(filePath);
+  }
+  return filePath;
+};
+var flushState = async (state) => {
+  if (state.closed || state.buffer.length === 0) {
+    return;
+  }
+  const chunk = state.buffer.join("");
+  state.buffer = [];
+  if (!state.stream.write(chunk)) {
+    await once(state.stream, "drain");
+  }
+};
+var flushAiDatasetWriter = async (filePath) => {
+  const state = writerByPath.get(filePath);
+  if (!state || state.closed) {
+    return;
+  }
+  state.writeQueue = state.writeQueue.then(() => flushState(state));
+  await state.writeQueue;
+};
+var closeState = async (state) => {
+  if (state.closed) {
+    return;
+  }
+  await flushState(state);
+  state.closed = true;
+  state.stream.end();
+  await Promise.all([
+    once(state.stream, "finish"),
+    once(state.stream, "close")
+  ]);
+};
+var closeAiDatasetWriter = async (filePath) => {
+  const state = writerByPath.get(filePath);
+  if (!state) {
+    return;
+  }
+  state.writeQueue = state.writeQueue.then(() => closeState(state));
+  await state.writeQueue;
+  writerByPath.delete(filePath);
+};
+var closeAllAiDatasetWriters = async () => {
+  const filePaths = [...writerByPath.keys()];
+  for (const filePath of filePaths) {
+    await closeAiDatasetWriter(filePath);
+  }
+};
+var listAiChunkFiles = async (params) => {
+  const { strategyName, outDir = DEFAULT_DIR } = params;
+  const prefix = `ai-dataset-${toFileToken(strategyName)}-chunk-`;
+  let entries = [];
+  try {
+    entries = await fs.readdir(outDir);
+  } catch {
+    return [];
+  }
+  return entries.filter((name) => name.startsWith(prefix) && name.endsWith(".jsonl")).map((name) => path.join(outDir, name)).sort();
+};
+var listAiChunkStrategies = async (params) => {
+  const outDir = params?.outDir ?? DEFAULT_DIR;
+  let entries = [];
+  try {
+    entries = await fs.readdir(outDir);
+  } catch {
+    return [];
+  }
+  return [
+    ...new Set(
+      entries.map((name) => name.match(AI_CHUNK_FILE_RE)?.[1] || "").filter(Boolean)
+    )
+  ].sort();
+};
+var parseAiDatasetLine = (line, filePath) => {
+  try {
+    return JSON.parse(line);
+  } catch (error) {
+    const message = error?.message || String(error);
+    throw new Error(
+      `Failed to parse AI dataset row from ${filePath}: ${message}`
+    );
+  }
+};
+var getAiDatasetSortKey = (row) => {
+  const timestamp = Number(row.timestamp);
+  return {
+    timestamp: Number.isFinite(timestamp) ? timestamp : Number.MAX_SAFE_INTEGER,
+    symbol: String(row.symbol || ""),
+    signalId: String(row.signalId || "")
+  };
+};
+var compareAiDatasetSortKeys = (left, right) => {
+  if (left.timestamp !== right.timestamp) {
+    return left.timestamp - right.timestamp;
+  }
+  const symbolCompare = left.symbol.localeCompare(right.symbol);
+  if (symbolCompare !== 0) {
+    return symbolCompare;
+  }
+  return left.signalId.localeCompare(right.signalId);
+};
+var compareSortableAiDatasetLines = (left, right) => {
+  const keyCompare = compareAiDatasetSortKeys(left.sortKey, right.sortKey);
+  if (keyCompare !== 0) {
+    return keyCompare;
+  }
+  return left.sourceIndex - right.sourceIndex;
+};
+var compareSortedRunHeads = (left, right) => {
+  const keyCompare = compareAiDatasetSortKeys(left.sortKey, right.sortKey);
+  if (keyCompare !== 0) {
+    return keyCompare;
+  }
+  return left.runIndex - right.runIndex;
+};
+var writeJsonlLines = async (params) => {
+  const { filePath, lines } = params;
+  await fs.mkdir(path.dirname(filePath), { recursive: true });
+  const stream = createWriteStream(filePath, { encoding: "utf8" });
+  const done = Promise.all([once(stream, "finish"), once(stream, "close")]);
+  try {
+    for (const line of lines) {
+      if (!stream.write(`${line}
+`)) {
+        await once(stream, "drain");
+      }
+    }
+  } finally {
+    stream.end();
+    await done;
+  }
+};
+var flushSortedRun = async (params) => {
+  const { tempDir, runIndex, entries } = params;
+  entries.sort(compareSortableAiDatasetLines);
+  const filePath = path.join(
+    tempDir,
+    `run-${String(runIndex).padStart(6, "0")}.jsonl`
+  );
+  await writeJsonlLines({
+    filePath,
+    lines: entries.map(({ line }) => line)
+  });
+  return filePath;
+};
+var readNextNonEmptyLine = async (iterator) => {
+  while (true) {
+    const next = await iterator.next();
+    if (next.done) {
+      return null;
+    }
+    const trimmed = String(next.value || "").trim();
+    if (trimmed) {
+      return trimmed;
+    }
+  }
+};
+var mergeSortedRuns = async (params) => {
+  const { runPaths, outPath } = params;
+  await fs.mkdir(path.dirname(outPath), { recursive: true });
+  const output = createWriteStream(outPath, { encoding: "utf8" });
+  const outputDone = Promise.all([
+    once(output, "finish"),
+    once(output, "close")
+  ]);
+  const heads = [];
+  try {
+    for (let runIndex = 0; runIndex < runPaths.length; runIndex += 1) {
+      const runPath = runPaths[runIndex];
+      const reader = readline.createInterface({
+        input: createReadStream(runPath, { encoding: "utf8" }),
+        crlfDelay: Infinity
+      });
+      const iterator = reader[Symbol.asyncIterator]();
+      const line = await readNextNonEmptyLine(iterator);
+      if (!line) {
+        reader.close();
+        continue;
+      }
+      heads.push({
+        runIndex,
+        line,
+        sortKey: getAiDatasetSortKey(parseAiDatasetLine(line, runPath)),
+        iterator,
+        reader
+      });
+    }
+    while (heads.length > 0) {
+      let bestIndex = 0;
+      for (let index = 1; index < heads.length; index += 1) {
+        if (compareSortedRunHeads(heads[index], heads[bestIndex]) < 0) {
+          bestIndex = index;
+        }
+      }
+      const best = heads[bestIndex];
+      if (!output.write(`${best.line}
+`)) {
+        await once(output, "drain");
+      }
+      const nextLine = await readNextNonEmptyLine(best.iterator);
+      if (!nextLine) {
+        best.reader.close();
+        heads.splice(bestIndex, 1);
+        continue;
+      }
+      best.line = nextLine;
+      best.sortKey = getAiDatasetSortKey(
+        parseAiDatasetLine(nextLine, runPaths[best.runIndex])
+      );
+    }
+  } finally {
+    for (const head of heads) {
+      head.reader.close();
+    }
+    output.end();
+    await outputDone;
+  }
+};
+var mergeAiJsonlFiles = async (params) => {
+  const {
+    filePaths,
+    outPath,
+    maxRowsInMemory = AI_MERGE_SORT_RUN_MAX_ROWS,
+    maxBytesInMemory = AI_MERGE_SORT_RUN_MAX_BYTES
+  } = params;
+  const tempDir = path.join(
+    path.dirname(outPath),
+    `.ai-merge-${path.basename(outPath)}-${Date.now()}-${process.pid}`
+  );
+  let batch = [];
+  let batchBytes = 0;
+  let sourceIndex = 0;
+  let runIndex = 0;
+  const runPaths = [];
+  const flushBatch = async () => {
+    if (!batch.length) {
+      return;
+    }
+    runPaths.push(
+      await flushSortedRun({
+        tempDir,
+        runIndex,
+        entries: batch
+      })
+    );
+    runIndex += 1;
+    batch = [];
+    batchBytes = 0;
+  };
+  try {
+    await fs.mkdir(tempDir, { recursive: true });
+    for (const filePath of filePaths) {
+      const reader = readline.createInterface({
+        input: createReadStream(filePath, { encoding: "utf8" }),
+        crlfDelay: Infinity
+      });
+      try {
+        for await (const line of reader) {
+          const trimmed = line.trim();
+          if (!trimmed) {
+            continue;
+          }
+          batch.push({
+            line: trimmed,
+            sortKey: getAiDatasetSortKey(parseAiDatasetLine(trimmed, filePath)),
+            sourceIndex
+          });
+          sourceIndex += 1;
+          batchBytes += Buffer.byteLength(trimmed, "utf8") + 1;
+          if (batch.length >= maxRowsInMemory || batchBytes >= maxBytesInMemory) {
+            await flushBatch();
+          }
+        }
+      } finally {
+        reader.close();
+      }
+    }
+    await flushBatch();
+    if (!runPaths.length) {
+      await fs.mkdir(path.dirname(outPath), { recursive: true });
+      await fs.writeFile(outPath, "", "utf8");
+      return;
+    }
+    await mergeSortedRuns({
+      runPaths,
+      outPath
+    });
+  } finally {
+    await fs.rm(tempDir, { recursive: true, force: true });
+  }
+};
+var readAiDatasetRows = async (params) => {
+  const { filePath, limitFromEnd = 0, skipFromEnd = 0 } = params;
+  const rows = [];
+  const recentLines = [];
+  let totalRows = 0;
+  const recentWindowLimit = limitFromEnd > 0 ? limitFromEnd + Math.max(0, skipFromEnd) : 0;
+  const reader = readline.createInterface({
+    input: createReadStream(filePath, { encoding: "utf8" }),
+    crlfDelay: Infinity
+  });
+  for await (const line of reader) {
+    const trimmed = line.trim();
+    if (!trimmed) {
+      continue;
+    }
+    totalRows += 1;
+    if (limitFromEnd > 0) {
+      if (recentLines.length === recentWindowLimit) {
+        recentLines.shift();
+      }
+      recentLines.push(trimmed);
+    } else {
+      const row = JSON.parse(trimmed);
+      rows.push(row);
+    }
+  }
+  const effectiveSkip = Math.max(0, skipFromEnd);
+  const selectedRecentLines = effectiveSkip > 0 ? recentLines.slice(0, Math.max(0, recentLines.length - effectiveSkip)) : recentLines;
+  const selectedRows = limitFromEnd > 0 ? selectedRecentLines.map((line) => JSON.parse(line)) : effectiveSkip > 0 ? rows.slice(0, Math.max(0, rows.length - effectiveSkip)) : rows;
+  return {
+    rows: selectedRows,
+    totalRows
+  };
+};
+export {
+  appendAiDatasetRow,
+  closeAiDatasetWriter,
+  closeAllAiDatasetWriters,
+  flushAiDatasetWriter,
+  getAiChunkFilePath,
+  listAiChunkFiles,
+  listAiChunkStrategies,
+  mergeAiJsonlFiles,
+  mergeJsonlFiles,
+  readAiDatasetRows,
+  toFileToken
+};

package/dist/chunk-EFARW5QE.mjs

@@ -0,0 +1,297 @@
+// src/redis.ts
+import Redis from "ioredis";
+var TTL_1D = 86400;
+var toJson = (value) => JSON.stringify(value);
+var logger = {
+  log: (level, message, ...args) => {
+    const method = level === "error" ? "error" : level === "warn" ? "warn" : "log";
+    console[method](`[infra:redis] ${message}`, ...args);
+  }
+};
+var redisConnectionWarningShown = false;
+var redisUnavailable = false;
+var isRedisConnectivityError = (error) => /ECONNREFUSED|ENOTFOUND|EAI_AGAIN|ETIMEDOUT|MaxRetriesPerRequestError|Connection is closed|Stream isn't writeable/i.test(
+  error.message
+);
+var toNonNegativeInt = (value, fallback) => {
+  const parsed = Number.parseInt(String(value ?? ""), 10);
+  return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback;
+};
+var toPositiveInt = (value, fallback) => {
+  const parsed = Number.parseInt(String(value ?? ""), 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+};
+var markRedisUnavailable = (error) => {
+  redisUnavailable = true;
+  if (redisConnectionWarningShown) return;
+  redisConnectionWarningShown = true;
+  logger.log(
+    "warn",
+    "Redis is unavailable: %s. Cache-dependent features are temporarily disabled.",
+    error.message
+  );
+};
+var getRedis = () => {
+  if (!global.__redis__) {
+    const host = process.env.REDIS_HOST || "127.0.0.1";
+    const port = toPositiveInt(process.env.REDIS_PORT, 6379);
+    const connectTimeout = toPositiveInt(
+      process.env.REDIS_CONNECT_TIMEOUT_MS,
+      3e3
+    );
+    const maxRetriesPerRequest = toNonNegativeInt(
+      process.env.REDIS_MAX_RETRIES_PER_REQUEST,
+      1
+    );
+    global.__redis__ = new Redis({
+      host,
+      port,
+      connectTimeout,
+      maxRetriesPerRequest,
+      enableOfflineQueue: false,
+      retryStrategy: (attempt) => Math.min(attempt * 200, 2e3)
+    });
+    global.__redis__.on("error", (error) => {
+      if (isRedisConnectivityError(error)) {
+        markRedisUnavailable(error);
+        return;
+      }
+      logger.log("error", "Redis client error: %s", String(error));
+    });
+    global.__redis__.on("ready", () => {
+      redisUnavailable = false;
+      if (redisConnectionWarningShown) {
+        redisConnectionWarningShown = false;
+        logger.log("info", "Redis connection restored");
+      }
+    });
+  }
+  return global.__redis__;
+};
+var sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+var getRedisStatus = (redis) => String(redis.status ?? "ready");
+var waitForRedisReady = async (redis) => {
+  if (getRedisStatus(redis) === "ready") {
+    return true;
+  }
+  const readyTimeoutMs = toPositiveInt(
+    process.env.REDIS_READY_TIMEOUT_MS,
+    toPositiveInt(process.env.REDIS_CONNECT_TIMEOUT_MS, 3e3)
+  );
+  const startedAt = Date.now();
+  while (Date.now() - startedAt < readyTimeoutMs) {
+    if (getRedisStatus(redis) === "ready") {
+      return true;
+    }
+    if (getRedisStatus(redis) === "end") {
+      return false;
+    }
+    await sleep(50);
+  }
+  return getRedisStatus(redis) === "ready";
+};
+var getReadyRedis = async () => {
+  if (redisUnavailable) return null;
+  const redis = getRedis();
+  const ready = await waitForRedisReady(redis);
+  if (ready) {
+    return redis;
+  }
+  markRedisUnavailable(
+    new Error(`Redis is not ready (status=${getRedisStatus(redis)})`)
+  );
+  return null;
+};
+var toResultString = (result) => {
+  if (result == null) return null;
+  if (typeof result === "string") return result;
+  if (Buffer.isBuffer(result)) return result.toString("utf8");
+  return String(result);
+};
+var DEFAULT_OPTIONS = {
+  expire: TTL_1D
+};
+var parseJsonOrDeleteKey = async (redis, key, raw, fallback) => {
+  try {
+    return JSON.parse(raw);
+  } catch (e) {
+    logger.log("error", "failed JSON.parse(%s): %s", key, String(e));
+    await redis.del(key);
+    return fallback;
+  }
+};
+var getKeys = async (prefix) => {
+  if (redisUnavailable) return [];
+  const redis = await getReadyRedis();
+  if (!redis) return [];
+  const keys = [];
+  try {
+    let cursor = "0";
+    do {
+      const [nextCursor, batch] = await redis.scan(
+        cursor,
+        "MATCH",
+        `${prefix}*`,
+        "COUNT",
+        "200"
+      );
+      cursor = nextCursor;
+      for (const key of batch) {
+        if (key.startsWith(prefix)) {
+          keys.push(key);
+        }
+      }
+    } while (cursor !== "0");
+  } catch (e) {
+    if (e instanceof Error && isRedisConnectivityError(e)) {
+      markRedisUnavailable(e);
+      return [];
+    }
+    logger.log("warn", "failed SCAN for %s: %s", prefix, String(e));
+    return [];
+  }
+  return keys;
+};
+var getData = async (key, fallback = []) => {
+  if (redisUnavailable) return fallback;
+  const redis = await getReadyRedis();
+  if (!redis) return fallback;
+  try {
+    const rawJson = await redis.call("JSON.GET", key);
+    const raw = toResultString(rawJson);
+    if (raw == null) return fallback;
+    return parseJsonOrDeleteKey(redis, key, raw, fallback);
+  } catch (e) {
+    if (e instanceof Error && isRedisConnectivityError(e)) {
+      markRedisUnavailable(e);
+      return fallback;
+    }
+    logger.log(
+      "error",
+      "failed JSON.GET %s: %s (fallback to GET)",
+      key,
+      String(e)
+    );
+  }
+  try {
+    const raw = await redis.get(key);
+    if (raw == null) return fallback;
+    return parseJsonOrDeleteKey(redis, key, raw, fallback);
+  } catch (e) {
+    if (e instanceof Error && isRedisConnectivityError(e)) {
+      markRedisUnavailable(e);
+      return fallback;
+    }
+    logger.log("error", "failed GET %s: %s", key, String(e));
+    return fallback;
+  }
+};
+var delKey = async (key) => {
+  return delKeyWithOptions(key);
+};
+var RedisWriteBlockedError = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "RedisWriteBlockedError";
+  }
+};
+var delKeyWithOptions = async (key, options = {}) => {
+  if (redisUnavailable) return false;
+  const { raiseOnMisconf = false } = options;
+  const redis = await getReadyRedis();
+  if (!redis) return false;
+  try {
+    const result = await redis.del(key);
+    if (result === 1) {
+      return true;
+    }
+    return false;
+  } catch (e) {
+    if (e instanceof Error && isRedisConnectivityError(e)) {
+      markRedisUnavailable(e);
+      return false;
+    }
+    const msg = String(e);
+    if (raiseOnMisconf && msg.includes("MISCONF")) {
+      throw new RedisWriteBlockedError(msg);
+    }
+    logger.log("error", "failed DEL %s: %s", key, String(e));
+    return false;
+  }
+};
+var setData = async (key, data, options = {}) => {
+  if (redisUnavailable) return;
+  const { expire } = { ...DEFAULT_OPTIONS, ...options };
+  const redis = await getReadyRedis();
+  if (!redis) return;
+  const value = toJson(data);
+  try {
+    await redis.call("JSON.SET", key, "$", value);
+    if (expire) {
+      await redis.expire(key, expire);
+    }
+  } catch (e) {
+    if (e instanceof Error && isRedisConnectivityError(e)) {
+      markRedisUnavailable(e);
+      return;
+    }
+    logger.log(
+      "error",
+      "failed JSON.SET %s: %s (fallback to SET)",
+      key,
+      String(e)
+    );
+    try {
+      if (expire) {
+        await redis.set(key, value, "EX", expire);
+      } else {
+        await redis.set(key, value);
+      }
+    } catch (e2) {
+      if (e2 instanceof Error && isRedisConnectivityError(e2)) {
+        markRedisUnavailable(e2);
+        return;
+      }
+      logger.log("error", "failed SET %s: %s", key, String(e2));
+    }
+  }
+};
+var redisKeys = {
+  users: () => "users:index:",
+  user: (userName) => `users:index:${userName}`,
+  bots: (userName) => `users:${userName}:bots`,
+  botsPrefix: () => "users:",
+  bot: (userName, botId) => `users:${userName}:bots:${botId}`,
+  backtestConfig: (userName, config) => `users:${userName}:backtests:configs:${config}`,
+  strategies: (userName) => `users:${userName}:strategies`,
+  strategyConfig: (userName, strategyName) => `users:${userName}:strategies:${strategyName}:config`,
+  strategyResults: (userName, strategyName) => `users:${userName}:strategies:${strategyName}:results`,
+  tests: (userName, strategyName) => strategyName ? `users:${userName}:tests:${strategyName}` : `users:${userName}:tests:`,
+  testOrders: (userName, strategyName, testName) => `users:${userName}:tests:${strategyName}:${testName}:orders`,
+  testConfig: (userName, strategyName, testName) => `users:${userName}:tests:${strategyName}:${testName}:config`,
+  testStat: (userName, strategyName, testName) => `users:${userName}:tests:${strategyName}:${testName}:stat`,
+  cacheChunk: (userName, chunkId) => `users:${userName}:cache:tests:chunks:${chunkId}`,
+  cacheOrders: (userName, orderLogId) => `users:${userName}:cache:tests:orders:${orderLogId}`,
+  cachePositions: (userName, orderLogId) => `users:${userName}:cache:tests:positions:${orderLogId}`,
+  signal: (symbol, signalId) => `signals:${symbol}:${signalId}`,
+  signalsBySymbol: (symbol) => `signals:${symbol}:`,
+  storeSignal: (symbol, signalId) => `store:signals:${symbol}:${signalId}`,
+  analysis: (symbol, signalId) => `analysis:${symbol}:${signalId}`,
+  backtestResults: (userName, config, timestamp) => `users:${userName}:backtests:results:${config}:${timestamp}`,
+  mlSignalsByStrategy: (strategyName) => `ml:${strategyName}:signals:`,
+  mlSignals: () => "ml:",
+  mlSignal: (strategyName, signalId) => `ml:${strategyName}:signals:${signalId}`,
+  mlResultsByStrategy: (strategyName) => `ml:${strategyName}:results:`,
+  mlResults: () => "ml:",
+  mlResult: (strategyName, signalId) => `ml:${strategyName}:results:${signalId}`
+};
+
+export {
+  getKeys,
+  getData,
+  delKey,
+  RedisWriteBlockedError,
+  delKeyWithOptions,
+  setData,
+  redisKeys
+};