@tradejs/cli 2.0.5 → 2.0.7

package/dist/scripts/aiExport.js

@@ -34,9 +34,9 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 ));
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 
-// ../../node_modules/debug/node_modules/ms/index.js
+// ../../node_modules/ms/index.js
 var require_ms = __commonJS({
-  "../../node_modules/debug/node_modules/ms/index.js"(exports2, module2) {
+  "../../node_modules/ms/index.js"(exports2, module2) {
     "use strict";
     var s = 1e3;
     var m = s * 60;
@@ -151,9 +151,9 @@ var require_ms = __commonJS({
   }
 });
 
-// ../../node_modules/debug/src/common.js
+// ../../node_modules/list-it/node_modules/debug/src/common.js
 var require_common = __commonJS({
-  "../../node_modules/debug/src/common.js"(exports2, module2) {
+  "../../node_modules/list-it/node_modules/debug/src/common.js"(exports2, module2) {
     "use strict";
     function setup(env) {
       createDebug.debug = createDebug;
@@ -255,50 +255,64 @@ var require_common = __commonJS({
         createDebug.namespaces = namespaces;
         createDebug.names = [];
         createDebug.skips = [];
-        let i;
-        const split = (typeof namespaces === "string" ? namespaces : "").split(/[\s,]+/);
-        const len = split.length;
-        for (i = 0; i < len; i++) {
-          if (!split[i]) {
-            continue;
+        const split = (typeof namespaces === "string" ? namespaces : "").trim().replace(/\s+/g, ",").split(",").filter(Boolean);
+        for (const ns of split) {
+          if (ns[0] === "-") {
+            createDebug.skips.push(ns.slice(1));
+          } else {
+            createDebug.names.push(ns);
           }
-          namespaces = split[i].replace(/\*/g, ".*?");
-          if (namespaces[0] === "-") {
-            createDebug.skips.push(new RegExp("^" + namespaces.slice(1) + "$"));
+        }
+      }
+      function matchesTemplate(search, template) {
+        let searchIndex = 0;
+        let templateIndex = 0;
+        let starIndex = -1;
+        let matchIndex = 0;
+        while (searchIndex < search.length) {
+          if (templateIndex < template.length && (template[templateIndex] === search[searchIndex] || template[templateIndex] === "*")) {
+            if (template[templateIndex] === "*") {
+              starIndex = templateIndex;
+              matchIndex = searchIndex;
+              templateIndex++;
+            } else {
+              searchIndex++;
+              templateIndex++;
+            }
+          } else if (starIndex !== -1) {
+            templateIndex = starIndex + 1;
+            matchIndex++;
+            searchIndex = matchIndex;
           } else {
-            createDebug.names.push(new RegExp("^" + namespaces + "$"));
+            return false;
           }
         }
+        while (templateIndex < template.length && template[templateIndex] === "*") {
+          templateIndex++;
+        }
+        return templateIndex === template.length;
       }
       function disable() {
         const namespaces = [
-          ...createDebug.names.map(toNamespace),
-          ...createDebug.skips.map(toNamespace).map((namespace) => "-" + namespace)
+          ...createDebug.names,
+          ...createDebug.skips.map((namespace) => "-" + namespace)
         ].join(",");
         createDebug.enable("");
         return namespaces;
       }
       function enabled(name) {
-        if (name[name.length - 1] === "*") {
-          return true;
-        }
-        let i;
-        let len;
-        for (i = 0, len = createDebug.skips.length; i < len; i++) {
-          if (createDebug.skips[i].test(name)) {
+        for (const skip of createDebug.skips) {
+          if (matchesTemplate(name, skip)) {
             return false;
           }
         }
-        for (i = 0, len = createDebug.names.length; i < len; i++) {
-          if (createDebug.names[i].test(name)) {
+        for (const ns of createDebug.names) {
+          if (matchesTemplate(name, ns)) {
             return true;
           }
         }
         return false;
       }
-      function toNamespace(regexp) {
-        return regexp.toString().substring(2, regexp.toString().length - 2).replace(/\.\*\?$/, "*");
-      }
       function coerce(val) {
         if (val instanceof Error) {
           return val.stack || val.message;
@@ -315,9 +329,9 @@ var require_common = __commonJS({
   }
 });
 
-// ../../node_modules/debug/src/browser.js
+// ../../node_modules/list-it/node_modules/debug/src/browser.js
 var require_browser = __commonJS({
-  "../../node_modules/debug/src/browser.js"(exports2, module2) {
+  "../../node_modules/list-it/node_modules/debug/src/browser.js"(exports2, module2) {
     "use strict";
     exports2.formatArgs = formatArgs;
     exports2.save = save;
@@ -418,10 +432,11 @@ var require_browser = __commonJS({
       if (typeof navigator !== "undefined" && navigator.userAgent && navigator.userAgent.toLowerCase().match(/(edge|trident)\/(\d+)/)) {
         return false;
       }
+      let m;
       return typeof document !== "undefined" && document.documentElement && document.documentElement.style && document.documentElement.style.WebkitAppearance || // Is firebug? http://stackoverflow.com/a/398120/376773
       typeof window !== "undefined" && window.console && (window.console.firebug || window.console.exception && window.console.table) || // Is firefox >= v31?
       // https://developer.mozilla.org/en-US/docs/Tools/Web_Console#Styling_messages
-      typeof navigator !== "undefined" && navigator.userAgent && navigator.userAgent.toLowerCase().match(/firefox\/(\d+)/) && parseInt(RegExp.$1, 10) >= 31 || // Double check webkit in userAgent just in case we are in a worker
+      typeof navigator !== "undefined" && navigator.userAgent && (m = navigator.userAgent.toLowerCase().match(/firefox\/(\d+)/)) && parseInt(m[1], 10) >= 31 || // Double check webkit in userAgent just in case we are in a worker
       typeof navigator !== "undefined" && navigator.userAgent && navigator.userAgent.toLowerCase().match(/applewebkit\/(\d+)/);
     }
     function formatArgs(args2) {
@@ -459,7 +474,7 @@ var require_browser = __commonJS({
     function load() {
       let r;
       try {
-        r = exports2.storage.getItem("debug");
+        r = exports2.storage.getItem("debug") || exports2.storage.getItem("DEBUG");
       } catch (error) {
       }
       if (!r && typeof process !== "undefined" && "env" in process) {
@@ -600,9 +615,9 @@ var require_supports_color = __commonJS({
   }
 });
 
-// ../../node_modules/debug/src/node.js
+// ../../node_modules/list-it/node_modules/debug/src/node.js
 var require_node = __commonJS({
-  "../../node_modules/debug/src/node.js"(exports2, module2) {
+  "../../node_modules/list-it/node_modules/debug/src/node.js"(exports2, module2) {
     "use strict";
     var tty = require("tty");
     var util = require("util");
@@ -743,7 +758,7 @@ var require_node = __commonJS({
       return (/* @__PURE__ */ new Date()).toISOString() + " ";
     }
     function log(...args2) {
-      return process.stderr.write(util.format(...args2) + "\n");
+      return process.stderr.write(util.formatWithOptions(exports2.inspectOpts, ...args2) + "\n");
     }
     function save(namespaces) {
       if (namespaces) {
@@ -775,9 +790,9 @@ var require_node = __commonJS({
   }
 });
 
-// ../../node_modules/debug/src/index.js
+// ../../node_modules/list-it/node_modules/debug/src/index.js
 var require_src = __commonJS({
-  "../../node_modules/debug/src/index.js"(exports2, module2) {
+  "../../node_modules/list-it/node_modules/debug/src/index.js"(exports2, module2) {
     "use strict";
     if (typeof process === "undefined" || process.type === "renderer" || process.browser === true || process.__nwjs) {
       module2.exports = require_browser();
@@ -1626,6 +1641,11 @@ __export(aiExport_exports, {
 module.exports = __toCommonJS(aiExport_exports);
 var import_args = __toESM(require("args"));
 var import_chalk2 = __toESM(require("chalk"));
+var import_promises2 = __toESM(require("fs/promises"));
+var import_progress = __toESM(require("progress"));
+var import_ai2 = require("@tradejs/infra/ai");
+
+// src/lib/aiExport.ts
 var import_promises = __toESM(require("fs/promises"));
 var import_path = __toESM(require("path"));
 var import_ai = require("@tradejs/infra/ai");
@@ -1669,6 +1689,139 @@ var loadBacktestCheckpointResults = async ({
   import_redis.redisKeys.backtestRunResults(userName, runId)
 );
 
+// src/lib/aiExport.ts
+var resolveRunScopedAiChunks = async ({
+  outDir,
+  requestedRunId,
+  strategyName,
+  userName
+}) => {
+  const resolvedRunId = String(requestedRunId || "").trim();
+  const runIds = resolvedRunId ? [resolvedRunId] : await (0, import_ai.listAiChunkRunIds)({ strategyName, outDir });
+  const runId = runIds.at(-1) ?? "";
+  if (!runId) {
+    return null;
+  }
+  const chunkFiles = await (0, import_ai.listAiChunkFiles)({ strategyName, outDir, runId });
+  if (!chunkFiles.length) {
+    return {
+      runId,
+      chunkFiles,
+      completedAttemptKeys: /* @__PURE__ */ new Set()
+    };
+  }
+  const completed = await loadBacktestCheckpointResults({ runId, userName });
+  return {
+    runId,
+    chunkFiles,
+    completedAttemptKeys: buildCompletedBacktestDatasetAttemptKeys(completed)
+  };
+};
+var exportAiStrategy = async (params) => {
+  const {
+    outDir,
+    strategyName,
+    keepChunks,
+    requestedRunId,
+    userName,
+    now = Date.now
+  } = params;
+  const partMonths = Math.max(0, Math.trunc(Number(params.partMonths) || 0));
+  const runScopedChunks = await resolveRunScopedAiChunks({
+    outDir,
+    requestedRunId,
+    strategyName,
+    userName
+  });
+  const chunkFiles = runScopedChunks ? runScopedChunks.chunkFiles : await (0, import_ai.listAiChunkFiles)({ strategyName, outDir });
+  if (!chunkFiles.length) {
+    return null;
+  }
+  if (runScopedChunks && runScopedChunks.completedAttemptKeys.size === 0) {
+    throw new Error(
+      `No completed checkpoint attempts found for AI export run "${runScopedChunks.runId}"`
+    );
+  }
+  const completedAttemptCount = runScopedChunks?.completedAttemptKeys.size ?? void 0;
+  const mergedPath = import_path.default.join(
+    outDir,
+    `ai-dataset-${(0, import_ai.toFileToken)(strategyName)}-merged-${now()}.jsonl`
+  );
+  await (0, import_ai.mergeAiJsonlFiles)({
+    filePaths: chunkFiles,
+    outPath: mergedPath,
+    shouldIncludeRow: runScopedChunks ? (row) => isBacktestDatasetRowForCompletedAttempt(
+      row,
+      runScopedChunks.completedAttemptKeys
+    ) : void 0
+  });
+  runScopedChunks?.completedAttemptKeys.clear();
+  const splitResult = partMonths > 0 ? await (0, import_ai.splitAiMergedDatasetFile)({
+    filePath: mergedPath,
+    monthsPerPart: partMonths
+  }) : {
+    partPaths: [mergedPath],
+    partCount: 1,
+    splitApplied: false
+  };
+  const shouldDeleteChunks = !keepChunks;
+  if (shouldDeleteChunks) {
+    for (const filePath of chunkFiles) {
+      await import_promises.default.rm(filePath, { force: true });
+    }
+  }
+  return {
+    strategyName,
+    sourceChunkCount: chunkFiles.length,
+    deleteChunks: shouldDeleteChunks,
+    partMonths,
+    partPaths: splitResult.partPaths,
+    partCount: splitResult.partCount,
+    splitApplied: splitResult.splitApplied,
+    runId: runScopedChunks?.runId,
+    completedAttemptCount
+  };
+};
+var exportAiStrategiesSequentially = async (params, dependencies = {}) => {
+  const { strategyNames, onProgress, ...strategyParams } = params;
+  const runExport = dependencies.exportStrategy ?? exportAiStrategy;
+  const results = [];
+  const total = strategyNames.length;
+  for (let index = 0; index < strategyNames.length; index += 1) {
+    const strategyName = strategyNames[index];
+    onProgress?.({
+      strategyName,
+      current: index,
+      total,
+      status: "started"
+    });
+    try {
+      const result = await runExport({
+        ...strategyParams,
+        strategyName
+      });
+      if (result) {
+        results.push(result);
+      }
+      onProgress?.({
+        strategyName,
+        current: index + 1,
+        total,
+        status: result ? "completed" : "skipped"
+      });
+    } catch (error) {
+      onProgress?.({
+        strategyName,
+        current: index,
+        total,
+        status: "failed"
+      });
+      throw error;
+    }
+  }
+  return results;
+};
+
 // src/scripts/selectStrategy.ts
 var import_readline = __toESM(require("readline"));
 var import_chalk = __toESM(require("chalk"));
@@ -1735,12 +1888,14 @@ var selectStrategy = async (promptLabel = "Select strategy", options) => {
 };
 
 // src/scripts/resolveExportStrategy.ts
+var ALL_EXPORT_STRATEGIES = "all";
 var resolveExportStrategy = async ({
   explicitStrategy,
   outDir,
   datasetLabel,
   promptLabel,
-  listStrategies
+  listStrategies,
+  includeAllOption = false
 }) => {
   const raw = String(explicitStrategy || "").trim();
   if (raw) {
@@ -1758,8 +1913,9 @@ var resolveExportStrategy = async ({
       `Multiple ${datasetLabel} chunk strategies found in ${outDir}: ${availableStrategies.join(", ")}. Pass --strategy.`
     );
   }
+  const promptStrategies = includeAllOption ? [...availableStrategies, ALL_EXPORT_STRATEGIES] : availableStrategies;
   return selectStrategy(promptLabel, {
-    strategies: availableStrategies,
+    strategies: promptStrategies,
     defaultStrategy: availableStrategies[0]
   });
 };
@@ -1788,110 +1944,81 @@ import_args.default.option(
   "root"
 );
 var flags = import_args.default.parse(process.argv);
-var resolveRunScopedAiChunks = async ({
-  outDir,
-  strategyName
-}) => {
-  const requestedRunId = typeof flags.runId === "string" && flags.runId.trim() ? flags.runId.trim() : "";
-  const runIds = requestedRunId ? [requestedRunId] : await (0, import_ai.listAiChunkRunIds)({ strategyName, outDir });
-  const runId = runIds.at(-1) ?? "";
-  if (!runId) {
-    return null;
-  }
-  const chunkFiles = await (0, import_ai.listAiChunkFiles)({ strategyName, outDir, runId });
-  if (!chunkFiles.length) {
-    return {
-      runId,
-      chunkFiles,
-      completedAttemptKeys: /* @__PURE__ */ new Set()
-    };
-  }
-  const userName = String(flags.user || "root");
-  const completed = await loadBacktestCheckpointResults({ runId, userName });
-  return {
-    runId,
-    chunkFiles,
-    completedAttemptKeys: buildCompletedBacktestDatasetAttemptKeys(completed)
-  };
+var printExportResult = (result) => {
+  console.log(
+    import_chalk2.default.green(
+      result.splitApplied ? `Merged AI dataset for ${result.strategyName} saved as ${result.partCount} part files` : `Merged AI dataset saved: ${result.partPaths[0]}`
+    )
+  );
+  result.partPaths.forEach((filePath, index) => {
+    console.log(import_chalk2.default.gray(`part${index + 1}: ${filePath}`));
+  });
+  console.log(
+    import_chalk2.default.gray(
+      `strategy=${result.strategyName}, source_chunks=${result.sourceChunkCount}, deleteChunks=${Boolean(
+        result.deleteChunks
+      )}, partMonths=${result.partMonths}, partCount=${result.partCount}${result.runId ? `, runId=${result.runId}, completedAttempts=${result.completedAttemptCount}` : ""}`
+    )
+  );
 };
 var main = async () => {
   const outDir = String(flags.outDir || "data/ai/export");
-  await import_promises.default.mkdir(outDir, { recursive: true });
+  await import_promises2.default.mkdir(outDir, { recursive: true });
   const strategyName = await resolveExportStrategy({
     explicitStrategy: String(flags.strategy || ""),
     outDir,
     datasetLabel: "AI",
     promptLabel: "Select AI export strategy",
-    listStrategies: import_ai.listAiChunkStrategies
+    listStrategies: import_ai2.listAiChunkStrategies,
+    includeAllOption: true
   });
   if (!strategyName) {
     console.log(import_chalk2.default.yellow(`No AI chunk files found in ${outDir}`));
-    process.exit(0);
+    return;
   }
-  const runScopedChunks = await resolveRunScopedAiChunks({
-    strategyName,
-    outDir
-  });
-  const chunkFiles = runScopedChunks ? runScopedChunks.chunkFiles : await (0, import_ai.listAiChunkFiles)({
-    strategyName,
-    outDir
+  const strategyNames = strategyName === ALL_EXPORT_STRATEGIES ? await (0, import_ai2.listAiChunkStrategies)({ outDir }) : [strategyName];
+  if (!strategyNames.length) {
+    console.log(import_chalk2.default.yellow(`No AI chunk files found in ${outDir}`));
+    return;
+  }
+  const progressBar = strategyNames.length > 1 ? new import_progress.default(":current/:total [:bar] :percent :strategy :status", {
+    total: strategyNames.length,
+    width: 30
+  }) : null;
+  const results = await exportAiStrategiesSequentially({
+    outDir,
+    strategyNames,
+    keepChunks: Boolean(flags.keepChunks),
+    partMonths: Number(flags.partMonths),
+    requestedRunId: typeof flags.runId === "string" ? flags.runId.trim() : void 0,
+    userName: String(flags.user || "root"),
+    onProgress: progressBar ? ({ strategyName: currentStrategy, status }) => {
+      const progressStatus = status === "started" ? "exporting" : status;
+      if (status === "started" || status === "failed") {
+        progressBar.tick(0, {
+          strategy: currentStrategy,
+          status: progressStatus
+        });
+        if (status === "failed") {
+          progressBar.terminate();
+        }
+        return;
+      }
+      progressBar.tick(1, {
+        strategy: currentStrategy,
+        status: progressStatus
+      });
+    } : void 0
   });
-  if (!chunkFiles.length) {
+  if (!results.length) {
     console.log(
       import_chalk2.default.yellow(
-        `No chunk files found for strategy "${strategyName}" in ${outDir}`
+        strategyNames.length === 1 ? `No chunk files found for strategy "${strategyNames[0]}" in ${outDir}` : `No AI chunk files were exported from ${outDir}`
       )
     );
-    process.exit(0);
+    return;
   }
-  if (runScopedChunks && runScopedChunks.completedAttemptKeys.size === 0) {
-    throw new Error(
-      `No completed checkpoint attempts found for AI export run "${runScopedChunks.runId}"`
-    );
-  }
-  const mergedPath = import_path.default.join(
-    outDir,
-    `ai-dataset-${(0, import_ai.toFileToken)(strategyName)}-merged-${Date.now()}.jsonl`
-  );
-  await (0, import_ai.mergeAiJsonlFiles)({
-    filePaths: chunkFiles,
-    outPath: mergedPath,
-    shouldIncludeRow: runScopedChunks ? (row) => isBacktestDatasetRowForCompletedAttempt(
-      row,
-      runScopedChunks.completedAttemptKeys
-    ) : void 0
-  });
-  const partMonths = Math.max(0, Math.trunc(Number(flags.partMonths) || 0));
-  const splitResult = partMonths > 0 ? await (0, import_ai.splitAiMergedDatasetFile)({
-    filePath: mergedPath,
-    monthsPerPart: partMonths
-  }) : {
-    partPaths: [mergedPath],
-    partCount: 1,
-    splitApplied: false
-  };
-  const shouldDeleteChunks = !Boolean(flags.keepChunks);
-  if (shouldDeleteChunks) {
-    for (const filePath of chunkFiles) {
-      await import_promises.default.rm(filePath, { force: true });
-    }
-  }
-  console.log(
-    import_chalk2.default.green(
-      splitResult.splitApplied ? `Merged AI dataset saved as ${splitResult.partCount} part files` : `Merged AI dataset saved: ${splitResult.partPaths[0]}`
-    )
-  );
-  splitResult.partPaths.forEach((filePath, index) => {
-    console.log(import_chalk2.default.gray(`part${index + 1}: ${filePath}`));
-  });
-  console.log(
-    import_chalk2.default.gray(
-      `strategy=${strategyName}, source_chunks=${chunkFiles.length}, deleteChunks=${Boolean(
-        shouldDeleteChunks
-      )}, partMonths=${partMonths}, partCount=${splitResult.partCount}${runScopedChunks ? `, runId=${runScopedChunks.runId}, completedAttempts=${runScopedChunks.completedAttemptKeys.size}` : ""}`
-    )
-  );
-  process.exit(0);
+  results.forEach(printExportResult);
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {