@tradejs/cli 2.0.15 → 2.0.16

package/dist/scripts/runtimeEvidenceSync.js

@@ -0,0 +1,335 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/scripts/runtimeEvidenceSync.ts
+var runtimeEvidenceSync_exports = {};
+__export(runtimeEvidenceSync_exports, {
+  main: () => main,
+  runtimeEvidenceSync: () => runtimeEvidenceSync
+});
+module.exports = __toCommonJS(runtimeEvidenceSync_exports);
+var import_node_path3 = __toESM(require("path"));
+var import_args = __toESM(require("args"));
+var import_chalk = __toESM(require("chalk"));
+
+// src/lib/runtimeEvidenceSync.ts
+var import_node_child_process = require("child_process");
+var import_promises2 = __toESM(require("fs/promises"));
+var import_node_path2 = __toESM(require("path"));
+var import_node_util = require("util");
+
+// src/lib/runtimeEvidenceArtifacts.ts
+var import_node_crypto = require("crypto");
+var import_promises = __toESM(require("fs/promises"));
+var import_node_path = __toESM(require("path"));
+var RUNTIME_EVIDENCE_MANIFEST_FILE = "manifest.json";
+var RUNTIME_EVIDENCE_PAYLOAD_FILE = "runtime-evidence.json";
+var RUNTIME_EVIDENCE_COMPLETE_FILE = ".complete";
+var asRecord = (value) => value && typeof value === "object" && !Array.isArray(value) ? value : null;
+var sha256 = (value) => (0, import_node_crypto.createHash)("sha256").update(value).digest("hex");
+var assertFiniteTimestamp = (value, label) => {
+  if (typeof value !== "number" || !Number.isFinite(value)) {
+    throw new Error(`Invalid runtime evidence ${label}`);
+  }
+  return value;
+};
+var parseManifest = (value) => {
+  const manifest = asRecord(value);
+  const window = asRecord(manifest?.window);
+  const payload = asRecord(manifest?.payload);
+  const counts = asRecord(manifest?.counts);
+  if (manifest?.schemaVersion !== 1 || manifest.reportType !== "runtime-evidence" || typeof manifest.artifactId !== "string" || typeof manifest.deploymentId !== "string" || typeof manifest.userName !== "string" || typeof manifest.createdAt !== "number" || payload?.file !== RUNTIME_EVIDENCE_PAYLOAD_FILE || typeof payload.sha256 !== "string" || !/^[a-f0-9]{64}$/.test(payload.sha256) || typeof payload.bytes !== "number" || !Array.isArray(manifest.lineageKeys)) {
+    throw new Error("Invalid runtime evidence manifest");
+  }
+  return {
+    schemaVersion: 1,
+    artifactId: manifest.artifactId,
+    reportType: "runtime-evidence",
+    deploymentId: manifest.deploymentId,
+    userName: manifest.userName,
+    createdAt: manifest.createdAt,
+    window: {
+      startTime: assertFiniteTimestamp(window?.startTime, "window.startTime"),
+      endTime: assertFiniteTimestamp(window?.endTime, "window.endTime")
+    },
+    payload: {
+      file: RUNTIME_EVIDENCE_PAYLOAD_FILE,
+      sha256: payload.sha256,
+      bytes: payload.bytes
+    },
+    counts: Object.fromEntries(
+      Object.entries(counts ?? {}).filter(
+        (entry) => typeof entry[1] === "number" && Number.isFinite(entry[1])
+      )
+    ),
+    lineageKeys: manifest.lineageKeys.filter(
+      (value2) => typeof value2 === "string" && value2.length > 0
+    )
+  };
+};
+var verifyRuntimeEvidenceBundle = async (bundleDir) => {
+  const manifestPath = import_node_path.default.join(bundleDir, RUNTIME_EVIDENCE_MANIFEST_FILE);
+  const payloadPath = import_node_path.default.join(bundleDir, RUNTIME_EVIDENCE_PAYLOAD_FILE);
+  const completePath = import_node_path.default.join(bundleDir, RUNTIME_EVIDENCE_COMPLETE_FILE);
+  const [manifestText, payload, completeText] = await Promise.all([
+    import_promises.default.readFile(manifestPath, "utf8"),
+    import_promises.default.readFile(payloadPath),
+    import_promises.default.readFile(completePath, "utf8")
+  ]);
+  const manifest = parseManifest(JSON.parse(manifestText));
+  if (completeText.trim() !== manifest.artifactId) {
+    throw new Error(`Invalid completion marker in ${bundleDir}`);
+  }
+  if (payload.byteLength !== manifest.payload.bytes) {
+    throw new Error(`Runtime evidence size mismatch in ${bundleDir}`);
+  }
+  if (sha256(payload) !== manifest.payload.sha256) {
+    throw new Error(`Runtime evidence checksum mismatch in ${bundleDir}`);
+  }
+  const artifact = asRecord(JSON.parse(payload.toString("utf8")));
+  const artifactWindow = asRecord(artifact?.window);
+  if (artifact?.reportType !== "runtime-evidence" || artifactWindow?.startTime !== manifest.window.startTime || artifactWindow.endTime !== manifest.window.endTime) {
+    throw new Error(`Runtime evidence payload mismatch in ${bundleDir}`);
+  }
+  return {
+    bundleDir,
+    manifestPath,
+    payloadPath,
+    manifest,
+    artifact
+  };
+};
+var discoverRuntimeEvidenceBundles = async (rootDir) => {
+  const bundles = [];
+  const visit = async (directory, depth) => {
+    if (depth > 8) return;
+    let entries;
+    try {
+      entries = await import_promises.default.readdir(directory, { withFileTypes: true });
+    } catch (error) {
+      if (error.code === "ENOENT") return;
+      throw error;
+    }
+    const names = new Set(entries.map((entry) => entry.name));
+    if (names.has(RUNTIME_EVIDENCE_MANIFEST_FILE) && names.has(RUNTIME_EVIDENCE_PAYLOAD_FILE) && names.has(RUNTIME_EVIDENCE_COMPLETE_FILE)) {
+      bundles.push(directory);
+      return;
+    }
+    await Promise.all(
+      entries.filter(
+        (entry) => entry.isDirectory() && !entry.name.startsWith(".") && !entry.name.includes(".tmp-")
+      ).map((entry) => visit(import_node_path.default.join(directory, entry.name), depth + 1))
+    );
+  };
+  await visit(rootDir, 0);
+  return bundles.sort();
+};
+
+// src/lib/runtimeEvidenceSync.ts
+var execFileAsync = (0, import_node_util.promisify)(import_node_child_process.execFile);
+var isExistingDestinationError = (error) => {
+  const code = error.code;
+  return code === "EEXIST" || code === "ENOTEMPTY";
+};
+var safeSegment = (value, fallback) => {
+  const normalized = value.trim().replace(/[^a-zA-Z0-9._-]+/g, "-");
+  return normalized && normalized !== "." && normalized !== ".." ? normalized : fallback;
+};
+var withTrailingSlash = (value) => value.endsWith("/") ? value : `${value}/`;
+var dateParts = (timestamp) => {
+  const date = new Date(timestamp);
+  return [
+    String(date.getUTCFullYear()),
+    String(date.getUTCMonth() + 1).padStart(2, "0"),
+    String(date.getUTCDate()).padStart(2, "0")
+  ];
+};
+var buildRuntimeEvidenceRsyncArgs = ({
+  source,
+  destination
+}) => [
+  "--recursive",
+  "--times",
+  "--partial",
+  "--partial-dir=.rsync-partial",
+  "--prune-empty-dirs",
+  withTrailingSlash(source),
+  withTrailingSlash(destination)
+];
+var receiptPath = (receiptsRoot, artifactId) => import_node_path2.default.join(receiptsRoot, `${safeSegment(artifactId, "artifact")}.json`);
+var listPendingRuntimeEvidenceBundles = async ({
+  evidenceRoot,
+  deploymentId
+}) => {
+  const safeDeploymentId = safeSegment(deploymentId, "production");
+  const artifactsRoot = import_node_path2.default.join(evidenceRoot, "artifacts", safeDeploymentId);
+  const receiptsRoot = import_node_path2.default.join(evidenceRoot, "receipts", safeDeploymentId);
+  const bundles = await discoverRuntimeEvidenceBundles(artifactsRoot);
+  const pending = [];
+  for (const bundleDir of bundles) {
+    const verified = await verifyRuntimeEvidenceBundle(bundleDir);
+    if (verified.manifest.deploymentId !== safeDeploymentId) continue;
+    try {
+      await import_promises2.default.access(receiptPath(receiptsRoot, verified.manifest.artifactId));
+    } catch (error) {
+      if (error.code !== "ENOENT") throw error;
+      pending.push(verified);
+    }
+  }
+  return pending.sort(
+    (left, right) => left.manifest.window.startTime - right.manifest.window.startTime || left.manifest.createdAt - right.manifest.createdAt
+  );
+};
+var syncRuntimeEvidenceBundles = async ({
+  source,
+  evidenceRoot,
+  deploymentId,
+  runRsync = async (args2) => {
+    const result = await execFileAsync("rsync", args2, {
+      maxBuffer: 10 * 1024 * 1024
+    });
+    return { stdout: result.stdout, stderr: result.stderr };
+  }
+}) => {
+  const safeDeploymentId = safeSegment(deploymentId, "production");
+  const inboxRoot = import_node_path2.default.join(evidenceRoot, "inbox", safeDeploymentId);
+  const artifactsRoot = import_node_path2.default.join(evidenceRoot, "artifacts", safeDeploymentId);
+  await Promise.all([
+    import_promises2.default.mkdir(inboxRoot, { recursive: true }),
+    import_promises2.default.mkdir(artifactsRoot, { recursive: true })
+  ]);
+  const rsyncArgs = buildRuntimeEvidenceRsyncArgs({
+    source,
+    destination: inboxRoot
+  });
+  const transfer = await runRsync(rsyncArgs);
+  const discovered = await discoverRuntimeEvidenceBundles(inboxRoot);
+  const received = [];
+  for (const bundleDir of discovered) {
+    const verified = await verifyRuntimeEvidenceBundle(bundleDir);
+    if (verified.manifest.deploymentId !== safeDeploymentId) {
+      throw new Error(
+        `Artifact deployment mismatch: expected ${safeDeploymentId}, received ${verified.manifest.deploymentId}`
+      );
+    }
+    const destinationDir = import_node_path2.default.join(
+      artifactsRoot,
+      ...dateParts(verified.manifest.window.endTime),
+      safeSegment(verified.manifest.artifactId, "artifact")
+    );
+    await import_promises2.default.mkdir(import_node_path2.default.dirname(destinationDir), { recursive: true });
+    try {
+      await import_promises2.default.rename(bundleDir, destinationDir);
+    } catch (error) {
+      if (!isExistingDestinationError(error)) throw error;
+      const existing = await verifyRuntimeEvidenceBundle(destinationDir);
+      if (existing.manifest.payload.sha256 !== verified.manifest.payload.sha256) {
+        throw new Error(
+          `Conflicting runtime evidence artifact ${verified.manifest.artifactId}`
+        );
+      }
+      await import_promises2.default.rm(bundleDir, { recursive: true, force: true });
+    }
+    received.push(await verifyRuntimeEvidenceBundle(destinationDir));
+  }
+  return {
+    rsyncArgs,
+    stdout: String(transfer.stdout ?? ""),
+    stderr: String(transfer.stderr ?? ""),
+    received,
+    pending: await listPendingRuntimeEvidenceBundles({
+      evidenceRoot,
+      deploymentId: safeDeploymentId
+    })
+  };
+};
+
+// src/scripts/runtimeEvidenceSync.ts
+import_args.default.option(
+  "source",
+  "Rsync source ending at the deployment ready directory",
+  process.env.RUNTIME_EVIDENCE_RSYNC_SOURCE
+);
+import_args.default.option(
+  "deployment",
+  "Deployment id expected in artifact manifests",
+  process.env.RUNTIME_EVIDENCE_DEPLOYMENT_ID || "production"
+);
+import_args.default.option(
+  "dir",
+  "Local evidence storage root",
+  process.env.RUNTIME_EVIDENCE_LOCAL_DIR || "data/runtime-evidence"
+);
+var flags = import_args.default.parse(process.argv);
+var projectRoot = String(process.env.PROJECT_CWD || process.cwd()).trim() || process.cwd();
+var runtimeEvidenceSync = async () => {
+  const source = String(flags.source ?? "").trim();
+  if (!source) {
+    throw new Error(
+      "Provide --source or RUNTIME_EVIDENCE_RSYNC_SOURCE for runtime evidence sync."
+    );
+  }
+  const evidenceRoot = import_node_path3.default.resolve(projectRoot, String(flags.dir));
+  const result = await syncRuntimeEvidenceBundles({
+    source,
+    evidenceRoot,
+    deploymentId: String(flags.deployment)
+  });
+  console.log(
+    import_chalk.default.green(
+      `Runtime evidence sync completed: received=${result.received.length} pending=${result.pending.length}`
+    )
+  );
+  console.log(
+    JSON.stringify(
+      {
+        evidenceRoot,
+        received: result.received.map((bundle) => ({
+          artifactId: bundle.manifest.artifactId,
+          bundleDir: bundle.bundleDir,
+          payloadPath: bundle.payloadPath
+        })),
+        pending: result.pending.map((bundle) => ({
+          artifactId: bundle.manifest.artifactId,
+          bundleDir: bundle.bundleDir,
+          payloadPath: bundle.payloadPath,
+          window: bundle.manifest.window
+        }))
+      },
+      null,
+      2
+    )
+  );
+};
+var main = runtimeEvidenceSync;
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  main,
+  runtimeEvidenceSync
+});

package/dist/scripts/runtimeParity.js

@@ -2351,6 +2351,8 @@ var timeOperation = async (label, operation, log = console.log) => {
 
 // src/lib/marketContextPrepare.ts
 var import_chalk4 = __toESM(require("chalk"));
+var import_timescale5 = require("@tradejs/infra/timescale");
+var import_registry = require("@tradejs/node/registry");
 
 // src/lib/binanceMarketContextBackfill.ts
 var import_progress = __toESM(require("progress"));
@@ -5592,6 +5594,9 @@ var backfillHyperliquidWhaleContext = async (params) => {
 };
 
 // src/lib/marketContextPrepare.ts
+var uniqueReasons = (reasons) => [
+  ...new Set(reasons)
+];
 var shouldPrepareDerivativesContextForRun = (params) => {
   if (params.universe === "tradfi") return false;
   if (params.mode === "signals") {
@@ -5599,6 +5604,9 @@ var shouldPrepareDerivativesContextForRun = (params) => {
       cacheOnly: params.cacheOnly
     });
   }
+  if (params.mode === "backtest" && !params.aiEnabled && !params.mlEnabled) {
+    return false;
+  }
   return shouldBackfillDerivativesContextForBacktest({
     aiEnabled: Boolean(params.aiEnabled),
     cacheOnly: params.cacheOnly,
@@ -5608,6 +5616,7 @@ var shouldPrepareDerivativesContextForRun = (params) => {
 var shouldPrepareBinanceMarketContextForRun = (params) => {
   if (params.universe === "tradfi") return false;
   if (params.mode === "backtest") {
+    if (!params.aiEnabled && !params.mlEnabled) return false;
     return shouldBackfillBinanceMarketContextForBacktest({
       aiEnabled: Boolean(params.aiEnabled),
       cacheOnly: params.cacheOnly,
@@ -5626,6 +5635,7 @@ var shouldPrepareBinanceMarketContextForRun = (params) => {
 var shouldPrepareCoinMarketCapContextForRun = (params) => {
   if (params.universe === "tradfi") return false;
   if (params.mode === "backtest") {
+    if (!params.aiEnabled && !params.mlEnabled) return false;
     return shouldBackfillCoinMarketCapContextForBacktest({
       aiEnabled: Boolean(params.aiEnabled),
       cacheOnly: params.cacheOnly,
@@ -5641,10 +5651,8 @@ var shouldPrepareCoinMarketCapContextForRun = (params) => {
     cacheOnly: params.cacheOnly
   });
 };
-var shouldPrepareHyperliquidWhaleContextForRun = (params) => {
-  const strategyRequiresContext = params.strategyNames?.includes(
-    "HyperliquidConsensus"
-  );
+var shouldPrepareHyperliquidWhaleContextForRun = (params, coreContextSources = []) => {
+  const strategyRequiresContext = coreContextSources.includes("hyperliquidWhales");
   if (params.universe === "tradfi" || params.mode === "signals" || params.cacheOnly || !params.aiEnabled && !params.mlEnabled && !strategyRequiresContext) {
     return false;
   }
@@ -5655,6 +5663,64 @@ var shouldPrepareHyperliquidWhaleContextForRun = (params) => {
   if (strategyRequiresContext) return true;
   return ["1", "true", "yes", "on"].includes(configured);
 };
+var resolveCoreContextSources = async (params) => {
+  const strategyNames = [...new Set(params.strategyNames ?? [])];
+  if (!strategyNames.length) return [];
+  await (0, import_registry.ensureStrategyPluginsLoaded)(params.projectRoot);
+  return [
+    ...new Set(
+      strategyNames.flatMap(
+        (strategyName) => (0, import_registry.getStrategyManifest)(strategyName, params.projectRoot)?.contextRequirements?.core ?? []
+      )
+    )
+  ];
+};
+var resolveMarketContextRunRequirements = async (params) => {
+  const coreContextSources = await resolveCoreContextSources(params);
+  const paramsWithCoreDemand = (source) => coreContextSources.includes(source) ? { ...params, aiEnabled: true } : params;
+  const isCrypto = params.universe !== "tradfi";
+  const capturesPayload = Boolean(params.aiEnabled || params.mlEnabled);
+  const runtimeReadsContext = params.mode !== "backtest";
+  const reasonsFor = (source) => uniqueReasons([
+    ...coreContextSources.includes(source) ? ["core"] : [],
+    ...params.aiEnabled ? ["ai"] : [],
+    ...params.mlEnabled ? ["ml"] : [],
+    ...runtimeReadsContext ? ["runtime"] : []
+  ]);
+  const readsStandardContext = isCrypto && (capturesPayload || runtimeReadsContext);
+  const readsHyperliquid = isCrypto && (readsStandardContext || coreContextSources.includes("hyperliquidWhales"));
+  return {
+    binance: {
+      read: readsStandardContext || coreContextSources.includes("binance"),
+      backfill: shouldPrepareBinanceMarketContextForRun(
+        paramsWithCoreDemand("binance")
+      ),
+      requiredBy: reasonsFor("binance")
+    },
+    coinmarketcap: {
+      read: readsStandardContext || coreContextSources.includes("coinmarketcap"),
+      backfill: shouldPrepareCoinMarketCapContextForRun(
+        paramsWithCoreDemand("coinmarketcap")
+      ),
+      requiredBy: reasonsFor("coinmarketcap")
+    },
+    derivatives: {
+      read: readsStandardContext || coreContextSources.includes("derivatives"),
+      backfill: shouldPrepareDerivativesContextForRun(
+        paramsWithCoreDemand("derivatives")
+      ),
+      requiredBy: reasonsFor("derivatives")
+    },
+    hyperliquidWhales: {
+      read: readsHyperliquid,
+      backfill: shouldPrepareHyperliquidWhaleContextForRun(
+        params,
+        coreContextSources
+      ),
+      requiredBy: reasonsFor("hyperliquidWhales")
+    }
+  };
+};
 var resolveCoinMarketCapBackfillForMode = (mode) => mode === "backtest" ? backfillCoinMarketCapContextForBacktest : mode === "signals" ? backfillCoinMarketCapContextForSignals : backfillCoinMarketCapContextForReplay;
 var buildCoinMarketCapBackfillParams = (params) => ({
   userName: params.userName,
@@ -5683,7 +5749,15 @@ var prepareMarketContextForRun = async (params) => {
     console.log(import_chalk4.default.gray(message));
   });
   const timeOperation2 = (label, operation) => timeOperation(label, operation, log);
-  if (shouldPrepareDerivativesContextForRun(params)) {
+  const requirements = await resolveMarketContextRunRequirements(params);
+  const readableSources = Object.entries(requirements).filter(([, requirement]) => requirement.read).map(([source]) => source);
+  if (readableSources.length) {
+    await timeOperation2(
+      "market context schema ensure",
+      () => (0, import_timescale5.ensureMarketContextSchemas)(readableSources)
+    );
+  }
+  if (requirements.derivatives.backfill) {
     await timeOperation2(
       "derivatives context backfill",
       () => (params.mode === "signals" ? backfillDerivativesContextForSignals : backfillDerivativesContextForBacktest)(
@@ -5691,7 +5765,7 @@ var prepareMarketContextForRun = async (params) => {
       )
     );
   }
-  if (shouldPrepareHyperliquidWhaleContextForRun(params)) {
+  if (requirements.hyperliquidWhales.backfill) {
     await timeOperation2(
       "hyperliquid whale context backfill",
       () => backfillHyperliquidWhaleContext({
@@ -5703,13 +5777,13 @@ var prepareMarketContextForRun = async (params) => {
       })
     );
   }
-  if (shouldPrepareBinanceMarketContextForRun(params)) {
+  if (requirements.binance.backfill) {
     await timeOperation2("binance market context backfill", () => {
       const backfill = params.mode === "backtest" ? backfillBinanceMarketContextForBacktest : params.mode === "signals" ? backfillBinanceMarketContextForSignals : backfillBinanceMarketContextForReplay;
       return backfill(buildBinanceMarketBackfillParams(params));
     });
   }
-  if (shouldPrepareCoinMarketCapContextForRun(params)) {
+  if (requirements.coinmarketcap.backfill) {
     await timeOperation2(
       "coinmarketcap historical context backfill",
       () => resolveCoinMarketCapBackfillForMode(params.mode)(
@@ -5717,6 +5791,7 @@ var prepareMarketContextForRun = async (params) => {
       )
     );
   }
+  return requirements;
 };
 
 // src/lib/marketData/historyPrepare.ts