dukascopy-node-plus 1.0.0

package/dist/esm/cli/index.js

@@ -0,0 +1,462 @@
+#!/usr/bin/env node
+import {
+  BufferFetcher,
+  CacheManager,
+  VolumeUnit,
+  __require,
+  formatBytes,
+  generateUrls,
+  getDateTimeFormatOptions,
+  getFormattedDate,
+  instrumentMetaData,
+  normaliseDates,
+  processData,
+  schema,
+  validateConfig,
+  version
+} from "../chunk-QOULS5Y3.js";
+
+// src/cli/cli.ts
+import { resolve, join } from "path";
+import os from "os";
+
+// src/cli/progress.ts
+import { Bar } from "cli-progress";
+var chalk = __require("chalk");
+var progressBar = new Bar({
+  format: "|" + chalk.green("{bar}") + "| {percentage}%",
+  barCompleteChar: "\u2588",
+  barIncompleteChar: "\u2591",
+  hideCursor: true,
+  fps: 24,
+  barsize: 45
+});
+
+// src/cli/cli.ts
+import { createWriteStream } from "fs";
+import { ensureDir, ensureFile, stat } from "fs-extra";
+
+// src/cli/config.ts
+import { program } from "commander";
+var now = "now";
+var commanderSchema = program.option("-d, --debug", "Output extra debugging", false).option("-s, --silent", "Hides the search config in the CLI output", false).requiredOption("-i, --instrument <value>", "Trading instrument").requiredOption("-from, --date-from <value>", "From date (yyyy-mm-dd)").option("-to, --date-to <value>", `To date (yyyy-mm-dd or '${now}')`, now).option(
+  "-t, --timeframe <value>",
+  "Timeframe aggregation (tick, s1, m1, m5, m15, m30, h1, h4, d1, mn1)",
+  "d1" /* d1 */
+).option("-p, --price-type <value>", "Price type: (bid, ask)", "bid" /* bid */).option("-utc, --utc-offset <value>", "UTC offset in minutes", Number, 0).option("-v, --volumes", "Include volumes", false).option(
+  "-vu, --volume-units <value>",
+  "Volume units (millions, thousands, units)",
+  VolumeUnit.millions
+).option("-fl, --flats", "Include flats (0 volumes)", false).option("-f, --format <value>", "Output format (csv, json, array)", "json" /* json */).option("-dir, --directory <value>", "Download directory", "./download").option("-bs, --batch-size <value>", "Batch size of downloaded artifacts", Number, 10).option("-bp, --batch-pause <value>", "Pause between batches in ms", Number, 1e3).option("-ch, --cache", "Use cache", false).option("-chpath, --cache-path <value>", "Folder path for cache data", "./.dukascopy-cache").option("-df, --date-format <value>", "Date format", "").option("-tz, --time-zone <value>", "Timezone", "").option("-r, --retries <value>", "Number of retries for a failed artifact download", Number, 0).option("-rp, --retry-pause <value>", "Pause between retries in milliseconds", Number, 500).option(
+  "-re, --retry-on-empty",
+  "A flag indicating whether requests with successful but empty (0 Bytes) responses should be retried. If `retries` is `0` this parameter will be ignored",
+  false
+).option(
+  "-fr, --no-fail-after-retries",
+  "A flag indicating whether the process should fail after all retries have been exhausted. If `retries` is `0` this parameter will be ignored"
+).option("-fn, --file-name <value>", "Custom file name for the generated file", "").option(
+  "-in, --inline",
+  "Makes files smaller in size by removing new lines in the output (works only with json and array formats)",
+  false
+);
+function getConfigFromCliArgs(argv) {
+  const options = commanderSchema.parse(argv).opts();
+  if (options.dateTo === now) {
+    options.dateTo = new Date();
+  }
+  const cliConfig = {
+    instrument: options.instrument,
+    dates: {
+      from: options.dateFrom,
+      to: options.dateTo
+    },
+    timeframe: options.timeframe,
+    priceType: options.priceType,
+    utcOffset: options.utcOffset,
+    volumes: options.volumes,
+    volumeUnits: options.volumeUnits,
+    ignoreFlats: !options.flats,
+    dir: options.directory,
+    silent: options.silent,
+    format: options.format,
+    batchSize: options.batchSize,
+    pauseBetweenBatchesMs: options.batchPause,
+    useCache: options.cache,
+    cacheFolderPath: options.cachePath,
+    retryCount: options.retries,
+    failAfterRetryCount: options.failAfterRetries,
+    retryOnEmpty: options.retryOnEmpty,
+    pauseBetweenRetriesMs: options.retryPause,
+    debug: options.debug,
+    inline: options.inline,
+    fileName: options.fileName,
+    dateFormat: options.dateFormat,
+    timeZone: options.timeZone
+  };
+  const cliSchema = {
+    ...schema,
+    ...{
+      dir: { type: "string", required: true },
+      silent: { type: "boolean", required: false },
+      debug: { type: "boolean", required: false },
+      inline: { type: "boolean", required: false },
+      fileName: { type: "string", required: false },
+      dateFormat: { type: "string", required: false },
+      timeZone: { type: "string", required: false }
+    }
+  };
+  return validateConfig(cliConfig, cliSchema);
+}
+
+// src/cli/printer.ts
+var chalk2 = __require("chalk");
+var log = console.log;
+function printSpacer() {
+  log();
+}
+function printDivider() {
+  log(chalk2.gray("----------------------------------------------------"));
+}
+function printHeader(searchConfig, adjustedStartDate, adjustedEndDate) {
+  const { instrument, timeframe, priceType, utcOffset, volumes, ignoreFlats, format } = searchConfig;
+  const dateTimeFormatOptions = getDateTimeFormatOptions(timeframe);
+  printDivider();
+  log(chalk2.whiteBright("Downloading historical price data for:"));
+  printDivider();
+  log("Instrument:    ", chalk2.bold(chalk2.yellow(instrumentMetaData[instrument].description)));
+  log("Timeframe:     ", chalk2.bold(chalk2.yellow(timeframe)));
+  log(
+    "From date:     ",
+    chalk2.bold(chalk2.yellow(getFormattedDate(adjustedStartDate, dateTimeFormatOptions)))
+  );
+  log(
+    "To date:       ",
+    chalk2.bold(chalk2.yellow(getFormattedDate(adjustedEndDate, dateTimeFormatOptions)))
+  );
+  if (timeframe !== "tick") {
+    log("Price type:    ", chalk2.bold(chalk2.yellow(priceType)));
+  }
+  log("Volumes:       ", chalk2.bold(chalk2.yellow(volumes)));
+  log("UTC Offset:    ", chalk2.bold(chalk2.yellow(utcOffset)));
+  log("Include flats: ", chalk2.bold(chalk2.yellow(!ignoreFlats)));
+  log("Format:        ", chalk2.bold(chalk2.yellow(format)));
+  printDivider();
+}
+function printErrors(header, errorMessage) {
+  log(chalk2.redBright(header));
+  [].concat(errorMessage).forEach((error) => log(chalk2.red(` > ${error}`)));
+  printSpacer();
+}
+function printSuccess(text) {
+  printDivider();
+  log(chalk2.greenBright(text));
+  printSpacer();
+}
+function printGeneral(text) {
+  log(text);
+  printSpacer();
+}
+
+// src/cli/cli.ts
+import chalk3 from "chalk";
+import debug from "debug";
+import dayjs from "dayjs";
+import utc from "dayjs/plugin/utc";
+import tz from "dayjs/plugin/timezone";
+
+// src/stream-writer/index.ts
+import fs from "fs";
+var BatchStreamWriter = class {
+  constructor(options) {
+    this.isFileEmpty = true;
+    this.fileWriteStream = options.fileWriteStream;
+    this.timeframe = options.timeframe;
+    this.format = options.format;
+    this.isInline = options.isInline;
+    this.volumes = Boolean(options.volumes);
+    this.startDateTs = options.startDateTs;
+    this.endDateTs = options.endDateTs;
+    this.bodyHeaders = this.initHeaders();
+  }
+  initHeaders() {
+    const bodyHeaders = this.timeframe === "tick" /* tick */ ? ["timestamp", "askPrice", "bidPrice", "askVolume", "bidVolume"] : ["timestamp", "open", "high", "low", "close", "volume"];
+    if (!this.volumes) {
+      bodyHeaders.pop();
+      if (this.timeframe === "tick" /* tick */) {
+        bodyHeaders.pop();
+      }
+    }
+    return bodyHeaders;
+  }
+  async writeBatch(batch, dateFormatter) {
+    const batchWithinRange = [];
+    for (let j = 0; j < batch.length; j++) {
+      const item = batch[j];
+      const isItemInRange = item.length > 0 && item[0] >= this.startDateTs && item[0] < this.endDateTs;
+      if (isItemInRange) {
+        if (dateFormatter) {
+          item[0] = dateFormatter(item[0]);
+        }
+        batchWithinRange.push(item);
+      }
+    }
+    for (let i = 0; i < batchWithinRange.length; i++) {
+      let item = batchWithinRange[i];
+      const isFirstItem = i === 0;
+      const isLastItem = i === batchWithinRange.length - 1;
+      const shouldOpen = isFirstItem && this.isFileEmpty;
+      let body = "";
+      if (this.format === "csv" /* csv */) {
+        if (shouldOpen) {
+          const csvHeaders = this.bodyHeaders.join(",");
+          body += `${csvHeaders}
+`;
+        }
+        body += item.join(",") + "\n";
+      } else {
+        if (shouldOpen) {
+          body += "[" + (!this.isInline ? "\n" : "");
+        }
+        if (isFirstItem && !this.isFileEmpty) {
+          body += "," + (!this.isInline ? "\n" : "");
+        }
+        if (dateFormatter && (this.format === "json" /* json */ || this.format === "array" /* array */)) {
+          item[0] = `"${item[0]}"`;
+        }
+        if (this.format === "json" /* json */) {
+          const jsonObjectBody = item.map((val, i2) => `"${this.bodyHeaders[i2]}":${val}`).join(",");
+          body += `{${jsonObjectBody}}`;
+        } else {
+          const arrayBody = item.join(",");
+          body += `[${arrayBody}]`;
+        }
+        body += (!isLastItem ? "," : "") + (!isLastItem && !this.isInline ? "\n" : "");
+      }
+      if (!body) {
+        continue;
+      }
+      const ableToWrite = this.fileWriteStream.write(body);
+      if (ableToWrite) {
+        if (this.isFileEmpty) {
+          this.isFileEmpty = false;
+        }
+      } else {
+        await new Promise((resolve2) => {
+          this.fileWriteStream.once("drain", resolve2);
+        });
+      }
+    }
+    return true;
+  }
+  async closeBatchFile() {
+    if ((this.format === "json" /* json */ || this.format === "array" /* array */) && !this.isFileEmpty) {
+      const body = this.isInline ? "]" : "\n]";
+      const ableToWrite = this.fileWriteStream.write(body);
+      if (!ableToWrite) {
+        await new Promise((resolve2) => {
+          this.fileWriteStream.once("drain", resolve2);
+        });
+      }
+    }
+    this.fileWriteStream.end();
+    return true;
+  }
+};
+
+// src/utils/formatTimeDuration.ts
+function formatTimeDuration(durationMs) {
+  if (durationMs < 1e3) {
+    return `${durationMs}ms`;
+  } else if (durationMs < 6e4) {
+    return `${(durationMs / 1e3).toFixed(1)}s`;
+  } else if (durationMs < 36e5) {
+    const min = Math.floor(durationMs / 6e4);
+    const sec = Math.floor((durationMs - min * 6e4) / 1e3);
+    return `${min}m ${sec}s`;
+  } else {
+    const hours = Math.floor(durationMs / 36e5);
+    const min = Math.floor((durationMs - hours * 36e5) / 6e4);
+    const sec = Math.floor((durationMs - hours * 36e5 - min * 6e4) / 1e3);
+    return `${hours}h ${min}m ${sec}s`;
+  }
+}
+
+// src/cli/cli.ts
+dayjs.extend(utc);
+dayjs.extend(tz);
+var DEBUG_NAMESPACE = "dukascopy-node:cli";
+async function run(argv) {
+  const { input, isValid, validationErrors } = getConfigFromCliArgs(argv);
+  let {
+    instrument,
+    dates: { from: fromDate, to: toDate },
+    timeframe,
+    priceType,
+    utcOffset,
+    volumes,
+    volumeUnits,
+    ignoreFlats,
+    format,
+    batchSize,
+    pauseBetweenBatchesMs,
+    useCache,
+    cacheFolderPath,
+    dir,
+    silent,
+    debug: isDebugActive,
+    inline,
+    retryCount,
+    failAfterRetryCount,
+    retryOnEmpty,
+    pauseBetweenRetriesMs,
+    fileName: customFileName,
+    dateFormat,
+    timeZone
+  } = input;
+  if (isDebugActive) {
+    debug.enable(`${DEBUG_NAMESPACE}:*`);
+  } else {
+    if (process.env.DEBUG) {
+      isDebugActive = true;
+      debug.enable(process.env.DEBUG);
+    }
+  }
+  const downloadStartTs = Date.now();
+  try {
+    debug(`${DEBUG_NAMESPACE}:version`)(version);
+    debug(`${DEBUG_NAMESPACE}:nodejs`)(process.version);
+    debug(`${DEBUG_NAMESPACE}:os`)(`${os.type()}, ${os.release()} (${os.platform()})`);
+    debug(`${DEBUG_NAMESPACE}:config`)("%O", {
+      input,
+      isValid,
+      validationErrors
+    });
+    if (isValid) {
+      const [startDate, endDate] = normaliseDates({
+        instrument,
+        startDate: fromDate,
+        endDate: toDate,
+        timeframe,
+        utcOffset
+      });
+      const fileExtension = format === "csv" /* csv */ ? "csv" /* csv */ : "json" /* json */;
+      const dateRangeStr = [startDate, endDate].map((date) => {
+        let cutoff = 10;
+        const hasHours = date.getUTCHours() !== 0;
+        const hasMinutes = date.getUTCMinutes() !== 0;
+        if (hasHours) {
+          cutoff = 13;
+        }
+        if (hasMinutes) {
+          cutoff = 16;
+        }
+        return date.toISOString().slice(0, cutoff);
+      }).join("-");
+      const fileName = customFileName ? `${customFileName}.${fileExtension}` : `${instrument}-${timeframe}${timeframe === "tick" ? "" : "-" + priceType}-${dateRangeStr}.${fileExtension}`;
+      const folderPath = resolve(process.cwd(), dir);
+      const filePath = resolve(folderPath, fileName);
+      if (!isDebugActive) {
+        silent ? printDivider() : printHeader(input, startDate, endDate);
+      }
+      const urls = generateUrls({ instrument, timeframe, priceType, startDate, endDate });
+      debug(`${DEBUG_NAMESPACE}:urls`)(`Generated ${urls.length} urls`);
+      debug(`${DEBUG_NAMESPACE}:urls`)(`%O`, urls);
+      let step = 0;
+      if (!isDebugActive) {
+        progressBar.start(urls.length, step);
+      }
+      await ensureDir(folderPath);
+      const fileWriteStream = createWriteStream(filePath, { flags: "w+" });
+      fileWriteStream.on("finish", async () => {
+        const downloadEndTs = Date.now();
+        if (!isDebugActive) {
+          progressBar.stop();
+        }
+        const relativeFilePath = join(dir, fileName);
+        await ensureFile(filePath);
+        const { size } = await stat(filePath);
+        printSuccess(`\u221A File saved: ${chalk3.bold(relativeFilePath)} (${formatBytes(size)})`);
+        printGeneral(`Download time: ${formatTimeDuration(downloadEndTs - downloadStartTs)}`);
+      });
+      const batchStreamWriter = new BatchStreamWriter({
+        fileWriteStream,
+        timeframe,
+        format,
+        isInline: inline,
+        volumes,
+        startDateTs: +startDate,
+        endDateTs: +endDate
+      });
+      const bufferFetcher = new BufferFetcher({
+        batchSize,
+        pauseBetweenBatchesMs,
+        cacheManager: useCache ? new CacheManager({ cacheFolderPath }) : void 0,
+        retryCount,
+        retryOnEmpty,
+        failAfterRetryCount,
+        pauseBetweenRetriesMs,
+        onItemFetch: (url, buffer, isCacheHit) => {
+          debug(`${DEBUG_NAMESPACE}:fetcher`)(
+            url,
+            `| ${formatBytes(buffer.length)} |`,
+            `${isCacheHit ? "cache" : "network"}`
+          );
+          if (!isDebugActive) {
+            step += 1;
+            progressBar.update(step);
+          }
+        },
+        onBatchFetch: async (bufferObjects, isLastBatch) => {
+          const filteredBatchData = [];
+          for (let j = 0, m = bufferObjects.length; j < m; j++) {
+            if (bufferObjects[j].buffer.length > 0) {
+              filteredBatchData.push(bufferObjects[j]);
+            }
+          }
+          if (filteredBatchData.length) {
+            const processedBatch = processData({
+              instrument,
+              requestedTimeframe: timeframe,
+              bufferObjects: filteredBatchData,
+              priceType,
+              volumes,
+              volumeUnits,
+              ignoreFlats
+            });
+            await batchStreamWriter.writeBatch(
+              processedBatch,
+              dateFormat ? (timeStamp) => {
+                if (dateFormat === "iso") {
+                  return new Date(timeStamp).toISOString();
+                }
+                if (timeZone) {
+                  return dayjs(timeStamp).tz(timeZone).format(dateFormat);
+                }
+                return dayjs(timeStamp).utc().format(dateFormat);
+              } : void 0
+            );
+          }
+          if (isLastBatch) {
+            await batchStreamWriter.closeBatchFile();
+          }
+        }
+      });
+      await bufferFetcher.fetch_optimized(urls);
+    } else {
+      printErrors(
+        "Search config invalid:",
+        validationErrors.map((err) => (err == null ? void 0 : err.message) || "")
+      );
+      process.exit(0);
+    }
+  } catch (err) {
+    const errorMsg = err instanceof Error ? err.message : JSON.stringify(err);
+    printErrors("\nSomething went wrong:", errorMsg);
+    process.exit(0);
+  }
+}
+
+// src/cli/index.ts
+(async () => {
+  await run(process.argv);
+})();