dukascopy-node-plus 1.1.0 → 1.1.2

package/dist/cli/index.js

@@ -13883,7 +13883,7 @@ var import_utc = __toESM(require("dayjs/plugin/utc"));
 var import_timezone = __toESM(require("dayjs/plugin/timezone"));
 
 // package.json
-var version = "1.1.0";
+var version = "1.1.2";
 
 // src/stream-writer/index.ts
 var import_fs = __toESM(require("fs"));

package/dist/esm/chunk-476J4KQE.js

@@ -0,0 +1,35 @@
+// src/output-formatter/index.ts
+var candleHeaders = ["timestamp", "open", "high", "low", "close", "volume"];
+var tickHeaders = ["timestamp", "askPrice", "bidPrice", "askVolume", "bidVolume"];
+function formatOutput({
+  processedData,
+  format,
+  timeframe
+}) {
+  if (processedData.length === 0) {
+    return [];
+  }
+  const bodyHeaders = timeframe === "tick" /* tick */ ? tickHeaders : candleHeaders;
+  if (format === "json" /* json */) {
+    const data = processedData.map((arr) => {
+      return arr.reduce((all, item, i) => {
+        const name = bodyHeaders[i];
+        all[name] = item;
+        return all;
+      }, {});
+    });
+    return data;
+  }
+  if (format === "csv" /* csv */) {
+    const csvHeaders = bodyHeaders.filter((_, i) => processedData[0][i] !== void 0);
+    const csv = [csvHeaders, ...processedData].map((arr) => arr.join(",")).join("\n");
+    return csv;
+  }
+  return processedData;
+}
+
+export {
+  candleHeaders,
+  tickHeaders,
+  formatOutput
+};

package/dist/esm/{chunk-DWZJECKF.js → chunk-7MFC3GUP.js}

@@ -1,5 +1,5 @@
 // package.json
-var version = "1.1.0";
+var version = "1.1.2";
 
 export {
   version

package/dist/esm/chunk-MTHVPJMW.js

@@ -0,0 +1,113 @@
+import {
+  version
+} from "./chunk-7MFC3GUP.js";
+import {
+  formatOutput
+} from "./chunk-476J4KQE.js";
+import {
+  BufferFetcher,
+  CacheManager,
+  formatBytes,
+  generateUrls,
+  normaliseDates,
+  processData,
+  validateConfigNode
+} from "./chunk-D3M3SVMW.js";
+
+// src/getHistoricalRates.ts
+import debug from "debug";
+import os from "os";
+var DEBUG_NAMESPACE = "dukascopy-node";
+async function getHistoricalRates(config) {
+  debug(`${DEBUG_NAMESPACE}:version`)(version);
+  debug(`${DEBUG_NAMESPACE}:nodejs`)(process.version);
+  debug(`${DEBUG_NAMESPACE}:os`)(`${os.type()}, ${os.release()} (${os.platform()})`);
+  const { input, isValid, validationErrors } = validateConfigNode(config);
+  debug(`${DEBUG_NAMESPACE}:config`)("%O", {
+    input,
+    isValid,
+    validationErrors
+  });
+  if (!isValid) {
+    throw { validationErrors };
+  }
+  const {
+    instrument,
+    dates: { from, to },
+    timeframe,
+    priceType,
+    volumes,
+    volumeUnits,
+    utcOffset,
+    ignoreFlats,
+    format,
+    batchSize,
+    pauseBetweenBatchesMs,
+    useCache,
+    cacheFolderPath,
+    retryCount,
+    pauseBetweenRetriesMs,
+    retryOnEmpty
+  } = input;
+  const [startDate, endDate] = normaliseDates({
+    instrument,
+    startDate: from,
+    endDate: to,
+    timeframe,
+    utcOffset
+  });
+  const urls = generateUrls({
+    instrument,
+    timeframe,
+    priceType,
+    startDate,
+    endDate
+  });
+  debug(`${DEBUG_NAMESPACE}:urls`)(`Generated ${urls.length} urls`);
+  debug(`${DEBUG_NAMESPACE}:urls`)(`%O`, urls);
+  const onItemFetch = process.env.DEBUG ? (url, buffer, isCacheHit) => {
+    debug(`${DEBUG_NAMESPACE}:fetcher`)(
+      url,
+      `| ${formatBytes(buffer.length)} |`,
+      `${isCacheHit ? "cache" : "network"}`
+    );
+  } : void 0;
+  const bufferFetcher = new BufferFetcher({
+    batchSize,
+    pauseBetweenBatchesMs,
+    cacheManager: useCache ? new CacheManager({ cacheFolderPath }) : void 0,
+    retryCount,
+    pauseBetweenRetriesMs,
+    onItemFetch,
+    retryOnEmpty
+  });
+  const bufferredData = await bufferFetcher.fetch(urls);
+  const processedData = processData({
+    instrument,
+    requestedTimeframe: timeframe,
+    bufferObjects: bufferredData,
+    priceType,
+    volumes,
+    volumeUnits,
+    ignoreFlats
+  });
+  const [startDateMs, endDateMs] = [+startDate, +endDate];
+  const filteredData = processedData.filter(
+    ([timestamp]) => timestamp && timestamp >= startDateMs && timestamp < endDateMs
+  );
+  debug(`${DEBUG_NAMESPACE}:data`)(
+    `Generated ${filteredData.length} ${timeframe === "tick" /* tick */ ? "ticks" : "OHLC candles"}`
+  );
+  const formattedData = formatOutput({
+    processedData: filteredData,
+    format,
+    timeframe
+  });
+  return formattedData;
+}
+var getHistoricRates = getHistoricalRates;
+
+export {
+  getHistoricalRates,
+  getHistoricRates
+};

package/dist/esm/chunk-NBUGCZCI.js

@@ -0,0 +1,164 @@
+import {
+  candleHeaders,
+  tickHeaders
+} from "./chunk-476J4KQE.js";
+import {
+  BufferFetcher,
+  CacheManager,
+  formatBytes,
+  generateUrls,
+  normaliseDates,
+  processData,
+  validateConfigNode
+} from "./chunk-D3M3SVMW.js";
+
+// src/getHistoricalRatesToStream.ts
+import debug from "debug";
+import { Readable, Transform } from "stream";
+import { pipeline } from "stream/promises";
+var DEBUG_NAMESPACE = "dukascopy-node";
+function getHistoricalRatesToStream(config) {
+  const stream = new Readable({
+    objectMode: true,
+    async read(_size) {
+    }
+  });
+  try {
+    const { input, isValid, validationErrors } = validateConfigNode(config);
+    debug(`${DEBUG_NAMESPACE}:config`)("%O", {
+      input,
+      isValid,
+      validationErrors
+    });
+    if (!isValid) {
+      stream.emit("error", { validationErrors });
+      stream.push(null);
+      return stream;
+    }
+    const {
+      instrument,
+      dates: { from, to },
+      timeframe,
+      priceType,
+      volumes,
+      volumeUnits,
+      utcOffset,
+      ignoreFlats,
+      format,
+      batchSize,
+      pauseBetweenBatchesMs,
+      useCache,
+      cacheFolderPath,
+      retryCount,
+      pauseBetweenRetriesMs,
+      retryOnEmpty
+    } = input;
+    const [startDate, endDate] = normaliseDates({
+      instrument,
+      startDate: from,
+      endDate: to,
+      timeframe,
+      utcOffset
+    });
+    const [startDateMs, endDateMs] = [+startDate, +endDate];
+    const onItemFetch = process.env.DEBUG ? (url, buffer, isCacheHit) => {
+      debug(`${DEBUG_NAMESPACE}:fetcher`)(
+        url,
+        `| ${formatBytes(buffer.length)} |`,
+        `${isCacheHit ? "cache" : "network"}`
+      );
+    } : void 0;
+    const bufferFetcher = new BufferFetcher({
+      batchSize,
+      pauseBetweenBatchesMs,
+      cacheManager: useCache ? new CacheManager({ cacheFolderPath }) : void 0,
+      retryCount,
+      pauseBetweenRetriesMs,
+      onItemFetch,
+      retryOnEmpty
+    });
+    const bodyHeaders = timeframe === "tick" /* tick */ ? tickHeaders : candleHeaders;
+    let firstLine = true;
+    let urlsProcessed = 0;
+    const urlsforFetchingData = generateUrls({
+      instrument: input.instrument,
+      timeframe: input.timeframe,
+      priceType: input.priceType,
+      startDate,
+      endDate
+    });
+    const promiseProcessedData = urlsforFetchingData.map((url) => {
+      return new Promise((resolve, reject) => {
+        bufferFetcher.fetch([url]).then((bufferObjects) => {
+          try {
+            const processedData = processData({
+              instrument: input.instrument,
+              requestedTimeframe: input.timeframe,
+              bufferObjects,
+              priceType: input.priceType,
+              volumes: input.volumes,
+              volumeUnits: input.volumeUnits,
+              ignoreFlats: input.ignoreFlats
+            });
+            resolve(processedData);
+          } catch (err) {
+            reject(err);
+          }
+        }).catch((err) => {
+          reject(err);
+        });
+      });
+    });
+    pipeline(
+      Readable.from(promiseProcessedData),
+      new Transform({
+        objectMode: true,
+        transform: async (processedDataPr, _, callback) => {
+          const processedData = await processedDataPr;
+          try {
+            processedData.forEach((item) => {
+              const [timestamp] = item;
+              if (timestamp && timestamp >= startDateMs && timestamp < endDateMs) {
+                if (input.format === "array" /* array */) {
+                  stream.push(item);
+                } else if (format === "json" /* json */) {
+                  const data = item.reduce((all, item2, i) => {
+                    const name = bodyHeaders[i];
+                    all[name] = item2;
+                    return all;
+                  }, {});
+                  stream.push(data);
+                } else if (format === "csv" /* csv */) {
+                  if (firstLine) {
+                    const csvHeaders = bodyHeaders.join(",");
+                    stream.push(csvHeaders);
+                    firstLine = false;
+                  }
+                  stream.push(`
+${item.join(",")}`);
+                }
+              }
+            });
+            if (++urlsProcessed === urlsforFetchingData.length) {
+              stream.push(null);
+            }
+            callback();
+          } catch (err) {
+            callback(err);
+          }
+        }
+      })
+    ).catch((err) => {
+      stream.emit("error", err);
+      stream.push(null);
+    });
+  } catch (err) {
+    stream.emit("error", err);
+    stream.push(null);
+  }
+  return stream;
+}
+
+export {
+  getHistoricalRatesToStream
+};

package/dist/esm/cli/index.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import {
   version
-} from "../chunk-DWZJECKF.js";
+} from "../chunk-7MFC3GUP.js";
 import {
   BufferFetcher,
   CacheManager,

package/dist/esm/index.example.nostream.js

@@ -0,0 +1,38 @@
+import {
+  getHistoricalRates
+} from "./chunk-MTHVPJMW.js";
+import "./chunk-7MFC3GUP.js";
+import "./chunk-476J4KQE.js";
+import "./chunk-D3M3SVMW.js";
+
+// src/index.example.nostream.ts
+var printMemory = () => {
+  const formatMemoryUsage = (data) => `${Math.round(data / 1024 / 1024 * 100) / 100} MB`;
+  const memoryData = process.memoryUsage();
+  const memoryUsage = {
+    rss: `${formatMemoryUsage(memoryData.rss)} -> Resident Set Size - total memory allocated for the process execution`,
+    heapTotal: `${formatMemoryUsage(memoryData.heapTotal)} -> total size of the allocated heap`,
+    heapUsed: `${formatMemoryUsage(memoryData.heapUsed)} -> actual memory used during the execution`,
+    external: `${formatMemoryUsage(memoryData.external)} -> V8 external memory`
+  };
+  console.log(memoryUsage);
+};
+(async () => {
+  try {
+    printMemory();
+    const data = await getHistoricalRates({
+      instrument: "eurusd",
+      dates: {
+        from: /* @__PURE__ */ new Date("2021-02-01"),
+        to: /* @__PURE__ */ new Date("2021-06-03")
+      },
+      timeframe: "m1",
+      format: "csv"
+    });
+    process.stdout.write(data);
+    console.log();
+    printMemory();
+  } catch (error) {
+    console.log("error", error);
+  }
+})();

package/dist/esm/index.example.stream.js

@@ -0,0 +1,41 @@
+import {
+  getHistoricalRatesToStream
+} from "./chunk-NBUGCZCI.js";
+import "./chunk-476J4KQE.js";
+import "./chunk-D3M3SVMW.js";
+
+// src/index.example.stream.ts
+var printMemory = () => {
+  const formatMemoryUsage = (data) => `${Math.round(data / 1024 / 1024 * 100) / 100} MB`;
+  const memoryData = process.memoryUsage();
+  const memoryUsage = {
+    rss: `${formatMemoryUsage(memoryData.rss)} -> Resident Set Size - total memory allocated for the process execution`,
+    heapTotal: `${formatMemoryUsage(memoryData.heapTotal)} -> total size of the allocated heap`,
+    heapUsed: `${formatMemoryUsage(memoryData.heapUsed)} -> actual memory used during the execution`,
+    external: `${formatMemoryUsage(memoryData.external)} -> V8 external memory`
+  };
+  console.log(memoryUsage);
+};
+(async () => {
+  try {
+    printMemory();
+    const data = await getHistoricalRatesToStream({
+      instrument: "eurusd",
+      dates: {
+        from: /* @__PURE__ */ new Date("2021-02-01"),
+        to: /* @__PURE__ */ new Date("2021-06-03")
+      },
+      timeframe: "m1",
+      format: "csv"
+    });
+    data.on("data", (chunk) => {
+      process.stdout.write(chunk);
+    });
+    data.on("end", () => {
+      console.log();
+      printMemory();
+    });
+  } catch (error) {
+    console.log("error", error);
+  }
+})();

package/dist/esm/index.js

@@ -1,10 +1,14 @@
 import {
-  version
-} from "./chunk-DWZJECKF.js";
-import {
-  formatOutput,
   getHistoricalRatesToStream
-} from "./chunk-KWYYNULA.js";
+} from "./chunk-NBUGCZCI.js";
+import {
+  getHistoricRates,
+  getHistoricalRates
+} from "./chunk-MTHVPJMW.js";
+import "./chunk-7MFC3GUP.js";
+import {
+  formatOutput
+} from "./chunk-476J4KQE.js";
 import {
   BufferFetcher,
   CacheManager,
@@ -14,7 +18,6 @@ import {
   Timeframe,
   URL_ROOT,
   defaultConfig,
-  formatBytes,
   generateUrls,
   instrumentMetaData,
   normaliseDates,
@@ -27,99 +30,6 @@ import {
 // src/index.ts
 import { RuleDate, RuleBoolean, RuleNumber, RuleString, RuleObject } from "fastest-validator";
 
-// src/getHistoricalRates.ts
-import debug from "debug";
-import os from "os";
-var DEBUG_NAMESPACE = "dukascopy-node";
-async function getHistoricalRates(config) {
-  debug(`${DEBUG_NAMESPACE}:version`)(version);
-  debug(`${DEBUG_NAMESPACE}:nodejs`)(process.version);
-  debug(`${DEBUG_NAMESPACE}:os`)(`${os.type()}, ${os.release()} (${os.platform()})`);
-  const { input, isValid, validationErrors } = validateConfigNode(config);
-  debug(`${DEBUG_NAMESPACE}:config`)("%O", {
-    input,
-    isValid,
-    validationErrors
-  });
-  if (!isValid) {
-    throw { validationErrors };
-  }
-  const {
-    instrument,
-    dates: { from, to },
-    timeframe,
-    priceType,
-    volumes,
-    volumeUnits,
-    utcOffset,
-    ignoreFlats,
-    format,
-    batchSize,
-    pauseBetweenBatchesMs,
-    useCache,
-    cacheFolderPath,
-    retryCount,
-    pauseBetweenRetriesMs,
-    retryOnEmpty
-  } = input;
-  const [startDate, endDate] = normaliseDates({
-    instrument,
-    startDate: from,
-    endDate: to,
-    timeframe,
-    utcOffset
-  });
-  const urls = generateUrls({
-    instrument,
-    timeframe,
-    priceType,
-    startDate,
-    endDate
-  });
-  debug(`${DEBUG_NAMESPACE}:urls`)(`Generated ${urls.length} urls`);
-  debug(`${DEBUG_NAMESPACE}:urls`)(`%O`, urls);
-  const onItemFetch = process.env.DEBUG ? (url, buffer, isCacheHit) => {
-    debug(`${DEBUG_NAMESPACE}:fetcher`)(
-      url,
-      `| ${formatBytes(buffer.length)} |`,
-      `${isCacheHit ? "cache" : "network"}`
-    );
-  } : void 0;
-  const bufferFetcher = new BufferFetcher({
-    batchSize,
-    pauseBetweenBatchesMs,
-    cacheManager: useCache ? new CacheManager({ cacheFolderPath }) : void 0,
-    retryCount,
-    pauseBetweenRetriesMs,
-    onItemFetch,
-    retryOnEmpty
-  });
-  const bufferredData = await bufferFetcher.fetch(urls);
-  const processedData = processData({
-    instrument,
-    requestedTimeframe: timeframe,
-    bufferObjects: bufferredData,
-    priceType,
-    volumes,
-    volumeUnits,
-    ignoreFlats
-  });
-  const [startDateMs, endDateMs] = [+startDate, +endDate];
-  const filteredData = processedData.filter(
-    ([timestamp]) => timestamp && timestamp >= startDateMs && timestamp < endDateMs
-  );
-  debug(`${DEBUG_NAMESPACE}:data`)(
-    `Generated ${filteredData.length} ${timeframe === "tick" /* tick */ ? "ticks" : "OHLC candles"}`
-  );
-  const formattedData = formatOutput({
-    processedData: filteredData,
-    format,
-    timeframe
-  });
-  return formattedData;
-}
-var getHistoricRates = getHistoricalRates;
-
 // src/getCurrentRates.ts
 import fetch from "node-fetch";
 var timeframeMap = {

package/dist/index.d.mts

@@ -6798,6 +6798,6 @@ declare function getCurrentRates(config: CurrentRatesConfigJsonItem): Promise<Js
 declare function getCurrentRates(config: CurrentRatesConfigJsonTickItem): Promise<JsonItemTick[]>;
 declare function getCurrentRates(config: CurrentRatesConfigCsv): Promise<string>;
 
-declare function getHistoricalRatesToStream(config: Config): Promise<Readable>;
+declare function getHistoricalRatesToStream(config: Config): Readable;
 
 export { type ArrayItem, type ArrayTickItem, BufferFetcher, CacheManager, type Config, type ConfigValidationError, type CurrentRatesConfig, type CurrentRatesConfigArrayItem, type CurrentRatesConfigArrayTickItem, type CurrentRatesConfigCsv, type CurrentRatesConfigJsonItem, type CurrentRatesConfigJsonTickItem, type DefaultConfig, Format, type FormatType, type InputSchema, Instrument, instrumentMetaData as InstrumentMetaData, type InstrumentType, type JsonItem, type JsonItemTick, type Output, Price, type PriceType, type SanitizedConfig, Timeframe, type TimeframeType, URL_ROOT, type ValidateConfigOutput, defaultConfig, formatOutput, generateUrls, getCurrentRates, getHistoricRates, getHistoricalRates, getHistoricalRatesToStream, normaliseDates, processData, schema, validateConfig, validateConfigNode };

package/dist/index.d.ts

@@ -6798,6 +6798,6 @@ declare function getCurrentRates(config: CurrentRatesConfigJsonItem): Promise<Js
 declare function getCurrentRates(config: CurrentRatesConfigJsonTickItem): Promise<JsonItemTick[]>;
 declare function getCurrentRates(config: CurrentRatesConfigCsv): Promise<string>;
 
-declare function getHistoricalRatesToStream(config: Config): Promise<Readable>;
+declare function getHistoricalRatesToStream(config: Config): Readable;
 
 export { type ArrayItem, type ArrayTickItem, BufferFetcher, CacheManager, type Config, type ConfigValidationError, type CurrentRatesConfig, type CurrentRatesConfigArrayItem, type CurrentRatesConfigArrayTickItem, type CurrentRatesConfigCsv, type CurrentRatesConfigJsonItem, type CurrentRatesConfigJsonTickItem, type DefaultConfig, Format, type FormatType, type InputSchema, Instrument, instrumentMetaData as InstrumentMetaData, type InstrumentType, type JsonItem, type JsonItemTick, type Output, Price, type PriceType, type SanitizedConfig, Timeframe, type TimeframeType, URL_ROOT, type ValidateConfigOutput, defaultConfig, formatOutput, generateUrls, getCurrentRates, getHistoricRates, getHistoricalRates, getHistoricalRatesToStream, normaliseDates, processData, schema, validateConfig, validateConfigNode };