dukascopy-node-plus 1.1.0 → 1.1.2

package/dist/index.example.stream.d.mts

@@ -0,0 +1,2 @@
+
+export {  }

package/dist/index.example.stream.d.ts

@@ -0,0 +1,2 @@
+
+export {  }

package/dist/{index.example.js → index.example.stream.js}

@@ -13706,158 +13706,178 @@ var import_debug = __toESM(require("debug"));
 var import_stream = require("stream");
 var import_promises = require("stream/promises");
 var DEBUG_NAMESPACE = "dukascopy-node";
-async function getHistoricalRatesToStream(config) {
+function getHistoricalRatesToStream(config) {
   const stream = new import_stream.Readable({
     objectMode: true,
     async read(_size) {
-      try {
-        const { input, isValid, validationErrors } = validateConfigNode(config);
-        (0, import_debug.default)(`${DEBUG_NAMESPACE}:config`)("%O", {
-          input,
-          isValid,
-          validationErrors
-        });
-        if (!isValid) {
-          this.emit("error", { validationErrors });
-          this.push(null);
-          return;
-        }
-        const {
-          instrument,
-          dates: { from, to },
-          timeframe,
-          priceType,
-          volumes,
-          volumeUnits,
-          utcOffset,
-          ignoreFlats,
-          format,
-          batchSize,
-          pauseBetweenBatchesMs,
-          useCache,
-          cacheFolderPath,
-          retryCount,
-          pauseBetweenRetriesMs,
-          retryOnEmpty
-        } = input;
-        const [startDate, endDate] = normaliseDates({
-          instrument,
-          startDate: from,
-          endDate: to,
-          timeframe,
-          utcOffset
-        });
-        const [startDateMs, endDateMs] = [+startDate, +endDate];
-        const onItemFetch = process.env.DEBUG ? (url, buffer, isCacheHit) => {
-          (0, import_debug.default)(`${DEBUG_NAMESPACE}:fetcher`)(
-            url,
-            `| ${formatBytes(buffer.length)} |`,
-            `${isCacheHit ? "cache" : "network"}`
-          );
-        } : void 0;
-        const bufferFetcher = new BufferFetcher({
-          batchSize,
-          pauseBetweenBatchesMs,
-          cacheManager: useCache ? new CacheManager({ cacheFolderPath }) : void 0,
-          retryCount,
-          pauseBetweenRetriesMs,
-          onItemFetch,
-          retryOnEmpty
-        });
-        let firstLine = true;
-        let urlsProcessed = 0;
-        const urlsforFetchingData = generateUrls({
-          instrument: input.instrument,
-          timeframe: input.timeframe,
-          priceType: input.priceType,
-          startDate,
-          endDate
+    }
+  });
+  try {
+    const { input, isValid, validationErrors } = validateConfigNode(config);
+    (0, import_debug.default)(`${DEBUG_NAMESPACE}:config`)("%O", {
+      input,
+      isValid,
+      validationErrors
+    });
+    if (!isValid) {
+      stream.emit("error", { validationErrors });
+      stream.push(null);
+      return stream;
+    }
+    const {
+      instrument,
+      dates: { from, to },
+      timeframe,
+      priceType,
+      volumes,
+      volumeUnits,
+      utcOffset,
+      ignoreFlats,
+      format,
+      batchSize,
+      pauseBetweenBatchesMs,
+      useCache,
+      cacheFolderPath,
+      retryCount,
+      pauseBetweenRetriesMs,
+      retryOnEmpty
+    } = input;
+    const [startDate, endDate] = normaliseDates({
+      instrument,
+      startDate: from,
+      endDate: to,
+      timeframe,
+      utcOffset
+    });
+    const [startDateMs, endDateMs] = [+startDate, +endDate];
+    const onItemFetch = process.env.DEBUG ? (url, buffer, isCacheHit) => {
+      (0, import_debug.default)(`${DEBUG_NAMESPACE}:fetcher`)(
+        url,
+        `| ${formatBytes(buffer.length)} |`,
+        `${isCacheHit ? "cache" : "network"}`
+      );
+    } : void 0;
+    const bufferFetcher = new BufferFetcher({
+      batchSize,
+      pauseBetweenBatchesMs,
+      cacheManager: useCache ? new CacheManager({ cacheFolderPath }) : void 0,
+      retryCount,
+      pauseBetweenRetriesMs,
+      onItemFetch,
+      retryOnEmpty
+    });
+    const bodyHeaders = timeframe === "tick" /* tick */ ? tickHeaders : candleHeaders;
+    let firstLine = true;
+    let urlsProcessed = 0;
+    const urlsforFetchingData = generateUrls({
+      instrument: input.instrument,
+      timeframe: input.timeframe,
+      priceType: input.priceType,
+      startDate,
+      endDate
+    });
+    const promiseProcessedData = urlsforFetchingData.map((url) => {
+      return new Promise((resolve2, reject) => {
+        bufferFetcher.fetch([url]).then((bufferObjects) => {
+          try {
+            const processedData = processData({
+              instrument: input.instrument,
+              requestedTimeframe: input.timeframe,
+              bufferObjects,
+              priceType: input.priceType,
+              volumes: input.volumes,
+              volumeUnits: input.volumeUnits,
+              ignoreFlats: input.ignoreFlats
+            });
+            resolve2(processedData);
+          } catch (err) {
+            reject(err);
+          }
+        }).catch((err) => {
+          reject(err);
         });
-        await (0, import_promises.pipeline)(
-          import_stream.Readable.from(urlsforFetchingData),
-          new import_stream.Transform({
-            objectMode: true,
-            transform: async (url, _, callback) => {
-              const bufferObject = {
-                url,
-                buffer: await bufferFetcher.fetchBuffer(url),
-                isCacheHit: useCache
-              };
-              try {
-                const processedData = processData({
-                  instrument: input.instrument,
-                  requestedTimeframe: input.timeframe,
-                  bufferObjects: [bufferObject],
-                  priceType: input.priceType,
-                  volumes: input.volumes,
-                  volumeUnits: input.volumeUnits,
-                  ignoreFlats: input.ignoreFlats
-                });
-                processedData.forEach((item) => {
-                  const [timestamp] = item;
-                  if (timestamp && timestamp >= startDateMs && timestamp < endDateMs) {
-                    if (input.format === "array" /* array */) {
-                      this.push(item);
-                    } else {
-                      const bodyHeaders = timeframe === "tick" /* tick */ ? tickHeaders : candleHeaders;
-                      if (format === "json" /* json */) {
-                        const data = item.reduce((all, item2, i) => {
-                          const name = bodyHeaders[i];
-                          all[name] = item2;
-                          return all;
-                        }, {});
-                        this.push(data);
-                      } else if (format === "csv" /* csv */) {
-                        if (firstLine) {
-                          const csvHeaders = bodyHeaders.join(",");
-                          this.push(csvHeaders);
-                          firstLine = false;
-                        }
-                        this.push(item.join(","));
-                      }
-                    }
+      });
+    });
+    (0, import_promises.pipeline)(
+      import_stream.Readable.from(promiseProcessedData),
+      new import_stream.Transform({
+        objectMode: true,
+        transform: async (processedDataPr, _, callback) => {
+          const processedData = await processedDataPr;
+          try {
+            processedData.forEach((item) => {
+              const [timestamp] = item;
+              if (timestamp && timestamp >= startDateMs && timestamp < endDateMs) {
+                if (input.format === "array" /* array */) {
+                  stream.push(item);
+                } else if (format === "json" /* json */) {
+                  const data = item.reduce((all, item2, i) => {
+                    const name = bodyHeaders[i];
+                    all[name] = item2;
+                    return all;
+                  }, {});
+                  stream.push(data);
+                } else if (format === "csv" /* csv */) {
+                  if (firstLine) {
+                    const csvHeaders = bodyHeaders.join(",");
+                    stream.push(csvHeaders);
+                    firstLine = false;
                   }
-                });
-                callback();
-              } catch (err) {
-                callback(err);
+                  stream.push(`
+${item.join(",")}`);
+                }
               }
+            });
+            if (++urlsProcessed === urlsforFetchingData.length) {
+              stream.push(null);
             }
-          })
-        ).then(() => {
-          if (++urlsProcessed === urlsforFetchingData.length) {
-            this.push(null);
+            callback();
+          } catch (err) {
+            callback(err);
           }
-        }).catch((err) => {
-          this.emit("error", err);
-          this.push(null);
-        });
-      } catch (err) {
-        this.emit("error", err);
-        this.push(null);
-      }
-    }
-  });
+        }
+      })
+    ).catch((err) => {
+      stream.emit("error", err);
+      stream.push(null);
+    });
+  } catch (err) {
+    stream.emit("error", err);
+    stream.push(null);
+  }
   return stream;
 }
 
-// src/index.example.ts
+// src/index.example.stream.ts
+var printMemory = () => {
+  const formatMemoryUsage = (data) => `${Math.round(data / 1024 / 1024 * 100) / 100} MB`;
+  const memoryData = process.memoryUsage();
+  const memoryUsage = {
+    rss: `${formatMemoryUsage(memoryData.rss)} -> Resident Set Size - total memory allocated for the process execution`,
+    heapTotal: `${formatMemoryUsage(memoryData.heapTotal)} -> total size of the allocated heap`,
+    heapUsed: `${formatMemoryUsage(memoryData.heapUsed)} -> actual memory used during the execution`,
+    external: `${formatMemoryUsage(memoryData.external)} -> V8 external memory`
+  };
+  console.log(memoryUsage);
+};
 (async () => {
   try {
+    printMemory();
     const data = await getHistoricalRatesToStream({
       instrument: "eurusd",
       dates: {
         from: /* @__PURE__ */ new Date("2021-02-01"),
-        to: /* @__PURE__ */ new Date("2021-03-01")
+        to: /* @__PURE__ */ new Date("2021-06-03")
       },
-      timeframe: "d1",
+      timeframe: "m1",
       format: "csv"
     });
     data.on("data", (chunk) => {
-      console.log(chunk);
+      process.stdout.write(chunk);
     });
     data.on("end", () => {
-      console.log("end");
+      console.log();
+      printMemory();
     });
   } catch (error) {
     console.log("error", error);

package/dist/index.js

@@ -13767,7 +13767,7 @@ function formatBytes(bytes, decimals = 2) {
 }
 
 // package.json
-var version = "1.1.0";
+var version = "1.1.2";
 
 // src/getHistoricalRates.ts
 var import_debug = __toESM(require("debug"));
@@ -13990,138 +13990,145 @@ var import_debug2 = __toESM(require("debug"));
 var import_stream = require("stream");
 var import_promises = require("stream/promises");
 var DEBUG_NAMESPACE2 = "dukascopy-node";
-async function getHistoricalRatesToStream(config) {
+function getHistoricalRatesToStream(config) {
   const stream = new import_stream.Readable({
     objectMode: true,
     async read(_size) {
-      try {
-        const { input, isValid, validationErrors } = validateConfigNode(config);
-        (0, import_debug2.default)(`${DEBUG_NAMESPACE2}:config`)("%O", {
-          input,
-          isValid,
-          validationErrors
-        });
-        if (!isValid) {
-          this.emit("error", { validationErrors });
-          this.push(null);
-          return;
-        }
-        const {
-          instrument,
-          dates: { from, to },
-          timeframe,
-          priceType,
-          volumes,
-          volumeUnits,
-          utcOffset,
-          ignoreFlats,
-          format,
-          batchSize,
-          pauseBetweenBatchesMs,
-          useCache,
-          cacheFolderPath,
-          retryCount,
-          pauseBetweenRetriesMs,
-          retryOnEmpty
-        } = input;
-        const [startDate, endDate] = normaliseDates({
-          instrument,
-          startDate: from,
-          endDate: to,
-          timeframe,
-          utcOffset
-        });
-        const [startDateMs, endDateMs] = [+startDate, +endDate];
-        const onItemFetch = process.env.DEBUG ? (url, buffer, isCacheHit) => {
-          (0, import_debug2.default)(`${DEBUG_NAMESPACE2}:fetcher`)(
-            url,
-            `| ${formatBytes(buffer.length)} |`,
-            `${isCacheHit ? "cache" : "network"}`
-          );
-        } : void 0;
-        const bufferFetcher = new BufferFetcher({
-          batchSize,
-          pauseBetweenBatchesMs,
-          cacheManager: useCache ? new CacheManager({ cacheFolderPath }) : void 0,
-          retryCount,
-          pauseBetweenRetriesMs,
-          onItemFetch,
-          retryOnEmpty
-        });
-        let firstLine = true;
-        let urlsProcessed = 0;
-        const urlsforFetchingData = generateUrls({
-          instrument: input.instrument,
-          timeframe: input.timeframe,
-          priceType: input.priceType,
-          startDate,
-          endDate
+    }
+  });
+  try {
+    const { input, isValid, validationErrors } = validateConfigNode(config);
+    (0, import_debug2.default)(`${DEBUG_NAMESPACE2}:config`)("%O", {
+      input,
+      isValid,
+      validationErrors
+    });
+    if (!isValid) {
+      stream.emit("error", { validationErrors });
+      stream.push(null);
+      return stream;
+    }
+    const {
+      instrument,
+      dates: { from, to },
+      timeframe,
+      priceType,
+      volumes,
+      volumeUnits,
+      utcOffset,
+      ignoreFlats,
+      format,
+      batchSize,
+      pauseBetweenBatchesMs,
+      useCache,
+      cacheFolderPath,
+      retryCount,
+      pauseBetweenRetriesMs,
+      retryOnEmpty
+    } = input;
+    const [startDate, endDate] = normaliseDates({
+      instrument,
+      startDate: from,
+      endDate: to,
+      timeframe,
+      utcOffset
+    });
+    const [startDateMs, endDateMs] = [+startDate, +endDate];
+    const onItemFetch = process.env.DEBUG ? (url, buffer, isCacheHit) => {
+      (0, import_debug2.default)(`${DEBUG_NAMESPACE2}:fetcher`)(
+        url,
+        `| ${formatBytes(buffer.length)} |`,
+        `${isCacheHit ? "cache" : "network"}`
+      );
+    } : void 0;
+    const bufferFetcher = new BufferFetcher({
+      batchSize,
+      pauseBetweenBatchesMs,
+      cacheManager: useCache ? new CacheManager({ cacheFolderPath }) : void 0,
+      retryCount,
+      pauseBetweenRetriesMs,
+      onItemFetch,
+      retryOnEmpty
+    });
+    const bodyHeaders = timeframe === "tick" /* tick */ ? tickHeaders : candleHeaders;
+    let firstLine = true;
+    let urlsProcessed = 0;
+    const urlsforFetchingData = generateUrls({
+      instrument: input.instrument,
+      timeframe: input.timeframe,
+      priceType: input.priceType,
+      startDate,
+      endDate
+    });
+    const promiseProcessedData = urlsforFetchingData.map((url) => {
+      return new Promise((resolve2, reject) => {
+        bufferFetcher.fetch([url]).then((bufferObjects) => {
+          try {
+            const processedData = processData({
+              instrument: input.instrument,
+              requestedTimeframe: input.timeframe,
+              bufferObjects,
+              priceType: input.priceType,
+              volumes: input.volumes,
+              volumeUnits: input.volumeUnits,
+              ignoreFlats: input.ignoreFlats
+            });
+            resolve2(processedData);
+          } catch (err) {
+            reject(err);
+          }
+        }).catch((err) => {
+          reject(err);
         });
-        await (0, import_promises.pipeline)(
-          import_stream.Readable.from(urlsforFetchingData),
-          new import_stream.Transform({
-            objectMode: true,
-            transform: async (url, _, callback) => {
-              const bufferObject = {
-                url,
-                buffer: await bufferFetcher.fetchBuffer(url),
-                isCacheHit: useCache
-              };
-              try {
-                const processedData = processData({
-                  instrument: input.instrument,
-                  requestedTimeframe: input.timeframe,
-                  bufferObjects: [bufferObject],
-                  priceType: input.priceType,
-                  volumes: input.volumes,
-                  volumeUnits: input.volumeUnits,
-                  ignoreFlats: input.ignoreFlats
-                });
-                processedData.forEach((item) => {
-                  const [timestamp] = item;
-                  if (timestamp && timestamp >= startDateMs && timestamp < endDateMs) {
-                    if (input.format === "array" /* array */) {
-                      this.push(item);
-                    } else {
-                      const bodyHeaders = timeframe === "tick" /* tick */ ? tickHeaders : candleHeaders;
-                      if (format === "json" /* json */) {
-                        const data = item.reduce((all, item2, i) => {
-                          const name = bodyHeaders[i];
-                          all[name] = item2;
-                          return all;
-                        }, {});
-                        this.push(data);
-                      } else if (format === "csv" /* csv */) {
-                        if (firstLine) {
-                          const csvHeaders = bodyHeaders.join(",");
-                          this.push(csvHeaders);
-                          firstLine = false;
-                        }
-                        this.push(item.join(","));
-                      }
-                    }
+      });
+    });
+    (0, import_promises.pipeline)(
+      import_stream.Readable.from(promiseProcessedData),
+      new import_stream.Transform({
+        objectMode: true,
+        transform: async (processedDataPr, _, callback) => {
+          const processedData = await processedDataPr;
+          try {
+            processedData.forEach((item) => {
+              const [timestamp] = item;
+              if (timestamp && timestamp >= startDateMs && timestamp < endDateMs) {
+                if (input.format === "array" /* array */) {
+                  stream.push(item);
+                } else if (format === "json" /* json */) {
+                  const data = item.reduce((all, item2, i) => {
+                    const name = bodyHeaders[i];
+                    all[name] = item2;
+                    return all;
+                  }, {});
+                  stream.push(data);
+                } else if (format === "csv" /* csv */) {
+                  if (firstLine) {
+                    const csvHeaders = bodyHeaders.join(",");
+                    stream.push(csvHeaders);
+                    firstLine = false;
                   }
-                });
-                callback();
-              } catch (err) {
-                callback(err);
+                  stream.push(`
+${item.join(",")}`);
+                }
               }
+            });
+            if (++urlsProcessed === urlsforFetchingData.length) {
+              stream.push(null);
             }
-          })
-        ).then(() => {
-          if (++urlsProcessed === urlsforFetchingData.length) {
-            this.push(null);
+            callback();
+          } catch (err) {
+            callback(err);
           }
-        }).catch((err) => {
-          this.emit("error", err);
-          this.push(null);
-        });
-      } catch (err) {
-        this.emit("error", err);
-        this.push(null);
-      }
-    }
-  });
+        }
+      })
+    ).catch((err) => {
+      stream.emit("error", err);
+      stream.push(null);
+    });
+  } catch (err) {
+    stream.emit("error", err);
+    stream.push(null);
+  }
   return stream;
 }
 // Annotate the CommonJS export names for ESM import in node:

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "dukascopy-node-plus",
-	"version": "1.1.0",
+	"version": "1.1.2",
 	"description": "Node.js library for downloading historical market tick data for for Crypto, Stocks, ETFs, CFDs, Forex",
 	"main": "dist/index.js",
 	"module": "dist/esm/index.js",
@@ -13,7 +13,7 @@
 		"dukascopy-node": "./dist/cli/index.js"
 	},
 	"scripts": {
-		"build": "rm -rf dist && tsup-node src/index.ts src/cli/index.ts src/index.example.ts --format esm,cjs --dts --legacy-output",
+		"build": "rm -rf dist && tsup-node src/index.ts src/cli/index.ts src/index.example.stream.ts src/index.example.nostream.ts --format esm,cjs --dts --legacy-output",
 		"test": "vitest run",
 		"coverage": "vitest run --coverage",
 		"format": "prettier --config .prettierrc 'src/**/*.ts' --write",