node-es-transformer 1.0.0-alpha11 → 1.0.0-alpha12

package/dist/node-es-transformer.cjs.js

@@ -120,6 +120,8 @@ var EventEmitter = require('events');
 
 var queueEmitter = new EventEmitter();
 
+var parallelCalls = 1;
+
 // a simple helper queue to bulk index documents
 function indexQueueFactory(ref) {
   var client = ref.targetClient;
@@ -130,36 +132,54 @@ function indexQueueFactory(ref) {
 
   var buffer = [];
   var queue = [];
-  var ingesting = false;
+  var ingesting = 0;
+  var ingestTimes = [];
 
-  var ingest = async function (b) {
+  var ingest = function (b) {
     if (typeof b !== 'undefined') {
       queue.push(b);
       queueEmitter.emit('queue-size', queue.length);
     }
 
-    if (ingesting === false) {
+    if (ingestTimes.length > 5) { ingestTimes = ingestTimes.slice(-5); }
+
+    if (ingesting < parallelCalls) {
       var docs = queue.shift();
+
       queueEmitter.emit('queue-size', queue.length);
-      ingesting = true;
+      if (queue.length <= 5) {
+        queueEmitter.emit('resume');
+      }
+
+      ingesting += 1;
+
       if (verbose)
         { console.log(("bulk ingest docs: " + (docs.length / 2) + ", queue length: " + (queue.length))); }
 
-      try {
-        await client.bulk({ body: docs });
-        ingesting = false;
+      var start = Date.now();
+      client.bulk({ body: docs }).then(function () {
+        var end = Date.now();
+        var delta = end - start;
+        ingestTimes.push(delta);
+        ingesting -= 1;
+
+        var ingestTimesMovingAverage =
+          ingestTimes.length > 0 ? ingestTimes.reduce(function (p, c) { return p + c; }, 0) / ingestTimes.length : 0;
+        var ingestTimesMovingAverageSeconds = Math.floor(ingestTimesMovingAverage / 1000);
+
+        if (ingestTimes.length > 0 && ingestTimesMovingAverageSeconds < 30 && parallelCalls < 10) {
+          parallelCalls += 1;
+        } else if (
+          ingestTimes.length > 0 &&
+          ingestTimesMovingAverageSeconds >= 30 &&
+          parallelCalls > 1
+        ) {
+          parallelCalls -= 1;
+        }
         if (queue.length > 0) {
           ingest();
         }
-      } catch (err) {
-        console.log('bulk index error', err);
-      }
-    }
-
-    // console.log(`ingest: queue.length ${queue.length}`);
-    if (queue.length === 0) {
-      queueEmitter.emit('queue-size', 0);
-      queueEmitter.emit('resume');
+      });
     }
   };
 
@@ -190,7 +210,7 @@ function indexQueueFactory(ref) {
   };
 }
 
-var MAX_QUEUE_SIZE = 5;
+var MAX_QUEUE_SIZE = 15;
 
 // create a new progress bar instance and use shades_classic theme
 var progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);

package/dist/node-es-transformer.esm.js

@@ -116,6 +116,8 @@ var EventEmitter = require('events');
 
 var queueEmitter = new EventEmitter();
 
+var parallelCalls = 1;
+
 // a simple helper queue to bulk index documents
 function indexQueueFactory(ref) {
   var client = ref.targetClient;
@@ -126,36 +128,54 @@ function indexQueueFactory(ref) {
 
   var buffer = [];
   var queue = [];
-  var ingesting = false;
+  var ingesting = 0;
+  var ingestTimes = [];
 
-  var ingest = async function (b) {
+  var ingest = function (b) {
     if (typeof b !== 'undefined') {
       queue.push(b);
       queueEmitter.emit('queue-size', queue.length);
     }
 
-    if (ingesting === false) {
+    if (ingestTimes.length > 5) { ingestTimes = ingestTimes.slice(-5); }
+
+    if (ingesting < parallelCalls) {
       var docs = queue.shift();
+
       queueEmitter.emit('queue-size', queue.length);
-      ingesting = true;
+      if (queue.length <= 5) {
+        queueEmitter.emit('resume');
+      }
+
+      ingesting += 1;
+
       if (verbose)
         { console.log(("bulk ingest docs: " + (docs.length / 2) + ", queue length: " + (queue.length))); }
 
-      try {
-        await client.bulk({ body: docs });
-        ingesting = false;
+      var start = Date.now();
+      client.bulk({ body: docs }).then(function () {
+        var end = Date.now();
+        var delta = end - start;
+        ingestTimes.push(delta);
+        ingesting -= 1;
+
+        var ingestTimesMovingAverage =
+          ingestTimes.length > 0 ? ingestTimes.reduce(function (p, c) { return p + c; }, 0) / ingestTimes.length : 0;
+        var ingestTimesMovingAverageSeconds = Math.floor(ingestTimesMovingAverage / 1000);
+
+        if (ingestTimes.length > 0 && ingestTimesMovingAverageSeconds < 30 && parallelCalls < 10) {
+          parallelCalls += 1;
+        } else if (
+          ingestTimes.length > 0 &&
+          ingestTimesMovingAverageSeconds >= 30 &&
+          parallelCalls > 1
+        ) {
+          parallelCalls -= 1;
+        }
         if (queue.length > 0) {
           ingest();
         }
-      } catch (err) {
-        console.log('bulk index error', err);
-      }
-    }
-
-    // console.log(`ingest: queue.length ${queue.length}`);
-    if (queue.length === 0) {
-      queueEmitter.emit('queue-size', 0);
-      queueEmitter.emit('resume');
+      });
     }
   };
 
@@ -186,7 +206,7 @@ function indexQueueFactory(ref) {
   };
 }
 
-var MAX_QUEUE_SIZE = 5;
+var MAX_QUEUE_SIZE = 15;
 
 // create a new progress bar instance and use shades_classic theme
 var progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);

package/package.json

@@ -14,7 +14,7 @@
   "license": "Apache-2.0",
   "author": "Walter Rafelsberger <walter@rafelsberger.at>",
   "contributors": [],
-  "version": "1.0.0-alpha11",
+  "version": "1.0.0-alpha12",
   "main": "dist/node-es-transformer.cjs.js",
   "module": "dist/node-es-transformer.esm.js",
   "dependencies": {

package/CHANGELOG.md

@@ -1,32 +0,0 @@
-# Changelog
-
-All notable changes to this project will be documented in this file. See [commit-and-tag-version](https://github.com/absolute-version/commit-and-tag-version) for commit guidelines.
-
-## [1.0.0-alpha11](https://github.com/walterra/node-es-transformer/compare/v1.0.0-alpha10...v1.0.0-alpha11) (2023-10-12)
-
-### Features
-
-- new option 'indexMappingTotalFieldsLimit' ([92edad1](https://github.com/walterra/node-es-transformer/commit/92edad18da7186d3881fc181e6e88b7929bed2d4))
-
-### Bug Fixes
-
-- fixes bufferSize to be applied to index reader too ([ffc3749](https://github.com/walterra/node-es-transformer/commit/ffc3749e296cd39f39924571c197986addc756ff))
-
-## [`v1.0.0-alpha10`](https://github.com/walterra/node-es-transformer/releases/tag/v1.0.0-alpha10)
-
-- New option `mappingsOverride` (0b951e1).
-- New option `query` (45f91db).
-
-## [`v1.0.0-alpha9`](https://github.com/walterra/node-es-transformer/releases/tag/v1.0.0-alpha9)
-
-- Source and target configs are now expected to be passed in as complete client configs instead of individual parameters (5e6d0c7).
-
-## [`v1.0.0-alpha8`](https://github.com/walterra/node-es-transformer/releases/tag/v1.0.0-alpha8)
-
-- Exposes events and introduces `finish` event (a3e5810).
-- Drop support for `_type` from `6.x` indices (3a26a84).
-
-## [`v1.0.0-alpha7`](https://github.com/walterra/node-es-transformer/releases/tag/v1.0.0-alpha7)
-
-- This version locks down `event-stream` to version `3.3.4` because of the security issue described here: https://github.com/dominictarr/event-stream/issues/116
-- Last version to support `_type` from `6.x` indices.