node-es-transformer 1.0.0-beta2 → 1.0.0-beta4

package/README.md

@@ -14,23 +14,12 @@ If you're looking for a nodejs based tool which allows you to ingest large CSV/J
 
 While I'd generally recommend using [Logstash](https://www.elastic.co/products/logstash), [filebeat](https://www.elastic.co/products/beats/filebeat), [Ingest Nodes](https://www.elastic.co/guide/en/elasticsearch/reference/master/ingest.html), [Elastic Agent](https://www.elastic.co/guide/en/fleet/current/fleet-overview.html) or [Elasticsearch Transforms](https://www.elastic.co/guide/en/elasticsearch/reference/current/transforms.html) for established use cases, this tool may be of help especially if you feel more at home in the JavaScript/nodejs universe and have use cases with customized ingestion and data transformation needs.
 
-**This is experimental code, use at your own risk. Nonetheless, I encourage you to give it a try so I can gather some feedback.**
-
-### So why is this still _alpha_?
-
-- The API is not quite final and might change from release to release.
-- The code needs some more safety measures to avoid some possible accidental data loss scenarios.
-- No test coverage yet.
-
----
-
-Now that we've talked about the caveats, let's have a look what you actually get with this tool:
-
 ## Features
 
 - Buffering/Streaming for both reading and indexing. Files are read using streaming and Elasticsearch ingestion is done using buffered bulk indexing. This is tailored towards ingestion of large files. Successfully tested so far with JSON and CSV files in the range of 20-30 GBytes. On a single machine running both `node-es-transformer` and Elasticsearch ingestion rates up to 20k documents/second were achieved (2,9 GHz Intel Core i7, 16GByte RAM, SSD), depending on document size.
 - Supports wildcards to ingest/transform a range of files in one go.
 - Supports fetching documents from existing indices using search/scroll. This allows you to reindex with custom data transformations just using JavaScript in the `transform` callback.
+- Supports ingesting docs based on a nodejs stream.
 - The `transform` callback gives you each source document, but you can split it up in multiple ones and return an array of documents. An example use case for this: Each source document is a Tweet and you want to transform that into an entity centric index based on Hashtags.
 
 ## Getting started
@@ -110,10 +99,12 @@ transformer({
 
 - `deleteIndex`: Setting to automatically delete an existing index, default is `false`.
 - `sourceClientConfig`/`targetClientConfig`: Optional Elasticsearch client options, defaults to `{ node: 'http://localhost:9200' }`.
-- `bufferSize`: The amount of documents inserted with each Elasticsearch bulk insert request, default is `1000`.
-- `fileName`: Source filename to ingest, supports wildcards. If this is set, `sourceIndexName` is not allowed.
+- `bufferSize`: The threshold to flush bulk index request in KBytes, defaults to `5120`.
+- `searchSize`: The amount of documents to be fetched with each search request when reindexing from another source index.
+- `fileName`: Source filename to ingest, supports wildcards. If this is set, `sourceIndexName` and `stream` are not allowed.
+- `stream`: Source nodejs stream to ingest. If this is set, `sourceIndexName` and `fileName` are not allowed.
 - `splitRegex`: Custom line split regex, defaults to `/\n/`.
-- `sourceIndexName`: The source Elasticsearch index to reindex from. If this is set, `fileName` is not allowed.
+- `sourceIndexName`: The source Elasticsearch index to reindex from. If this is set, `fileName` and `stream` are not allowed.
 - `targetIndexName`: The target Elasticsearch index where documents will be indexed.
 - `mappings`: Optional Elasticsearch document mappings. If not set and you're reindexing from another index, the mappings from the existing index will be used.
 - `mappingsOverride`: If you're reindexing and this is set to `true`, `mappings` will be applied on top of the source index's mappings. Defaults to `false`.
@@ -147,10 +138,10 @@ yarn
 
 ```bash
 # Download the docker image
-docker pull docker.elastic.co/elasticsearch/elasticsearch:8.10.4
+docker pull docker.elastic.co/elasticsearch/elasticsearch:8.17.0
 
 # Run the container
-docker run --name es01 --net elastic -p 9200:9200 -it -m 1GB -e "discovery.type=single-node" -e "xpack.security.enabled=false" docker.elastic.co/elasticsearch/elasticsearch:8.10.4
+docker run --name es01 --net elastic -p 9200:9200 -it -m 1GB -e "discovery.type=single-node" -e "xpack.security.enabled=false" docker.elastic.co/elasticsearch/elasticsearch:8.17.0
 ```
 
 To commit, use `cz`. To prepare a release, use e.g. `yarn release -- --release-as 1.0.0-beta2`.

package/dist/node-es-transformer.cjs.js

@@ -5,10 +5,20 @@ function _interopDefault (ex) { return (ex && (typeof ex === 'object') && 'defau
 var fs = _interopDefault(require('fs'));
 var es = _interopDefault(require('event-stream'));
 var glob = _interopDefault(require('glob'));
+var split = _interopDefault(require('split2'));
+var stream = require('stream');
 var cliProgress = _interopDefault(require('cli-progress'));
 var elasticsearch = _interopDefault(require('@elastic/elasticsearch'));
 
-var DEFAULT_BUFFER_SIZE = 1000;
+// In earlier versions this was used to set the number of docs to index in a
+// single bulk request. Since we switched to use the helpers.bulk() method from
+// the ES client, this now translates to the `flushBytes` option of the helper.
+// However, for kind of a backwards compability with the old values, this uses
+// KBytes instead of Bytes. It will be multiplied by 1024 in the index queue.
+var DEFAULT_BUFFER_SIZE = 5120;
+
+// The default number of docs to fetch in a single search request when reindexing.
+var DEFAULT_SEARCH_SIZE = 1000;
 
 function createMappingFactory(ref) {
   var sourceClient = ref.sourceClient;
@@ -19,6 +29,7 @@ function createMappingFactory(ref) {
   var mappingsOverride = ref.mappingsOverride;
   var indexMappingTotalFieldsLimit = ref.indexMappingTotalFieldsLimit;
   var verbose = ref.verbose;
+  var deleteIndex = ref.deleteIndex;
 
   return async function () {
     var targetMappings = mappingsOverride ? undefined : mappings;
@@ -28,7 +39,14 @@ function createMappingFactory(ref) {
         var mapping = await sourceClient.indices.getMapping({
           index: sourceIndexName,
         });
-        targetMappings = mapping[sourceIndexName].mappings;
+        if (mapping[sourceIndexName]) {
+          targetMappings = mapping[sourceIndexName].mappings;
+        } else {
+          var allMappings = Object.values(mapping);
+          if (allMappings.length > 0) {
+            targetMappings = Object.values(mapping)[0].mappings;
+          }
+        }
       } catch (err) {
         console.log('Error reading source mapping', err);
         return;
@@ -43,18 +61,28 @@ function createMappingFactory(ref) {
       }
 
       try {
-        var resp = await targetClient.indices.create({
-          index: targetIndexName,
-          body: Object.assign({}, {mappings: targetMappings},
-            (indexMappingTotalFieldsLimit !== undefined
-              ? {
-                  settings: {
-                    'index.mapping.total_fields.limit': indexMappingTotalFieldsLimit,
-                  },
-                }
-              : {})),
-        });
-        if (verbose) { console.log('Created target mapping', resp); }
+        var indexExists = await targetClient.indices.exists({ index: targetIndexName });
+
+        if (indexExists === true && deleteIndex === true) {
+          await targetClient.indices.delete({ index: targetIndexName });
+        }
+
+        if (indexExists === false || deleteIndex === true) {
+          var resp = await targetClient.indices.create({
+            index: targetIndexName,
+            body: Object.assign({}, {mappings: targetMappings},
+              (indexMappingTotalFieldsLimit !== undefined
+                ? {
+                    settings: {
+                      'index.mapping.total_fields.limit': indexMappingTotalFieldsLimit,
+                      'index.number_of_shards': 1,
+                      'index.number_of_replicas': 0,
+                    },
+                  }
+                : {})),
+          });
+          if (verbose) { console.log('Created target mapping', resp); }
+        }
       } catch (err) {
         console.log('Error creating target mapping', err);
       }
@@ -62,17 +90,14 @@ function createMappingFactory(ref) {
   };
 }
 
-var MAX_QUEUE_SIZE = 15;
-
 function fileReaderFactory(indexer, fileName, transform, splitRegex, verbose) {
   function startIndex(files) {
-    var ingestQueueSize = 0;
     var finished = false;
 
     var file = files.shift();
     var s = fs
       .createReadStream(file)
-      .pipe(es.split(splitRegex))
+      .pipe(split(splitRegex))
       .pipe(
         es
           .mapSync(function (line) {
@@ -120,20 +145,13 @@ function fileReaderFactory(indexer, fileName, transform, splitRegex, verbose) {
           })
       );
 
-    indexer.queueEmitter.on('queue-size', async function (size) {
+    indexer.queueEmitter.on('pause', function () {
       if (finished) { return; }
-      ingestQueueSize = size;
-
-      if (ingestQueueSize < MAX_QUEUE_SIZE) {
-        s.resume();
-      } else {
-        s.pause();
-      }
+      s.pause();
     });
 
     indexer.queueEmitter.on('resume', function () {
       if (finished) { return; }
-      ingestQueueSize = 0;
       s.resume();
     });
   }
@@ -149,7 +167,7 @@ var EventEmitter = require('events');
 
 var queueEmitter = new EventEmitter();
 
-var parallelCalls = 1;
+var parallelCalls = 5;
 
 // a simple helper queue to bulk index documents
 function indexQueueFactory(ref) {
@@ -159,78 +177,74 @@ function indexQueueFactory(ref) {
   var skipHeader = ref.skipHeader; if ( skipHeader === void 0 ) skipHeader = false;
   var verbose = ref.verbose; if ( verbose === void 0 ) verbose = true;
 
-  var buffer = [];
-  var queue = [];
-  var ingesting = 0;
-  var ingestTimes = [];
-  var finished = false;
+  var flushBytes = bufferSize * 1024; // Convert KB to Bytes
+  var highWaterMark = flushBytes * parallelCalls;
 
-  var ingest = function (b) {
-    if (typeof b !== 'undefined') {
-      queue.push(b);
-      queueEmitter.emit('queue-size', queue.length);
-    }
+  // Create a Readable stream
+  var stream$$1 = new stream.Readable({
+    read: function read() {}, // Implement read but we manage pushing manually
+    highWaterMark: highWaterMark, // Buffer size for backpressure management
+  });
 
-    if (ingestTimes.length > 5) { ingestTimes = ingestTimes.slice(-5); }
+  async function* ndjsonStreamIterator(readableStream) {
+    var buffer = ''; // To hold the incomplete data
+    var skippedHeader = false;
 
-    if (ingesting < parallelCalls) {
-      var docs = queue.shift();
+    // Iterate over the stream using async iteration
+    for await (var chunk of readableStream) {
+      buffer += chunk.toString(); // Accumulate the chunk data in the buffer
 
-      queueEmitter.emit('queue-size', queue.length);
-      if (queue.length <= 5) {
-        queueEmitter.emit('resume');
-      }
+      // Split the buffer into lines (NDJSON items)
+      var lines = buffer.split('\n');
 
-      ingesting += 1;
-
-      if (verbose)
-        { console.log(("bulk ingest docs: " + (docs.length / 2) + ", queue length: " + (queue.length))); }
-
-      var start = Date.now();
-      client
-        .bulk({ body: docs })
-        .then(function () {
-          var end = Date.now();
-          var delta = end - start;
-          ingestTimes.push(delta);
-          ingesting -= 1;
-
-          var ingestTimesMovingAverage =
-            ingestTimes.length > 0
-              ? ingestTimes.reduce(function (p, c) { return p + c; }, 0) / ingestTimes.length
-              : 0;
-          var ingestTimesMovingAverageSeconds = Math.floor(ingestTimesMovingAverage / 1000);
-
-          if (
-            ingestTimes.length > 0 &&
-            ingestTimesMovingAverageSeconds < 30 &&
-            parallelCalls < 10
-          ) {
-            parallelCalls += 1;
-          } else if (
-            ingestTimes.length > 0 &&
-            ingestTimesMovingAverageSeconds >= 30 &&
-            parallelCalls > 1
-          ) {
-            parallelCalls -= 1;
-          }
+      // The last line might be incomplete, so hold it back in the buffer
+      buffer = lines.pop();
 
-          if (queue.length > 0) {
-            ingest();
-          } else if (queue.length === 0 && finished) {
-            queueEmitter.emit('finish');
-          }
-        })
-        .catch(function (error) {
-          console.error(error);
-          ingesting -= 1;
-          parallelCalls = 1;
-          if (queue.length > 0) {
-            ingest();
+      // Yield each complete JSON object
+      for (var line of lines) {
+        if (line.trim()) {
+          try {
+            if (!skipHeader || (skipHeader && !skippedHeader)) {
+              yield JSON.parse(line); // Parse and yield the JSON object
+              skippedHeader = true;
+            }
+          } catch (err) {
+            // Handle JSON parse errors if necessary
+            console.error('Failed to parse JSON:', err);
           }
-        });
+        }
+      }
     }
-  };
+
+    // Handle any remaining data in the buffer after the stream ends
+    if (buffer.trim()) {
+      try {
+        yield JSON.parse(buffer);
+      } catch (err) {
+        console.error('Failed to parse final JSON:', err);
+      }
+    }
+  }
+
+  var finished = false;
+
+  // Async IIFE to start bulk indexing
+  (async function () {
+    await client.helpers.bulk({
+      concurrency: parallelCalls,
+      flushBytes: flushBytes,
+      flushInterval: 1000,
+      refreshOnCompletion: true,
+      datasource: ndjsonStreamIterator(stream$$1),
+      onDocument: function onDocument(doc) {
+        return {
+          index: { _index: targetIndexName },
+        };
+      },
+    });
+
+    queueEmitter.emit('finish');
+  })();
 
   return {
     add: function (doc) {
@@ -238,37 +252,22 @@ function indexQueueFactory(ref) {
         throw new Error('Unexpected doc added after indexer should finish.');
       }
 
-      if (!skipHeader) {
-        var header = { index: { _index: targetIndexName } };
-        buffer.push(header);
-      }
-      buffer.push(doc);
-
-      if (queue.length === 0) {
-        queueEmitter.emit('resume');
-      }
-
-      if (buffer.length >= bufferSize * 2) {
-        ingest(buffer);
-        buffer = [];
+      var canContinue = stream$$1.push(((JSON.stringify(doc)) + "\n"));
+      if (!canContinue) {
+        queueEmitter.emit('pause');
+        stream$$1.once('drain', function () {
+          queueEmitter.emit('resume');
+        });
       }
     },
     finish: function () {
       finished = true;
-
-      if (buffer.length > 0) {
-        ingest(buffer);
-        buffer = [];
-      } else if (queue.length === 0 && ingesting === 0) {
-        queueEmitter.emit('finish');
-      }
+      stream$$1.push(null);
     },
     queueEmitter: queueEmitter,
   };
 }
 
-var MAX_QUEUE_SIZE$1 = 15;
-
 // create a new progress bar instance and use shades_classic theme
 var progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
 
@@ -278,32 +277,33 @@ function indexReaderFactory(
   transform,
   client,
   query,
-  bufferSize,
+  searchSize,
   populatedFields
 ) {
-  if ( bufferSize === void 0 ) bufferSize = DEFAULT_BUFFER_SIZE;
+  if ( searchSize === void 0 ) searchSize = DEFAULT_SEARCH_SIZE;
   if ( populatedFields === void 0 ) populatedFields = false;
 
   return async function indexReader() {
-    var responseQueue = [];
     var docsNum = 0;
+    var scrollId;
+    var finished = false;
+    var readActive = false;
+    var backPressurePause = false;
 
     async function fetchPopulatedFields() {
       try {
-        var response = await client.search({
-          index: sourceIndexName,
-          size: bufferSize,
-          query: {
-            function_score: {
-              query: query,
-              random_score: {},
-            },
+        // Get all populated fields from the index
+        var response = await client.fieldCaps(
+          {
+            index: sourceIndexName,
+            fields: '*',
+            include_empty_fields: false,
+            filters: '-metadata',
           },
-        });
+          { maxRetries: 0 }
+        );
 
-        // Get all field names for each returned doc and flatten it
-        // to a list of unique field names used across all docs.
-        return new Set(response.hits.hits.map(function (d) { return Object.keys(d._source); }).flat(1));
+        return Object.keys(response.fields);
       } catch (e) {
         console.log('error', e);
       }
@@ -312,7 +312,7 @@ function indexReaderFactory(
     function search(fields) {
       return client.search(Object.assign({}, {index: sourceIndexName,
         scroll: '600s',
-        size: bufferSize,
+        size: searchSize,
         query: query},
         (fields ? { _source: fields } : {})));
     }
@@ -329,21 +329,14 @@ function indexReaderFactory(
     // identify populated fields
     if (populatedFields) {
       fieldsWithData = await fetchPopulatedFields();
-      console.log('fieldsWithData', fieldsWithData);
     }
 
-    // start things off by searching, setting a scroll timeout, and pushing
-    // our first response into the queue to be processed
-    var se = await search(fieldsWithData);
-    responseQueue.push(se);
-    progressBar.start(se.hits.total.value, 0);
-    console.log('se', se.hits.hits[0]);
+    await fetchNextResponse();
 
     function processHit(hit) {
       docsNum += 1;
       try {
         var doc = typeof transform === 'function' ? transform(hit._source) : hit._source; // eslint-disable-line no-underscore-dangle
-        // console.log('doc', doc);
 
         // if doc is undefined we'll skip indexing it
         if (typeof doc === 'undefined') {
@@ -363,68 +356,116 @@ function indexReaderFactory(
       }
     }
 
-    var ingestQueueSize = 0;
-    var scrollId = se._scroll_id; // eslint-disable-line no-underscore-dangle
-    var readActive = false;
-
-    async function processResponseQueue() {
-      while (responseQueue.length) {
-        readActive = true;
-        var response = responseQueue.shift();
+    async function fetchNextResponse() {
+      readActive = true;
 
-        // collect the docs from this response
-        response.hits.hits.forEach(processHit);
+      var sc = scrollId ? await scroll(scrollId) : await search(fieldsWithData);
 
-        progressBar.update(docsNum);
+      if (!scrollId) {
+        progressBar.start(sc.hits.total.value, 0);
+      }
 
-        // check to see if we have collected all of the docs
-        if (response.hits.total.value === docsNum) {
-          indexer.finish();
-          break;
-        }
+      scrollId = sc._scroll_id;
+      readActive = false;
 
-        if (ingestQueueSize < MAX_QUEUE_SIZE$1) {
-          // get the next response if there are more docs to fetch
-          var sc = await scroll(response._scroll_id); // eslint-disable-line no-await-in-loop,no-underscore-dangle,max-len
-          scrollId = sc._scroll_id; // eslint-disable-line no-underscore-dangle
-          responseQueue.push(sc);
-        } else {
-          readActive = false;
-        }
-      }
+      processResponse(sc);
     }
 
-    indexer.queueEmitter.on('queue-size', async function (size) {
-      ingestQueueSize = size;
+    async function processResponse(response) {
+      // collect the docs from this response
+      response.hits.hits.forEach(processHit);
+
+      progressBar.update(docsNum);
+
+      // check to see if we have collected all of the docs
+      if (response.hits.total.value === docsNum) {
+        indexer.finish();
+        return;
+      }
 
-      if (!readActive && ingestQueueSize < MAX_QUEUE_SIZE$1) {
-        // get the next response if there are more docs to fetch
-        var sc = await scroll(scrollId); // eslint-disable-line no-await-in-loop,no-underscore-dangle,max-len
-        scrollId = sc._scroll_id; // eslint-disable-line no-underscore-dangle
-        responseQueue.push(sc);
-        processResponseQueue();
+      if (!backPressurePause) {
+        await fetchNextResponse();
       }
+    }
+
+    indexer.queueEmitter.on('pause', async function () {
+      backPressurePause = true;
     });
 
     indexer.queueEmitter.on('resume', async function () {
-      ingestQueueSize = 0;
+      backPressurePause = false;
 
-      if (readActive) {
+      if (readActive || finished) {
         return;
       }
 
-      // get the next response if there are more docs to fetch
-      var sc = await scroll(scrollId); // eslint-disable-line no-await-in-loop,no-underscore-dangle,max-len
-      scrollId = sc._scroll_id; // eslint-disable-line no-underscore-dangle
-      responseQueue.push(sc);
-      processResponseQueue();
+      await fetchNextResponse();
     });
 
     indexer.queueEmitter.on('finish', function () {
+      finished = true;
       progressBar.stop();
     });
+  };
+}
+
+function streamReaderFactory(indexer, stream$$1, transform, splitRegex, verbose) {
+  function startIndex() {
+    var finished = false;
+
+    var s = stream$$1.pipe(split(splitRegex)).pipe(
+      es
+        .mapSync(function (line) {
+          try {
+            // skip empty lines
+            if (line === '') {
+              return;
+            }
+
+            var doc =
+              typeof transform === 'function' ? JSON.stringify(transform(JSON.parse(line))) : line;
 
-    processResponseQueue();
+            // if doc is undefined we'll skip indexing it
+            if (typeof doc === 'undefined') {
+              s.resume();
+              return;
+            }
+
+            // the transform callback may return an array of docs so we can emit
+            // multiple docs from a single line
+            if (Array.isArray(doc)) {
+              doc.forEach(function (d) { return indexer.add(d); });
+              return;
+            }
+
+            indexer.add(doc);
+          } catch (e) {
+            console.log('error', e);
+          }
+        })
+        .on('error', function (err) {
+          console.log('Error while reading stream.', err);
+        })
+        .on('end', function () {
+          if (verbose) { console.log('Read entire stream.'); }
+          indexer.finish();
+          finished = true;
+        })
+    );
+
+    indexer.queueEmitter.on('pause', function () {
+      if (finished) { return; }
+      s.pause();
+    });
+
+    indexer.queueEmitter.on('resume', function () {
+      if (finished) { return; }
+      s.resume();
+    });
+  }
+
+  return function () {
+    startIndex();
   };
 }
 
@@ -433,6 +474,8 @@ async function transformer(ref) {
   var sourceClientConfig = ref.sourceClientConfig;
   var targetClientConfig = ref.targetClientConfig;
   var bufferSize = ref.bufferSize; if ( bufferSize === void 0 ) bufferSize = DEFAULT_BUFFER_SIZE;
+  var searchSize = ref.searchSize; if ( searchSize === void 0 ) searchSize = DEFAULT_SEARCH_SIZE;
+  var stream$$1 = ref.stream;
   var fileName = ref.fileName;
   var splitRegex = ref.splitRegex; if ( splitRegex === void 0 ) splitRegex = /\n/;
   var sourceIndexName = ref.sourceIndexName;
@@ -468,6 +511,7 @@ async function transformer(ref) {
     mappingsOverride: mappingsOverride,
     indexMappingTotalFieldsLimit: indexMappingTotalFieldsLimit,
     verbose: verbose,
+    deleteIndex: deleteIndex,
   });
   var indexer = indexQueueFactory({
     targetClient: targetClient,
@@ -482,8 +526,12 @@ async function transformer(ref) {
       throw Error('Only either one of fileName or sourceIndexName can be specified.');
     }
 
-    if (typeof fileName === 'undefined' && typeof sourceIndexName === 'undefined') {
-      throw Error('Either fileName or sourceIndexName must be specified.');
+    if (
+      (typeof fileName !== 'undefined' && typeof sourceIndexName !== 'undefined') ||
+      (typeof fileName !== 'undefined' && typeof stream$$1 !== 'undefined') ||
+      (typeof sourceIndexName !== 'undefined' && typeof stream$$1 !== 'undefined')
+    ) {
+      throw Error('Only one of fileName, sourceIndexName, or stream can be specified.');
     }
 
     if (typeof fileName !== 'undefined') {
@@ -497,11 +545,15 @@ async function transformer(ref) {
         transform,
         sourceClient,
         query,
-        bufferSize,
+        searchSize,
         populatedFields
       );
     }
 
+    if (typeof stream$$1 !== 'undefined') {
+      return streamReaderFactory(indexer, stream$$1, transform, splitRegex, verbose);
+    }
+
     return null;
   }
 

package/dist/node-es-transformer.esm.js

@@ -1,10 +1,20 @@
 import fs from 'fs';
 import es from 'event-stream';
 import glob from 'glob';
+import split from 'split2';
+import { Readable } from 'stream';
 import cliProgress from 'cli-progress';
 import elasticsearch from '@elastic/elasticsearch';
 
-var DEFAULT_BUFFER_SIZE = 1000;
+// In earlier versions this was used to set the number of docs to index in a
+// single bulk request. Since we switched to use the helpers.bulk() method from
+// the ES client, this now translates to the `flushBytes` option of the helper.
+// However, for kind of a backwards compability with the old values, this uses
+// KBytes instead of Bytes. It will be multiplied by 1024 in the index queue.
+var DEFAULT_BUFFER_SIZE = 5120;
+
+// The default number of docs to fetch in a single search request when reindexing.
+var DEFAULT_SEARCH_SIZE = 1000;
 
 function createMappingFactory(ref) {
   var sourceClient = ref.sourceClient;
@@ -15,6 +25,7 @@ function createMappingFactory(ref) {
   var mappingsOverride = ref.mappingsOverride;
   var indexMappingTotalFieldsLimit = ref.indexMappingTotalFieldsLimit;
   var verbose = ref.verbose;
+  var deleteIndex = ref.deleteIndex;
 
   return async function () {
     var targetMappings = mappingsOverride ? undefined : mappings;
@@ -24,7 +35,14 @@ function createMappingFactory(ref) {
         var mapping = await sourceClient.indices.getMapping({
           index: sourceIndexName,
         });
-        targetMappings = mapping[sourceIndexName].mappings;
+        if (mapping[sourceIndexName]) {
+          targetMappings = mapping[sourceIndexName].mappings;
+        } else {
+          var allMappings = Object.values(mapping);
+          if (allMappings.length > 0) {
+            targetMappings = Object.values(mapping)[0].mappings;
+          }
+        }
       } catch (err) {
         console.log('Error reading source mapping', err);
         return;
@@ -39,18 +57,28 @@ function createMappingFactory(ref) {
       }
 
       try {
-        var resp = await targetClient.indices.create({
-          index: targetIndexName,
-          body: Object.assign({}, {mappings: targetMappings},
-            (indexMappingTotalFieldsLimit !== undefined
-              ? {
-                  settings: {
-                    'index.mapping.total_fields.limit': indexMappingTotalFieldsLimit,
-                  },
-                }
-              : {})),
-        });
-        if (verbose) { console.log('Created target mapping', resp); }
+        var indexExists = await targetClient.indices.exists({ index: targetIndexName });
+
+        if (indexExists === true && deleteIndex === true) {
+          await targetClient.indices.delete({ index: targetIndexName });
+        }
+
+        if (indexExists === false || deleteIndex === true) {
+          var resp = await targetClient.indices.create({
+            index: targetIndexName,
+            body: Object.assign({}, {mappings: targetMappings},
+              (indexMappingTotalFieldsLimit !== undefined
+                ? {
+                    settings: {
+                      'index.mapping.total_fields.limit': indexMappingTotalFieldsLimit,
+                      'index.number_of_shards': 1,
+                      'index.number_of_replicas': 0,
+                    },
+                  }
+                : {})),
+          });
+          if (verbose) { console.log('Created target mapping', resp); }
+        }
       } catch (err) {
         console.log('Error creating target mapping', err);
       }
@@ -58,17 +86,14 @@ function createMappingFactory(ref) {
   };
 }
 
-var MAX_QUEUE_SIZE = 15;
-
 function fileReaderFactory(indexer, fileName, transform, splitRegex, verbose) {
   function startIndex(files) {
-    var ingestQueueSize = 0;
     var finished = false;
 
     var file = files.shift();
     var s = fs
       .createReadStream(file)
-      .pipe(es.split(splitRegex))
+      .pipe(split(splitRegex))
       .pipe(
         es
           .mapSync(function (line) {
@@ -116,20 +141,13 @@ function fileReaderFactory(indexer, fileName, transform, splitRegex, verbose) {
           })
       );
 
-    indexer.queueEmitter.on('queue-size', async function (size) {
+    indexer.queueEmitter.on('pause', function () {
       if (finished) { return; }
-      ingestQueueSize = size;
-
-      if (ingestQueueSize < MAX_QUEUE_SIZE) {
-        s.resume();
-      } else {
-        s.pause();
-      }
+      s.pause();
     });
 
     indexer.queueEmitter.on('resume', function () {
       if (finished) { return; }
-      ingestQueueSize = 0;
       s.resume();
     });
   }
@@ -145,7 +163,7 @@ var EventEmitter = require('events');
 
 var queueEmitter = new EventEmitter();
 
-var parallelCalls = 1;
+var parallelCalls = 5;
 
 // a simple helper queue to bulk index documents
 function indexQueueFactory(ref) {
@@ -155,78 +173,74 @@ function indexQueueFactory(ref) {
   var skipHeader = ref.skipHeader; if ( skipHeader === void 0 ) skipHeader = false;
   var verbose = ref.verbose; if ( verbose === void 0 ) verbose = true;
 
-  var buffer = [];
-  var queue = [];
-  var ingesting = 0;
-  var ingestTimes = [];
-  var finished = false;
+  var flushBytes = bufferSize * 1024; // Convert KB to Bytes
+  var highWaterMark = flushBytes * parallelCalls;
 
-  var ingest = function (b) {
-    if (typeof b !== 'undefined') {
-      queue.push(b);
-      queueEmitter.emit('queue-size', queue.length);
-    }
+  // Create a Readable stream
+  var stream = new Readable({
+    read: function read() {}, // Implement read but we manage pushing manually
+    highWaterMark: highWaterMark, // Buffer size for backpressure management
+  });
 
-    if (ingestTimes.length > 5) { ingestTimes = ingestTimes.slice(-5); }
+  async function* ndjsonStreamIterator(readableStream) {
+    var buffer = ''; // To hold the incomplete data
+    var skippedHeader = false;
 
-    if (ingesting < parallelCalls) {
-      var docs = queue.shift();
+    // Iterate over the stream using async iteration
+    for await (var chunk of readableStream) {
+      buffer += chunk.toString(); // Accumulate the chunk data in the buffer
 
-      queueEmitter.emit('queue-size', queue.length);
-      if (queue.length <= 5) {
-        queueEmitter.emit('resume');
-      }
+      // Split the buffer into lines (NDJSON items)
+      var lines = buffer.split('\n');
 
-      ingesting += 1;
-
-      if (verbose)
-        { console.log(("bulk ingest docs: " + (docs.length / 2) + ", queue length: " + (queue.length))); }
-
-      var start = Date.now();
-      client
-        .bulk({ body: docs })
-        .then(function () {
-          var end = Date.now();
-          var delta = end - start;
-          ingestTimes.push(delta);
-          ingesting -= 1;
-
-          var ingestTimesMovingAverage =
-            ingestTimes.length > 0
-              ? ingestTimes.reduce(function (p, c) { return p + c; }, 0) / ingestTimes.length
-              : 0;
-          var ingestTimesMovingAverageSeconds = Math.floor(ingestTimesMovingAverage / 1000);
-
-          if (
-            ingestTimes.length > 0 &&
-            ingestTimesMovingAverageSeconds < 30 &&
-            parallelCalls < 10
-          ) {
-            parallelCalls += 1;
-          } else if (
-            ingestTimes.length > 0 &&
-            ingestTimesMovingAverageSeconds >= 30 &&
-            parallelCalls > 1
-          ) {
-            parallelCalls -= 1;
-          }
+      // The last line might be incomplete, so hold it back in the buffer
+      buffer = lines.pop();
 
-          if (queue.length > 0) {
-            ingest();
-          } else if (queue.length === 0 && finished) {
-            queueEmitter.emit('finish');
-          }
-        })
-        .catch(function (error) {
-          console.error(error);
-          ingesting -= 1;
-          parallelCalls = 1;
-          if (queue.length > 0) {
-            ingest();
+      // Yield each complete JSON object
+      for (var line of lines) {
+        if (line.trim()) {
+          try {
+            if (!skipHeader || (skipHeader && !skippedHeader)) {
+              yield JSON.parse(line); // Parse and yield the JSON object
+              skippedHeader = true;
+            }
+          } catch (err) {
+            // Handle JSON parse errors if necessary
+            console.error('Failed to parse JSON:', err);
           }
-        });
+        }
+      }
     }
-  };
+
+    // Handle any remaining data in the buffer after the stream ends
+    if (buffer.trim()) {
+      try {
+        yield JSON.parse(buffer);
+      } catch (err) {
+        console.error('Failed to parse final JSON:', err);
+      }
+    }
+  }
+
+  var finished = false;
+
+  // Async IIFE to start bulk indexing
+  (async function () {
+    await client.helpers.bulk({
+      concurrency: parallelCalls,
+      flushBytes: flushBytes,
+      flushInterval: 1000,
+      refreshOnCompletion: true,
+      datasource: ndjsonStreamIterator(stream),
+      onDocument: function onDocument(doc) {
+        return {
+          index: { _index: targetIndexName },
+        };
+      },
+    });
+
+    queueEmitter.emit('finish');
+  })();
 
   return {
     add: function (doc) {
@@ -234,37 +248,22 @@ function indexQueueFactory(ref) {
         throw new Error('Unexpected doc added after indexer should finish.');
       }
 
-      if (!skipHeader) {
-        var header = { index: { _index: targetIndexName } };
-        buffer.push(header);
-      }
-      buffer.push(doc);
-
-      if (queue.length === 0) {
-        queueEmitter.emit('resume');
-      }
-
-      if (buffer.length >= bufferSize * 2) {
-        ingest(buffer);
-        buffer = [];
+      var canContinue = stream.push(((JSON.stringify(doc)) + "\n"));
+      if (!canContinue) {
+        queueEmitter.emit('pause');
+        stream.once('drain', function () {
+          queueEmitter.emit('resume');
+        });
       }
     },
     finish: function () {
       finished = true;
-
-      if (buffer.length > 0) {
-        ingest(buffer);
-        buffer = [];
-      } else if (queue.length === 0 && ingesting === 0) {
-        queueEmitter.emit('finish');
-      }
+      stream.push(null);
     },
     queueEmitter: queueEmitter,
   };
 }
 
-var MAX_QUEUE_SIZE$1 = 15;
-
 // create a new progress bar instance and use shades_classic theme
 var progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
 
@@ -274,32 +273,33 @@ function indexReaderFactory(
   transform,
   client,
   query,
-  bufferSize,
+  searchSize,
   populatedFields
 ) {
-  if ( bufferSize === void 0 ) bufferSize = DEFAULT_BUFFER_SIZE;
+  if ( searchSize === void 0 ) searchSize = DEFAULT_SEARCH_SIZE;
   if ( populatedFields === void 0 ) populatedFields = false;
 
   return async function indexReader() {
-    var responseQueue = [];
     var docsNum = 0;
+    var scrollId;
+    var finished = false;
+    var readActive = false;
+    var backPressurePause = false;
 
     async function fetchPopulatedFields() {
       try {
-        var response = await client.search({
-          index: sourceIndexName,
-          size: bufferSize,
-          query: {
-            function_score: {
-              query: query,
-              random_score: {},
-            },
+        // Get all populated fields from the index
+        var response = await client.fieldCaps(
+          {
+            index: sourceIndexName,
+            fields: '*',
+            include_empty_fields: false,
+            filters: '-metadata',
           },
-        });
+          { maxRetries: 0 }
+        );
 
-        // Get all field names for each returned doc and flatten it
-        // to a list of unique field names used across all docs.
-        return new Set(response.hits.hits.map(function (d) { return Object.keys(d._source); }).flat(1));
+        return Object.keys(response.fields);
       } catch (e) {
         console.log('error', e);
       }
@@ -308,7 +308,7 @@ function indexReaderFactory(
     function search(fields) {
       return client.search(Object.assign({}, {index: sourceIndexName,
         scroll: '600s',
-        size: bufferSize,
+        size: searchSize,
         query: query},
         (fields ? { _source: fields } : {})));
     }
@@ -325,21 +325,14 @@ function indexReaderFactory(
     // identify populated fields
     if (populatedFields) {
       fieldsWithData = await fetchPopulatedFields();
-      console.log('fieldsWithData', fieldsWithData);
     }
 
-    // start things off by searching, setting a scroll timeout, and pushing
-    // our first response into the queue to be processed
-    var se = await search(fieldsWithData);
-    responseQueue.push(se);
-    progressBar.start(se.hits.total.value, 0);
-    console.log('se', se.hits.hits[0]);
+    await fetchNextResponse();
 
     function processHit(hit) {
       docsNum += 1;
       try {
         var doc = typeof transform === 'function' ? transform(hit._source) : hit._source; // eslint-disable-line no-underscore-dangle
-        // console.log('doc', doc);
 
         // if doc is undefined we'll skip indexing it
         if (typeof doc === 'undefined') {
@@ -359,68 +352,116 @@ function indexReaderFactory(
       }
     }
 
-    var ingestQueueSize = 0;
-    var scrollId = se._scroll_id; // eslint-disable-line no-underscore-dangle
-    var readActive = false;
-
-    async function processResponseQueue() {
-      while (responseQueue.length) {
-        readActive = true;
-        var response = responseQueue.shift();
+    async function fetchNextResponse() {
+      readActive = true;
 
-        // collect the docs from this response
-        response.hits.hits.forEach(processHit);
+      var sc = scrollId ? await scroll(scrollId) : await search(fieldsWithData);
 
-        progressBar.update(docsNum);
+      if (!scrollId) {
+        progressBar.start(sc.hits.total.value, 0);
+      }
 
-        // check to see if we have collected all of the docs
-        if (response.hits.total.value === docsNum) {
-          indexer.finish();
-          break;
-        }
+      scrollId = sc._scroll_id;
+      readActive = false;
 
-        if (ingestQueueSize < MAX_QUEUE_SIZE$1) {
-          // get the next response if there are more docs to fetch
-          var sc = await scroll(response._scroll_id); // eslint-disable-line no-await-in-loop,no-underscore-dangle,max-len
-          scrollId = sc._scroll_id; // eslint-disable-line no-underscore-dangle
-          responseQueue.push(sc);
-        } else {
-          readActive = false;
-        }
-      }
+      processResponse(sc);
     }
 
-    indexer.queueEmitter.on('queue-size', async function (size) {
-      ingestQueueSize = size;
+    async function processResponse(response) {
+      // collect the docs from this response
+      response.hits.hits.forEach(processHit);
+
+      progressBar.update(docsNum);
+
+      // check to see if we have collected all of the docs
+      if (response.hits.total.value === docsNum) {
+        indexer.finish();
+        return;
+      }
 
-      if (!readActive && ingestQueueSize < MAX_QUEUE_SIZE$1) {
-        // get the next response if there are more docs to fetch
-        var sc = await scroll(scrollId); // eslint-disable-line no-await-in-loop,no-underscore-dangle,max-len
-        scrollId = sc._scroll_id; // eslint-disable-line no-underscore-dangle
-        responseQueue.push(sc);
-        processResponseQueue();
+      if (!backPressurePause) {
+        await fetchNextResponse();
       }
+    }
+
+    indexer.queueEmitter.on('pause', async function () {
+      backPressurePause = true;
     });
 
     indexer.queueEmitter.on('resume', async function () {
-      ingestQueueSize = 0;
+      backPressurePause = false;
 
-      if (readActive) {
+      if (readActive || finished) {
         return;
       }
 
-      // get the next response if there are more docs to fetch
-      var sc = await scroll(scrollId); // eslint-disable-line no-await-in-loop,no-underscore-dangle,max-len
-      scrollId = sc._scroll_id; // eslint-disable-line no-underscore-dangle
-      responseQueue.push(sc);
-      processResponseQueue();
+      await fetchNextResponse();
     });
 
     indexer.queueEmitter.on('finish', function () {
+      finished = true;
       progressBar.stop();
     });
+  };
+}
+
+function streamReaderFactory(indexer, stream, transform, splitRegex, verbose) {
+  function startIndex() {
+    var finished = false;
+
+    var s = stream.pipe(split(splitRegex)).pipe(
+      es
+        .mapSync(function (line) {
+          try {
+            // skip empty lines
+            if (line === '') {
+              return;
+            }
+
+            var doc =
+              typeof transform === 'function' ? JSON.stringify(transform(JSON.parse(line))) : line;
 
-    processResponseQueue();
+            // if doc is undefined we'll skip indexing it
+            if (typeof doc === 'undefined') {
+              s.resume();
+              return;
+            }
+
+            // the transform callback may return an array of docs so we can emit
+            // multiple docs from a single line
+            if (Array.isArray(doc)) {
+              doc.forEach(function (d) { return indexer.add(d); });
+              return;
+            }
+
+            indexer.add(doc);
+          } catch (e) {
+            console.log('error', e);
+          }
+        })
+        .on('error', function (err) {
+          console.log('Error while reading stream.', err);
+        })
+        .on('end', function () {
+          if (verbose) { console.log('Read entire stream.'); }
+          indexer.finish();
+          finished = true;
+        })
+    );
+
+    indexer.queueEmitter.on('pause', function () {
+      if (finished) { return; }
+      s.pause();
+    });
+
+    indexer.queueEmitter.on('resume', function () {
+      if (finished) { return; }
+      s.resume();
+    });
+  }
+
+  return function () {
+    startIndex();
   };
 }
 
@@ -429,6 +470,8 @@ async function transformer(ref) {
   var sourceClientConfig = ref.sourceClientConfig;
   var targetClientConfig = ref.targetClientConfig;
   var bufferSize = ref.bufferSize; if ( bufferSize === void 0 ) bufferSize = DEFAULT_BUFFER_SIZE;
+  var searchSize = ref.searchSize; if ( searchSize === void 0 ) searchSize = DEFAULT_SEARCH_SIZE;
+  var stream = ref.stream;
   var fileName = ref.fileName;
   var splitRegex = ref.splitRegex; if ( splitRegex === void 0 ) splitRegex = /\n/;
   var sourceIndexName = ref.sourceIndexName;
@@ -464,6 +507,7 @@ async function transformer(ref) {
     mappingsOverride: mappingsOverride,
     indexMappingTotalFieldsLimit: indexMappingTotalFieldsLimit,
     verbose: verbose,
+    deleteIndex: deleteIndex,
   });
   var indexer = indexQueueFactory({
     targetClient: targetClient,
@@ -478,8 +522,12 @@ async function transformer(ref) {
       throw Error('Only either one of fileName or sourceIndexName can be specified.');
     }
 
-    if (typeof fileName === 'undefined' && typeof sourceIndexName === 'undefined') {
-      throw Error('Either fileName or sourceIndexName must be specified.');
+    if (
+      (typeof fileName !== 'undefined' && typeof sourceIndexName !== 'undefined') ||
+      (typeof fileName !== 'undefined' && typeof stream !== 'undefined') ||
+      (typeof sourceIndexName !== 'undefined' && typeof stream !== 'undefined')
+    ) {
+      throw Error('Only one of fileName, sourceIndexName, or stream can be specified.');
     }
 
     if (typeof fileName !== 'undefined') {
@@ -493,11 +541,15 @@ async function transformer(ref) {
         transform,
         sourceClient,
         query,
-        bufferSize,
+        searchSize,
         populatedFields
       );
     }
 
+    if (typeof stream !== 'undefined') {
+      return streamReaderFactory(indexer, stream, transform, splitRegex, verbose);
+    }
+
     return null;
   }
 

package/package.json

@@ -14,20 +14,21 @@
   "license": "Apache-2.0",
   "author": "Walter Rafelsberger <walter@rafelsberger.at>",
   "contributors": [],
-  "version": "1.0.0-beta2",
+  "version": "1.0.0-beta4",
   "main": "dist/node-es-transformer.cjs.js",
   "module": "dist/node-es-transformer.esm.js",
   "dependencies": {
-    "@elastic/elasticsearch": "^8.10.0",
+    "@elastic/elasticsearch": "^8.17.0",
     "cli-progress": "^3.12.0",
     "event-stream": "3.3.4",
-    "glob": "7.1.2"
+    "git-cz": "^4.9.0",
+    "glob": "7.1.2",
+    "split2": "^4.2.0"
   },
   "devDependencies": {
     "acorn": "^6.4.2",
     "async-retry": "^1.3.3",
     "commit-and-tag-version": "^11.3.0",
-    "cz-conventional-changelog": "^3.3.0",
     "eslint": "^8.51.0",
     "eslint-config-airbnb": "19.0.4",
     "eslint-config-prettier": "^9.0.0",
@@ -57,7 +58,7 @@
   ],
   "config": {
     "commitizen": {
-      "path": "./node_modules/cz-conventional-changelog"
+      "path": "git-cz"
     }
   },
   "jest": {