node-es-transformer 1.0.0-alpha10 → 1.0.0-alpha12

package/README.md

@@ -1,6 +1,7 @@
 [![npm](https://img.shields.io/npm/v/node-es-transformer.svg?maxAge=2592000)](https://www.npmjs.com/package/node-es-transformer)
 [![npm](https://img.shields.io/npm/l/node-es-transformer.svg?maxAge=2592000)](https://www.npmjs.com/package/node-es-transformer)
 [![npm](https://img.shields.io/npm/dt/node-es-transformer.svg?maxAge=2592000)](https://www.npmjs.com/package/node-es-transformer)
+[![Commitizen friendly](https://img.shields.io/badge/commitizen-friendly-brightgreen.svg)](http://commitizen.github.io/cz-cli/)
 
 # node-es-transformer
 
@@ -115,6 +116,7 @@ transformer({
 - `targetIndexName`: The target Elasticsearch index where documents will be indexed.
 - `mappings`: Optional Elasticsearch document mappings. If not set and you're reindexing from another index, the mappings from the existing index will be used.
 - `mappingsOverride`: If you're reindexing and this is set to `true`, `mappings` will be applied on top of the source index's mappings. Defaults to `false`.
+- `indexMappingTotalFieldsLimit`: Optional field limit for the target index to be created that will be passed on as the `index.mapping.total_fields.limit` setting.
 - `query`: Optional Elasticsearch [DSL query](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html) to filter documents from the source index.
 - `skipHeader`: If true, skips the first line of the source file. Defaults to `false`.
 - `transform(line)`: A callback function which allows the transformation of a source line into one or several documents.

package/dist/node-es-transformer.cjs.js

@@ -8,6 +8,8 @@ var glob = _interopDefault(require('glob'));
 var cliProgress = _interopDefault(require('cli-progress'));
 var elasticsearch = _interopDefault(require('@elastic/elasticsearch'));
 
+var DEFAULT_BUFFER_SIZE = 1000;
+
 function createMappingFactory(ref) {
   var sourceClient = ref.sourceClient;
   var sourceIndexName = ref.sourceIndexName;
@@ -15,6 +17,7 @@ function createMappingFactory(ref) {
   var targetIndexName = ref.targetIndexName;
   var mappings = ref.mappings;
   var mappingsOverride = ref.mappingsOverride;
+  var indexMappingTotalFieldsLimit = ref.indexMappingTotalFieldsLimit;
   var verbose = ref.verbose;
 
   return async function () {
@@ -22,7 +25,9 @@ function createMappingFactory(ref) {
 
     if (sourceClient && sourceIndexName && typeof targetMappings === 'undefined') {
       try {
-        var mapping = await sourceClient.indices.getMapping({ index: sourceIndexName });
+        var mapping = await sourceClient.indices.getMapping({
+          index: sourceIndexName,
+        });
         targetMappings = mapping[sourceIndexName].mappings;
       } catch (err) {
         console.log('Error reading source mapping', err);
@@ -38,12 +43,17 @@ function createMappingFactory(ref) {
       }
 
       try {
-        var resp = await targetClient.indices.create(
-          {
-            index: targetIndexName,
-            body: { mappings: targetMappings },
-          }
-        );
+        var resp = await targetClient.indices.create({
+          index: targetIndexName,
+          body: Object.assign({}, {mappings: targetMappings},
+            (indexMappingTotalFieldsLimit !== undefined
+              ? {
+                  settings: {
+                    'index.mapping.total_fields.limit': indexMappingTotalFieldsLimit,
+                  },
+                }
+              : {})),
+        });
         if (verbose) { console.log('Created target mapping', resp); }
       } catch (err) {
         console.log('Error creating target mapping', err);
@@ -55,40 +65,44 @@ function createMappingFactory(ref) {
 function fileReaderFactory(indexer, fileName, transform, splitRegex, verbose) {
   function startIndex(files) {
     var file = files.shift();
-    var s = fs.createReadStream(file)
+    var s = fs
+      .createReadStream(file)
       .pipe(es.split(splitRegex))
-      .pipe(es.mapSync(function (line) {
-        s.pause();
-        try {
-          var doc = (typeof transform === 'function') ? transform(line) : line;
-          // if doc is undefined we'll skip indexing it
-          if (typeof doc === 'undefined') {
-            s.resume();
-            return;
-          }
-
-          // the transform callback may return an array of docs so we can emit
-          // multiple docs from a single line
-          if (Array.isArray(doc)) {
-            doc.forEach(function (d) { return indexer.add(d); });
-            return;
-          }
-
-          indexer.add(doc);
-        } catch (e) {
-          console.log('error', e);
-        }
-      })
-        .on('error', function (err) {
-          console.log('Error while reading file.', err);
-        })
-        .on('end', function () {
-          if (verbose) { console.log('Read entire file: ', file); }
-          indexer.finish();
-          if (files.length > 0) {
-            startIndex(files);
-          }
-        }));
+      .pipe(
+        es
+          .mapSync(function (line) {
+            s.pause();
+            try {
+              var doc = typeof transform === 'function' ? transform(line) : line;
+              // if doc is undefined we'll skip indexing it
+              if (typeof doc === 'undefined') {
+                s.resume();
+                return;
+              }
+
+              // the transform callback may return an array of docs so we can emit
+              // multiple docs from a single line
+              if (Array.isArray(doc)) {
+                doc.forEach(function (d) { return indexer.add(d); });
+                return;
+              }
+
+              indexer.add(doc);
+            } catch (e) {
+              console.log('error', e);
+            }
+          })
+          .on('error', function (err) {
+            console.log('Error while reading file.', err);
+          })
+          .on('end', function () {
+            if (verbose) { console.log('Read entire file: ', file); }
+            indexer.finish();
+            if (files.length > 0) {
+              startIndex(files);
+            }
+          })
+      );
 
     indexer.queueEmitter.on('resume', function () {
       s.resume();
@@ -106,45 +120,66 @@ var EventEmitter = require('events');
 
 var queueEmitter = new EventEmitter();
 
+var parallelCalls = 1;
+
 // a simple helper queue to bulk index documents
 function indexQueueFactory(ref) {
   var client = ref.targetClient;
   var targetIndexName = ref.targetIndexName;
-  var bufferSize = ref.bufferSize; if ( bufferSize === void 0 ) bufferSize = 1000;
+  var bufferSize = ref.bufferSize; if ( bufferSize === void 0 ) bufferSize = DEFAULT_BUFFER_SIZE;
   var skipHeader = ref.skipHeader; if ( skipHeader === void 0 ) skipHeader = false;
   var verbose = ref.verbose; if ( verbose === void 0 ) verbose = true;
 
   var buffer = [];
   var queue = [];
-  var ingesting = false;
+  var ingesting = 0;
+  var ingestTimes = [];
 
-  var ingest = async function (b) {
+  var ingest = function (b) {
     if (typeof b !== 'undefined') {
       queue.push(b);
       queueEmitter.emit('queue-size', queue.length);
     }
 
-    if (ingesting === false) {
+    if (ingestTimes.length > 5) { ingestTimes = ingestTimes.slice(-5); }
+
+    if (ingesting < parallelCalls) {
       var docs = queue.shift();
+
       queueEmitter.emit('queue-size', queue.length);
-      ingesting = true;
-      if (verbose) { console.log(("bulk ingest docs: " + (docs.length / 2) + ", queue length: " + (queue.length))); }
+      if (queue.length <= 5) {
+        queueEmitter.emit('resume');
+      }
 
-      try {
-        await client.bulk({ body: docs });
-        ingesting = false;
+      ingesting += 1;
+
+      if (verbose)
+        { console.log(("bulk ingest docs: " + (docs.length / 2) + ", queue length: " + (queue.length))); }
+
+      var start = Date.now();
+      client.bulk({ body: docs }).then(function () {
+        var end = Date.now();
+        var delta = end - start;
+        ingestTimes.push(delta);
+        ingesting -= 1;
+
+        var ingestTimesMovingAverage =
+          ingestTimes.length > 0 ? ingestTimes.reduce(function (p, c) { return p + c; }, 0) / ingestTimes.length : 0;
+        var ingestTimesMovingAverageSeconds = Math.floor(ingestTimesMovingAverage / 1000);
+
+        if (ingestTimes.length > 0 && ingestTimesMovingAverageSeconds < 30 && parallelCalls < 10) {
+          parallelCalls += 1;
+        } else if (
+          ingestTimes.length > 0 &&
+          ingestTimesMovingAverageSeconds >= 30 &&
+          parallelCalls > 1
+        ) {
+          parallelCalls -= 1;
+        }
         if (queue.length > 0) {
           ingest();
         }
-      } catch (err) {
-        console.log('bulk index error', err);
-      }
-    }
-
-    // console.log(`ingest: queue.length ${queue.length}`);
-    if (queue.length === 0) {
-      queueEmitter.emit('queue-size', 0);
-      queueEmitter.emit('resume');
+      });
     }
   };
 
@@ -161,7 +196,7 @@ function indexQueueFactory(ref) {
         queueEmitter.emit('resume');
       }
 
-      if (buffer.length >= (bufferSize * 2)) {
+      if (buffer.length >= bufferSize * 2) {
         ingest(buffer);
         buffer = [];
       }
@@ -175,12 +210,21 @@ function indexQueueFactory(ref) {
   };
 }
 
-var MAX_QUEUE_SIZE = 5;
+var MAX_QUEUE_SIZE = 15;
 
 // create a new progress bar instance and use shades_classic theme
 var progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
 
-function indexReaderFactory(indexer, sourceIndexName, transform, client, query) {
+function indexReaderFactory(
+  indexer,
+  sourceIndexName,
+  transform,
+  client,
+  query,
+  bufferSize
+) {
+  if ( bufferSize === void 0 ) bufferSize = DEFAULT_BUFFER_SIZE;
+
   return async function indexReader() {
     var responseQueue = [];
     var docsNum = 0;
@@ -189,7 +233,7 @@ function indexReaderFactory(indexer, sourceIndexName, transform, client, query)
       return client.search({
         index: sourceIndexName,
         scroll: '30s',
-        size: 10000,
+        size: bufferSize,
         query: query,
       });
     }
@@ -210,7 +254,7 @@ function indexReaderFactory(indexer, sourceIndexName, transform, client, query)
     function processHit(hit) {
       docsNum += 1;
       try {
-        var doc = (typeof transform === 'function') ? transform(hit._source) : hit._source; // eslint-disable-line no-underscore-dangle
+        var doc = typeof transform === 'function' ? transform(hit._source) : hit._source; // eslint-disable-line no-underscore-dangle
         // if doc is undefined we'll skip indexing it
         if (typeof doc === 'undefined') {
           return;
@@ -252,7 +296,7 @@ function indexReaderFactory(indexer, sourceIndexName, transform, client, query)
         }
 
         if (ingestQueueSize < MAX_QUEUE_SIZE) {
-        // get the next response if there are more docs to fetch
+          // get the next response if there are more docs to fetch
           var sc = await scroll(response._scroll_id); // eslint-disable-line no-await-in-loop,no-underscore-dangle,max-len
           scrollId = sc._scroll_id; // eslint-disable-line no-underscore-dangle
           responseQueue.push(sc);
@@ -266,7 +310,7 @@ function indexReaderFactory(indexer, sourceIndexName, transform, client, query)
       ingestQueueSize = size;
 
       if (!readActive && ingestQueueSize < MAX_QUEUE_SIZE) {
-      // get the next response if there are more docs to fetch
+        // get the next response if there are more docs to fetch
         var sc = await scroll(scrollId); // eslint-disable-line no-await-in-loop,no-underscore-dangle,max-len
         scrollId = sc._scroll_id; // eslint-disable-line no-underscore-dangle
         responseQueue.push(sc);
@@ -296,13 +340,14 @@ async function transformer(ref) {
   var deleteIndex = ref.deleteIndex; if ( deleteIndex === void 0 ) deleteIndex = false;
   var sourceClientConfig = ref.sourceClientConfig;
   var targetClientConfig = ref.targetClientConfig;
-  var bufferSize = ref.bufferSize; if ( bufferSize === void 0 ) bufferSize = 1000;
+  var bufferSize = ref.bufferSize; if ( bufferSize === void 0 ) bufferSize = DEFAULT_BUFFER_SIZE;
   var fileName = ref.fileName;
   var splitRegex = ref.splitRegex; if ( splitRegex === void 0 ) splitRegex = /\n/;
   var sourceIndexName = ref.sourceIndexName;
   var targetIndexName = ref.targetIndexName;
   var mappings = ref.mappings;
   var mappingsOverride = ref.mappingsOverride; if ( mappingsOverride === void 0 ) mappingsOverride = false;
+  var indexMappingTotalFieldsLimit = ref.indexMappingTotalFieldsLimit;
   var query = ref.query;
   var skipHeader = ref.skipHeader; if ( skipHeader === void 0 ) skipHeader = false;
   var transform = ref.transform;
@@ -328,6 +373,7 @@ async function transformer(ref) {
     targetIndexName: targetIndexName,
     mappings: mappings,
     mappingsOverride: mappingsOverride,
+    indexMappingTotalFieldsLimit: indexMappingTotalFieldsLimit,
     verbose: verbose,
   });
   var indexer = indexQueueFactory({
@@ -339,30 +385,16 @@ async function transformer(ref) {
   });
 
   function getReader() {
-    if (
-      typeof fileName !== 'undefined'
-      && typeof sourceIndexName !== 'undefined'
-    ) {
-      throw Error(
-        'Only either one of fileName or sourceIndexName can be specified.'
-      );
+    if (typeof fileName !== 'undefined' && typeof sourceIndexName !== 'undefined') {
+      throw Error('Only either one of fileName or sourceIndexName can be specified.');
     }
 
-    if (
-      typeof fileName === 'undefined'
-      && typeof sourceIndexName === 'undefined'
-    ) {
+    if (typeof fileName === 'undefined' && typeof sourceIndexName === 'undefined') {
       throw Error('Either fileName or sourceIndexName must be specified.');
     }
 
     if (typeof fileName !== 'undefined') {
-      return fileReaderFactory(
-        indexer,
-        fileName,
-        transform,
-        splitRegex,
-        verbose
-      );
+      return fileReaderFactory(indexer, fileName, transform, splitRegex, verbose);
     }
 
     if (typeof sourceIndexName !== 'undefined') {
@@ -371,7 +403,8 @@ async function transformer(ref) {
         sourceIndexName,
         transform,
         sourceClient,
-        query
+        query,
+        bufferSize
       );
     }
 

package/dist/node-es-transformer.esm.js

@@ -4,6 +4,8 @@ import glob from 'glob';
 import cliProgress from 'cli-progress';
 import elasticsearch from '@elastic/elasticsearch';
 
+var DEFAULT_BUFFER_SIZE = 1000;
+
 function createMappingFactory(ref) {
   var sourceClient = ref.sourceClient;
   var sourceIndexName = ref.sourceIndexName;
@@ -11,6 +13,7 @@ function createMappingFactory(ref) {
   var targetIndexName = ref.targetIndexName;
   var mappings = ref.mappings;
   var mappingsOverride = ref.mappingsOverride;
+  var indexMappingTotalFieldsLimit = ref.indexMappingTotalFieldsLimit;
   var verbose = ref.verbose;
 
   return async function () {
@@ -18,7 +21,9 @@ function createMappingFactory(ref) {
 
     if (sourceClient && sourceIndexName && typeof targetMappings === 'undefined') {
       try {
-        var mapping = await sourceClient.indices.getMapping({ index: sourceIndexName });
+        var mapping = await sourceClient.indices.getMapping({
+          index: sourceIndexName,
+        });
         targetMappings = mapping[sourceIndexName].mappings;
       } catch (err) {
         console.log('Error reading source mapping', err);
@@ -34,12 +39,17 @@ function createMappingFactory(ref) {
       }
 
       try {
-        var resp = await targetClient.indices.create(
-          {
-            index: targetIndexName,
-            body: { mappings: targetMappings },
-          }
-        );
+        var resp = await targetClient.indices.create({
+          index: targetIndexName,
+          body: Object.assign({}, {mappings: targetMappings},
+            (indexMappingTotalFieldsLimit !== undefined
+              ? {
+                  settings: {
+                    'index.mapping.total_fields.limit': indexMappingTotalFieldsLimit,
+                  },
+                }
+              : {})),
+        });
         if (verbose) { console.log('Created target mapping', resp); }
       } catch (err) {
         console.log('Error creating target mapping', err);
@@ -51,40 +61,44 @@ function createMappingFactory(ref) {
 function fileReaderFactory(indexer, fileName, transform, splitRegex, verbose) {
   function startIndex(files) {
     var file = files.shift();
-    var s = fs.createReadStream(file)
+    var s = fs
+      .createReadStream(file)
       .pipe(es.split(splitRegex))
-      .pipe(es.mapSync(function (line) {
-        s.pause();
-        try {
-          var doc = (typeof transform === 'function') ? transform(line) : line;
-          // if doc is undefined we'll skip indexing it
-          if (typeof doc === 'undefined') {
-            s.resume();
-            return;
-          }
-
-          // the transform callback may return an array of docs so we can emit
-          // multiple docs from a single line
-          if (Array.isArray(doc)) {
-            doc.forEach(function (d) { return indexer.add(d); });
-            return;
-          }
-
-          indexer.add(doc);
-        } catch (e) {
-          console.log('error', e);
-        }
-      })
-        .on('error', function (err) {
-          console.log('Error while reading file.', err);
-        })
-        .on('end', function () {
-          if (verbose) { console.log('Read entire file: ', file); }
-          indexer.finish();
-          if (files.length > 0) {
-            startIndex(files);
-          }
-        }));
+      .pipe(
+        es
+          .mapSync(function (line) {
+            s.pause();
+            try {
+              var doc = typeof transform === 'function' ? transform(line) : line;
+              // if doc is undefined we'll skip indexing it
+              if (typeof doc === 'undefined') {
+                s.resume();
+                return;
+              }
+
+              // the transform callback may return an array of docs so we can emit
+              // multiple docs from a single line
+              if (Array.isArray(doc)) {
+                doc.forEach(function (d) { return indexer.add(d); });
+                return;
+              }
+
+              indexer.add(doc);
+            } catch (e) {
+              console.log('error', e);
+            }
+          })
+          .on('error', function (err) {
+            console.log('Error while reading file.', err);
+          })
+          .on('end', function () {
+            if (verbose) { console.log('Read entire file: ', file); }
+            indexer.finish();
+            if (files.length > 0) {
+              startIndex(files);
+            }
+          })
+      );
 
     indexer.queueEmitter.on('resume', function () {
       s.resume();
@@ -102,45 +116,66 @@ var EventEmitter = require('events');
 
 var queueEmitter = new EventEmitter();
 
+var parallelCalls = 1;
+
 // a simple helper queue to bulk index documents
 function indexQueueFactory(ref) {
   var client = ref.targetClient;
   var targetIndexName = ref.targetIndexName;
-  var bufferSize = ref.bufferSize; if ( bufferSize === void 0 ) bufferSize = 1000;
+  var bufferSize = ref.bufferSize; if ( bufferSize === void 0 ) bufferSize = DEFAULT_BUFFER_SIZE;
   var skipHeader = ref.skipHeader; if ( skipHeader === void 0 ) skipHeader = false;
   var verbose = ref.verbose; if ( verbose === void 0 ) verbose = true;
 
   var buffer = [];
   var queue = [];
-  var ingesting = false;
+  var ingesting = 0;
+  var ingestTimes = [];
 
-  var ingest = async function (b) {
+  var ingest = function (b) {
     if (typeof b !== 'undefined') {
       queue.push(b);
       queueEmitter.emit('queue-size', queue.length);
     }
 
-    if (ingesting === false) {
+    if (ingestTimes.length > 5) { ingestTimes = ingestTimes.slice(-5); }
+
+    if (ingesting < parallelCalls) {
       var docs = queue.shift();
+
       queueEmitter.emit('queue-size', queue.length);
-      ingesting = true;
-      if (verbose) { console.log(("bulk ingest docs: " + (docs.length / 2) + ", queue length: " + (queue.length))); }
+      if (queue.length <= 5) {
+        queueEmitter.emit('resume');
+      }
 
-      try {
-        await client.bulk({ body: docs });
-        ingesting = false;
+      ingesting += 1;
+
+      if (verbose)
+        { console.log(("bulk ingest docs: " + (docs.length / 2) + ", queue length: " + (queue.length))); }
+
+      var start = Date.now();
+      client.bulk({ body: docs }).then(function () {
+        var end = Date.now();
+        var delta = end - start;
+        ingestTimes.push(delta);
+        ingesting -= 1;
+
+        var ingestTimesMovingAverage =
+          ingestTimes.length > 0 ? ingestTimes.reduce(function (p, c) { return p + c; }, 0) / ingestTimes.length : 0;
+        var ingestTimesMovingAverageSeconds = Math.floor(ingestTimesMovingAverage / 1000);
+
+        if (ingestTimes.length > 0 && ingestTimesMovingAverageSeconds < 30 && parallelCalls < 10) {
+          parallelCalls += 1;
+        } else if (
+          ingestTimes.length > 0 &&
+          ingestTimesMovingAverageSeconds >= 30 &&
+          parallelCalls > 1
+        ) {
+          parallelCalls -= 1;
+        }
         if (queue.length > 0) {
           ingest();
         }
-      } catch (err) {
-        console.log('bulk index error', err);
-      }
-    }
-
-    // console.log(`ingest: queue.length ${queue.length}`);
-    if (queue.length === 0) {
-      queueEmitter.emit('queue-size', 0);
-      queueEmitter.emit('resume');
+      });
     }
   };
 
@@ -157,7 +192,7 @@ function indexQueueFactory(ref) {
         queueEmitter.emit('resume');
       }
 
-      if (buffer.length >= (bufferSize * 2)) {
+      if (buffer.length >= bufferSize * 2) {
         ingest(buffer);
         buffer = [];
       }
@@ -171,12 +206,21 @@ function indexQueueFactory(ref) {
   };
 }
 
-var MAX_QUEUE_SIZE = 5;
+var MAX_QUEUE_SIZE = 15;
 
 // create a new progress bar instance and use shades_classic theme
 var progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
 
-function indexReaderFactory(indexer, sourceIndexName, transform, client, query) {
+function indexReaderFactory(
+  indexer,
+  sourceIndexName,
+  transform,
+  client,
+  query,
+  bufferSize
+) {
+  if ( bufferSize === void 0 ) bufferSize = DEFAULT_BUFFER_SIZE;
+
   return async function indexReader() {
     var responseQueue = [];
     var docsNum = 0;
@@ -185,7 +229,7 @@ function indexReaderFactory(indexer, sourceIndexName, transform, client, query)
       return client.search({
         index: sourceIndexName,
         scroll: '30s',
-        size: 10000,
+        size: bufferSize,
         query: query,
       });
     }
@@ -206,7 +250,7 @@ function indexReaderFactory(indexer, sourceIndexName, transform, client, query)
     function processHit(hit) {
       docsNum += 1;
       try {
-        var doc = (typeof transform === 'function') ? transform(hit._source) : hit._source; // eslint-disable-line no-underscore-dangle
+        var doc = typeof transform === 'function' ? transform(hit._source) : hit._source; // eslint-disable-line no-underscore-dangle
         // if doc is undefined we'll skip indexing it
         if (typeof doc === 'undefined') {
           return;
@@ -248,7 +292,7 @@ function indexReaderFactory(indexer, sourceIndexName, transform, client, query)
         }
 
         if (ingestQueueSize < MAX_QUEUE_SIZE) {
-        // get the next response if there are more docs to fetch
+          // get the next response if there are more docs to fetch
           var sc = await scroll(response._scroll_id); // eslint-disable-line no-await-in-loop,no-underscore-dangle,max-len
           scrollId = sc._scroll_id; // eslint-disable-line no-underscore-dangle
           responseQueue.push(sc);
@@ -262,7 +306,7 @@ function indexReaderFactory(indexer, sourceIndexName, transform, client, query)
       ingestQueueSize = size;
 
       if (!readActive && ingestQueueSize < MAX_QUEUE_SIZE) {
-      // get the next response if there are more docs to fetch
+        // get the next response if there are more docs to fetch
         var sc = await scroll(scrollId); // eslint-disable-line no-await-in-loop,no-underscore-dangle,max-len
         scrollId = sc._scroll_id; // eslint-disable-line no-underscore-dangle
         responseQueue.push(sc);
@@ -292,13 +336,14 @@ async function transformer(ref) {
   var deleteIndex = ref.deleteIndex; if ( deleteIndex === void 0 ) deleteIndex = false;
   var sourceClientConfig = ref.sourceClientConfig;
   var targetClientConfig = ref.targetClientConfig;
-  var bufferSize = ref.bufferSize; if ( bufferSize === void 0 ) bufferSize = 1000;
+  var bufferSize = ref.bufferSize; if ( bufferSize === void 0 ) bufferSize = DEFAULT_BUFFER_SIZE;
   var fileName = ref.fileName;
   var splitRegex = ref.splitRegex; if ( splitRegex === void 0 ) splitRegex = /\n/;
   var sourceIndexName = ref.sourceIndexName;
   var targetIndexName = ref.targetIndexName;
   var mappings = ref.mappings;
   var mappingsOverride = ref.mappingsOverride; if ( mappingsOverride === void 0 ) mappingsOverride = false;
+  var indexMappingTotalFieldsLimit = ref.indexMappingTotalFieldsLimit;
   var query = ref.query;
   var skipHeader = ref.skipHeader; if ( skipHeader === void 0 ) skipHeader = false;
   var transform = ref.transform;
@@ -324,6 +369,7 @@ async function transformer(ref) {
     targetIndexName: targetIndexName,
     mappings: mappings,
     mappingsOverride: mappingsOverride,
+    indexMappingTotalFieldsLimit: indexMappingTotalFieldsLimit,
     verbose: verbose,
   });
   var indexer = indexQueueFactory({
@@ -335,30 +381,16 @@ async function transformer(ref) {
   });
 
   function getReader() {
-    if (
-      typeof fileName !== 'undefined'
-      && typeof sourceIndexName !== 'undefined'
-    ) {
-      throw Error(
-        'Only either one of fileName or sourceIndexName can be specified.'
-      );
+    if (typeof fileName !== 'undefined' && typeof sourceIndexName !== 'undefined') {
+      throw Error('Only either one of fileName or sourceIndexName can be specified.');
     }
 
-    if (
-      typeof fileName === 'undefined'
-      && typeof sourceIndexName === 'undefined'
-    ) {
+    if (typeof fileName === 'undefined' && typeof sourceIndexName === 'undefined') {
       throw Error('Either fileName or sourceIndexName must be specified.');
     }
 
     if (typeof fileName !== 'undefined') {
-      return fileReaderFactory(
-        indexer,
-        fileName,
-        transform,
-        splitRegex,
-        verbose
-      );
+      return fileReaderFactory(indexer, fileName, transform, splitRegex, verbose);
     }
 
     if (typeof sourceIndexName !== 'undefined') {
@@ -367,7 +399,8 @@ async function transformer(ref) {
         sourceIndexName,
         transform,
         sourceClient,
-        query
+        query,
+        bufferSize
       );
     }
 

package/package.json

@@ -14,7 +14,7 @@
   "license": "Apache-2.0",
   "author": "Walter Rafelsberger <walter@rafelsberger.at>",
   "contributors": [],
-  "version": "1.0.0-alpha10",
+  "version": "1.0.0-alpha12",
   "main": "dist/node-es-transformer.cjs.js",
   "module": "dist/node-es-transformer.esm.js",
   "dependencies": {
@@ -25,11 +25,16 @@
   },
   "devDependencies": {
     "acorn": "^6.4.2",
+    "commit-and-tag-version": "^11.3.0",
+    "cz-conventional-changelog": "^3.3.0",
     "eslint": "8.2.0",
     "eslint-config-airbnb": "19.0.4",
+    "eslint-config-prettier": "^9.0.0",
     "eslint-plugin-import": "2.27.5",
     "eslint-plugin-jsx-a11y": "6.7.1",
+    "eslint-plugin-prettier": "^3.3.1",
     "eslint-plugin-react": "7.32.2",
+    "prettier": "^2.2.1",
     "rollup": "0.66.6",
     "rollup-plugin-buble": "0.19.6",
     "rollup-plugin-commonjs": "8.0.2",
@@ -39,9 +44,15 @@
     "build": "rollup -c",
     "dev": "rollup -c -w",
     "test": "node test/test.js",
-    "pretest": "npm run build"
+    "pretest": "npm run build",
+    "release": "commit-and-tag-version"
   },
   "files": [
     "dist"
-  ]
+  ],
+  "config": {
+    "commitizen": {
+      "path": "./node_modules/cz-conventional-changelog"
+    }
+  }
 }