node-es-transformer 1.0.0-alpha10 → 1.0.0-alpha11

package/CHANGELOG.md

@@ -0,0 +1,32 @@
+# Changelog
+
+All notable changes to this project will be documented in this file. See [commit-and-tag-version](https://github.com/absolute-version/commit-and-tag-version) for commit guidelines.
+
+## [1.0.0-alpha11](https://github.com/walterra/node-es-transformer/compare/v1.0.0-alpha10...v1.0.0-alpha11) (2023-10-12)
+
+### Features
+
+- new option 'indexMappingTotalFieldsLimit' ([92edad1](https://github.com/walterra/node-es-transformer/commit/92edad18da7186d3881fc181e6e88b7929bed2d4))
+
+### Bug Fixes
+
+- fixes bufferSize to be applied to index reader too ([ffc3749](https://github.com/walterra/node-es-transformer/commit/ffc3749e296cd39f39924571c197986addc756ff))
+
+## [`v1.0.0-alpha10`](https://github.com/walterra/node-es-transformer/releases/tag/v1.0.0-alpha10)
+
+- New option `mappingsOverride` (0b951e1).
+- New option `query` (45f91db).
+
+## [`v1.0.0-alpha9`](https://github.com/walterra/node-es-transformer/releases/tag/v1.0.0-alpha9)
+
+- Source and target configs are now expected to be passed in as complete client configs instead of individual parameters (5e6d0c7).
+
+## [`v1.0.0-alpha8`](https://github.com/walterra/node-es-transformer/releases/tag/v1.0.0-alpha8)
+
+- Exposes events and introduces `finish` event (a3e5810).
+- Drop support for `_type` from `6.x` indices (3a26a84).
+
+## [`v1.0.0-alpha7`](https://github.com/walterra/node-es-transformer/releases/tag/v1.0.0-alpha7)
+
+- This version locks down `event-stream` to version `3.3.4` because of the security issue described here: https://github.com/dominictarr/event-stream/issues/116
+- Last version to support `_type` from `6.x` indices.

package/README.md

@@ -1,6 +1,7 @@
 [![npm](https://img.shields.io/npm/v/node-es-transformer.svg?maxAge=2592000)](https://www.npmjs.com/package/node-es-transformer)
 [![npm](https://img.shields.io/npm/l/node-es-transformer.svg?maxAge=2592000)](https://www.npmjs.com/package/node-es-transformer)
 [![npm](https://img.shields.io/npm/dt/node-es-transformer.svg?maxAge=2592000)](https://www.npmjs.com/package/node-es-transformer)
+[![Commitizen friendly](https://img.shields.io/badge/commitizen-friendly-brightgreen.svg)](http://commitizen.github.io/cz-cli/)
 
 # node-es-transformer
 
@@ -115,6 +116,7 @@ transformer({
 - `targetIndexName`: The target Elasticsearch index where documents will be indexed.
 - `mappings`: Optional Elasticsearch document mappings. If not set and you're reindexing from another index, the mappings from the existing index will be used.
 - `mappingsOverride`: If you're reindexing and this is set to `true`, `mappings` will be applied on top of the source index's mappings. Defaults to `false`.
+- `indexMappingTotalFieldsLimit`: Optional field limit for the target index to be created that will be passed on as the `index.mapping.total_fields.limit` setting.
 - `query`: Optional Elasticsearch [DSL query](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html) to filter documents from the source index.
 - `skipHeader`: If true, skips the first line of the source file. Defaults to `false`.
 - `transform(line)`: A callback function which allows the transformation of a source line into one or several documents.

package/dist/node-es-transformer.cjs.js

@@ -8,6 +8,8 @@ var glob = _interopDefault(require('glob'));
 var cliProgress = _interopDefault(require('cli-progress'));
 var elasticsearch = _interopDefault(require('@elastic/elasticsearch'));
 
+var DEFAULT_BUFFER_SIZE = 1000;
+
 function createMappingFactory(ref) {
   var sourceClient = ref.sourceClient;
   var sourceIndexName = ref.sourceIndexName;
@@ -15,6 +17,7 @@ function createMappingFactory(ref) {
   var targetIndexName = ref.targetIndexName;
   var mappings = ref.mappings;
   var mappingsOverride = ref.mappingsOverride;
+  var indexMappingTotalFieldsLimit = ref.indexMappingTotalFieldsLimit;
   var verbose = ref.verbose;
 
   return async function () {
@@ -22,7 +25,9 @@ function createMappingFactory(ref) {
 
     if (sourceClient && sourceIndexName && typeof targetMappings === 'undefined') {
       try {
-        var mapping = await sourceClient.indices.getMapping({ index: sourceIndexName });
+        var mapping = await sourceClient.indices.getMapping({
+          index: sourceIndexName,
+        });
         targetMappings = mapping[sourceIndexName].mappings;
       } catch (err) {
         console.log('Error reading source mapping', err);
@@ -38,12 +43,17 @@ function createMappingFactory(ref) {
       }
 
       try {
-        var resp = await targetClient.indices.create(
-          {
-            index: targetIndexName,
-            body: { mappings: targetMappings },
-          }
-        );
+        var resp = await targetClient.indices.create({
+          index: targetIndexName,
+          body: Object.assign({}, {mappings: targetMappings},
+            (indexMappingTotalFieldsLimit !== undefined
+              ? {
+                  settings: {
+                    'index.mapping.total_fields.limit': indexMappingTotalFieldsLimit,
+                  },
+                }
+              : {})),
+        });
         if (verbose) { console.log('Created target mapping', resp); }
       } catch (err) {
         console.log('Error creating target mapping', err);
@@ -55,40 +65,44 @@ function createMappingFactory(ref) {
 function fileReaderFactory(indexer, fileName, transform, splitRegex, verbose) {
   function startIndex(files) {
     var file = files.shift();
-    var s = fs.createReadStream(file)
+    var s = fs
+      .createReadStream(file)
       .pipe(es.split(splitRegex))
-      .pipe(es.mapSync(function (line) {
-        s.pause();
-        try {
-          var doc = (typeof transform === 'function') ? transform(line) : line;
-          // if doc is undefined we'll skip indexing it
-          if (typeof doc === 'undefined') {
-            s.resume();
-            return;
-          }
-
-          // the transform callback may return an array of docs so we can emit
-          // multiple docs from a single line
-          if (Array.isArray(doc)) {
-            doc.forEach(function (d) { return indexer.add(d); });
-            return;
-          }
-
-          indexer.add(doc);
-        } catch (e) {
-          console.log('error', e);
-        }
-      })
-        .on('error', function (err) {
-          console.log('Error while reading file.', err);
-        })
-        .on('end', function () {
-          if (verbose) { console.log('Read entire file: ', file); }
-          indexer.finish();
-          if (files.length > 0) {
-            startIndex(files);
-          }
-        }));
+      .pipe(
+        es
+          .mapSync(function (line) {
+            s.pause();
+            try {
+              var doc = typeof transform === 'function' ? transform(line) : line;
+              // if doc is undefined we'll skip indexing it
+              if (typeof doc === 'undefined') {
+                s.resume();
+                return;
+              }
+
+              // the transform callback may return an array of docs so we can emit
+              // multiple docs from a single line
+              if (Array.isArray(doc)) {
+                doc.forEach(function (d) { return indexer.add(d); });
+                return;
+              }
+
+              indexer.add(doc);
+            } catch (e) {
+              console.log('error', e);
+            }
+          })
+          .on('error', function (err) {
+            console.log('Error while reading file.', err);
+          })
+          .on('end', function () {
+            if (verbose) { console.log('Read entire file: ', file); }
+            indexer.finish();
+            if (files.length > 0) {
+              startIndex(files);
+            }
+          })
+      );
 
     indexer.queueEmitter.on('resume', function () {
       s.resume();
@@ -110,7 +124,7 @@ var queueEmitter = new EventEmitter();
 function indexQueueFactory(ref) {
   var client = ref.targetClient;
   var targetIndexName = ref.targetIndexName;
-  var bufferSize = ref.bufferSize; if ( bufferSize === void 0 ) bufferSize = 1000;
+  var bufferSize = ref.bufferSize; if ( bufferSize === void 0 ) bufferSize = DEFAULT_BUFFER_SIZE;
   var skipHeader = ref.skipHeader; if ( skipHeader === void 0 ) skipHeader = false;
   var verbose = ref.verbose; if ( verbose === void 0 ) verbose = true;
 
@@ -128,7 +142,8 @@ function indexQueueFactory(ref) {
       var docs = queue.shift();
       queueEmitter.emit('queue-size', queue.length);
       ingesting = true;
-      if (verbose) { console.log(("bulk ingest docs: " + (docs.length / 2) + ", queue length: " + (queue.length))); }
+      if (verbose)
+        { console.log(("bulk ingest docs: " + (docs.length / 2) + ", queue length: " + (queue.length))); }
 
       try {
         await client.bulk({ body: docs });
@@ -161,7 +176,7 @@ function indexQueueFactory(ref) {
         queueEmitter.emit('resume');
       }
 
-      if (buffer.length >= (bufferSize * 2)) {
+      if (buffer.length >= bufferSize * 2) {
         ingest(buffer);
         buffer = [];
       }
@@ -180,7 +195,16 @@ var MAX_QUEUE_SIZE = 5;
 // create a new progress bar instance and use shades_classic theme
 var progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
 
-function indexReaderFactory(indexer, sourceIndexName, transform, client, query) {
+function indexReaderFactory(
+  indexer,
+  sourceIndexName,
+  transform,
+  client,
+  query,
+  bufferSize
+) {
+  if ( bufferSize === void 0 ) bufferSize = DEFAULT_BUFFER_SIZE;
+
   return async function indexReader() {
     var responseQueue = [];
     var docsNum = 0;
@@ -189,7 +213,7 @@ function indexReaderFactory(indexer, sourceIndexName, transform, client, query)
       return client.search({
         index: sourceIndexName,
         scroll: '30s',
-        size: 10000,
+        size: bufferSize,
         query: query,
       });
     }
@@ -210,7 +234,7 @@ function indexReaderFactory(indexer, sourceIndexName, transform, client, query)
     function processHit(hit) {
       docsNum += 1;
       try {
-        var doc = (typeof transform === 'function') ? transform(hit._source) : hit._source; // eslint-disable-line no-underscore-dangle
+        var doc = typeof transform === 'function' ? transform(hit._source) : hit._source; // eslint-disable-line no-underscore-dangle
         // if doc is undefined we'll skip indexing it
         if (typeof doc === 'undefined') {
           return;
@@ -252,7 +276,7 @@ function indexReaderFactory(indexer, sourceIndexName, transform, client, query)
         }
 
         if (ingestQueueSize < MAX_QUEUE_SIZE) {
-        // get the next response if there are more docs to fetch
+          // get the next response if there are more docs to fetch
           var sc = await scroll(response._scroll_id); // eslint-disable-line no-await-in-loop,no-underscore-dangle,max-len
           scrollId = sc._scroll_id; // eslint-disable-line no-underscore-dangle
           responseQueue.push(sc);
@@ -266,7 +290,7 @@ function indexReaderFactory(indexer, sourceIndexName, transform, client, query)
       ingestQueueSize = size;
 
       if (!readActive && ingestQueueSize < MAX_QUEUE_SIZE) {
-      // get the next response if there are more docs to fetch
+        // get the next response if there are more docs to fetch
         var sc = await scroll(scrollId); // eslint-disable-line no-await-in-loop,no-underscore-dangle,max-len
         scrollId = sc._scroll_id; // eslint-disable-line no-underscore-dangle
         responseQueue.push(sc);
@@ -296,13 +320,14 @@ async function transformer(ref) {
   var deleteIndex = ref.deleteIndex; if ( deleteIndex === void 0 ) deleteIndex = false;
   var sourceClientConfig = ref.sourceClientConfig;
   var targetClientConfig = ref.targetClientConfig;
-  var bufferSize = ref.bufferSize; if ( bufferSize === void 0 ) bufferSize = 1000;
+  var bufferSize = ref.bufferSize; if ( bufferSize === void 0 ) bufferSize = DEFAULT_BUFFER_SIZE;
   var fileName = ref.fileName;
   var splitRegex = ref.splitRegex; if ( splitRegex === void 0 ) splitRegex = /\n/;
   var sourceIndexName = ref.sourceIndexName;
   var targetIndexName = ref.targetIndexName;
   var mappings = ref.mappings;
   var mappingsOverride = ref.mappingsOverride; if ( mappingsOverride === void 0 ) mappingsOverride = false;
+  var indexMappingTotalFieldsLimit = ref.indexMappingTotalFieldsLimit;
   var query = ref.query;
   var skipHeader = ref.skipHeader; if ( skipHeader === void 0 ) skipHeader = false;
   var transform = ref.transform;
@@ -328,6 +353,7 @@ async function transformer(ref) {
     targetIndexName: targetIndexName,
     mappings: mappings,
     mappingsOverride: mappingsOverride,
+    indexMappingTotalFieldsLimit: indexMappingTotalFieldsLimit,
     verbose: verbose,
   });
   var indexer = indexQueueFactory({
@@ -339,30 +365,16 @@ async function transformer(ref) {
   });
 
   function getReader() {
-    if (
-      typeof fileName !== 'undefined'
-      && typeof sourceIndexName !== 'undefined'
-    ) {
-      throw Error(
-        'Only either one of fileName or sourceIndexName can be specified.'
-      );
+    if (typeof fileName !== 'undefined' && typeof sourceIndexName !== 'undefined') {
+      throw Error('Only either one of fileName or sourceIndexName can be specified.');
     }
 
-    if (
-      typeof fileName === 'undefined'
-      && typeof sourceIndexName === 'undefined'
-    ) {
+    if (typeof fileName === 'undefined' && typeof sourceIndexName === 'undefined') {
       throw Error('Either fileName or sourceIndexName must be specified.');
     }
 
     if (typeof fileName !== 'undefined') {
-      return fileReaderFactory(
-        indexer,
-        fileName,
-        transform,
-        splitRegex,
-        verbose
-      );
+      return fileReaderFactory(indexer, fileName, transform, splitRegex, verbose);
     }
 
     if (typeof sourceIndexName !== 'undefined') {
@@ -371,7 +383,8 @@ async function transformer(ref) {
         sourceIndexName,
         transform,
         sourceClient,
-        query
+        query,
+        bufferSize
       );
     }
 

package/dist/node-es-transformer.esm.js

@@ -4,6 +4,8 @@ import glob from 'glob';
 import cliProgress from 'cli-progress';
 import elasticsearch from '@elastic/elasticsearch';
 
+var DEFAULT_BUFFER_SIZE = 1000;
+
 function createMappingFactory(ref) {
   var sourceClient = ref.sourceClient;
   var sourceIndexName = ref.sourceIndexName;
@@ -11,6 +13,7 @@ function createMappingFactory(ref) {
   var targetIndexName = ref.targetIndexName;
   var mappings = ref.mappings;
   var mappingsOverride = ref.mappingsOverride;
+  var indexMappingTotalFieldsLimit = ref.indexMappingTotalFieldsLimit;
   var verbose = ref.verbose;
 
   return async function () {
@@ -18,7 +21,9 @@ function createMappingFactory(ref) {
 
     if (sourceClient && sourceIndexName && typeof targetMappings === 'undefined') {
       try {
-        var mapping = await sourceClient.indices.getMapping({ index: sourceIndexName });
+        var mapping = await sourceClient.indices.getMapping({
+          index: sourceIndexName,
+        });
         targetMappings = mapping[sourceIndexName].mappings;
       } catch (err) {
         console.log('Error reading source mapping', err);
@@ -34,12 +39,17 @@ function createMappingFactory(ref) {
       }
 
       try {
-        var resp = await targetClient.indices.create(
-          {
-            index: targetIndexName,
-            body: { mappings: targetMappings },
-          }
-        );
+        var resp = await targetClient.indices.create({
+          index: targetIndexName,
+          body: Object.assign({}, {mappings: targetMappings},
+            (indexMappingTotalFieldsLimit !== undefined
+              ? {
+                  settings: {
+                    'index.mapping.total_fields.limit': indexMappingTotalFieldsLimit,
+                  },
+                }
+              : {})),
+        });
         if (verbose) { console.log('Created target mapping', resp); }
       } catch (err) {
         console.log('Error creating target mapping', err);
@@ -51,40 +61,44 @@ function createMappingFactory(ref) {
 function fileReaderFactory(indexer, fileName, transform, splitRegex, verbose) {
   function startIndex(files) {
     var file = files.shift();
-    var s = fs.createReadStream(file)
+    var s = fs
+      .createReadStream(file)
       .pipe(es.split(splitRegex))
-      .pipe(es.mapSync(function (line) {
-        s.pause();
-        try {
-          var doc = (typeof transform === 'function') ? transform(line) : line;
-          // if doc is undefined we'll skip indexing it
-          if (typeof doc === 'undefined') {
-            s.resume();
-            return;
-          }
-
-          // the transform callback may return an array of docs so we can emit
-          // multiple docs from a single line
-          if (Array.isArray(doc)) {
-            doc.forEach(function (d) { return indexer.add(d); });
-            return;
-          }
-
-          indexer.add(doc);
-        } catch (e) {
-          console.log('error', e);
-        }
-      })
-        .on('error', function (err) {
-          console.log('Error while reading file.', err);
-        })
-        .on('end', function () {
-          if (verbose) { console.log('Read entire file: ', file); }
-          indexer.finish();
-          if (files.length > 0) {
-            startIndex(files);
-          }
-        }));
+      .pipe(
+        es
+          .mapSync(function (line) {
+            s.pause();
+            try {
+              var doc = typeof transform === 'function' ? transform(line) : line;
+              // if doc is undefined we'll skip indexing it
+              if (typeof doc === 'undefined') {
+                s.resume();
+                return;
+              }
+
+              // the transform callback may return an array of docs so we can emit
+              // multiple docs from a single line
+              if (Array.isArray(doc)) {
+                doc.forEach(function (d) { return indexer.add(d); });
+                return;
+              }
+
+              indexer.add(doc);
+            } catch (e) {
+              console.log('error', e);
+            }
+          })
+          .on('error', function (err) {
+            console.log('Error while reading file.', err);
+          })
+          .on('end', function () {
+            if (verbose) { console.log('Read entire file: ', file); }
+            indexer.finish();
+            if (files.length > 0) {
+              startIndex(files);
+            }
+          })
+      );
 
     indexer.queueEmitter.on('resume', function () {
       s.resume();
@@ -106,7 +120,7 @@ var queueEmitter = new EventEmitter();
 function indexQueueFactory(ref) {
   var client = ref.targetClient;
   var targetIndexName = ref.targetIndexName;
-  var bufferSize = ref.bufferSize; if ( bufferSize === void 0 ) bufferSize = 1000;
+  var bufferSize = ref.bufferSize; if ( bufferSize === void 0 ) bufferSize = DEFAULT_BUFFER_SIZE;
   var skipHeader = ref.skipHeader; if ( skipHeader === void 0 ) skipHeader = false;
   var verbose = ref.verbose; if ( verbose === void 0 ) verbose = true;
 
@@ -124,7 +138,8 @@ function indexQueueFactory(ref) {
       var docs = queue.shift();
       queueEmitter.emit('queue-size', queue.length);
       ingesting = true;
-      if (verbose) { console.log(("bulk ingest docs: " + (docs.length / 2) + ", queue length: " + (queue.length))); }
+      if (verbose)
+        { console.log(("bulk ingest docs: " + (docs.length / 2) + ", queue length: " + (queue.length))); }
 
       try {
         await client.bulk({ body: docs });
@@ -157,7 +172,7 @@ function indexQueueFactory(ref) {
         queueEmitter.emit('resume');
       }
 
-      if (buffer.length >= (bufferSize * 2)) {
+      if (buffer.length >= bufferSize * 2) {
         ingest(buffer);
         buffer = [];
       }
@@ -176,7 +191,16 @@ var MAX_QUEUE_SIZE = 5;
 // create a new progress bar instance and use shades_classic theme
 var progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
 
-function indexReaderFactory(indexer, sourceIndexName, transform, client, query) {
+function indexReaderFactory(
+  indexer,
+  sourceIndexName,
+  transform,
+  client,
+  query,
+  bufferSize
+) {
+  if ( bufferSize === void 0 ) bufferSize = DEFAULT_BUFFER_SIZE;
+
   return async function indexReader() {
     var responseQueue = [];
     var docsNum = 0;
@@ -185,7 +209,7 @@ function indexReaderFactory(indexer, sourceIndexName, transform, client, query)
       return client.search({
         index: sourceIndexName,
         scroll: '30s',
-        size: 10000,
+        size: bufferSize,
         query: query,
       });
     }
@@ -206,7 +230,7 @@ function indexReaderFactory(indexer, sourceIndexName, transform, client, query)
     function processHit(hit) {
       docsNum += 1;
       try {
-        var doc = (typeof transform === 'function') ? transform(hit._source) : hit._source; // eslint-disable-line no-underscore-dangle
+        var doc = typeof transform === 'function' ? transform(hit._source) : hit._source; // eslint-disable-line no-underscore-dangle
         // if doc is undefined we'll skip indexing it
         if (typeof doc === 'undefined') {
           return;
@@ -248,7 +272,7 @@ function indexReaderFactory(indexer, sourceIndexName, transform, client, query)
         }
 
         if (ingestQueueSize < MAX_QUEUE_SIZE) {
-        // get the next response if there are more docs to fetch
+          // get the next response if there are more docs to fetch
           var sc = await scroll(response._scroll_id); // eslint-disable-line no-await-in-loop,no-underscore-dangle,max-len
           scrollId = sc._scroll_id; // eslint-disable-line no-underscore-dangle
           responseQueue.push(sc);
@@ -262,7 +286,7 @@ function indexReaderFactory(indexer, sourceIndexName, transform, client, query)
       ingestQueueSize = size;
 
       if (!readActive && ingestQueueSize < MAX_QUEUE_SIZE) {
-      // get the next response if there are more docs to fetch
+        // get the next response if there are more docs to fetch
         var sc = await scroll(scrollId); // eslint-disable-line no-await-in-loop,no-underscore-dangle,max-len
         scrollId = sc._scroll_id; // eslint-disable-line no-underscore-dangle
         responseQueue.push(sc);
@@ -292,13 +316,14 @@ async function transformer(ref) {
   var deleteIndex = ref.deleteIndex; if ( deleteIndex === void 0 ) deleteIndex = false;
   var sourceClientConfig = ref.sourceClientConfig;
   var targetClientConfig = ref.targetClientConfig;
-  var bufferSize = ref.bufferSize; if ( bufferSize === void 0 ) bufferSize = 1000;
+  var bufferSize = ref.bufferSize; if ( bufferSize === void 0 ) bufferSize = DEFAULT_BUFFER_SIZE;
   var fileName = ref.fileName;
   var splitRegex = ref.splitRegex; if ( splitRegex === void 0 ) splitRegex = /\n/;
   var sourceIndexName = ref.sourceIndexName;
   var targetIndexName = ref.targetIndexName;
   var mappings = ref.mappings;
   var mappingsOverride = ref.mappingsOverride; if ( mappingsOverride === void 0 ) mappingsOverride = false;
+  var indexMappingTotalFieldsLimit = ref.indexMappingTotalFieldsLimit;
   var query = ref.query;
   var skipHeader = ref.skipHeader; if ( skipHeader === void 0 ) skipHeader = false;
   var transform = ref.transform;
@@ -324,6 +349,7 @@ async function transformer(ref) {
     targetIndexName: targetIndexName,
     mappings: mappings,
     mappingsOverride: mappingsOverride,
+    indexMappingTotalFieldsLimit: indexMappingTotalFieldsLimit,
     verbose: verbose,
   });
   var indexer = indexQueueFactory({
@@ -335,30 +361,16 @@ async function transformer(ref) {
   });
 
   function getReader() {
-    if (
-      typeof fileName !== 'undefined'
-      && typeof sourceIndexName !== 'undefined'
-    ) {
-      throw Error(
-        'Only either one of fileName or sourceIndexName can be specified.'
-      );
+    if (typeof fileName !== 'undefined' && typeof sourceIndexName !== 'undefined') {
+      throw Error('Only either one of fileName or sourceIndexName can be specified.');
     }
 
-    if (
-      typeof fileName === 'undefined'
-      && typeof sourceIndexName === 'undefined'
-    ) {
+    if (typeof fileName === 'undefined' && typeof sourceIndexName === 'undefined') {
       throw Error('Either fileName or sourceIndexName must be specified.');
     }
 
     if (typeof fileName !== 'undefined') {
-      return fileReaderFactory(
-        indexer,
-        fileName,
-        transform,
-        splitRegex,
-        verbose
-      );
+      return fileReaderFactory(indexer, fileName, transform, splitRegex, verbose);
     }
 
     if (typeof sourceIndexName !== 'undefined') {
@@ -367,7 +379,8 @@ async function transformer(ref) {
         sourceIndexName,
         transform,
         sourceClient,
-        query
+        query,
+        bufferSize
       );
     }
 

package/package.json

@@ -14,7 +14,7 @@
   "license": "Apache-2.0",
   "author": "Walter Rafelsberger <walter@rafelsberger.at>",
   "contributors": [],
-  "version": "1.0.0-alpha10",
+  "version": "1.0.0-alpha11",
   "main": "dist/node-es-transformer.cjs.js",
   "module": "dist/node-es-transformer.esm.js",
   "dependencies": {
@@ -25,11 +25,16 @@
   },
   "devDependencies": {
     "acorn": "^6.4.2",
+    "commit-and-tag-version": "^11.3.0",
+    "cz-conventional-changelog": "^3.3.0",
     "eslint": "8.2.0",
     "eslint-config-airbnb": "19.0.4",
+    "eslint-config-prettier": "^9.0.0",
     "eslint-plugin-import": "2.27.5",
     "eslint-plugin-jsx-a11y": "6.7.1",
+    "eslint-plugin-prettier": "^3.3.1",
     "eslint-plugin-react": "7.32.2",
+    "prettier": "^2.2.1",
     "rollup": "0.66.6",
     "rollup-plugin-buble": "0.19.6",
     "rollup-plugin-commonjs": "8.0.2",
@@ -39,9 +44,15 @@
     "build": "rollup -c",
     "dev": "rollup -c -w",
     "test": "node test/test.js",
-    "pretest": "npm run build"
+    "pretest": "npm run build",
+    "release": "commit-and-tag-version"
   },
   "files": [
     "dist"
-  ]
+  ],
+  "config": {
+    "commitizen": {
+      "path": "./node_modules/cz-conventional-changelog"
+    }
+  }
 }