node-es-transformer 1.0.0-beta1 → 1.0.0-beta2

package/README.md

@@ -118,6 +118,7 @@ transformer({
 - `mappings`: Optional Elasticsearch document mappings. If not set and you're reindexing from another index, the mappings from the existing index will be used.
 - `mappingsOverride`: If you're reindexing and this is set to `true`, `mappings` will be applied on top of the source index's mappings. Defaults to `false`.
 - `indexMappingTotalFieldsLimit`: Optional field limit for the target index to be created that will be passed on as the `index.mapping.total_fields.limit` setting.
+- `populatedFields`: If `true`, fetches a set of random documents to identify which fields are actually used by documents. Can be useful for indices with lots of field mappings to increase query/reindex performance. Defaults to `false`.
 - `query`: Optional Elasticsearch [DSL query](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html) to filter documents from the source index.
 - `skipHeader`: If true, skips the first line of the source file. Defaults to `false`.
 - `transform(line)`: A callback function which allows the transformation of a source line into one or several documents.
@@ -152,6 +153,8 @@ docker pull docker.elastic.co/elasticsearch/elasticsearch:8.10.4
 docker run --name es01 --net elastic -p 9200:9200 -it -m 1GB -e "discovery.type=single-node" -e "xpack.security.enabled=false" docker.elastic.co/elasticsearch/elasticsearch:8.10.4
 ```
 
+To commit, use `cz`. To prepare a release, use e.g. `yarn release -- --release-as 1.0.0-beta2`.
+
 ## License
 
 [Apache 2.0](LICENSE).

package/dist/node-es-transformer.cjs.js

@@ -77,7 +77,16 @@ function fileReaderFactory(indexer, fileName, transform, splitRegex, verbose) {
         es
           .mapSync(function (line) {
             try {
-              var doc = typeof transform === 'function' ? transform(line) : line;
+              // skip empty lines
+              if (line === '') {
+                return;
+              }
+
+              var doc =
+                typeof transform === 'function'
+                  ? JSON.stringify(transform(JSON.parse(line)))
+                  : line;
+
               // if doc is undefined we'll skip indexing it
               if (typeof doc === 'undefined') {
                 s.resume();
@@ -269,40 +278,73 @@ function indexReaderFactory(
   transform,
   client,
   query,
-  bufferSize
+  bufferSize,
+  populatedFields
 ) {
   if ( bufferSize === void 0 ) bufferSize = DEFAULT_BUFFER_SIZE;
+  if ( populatedFields === void 0 ) populatedFields = false;
 
   return async function indexReader() {
     var responseQueue = [];
     var docsNum = 0;
 
-    function search() {
-      return client.search({
-        index: sourceIndexName,
-        scroll: '30s',
+    async function fetchPopulatedFields() {
+      try {
+        var response = await client.search({
+          index: sourceIndexName,
+          size: bufferSize,
+          query: {
+            function_score: {
+              query: query,
+              random_score: {},
+            },
+          },
+        });
+
+        // Get all field names for each returned doc and flatten it
+        // to a list of unique field names used across all docs.
+        return new Set(response.hits.hits.map(function (d) { return Object.keys(d._source); }).flat(1));
+      } catch (e) {
+        console.log('error', e);
+      }
+    }
+
+    function search(fields) {
+      return client.search(Object.assign({}, {index: sourceIndexName,
+        scroll: '600s',
         size: bufferSize,
-        query: query,
-      });
+        query: query},
+        (fields ? { _source: fields } : {})));
     }
 
     function scroll(id) {
       return client.scroll({
         scroll_id: id,
-        scroll: '30s',
+        scroll: '600s',
       });
     }
 
+    var fieldsWithData;
+
+    // identify populated fields
+    if (populatedFields) {
+      fieldsWithData = await fetchPopulatedFields();
+      console.log('fieldsWithData', fieldsWithData);
+    }
+
     // start things off by searching, setting a scroll timeout, and pushing
     // our first response into the queue to be processed
-    var se = await search();
+    var se = await search(fieldsWithData);
     responseQueue.push(se);
     progressBar.start(se.hits.total.value, 0);
+    console.log('se', se.hits.hits[0]);
 
     function processHit(hit) {
       docsNum += 1;
       try {
         var doc = typeof transform === 'function' ? transform(hit._source) : hit._source; // eslint-disable-line no-underscore-dangle
+        // console.log('doc', doc);
+
         // if doc is undefined we'll skip indexing it
         if (typeof doc === 'undefined') {
           return;
@@ -398,6 +440,7 @@ async function transformer(ref) {
   var mappings = ref.mappings;
   var mappingsOverride = ref.mappingsOverride; if ( mappingsOverride === void 0 ) mappingsOverride = false;
   var indexMappingTotalFieldsLimit = ref.indexMappingTotalFieldsLimit;
+  var populatedFields = ref.populatedFields; if ( populatedFields === void 0 ) populatedFields = false;
   var query = ref.query;
   var skipHeader = ref.skipHeader; if ( skipHeader === void 0 ) skipHeader = false;
   var transform = ref.transform;
@@ -454,7 +497,8 @@ async function transformer(ref) {
         transform,
         sourceClient,
         query,
-        bufferSize
+        bufferSize,
+        populatedFields
       );
     }
 

package/dist/node-es-transformer.esm.js

@@ -73,7 +73,16 @@ function fileReaderFactory(indexer, fileName, transform, splitRegex, verbose) {
         es
           .mapSync(function (line) {
             try {
-              var doc = typeof transform === 'function' ? transform(line) : line;
+              // skip empty lines
+              if (line === '') {
+                return;
+              }
+
+              var doc =
+                typeof transform === 'function'
+                  ? JSON.stringify(transform(JSON.parse(line)))
+                  : line;
+
               // if doc is undefined we'll skip indexing it
               if (typeof doc === 'undefined') {
                 s.resume();
@@ -265,40 +274,73 @@ function indexReaderFactory(
   transform,
   client,
   query,
-  bufferSize
+  bufferSize,
+  populatedFields
 ) {
   if ( bufferSize === void 0 ) bufferSize = DEFAULT_BUFFER_SIZE;
+  if ( populatedFields === void 0 ) populatedFields = false;
 
   return async function indexReader() {
     var responseQueue = [];
     var docsNum = 0;
 
-    function search() {
-      return client.search({
-        index: sourceIndexName,
-        scroll: '30s',
+    async function fetchPopulatedFields() {
+      try {
+        var response = await client.search({
+          index: sourceIndexName,
+          size: bufferSize,
+          query: {
+            function_score: {
+              query: query,
+              random_score: {},
+            },
+          },
+        });
+
+        // Get all field names for each returned doc and flatten it
+        // to a list of unique field names used across all docs.
+        return new Set(response.hits.hits.map(function (d) { return Object.keys(d._source); }).flat(1));
+      } catch (e) {
+        console.log('error', e);
+      }
+    }
+
+    function search(fields) {
+      return client.search(Object.assign({}, {index: sourceIndexName,
+        scroll: '600s',
         size: bufferSize,
-        query: query,
-      });
+        query: query},
+        (fields ? { _source: fields } : {})));
     }
 
     function scroll(id) {
       return client.scroll({
         scroll_id: id,
-        scroll: '30s',
+        scroll: '600s',
       });
     }
 
+    var fieldsWithData;
+
+    // identify populated fields
+    if (populatedFields) {
+      fieldsWithData = await fetchPopulatedFields();
+      console.log('fieldsWithData', fieldsWithData);
+    }
+
     // start things off by searching, setting a scroll timeout, and pushing
     // our first response into the queue to be processed
-    var se = await search();
+    var se = await search(fieldsWithData);
     responseQueue.push(se);
     progressBar.start(se.hits.total.value, 0);
+    console.log('se', se.hits.hits[0]);
 
     function processHit(hit) {
       docsNum += 1;
       try {
         var doc = typeof transform === 'function' ? transform(hit._source) : hit._source; // eslint-disable-line no-underscore-dangle
+        // console.log('doc', doc);
+
         // if doc is undefined we'll skip indexing it
         if (typeof doc === 'undefined') {
           return;
@@ -394,6 +436,7 @@ async function transformer(ref) {
   var mappings = ref.mappings;
   var mappingsOverride = ref.mappingsOverride; if ( mappingsOverride === void 0 ) mappingsOverride = false;
   var indexMappingTotalFieldsLimit = ref.indexMappingTotalFieldsLimit;
+  var populatedFields = ref.populatedFields; if ( populatedFields === void 0 ) populatedFields = false;
   var query = ref.query;
   var skipHeader = ref.skipHeader; if ( skipHeader === void 0 ) skipHeader = false;
   var transform = ref.transform;
@@ -450,7 +493,8 @@ async function transformer(ref) {
         transform,
         sourceClient,
         query,
-        bufferSize
+        bufferSize,
+        populatedFields
       );
     }
 

package/package.json

@@ -14,7 +14,7 @@
   "license": "Apache-2.0",
   "author": "Walter Rafelsberger <walter@rafelsberger.at>",
   "contributors": [],
-  "version": "1.0.0-beta1",
+  "version": "1.0.0-beta2",
   "main": "dist/node-es-transformer.cjs.js",
   "module": "dist/node-es-transformer.esm.js",
   "dependencies": {
@@ -25,6 +25,7 @@
   },
   "devDependencies": {
     "acorn": "^6.4.2",
+    "async-retry": "^1.3.3",
     "commit-and-tag-version": "^11.3.0",
     "cz-conventional-changelog": "^3.3.0",
     "eslint": "^8.51.0",
@@ -35,7 +36,6 @@
     "eslint-plugin-jsx-a11y": "6.7.1",
     "eslint-plugin-prettier": "^3.3.1",
     "eslint-plugin-react": "7.32.2",
-    "frisby": "^2.1.3",
     "jest": "^29.7.0",
     "prettier": "^2.2.1",
     "rollup": "0.66.6",
@@ -46,7 +46,7 @@
   "scripts": {
     "build": "rollup -c",
     "dev": "rollup -c -w",
-    "test": "jest",
+    "test": "jest --runInBand --detectOpenHandles --forceExit",
     "pretest": "npm run build",
     "release": "commit-and-tag-version",
     "create-sample-data-10000": "node scripts/create_sample_data_10000",
@@ -59,5 +59,10 @@
     "commitizen": {
       "path": "./node_modules/cz-conventional-changelog"
     }
+  },
+  "jest": {
+    "testMatch": [
+      "**/__tests__/**/*.test.js"
+    ]
   }
 }