@terascope/elasticsearch-api 3.6.1 → 3.7.0

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/elasticsearch-api",
     "displayName": "Elasticsearch API",
-    "version": "3.6.1",
+    "version": "3.7.0",
     "description": "Elasticsearch client api used across multiple services, handles retries and exponential backoff",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-api#readme",
     "bugs": {
@@ -13,17 +13,19 @@
     "main": "index.js",
     "typings": "types/index.d.ts",
     "scripts": {
-        "test": "ts-scripts test . --",
-        "test:7": "ELASTICSEARCH_VERSION='7.9.3' yarn run test",
-        "test:8": "ELASTICSEARCH_VERSION='8.1.2' yarn run test",
-        "test:debug": "ts-scripts test --debug . --",
-        "test:legacy": "LEGACY_CLIENT=true yarn run test:debug",
-        "test:opensearch": "TEST_OPENSEARCH='true' RESTRAINED_OPENSEARCH='true' yarn run test",
-        "test:watch": "ts-scripts test --watch . --"
+        "test": "TEST_RESTRAINED_ELASTICSEARCH='true' ts-scripts test . --",
+        "test:debug": "TEST_RESTRAINED_ELASTICSEARCH='true' ts-scripts test --debug . --",
+        "test:elasticsearch6": "TEST_RESTRAINED_ELASTICSEARCH='true' ts-scripts test . --",
+        "test:elasticsearch7": "TEST_RESTRAINED_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' ts-scripts test . --",
+        "test:elasticsearch8": "TEST_RESTRAINED_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='8.1.2' ts-scripts test . --",
+        "test:legacy": "TEST_RESTRAINED_ELASTICSEARCH='true' LEGACY_CLIENT=true ts-scripts test . --",
+        "test:opensearch1": "TEST_RESTRAINED_OPENSEARCH='true' ts-scripts test --debug . --",
+        "test:opensearch2": "TEST_RESTRAINED_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' ts-scripts test --debug . --",
+        "test:watch": "TEST_RESTRAINED_ELASTICSEARCH='true' ts-scripts test --watch . --"
     },
     "dependencies": {
         "@terascope/types": "^0.11.2",
-        "@terascope/utils": "^0.46.0",
+        "@terascope/utils": "^0.47.0",
         "bluebird": "^3.7.2",
         "setimmediate": "^1.0.5"
     },
@@ -31,7 +33,7 @@
         "@opensearch-project/opensearch": "^1.2.0",
         "@types/elasticsearch": "^5.0.40",
         "elasticsearch": "^15.4.1",
-        "elasticsearch-store": "^0.67.2",
+        "elasticsearch-store": "^0.68.0",
         "elasticsearch6": "npm:@elastic/elasticsearch@^6.7.0",
         "elasticsearch7": "npm:@elastic/elasticsearch@^7.0.0",
         "elasticsearch8": "npm:@elastic/elasticsearch@^8.0.0"

package/test/bulk-send-dlq-spec.js

@@ -9,37 +9,18 @@ const { ElasticsearchTestHelpers } = require('elasticsearch-store');
 const elasticsearchAPI = require('../index');
 
 const {
-    makeClient, cleanupIndex,
-    EvenDateData, TEST_INDEX_PREFIX,
-    createMappingFromDatatype
+    makeClient, cleanupIndex, EvenDateData,
+    TEST_INDEX_PREFIX, createMappingFromDatatype,
+    formatUploadData
 } = ElasticsearchTestHelpers;
 
 const THREE_MINUTES = 3 * 60 * 1000;
 
 jest.setTimeout(THREE_MINUTES + 60000);
 
-function formatUploadData(
-    index, data, isES8ClientTest = false
-) {
-    const results = [];
-
-    data.forEach((record, i) => {
-        const meta = { _index: index, _id: i + 1 };
-
-        if (!isES8ClientTest) {
-            meta._type = '_doc';
-        }
-
-        results.push({ action: { index: meta }, data: record });
-    });
-
-    return results;
-}
-
 describe('bulkSend', () => {
     let client;
     let api;
-    let isElasticsearch8 = false;
 
     beforeAll(async () => {
         client = await makeClient();
@@ -52,7 +33,6 @@ describe('bulkSend', () => {
         beforeAll(async () => {
             await cleanupIndex(client, index);
             api = elasticsearchAPI(client, logger, { _dead_letter_action: 'kafka_dead_letter' });
-            isElasticsearch8 = api.isElasticsearch8();
 
             const overrides = {
                 settings: {
@@ -79,7 +59,7 @@ describe('bulkSend', () => {
 
             const docs = cloneDeep(EvenDateData.data.slice(0, 2));
 
-            const result = await diffApi.bulkSend(formatUploadData(index, docs, isElasticsearch8));
+            const result = await diffApi.bulkSend(formatUploadData(index, docs, true));
 
             expect(result).toBe(2);
         });
@@ -91,7 +71,7 @@ describe('bulkSend', () => {
 
             docs[0].bytes = 'this is a bad value';
 
-            await expect(diffApi.bulkSend(formatUploadData(index, docs, isElasticsearch8)))
+            await expect(diffApi.bulkSend(formatUploadData(index, docs, true)))
                 .rejects.toThrow();
         });
 
@@ -101,7 +81,7 @@ describe('bulkSend', () => {
 
             docs[0].bytes = 'this is a bad value';
 
-            const result = await api.bulkSend(formatUploadData(index, docs, isElasticsearch8));
+            const result = await api.bulkSend(formatUploadData(index, docs, true));
 
             // 1 good doc  - so only 1 row affected
             expect(result).toBe(1);
@@ -113,7 +93,7 @@ describe('bulkSend', () => {
         it('should return a count if not un-retryable records if dlq is set', async () => {
             const docs = cloneDeep(EvenDateData.data.slice(0, 2));
 
-            const result = await api.bulkSend(formatUploadData(index, docs, isElasticsearch8));
+            const result = await api.bulkSend(formatUploadData(index, docs, true));
 
             expect(result).toBe(2);
         });

package/test/bulk-send-limit-spec.js

@@ -10,35 +10,16 @@ const elasticsearchAPI = require('../index');
 
 const {
     makeClient, cleanupIndex, waitForData,
-    EvenDateData, TEST_INDEX_PREFIX
+    EvenDateData, TEST_INDEX_PREFIX, formatUploadData
 } = ElasticsearchTestHelpers;
 
 const THREE_MINUTES = 3 * 60 * 1000;
 
 jest.setTimeout(THREE_MINUTES + 60000);
 
-function formatUploadData(
-    index, data, isES8ClientTest = false
-) {
-    const results = [];
-
-    data.forEach((record) => {
-        const meta = { _index: index };
-
-        if (!isES8ClientTest) {
-            meta._type = '_doc';
-        }
-
-        results.push({ action: { index: meta }, data: record });
-    });
-
-    return results;
-}
-
 describe('bulkSend', () => {
     let client;
     let api;
-    let isElasticsearch8 = false;
 
     beforeAll(async () => {
         client = await makeClient();
@@ -51,7 +32,6 @@ describe('bulkSend', () => {
         beforeAll(async () => {
             await cleanupIndex(client, index);
             api = elasticsearchAPI(client, logger);
-            isElasticsearch8 = api.isElasticsearch8();
         });
 
         afterAll(async () => {
@@ -62,7 +42,7 @@ describe('bulkSend', () => {
             const chunkedData = chunk(EvenDateData.data, 50);
 
             await pMap(chunkedData, async (cData) => {
-                const formattedData = formatUploadData(index, cData, isElasticsearch8);
+                const formattedData = formatUploadData(index, cData, true);
                 return api.bulkSend(formattedData);
             }, { concurrency: 9 });