@terascope/elasticsearch-api 3.1.1 → 3.2.2

package/index.js

@@ -1,5 +1,9 @@
 'use strict';
 
+// polyfill because opensearch has references to an api that won't exist
+// on the client side, should be able to remove in the future
+require('setimmediate');
+
 const Promise = require('bluebird');
 const {
     isTest,
@@ -28,6 +32,7 @@ const { ElasticsearchDistribution } = require('@terascope/types');
 const { inspect } = require('util');
 
 const DOCUMENT_EXISTS = 409;
+const TOO_MANY_REQUESTS = 429;
 
 // Module to manage persistence in Elasticsearch.
 // All functions in this module return promises that must be resolved to get the final result.
@@ -102,7 +107,10 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
 
             function _runRequest() {
                 clientBase[endpoint](query)
-                    .then(resolve)
+                    .then((rawResponse) => {
+                        const response = get(rawResponse, 'body', rawResponse);
+                        resolve(response);
+                    })
                     .catch(errHandler);
             }
 
@@ -290,7 +298,7 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
                     continue;
                 }
 
-                if (item.error.type === 'es_rejected_execution_exception') {
+                if (item.status === TOO_MANY_REQUESTS || item.error.type === 'es_rejected_execution_exception') {
                     if (actionRecords[i] == null) {
                         // this error should not happen in production,
                         // only in tests where the bulk function is mocked
@@ -927,6 +935,13 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
         return distribution === ElasticsearchDistribution.elasticsearch && parsedVersion === 6;
     }
 
+    function isElasticsearch8() {
+        const { distribution, version: esVersion } = getClientMetadata();
+        const parsedVersion = toNumber(esVersion.split('.', 1)[0]);
+
+        return distribution === ElasticsearchDistribution.elasticsearch && parsedVersion === 8;
+    }
+
     function _fixMappingRequest(_params, isTemplate) {
         if (!_params || !_params.body) {
             throw new Error('Invalid mapping request');
@@ -956,6 +971,10 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
             }
         }
 
+        if (isElasticsearch8(client)) {
+            delete defaultParams.includeTypeName;
+        }
+
         return Object.assign({}, defaultParams, params);
     }
 
@@ -1049,7 +1068,7 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
                 });
                 return Promise.reject(error);
             });
-        });
+        }).catch((err) => Promise.reject(err));
     }
 
     function _verifyMapping(query, configMapping, recordType) {

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/elasticsearch-api",
     "displayName": "Elasticsearch API",
-    "version": "3.1.1",
+    "version": "3.2.2",
     "description": "Elasticsearch client api used across multiple services, handles retries and exponential backoff",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-api#readme",
     "bugs": {
@@ -14,15 +14,25 @@
     "typings": "types/index.d.ts",
     "scripts": {
         "test": "ts-scripts test . --",
+        "test:7": "ELASTICSEARCH_VERSION='7.9.3' yarn run test",
+        "test:8": "ELASTICSEARCH_VERSION='8.1.2' yarn run test",
         "test:debug": "ts-scripts test --debug . --",
+        "test:legacy": "LEGACY_CLIENT=true yarn run test",
+        "test:opensearch": "TEST_OPENSEARCH='true' yarn run test",
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/utils": "^0.44.2",
-        "bluebird": "^3.7.2"
+        "@terascope/utils": "^0.44.4",
+        "bluebird": "^3.7.2",
+        "setimmediate": "^1.0.5"
     },
     "devDependencies": {
-        "@types/elasticsearch": "^5.0.40"
+        "@opensearch-project/opensearch": "^1.0.2",
+        "@types/elasticsearch": "^5.0.40",
+        "elasticsearch": "^15.4.1",
+        "elasticsearch6": "npm:@elastic/elasticsearch@^6.7.0",
+        "elasticsearch7": "npm:@elastic/elasticsearch@^7.0.0",
+        "elasticsearch8": "npm:@elastic/elasticsearch@^8.0.0"
     },
     "engines": {
         "node": "^12.22.0 || >=14.17.0",
@@ -33,6 +43,6 @@
         "registry": "https://registry.npmjs.org/"
     },
     "terascope": {
-        "testSuite": "unit-a"
+        "testSuite": "restrained"
     }
 }

package/test/bulk-send-limit-spec.js

@@ -0,0 +1,43 @@
+'use strict';
+
+const { debugLogger, chunk, pMap } = require('@terascope/utils');
+const elasticsearchAPI = require('../index');
+const { data } = require('./helpers/data');
+const { TEST_INDEX_PREFIX } = require('./helpers/config');
+const {
+    makeClient, cleanupIndex,
+    waitForData, formatUploadData
+} = require('./helpers/elasticsearch');
+
+const THREE_MINUTES = 3 * 60 * 1000;
+
+jest.setTimeout(THREE_MINUTES + 30000);
+
+describe('bulkSend can work with congested queues', () => {
+    const logger = debugLogger('congested_test');
+    const index = `${TEST_INDEX_PREFIX}_congested_queues_`;
+
+    let client;
+    let api;
+
+    beforeAll(async () => {
+        client = await makeClient();
+        await cleanupIndex(client, index);
+        api = elasticsearchAPI(client, logger);
+    });
+
+    afterAll(async () => {
+        await cleanupIndex(client, index);
+    });
+
+    it('can get correct data even with congested queues', async () => {
+        const chunkedData = chunk(data, 50);
+
+        await pMap(chunkedData, async (cData) => {
+            const formattedData = formatUploadData(index, cData);
+            return api.bulkSend(formattedData);
+        }, { concurrency: 9 });
+
+        await waitForData(client, index, data.length, logger, THREE_MINUTES);
+    });
+});

package/test/helpers/config.js

@@ -0,0 +1,22 @@
+'use strict';
+
+const {
+    TEST_INDEX_PREFIX = 'teratest_',
+    ELASTICSEARCH_HOST = 'http://localhost:9200',
+    ELASTICSEARCH_API_VERSION = '6.5',
+    ELASTICSEARCH_VERSION = '6.8.6',
+    OPENSEARCH_USER = 'admin',
+    OPENSEARCH_PASSWORD = 'admin',
+    OPENSEARCH_VERSION = '1.3.0',
+    RESTRAINED_OPENSEARCH_PORT = process.env.RESTRAINED_OPENSEARCH_PORT || '49206',
+    RESTRAINED_OPENSEARCH_HOST = `http://${OPENSEARCH_USER}:${OPENSEARCH_PASSWORD}@http://localhost:${RESTRAINED_OPENSEARCH_PORT}`,
+} = process.env;
+
+module.exports = {
+    TEST_INDEX_PREFIX,
+    ELASTICSEARCH_HOST,
+    ELASTICSEARCH_API_VERSION,
+    ELASTICSEARCH_VERSION,
+    RESTRAINED_OPENSEARCH_HOST,
+    OPENSEARCH_VERSION
+};