@terascope/elasticsearch-api 3.0.2 → 3.2.0

package/index.js

@@ -1,5 +1,9 @@
 'use strict';
 
+// polyfill because opensearch has references to an api that won't exist
+// on the client side, should be able to remove in the future
+require('setimmediate');
+
 const Promise = require('bluebird');
 const {
     isTest,
@@ -23,9 +27,12 @@ const {
     getTypeOf,
     isProd
 } = require('@terascope/utils');
+const { ElasticsearchDistribution } = require('@terascope/types');
+
 const { inspect } = require('util');
 
 const DOCUMENT_EXISTS = 409;
+const TOO_MANY_REQUESTS = 429;
 
 // Module to manage persistence in Elasticsearch.
 // All functions in this module return promises that must be resolved to get the final result.
@@ -73,8 +80,7 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
     }
 
     function search(query) {
-        const esVersion = getESVersion();
-        if (esVersion >= 7) {
+        if (!isElasticsearch6()) {
             if (query._sourceExclude) {
                 query._sourceExcludes = query._sourceExclude.slice();
                 delete query._sourceExclude;
@@ -101,7 +107,10 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
 
             function _runRequest() {
                 clientBase[endpoint](query)
-                    .then(resolve)
+                    .then((rawResponse) => {
+                        const response = get(rawResponse, 'body', rawResponse);
+                        resolve(response);
+                    })
                     .catch(errHandler);
             }
 
@@ -228,31 +237,29 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
             );
             return Promise.resolve(true);
         }
-        return client.cluster.stats({}).then((data) => {
-            if (!_checkVersion(data.nodes.versions[0])) {
-                return Promise.resolve();
-            }
-            return client.indices
-                .getSettings({})
-                .then((results) => {
-                    const resultIndex = _verifyIndex(results, config.index);
-                    if (resultIndex.found) {
-                        resultIndex.indexWindowSize.forEach((ind) => {
-                            logger.warn(
-                                `max_result_window for index: ${ind.name} is set at ${ind.windowSize}. On very large indices it is possible that a slice can not be divided to stay below this limit. If that occurs an error will be thrown by Elasticsearch and the slice can not be processed. Increasing max_result_window in the Elasticsearch index settings will resolve the problem.`
-                            );
-                        });
-                    } else {
-                        const error = new TSError('index specified in reader does not exist', {
-                            statusCode: 404,
-                        });
-                        return Promise.reject(error);
-                    }
 
-                    return Promise.resolve();
-                })
-                .catch((err) => Promise.reject(new TSError(err)));
-        });
+        return client.indices
+            .getSettings({})
+            .then((results) => {
+                const settingsData = results.body && results.meta ? results.body : results;
+                const resultIndex = _verifyIndex(settingsData, config.index);
+
+                if (resultIndex.found) {
+                    resultIndex.indexWindowSize.forEach((ind) => {
+                        logger.warn(
+                            `max_result_window for index: ${ind.name} is set at ${ind.windowSize}. On very large indices it is possible that a slice can not be divided to stay below this limit. If that occurs an error will be thrown by Elasticsearch and the slice can not be processed. Increasing max_result_window in the Elasticsearch index settings will resolve the problem.`
+                        );
+                    });
+                } else {
+                    const error = new TSError('index specified in reader does not exist', {
+                        statusCode: 404,
+                    });
+                    return Promise.reject(error);
+                }
+
+                return Promise.resolve();
+            })
+            .catch((err) => Promise.reject(new TSError(err)));
     }
 
     function putTemplate(template, name) {
@@ -291,7 +298,7 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
                     continue;
                 }
 
-                if (item.error.type === 'es_rejected_execution_exception') {
+                if (item.status === TOO_MANY_REQUESTS || item.error.type === 'es_rejected_execution_exception') {
                     if (actionRecords[i] == null) {
                         // this error should not happen in production,
                         // only in tests where the bulk function is mocked
@@ -340,7 +347,7 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
                 throw new Error(`Bulk send record is missing the action property${dbg}`);
             }
 
-            if (getESVersion() >= 7) {
+            if (!isElasticsearch6()) {
                 const actionKey = getFirstKey(record.action);
                 const { _type, ...withoutTypeAction } = record.action[actionKey];
                 // if data is specified return both
@@ -357,10 +364,11 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
             return record.data ? [record.action, record.data] : [record.action];
         });
 
-        const result = await _clientRequest('bulk', { body });
+        const response = await _clientRequest('bulk', { body });
+        const results = response.body ? response.body : response;
 
-        if (!result.errors) {
-            return result.items.reduce((c, item) => {
+        if (!results.errors) {
+            return results.items.reduce((c, item) => {
                 const [value] = Object.values(item);
                 // ignore non-successful status codes
                 if (value.status != null && value.status >= 400) return c;
@@ -370,7 +378,7 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
 
         const {
             retry, successful, error, reason
-        } = _filterRetryRecords(actionRecords, result);
+        } = _filterRetryRecords(actionRecords, results);
 
         if (error) {
             throw new Error(`bulk send error: ${reason}`);
@@ -631,14 +639,19 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
                 client
                     .search(queryParam)
                     .then((data) => {
-                        const { failures, failed } = data._shards;
+                        const failuresReasons = [];
+                        const results = data.body ? data.body : data;
+                        const { failures, failed } = results._shards;
+
                         if (!failed) {
-                            resolve(data);
+                            resolve(results);
                             return;
                         }
 
+                        failuresReasons.push(...failures);
+
                         const reasons = uniq(
-                            flatten(failures.map((shard) => shard.reason.type))
+                            flatten(failuresReasons.map((shard) => shard.reason.type))
                         );
 
                         if (
@@ -665,13 +678,13 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
     }
 
     function _esV7adjustments(queryParam) {
-        if (getESVersion() >= 7) {
+        if (!isElasticsearch6()) {
             queryParam.trackTotalHits = true;
         }
     }
 
     function _adjustTypeForEs7(query) {
-        if (getESVersion() >= 7) {
+        if (!isElasticsearch6()) {
             if (Array.isArray(query)) {
                 return _removeTypeFromBulkRequest(query);
             }
@@ -682,7 +695,7 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
     }
 
     function _removeTypeFromBulkRequest(query) {
-        if (getESVersion() < 7) return query;
+        if (isElasticsearch6()) return query;
 
         return query.map((queryItem) => {
             if (isSimpleObject(queryItem)) {
@@ -812,11 +825,6 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
         };
     }
 
-    function _checkVersion(str) {
-        const num = Number(str.replace(/\./g, ''));
-        return num >= 210;
-    }
-
     function isAvailable(index, recordType) {
         const query = {
             index,
@@ -870,7 +878,7 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
                 });
         });
     }
-
+    // TODO: verifyClient needs to be checked with new client usage
     /**
      * Verify the state of the underlying es client.
      * Will throw error if in fatal state, like the connection is closed.
@@ -894,16 +902,46 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
 
         return true;
     }
-
+    /** This is deprecated as an external api,
+     * please use getClientMetadata
+     * */
     function getESVersion() {
-        const esVersion = get(client, 'transport._config.apiVersion', '6.5');
+        const newClientVersion = get(client, '__meta.version');
+        const esVersion = newClientVersion || get(client, 'transport._config.apiVersion', '6.5');
+
         if (esVersion && isString(esVersion)) {
             const [majorVersion] = esVersion.split('.');
             return toNumber(majorVersion);
         }
+
         return 6;
     }
 
+    function getClientMetadata() {
+        const newClientVersion = get(client, '__meta.version');
+        const esVersion = newClientVersion || get(client, 'transport._config.apiVersion', '6.5');
+        const distribution = get(client, '__meta.distribution', ElasticsearchDistribution.elasticsearch);
+
+        return {
+            distribution,
+            version: esVersion
+        };
+    }
+
+    function isElasticsearch6() {
+        const { distribution, version: esVersion } = getClientMetadata();
+        const parsedVersion = toNumber(esVersion.split('.', 1)[0]);
+
+        return distribution === ElasticsearchDistribution.elasticsearch && parsedVersion === 6;
+    }
+
+    function isElasticsearch8() {
+        const { distribution, version: esVersion } = getClientMetadata();
+        const parsedVersion = toNumber(esVersion.split('.', 1)[0]);
+
+        return distribution === ElasticsearchDistribution.elasticsearch && parsedVersion === 8;
+    }
+
     function _fixMappingRequest(_params, isTemplate) {
         if (!_params || !_params.body) {
             throw new Error('Invalid mapping request');
@@ -911,17 +949,14 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
         const params = cloneDeep(_params);
         const defaultParams = {};
 
-        const esVersion = getESVersion();
-        if (esVersion >= 6) {
-            if (params.body.template) {
-                if (isTemplate) {
-                    params.body.index_patterns = castArray(params.body.template).slice();
-                }
-                delete params.body.template;
+        if (params.body.template != null) {
+            if (isTemplate && params.body.index_patterns == null) {
+                params.body.index_patterns = castArray(params.body.template).slice();
             }
+            delete params.body.template;
         }
 
-        if (esVersion >= 7) {
+        if (!isElasticsearch6()) {
             const typeMappings = get(params.body, 'mappings', {});
             if (typeMappings.properties) {
                 defaultParams.includeTypeName = false;
@@ -935,6 +970,11 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
                 });
             }
         }
+
+        if (isElasticsearch8(client)) {
+            delete defaultParams.includeTypeName;
+        }
+
         return Object.assign({}, defaultParams, params);
     }
 
@@ -1028,13 +1068,13 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
                 });
                 return Promise.reject(error);
             });
-        });
+        }).catch((err) => Promise.reject(err));
     }
 
     function _verifyMapping(query, configMapping, recordType) {
         const params = Object.assign({}, query);
-        const esVersion = getESVersion();
-        if (esVersion > 6) {
+
+        if (!isElasticsearch6()) {
             if (recordType) {
                 params.includeTypeName = true;
             }
@@ -1202,12 +1242,14 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
         indexRecovery,
         indexSetup,
         verifyClient,
-        getESVersion,
         validateGeoParameters,
+        getClientMetadata,
+        isElasticsearch6,
         // The APIs below are deprecated and should be removed.
         index_exists: indexExists,
         index_create: indexCreate,
         index_refresh: indexRefresh,
         index_recovery: indexRecovery,
+        getESVersion,
     };
 };

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/elasticsearch-api",
     "displayName": "Elasticsearch API",
-    "version": "3.0.2",
+    "version": "3.2.0",
     "description": "Elasticsearch client api used across multiple services, handles retries and exponential backoff",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-api#readme",
     "bugs": {
@@ -14,15 +14,25 @@
     "typings": "types/index.d.ts",
     "scripts": {
         "test": "ts-scripts test . --",
+        "test:7": "ELASTICSEARCH_VERSION='7.9.3' yarn run test",
+        "test:8": "ELASTICSEARCH_VERSION='8.1.2' yarn run test",
         "test:debug": "ts-scripts test --debug . --",
+        "test:legacy": "LEGACY_CLIENT=true yarn run test",
+        "test:opensearch": "TEST_OPENSEARCH='true' yarn run test",
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/utils": "^0.44.1",
-        "bluebird": "^3.7.2"
+        "@terascope/utils": "^0.44.2",
+        "bluebird": "^3.7.2",
+        "setimmediate": "^1.0.5"
     },
     "devDependencies": {
-        "@types/elasticsearch": "^5.0.40"
+        "@opensearch-project/opensearch": "^1.0.2",
+        "@types/elasticsearch": "^5.0.40",
+        "elasticsearch": "^15.4.1",
+        "elasticsearch6": "npm:@elastic/elasticsearch@^6.7.0",
+        "elasticsearch7": "npm:@elastic/elasticsearch@^7.0.0",
+        "elasticsearch8": "npm:@elastic/elasticsearch@^8.0.0"
     },
     "engines": {
         "node": "^12.22.0 || >=14.17.0",
@@ -33,6 +43,6 @@
         "registry": "https://registry.npmjs.org/"
     },
     "terascope": {
-        "testSuite": "unit-a"
+        "testSuite": "restrained"
     }
 }

package/test/api-spec.js

@@ -360,7 +360,7 @@ describe('elasticsearch-api', () => {
             stats: () => Promise.resolve({ _nodes: {} })
         },
         cluster: {
-            stats: () => Promise.resolve({ nodes: { versions: ['5.4.1'] } })
+            stats: () => Promise.resolve({ nodes: { versions: ['6.8.1'] } })
         },
         __testing: {
             start: 100,

package/test/bulk-send-limit-spec.js

@@ -0,0 +1,43 @@
+'use strict';
+
+const { debugLogger, chunk, pMap } = require('@terascope/utils');
+const elasticsearchAPI = require('../index');
+const { data } = require('./helpers/data');
+const { TEST_INDEX_PREFIX } = require('./helpers/config');
+const {
+    makeClient, cleanupIndex,
+    waitForData, formatUploadData
+} = require('./helpers/elasticsearch');
+
+const THREE_MINUTES = 3 * 60 * 1000;
+
+jest.setTimeout(THREE_MINUTES + 30000);
+
+describe('bulkSend can work with congested queues', () => {
+    const logger = debugLogger('congested_test');
+    const index = `${TEST_INDEX_PREFIX}_congested_queues_`;
+
+    let client;
+    let api;
+
+    beforeAll(async () => {
+        client = await makeClient();
+        await cleanupIndex(client, index);
+        api = elasticsearchAPI(client, logger);
+    });
+
+    afterAll(async () => {
+        await cleanupIndex(client, index);
+    });
+
+    it('can get correct data even with congested queues', async () => {
+        const chunkedData = chunk(data, 50);
+
+        await pMap(chunkedData, async (cData) => {
+            const formattedData = formatUploadData(index, cData);
+            return api.bulkSend(formattedData);
+        }, { concurrency: 9 });
+
+        await waitForData(client, index, data.length, logger, THREE_MINUTES);
+    });
+});

package/test/helpers/config.js

@@ -0,0 +1,22 @@
+'use strict';
+
+const {
+    TEST_INDEX_PREFIX = 'teratest_',
+    ELASTICSEARCH_HOST = 'http://localhost:9200',
+    ELASTICSEARCH_API_VERSION = '6.5',
+    ELASTICSEARCH_VERSION = '6.8.6',
+    OPENSEARCH_USER = 'admin',
+    OPENSEARCH_PASSWORD = 'admin',
+    OPENSEARCH_VERSION = '1.3.0',
+    RESTRAINED_OPENSEARCH_PORT = process.env.RESTRAINED_OPENSEARCH_PORT || '49206',
+    RESTRAINED_OPENSEARCH_HOST = `http://${OPENSEARCH_USER}:${OPENSEARCH_PASSWORD}@http://localhost:${RESTRAINED_OPENSEARCH_PORT}`,
+} = process.env;
+
+module.exports = {
+    TEST_INDEX_PREFIX,
+    ELASTICSEARCH_HOST,
+    ELASTICSEARCH_API_VERSION,
+    ELASTICSEARCH_VERSION,
+    RESTRAINED_OPENSEARCH_HOST,
+    OPENSEARCH_VERSION
+};