@terascope/elasticsearch-api 3.3.1 → 3.3.3

package/index.js

@@ -1,3 +1,5 @@
+/* eslint-disable camelcase */
+
 'use strict';
 
 // polyfill because opensearch has references to an api that won't exist
@@ -6,31 +8,15 @@ require('setimmediate');
 
 const Promise = require('bluebird');
 const {
-    isTest,
-    TSError,
-    isFatalError,
-    parseError,
-    getBackoffDelay,
-    isRetryableError,
-    get,
-    toNumber,
-    isString,
-    isSimpleObject,
-    castArray,
-    flatten,
-    toBoolean,
-    uniq,
-    random,
-    cloneDeep,
-    DataEntity,
-    isDeepEqual,
-    getTypeOf,
-    isProd
+    isTest, TSError, isFatalError,
+    parseError, getBackoffDelay, isRetryableError,
+    get, toNumber, isString, isSimpleObject,
+    castArray, flatten, toBoolean,
+    uniq, random, cloneDeep, DataEntity,
+    isDeepEqual, getTypeOf, isProd
 } = require('@terascope/utils');
 const { ElasticsearchDistribution } = require('@terascope/types');
 
-const { inspect } = require('util');
-
 const DOCUMENT_EXISTS = 409;
 const TOO_MANY_REQUESTS = 429;
 
@@ -55,9 +41,13 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
 
     const { connection = 'unknown' } = config;
 
-    function count(query) {
+    async function count(query) {
         query.size = 0;
-        return _searchES(query).then((data) => get(data, 'hits.total.value', get(data, 'hits.total')));
+        const response = await _searchES(query);
+
+        const data = get(response, 'hits.total.value', get(response, 'hits.total'));
+
+        return data;
     }
 
     function convertDocToDataEntity(doc) {
@@ -80,18 +70,24 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
     }
 
     function search(query) {
-        if (!isElasticsearch6()) {
-            if (query._sourceExclude) {
-                query._sourceExcludes = query._sourceExclude.slice();
-                delete query._sourceExclude;
-            }
-            if (query._sourceInclude) {
-                query._sourceIncludes = query._sourceInclude.slice();
-                delete query._sourceInclude;
-            }
+        const {
+            _sourceInclude, _source_includes,
+            _sourceExclude, _source_excludes,
+            ...safeQuery
+        } = query;
+
+        const sourceIncludes = _sourceInclude || _source_includes;
+        const sourceExcludes = _sourceExclude || _source_excludes;
+
+        if (sourceIncludes) {
+            safeQuery._source_includes = sourceIncludes;
+        }
+
+        if (sourceExcludes) {
+            safeQuery._source_excludes = sourceExcludes;
         }
 
-        return _searchES(query).then((data) => {
+        return _searchES(safeQuery).then((data) => {
             if (config.full_response) {
                 return data;
             }
@@ -342,7 +338,7 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
             if (record.action == null) {
                 let dbg = '';
                 if (!isProd) {
-                    dbg = `, dbg: ${inspect({ record, index })}`;
+                    dbg = `, dbg: ${JSON.stringify({ record, index })}`;
                 }
                 throw new Error(`Bulk send record is missing the action property${dbg}`);
             }
@@ -796,19 +792,33 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
         });
     }
 
+    function isConnectionErrorMessage(err) {
+        const msg = get(err, 'message', '');
+        return msg.includes('No Living connections') || msg.includes('ECONNREFUSED');
+    }
+
+    function isErrorRetryable(err) {
+        const checkErrorMsg = isRetryableError(err);
+
+        if (checkErrorMsg) {
+            return true;
+        }
+
+        const isRejectedError = get(err, 'body.error.type') === 'es_rejected_execution_exception';
+        const isConnectionError = isConnectionErrorMessage(err);
+
+        if (isRejectedError || isConnectionError) {
+            return true;
+        }
+
+        return false;
+    }
+
     function _errorHandler(fn, data, reject, fnName = '->unknown()') {
         const retry = _retryFn(fn, data, reject);
+
         return function _errorHandlerFn(err) {
-            let retryable = false;
-            if (isRetryableError(err)) {
-                retryable = true;
-            } else {
-                const isRejectedError = get(err, 'body.error.type') === 'es_rejected_execution_exception';
-                const isConnectionError = get(err, 'message', '').includes('No Living connections');
-                if (isRejectedError || isConnectionError) {
-                    retryable = true;
-                }
-            }
+            const retryable = isErrorRetryable(err);
 
             if (retryable) {
                 retry();
@@ -906,8 +916,12 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
      * please use getClientMetadata
      * */
     function getESVersion() {
-        const newClientVersion = get(client, '__meta.version');
-        const esVersion = newClientVersion || get(client, 'transport._config.apiVersion', '6.5');
+        const newClientVersion = get(client, '__meta.majorVersion');
+
+        if (newClientVersion) return newClientVersion;
+
+        // legacy
+        const esVersion = get(client, 'transport._config.apiVersion', '6.5');
 
         if (esVersion && isString(esVersion)) {
             const [majorVersion] = esVersion.split('.');
@@ -918,28 +932,30 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
     }
 
     function getClientMetadata() {
-        const newClientVersion = get(client, '__meta.version');
-        const esVersion = newClientVersion || get(client, 'transport._config.apiVersion', '6.5');
-        const distribution = get(client, '__meta.distribution', ElasticsearchDistribution.elasticsearch);
+        if (client.__meta) {
+            return client.__meta;
+        }
+
+        const esVersion = get(client, 'transport._config.apiVersion', '6.5');
+        const distribution = ElasticsearchDistribution.elasticsearch;
+        const [majorVersion = 6, minorVersion = 5] = esVersion.split('.').map(toNumber);
 
         return {
             distribution,
-            version: esVersion
+            version: esVersion,
+            majorVersion,
+            minorVersion
         };
     }
 
     function isElasticsearch6() {
-        const { distribution, version: esVersion } = getClientMetadata();
-        const parsedVersion = toNumber(esVersion.split('.', 1)[0]);
-
-        return distribution === ElasticsearchDistribution.elasticsearch && parsedVersion === 6;
+        const { distribution, majorVersion } = getClientMetadata();
+        return distribution === ElasticsearchDistribution.elasticsearch && majorVersion === 6;
     }
 
     function isElasticsearch8() {
-        const { distribution, version: esVersion } = getClientMetadata();
-        const parsedVersion = toNumber(esVersion.split('.', 1)[0]);
-
-        return distribution === ElasticsearchDistribution.elasticsearch && parsedVersion === 8;
+        const { distribution, majorVersion } = getClientMetadata();
+        return distribution === ElasticsearchDistribution.elasticsearch && majorVersion === 8;
     }
 
     function _fixMappingRequest(_params, isTemplate) {
@@ -1245,11 +1261,13 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
         validateGeoParameters,
         getClientMetadata,
         isElasticsearch6,
+        isElasticsearch8,
         // The APIs below are deprecated and should be removed.
         index_exists: indexExists,
         index_create: indexCreate,
         index_refresh: indexRefresh,
         index_recovery: indexRecovery,
         getESVersion,
+        isErrorRetryable
     };
 };

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/elasticsearch-api",
     "displayName": "Elasticsearch API",
-    "version": "3.3.1",
+    "version": "3.3.3",
     "description": "Elasticsearch client api used across multiple services, handles retries and exponential backoff",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-api#readme",
     "bugs": {
@@ -18,18 +18,20 @@
         "test:8": "ELASTICSEARCH_VERSION='8.1.2' yarn run test",
         "test:debug": "ts-scripts test --debug . --",
         "test:legacy": "LEGACY_CLIENT=true yarn run test",
-        "test:opensearch": "TEST_OPENSEARCH='true' yarn run test",
+        "test:opensearch": "TEST_OPENSEARCH='true' RESTRAINED_OPENSEARCH='true' yarn run test",
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
+        "@terascope/types": "^0.11.0",
         "@terascope/utils": "^0.45.1",
         "bluebird": "^3.7.2",
         "setimmediate": "^1.0.5"
     },
     "devDependencies": {
-        "@opensearch-project/opensearch": "^1.0.2",
+        "@opensearch-project/opensearch": "^1.1.0",
         "@types/elasticsearch": "^5.0.40",
         "elasticsearch": "^15.4.1",
+        "elasticsearch-store": "^0.65.1",
         "elasticsearch6": "npm:@elastic/elasticsearch@^6.7.0",
         "elasticsearch7": "npm:@elastic/elasticsearch@^7.0.0",
         "elasticsearch8": "npm:@elastic/elasticsearch@^8.0.0"

package/test/bulk-send-limit-spec.js

@@ -1,29 +1,49 @@
 'use strict';
 
 const { debugLogger, chunk, pMap } = require('@terascope/utils');
+const { ElasticsearchTestHelpers } = require('elasticsearch-store');
 const elasticsearchAPI = require('../index');
-const { data } = require('./helpers/data');
-const { TEST_INDEX_PREFIX } = require('./helpers/config');
+
 const {
-    makeClient, cleanupIndex,
-    waitForData, formatUploadData
-} = require('./helpers/elasticsearch');
+    makeClient, cleanupIndex, waitForData,
+    EvenDateData, TEST_INDEX_PREFIX
+} = ElasticsearchTestHelpers;
 
 const THREE_MINUTES = 3 * 60 * 1000;
 
 jest.setTimeout(THREE_MINUTES + 30000);
 
+function formatUploadData(
+    index, data, isES8ClientTest = false
+) {
+    const results = [];
+
+    data.forEach((record) => {
+        const meta = { _index: index };
+
+        if (!isES8ClientTest) {
+            meta._type = '_doc';
+        }
+
+        results.push({ action: { index: meta }, data: record });
+    });
+
+    return results;
+}
+
 describe('bulkSend can work with congested queues', () => {
     const logger = debugLogger('congested_test');
     const index = `${TEST_INDEX_PREFIX}_congested_queues_`;
 
     let client;
     let api;
+    let isElasticsearch8 = false;
 
     beforeAll(async () => {
         client = await makeClient();
         await cleanupIndex(client, index);
         api = elasticsearchAPI(client, logger);
+        isElasticsearch8 = api.isElasticsearch8();
     });
 
     afterAll(async () => {
@@ -31,13 +51,13 @@ describe('bulkSend can work with congested queues', () => {
     });
 
     it('can get correct data even with congested queues', async () => {
-        const chunkedData = chunk(data, 50);
+        const chunkedData = chunk(EvenDateData.data, 50);
 
         await pMap(chunkedData, async (cData) => {
-            const formattedData = formatUploadData(index, cData);
+            const formattedData = formatUploadData(index, cData, isElasticsearch8);
             return api.bulkSend(formattedData);
         }, { concurrency: 9 });
 
-        await waitForData(client, index, data.length, logger, THREE_MINUTES);
+        await waitForData(client, index, EvenDateData.data.length, logger, THREE_MINUTES);
     });
 });

package/test/retry-spec.js

@@ -0,0 +1,38 @@
+'use strict';
+
+const { ElasticsearchTestHelpers } = require('elasticsearch-store');
+const { debugLogger } = require('@terascope/utils');
+const opensearch = require('@opensearch-project/opensearch');
+const API = require('../index');
+
+const logger = debugLogger('retry spec');
+const port = '1111';
+
+const { data } = ElasticsearchTestHelpers.EvenDateData;
+
+const body = ElasticsearchTestHelpers.formatUploadData('retry-error-test', data);
+
+describe('retry behavior', () => {
+    it('will work with opensearch when connection cannot be established', async () => {
+        expect.assertions(4);
+        const client = new opensearch.Client({ node: `http://127.0.0.1:${port}` });
+        const { isErrorRetryable } = API(client, logger);
+
+        try {
+            await client.bulk({ body });
+            throw Error('should have thrown');
+        } catch (err) {
+            expect(err).toBeDefined();
+            expect(isErrorRetryable(err)).toBeTrue();
+        }
+
+        // we try again as we have seen the second call be marked as unretryable
+        try {
+            await client.bulk({ body });
+            throw Error('should have thrown');
+        } catch (err) {
+            expect(err).toBeDefined();
+            expect(isErrorRetryable(err)).toBeTrue();
+        }
+    });
+});

package/types/index.d.ts

@@ -1,7 +1,7 @@
 // Type definitions for elasticsearch-api
 // Project: @terascope/elasticsearch-api
 
-import * as es from 'elasticsearch';
+import { ClientParams, ClientResponse } from '@terascope/types';
 import { Logger } from '@terascope/utils';
 import { ClientMetadata } from '@terascope/types'
 
@@ -17,16 +17,16 @@ declare namespace elasticsearchAPI {
     }
 
     export interface Client {
-        search: (query: es.SearchParams) => Promise<es.SearchResponse | any[]>;
-        count: (query: es.CountParams) => Promise<number>;
-        get: (query: es.GetParams, fullResponse?: boolean) => Promise<any>;
-        mget: (query: es.MGetParams) => Promise<any>;
-        index: (query: es.IndexDocumentParams) => Promise<any>;
-        indexWithId: (query: es.IndexDocumentParams) => Promise<any>;
+        search: (query: ClientParams.SearchParams) => Promise<ClientResponse.SearchResponse | any[]>;
+        count: (query: ClientParams.CountParams) => Promise<number>;
+        get: (query: ClientParams.GetParams, fullResponse?: boolean) => Promise<any>;
+        mget: (query: ClientParams.MGetParams) => Promise<any>;
+        index: (query: ClientParams.IndexParams) => Promise<any>;
+        indexWithId: (query: ClientParams.IndexParams) => Promise<any>;
         isAvailable: (index: string, recordType?: string) => Promise<any>;
-        create: (query: es.CreateDocumentParams) => Promise<any>;
-        update: (query: es.UpdateDocumentParams) => Promise<any>;
-        remove: (query: es.DeleteDocumentParams) => Promise<es.DeleteDocumentResponse>;
+        create: (query: ClientParams.CreateParams) => Promise<any>;
+        update: (query: ClientParams.UpdateParams) => Promise<any>;
+        remove: (query: ClientParams.DeleteParams) => Promise<es.DeleteDocumentResponse>;
         version: () => Promise<boolean>;
         putTemplate: (template: any, name: string) => Promise<any>;
         /**
@@ -37,11 +37,11 @@ declare namespace elasticsearchAPI {
         bulkSend: (data: BulkRecord[]) => Promise<number>;
         nodeInfo: (query: any) => Promise<any>;
         nodeStats: (query: any) => Promise<any>;
-        buildQuery: (opConfig: Config, msg: any) => es.SearchParams;
-        indexExists: (query: es.ExistsParams) => Promise<boolean>;
-        indexCreate: (query: es.IndicesCreateParams) => Promise<any>;
-        indexRefresh: (query: es.IndicesRefreshParams) => Promise<any>;
-        indexRecovery: (query: es.IndicesRecoveryParams) => Promise<any>;
+        buildQuery: (opConfig: Config, msg: any) => ClientParams.SearchParams;
+        indexExists: (query: ClientParams.ExistsParams) => Promise<boolean>;
+        indexCreate: (query: ClientParams.IndicesCreateParams) => Promise<any>;
+        indexRefresh: (query: ClientParams.IndicesRefreshParams) => Promise<any>;
+        indexRecovery: (query: ClientParams.IndicesRecoveryParams) => Promise<any>;
         indexSetup: (clusterName, newIndex, migrantIndexName, mapping, recordType, clientName) => Promise<boolean>;
         verifyClient: () => boolean;
         validateGeoParameters: (opConfig: any) => any;

package/test/helpers/config.js

@@ -1,22 +0,0 @@
-'use strict';
-
-const {
-    TEST_INDEX_PREFIX = 'teratest_',
-    ELASTICSEARCH_HOST = 'http://localhost:9200',
-    ELASTICSEARCH_API_VERSION = '6.5',
-    ELASTICSEARCH_VERSION = '6.8.6',
-    OPENSEARCH_USER = 'admin',
-    OPENSEARCH_PASSWORD = 'admin',
-    OPENSEARCH_VERSION = '1.3.0',
-    RESTRAINED_OPENSEARCH_PORT = process.env.RESTRAINED_OPENSEARCH_PORT || '49206',
-    RESTRAINED_OPENSEARCH_HOST = `http://${OPENSEARCH_USER}:${OPENSEARCH_PASSWORD}@http://localhost:${RESTRAINED_OPENSEARCH_PORT}`,
-} = process.env;
-
-module.exports = {
-    TEST_INDEX_PREFIX,
-    ELASTICSEARCH_HOST,
-    ELASTICSEARCH_API_VERSION,
-    ELASTICSEARCH_VERSION,
-    RESTRAINED_OPENSEARCH_HOST,
-    OPENSEARCH_VERSION
-};