@terascope/elasticsearch-api 3.0.2 → 3.1.0

package/index.js

@@ -23,6 +23,8 @@ const {
     getTypeOf,
     isProd
 } = require('@terascope/utils');
+const { ElasticsearchDistribution } = require('@terascope/types');
+
 const { inspect } = require('util');
 
 const DOCUMENT_EXISTS = 409;
@@ -73,8 +75,7 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
     }
 
     function search(query) {
-        const esVersion = getESVersion();
-        if (esVersion >= 7) {
+        if (!isElasticsearch6()) {
             if (query._sourceExclude) {
                 query._sourceExcludes = query._sourceExclude.slice();
                 delete query._sourceExclude;
@@ -228,31 +229,29 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
             );
             return Promise.resolve(true);
         }
-        return client.cluster.stats({}).then((data) => {
-            if (!_checkVersion(data.nodes.versions[0])) {
-                return Promise.resolve();
-            }
-            return client.indices
-                .getSettings({})
-                .then((results) => {
-                    const resultIndex = _verifyIndex(results, config.index);
-                    if (resultIndex.found) {
-                        resultIndex.indexWindowSize.forEach((ind) => {
-                            logger.warn(
-                                `max_result_window for index: ${ind.name} is set at ${ind.windowSize}. On very large indices it is possible that a slice can not be divided to stay below this limit. If that occurs an error will be thrown by Elasticsearch and the slice can not be processed. Increasing max_result_window in the Elasticsearch index settings will resolve the problem.`
-                            );
-                        });
-                    } else {
-                        const error = new TSError('index specified in reader does not exist', {
-                            statusCode: 404,
-                        });
-                        return Promise.reject(error);
-                    }
 
-                    return Promise.resolve();
-                })
-                .catch((err) => Promise.reject(new TSError(err)));
-        });
+        return client.indices
+            .getSettings({})
+            .then((results) => {
+                const settingsData = results.body && results.meta ? results.body : results;
+                const resultIndex = _verifyIndex(settingsData, config.index);
+
+                if (resultIndex.found) {
+                    resultIndex.indexWindowSize.forEach((ind) => {
+                        logger.warn(
+                            `max_result_window for index: ${ind.name} is set at ${ind.windowSize}. On very large indices it is possible that a slice can not be divided to stay below this limit. If that occurs an error will be thrown by Elasticsearch and the slice can not be processed. Increasing max_result_window in the Elasticsearch index settings will resolve the problem.`
+                        );
+                    });
+                } else {
+                    const error = new TSError('index specified in reader does not exist', {
+                        statusCode: 404,
+                    });
+                    return Promise.reject(error);
+                }
+
+                return Promise.resolve();
+            })
+            .catch((err) => Promise.reject(new TSError(err)));
     }
 
     function putTemplate(template, name) {
@@ -340,7 +339,7 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
                 throw new Error(`Bulk send record is missing the action property${dbg}`);
             }
 
-            if (getESVersion() >= 7) {
+            if (!isElasticsearch6()) {
                 const actionKey = getFirstKey(record.action);
                 const { _type, ...withoutTypeAction } = record.action[actionKey];
                 // if data is specified return both
@@ -357,10 +356,11 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
             return record.data ? [record.action, record.data] : [record.action];
         });
 
-        const result = await _clientRequest('bulk', { body });
+        const response = await _clientRequest('bulk', { body });
+        const results = response.body ? response.body : response;
 
-        if (!result.errors) {
-            return result.items.reduce((c, item) => {
+        if (!results.errors) {
+            return results.items.reduce((c, item) => {
                 const [value] = Object.values(item);
                 // ignore non-successful status codes
                 if (value.status != null && value.status >= 400) return c;
@@ -370,7 +370,7 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
 
         const {
             retry, successful, error, reason
-        } = _filterRetryRecords(actionRecords, result);
+        } = _filterRetryRecords(actionRecords, results);
 
         if (error) {
             throw new Error(`bulk send error: ${reason}`);
@@ -631,14 +631,19 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
                 client
                     .search(queryParam)
                     .then((data) => {
-                        const { failures, failed } = data._shards;
+                        const failuresReasons = [];
+                        const results = data.body ? data.body : data;
+                        const { failures, failed } = results._shards;
+
                         if (!failed) {
-                            resolve(data);
+                            resolve(results);
                             return;
                         }
 
+                        failuresReasons.push(...failures);
+
                         const reasons = uniq(
-                            flatten(failures.map((shard) => shard.reason.type))
+                            flatten(failuresReasons.map((shard) => shard.reason.type))
                         );
 
                         if (
@@ -665,13 +670,13 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
     }
 
     function _esV7adjustments(queryParam) {
-        if (getESVersion() >= 7) {
+        if (!isElasticsearch6()) {
             queryParam.trackTotalHits = true;
         }
     }
 
     function _adjustTypeForEs7(query) {
-        if (getESVersion() >= 7) {
+        if (!isElasticsearch6()) {
             if (Array.isArray(query)) {
                 return _removeTypeFromBulkRequest(query);
             }
@@ -682,7 +687,7 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
     }
 
     function _removeTypeFromBulkRequest(query) {
-        if (getESVersion() < 7) return query;
+        if (isElasticsearch6()) return query;
 
         return query.map((queryItem) => {
             if (isSimpleObject(queryItem)) {
@@ -812,11 +817,6 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
         };
     }
 
-    function _checkVersion(str) {
-        const num = Number(str.replace(/\./g, ''));
-        return num >= 210;
-    }
-
     function isAvailable(index, recordType) {
         const query = {
             index,
@@ -870,7 +870,7 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
                 });
         });
     }
-
+    // TODO: verifyClient needs to be checked with new client usage
     /**
      * Verify the state of the underlying es client.
      * Will throw error if in fatal state, like the connection is closed.
@@ -894,16 +894,39 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
 
         return true;
     }
-
+    /** This is deprecated as an external api,
+     * please use getClientMetadata
+     * */
     function getESVersion() {
-        const esVersion = get(client, 'transport._config.apiVersion', '6.5');
+        const newClientVersion = get(client, '__meta.version');
+        const esVersion = newClientVersion || get(client, 'transport._config.apiVersion', '6.5');
+
         if (esVersion && isString(esVersion)) {
             const [majorVersion] = esVersion.split('.');
             return toNumber(majorVersion);
         }
+
         return 6;
     }
 
+    function getClientMetadata() {
+        const newClientVersion = get(client, '__meta.version');
+        const esVersion = newClientVersion || get(client, 'transport._config.apiVersion', '6.5');
+        const distribution = get(client, '__meta.distribution', ElasticsearchDistribution.elasticsearch);
+
+        return {
+            distribution,
+            version: esVersion
+        };
+    }
+
+    function isElasticsearch6() {
+        const { distribution, version: esVersion } = getClientMetadata();
+        const parsedVersion = toNumber(esVersion.split('.', 1)[0]);
+
+        return distribution === ElasticsearchDistribution.elasticsearch && parsedVersion === 6;
+    }
+
     function _fixMappingRequest(_params, isTemplate) {
         if (!_params || !_params.body) {
             throw new Error('Invalid mapping request');
@@ -911,17 +934,14 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
         const params = cloneDeep(_params);
         const defaultParams = {};
 
-        const esVersion = getESVersion();
-        if (esVersion >= 6) {
-            if (params.body.template) {
-                if (isTemplate) {
-                    params.body.index_patterns = castArray(params.body.template).slice();
-                }
-                delete params.body.template;
+        if (params.body.template != null) {
+            if (isTemplate && params.body.index_patterns == null) {
+                params.body.index_patterns = castArray(params.body.template).slice();
             }
+            delete params.body.template;
         }
 
-        if (esVersion >= 7) {
+        if (!isElasticsearch6()) {
             const typeMappings = get(params.body, 'mappings', {});
             if (typeMappings.properties) {
                 defaultParams.includeTypeName = false;
@@ -935,6 +955,7 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
                 });
             }
         }
+
         return Object.assign({}, defaultParams, params);
     }
 
@@ -1033,8 +1054,8 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
 
     function _verifyMapping(query, configMapping, recordType) {
         const params = Object.assign({}, query);
-        const esVersion = getESVersion();
-        if (esVersion > 6) {
+
+        if (!isElasticsearch6()) {
             if (recordType) {
                 params.includeTypeName = true;
             }
@@ -1202,12 +1223,14 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
         indexRecovery,
         indexSetup,
         verifyClient,
-        getESVersion,
         validateGeoParameters,
+        getClientMetadata,
+        isElasticsearch6,
         // The APIs below are deprecated and should be removed.
         index_exists: indexExists,
         index_create: indexCreate,
         index_refresh: indexRefresh,
         index_recovery: indexRecovery,
+        getESVersion,
     };
 };

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/elasticsearch-api",
     "displayName": "Elasticsearch API",
-    "version": "3.0.2",
+    "version": "3.1.0",
     "description": "Elasticsearch client api used across multiple services, handles retries and exponential backoff",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-api#readme",
     "bugs": {

package/test/api-spec.js

@@ -360,7 +360,7 @@ describe('elasticsearch-api', () => {
             stats: () => Promise.resolve({ _nodes: {} })
         },
         cluster: {
-            stats: () => Promise.resolve({ nodes: { versions: ['5.4.1'] } })
+            stats: () => Promise.resolve({ nodes: { versions: ['6.8.1'] } })
         },
         __testing: {
             start: 100,

package/types/index.d.ts

@@ -3,6 +3,7 @@
 
 import * as es from 'elasticsearch';
 import { Logger } from '@terascope/utils';
+import { ClientMetadata } from '@terascope/types'
 
 export = elasticsearchAPI;
 
@@ -44,7 +45,10 @@ declare namespace elasticsearchAPI {
         indexSetup: (clusterName, newIndex, migrantIndexName, mapping, recordType, clientName) => Promise<boolean>;
         verifyClient: () => boolean;
         validateGeoParameters: (opConfig: any) => any;
+        /** This api is deprecated, please use getClientMetadata */
         getESVersion: () => number;
+        getClientMetadata: () => ClientMetadata;
+        isElasticsearch6: () => boolean
     }
 
     /**