@terascope/elasticsearch-api 3.0.0 → 3.1.0

package/index.js

@@ -20,8 +20,12 @@ const {
     cloneDeep,
     DataEntity,
     isDeepEqual,
-    getTypeOf
+    getTypeOf,
+    isProd
 } = require('@terascope/utils');
+const { ElasticsearchDistribution } = require('@terascope/types');
+
+const { inspect } = require('util');
 
 const DOCUMENT_EXISTS = 409;
 
@@ -71,8 +75,7 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
     }
 
     function search(query) {
-        const esVersion = getESVersion();
-        if (esVersion >= 7) {
+        if (!isElasticsearch6()) {
             if (query._sourceExclude) {
                 query._sourceExcludes = query._sourceExclude.slice();
                 delete query._sourceExclude;
@@ -226,31 +229,29 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
             );
             return Promise.resolve(true);
         }
-        return client.cluster.stats({}).then((data) => {
-            if (!_checkVersion(data.nodes.versions[0])) {
-                return Promise.resolve();
-            }
-            return client.indices
-                .getSettings({})
-                .then((results) => {
-                    const resultIndex = _verifyIndex(results, config.index);
-                    if (resultIndex.found) {
-                        resultIndex.indexWindowSize.forEach((ind) => {
-                            logger.warn(
-                                `max_result_window for index: ${ind.name} is set at ${ind.windowSize}. On very large indices it is possible that a slice can not be divided to stay below this limit. If that occurs an error will be thrown by Elasticsearch and the slice can not be processed. Increasing max_result_window in the Elasticsearch index settings will resolve the problem.`
-                            );
-                        });
-                    } else {
-                        const error = new TSError('index specified in reader does not exist', {
-                            statusCode: 404,
-                        });
-                        return Promise.reject(error);
-                    }
 
-                    return Promise.resolve();
-                })
-                .catch((err) => Promise.reject(new TSError(err)));
-        });
+        return client.indices
+            .getSettings({})
+            .then((results) => {
+                const settingsData = results.body && results.meta ? results.body : results;
+                const resultIndex = _verifyIndex(settingsData, config.index);
+
+                if (resultIndex.found) {
+                    resultIndex.indexWindowSize.forEach((ind) => {
+                        logger.warn(
+                            `max_result_window for index: ${ind.name} is set at ${ind.windowSize}. On very large indices it is possible that a slice can not be divided to stay below this limit. If that occurs an error will be thrown by Elasticsearch and the slice can not be processed. Increasing max_result_window in the Elasticsearch index settings will resolve the problem.`
+                        );
+                    });
+                } else {
+                    const error = new TSError('index specified in reader does not exist', {
+                        statusCode: 404,
+                    });
+                    return Promise.reject(error);
+                }
+
+                return Promise.resolve();
+            })
+            .catch((err) => Promise.reject(new TSError(err)));
     }
 
     function putTemplate(template, name) {
@@ -264,7 +265,12 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
      * When the bulk request has errors this will find the actions
      * records to retry.
      *
-     * @returns {{ retry: Record<string, any>[], error: boolean, reason?: string }}
+     * @returns {{
+     *    retry: Record<string, any>[],
+     *    successful: number,
+     *    error: boolean,
+     *    reason?: string
+     * }}
     */
     function _filterRetryRecords(actionRecords, result) {
         const retry = [];
@@ -272,6 +278,7 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
 
         let nonRetriableError = false;
         let reason = '';
+        let successful = 0;
 
         for (let i = 0; i < items.length; i++) {
             // key could either be create or delete etc, just want the actual data at the value spot
@@ -300,14 +307,18 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
                     reason = `${item.error.type}--${item.error.reason}`;
                     break;
                 }
+            } else if (item.status == null || item.status < 400) {
+                successful++;
             }
         }
 
         if (nonRetriableError) {
-            return { retry: [], error: true, reason };
+            return {
+                retry: [], successful, error: true, reason
+            };
         }
 
-        return { retry, error: false };
+        return { retry, successful, error: false };
     }
 
     function getFirstKey(obj) {
@@ -319,49 +330,60 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
      * @returns {Promise<number>}
     */
     async function _bulkSend(actionRecords, previousCount = 0, previousRetryDelay = 0) {
-        const body = actionRecords.flatMap(({ action, data }) => {
-            if (action == null) {
-                throw new Error('Bulk send record is missing the action property');
+        const body = actionRecords.flatMap((record, index) => {
+            if (record.action == null) {
+                let dbg = '';
+                if (!isProd) {
+                    dbg = `, dbg: ${inspect({ record, index })}`;
+                }
+                throw new Error(`Bulk send record is missing the action property${dbg}`);
             }
-            if (getESVersion() >= 7) {
-                const actionKey = getFirstKey(action);
-                const { _type, ...withoutTypeAction } = action[actionKey];
+
+            if (!isElasticsearch6()) {
+                const actionKey = getFirstKey(record.action);
+                const { _type, ...withoutTypeAction } = record.action[actionKey];
                 // if data is specified return both
-                return data ? [{
-                    ...action,
+                return record.data ? [{
+                    ...record.action,
                     [actionKey]: withoutTypeAction
-                }, data] : [{
-                    ...action,
+                }, record.data] : [{
+                    ...record.action,
                     [actionKey]: withoutTypeAction
                 }];
             }
 
             // if data is specified return both
-            return data ? [action, data] : [action];
+            return record.data ? [record.action, record.data] : [record.action];
         });
 
-        const result = await _clientRequest('bulk', { body });
-        if (result.errors) {
-            const { retry, error, reason } = _filterRetryRecords(actionRecords, result);
+        const response = await _clientRequest('bulk', { body });
+        const results = response.body ? response.body : response;
 
-            if (error) {
-                throw new TSError(reason, {
-                    retryable: false
-                });
-            }
+        if (!results.errors) {
+            return results.items.reduce((c, item) => {
+                const [value] = Object.values(item);
+                // ignore non-successful status codes
+                if (value.status != null && value.status >= 400) return c;
+                return c + 1;
+            }, 0);
+        }
 
-            if (retry.length === 0) {
-                return previousCount;
-            }
+        const {
+            retry, successful, error, reason
+        } = _filterRetryRecords(actionRecords, results);
 
-            warning();
+        if (error) {
+            throw new Error(`bulk send error: ${reason}`);
+        }
 
-            const nextRetryDelay = await _awaitRetry(previousRetryDelay);
-            const diff = actionRecords.length - retry.length;
-            return _bulkSend(retry, previousCount + diff, nextRetryDelay);
+        if (retry.length === 0) {
+            return previousCount + successful;
         }
 
-        return actionRecords.length;
+        warning();
+
+        const nextRetryDelay = await _awaitRetry(previousRetryDelay);
+        return _bulkSend(retry, previousCount + successful, nextRetryDelay);
     }
 
     /**
@@ -609,14 +631,19 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
                 client
                     .search(queryParam)
                     .then((data) => {
-                        const { failures, failed } = data._shards;
+                        const failuresReasons = [];
+                        const results = data.body ? data.body : data;
+                        const { failures, failed } = results._shards;
+
                         if (!failed) {
-                            resolve(data);
+                            resolve(results);
                             return;
                         }
 
+                        failuresReasons.push(...failures);
+
                         const reasons = uniq(
-                            flatten(failures.map((shard) => shard.reason.type))
+                            flatten(failuresReasons.map((shard) => shard.reason.type))
                         );
 
                         if (
@@ -643,13 +670,13 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
     }
 
     function _esV7adjustments(queryParam) {
-        if (getESVersion() >= 7) {
+        if (!isElasticsearch6()) {
             queryParam.trackTotalHits = true;
         }
     }
 
     function _adjustTypeForEs7(query) {
-        if (getESVersion() >= 7) {
+        if (!isElasticsearch6()) {
             if (Array.isArray(query)) {
                 return _removeTypeFromBulkRequest(query);
             }
@@ -660,7 +687,7 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
     }
 
     function _removeTypeFromBulkRequest(query) {
-        if (getESVersion() < 7) return query;
+        if (isElasticsearch6()) return query;
 
         return query.map((queryItem) => {
             if (isSimpleObject(queryItem)) {
@@ -790,11 +817,6 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
         };
     }
 
-    function _checkVersion(str) {
-        const num = Number(str.replace(/\./g, ''));
-        return num >= 210;
-    }
-
     function isAvailable(index, recordType) {
         const query = {
             index,
@@ -848,7 +870,7 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
                 });
         });
     }
-
+    // TODO: verifyClient needs to be checked with new client usage
     /**
      * Verify the state of the underlying es client.
      * Will throw error if in fatal state, like the connection is closed.
@@ -872,16 +894,39 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
 
         return true;
     }
-
+    /** This is deprecated as an external api,
+     * please use getClientMetadata
+     * */
     function getESVersion() {
-        const esVersion = get(client, 'transport._config.apiVersion', '6.5');
+        const newClientVersion = get(client, '__meta.version');
+        const esVersion = newClientVersion || get(client, 'transport._config.apiVersion', '6.5');
+
         if (esVersion && isString(esVersion)) {
             const [majorVersion] = esVersion.split('.');
             return toNumber(majorVersion);
         }
+
         return 6;
     }
 
+    function getClientMetadata() {
+        const newClientVersion = get(client, '__meta.version');
+        const esVersion = newClientVersion || get(client, 'transport._config.apiVersion', '6.5');
+        const distribution = get(client, '__meta.distribution', ElasticsearchDistribution.elasticsearch);
+
+        return {
+            distribution,
+            version: esVersion
+        };
+    }
+
+    function isElasticsearch6() {
+        const { distribution, version: esVersion } = getClientMetadata();
+        const parsedVersion = toNumber(esVersion.split('.', 1)[0]);
+
+        return distribution === ElasticsearchDistribution.elasticsearch && parsedVersion === 6;
+    }
+
     function _fixMappingRequest(_params, isTemplate) {
         if (!_params || !_params.body) {
             throw new Error('Invalid mapping request');
@@ -889,17 +934,14 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
         const params = cloneDeep(_params);
         const defaultParams = {};
 
-        const esVersion = getESVersion();
-        if (esVersion >= 6) {
-            if (params.body.template) {
-                if (isTemplate) {
-                    params.body.index_patterns = castArray(params.body.template).slice();
-                }
-                delete params.body.template;
+        if (params.body.template != null) {
+            if (isTemplate && params.body.index_patterns == null) {
+                params.body.index_patterns = castArray(params.body.template).slice();
             }
+            delete params.body.template;
         }
 
-        if (esVersion >= 7) {
+        if (!isElasticsearch6()) {
             const typeMappings = get(params.body, 'mappings', {});
             if (typeMappings.properties) {
                 defaultParams.includeTypeName = false;
@@ -913,6 +955,7 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
                 });
             }
         }
+
         return Object.assign({}, defaultParams, params);
     }
 
@@ -1011,8 +1054,8 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
 
     function _verifyMapping(query, configMapping, recordType) {
         const params = Object.assign({}, query);
-        const esVersion = getESVersion();
-        if (esVersion > 6) {
+
+        if (!isElasticsearch6()) {
             if (recordType) {
                 params.includeTypeName = true;
             }
@@ -1180,12 +1223,14 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
         indexRecovery,
         indexSetup,
         verifyClient,
-        getESVersion,
         validateGeoParameters,
+        getClientMetadata,
+        isElasticsearch6,
         // The APIs below are deprecated and should be removed.
         index_exists: indexExists,
         index_create: indexCreate,
         index_refresh: indexRefresh,
         index_recovery: indexRecovery,
+        getESVersion,
     };
 };

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/elasticsearch-api",
     "displayName": "Elasticsearch API",
-    "version": "3.0.0",
+    "version": "3.1.0",
     "description": "Elasticsearch client api used across multiple services, handles retries and exponential backoff",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-api#readme",
     "bugs": {
@@ -18,7 +18,7 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/utils": "^0.43.3",
+        "@terascope/utils": "^0.44.1",
         "bluebird": "^3.7.2"
     },
     "devDependencies": {

package/test/api-spec.js

@@ -1,7 +1,9 @@
 'use strict';
 
 const Promise = require('bluebird');
-const { debugLogger, cloneDeep, DataEntity } = require('@terascope/utils');
+const {
+    debugLogger, cloneDeep, DataEntity, isEmpty
+} = require('@terascope/utils');
 const esApi = require('..');
 
 describe('elasticsearch-api', () => {
@@ -189,19 +191,63 @@ describe('elasticsearch-api', () => {
 
     function createBulkResponse(results) {
         const response = { took: 22, errors: false, items: results };
-        if (bulkError) {
+        if (!isEmpty(bulkError)) {
             response.errors = true;
-            response.items = results.body.flatMap((obj, index) => {
+            let i = -1;
+            response.items = results.body.flatMap((obj) => {
                 if (!obj.index && !obj.update && !obj.create && !obj.delete) {
                     // ignore the non-metadata objects
                     return [];
                 }
-                Object.values(obj).forEach((value) => {
-                    Object.assign(value, {
-                        error: { type: bulkError[index] || 'someType', reason: 'someReason' }
-                    });
-                });
-                return [obj];
+                i++;
+                const [key, value] = Object.entries(obj)[0];
+                return [{
+                    [key]: {
+                        _index: value._index,
+                        _type: value._type,
+                        _id: String(i),
+                        _version: 1,
+                        result: `${key}d`,
+                        error: { type: bulkError[i] || 'someType', reason: 'someReason' },
+                        _shards: {
+                            total: 2,
+                            successful: 1,
+                            failed: 0
+                        },
+                        status: 400,
+                        _seq_no: 2,
+                        _primary_term: 3
+                    }
+                }];
+            });
+        } else {
+            response.errors = false;
+            let i = -1;
+            response.items = results.body.flatMap((obj) => {
+                if (!obj.index && !obj.update && !obj.create && !obj.delete) {
+                    // ignore the non-metadata objects
+                    return [];
+                }
+
+                i++;
+                const [key, value] = Object.entries(obj)[0];
+                return [{
+                    [key]: {
+                        _index: value._index,
+                        _type: value._type,
+                        _id: String(i),
+                        _version: 1,
+                        result: `${key}d`,
+                        _shards: {
+                            total: 2,
+                            successful: 1,
+                            failed: 0
+                        },
+                        status: 200,
+                        _seq_no: 2,
+                        _primary_term: 3
+                    }
+                }];
             });
         }
         return response;
@@ -314,7 +360,7 @@ describe('elasticsearch-api', () => {
             stats: () => Promise.resolve({ _nodes: {} })
         },
         cluster: {
-            stats: () => Promise.resolve({ nodes: { versions: ['5.4.1'] } })
+            stats: () => Promise.resolve({ nodes: { versions: ['6.8.1'] } })
         },
         __testing: {
             start: 100,
@@ -824,27 +870,20 @@ describe('elasticsearch-api', () => {
         bulkError = [
             'es_rejected_execution_exception',
             'es_rejected_execution_exception',
-            'es_rejected_execution_exception'
         ];
 
-        const [results] = await Promise.all([
-            api.bulkSend(myBulkData),
-            waitFor(20, () => {
-                bulkError = false;
-            })
-        ]);
+        waitFor(20, () => {
+            bulkError = false;
+        });
+        const result = await api.bulkSend(myBulkData);
 
-        expect(results).toBeTruthy();
-        bulkError = ['some_thing_else', 'some_thing_else', 'some_thing_else'];
+        expect(result).toBe(2);
 
-        return expect(
-            Promise.all([
-                api.bulkSend(myBulkData),
-                waitFor(20, () => {
-                    bulkError = false;
-                })
-            ])
-        ).rejects.toThrow(/some_thing_else--someReason/);
+        bulkError = ['some_thing_else', 'some_thing_else'];
+
+        await expect(
+            api.bulkSend(myBulkData)
+        ).rejects.toThrow('bulk send error: some_thing_else--someReason');
     });
 
     it('can call buildQuery for geo queries', () => {

package/types/index.d.ts

@@ -3,6 +3,7 @@
 
 import * as es from 'elasticsearch';
 import { Logger } from '@terascope/utils';
+import { ClientMetadata } from '@terascope/types'
 
 export = elasticsearchAPI;
 
@@ -44,7 +45,10 @@ declare namespace elasticsearchAPI {
         indexSetup: (clusterName, newIndex, migrantIndexName, mapping, recordType, clientName) => Promise<boolean>;
         verifyClient: () => boolean;
         validateGeoParameters: (opConfig: any) => any;
+        /** This api is deprecated, please use getClientMetadata */
         getESVersion: () => number;
+        getClientMetadata: () => ClientMetadata;
+        isElasticsearch6: () => boolean
     }
 
     /**