@terascope/elasticsearch-api 2.24.2 → 3.0.2

package/index.js

@@ -19,14 +19,17 @@ const {
     random,
     cloneDeep,
     DataEntity,
-    isDeepEqual
+    isDeepEqual,
+    getTypeOf,
+    isProd
 } = require('@terascope/utils');
+const { inspect } = require('util');
 
 const DOCUMENT_EXISTS = 409;
 
 // Module to manage persistence in Elasticsearch.
 // All functions in this module return promises that must be resolved to get the final result.
-module.exports = function elasticsearchApi(client = {}, logger, _opConfig) {
+module.exports = function elasticsearchApi(client, logger, _opConfig) {
     const config = _opConfig || {};
     if (!client) {
         throw new Error('Elasticsearch API requires client');
@@ -98,7 +101,7 @@ module.exports = function elasticsearchApi(client = {}, logger, _opConfig) {
 
             function _runRequest() {
                 clientBase[endpoint](query)
-                    .then((result) => resolve(result))
+                    .then(resolve)
                     .catch(errHandler);
             }
 
@@ -211,7 +214,7 @@ module.exports = function elasticsearchApi(client = {}, logger, _opConfig) {
     }
 
     function version() {
-        const wildCardRegex = RegExp(/\*/g);
+        const wildCardRegex = /\*/g;
         const isWildCardRegexSearch = config.index.match(wildCardRegex);
         // We cannot reliable search index queries with wildcards
         // for existence or max_result_window, it could be
@@ -259,12 +262,26 @@ module.exports = function elasticsearchApi(client = {}, logger, _opConfig) {
         );
     }
 
-    function _filterResponse(data, results) {
+    /**
+     * When the bulk request has errors this will find the actions
+     * records to retry.
+     *
+     * @returns {{
+     *    retry: Record<string, any>[],
+     *    successful: number,
+     *    error: boolean,
+     *    reason?: string
+     * }}
+    */
+    function _filterRetryRecords(actionRecords, result) {
         const retry = [];
-        const { items } = results;
+        const { items } = result;
+
         let nonRetriableError = false;
         let reason = '';
-        for (let i = 0; i < items.length; i += 1) {
+        let successful = 0;
+
+        for (let i = 0; i < items.length; i++) {
             // key could either be create or delete etc, just want the actual data at the value spot
             const item = Object.values(items[i])[0];
             if (item.error) {
@@ -275,11 +292,14 @@ module.exports = function elasticsearchApi(client = {}, logger, _opConfig) {
                 }
 
                 if (item.error.type === 'es_rejected_execution_exception') {
-                    if (i === 0) {
-                        retry.push(data[0], data[1]);
-                    } else {
-                        retry.push(data[i * 2], data[i * 2 + 1]);
+                    if (actionRecords[i] == null) {
+                        // this error should not happen in production,
+                        // only in tests where the bulk function is mocked
+                        throw new Error(`Invalid item index (${i}), not found in bulk send records (length: ${actionRecords.length})`);
                     }
+                    // the index in the item list will match the index in the
+                    // input records
+                    retry.push(actionRecords[i]);
                 } else if (
                     item.error.type !== 'document_already_exists_exception'
                     && item.error.type !== 'document_missing_exception'
@@ -288,55 +308,95 @@ module.exports = function elasticsearchApi(client = {}, logger, _opConfig) {
                     reason = `${item.error.type}--${item.error.reason}`;
                     break;
                 }
+            } else if (item.status == null || item.status < 400) {
+                successful++;
             }
         }
 
         if (nonRetriableError) {
-            return { data: [], error: true, reason };
+            return {
+                retry: [], successful, error: true, reason
+            };
         }
 
-        return { data: retry, error: false };
+        return { retry, successful, error: false };
     }
 
-    function bulkSend(data) {
-        return new Promise((resolve, reject) => {
-            const retry = _retryFn(_sendData, data, reject);
-
-            function _sendData(formattedData) {
-                return _clientRequest('bulk', { body: formattedData })
-                    .then((results) => {
-                        if (results.errors) {
-                            const response = _filterResponse(formattedData, results);
-
-                            if (response.error) {
-                                reject(new TSError(response.reason, {
-                                    retryable: false
-                                }));
-                            } else if (response.data.length === 0) {
-                                // may get doc already created error, if so just return
-                                resolve(results);
-                            } else {
-                                warning();
-                                retry(response.data);
-                            }
-                        } else {
-                            resolve(results);
-                        }
-                    })
-                    .catch((err) => {
-                        const error = new TSError(err, {
-                            reason: 'bulk sender error',
-                            context: {
-                                connection,
-                            },
-                        });
+    function getFirstKey(obj) {
+        return Object.keys(obj)[0];
+    }
 
-                        reject(error);
-                    });
+    /**
+     * @param data {Array<{ action: data }>}
+     * @returns {Promise<number>}
+    */
+    async function _bulkSend(actionRecords, previousCount = 0, previousRetryDelay = 0) {
+        const body = actionRecords.flatMap((record, index) => {
+            if (record.action == null) {
+                let dbg = '';
+                if (!isProd) {
+                    dbg = `, dbg: ${inspect({ record, index })}`;
+                }
+                throw new Error(`Bulk send record is missing the action property${dbg}`);
             }
 
-            _sendData(_adjustTypeForEs7(data));
+            if (getESVersion() >= 7) {
+                const actionKey = getFirstKey(record.action);
+                const { _type, ...withoutTypeAction } = record.action[actionKey];
+                // if data is specified return both
+                return record.data ? [{
+                    ...record.action,
+                    [actionKey]: withoutTypeAction
+                }, record.data] : [{
+                    ...record.action,
+                    [actionKey]: withoutTypeAction
+                }];
+            }
+
+            // if data is specified return both
+            return record.data ? [record.action, record.data] : [record.action];
         });
+
+        const result = await _clientRequest('bulk', { body });
+
+        if (!result.errors) {
+            return result.items.reduce((c, item) => {
+                const [value] = Object.values(item);
+                // ignore non-successful status codes
+                if (value.status != null && value.status >= 400) return c;
+                return c + 1;
+            }, 0);
+        }
+
+        const {
+            retry, successful, error, reason
+        } = _filterRetryRecords(actionRecords, result);
+
+        if (error) {
+            throw new Error(`bulk send error: ${reason}`);
+        }
+
+        if (retry.length === 0) {
+            return previousCount + successful;
+        }
+
+        warning();
+
+        const nextRetryDelay = await _awaitRetry(previousRetryDelay);
+        return _bulkSend(retry, previousCount + successful, nextRetryDelay);
+    }
+
+    /**
+     * The new and improved bulk send with proper retry support
+     *
+     * @returns {Promise<number>} the number of affected rows
+    */
+    function bulkSend(data) {
+        if (!Array.isArray(data)) {
+            throw new Error(`Expected bulkSend to receive an array, got ${data} (${getTypeOf(data)})`);
+        }
+
+        return Promise.resolve(_bulkSend(data));
     }
 
     function _warn(warnLogger, msg) {
@@ -622,6 +682,8 @@ module.exports = function elasticsearchApi(client = {}, logger, _opConfig) {
     }
 
     function _removeTypeFromBulkRequest(query) {
+        if (getESVersion() < 7) return query;
+
         return query.map((queryItem) => {
             if (isSimpleObject(queryItem)) {
                 // get the metadata and ignore the record
@@ -696,17 +758,29 @@ module.exports = function elasticsearchApi(client = {}, logger, _opConfig) {
         return (_data) => {
             const args = _data || data;
 
+            _awaitRetry(delay)
+                .then((newDelay) => {
+                    delay = newDelay;
+                    fn(args);
+                })
+                .catch(reject);
+        };
+    }
+
+    /**
+     * @returns {Promise<number>} the delayed time
+    */
+    async function _awaitRetry(previousDelay = 0) {
+        return new Promise((resolve, reject) => {
             waitForClient((elapsed) => {
-                delay = getBackoffDelay(delay, 2, retryLimit, retryStart);
+                const delay = getBackoffDelay(previousDelay, 2, retryLimit, retryStart);
 
                 let timeoutMs = delay - elapsed;
                 if (timeoutMs < 1) timeoutMs = 1;
 
-                setTimeout(() => {
-                    fn(args);
-                }, timeoutMs);
+                setTimeout(resolve, timeoutMs, delay);
             }, reject);
-        };
+        });
     }
 
     function _errorHandler(fn, data, reject, fnName = '->unknown()') {

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/elasticsearch-api",
     "displayName": "Elasticsearch API",
-    "version": "2.24.2",
+    "version": "3.0.2",
     "description": "Elasticsearch client api used across multiple services, handles retries and exponential backoff",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-api#readme",
     "bugs": {
@@ -18,11 +18,11 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/utils": "^0.43.2",
+        "@terascope/utils": "^0.44.1",
         "bluebird": "^3.7.2"
     },
     "devDependencies": {
-        "@types/elasticsearch": "^5.0.37"
+        "@types/elasticsearch": "^5.0.40"
     },
     "engines": {
         "node": "^12.22.0 || >=14.17.0",

package/test/api-spec.js

@@ -1,7 +1,9 @@
 'use strict';
 
 const Promise = require('bluebird');
-const { debugLogger, cloneDeep, DataEntity } = require('@terascope/utils');
+const {
+    debugLogger, cloneDeep, DataEntity, isEmpty
+} = require('@terascope/utils');
 const esApi = require('..');
 
 describe('elasticsearch-api', () => {
@@ -189,15 +191,63 @@ describe('elasticsearch-api', () => {
 
     function createBulkResponse(results) {
         const response = { took: 22, errors: false, items: results };
-        if (bulkError) {
+        if (!isEmpty(bulkError)) {
             response.errors = true;
-            response.items = results.body.map((obj, index) => {
-                Object.entries(obj).forEach(([key, value]) => {
-                    obj[key] = Object.assign(value, {
-                        error: { type: bulkError[index] || 'someType', reason: 'someReason' }
-                    });
-                });
-                return obj;
+            let i = -1;
+            response.items = results.body.flatMap((obj) => {
+                if (!obj.index && !obj.update && !obj.create && !obj.delete) {
+                    // ignore the non-metadata objects
+                    return [];
+                }
+                i++;
+                const [key, value] = Object.entries(obj)[0];
+                return [{
+                    [key]: {
+                        _index: value._index,
+                        _type: value._type,
+                        _id: String(i),
+                        _version: 1,
+                        result: `${key}d`,
+                        error: { type: bulkError[i] || 'someType', reason: 'someReason' },
+                        _shards: {
+                            total: 2,
+                            successful: 1,
+                            failed: 0
+                        },
+                        status: 400,
+                        _seq_no: 2,
+                        _primary_term: 3
+                    }
+                }];
+            });
+        } else {
+            response.errors = false;
+            let i = -1;
+            response.items = results.body.flatMap((obj) => {
+                if (!obj.index && !obj.update && !obj.create && !obj.delete) {
+                    // ignore the non-metadata objects
+                    return [];
+                }
+
+                i++;
+                const [key, value] = Object.entries(obj)[0];
+                return [{
+                    [key]: {
+                        _index: value._index,
+                        _type: value._type,
+                        _id: String(i),
+                        _version: 1,
+                        result: `${key}d`,
+                        _shards: {
+                            total: 2,
+                            successful: 1,
+                            failed: 0
+                        },
+                        status: 200,
+                        _seq_no: 2,
+                        _primary_term: 3
+                    }
+                }];
             });
         }
         return response;
@@ -323,45 +373,25 @@ describe('elasticsearch-api', () => {
         expect(() => {
             api = esApi(client, logger);
         }).not.toThrow();
-        expect(api).toBeDefined();
         expect(typeof api).toEqual('object');
-        expect(api.search).toBeDefined();
         expect(typeof api.search).toEqual('function');
-        expect(api.count).toBeDefined();
         expect(typeof api.count).toEqual('function');
-        expect(api.get).toBeDefined();
         expect(typeof api.get).toEqual('function');
-        expect(api.index).toBeDefined();
         expect(typeof api.index).toEqual('function');
-        expect(api.indexWithId).toBeDefined();
         expect(typeof api.indexWithId).toEqual('function');
-        expect(api.create).toBeDefined();
         expect(typeof api.create).toEqual('function');
-        expect(api.update).toBeDefined();
         expect(typeof api.update).toEqual('function');
-        expect(api.remove).toBeDefined();
         expect(typeof api.remove).toEqual('function');
-        expect(api.version).toBeDefined();
         expect(typeof api.version).toEqual('function');
-        expect(api.putTemplate).toBeDefined();
         expect(typeof api.putTemplate).toEqual('function');
-        expect(api.bulkSend).toBeDefined();
         expect(typeof api.bulkSend).toEqual('function');
-        expect(api.nodeInfo).toBeDefined();
         expect(typeof api.nodeInfo).toEqual('function');
-        expect(api.nodeStats).toBeDefined();
         expect(typeof api.nodeStats).toEqual('function');
-        expect(api.buildQuery).toBeDefined();
         expect(typeof api.buildQuery).toEqual('function');
-        expect(api.indexExists).toBeDefined();
         expect(typeof api.indexExists).toEqual('function');
-        expect(api.indexCreate).toBeDefined();
         expect(typeof api.indexCreate).toEqual('function');
-        expect(api.indexRefresh).toBeDefined();
         expect(typeof api.indexRefresh).toEqual('function');
-        expect(api.indexRecovery).toBeDefined();
         expect(typeof api.indexRecovery).toEqual('function');
-        expect(api.indexSetup).toBeDefined();
         expect(typeof api.indexSetup).toEqual('function');
     });
 
@@ -717,30 +747,48 @@ describe('elasticsearch-api', () => {
 
     it('can call bulkSend', async () => {
         const api = esApi(client, logger);
-        const myBulkData = [
-            { index: { _index: 'some_index', _type: 'events', _id: 1 } },
-            { title: 'foo' },
-            { delete: { _index: 'some_index', _type: 'events', _id: 5 } }
-        ];
 
-        const results = await api.bulkSend(myBulkData);
-        return expect(results).toBeTruthy();
+        const result = await api.bulkSend([
+            {
+                action: {
+                    index: { _index: 'some_index', _type: 'events', _id: 1 }
+                },
+                data: { title: 'foo' }
+            },
+            {
+                action: {
+                    delete: { _index: 'some_index', _type: 'events', _id: 5 }
+                }
+            }
+        ]);
+        expect(bulkData).toEqual({
+            body: [
+                { index: { _index: 'some_index', _type: 'events', _id: 1 } },
+                { title: 'foo' },
+                { delete: { _index: 'some_index', _type: 'events', _id: 5 } }
+            ]
+        });
+        return expect(result).toBe(2);
     });
 
-    it('can remove type from bulk send', async () => {
+    it('can remove type from bulkSend', async () => {
         const es7client = cloneDeep(client);
 
         es7client.transport._config = { apiVersion: '7.0' };
 
         const api = esApi(es7client, logger);
 
-        const myBulkData = [
-            { index: { _index: 'some_index', _type: 'events', _id: 1 } },
-            { title: 'foo' },
-            { delete: { _index: 'some_index', _type: 'events', _id: 5 } }
-        ];
-
-        await api.bulkSend(myBulkData);
+        await api.bulkSend([{
+            action: {
+                index: { _index: 'some_index', _type: 'events', _id: 1 }
+            },
+            data: { title: 'foo' }
+        },
+        {
+            action: {
+                delete: { _index: 'some_index', _type: 'events', _id: 5 }
+            }
+        }]);
         expect(bulkData).toEqual({
             body: [
                 { index: { _index: 'some_index', _id: 1 } },
@@ -750,20 +798,24 @@ describe('elasticsearch-api', () => {
         });
     });
 
-    it('will not remove _type from record in a bulk send', async () => {
+    it('will not remove _type from record in a bulkSend', async () => {
         const es7client = cloneDeep(client);
 
         es7client.transport._config = { apiVersion: '7.0' };
 
         const api = esApi(es7client, logger);
 
-        const myBulkData = [
-            { delete: { _index: 'some_index', _type: 'events', _id: 5 } },
-            { index: { _index: 'some_index', _type: 'events', _id: 1 } },
-            { title: 'foo', _type: 'doc', name: 'joe' }
-        ];
-
-        await api.bulkSend(myBulkData);
+        await api.bulkSend([{
+            action: {
+                delete: { _index: 'some_index', _type: 'events', _id: 5 }
+            },
+        },
+        {
+            action: {
+                index: { _index: 'some_index', _type: 'events', _id: 1 }
+            },
+            data: { title: 'foo', _type: 'doc', name: 'joe' }
+        }]);
         expect(bulkData).toEqual({
             body: [
                 { delete: { _index: 'some_index', _id: 5 } },
@@ -773,20 +825,25 @@ describe('elasticsearch-api', () => {
         });
     });
 
-    it('will not err if no _type in es7 bulk request metadata', async () => {
+    it('will not err if no _type in es7 bulkSend request metadata', async () => {
         const es7client = cloneDeep(client);
 
         es7client.transport._config = { apiVersion: '7.0' };
 
         const api = esApi(es7client, logger);
 
-        const myBulkData = [
-            { delete: { _index: 'some_index', _id: 5 } },
-            { index: { _index: 'some_index', _id: 1 } },
-            { title: 'foo', _type: 'doc', name: 'joe' }
-        ];
+        await api.bulkSend([{
+            action: {
+                delete: { _index: 'some_index', _type: 'events', _id: 5 }
+            },
+        },
+        {
+            action: {
+                index: { _index: 'some_index', _type: 'events', _id: 1 }
+            },
+            data: { title: 'foo', _type: 'doc', name: 'joe' }
+        }]);
 
-        await api.bulkSend(myBulkData);
         expect(bulkData).toEqual({
             body: [
                 { delete: { _index: 'some_index', _id: 5 } },
@@ -798,36 +855,35 @@ describe('elasticsearch-api', () => {
 
     it('can call bulkSend with errors', async () => {
         const api = esApi(client, logger);
-        const myBulkData = [
-            { index: { _index: 'some_index', _type: 'events', _id: 1 } },
-            { title: 'foo' },
-            { delete: { _index: 'some_index', _type: 'events', _id: 5 } }
-        ];
+        const myBulkData = [{
+            action: {
+                index: { _index: 'some_index', _type: 'events', _id: 1 }
+            },
+            data: { title: 'foo' }
+        },
+        {
+            action: {
+                delete: { _index: 'some_index', _type: 'events', _id: 5 }
+            }
+        }];
 
         bulkError = [
             'es_rejected_execution_exception',
             'es_rejected_execution_exception',
-            'es_rejected_execution_exception'
         ];
 
-        const [results] = await Promise.all([
-            api.bulkSend(myBulkData),
-            waitFor(20, () => {
-                bulkError = false;
-            })
-        ]);
+        waitFor(20, () => {
+            bulkError = false;
+        });
+        const result = await api.bulkSend(myBulkData);
 
-        expect(results).toBeTruthy();
-        bulkError = ['some_thing_else', 'some_thing_else', 'some_thing_else'];
+        expect(result).toBe(2);
 
-        return expect(
-            Promise.all([
-                api.bulkSend(myBulkData),
-                waitFor(20, () => {
-                    bulkError = false;
-                })
-            ])
-        ).rejects.toThrow(/some_thing_else--someReason/);
+        bulkError = ['some_thing_else', 'some_thing_else'];
+
+        await expect(
+            api.bulkSend(myBulkData)
+        ).rejects.toThrow('bulk send error: some_thing_else--someReason');
     });
 
     it('can call buildQuery for geo queries', () => {

package/types/index.d.ts

@@ -28,7 +28,12 @@ declare namespace elasticsearchAPI {
         remove: (query: es.DeleteDocumentParams) => Promise<es.DeleteDocumentResponse>;
         version: () => Promise<boolean>;
         putTemplate: (template: any, name: string) => Promise<any>;
-        bulkSend: (data: any[]) => Promise<any>;
+        /**
+         * The new and improved bulk send with proper retry support
+         *
+         * @returns the number of affected rows
+        */
+        bulkSend: (data: BulkRecord[]) => Promise<number>;
         nodeInfo: (query: any) => Promise<any>;
         nodeStats: (query: any) => Promise<any>;
         buildQuery: (opConfig: Config, msg: any) => es.SearchParams;
@@ -41,4 +46,32 @@ declare namespace elasticsearchAPI {
         validateGeoParameters: (opConfig: any) => any;
         getESVersion: () => number;
     }
+
+    /**
+     * This is used for improved bulk sending function
+    */
+    export interface BulkRecord {
+        action: AnyBulkAction;
+        data?: UpdateConfig | DataEntity;
+    }
+
+    /**
+     * This is used for improved bulk sending function
+    */
+    export interface AnyBulkAction  {
+        update?: Partial<BulkActionMetadata>;
+        index?: Partial<BulkActionMetadata>;
+        create?: Partial<BulkActionMetadata>;
+        delete?: Partial<BulkActionMetadata>;
+    }
+
+    /**
+     * This is used for improved bulk sending function
+    */
+    export interface BulkActionMetadata {
+        _index: string;
+        _type: string;
+        _id: string | number;
+        retry_on_conflict?: number;
+    }
 }