@terascope/elasticsearch-api 3.4.0 → 3.5.1

package/index.js

@@ -278,7 +278,6 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
     */
     function _filterRetryRecords(actionRecords, result) {
         const retry = [];
-        const deadLetter = [];
         const { items } = result;
 
         let nonRetriableError = false;
@@ -312,9 +311,10 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
                     reason = `${item.error.type}--${item.error.reason}`;
 
                     if (config._dead_letter_action === 'kafka_dead_letter') {
-                        deadLetter.push({ doc: actionRecords[i].data, reason });
+                        actionRecords[i].data.setMetadata('_bulk_sender_rejection', reason);
                         continue;
                     }
+
                     break;
                 }
             } else if (item.status == null || item.status < 400) {
@@ -323,8 +323,11 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
         }
 
         if (nonRetriableError) {
+            // if dlq active still attempt the retries
+            const retryOnError = config._dead_letter_action === 'kafka_dead_letter' ? retry : [];
+
             return {
-                retry: [], successful, error: true, reason, deadLetter
+                retry: retryOnError, successful, error: true, reason
             };
         }
 
@@ -337,7 +340,7 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
 
     /**
      * @param data {Array<{ action: data }>}
-     * @returns {Promise<{ recordCount: number, deadLetter: record[] }>}
+     * @returns {Promise<number>}
     */
     async function _bulkSend(actionRecords, previousCount = 0, previousRetryDelay = 0) {
         const body = actionRecords.flatMap((record, index) => {
@@ -349,7 +352,18 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
                 throw new Error(`Bulk send record is missing the action property${dbg}`);
             }
 
-            if (!isElasticsearch6()) return _nonEs6Prep(record);
+            if (!isElasticsearch6()) {
+                const actionKey = getFirstKey(record.action);
+                const { _type, ...withoutTypeAction } = record.action[actionKey];
+                // if data is specified return both
+                return record.data ? [{
+                    ...record.action,
+                    [actionKey]: withoutTypeAction
+                }, record.data] : [{
+                    ...record.action,
+                    [actionKey]: withoutTypeAction
+                }];
+            }
 
             // if data is specified return both
             return record.data ? [record.action, record.data] : [record.action];
@@ -358,51 +372,41 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
         const response = await _clientRequest('bulk', { body });
         const results = response.body ? response.body : response;
 
-        if (!results.errors) return { recordCount: _affectedRowsCount(results) };
+        if (!results.errors) {
+            return results.items.reduce((c, item) => {
+                const [value] = Object.values(item);
+                // ignore non-successful status codes
+                if (value.status != null && value.status >= 400) return c;
+                return c + 1;
+            }, 0);
+        }
 
         const {
-            retry, successful, error, reason, deadLetter
+            retry, successful, error, reason
         } = _filterRetryRecords(actionRecords, results);
 
-        if (error) {
-            if (config._dead_letter_action === 'kafka_dead_letter') {
-                return {
-                    recordCount: previousCount + successful,
-                    deadLetter
-                };
-            }
-
+        if (error && config._dead_letter_action !== 'kafka_dead_letter') {
             throw new Error(`bulk send error: ${reason}`);
         }
 
         if (retry.length === 0) {
-            return { recordCount: previousCount + successful };
+            return previousCount + successful;
         }
 
-        warning();
-
-        const nextRetryDelay = await _awaitRetry(previousRetryDelay);
-        return _bulkSend(retry, previousCount + successful, nextRetryDelay);
+        return _handleRetries(retry, previousCount + successful, previousRetryDelay);
     }
 
-    function _nonEs6Prep(record) {
-        const actionKey = getFirstKey(record.action);
-
-        const { _type, ...withoutTypeAction } = record.action[actionKey];
-        // if data is specified return both
-
-        const body = [{ ...record.action, [actionKey]: withoutTypeAction }];
-
-        if (record.data != null) body.push(record.data);
+    async function _handleRetries(retry, affectedCount, previousRetryDelay) {
+        warning();
 
-        return body;
+        const nextRetryDelay = await _awaitRetry(previousRetryDelay);
+        return _bulkSend(retry, affectedCount, nextRetryDelay);
     }
 
     /**
      * The new and improved bulk send with proper retry support
      *
-     * @returns {Promise<{ recordCount: number, deadLetter: record[] }>}
-     * the number of affected rows and records for kafka dead letter queue
+     * @returns {Promise<number>} the number of affected rows
     */
     function bulkSend(data) {
         if (!Array.isArray(data)) {
@@ -412,15 +416,6 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
         return Promise.resolve(_bulkSend(data));
     }
 
-    function _affectedRowsCount(results) {
-        return results.items.reduce((c, item) => {
-            const [value] = Object.values(item);
-            // ignore non-successful status codes
-            if (value.status != null && value.status >= 400) return c;
-            return c + 1;
-        }, 0);
-    }
-
     function _warn(warnLogger, msg) {
         let _lastTime = null;
         return () => {

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/elasticsearch-api",
     "displayName": "Elasticsearch API",
-    "version": "3.4.0",
+    "version": "3.5.1",
     "description": "Elasticsearch client api used across multiple services, handles retries and exponential backoff",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-api#readme",
     "bugs": {
@@ -23,7 +23,7 @@
     },
     "dependencies": {
         "@terascope/types": "^0.11.2",
-        "@terascope/utils": "^0.45.5",
+        "@terascope/utils": "^0.45.6",
         "bluebird": "^3.7.2",
         "setimmediate": "^1.0.5"
     },
@@ -31,7 +31,7 @@
         "@opensearch-project/opensearch": "^1.1.0",
         "@types/elasticsearch": "^5.0.40",
         "elasticsearch": "^15.4.1",
-        "elasticsearch-store": "^0.65.5",
+        "elasticsearch-store": "^0.66.1",
         "elasticsearch6": "npm:@elastic/elasticsearch@^6.7.0",
         "elasticsearch7": "npm:@elastic/elasticsearch@^7.0.0",
         "elasticsearch8": "npm:@elastic/elasticsearch@^8.0.0"

package/test/api-spec.js

@@ -768,7 +768,7 @@ describe('elasticsearch-api', () => {
                 { delete: { _index: 'some_index', _type: 'events', _id: 5 } }
             ]
         });
-        return expect(result).toEqual({ recordCount: 2 });
+        return expect(result).toBe(2);
     });
 
     it('can remove type from bulkSend', async () => {
@@ -877,7 +877,7 @@ describe('elasticsearch-api', () => {
         });
         const result = await api.bulkSend(myBulkData);
 
-        expect(result).toEqual({ recordCount: 2 });
+        expect(result).toBe(2);
 
         bulkError = ['some_thing_else', 'some_thing_else'];
 

package/test/{bulk-send-dlq.js → bulk-send-dlq-spec.js}

@@ -14,7 +14,9 @@ const {
     createMappingFromDatatype
 } = ElasticsearchTestHelpers;
 
-jest.setTimeout(10000);
+const THREE_MINUTES = 3 * 60 * 1000;
+
+jest.setTimeout(THREE_MINUTES + 60000);
 
 function formatUploadData(
     index, data, isES8ClientTest = false
@@ -72,35 +74,48 @@ describe('bulkSend', () => {
             await cleanupIndex(client, index);
         });
 
-        it('returns records that cannot be tried again if dlq config is set', async () => {
+        it('should send records if no issues and dlq not set', async () => {
+            const diffApi = elasticsearchAPI(client, logger);
+
+            const docs = cloneDeep(EvenDateData.data.slice(0, 2));
+
+            const result = await diffApi.bulkSend(formatUploadData(index, docs, isElasticsearch8));
+
+            expect(result).toBe(2);
+        });
+
+        it('should throw an error if dlq is not set', async () => {
+            const diffApi = elasticsearchAPI(client, logger);
+
             const docs = cloneDeep(EvenDateData.data.slice(0, 2));
 
             docs[0].bytes = 'this is a bad value';
 
-            const result = await api.bulkSend(formatUploadData(index, docs, isElasticsearch8));
+            await expect(diffApi.bulkSend(formatUploadData(index, docs, isElasticsearch8)))
+                .rejects.toThrow();
+        });
+
+        it('should update metadata for records that are not retryable and return results for successful updates if dlq is set', async () => {
+            const docs = cloneDeep(EvenDateData.data.slice(0, 2))
+                .map((doc, i) => DataEntity.make(doc, { _key: i + 1 }));
 
-            expect(result.recordCount).toBe(1);
+            docs[0].bytes = 'this is a bad value';
+
+            const result = await api.bulkSend(formatUploadData(index, docs, isElasticsearch8));
 
-            expect(result.deadLetter[0].doc).toEqual(DataEntity.make({
-                ip: '120.67.248.156',
-                userAgent: 'Mozilla/5.0 (Windows; U; Windows NT 6.1) AppleWebKit/533.1.2 (KHTML, like Gecko) Chrome/35.0.894.0 Safari/533.1.2',
-                url: 'http://lucious.biz',
-                uuid: 'b23a8550-0081-453f-9e80-93a90782a5bd',
-                created: '2019-04-26T15:00:23.225+00:00',
-                ipv6: '9e79:7798:585a:b847:f1c4:81eb:0c3d:7eb8',
-                location: '50.15003, -94.89355',
-                bytes: 'this is a bad value'
-            }));
+            // 1 good doc  - so only 1 row affected
+            expect(result).toBe(1);
 
-            expect(result.deadLetter[0].reason).toBeDefined();
+            expect(docs[0].getMetadata('_bulk_sender_rejection')).toInclude('mapper_parsing_exception--failed to parse field [bytes]');
+            expect(docs[1].getMetadata('_bulk_sender_rejection')).toBeUndefined();
         });
 
-        it('should return a count if not un-retryable records', async () => {
+        it('should return a count if not un-retryable records if dlq is set', async () => {
             const docs = cloneDeep(EvenDateData.data.slice(0, 2));
 
             const result = await api.bulkSend(formatUploadData(index, docs, isElasticsearch8));
 
-            expect(result).toEqual({ recordCount: 2 });
+            expect(result).toBe(2);
         });
     });
 });

package/types/index.d.ts

@@ -32,9 +32,9 @@ declare namespace elasticsearchAPI {
         /**
          * The new and improved bulk send with proper retry support
          *
-         * @returns the number of affected rows, and deadLetter records if config is set
+         * @returns the number of affected rows
         */
-        bulkSend: (data: BulkRecord[]) => Promise<{ recordCount: number; deadLetter?: any[]; }>;
+        bulkSend: (data: BulkRecord[]) => Promise<number>;
         nodeInfo: (query: any) => Promise<any>;
         nodeStats: (query: any) => Promise<any>;
         buildQuery: (opConfig: Config, msg: any) => ClientParams.SearchParams;