@terascope/elasticsearch-api 3.3.7 → 3.5.0

package/index.js

@@ -309,6 +309,12 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
                 ) {
                     nonRetriableError = true;
                     reason = `${item.error.type}--${item.error.reason}`;
+
+                    if (config._dead_letter_action === 'kafka_dead_letter') {
+                        actionRecords[i].data.setMetadata('_bulk_sender_rejection', reason);
+                        continue;
+                    }
+
                     break;
                 }
             } else if (item.status == null || item.status < 400) {
@@ -317,8 +323,11 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
         }
 
         if (nonRetriableError) {
+            // if dlq active still attempt the retries
+            const retryOnError = config._dead_letter_action === 'kafka_dead_letter' ? retry : [];
+
             return {
-                retry: [], successful, error: true, reason
+                retry: retryOnError, successful, error: true, reason
             };
         }
 
@@ -376,7 +385,7 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
             retry, successful, error, reason
         } = _filterRetryRecords(actionRecords, results);
 
-        if (error) {
+        if (error && config._dead_letter_action !== 'kafka_dead_letter') {
             throw new Error(`bulk send error: ${reason}`);
         }
 
@@ -384,10 +393,14 @@ module.exports = function elasticsearchApi(client, logger, _opConfig) {
             return previousCount + successful;
         }
 
+        return _handleRetries(retry, previousCount + successful, previousRetryDelay);
+    }
+
+    async function _handleRetries(retry, affectedCount, previousRetryDelay) {
         warning();
 
         const nextRetryDelay = await _awaitRetry(previousRetryDelay);
-        return _bulkSend(retry, previousCount + successful, nextRetryDelay);
+        return _bulkSend(retry, affectedCount, nextRetryDelay);
     }
 
     /**

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/elasticsearch-api",
     "displayName": "Elasticsearch API",
-    "version": "3.3.7",
+    "version": "3.5.0",
     "description": "Elasticsearch client api used across multiple services, handles retries and exponential backoff",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-api#readme",
     "bugs": {

package/test/bulk-send-dlq-spec.js

@@ -0,0 +1,119 @@
+'use strict';
+
+const {
+    debugLogger,
+    cloneDeep,
+    DataEntity
+} = require('@terascope/utils');
+const { ElasticsearchTestHelpers } = require('elasticsearch-store');
+const elasticsearchAPI = require('../index');
+
+const {
+    makeClient, cleanupIndex,
+    EvenDateData, TEST_INDEX_PREFIX,
+    createMappingFromDatatype
+} = ElasticsearchTestHelpers;
+
+jest.setTimeout(10000);
+
+function formatUploadData(
+    index, data, isES8ClientTest = false
+) {
+    const results = [];
+
+    data.forEach((record, i) => {
+        const meta = { _index: index, _id: i + 1 };
+
+        if (!isES8ClientTest) {
+            meta._type = '_doc';
+        }
+
+        results.push({ action: { index: meta }, data: record });
+    });
+
+    return results;
+}
+
+describe('bulkSend', () => {
+    let client;
+    let api;
+    let isElasticsearch8 = false;
+
+    beforeAll(async () => {
+        client = await makeClient();
+    });
+
+    describe('can return non-retryable records', () => {
+        const logger = debugLogger('congested_test');
+        const index = `${TEST_INDEX_PREFIX}_non-retryable-records`;
+
+        beforeAll(async () => {
+            await cleanupIndex(client, index);
+            api = elasticsearchAPI(client, logger, { _dead_letter_action: 'kafka_dead_letter' });
+            isElasticsearch8 = api.isElasticsearch8();
+
+            const overrides = {
+                settings: {
+                    'index.number_of_shards': 1,
+                    'index.number_of_replicas': 0,
+                },
+            };
+
+            const mapping = await createMappingFromDatatype(
+                client, EvenDateData.EvenDataType, '_doc', overrides
+            );
+
+            mapping.index = index;
+
+            await client.indices.create(mapping);
+        });
+
+        afterAll(async () => {
+            await cleanupIndex(client, index);
+        });
+
+        it('should send records if no issues and dlq not set', async () => {
+            const diffApi = elasticsearchAPI(client, logger);
+
+            const docs = cloneDeep(EvenDateData.data.slice(0, 2));
+
+            const result = await diffApi.bulkSend(formatUploadData(index, docs, isElasticsearch8));
+
+            expect(result).toBe(2);
+        });
+
+        it('should throw an error if dlq is not set', async () => {
+            const diffApi = elasticsearchAPI(client, logger);
+
+            const docs = cloneDeep(EvenDateData.data.slice(0, 2));
+
+            docs[0].bytes = 'this is a bad value';
+
+            await expect(diffApi.bulkSend(formatUploadData(index, docs, isElasticsearch8)))
+                .rejects.toThrow();
+        });
+
+        it('should update metadata for records that are not retryable and return results for successful updates if dlq is set', async () => {
+            const docs = cloneDeep(EvenDateData.data.slice(0, 2))
+                .map((doc, i) => DataEntity.make(doc, { _key: i + 1 }));
+
+            docs[0].bytes = 'this is a bad value';
+
+            const result = await api.bulkSend(formatUploadData(index, docs, isElasticsearch8));
+
+            // 1 good doc  - so only 1 row affected
+            expect(result).toBe(1);
+
+            expect(docs[0].getMetadata('_bulk_sender_rejection')).toInclude('mapper_parsing_exception--failed to parse field [bytes]');
+            expect(docs[1].getMetadata('_bulk_sender_rejection')).toBeUndefined();
+        });
+
+        it('should return a count if not un-retryable records if dlq is set', async () => {
+            const docs = cloneDeep(EvenDateData.data.slice(0, 2));
+
+            const result = await api.bulkSend(formatUploadData(index, docs, isElasticsearch8));
+
+            expect(result).toBe(2);
+        });
+    });
+});

package/test/bulk-send-limit-spec.js

@@ -1,6 +1,10 @@
 'use strict';
 
-const { debugLogger, chunk, pMap } = require('@terascope/utils');
+const {
+    debugLogger,
+    chunk,
+    pMap
+} = require('@terascope/utils');
 const { ElasticsearchTestHelpers } = require('elasticsearch-store');
 const elasticsearchAPI = require('../index');
 
@@ -11,7 +15,7 @@ const {
 
 const THREE_MINUTES = 3 * 60 * 1000;
 
-jest.setTimeout(THREE_MINUTES + 30000);
+jest.setTimeout(THREE_MINUTES + 60000);
 
 function formatUploadData(
     index, data, isES8ClientTest = false
@@ -31,33 +35,38 @@ function formatUploadData(
     return results;
 }
 
-describe('bulkSend can work with congested queues', () => {
-    const logger = debugLogger('congested_test');
-    const index = `${TEST_INDEX_PREFIX}_congested_queues_`;
-
+describe('bulkSend', () => {
     let client;
     let api;
     let isElasticsearch8 = false;
 
     beforeAll(async () => {
         client = await makeClient();
-        await cleanupIndex(client, index);
-        api = elasticsearchAPI(client, logger);
-        isElasticsearch8 = api.isElasticsearch8();
     });
 
-    afterAll(async () => {
-        await cleanupIndex(client, index);
-    });
+    describe('can work with congested queues', () => {
+        const logger = debugLogger('congested_test');
+        const index = `${TEST_INDEX_PREFIX}_congested_queues_`;
+
+        beforeAll(async () => {
+            await cleanupIndex(client, index);
+            api = elasticsearchAPI(client, logger);
+            isElasticsearch8 = api.isElasticsearch8();
+        });
+
+        afterAll(async () => {
+            await cleanupIndex(client, index);
+        });
 
-    it('can get correct data even with congested queues', async () => {
-        const chunkedData = chunk(EvenDateData.data, 50);
+        it('can get correct data even with congested queues', async () => {
+            const chunkedData = chunk(EvenDateData.data, 50);
 
-        await pMap(chunkedData, async (cData) => {
-            const formattedData = formatUploadData(index, cData, isElasticsearch8);
-            return api.bulkSend(formattedData);
-        }, { concurrency: 9 });
+            await pMap(chunkedData, async (cData) => {
+                const formattedData = formatUploadData(index, cData, isElasticsearch8);
+                return api.bulkSend(formattedData);
+            }, { concurrency: 9 });
 
-        await waitForData(client, index, EvenDateData.data.length, logger, THREE_MINUTES);
+            await waitForData(client, index, EvenDateData.data.length, logger, THREE_MINUTES);
+        });
     });
 });

package/types/index.d.ts

@@ -62,7 +62,7 @@ declare namespace elasticsearchAPI {
     /**
      * This is used for improved bulk sending function
     */
-    export interface AnyBulkAction  {
+    export interface AnyBulkAction {
         update?: Partial<BulkActionMetadata>;
         index?: Partial<BulkActionMetadata>;
         create?: Partial<BulkActionMetadata>;