idea-aws 3.7.1 → 3.7.2

package/dist/src/dynamoDB.d.ts

@@ -63,18 +63,21 @@ export declare class DynamoDB {
      * @param ignoreErr if set, ignore the errors and continue the bulk op.
      */
     batchGet(table: string, keys: DDB.DocumentClient.Key[], ignoreErr?: boolean): Promise<DDB.DocumentClient.AttributeMap[]>;
-    protected batchGetHelper(table: string, keys: DDB.DocumentClient.Key[], resultElements: DDB.DocumentClient.AttributeMap[], ignoreErr: boolean, currentChunk: number, chunkSize: number): Promise<DDB.DocumentClient.AttributeMap[]>;
+    protected batchGetHelper(table: string, keys: DDB.DocumentClient.Key[], resultElements: DDB.DocumentClient.AttributeMap[], ignoreErr: boolean, currentChunk?: number, chunkSize?: number): Promise<DDB.DocumentClient.AttributeMap[]>;
     /**
      * Put an array of items in a DynamoDb table, avoiding the limits of DynamoDB's BatchWriteItem.
-     * @param ignoreErr if true, ignore the errors and continue the bulk op
+     * In case of errors, it will retry with a random back-off mechanism until the timeout.
+     * Therefore, in case of timeout, there may be some elements written and some not.
      */
-    batchPut(table: string, items: DDB.DocumentClient.AttributeMap[], ignoreErr?: boolean): Promise<void>;
+    batchPut(table: string, items: DDB.DocumentClient.AttributeMap[]): Promise<void>;
     /**
      * Delete an array of items from a DynamoDb table, avoiding the limits of DynamoDB's BatchWriteItem.
-     * @param ignoreErr if true, ignore the errors and continue the bulk op.
+     * In case of errors, it will retry with a random back-off mechanism until the timeout.
+     * Therefore, in case of timeout, there may be some elements deleted and some not.
      */
-    batchDelete(table: string, keys: DDB.DocumentClient.Key[], ignoreErr?: boolean): Promise<void>;
-    protected batchWriteHelper(table: string, items: DDB.DocumentClient.AttributeMap[], isPut: boolean, ignoreErr: boolean, currentChunk: number, chunkSize: number): Promise<void>;
+    batchDelete(table: string, keys: DDB.DocumentClient.Key[]): Promise<void>;
+    protected batchWriteHelper(table: string, itemsOrKeys: DDB.DocumentClient.AttributeMap[] | DDB.DocumentClient.Key[], isPut: boolean, currentChunk?: number, chunkSize?: number): Promise<void>;
+    protected batchWriteChunkWithRetries(table: string, params: DDB.DocumentClient.BatchWriteItemInput): Promise<void>;
     /**
      * Query a DynamoDb table, avoiding the limits of DynamoDB's Query.
      * @param params the params to apply to DynamoDB's function

package/dist/src/dynamoDB.js

@@ -157,9 +157,9 @@ class DynamoDB {
             (0, idea_toolbox_1.logger)(`BATCH GET ${table}`, null, 'No elements to get');
             return [];
         }
-        await this.batchGetHelper(table, keys, [], Boolean(ignoreErr), 0, 100);
+        await this.batchGetHelper(table, keys, [], Boolean(ignoreErr));
     }
-    async batchGetHelper(table, keys, resultElements, ignoreErr, currentChunk, chunkSize) {
+    async batchGetHelper(table, keys, resultElements, ignoreErr, currentChunk = 0, chunkSize = 100) {
         const batch = { RequestItems: {} };
         batch.RequestItems[table] = { Keys: [] };
         batch.RequestItems[table].Keys = keys.slice(currentChunk, currentChunk + chunkSize);
@@ -182,48 +182,57 @@ class DynamoDB {
     }
     /**
      * Put an array of items in a DynamoDb table, avoiding the limits of DynamoDB's BatchWriteItem.
-     * @param ignoreErr if true, ignore the errors and continue the bulk op
+     * In case of errors, it will retry with a random back-off mechanism until the timeout.
+     * Therefore, in case of timeout, there may be some elements written and some not.
      */
-    async batchPut(table, items, ignoreErr) {
+    async batchPut(table, items) {
         if (!items.length)
             return (0, idea_toolbox_1.logger)(`BATCH WRITE (PUT) ${table}`, null, 'No elements to write');
-        await this.batchWriteHelper(table, items, true, Boolean(ignoreErr), 0, 25);
+        await this.batchWriteHelper(table, items, true);
     }
     /**
      * Delete an array of items from a DynamoDb table, avoiding the limits of DynamoDB's BatchWriteItem.
-     * @param ignoreErr if true, ignore the errors and continue the bulk op.
+     * In case of errors, it will retry with a random back-off mechanism until the timeout.
+     * Therefore, in case of timeout, there may be some elements deleted and some not.
      */
-    async batchDelete(table, keys, ignoreErr) {
+    async batchDelete(table, keys) {
         if (!keys.length)
             return (0, idea_toolbox_1.logger)(`BATCH WRITE (DELETE) ${table}`, null, 'No elements to write');
-        await this.batchWriteHelper(table, keys, false, Boolean(ignoreErr), 0, 25);
+        await this.batchWriteHelper(table, keys, false);
     }
-    async batchWriteHelper(table, items, isPut, ignoreErr, currentChunk, chunkSize) {
-        const batch = { RequestItems: {} };
-        if (isPut) {
-            batch.RequestItems[table] = items
-                .slice(currentChunk, currentChunk + chunkSize)
-                .map(i => ({ PutRequest: { Item: i } }));
-        }
-        else {
-            // isDelete
-            batch.RequestItems[table] = items
-                .slice(currentChunk, currentChunk + chunkSize)
-                .map(k => ({ DeleteRequest: { Key: k } }));
-        }
-        (0, idea_toolbox_1.logger)(`BATCH WRITE (${isPut ? 'PUT' : 'DELETE'}) ${table}`, null, `${currentChunk} of ${items.length}`);
-        try {
-            await this.dynamo.batchWrite(batch).promise();
-        }
-        catch (err) {
-            if (!ignoreErr)
-                throw err;
-        }
+    async batchWriteHelper(table, itemsOrKeys, isPut, currentChunk = 0, chunkSize = 25) {
+        (0, idea_toolbox_1.logger)(`BATCH WRITE (${isPut ? 'PUT' : 'DELETE'}) ${table}`, null, `${currentChunk} of ${itemsOrKeys.length}`);
+        let requests;
+        if (isPut)
+            requests = itemsOrKeys.slice(currentChunk, currentChunk + chunkSize).map(i => ({ PutRequest: { Item: i } }));
+        // isDelete
+        else
+            requests = itemsOrKeys.slice(currentChunk, currentChunk + chunkSize).map(k => ({ DeleteRequest: { Key: k } }));
+        const batch = { RequestItems: { [table]: requests } };
+        await this.batchWriteChunkWithRetries(table, batch);
         // if there are still chunks to manage, go on recursively
-        if (currentChunk + chunkSize < items.length)
-            await this.batchWriteHelper(table, items, isPut, ignoreErr, currentChunk + chunkSize, chunkSize);
-        // no more chunks to manage: we're done
-        return;
+        if (currentChunk + chunkSize < itemsOrKeys.length)
+            await this.batchWriteHelper(table, itemsOrKeys, isPut, currentChunk + chunkSize, chunkSize);
+    }
+    async batchWriteChunkWithRetries(table, params) {
+        const getRandomInt = (max) => Math.floor(Math.random() * max);
+        const wait = (seconds) => new Promise(x => setTimeout(() => x(), seconds * 1000));
+        let attempts = 0;
+        do {
+            const response = await this.dynamo.batchWrite(params).promise();
+            if (response.UnprocessedItems &&
+                response.UnprocessedItems[table] &&
+                response.UnprocessedItems[table].length > 0) {
+                params.RequestItems = response.UnprocessedItems;
+                attempts++;
+                const waitSeconds = getRandomInt(attempts * 5);
+                (0, idea_toolbox_1.logger)('BATCH WRITE THROTTLED', null, `Waiting ${waitSeconds} seconds to retry`);
+                await wait(waitSeconds);
+            }
+            else {
+                params.RequestItems = null;
+            }
+        } while (params.RequestItems);
     }
     /**
      * Query a DynamoDb table, avoiding the limits of DynamoDB's Query.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "idea-aws",
-  "version": "3.7.1",
+  "version": "3.7.2",
   "description": "AWS wrappers to use in IDEA's back-ends",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",