ag-common 0.0.732 → 0.0.733

package/dist/api/helpers/dynamo.d.ts

@@ -28,6 +28,7 @@ interface ScanOptions {
     filter?: DynamoFilter;
     requiredAttributeList?: string[];
     indexName?: string;
+    alwaysRetry?: boolean;
 }
 interface DynamoQueryParams {
     tableName: string;
@@ -40,9 +41,7 @@ interface DynamoQueryParams {
     indexName?: string;
     limit?: number;
     startKey?: Key;
-    filterName?: string;
-    filterValue?: unknown;
-    filterOperator?: string;
+    filter?: DynamoFilter;
     sortAscending?: boolean;
 }
 export declare let dynamoDb: DynamoDBDocument;

package/dist/api/helpers/dynamo.js

@@ -76,6 +76,7 @@ exports.putDynamo = putDynamo;
 const batchWrite = (tableName, items, opt) => __awaiter(void 0, void 0, void 0, function* () {
     try {
         const { batchSize = 20 } = opt !== null && opt !== void 0 ? opt : {};
+        var processed = 0;
         const chunked = (0, array_1.chunk)(items, batchSize);
         yield (0, async_1.asyncForEach)(chunked, (chunk) => __awaiter(void 0, void 0, void 0, function* () {
             const params = new lib_dynamodb_1.BatchWriteCommand({
@@ -83,9 +84,10 @@ const batchWrite = (tableName, items, opt) => __awaiter(void 0, void 0, void 0,
                     [tableName]: chunk.map((Item) => ({ PutRequest: { Item } })),
                 },
             });
-            yield (0, withRetry_1.withRetry)(() => exports.dynamoDb.send(params), 'batchWrite', {
+            yield (0, withRetry_1.withRetry)(() => exports.dynamoDb.send(params), `batchwrite ${processed}/${items.length}. size=${batchSize}`, {
                 maxRetries: (opt === null || opt === void 0 ? void 0 : opt.alwaysRetry) ? null : undefined,
             });
+            processed += chunk.length;
         }));
         return { data: undefined };
     }
@@ -105,6 +107,7 @@ const batchDelete = (params) => __awaiter(void 0, void 0, void 0, function* () {
     try {
         const { batchSize = 20, alwaysRetry = false } = (_a = params.opt) !== null && _a !== void 0 ? _a : {};
         const chunked = (0, array_1.chunk)(params.keys, batchSize);
+        var processed = 0;
         yield (0, async_1.asyncForEach)(chunked, (chunk) => __awaiter(void 0, void 0, void 0, function* () {
             const command = new lib_dynamodb_1.BatchWriteCommand({
                 RequestItems: {
@@ -113,9 +116,10 @@ const batchDelete = (params) => __awaiter(void 0, void 0, void 0, function* () {
                     })),
                 },
             });
-            yield (0, withRetry_1.withRetry)(() => exports.dynamoDb.send(command), 'batchDelete', {
+            yield (0, withRetry_1.withRetry)(() => exports.dynamoDb.send(command), `batchdelete ${processed}/${params.keys.length}. size=${batchSize}`, {
                 maxRetries: alwaysRetry ? null : undefined,
             });
+            processed += chunk.length;
         }));
         return { data: undefined };
     }
@@ -151,7 +155,9 @@ const scan = (tableName, options) => __awaiter(void 0, void 0, void 0, function*
                     .map((_, index) => `#proj${index}`)
                     .join(', '),
             })), { ExclusiveStartKey }));
-            const result = yield (0, withRetry_1.withRetry)(() => exports.dynamoDb.send(params), 'scan');
+            const result = yield (0, withRetry_1.withRetry)(() => exports.dynamoDb.send(params), `scan. already seen=${Items.length}`, {
+                maxRetries: (options === null || options === void 0 ? void 0 : options.alwaysRetry) ? null : undefined,
+            });
             if (result.Items) {
                 Items.push(...result.Items);
             }
@@ -194,7 +200,9 @@ function scanWithGenerator(tableName, options) {
                         .map((_, index) => `#proj${index}`)
                         .join(', '),
                 })), { ExclusiveStartKey: exclusiveStartKey }));
-                const result = yield __await((0, withRetry_1.withRetry)(() => exports.dynamoDb.send(params), 'scanWithGenerator'));
+                const result = yield __await((0, withRetry_1.withRetry)(() => exports.dynamoDb.send(params), `scanWithGenerator. already seen=${items.length}`, {
+                    maxRetries: (options === null || options === void 0 ? void 0 : options.alwaysRetry) ? null : undefined,
+                }));
                 if (result.Items) {
                     items.push(...result.Items);
                     // Process items in chunks of BATCH_SIZE
@@ -250,7 +258,7 @@ const getItemDynamo = (params) => __awaiter(void 0, void 0, void 0, function* ()
 });
 exports.getItemDynamo = getItemDynamo;
 const queryDynamo = (params) => __awaiter(void 0, void 0, void 0, function* () {
-    var _a, _b, _c;
+    var _a, _b, _c, _d;
     try {
         let kce = `#${params.pkName.toLowerCase()} ${(_a = params.pkOperator) !== null && _a !== void 0 ? _a : '='} :${params.pkName.toLowerCase()}`;
         const ean = {
@@ -279,29 +287,12 @@ const queryDynamo = (params) => __awaiter(void 0, void 0, void 0, function* () {
                 eav[`:${skName.toLowerCase()}`] = skValue;
             }
         }
-        let FilterExpression;
-        if (params.filterName && params.filterValue !== undefined) {
-            ean[`#${params.filterName.toLowerCase()}`] = params.filterName;
-            eav[`:${params.filterName.toLowerCase()}`] = params.filterValue;
-            FilterExpression =
-                params.filterOperator === 'contains'
-                    ? `contains(#${params.filterName.toLowerCase()}, :${params.filterName.toLowerCase()})`
-                    : `#${params.filterName.toLowerCase()} ${(_b = params.filterOperator) !== null && _b !== void 0 ? _b : '='} :${params.filterName.toLowerCase()}`;
-        }
         const items = [];
         let { startKey } = params;
         do {
-            const queryParams = new lib_dynamodb_1.QueryCommand({
-                TableName: params.tableName,
-                KeyConditionExpression: kce,
-                ExpressionAttributeNames: ean,
-                ExpressionAttributeValues: eav,
-                ScanIndexForward: (_c = params.sortAscending) !== null && _c !== void 0 ? _c : true,
-                Limit: params.limit,
-                IndexName: params.indexName,
-                ExclusiveStartKey: startKey,
-                FilterExpression,
-            });
+            const queryParams = new lib_dynamodb_1.QueryCommand(Object.assign({ TableName: params.tableName, KeyConditionExpression: kce, ExpressionAttributeNames: Object.assign(Object.assign({}, ean), (_b = params.filter) === null || _b === void 0 ? void 0 : _b.attrNames), ExpressionAttributeValues: Object.assign(Object.assign({}, eav), (_c = params.filter) === null || _c === void 0 ? void 0 : _c.attrValues), ScanIndexForward: (_d = params.sortAscending) !== null && _d !== void 0 ? _d : true, Limit: params.limit, IndexName: params.indexName, ExclusiveStartKey: startKey }, (params.filter && Object.assign({ FilterExpression: params.filter.filterExpression }, (params.filter.attrValues && {
+                ExpressionAttributeValues: Object.assign(Object.assign({}, eav), params.filter.attrValues),
+            })))));
             const result = yield (0, withRetry_1.withRetry)(() => exports.dynamoDb.send(queryParams), 'queryDynamo');
             if (result.Items) {
                 items.push(...result.Items);

package/dist/api/helpers/withRetry.js

@@ -14,7 +14,7 @@ const log_1 = require("../../common/helpers/log");
 const sleep_1 = require("../../common/helpers/sleep");
 const withRetry = (operation, operationName, opt) => __awaiter(void 0, void 0, void 0, function* () {
     let retryCount = 0;
-    var baseDelay = 1000;
+    const baseDelay = 2000;
     let { maxRetries = 3 } = opt !== null && opt !== void 0 ? opt : {};
     // eslint-disable-next-line
     while (true) {
@@ -32,7 +32,7 @@ const withRetry = (operation, operationName, opt) => __awaiter(void 0, void 0, v
                     (0, log_1.warn)(`${operationName}: Max retries exceeded`);
                     throw error;
                 }
-                const delay = baseDelay * retryCount;
+                const delay = baseDelay + retryCount * 1000;
                 (0, log_1.warn)(`${operationName}: Throttled. Retry ${retryCount}. Sleeping for ${delay}ms`);
                 yield (0, sleep_1.sleep)(delay);
                 continue;

package/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.0.732",
+  "version": "0.0.733",
   "name": "ag-common",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",