ag-common 0.0.725 → 0.0.727

package/dist/api/helpers/dynamo.d.ts

@@ -16,9 +16,17 @@ interface DynamoFilter {
     attrValues?: Record<string, unknown>;
 }
 interface ScanOptions {
+    /** eg
+     * filter: {
+        filterExpression: '#feedIcon = :empty',
+        attrNames: { '#feedIcon': 'feedIcon' },
+        attrValues: {
+          ':empty': '',
+        },
+      },
+     */
     filter?: DynamoFilter;
     requiredAttributeList?: string[];
-    limit?: number;
 }
 interface DynamoQueryParams {
     tableName: string;
@@ -48,6 +56,10 @@ export declare const batchDelete: (params: {
     pkName: string;
 }) => Promise<DynamoDBResult<void>>;
 export declare const scan: <T>(tableName: string, options?: ScanOptions) => Promise<DynamoDBResult<T[]>>;
+export declare function scanWithGenerator<T>(tableName: string, options?: ScanOptions & {
+    /** how many to return in scan generator. default 100 */
+    BATCH_SIZE?: number;
+}): AsyncGenerator<T[], void, unknown>;
 export declare const getItemsDynamo: <T>(params: {
     tableName: string;
     items: {

package/dist/api/helpers/dynamo.js

@@ -8,8 +8,21 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+var __await = (this && this.__await) || function (v) { return this instanceof __await ? (this.v = v, this) : new __await(v); }
+var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var g = generator.apply(thisArg, _arguments || []), i, q = [];
+    return i = {}, verb("next"), verb("throw"), verb("return", awaitReturn), i[Symbol.asyncIterator] = function () { return this; }, i;
+    function awaitReturn(f) { return function (v) { return Promise.resolve(v).then(f, reject); }; }
+    function verb(n, f) { if (g[n]) { i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; if (f) i[n] = f(i[n]); } }
+    function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } }
+    function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); }
+    function fulfill(value) { resume("next", value); }
+    function reject(value) { resume("throw", value); }
+    function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); }
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getDynamoUpdates = exports.wipeTable = exports.getDynamoTtlMinutes = exports.getDynamoTtlDays = exports.queryDynamo = exports.getItemDynamo = exports.getItemsDynamo = exports.scan = exports.batchDelete = exports.batchWrite = exports.putDynamo = exports.setDynamo = exports.dynamoDb = void 0;
+exports.getDynamoUpdates = exports.wipeTable = exports.getDynamoTtlMinutes = exports.getDynamoTtlDays = exports.queryDynamo = exports.getItemDynamo = exports.getItemsDynamo = exports.scanWithGenerator = exports.scan = exports.batchDelete = exports.batchWrite = exports.putDynamo = exports.setDynamo = exports.dynamoDb = void 0;
 const client_dynamodb_1 = require("@aws-sdk/client-dynamodb");
 const lib_dynamodb_1 = require("@aws-sdk/lib-dynamodb");
 const array_1 = require("../../common/helpers/array");
@@ -118,7 +131,7 @@ const scan = (tableName, options) => __awaiter(void 0, void 0, void 0, function*
         }, {});
         const expressionAttributeNames = Object.assign(Object.assign({}, projectionAttrs), (_b = options === null || options === void 0 ? void 0 : options.filter) === null || _b === void 0 ? void 0 : _b.attrNames);
         do {
-            const params = new lib_dynamodb_1.ScanCommand(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({ TableName: tableName }, ((options === null || options === void 0 ? void 0 : options.filter) && Object.assign({ FilterExpression: options.filter.filterExpression }, (options.filter.attrValues && {
+            const params = new lib_dynamodb_1.ScanCommand(Object.assign(Object.assign(Object.assign(Object.assign({ TableName: tableName }, ((options === null || options === void 0 ? void 0 : options.filter) && Object.assign({ FilterExpression: options.filter.filterExpression }, (options.filter.attrValues && {
                 ExpressionAttributeValues: options.filter.attrValues,
             })))), (Object.keys(expressionAttributeNames).length > 0 && {
                 ExpressionAttributeNames: expressionAttributeNames,
@@ -126,28 +139,65 @@ const scan = (tableName, options) => __awaiter(void 0, void 0, void 0, function*
                 ProjectionExpression: options.requiredAttributeList
                     .map((_, index) => `#proj${index}`)
                     .join(', '),
-            })), { ExclusiveStartKey }), ((options === null || options === void 0 ? void 0 : options.limit) &&
-                Items.length < options.limit && {
-                Limit: options.limit - Items.length,
-            })));
+            })), { ExclusiveStartKey }));
             const result = yield withRetry(() => exports.dynamoDb.send(params), 'scan');
             if (result.Items) {
                 Items.push(...result.Items);
             }
             ExclusiveStartKey = result.LastEvaluatedKey;
-            if ((options === null || options === void 0 ? void 0 : options.limit) && Items.length >= options.limit) {
-                break;
-            }
         } while (ExclusiveStartKey);
-        return {
-            data: (options === null || options === void 0 ? void 0 : options.limit) ? Items.slice(0, options.limit) : Items,
-        };
+        return { data: Items };
     }
     catch (e) {
         return { error: e.toString() };
     }
 });
 exports.scan = scan;
+function scanWithGenerator(tableName, options) {
+    return __asyncGenerator(this, arguments, function* scanWithGenerator_1() {
+        var _a, _b, _c;
+        const BATCH_SIZE = (_a = options === null || options === void 0 ? void 0 : options.BATCH_SIZE) !== null && _a !== void 0 ? _a : 100;
+        let items = [];
+        let exclusiveStartKey;
+        try {
+            const projectionAttrs = (_b = options === null || options === void 0 ? void 0 : options.requiredAttributeList) === null || _b === void 0 ? void 0 : _b.reduce((acc, attr, index) => {
+                acc[`#proj${index}`] = attr;
+                return acc;
+            }, {});
+            const expressionAttributeNames = Object.assign(Object.assign({}, projectionAttrs), (_c = options === null || options === void 0 ? void 0 : options.filter) === null || _c === void 0 ? void 0 : _c.attrNames);
+            do {
+                const params = new lib_dynamodb_1.ScanCommand(Object.assign(Object.assign(Object.assign(Object.assign({ TableName: tableName, Limit: BATCH_SIZE }, ((options === null || options === void 0 ? void 0 : options.filter) && Object.assign({ FilterExpression: options.filter.filterExpression }, (options.filter.attrValues && {
+                    ExpressionAttributeValues: options.filter.attrValues,
+                })))), (Object.keys(expressionAttributeNames).length > 0 && {
+                    ExpressionAttributeNames: expressionAttributeNames,
+                })), ((options === null || options === void 0 ? void 0 : options.requiredAttributeList) && {
+                    ProjectionExpression: options.requiredAttributeList
+                        .map((_, index) => `#proj${index}`)
+                        .join(', '),
+                })), { ExclusiveStartKey: exclusiveStartKey }));
+                const result = yield __await(withRetry(() => exports.dynamoDb.send(params), 'scanWithGenerator'));
+                if (result.Items) {
+                    items.push(...result.Items);
+                    // Process items in chunks of BATCH_SIZE
+                    while (items.length >= BATCH_SIZE) {
+                        const batch = items.slice(0, BATCH_SIZE);
+                        items = items.slice(BATCH_SIZE);
+                        yield yield __await(batch);
+                    }
+                }
+                exclusiveStartKey = result.LastEvaluatedKey;
+            } while (exclusiveStartKey);
+            // Yield any remaining items
+            if (items.length > 0) {
+                yield yield __await(items);
+            }
+        }
+        catch (e) {
+            throw new Error(`Scan generator error: ${e.toString()}`);
+        }
+    });
+}
+exports.scanWithGenerator = scanWithGenerator;
 const getItemsDynamo = (params) => __awaiter(void 0, void 0, void 0, function* () {
     var _c, _d;
     try {

package/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.0.725",
+  "version": "0.0.727",
   "name": "ag-common",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",