ag-common 0.0.725 → 0.0.726

package/dist/api/helpers/dynamo.d.ts

@@ -16,6 +16,15 @@ interface DynamoFilter {
     attrValues?: Record<string, unknown>;
 }
 interface ScanOptions {
+    /** eg
+     * filter: {
+        filterExpression: '#feedIcon = :empty',
+        attrNames: { '#feedIcon': 'feedIcon' },
+        attrValues: {
+          ':empty': '',
+        },
+      },
+     */
     filter?: DynamoFilter;
     requiredAttributeList?: string[];
     limit?: number;
@@ -48,6 +57,7 @@ export declare const batchDelete: (params: {
     pkName: string;
 }) => Promise<DynamoDBResult<void>>;
 export declare const scan: <T>(tableName: string, options?: ScanOptions) => Promise<DynamoDBResult<T[]>>;
+export declare function scanWithGenerator<T>(tableName: string, options?: ScanOptions): AsyncGenerator<T[], void, unknown>;
 export declare const getItemsDynamo: <T>(params: {
     tableName: string;
     items: {

package/dist/api/helpers/dynamo.js

@@ -8,8 +8,21 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+var __await = (this && this.__await) || function (v) { return this instanceof __await ? (this.v = v, this) : new __await(v); }
+var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var g = generator.apply(thisArg, _arguments || []), i, q = [];
+    return i = {}, verb("next"), verb("throw"), verb("return", awaitReturn), i[Symbol.asyncIterator] = function () { return this; }, i;
+    function awaitReturn(f) { return function (v) { return Promise.resolve(v).then(f, reject); }; }
+    function verb(n, f) { if (g[n]) { i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; if (f) i[n] = f(i[n]); } }
+    function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } }
+    function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); }
+    function fulfill(value) { resume("next", value); }
+    function reject(value) { resume("throw", value); }
+    function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); }
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getDynamoUpdates = exports.wipeTable = exports.getDynamoTtlMinutes = exports.getDynamoTtlDays = exports.queryDynamo = exports.getItemDynamo = exports.getItemsDynamo = exports.scan = exports.batchDelete = exports.batchWrite = exports.putDynamo = exports.setDynamo = exports.dynamoDb = void 0;
+exports.getDynamoUpdates = exports.wipeTable = exports.getDynamoTtlMinutes = exports.getDynamoTtlDays = exports.queryDynamo = exports.getItemDynamo = exports.getItemsDynamo = exports.scanWithGenerator = exports.scan = exports.batchDelete = exports.batchWrite = exports.putDynamo = exports.setDynamo = exports.dynamoDb = void 0;
 const client_dynamodb_1 = require("@aws-sdk/client-dynamodb");
 const lib_dynamodb_1 = require("@aws-sdk/lib-dynamodb");
 const array_1 = require("../../common/helpers/array");
@@ -148,6 +161,68 @@ const scan = (tableName, options) => __awaiter(void 0, void 0, void 0, function*
     }
 });
 exports.scan = scan;
+function scanWithGenerator(tableName, options) {
+    return __asyncGenerator(this, arguments, function* scanWithGenerator_1() {
+        var _a, _b;
+        const BATCH_SIZE = 25;
+        let items = [];
+        let exclusiveStartKey;
+        let totalItems = 0;
+        try {
+            const projectionAttrs = (_a = options === null || options === void 0 ? void 0 : options.requiredAttributeList) === null || _a === void 0 ? void 0 : _a.reduce((acc, attr, index) => {
+                acc[`#proj${index}`] = attr;
+                return acc;
+            }, {});
+            const expressionAttributeNames = Object.assign(Object.assign({}, projectionAttrs), (_b = options === null || options === void 0 ? void 0 : options.filter) === null || _b === void 0 ? void 0 : _b.attrNames);
+            do {
+                const params = new lib_dynamodb_1.ScanCommand(Object.assign(Object.assign(Object.assign(Object.assign({ TableName: tableName, Limit: BATCH_SIZE }, ((options === null || options === void 0 ? void 0 : options.filter) && Object.assign({ FilterExpression: options.filter.filterExpression }, (options.filter.attrValues && {
+                    ExpressionAttributeValues: options.filter.attrValues,
+                })))), (Object.keys(expressionAttributeNames).length > 0 && {
+                    ExpressionAttributeNames: expressionAttributeNames,
+                })), ((options === null || options === void 0 ? void 0 : options.requiredAttributeList) && {
+                    ProjectionExpression: options.requiredAttributeList
+                        .map((_, index) => `#proj${index}`)
+                        .join(', '),
+                })), { ExclusiveStartKey: exclusiveStartKey }));
+                const result = yield __await(withRetry(() => exports.dynamoDb.send(params), 'scanWithGenerator'));
+                if (result.Items) {
+                    items.push(...result.Items);
+                    // Process items in chunks of BATCH_SIZE
+                    while (items.length >= BATCH_SIZE) {
+                        const batch = items.slice(0, BATCH_SIZE);
+                        items = items.slice(BATCH_SIZE);
+                        totalItems += batch.length;
+                        // If we've reached the limit, yield the final batch and return
+                        if ((options === null || options === void 0 ? void 0 : options.limit) && totalItems >= options.limit) {
+                            const remainingCount = options.limit - (totalItems - batch.length);
+                            yield yield __await(batch.slice(0, remainingCount));
+                            return yield __await(void 0);
+                        }
+                        yield yield __await(batch);
+                    }
+                }
+                exclusiveStartKey = result.LastEvaluatedKey;
+            } while (exclusiveStartKey &&
+                (!(options === null || options === void 0 ? void 0 : options.limit) || totalItems < options.limit));
+            // Yield any remaining items
+            if (items.length > 0) {
+                if (options === null || options === void 0 ? void 0 : options.limit) {
+                    const remainingCount = options.limit - totalItems;
+                    if (remainingCount > 0) {
+                        yield yield __await(items.slice(0, remainingCount));
+                    }
+                }
+                else {
+                    yield yield __await(items);
+                }
+            }
+        }
+        catch (e) {
+            throw new Error(`Scan generator error: ${e.toString()}`);
+        }
+    });
+}
+exports.scanWithGenerator = scanWithGenerator;
 const getItemsDynamo = (params) => __awaiter(void 0, void 0, void 0, function* () {
     var _c, _d;
     try {

package/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.0.725",
+  "version": "0.0.726",
   "name": "ag-common",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",