ag-common 0.0.802 → 0.0.804

package/dist/api/helpers/dynamo/delete.js

@@ -53,7 +53,7 @@ const wipeTable = (tableName) => __awaiter(void 0, void 0, void 0, function* ()
     var _a, e_1, _b, _c;
     try {
         const generator = (0, get_1.scanWithGenerator)(tableName, {
-            BATCH_SIZE: 100, // Process in chunks of 100 items
+            limit: 100, // Process in chunks of 100 items
             maxRetries: null, // Always retry on 429 since we want to ensure complete deletion
         });
         try {

package/dist/api/helpers/dynamo/get.d.ts

@@ -51,11 +51,5 @@ export declare const scan: <T>(tableName: string, options?: ScanOptions) => Prom
  * @returns Promise resolving to query results or error
  */
 export declare const batchQueryDynamo: <T>(params: DynamoBatchQueryParams) => Promise<DynamoDBResult<T[]>>;
-export declare function queryWithGenerator<T>(params: DynamoQueryParams & {
-    /** how many to return in query generator. default 100 */
-    BATCH_SIZE?: number;
-}): AsyncGenerator<T[], void, unknown>;
-export declare function scanWithGenerator<T>(tableName: string, options?: ScanOptions & {
-    /** how many to return in scan generator. default 100 */
-    BATCH_SIZE?: number;
-}): AsyncGenerator<T[], void, unknown>;
+export declare function queryWithGenerator<T>(params: DynamoQueryParams): AsyncGenerator<T[], void, unknown>;
+export declare function scanWithGenerator<T>(tableName: string, options?: ScanOptions): AsyncGenerator<T[], void, unknown>;

package/dist/api/helpers/dynamo/get.js

@@ -33,7 +33,7 @@ const isError = (result) => 'error' in result;
  * Helper function that builds the query parameters and executes the query
  */
 const executeQuery = (params, exclusiveStartKey) => __awaiter(void 0, void 0, void 0, function* () {
-    var _a, _b, _c, _d, _e;
+    var _a, _b, _c, _d;
     let kce = `#${params.pkName.toLowerCase()} ${(_a = params.pkOperator) !== null && _a !== void 0 ? _a : '='} :${params.pkName.toLowerCase()}`;
     const ean = {
         [`#${params.pkName.toLowerCase()}`]: params.pkName,
@@ -61,7 +61,7 @@ const executeQuery = (params, exclusiveStartKey) => __awaiter(void 0, void 0, vo
             eav[`:${skName.toLowerCase()}`] = skValue;
         }
     }
-    const queryParams = Object.assign({ TableName: params.tableName, KeyConditionExpression: kce, ExpressionAttributeNames: Object.assign(Object.assign({}, ean), (_b = params.filter) === null || _b === void 0 ? void 0 : _b.attrNames), ExpressionAttributeValues: Object.assign(Object.assign({}, eav), (_c = params.filter) === null || _c === void 0 ? void 0 : _c.attrValues), ScanIndexForward: (_d = params.sortAscending) !== null && _d !== void 0 ? _d : true, Limit: (_e = params.BATCH_SIZE) !== null && _e !== void 0 ? _e : params.limit, IndexName: params.indexName, ExclusiveStartKey: exclusiveStartKey }, (params.filter && Object.assign({ FilterExpression: params.filter.filterExpression }, (params.filter.attrValues && {
+    const queryParams = Object.assign({ TableName: params.tableName, KeyConditionExpression: kce, ExpressionAttributeNames: Object.assign(Object.assign({}, ean), (_b = params.filter) === null || _b === void 0 ? void 0 : _b.attrNames), ExpressionAttributeValues: Object.assign(Object.assign({}, eav), (_c = params.filter) === null || _c === void 0 ? void 0 : _c.attrValues), ScanIndexForward: (_d = params.sortAscending) !== null && _d !== void 0 ? _d : true, Limit: params.limit === -1 ? undefined : params.limit, IndexName: params.indexName, ExclusiveStartKey: exclusiveStartKey }, (params.filter && Object.assign({ FilterExpression: params.filter.filterExpression }, (params.filter.attrValues && {
         ExpressionAttributeValues: Object.assign(Object.assign({}, eav), params.filter.attrValues),
     }))));
     return (0, withRetry_1.withRetry)(() => _1.dynamoDb.send(new lib_dynamodb_1.QueryCommand(queryParams)), 'queryDynamo:' + JSON.stringify(params), {
@@ -78,7 +78,7 @@ const executeScan = (tableName, options, exclusiveStartKey) => __awaiter(void 0,
         return acc;
     }, {});
     const expressionAttributeNames = Object.assign(Object.assign({}, projectionAttrs), (_b = options === null || options === void 0 ? void 0 : options.filter) === null || _b === void 0 ? void 0 : _b.attrNames);
-    const scanParams = Object.assign(Object.assign(Object.assign(Object.assign({ TableName: tableName, IndexName: options === null || options === void 0 ? void 0 : options.indexName, Limit: options === null || options === void 0 ? void 0 : options.BATCH_SIZE }, ((options === null || options === void 0 ? void 0 : options.filter) && Object.assign({ FilterExpression: options.filter.filterExpression }, (options.filter.attrValues && {
+    const scanParams = Object.assign(Object.assign(Object.assign(Object.assign({ TableName: tableName, IndexName: options === null || options === void 0 ? void 0 : options.indexName, Limit: (options === null || options === void 0 ? void 0 : options.limit) === -1 ? undefined : options === null || options === void 0 ? void 0 : options.limit }, ((options === null || options === void 0 ? void 0 : options.filter) && Object.assign({ FilterExpression: options.filter.filterExpression }, (options.filter.attrValues && {
         ExpressionAttributeValues: options.filter.attrValues,
     })))), (Object.keys(expressionAttributeNames).length > 0 && {
         ExpressionAttributeNames: expressionAttributeNames,
@@ -128,24 +128,25 @@ const queryDynamo = (params) => __awaiter(void 0, void 0, void 0, function* () {
     try {
         const items = [];
         let lastEvaluatedKey = params.exclusiveStartKey;
+        const hasLimit = params.limit && params.limit > 0;
         do {
             const result = yield executeQuery(params, lastEvaluatedKey);
             if (result.Items) {
                 items.push(...result.Items);
             }
             lastEvaluatedKey = result.LastEvaluatedKey;
-            // If we have a limit and we've reached it, stop processing
-            if (params.limit && items.length >= params.limit) {
+            // If limit is undefined, we stop after one query (default to single page).
+            // This satisfies: "other never endlessly query" (unless limit is -1).
+            if (params.limit === undefined || params.limit === null) {
+                break;
+            }
+            // If limit is > 0 and we've reached it, stop processing
+            if (hasLimit && items.length >= params.limit) {
                 return {
                     data: items.slice(0, params.limit),
                     lastEvaluatedKey,
                 };
             }
-            // If a start key was provided but no limit, we don't want to endlessly page
-            // so we stop after the first batch
-            if (params.exclusiveStartKey && !params.limit) {
-                break;
-            }
         } while (lastEvaluatedKey && Object.keys(lastEvaluatedKey).length > 0);
         return { data: items, lastEvaluatedKey };
     }
@@ -164,8 +165,20 @@ const scan = (tableName, options) => __awaiter(void 0, void 0, void 0, function*
                 Items.push(...result.Items);
             }
             ExclusiveStartKey = result.LastEvaluatedKey;
+            // If limit is undefined, we stop after one query (default to single page).
+            if ((options === null || options === void 0 ? void 0 : options.limit) === undefined) {
+                break;
+            }
+            if ((options === null || options === void 0 ? void 0 : options.limit) &&
+                options.limit > 0 &&
+                Items.length >= options.limit) {
+                return {
+                    data: Items.slice(0, options.limit),
+                    lastEvaluatedKey: ExclusiveStartKey,
+                };
+            }
         } while (ExclusiveStartKey);
-        return { data: Items };
+        return { data: Items, lastEvaluatedKey: ExclusiveStartKey };
     }
     catch (e) {
         return { error: e.toString() };
@@ -281,7 +294,7 @@ const executePartiQLQuery = (params) => __awaiter(void 0, void 0, void 0, functi
             const executeParams = {
                 Statement: statement,
                 Parameters: parameters,
-                Limit: params.limit,
+                Limit: params.limit === -1 ? undefined : params.limit,
                 NextToken: nextToken,
             };
             const result = yield (0, withRetry_1.withRetry)(() => _1.dynamoDb.send(new lib_dynamodb_1.ExecuteStatementCommand(executeParams)), 'batchQueryDynamo:' + JSON.stringify(params), {
@@ -309,7 +322,7 @@ const executePartiQLQuery = (params) => __awaiter(void 0, void 0, void 0, functi
 function queryWithGenerator(params) {
     return __asyncGenerator(this, arguments, function* queryWithGenerator_1() {
         var _a;
-        const BATCH_SIZE = (_a = params.BATCH_SIZE) !== null && _a !== void 0 ? _a : 100;
+        const limit = (_a = params.limit) !== null && _a !== void 0 ? _a : 100;
         let items = [];
         let lastEvaluatedKey = params.exclusiveStartKey;
         try {
@@ -318,9 +331,13 @@ function queryWithGenerator(params) {
                 if (result.Items) {
                     items.push(...result.Items);
                     // Process items in chunks of BATCH_SIZE
-                    while (items.length >= BATCH_SIZE) {
-                        const batch = items.slice(0, BATCH_SIZE);
-                        items = items.slice(BATCH_SIZE);
+                    if (!limit) {
+                        yield yield __await(items);
+                        return yield __await(void 0);
+                    }
+                    while (items.length >= limit) {
+                        const batch = items.slice(0, limit);
+                        items = items.slice(limit);
                         yield yield __await(batch);
                     }
                 }
@@ -339,7 +356,7 @@ function queryWithGenerator(params) {
 function scanWithGenerator(tableName, options) {
     return __asyncGenerator(this, arguments, function* scanWithGenerator_1() {
         var _a;
-        const BATCH_SIZE = (_a = options === null || options === void 0 ? void 0 : options.BATCH_SIZE) !== null && _a !== void 0 ? _a : 100;
+        const limit = (_a = options === null || options === void 0 ? void 0 : options.limit) !== null && _a !== void 0 ? _a : 100;
         let items = [];
         let exclusiveStartKey;
         try {
@@ -347,10 +364,14 @@ function scanWithGenerator(tableName, options) {
                 const result = yield __await(executeScan(tableName, options, exclusiveStartKey));
                 if (result.Items) {
                     items.push(...result.Items);
-                    // Process items in chunks of BATCH_SIZE
-                    while (items.length >= BATCH_SIZE) {
-                        const batch = items.slice(0, BATCH_SIZE);
-                        items = items.slice(BATCH_SIZE);
+                    // Process items in chunks of limit
+                    if (!limit) {
+                        yield yield __await(items);
+                        return yield __await(void 0);
+                    }
+                    while (items.length >= limit) {
+                        const batch = items.slice(0, limit);
+                        items = items.slice(limit);
                         yield yield __await(batch);
                     }
                 }

package/dist/api/helpers/dynamo/types.d.ts

@@ -25,7 +25,17 @@ export interface ScanOptions {
     filter?: DynamoFilter;
     requiredAttributeList?: string[];
     indexName?: string;
+    /**
+     * - number > 0: Limit the total number of items returned.
+     * - -1: Endlessly query until all items are retrieved.
+     * - null | undefined: Fetch only a single page (one network request).
+     */
+    limit?: number;
     maxRetries?: number | null;
+    /**
+     * The key to start/resume from.
+     */
+    exclusiveStartKey?: Key;
 }
 export interface DynamoQueryParams {
     tableName: string;
@@ -36,11 +46,19 @@ export interface DynamoQueryParams {
     skValue?: string | number | [string | number, string | number];
     skOperator?: '=' | '<' | '>' | '<=' | '>=' | 'BETWEEN' | 'BEGINS_WITH';
     indexName?: string;
+    /**
+     * - number > 0: Limit the total number of items returned.
+     * - -1: Endlessly query until all items are retrieved.
+     * - null | undefined: Fetch only a single page (one network request).
+     */
     limit?: number;
     filter?: DynamoFilter;
     sortAscending?: boolean;
     /** default 3, set to null to disable retries */
     maxRetries?: number | null;
+    /**
+     * The key to start/resume from.
+     */
     exclusiveStartKey?: Key;
 }
 export interface DynamoBatchQueryParams {

package/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.0.802",
+  "version": "0.0.804",
   "name": "ag-common",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",