ag-common 0.0.902 → 0.0.903

package/dist/api/helpers/dynamo/batch.d.ts

@@ -0,0 +1,8 @@
+import { type BatchWriteCommandInput } from "@aws-sdk/lib-dynamodb";
+export type DynamoWriteRequest = NonNullable<NonNullable<BatchWriteCommandInput["RequestItems"]>[string]>[number];
+export declare const sendAllBatchRequests: ({ maxRetries, operationName, requests, tableName, }: {
+    maxRetries?: number | null;
+    operationName: string;
+    requests: DynamoWriteRequest[];
+    tableName: string;
+}) => Promise<void>;

package/dist/api/helpers/dynamo/batch.js

@@ -0,0 +1,28 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sendAllBatchRequests = void 0;
+const lib_dynamodb_1 = require("@aws-sdk/lib-dynamodb");
+const _1 = require(".");
+const sleep_1 = require("../../../common/helpers/sleep");
+const withRetry_1 = require("../../../common/helpers/withRetry");
+const sendAllBatchRequests = async ({ maxRetries = 3, operationName, requests, tableName, }) => {
+    let pending = requests;
+    let unprocessedRetries = 0;
+    while (pending.length > 0) {
+        // oxlint-disable-next-line no-await-in-loop -- each request depends on DynamoDB's unprocessed set
+        const result = await (0, withRetry_1.withRetry)(() => _1.dynamoDb.send(new lib_dynamodb_1.BatchWriteCommand({
+            RequestItems: { [tableName]: pending },
+        })), operationName, { maxRetries });
+        pending = result.UnprocessedItems?.[tableName] ?? [];
+        if (pending.length === 0) {
+            return;
+        }
+        if (maxRetries !== null && unprocessedRetries >= maxRetries) {
+            throw new Error(`${operationName}: ${pending.length} items remained unprocessed`);
+        }
+        unprocessedRetries += 1;
+        // oxlint-disable-next-line no-await-in-loop -- unprocessed writes require backoff before retrying
+        await (0, sleep_1.sleep)(Math.min(2 ** unprocessedRetries * 25, 2000));
+    }
+};
+exports.sendAllBatchRequests = sendAllBatchRequests;

package/dist/api/helpers/dynamo/delete.js

@@ -1,11 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.wipeTable = exports.batchDelete = void 0;
-const lib_dynamodb_1 = require("@aws-sdk/lib-dynamodb");
-const _1 = require(".");
 const array_1 = require("../../../common/helpers/array");
 const async_1 = require("../../../common/helpers/async");
-const withRetry_1 = require("../../../common/helpers/withRetry");
+const batch_1 = require("./batch");
 const get_1 = require("./get");
 const batchDelete = async (params) => {
     try {
@@ -13,15 +11,13 @@ const batchDelete = async (params) => {
         const chunked = (0, array_1.chunk)(params.keys, batchSize);
         let processed = 0;
         await (0, async_1.asyncForEach)(chunked, async (chunk) => {
-            const batchDeleteParams = {
-                RequestItems: {
-                    [params.tableName]: chunk.map((key) => ({
-                        DeleteRequest: { Key: { [params.pkName]: key } },
-                    })),
-                },
-            };
-            await (0, withRetry_1.withRetry)(() => _1.dynamoDb.send(new lib_dynamodb_1.BatchWriteCommand(batchDeleteParams)), `batchdelete ${processed}/${params.keys.length}. size=${batchSize}`, {
-                maxRetries: maxRetries === undefined ? 3 : maxRetries,
+            await (0, batch_1.sendAllBatchRequests)({
+                tableName: params.tableName,
+                requests: chunk.map((key) => ({
+                    DeleteRequest: { Key: { [params.pkName]: key } },
+                })),
+                operationName: `batchdelete ${processed}/${params.keys.length}. size=${batchSize}`,
+                maxRetries,
             });
             processed += chunk.length;
         });

package/dist/api/helpers/dynamo/get.js

@@ -38,11 +38,16 @@ const executeQuery = async (params, exclusiveStartKey) => {
             eav[`:${skName.toLowerCase()}`] = skValue;
         }
     }
+    const projectionAttrs = params.requiredAttributeList?.reduce((acc, attr, index) => {
+        acc[`#proj${index}`] = attr;
+        return acc;
+    }, {});
     const queryParams = {
         TableName: params.tableName,
         KeyConditionExpression: kce,
         ExpressionAttributeNames: {
             ...ean,
+            ...projectionAttrs,
             ...params.filter?.attrNames,
         },
         ExpressionAttributeValues: {
@@ -53,6 +58,11 @@ const executeQuery = async (params, exclusiveStartKey) => {
         Limit: params.limit === -1 ? undefined : params.limit,
         IndexName: params.indexName,
         ExclusiveStartKey: exclusiveStartKey,
+        ...(params.requiredAttributeList && {
+            ProjectionExpression: params.requiredAttributeList
+                .map((_, index) => `#proj${index}`)
+                .join(", "),
+        }),
         ...(params.filter && {
             FilterExpression: params.filter.filterExpression,
             ...(params.filter.attrValues && {

package/dist/api/helpers/dynamo/set.js

@@ -6,6 +6,7 @@ const _1 = require(".");
 const array_1 = require("../../../common/helpers/array");
 const async_1 = require("../../../common/helpers/async");
 const withRetry_1 = require("../../../common/helpers/withRetry");
+const batch_1 = require("./batch");
 const putDynamo = async (item, tableName, opt) => {
     const putParams = {
         TableName: tableName,
@@ -32,13 +33,11 @@ const batchWrite = async (tableName, items, opt) => {
         let processed = 0;
         const chunked = (0, array_1.chunk)(items, batchSize);
         await (0, async_1.asyncForEach)(chunked, async (chunk) => {
-            const batchWriteParams = {
-                RequestItems: {
-                    [tableName]: chunk.map((Item) => ({ PutRequest: { Item } })),
-                },
-            };
-            await (0, withRetry_1.withRetry)(() => _1.dynamoDb.send(new lib_dynamodb_1.BatchWriteCommand(batchWriteParams)), `batchwrite ${processed}/${items.length}. size=${batchSize}`, {
-                maxRetries: opt?.maxRetries === undefined ? 3 : opt.maxRetries,
+            await (0, batch_1.sendAllBatchRequests)({
+                tableName,
+                requests: chunk.map((Item) => ({ PutRequest: { Item } })),
+                operationName: `batchwrite ${processed}/${items.length}. size=${batchSize}`,
+                maxRetries: opt?.maxRetries,
             });
             processed += chunk.length;
         });

package/dist/api/helpers/dynamo/types.d.ts

@@ -53,6 +53,8 @@ export interface DynamoQueryParams {
      */
     limit?: number;
     filter?: DynamoFilter;
+    /** Return only these attributes. */
+    requiredAttributeList?: string[];
     sortAscending?: boolean;
     /** default 3, set to null to disable retries */
     maxRetries?: number | null;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ag-common",
-  "version": "0.0.902",
+  "version": "0.0.903",
   "license": "ISC",
   "author": "admin@gec.dev",
   "repository": {