ag-common 0.0.724 → 0.0.726

package/dist/api/helpers/dynamo.d.ts

@@ -1,77 +1,90 @@
 import { DynamoDBDocument } from '@aws-sdk/lib-dynamodb';
 import type { AwsCredentialIdentity } from '@smithy/types';
-import type { IQueryDynamo, Key } from '../types';
-export declare const setDynamo: (region: string, credentials?: AwsCredentialIdentity) => DynamoDBDocument;
+type DynamoDBError = {
+    error: string;
+};
+type DynamoDBSuccess<T> = {
+    data: T;
+};
+type DynamoDBResult<T> = DynamoDBSuccess<T> | DynamoDBError;
+interface Key {
+    [key: string]: string | number;
+}
+interface DynamoFilter {
+    filterExpression: string;
+    attrNames: Record<string, string>;
+    attrValues?: Record<string, unknown>;
+}
+interface ScanOptions {
+    /** eg
+     * filter: {
+        filterExpression: '#feedIcon = :empty',
+        attrNames: { '#feedIcon': 'feedIcon' },
+        attrValues: {
+          ':empty': '',
+        },
+      },
+     */
+    filter?: DynamoFilter;
+    requiredAttributeList?: string[];
+    limit?: number;
+}
+interface DynamoQueryParams {
+    tableName: string;
+    pkName: string;
+    pkValue: string | number;
+    pkOperator?: '=' | '<' | '>' | '<=' | '>=';
+    skName?: string;
+    skValue?: string | number | [string | number, string | number];
+    skOperator?: '=' | '<' | '>' | '<=' | '>=' | 'BETWEEN' | 'BEGINS_WITH';
+    indexName?: string;
+    limit?: number;
+    startKey?: Key;
+    filterName?: string;
+    filterValue?: unknown;
+    filterOperator?: string;
+    sortAscending?: boolean;
+}
 export declare let dynamoDb: DynamoDBDocument;
-export declare const putDynamo: <T extends Record<string, any>>(item: T, tableName: string, opt?: {
-    /** if provided, will assert this PK value doesnt already exist */
+export declare const setDynamo: (region: string, credentials?: AwsCredentialIdentity) => DynamoDBDocument;
+export declare const putDynamo: <T extends Record<string, unknown>>(item: T, tableName: string, opt?: {
     pkName?: string;
-}) => Promise<{
-    error?: string;
-}>;
-export declare const batchWrite: <T extends Record<string, any>>(tableName: string, itemsIn: T[]) => Promise<{
-    error?: string;
-}>;
-export declare const batchDelete: ({ tableName, keys, pkName, }: {
+}) => Promise<DynamoDBResult<void>>;
+export declare const batchWrite: <T extends Record<string, unknown>>(tableName: string, items: T[]) => Promise<DynamoDBResult<void>>;
+export declare const batchDelete: (params: {
     tableName: string;
     keys: string[];
     pkName: string;
-}) => Promise<{
-    error?: string;
-}>;
-export declare const scan: <T>(tableName: string, opt?: {
-    filter?: {
-        filterExpression: string;
-        attrNames: Record<string, string>;
-        attrValues?: Record<string, string>;
-    };
-    /** ProjectionExpression. will csv values */
-    requiredAttributeList?: string[];
-    /** default =ALL */
-    limit?: number;
-}) => Promise<{
-    data: T[];
-} | {
-    error: string;
-}>;
-export declare const getItemsDynamo: <T>({ tableName, items, }: {
+}) => Promise<DynamoDBResult<void>>;
+export declare const scan: <T>(tableName: string, options?: ScanOptions) => Promise<DynamoDBResult<T[]>>;
+export declare function scanWithGenerator<T>(tableName: string, options?: ScanOptions): AsyncGenerator<T[], void, unknown>;
+export declare const getItemsDynamo: <T>(params: {
+    tableName: string;
     items: {
         pkName: string;
         pkValue: string;
     }[];
+}) => Promise<DynamoDBResult<T[]>>;
+export declare const getItemDynamo: <T>(params: {
     tableName: string;
-}) => Promise<{
-    data: T[];
-} | {
-    error: string;
-}>;
-export declare const getItemDynamo: <T>({ tableName, pkName, pkValue, }: {
     pkName: string;
     pkValue: string;
-    tableName: string;
-}) => Promise<{
-    data: T;
-} | {
-    error: string;
-}>;
-export declare const queryDynamo: <T>({ tableName, pkName, pkValue, pkOperator, skName, skValue, skOperator, indexName, limit, startKey, filterName, filterValue, filterOperator, sortAscending, }: IQueryDynamo) => Promise<{
+}) => Promise<DynamoDBResult<T>>;
+export declare const queryDynamo: <T>(params: DynamoQueryParams) => Promise<{
     data: T[];
     startKey?: Key;
 } | {
     error: string;
 }>;
 export declare const getDynamoTtlDays: (days: number) => number;
-export declare const getDynamoTtlMinutes: (mins: number) => number;
-export declare const wipeTable: (tableName: string) => Promise<{
-    error?: string;
-}>;
-/** gets all fields in dynamokeys, and moves them into update expressions. eg will turn item.yourFieldName, into a dynamo write into field "yourFieldName" */
-export declare const getDynamoUpdates: (item: Record<string, string | number | boolean>, opt?: {
-    /** default PK. will also exclude null or undefined */
+export declare const getDynamoTtlMinutes: (minutes: number) => number;
+export declare const wipeTable: (tableName: string) => Promise<DynamoDBResult<void>>;
+export declare const getDynamoUpdates: <T extends Record<string, unknown>>(item: T, options?: {
     excludeKeys?: string[];
 }) => {
     UpdateExpression: string;
     ExpressionAttributeNames: Record<string, string>;
-    ExpressionAttributeValues: Record<string, string | number | boolean>;
+    ExpressionAttributeValues: Record<string, unknown>;
     ReturnValues: 'UPDATED_NEW';
 };
+export {};

package/dist/api/helpers/dynamo.js

@@ -1,8 +1,4 @@
 "use strict";
-/* eslint-disable guard-for-in */
-/* eslint-disable no-restricted-syntax */
-/* eslint-disable no-await-in-loop */
-/* eslint-disable prefer-const */
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -12,353 +8,368 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+var __await = (this && this.__await) || function (v) { return this instanceof __await ? (this.v = v, this) : new __await(v); }
+var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var g = generator.apply(thisArg, _arguments || []), i, q = [];
+    return i = {}, verb("next"), verb("throw"), verb("return", awaitReturn), i[Symbol.asyncIterator] = function () { return this; }, i;
+    function awaitReturn(f) { return function (v) { return Promise.resolve(v).then(f, reject); }; }
+    function verb(n, f) { if (g[n]) { i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; if (f) i[n] = f(i[n]); } }
+    function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } }
+    function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); }
+    function fulfill(value) { resume("next", value); }
+    function reject(value) { resume("throw", value); }
+    function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); }
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getDynamoUpdates = exports.wipeTable = exports.getDynamoTtlMinutes = exports.getDynamoTtlDays = exports.queryDynamo = exports.getItemDynamo = exports.getItemsDynamo = exports.scan = exports.batchDelete = exports.batchWrite = exports.putDynamo = exports.dynamoDb = exports.setDynamo = void 0;
+exports.getDynamoUpdates = exports.wipeTable = exports.getDynamoTtlMinutes = exports.getDynamoTtlDays = exports.queryDynamo = exports.getItemDynamo = exports.getItemsDynamo = exports.scanWithGenerator = exports.scan = exports.batchDelete = exports.batchWrite = exports.putDynamo = exports.setDynamo = exports.dynamoDb = void 0;
 const client_dynamodb_1 = require("@aws-sdk/client-dynamodb");
 const lib_dynamodb_1 = require("@aws-sdk/lib-dynamodb");
-// ES6 import
 const array_1 = require("../../common/helpers/array");
 const async_1 = require("../../common/helpers/async");
 const log_1 = require("../../common/helpers/log");
 const sleep_1 = require("../../common/helpers/sleep");
-const trim_1 = require("../../common/helpers/string/trim");
+const RETRY_CONFIG = {
+    maxRetries: 3,
+    baseDelay: 2000,
+};
+const isError = (result) => 'error' in result;
+const withRetry = (operation, operationName) => __awaiter(void 0, void 0, void 0, function* () {
+    let retryCount = 0;
+    // eslint-disable-next-line
+    while (true) {
+        try {
+            return yield operation();
+        }
+        catch (e) {
+            const error = e;
+            const errorString = error.toString();
+            if (errorString.includes('429') ||
+                errorString.includes('ProvisionedThroughputExceeded')) {
+                retryCount++;
+                if (retryCount >= RETRY_CONFIG.maxRetries) {
+                    (0, log_1.warn)(`${operationName}: Max retries exceeded`);
+                    throw error;
+                }
+                const delay = RETRY_CONFIG.baseDelay * Math.pow(2, retryCount - 1);
+                (0, log_1.warn)(`${operationName}: Throttled. Retry ${retryCount}`);
+                yield (0, sleep_1.sleep)(delay);
+                continue;
+            }
+            throw error;
+        }
+    }
+});
 const setDynamo = (region, credentials) => {
-    let raw = new client_dynamodb_1.DynamoDBClient({ region, credentials });
-    const ddbDocClient = lib_dynamodb_1.DynamoDBDocument.from(raw, {
+    const client = new client_dynamodb_1.DynamoDBClient({ region, credentials });
+    exports.dynamoDb = lib_dynamodb_1.DynamoDBDocument.from(client, {
         marshallOptions: { removeUndefinedValues: true },
     });
-    exports.dynamoDb = ddbDocClient;
-    return ddbDocClient;
+    return exports.dynamoDb;
 };
 exports.setDynamo = setDynamo;
 exports.dynamoDb = (0, exports.setDynamo)('ap-southeast-2');
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
 const putDynamo = (item, tableName, opt) => __awaiter(void 0, void 0, void 0, function* () {
-    let params = new lib_dynamodb_1.PutCommand(Object.assign({ TableName: tableName, Item: item }, ((opt === null || opt === void 0 ? void 0 : opt.pkName) && {
+    const params = new lib_dynamodb_1.PutCommand(Object.assign({ TableName: tableName, Item: item }, ((opt === null || opt === void 0 ? void 0 : opt.pkName) && {
         ConditionExpression: `attribute_not_exists(${opt.pkName})`,
     })));
-    (0, log_1.debug)(`running dynamo put=${JSON.stringify(params, null, 2)}`);
     try {
-        yield exports.dynamoDb.send(params);
-        return {};
+        yield withRetry(() => exports.dynamoDb.send(params), 'putDynamo');
+        return { data: undefined };
     }
     catch (e) {
-        (0, log_1.warn)('putDynamo error', e);
         return { error: e.toString() };
     }
 });
 exports.putDynamo = putDynamo;
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-const batchWrite = (tableName, itemsIn) => __awaiter(void 0, void 0, void 0, function* () {
-    //batch up to 20, so we can retry.
-    let chunked = (0, array_1.chunk)(itemsIn, 20);
+const batchWrite = (tableName, items) => __awaiter(void 0, void 0, void 0, function* () {
     try {
-        yield (0, async_1.asyncForEach)(chunked, (items) => __awaiter(void 0, void 0, void 0, function* () {
-            let retryCount = 0;
-            const retryMax = 3;
+        const chunked = (0, array_1.chunk)(items, 20);
+        yield (0, async_1.asyncForEach)(chunked, (chunk) => __awaiter(void 0, void 0, void 0, function* () {
             const params = new lib_dynamodb_1.BatchWriteCommand({
                 RequestItems: {
-                    [`${tableName}`]: items.map((Item) => ({
-                        PutRequest: { Item },
-                    })),
+                    [tableName]: chunk.map((Item) => ({ PutRequest: { Item } })),
                 },
             });
-            (0, log_1.debug)(`running dynamo batchWrite=${JSON.stringify(params, null, 2)}`);
-            // eslint-disable-next-line
-            while (true) {
-                try {
-                    yield exports.dynamoDb.send(params);
-                    return {};
-                }
-                catch (e) {
-                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                    const es = e.toString();
-                    let msg = es;
-                    (0, log_1.warn)('dynamo write error', msg);
-                    if (es.indexOf('429') !== -1 ||
-                        es.indexOf('ProvisionedThroughputExceeded') !== -1) {
-                        retryCount += 1;
-                        msg = `batch write throttled. retry ${retryCount}/${retryMax}`;
-                        (0, log_1.warn)(msg);
-                        if (retryCount >= retryMax) {
-                            throw new Error(`Max retries (${retryMax}) exceeded: ${es}`);
-                        }
-                        yield (0, sleep_1.sleep)(2000);
-                        // Continue the while loop to retry
-                        continue;
-                    }
-                    // For non-throttling errors, throw immediately
-                    throw e;
-                }
-            }
+            yield withRetry(() => exports.dynamoDb.send(params), 'batchWrite');
         }));
-        return {};
+        return { data: undefined };
     }
     catch (e) {
-        (0, log_1.warn)('batchWrite error', e);
         return { error: e.toString() };
     }
 });
 exports.batchWrite = batchWrite;
-const batchDelete = (_a) => __awaiter(void 0, [_a], void 0, function* ({ tableName, keys, pkName, }) {
-    // batch up to 20, so we can retry
-    let chunked = (0, array_1.chunk)(keys, 20);
+const batchDelete = (params) => __awaiter(void 0, void 0, void 0, function* () {
     try {
-        yield (0, async_1.asyncForEach)(chunked, (items) => __awaiter(void 0, void 0, void 0, function* () {
-            let retryCount = 0;
-            const retryMax = 3;
-            const params = new lib_dynamodb_1.BatchWriteCommand({
+        const chunked = (0, array_1.chunk)(params.keys, 20);
+        yield (0, async_1.asyncForEach)(chunked, (chunk) => __awaiter(void 0, void 0, void 0, function* () {
+            const command = new lib_dynamodb_1.BatchWriteCommand({
                 RequestItems: {
-                    [`${tableName}`]: items.map((key) => ({
-                        DeleteRequest: { Key: { [`${pkName}`]: key } },
+                    [params.tableName]: chunk.map((key) => ({
+                        DeleteRequest: { Key: { [params.pkName]: key } },
                     })),
                 },
             });
-            (0, log_1.debug)(`running dynamo batch delete=${JSON.stringify(params, null, 2)}`);
-            while (retryCount < retryMax) {
-                try {
-                    yield exports.dynamoDb.send(params);
-                    return {};
-                }
-                catch (e) {
-                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                    const es = e.toString();
-                    let shouldRetry = false;
-                    if (es.indexOf('429') !== -1 ||
-                        es.indexOf('ProvisionedThroughputExceeded') !== -1) {
-                        shouldRetry = true;
-                        retryCount += 1;
-                        const msg = `batch delete write throttled. retry ${retryCount}/${retryMax}`;
-                        (0, log_1.warn)('dynamo write error', msg);
-                    }
-                    else {
-                        // Non-retryable error
-                        throw e;
-                    }
-                    if (shouldRetry) {
-                        if (retryCount >= retryMax) {
-                            (0, log_1.warn)(`Max retries (${retryMax}) reached, giving up`);
-                            throw e;
-                        }
-                        (0, log_1.warn)(`dynamo retry ${retryCount}/${retryMax}`);
-                        yield (0, sleep_1.sleep)(2000 * Math.pow(2, retryCount - 1)); // Exponential backoff
-                        continue;
-                    }
-                }
-            }
+            yield withRetry(() => exports.dynamoDb.send(command), 'batchDelete');
         }));
-        return {};
+        return { data: undefined };
     }
     catch (e) {
-        (0, log_1.warn)('batchDelete error', e);
         return { error: e.toString() };
     }
 });
 exports.batchDelete = batchDelete;
-const scan = (tableName, opt) => __awaiter(void 0, void 0, void 0, function* () {
-    var _b, _c;
+const scan = (tableName, options) => __awaiter(void 0, void 0, void 0, function* () {
+    var _a, _b;
     try {
+        const Items = [];
         let ExclusiveStartKey;
-        let Items = [];
-        // Handle projection attributes
-        const projectionAttrs = (_b = opt === null || opt === void 0 ? void 0 : opt.requiredAttributeList) === null || _b === void 0 ? void 0 : _b.reduce((acc, attr, index) => {
-            const escapedName = `#proj${index}`;
-            acc[escapedName] = attr;
+        const projectionAttrs = (_a = options === null || options === void 0 ? void 0 : options.requiredAttributeList) === null || _a === void 0 ? void 0 : _a.reduce((acc, attr, index) => {
+            acc[`#proj${index}`] = attr;
             return acc;
         }, {});
-        // Merge projection attribute names with filter attribute names
-        const expressionAttributeNames = Object.assign(Object.assign({}, projectionAttrs), (_c = opt === null || opt === void 0 ? void 0 : opt.filter) === null || _c === void 0 ? void 0 : _c.attrNames);
+        const expressionAttributeNames = Object.assign(Object.assign({}, projectionAttrs), (_b = options === null || options === void 0 ? void 0 : options.filter) === null || _b === void 0 ? void 0 : _b.attrNames);
         do {
-            const params = new lib_dynamodb_1.ScanCommand(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({ TableName: tableName }, ((opt === null || opt === void 0 ? void 0 : opt.filter) && Object.assign({ FilterExpression: opt.filter.filterExpression }, (opt.filter.attrValues && {
-                ExpressionAttributeValues: opt.filter.attrValues,
+            const params = new lib_dynamodb_1.ScanCommand(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({ TableName: tableName }, ((options === null || options === void 0 ? void 0 : options.filter) && Object.assign({ FilterExpression: options.filter.filterExpression }, (options.filter.attrValues && {
+                ExpressionAttributeValues: options.filter.attrValues,
             })))), (Object.keys(expressionAttributeNames).length > 0 && {
                 ExpressionAttributeNames: expressionAttributeNames,
-            })), ((opt === null || opt === void 0 ? void 0 : opt.requiredAttributeList) && {
-                ProjectionExpression: opt.requiredAttributeList
+            })), ((options === null || options === void 0 ? void 0 : options.requiredAttributeList) && {
+                ProjectionExpression: options.requiredAttributeList
                     .map((_, index) => `#proj${index}`)
                     .join(', '),
-            })), { ExclusiveStartKey }), (((opt === null || opt === void 0 ? void 0 : opt.limit) && Items.length < opt.limit) && {
-                Limit: opt.limit - Items.length
+            })), { ExclusiveStartKey }), ((options === null || options === void 0 ? void 0 : options.limit) &&
+                Items.length < options.limit && {
+                Limit: options.limit - Items.length,
             })));
-            (0, log_1.debug)(`running dynamo scan=${JSON.stringify(params, null, 2)}`);
-            const { Items: newItems = [], LastEvaluatedKey, } = yield exports.dynamoDb.send(params);
-            ExclusiveStartKey = LastEvaluatedKey;
-            Items.push(...newItems);
-            // Break the loop if we've reached the limit
-            if ((opt === null || opt === void 0 ? void 0 : opt.limit) && Items.length >= opt.limit) {
+            const result = yield withRetry(() => exports.dynamoDb.send(params), 'scan');
+            if (result.Items) {
+                Items.push(...result.Items);
+            }
+            ExclusiveStartKey = result.LastEvaluatedKey;
+            if ((options === null || options === void 0 ? void 0 : options.limit) && Items.length >= options.limit) {
                 break;
             }
         } while (ExclusiveStartKey);
-        // Ensure we don't return more items than the limit
-        if ((opt === null || opt === void 0 ? void 0 : opt.limit) && Items.length > opt.limit) {
-            Items = Items.slice(0, opt.limit);
-        }
-        (0, log_1.debug)(`dynamo scan against ${tableName} ok, count=${Items.length}`);
-        return { data: Items };
+        return {
+            data: (options === null || options === void 0 ? void 0 : options.limit) ? Items.slice(0, options.limit) : Items,
+        };
     }
     catch (e) {
-        (0, log_1.warn)('scan error:', e);
         return { error: e.toString() };
     }
 });
 exports.scan = scan;
-const getItemsDynamo = (_d) => __awaiter(void 0, [_d], void 0, function* ({ tableName, items, }) {
-    var _e, _f;
-    const params = new lib_dynamodb_1.BatchGetCommand({
-        RequestItems: {
-            [tableName]: {
-                Keys: items.map(({ pkName, pkValue }) => ({
-                    [pkName]: pkValue,
-                })),
-            },
-        },
+function scanWithGenerator(tableName, options) {
+    return __asyncGenerator(this, arguments, function* scanWithGenerator_1() {
+        var _a, _b;
+        const BATCH_SIZE = 25;
+        let items = [];
+        let exclusiveStartKey;
+        let totalItems = 0;
+        try {
+            const projectionAttrs = (_a = options === null || options === void 0 ? void 0 : options.requiredAttributeList) === null || _a === void 0 ? void 0 : _a.reduce((acc, attr, index) => {
+                acc[`#proj${index}`] = attr;
+                return acc;
+            }, {});
+            const expressionAttributeNames = Object.assign(Object.assign({}, projectionAttrs), (_b = options === null || options === void 0 ? void 0 : options.filter) === null || _b === void 0 ? void 0 : _b.attrNames);
+            do {
+                const params = new lib_dynamodb_1.ScanCommand(Object.assign(Object.assign(Object.assign(Object.assign({ TableName: tableName, Limit: BATCH_SIZE }, ((options === null || options === void 0 ? void 0 : options.filter) && Object.assign({ FilterExpression: options.filter.filterExpression }, (options.filter.attrValues && {
+                    ExpressionAttributeValues: options.filter.attrValues,
+                })))), (Object.keys(expressionAttributeNames).length > 0 && {
+                    ExpressionAttributeNames: expressionAttributeNames,
+                })), ((options === null || options === void 0 ? void 0 : options.requiredAttributeList) && {
+                    ProjectionExpression: options.requiredAttributeList
+                        .map((_, index) => `#proj${index}`)
+                        .join(', '),
+                })), { ExclusiveStartKey: exclusiveStartKey }));
+                const result = yield __await(withRetry(() => exports.dynamoDb.send(params), 'scanWithGenerator'));
+                if (result.Items) {
+                    items.push(...result.Items);
+                    // Process items in chunks of BATCH_SIZE
+                    while (items.length >= BATCH_SIZE) {
+                        const batch = items.slice(0, BATCH_SIZE);
+                        items = items.slice(BATCH_SIZE);
+                        totalItems += batch.length;
+                        // If we've reached the limit, yield the final batch and return
+                        if ((options === null || options === void 0 ? void 0 : options.limit) && totalItems >= options.limit) {
+                            const remainingCount = options.limit - (totalItems - batch.length);
+                            yield yield __await(batch.slice(0, remainingCount));
+                            return yield __await(void 0);
+                        }
+                        yield yield __await(batch);
+                    }
+                }
+                exclusiveStartKey = result.LastEvaluatedKey;
+            } while (exclusiveStartKey &&
+                (!(options === null || options === void 0 ? void 0 : options.limit) || totalItems < options.limit));
+            // Yield any remaining items
+            if (items.length > 0) {
+                if (options === null || options === void 0 ? void 0 : options.limit) {
+                    const remainingCount = options.limit - totalItems;
+                    if (remainingCount > 0) {
+                        yield yield __await(items.slice(0, remainingCount));
+                    }
+                }
+                else {
+                    yield yield __await(items);
+                }
+            }
+        }
+        catch (e) {
+            throw new Error(`Scan generator error: ${e.toString()}`);
+        }
     });
+}
+exports.scanWithGenerator = scanWithGenerator;
+const getItemsDynamo = (params) => __awaiter(void 0, void 0, void 0, function* () {
+    var _c, _d;
     try {
-        let res = yield exports.dynamoDb.send(params);
-        let data = (_f = (_e = res.Responses) === null || _e === void 0 ? void 0 : _e[tableName].map((r) => r)) !== null && _f !== void 0 ? _f : [];
-        return { data };
+        const command = new lib_dynamodb_1.BatchGetCommand({
+            RequestItems: {
+                [params.tableName]: {
+                    Keys: params.items.map(({ pkName, pkValue }) => ({
+                        [pkName]: pkValue,
+                    })),
+                },
+            },
+        });
+        const result = yield withRetry(() => exports.dynamoDb.send(command), 'getItemsDynamo');
+        return {
+            data: (_d = (_c = result.Responses) === null || _c === void 0 ? void 0 : _c[params.tableName]) !== null && _d !== void 0 ? _d : [],
+        };
     }
     catch (e) {
-        (0, log_1.warn)('getItemsDynamo error:', e);
         return { error: e.toString() };
-        //
     }
 });
 exports.getItemsDynamo = getItemsDynamo;
-const getItemDynamo = (_g) => __awaiter(void 0, [_g], void 0, function* ({ tableName, pkName, pkValue, }) {
-    let r = yield (0, exports.getItemsDynamo)({ tableName, items: [{ pkName, pkValue }] });
-    if ('error' in r) {
-        return { error: r.error };
+const getItemDynamo = (params) => __awaiter(void 0, void 0, void 0, function* () {
+    const result = yield (0, exports.getItemsDynamo)({
+        tableName: params.tableName,
+        items: [{ pkName: params.pkName, pkValue: params.pkValue }],
+    });
+    if (isError(result)) {
+        return result;
     }
-    return { data: r.data[0] };
+    return { data: result.data[0] };
 });
 exports.getItemDynamo = getItemDynamo;
-const queryDynamo = (_h) => __awaiter(void 0, [_h], void 0, function* ({ tableName, pkName, pkValue, pkOperator = '=', skName, skValue, skOperator = '=', indexName, limit = 1000, startKey, filterName, filterValue, filterOperator = '=', sortAscending = true, }) {
-    let kce = `#${pkName.toLowerCase()} ${pkOperator} :${pkName.toLowerCase()}`;
-    const ean = { [`#${pkName.toLowerCase()}`]: pkName };
-    const eav = {
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        [`:${pkName.toLowerCase()}`]: pkValue,
-    };
-    if (skName) {
-        if (skOperator === 'BETWEEN') {
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            let [sp0, sp1] = skValue;
-            const skName1 = `${skName.toLowerCase()}1`;
-            const skName2 = `${skName.toLowerCase()}2`;
-            kce += ` and #${skName.toLowerCase()} ${skOperator} :${skName1.toLowerCase()} AND :${skName2.toLowerCase()}`;
-            ean[`#${skName.toLowerCase()}`] = skName;
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            eav[`:${skName1}`] = sp0;
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            eav[`:${skName2}`] = sp1;
-        }
-        else if (skOperator === 'BEGINS_WITH') {
-            kce += ` and ${skOperator.toLowerCase()}(#${skName.toLowerCase()}, :${skName.toLowerCase()})`;
-            ean[`#${skName.toLowerCase()}`] = skName;
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            eav[`:${skName.toLowerCase()}`] = skValue;
+const queryDynamo = (params) => __awaiter(void 0, void 0, void 0, function* () {
+    var _e, _f, _g;
+    try {
+        let kce = `#${params.pkName.toLowerCase()} ${(_e = params.pkOperator) !== null && _e !== void 0 ? _e : '='} :${params.pkName.toLowerCase()}`;
+        const ean = {
+            [`#${params.pkName.toLowerCase()}`]: params.pkName,
+        };
+        const eav = {
+            [`:${params.pkName.toLowerCase()}`]: params.pkValue,
+        };
+        if (params.skName && params.skValue !== undefined) {
+            const { skName, skValue, skOperator = '=' } = params;
+            if (skOperator === 'BETWEEN' && Array.isArray(skValue)) {
+                const [start, end] = skValue;
+                kce += ` AND #${skName.toLowerCase()} BETWEEN :${skName}1 AND :${skName}2`;
+                ean[`#${skName.toLowerCase()}`] = skName;
+                eav[`:${skName}1`] = start;
+                eav[`:${skName}2`] = end;
+            }
+            else if (skOperator === 'BEGINS_WITH') {
+                kce += ` AND begins_with(#${skName.toLowerCase()}, :${skName.toLowerCase()})`;
+                ean[`#${skName.toLowerCase()}`] = skName;
+                eav[`:${skName.toLowerCase()}`] = skValue;
+            }
+            else {
+                kce += ` AND #${skName.toLowerCase()} ${skOperator} :${skName.toLowerCase()}`;
+                ean[`#${skName.toLowerCase()}`] = skName;
+                eav[`:${skName.toLowerCase()}`] = skValue;
+            }
         }
-        else {
-            kce += ` and #${skName.toLowerCase()} ${skOperator} :${skName.toLowerCase()}`;
-            ean[`#${skName.toLowerCase()}`] = skName;
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            eav[`:${skName.toLowerCase()}`] = skValue;
+        let FilterExpression;
+        if (params.filterName && params.filterValue !== undefined) {
+            ean[`#${params.filterName.toLowerCase()}`] = params.filterName;
+            eav[`:${params.filterName.toLowerCase()}`] = params.filterValue;
+            FilterExpression =
+                params.filterOperator === 'contains'
+                    ? `contains(#${params.filterName.toLowerCase()}, :${params.filterName.toLowerCase()})`
+                    : `#${params.filterName.toLowerCase()} ${(_f = params.filterOperator) !== null && _f !== void 0 ? _f : '='} :${params.filterName.toLowerCase()}`;
         }
+        const items = [];
+        let { startKey } = params;
+        do {
+            const queryParams = new lib_dynamodb_1.QueryCommand({
+                TableName: params.tableName,
+                KeyConditionExpression: kce,
+                ExpressionAttributeNames: ean,
+                ExpressionAttributeValues: eav,
+                ScanIndexForward: (_g = params.sortAscending) !== null && _g !== void 0 ? _g : true,
+                Limit: params.limit,
+                IndexName: params.indexName,
+                ExclusiveStartKey: startKey,
+                FilterExpression,
+            });
+            const result = yield withRetry(() => exports.dynamoDb.send(queryParams), 'queryDynamo');
+            if (result.Items) {
+                items.push(...result.Items);
+            }
+            startKey = result.LastEvaluatedKey;
+            if (params.limit && items.length >= params.limit) {
+                return {
+                    data: items.slice(0, params.limit),
+                    startKey,
+                };
+            }
+        } while (startKey && Object.keys(startKey).length > 0);
+        return { data: items };
     }
-    let FilterExpression;
-    if (filterName) {
-        ean[`#${filterName.toLowerCase()}`] = filterName;
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        eav[`:${filterName.toLowerCase()}`] = filterValue;
-        FilterExpression = `#${filterName.toLowerCase()} ${filterOperator} :${filterName.toLowerCase()}`;
-        if (filterOperator === 'contains') {
-            FilterExpression = `contains(#${filterName.toLowerCase()}, :${filterName.toLowerCase()})`;
-        }
+    catch (e) {
+        return { error: e.toString() };
     }
-    const Items = [];
-    do {
-        const params = new lib_dynamodb_1.QueryCommand(Object.assign(Object.assign(Object.assign(Object.assign({ TableName: tableName, KeyConditionExpression: kce, ExpressionAttributeNames: ean, ExpressionAttributeValues: eav, ScanIndexForward: sortAscending }, (limit !== null && { Limit: limit || 1000 })), (indexName && { IndexName: indexName })), (startKey && {
-            ExclusiveStartKey: startKey,
-        })), (FilterExpression && { FilterExpression })));
-        let lek;
-        let newItems;
-        try {
-            ({
-                Items: newItems,
-                LastEvaluatedKey: lek,
-                // eslint-disable-next-line no-await-in-loop
-            } = yield exports.dynamoDb.send(params));
-            if (newItems) {
-                Items.push(...newItems.map((i) => i));
-            }
-        }
-        catch (e) {
-            (0, log_1.warn)('error. query params=', JSON.stringify(params), e);
-            return { error: e.toString() };
-        }
-        startKey = lek;
-        (0, log_1.debug)(`dynamo query against ${params.input.TableName} ok, count=${newItems === null || newItems === void 0 ? void 0 : newItems.length} ${JSON.stringify(params)}`, ` next startkey=${startKey}`);
-        if (!!limit && Items.length > limit) {
-            return { data: Items, startKey };
-        }
-    } while (startKey && Object.keys(startKey).length > 0);
-    return { data: Items };
 });
 exports.queryDynamo = queryDynamo;
-const getDynamoTtlDays = (days) => Math.ceil(new Date().getTime() / 1000) + days * 86400;
+const getDynamoTtlDays = (days) => Math.ceil(Date.now() / 1000) + days * 86400;
 exports.getDynamoTtlDays = getDynamoTtlDays;
-const getDynamoTtlMinutes = (mins) => Math.ceil(new Date().getTime() / 1000) + mins * 60;
+const getDynamoTtlMinutes = (minutes) => Math.ceil(Date.now() / 1000) + minutes * 60;
 exports.getDynamoTtlMinutes = getDynamoTtlMinutes;
 const wipeTable = (tableName) => __awaiter(void 0, void 0, void 0, function* () {
+    var _h, _j, _k;
     try {
-        let infoV = yield exports.dynamoDb.send(new client_dynamodb_1.DescribeTableCommand({ TableName: tableName }));
-        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-        // @ts-ignore
-        let keyHash = infoV.Table.KeySchema.find((k) => k.KeyType === 'HASH').AttributeName;
+        const info = yield withRetry(() => exports.dynamoDb.send(new client_dynamodb_1.DescribeTableCommand({ TableName: tableName })), 'wipeTable-describe');
+        const keyHash = (_k = (_j = (_h = info.Table) === null || _h === void 0 ? void 0 : _h.KeySchema) === null || _j === void 0 ? void 0 : _j.find((k) => k.KeyType === 'HASH')) === null || _k === void 0 ? void 0 : _k.AttributeName;
         if (!keyHash) {
-            throw new Error('couldnt find keyHash');
-        }
-        let allraw = yield (0, exports.scan)(tableName);
-        if ('error' in allraw) {
-            throw allraw.error;
+            throw new Error('Could not find hash key');
         }
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        let all = allraw.data.map((d) => d) || [];
-        (0, log_1.warn)(`will delete ${all.length} items from ${tableName}`);
+        const scanResult = yield (0, exports.scan)(tableName);
+        if (isError(scanResult)) {
+            throw new Error(scanResult.error);
+        }
         yield (0, exports.batchDelete)({
             tableName,
-            keys: all.map((s) => s[keyHash]),
-            pkName: 'PK',
+            keys: scanResult.data.map((item) => item[keyHash]),
+            pkName: keyHash,
         });
-        (0, log_1.warn)(`cleared table ${tableName}`);
-        return {};
+        return { data: undefined };
     }
     catch (e) {
-        (0, log_1.warn)('wipeTable error:', e);
         return { error: e.toString() };
     }
 });
 exports.wipeTable = wipeTable;
-/** gets all fields in dynamokeys, and moves them into update expressions. eg will turn item.yourFieldName, into a dynamo write into field "yourFieldName" */
-const getDynamoUpdates = (item, opt) => {
+const getDynamoUpdates = (item, options) => {
     var _a;
-    let ek = (_a = opt === null || opt === void 0 ? void 0 : opt.excludeKeys) !== null && _a !== void 0 ? _a : ['PK'];
-    ek = ek.map((r) => r.toLowerCase());
-    let UpdateExpression = `SET `;
+    const excludeKeys = ((_a = options === null || options === void 0 ? void 0 : options.excludeKeys) !== null && _a !== void 0 ? _a : ['PK']).map((k) => k.toLowerCase());
+    const validEntries = Object.entries(item).filter(([key, value]) => !excludeKeys.includes(key.toLowerCase()) && value != null);
     const ExpressionAttributeNames = {};
     const ExpressionAttributeValues = {};
-    //
-    const cleanedKeys = Object.entries(item).filter(([k]) => !ek.includes(k.toLowerCase()));
-    cleanedKeys
-        .filter(([_k, v]) => v !== null && v !== undefined)
-        .forEach(([k, v]) => {
-        UpdateExpression += `#${k} = :${k}, `;
-        ExpressionAttributeNames[`#${k}`] = k;
-        ExpressionAttributeValues[`:${k}`] = v;
-    });
+    const UpdateExpression = validEntries.reduce((expr, [key, value], index) => {
+        ExpressionAttributeNames[`#${key}`] = key;
+        ExpressionAttributeValues[`:${key}`] = value;
+        return `${expr}${index > 0 ? ', ' : ''}#${key} = :${key}`;
+    }, 'SET ');
     return {
-        UpdateExpression: (0, trim_1.trimSide)(UpdateExpression, false, ' ', ','),
+        UpdateExpression,
         ExpressionAttributeNames,
         ExpressionAttributeValues,
         ReturnValues: 'UPDATED_NEW',

package/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.0.724",
+  "version": "0.0.726",
   "name": "ag-common",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",