ag-common 0.0.459 → 0.0.461

package/dist/api/helpers/dynamo.d.ts

@@ -1,58 +1,52 @@
-import DynamoDB from 'aws-sdk/clients/dynamodb';
+import { DynamoDBDocument } from '@aws-sdk/lib-dynamodb';
 import { DYNAMOKEYS, IQueryDynamo, Key } from '../types';
-export declare let dynamoDb: DynamoDB.DocumentClient;
-export declare const setDynamo: (region: string) => void;
-export declare const putDynamo: <T>(item: T, tableName: string, opt?: {
+export declare const setDynamo: (region: string) => DynamoDBDocument;
+export declare let dynamoDb: DynamoDBDocument;
+export declare const putDynamo: <T extends Record<string, any>>(item: T, tableName: string, opt?: {
     /** if provided, will assert this PK value doesnt already exist */
     pkName?: string;
 }) => Promise<{
-    error?: string | undefined;
-    data?: T | undefined;
+    error?: string;
 }>;
-export declare const batchWrite: <T extends {}>(tableName: string, itemsIn: T[], breakOnError?: boolean) => Promise<{
-    error?: string | undefined;
+export declare const batchWrite: <T extends Record<string, any>>(tableName: string, itemsIn: T[]) => Promise<{
+    error?: string;
 }>;
-export declare const batchDelete: ({ tableName, breakOnError, pkName, keys, rangeName, rangeKeys, }: {
-    pkName: string;
-    breakOnError?: boolean | undefined;
+export declare const batchDelete: ({ tableName, keys, pkName, }: {
     tableName: string;
     keys: string[];
-    rangeName?: string | undefined;
-    rangeKeys?: string[] | undefined;
-}) => Promise<{
-    error: string;
-} | {
-    error?: undefined;
-}>;
-export declare const scan: <T>(tableName: string, opt?: {
-    filter?: {
-        filterExpression: string;
-        attrNames: Record<string, string>;
-        attrValues: Record<string, string>;
-    };
-    /** ProjectionExpression. will csv values */
-    requiredAttributeList?: string[];
-}) => Promise<T[]>;
-export declare const getItemDynamo: <T>({ tableName, pkName, pkValue, }: {
     pkName: string;
-    pkValue: string;
-    tableName: string;
-}) => Promise<T>;
+}) => Promise<{}>;
+export declare const scan: <T>(tableName: string) => Promise<{
+    data?: T[] | undefined;
+    error?: string | undefined;
+}>;
 export declare const getItemsDynamo: <T>({ tableName, items, }: {
     items: {
         pkName: string;
         pkValue: string;
     }[];
     tableName: string;
-}) => Promise<T[]>;
-export declare const queryDynamo: <T>({ tableName, pkName, pkValue, pkOperator, skName, skValue, skOperator, indexName, count, startKey: startKeyIn, filterName, filterValue, filterOperator, sortAscending, }: IQueryDynamo) => Promise<{
-    Items: T[];
+}) => Promise<{
+    data?: T[] | undefined;
+    error?: string | undefined;
+}>;
+export declare const getItemDynamo: <T>({ tableName, pkName, pkValue, }: {
+    pkName: string;
+    pkValue: string;
+    tableName: string;
+}) => Promise<{
+    data?: T | undefined;
+    error?: string | undefined;
+}>;
+export declare const queryDynamo: <T>({ tableName, pkName, pkValue, pkOperator, skName, skValue, skOperator, indexName, count, startKeyPk, filterName, filterValue, filterOperator, sortAscending, }: IQueryDynamo) => Promise<{
+    Items?: T[] | undefined;
     startKey?: Key | undefined;
+    error?: string | undefined;
 }>;
 export declare const getDynamoTtlDays: (days: number) => number;
 export declare const getDynamoTtlMinutes: (mins: number) => number;
 export declare const wipeTable: (tableName: string) => Promise<{
-    errors?: string[] | undefined;
+    errors?: string[];
 }>;
 export declare const getDynamoUpdates: (items: DYNAMOKEYS) => {
     UpdateExpression: string;

package/dist/api/helpers/dynamo.js

@@ -1,27 +1,8 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
+/* eslint-disable guard-for-in */
+/* eslint-disable no-restricted-syntax */
+/* eslint-disable no-await-in-loop */
+/* eslint-disable prefer-const */
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -32,53 +13,56 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getDynamoUpdates = exports.wipeTable = exports.getDynamoTtlMinutes = exports.getDynamoTtlDays = exports.queryDynamo = exports.getItemsDynamo = exports.getItemDynamo = exports.scan = exports.batchDelete = exports.batchWrite = exports.putDynamo = exports.setDynamo = exports.dynamoDb = void 0;
-/* eslint-disable guard-for-in */
-/* eslint-disable no-restricted-syntax */
-/* eslint-disable no-await-in-loop */
-/* eslint-disable prefer-const */
-const dynamodb_1 = __importStar(require("aws-sdk/clients/dynamodb"));
+exports.getDynamoUpdates = exports.wipeTable = exports.getDynamoTtlMinutes = exports.getDynamoTtlDays = exports.queryDynamo = exports.getItemDynamo = exports.getItemsDynamo = exports.scan = exports.batchDelete = exports.batchWrite = exports.putDynamo = exports.dynamoDb = exports.setDynamo = void 0;
+const client_dynamodb_1 = require("@aws-sdk/client-dynamodb");
+const lib_dynamodb_1 = require("@aws-sdk/lib-dynamodb");
+// ES6 import
 const array_1 = require("../../common/helpers/array");
 const async_1 = require("../../common/helpers/async");
 const log_1 = require("../../common/helpers/log");
 const sleep_1 = require("../../common/helpers/sleep");
-// eslint-disable-next-line import/no-mutable-exports
-exports.dynamoDb = new dynamodb_1.DocumentClient();
 const setDynamo = (region) => {
-    exports.dynamoDb = new dynamodb_1.DocumentClient({ region });
+    let raw = new client_dynamodb_1.DynamoDBClient({ region });
+    const ddbDocClient = lib_dynamodb_1.DynamoDBDocument.from(raw);
+    return ddbDocClient;
 };
 exports.setDynamo = setDynamo;
+exports.dynamoDb = (0, exports.setDynamo)('ap-southeast-2');
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
 const putDynamo = (item, tableName, opt) => __awaiter(void 0, void 0, void 0, function* () {
-    const params = Object.assign({ TableName: tableName, 
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        Item: item }, ((opt === null || opt === void 0 ? void 0 : opt.pkName) && {
+    let params = new lib_dynamodb_1.PutCommand(Object.assign({ TableName: tableName, Item: item }, ((opt === null || opt === void 0 ? void 0 : opt.pkName) && {
         ConditionExpression: `attribute_not_exists(${opt.pkName})`,
-    }));
-    (0, log_1.info)(`running dynamo put=${JSON.stringify(params, null, 2)}`);
-    // write the todo to the database
-    const put = yield exports.dynamoDb.put(params).promise();
-    if (put.$response.error && put.$response.error.statusCode) {
-        return { error: put.$response.error.message };
+    })));
+    (0, log_1.debug)(`running dynamo put=${JSON.stringify(params, null, 2)}`);
+    try {
+        let r1 = yield exports.dynamoDb.send(params);
+        console.log('r1=', r1);
+        return {};
+    }
+    catch (e) {
+        (0, log_1.warn)('put error', e);
+        return { error: e.toString() };
     }
-    // put never returns into, so just use what we have already
-    return { data: item };
 });
 exports.putDynamo = putDynamo;
-let batchWriteRaw = (req) => __awaiter(void 0, void 0, void 0, function* () {
-    let count = 0;
-    let max = 5;
-    // eslint-disable-next-line no-constant-condition
-    while (true)
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const batchWrite = (tableName, itemsIn) => __awaiter(void 0, void 0, void 0, function* () {
+    //batch up to 20, so we can retry.
+    let chunked = (0, array_1.chunk)(itemsIn, 20);
+    yield (0, async_1.asyncForEach)(chunked, (items) => __awaiter(void 0, void 0, void 0, function* () {
+        let retryCount = 0;
+        let retryMax = 3;
+        let params = new lib_dynamodb_1.BatchWriteCommand({
+            RequestItems: {
+                [`${tableName}`]: items.map((Item) => ({
+                    PutRequest: { Item: Item },
+                })),
+            },
+        });
+        (0, log_1.debug)(`running dynamo batchWrite=${JSON.stringify(params, null, 2)}`);
         try {
-            const res = yield exports.dynamoDb
-                .batchWrite({
-                RequestItems: req,
-            })
-                .promise();
-            if (res.$response.error) {
-                throw new Error(res.$response.error.message);
-            }
-            return res;
+            yield exports.dynamoDb.send(params);
+            return {};
         }
         catch (e) {
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -87,168 +71,110 @@ let batchWriteRaw = (req) => __awaiter(void 0, void 0, void 0, function* () {
             (0, log_1.warn)('dynamo write error', msg);
             if (es.indexOf('429') !== -1 ||
                 es.indexOf('ProvisionedThroughputExceeded') !== -1) {
-                count += 1;
-                msg = `batch write throttled. retry ${count}/${max}`;
+                retryCount += 1;
+                msg = `batch write throttled. retry ${retryCount}/${retryMax}`;
             }
             else {
                 throw e;
             }
-            if (count >= max) {
+            if (retryCount >= retryMax) {
                 throw e;
             }
-            (0, log_1.warn)(`dynamo retry ${count}/${max}`);
+            (0, log_1.warn)(`dynamo retry ${retryCount}/${retryMax}`);
             yield (0, sleep_1.sleep)(2000);
         }
-});
-// eslint-disable-next-line @typescript-eslint/ban-types
-const batchWrite = (tableName, itemsIn, breakOnError = false) => __awaiter(void 0, void 0, void 0, function* () {
-    var _a, _b;
-    let items = JSON.parse(JSON.stringify(itemsIn));
-    (0, log_1.debug)(`push to dynamo:${tableName} - count=${itemsIn.length}`);
-    const error = [];
-    while (items.length > 0) {
-        const { part, rest } = (0, array_1.take)(items, 25);
-        // eslint-disable-next-line no-param-reassign
-        items = rest;
-        let req = {
-            [`${tableName}`]: part.map((item) => ({
-                PutRequest: { Item: item },
-            })),
-        };
-        let res = yield batchWriteRaw(req);
-        const newError = (_b = (_a = res.$response) === null || _a === void 0 ? void 0 : _a.error) !== null && _b !== void 0 ? _b : null;
-        if (newError) {
-            error.push(newError);
-            if (breakOnError) {
-                items = [];
-            }
-        }
-    }
-    if ((error === null || error === void 0 ? void 0 : error.length) > 0) {
-        const me = error.join('\n');
-        (0, log_1.error)(`batch write error=${me}`);
-        return { error: me };
-    }
+    }));
     return {};
 });
 exports.batchWrite = batchWrite;
-const batchDelete = ({ tableName, breakOnError = true, pkName, keys, rangeName, rangeKeys, }) => __awaiter(void 0, void 0, void 0, function* () {
-    (0, log_1.info)(`wipe keys dynamo:${tableName} - count=${keys.length}`);
-    const error = [];
-    let breakV = false;
-    yield (0, async_1.asyncForEach)(keys, (key, i) => __awaiter(void 0, void 0, void 0, function* () {
-        var _c, _d;
-        if (breakV) {
-            return;
+const batchDelete = ({ tableName, keys, pkName, }) => __awaiter(void 0, void 0, void 0, function* () {
+    //batch up to 20, so we can retry.
+    let chunked = (0, array_1.chunk)(keys, 20);
+    yield (0, async_1.asyncForEach)(chunked, (items) => __awaiter(void 0, void 0, void 0, function* () {
+        let retryCount = 0;
+        let retryMax = 3;
+        let params = new lib_dynamodb_1.BatchWriteCommand({
+            RequestItems: {
+                [`${tableName}`]: items.map((key) => ({
+                    DeleteRequest: { Key: { [`${pkName}`]: key } },
+                })),
+            },
+        });
+        (0, log_1.debug)(`running dynamo batch delete=${JSON.stringify(params, null, 2)}`);
+        try {
+            yield exports.dynamoDb.send(params);
+            return {};
         }
-        let params = {
-            TableName: tableName,
-            Key: { [pkName]: key },
-        };
-        if (rangeName) {
-            let rangeValue = rangeKeys === null || rangeKeys === void 0 ? void 0 : rangeKeys[i];
-            if (rangeValue) {
-                params.Key[rangeName] = rangeValue;
+        catch (e) {
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            let es = e.toString();
+            let msg = es;
+            (0, log_1.warn)('dynamo write error', msg);
+            if (es.indexOf('429') !== -1 ||
+                es.indexOf('ProvisionedThroughputExceeded') !== -1) {
+                retryCount += 1;
+                msg = `batch delete write throttled. retry ${retryCount}/${retryMax}`;
             }
-        }
-        const res = yield exports.dynamoDb.delete(params).promise();
-        const newError = (_d = (_c = res.$response) === null || _c === void 0 ? void 0 : _c.error) !== null && _d !== void 0 ? _d : null;
-        if (newError) {
-            error.push(newError);
-            if (breakOnError) {
-                breakV = true;
+            else {
+                throw e;
+            }
+            if (retryCount >= retryMax) {
+                throw e;
             }
+            (0, log_1.warn)(`dynamo retry ${retryCount}/${retryMax}`);
+            yield (0, sleep_1.sleep)(2000);
         }
     }));
-    if ((error === null || error === void 0 ? void 0 : error.length) > 0) {
-        const me = error.join('\n');
-        (0, log_1.error)(`batch write error=${me}`);
-        return { error: me };
-    }
     return {};
 });
 exports.batchDelete = batchDelete;
-const scan = (tableName, opt) => __awaiter(void 0, void 0, void 0, function* () {
-    const Items = [];
-    let ExclusiveStartKey;
-    do {
-        let params = {
-            TableName: tableName,
-            ExclusiveStartKey,
-        };
-        if (opt === null || opt === void 0 ? void 0 : opt.filter) {
-            params.FilterExpression = opt.filter.filterExpression;
-            params.ExpressionAttributeNames = opt.filter.attrNames;
-            params.ExpressionAttributeValues = opt.filter.attrValues;
-        }
-        if (opt === null || opt === void 0 ? void 0 : opt.requiredAttributeList) {
-            params.ProjectionExpression = opt.requiredAttributeList.join(', ');
-        }
-        const { Items: newitems, LastEvaluatedKey, $response,
-        // eslint-disable-next-line no-await-in-loop
-         } = yield exports.dynamoDb.scan(params).promise();
-        ExclusiveStartKey = LastEvaluatedKey;
-        if ($response.error && $response.error.statusCode) {
-            throw new Error($response.error.message);
-        }
-        if (newitems) {
-            Items.push(...newitems.map((r) => r));
-        }
-    } while (ExclusiveStartKey);
-    (0, log_1.info)(`dynamo scan against ${tableName} ok, count=${Items === null || Items === void 0 ? void 0 : Items.length}`);
-    return Items;
-});
-exports.scan = scan;
-const getItemDynamo = ({ tableName, pkName, pkValue, }) => __awaiter(void 0, void 0, void 0, function* () {
-    const params = {
-        Key: { [pkName]: pkValue },
+const scan = (tableName) => __awaiter(void 0, void 0, void 0, function* () {
+    var _a;
+    let params = new lib_dynamodb_1.ScanCommand({
         TableName: tableName,
-    };
+    });
+    (0, log_1.debug)(`running dynamo scan=${JSON.stringify(params, null, 2)}`);
     try {
-        const res = yield exports.dynamoDb.get(params).promise();
-        const ret = res.Item;
-        (0, log_1.debug)(`got dynamo getitem=${JSON.stringify(params, null, 2)}`);
-        return ret;
+        let ret = yield exports.dynamoDb.send(params);
+        let items = ((_a = ret.Items) !== null && _a !== void 0 ? _a : []).map((r) => r);
+        return { data: items };
     }
     catch (e) {
-        (0, log_1.error)(e);
-        throw e;
+        return { error: e.toString() };
     }
 });
-exports.getItemDynamo = getItemDynamo;
+exports.scan = scan;
 const getItemsDynamo = ({ tableName, items, }) => __awaiter(void 0, void 0, void 0, function* () {
-    var _e, _f;
-    const params = {
+    var _b, _c;
+    const params = new lib_dynamodb_1.BatchGetCommand({
         RequestItems: {
             [tableName]: {
                 Keys: items.map(({ pkName, pkValue }) => ({
-                    [pkName]: { S: pkValue },
+                    [pkName]: pkValue,
                 })),
             },
         },
-    };
-    const dbRaw = new dynamodb_1.default({ apiVersion: '2012-10-08' });
+    });
     try {
-        const res = yield dbRaw.batchGetItem(params).promise();
-        (0, log_1.debug)(`got dynamo getitems=${JSON.stringify(res, null, 2)}`);
-        let ret = ((_f = (_e = res.Responses) === null || _e === void 0 ? void 0 : _e[tableName]) === null || _f === void 0 ? void 0 : _f.map((s) => dynamodb_1.Converter.unmarshall(s))) ||
-            [];
-        return ret;
+        let res = yield exports.dynamoDb.send(params);
+        let data = (_c = (_b = res.Responses) === null || _b === void 0 ? void 0 : _b[tableName].map((r) => r)) !== null && _c !== void 0 ? _c : [];
+        return { data };
     }
     catch (e) {
-        let msg = `error with getitems query:` +
-            JSON.stringify(params, null, 2) +
-            '\n' +
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            e.toString();
-        (0, log_1.error)(msg);
-        throw e;
+        (0, log_1.warn)('get error:', e);
+        return { error: e.toString() };
+        //
     }
 });
 exports.getItemsDynamo = getItemsDynamo;
-const queryDynamo = ({ tableName, pkName, pkValue, pkOperator = '=', skName, skValue, skOperator = '=', indexName, count = 1000, startKey: startKeyIn, filterName, filterValue, filterOperator = '=', sortAscending = true, }) => __awaiter(void 0, void 0, void 0, function* () {
-    var _g;
-    let startKey = startKeyIn;
+const getItemDynamo = ({ tableName, pkName, pkValue, }) => __awaiter(void 0, void 0, void 0, function* () {
+    var _d;
+    let r = yield (0, exports.getItemsDynamo)({ tableName, items: [{ pkName, pkValue }] });
+    return { data: (_d = r.data) === null || _d === void 0 ? void 0 : _d[0], error: r.error };
+});
+exports.getItemDynamo = getItemDynamo;
+const queryDynamo = ({ tableName, pkName, pkValue, pkOperator = '=', skName, skValue, skOperator = '=', indexName, count = 1000, startKeyPk, filterName, filterValue, filterOperator = '=', sortAscending = true, }) => __awaiter(void 0, void 0, void 0, function* () {
+    var _e;
     let kce = `#${pkName.toLowerCase()} ${pkOperator} :${pkName.toLowerCase()}`;
     const ean = { [`#${pkName.toLowerCase()}`]: pkName };
     const eav = {
@@ -292,50 +218,32 @@ const queryDynamo = ({ tableName, pkName, pkValue, pkOperator = '=', skName, skV
         }
     }
     const Items = [];
+    let startKey = startKeyPk
+        ? { [`${pkName}`]: { S: startKeyPk } }
+        : undefined;
     do {
-        const params = {
-            TableName: tableName,
-            KeyConditionExpression: kce,
-            ExpressionAttributeNames: ean,
-            ExpressionAttributeValues: eav,
+        const params = new lib_dynamodb_1.QueryCommand(Object.assign(Object.assign(Object.assign(Object.assign({ TableName: tableName, KeyConditionExpression: kce, ExpressionAttributeNames: ean, ExpressionAttributeValues: eav }, (startKey && {
             ExclusiveStartKey: startKey,
-            FilterExpression,
-            ScanIndexForward: sortAscending,
-        };
-        if (count > 0) {
-            params.Limit = count;
-        }
-        else {
-            params.Limit = 1000;
-        }
-        if (indexName) {
-            params.IndexName = indexName;
-        }
-        let newitems;
+        })), (FilterExpression && { FilterExpression })), { ScanIndexForward: sortAscending, Limit: count > 0 ? count : 1000 }), (indexName && { IndexName: indexName })));
         let lek;
-        let $response;
+        let newItems;
         try {
             ({
-                Items: newitems,
+                Items: newItems,
                 LastEvaluatedKey: lek,
-                $response,
                 // eslint-disable-next-line no-await-in-loop
-            } = yield exports.dynamoDb.query(params).promise());
+            } = yield exports.dynamoDb.send(params));
+            if (newItems) {
+                Items.push(...newItems.map((i) => i));
+            }
         }
         catch (e) {
-            (0, log_1.error)('error. query params=', JSON.stringify(params), e);
-            throw e;
+            (0, log_1.warn)('error. query params=', JSON.stringify(params), e);
+            return { error: e.toString() };
         }
         startKey = lek;
-        if ($response.error) {
-            (0, log_1.error)('error. query params=', JSON.stringify(params));
-            throw new Error($response.error.message);
-        }
-        (0, log_1.debug)(`dynamo query against ${params === null || params === void 0 ? void 0 : params.TableName} ok, count=${newitems === null || newitems === void 0 ? void 0 : newitems.length} ${JSON.stringify(params)}`, ` next startkey=${startKey}`);
-        if (newitems) {
-            Items.push(...newitems.map((r) => r));
-        }
-        if (count > 0 && ((_g = newitems === null || newitems === void 0 ? void 0 : newitems.length) !== null && _g !== void 0 ? _g : 0) >= count) {
+        (0, log_1.debug)(`dynamo query against ${params === null || params === void 0 ? void 0 : params.input.TableName} ok, count=${newItems === null || newItems === void 0 ? void 0 : newItems.length} ${JSON.stringify(params)}`, ` next startkey=${startKey}`);
+        if (count > 0 && ((_e = newItems === null || newItems === void 0 ? void 0 : newItems.length) !== null && _e !== void 0 ? _e : 0) >= count) {
             return { Items, startKey };
         }
     } while (startKey && Object.keys(startKey).length > 0);
@@ -347,51 +255,22 @@ exports.getDynamoTtlDays = getDynamoTtlDays;
 const getDynamoTtlMinutes = (mins) => Math.ceil(new Date().getTime() / 1000) + mins * 60;
 exports.getDynamoTtlMinutes = getDynamoTtlMinutes;
 const wipeTable = (tableName) => __awaiter(void 0, void 0, void 0, function* () {
-    var _h;
-    const dbRaw = new dynamodb_1.default({ apiVersion: '2012-10-08' });
-    let infoV = yield dbRaw
-        .describeTable({
-        TableName: tableName,
-    })
-        .promise();
+    var _f, _g;
+    let infoV = yield exports.dynamoDb.send(new client_dynamodb_1.DescribeTableCommand({ TableName: tableName }));
     // eslint-disable-next-line @typescript-eslint/ban-ts-comment
     // @ts-ignore
     let keyHash = infoV.Table.KeySchema.find((k) => k.KeyType === 'HASH').AttributeName;
-    const params = {
-        TableName: tableName,
-        ExclusiveStartKey: undefined,
-    };
-    let all = [];
-    let working;
-    do {
-        working = yield dbRaw.scan(params).promise();
-        (_h = working.Items) === null || _h === void 0 ? void 0 : _h.forEach((item) => all.push(item));
-        params.ExclusiveStartKey = working.LastEvaluatedKey;
-    } while (typeof working.LastEvaluatedKey !== 'undefined');
-    (0, log_1.warn)(`will delete ${all === null || all === void 0 ? void 0 : all.length} items from ${tableName}`);
-    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-    // @ts-ignore
-    const datagr = (0, array_1.chunk)(all, 25);
-    let res = yield Promise.all(datagr.map((group) => dbRaw
-        .batchWriteItem({
-        RequestItems: {
-            [`${tableName}`]: group.map((s) => ({
-                DeleteRequest: {
-                    Key: {
-                        [`${keyHash}`]: s[keyHash],
-                    },
-                },
-            })),
-        },
-    })
-        .promise()));
-    let errors = res
-        .map((r) => { var _a, _b; return ((_b = (((_a = r === null || r === void 0 ? void 0 : r.$response) === null || _a === void 0 ? void 0 : _a.error) || {})) === null || _b === void 0 ? void 0 : _b.message) || undefined; })
-        .filter(array_1.notEmpty);
-    if (errors.length > 0) {
-        (0, log_1.error)('errors=', JSON.stringify(errors));
-        return { errors };
+    if (!keyHash) {
+        throw new Error('couldnt find keyHash');
     }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    let all = ((_g = (_f = (yield (0, exports.scan)(tableName))) === null || _f === void 0 ? void 0 : _f.data) === null || _g === void 0 ? void 0 : _g.map((d) => d)) || [];
+    (0, log_1.warn)(`will delete ${all === null || all === void 0 ? void 0 : all.length} items from ${tableName}`);
+    yield (0, exports.batchDelete)({
+        tableName,
+        keys: all.map((s) => s[keyHash]),
+        pkName: 'PK',
+    });
     (0, log_1.warn)(`cleared table ${tableName}`);
     return {};
 });

package/dist/api/helpers/index.d.ts

@@ -2,13 +2,13 @@ export * from './api';
 export * from './aws';
 export * from './cloudfront';
 export * from './dynamo';
-export * from './dynamoInfra';
 export * from './enforceDynamoProvisionCap';
 export * from './openApiHelpers';
 export * from './s3';
 export * from './ses';
 export * from './sqs';
 export * from './ssm';
+export * from './ssmInfra';
 export * from './sts';
 export * from './validateOpenApi';
 export * from './validations';

package/dist/api/helpers/index.js

@@ -18,13 +18,13 @@ __exportStar(require("./api"), exports);
 __exportStar(require("./aws"), exports);
 __exportStar(require("./cloudfront"), exports);
 __exportStar(require("./dynamo"), exports);
-__exportStar(require("./dynamoInfra"), exports);
 __exportStar(require("./enforceDynamoProvisionCap"), exports);
 __exportStar(require("./openApiHelpers"), exports);
 __exportStar(require("./s3"), exports);
 __exportStar(require("./ses"), exports);
 __exportStar(require("./sqs"), exports);
 __exportStar(require("./ssm"), exports);
+__exportStar(require("./ssmInfra"), exports);
 __exportStar(require("./sts"), exports);
 __exportStar(require("./validateOpenApi"), exports);
 __exportStar(require("./validations"), exports);

package/dist/api/helpers/openApiHelpers.js

@@ -52,8 +52,8 @@ const setupLambda = ({ lambdaConfig, pathV, verb, seenPermissions, authorizers,
         seenPermissions.default = true;
     }
     //
-    const readTables = (0, array_1.distinctBy)([...(((_a = def === null || def === void 0 ? void 0 : def.dynamo) === null || _a === void 0 ? void 0 : _a.reads) || []), ...(((_b = lp === null || lp === void 0 ? void 0 : lp.dynamo) === null || _b === void 0 ? void 0 : _b.reads) || [])], (s) => s.shortName);
-    const writeTables = (0, array_1.distinctBy)([...(((_c = def === null || def === void 0 ? void 0 : def.dynamo) === null || _c === void 0 ? void 0 : _c.writes) || []), ...(((_d = lp === null || lp === void 0 ? void 0 : lp.dynamo) === null || _d === void 0 ? void 0 : _d.writes) || [])], (s) => s.shortName);
+    const readTables = (0, array_1.distinctBy)([...(((_a = def === null || def === void 0 ? void 0 : def.dynamo) === null || _a === void 0 ? void 0 : _a.reads) || []), ...(((_b = lp === null || lp === void 0 ? void 0 : lp.dynamo) === null || _b === void 0 ? void 0 : _b.reads) || [])], (s) => s.tableName);
+    const writeTables = (0, array_1.distinctBy)([...(((_c = def === null || def === void 0 ? void 0 : def.dynamo) === null || _c === void 0 ? void 0 : _c.writes) || []), ...(((_d = lp === null || lp === void 0 ? void 0 : lp.dynamo) === null || _d === void 0 ? void 0 : _d.writes) || [])], (s) => s.tableName);
     const policies = [...(def.policies || []), ...((lp === null || lp === void 0 ? void 0 : lp.policies) || [])].filter(array_1.notEmpty);
     const layers = [...(def.layers || []), ...((lp === null || lp === void 0 ? void 0 : lp.layers) || [])].filter(array_1.notEmpty);
     const memory = (lp === null || lp === void 0 ? void 0 : lp.memory) || (def === null || def === void 0 ? void 0 : def.memory) || 128;
@@ -79,11 +79,7 @@ const setupLambda = ({ lambdaConfig, pathV, verb, seenPermissions, authorizers,
         ? undefined
         : authorizers === null || authorizers === void 0 ? void 0 : authorizers[authorizerName];
     const env = Object.assign(Object.assign({}, (def.env || {})), ((lp === null || lp === void 0 ? void 0 : lp.env) || {}));
-    const tables = [...readTables, ...writeTables];
     const environment = env;
-    Object.values(tables).forEach((v) => {
-        environment[v.shortName] = v.table.tableName;
-    });
     return {
         environment,
         readTables,
@@ -149,8 +145,8 @@ const openApiImpl = (p) => {
                 logRetention: aws_cdk_lib_1.aws_logs.RetentionDays.FIVE_DAYS,
                 layers: lc.layers,
             });
-            lc.readTables.forEach((t) => t.table.grantReadData(lambdaV));
-            lc.writeTables.forEach((t) => t.table.grantReadWriteData(lambdaV));
+            lc.readTables.forEach((t) => t.grantReadData(lambdaV));
+            lc.writeTables.forEach((t) => t.grantReadWriteData(lambdaV));
             lc.policies.forEach((p1) => lambdaV.addToRolePolicy(p1));
             //
             apiPath.addMethod(verb.toUpperCase(), new aws_cdk_lib_1.aws_apigateway.LambdaIntegration(lambdaV, {}), { authorizer: lc.authorizer });

package/dist/api/helpers/ssm.d.ts

@@ -15,6 +15,12 @@ export declare const loadSsmString: ({ stack, path, }: {
     /** eg `/${NODE_ENV}/${shortStackName}/service/key` */
     path: string;
 }) => string;
+/** load a string from ssm at synth time*/
+export declare const loadSsmStringSynth: ({ stack, path, }: {
+    stack: Stack;
+    /** eg `/${NODE_ENV}/${shortStackName}/service/key` */
+    path: string;
+}) => string;
 /** load a csv from ssm */
 export declare const loadSsmStringList: ({ stack, path, }: {
     stack: Stack;

package/dist/api/helpers/ssm.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.loadSsmStringList = exports.loadSsmString = exports.saveToSsm = void 0;
+exports.loadSsmStringList = exports.loadSsmStringSynth = exports.loadSsmString = exports.saveToSsm = void 0;
 const aws_cdk_lib_1 = require("aws-cdk-lib");
 const base64_1 = require("../../common/helpers/string/base64");
 /**
@@ -21,6 +21,9 @@ exports.saveToSsm = saveToSsm;
 /** load a string from ssm */
 const loadSsmString = ({ stack, path, }) => aws_cdk_lib_1.aws_ssm.StringParameter.valueForStringParameter(stack, path);
 exports.loadSsmString = loadSsmString;
+/** load a string from ssm at synth time*/
+const loadSsmStringSynth = ({ stack, path, }) => aws_cdk_lib_1.aws_ssm.StringParameter.valueFromLookup(stack, path);
+exports.loadSsmStringSynth = loadSsmStringSynth;
 /** load a csv from ssm */
 const loadSsmStringList = ({ stack, path, }) => aws_cdk_lib_1.Fn.split(',', (0, exports.loadSsmString)({ stack, path }));
 exports.loadSsmStringList = loadSsmStringList;

package/dist/api/helpers/{dynamoInfra.d.ts → ssmInfra/dynamo.d.ts}

@@ -1,15 +1,13 @@
 import { aws_dynamodb as dynamodb, Stack } from 'aws-cdk-lib';
-import { IGeneratedDynamoData } from '../types';
 export declare const generateTableRef: ({ stack, baseKey, hasStream, hasGSI, }: {
     hasStream: boolean;
     hasGSI: boolean;
     stack: Stack;
     baseKey: string;
-}) => IGeneratedDynamoData;
-export declare const generateSSMParams: ({ stack, table, baseKey, gsi, shortName, }: {
+}) => dynamodb.ITable;
+export declare const generateDynamoSSMParams: ({ stack, table, baseKey, gsi, }: {
     stack: Stack;
     table: dynamodb.ITable;
     baseKey: string;
     gsi?: string[] | undefined;
-    shortName: string;
 }) => void;

package/dist/api/helpers/ssmInfra/dynamo.js

@@ -0,0 +1,34 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.generateDynamoSSMParams = exports.generateTableRef = void 0;
+const aws_cdk_lib_1 = require("aws-cdk-lib");
+const ssm_1 = require("../ssm");
+const generateTableRef = ({ stack, baseKey, hasStream, hasGSI, }) => {
+    const tableArn = (0, ssm_1.loadSsmString)({ stack, path: `${baseKey}/arn` });
+    const ret = aws_cdk_lib_1.aws_dynamodb.Table.fromTableAttributes(stack, baseKey, Object.assign(Object.assign({ tableArn }, (hasStream && {
+        tableStreamArn: (0, ssm_1.loadSsmString)({ stack, path: `${baseKey}/streamArn` }),
+    })), (hasGSI && {
+        globalIndexes: (0, ssm_1.loadSsmStringList)({ stack, path: `${baseKey}/gsi` }),
+    })));
+    return ret;
+};
+exports.generateTableRef = generateTableRef;
+const generateDynamoSSMParams = ({ stack, table, baseKey, gsi, }) => {
+    (0, ssm_1.saveToSsm)({ stack, path: `${baseKey}/arn`, value: table.tableArn });
+    (0, ssm_1.saveToSsm)({ stack, path: `${baseKey}/name`, value: table.tableName });
+    if (table.tableStreamArn) {
+        (0, ssm_1.saveToSsm)({
+            stack,
+            path: `${baseKey}/streamArn`,
+            value: table.tableStreamArn,
+        });
+    }
+    if (gsi && (gsi === null || gsi === void 0 ? void 0 : gsi.length) > 0) {
+        (0, ssm_1.saveToSsm)({
+            stack,
+            path: `${baseKey}/gsi`,
+            value: gsi.join(','),
+        });
+    }
+};
+exports.generateDynamoSSMParams = generateDynamoSSMParams;

package/dist/api/helpers/ssmInfra/index.d.ts

@@ -0,0 +1,3 @@
+export * from './dynamo';
+export * from './s3';
+export * from './sqs';

package/dist/api/helpers/ssmInfra/index.js

@@ -0,0 +1,19 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./dynamo"), exports);
+__exportStar(require("./s3"), exports);
+__exportStar(require("./sqs"), exports);

package/dist/api/helpers/ssmInfra/s3.d.ts

@@ -0,0 +1,10 @@
+import { aws_s3 as s3, Stack } from 'aws-cdk-lib';
+export declare const generateS3SSMParams: ({ stack, bucket, baseKey, }: {
+    stack: Stack;
+    bucket: s3.IBucket;
+    baseKey: string;
+}) => void;
+export declare const generateBucketRef: ({ stack, baseKey, }: {
+    stack: Stack;
+    baseKey: string;
+}) => s3.IBucket;

package/dist/api/helpers/ssmInfra/s3.js

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.generateBucketRef = exports.generateS3SSMParams = void 0;
+const aws_cdk_lib_1 = require("aws-cdk-lib");
+const ssm_1 = require("../ssm");
+const generateS3SSMParams = ({ stack, bucket, baseKey, }) => {
+    (0, ssm_1.saveToSsm)({ stack, path: `${baseKey}/arn`, value: bucket.bucketArn });
+    (0, ssm_1.saveToSsm)({ stack, path: `${baseKey}/name`, value: bucket.bucketName });
+};
+exports.generateS3SSMParams = generateS3SSMParams;
+const generateBucketRef = ({ stack, baseKey, }) => {
+    const bucketName = (0, ssm_1.loadSsmString)({ stack, path: `${baseKey}/name` });
+    return aws_cdk_lib_1.aws_s3.Bucket.fromBucketAttributes(stack, baseKey, {
+        bucketArn: (0, ssm_1.loadSsmString)({ stack, path: `${baseKey}/arn` }),
+        bucketName,
+    });
+};
+exports.generateBucketRef = generateBucketRef;

package/dist/api/helpers/ssmInfra/sqs.d.ts

@@ -0,0 +1,10 @@
+import { aws_sqs as sqs, Stack } from 'aws-cdk-lib';
+export declare const generateSqsSSMParams: ({ stack, queue, baseKey, }: {
+    stack: Stack;
+    queue: sqs.IQueue;
+    baseKey: string;
+}) => void;
+export declare const generateQueueRef: ({ stack, baseKey, }: {
+    stack: Stack;
+    baseKey: string;
+}) => sqs.IQueue;

package/dist/api/helpers/ssmInfra/sqs.js

@@ -0,0 +1,20 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.generateQueueRef = exports.generateSqsSSMParams = void 0;
+const aws_cdk_lib_1 = require("aws-cdk-lib");
+const ssm_1 = require("../ssm");
+const generateSqsSSMParams = ({ stack, queue, baseKey, }) => {
+    (0, ssm_1.saveToSsm)({ stack, path: `${baseKey}/arn`, value: queue.queueArn });
+    (0, ssm_1.saveToSsm)({ stack, path: `${baseKey}/name`, value: queue.queueName });
+    (0, ssm_1.saveToSsm)({ stack, path: `${baseKey}/url`, value: queue.queueUrl });
+};
+exports.generateSqsSSMParams = generateSqsSSMParams;
+const generateQueueRef = ({ stack, baseKey, }) => {
+    const queueArn = (0, ssm_1.loadSsmString)({ stack, path: `${baseKey}/arn` });
+    return aws_cdk_lib_1.aws_sqs.Queue.fromQueueAttributes(stack, baseKey, {
+        queueArn,
+        queueName: (0, ssm_1.loadSsmString)({ stack, path: `${baseKey}/name` }),
+        queueUrl: (0, ssm_1.loadSsmString)({ stack, path: `${baseKey}/url` }),
+    });
+};
+exports.generateQueueRef = generateQueueRef;

package/dist/api/types/aws.d.ts

@@ -1,3 +1,4 @@
+import { AttributeValue } from '@aws-sdk/client-dynamodb/dist-types/models/models_0';
 export interface APIGatewayProxyResult {
     statusCode: number;
     headers?: {
@@ -30,20 +31,6 @@ export interface APIGatewayEvent {
         httpMethod: string;
     };
 }
-interface AttributeValue {
-    B?: string | undefined;
-    BS?: string[] | undefined;
-    BOOL?: boolean | undefined;
-    L?: AttributeValue[] | undefined;
-    M?: {
-        [id: string]: AttributeValue;
-    } | undefined;
-    N?: string | undefined;
-    NS?: string[] | undefined;
-    NULL?: boolean | undefined;
-    S?: string | undefined;
-    SS?: string[] | undefined;
-}
 export interface AppSyncResolverEvent<TArguments, TSource = Record<string, any> | null> {
     arguments: TArguments;
     identity?: any;
@@ -100,4 +87,3 @@ export interface DynamoDBStreamEvent {
 export type Key = {
     [key: string]: AttributeValue;
 };
-export {};

package/dist/api/types/index.d.ts

@@ -1,5 +1,4 @@
 import { aws_dynamodb as dynamodb, aws_iam as iam, aws_lambda as lambda } from 'aws-cdk-lib';
-import { Key } from './aws';
 export interface DYNAMOKEYS {
     type: string;
     L1: string;
@@ -13,14 +12,10 @@ export interface DYNAMOKEYS {
     PK3?: string;
     PK4?: string;
 }
-export interface IGeneratedDynamoData {
-    table: dynamodb.ITable;
-    shortName: string;
-}
 export interface ILambdaConfig {
     dynamo?: {
-        reads?: IGeneratedDynamoData[];
-        writes?: IGeneratedDynamoData[];
+        reads?: dynamodb.ITable[];
+        writes?: dynamodb.ITable[];
     };
     policies?: iam.PolicyStatement[];
     env?: Record<string, string>;
@@ -64,7 +59,7 @@ export interface IQueryDynamo {
     indexName?: string;
     /** default 1000 */
     count?: number;
-    startKey?: Key;
+    startKeyPk?: string;
     filterName?: string;
     filterValue?: string | number | boolean;
     /** default, = */

package/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.0.459",
+  "version": "0.0.461",
   "name": "ag-common",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -9,14 +9,18 @@
   "scripts": {
     "preinstall": "npx only-allow pnpm",
     "format": "npx eslint --ext .ts,.tsx ./src --fix && npx eslint --ext .ts,.tsx ./stories --fix",
-    "build": "rimraf dist && npm run lint && tsc",
-    "lint": "npx eslint --ext .ts,.tsx ./src",
+    "build": "rimraf dist && tsc -p tsconfig.build.json",
+    "lint": "npx eslint --ext .ts,.tsx .",
     "start": "cross-env BROWSER=none cross-env storybook dev -p 6006",
-    "build-storybook": "storybook build -o docs --quiet"
+    "build-storybook": "storybook build -o docs --quiet",
+    "test-run": "ts-node -- src/api/helpers/dynamo.test.ts"
   },
   "dependencies": {
+    "@aws-sdk/client-dynamodb": "^3.379.1",
+    "@aws-sdk/lib-dynamodb": "^3.379.1",
+    "@aws-sdk/util-dynamodb": "^3.379.1",
     "aws-cdk-lib": ">=2",
-    "aws-sdk": ">=2",
+    "aws-sdk": ">=2.1368.0",
     "axios": ">=1",
     "constructs": ">=10",
     "jsonwebtoken": ">=9",
@@ -42,14 +46,17 @@
     "@storybook/react": "7.1.1",
     "@storybook/react-webpack5": "7.1.1",
     "@storybook/theming": "7.1.1",
+    "@types/jest": "^29.5.3",
     "@types/jsonwebtoken": "9.0.2",
     "@types/node": "20.4.5",
     "@types/react": "18.2.17",
     "@types/react-dom": "18.2.7",
     "cross-env": "7.0.3",
     "eslint-config-e7npm": "0.0.13",
+    "jest": "^29.6.2",
     "rimraf": "5.0.1",
-    "storybook": "7.1.1"
+    "storybook": "7.1.1",
+    "ts-jest": "^29.1.1"
   },
   "files": [
     "dist/**/*",

package/dist/api/helpers/dynamoInfra.js

@@ -1,47 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.generateSSMParams = exports.generateTableRef = void 0;
-const aws_cdk_lib_1 = require("aws-cdk-lib");
-const ssm_1 = require("./ssm");
-const generateTableRef = ({ stack, baseKey, hasStream, hasGSI, }) => {
-    const tableStreamArn = !hasStream
-        ? undefined
-        : (0, ssm_1.loadSsmString)({ stack, path: `${baseKey}/streamArn` });
-    const globalIndexes = !hasGSI
-        ? undefined
-        : (0, ssm_1.loadSsmStringList)({ stack, path: `${baseKey}/gsi` });
-    const shortName = aws_cdk_lib_1.aws_ssm.StringParameter.valueFromLookup(stack, `${baseKey}/shortName`);
-    const table = aws_cdk_lib_1.aws_dynamodb.Table.fromTableAttributes(stack, baseKey, {
-        tableArn: (0, ssm_1.loadSsmString)({ stack, path: `${baseKey}/arn` }),
-        tableStreamArn,
-        globalIndexes,
-    });
-    return {
-        shortName,
-        table,
-    };
-};
-exports.generateTableRef = generateTableRef;
-const generateSSMParams = ({ stack, table, baseKey, gsi, shortName, }) => {
-    (0, ssm_1.saveToSsm)({ stack, path: `${baseKey}/arn`, value: table.tableArn });
-    if (table.tableStreamArn) {
-        (0, ssm_1.saveToSsm)({
-            stack,
-            path: `${baseKey}/streamArn`,
-            value: table.tableStreamArn,
-        });
-    }
-    if (gsi && (gsi === null || gsi === void 0 ? void 0 : gsi.length) > 0) {
-        (0, ssm_1.saveToSsm)({
-            stack,
-            path: `${baseKey}/gsi`,
-            value: gsi.join(','),
-        });
-    }
-    (0, ssm_1.saveToSsm)({
-        stack,
-        path: `${baseKey}/shortName`,
-        value: shortName,
-    });
-};
-exports.generateSSMParams = generateSSMParams;