ag-common 0.0.461 → 0.0.463

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/dist/api/helpers/dynamo.d.ts +14 -4
  2. package/dist/api/helpers/dynamo.js +137 -106
  3. package/package.json +1 -1
package/dist/api/helpers/dynamo.d.ts CHANGED
@@ -15,8 +15,18 @@ export declare const batchDelete: ({ tableName, keys, pkName, }: {
15
15
  tableName: string;
16
16
  keys: string[];
17
17
  pkName: string;
18
- }) => Promise<{}>;
19
- export declare const scan: <T>(tableName: string) => Promise<{
18
+ }) => Promise<{
19
+ error?: string;
20
+ }>;
21
+ export declare const scan: <T>(tableName: string, opt?: {
22
+ filter?: {
23
+ filterExpression: string;
24
+ attrNames: Record<string, string>;
25
+ attrValues: Record<string, string>;
26
+ };
27
+ /** ProjectionExpression. will csv values */
28
+ requiredAttributeList?: string[];
29
+ }) => Promise<{
20
30
  data?: T[] | undefined;
21
31
  error?: string | undefined;
22
32
  }>;
@@ -39,14 +49,14 @@ export declare const getItemDynamo: <T>({ tableName, pkName, pkValue, }: {
39
49
  error?: string | undefined;
40
50
  }>;
41
51
  export declare const queryDynamo: <T>({ tableName, pkName, pkValue, pkOperator, skName, skValue, skOperator, indexName, count, startKeyPk, filterName, filterValue, filterOperator, sortAscending, }: IQueryDynamo) => Promise<{
42
- Items?: T[] | undefined;
52
+ data?: T[] | undefined;
43
53
  startKey?: Key | undefined;
44
54
  error?: string | undefined;
45
55
  }>;
46
56
  export declare const getDynamoTtlDays: (days: number) => number;
47
57
  export declare const getDynamoTtlMinutes: (mins: number) => number;
48
58
  export declare const wipeTable: (tableName: string) => Promise<{
49
- errors?: string[];
59
+ error?: string;
50
60
  }>;
51
61
  export declare const getDynamoUpdates: (items: DYNAMOKEYS) => {
52
62
  UpdateExpression: string;
package/dist/api/helpers/dynamo.js CHANGED
@@ -35,12 +35,11 @@ const putDynamo = (item, tableName, opt) => __awaiter(void 0, void 0, void 0, fu
35
35
  })));
36
36
  (0, log_1.debug)(`running dynamo put=${JSON.stringify(params, null, 2)}`);
37
37
  try {
38
- let r1 = yield exports.dynamoDb.send(params);
39
- console.log('r1=', r1);
38
+ yield exports.dynamoDb.send(params);
40
39
  return {};
41
40
  }
42
41
  catch (e) {
43
- (0, log_1.warn)('put error', e);
42
+ (0, log_1.warn)('putDynamo error', e);
44
43
  return { error: e.toString() };
45
44
  }
46
45
  });
@@ -49,103 +48,129 @@ exports.putDynamo = putDynamo;
49
48
  const batchWrite = (tableName, itemsIn) => __awaiter(void 0, void 0, void 0, function* () {
50
49
  //batch up to 20, so we can retry.
51
50
  let chunked = (0, array_1.chunk)(itemsIn, 20);
52
- yield (0, async_1.asyncForEach)(chunked, (items) => __awaiter(void 0, void 0, void 0, function* () {
53
- let retryCount = 0;
54
- let retryMax = 3;
55
- let params = new lib_dynamodb_1.BatchWriteCommand({
56
- RequestItems: {
57
- [`${tableName}`]: items.map((Item) => ({
58
- PutRequest: { Item: Item },
59
- })),
60
- },
61
- });
62
- (0, log_1.debug)(`running dynamo batchWrite=${JSON.stringify(params, null, 2)}`);
63
- try {
64
- yield exports.dynamoDb.send(params);
65
- return {};
66
- }
67
- catch (e) {
68
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
69
- let es = e.toString();
70
- let msg = es;
71
- (0, log_1.warn)('dynamo write error', msg);
72
- if (es.indexOf('429') !== -1 ||
73
- es.indexOf('ProvisionedThroughputExceeded') !== -1) {
74
- retryCount += 1;
75
- msg = `batch write throttled. retry ${retryCount}/${retryMax}`;
76
- }
77
- else {
78
- throw e;
51
+ try {
52
+ yield (0, async_1.asyncForEach)(chunked, (items) => __awaiter(void 0, void 0, void 0, function* () {
53
+ let retryCount = 0;
54
+ let retryMax = 3;
55
+ let params = new lib_dynamodb_1.BatchWriteCommand({
56
+ RequestItems: {
57
+ [`${tableName}`]: items.map((Item) => ({
58
+ PutRequest: { Item: Item },
59
+ })),
60
+ },
61
+ });
62
+ (0, log_1.debug)(`running dynamo batchWrite=${JSON.stringify(params, null, 2)}`);
63
+ try {
64
+ yield exports.dynamoDb.send(params);
65
+ return {};
79
66
  }
80
- if (retryCount >= retryMax) {
81
- throw e;
67
+ catch (e) {
68
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
69
+ let es = e.toString();
70
+ let msg = es;
71
+ (0, log_1.warn)('dynamo write error', msg);
72
+ if (es.indexOf('429') !== -1 ||
73
+ es.indexOf('ProvisionedThroughputExceeded') !== -1) {
74
+ retryCount += 1;
75
+ msg = `batch write throttled. retry ${retryCount}/${retryMax}`;
76
+ }
77
+ else {
78
+ throw e;
79
+ }
80
+ if (retryCount >= retryMax) {
81
+ throw e;
82
+ }
83
+ (0, log_1.warn)(`dynamo retry ${retryCount}/${retryMax}`);
84
+ yield (0, sleep_1.sleep)(2000);
82
85
  }
83
- (0, log_1.warn)(`dynamo retry ${retryCount}/${retryMax}`);
84
- yield (0, sleep_1.sleep)(2000);
85
- }
86
- }));
87
- return {};
86
+ }));
87
+ return {};
88
+ }
89
+ catch (e) {
90
+ (0, log_1.warn)('batchWrite error', e);
91
+ return { error: e.toString() };
92
+ }
88
93
  });
89
94
  exports.batchWrite = batchWrite;
90
95
  const batchDelete = ({ tableName, keys, pkName, }) => __awaiter(void 0, void 0, void 0, function* () {
91
96
  //batch up to 20, so we can retry.
92
97
  let chunked = (0, array_1.chunk)(keys, 20);
93
- yield (0, async_1.asyncForEach)(chunked, (items) => __awaiter(void 0, void 0, void 0, function* () {
94
- let retryCount = 0;
95
- let retryMax = 3;
96
- let params = new lib_dynamodb_1.BatchWriteCommand({
97
- RequestItems: {
98
- [`${tableName}`]: items.map((key) => ({
99
- DeleteRequest: { Key: { [`${pkName}`]: key } },
100
- })),
101
- },
102
- });
103
- (0, log_1.debug)(`running dynamo batch delete=${JSON.stringify(params, null, 2)}`);
104
- try {
105
- yield exports.dynamoDb.send(params);
106
- return {};
107
- }
108
- catch (e) {
109
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
110
- let es = e.toString();
111
- let msg = es;
112
- (0, log_1.warn)('dynamo write error', msg);
113
- if (es.indexOf('429') !== -1 ||
114
- es.indexOf('ProvisionedThroughputExceeded') !== -1) {
115
- retryCount += 1;
116
- msg = `batch delete write throttled. retry ${retryCount}/${retryMax}`;
117
- }
118
- else {
119
- throw e;
98
+ try {
99
+ yield (0, async_1.asyncForEach)(chunked, (items) => __awaiter(void 0, void 0, void 0, function* () {
100
+ let retryCount = 0;
101
+ let retryMax = 3;
102
+ let params = new lib_dynamodb_1.BatchWriteCommand({
103
+ RequestItems: {
104
+ [`${tableName}`]: items.map((key) => ({
105
+ DeleteRequest: { Key: { [`${pkName}`]: key } },
106
+ })),
107
+ },
108
+ });
109
+ (0, log_1.debug)(`running dynamo batch delete=${JSON.stringify(params, null, 2)}`);
110
+ try {
111
+ yield exports.dynamoDb.send(params);
112
+ return {};
120
113
  }
121
- if (retryCount >= retryMax) {
122
- throw e;
114
+ catch (e) {
115
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
116
+ let es = e.toString();
117
+ let msg = es;
118
+ (0, log_1.warn)('dynamo write error', msg);
119
+ if (es.indexOf('429') !== -1 ||
120
+ es.indexOf('ProvisionedThroughputExceeded') !== -1) {
121
+ retryCount += 1;
122
+ msg = `batch delete write throttled. retry ${retryCount}/${retryMax}`;
123
+ }
124
+ else {
125
+ throw e;
126
+ }
127
+ if (retryCount >= retryMax) {
128
+ throw e;
129
+ }
130
+ (0, log_1.warn)(`dynamo retry ${retryCount}/${retryMax}`);
131
+ yield (0, sleep_1.sleep)(2000);
123
132
  }
124
- (0, log_1.warn)(`dynamo retry ${retryCount}/${retryMax}`);
125
- yield (0, sleep_1.sleep)(2000);
126
- }
127
- }));
128
- return {};
133
+ }));
134
+ return {};
135
+ }
136
+ catch (e) {
137
+ (0, log_1.warn)('batchDelete error', e);
138
+ return { error: e.toString() };
139
+ }
129
140
  });
130
141
  exports.batchDelete = batchDelete;
131
- const scan = (tableName) => __awaiter(void 0, void 0, void 0, function* () {
132
- var _a;
133
- let params = new lib_dynamodb_1.ScanCommand({
134
- TableName: tableName,
135
- });
136
- (0, log_1.debug)(`running dynamo scan=${JSON.stringify(params, null, 2)}`);
142
+ const scan = (tableName, opt) => __awaiter(void 0, void 0, void 0, function* () {
137
143
  try {
138
- let ret = yield exports.dynamoDb.send(params);
139
- let items = ((_a = ret.Items) !== null && _a !== void 0 ? _a : []).map((r) => r);
140
- return { data: items };
144
+ let ExclusiveStartKey;
145
+ const Items = [];
146
+ do {
147
+ let params = new lib_dynamodb_1.ScanCommand(Object.assign(Object.assign(Object.assign({ TableName: tableName }, ((opt === null || opt === void 0 ? void 0 : opt.filter) && {
148
+ FilterExpression: opt.filter.filterExpression,
149
+ ExpressionAttributeNames: opt.filter.attrNames,
150
+ ExpressionAttributeValues: opt.filter.attrValues,
151
+ })), ((opt === null || opt === void 0 ? void 0 : opt.requiredAttributeList) && {
152
+ ProjectionExpression: opt.requiredAttributeList.join(', '),
153
+ })), { ExclusiveStartKey }));
154
+ (0, log_1.trace)(`running dynamo scan=${JSON.stringify(params, null, 2)}`);
155
+ const { Items: newitems, LastEvaluatedKey,
156
+ // eslint-disable-next-line no-await-in-loop
157
+ } = yield exports.dynamoDb.send(params);
158
+ ExclusiveStartKey = LastEvaluatedKey;
159
+ if (newitems) {
160
+ Items.push(...newitems.map((r) => r));
161
+ }
162
+ } while (ExclusiveStartKey);
163
+ (0, log_1.trace)(`dynamo scan against ${tableName} ok, count=${Items === null || Items === void 0 ? void 0 : Items.length}`);
164
+ return { data: Items };
141
165
  }
142
166
  catch (e) {
167
+ (0, log_1.warn)('scan error:', e);
143
168
  return { error: e.toString() };
144
169
  }
145
170
  });
146
171
  exports.scan = scan;
147
172
  const getItemsDynamo = ({ tableName, items, }) => __awaiter(void 0, void 0, void 0, function* () {
148
- var _b, _c;
173
+ var _a, _b;
149
174
  const params = new lib_dynamodb_1.BatchGetCommand({
150
175
  RequestItems: {
151
176
  [tableName]: {
@@ -157,24 +182,24 @@ const getItemsDynamo = ({ tableName, items, }) => __awaiter(void 0, void 0, void
157
182
  });
158
183
  try {
159
184
  let res = yield exports.dynamoDb.send(params);
160
- let data = (_c = (_b = res.Responses) === null || _b === void 0 ? void 0 : _b[tableName].map((r) => r)) !== null && _c !== void 0 ? _c : [];
185
+ let data = (_b = (_a = res.Responses) === null || _a === void 0 ? void 0 : _a[tableName].map((r) => r)) !== null && _b !== void 0 ? _b : [];
161
186
  return { data };
162
187
  }
163
188
  catch (e) {
164
- (0, log_1.warn)('get error:', e);
189
+ (0, log_1.warn)('getItemsDynamo error:', e);
165
190
  return { error: e.toString() };
166
191
  //
167
192
  }
168
193
  });
169
194
  exports.getItemsDynamo = getItemsDynamo;
170
195
  const getItemDynamo = ({ tableName, pkName, pkValue, }) => __awaiter(void 0, void 0, void 0, function* () {
171
- var _d;
196
+ var _c;
172
197
  let r = yield (0, exports.getItemsDynamo)({ tableName, items: [{ pkName, pkValue }] });
173
- return { data: (_d = r.data) === null || _d === void 0 ? void 0 : _d[0], error: r.error };
198
+ return { data: (_c = r.data) === null || _c === void 0 ? void 0 : _c[0], error: r.error };
174
199
  });
175
200
  exports.getItemDynamo = getItemDynamo;
176
201
  const queryDynamo = ({ tableName, pkName, pkValue, pkOperator = '=', skName, skValue, skOperator = '=', indexName, count = 1000, startKeyPk, filterName, filterValue, filterOperator = '=', sortAscending = true, }) => __awaiter(void 0, void 0, void 0, function* () {
177
- var _e;
202
+ var _d;
178
203
  let kce = `#${pkName.toLowerCase()} ${pkOperator} :${pkName.toLowerCase()}`;
179
204
  const ean = { [`#${pkName.toLowerCase()}`]: pkName };
180
205
  const eav = {
@@ -243,11 +268,11 @@ const queryDynamo = ({ tableName, pkName, pkValue, pkOperator = '=', skName, skV
243
268
  }
244
269
  startKey = lek;
245
270
  (0, log_1.debug)(`dynamo query against ${params === null || params === void 0 ? void 0 : params.input.TableName} ok, count=${newItems === null || newItems === void 0 ? void 0 : newItems.length} ${JSON.stringify(params)}`, ` next startkey=${startKey}`);
246
- if (count > 0 && ((_e = newItems === null || newItems === void 0 ? void 0 : newItems.length) !== null && _e !== void 0 ? _e : 0) >= count) {
247
- return { Items, startKey };
271
+ if (count > 0 && ((_d = newItems === null || newItems === void 0 ? void 0 : newItems.length) !== null && _d !== void 0 ? _d : 0) >= count) {
272
+ return { data: Items, startKey };
248
273
  }
249
274
  } while (startKey && Object.keys(startKey).length > 0);
250
- return { Items, startKey: undefined };
275
+ return { data: Items, startKey: undefined };
251
276
  });
252
277
  exports.queryDynamo = queryDynamo;
253
278
  const getDynamoTtlDays = (days) => Math.ceil(new Date().getTime() / 1000) + days * 86400;
@@ -255,24 +280,30 @@ exports.getDynamoTtlDays = getDynamoTtlDays;
255
280
  const getDynamoTtlMinutes = (mins) => Math.ceil(new Date().getTime() / 1000) + mins * 60;
256
281
  exports.getDynamoTtlMinutes = getDynamoTtlMinutes;
257
282
  const wipeTable = (tableName) => __awaiter(void 0, void 0, void 0, function* () {
258
- var _f, _g;
259
- let infoV = yield exports.dynamoDb.send(new client_dynamodb_1.DescribeTableCommand({ TableName: tableName }));
260
- // eslint-disable-next-line @typescript-eslint/ban-ts-comment
261
- // @ts-ignore
262
- let keyHash = infoV.Table.KeySchema.find((k) => k.KeyType === 'HASH').AttributeName;
263
- if (!keyHash) {
264
- throw new Error('couldnt find keyHash');
283
+ var _e, _f;
284
+ try {
285
+ let infoV = yield exports.dynamoDb.send(new client_dynamodb_1.DescribeTableCommand({ TableName: tableName }));
286
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
287
+ // @ts-ignore
288
+ let keyHash = infoV.Table.KeySchema.find((k) => k.KeyType === 'HASH').AttributeName;
289
+ if (!keyHash) {
290
+ throw new Error('couldnt find keyHash');
291
+ }
292
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
293
+ let all = ((_f = (_e = (yield (0, exports.scan)(tableName))) === null || _e === void 0 ? void 0 : _e.data) === null || _f === void 0 ? void 0 : _f.map((d) => d)) || [];
294
+ (0, log_1.warn)(`will delete ${all === null || all === void 0 ? void 0 : all.length} items from ${tableName}`);
295
+ yield (0, exports.batchDelete)({
296
+ tableName,
297
+ keys: all.map((s) => s[keyHash]),
298
+ pkName: 'PK',
299
+ });
300
+ (0, log_1.warn)(`cleared table ${tableName}`);
301
+ return {};
302
+ }
303
+ catch (e) {
304
+ (0, log_1.warn)('wipeTable error:', e);
305
+ return { error: e.toString() };
265
306
  }
266
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
267
- let all = ((_g = (_f = (yield (0, exports.scan)(tableName))) === null || _f === void 0 ? void 0 : _f.data) === null || _g === void 0 ? void 0 : _g.map((d) => d)) || [];
268
- (0, log_1.warn)(`will delete ${all === null || all === void 0 ? void 0 : all.length} items from ${tableName}`);
269
- yield (0, exports.batchDelete)({
270
- tableName,
271
- keys: all.map((s) => s[keyHash]),
272
- pkName: 'PK',
273
- });
274
- (0, log_1.warn)(`cleared table ${tableName}`);
275
- return {};
276
307
  });
277
308
  exports.wipeTable = wipeTable;
278
309
  const getDynamoUpdates = (items) => {
package/package.json CHANGED
@@ -1,5 +1,5 @@
1
1
  {
2
- "version": "0.0.461",
2
+ "version": "0.0.463",
3
3
  "name": "ag-common",
4
4
  "main": "./dist/index.js",
5
5
  "types": "./dist/index.d.ts",