@jzhuo3/dynamodb-lib 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PUBLISHING.md +9 -1
- package/README.md +25 -0
- package/dist/cjs/index.d.ts +22 -0
- package/dist/cjs/index.js +128 -102
- package/dist/cjs/index.js.map +1 -1
- package/dist/esm/index.d.ts +22 -0
- package/dist/esm/index.js +129 -103
- package/dist/esm/index.js.map +1 -1
- package/doc/service.md +36 -0
- package/package.json +4 -3
package/dist/esm/index.d.ts
CHANGED
|
@@ -135,6 +135,20 @@ export declare function ListPrepend(attribute: string, value: Array<any>, upsert
|
|
|
135
135
|
export declare function Remove(attribute: string): DynamoDBExpression;
|
|
136
136
|
export declare function SetAdd(attribute: string, value: Set<any>): DynamoDBExpression;
|
|
137
137
|
export declare function SetDelete(attribute: string, value: Set<any>): DynamoDBExpression;
|
|
138
|
+
/** JSON-compatible value used by expression descriptions. */
|
|
139
|
+
export type DynamoDBExpressionSerializedValue = null | boolean | number | string | DynamoDBExpressionSerializedValue[] | {
|
|
140
|
+
[key: string]: DynamoDBExpressionSerializedValue;
|
|
141
|
+
};
|
|
142
|
+
export type DynamoDBExpressionDescription = {
|
|
143
|
+
/** DynamoDB expression string, e.g. `#attr0 = :val0`. */
|
|
144
|
+
expression: string;
|
|
145
|
+
/** Placeholder -> attribute name (e.g. `#attr0` -> `VersionStamp`). */
|
|
146
|
+
expressionAttributeNames: Record<string, string>;
|
|
147
|
+
/** Placeholder -> JSON-compatible value; native non-JSON values use explicit `$type` tags. */
|
|
148
|
+
expressionAttributeValues: Record<string, DynamoDBExpressionSerializedValue>;
|
|
149
|
+
/** Which update clause the expression belongs to (SET/REMOVE/ADD/DELETE). */
|
|
150
|
+
updateExpressionGroup?: 'SET' | 'REMOVE' | 'ADD' | 'DELETE';
|
|
151
|
+
};
|
|
138
152
|
export declare class DynamoDBExpression {
|
|
139
153
|
expressionAttributeNameMap: Map<string, string>;
|
|
140
154
|
expressionAttributeValueMap: Map<string, any> | null;
|
|
@@ -143,6 +157,14 @@ export declare class DynamoDBExpression {
|
|
|
143
157
|
constructor(expressionAttributeNameMap?: Map<string, string>, // e.g. #attr0 -> attribute
|
|
144
158
|
expressionAttributeValueMap?: Map<string, any> | null, // e.g. :val0 -> value
|
|
145
159
|
expression?: string, updateExpressionGroup?: "SET" | "REMOVE" | "ADD" | "DELETE" | undefined);
|
|
160
|
+
/**
|
|
161
|
+
* Detached, JSON-serializable description of the expression. Intended for logging,
|
|
162
|
+
* debugging, and unit tests that need to assert on a built expression
|
|
163
|
+
* without reaching into the internal Maps.
|
|
164
|
+
* Sets, bigint, binary, Maps, NumberValue, undefined and non-finite numbers
|
|
165
|
+
* use explicit `$type` tags. Circular and unsupported values throw TypeError.
|
|
166
|
+
*/
|
|
167
|
+
describe(): DynamoDBExpressionDescription;
|
|
146
168
|
}
|
|
147
169
|
export declare enum DynamoDBTransactionMode {
|
|
148
170
|
READ = "READ",
|
package/dist/esm/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Agent } from 'https';
|
|
2
2
|
import { DynamoDBClient, ReturnConsumedCapacity, } from '@aws-sdk/client-dynamodb';
|
|
3
|
-
import { DynamoDBDocumentClient, GetCommand, BatchGetCommand, QueryCommand, PutCommand, BatchWriteCommand, UpdateCommand, DeleteCommand, TransactGetCommand, TransactWriteCommand, } from '@aws-sdk/lib-dynamodb';
|
|
3
|
+
import { DynamoDBDocumentClient, NumberValue, GetCommand, BatchGetCommand, QueryCommand, PutCommand, BatchWriteCommand, UpdateCommand, DeleteCommand, TransactGetCommand, TransactWriteCommand, } from '@aws-sdk/lib-dynamodb';
|
|
4
4
|
import { diff, camel, snake } from 'radash';
|
|
5
5
|
const silentLogger = { debug() { }, warn() { }, error() { } };
|
|
6
6
|
export class DynamoDBError extends Error {
|
|
@@ -324,6 +324,55 @@ export function SetDelete(attribute, value) {
|
|
|
324
324
|
const expression = `${path} :val0`;
|
|
325
325
|
return new DynamoDBExpression(expressionAttributeNameMap, expressionAttributeValueMap, expression, 'DELETE');
|
|
326
326
|
}
|
|
327
|
+
function serializeExpressionValue(value, ancestors = new Set()) {
|
|
328
|
+
if (value === null || typeof value === 'string' || typeof value === 'boolean')
|
|
329
|
+
return value;
|
|
330
|
+
if (typeof value === 'number') {
|
|
331
|
+
return Number.isFinite(value) ? value : { $type: 'Number', value: String(value) };
|
|
332
|
+
}
|
|
333
|
+
if (typeof value === 'bigint')
|
|
334
|
+
return { $type: 'BigInt', value: value.toString() };
|
|
335
|
+
if (value === undefined)
|
|
336
|
+
return { $type: 'Undefined' };
|
|
337
|
+
if (typeof value !== 'object') {
|
|
338
|
+
throw new TypeError(`Cannot describe expression value of type ${typeof value}`);
|
|
339
|
+
}
|
|
340
|
+
if (ancestors.has(value))
|
|
341
|
+
throw new TypeError('Cannot describe circular expression value');
|
|
342
|
+
ancestors.add(value);
|
|
343
|
+
try {
|
|
344
|
+
if (value instanceof NumberValue)
|
|
345
|
+
return { $type: 'NumberValue', value: value.toString() };
|
|
346
|
+
if (value instanceof ArrayBuffer || ArrayBuffer.isView(value)) {
|
|
347
|
+
const bytes = value instanceof ArrayBuffer
|
|
348
|
+
? Buffer.from(value)
|
|
349
|
+
: Buffer.from(value.buffer, value.byteOffset, value.byteLength);
|
|
350
|
+
return { $type: 'Binary', encoding: 'base64', value: bytes.toString('base64') };
|
|
351
|
+
}
|
|
352
|
+
if (value instanceof Set) {
|
|
353
|
+
return { $type: 'Set', values: Array.from(value, item => serializeExpressionValue(item, ancestors)) };
|
|
354
|
+
}
|
|
355
|
+
if (Array.isArray(value))
|
|
356
|
+
return Array.from(value, item => serializeExpressionValue(item, ancestors));
|
|
357
|
+
if (value instanceof Map) {
|
|
358
|
+
return { $type: 'Map', entries: Array.from(value, ([key, item]) => [
|
|
359
|
+
serializeExpressionValue(key, ancestors), serializeExpressionValue(item, ancestors),
|
|
360
|
+
]) };
|
|
361
|
+
}
|
|
362
|
+
if (Object.getPrototypeOf(value) !== Object.prototype && Object.getPrototypeOf(value) !== null) {
|
|
363
|
+
throw new TypeError('Cannot describe unsupported expression value object');
|
|
364
|
+
}
|
|
365
|
+
if (Object.getOwnPropertySymbols(value).some(key => Object.prototype.propertyIsEnumerable.call(value, key))) {
|
|
366
|
+
throw new TypeError('Cannot describe expression value with symbol keys');
|
|
367
|
+
}
|
|
368
|
+
return Object.fromEntries(Object.entries(value).map(([key, item]) => [
|
|
369
|
+
key, serializeExpressionValue(item, ancestors),
|
|
370
|
+
]));
|
|
371
|
+
}
|
|
372
|
+
finally {
|
|
373
|
+
ancestors.delete(value);
|
|
374
|
+
}
|
|
375
|
+
}
|
|
327
376
|
export class DynamoDBExpression {
|
|
328
377
|
expressionAttributeNameMap;
|
|
329
378
|
expressionAttributeValueMap;
|
|
@@ -337,6 +386,27 @@ export class DynamoDBExpression {
|
|
|
337
386
|
this.expression = expression;
|
|
338
387
|
this.updateExpressionGroup = updateExpressionGroup;
|
|
339
388
|
}
|
|
389
|
+
/**
|
|
390
|
+
* Detached, JSON-serializable description of the expression. Intended for logging,
|
|
391
|
+
* debugging, and unit tests that need to assert on a built expression
|
|
392
|
+
* without reaching into the internal Maps.
|
|
393
|
+
* Sets, bigint, binary, Maps, NumberValue, undefined and non-finite numbers
|
|
394
|
+
* use explicit `$type` tags. Circular and unsupported values throw TypeError.
|
|
395
|
+
*/
|
|
396
|
+
describe() {
|
|
397
|
+
return {
|
|
398
|
+
expression: this.expression,
|
|
399
|
+
expressionAttributeNames: Object.fromEntries(this.expressionAttributeNameMap),
|
|
400
|
+
expressionAttributeValues: this.expressionAttributeValueMap
|
|
401
|
+
? Object.fromEntries(Array.from(this.expressionAttributeValueMap, ([key, value]) => [
|
|
402
|
+
key, serializeExpressionValue(value),
|
|
403
|
+
]))
|
|
404
|
+
: {},
|
|
405
|
+
...(this.updateExpressionGroup !== undefined
|
|
406
|
+
? { updateExpressionGroup: this.updateExpressionGroup }
|
|
407
|
+
: {}),
|
|
408
|
+
};
|
|
409
|
+
}
|
|
340
410
|
}
|
|
341
411
|
export var DynamoDBTransactionMode;
|
|
342
412
|
(function (DynamoDBTransactionMode) {
|
|
@@ -388,6 +458,7 @@ export class DynamoDBTransaction {
|
|
|
388
458
|
if (i === retries - 1) {
|
|
389
459
|
throw error;
|
|
390
460
|
}
|
|
461
|
+
this.logger.warn('DynamoDB retrying transaction conflict', { mode: this.mode, attempt: i + 1, maxAttempts: retries, delayMs: delay * (i + 1) });
|
|
391
462
|
await new Promise(resolve => setTimeout(resolve, delay * (i + 1)));
|
|
392
463
|
}
|
|
393
464
|
else {
|
|
@@ -636,10 +707,6 @@ export class DynamoDBTransaction {
|
|
|
636
707
|
}
|
|
637
708
|
ExpressionAttributeValues[attributeValueVariable] = value;
|
|
638
709
|
});
|
|
639
|
-
this.logger.debug('Update Expression: %s', UpdateExpression);
|
|
640
|
-
this.logger.debug('Condition Expression: %s', ConditionExpression);
|
|
641
|
-
this.logger.debug('Expression Attribute Names: %o', ExpressionAttributeNames);
|
|
642
|
-
this.logger.debug('Expression Attribute Values: %o', ExpressionAttributeValues);
|
|
643
710
|
this.transactWriteItems.push({
|
|
644
711
|
Update: {
|
|
645
712
|
TableName: table.name,
|
|
@@ -715,6 +782,7 @@ export class DynamoDBTransaction {
|
|
|
715
782
|
const result = {
|
|
716
783
|
numberOfTransactItems: 0,
|
|
717
784
|
};
|
|
785
|
+
this.logger.debug('DynamoDB transaction started', { mode: this.mode, itemCount: this.mode === DynamoDBTransactionMode.READ ? this.transactGetItems.length : this.transactWriteItems.length });
|
|
718
786
|
switch (this.mode) {
|
|
719
787
|
case DynamoDBTransactionMode.READ:
|
|
720
788
|
command = new TransactGetCommand({
|
|
@@ -726,21 +794,21 @@ export class DynamoDBTransaction {
|
|
|
726
794
|
const { Responses } = (await this.txConflictRetryWrapper(command, this.txRetryLimit, this.txRetryDelay).catch((e) => {
|
|
727
795
|
this._cleanup();
|
|
728
796
|
if (isTransactionConflictError(e)) {
|
|
729
|
-
this.logger.error('
|
|
797
|
+
this.logger.error('DynamoDB transaction conflict retry limit reached', { errorName: e.name });
|
|
730
798
|
throw new DynamoDBError(409, {
|
|
731
799
|
message: e.message,
|
|
732
800
|
cause: e,
|
|
733
801
|
});
|
|
734
802
|
}
|
|
735
803
|
else if (e.name === 'TransactionCanceledException') {
|
|
736
|
-
this.logger.error(e.
|
|
804
|
+
this.logger.error('DynamoDB transaction canceled', { errorName: e.name });
|
|
737
805
|
throw new DynamoDBError(400, {
|
|
738
806
|
message: e.message,
|
|
739
807
|
cause: e,
|
|
740
808
|
});
|
|
741
809
|
}
|
|
742
810
|
else {
|
|
743
|
-
this.logger.error('Transaction failed
|
|
811
|
+
this.logger.error('Transaction failed', { errorName: e.name });
|
|
744
812
|
throw new DynamoDBError(500, {
|
|
745
813
|
message: e.message,
|
|
746
814
|
cause: e,
|
|
@@ -789,28 +857,28 @@ export class DynamoDBTransaction {
|
|
|
789
857
|
await this.txConflictRetryWrapper(command, this.txRetryLimit, this.txRetryDelay).catch((e) => {
|
|
790
858
|
this._cleanup();
|
|
791
859
|
if (isTransactionConflictError(e)) {
|
|
792
|
-
this.logger.error('
|
|
860
|
+
this.logger.error('DynamoDB transaction conflict retry limit reached', { errorName: e.name });
|
|
793
861
|
throw new DynamoDBError(409, {
|
|
794
862
|
message: e.message,
|
|
795
863
|
cause: e,
|
|
796
864
|
});
|
|
797
865
|
}
|
|
798
866
|
else if (e.name === 'TransactionCanceledException') {
|
|
799
|
-
this.logger.error(e.
|
|
867
|
+
this.logger.error('DynamoDB transaction canceled', { errorName: e.name });
|
|
800
868
|
throw new DynamoDBError(400, {
|
|
801
869
|
message: e.message,
|
|
802
870
|
cause: e,
|
|
803
871
|
});
|
|
804
872
|
}
|
|
805
873
|
else if (e.name === 'ValidationException') {
|
|
806
|
-
this.logger.error('The request parameters are invalid
|
|
874
|
+
this.logger.error('The request parameters are invalid', { errorName: e.name });
|
|
807
875
|
throw new DynamoDBError(400, {
|
|
808
876
|
message: 'Invalid request parameters',
|
|
809
877
|
cause: e,
|
|
810
878
|
});
|
|
811
879
|
}
|
|
812
880
|
else {
|
|
813
|
-
this.logger.error('Transaction failed
|
|
881
|
+
this.logger.error('Transaction failed', { errorName: e.name });
|
|
814
882
|
throw new DynamoDBError(500, {
|
|
815
883
|
message: e.message,
|
|
816
884
|
cause: e,
|
|
@@ -823,6 +891,7 @@ export class DynamoDBTransaction {
|
|
|
823
891
|
default:
|
|
824
892
|
throw new Error('Invalid transaction mode');
|
|
825
893
|
}
|
|
894
|
+
this.logger.debug('DynamoDB transaction completed', { mode: this.mode, itemCount: result.numberOfTransactItems });
|
|
826
895
|
return result;
|
|
827
896
|
}
|
|
828
897
|
}
|
|
@@ -891,8 +960,7 @@ export default class DynamoDBService {
|
|
|
891
960
|
ExpressionAttributeNames[attributeNameVariable] = attributeName;
|
|
892
961
|
});
|
|
893
962
|
}
|
|
894
|
-
this.logger.debug('
|
|
895
|
-
this.logger.debug('Expression Attribute Names: %o', ExpressionAttributeNames);
|
|
963
|
+
this.logger.debug('DynamoDB request started', { operation: 'getOne', tableKey });
|
|
896
964
|
const getCommand = new GetCommand({
|
|
897
965
|
TableName: table.name,
|
|
898
966
|
Key: keys,
|
|
@@ -907,22 +975,14 @@ export default class DynamoDBService {
|
|
|
907
975
|
.send(getCommand)
|
|
908
976
|
.catch((e) => {
|
|
909
977
|
if (e.name === 'ValidationException') {
|
|
910
|
-
this.logger.error('The request parameters are invalid
|
|
911
|
-
tableKey,
|
|
912
|
-
primaryKey,
|
|
913
|
-
opts,
|
|
914
|
-
});
|
|
978
|
+
this.logger.error('The request parameters are invalid', { errorName: e.name, tableKey });
|
|
915
979
|
throw new DynamoDBError(400, {
|
|
916
980
|
message: 'Invalid request parameters',
|
|
917
981
|
cause: e,
|
|
918
982
|
});
|
|
919
983
|
}
|
|
920
984
|
else {
|
|
921
|
-
this.logger.error('Failed to get item
|
|
922
|
-
tableKey,
|
|
923
|
-
primaryKey,
|
|
924
|
-
opts,
|
|
925
|
-
});
|
|
985
|
+
this.logger.error('Failed to get item', { errorName: e.name, tableKey });
|
|
926
986
|
throw new DynamoDBError(500, {
|
|
927
987
|
message: 'Failed to get item',
|
|
928
988
|
cause: e,
|
|
@@ -933,6 +993,7 @@ export default class DynamoDBService {
|
|
|
933
993
|
if (item && table.timeToLiveAttribute && item[table.timeToLiveAttribute] !== undefined) {
|
|
934
994
|
const now = getUnixTime(new Date());
|
|
935
995
|
if (item[table.timeToLiveAttribute] <= now) {
|
|
996
|
+
this.logger.debug('DynamoDB getOne completed', { tableKey, outcome: 'expired' });
|
|
936
997
|
return null;
|
|
937
998
|
}
|
|
938
999
|
// Delete the time to live attribute from the item if not requested
|
|
@@ -940,6 +1001,7 @@ export default class DynamoDBService {
|
|
|
940
1001
|
delete item[table.timeToLiveAttribute];
|
|
941
1002
|
}
|
|
942
1003
|
}
|
|
1004
|
+
this.logger.debug('DynamoDB getOne completed', { tableKey, outcome: item ? 'found' : 'not-found' });
|
|
943
1005
|
// Item will be undefined if not found
|
|
944
1006
|
// Convert keys back to camelCase if camelOrSnake is 'camel' or to snake_case if camelOrSnake is 'snake'
|
|
945
1007
|
if (item) {
|
|
@@ -973,7 +1035,6 @@ export default class DynamoDBService {
|
|
|
973
1035
|
const keys = primaryKeys.map((primaryKey, indx) => {
|
|
974
1036
|
const isArrayOfKeys = Array.isArray(primaryKey);
|
|
975
1037
|
const pkStr = serializeKey(primaryKey);
|
|
976
|
-
this.logger.debug('Primary Key: %s', pkStr);
|
|
977
1038
|
if (pkIndexMap.has(pkStr)) {
|
|
978
1039
|
throw new Error(`Duplicate primary key: ${pkStr}`);
|
|
979
1040
|
}
|
|
@@ -1022,13 +1083,12 @@ export default class DynamoDBService {
|
|
|
1022
1083
|
ExpressionAttributeNames[attributeNameVariable] = attributeName;
|
|
1023
1084
|
});
|
|
1024
1085
|
}
|
|
1025
|
-
this.logger.debug('Projection Expression: %s', ProjectionExpression);
|
|
1026
|
-
this.logger.debug('Expression Attribute Names: %o', ExpressionAttributeNames);
|
|
1027
1086
|
const results = new Array(primaryKeys.length).fill(null);
|
|
1028
1087
|
const now = getUnixTime(new Date());
|
|
1029
1088
|
let unprocessedAttempt = 0;
|
|
1030
1089
|
while (keys.length > 0) {
|
|
1031
1090
|
const batch = keys.splice(0, 100);
|
|
1091
|
+
this.logger.debug('DynamoDB batch request started', { operation: 'getMany', tableKey, itemCount: batch.length });
|
|
1032
1092
|
const batchGetCommand = new BatchGetCommand({
|
|
1033
1093
|
RequestItems: {
|
|
1034
1094
|
[table.name]: {
|
|
@@ -1042,19 +1102,23 @@ export default class DynamoDBService {
|
|
|
1042
1102
|
},
|
|
1043
1103
|
ReturnConsumedCapacity: opts.returnConsumedCapacity ?? ReturnConsumedCapacity.NONE,
|
|
1044
1104
|
});
|
|
1045
|
-
const { Responses: responses, UnprocessedKeys: rawUnprocessedKeys } = await this.ddbDocClient.send(batchGetCommand)
|
|
1105
|
+
const { Responses: responses, UnprocessedKeys: rawUnprocessedKeys } = await this.ddbDocClient.send(batchGetCommand).catch((error) => {
|
|
1106
|
+
this.logger.error('Failed to get items', { tableKey, errorName: error.name });
|
|
1107
|
+
throw error;
|
|
1108
|
+
});
|
|
1046
1109
|
const items = responses && responses[table.name] ? responses[table.name] : [];
|
|
1047
1110
|
const unprocessedKeys = (rawUnprocessedKeys && rawUnprocessedKeys[table.name]?.Keys) || [];
|
|
1048
|
-
this.logger.debug(`GetMany Responsed Items: %o`, items);
|
|
1049
|
-
this.logger.debug('Now: %d', now);
|
|
1050
1111
|
if (unprocessedKeys.length > 0) {
|
|
1051
1112
|
if (unprocessedAttempt >= MAX_UNPROCESSED_ATTEMPTS) {
|
|
1113
|
+
this.logger.error('DynamoDB batch retry limit reached', { operation: 'getMany', tableKey, retryCount: unprocessedAttempt });
|
|
1052
1114
|
throw new DynamoDBError(500, {
|
|
1053
1115
|
message: 'Failed to retrieve items after multiple attempts',
|
|
1054
1116
|
cause: new Error('Too many unprocessed items'),
|
|
1055
1117
|
});
|
|
1056
1118
|
}
|
|
1057
|
-
|
|
1119
|
+
const delayMs = backoffDelayMs(unprocessedAttempt);
|
|
1120
|
+
this.logger.warn('DynamoDB retrying unprocessed batch items', { operation: 'getMany', tableKey, itemCount: unprocessedKeys.length, attempt: unprocessedAttempt + 1, delayMs });
|
|
1121
|
+
await sleep(delayMs);
|
|
1058
1122
|
unprocessedAttempt += 1;
|
|
1059
1123
|
}
|
|
1060
1124
|
else {
|
|
@@ -1077,7 +1141,6 @@ export default class DynamoDBService {
|
|
|
1077
1141
|
const pkStr = serializeKey(table.sortKey
|
|
1078
1142
|
? [item[table.partitionKey], item[table.sortKey]]
|
|
1079
1143
|
: item[table.partitionKey]);
|
|
1080
|
-
this.logger.debug('Primary Key: %s', pkStr);
|
|
1081
1144
|
// Delete the primary key values from the item if not requested
|
|
1082
1145
|
if (optIncludePartitionKey === false) {
|
|
1083
1146
|
delete item[table.partitionKey];
|
|
@@ -1104,6 +1167,7 @@ export default class DynamoDBService {
|
|
|
1104
1167
|
}
|
|
1105
1168
|
});
|
|
1106
1169
|
}
|
|
1170
|
+
this.logger.debug('DynamoDB getMany completed', { tableKey, requestedCount: primaryKeys.length, returnedCount: results.filter(item => item !== null).length });
|
|
1107
1171
|
return results;
|
|
1108
1172
|
}
|
|
1109
1173
|
async find(tableKey, keyCondition, filter = null, opts = {}, camelOrSnake = 'camel') {
|
|
@@ -1266,7 +1330,6 @@ export default class DynamoDBService {
|
|
|
1266
1330
|
indexName: indexToScan.indexName,
|
|
1267
1331
|
projectionType: indexToScan
|
|
1268
1332
|
.projectionType,
|
|
1269
|
-
projectionFields,
|
|
1270
1333
|
});
|
|
1271
1334
|
}
|
|
1272
1335
|
break;
|
|
@@ -1296,12 +1359,11 @@ export default class DynamoDBService {
|
|
|
1296
1359
|
]).length > 0;
|
|
1297
1360
|
}
|
|
1298
1361
|
if (hasFieldNotInsideGSIProjection) {
|
|
1299
|
-
this.logger.error('Some of the projection fields are not included in the GSI projection,
|
|
1362
|
+
this.logger.error('Some of the projection fields are not included in the GSI projection, so the query cannot be executed', {
|
|
1300
1363
|
tableKey,
|
|
1301
1364
|
indexName: indexToScan.indexName,
|
|
1302
1365
|
projectionType: indexToScan
|
|
1303
1366
|
.projectionType,
|
|
1304
|
-
projectionFields,
|
|
1305
1367
|
});
|
|
1306
1368
|
throw new Error('All projection fields must be included in the GSI projection');
|
|
1307
1369
|
}
|
|
@@ -1358,11 +1420,6 @@ export default class DynamoDBService {
|
|
|
1358
1420
|
}
|
|
1359
1421
|
ExpressionAttributeValues[attributeValueVariable] = value;
|
|
1360
1422
|
});
|
|
1361
|
-
this.logger.debug('Projection Expression: %s', ProjectionExpression);
|
|
1362
|
-
this.logger.debug('Expression Attribute Names: %o', ExpressionAttributeNames);
|
|
1363
|
-
this.logger.debug('Expression Attribute Values: %o', ExpressionAttributeValues);
|
|
1364
|
-
this.logger.debug('Key Condition Expression: %s', KeyConditionExpression);
|
|
1365
|
-
this.logger.debug('Filter Expression: %s', FilterExpression);
|
|
1366
1423
|
const numOfNeededItems = opts.limit; // May be undefined then no limit
|
|
1367
1424
|
const result = {
|
|
1368
1425
|
items: [],
|
|
@@ -1424,32 +1481,24 @@ export default class DynamoDBService {
|
|
|
1424
1481
|
: {}),
|
|
1425
1482
|
};
|
|
1426
1483
|
while (true) {
|
|
1484
|
+
this.logger.debug('DynamoDB query page requested', { tableKey, hasCursor: !!params.ExclusiveStartKey, hasFilter: !!params.FilterExpression });
|
|
1427
1485
|
const { Items, LastEvaluatedKey } = await this.ddbDocClient
|
|
1428
1486
|
.send(new QueryCommand(params))
|
|
1429
1487
|
.catch((e) => {
|
|
1430
1488
|
if (e.name === 'ValidationException') {
|
|
1431
|
-
this.logger.error('The request parameters are invalid
|
|
1432
|
-
tableKey,
|
|
1433
|
-
keyCondition,
|
|
1434
|
-
filter,
|
|
1435
|
-
opts,
|
|
1436
|
-
});
|
|
1489
|
+
this.logger.error('The request parameters are invalid', { errorName: e.name, tableKey });
|
|
1437
1490
|
throw new DynamoDBError(400, {
|
|
1438
1491
|
message: 'Invalid request parameters',
|
|
1439
1492
|
cause: e,
|
|
1440
1493
|
});
|
|
1441
1494
|
}
|
|
1442
|
-
this.logger.error('Failed to query items
|
|
1443
|
-
tableKey,
|
|
1444
|
-
keyCondition,
|
|
1445
|
-
filter,
|
|
1446
|
-
opts,
|
|
1447
|
-
});
|
|
1495
|
+
this.logger.error('Failed to query items', { errorName: e.name, tableKey });
|
|
1448
1496
|
throw new DynamoDBError(500, {
|
|
1449
1497
|
message: 'Failed to query items',
|
|
1450
1498
|
cause: e,
|
|
1451
1499
|
});
|
|
1452
1500
|
});
|
|
1501
|
+
this.logger.debug('DynamoDB query page received', { tableKey, itemCount: Items?.length ?? 0, hasMore: !!LastEvaluatedKey });
|
|
1453
1502
|
if (LastEvaluatedKey) {
|
|
1454
1503
|
params.ExclusiveStartKey = LastEvaluatedKey;
|
|
1455
1504
|
}
|
|
@@ -1594,6 +1643,7 @@ export default class DynamoDBService {
|
|
|
1594
1643
|
ExpressionAttributeNames[key] = value;
|
|
1595
1644
|
});
|
|
1596
1645
|
}
|
|
1646
|
+
this.logger.debug('DynamoDB request started', { operation: 'createOne', tableKey });
|
|
1597
1647
|
const putCommand = new PutCommand({
|
|
1598
1648
|
TableName: table.name,
|
|
1599
1649
|
Item: normalizedItem,
|
|
@@ -1646,22 +1696,18 @@ export default class DynamoDBService {
|
|
|
1646
1696
|
const error = e;
|
|
1647
1697
|
if (error.name === 'ConditionalCheckFailedException' &&
|
|
1648
1698
|
opts.onConflict === 'ignore') {
|
|
1649
|
-
this.logger.debug('
|
|
1650
|
-
(table.sortKey ? `:${normalizedItem[table.sortKey]}` : ''));
|
|
1699
|
+
this.logger.debug('DynamoDB create skipped: item already exists', { tableKey });
|
|
1651
1700
|
result.created = false;
|
|
1652
1701
|
}
|
|
1653
1702
|
else {
|
|
1654
|
-
this.logger.error('Failed to create item
|
|
1655
|
-
tableKey,
|
|
1656
|
-
item: normalizedItem,
|
|
1657
|
-
opts,
|
|
1658
|
-
});
|
|
1703
|
+
this.logger.error('Failed to create item', { errorName: error.name, tableKey });
|
|
1659
1704
|
throw new DynamoDBError(500, {
|
|
1660
1705
|
message: 'Failed to create item',
|
|
1661
1706
|
cause: error,
|
|
1662
1707
|
});
|
|
1663
1708
|
}
|
|
1664
1709
|
}
|
|
1710
|
+
this.logger.debug('DynamoDB createOne completed', { tableKey, created: result.created });
|
|
1665
1711
|
return result;
|
|
1666
1712
|
}
|
|
1667
1713
|
async createMany(tableKey, primaryKeys, items, opts = {}) {
|
|
@@ -1706,6 +1752,7 @@ export default class DynamoDBService {
|
|
|
1706
1752
|
let unprocessedAttempt = 0;
|
|
1707
1753
|
while (normalizedItems.length > 0) {
|
|
1708
1754
|
const batch = normalizedItems.splice(0, 25);
|
|
1755
|
+
this.logger.debug('DynamoDB batch request started', { operation: 'createMany', tableKey, itemCount: batch.length });
|
|
1709
1756
|
const batchWriteCommand = new BatchWriteCommand({
|
|
1710
1757
|
RequestItems: {
|
|
1711
1758
|
[table.name]: batch.map(item => ({
|
|
@@ -1719,11 +1766,7 @@ export default class DynamoDBService {
|
|
|
1719
1766
|
const { UnprocessedItems } = await this.ddbDocClient
|
|
1720
1767
|
.send(batchWriteCommand)
|
|
1721
1768
|
.catch((e) => {
|
|
1722
|
-
this.logger.error('Failed to create items
|
|
1723
|
-
tableKey,
|
|
1724
|
-
items: batch,
|
|
1725
|
-
opts,
|
|
1726
|
-
});
|
|
1769
|
+
this.logger.error('Failed to create items', { errorName: e.name, tableKey });
|
|
1727
1770
|
throw new DynamoDBError(500, {
|
|
1728
1771
|
message: 'Failed to create items',
|
|
1729
1772
|
cause: e,
|
|
@@ -1732,12 +1775,15 @@ export default class DynamoDBService {
|
|
|
1732
1775
|
const unprocessedItems = UnprocessedItems?.[table.name];
|
|
1733
1776
|
if (unprocessedItems !== undefined && unprocessedItems.length > 0) {
|
|
1734
1777
|
if (unprocessedAttempt >= MAX_UNPROCESSED_ATTEMPTS) {
|
|
1778
|
+
this.logger.error('DynamoDB batch retry limit reached', { operation: 'createMany', tableKey, retryCount: unprocessedAttempt });
|
|
1735
1779
|
throw new DynamoDBError(500, {
|
|
1736
1780
|
message: 'Failed to create items after multiple attempts',
|
|
1737
1781
|
cause: new Error('Too many unprocessed items'),
|
|
1738
1782
|
});
|
|
1739
1783
|
}
|
|
1740
|
-
|
|
1784
|
+
const delayMs = backoffDelayMs(unprocessedAttempt);
|
|
1785
|
+
this.logger.warn('DynamoDB retrying unprocessed batch items', { operation: 'createMany', tableKey, itemCount: unprocessedItems.length, attempt: unprocessedAttempt + 1, delayMs });
|
|
1786
|
+
await sleep(delayMs);
|
|
1741
1787
|
unprocessedAttempt += 1;
|
|
1742
1788
|
normalizedItems.push(...unprocessedItems
|
|
1743
1789
|
.map(item => item.PutRequest?.Item)
|
|
@@ -1747,6 +1793,7 @@ export default class DynamoDBService {
|
|
|
1747
1793
|
unprocessedAttempt = 0;
|
|
1748
1794
|
}
|
|
1749
1795
|
}
|
|
1796
|
+
this.logger.debug('DynamoDB createMany completed', { tableKey, itemCount: result.numberOfItems });
|
|
1750
1797
|
return result;
|
|
1751
1798
|
}
|
|
1752
1799
|
async update(tableKey, primaryKey, commands, condition = null, opts = {}, camelOrSnake = 'camel') {
|
|
@@ -1822,10 +1869,7 @@ export default class DynamoDBService {
|
|
|
1822
1869
|
}
|
|
1823
1870
|
ExpressionAttributeValues[attributeValueVariable] = value;
|
|
1824
1871
|
});
|
|
1825
|
-
this.logger.debug('
|
|
1826
|
-
this.logger.debug('Condition Expression: %s', ConditionExpression);
|
|
1827
|
-
this.logger.debug('Expression Attribute Names: %o', ExpressionAttributeNames);
|
|
1828
|
-
this.logger.debug('Expression Attribute Values: %o', ExpressionAttributeValues);
|
|
1872
|
+
this.logger.debug('DynamoDB request started', { operation: 'update', tableKey });
|
|
1829
1873
|
const updateCommand = new UpdateCommand({
|
|
1830
1874
|
TableName: table.name,
|
|
1831
1875
|
Key: keys,
|
|
@@ -1859,29 +1903,17 @@ export default class DynamoDBService {
|
|
|
1859
1903
|
catch (e) {
|
|
1860
1904
|
const error = e;
|
|
1861
1905
|
if (error.name === 'ValidationException') {
|
|
1862
|
-
this.logger.error('The request parameters are invalid
|
|
1863
|
-
tableKey,
|
|
1864
|
-
primaryKey,
|
|
1865
|
-
commands,
|
|
1866
|
-
condition,
|
|
1867
|
-
opts,
|
|
1868
|
-
});
|
|
1906
|
+
this.logger.error('The request parameters are invalid', { errorName: error.name, tableKey });
|
|
1869
1907
|
throw new DynamoDBError(400, {
|
|
1870
1908
|
message: 'Invalid request parameters',
|
|
1871
1909
|
cause: e,
|
|
1872
1910
|
});
|
|
1873
1911
|
}
|
|
1874
1912
|
else if (error.name === 'ConditionalCheckFailedException') {
|
|
1875
|
-
this.logger.debug('
|
|
1913
|
+
this.logger.debug('DynamoDB update skipped: condition not met', { tableKey });
|
|
1876
1914
|
}
|
|
1877
1915
|
else {
|
|
1878
|
-
this.logger.error('Failed to update item
|
|
1879
|
-
tableKey,
|
|
1880
|
-
primaryKey,
|
|
1881
|
-
commands,
|
|
1882
|
-
condition,
|
|
1883
|
-
opts,
|
|
1884
|
-
});
|
|
1916
|
+
this.logger.error('Failed to update item', { errorName: error.name, tableKey });
|
|
1885
1917
|
throw new DynamoDBError(500, {
|
|
1886
1918
|
message: 'Failed to update item',
|
|
1887
1919
|
cause: e,
|
|
@@ -1907,6 +1939,7 @@ export default class DynamoDBService {
|
|
|
1907
1939
|
result.item = newItem;
|
|
1908
1940
|
}
|
|
1909
1941
|
}
|
|
1942
|
+
this.logger.debug('DynamoDB update completed', { tableKey, updatedOrCreated: result.updatedOrCreated });
|
|
1910
1943
|
return result;
|
|
1911
1944
|
}
|
|
1912
1945
|
async deleteOne(tableKey, primaryKey, condition = null, opts = {}, camelOrSnake = 'camel') {
|
|
@@ -1942,6 +1975,7 @@ export default class DynamoDBService {
|
|
|
1942
1975
|
}
|
|
1943
1976
|
ConditionExpression = condition.expression;
|
|
1944
1977
|
}
|
|
1978
|
+
this.logger.debug('DynamoDB request started', { operation: 'deleteOne', tableKey });
|
|
1945
1979
|
const deleteCommand = new DeleteCommand({
|
|
1946
1980
|
TableName: table.name,
|
|
1947
1981
|
Key: keys,
|
|
@@ -1969,27 +2003,17 @@ export default class DynamoDBService {
|
|
|
1969
2003
|
catch (e) {
|
|
1970
2004
|
const error = e;
|
|
1971
2005
|
if (error.name === 'ValidationException') {
|
|
1972
|
-
this.logger.error('The request parameters are invalid
|
|
1973
|
-
tableKey,
|
|
1974
|
-
primaryKey,
|
|
1975
|
-
condition,
|
|
1976
|
-
opts,
|
|
1977
|
-
});
|
|
2006
|
+
this.logger.error('The request parameters are invalid', { errorName: error.name, tableKey });
|
|
1978
2007
|
throw new DynamoDBError(400, {
|
|
1979
2008
|
message: 'Invalid request parameters',
|
|
1980
2009
|
cause: e,
|
|
1981
2010
|
});
|
|
1982
2011
|
}
|
|
1983
2012
|
else if (error.name === 'ConditionalCheckFailedException') {
|
|
1984
|
-
this.logger.debug('
|
|
2013
|
+
this.logger.debug('DynamoDB delete skipped: condition not met', { tableKey });
|
|
1985
2014
|
}
|
|
1986
2015
|
else {
|
|
1987
|
-
this.logger.error('Failed to delete item
|
|
1988
|
-
tableKey,
|
|
1989
|
-
primaryKey,
|
|
1990
|
-
condition,
|
|
1991
|
-
opts,
|
|
1992
|
-
});
|
|
2016
|
+
this.logger.error('Failed to delete item', { errorName: error.name, tableKey });
|
|
1993
2017
|
throw new DynamoDBError(500, {
|
|
1994
2018
|
message: 'Failed to delete item',
|
|
1995
2019
|
cause: e,
|
|
@@ -2015,6 +2039,7 @@ export default class DynamoDBService {
|
|
|
2015
2039
|
result.item = newItem;
|
|
2016
2040
|
}
|
|
2017
2041
|
}
|
|
2042
|
+
this.logger.debug('DynamoDB deleteOne completed', { tableKey, deleted: result.deleted });
|
|
2018
2043
|
return result;
|
|
2019
2044
|
}
|
|
2020
2045
|
async deleteMany(tableKey, primaryKeys, opts = {}) {
|
|
@@ -2039,6 +2064,7 @@ export default class DynamoDBService {
|
|
|
2039
2064
|
let unprocessedAttempt = 0;
|
|
2040
2065
|
while (keys.length > 0) {
|
|
2041
2066
|
const batch = keys.splice(0, 25);
|
|
2067
|
+
this.logger.debug('DynamoDB batch request started', { operation: 'deleteMany', tableKey, itemCount: batch.length });
|
|
2042
2068
|
const batchWriteCommand = new BatchWriteCommand({
|
|
2043
2069
|
RequestItems: {
|
|
2044
2070
|
[table.name]: batch.map(key => ({
|
|
@@ -2052,11 +2078,7 @@ export default class DynamoDBService {
|
|
|
2052
2078
|
const { UnprocessedItems } = await this.ddbDocClient
|
|
2053
2079
|
.send(batchWriteCommand)
|
|
2054
2080
|
.catch((e) => {
|
|
2055
|
-
this.logger.error('Failed to delete items
|
|
2056
|
-
tableKey,
|
|
2057
|
-
keys: batch,
|
|
2058
|
-
opts,
|
|
2059
|
-
});
|
|
2081
|
+
this.logger.error('Failed to delete items', { errorName: e.name, tableKey });
|
|
2060
2082
|
throw new DynamoDBError(500, {
|
|
2061
2083
|
message: 'Failed to delete items',
|
|
2062
2084
|
cause: e,
|
|
@@ -2065,12 +2087,15 @@ export default class DynamoDBService {
|
|
|
2065
2087
|
const unprocessedItems = UnprocessedItems?.[table.name];
|
|
2066
2088
|
if (unprocessedItems !== undefined && unprocessedItems.length > 0) {
|
|
2067
2089
|
if (unprocessedAttempt >= MAX_UNPROCESSED_ATTEMPTS) {
|
|
2090
|
+
this.logger.error('DynamoDB batch retry limit reached', { operation: 'deleteMany', tableKey, retryCount: unprocessedAttempt });
|
|
2068
2091
|
throw new DynamoDBError(500, {
|
|
2069
2092
|
message: 'Failed to delete items after multiple attempts',
|
|
2070
2093
|
cause: new Error('Too many unprocessed items'),
|
|
2071
2094
|
});
|
|
2072
2095
|
}
|
|
2073
|
-
|
|
2096
|
+
const delayMs = backoffDelayMs(unprocessedAttempt);
|
|
2097
|
+
this.logger.warn('DynamoDB retrying unprocessed batch items', { operation: 'deleteMany', tableKey, itemCount: unprocessedItems.length, attempt: unprocessedAttempt + 1, delayMs });
|
|
2098
|
+
await sleep(delayMs);
|
|
2074
2099
|
unprocessedAttempt += 1;
|
|
2075
2100
|
keys.push(...unprocessedItems
|
|
2076
2101
|
.map(item => item.DeleteRequest?.Key)
|
|
@@ -2080,6 +2105,7 @@ export default class DynamoDBService {
|
|
|
2080
2105
|
unprocessedAttempt = 0;
|
|
2081
2106
|
}
|
|
2082
2107
|
}
|
|
2108
|
+
this.logger.debug('DynamoDB deleteMany completed', { tableKey, itemCount: result.numberOfItems });
|
|
2083
2109
|
return result;
|
|
2084
2110
|
}
|
|
2085
2111
|
transaction(opts) {
|