@ftschopp/dynatable-core 1.4.3 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/dist/builders/batch-get/create-batch-get-builder.d.ts +8 -1
- package/dist/builders/batch-get/create-batch-get-builder.d.ts.map +1 -1
- package/dist/builders/batch-get/create-batch-get-builder.js +84 -34
- package/dist/builders/batch-get/types.d.ts +26 -3
- package/dist/builders/batch-get/types.d.ts.map +1 -1
- package/dist/builders/batch-write/create-batch-write-builder.d.ts +9 -1
- package/dist/builders/batch-write/create-batch-write-builder.d.ts.map +1 -1
- package/dist/builders/batch-write/create-batch-write-builder.js +43 -14
- package/dist/builders/batch-write/types.d.ts +31 -3
- package/dist/builders/batch-write/types.d.ts.map +1 -1
- package/dist/builders/shared/batch.d.ts +46 -0
- package/dist/builders/shared/batch.d.ts.map +1 -0
- package/dist/builders/shared/batch.js +101 -0
- package/dist/builders/shared/index.d.ts +1 -0
- package/dist/builders/shared/index.d.ts.map +1 -1
- package/dist/builders/shared/index.js +11 -0
- package/dist/builders/update/create-update-builder.d.ts.map +1 -1
- package/dist/builders/update/create-update-builder.js +25 -0
- package/dist/utils/model-utils.d.ts +9 -3
- package/dist/utils/model-utils.d.ts.map +1 -1
- package/dist/utils/model-utils.js +15 -7
- package/package.json +9 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,17 @@
|
|
|
1
|
+
# @ftschopp/dynatable-core [1.5.0](https://github.com/ftschopp/dynatable/compare/@ftschopp/dynatable-core@1.4.4...@ftschopp/dynatable-core@1.5.0) (2026-05-09)
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
### Features
|
|
5
|
+
|
|
6
|
+
* **core:** chunk and retry BatchGet/BatchWrite to handle DynamoDB limits ([#26](https://github.com/ftschopp/dynatable/issues/26)) ([5c7b1cc](https://github.com/ftschopp/dynatable/commit/5c7b1cc0044d0ae6a213735a40f20ef0232dbceb)), closes [#6](https://github.com/ftschopp/dynatable/issues/6)
|
|
7
|
+
|
|
8
|
+
## @ftschopp/dynatable-core [1.4.4](https://github.com/ftschopp/dynatable/compare/@ftschopp/dynatable-core@1.4.3...@ftschopp/dynatable-core@1.4.4) (2026-05-09)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
### Bug Fixes
|
|
12
|
+
|
|
13
|
+
* **core:** detect SET-collisions on auto-recomputed index keys; stripInternalKeys recurses ([#38](https://github.com/ftschopp/dynatable/issues/38)) ([d609c5b](https://github.com/ftschopp/dynatable/commit/d609c5bbeb566145ec034ec9883246a6d77d7cf6)), closes [#17](https://github.com/ftschopp/dynatable/issues/17) [#GSI1](https://github.com/ftschopp/dynatable/issues/GSI1) [#GSI1](https://github.com/ftschopp/dynatable/issues/GSI1)
|
|
14
|
+
|
|
1
15
|
## @ftschopp/dynatable-core [1.4.3](https://github.com/ftschopp/dynatable/compare/@ftschopp/dynatable-core@1.4.2...@ftschopp/dynatable-core@1.4.3) (2026-05-09)
|
|
2
16
|
|
|
3
17
|
|
|
@@ -4,9 +4,14 @@ import { DynamoDBLogger } from '../../utils/dynamodb-logger';
|
|
|
4
4
|
/**
|
|
5
5
|
* Creates a BatchGetBuilder to retrieve multiple items by their keys.
|
|
6
6
|
*
|
|
7
|
+
* `execute()` transparently chunks the request to at most 100 keys per
|
|
8
|
+
* SDK call (DynamoDB's `BatchGetItem` limit) and retries any
|
|
9
|
+
* UnprocessedKeys with exponential backoff. The `dbParams()` method
|
|
10
|
+
* still returns the un-chunked request — see its JSDoc.
|
|
11
|
+
*
|
|
7
12
|
* @param requestItems - An object where keys are table names and values are objects containing Keys array
|
|
8
13
|
* @param client - DynamoDB client instance
|
|
9
|
-
* @param options - Optional configuration (projection, consistentRead)
|
|
14
|
+
* @param options - Optional configuration (projection, consistentRead, maxRetries, retryBackoffMs)
|
|
10
15
|
* @param logger - Optional logger instance
|
|
11
16
|
*/
|
|
12
17
|
export declare function createBatchGetBuilder<Model>(requestItems: Record<string, {
|
|
@@ -14,5 +19,7 @@ export declare function createBatchGetBuilder<Model>(requestItems: Record<string
|
|
|
14
19
|
}>, client: DynamoDBClient, options?: {
|
|
15
20
|
projection?: (keyof Model)[];
|
|
16
21
|
consistentRead?: boolean;
|
|
22
|
+
maxRetries?: number;
|
|
23
|
+
retryBackoffMs?: number;
|
|
17
24
|
}, logger?: DynamoDBLogger): BatchGetBuilder<Model>;
|
|
18
25
|
//# sourceMappingURL=create-batch-get-builder.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"create-batch-get-builder.d.ts","sourceRoot":"","sources":["../../../src/builders/batch-get/create-batch-get-builder.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"create-batch-get-builder.d.ts","sourceRoot":"","sources":["../../../src/builders/batch-get/create-batch-get-builder.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAiB1D,OAAO,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAsC7D;;;;;;;;;;;;GAYG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EACzC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE;IAAE,IAAI,EAAE,GAAG,EAAE,CAAA;CAAE,CAAC,EAC7C,MAAM,EAAE,cAAc,EACtB,OAAO,CAAC,EAAE;IACR,UAAU,CAAC,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;IAC7B,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,EACD,MAAM,CAAC,EAAE,cAAc,GACtB,eAAe,CAAC,KAAK,CAAC,CAqIxB"}
|
|
@@ -3,70 +3,120 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.createBatchGetBuilder = createBatchGetBuilder;
|
|
4
4
|
const lib_dynamodb_1 = require("@aws-sdk/lib-dynamodb");
|
|
5
5
|
const shared_1 = require("../shared");
|
|
6
|
+
/**
|
|
7
|
+
* Builds the per-table request shape with projection placeholders applied.
|
|
8
|
+
*
|
|
9
|
+
* Resolves the same `#-placeholder` projection (from #5) for every table in
|
|
10
|
+
* the request — the projection is a property of the builder, not per-table.
|
|
11
|
+
*/
|
|
12
|
+
function buildEnhancedRequestItems(requestItems, projection, isConsistent) {
|
|
13
|
+
const proj = projection.length > 0 ? (0, shared_1.buildProjectionExpression)(projection) : undefined;
|
|
14
|
+
return Object.entries(requestItems).reduce((acc, [tableName, tableRequest]) => {
|
|
15
|
+
acc[tableName] = {
|
|
16
|
+
Keys: tableRequest.Keys,
|
|
17
|
+
...(proj && {
|
|
18
|
+
ProjectionExpression: proj.ProjectionExpression,
|
|
19
|
+
ExpressionAttributeNames: proj.ExpressionAttributeNames,
|
|
20
|
+
}),
|
|
21
|
+
...(isConsistent && { ConsistentRead: true }),
|
|
22
|
+
};
|
|
23
|
+
return acc;
|
|
24
|
+
}, {});
|
|
25
|
+
}
|
|
6
26
|
/**
|
|
7
27
|
* Creates a BatchGetBuilder to retrieve multiple items by their keys.
|
|
8
28
|
*
|
|
29
|
+
* `execute()` transparently chunks the request to at most 100 keys per
|
|
30
|
+
* SDK call (DynamoDB's `BatchGetItem` limit) and retries any
|
|
31
|
+
* UnprocessedKeys with exponential backoff. The `dbParams()` method
|
|
32
|
+
* still returns the un-chunked request — see its JSDoc.
|
|
33
|
+
*
|
|
9
34
|
* @param requestItems - An object where keys are table names and values are objects containing Keys array
|
|
10
35
|
* @param client - DynamoDB client instance
|
|
11
|
-
* @param options - Optional configuration (projection, consistentRead)
|
|
36
|
+
* @param options - Optional configuration (projection, consistentRead, maxRetries, retryBackoffMs)
|
|
12
37
|
* @param logger - Optional logger instance
|
|
13
38
|
*/
|
|
14
39
|
function createBatchGetBuilder(requestItems, client, options, logger) {
|
|
15
40
|
const projection = options?.projection ?? [];
|
|
16
41
|
const isConsistent = options?.consistentRead ?? false;
|
|
42
|
+
const maxRetries = options?.maxRetries ?? shared_1.DEFAULT_MAX_RETRIES;
|
|
43
|
+
const retryBackoff = options?.retryBackoffMs ?? shared_1.DEFAULT_RETRY_BACKOFF_MS;
|
|
17
44
|
return {
|
|
18
|
-
/**
|
|
19
|
-
* Select only specific attributes to retrieve.
|
|
20
|
-
*/
|
|
21
45
|
select(attrs) {
|
|
22
46
|
return createBatchGetBuilder(requestItems, client, {
|
|
23
47
|
projection: attrs,
|
|
24
48
|
consistentRead: isConsistent,
|
|
49
|
+
maxRetries,
|
|
50
|
+
retryBackoffMs: retryBackoff,
|
|
25
51
|
}, logger);
|
|
26
52
|
},
|
|
27
|
-
/**
|
|
28
|
-
* Enable strongly consistent read.
|
|
29
|
-
*/
|
|
30
53
|
consistentRead() {
|
|
31
54
|
return createBatchGetBuilder(requestItems, client, {
|
|
32
55
|
projection,
|
|
33
56
|
consistentRead: true,
|
|
57
|
+
maxRetries,
|
|
58
|
+
retryBackoffMs: retryBackoff,
|
|
59
|
+
}, logger);
|
|
60
|
+
},
|
|
61
|
+
maxRetries(n) {
|
|
62
|
+
return createBatchGetBuilder(requestItems, client, {
|
|
63
|
+
projection,
|
|
64
|
+
consistentRead: isConsistent,
|
|
65
|
+
maxRetries: n,
|
|
66
|
+
retryBackoffMs: retryBackoff,
|
|
67
|
+
}, logger);
|
|
68
|
+
},
|
|
69
|
+
retryBackoffMs(ms) {
|
|
70
|
+
return createBatchGetBuilder(requestItems, client, {
|
|
71
|
+
projection,
|
|
72
|
+
consistentRead: isConsistent,
|
|
73
|
+
maxRetries,
|
|
74
|
+
retryBackoffMs: ms,
|
|
34
75
|
}, logger);
|
|
35
76
|
},
|
|
36
|
-
/**
|
|
37
|
-
* Build the underlying DynamoDB input parameters.
|
|
38
|
-
*/
|
|
39
77
|
dbParams() {
|
|
40
|
-
// Build the projection once (with #-placeholders so reserved words
|
|
41
|
-
// like name/date/status/type don't blow up at DynamoDB) and reuse
|
|
42
|
-
// it per-table.
|
|
43
|
-
const proj = projection.length > 0 ? (0, shared_1.buildProjectionExpression)(projection.map(String)) : undefined;
|
|
44
|
-
const enhancedRequestItems = Object.entries(requestItems).reduce((acc, [tableName, tableRequest]) => {
|
|
45
|
-
acc[tableName] = {
|
|
46
|
-
Keys: tableRequest.Keys,
|
|
47
|
-
...(proj && {
|
|
48
|
-
ProjectionExpression: proj.ProjectionExpression,
|
|
49
|
-
ExpressionAttributeNames: proj.ExpressionAttributeNames,
|
|
50
|
-
}),
|
|
51
|
-
...(isConsistent && { ConsistentRead: true }),
|
|
52
|
-
};
|
|
53
|
-
return acc;
|
|
54
|
-
}, {});
|
|
55
78
|
return {
|
|
56
|
-
RequestItems:
|
|
79
|
+
RequestItems: buildEnhancedRequestItems(requestItems, projection.map(String), isConsistent),
|
|
57
80
|
};
|
|
58
81
|
},
|
|
59
|
-
/**
|
|
60
|
-
* Execute the BatchGetItem command and return the items as a flat array.
|
|
61
|
-
*/
|
|
62
82
|
async execute() {
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
83
|
+
// Chunk only the Keys (the projection / consistentRead options are
|
|
84
|
+
// re-applied per chunk).
|
|
85
|
+
const flatKeys = {};
|
|
86
|
+
for (const [tableName, tr] of Object.entries(requestItems)) {
|
|
87
|
+
flatKeys[tableName] = tr.Keys;
|
|
88
|
+
}
|
|
89
|
+
const chunks = (0, shared_1.chunkRequestItems)(flatKeys, shared_1.BATCH_GET_LIMIT);
|
|
90
|
+
if (chunks.length === 0)
|
|
67
91
|
return [];
|
|
92
|
+
const aggregated = [];
|
|
93
|
+
for (const chunk of chunks) {
|
|
94
|
+
const chunkRequest = {};
|
|
95
|
+
for (const [tableName, keys] of Object.entries(chunk)) {
|
|
96
|
+
chunkRequest[tableName] = { Keys: keys };
|
|
97
|
+
}
|
|
98
|
+
let pending = buildEnhancedRequestItems(chunkRequest, projection.map(String), isConsistent);
|
|
99
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
100
|
+
const params = { RequestItems: pending };
|
|
101
|
+
const result = await client.send(new lib_dynamodb_1.BatchGetCommand(params));
|
|
102
|
+
logger?.log('BatchGetCommand', params, result);
|
|
103
|
+
if (result.Responses) {
|
|
104
|
+
for (const items of Object.values(result.Responses)) {
|
|
105
|
+
aggregated.push(...items);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
const unprocessed = result.UnprocessedKeys;
|
|
109
|
+
if ((0, shared_1.isUnprocessedEmpty)(unprocessed)) {
|
|
110
|
+
break;
|
|
111
|
+
}
|
|
112
|
+
if (attempt === maxRetries) {
|
|
113
|
+
throw new shared_1.BatchUnprocessedError(`BatchGet still has unprocessed keys after ${maxRetries} retries.`, unprocessed);
|
|
114
|
+
}
|
|
115
|
+
await (0, shared_1.sleep)((0, shared_1.backoffMs)(attempt, retryBackoff));
|
|
116
|
+
pending = unprocessed;
|
|
117
|
+
}
|
|
68
118
|
}
|
|
69
|
-
return
|
|
119
|
+
return aggregated;
|
|
70
120
|
},
|
|
71
121
|
};
|
|
72
122
|
}
|
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
import { BatchGetItemCommandInput } from '@aws-sdk/client-dynamodb';
|
|
2
2
|
/**
|
|
3
|
-
* BatchGetBuilder allows you to retrieve multiple items from one or more
|
|
4
|
-
* in a single
|
|
3
|
+
* BatchGetBuilder allows you to retrieve multiple items from one or more
|
|
4
|
+
* tables in a single logical operation.
|
|
5
|
+
*
|
|
6
|
+
* `execute()` chunks the request into sub-requests of at most 100 keys
|
|
7
|
+
* (DynamoDB's hard BatchGetItem limit) and retries any UnprocessedKeys
|
|
8
|
+
* with exponential backoff. After the retry budget is exhausted, a
|
|
9
|
+
* `BatchUnprocessedError` is thrown carrying whatever could not be retrieved.
|
|
5
10
|
*/
|
|
6
11
|
export type BatchGetBuilder<Model> = {
|
|
7
12
|
/**
|
|
@@ -12,12 +17,30 @@ export type BatchGetBuilder<Model> = {
|
|
|
12
17
|
* Enable strongly consistent read for all items.
|
|
13
18
|
*/
|
|
14
19
|
consistentRead(): BatchGetBuilder<Model>;
|
|
20
|
+
/**
|
|
21
|
+
* Configure the maximum number of retry attempts per chunk when the
|
|
22
|
+
* response carries UnprocessedKeys. Defaults to 3.
|
|
23
|
+
*/
|
|
24
|
+
maxRetries(n: number): BatchGetBuilder<Model>;
|
|
25
|
+
/**
|
|
26
|
+
* Configure the initial exponential-backoff delay in milliseconds.
|
|
27
|
+
* Subsequent retries wait `initial * 2^attempt` ms. Defaults to 50ms.
|
|
28
|
+
*/
|
|
29
|
+
retryBackoffMs(ms: number): BatchGetBuilder<Model>;
|
|
15
30
|
/**
|
|
16
31
|
* Build the underlying DynamoDB BatchGetItem input parameters.
|
|
32
|
+
*
|
|
33
|
+
* **Note:** this returns the request as a single, unchunked
|
|
34
|
+
* `BatchGetItem` input. If you have more than 100 keys total, sending
|
|
35
|
+
* this directly to the SDK will fail at runtime — `execute()` is what
|
|
36
|
+
* applies the chunking and retry logic.
|
|
17
37
|
*/
|
|
18
38
|
dbParams(): BatchGetItemCommandInput;
|
|
19
39
|
/**
|
|
20
|
-
* Execute the BatchGetItem
|
|
40
|
+
* Execute the BatchGetItem operation and return the items as a flat
|
|
41
|
+
* array, aggregated across all chunks. Order across chunks is preserved
|
|
42
|
+
* but the order *within* a chunk is whatever DynamoDB returns
|
|
43
|
+
* (BatchGetItem does not guarantee item order).
|
|
21
44
|
*/
|
|
22
45
|
execute(): Promise<Model[]>;
|
|
23
46
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/builders/batch-get/types.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,wBAAwB,EAAE,MAAM,0BAA0B,CAAC;AAEpE
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/builders/batch-get/types.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,wBAAwB,EAAE,MAAM,0BAA0B,CAAC;AAEpE;;;;;;;;GAQG;AACH,MAAM,MAAM,eAAe,CAAC,KAAK,IAAI;IACnC;;OAEG;IACH,MAAM,CAAC,KAAK,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IAEvD;;OAEG;IACH,cAAc,IAAI,eAAe,CAAC,KAAK,CAAC,CAAC;IAEzC;;;OAGG;IACH,UAAU,CAAC,CAAC,EAAE,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IAE9C;;;OAGG;IACH,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IAEnD;;;;;;;OAOG;IACH,QAAQ,IAAI,wBAAwB,CAAC;IAErC;;;;;OAKG;IACH,OAAO,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;CAC7B,CAAC"}
|
|
@@ -4,9 +4,17 @@ import { DynamoDBLogger } from '../../utils/dynamodb-logger';
|
|
|
4
4
|
/**
|
|
5
5
|
* Creates a BatchWriteBuilder to put or delete multiple items.
|
|
6
6
|
*
|
|
7
|
+
* `execute()` transparently chunks `requestItems` into sub-requests of at
|
|
8
|
+
* most 25 items (DynamoDB's `BatchWriteItem` limit) and retries any
|
|
9
|
+
* UnprocessedItems with exponential backoff. The `dbParams()` method
|
|
10
|
+
* still returns the un-chunked request — see its JSDoc for details.
|
|
11
|
+
*
|
|
7
12
|
* @param requestItems - An object where keys are table names and values are arrays of WriteRequest
|
|
8
13
|
* @param client - DynamoDB client instance
|
|
9
14
|
* @param logger - Optional logger instance
|
|
10
15
|
*/
|
|
11
|
-
export declare function createBatchWriteBuilder(requestItems: Record<string, WriteRequest[]>, client: DynamoDBClient, logger?: DynamoDBLogger
|
|
16
|
+
export declare function createBatchWriteBuilder(requestItems: Record<string, WriteRequest[]>, client: DynamoDBClient, logger?: DynamoDBLogger, options?: {
|
|
17
|
+
maxRetries?: number;
|
|
18
|
+
retryBackoffMs?: number;
|
|
19
|
+
}): BatchWriteBuilder;
|
|
12
20
|
//# sourceMappingURL=create-batch-write-builder.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"create-batch-write-builder.d.ts","sourceRoot":"","sources":["../../../src/builders/batch-write/create-batch-write-builder.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"create-batch-write-builder.d.ts","sourceRoot":"","sources":["../../../src/builders/batch-write/create-batch-write-builder.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAgB1D,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAE7D;;;;;;;;;;;GAWG;AACH,wBAAgB,uBAAuB,CACrC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,EAAE,CAAC,EAC5C,MAAM,EAAE,cAAc,EACtB,MAAM,CAAC,EAAE,cAAc,EACvB,OAAO,CAAC,EAAE;IACR,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,GACA,iBAAiB,CAkEnB"}
|
|
@@ -2,34 +2,63 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.createBatchWriteBuilder = createBatchWriteBuilder;
|
|
4
4
|
const lib_dynamodb_1 = require("@aws-sdk/lib-dynamodb");
|
|
5
|
+
const shared_1 = require("../shared");
|
|
5
6
|
/**
|
|
6
7
|
* Creates a BatchWriteBuilder to put or delete multiple items.
|
|
7
8
|
*
|
|
9
|
+
* `execute()` transparently chunks `requestItems` into sub-requests of at
|
|
10
|
+
* most 25 items (DynamoDB's `BatchWriteItem` limit) and retries any
|
|
11
|
+
* UnprocessedItems with exponential backoff. The `dbParams()` method
|
|
12
|
+
* still returns the un-chunked request — see its JSDoc for details.
|
|
13
|
+
*
|
|
8
14
|
* @param requestItems - An object where keys are table names and values are arrays of WriteRequest
|
|
9
15
|
* @param client - DynamoDB client instance
|
|
10
16
|
* @param logger - Optional logger instance
|
|
11
17
|
*/
|
|
12
|
-
function createBatchWriteBuilder(requestItems, client, logger) {
|
|
18
|
+
function createBatchWriteBuilder(requestItems, client, logger, options) {
|
|
19
|
+
const maxRetries = options?.maxRetries ?? shared_1.DEFAULT_MAX_RETRIES;
|
|
20
|
+
const retryBackoff = options?.retryBackoffMs ?? shared_1.DEFAULT_RETRY_BACKOFF_MS;
|
|
13
21
|
return {
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
22
|
+
maxRetries(n) {
|
|
23
|
+
return createBatchWriteBuilder(requestItems, client, logger, {
|
|
24
|
+
maxRetries: n,
|
|
25
|
+
retryBackoffMs: retryBackoff,
|
|
26
|
+
});
|
|
27
|
+
},
|
|
28
|
+
retryBackoffMs(ms) {
|
|
29
|
+
return createBatchWriteBuilder(requestItems, client, logger, {
|
|
30
|
+
maxRetries,
|
|
31
|
+
retryBackoffMs: ms,
|
|
32
|
+
});
|
|
33
|
+
},
|
|
17
34
|
dbParams() {
|
|
18
35
|
return {
|
|
19
36
|
RequestItems: requestItems,
|
|
20
37
|
};
|
|
21
38
|
},
|
|
22
|
-
/**
|
|
23
|
-
* Execute the BatchWriteItem command.
|
|
24
|
-
*/
|
|
25
39
|
async execute() {
|
|
26
|
-
const
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
40
|
+
const chunks = (0, shared_1.chunkRequestItems)(requestItems, shared_1.BATCH_WRITE_LIMIT);
|
|
41
|
+
if (chunks.length === 0) {
|
|
42
|
+
return {};
|
|
43
|
+
}
|
|
44
|
+
for (const chunk of chunks) {
|
|
45
|
+
let pending = chunk;
|
|
46
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
47
|
+
const params = { RequestItems: pending };
|
|
48
|
+
const result = await client.send(new lib_dynamodb_1.BatchWriteCommand(params));
|
|
49
|
+
logger?.log('BatchWriteCommand', params, result);
|
|
50
|
+
const unprocessed = result.UnprocessedItems;
|
|
51
|
+
if ((0, shared_1.isUnprocessedEmpty)(unprocessed)) {
|
|
52
|
+
break; // chunk fully processed
|
|
53
|
+
}
|
|
54
|
+
if (attempt === maxRetries) {
|
|
55
|
+
throw new shared_1.BatchUnprocessedError(`BatchWrite still has unprocessed items after ${maxRetries} retries.`, unprocessed);
|
|
56
|
+
}
|
|
57
|
+
await (0, shared_1.sleep)((0, shared_1.backoffMs)(attempt, retryBackoff));
|
|
58
|
+
pending = unprocessed;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return {};
|
|
33
62
|
},
|
|
34
63
|
};
|
|
35
64
|
}
|
|
@@ -12,16 +12,44 @@ export type WriteRequest = {
|
|
|
12
12
|
};
|
|
13
13
|
/**
|
|
14
14
|
* BatchWriteBuilder allows you to put or delete multiple items from one or more tables
|
|
15
|
-
* in a single
|
|
15
|
+
* in a single logical operation.
|
|
16
|
+
*
|
|
17
|
+
* `execute()` chunks the request into sub-requests of at most 25 items
|
|
18
|
+
* (DynamoDB's hard BatchWriteItem limit) and retries any UnprocessedItems
|
|
19
|
+
* with exponential backoff. After the retry budget is exhausted, a
|
|
20
|
+
* `BatchUnprocessedError` is thrown carrying whatever could not be processed.
|
|
16
21
|
*/
|
|
17
22
|
export type BatchWriteBuilder = {
|
|
23
|
+
/**
|
|
24
|
+
* Configure the maximum number of retry attempts per chunk when the
|
|
25
|
+
* response carries UnprocessedItems. Defaults to 3.
|
|
26
|
+
*/
|
|
27
|
+
maxRetries(n: number): BatchWriteBuilder;
|
|
28
|
+
/**
|
|
29
|
+
* Configure the initial exponential-backoff delay in milliseconds.
|
|
30
|
+
* Subsequent retries wait `initial * 2^attempt` ms. Defaults to 50ms.
|
|
31
|
+
*/
|
|
32
|
+
retryBackoffMs(ms: number): BatchWriteBuilder;
|
|
18
33
|
/**
|
|
19
34
|
* Build the underlying DynamoDB BatchWriteItem input parameters.
|
|
35
|
+
*
|
|
36
|
+
* **Note:** this returns the request as a single, unchunked
|
|
37
|
+
* `BatchWriteItem` input. If you have more than 25 items total, sending
|
|
38
|
+
* this directly to the SDK will fail at runtime — `execute()` is what
|
|
39
|
+
* applies the chunking and retry logic.
|
|
20
40
|
*/
|
|
21
41
|
dbParams(): BatchWriteItemCommandInput;
|
|
22
42
|
/**
|
|
23
|
-
* Execute the BatchWriteItem
|
|
24
|
-
*
|
|
43
|
+
* Execute the BatchWriteItem operation.
|
|
44
|
+
*
|
|
45
|
+
* Internally chunks `RequestItems` to at most 25 items per SDK call and
|
|
46
|
+
* retries any UnprocessedItems with exponential backoff (default 3
|
|
47
|
+
* retries, configurable via {@link maxRetries}). Throws
|
|
48
|
+
* `BatchUnprocessedError` if items remain unprocessed after the budget.
|
|
49
|
+
*
|
|
50
|
+
* On success the resolved value is `{}` — there is nothing left to
|
|
51
|
+
* report. The legacy `unprocessedItems` field is no longer populated
|
|
52
|
+
* because partial success now surfaces as a thrown error.
|
|
25
53
|
*/
|
|
26
54
|
execute(): Promise<{
|
|
27
55
|
unprocessedItems?: Record<string, WriteRequest[]>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/builders/batch-write/types.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,0BAA0B,EAAE,MAAM,0BAA0B,CAAC;AAEtE;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB,UAAU,CAAC,EAAE;QACX,IAAI,EAAE,GAAG,CAAC;KACX,CAAC;IACF,aAAa,CAAC,EAAE;QACd,GAAG,EAAE,GAAG,CAAC;KACV,CAAC;CACH,CAAC;AAEF
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/builders/batch-write/types.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,0BAA0B,EAAE,MAAM,0BAA0B,CAAC;AAEtE;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB,UAAU,CAAC,EAAE;QACX,IAAI,EAAE,GAAG,CAAC;KACX,CAAC;IACF,aAAa,CAAC,EAAE;QACd,GAAG,EAAE,GAAG,CAAC;KACV,CAAC;CACH,CAAC;AAEF;;;;;;;;GAQG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;;OAGG;IACH,UAAU,CAAC,CAAC,EAAE,MAAM,GAAG,iBAAiB,CAAC;IAEzC;;;OAGG;IACH,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,iBAAiB,CAAC;IAE9C;;;;;;;OAOG;IACH,QAAQ,IAAI,0BAA0B,CAAC;IAEvC;;;;;;;;;;;OAWG;IACH,OAAO,IAAI,OAAO,CAAC;QACjB,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;KACnD,CAAC,CAAC;CACJ,CAAC"}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared helpers for BatchGet / BatchWrite chunking + retry.
|
|
3
|
+
*
|
|
4
|
+
* DynamoDB caps batch operations at 100 items per BatchGetItem and 25 items
|
|
5
|
+
* per BatchWriteItem, *across all tables combined*. The service can also
|
|
6
|
+
* return UnprocessedItems / UnprocessedKeys when throttled or when the 16MB
|
|
7
|
+
* response limit is hit. This module centralises the chunking, the
|
|
8
|
+
* exponential-backoff retry, and the typed error so the two builders behave
|
|
9
|
+
* the same way.
|
|
10
|
+
*/
|
|
11
|
+
export declare const BATCH_WRITE_LIMIT = 25;
|
|
12
|
+
export declare const BATCH_GET_LIMIT = 100;
|
|
13
|
+
export declare const DEFAULT_MAX_RETRIES = 3;
|
|
14
|
+
export declare const DEFAULT_RETRY_BACKOFF_MS = 50;
|
|
15
|
+
/**
|
|
16
|
+
* Thrown when a BatchGet / BatchWrite still has unprocessed items / keys
|
|
17
|
+
* after the configured number of retries.
|
|
18
|
+
*
|
|
19
|
+
* The `unprocessed` payload mirrors the shape DynamoDB itself uses for
|
|
20
|
+
* `UnprocessedItems` / `UnprocessedKeys`, so the caller can re-issue the
|
|
21
|
+
* request manually if they want.
|
|
22
|
+
*/
|
|
23
|
+
export declare class BatchUnprocessedError extends Error {
|
|
24
|
+
readonly unprocessed: Record<string, any>;
|
|
25
|
+
constructor(message: string, unprocessed: Record<string, any>);
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Splits a `(tableName -> items[])` map into chunks of at most `limit` items
|
|
29
|
+
* total across all tables. Each chunk is itself a `(tableName -> items[])` map,
|
|
30
|
+
* preserving the original per-table grouping.
|
|
31
|
+
*
|
|
32
|
+
* Items are emitted in iteration order: every item of the first table, then
|
|
33
|
+
* the second, etc. The relative order within a table is preserved.
|
|
34
|
+
*/
|
|
35
|
+
export declare function chunkRequestItems<T>(requestItems: Record<string, T[]>, limit: number): Record<string, T[]>[];
|
|
36
|
+
/**
|
|
37
|
+
* Returns true when the SDK's UnprocessedItems / UnprocessedKeys map is
|
|
38
|
+
* empty (either undefined, no keys, or every per-table array empty).
|
|
39
|
+
*/
|
|
40
|
+
export declare function isUnprocessedEmpty(unprocessed: Record<string, any> | undefined): boolean;
|
|
41
|
+
export declare function sleep(ms: number): Promise<void>;
|
|
42
|
+
/**
|
|
43
|
+
* Backoff for retry attempt N (0-indexed): `initial * 2^N` ms.
|
|
44
|
+
*/
|
|
45
|
+
export declare function backoffMs(attempt: number, initial: number): number;
|
|
46
|
+
//# sourceMappingURL=batch.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"batch.d.ts","sourceRoot":"","sources":["../../../src/builders/shared/batch.ts"],"names":[],"mappings":"AACA;;;;;;;;;GASG;AAEH,eAAO,MAAM,iBAAiB,KAAK,CAAC;AACpC,eAAO,MAAM,eAAe,MAAM,CAAC;AACnC,eAAO,MAAM,mBAAmB,IAAI,CAAC;AACrC,eAAO,MAAM,wBAAwB,KAAK,CAAC;AAE3C;;;;;;;GAOG;AACH,qBAAa,qBAAsB,SAAQ,KAAK;IAC9C,SAAgB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBAErC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;CAS9D;AAED;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAAC,CAAC,EACjC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,EACjC,KAAK,EAAE,MAAM,GACZ,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,CAoBvB;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS,GAAG,OAAO,CAUxF;AAED,wBAAgB,KAAK,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE/C;AAED;;GAEG;AACH,wBAAgB,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAElE"}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
3
|
+
/**
|
|
4
|
+
* Shared helpers for BatchGet / BatchWrite chunking + retry.
|
|
5
|
+
*
|
|
6
|
+
* DynamoDB caps batch operations at 100 items per BatchGetItem and 25 items
|
|
7
|
+
* per BatchWriteItem, *across all tables combined*. The service can also
|
|
8
|
+
* return UnprocessedItems / UnprocessedKeys when throttled or when the 16MB
|
|
9
|
+
* response limit is hit. This module centralises the chunking, the
|
|
10
|
+
* exponential-backoff retry, and the typed error so the two builders behave
|
|
11
|
+
* the same way.
|
|
12
|
+
*/
|
|
13
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
|
+
exports.BatchUnprocessedError = exports.DEFAULT_RETRY_BACKOFF_MS = exports.DEFAULT_MAX_RETRIES = exports.BATCH_GET_LIMIT = exports.BATCH_WRITE_LIMIT = void 0;
|
|
15
|
+
exports.chunkRequestItems = chunkRequestItems;
|
|
16
|
+
exports.isUnprocessedEmpty = isUnprocessedEmpty;
|
|
17
|
+
exports.sleep = sleep;
|
|
18
|
+
exports.backoffMs = backoffMs;
|
|
19
|
+
exports.BATCH_WRITE_LIMIT = 25;
|
|
20
|
+
exports.BATCH_GET_LIMIT = 100;
|
|
21
|
+
exports.DEFAULT_MAX_RETRIES = 3;
|
|
22
|
+
exports.DEFAULT_RETRY_BACKOFF_MS = 50;
|
|
23
|
+
/**
|
|
24
|
+
* Thrown when a BatchGet / BatchWrite still has unprocessed items / keys
|
|
25
|
+
* after the configured number of retries.
|
|
26
|
+
*
|
|
27
|
+
* The `unprocessed` payload mirrors the shape DynamoDB itself uses for
|
|
28
|
+
* `UnprocessedItems` / `UnprocessedKeys`, so the caller can re-issue the
|
|
29
|
+
* request manually if they want.
|
|
30
|
+
*/
|
|
31
|
+
class BatchUnprocessedError extends Error {
|
|
32
|
+
unprocessed;
|
|
33
|
+
constructor(message, unprocessed) {
|
|
34
|
+
super(message);
|
|
35
|
+
this.name = 'BatchUnprocessedError';
|
|
36
|
+
this.unprocessed = unprocessed;
|
|
37
|
+
// Preserve stack on V8.
|
|
38
|
+
if (typeof Error.captureStackTrace === 'function') {
|
|
39
|
+
Error.captureStackTrace(this, BatchUnprocessedError);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
exports.BatchUnprocessedError = BatchUnprocessedError;
|
|
44
|
+
/**
|
|
45
|
+
* Splits a `(tableName -> items[])` map into chunks of at most `limit` items
|
|
46
|
+
* total across all tables. Each chunk is itself a `(tableName -> items[])` map,
|
|
47
|
+
* preserving the original per-table grouping.
|
|
48
|
+
*
|
|
49
|
+
* Items are emitted in iteration order: every item of the first table, then
|
|
50
|
+
* the second, etc. The relative order within a table is preserved.
|
|
51
|
+
*/
|
|
52
|
+
function chunkRequestItems(requestItems, limit) {
|
|
53
|
+
const flat = [];
|
|
54
|
+
for (const [tableName, items] of Object.entries(requestItems)) {
|
|
55
|
+
for (const item of items) {
|
|
56
|
+
flat.push({ tableName, item });
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
if (flat.length === 0)
|
|
60
|
+
return [];
|
|
61
|
+
const chunks = [];
|
|
62
|
+
for (let i = 0; i < flat.length; i += limit) {
|
|
63
|
+
const slice = flat.slice(i, i + limit);
|
|
64
|
+
const grouped = {};
|
|
65
|
+
for (const { tableName, item } of slice) {
|
|
66
|
+
if (!grouped[tableName])
|
|
67
|
+
grouped[tableName] = [];
|
|
68
|
+
grouped[tableName].push(item);
|
|
69
|
+
}
|
|
70
|
+
chunks.push(grouped);
|
|
71
|
+
}
|
|
72
|
+
return chunks;
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Returns true when the SDK's UnprocessedItems / UnprocessedKeys map is
|
|
76
|
+
* empty (either undefined, no keys, or every per-table array empty).
|
|
77
|
+
*/
|
|
78
|
+
function isUnprocessedEmpty(unprocessed) {
|
|
79
|
+
if (!unprocessed)
|
|
80
|
+
return true;
|
|
81
|
+
const keys = Object.keys(unprocessed);
|
|
82
|
+
if (keys.length === 0)
|
|
83
|
+
return true;
|
|
84
|
+
return keys.every((k) => {
|
|
85
|
+
const v = unprocessed[k];
|
|
86
|
+
if (Array.isArray(v))
|
|
87
|
+
return v.length === 0;
|
|
88
|
+
if (v && Array.isArray(v.Keys))
|
|
89
|
+
return v.Keys.length === 0;
|
|
90
|
+
return false;
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
function sleep(ms) {
|
|
94
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Backoff for retry attempt N (0-indexed): `initial * 2^N` ms.
|
|
98
|
+
*/
|
|
99
|
+
function backoffMs(attempt, initial) {
|
|
100
|
+
return initial * 2 ** attempt;
|
|
101
|
+
}
|
|
@@ -2,4 +2,5 @@ export * from './types';
|
|
|
2
2
|
export * from './operators';
|
|
3
3
|
export * from './conditions';
|
|
4
4
|
export * from './projection';
|
|
5
|
+
export { BatchUnprocessedError, BATCH_WRITE_LIMIT, BATCH_GET_LIMIT, DEFAULT_MAX_RETRIES, DEFAULT_RETRY_BACKOFF_MS, chunkRequestItems, isUnprocessedEmpty, backoffMs, sleep, } from './batch';
|
|
5
6
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/builders/shared/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,aAAa,CAAC;AAC5B,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/builders/shared/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,aAAa,CAAC;AAC5B,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,OAAO,EACL,qBAAqB,EACrB,iBAAiB,EACjB,eAAe,EACf,mBAAmB,EACnB,wBAAwB,EACxB,iBAAiB,EACjB,kBAAkB,EAClB,SAAS,EACT,KAAK,GACN,MAAM,SAAS,CAAC"}
|
|
@@ -14,7 +14,18 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
14
14
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
exports.sleep = exports.backoffMs = exports.isUnprocessedEmpty = exports.chunkRequestItems = exports.DEFAULT_RETRY_BACKOFF_MS = exports.DEFAULT_MAX_RETRIES = exports.BATCH_GET_LIMIT = exports.BATCH_WRITE_LIMIT = exports.BatchUnprocessedError = void 0;
|
|
17
18
|
__exportStar(require("./types"), exports);
|
|
18
19
|
__exportStar(require("./operators"), exports);
|
|
19
20
|
__exportStar(require("./conditions"), exports);
|
|
20
21
|
__exportStar(require("./projection"), exports);
|
|
22
|
+
var batch_1 = require("./batch");
|
|
23
|
+
Object.defineProperty(exports, "BatchUnprocessedError", { enumerable: true, get: function () { return batch_1.BatchUnprocessedError; } });
|
|
24
|
+
Object.defineProperty(exports, "BATCH_WRITE_LIMIT", { enumerable: true, get: function () { return batch_1.BATCH_WRITE_LIMIT; } });
|
|
25
|
+
Object.defineProperty(exports, "BATCH_GET_LIMIT", { enumerable: true, get: function () { return batch_1.BATCH_GET_LIMIT; } });
|
|
26
|
+
Object.defineProperty(exports, "DEFAULT_MAX_RETRIES", { enumerable: true, get: function () { return batch_1.DEFAULT_MAX_RETRIES; } });
|
|
27
|
+
Object.defineProperty(exports, "DEFAULT_RETRY_BACKOFF_MS", { enumerable: true, get: function () { return batch_1.DEFAULT_RETRY_BACKOFF_MS; } });
|
|
28
|
+
Object.defineProperty(exports, "chunkRequestItems", { enumerable: true, get: function () { return batch_1.chunkRequestItems; } });
|
|
29
|
+
Object.defineProperty(exports, "isUnprocessedEmpty", { enumerable: true, get: function () { return batch_1.isUnprocessedEmpty; } });
|
|
30
|
+
Object.defineProperty(exports, "backoffMs", { enumerable: true, get: function () { return batch_1.backoffMs; } });
|
|
31
|
+
Object.defineProperty(exports, "sleep", { enumerable: true, get: function () { return batch_1.sleep; } });
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"create-update-builder.d.ts","sourceRoot":"","sources":["../../../src/builders/update/create-update-builder.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAE1D,OAAO,EAAgC,SAAS,EAA4B,MAAM,WAAW,CAAC;AAC9F,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACpE,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"create-update-builder.d.ts","sourceRoot":"","sources":["../../../src/builders/update/create-update-builder.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAE1D,OAAO,EAAgC,SAAS,EAA4B,MAAM,WAAW,CAAC;AAC9F,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACpE,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAa7D;;;;;;;;GAQG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EACvC,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,EACnB,MAAM,EAAE,cAAc,EACtB,cAAc,GAAE,SAAS,EAAO,EAChC,aAAa,GAAE;IACb,GAAG,EAAE,YAAY,EAAE,CAAC;IACpB,MAAM,EAAE,YAAY,EAAE,CAAC;IACvB,GAAG,EAAE,YAAY,EAAE,CAAC;IACpB,MAAM,EAAE,YAAY,EAAE,CAAC;CACuB,EAChD,UAAU,GAAE,MAAM,GAAG,SAAS,GAAG,SAAS,GAAG,aAAa,GAAG,aAAsB,EACnF,YAAY,SAAI,EAChB,gBAAgB,UAAQ,EACxB,MAAM,CAAC,EAAE,cAAc,EACvB,YAAY,CAAC,EAAE,YAAY,EAC3B,SAAS,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM,GAClC,aAAa,CAAC,KAAK,CAAC,CAsWtB"}
|
|
@@ -4,6 +4,15 @@ exports.createUpdateBuilder = createUpdateBuilder;
|
|
|
4
4
|
const lib_dynamodb_1 = require("@aws-sdk/lib-dynamodb");
|
|
5
5
|
const shared_1 = require("../shared");
|
|
6
6
|
const model_utils_1 = require("../../utils/model-utils");
|
|
7
|
+
/**
|
|
8
|
+
* Pull the LHS attribute name out of a SET-action expression like
|
|
9
|
+
* `#GSI1PK = :v_0` → `'GSI1PK'`. Returns `undefined` for shapes the
|
|
10
|
+
* builder doesn't emit so callers can ignore them safely.
|
|
11
|
+
*/
|
|
12
|
+
function extractAttrName(action) {
|
|
13
|
+
const m = action.expression.match(/^\s*#([A-Za-z0-9_]+)\s*=/);
|
|
14
|
+
return m?.[1];
|
|
15
|
+
}
|
|
7
16
|
/**
|
|
8
17
|
* Creates an UpdateBuilder for an item key and table.
|
|
9
18
|
*
|
|
@@ -120,6 +129,22 @@ function createUpdateBuilder(tableName, key, client, prevConditions = [], update
|
|
|
120
129
|
`but the templates cannot be fully resolved from the update payload. ` +
|
|
121
130
|
`Include the missing fields in .set():\n${details}`);
|
|
122
131
|
}
|
|
132
|
+
// If the user's `.set()` already targets the same index-key
|
|
133
|
+
// attribute (e.g. `.set('GSI1PK', 'foo')`), refuse to silently
|
|
134
|
+
// emit a second SET against the same path. DynamoDB rejects
|
|
135
|
+
// `SET #GSI1PK = :a, #GSI1PK = :b` outright, and namespacing the
|
|
136
|
+
// placeholders only hides which one wins. Force the caller to
|
|
137
|
+
// resolve the conflict explicitly.
|
|
138
|
+
const userSetKeys = new Set(actionsToProcess.set.map(extractAttrName));
|
|
139
|
+
userSetKeys.delete(undefined);
|
|
140
|
+
const conflicts = Object.keys(idxActions).filter((k) => userSetKeys.has(k));
|
|
141
|
+
if (conflicts.length > 0) {
|
|
142
|
+
throw new Error(`Update would write the same secondary-index key twice: ` +
|
|
143
|
+
`[${conflicts.join(', ')}] is recomputed from the index template AND ` +
|
|
144
|
+
`set explicitly via .set(). Either remove the explicit .set() and let ` +
|
|
145
|
+
`the recomputation handle it, or include all template variables in ` +
|
|
146
|
+
`.set() so you take full control.`);
|
|
147
|
+
}
|
|
123
148
|
for (const [indexName, resolved] of Object.entries(idxActions)) {
|
|
124
149
|
const valueName = getUniqueValueName(indexName);
|
|
125
150
|
actionsToProcess.set.push({
|
|
@@ -50,9 +50,15 @@ export declare const applyPostDefaults: <M extends ModelDefinition>(model: M, va
|
|
|
50
50
|
timestamps?: boolean;
|
|
51
51
|
}) => InferModel<M>;
|
|
52
52
|
/**
|
|
53
|
-
* Removes internal DynamoDB keys from an item or array of items
|
|
54
|
-
*
|
|
55
|
-
*
|
|
53
|
+
* Removes internal DynamoDB keys from an item or array of items.
|
|
54
|
+
*
|
|
55
|
+
* Recurses into nested plain objects and arrays so an `_type` / `PK` / `SK`
|
|
56
|
+
* field embedded inside an attribute value (e.g. a denormalized snapshot
|
|
57
|
+
* of another entity) is also stripped. Class instances such as `Date` are
|
|
58
|
+
* passed through untouched — recursing into them would lose their type.
|
|
59
|
+
*
|
|
60
|
+
* @param data - Single item, array of items, or any nested value from DynamoDB
|
|
61
|
+
* @returns Data with internal keys removed at every level
|
|
56
62
|
*/
|
|
57
63
|
export declare const stripInternalKeys: <T>(data: T | T[] | undefined) => T | T[] | undefined;
|
|
58
64
|
//# sourceMappingURL=model-utils.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"model-utils.d.ts","sourceRoot":"","sources":["../../src/utils/model-utils.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAiB,eAAe,EAAE,MAAM,cAAc,CAAC;AAG1E;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,GAAI,UAAU,MAAM,KAAG,MAAM,EAG5D,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,eAAe,GAAI,UAAU,MAAM,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAG,MAoB7E,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,WAAW,GAAI,CAAC,SAAS,eAAe,EACnD,OAAO,CAAC,EACR,OAAO,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC1B,OAAM,KAAK,GAAG,OAAO,GAAG,MAAc,KACrC,MAAM,CAAC,MAAM,EAAE,MAAM,CAWvB,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,MAAM,mBAAmB,GAAG;IAChC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAA;KAAE,EAAE,CAAC;CACnE,CAAC;AAEF;;;;;;;;;;GAUG;AACH,eAAO,MAAM,mBAAmB,GAAI,CAAC,SAAS,eAAe,EAC3D,OAAO,CAAC,EACR,SAAS,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC5B,SAAS,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAC3B,mBAwBF,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,iBAAiB,GAAI,CAAC,SAAS,eAAe,EACzD,OAAO,CAAC,EACR,eAAe,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAClC,UAAU;IAAE,QAAQ,CAAC,EAAE,OAAO,CAAC;IAAC,UAAU,CAAC,EAAE,OAAO,CAAA;CAAE,KACrD,UAAU,CAAC,CAAC,CAiCd,CAAC;AAOF
|
|
1
|
+
{"version":3,"file":"model-utils.d.ts","sourceRoot":"","sources":["../../src/utils/model-utils.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAiB,eAAe,EAAE,MAAM,cAAc,CAAC;AAG1E;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,GAAI,UAAU,MAAM,KAAG,MAAM,EAG5D,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,eAAe,GAAI,UAAU,MAAM,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAG,MAoB7E,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,WAAW,GAAI,CAAC,SAAS,eAAe,EACnD,OAAO,CAAC,EACR,OAAO,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC1B,OAAM,KAAK,GAAG,OAAO,GAAG,MAAc,KACrC,MAAM,CAAC,MAAM,EAAE,MAAM,CAWvB,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,MAAM,mBAAmB,GAAG;IAChC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAA;KAAE,EAAE,CAAC;CACnE,CAAC;AAEF;;;;;;;;;;GAUG;AACH,eAAO,MAAM,mBAAmB,GAAI,CAAC,SAAS,eAAe,EAC3D,OAAO,CAAC,EACR,SAAS,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC5B,SAAS,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAC3B,mBAwBF,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,iBAAiB,GAAI,CAAC,SAAS,eAAe,EACzD,OAAO,CAAC,EACR,eAAe,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAClC,UAAU;IAAE,QAAQ,CAAC,EAAE,OAAO,CAAC;IAAC,UAAU,CAAC,EAAE,OAAO,CAAA;CAAE,KACrD,UAAU,CAAC,CAAC,CAiCd,CAAC;AAOF;;;;;;;;;;GAUG;AACH,eAAO,MAAM,iBAAiB,GAAI,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,GAAG,SAAS,KAAG,CAAC,GAAG,CAAC,EAAE,GAAG,SAqB1E,CAAC"}
|
|
@@ -125,9 +125,15 @@ exports.applyPostDefaults = applyPostDefaults;
|
|
|
125
125
|
*/
|
|
126
126
|
const INTERNAL_KEYS = ['PK', 'SK', '_type'];
|
|
127
127
|
/**
|
|
128
|
-
* Removes internal DynamoDB keys from an item or array of items
|
|
129
|
-
*
|
|
130
|
-
*
|
|
128
|
+
* Removes internal DynamoDB keys from an item or array of items.
|
|
129
|
+
*
|
|
130
|
+
* Recurses into nested plain objects and arrays so an `_type` / `PK` / `SK`
|
|
131
|
+
* field embedded inside an attribute value (e.g. a denormalized snapshot
|
|
132
|
+
* of another entity) is also stripped. Class instances such as `Date` are
|
|
133
|
+
* passed through untouched — recursing into them would lose their type.
|
|
134
|
+
*
|
|
135
|
+
* @param data - Single item, array of items, or any nested value from DynamoDB
|
|
136
|
+
* @returns Data with internal keys removed at every level
|
|
131
137
|
*/
|
|
132
138
|
const stripInternalKeys = (data) => {
|
|
133
139
|
if (data === undefined || data === null) {
|
|
@@ -136,12 +142,14 @@ const stripInternalKeys = (data) => {
|
|
|
136
142
|
if (Array.isArray(data)) {
|
|
137
143
|
return data.map((item) => (0, exports.stripInternalKeys)(item));
|
|
138
144
|
}
|
|
139
|
-
|
|
145
|
+
// Only recurse into plain objects; preserve Date, Buffer, Set, Map,
|
|
146
|
+
// and anything else with a non-Object prototype.
|
|
147
|
+
if (typeof data === 'object' && Object.getPrototypeOf(data) === Object.prototype) {
|
|
140
148
|
const cleaned = {};
|
|
141
149
|
for (const [key, value] of Object.entries(data)) {
|
|
142
|
-
if (
|
|
143
|
-
|
|
144
|
-
|
|
150
|
+
if (INTERNAL_KEYS.includes(key))
|
|
151
|
+
continue;
|
|
152
|
+
cleaned[key] = (0, exports.stripInternalKeys)(value);
|
|
145
153
|
}
|
|
146
154
|
return cleaned;
|
|
147
155
|
}
|
package/package.json
CHANGED
|
@@ -1,9 +1,16 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ftschopp/dynatable-core",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.5.0",
|
|
4
4
|
"description": "Core library for DynamoDB single table design",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
7
|
+
"exports": {
|
|
8
|
+
".": {
|
|
9
|
+
"types": "./dist/index.d.ts",
|
|
10
|
+
"default": "./dist/index.js"
|
|
11
|
+
},
|
|
12
|
+
"./package.json": "./package.json"
|
|
13
|
+
},
|
|
7
14
|
"files": [
|
|
8
15
|
"dist",
|
|
9
16
|
"README.md",
|
|
@@ -15,6 +22,7 @@
|
|
|
15
22
|
"scripts": {
|
|
16
23
|
"build": "tsc && tsc-alias",
|
|
17
24
|
"clean": "rm -rf .turbo && rm -rf node_modules && rm -rf dist",
|
|
25
|
+
"lint": "eslint src",
|
|
18
26
|
"test": "jest",
|
|
19
27
|
"prepublishOnly": "yarn build"
|
|
20
28
|
},
|