@twin.org/entity-storage-connector-dynamodb 0.0.3-next.9 → 0.9.0-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -2
- package/dist/es/dynamoDbEntityStorageConnector.js +847 -96
- package/dist/es/dynamoDbEntityStorageConnector.js.map +1 -1
- package/dist/es/models/IDynamoDbEntityStorageConnectorConfig.js.map +1 -1
- package/dist/types/dynamoDbEntityStorageConnector.d.ts +63 -5
- package/dist/types/models/IDynamoDbEntityStorageConnectorConfig.d.ts +5 -0
- package/docs/changelog.md +646 -67
- package/docs/reference/classes/DynamoDbEntityStorageConnector.md +276 -12
- package/docs/reference/interfaces/IDynamoDbEntityStorageConnectorConfig.md +9 -0
- package/locales/en.json +17 -3
- package/package.json +12 -12
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
// Copyright 2024 IOTA Stiftung.
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0.
|
|
3
|
-
import { DynamoDB, QueryCommand, waitUntilTableExists } from "@aws-sdk/client-dynamodb";
|
|
4
|
-
import { DeleteCommand, DynamoDBDocumentClient, GetCommand, PutCommand } from "@aws-sdk/lib-dynamodb";
|
|
3
|
+
import { BatchWriteItemCommand, DynamoDB, QueryCommand, ScanCommand as RawScanCommand, waitUntilTableExists, waitUntilTableNotExists } from "@aws-sdk/client-dynamodb";
|
|
4
|
+
import { BatchWriteCommand, DeleteCommand, DynamoDBDocumentClient, GetCommand, PutCommand, ScanCommand } from "@aws-sdk/lib-dynamodb";
|
|
5
5
|
import { unmarshall } from "@aws-sdk/util-dynamodb";
|
|
6
6
|
import { ContextIdHelper, ContextIdStore } from "@twin.org/context";
|
|
7
|
-
import { BaseError, Coerce, ComponentFactory, Converter, GeneralError, Guards, Is, ObjectHelper } from "@twin.org/core";
|
|
7
|
+
import { BaseError, Coerce, ComponentFactory, Converter, GeneralError, Guards, HealthStatus, Is, ObjectHelper, Validation } from "@twin.org/core";
|
|
8
8
|
import { ComparisonOperator, EntitySchemaFactory, EntitySchemaHelper, LogicalOperator, SortDirection } from "@twin.org/entity";
|
|
9
|
+
import { EntityStorageHelper } from "@twin.org/entity-storage-models";
|
|
9
10
|
/**
|
|
10
11
|
* Class for performing entity storage operations using Dynamo DB.
|
|
11
12
|
*/
|
|
@@ -29,6 +30,11 @@ export class DynamoDbEntityStorageConnector {
|
|
|
29
30
|
* @internal
|
|
30
31
|
*/
|
|
31
32
|
static _PARTITION_KEY_VALUE = "root";
|
|
33
|
+
/**
|
|
34
|
+
* The name for the schema.
|
|
35
|
+
* @internal
|
|
36
|
+
*/
|
|
37
|
+
_entitySchemaName;
|
|
32
38
|
/**
|
|
33
39
|
* The schema for the entity.
|
|
34
40
|
* @internal
|
|
@@ -64,8 +70,9 @@ export class DynamoDbEntityStorageConnector {
|
|
|
64
70
|
}
|
|
65
71
|
Guards.stringValue(DynamoDbEntityStorageConnector.CLASS_NAME, "options.config.region", options.config.region);
|
|
66
72
|
Guards.stringValue(DynamoDbEntityStorageConnector.CLASS_NAME, "options.config.tableName", options.config.tableName);
|
|
67
|
-
this._entitySchema = EntitySchemaFactory.get(options.entitySchema);
|
|
68
73
|
this._partitionContextIds = options.partitionContextIds;
|
|
74
|
+
this._entitySchemaName = options.entitySchema;
|
|
75
|
+
this._entitySchema = EntitySchemaFactory.get(options.entitySchema);
|
|
69
76
|
this._primaryKey = EntitySchemaHelper.getPrimaryKey(this._entitySchema);
|
|
70
77
|
this._config = options.config;
|
|
71
78
|
this._config.endpoint = Is.stringValue(this._config.endpoint)
|
|
@@ -79,6 +86,35 @@ export class DynamoDbEntityStorageConnector {
|
|
|
79
86
|
className() {
|
|
80
87
|
return DynamoDbEntityStorageConnector.CLASS_NAME;
|
|
81
88
|
}
|
|
89
|
+
/**
|
|
90
|
+
* Returns the health status of the component.
|
|
91
|
+
* @returns The health status of the component.
|
|
92
|
+
*/
|
|
93
|
+
async health() {
|
|
94
|
+
try {
|
|
95
|
+
const dbConnection = this.createConnection();
|
|
96
|
+
await dbConnection.describeTable({ TableName: this._config.tableName });
|
|
97
|
+
return [
|
|
98
|
+
{
|
|
99
|
+
source: DynamoDbEntityStorageConnector.CLASS_NAME,
|
|
100
|
+
status: HealthStatus.Ok,
|
|
101
|
+
description: "healthDescription",
|
|
102
|
+
data: { tableName: this._config.tableName }
|
|
103
|
+
}
|
|
104
|
+
];
|
|
105
|
+
}
|
|
106
|
+
catch {
|
|
107
|
+
return [
|
|
108
|
+
{
|
|
109
|
+
source: DynamoDbEntityStorageConnector.CLASS_NAME,
|
|
110
|
+
status: HealthStatus.Error,
|
|
111
|
+
description: "healthDescription",
|
|
112
|
+
message: "connectionFailed",
|
|
113
|
+
data: { tableName: this._config.tableName }
|
|
114
|
+
}
|
|
115
|
+
];
|
|
116
|
+
}
|
|
117
|
+
}
|
|
82
118
|
/**
|
|
83
119
|
* Get the schema for the entities.
|
|
84
120
|
* @returns The schema for the entities.
|
|
@@ -174,7 +210,7 @@ export class DynamoDbEntityStorageConnector {
|
|
|
174
210
|
// Wait for table to exist
|
|
175
211
|
await waitUntilTableExists({
|
|
176
212
|
client: dbConnection,
|
|
177
|
-
maxWaitTime:
|
|
213
|
+
maxWaitTime: 60
|
|
178
214
|
}, {
|
|
179
215
|
TableName: this._config.tableName
|
|
180
216
|
});
|
|
@@ -250,8 +286,12 @@ export class DynamoDbEntityStorageConnector {
|
|
|
250
286
|
}
|
|
251
287
|
});
|
|
252
288
|
const response = await docClient.send(getCommand);
|
|
253
|
-
|
|
254
|
-
|
|
289
|
+
if (response.Item) {
|
|
290
|
+
return EntityStorageHelper.unPrepareEntity(response.Item, [
|
|
291
|
+
DynamoDbEntityStorageConnector._PARTITION_KEY
|
|
292
|
+
]);
|
|
293
|
+
}
|
|
294
|
+
return undefined;
|
|
255
295
|
}
|
|
256
296
|
const finalConditions = {
|
|
257
297
|
conditions: []
|
|
@@ -263,6 +303,13 @@ export class DynamoDbEntityStorageConnector {
|
|
|
263
303
|
value: id
|
|
264
304
|
});
|
|
265
305
|
}
|
|
306
|
+
else {
|
|
307
|
+
finalConditions.conditions.push({
|
|
308
|
+
property: this._primaryKey.property,
|
|
309
|
+
comparison: ComparisonOperator.Equals,
|
|
310
|
+
value: id
|
|
311
|
+
});
|
|
312
|
+
}
|
|
266
313
|
if (Is.arrayValue(conditions)) {
|
|
267
314
|
for (const c of conditions) {
|
|
268
315
|
finalConditions.conditions.push({
|
|
@@ -296,17 +343,21 @@ export class DynamoDbEntityStorageConnector {
|
|
|
296
343
|
Guards.object(DynamoDbEntityStorageConnector.CLASS_NAME, "entity", entity);
|
|
297
344
|
const contextIds = await ContextIdStore.getContextIds();
|
|
298
345
|
const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
|
|
299
|
-
|
|
300
|
-
|
|
346
|
+
const prepared = EntityStorageHelper.prepareEntity(entity, this._entitySchema, partitionKey
|
|
347
|
+
? [{ property: DynamoDbEntityStorageConnector._PARTITION_KEY, value: partitionKey }]
|
|
348
|
+
: [
|
|
349
|
+
{
|
|
350
|
+
property: DynamoDbEntityStorageConnector._PARTITION_KEY,
|
|
351
|
+
value: DynamoDbEntityStorageConnector._PARTITION_KEY_VALUE
|
|
352
|
+
}
|
|
353
|
+
], { nullBehavior: "omit" });
|
|
354
|
+
const id = prepared[this._primaryKey.property];
|
|
301
355
|
try {
|
|
302
356
|
const docClient = this.createDocClient();
|
|
303
357
|
const { conditionExpression, attributeNames, attributeValues } = this.buildConditionExpression(conditions);
|
|
304
358
|
const putCommand = new PutCommand({
|
|
305
359
|
TableName: this._config.tableName,
|
|
306
|
-
Item:
|
|
307
|
-
[DynamoDbEntityStorageConnector._PARTITION_KEY]: partitionKey ?? DynamoDbEntityStorageConnector._PARTITION_KEY_VALUE,
|
|
308
|
-
...entity
|
|
309
|
-
},
|
|
360
|
+
Item: prepared,
|
|
310
361
|
// Only set the condition expression if we have conditions to match
|
|
311
362
|
// and the primary key exists, otherwise we are creating a new object
|
|
312
363
|
ConditionExpression: Is.stringValue(conditionExpression)
|
|
@@ -331,6 +382,93 @@ export class DynamoDbEntityStorageConnector {
|
|
|
331
382
|
}, err);
|
|
332
383
|
}
|
|
333
384
|
}
|
|
385
|
+
/**
|
|
386
|
+
* Set multiple entities in a batch.
|
|
387
|
+
* @param entities The entities to set.
|
|
388
|
+
* @returns Nothing.
|
|
389
|
+
*/
|
|
390
|
+
async setBatch(entities) {
|
|
391
|
+
Guards.arrayValue(DynamoDbEntityStorageConnector.CLASS_NAME, "entities", entities);
|
|
392
|
+
const contextIds = await ContextIdStore.getContextIds();
|
|
393
|
+
const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
|
|
394
|
+
const preparedEntities = entities.map(entity => EntityStorageHelper.prepareEntity(entity, this._entitySchema, partitionKey
|
|
395
|
+
? [{ property: DynamoDbEntityStorageConnector._PARTITION_KEY, value: partitionKey }]
|
|
396
|
+
: [
|
|
397
|
+
{
|
|
398
|
+
property: DynamoDbEntityStorageConnector._PARTITION_KEY,
|
|
399
|
+
value: DynamoDbEntityStorageConnector._PARTITION_KEY_VALUE
|
|
400
|
+
}
|
|
401
|
+
], { nullBehavior: "omit" }));
|
|
402
|
+
try {
|
|
403
|
+
const docClient = this.createDocClient();
|
|
404
|
+
const chunkSize = 25;
|
|
405
|
+
for (let i = 0; i < preparedEntities.length; i += chunkSize) {
|
|
406
|
+
const chunk = preparedEntities.slice(i, i + chunkSize);
|
|
407
|
+
await docClient.send(new BatchWriteCommand({
|
|
408
|
+
RequestItems: {
|
|
409
|
+
[this._config.tableName]: chunk.map(entity => ({
|
|
410
|
+
PutRequest: {
|
|
411
|
+
Item: entity
|
|
412
|
+
}
|
|
413
|
+
}))
|
|
414
|
+
}
|
|
415
|
+
}));
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
catch (err) {
|
|
419
|
+
if (BaseError.isErrorCode(err, "ResourceNotFoundException")) {
|
|
420
|
+
throw new GeneralError(DynamoDbEntityStorageConnector.CLASS_NAME, "tableDoesNotExist", { tableName: this._config.tableName }, err);
|
|
421
|
+
}
|
|
422
|
+
throw new GeneralError(DynamoDbEntityStorageConnector.CLASS_NAME, "setBatchFailed", undefined, err);
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
/**
|
|
426
|
+
* Empty the entity storage.
|
|
427
|
+
* @returns Nothing.
|
|
428
|
+
*/
|
|
429
|
+
async empty() {
|
|
430
|
+
try {
|
|
431
|
+
const contextIds = await ContextIdStore.getContextIds();
|
|
432
|
+
const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
|
|
433
|
+
const pKey = partitionKey ?? DynamoDbEntityStorageConnector._PARTITION_KEY_VALUE;
|
|
434
|
+
const docClient = this.createDocClient();
|
|
435
|
+
const chunkSize = 25;
|
|
436
|
+
let exclusiveStartKey;
|
|
437
|
+
do {
|
|
438
|
+
const scanResult = await docClient.send(new ScanCommand({
|
|
439
|
+
TableName: this._config.tableName,
|
|
440
|
+
FilterExpression: "#partitionId = :partitionId",
|
|
441
|
+
ExpressionAttributeNames: {
|
|
442
|
+
"#partitionId": DynamoDbEntityStorageConnector._PARTITION_KEY
|
|
443
|
+
},
|
|
444
|
+
ExpressionAttributeValues: {
|
|
445
|
+
":partitionId": pKey
|
|
446
|
+
},
|
|
447
|
+
ExclusiveStartKey: exclusiveStartKey
|
|
448
|
+
}));
|
|
449
|
+
const items = scanResult.Items ?? [];
|
|
450
|
+
for (let i = 0; i < items.length; i += chunkSize) {
|
|
451
|
+
const chunk = items.slice(i, i + chunkSize);
|
|
452
|
+
await docClient.send(new BatchWriteCommand({
|
|
453
|
+
RequestItems: {
|
|
454
|
+
[this._config.tableName]: chunk.map((item) => ({
|
|
455
|
+
DeleteRequest: {
|
|
456
|
+
Key: {
|
|
457
|
+
[DynamoDbEntityStorageConnector._PARTITION_KEY]: item[DynamoDbEntityStorageConnector._PARTITION_KEY],
|
|
458
|
+
[this._primaryKey.property]: item[this._primaryKey.property]
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
}))
|
|
462
|
+
}
|
|
463
|
+
}));
|
|
464
|
+
}
|
|
465
|
+
exclusiveStartKey = scanResult.LastEvaluatedKey;
|
|
466
|
+
} while (exclusiveStartKey);
|
|
467
|
+
}
|
|
468
|
+
catch (err) {
|
|
469
|
+
throw new GeneralError(DynamoDbEntityStorageConnector.CLASS_NAME, "emptyFailed", undefined, err);
|
|
470
|
+
}
|
|
471
|
+
}
|
|
334
472
|
/**
|
|
335
473
|
* Remove the entity.
|
|
336
474
|
* @param id The id of the entity to remove.
|
|
@@ -370,6 +508,77 @@ export class DynamoDbEntityStorageConnector {
|
|
|
370
508
|
}, err);
|
|
371
509
|
}
|
|
372
510
|
}
|
|
511
|
+
/**
|
|
512
|
+
* Remove multiple entities by their IDs in a batch.
|
|
513
|
+
* @param ids The ids of the entities to remove.
|
|
514
|
+
* @returns Nothing.
|
|
515
|
+
*/
|
|
516
|
+
async removeBatch(ids) {
|
|
517
|
+
Guards.arrayValue(DynamoDbEntityStorageConnector.CLASS_NAME, "ids", ids);
|
|
518
|
+
const contextIds = await ContextIdStore.getContextIds();
|
|
519
|
+
const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
|
|
520
|
+
try {
|
|
521
|
+
const docClient = this.createDocClient();
|
|
522
|
+
const chunkSize = 25;
|
|
523
|
+
const primaryKeyProperty = this._primaryKey.property;
|
|
524
|
+
for (let i = 0; i < ids.length; i += chunkSize) {
|
|
525
|
+
const chunk = ids.slice(i, i + chunkSize);
|
|
526
|
+
await docClient.send(new BatchWriteCommand({
|
|
527
|
+
RequestItems: {
|
|
528
|
+
[this._config.tableName]: chunk.map(id => ({
|
|
529
|
+
DeleteRequest: {
|
|
530
|
+
Key: {
|
|
531
|
+
[primaryKeyProperty]: id,
|
|
532
|
+
[DynamoDbEntityStorageConnector._PARTITION_KEY]: partitionKey ?? DynamoDbEntityStorageConnector._PARTITION_KEY_VALUE
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
}))
|
|
536
|
+
}
|
|
537
|
+
}));
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
catch (err) {
|
|
541
|
+
throw new GeneralError(DynamoDbEntityStorageConnector.CLASS_NAME, "removeBatchFailed", undefined, err);
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
/**
|
|
545
|
+
* Teardown the entity storage by deleting the underlying table.
|
|
546
|
+
* @param nodeLoggingComponentType The node logging component type.
|
|
547
|
+
* @returns True if the teardown process was successful.
|
|
548
|
+
*/
|
|
549
|
+
async teardown(nodeLoggingComponentType) {
|
|
550
|
+
const nodeLogging = ComponentFactory.getIfExists(nodeLoggingComponentType);
|
|
551
|
+
await nodeLogging?.log({
|
|
552
|
+
level: "info",
|
|
553
|
+
source: DynamoDbEntityStorageConnector.CLASS_NAME,
|
|
554
|
+
ts: Date.now(),
|
|
555
|
+
message: "tableDeleting",
|
|
556
|
+
data: { tableName: this._config.tableName }
|
|
557
|
+
});
|
|
558
|
+
try {
|
|
559
|
+
const dbConnection = this.createConnection();
|
|
560
|
+
await dbConnection.deleteTable({ TableName: this._config.tableName });
|
|
561
|
+
await waitUntilTableNotExists({ client: dbConnection, maxWaitTime: 60 }, { TableName: this._config.tableName });
|
|
562
|
+
await nodeLogging?.log({
|
|
563
|
+
level: "info",
|
|
564
|
+
source: DynamoDbEntityStorageConnector.CLASS_NAME,
|
|
565
|
+
ts: Date.now(),
|
|
566
|
+
message: "tableDeleted",
|
|
567
|
+
data: { tableName: this._config.tableName }
|
|
568
|
+
});
|
|
569
|
+
return true;
|
|
570
|
+
}
|
|
571
|
+
catch (err) {
|
|
572
|
+
await nodeLogging?.log({
|
|
573
|
+
level: "error",
|
|
574
|
+
source: DynamoDbEntityStorageConnector.CLASS_NAME,
|
|
575
|
+
ts: Date.now(),
|
|
576
|
+
message: "teardownFailed",
|
|
577
|
+
error: BaseError.fromError(err)
|
|
578
|
+
});
|
|
579
|
+
return false;
|
|
580
|
+
}
|
|
581
|
+
}
|
|
373
582
|
/**
|
|
374
583
|
* Find all the entities which match the conditions.
|
|
375
584
|
* @param conditions The conditions to match for the entities.
|
|
@@ -383,18 +592,204 @@ export class DynamoDbEntityStorageConnector {
|
|
|
383
592
|
async query(conditions, sortProperties, properties, cursor, limit) {
|
|
384
593
|
const contextIds = await ContextIdStore.getContextIds();
|
|
385
594
|
const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
|
|
595
|
+
if (!Is.empty(limit)) {
|
|
596
|
+
const validationFailures = [];
|
|
597
|
+
Validation.integer("limit", limit, validationFailures, undefined, { minValue: 1 });
|
|
598
|
+
Validation.asValidationError(DynamoDbEntityStorageConnector.CLASS_NAME, "query", validationFailures);
|
|
599
|
+
}
|
|
600
|
+
EntityStorageHelper.validateSortProperties(this._entitySchema, sortProperties);
|
|
601
|
+
EntityStorageHelper.validateProperties(this._entitySchema, properties);
|
|
386
602
|
return this.internalQuery(conditions, sortProperties, properties, cursor, limit, undefined, partitionKey);
|
|
387
603
|
}
|
|
388
604
|
/**
|
|
389
|
-
*
|
|
390
|
-
* @
|
|
605
|
+
* Count all the entities which match the conditions.
|
|
606
|
+
* @param conditions The optional conditions to match for the entities.
|
|
607
|
+
* @returns The total count of entities in the storage.
|
|
391
608
|
*/
|
|
392
|
-
async
|
|
609
|
+
async count(conditions) {
|
|
393
610
|
try {
|
|
611
|
+
const contextIds = await ContextIdStore.getContextIds();
|
|
612
|
+
const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
|
|
613
|
+
const attributeNames = {
|
|
614
|
+
"#partitionId": DynamoDbEntityStorageConnector._PARTITION_KEY
|
|
615
|
+
};
|
|
616
|
+
const attributeValues = {
|
|
617
|
+
":partitionId": {
|
|
618
|
+
S: partitionKey ?? DynamoDbEntityStorageConnector._PARTITION_KEY_VALUE
|
|
619
|
+
}
|
|
620
|
+
};
|
|
621
|
+
const expressions = this.buildQueryParameters("", conditions, attributeNames, attributeValues);
|
|
622
|
+
if (expressions.noResults) {
|
|
623
|
+
return 0;
|
|
624
|
+
}
|
|
394
625
|
const dbConnection = this.createConnection();
|
|
395
|
-
|
|
626
|
+
let total = 0;
|
|
627
|
+
let exclusiveStartKey;
|
|
628
|
+
do {
|
|
629
|
+
const result = await dbConnection.send(new QueryCommand({
|
|
630
|
+
TableName: this._config.tableName,
|
|
631
|
+
Select: "COUNT",
|
|
632
|
+
KeyConditionExpression: "#partitionId = :partitionId",
|
|
633
|
+
FilterExpression: Is.stringValue(expressions.filterCondition)
|
|
634
|
+
? expressions.filterCondition
|
|
635
|
+
: undefined,
|
|
636
|
+
ExpressionAttributeNames: attributeNames,
|
|
637
|
+
ExpressionAttributeValues: attributeValues,
|
|
638
|
+
ExclusiveStartKey: exclusiveStartKey
|
|
639
|
+
}));
|
|
640
|
+
total += result.Count ?? 0;
|
|
641
|
+
exclusiveStartKey = result.LastEvaluatedKey;
|
|
642
|
+
} while (exclusiveStartKey);
|
|
643
|
+
return total;
|
|
644
|
+
}
|
|
645
|
+
catch (err) {
|
|
646
|
+
throw new GeneralError(DynamoDbEntityStorageConnector.CLASS_NAME, "countFailed", undefined, err);
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
/**
|
|
650
|
+
* Get a unique list of all the context ids from the storage.
|
|
651
|
+
* @returns The list of unique context ids.
|
|
652
|
+
*/
|
|
653
|
+
async getPartitionContextIds() {
|
|
654
|
+
if (!Is.arrayValue(this._partitionContextIds)) {
|
|
655
|
+
return [];
|
|
656
|
+
}
|
|
657
|
+
const contextIdsMap = {};
|
|
658
|
+
try {
|
|
659
|
+
const docClient = this.createDocClient();
|
|
660
|
+
let exclusiveStartKey;
|
|
661
|
+
do {
|
|
662
|
+
const scanResult = await docClient.send(new ScanCommand({
|
|
663
|
+
TableName: this._config.tableName,
|
|
664
|
+
ProjectionExpression: "#partitionId",
|
|
665
|
+
ExpressionAttributeNames: {
|
|
666
|
+
"#partitionId": DynamoDbEntityStorageConnector._PARTITION_KEY
|
|
667
|
+
},
|
|
668
|
+
ExclusiveStartKey: exclusiveStartKey
|
|
669
|
+
}));
|
|
670
|
+
for (const item of scanResult.Items ?? []) {
|
|
671
|
+
const partitionId = item[DynamoDbEntityStorageConnector._PARTITION_KEY];
|
|
672
|
+
if (Is.stringValue(partitionId) && !(partitionId in contextIdsMap)) {
|
|
673
|
+
contextIdsMap[partitionId] = ContextIdHelper.shortSplit(this._partitionContextIds ?? [], partitionId);
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
exclusiveStartKey = scanResult.LastEvaluatedKey;
|
|
677
|
+
} while (exclusiveStartKey);
|
|
678
|
+
}
|
|
679
|
+
catch (err) {
|
|
680
|
+
throw new GeneralError(DynamoDbEntityStorageConnector.CLASS_NAME, "getPartitionContextIdsFailed", undefined, err);
|
|
681
|
+
}
|
|
682
|
+
return Object.values(contextIdsMap);
|
|
683
|
+
}
|
|
684
|
+
/**
|
|
685
|
+
* Create the target connector for performing the migration it will use a temporary storage location.
|
|
686
|
+
* @param newEntitySchema The name of the new entity schema to create the connector for.
|
|
687
|
+
* @returns Connector for performing the migration.
|
|
688
|
+
*/
|
|
689
|
+
async createTargetConnector(newEntitySchema) {
|
|
690
|
+
// We create a new table for the migration with a unique name to avoid conflicts with the existing table
|
|
691
|
+
// This table will be swapped with the existing table once the migration is finalized.
|
|
692
|
+
const migrationTableName = `${this._config.tableName}Migration${Date.now()}`;
|
|
693
|
+
return new DynamoDbEntityStorageConnector({
|
|
694
|
+
entitySchema: newEntitySchema,
|
|
695
|
+
config: {
|
|
696
|
+
...this._config,
|
|
697
|
+
tableName: migrationTableName
|
|
698
|
+
},
|
|
699
|
+
partitionContextIds: this._partitionContextIds
|
|
700
|
+
});
|
|
701
|
+
}
|
|
702
|
+
/**
|
|
703
|
+
* Finalize the migration by tearing down the old connector and replacing it with the new one.
|
|
704
|
+
* @param targetConnector The target connector to finalize the migration with.
|
|
705
|
+
* @param options The options to control how the migration is finalized.
|
|
706
|
+
* @param loggingComponentType The logging component type to use for logging during the migration finalization.
|
|
707
|
+
* @returns A promise that resolves when the migration is finalized.
|
|
708
|
+
*/
|
|
709
|
+
async finalizeMigration(targetConnector, options, loggingComponentType) {
|
|
710
|
+
// There is no rename operation in DynamoDB so we have to create a new table with the original name and copy the data over
|
|
711
|
+
// Teardown the existing table with the original name to free up the name for the new table
|
|
712
|
+
await this.teardown(loggingComponentType);
|
|
713
|
+
// Create a new connector with the original table name but with the new schema
|
|
714
|
+
// and copy the data from the migration table to the new table using batch operations
|
|
715
|
+
const finalConnector = new DynamoDbEntityStorageConnector({
|
|
716
|
+
entitySchema: targetConnector._entitySchemaName,
|
|
717
|
+
config: this._config,
|
|
718
|
+
partitionContextIds: this._partitionContextIds
|
|
719
|
+
});
|
|
720
|
+
if (await finalConnector.bootstrap(loggingComponentType)) {
|
|
721
|
+
// Since there is no rename, we need to copy the data from the migration table to the new table
|
|
722
|
+
const partitions = await targetConnector.getPartitionContextIds();
|
|
723
|
+
const batchSize = options?.batchSize ?? DynamoDbEntityStorageConnector._DEFAULT_LIMIT;
|
|
724
|
+
await this.bulkCopy(targetConnector, finalConnector, partitions, batchSize);
|
|
725
|
+
await targetConnector.teardown(loggingComponentType);
|
|
726
|
+
return finalConnector;
|
|
727
|
+
}
|
|
728
|
+
throw new GeneralError(DynamoDbEntityStorageConnector.CLASS_NAME, "finalizeMigrationFailedBootstrap", undefined);
|
|
729
|
+
}
|
|
730
|
+
/**
|
|
731
|
+
* Cleanup the migration if a migration fails or needs to be aborted.
|
|
732
|
+
* @param targetConnector The target connector to cleanup the migration with.
|
|
733
|
+
* @param options The options to control how the migration is cleaned up.
|
|
734
|
+
* @param loggingComponentType The optional component type to use for logging the migration progress.
|
|
735
|
+
* @returns A promise that resolves when the migration is cleaned up.
|
|
736
|
+
*/
|
|
737
|
+
async cleanupMigration(targetConnector, options, loggingComponentType) {
|
|
738
|
+
// If something failed the only thing to cleanup is the migration table
|
|
739
|
+
await targetConnector?.teardown?.(loggingComponentType);
|
|
740
|
+
}
|
|
741
|
+
/**
|
|
742
|
+
* Copy all entities from sourceConnector to destConnector, paging through each partition.
|
|
743
|
+
* @param sourceConnector The connector to read entities from.
|
|
744
|
+
* @param destConnector The connector to write entities to.
|
|
745
|
+
* @param partitions The partition list returned by getPartitionContextIds.
|
|
746
|
+
* @param batchSize The number of entities to read per page.
|
|
747
|
+
* @internal
|
|
748
|
+
*/
|
|
749
|
+
async bulkCopy(sourceConnector, destConnector, partitions, batchSize) {
|
|
750
|
+
let partitionList;
|
|
751
|
+
if (Is.arrayValue(partitions)) {
|
|
752
|
+
partitionList = partitions;
|
|
753
|
+
}
|
|
754
|
+
else if (Is.arrayValue(sourceConnector._partitionContextIds)) {
|
|
755
|
+
partitionList = [];
|
|
756
|
+
}
|
|
757
|
+
else {
|
|
758
|
+
partitionList = [{}];
|
|
759
|
+
}
|
|
760
|
+
const dbConnection = sourceConnector.createConnection();
|
|
761
|
+
const chunkSize = 25;
|
|
762
|
+
for (let i = 0; i < partitionList.length; i++) {
|
|
763
|
+
const partitionKey = ContextIdHelper.combinedContextKey(partitionList[i], sourceConnector._partitionContextIds) ?? DynamoDbEntityStorageConnector._PARTITION_KEY_VALUE;
|
|
764
|
+
let exclusiveStartKey;
|
|
765
|
+
do {
|
|
766
|
+
const { Items: items, LastEvaluatedKey: lastKey } = await dbConnection.send(new QueryCommand({
|
|
767
|
+
TableName: sourceConnector._config.tableName,
|
|
768
|
+
KeyConditionExpression: `#${DynamoDbEntityStorageConnector._PARTITION_KEY} = :${DynamoDbEntityStorageConnector._PARTITION_KEY}`,
|
|
769
|
+
ExpressionAttributeNames: {
|
|
770
|
+
[`#${DynamoDbEntityStorageConnector._PARTITION_KEY}`]: DynamoDbEntityStorageConnector._PARTITION_KEY
|
|
771
|
+
},
|
|
772
|
+
ExpressionAttributeValues: {
|
|
773
|
+
[`:${DynamoDbEntityStorageConnector._PARTITION_KEY}`]: { S: partitionKey }
|
|
774
|
+
},
|
|
775
|
+
Limit: batchSize,
|
|
776
|
+
ExclusiveStartKey: exclusiveStartKey
|
|
777
|
+
}));
|
|
778
|
+
exclusiveStartKey = lastKey;
|
|
779
|
+
if (Is.arrayValue(items)) {
|
|
780
|
+
for (let j = 0; j < items.length; j += chunkSize) {
|
|
781
|
+
const chunk = items.slice(j, j + chunkSize);
|
|
782
|
+
await dbConnection.send(new BatchWriteItemCommand({
|
|
783
|
+
RequestItems: {
|
|
784
|
+
[destConnector._config.tableName]: chunk.map(item => ({
|
|
785
|
+
PutRequest: { Item: item }
|
|
786
|
+
}))
|
|
787
|
+
}
|
|
788
|
+
}));
|
|
789
|
+
}
|
|
790
|
+
}
|
|
791
|
+
} while (exclusiveStartKey);
|
|
396
792
|
}
|
|
397
|
-
catch { }
|
|
398
793
|
}
|
|
399
794
|
/**
|
|
400
795
|
* Create the parameters for a query.
|
|
@@ -402,6 +797,7 @@ export class DynamoDbEntityStorageConnector {
|
|
|
402
797
|
* @param condition The conditions to create the query from.
|
|
403
798
|
* @param attributeNames The attribute names to use in the query.
|
|
404
799
|
* @param attributeValues The attribute values to use in the query.
|
|
800
|
+
* @param secondaryIndex The optional secondary index to use for the query.
|
|
405
801
|
* @returns The condition clause.
|
|
406
802
|
* @internal
|
|
407
803
|
*/
|
|
@@ -410,19 +806,79 @@ export class DynamoDbEntityStorageConnector {
|
|
|
410
806
|
if (Is.undefined(condition)) {
|
|
411
807
|
return {
|
|
412
808
|
keyCondition: "",
|
|
413
|
-
filterCondition: ""
|
|
809
|
+
filterCondition: "",
|
|
810
|
+
requiresScan: false
|
|
414
811
|
};
|
|
415
812
|
}
|
|
416
813
|
if ("conditions" in condition) {
|
|
417
814
|
if (condition.conditions.length === 0) {
|
|
418
815
|
return {
|
|
419
816
|
keyCondition: "",
|
|
420
|
-
filterCondition: ""
|
|
817
|
+
filterCondition: "",
|
|
818
|
+
requiresScan: false
|
|
421
819
|
};
|
|
422
820
|
}
|
|
821
|
+
// Snapshot before the entire group. Used by the AND path to undo
|
|
822
|
+
// surviving siblings' attribute registrations when the AND is dead (#141).
|
|
823
|
+
const preGroupNames = new Set(Object.keys(attributeNames));
|
|
824
|
+
const preGroupValues = new Set(Object.keys(attributeValues));
|
|
423
825
|
// It's a group of comparisons, so check the individual items and combine with the logical operator
|
|
424
|
-
const joinConditions = condition.conditions.map(c =>
|
|
826
|
+
const joinConditions = condition.conditions.map(c => {
|
|
827
|
+
// Snapshot before each branch. When a branch is dead (noResults),
|
|
828
|
+
// undo its attribute registrations so the final expressions stay
|
|
829
|
+
// consistent — DynamoDB rejects unused ExpressionAttributeNames (#141).
|
|
830
|
+
const preBranchNames = new Set(Object.keys(attributeNames));
|
|
831
|
+
const preBranchValues = new Set(Object.keys(attributeValues));
|
|
832
|
+
const result = this.buildQueryParameters(objectPath, c, attributeNames, attributeValues, secondaryIndex);
|
|
833
|
+
if (result.noResults) {
|
|
834
|
+
for (const key of Object.keys(attributeNames)) {
|
|
835
|
+
if (!preBranchNames.has(key)) {
|
|
836
|
+
delete attributeNames[key];
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
for (const key of Object.keys(attributeValues)) {
|
|
840
|
+
if (!preBranchValues.has(key)) {
|
|
841
|
+
delete attributeValues[key];
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
}
|
|
845
|
+
return result;
|
|
846
|
+
});
|
|
425
847
|
const logicalOperator = this.mapConditionalOperator(condition.logicalOperator);
|
|
848
|
+
// DynamoDB does not support OR in KeyConditionExpression, so when the operator
|
|
849
|
+
// is OR we must move all conditions (including key conditions) into FilterExpression.
|
|
850
|
+
if (condition.logicalOperator === LogicalOperator.Or) {
|
|
851
|
+
// OR: only empty if ALL branches are guaranteed empty (e.g. all empty IN lists).
|
|
852
|
+
// If only some are empty they are naturally filtered out of `parts` below,
|
|
853
|
+
// which is correct — false OR x = x (#141).
|
|
854
|
+
if (joinConditions.every(j => j.noResults)) {
|
|
855
|
+
return { keyCondition: "", filterCondition: "", requiresScan: false, noResults: true };
|
|
856
|
+
}
|
|
857
|
+
const parts = joinConditions
|
|
858
|
+
.map(j => {
|
|
859
|
+
// A branch marked noResults (e.g. a dead AND group containing In [])
|
|
860
|
+
// must contribute nothing to the OR — false OR x = x (#141).
|
|
861
|
+
if (j.noResults) {
|
|
862
|
+
return "";
|
|
863
|
+
}
|
|
864
|
+
const subParts = [j.keyCondition.trim(), j.filterCondition.trim()].filter(s => s.length > 0);
|
|
865
|
+
if (subParts.length === 0) {
|
|
866
|
+
return "";
|
|
867
|
+
}
|
|
868
|
+
if (subParts.length === 1) {
|
|
869
|
+
return subParts[0];
|
|
870
|
+
}
|
|
871
|
+
return `(${subParts.join(" AND ")})`;
|
|
872
|
+
})
|
|
873
|
+
.filter(s => s.length > 0);
|
|
874
|
+
const hasKeyConditions = joinConditions.some(j => j.keyCondition.length > 0);
|
|
875
|
+
const filterCondition = parts.join(" OR ");
|
|
876
|
+
return {
|
|
877
|
+
keyCondition: "",
|
|
878
|
+
filterCondition: Is.stringValue(filterCondition) ? ` (${filterCondition}) ` : "",
|
|
879
|
+
requiresScan: hasKeyConditions
|
|
880
|
+
};
|
|
881
|
+
}
|
|
426
882
|
const keyCondition = joinConditions
|
|
427
883
|
.filter(j => j.keyCondition.length > 0)
|
|
428
884
|
.map(j => j.keyCondition)
|
|
@@ -431,18 +887,46 @@ export class DynamoDbEntityStorageConnector {
|
|
|
431
887
|
.filter(j => j.filterCondition.length > 0)
|
|
432
888
|
.map(j => j.filterCondition)
|
|
433
889
|
.join(` ${logicalOperator} `);
|
|
890
|
+
// AND: if any sub-condition is a guaranteed empty result (e.g. empty IN list),
|
|
891
|
+
// the whole AND group is also empty (#141). Restore the attribute maps to the
|
|
892
|
+
// pre-group snapshot so surviving siblings' registrations are also undone —
|
|
893
|
+
// per-branch cleanup above only undoes dead branches, not live ones whose AND
|
|
894
|
+
// partner was dead.
|
|
895
|
+
const noResults = joinConditions.some(j => j.noResults);
|
|
896
|
+
if (noResults) {
|
|
897
|
+
for (const key of Object.keys(attributeNames)) {
|
|
898
|
+
if (!preGroupNames.has(key)) {
|
|
899
|
+
delete attributeNames[key];
|
|
900
|
+
}
|
|
901
|
+
}
|
|
902
|
+
for (const key of Object.keys(attributeValues)) {
|
|
903
|
+
if (!preGroupValues.has(key)) {
|
|
904
|
+
delete attributeValues[key];
|
|
905
|
+
}
|
|
906
|
+
}
|
|
907
|
+
return { keyCondition: "", filterCondition: "", requiresScan: false, noResults: true };
|
|
908
|
+
}
|
|
434
909
|
return {
|
|
435
910
|
keyCondition: Is.stringValue(keyCondition) ? ` (${keyCondition}) ` : "",
|
|
436
|
-
filterCondition: Is.stringValue(filterCondition) ? ` (${filterCondition}) ` : ""
|
|
911
|
+
filterCondition: Is.stringValue(filterCondition) ? ` (${filterCondition}) ` : "",
|
|
912
|
+
requiresScan: joinConditions.some(j => j.requiresScan)
|
|
437
913
|
};
|
|
438
914
|
}
|
|
439
915
|
const schemaProp = this._entitySchema.properties?.find(p => p.property === condition.property);
|
|
916
|
+
// Empty IN list: DynamoDB has no `IN ()` syntax — short-circuit to empty result (#141).
|
|
917
|
+
if ("comparison" in condition &&
|
|
918
|
+
condition.comparison === ComparisonOperator.In &&
|
|
919
|
+
Is.array(condition.value) &&
|
|
920
|
+
condition.value.length === 0) {
|
|
921
|
+
return { keyCondition: "", filterCondition: "", requiresScan: false, noResults: true };
|
|
922
|
+
}
|
|
440
923
|
// It's a single value so just create the property comparison for the condition
|
|
441
924
|
const comparison = this.mapComparisonOperator(objectPath, condition, schemaProp?.type, attributeNames, attributeValues);
|
|
442
925
|
const isKey = schemaProp?.isPrimary ?? (schemaProp?.isSecondary && schemaProp?.property === secondaryIndex);
|
|
443
926
|
return {
|
|
444
927
|
keyCondition: isKey ? comparison : "",
|
|
445
|
-
filterCondition: !isKey ? comparison : ""
|
|
928
|
+
filterCondition: !isKey ? comparison : "",
|
|
929
|
+
requiresScan: false
|
|
446
930
|
};
|
|
447
931
|
}
|
|
448
932
|
/**
|
|
@@ -464,6 +948,8 @@ export class DynamoDbEntityStorageConnector {
|
|
|
464
948
|
prop += comparator.property;
|
|
465
949
|
let attributeName = this.populateAttributeNames(prop, attributeNames);
|
|
466
950
|
if (Is.empty(comparator.value)) {
|
|
951
|
+
// With "omit" storage, optional null/undefined fields are absent from the item entirely.
|
|
952
|
+
// attribute_not_exists matches absent attributes; attribute_exists matches present ones.
|
|
467
953
|
if (comparator.comparison === ComparisonOperator.Equals) {
|
|
468
954
|
return `attribute_not_exists(${attributeName})`;
|
|
469
955
|
}
|
|
@@ -471,7 +957,13 @@ export class DynamoDbEntityStorageConnector {
|
|
|
471
957
|
return `attribute_exists(${attributeName})`;
|
|
472
958
|
}
|
|
473
959
|
}
|
|
474
|
-
|
|
960
|
+
const basePropName = `:${attributeName.replace(/\./g, "").replace(/#/g, "")}`;
|
|
961
|
+
let propName = basePropName;
|
|
962
|
+
let propSuffix = 0;
|
|
963
|
+
while (!Is.undefined(attributeValues[propName])) {
|
|
964
|
+
propSuffix++;
|
|
965
|
+
propName = `${basePropName}${propSuffix}`;
|
|
966
|
+
}
|
|
475
967
|
if (Is.array(comparator.value)) {
|
|
476
968
|
const dbValues = comparator.value.map(v => this.propertyToDbValue(v, type));
|
|
477
969
|
const arrAttributeNames = [];
|
|
@@ -508,7 +1000,7 @@ export class DynamoDbEntityStorageConnector {
|
|
|
508
1000
|
return `contains(${attributeName}, ${propName})`;
|
|
509
1001
|
}
|
|
510
1002
|
else if (comparator.comparison === ComparisonOperator.NotIncludes) {
|
|
511
|
-
return `
|
|
1003
|
+
return `NOT contains(${attributeName}, ${propName})`;
|
|
512
1004
|
}
|
|
513
1005
|
else if (comparator.comparison === ComparisonOperator.In) {
|
|
514
1006
|
return `${propName} IN ${attributeName}`;
|
|
@@ -577,6 +1069,12 @@ export class DynamoDbEntityStorageConnector {
|
|
|
577
1069
|
else if (type === "boolean") {
|
|
578
1070
|
return { BOOL: Coerce.boolean(value) ?? false };
|
|
579
1071
|
}
|
|
1072
|
+
if (Is.boolean(value)) {
|
|
1073
|
+
return { BOOL: value };
|
|
1074
|
+
}
|
|
1075
|
+
else if (Is.number(value)) {
|
|
1076
|
+
return { N: value.toString() };
|
|
1077
|
+
}
|
|
580
1078
|
return { S: Coerce.string(value) ?? "" };
|
|
581
1079
|
}
|
|
582
1080
|
/**
|
|
@@ -608,9 +1106,9 @@ export class DynamoDbEntityStorageConnector {
|
|
|
608
1106
|
* @internal
|
|
609
1107
|
*/
|
|
610
1108
|
createConnectionConfig() {
|
|
611
|
-
const requestHandler =
|
|
612
|
-
requestTimeout: this._config.connectionTimeoutMs
|
|
613
|
-
|
|
1109
|
+
const requestHandler = Is.number(this._config.connectionTimeoutMs)
|
|
1110
|
+
? { requestTimeout: this._config.connectionTimeoutMs }
|
|
1111
|
+
: undefined;
|
|
614
1112
|
if (Is.stringValue(this._config.secretAccessKey) &&
|
|
615
1113
|
Is.stringValue(this._config.accessKeyId) &&
|
|
616
1114
|
this._config.authMode === "credentials") {
|
|
@@ -621,13 +1119,15 @@ export class DynamoDbEntityStorageConnector {
|
|
|
621
1119
|
},
|
|
622
1120
|
endpoint: this._config.endpoint,
|
|
623
1121
|
region: this._config.region,
|
|
624
|
-
requestHandler
|
|
1122
|
+
requestHandler,
|
|
1123
|
+
maxAttempts: this._config.maxAttempts
|
|
625
1124
|
};
|
|
626
1125
|
}
|
|
627
1126
|
return {
|
|
628
1127
|
endpoint: this._config.endpoint,
|
|
629
1128
|
region: this._config.region,
|
|
630
|
-
requestHandler
|
|
1129
|
+
requestHandler,
|
|
1130
|
+
maxAttempts: this._config.maxAttempts
|
|
631
1131
|
};
|
|
632
1132
|
}
|
|
633
1133
|
/**
|
|
@@ -639,8 +1139,9 @@ export class DynamoDbEntityStorageConnector {
|
|
|
639
1139
|
async tableExists(tableName) {
|
|
640
1140
|
try {
|
|
641
1141
|
const dbConnection = this.createConnection();
|
|
642
|
-
await dbConnection.describeTable({ TableName: tableName });
|
|
643
|
-
|
|
1142
|
+
const result = await dbConnection.describeTable({ TableName: tableName });
|
|
1143
|
+
// A table in DELETING state should not be treated as existing
|
|
1144
|
+
return result.Table?.TableStatus !== "DELETING";
|
|
644
1145
|
}
|
|
645
1146
|
catch {
|
|
646
1147
|
return false;
|
|
@@ -662,84 +1163,313 @@ export class DynamoDbEntityStorageConnector {
|
|
|
662
1163
|
async internalQuery(conditions, sortProperties, properties, cursor, limit, secondaryIndex, partitionKey) {
|
|
663
1164
|
try {
|
|
664
1165
|
const returnSize = limit ?? DynamoDbEntityStorageConnector._DEFAULT_LIMIT;
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
1166
|
+
const indexConfig = this.resolveQueryIndexConfig(sortProperties, secondaryIndex);
|
|
1167
|
+
const { attributeNames, attributeValues } = this.buildQueryAttributeMaps(partitionKey);
|
|
1168
|
+
const safeCursor = this.sanitizeCursorForPartition(cursor, attributeValues);
|
|
1169
|
+
const expressions = this.buildQueryParameters("", conditions, attributeNames, attributeValues, indexConfig.gsiAttribute);
|
|
1170
|
+
if (expressions.noResults) {
|
|
1171
|
+
return { entities: [], cursor: undefined };
|
|
1172
|
+
}
|
|
1173
|
+
if (expressions.requiresScan) {
|
|
1174
|
+
const scanResult = await this.executeScanFallback(expressions.filterCondition, properties, attributeNames, attributeValues, safeCursor, returnSize);
|
|
1175
|
+
return {
|
|
1176
|
+
entities: this.mapRawItemsToEntities(scanResult.rawItems),
|
|
1177
|
+
cursor: scanResult.cursor
|
|
1178
|
+
};
|
|
1179
|
+
}
|
|
1180
|
+
const keyExpression = this.buildKeyExpression(expressions.keyCondition);
|
|
1181
|
+
const queryProjection = this.buildProjectionExpression(properties, attributeNames);
|
|
1182
|
+
const queryResult = Is.stringValue(expressions.filterCondition)
|
|
1183
|
+
? await this.executeFilteredQuery(keyExpression, expressions.filterCondition, attributeNames, attributeValues, queryProjection, indexConfig.indexName, indexConfig.scanAscending, safeCursor, returnSize)
|
|
1184
|
+
: await this.executeUnfilteredQuery(keyExpression, attributeNames, attributeValues, queryProjection, indexConfig.indexName, indexConfig.scanAscending, safeCursor, returnSize);
|
|
1185
|
+
return {
|
|
1186
|
+
entities: this.mapRawItemsToEntities(queryResult.rawItems),
|
|
1187
|
+
cursor: queryResult.cursor
|
|
1188
|
+
};
|
|
1189
|
+
}
|
|
1190
|
+
catch (err) {
|
|
1191
|
+
if (BaseError.isErrorCode(err, "ResourceNotFoundException")) {
|
|
1192
|
+
throw new GeneralError(DynamoDbEntityStorageConnector.CLASS_NAME, "tableDoesNotExist", {
|
|
1193
|
+
tableName: this._config.tableName
|
|
1194
|
+
}, err);
|
|
1195
|
+
}
|
|
1196
|
+
throw new GeneralError(DynamoDbEntityStorageConnector.CLASS_NAME, "queryFailed", undefined, err);
|
|
1197
|
+
}
|
|
1198
|
+
}
|
|
1199
|
+
/**
|
|
1200
|
+
* Resolve index configuration from sort options and optional explicit secondary index.
|
|
1201
|
+
* @param sortProperties The optional sort order.
|
|
1202
|
+
* @param secondaryIndex The optional explicit secondary index.
|
|
1203
|
+
* @returns The resolved index name, GSI attribute and sort direction.
|
|
1204
|
+
* @throws GeneralError if more than one sort property is specified.
|
|
1205
|
+
* @internal
|
|
1206
|
+
*/
|
|
1207
|
+
resolveQueryIndexConfig(sortProperties, secondaryIndex) {
|
|
1208
|
+
let indexName = Is.stringValue(secondaryIndex)
|
|
1209
|
+
? `${secondaryIndex}Index`
|
|
1210
|
+
: undefined;
|
|
1211
|
+
let gsiAttribute = secondaryIndex;
|
|
1212
|
+
let scanAscending = true;
|
|
1213
|
+
if (Is.arrayValue(sortProperties)) {
|
|
1214
|
+
if (sortProperties.length > 1) {
|
|
1215
|
+
throw new GeneralError(DynamoDbEntityStorageConnector.CLASS_NAME, "sortSingle");
|
|
1216
|
+
}
|
|
1217
|
+
for (const sortProperty of sortProperties) {
|
|
1218
|
+
const propertySchema = this._entitySchema.properties?.find(e => e.property === sortProperty.property);
|
|
1219
|
+
if (propertySchema?.isPrimary) {
|
|
1220
|
+
indexName = undefined;
|
|
1221
|
+
gsiAttribute = undefined;
|
|
674
1222
|
}
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
(!propertySchema.isPrimary &&
|
|
679
|
-
!propertySchema.isSecondary &&
|
|
680
|
-
Is.empty(propertySchema.sortDirection))) {
|
|
681
|
-
throw new GeneralError(DynamoDbEntityStorageConnector.CLASS_NAME, "sortNotIndexed", {
|
|
682
|
-
property: sortProperty.property
|
|
683
|
-
});
|
|
684
|
-
}
|
|
685
|
-
indexName = propertySchema.isPrimary
|
|
686
|
-
? undefined
|
|
687
|
-
: `${sortProperty.property}Index`;
|
|
688
|
-
scanAscending = sortProperty.sortDirection === SortDirection.Ascending;
|
|
1223
|
+
else {
|
|
1224
|
+
indexName = `${sortProperty.property}Index`;
|
|
1225
|
+
gsiAttribute = sortProperty.property;
|
|
689
1226
|
}
|
|
1227
|
+
scanAscending = sortProperty.sortDirection === SortDirection.Ascending;
|
|
690
1228
|
}
|
|
691
|
-
|
|
692
|
-
|
|
1229
|
+
}
|
|
1230
|
+
return {
|
|
1231
|
+
indexName,
|
|
1232
|
+
gsiAttribute,
|
|
1233
|
+
scanAscending
|
|
1234
|
+
};
|
|
1235
|
+
}
|
|
1236
|
+
/**
|
|
1237
|
+
* Build the base attribute maps used for query/scan operations.
|
|
1238
|
+
* @param partitionKey The optional partition key.
|
|
1239
|
+
* @returns The query attribute maps.
|
|
1240
|
+
* @internal
|
|
1241
|
+
*/
|
|
1242
|
+
buildQueryAttributeMaps(partitionKey) {
|
|
1243
|
+
return {
|
|
1244
|
+
attributeNames: { "#partitionId": "partitionId" },
|
|
1245
|
+
attributeValues: {
|
|
693
1246
|
[`:${DynamoDbEntityStorageConnector._PARTITION_KEY}`]: {
|
|
694
1247
|
S: partitionKey ?? DynamoDbEntityStorageConnector._PARTITION_KEY_VALUE
|
|
695
1248
|
}
|
|
696
|
-
};
|
|
697
|
-
const expressions = this.buildQueryParameters("", conditions, attributeNames, attributeValues, secondaryIndex);
|
|
698
|
-
let keyExpression = "#partitionId = :partitionId";
|
|
699
|
-
if (expressions.keyCondition.length > 0) {
|
|
700
|
-
keyExpression += ` AND ${expressions.keyCondition}`;
|
|
701
1249
|
}
|
|
702
|
-
|
|
1250
|
+
};
|
|
1251
|
+
}
|
|
1252
|
+
/**
|
|
1253
|
+
* Build the key condition expression with the mandatory partition predicate.
|
|
1254
|
+
* @param keyCondition The optional extra key condition segment.
|
|
1255
|
+
* @returns The full key condition expression.
|
|
1256
|
+
* @internal
|
|
1257
|
+
*/
|
|
1258
|
+
buildKeyExpression(keyCondition) {
|
|
1259
|
+
let keyExpression = "#partitionId = :partitionId";
|
|
1260
|
+
if (keyCondition.length > 0) {
|
|
1261
|
+
keyExpression += ` AND ${keyCondition}`;
|
|
1262
|
+
}
|
|
1263
|
+
return keyExpression;
|
|
1264
|
+
}
|
|
1265
|
+
/**
|
|
1266
|
+
* Decode a paginated cursor into a DynamoDB ExclusiveStartKey.
|
|
1267
|
+
* @param cursor The encoded cursor.
|
|
1268
|
+
* @returns The decoded exclusive start key.
|
|
1269
|
+
* @internal
|
|
1270
|
+
*/
|
|
1271
|
+
decodeCursor(cursor) {
|
|
1272
|
+
return Is.empty(cursor) ? undefined : ObjectHelper.fromBytes(Converter.base64ToBytes(cursor));
|
|
1273
|
+
}
|
|
1274
|
+
/**
|
|
1275
|
+
* Encode a DynamoDB key to a cursor string.
|
|
1276
|
+
* @param key The key to encode.
|
|
1277
|
+
* @returns The encoded cursor.
|
|
1278
|
+
* @internal
|
|
1279
|
+
*/
|
|
1280
|
+
encodeCursor(key) {
|
|
1281
|
+
return Is.empty(key) ? undefined : Converter.bytesToBase64(ObjectHelper.toBytes(key));
|
|
1282
|
+
}
|
|
1283
|
+
/**
|
|
1284
|
+
* Return undefined if the cursor belongs to a different partition, preventing cross-partition leakage.
|
|
1285
|
+
* @param cursor The encoded cursor.
|
|
1286
|
+
* @param attributeValues The current query attribute values containing the expected partition key.
|
|
1287
|
+
* @returns The cursor if it matches the current partition, otherwise undefined.
|
|
1288
|
+
* @internal
|
|
1289
|
+
*/
|
|
1290
|
+
sanitizeCursorForPartition(cursor, attributeValues) {
|
|
1291
|
+
if (Is.empty(cursor)) {
|
|
1292
|
+
return undefined;
|
|
1293
|
+
}
|
|
1294
|
+
const decoded = this.decodeCursor(cursor);
|
|
1295
|
+
if (Is.empty(decoded)) {
|
|
1296
|
+
return undefined;
|
|
1297
|
+
}
|
|
1298
|
+
const expectedPartition = attributeValues[`:${DynamoDbEntityStorageConnector._PARTITION_KEY}`]?.S;
|
|
1299
|
+
const cursorPartition = decoded[DynamoDbEntityStorageConnector._PARTITION_KEY]?.S;
|
|
1300
|
+
return cursorPartition === expectedPartition ? cursor : undefined;
|
|
1301
|
+
}
|
|
1302
|
+
/**
|
|
1303
|
+
* Execute a scan fallback path for unsupported key-condition shapes.
|
|
1304
|
+
* @param filterCondition The optional filter expression part.
|
|
1305
|
+
* @param properties The projection properties.
|
|
1306
|
+
* @param attributeNames The expression attribute names.
|
|
1307
|
+
* @param attributeValues The expression attribute values.
|
|
1308
|
+
* @param cursor The optional cursor.
|
|
1309
|
+
* @param returnSize The requested page size.
|
|
1310
|
+
* @returns Raw items and an optional cursor.
|
|
1311
|
+
* @internal
|
|
1312
|
+
*/
|
|
1313
|
+
async executeScanFallback(filterCondition, properties, attributeNames, attributeValues, cursor, returnSize) {
|
|
1314
|
+
let scanFilter = "#partitionId = :partitionId";
|
|
1315
|
+
if (Is.stringValue(filterCondition)) {
|
|
1316
|
+
scanFilter += ` AND ${filterCondition.trim()}`;
|
|
1317
|
+
}
|
|
1318
|
+
const dbConnection = this.createConnection();
|
|
1319
|
+
const matchingItems = [];
|
|
1320
|
+
let scanStartKey = this.decodeCursor(cursor);
|
|
1321
|
+
let lastEvaluatedKey;
|
|
1322
|
+
// Scan in batches with a heuristic limit to avoid scanning excessive unmatched rows.
|
|
1323
|
+
// Use at least 2x returnSize per batch to balance accuracy and efficiency.
|
|
1324
|
+
const batchScanLimit = Math.max(returnSize * 2, 100);
|
|
1325
|
+
do {
|
|
1326
|
+
const scanResult = await dbConnection.send(new RawScanCommand({
|
|
1327
|
+
TableName: this._config.tableName,
|
|
1328
|
+
FilterExpression: scanFilter,
|
|
1329
|
+
ExpressionAttributeNames: attributeNames,
|
|
1330
|
+
ExpressionAttributeValues: attributeValues,
|
|
1331
|
+
ExclusiveStartKey: scanStartKey,
|
|
1332
|
+
Limit: batchScanLimit
|
|
1333
|
+
}));
|
|
1334
|
+
matchingItems.push(...(scanResult.Items ?? []));
|
|
1335
|
+
lastEvaluatedKey = scanResult.LastEvaluatedKey;
|
|
1336
|
+
scanStartKey = lastEvaluatedKey;
|
|
1337
|
+
// Early exit: stop scanning once we have enough filtered results to fill the page.
|
|
1338
|
+
if (matchingItems.length >= returnSize) {
|
|
1339
|
+
break;
|
|
1340
|
+
}
|
|
1341
|
+
} while (!Is.empty(lastEvaluatedKey));
|
|
1342
|
+
const hasMore = matchingItems.length > returnSize;
|
|
1343
|
+
const returnedRawItems = hasMore ? matchingItems.slice(0, returnSize) : matchingItems;
|
|
1344
|
+
let resultCursor;
|
|
1345
|
+
if (hasMore) {
|
|
1346
|
+
const lastRawItem = returnedRawItems[returnedRawItems.length - 1];
|
|
1347
|
+
const syntheticKey = {
|
|
1348
|
+
[DynamoDbEntityStorageConnector._PARTITION_KEY]: lastRawItem[DynamoDbEntityStorageConnector._PARTITION_KEY],
|
|
1349
|
+
[this._primaryKey.property]: lastRawItem[this._primaryKey.property]
|
|
1350
|
+
};
|
|
1351
|
+
resultCursor = this.encodeCursor(syntheticKey);
|
|
1352
|
+
}
|
|
1353
|
+
const projectedRawItems = Is.arrayValue(properties)
|
|
1354
|
+
? returnedRawItems.map(item => {
|
|
1355
|
+
const projected = {};
|
|
1356
|
+
for (const prop of properties) {
|
|
1357
|
+
const key = prop;
|
|
1358
|
+
if (!Is.undefined(item[key])) {
|
|
1359
|
+
projected[key] = item[key];
|
|
1360
|
+
}
|
|
1361
|
+
}
|
|
1362
|
+
if (!Is.undefined(item[DynamoDbEntityStorageConnector._PARTITION_KEY])) {
|
|
1363
|
+
projected[DynamoDbEntityStorageConnector._PARTITION_KEY] =
|
|
1364
|
+
item[DynamoDbEntityStorageConnector._PARTITION_KEY];
|
|
1365
|
+
}
|
|
1366
|
+
return projected;
|
|
1367
|
+
})
|
|
1368
|
+
: returnedRawItems;
|
|
1369
|
+
return {
|
|
1370
|
+
rawItems: projectedRawItems,
|
|
1371
|
+
cursor: resultCursor
|
|
1372
|
+
};
|
|
1373
|
+
}
|
|
1374
|
+
/**
|
|
1375
|
+
* Execute a query path without filter expression.
|
|
1376
|
+
* @param keyExpression The key condition expression.
|
|
1377
|
+
* @param attributeNames The expression attribute names.
|
|
1378
|
+
* @param attributeValues The expression attribute values.
|
|
1379
|
+
* @param projectionExpression The projection expression.
|
|
1380
|
+
* @param indexName The optional index name.
|
|
1381
|
+
* @param scanAscending The scan direction.
|
|
1382
|
+
* @param cursor The optional cursor.
|
|
1383
|
+
* @param returnSize The requested page size.
|
|
1384
|
+
* @returns Raw items and an optional cursor.
|
|
1385
|
+
* @internal
|
|
1386
|
+
*/
|
|
1387
|
+
async executeUnfilteredQuery(keyExpression, attributeNames, attributeValues, projectionExpression, indexName, scanAscending, cursor, returnSize) {
|
|
1388
|
+
const connection = this.createDocClient();
|
|
1389
|
+
const results = await connection.send(new QueryCommand({
|
|
1390
|
+
TableName: this._config.tableName,
|
|
1391
|
+
IndexName: indexName,
|
|
1392
|
+
KeyConditionExpression: keyExpression,
|
|
1393
|
+
ExpressionAttributeNames: attributeNames,
|
|
1394
|
+
ExpressionAttributeValues: attributeValues,
|
|
1395
|
+
ProjectionExpression: projectionExpression,
|
|
1396
|
+
Limit: returnSize,
|
|
1397
|
+
ScanIndexForward: scanAscending,
|
|
1398
|
+
ExclusiveStartKey: this.decodeCursor(cursor)
|
|
1399
|
+
}));
|
|
1400
|
+
const rawItems = (results.Items ?? []);
|
|
1401
|
+
let hasMore = false;
|
|
1402
|
+
if (rawItems.length === returnSize && !Is.empty(results.LastEvaluatedKey)) {
|
|
1403
|
+
const probe = await connection.send(new QueryCommand({
|
|
703
1404
|
TableName: this._config.tableName,
|
|
704
1405
|
IndexName: indexName,
|
|
705
1406
|
KeyConditionExpression: keyExpression,
|
|
706
|
-
FilterExpression: Is.stringValue(expressions.filterCondition)
|
|
707
|
-
? expressions.filterCondition
|
|
708
|
-
: undefined,
|
|
709
1407
|
ExpressionAttributeNames: attributeNames,
|
|
710
1408
|
ExpressionAttributeValues: attributeValues,
|
|
711
|
-
ProjectionExpression:
|
|
712
|
-
Limit:
|
|
1409
|
+
ProjectionExpression: projectionExpression,
|
|
1410
|
+
Limit: 1,
|
|
713
1411
|
ScanIndexForward: scanAscending,
|
|
714
|
-
ExclusiveStartKey:
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
});
|
|
718
|
-
const connection = this.createDocClient();
|
|
719
|
-
const results = await connection.send(query);
|
|
720
|
-
let entities = [];
|
|
721
|
-
if (Is.arrayValue(results.Items)) {
|
|
722
|
-
entities = results.Items.map(item => {
|
|
723
|
-
const unmarshalled = unmarshall(item);
|
|
724
|
-
delete unmarshalled[DynamoDbEntityStorageConnector._PARTITION_KEY];
|
|
725
|
-
return unmarshalled;
|
|
726
|
-
});
|
|
727
|
-
}
|
|
728
|
-
return {
|
|
729
|
-
entities,
|
|
730
|
-
cursor: Is.empty(results.LastEvaluatedKey)
|
|
731
|
-
? undefined
|
|
732
|
-
: Converter.bytesToBase64(ObjectHelper.toBytes(results.LastEvaluatedKey))
|
|
733
|
-
};
|
|
734
|
-
}
|
|
735
|
-
catch (err) {
|
|
736
|
-
if (BaseError.isErrorCode(err, "ResourceNotFoundException")) {
|
|
737
|
-
throw new GeneralError(DynamoDbEntityStorageConnector.CLASS_NAME, "tableDoesNotExist", {
|
|
738
|
-
tableName: this._config.tableName
|
|
739
|
-
}, err);
|
|
740
|
-
}
|
|
741
|
-
throw new GeneralError(DynamoDbEntityStorageConnector.CLASS_NAME, "queryFailed", undefined, err);
|
|
1412
|
+
ExclusiveStartKey: results.LastEvaluatedKey
|
|
1413
|
+
}));
|
|
1414
|
+
hasMore = (probe.Items?.length ?? 0) > 0;
|
|
742
1415
|
}
|
|
1416
|
+
return {
|
|
1417
|
+
rawItems,
|
|
1418
|
+
cursor: hasMore ? this.encodeCursor(results.LastEvaluatedKey) : undefined
|
|
1419
|
+
};
|
|
1420
|
+
}
|
|
1421
|
+
/**
|
|
1422
|
+
* Execute a query path with filter expression, continuing until page is full or exhausted.
|
|
1423
|
+
* @param keyExpression The key condition expression.
|
|
1424
|
+
* @param filterExpression The filter expression.
|
|
1425
|
+
* @param attributeNames The expression attribute names.
|
|
1426
|
+
* @param attributeValues The expression attribute values.
|
|
1427
|
+
* @param projectionExpression The projection expression.
|
|
1428
|
+
* @param indexName The optional index name.
|
|
1429
|
+
* @param scanAscending The scan direction.
|
|
1430
|
+
* @param cursor The optional cursor.
|
|
1431
|
+
* @param returnSize The requested page size.
|
|
1432
|
+
* @returns Raw items and an optional cursor.
|
|
1433
|
+
* @internal
|
|
1434
|
+
*/
|
|
1435
|
+
async executeFilteredQuery(keyExpression, filterExpression, attributeNames, attributeValues, projectionExpression, indexName, scanAscending, cursor, returnSize) {
|
|
1436
|
+
const connection = this.createDocClient();
|
|
1437
|
+
const returnedRawItems = [];
|
|
1438
|
+
let lastEvaluatedKey = this.decodeCursor(cursor);
|
|
1439
|
+
do {
|
|
1440
|
+
const results = await connection.send(new QueryCommand({
|
|
1441
|
+
TableName: this._config.tableName,
|
|
1442
|
+
IndexName: indexName,
|
|
1443
|
+
KeyConditionExpression: keyExpression,
|
|
1444
|
+
FilterExpression: filterExpression,
|
|
1445
|
+
ExpressionAttributeNames: attributeNames,
|
|
1446
|
+
ExpressionAttributeValues: attributeValues,
|
|
1447
|
+
ProjectionExpression: projectionExpression,
|
|
1448
|
+
Limit: returnSize - returnedRawItems.length,
|
|
1449
|
+
ScanIndexForward: scanAscending,
|
|
1450
|
+
ExclusiveStartKey: lastEvaluatedKey
|
|
1451
|
+
}));
|
|
1452
|
+
returnedRawItems.push(...(results.Items ?? []));
|
|
1453
|
+
lastEvaluatedKey = results.LastEvaluatedKey;
|
|
1454
|
+
} while (returnedRawItems.length < returnSize && !Is.empty(lastEvaluatedKey));
|
|
1455
|
+
return {
|
|
1456
|
+
rawItems: returnedRawItems,
|
|
1457
|
+
cursor: this.encodeCursor(lastEvaluatedKey)
|
|
1458
|
+
};
|
|
1459
|
+
}
|
|
1460
|
+
/**
|
|
1461
|
+
* Convert raw DynamoDB items into connector entities.
|
|
1462
|
+
* @param rawItems Raw DynamoDB items.
|
|
1463
|
+
* @returns The mapped entities.
|
|
1464
|
+
* @internal
|
|
1465
|
+
*/
|
|
1466
|
+
mapRawItemsToEntities(rawItems) {
|
|
1467
|
+
return rawItems.map(item => {
|
|
1468
|
+
const unmarshalled = unmarshall(item);
|
|
1469
|
+
return EntityStorageHelper.unPrepareEntity(unmarshalled, [
|
|
1470
|
+
DynamoDbEntityStorageConnector._PARTITION_KEY
|
|
1471
|
+
]);
|
|
1472
|
+
});
|
|
743
1473
|
}
|
|
744
1474
|
/**
|
|
745
1475
|
* Build the condition expression for the query.
|
|
@@ -773,5 +1503,26 @@ export class DynamoDbEntityStorageConnector {
|
|
|
773
1503
|
}
|
|
774
1504
|
return { conditionExpression, attributeNames, attributeValues };
|
|
775
1505
|
}
|
|
1506
|
+
/**
|
|
1507
|
+
* Build a ProjectionExpression string and register a safe alias in attributeNames for every
|
|
1508
|
+
* projected property, preventing ValidationException when a property name is a DynamoDB
|
|
1509
|
+
* reserved word (e.g. "role", "name", "status").
|
|
1510
|
+
* @param properties The properties to project, or undefined to return all attributes.
|
|
1511
|
+
* @param attributeNames The expression attribute names map to mutate with the aliases.
|
|
1512
|
+
* @returns The ProjectionExpression string, or undefined when no projection is needed.
|
|
1513
|
+
* @internal
|
|
1514
|
+
*/
|
|
1515
|
+
buildProjectionExpression(properties, attributeNames) {
|
|
1516
|
+
if (!Is.arrayValue(properties)) {
|
|
1517
|
+
return undefined;
|
|
1518
|
+
}
|
|
1519
|
+
return properties
|
|
1520
|
+
.map(p => {
|
|
1521
|
+
const alias = `#p_${p}`;
|
|
1522
|
+
attributeNames[alias] = p;
|
|
1523
|
+
return alias;
|
|
1524
|
+
})
|
|
1525
|
+
.join(", ");
|
|
1526
|
+
}
|
|
776
1527
|
}
|
|
777
1528
|
//# sourceMappingURL=dynamoDbEntityStorageConnector.js.map
|