@squiz/db-lib 1.73.0 → 1.75.0

package/src/dynamodb/AbstractDynamoDbRepository.spec.ts

@@ -38,6 +38,7 @@ interface ITestItem {
   country: string;
   data?: object;
   data2?: object;
+  email?: string;
 }
 
 class TestItem implements ITestItem {
@@ -46,6 +47,7 @@ class TestItem implements ITestItem {
   public country: string;
   public data: object;
   public data2?: object;
+  public email?: string;
 
   constructor(data: Partial<ITestItem> = {}) {
     this.name = data.name ?? 'default name';
@@ -66,6 +68,12 @@ class TestItem implements ITestItem {
     if (typeof this.data !== 'object' || Array.isArray(this.data)) {
       throw Error('Invalid "data"');
     }
+    if (data.email !== undefined) {
+      if (typeof data.email !== 'string') {
+        throw Error('Invalid "name"');
+      }
+      this.email = data.email;
+    }
   }
 }
 
@@ -99,9 +107,20 @@ const TEST_ITEM_ENTITY_DEFINITION = {
         attributeName: 'gsi1_sk',
       },
     },
+    'gsi2_pk-gsi2_sk-index': {
+      pk: {
+        format: 'email#{email}',
+        attributeName: 'gsi2_pk',
+      },
+      sk: {
+        format: '#meta',
+        attributeName: 'gsi2_sk',
+      },
+    },
   },
   // field to be stored as JSON string
   fieldsAsJsonString: ['data2'],
+  optionalIndexes: ['gsi2_pk-gsi2_sk-index'],
 };
 
 class TestItemRepository extends AbstractDynamoDbRepository<ITestItem, TestItem> {
@@ -176,6 +195,61 @@ describe('AbstractRepository', () => {
       );
     });
 
+    it('should create and return the item with multiple gsi fields', async () => {
+      ddbClientMock.on(PutCommand).resolves({
+        $metadata: {
+          httpStatusCode: 200,
+        },
+      });
+      const input: PutCommandInput = {
+        TableName: TABLE_NAME,
+        Item: {
+          pk: 'test_item#foo',
+          sk: '#meta',
+          gsi1_pk: 'country#au',
+          gsi1_sk: 'age#99',
+          gsi2_pk: 'email#foo@bar.xyz',
+          gsi2_sk: '#meta',
+
+          name: 'foo',
+          age: 99,
+          country: 'au',
+          data: {},
+          // "data2" property is defined to be stored as JSON string
+          data2: '{"foo":"bar","num":123}',
+          email: 'foo@bar.xyz',
+        },
+        ConditionExpression: `attribute_not_exists(pk)`,
+      };
+
+      const item = {
+        name: 'foo',
+        age: 99,
+        country: 'au',
+        data: {},
+        data2: {
+          foo: 'bar',
+          num: 123,
+        },
+        email: 'foo@bar.xyz',
+      };
+      const result = await repository.createItem(item);
+      expect(ddbClientMock).toHaveReceivedCommandWith(PutCommand, input);
+      expect(result).toEqual(
+        new TestItem({
+          name: 'foo',
+          age: 99,
+          country: 'au',
+          data: {},
+          data2: {
+            foo: 'bar',
+            num: 123,
+          },
+          email: 'foo@bar.xyz',
+        }),
+      );
+    });
+
     it('should throw error if invalid input', async () => {
       const item = {
         name: 'foo',
@@ -232,6 +306,8 @@ describe('AbstractRepository', () => {
         Attributes: {
           name: 'foo',
           age: 99,
+          // country attribute is part of gsi key
+          // hence updating this will also update gsi key value
           country: 'au-updated',
           data: {},
         },
@@ -239,12 +315,14 @@ describe('AbstractRepository', () => {
       const input: UpdateCommandInput = {
         TableName: TABLE_NAME,
         Key: { pk: 'test_item#foo', sk: '#meta' },
-        UpdateExpression: 'SET #country = :country',
+        UpdateExpression: 'SET #country = :country, #gsi1_pk = :gsi1_pk',
         ExpressionAttributeNames: {
           '#country': 'country',
+          '#gsi1_pk': 'gsi1_pk',
         },
         ExpressionAttributeValues: {
           ':country': 'au-updated',
+          ':gsi1_pk': 'country#au-updated',
         },
         ConditionExpression: `attribute_exists(pk)`,
       };
@@ -265,6 +343,60 @@ describe('AbstractRepository', () => {
       );
     });
 
+    it('should only update the changed attributes', async () => {
+      ddbClientMock.on(GetCommand).resolves({
+        $metadata: {
+          httpStatusCode: 200,
+        },
+        Item: {
+          name: 'foo',
+          age: 99,
+          country: 'au',
+          data: {},
+        },
+      });
+      ddbClientMock.on(UpdateCommand).resolves({
+        $metadata: {
+          httpStatusCode: 200,
+        },
+        Attributes: {
+          name: 'foo',
+          age: 99,
+          country: 'au',
+          data: { active: true },
+        },
+      });
+      const input: UpdateCommandInput = {
+        TableName: TABLE_NAME,
+        Key: { pk: 'test_item#foo', sk: '#meta' },
+        UpdateExpression: 'SET #data = :data',
+        ExpressionAttributeNames: {
+          '#data': 'data',
+        },
+        ExpressionAttributeValues: {
+          ':data': { active: true },
+        },
+        ConditionExpression: `attribute_exists(pk)`,
+      };
+
+      const updateItem = {
+        name: 'foo',
+        age: 99,
+        // this is the only change attribute value
+        data: { active: true },
+      };
+      const result = await repository.updateItem(updateItem);
+      expect(ddbClientMock).toHaveReceivedNthCommandWith(2, UpdateCommand, input);
+      expect(result).toEqual(
+        new TestItem({
+          name: 'foo',
+          age: 99,
+          country: 'au',
+          data: { active: true },
+        }),
+      );
+    });
+
     it('should not trigger update request if the input attributes are same as in the existing item', async () => {
       ddbClientMock.on(GetCommand).resolves({
         $metadata: {
@@ -559,6 +691,68 @@ describe('AbstractRepository', () => {
       ]);
     });
 
+    it('should remove duplicate items in BatchGetItem request', async () => {
+      ddbClientMock.on(BatchGetCommand).resolves({
+        $metadata: {
+          httpStatusCode: 200,
+        },
+        Responses: {
+          [TABLE_NAME]: [
+            {
+              name: 'foo',
+              age: 99,
+              country: 'au',
+              data: {},
+              data2: '{"foo":"bar","num":123}',
+            },
+            {
+              name: 'foo2',
+              age: 999,
+              country: 'au',
+              data: {},
+              data2: '{"foo":"bar","num":123}',
+            },
+          ],
+        },
+      });
+      const input: BatchGetCommandInput = {
+        RequestItems: {
+          [TABLE_NAME]: {
+            Keys: [
+              { pk: 'test_item#foo', sk: '#meta' },
+              { pk: 'test_item#foo2', sk: '#meta' },
+            ],
+          },
+        },
+      };
+
+      const requestItems = [{ name: 'foo' }, { name: 'foo2' }, { name: 'foo2' }];
+      const result = await repository.getItems(requestItems);
+      expect(ddbClientMock).toHaveReceivedCommandWith(BatchGetCommand, input);
+      expect(result).toEqual([
+        new TestItem({
+          name: 'foo',
+          age: 99,
+          country: 'au',
+          data: {},
+          data2: {
+            foo: 'bar',
+            num: 123,
+          },
+        }),
+        new TestItem({
+          name: 'foo2',
+          age: 999,
+          country: 'au',
+          data: {},
+          data2: {
+            foo: 'bar',
+            num: 123,
+          },
+        }),
+      ]);
+    });
+
     it('should retry if unprocessed keys returned', async () => {
       const input1: BatchGetCommandInput = {
         RequestItems: {
@@ -792,6 +986,37 @@ describe('AbstractRepository', () => {
       expect(ddbClientMock).toHaveReceivedCommandWith(BatchWriteCommand, input);
     });
 
+    it('should remove duplicate items in batchWrite() request', async () => {
+      ddbClientMock.on(BatchWriteCommand).resolves({
+        $metadata: {
+          httpStatusCode: 200,
+        },
+        ItemCollectionMetrics: {
+          [TABLE_NAME]: [{}],
+        },
+      });
+      const input: BatchWriteCommandInput = {
+        RequestItems: {
+          [TABLE_NAME]: [
+            {
+              DeleteRequest: {
+                Key: { pk: 'test_item#foo', sk: '#meta' },
+              },
+            },
+            {
+              DeleteRequest: {
+                Key: { pk: 'test_item#foo2', sk: '#meta' },
+              },
+            },
+          ],
+        },
+      };
+
+      const requestItems = [{ name: 'foo' }, { name: 'foo2' }, { name: 'foo2' }, { name: 'foo' }];
+      await repository.deleteItems(requestItems);
+      expect(ddbClientMock).toHaveReceivedCommandWith(BatchWriteCommand, input);
+    });
+
     it('should use re-try if unprocessed items returned', async () => {
       const input1: BatchWriteCommandInput = {
         RequestItems: {
@@ -1391,16 +1616,18 @@ describe('AbstractRepository', () => {
               ConditionExpression: 'attribute_exists(pk)',
               ExpressionAttributeNames: {
                 '#age': 'age',
+                '#gsi1_sk': 'gsi1_sk',
               },
               ExpressionAttributeValues: {
                 ':age': 55,
+                ':gsi1_sk': 'age#55',
               },
               Key: {
                 pk: 'test_item#foo2',
                 sk: '#meta',
               },
               TableName: 'test-table',
-              UpdateExpression: 'SET #age = :age',
+              UpdateExpression: 'SET #age = :age, #gsi1_sk = :gsi1_sk',
             },
           },
           {

package/src/dynamodb/AbstractDynamoDbRepository.ts

@@ -12,6 +12,7 @@ import {
 
 import { Transaction, DynamoDbManager, MissingKeyValuesError, InvalidDbSchemaError } from '..';
 import { InvalidDataFormatError } from '../error/InvalidDataFormatError';
+import { UnknownKeyAttributeError } from '../error/UnknownKeyAttributeError';
 
 export type QueryFilterTypeBeginsWith = {
   type: 'begins_with';
@@ -68,6 +69,7 @@ export type EntityDefinition = {
   keys: TableKeys;
   indexes: TableIndexes;
   fieldsAsJsonString: string[];
+  optionalIndexes?: (keyof TableIndexes)[];
 };
 
 const MAX_REATTEMPTS = 3;
@@ -80,6 +82,7 @@ export abstract class AbstractDynamoDbRepository<SHAPE extends object, DATA_CLAS
   protected keys: TableKeys;
   protected indexes: TableIndexes;
   protected keysFormat: KeysFormat;
+  protected optionalIndexes: (keyof TableIndexes)[] = [];
 
   // fields listed in this property are stored as a JSON string value in db
   protected fieldsAsJsonString: string[];
@@ -105,6 +108,10 @@ export abstract class AbstractDynamoDbRepository<SHAPE extends object, DATA_CLAS
       const index = this.indexes[key];
       this.keysFormat[index.pk.attributeName] = index.pk.format;
       this.keysFormat[index.sk.attributeName] = index.sk.format;
+
+      if (entityDefinition.optionalIndexes?.includes(key)) {
+        this.optionalIndexes.push(key);
+      }
     });
   }
 
@@ -144,10 +151,11 @@ export abstract class AbstractDynamoDbRepository<SHAPE extends object, DATA_CLAS
   public async getItems(items: Partial<SHAPE>[]): Promise<DATA_CLASS[]> {
     // this is the maximum items allowed by BatchGetItem()
     const batchSize = 100;
+    const keys = this.getItemsKeys(items);
 
     let result: DATA_CLASS[] = [];
-    for (let i = 0; i < items.length; i += batchSize) {
-      const batchResult = await this.getBatchItems(items.slice(i, i + batchSize));
+    for (let i = 0; i < keys.length; i += batchSize) {
+      const batchResult = await this.getBatchItems(keys.slice(i, i + batchSize));
       result = result.concat(batchResult);
     }
     return result;
@@ -159,12 +167,12 @@ export abstract class AbstractDynamoDbRepository<SHAPE extends object, DATA_CLAS
    * @param items
    * @returns
    */
-  private async getBatchItems(items: Partial<SHAPE>[]): Promise<DATA_CLASS[]> {
+  private async getBatchItems(keys: { [key: string]: string }[]): Promise<DATA_CLASS[]> {
     let resultItems: DATA_CLASS[] = [];
 
     let requestKeys: BatchGetCommandInput['RequestItems'] = {
       [this.tableName]: {
-        Keys: this.getBatchKeys(items),
+        Keys: keys,
       },
     };
     let reattemptsCount = 0;
@@ -196,9 +204,9 @@ export abstract class AbstractDynamoDbRepository<SHAPE extends object, DATA_CLAS
     // this is the maximum items allowed by BatchWriteItem()
     const batchSize = 25;
 
-    for (let i = 0; i < items.length; i += batchSize) {
-      const keys = this.getBatchKeys(items.slice(i, i + batchSize));
-      await this.deleteBatchItems(keys);
+    const keys = this.getItemsKeys(items);
+    for (let i = 0; i < keys.length; i += batchSize) {
+      await this.deleteBatchItems(keys.slice(i, i + batchSize));
     }
   }
 
@@ -224,7 +232,7 @@ export abstract class AbstractDynamoDbRepository<SHAPE extends object, DATA_CLAS
     }
   }
 
-  private getBatchKeys(items: Partial<SHAPE>[]) {
+  private getItemsKeys(items: Partial<SHAPE>[]) {
     const keys: { [key: string]: string }[] = [];
     for (const item of items) {
       keys.push({
@@ -232,12 +240,17 @@ export abstract class AbstractDynamoDbRepository<SHAPE extends object, DATA_CLAS
         [this.keys.sk.attributeName]: this.getSk(item),
       });
     }
-    // keys.push({
-    //   [this.keys.pk.attributeName]: 'foo1',
-    //   [this.keys.sk.attributeName]: 'foo2',
-    // });
-
-    return keys;
+    // filter duplicate items keys
+    return keys.filter((key, index) => {
+      return (
+        index ===
+        keys.findIndex(
+          (key2) =>
+            key[this.keys.pk.attributeName] === key2[this.keys.pk.attributeName] &&
+            key[this.keys.sk.attributeName] === key2[this.keys.sk.attributeName],
+        )
+      );
+    });
   }
 
   /**
@@ -330,12 +343,15 @@ export abstract class AbstractDynamoDbRepository<SHAPE extends object, DATA_CLAS
     this.assertValueMatchesModel(value);
 
     this.convertSelectedValuesToJsonString(newValue);
+    this.convertSelectedValuesToJsonString(oldValue as Record<string, unknown>);
 
     const updateExpression = [];
     const expressionAttributeNames: Record<string, string> = {};
     const expressionAttributeValues: Record<string, unknown> = {};
+    const updatedAttributes: string[] = [];
     for (const modelProperty of Object.keys(newValue)) {
-      const propValue = newValue[modelProperty as keyof SHAPE] ?? null;
+      const propValue = newValue[modelProperty as keyof SHAPE];
+
       if (propValue === oldValue[modelProperty as keyof SHAPE]) {
         // don't need to update the properties that are unchanged
         continue;
@@ -347,19 +363,35 @@ export abstract class AbstractDynamoDbRepository<SHAPE extends object, DATA_CLAS
       updateExpression.push(`${propName} = ${propValuePlaceHolder}`);
       expressionAttributeNames[propName] = modelProperty;
       expressionAttributeValues[propValuePlaceHolder] = propValue;
+      updatedAttributes.push(modelProperty);
     }
-    if (!updateExpression.length) {
+    if (!updatedAttributes.length) {
       // nothing to update
       return value;
     }
 
+    // also update the gsi attributes if needed
+    Object.keys(this.indexes).forEach((key) => {
+      const index = this.indexes[key];
+      [index.pk.attributeName, index.sk.attributeName].forEach((keyAttributeName) => {
+        const keyFormat = this.keysFormat[keyAttributeName];
+        if (updatedAttributes.find((attr) => keyFormat.search(`{${attr}}`) !== -1)) {
+          const propName = `#${keyAttributeName}`;
+          const propValuePlaceHolder = `:${keyAttributeName}`;
+          updateExpression.push(`${propName} = ${propValuePlaceHolder}`);
+          expressionAttributeNames[propName] = keyAttributeName;
+          expressionAttributeValues[propValuePlaceHolder] = this.getKey(value, keyAttributeName);
+        }
+      });
+    });
+
     const updateCommandInput = {
       TableName: this.tableName,
       Key: {
         [this.keys.pk.attributeName]: this.getPk(newValue),
         [this.keys.sk.attributeName]: this.getSk(newValue),
       },
-      UpdateExpression: 'SET ' + updateExpression.join(','),
+      UpdateExpression: 'SET ' + updateExpression.join(', '),
       ExpressionAttributeValues: expressionAttributeValues,
       ExpressionAttributeNames: expressionAttributeNames,
       ConditionExpression: `attribute_exists(${this.keys.pk.attributeName})`,
@@ -423,12 +455,20 @@ export abstract class AbstractDynamoDbRepository<SHAPE extends object, DATA_CLAS
       [this.keys.sk.attributeName]: this.getSk({ ...value, ...additionalValue }),
     };
 
-    Object.keys(this.indexes).forEach((key) => {
-      const index = this.indexes[key];
-      keyFields[index.pk.attributeName] = this.getKey({ ...value, ...additionalValue }, index.pk.attributeName);
-      keyFields[index.sk.attributeName] = this.getKey({ ...value, ...additionalValue }, index.sk.attributeName);
-    });
-
+    for (const [key, index] of Object.entries(this.indexes)) {
+      try {
+        keyFields[index.pk.attributeName] = this.getKey({ ...value, ...additionalValue }, index.pk.attributeName);
+        keyFields[index.sk.attributeName] = this.getKey({ ...value, ...additionalValue }, index.sk.attributeName);
+      } catch (e) {
+        if ((e as Error).name === 'MissingKeyValuesError' && this.optionalIndexes.includes(key)) {
+          // ignore optional index fields missing error
+          delete keyFields[index.pk.attributeName];
+          delete keyFields[index.pk.attributeName];
+          continue;
+        }
+        throw e;
+      }
+    }
     const putCommandInput: PutCommandInput = {
       TableName: this.tableName,
       Item: {
@@ -575,7 +615,7 @@ export abstract class AbstractDynamoDbRepository<SHAPE extends object, DATA_CLAS
   protected getKey(item: Partial<SHAPE>, attributeName: keyof KeysFormat): string {
     let keyFormat = this.keysFormat[attributeName];
     if (keyFormat == undefined || !keyFormat.length) {
-      throw new MissingKeyValuesError(
+      throw new UnknownKeyAttributeError(
         `Key format not defined or empty for key attribute '${attributeName}' in entity ${this.entityName}`,
       );
     }
@@ -611,17 +651,6 @@ export abstract class AbstractDynamoDbRepository<SHAPE extends object, DATA_CLAS
     return keyFormat;
   }
 
-  /**
-   * Whether the given property name is part of the entity's pk/sk string
-   * @param propertyName
-   * @returns boolean
-   */
-  private isPropertyPartOfKeys(propertyName: string) {
-    if (this.keysFormat[this.keys.pk.attributeName].search(`{${propertyName}}`) !== -1) return true;
-    if (this.keysFormat[this.keys.sk.attributeName].search(`{${propertyName}}`) !== -1) return true;
-    return false;
-  }
-
   /**
    * Validate the data matches with "DATA_MODEL"
    * @param value

package/src/error/UnknownKeyAttributeError.ts

@@ -0,0 +1,8 @@
+import { InternalServerError } from '@squiz/dx-common-lib';
+
+export class UnknownKeyAttributeError extends InternalServerError {
+  name = 'UnknownKeyAttributeError';
+  constructor(message: string) {
+    super(message);
+  }
+}