@squiz/db-lib 1.75.1 → 1.77.0

package/src/dynamodb/AbstractDynamoDbRepository.spec.ts

@@ -21,7 +21,8 @@ import {
 } from '@aws-sdk/lib-dynamodb';
 import { ConditionalCheckFailedException } from '@aws-sdk/client-dynamodb';
 import { DynamoDbManager, Transaction } from './DynamoDbManager';
-import { MissingKeyValuesError, InvalidDbSchemaError } from '..';
+import { MissingKeyValuesError } from '../error/MissingKeyValuesError';
+import { InvalidDbSchemaError } from '../error/InvalidDbSchemaError';
 
 import { mockClient } from 'aws-sdk-client-mock';
 import 'aws-sdk-client-mock-jest';
@@ -77,8 +78,32 @@ class TestItem implements ITestItem {
   }
 }
 
+interface ITestItem2 {
+  date: string;
+  info: {
+    id: string;
+    name: string;
+    age: number;
+    address: {
+      country: string;
+      city?: string;
+    };
+  };
+}
+
+class TestItem2 implements ITestItem2 {
+  public date: string;
+  public info: ITestItem2['info'];
+
+  constructor(data: Partial<ITestItem2> = {}) {
+    this.date = data.date ?? '';
+    this.info = data.info as ITestItem2['info'];
+  }
+}
+
 export type TestRepositories = {
   testItem: TestItemRepository;
+  testItem2: TestItem2Repository;
 };
 export type TestDbManager = DynamoDbManager<TestRepositories>;
 
@@ -123,15 +148,49 @@ const TEST_ITEM_ENTITY_DEFINITION = {
   optionalIndexes: ['gsi2_pk-gsi2_sk-index'],
 };
 
+const TEST_ITEM2_ENTITY_NAME = 'test-item-entity2';
+const TEST_ITEM2_ENTITY_DEFINITION = {
+  keys: {
+    pk: {
+      format: 'c#{info.address.country}',
+      attributeName: 'pk',
+    },
+    sk: {
+      format: 'ci#{info.address.city}#{info.id}',
+      attributeName: 'sk',
+    },
+  },
+  indexes: {
+    'gsi1_pk-gsi1_sk-index': {
+      pk: {
+        format: 'dc#{date}#{info.address.country}',
+        attributeName: 'gsi1_pk',
+      },
+      sk: {
+        format: '#meta',
+        attributeName: 'gsi1_sk',
+      },
+    },
+  },
+  fieldsAsJsonString: [],
+};
+
 class TestItemRepository extends AbstractDynamoDbRepository<ITestItem, TestItem> {
   constructor(tableName: string, dbManager: TestDbManager) {
     super(tableName, dbManager, TEST_ITEM_ENTITY_NAME, TEST_ITEM_ENTITY_DEFINITION, TestItem);
   }
 }
 
+class TestItem2Repository extends AbstractDynamoDbRepository<ITestItem2, TestItem2> {
+  constructor(tableName: string, dbManager: TestDbManager) {
+    super(tableName, dbManager, TEST_ITEM2_ENTITY_NAME, TEST_ITEM2_ENTITY_DEFINITION, TestItem2);
+  }
+}
+
 const ddbManager = new DynamoDbManager<TestRepositories>(ddbDoc, (dbManager: TestDbManager) => {
   return {
     testItem: new TestItemRepository(TABLE_NAME, dbManager),
+    testItem2: new TestItem2Repository(TABLE_NAME, dbManager),
   };
 });
 
@@ -1658,4 +1717,84 @@ describe('AbstractRepository', () => {
       });
     });
   });
+
+  describe('Entity definitions having keys with the object properties', () => {
+    it('should use the correct keys in the request', async () => {
+      const repository = new TestItem2Repository(TABLE_NAME, ddbManager);
+      ddbClientMock.on(PutCommand).resolves({
+        $metadata: {
+          httpStatusCode: 200,
+        },
+      });
+      const input: PutCommandInput = {
+        TableName: TABLE_NAME,
+        Item: {
+          pk: 'c#au',
+          sk: 'ci#syd#id-99',
+          gsi1_pk: 'dc#2025-02-26#au',
+          gsi1_sk: '#meta',
+
+          date: '2025-02-26',
+          info: {
+            id: 'id-99',
+            name: 'john',
+            age: 42,
+            address: {
+              country: 'au',
+              city: 'syd',
+            },
+          },
+        },
+        ConditionExpression: `attribute_not_exists(pk)`,
+      };
+
+      const item = {
+        date: '2025-02-26',
+        info: {
+          id: 'id-99',
+          name: 'john',
+          age: 42,
+          address: {
+            country: 'au',
+            city: 'syd',
+          },
+        },
+      };
+      const result = await repository.createItem(item);
+      expect(ddbClientMock).toHaveReceivedCommandWith(PutCommand, input);
+      expect(result).toEqual(
+        new TestItem2({
+          date: '2025-02-26',
+          info: {
+            id: 'id-99',
+            name: 'john',
+            age: 42,
+            address: {
+              country: 'au',
+              city: 'syd',
+            },
+          },
+        }),
+      );
+    });
+    it('should throw error if input does not includes key field(s)', async () => {
+      const repository = new TestItem2Repository(TABLE_NAME, ddbManager);
+      const partialItem = {
+        date: '2025-02-26',
+        info: {
+          id: 'id-99',
+          name: 'john',
+          age: 42,
+          address: {
+            country: 'au',
+          },
+        },
+      };
+      await expect(repository.getItem(partialItem)).rejects.toEqual(
+        new MissingKeyValuesError(
+          'Key field "info.address.city" must be specified in the input item in entity test-item-entity2',
+        ),
+      );
+    });
+  });
 });

package/src/dynamodb/AbstractDynamoDbRepository.ts

@@ -10,7 +10,9 @@ import {
   BatchGetCommandOutput,
 } from '@aws-sdk/lib-dynamodb';
 
-import { Transaction, DynamoDbManager, MissingKeyValuesError, InvalidDbSchemaError } from '..';
+import { Transaction, DynamoDbManager } from './DynamoDbManager';
+import { MissingKeyValuesError } from '../error/MissingKeyValuesError';
+import { InvalidDbSchemaError } from '../error/InvalidDbSchemaError';
 import { InvalidDataFormatError } from '../error/InvalidDataFormatError';
 import { UnknownKeyAttributeError } from '../error/UnknownKeyAttributeError';
 
@@ -441,6 +443,7 @@ export abstract class AbstractDynamoDbRepository<SHAPE extends object, DATA_CLAS
     value: DATA_CLASS,
     transaction: Transaction = {},
     additionalValue: Partial<SHAPE> = {},
+    options: { overrideExisting?: boolean } = { overrideExisting: false },
   ): Promise<DATA_CLASS> {
     this.assertValueMatchesModel(value);
 
@@ -475,7 +478,7 @@ export abstract class AbstractDynamoDbRepository<SHAPE extends object, DATA_CLAS
         ...keyFields,
         ...columns,
       },
-      ConditionExpression: `attribute_not_exists(${this.keys.pk.attributeName})`,
+      ConditionExpression: options.overrideExisting ? undefined : `attribute_not_exists(${this.keys.pk.attributeName})`,
     };
 
     if (transaction.id?.length) {
@@ -620,27 +623,34 @@ export abstract class AbstractDynamoDbRepository<SHAPE extends object, DATA_CLAS
       );
     }
 
-    const matches = keyFormat.match(/{[a-zA-Z]+?}/g);
-    const replacements: { property: keyof SHAPE; placeholder: string }[] = !matches
+    const matches = keyFormat.match(/{[a-zA-Z\\.]+?}/g);
+    const replacements: { property: string; placeholder: string }[] = !matches
       ? []
       : matches.map((match) => {
           return {
-            property: match.slice(1, -1) as keyof SHAPE,
+            property: match.slice(1, -1),
             placeholder: match,
           };
         });
 
     for (let i = 0; i < replacements.length; i++) {
-      const field = replacements[i].property;
-      if (item[field] === undefined) {
-        throw new MissingKeyValuesError(
-          `Key field "${String(field)}" must be specified in the input item in entity ${this.entityName}`,
-        );
+      let value = JSON.parse(JSON.stringify(item));
+      const fields = replacements[i].property.split('.');
+      for (let j = 0; j < fields.length; j++) {
+        const field = fields[j];
+        value = value[field];
+        if (value === undefined) {
+          throw new MissingKeyValuesError(
+            `Key field "${String(replacements[i].property)}" must be specified in the input item in entity ${
+              this.entityName
+            }`,
+          );
+        }
       }
-      keyFormat = keyFormat.replace(replacements[i].placeholder, String(item[field] ?? ''));
+      keyFormat = keyFormat.replace(replacements[i].placeholder, String(value ?? ''));
     }
 
-    const moreMatches = keyFormat.match(/{[a-zA-Z]+?}/g);
+    const moreMatches = keyFormat.match(/{[a-zA-Z\\.]+?}/g);
     if (moreMatches?.length) {
       throw new MissingKeyValuesError(
         `Cannot resolve key placeholder(s) for key attribute format '${this.keysFormat[attributeName]} in entity ${

package/src/externalized/ExternalizedDynamoDbRepository.spec.ts

@@ -0,0 +1,274 @@
+import { ExternalizedDynamoDbRepository } from './ExternalizedDynamoDbRepository';
+import { S3ExternalStorage, S3StorageLocation } from '../s3/S3ExternalStorage';
+import { DynamoDbManager } from '../dynamodb/DynamoDbManager';
+import { EntityDefinition } from '../dynamodb/AbstractDynamoDbRepository';
+
+// Mock model class for testing
+interface TestItemShape {
+  id: string;
+  name: string;
+  data: Array<{ key: string; value: object }>;
+}
+
+class TestItem implements TestItemShape {
+  id: string;
+  name: string;
+  data: Array<{ key: string; value: object }>;
+  storageLocation?: S3StorageLocation;
+
+  constructor(input: Record<string, unknown> = {}) {
+    this.id = (input.id as string) || '';
+    this.name = (input.name as string) || '';
+    this.data = (input.data as Array<{ key: string; value: object }>) || [];
+    // Note: storageLocation is intentionally not set from input to simulate model stripping it
+  }
+}
+
+// Test entity definition
+const testEntityDefinition: EntityDefinition = {
+  keys: {
+    pk: {
+      attributeName: 'pk',
+      format: 'ITEM#${id}',
+    },
+    sk: {
+      attributeName: 'sk',
+      format: 'ITEM#${id}',
+    },
+  },
+  indexes: {},
+  fieldsAsJsonString: ['data'],
+};
+
+// Concrete test repository
+class TestRepo extends ExternalizedDynamoDbRepository<TestItemShape, TestItem> {
+  public mockSuperUpdateItem?: jest.Mock;
+
+  constructor(dbManager: DynamoDbManager<any>, storage: S3ExternalStorage, overrides: Partial<EntityDefinition> = {}) {
+    super('test-table', dbManager, 'test_item', { ...testEntityDefinition, ...overrides }, TestItem, storage);
+  }
+
+  // Override updateItem to allow mocking super.updateItem
+  public async updateItem(value: Partial<TestItemShape>): Promise<TestItem | undefined> {
+    if (this.mockSuperUpdateItem) {
+      // Use mock if provided
+      const previousLocation = await (this as any).fetchStoredLocation(value);
+      const updatedItem = await this.mockSuperUpdateItem(value);
+      if (!updatedItem) {
+        return undefined;
+      }
+      const newLocation = (updatedItem as any).storageLocation;
+      if (previousLocation && previousLocation.key !== newLocation?.key) {
+        await (this as any).storage.delete(previousLocation);
+      }
+      return updatedItem;
+    }
+    return super.updateItem(value);
+  }
+}
+
+const createDbManager = (clientOverrides: Record<string, jest.Mock> = {}) =>
+  ({
+    client: {
+      get: jest.fn(),
+      put: jest.fn(),
+      delete: jest.fn(),
+      update: jest.fn(),
+      ...clientOverrides,
+    },
+    repositories: {} as any,
+    executeInTransaction: async <T>(fn: (transaction: any) => Promise<T>): Promise<T> => fn({}),
+    addWriteTransactionItem: jest.fn(),
+  } as unknown as DynamoDbManager<any>);
+
+const createStorage = () =>
+  ({
+    save: jest.fn(),
+    load: jest.fn(),
+    delete: jest.fn(),
+  } as unknown as S3ExternalStorage);
+
+const createTestItem = (overrides: Partial<TestItem> = {}): TestItem => {
+  const item = new TestItem({
+    id: 'item-1',
+    name: 'Test',
+    data: [{ key: 'sample', value: { main: [] } }],
+    ...overrides,
+  });
+  return item;
+};
+
+describe('ExternalizedDynamoDbRepository', () => {
+  it('externalizes data to S3 when prepareValueForStorage is called', async () => {
+    const storage = {
+      ...createStorage(),
+      save: jest.fn().mockResolvedValue({ location: { type: 's3', key: 'item-key' }, size: 500 }),
+    } as unknown as S3ExternalStorage;
+    const repo = new TestRepo(createDbManager(), storage);
+    const item = createTestItem();
+
+    const storedValue = await (repo as any).prepareValueForStorage(item);
+
+    expect(storage.save).toHaveBeenCalled();
+    // storedValue is a minimal payload - original item is NOT mutated
+    expect(storedValue.data).toEqual([]);
+    expect(storedValue.storageLocation).toEqual({ type: 's3', key: 'item-key' });
+  });
+
+  it('creates item with storageLocation preserved', async () => {
+    const client = createDbManager().client as any;
+    (client.put as jest.Mock).mockResolvedValue({});
+    const storage = {
+      ...createStorage(),
+      save: jest.fn().mockResolvedValue({ location: { type: 's3', key: 'item-key' }, size: 500 }),
+    } as unknown as S3ExternalStorage;
+
+    const repo = new TestRepo({ client } as any, storage);
+    const item = createTestItem();
+
+    // Use prepareValueForStorage to externalize data and set storageLocation
+    const preparedItem = await (repo as any).prepareValueForStorage(item);
+
+    // Verify the prepared value has minimal payload (empty data)
+    expect(preparedItem.data).toEqual([]);
+    expect(preparedItem.storageLocation).toEqual({ type: 's3', key: 'item-key' });
+
+    // Original item should NOT be mutated
+    expect(item.storageLocation).toBeUndefined();
+  });
+
+  it('loads payload from external storage when storageLocation is set', async () => {
+    const storage = createStorage();
+    const fullData = {
+      id: 'item-1',
+      name: 'From S3',
+      data: [{ key: 'sample', value: { main: [] } }],
+    };
+    (storage.load as jest.Mock).mockResolvedValue(fullData);
+
+    const repo = new TestRepo(createDbManager({ get: jest.fn() as any }), storage);
+    const result = await (repo as any).hydrateFromExternalStorage({
+      id: 'item-1',
+      name: 'Placeholder',
+      data: [],
+      storageLocation: { type: 's3', key: 'item-key' },
+    });
+
+    expect(storage.load).toHaveBeenCalledWith({ type: 's3', key: 'item-key' });
+    expect(result?.name).toBe('From S3');
+    expect(result?.data).toEqual([{ key: 'sample', value: { main: [] } }]);
+  });
+
+  it('deletes externalized payloads when deleting items', async () => {
+    const client = createDbManager({
+      get: jest.fn().mockResolvedValue({ Item: { storageLocation: { type: 's3', key: 'item-key' } } }),
+      delete: jest.fn().mockResolvedValue({}),
+    }).client as any;
+    const storage = createStorage();
+    const repo = new TestRepo({ client } as any, storage);
+
+    await repo.deleteItem({ id: 'item-1' });
+
+    expect(storage.delete).toHaveBeenCalledWith({ type: 's3', key: 'item-key' });
+  });
+
+  it('returns inline data when no storageLocation is present', async () => {
+    const storage = createStorage();
+    const repo = new TestRepo(createDbManager({ get: jest.fn() as any }), storage);
+    const inlineRecord = {
+      id: 'item-1',
+      name: 'Inline Item',
+      data: [{ key: 'sample', value: { main: [] } }],
+    };
+
+    const result = await (repo as any).hydrateFromExternalStorage(inlineRecord);
+
+    expect(storage.load).not.toHaveBeenCalled();
+    expect(result).toBeDefined();
+    expect(result?.name).toBe('Inline Item');
+    expect(result?.data).toEqual([{ key: 'sample', value: { main: [] } }]);
+  });
+
+  it('cleans up old S3 file when updating with new storageLocation', async () => {
+    const oldLocation = { type: 's3' as const, key: 'old-key' };
+    const newLocation = { type: 's3' as const, key: 'new-key' };
+
+    const client = createDbManager({
+      get: jest.fn().mockResolvedValue({
+        Item: {
+          id: 'item-1',
+          name: 'Test',
+          storageLocation: oldLocation,
+        },
+      }),
+      update: jest.fn().mockResolvedValue({
+        Attributes: {
+          id: 'item-1',
+          name: 'Updated',
+          storageLocation: newLocation,
+        },
+      }),
+    }).client as any;
+
+    const storage = createStorage();
+    const repo = new TestRepo({ client } as any, storage);
+
+    // Update without storageLocation (it will be fetched from DynamoDB)
+    const updateValue = {
+      id: 'item-1',
+      name: 'Updated',
+    };
+
+    // Create an updated item with the new storageLocation
+    const updatedItem = createTestItem({ id: 'item-1', name: 'Updated' });
+    (updatedItem as any).storageLocation = newLocation;
+
+    // Mock super.updateItem using the custom mock hook
+    repo.mockSuperUpdateItem = jest.fn().mockResolvedValue(updatedItem as any);
+
+    await repo.updateItem(updateValue);
+
+    expect(storage.delete).toHaveBeenCalledWith(oldLocation);
+  });
+
+  it('does not delete S3 file when storageLocation is unchanged', async () => {
+    const location = { type: 's3' as const, key: 'same-key' };
+
+    const client = createDbManager({
+      get: jest.fn().mockResolvedValue({
+        Item: {
+          id: 'item-1',
+          name: 'Test',
+          storageLocation: location,
+        },
+      }),
+      update: jest.fn().mockResolvedValue({
+        Attributes: {
+          id: 'item-1',
+          name: 'Updated',
+          storageLocation: location,
+        },
+      }),
+    }).client as any;
+
+    const storage = createStorage();
+    const repo = new TestRepo({ client } as any, storage);
+
+    // Update without storageLocation (it will be fetched from DynamoDB)
+    const updateValue = {
+      id: 'item-1',
+      name: 'Updated',
+    };
+
+    // Create an updated item with the same storageLocation
+    const updatedItem = createTestItem({ id: 'item-1', name: 'Updated' });
+    (updatedItem as any).storageLocation = location;
+
+    // Mock super.updateItem using the custom mock hook
+    repo.mockSuperUpdateItem = jest.fn().mockResolvedValue(updatedItem as any);
+
+    await repo.updateItem(updateValue);
+
+    expect(storage.delete).not.toHaveBeenCalled();
+  });
+});