@squiz/db-lib 1.71.2 → 1.72.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@squiz/db-lib",
-  "version": "1.71.2",
+  "version": "1.72.0",
   "description": "",
   "main": "lib/index.js",
   "private": false,
@@ -8,7 +8,7 @@
     "access": "public"
   },
   "scripts": {
-    "compile": "node build.js && tsc",
+    "compile": "tsc -b",
     "test": "jest -c jest.config.ts --testMatch=\"**/*.spec.ts\" --testMatch=\"!**/*.integration.spec.ts\" --passWithNoTests",
     "test:integration": "jest --testMatch=\"**/*.integration.spec.ts\"",
     "test:update-snapshots": "jest -c jest.config.ts --updateSnapshot",
@@ -19,7 +19,7 @@
   "devDependencies": {
     "@types/jest": "28.1.8",
     "@types/node": "20.12.4",
-    "@types/pg": "8.6.6",
+    "@types/pg": "^8.11.8",
     "aws-sdk-client-mock": "^4.0.0",
     "aws-sdk-client-mock-jest": "4.0.0",
     "eslint": "8.33.0",
@@ -36,9 +36,9 @@
     "@aws-sdk/client-secrets-manager": "3.651.1",
     "@aws-sdk/lib-dynamodb": "^3.651.1",
     "@opentelemetry/api": "^1.6.0",
-    "@squiz/dx-common-lib": "^1.66.4",
+    "@squiz/dx-common-lib": "^1.69.0",
     "@squiz/dx-logger-lib": "^1.64.0",
     "dotenv": "16.0.3",
-    "pg": "8.9.0"
+    "pg": "^8.12.0"
   }
 }

package/src/AbstractRepository.ts

@@ -1,4 +1,4 @@
-import { PoolClient, Pool } from 'pg';
+import { PoolClient, Pool, escapeIdentifier } from 'pg';
 import { Repositories } from './Repositories';
 import { InvalidUpdateValueError } from '@squiz/dx-common-lib';
 
@@ -56,19 +56,29 @@ export abstract class AbstractRepository<SHAPE extends object, DATA_CLASS extend
     return await this.pool.connect();
   }
 
+  private sanitiseValue(value: Partial<SHAPE>) {
+    const sanitisedValue = Object.entries(value)
+      .map(([key, value]) => [this.modelPropertyToSqlColumn[key as keyof SHAPE], value] as const)
+      .filter(([column, _v]) => !!column);
+
+    const columns = sanitisedValue.map(([column]) => escapeIdentifier(column));
+    const bindingParams = sanitisedValue.map((_, index) => `$${index + 1}`);
+    const values = sanitisedValue.map(([_column, value]) => value);
+
+    return {
+      columns,
+      bindingParams,
+      values,
+    };
+  }
+
   async create(value: SHAPE, transactionClient: PoolClient | null = null): Promise<SHAPE> {
     const valueAsClass = new this.classRef(value as Record<string, unknown>);
-    const columns = Object.keys(valueAsClass)
-      .map((a) => `"${this.modelPropertyToSqlColumn[a as keyof SHAPE]}"`)
-      .join(', ');
-
-    const values = Object.values(valueAsClass)
-      .map((a, index) => `$${index + 1}`)
-      .join(', ');
+    const { columns, bindingParams, values } = this.sanitiseValue(valueAsClass);
 
     const result = await this.executeQuery(
-      `INSERT INTO ${this.tableName} (${columns}) VALUES (${values}) RETURNING *`,
-      Object.values(valueAsClass),
+      `INSERT INTO ${this.tableName} (${columns.join(', ')}) VALUES (${bindingParams.join(', ')}) RETURNING *`,
+      values,
       transactionClient,
     );
 
@@ -84,20 +94,16 @@ export abstract class AbstractRepository<SHAPE extends object, DATA_CLASS extend
       throw new InvalidUpdateValueError('Failed updating the repository, update values cannot be empty');
     }
     const whereValues = Object.values(where);
-    const newValues = Object.values(newValue);
-
-    const setString = Object.keys(newValue)
-      .map((a, index) => `"${this.modelPropertyToSqlColumn[a as keyof SHAPE]}" = $${index + 1}`)
-      .join(', ');
-
-    const whereString = this.createWhereStringFromPartialModel(where, newValues.length);
+    const newValues = this.sanitiseValue(newValue);
+    const setValues = newValues.columns.map((c, i) => `${c} = ${newValues.bindingParams[i]}`);
+    const whereString = this.createWhereStringFromPartialModel(where, setValues.length);
 
     const result = await this.executeQuery(
       `UPDATE ${this.tableName} 
-        SET ${setString} 
+        SET ${setValues.join(', ')} 
         WHERE ${whereString}
         RETURNING *`,
-      [...Object.values(newValues), ...Object.values(whereValues)],
+      [...newValues.values, ...whereValues],
       transactionClient,
     );
 
@@ -112,7 +118,7 @@ export abstract class AbstractRepository<SHAPE extends object, DATA_CLASS extend
 
       const result = await client.query(`DELETE FROM ${this.tableName} WHERE ${whereString}`, Object.values(where));
 
-      return result.rowCount;
+      return result.rowCount ?? 0;
     } finally {
       if (client && !transactionClient) {
         client.release();

package/src/dynamodb/AbstractDynamoDbRepository.spec.ts

@@ -1,4 +1,4 @@
-import { AbstractDynamoDbRepository } from './AbstractDynamoDbRepository';
+import { AbstractDynamoDbRepository, QueryOptions } from './AbstractDynamoDbRepository';
 import {
   DeleteCommand,
   DeleteCommandInput,
@@ -14,6 +14,8 @@ import {
   UpdateCommandInput,
   TransactWriteCommand,
   TransactWriteCommandInput,
+  BatchGetCommand,
+  BatchGetCommandInput,
 } from '@aws-sdk/lib-dynamodb';
 import { ConditionalCheckFailedException } from '@aws-sdk/client-dynamodb';
 import { DynamoDbManager, Transaction } from './DynamoDbManager';
@@ -245,13 +247,11 @@ describe('AbstractRepository', () => {
         ConditionExpression: `attribute_exists(pk)`,
       };
 
-      const partialItemWithKeyFields = {
-        name: 'foo',
-      };
       const updateItem = {
+        name: 'foo',
         country: 'au-updated',
       };
-      const result = await repository.updateItem(partialItemWithKeyFields, updateItem);
+      const result = await repository.updateItem(updateItem);
       expect(ddbClientMock).toHaveReceivedNthCommandWith(2, UpdateCommand, input);
       expect(result).toEqual(
         new TestItem({
@@ -263,20 +263,49 @@ describe('AbstractRepository', () => {
       );
     });
 
-    it('should undefined if item does does not exist', async () => {
+    it('should not trigger update request if the input attributes are same as in the existing item', async () => {
       ddbClientMock.on(GetCommand).resolves({
         $metadata: {
           httpStatusCode: 200,
         },
+        Item: {
+          name: 'foo',
+          age: 99,
+          country: 'au',
+          data: {},
+        },
       });
+      ddbClientMock.on(UpdateCommand).rejects(new Error('updateItem() called when not expected'));
 
-      const partialItemWithKeyFields = {
+      // update input attributes are same as in the existing item
+      const updateItem = {
         name: 'foo',
+        country: 'au',
       };
+
+      const result = await repository.updateItem(updateItem);
+      expect(result).toEqual(
+        new TestItem({
+          name: 'foo',
+          age: 99,
+          country: 'au',
+          data: {},
+        }),
+      );
+    });
+
+    it('should return undefined if item does does not exist', async () => {
+      ddbClientMock.on(GetCommand).resolves({
+        $metadata: {
+          httpStatusCode: 200,
+        },
+      });
+
       const updateItem = {
+        name: 'foo',
         country: 'au-updated',
       };
-      const result = await repository.updateItem(partialItemWithKeyFields, updateItem);
+      const result = await repository.updateItem(updateItem);
       expect(result).toEqual(undefined);
     });
 
@@ -300,13 +329,11 @@ describe('AbstractRepository', () => {
         }),
       );
 
-      const partialItemWithKeyFields = {
-        name: 'foo',
-      };
       const updateItem = {
+        name: 'foo',
         country: 'au-updated',
       };
-      const result = await repository.updateItem(partialItemWithKeyFields, updateItem);
+      const result = await repository.updateItem(updateItem);
       expect(result).toEqual(undefined);
     });
 
@@ -323,15 +350,13 @@ describe('AbstractRepository', () => {
         },
       });
 
-      const partialItemWithKeyFields = {
-        name: 'foo',
-      };
       const updateItem = {
+        name: 'foo',
         country: 61, // should be "string" type
       };
-      await expect(
-        repository.updateItem(partialItemWithKeyFields, updateItem as unknown as Partial<TestItem>),
-      ).rejects.toEqual(new Error('Invalid "country"'));
+      await expect(repository.updateItem(updateItem as unknown as Partial<TestItem>)).rejects.toEqual(
+        new Error('Invalid "country"'),
+      );
     });
 
     it('should throw error if excess column in input', async () => {
@@ -347,26 +372,22 @@ describe('AbstractRepository', () => {
         },
       });
 
-      const partialItemWithKeyFields = {
-        name: 'foo',
-      };
       const updateItem = {
+        name: 'foo',
         country: 'au-updated',
         extra: '',
       };
-      await expect(
-        repository.updateItem(partialItemWithKeyFields, updateItem as unknown as Partial<TestItem>),
-      ).rejects.toEqual(new InvalidDbSchemaError('Excess properties in entity test-item-entity: extra'));
+      await expect(repository.updateItem(updateItem as unknown as Partial<TestItem>)).rejects.toEqual(
+        new InvalidDbSchemaError('Excess properties in entity test-item-entity: extra'),
+      );
     });
 
     it('should throw error if input does not includes key field(s)', async () => {
-      const partialItemWithKeyFields = {
-        age: 99,
-      };
       const updateItem = {
+        age: 99,
         country: 'au-updated', // should be "string" type
       };
-      await expect(repository.updateItem(partialItemWithKeyFields, updateItem)).rejects.toEqual(
+      await expect(repository.updateItem(updateItem)).rejects.toEqual(
         new MissingKeyValuesError('Key field "name" must be specified in the input item in entity test-item-entity'),
       );
     });
@@ -473,6 +494,120 @@ describe('AbstractRepository', () => {
     });
   });
 
+  describe('getItems()', () => {
+    it('should use BatchGetItem to get result', async () => {
+      ddbClientMock.on(BatchGetCommand).resolves({
+        $metadata: {
+          httpStatusCode: 200,
+        },
+        Responses: {
+          [TABLE_NAME]: [
+            {
+              name: 'foo',
+              age: 99,
+              country: 'au',
+              data: {},
+              data2: '{"foo":"bar","num":123}',
+            },
+            {
+              name: 'foo2',
+              age: 999,
+              country: 'au',
+              data: {},
+              data2: '{"foo":"bar","num":123}',
+            },
+          ],
+        },
+      });
+      const input: BatchGetCommandInput = {
+        RequestItems: {
+          [TABLE_NAME]: {
+            Keys: [
+              { pk: 'test_item#foo', sk: '#meta' },
+              { pk: 'test_item#foo2', sk: '#meta' },
+            ],
+          },
+        },
+      };
+
+      const requestItems = [{ name: 'foo' }, { name: 'foo2' }];
+      const result = await repository.getItems(requestItems);
+      expect(ddbClientMock).toHaveReceivedCommandWith(BatchGetCommand, input);
+      expect(result).toEqual([
+        new TestItem({
+          name: 'foo',
+          age: 99,
+          country: 'au',
+          data: {},
+          data2: {
+            foo: 'bar',
+            num: 123,
+          },
+        }),
+        new TestItem({
+          name: 'foo2',
+          age: 999,
+          country: 'au',
+          data: {},
+          data2: {
+            foo: 'bar',
+            num: 123,
+          },
+        }),
+      ]);
+    });
+
+    it('should request BatchGetItem in batch of 100 items to get result', async () => {
+      ddbClientMock.on(BatchGetCommand).resolves({
+        $metadata: {
+          httpStatusCode: 200,
+        },
+      });
+
+      const requestItems = [];
+      for (let i = 0; i < 120; i++) {
+        requestItems.push({ name: `foo${i}` });
+      }
+      // keys for first batch request
+      const keys1 = [];
+      for (let i = 0; i < 100; i++) {
+        keys1.push({ pk: `test_item#foo${i}`, sk: '#meta' });
+      }
+      // keys for second batch request
+      const keys2 = [];
+      for (let i = 100; i < 120; i++) {
+        keys2.push({ pk: `test_item#foo${i}`, sk: '#meta' });
+      }
+
+      const input1: BatchGetCommandInput = {
+        RequestItems: {
+          [TABLE_NAME]: {
+            Keys: keys1,
+          },
+        },
+      };
+      const input2: BatchGetCommandInput = {
+        RequestItems: {
+          [TABLE_NAME]: {
+            Keys: keys2,
+          },
+        },
+      };
+
+      await repository.getItems(requestItems);
+      expect(ddbClientMock).toHaveReceivedCommandTimes(BatchGetCommand, 2);
+      expect(ddbClientMock).toHaveReceivedNthCommandWith(1, BatchGetCommand, input1);
+      expect(ddbClientMock).toHaveReceivedNthCommandWith(2, BatchGetCommand, input2);
+    });
+
+    it('should throw error if any input item does not includes key field(s)', async () => {
+      const requestItems = [{ name: 'foo' }, { age: 22 }];
+      await expect(repository.getItems(requestItems)).rejects.toEqual(
+        new MissingKeyValuesError('Key field "name" must be specified in the input item in entity test-item-entity'),
+      );
+    });
+  });
+
   describe('queryItems()', () => {
     it('should return the items if found', async () => {
       ddbClientMock.on(QueryCommand).resolves({
@@ -555,8 +690,7 @@ describe('AbstractRepository', () => {
       };
 
       const partialItem = { name: 'foo' };
-      const useSortKey = true;
-      const _result = await repository.queryItems(partialItem, useSortKey);
+      const _result = await repository.queryItems(partialItem, { useSortKey: true });
       expect(ddbClientMock).toHaveReceivedCommandWith(QueryCommand, input);
     });
 
@@ -595,7 +729,7 @@ describe('AbstractRepository', () => {
 
       const partialItem = { country: 'au' };
       const useSortKey = false;
-      const result = await repository.queryItems(partialItem, useSortKey, index);
+      const result = await repository.queryItems(partialItem, { useSortKey, index });
       expect(ddbClientMock).toHaveReceivedCommandWith(QueryCommand, input);
       expect(result).toEqual([
         new TestItem({
@@ -636,14 +770,134 @@ describe('AbstractRepository', () => {
 
       const partialItem = { age: 99, country: 'au' };
       const useSortKey = true;
-      const _result = await repository.queryItems(partialItem, useSortKey, index);
+      const _result = await repository.queryItems(partialItem, { useSortKey, index });
+      expect(ddbClientMock).toHaveReceivedCommandWith(QueryCommand, input);
+    });
+
+    it('should set input query correctly when "filter - begins_with" query option is set', async () => {
+      ddbClientMock.on(QueryCommand).resolves({
+        $metadata: {
+          httpStatusCode: 200,
+        },
+      });
+      const input: QueryCommandInput = {
+        TableName: TABLE_NAME,
+        KeyConditionExpression: '#pkName = :pkValue AND begins_with(#skName, :skValue)',
+        ExpressionAttributeNames: {
+          '#pkName': 'pk',
+          '#skName': 'sk',
+        },
+        ExpressionAttributeValues: {
+          ':pkValue': 'test_item#foo',
+          ':skValue': 'keyword-x',
+        },
+      };
+
+      const partialItem = { name: 'foo' };
+      const queryOptions: QueryOptions = {
+        filter: {
+          type: 'begins_with',
+          keyword: 'keyword-x',
+        },
+      };
+      await repository.queryItems(partialItem, queryOptions);
+      expect(ddbClientMock).toHaveReceivedCommandWith(QueryCommand, input);
+    });
+
+    it('should throw error invalid "filter" query option is set', async () => {
+      ddbClientMock.on(QueryCommand).resolves({
+        $metadata: {
+          httpStatusCode: 200,
+        },
+      });
+      const partialItem = { name: 'foo' };
+      const queryOptions = {
+        filter: {
+          type: 'invalid-type',
+          keyword: 'keyword-x',
+        },
+      } as unknown as QueryOptions;
+      await expect(repository.queryItems(partialItem, queryOptions)).rejects.toEqual(
+        new Error(`Invalid query filter type: invalid-type`),
+      );
+    });
+
+    it('should set input query correctly when "limit" query option is set', async () => {
+      ddbClientMock.on(QueryCommand).resolves({
+        $metadata: {
+          httpStatusCode: 200,
+        },
+      });
+      const input: QueryCommandInput = {
+        TableName: TABLE_NAME,
+        KeyConditionExpression: '#pkName = :pkValue',
+        ExpressionAttributeNames: {
+          '#pkName': 'pk',
+        },
+        ExpressionAttributeValues: {
+          ':pkValue': 'test_item#foo',
+        },
+        Limit: 33,
+      };
+
+      const partialItem = { name: 'foo' };
+      const queryOptions = { limit: 33 };
+      await repository.queryItems(partialItem, queryOptions);
+      expect(ddbClientMock).toHaveReceivedCommandWith(QueryCommand, input);
+    });
+
+    it('should set input query correctly when "order asc" query option is set', async () => {
+      ddbClientMock.on(QueryCommand).resolves({
+        $metadata: {
+          httpStatusCode: 200,
+        },
+      });
+      const input: QueryCommandInput = {
+        TableName: TABLE_NAME,
+        KeyConditionExpression: '#pkName = :pkValue',
+        ExpressionAttributeNames: {
+          '#pkName': 'pk',
+        },
+        ExpressionAttributeValues: {
+          ':pkValue': 'test_item#foo',
+        },
+        ScanIndexForward: true,
+      };
+
+      const partialItem = { name: 'foo' };
+      const queryOptions = { order: 'asc' } as unknown as QueryOptions;
+      await repository.queryItems(partialItem, queryOptions);
+      expect(ddbClientMock).toHaveReceivedCommandWith(QueryCommand, input);
+    });
+
+    it('should set input query correctly when "order desc" query option is set', async () => {
+      ddbClientMock.on(QueryCommand).resolves({
+        $metadata: {
+          httpStatusCode: 200,
+        },
+      });
+      const input: QueryCommandInput = {
+        TableName: TABLE_NAME,
+        KeyConditionExpression: '#pkName = :pkValue',
+        ExpressionAttributeNames: {
+          '#pkName': 'pk',
+        },
+        ExpressionAttributeValues: {
+          ':pkValue': 'test_item#foo',
+        },
+        ScanIndexForward: false,
+      };
+
+      const partialItem = { name: 'foo' };
+      const queryOptions = { order: 'desc' } as unknown as QueryOptions;
+      await repository.queryItems(partialItem, queryOptions);
       expect(ddbClientMock).toHaveReceivedCommandWith(QueryCommand, input);
     });
 
     it('should throw error for invalid index query', async () => {
       const index = 'undefined-index';
       const partialItem = { age: 99, country: 'au' };
-      await expect(repository.queryItems(partialItem, false, index)).rejects.toEqual(
+      await expect(repository.queryItems(partialItem, { index })).rejects.toEqual(
         new MissingKeyValuesError(`Table index '${index}' not defined on entity test-item-entity`),
       );
     });
@@ -659,7 +913,7 @@ describe('AbstractRepository', () => {
       const partialItem = { name: 'foo' };
       const useSortKey = false;
       const index = 'gsi1_pk-gsi1_sk-index';
-      await expect(repository.queryItems(partialItem, useSortKey, index)).rejects.toEqual(
+      await expect(repository.queryItems(partialItem, { useSortKey, index })).rejects.toEqual(
         new MissingKeyValuesError(`Key field "country" must be specified in the input item in entity test-item-entity`),
       );
     });
@@ -668,7 +922,7 @@ describe('AbstractRepository', () => {
       const partialItem = { country: 'au' };
       const useSortKey = true;
       const index = 'gsi1_pk-gsi1_sk-index';
-      await expect(repository.queryItems(partialItem, useSortKey, index)).rejects.toEqual(
+      await expect(repository.queryItems(partialItem, { useSortKey, index })).rejects.toEqual(
         new MissingKeyValuesError(`Key field "age" must be specified in the input item in entity test-item-entity`),
       );
     });
@@ -752,8 +1006,6 @@ describe('AbstractRepository', () => {
         await repository.updateItem(
           {
             name: 'foo2',
-          },
-          {
             age: 55,
           },
           transaction,