@bitblit/ratchet-aws 6.0.146-alpha → 6.0.148-alpha

package/src/dao/s3-simple-dao.ts

@@ -0,0 +1,96 @@
+/*
+    Wrap S3 with an ability to store and retrieve objects cached as json files
+*/
+
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { StringRatchet } from '@bitblit/ratchet-common/lang/string-ratchet';
+import { SimpleDaoItem } from './simple-dao-item.js';
+import { DeleteObjectOutput, PutObjectOutput } from '@aws-sdk/client-s3';
+import { S3CacheRatchetLike } from '../s3/s3-cache-ratchet-like.js';
+
+export class S3SimpleDao<T extends SimpleDaoItem> {
+  constructor(
+    private cache: S3CacheRatchetLike,
+    private prefix?: string,
+  ) {
+    if (!cache) {
+      throw new Error('cache object may not be null');
+    }
+    if (!cache.getDefaultBucket()) {
+      throw new Error('Supplied cache must have default bucket set');
+    }
+  }
+
+  public buildPathPrefix(path: string): string {
+    let rval: string = '';
+    if (this.prefix) {
+      rval += this.prefix;
+    }
+    if (path) {
+      rval += path;
+    }
+    return rval;
+  }
+
+  public buildFullPath(id: string, path?: string): string {
+    let rval: string = this.buildPathPrefix(path);
+    if (rval.length > 0) {
+      rval += '/';
+    }
+    rval += id + '.json';
+    return rval;
+  }
+
+  public async exists(id: string, path?: string): Promise<boolean> {
+    const fullPath: string = this.buildFullPath(id, path);
+    Logger.debug('Check file existence : %s', fullPath);
+    return this.cache.fileExists(fullPath);
+  }
+
+  public async fetch(id: string, path?: string): Promise<T> {
+    const fullPath: string = this.buildFullPath(id, path);
+    Logger.debug('Fetching : %s', fullPath);
+    const rval: T = (await this.cache.fetchCacheFileAsObject(fullPath)) as T;
+    // Force-set id and path
+    rval.id = id;
+    rval.path = path;
+    return rval;
+  }
+
+  public async store(item: T, path?: string): Promise<T> {
+    item.id = item.id || StringRatchet.createType4Guid();
+    item.lastModifiedEpochMS = new Date().getTime();
+
+    const fullPath: string = this.buildFullPath(item.id, path);
+    Logger.debug('Storing : %s', fullPath);
+
+    const _stored: PutObjectOutput = await this.cache.writeObjectToCacheFile(fullPath, item);
+    const read: T = await this.fetch(item.id, path);
+    return read;
+  }
+
+  public async listItems(path?: string): Promise<string[]> {
+    const fullPath: string = this.buildPathPrefix(path);
+    Logger.debug('Listing : %s', fullPath);
+
+    const rval: string[] = await this.cache.directChildrenOfPrefix(fullPath);
+    return rval;
+  }
+
+  public async fetchItemsInPath(path?: string): Promise<T[]> {
+    const fullPath: string = this.buildPathPrefix(path);
+    Logger.debug('Full fetch of : %s', fullPath);
+    const items: string[] = await this.listItems(path);
+    const promises: Promise<T>[] = items.map((s) => this.fetch(s, path));
+    const rval: T[] = await Promise.all(promises);
+    return rval;
+  }
+
+  public async delete(id: string, path?: string): Promise<boolean> {
+    const fullPath: string = this.buildFullPath(id, path);
+    Logger.debug('Deleting : %s', fullPath);
+
+    const del: DeleteObjectOutput = await this.cache.removeCacheFile(fullPath);
+    return del != null;
+  }
+}

package/src/dao/simple-dao-item.ts

@@ -0,0 +1,13 @@
+/*
+    Objects implementing this interface can producer objects that are then cached in S3 (or elsewhere I suppose)
+    These objects are consumed by SimpleCache
+
+    T is the type of object cached
+    R is the type of object passed to the create process.  Could just be a string or something
+*/
+
+export interface SimpleDaoItem {
+  id: string;
+  path: string;
+  lastModifiedEpochMS: number;
+}

package/src/dynamodb/dynamo-ratchet-like.ts

@@ -0,0 +1,61 @@
+import { DynamoCountResult } from '../model/dynamo-count-result.js';
+import { DeleteCommandOutput, DynamoDBDocumentClient, PutCommandOutput, QueryCommandInput, ScanCommandInput } from '@aws-sdk/lib-dynamodb';
+
+export interface DynamoRatchetLike {
+  getDDB(): DynamoDBDocumentClient;
+  tableIsEmpty(tableName: string): Promise<boolean>;
+  // This basically wraps up scans and queries with a function that will auto-retry them if a
+  // Throughput exception is encountered (up to a limit) but lets other errors get thrown.
+  // Drop-in replacement to make sure that things do not fail just because of throughput issues
+  throughputSafeScanOrQuery<T, R>(proc: (T) => Promise<R>, input: T, maxTries?: number, inCurrentTry?: number): Promise<R>;
+  fullyExecuteQueryCount(qry: QueryCommandInput, delayMS?: number): Promise<DynamoCountResult>;
+  fullyExecuteQuery<T>(qry: QueryCommandInput, delayMS?: number, softLimit?: number): Promise<T[]>;
+  fullyExecuteProcessOverQuery<T>(
+    qry: QueryCommandInput,
+    proc: (val: T) => Promise<void>,
+    delayMS?: number,
+    softLimit?: number,
+  ): Promise<number>;
+
+  fullyExecuteScanCount(scan: ScanCommandInput, delayMS?: number): Promise<DynamoCountResult>;
+  fullyExecuteScan<T>(scan: ScanCommandInput, delayMS?: number, softLimit?: number): Promise<T[]>;
+  fullyExecuteProcessOverScan<T>(
+    scan: ScanCommandInput,
+    proc: (val: T) => Promise<void>,
+    delayMS?: number,
+    softLimit?: number,
+  ): Promise<number>;
+
+  writeAllInBatches<T>(tableName: string, elements: T[], batchSize: number): Promise<number>;
+  fetchFullObjectsMatchingKeysOnlyIndexQuery<T>(qry: QueryCommandInput, keyNames: string[], batchSize?: number): Promise<T[]>;
+  fetchAllInBatches<T>(tableName: string, inKeys: any[], batchSize: number): Promise<T[]>;
+  deleteAllInBatches(tableName: string, keys: any[], batchSize: number): Promise<number>;
+
+  simplePut(tableName: string, value: any, autoRetryCount?: number): Promise<PutCommandOutput>;
+  simplePutOnlyIfFieldIsNullOrUndefined(tableName: string, value: any, fieldName: string): Promise<boolean>;
+
+  // This works like simplePut, but if a collision is detected it adjusts the object and tries writing again
+  // The adjustment function MUST change one of the keys - otherwise this just runs forever (or until it hits "maxAdjusts")
+  simplePutWithCollisionAvoidance<T>(
+    tableName: string,
+    value: T,
+    keyNames: string[],
+    adjustFunction: (val: T) => T,
+    maxAdjusts?: number,
+    autoRetryCount?: number,
+  ): Promise<T>;
+
+  simpleGet<T>(tableName: string, keys: any, autoRetryCount?: number): Promise<T>;
+
+  simpleGetWithCounterDecrement<T>(
+    tableName: string,
+    keys: any,
+    counterAttributeName: string,
+    deleteOnZero: boolean,
+    autoRetryCount?: number,
+  ): Promise<T>;
+
+  simpleDelete(tableName: string, keys: any): Promise<DeleteCommandOutput>;
+
+  atomicCounter(tableName: string, keys: any, counterFieldName: string, increment?: number): Promise<number>;
+}

package/src/dynamodb/dynamo-ratchet.spec.ts

@@ -0,0 +1,206 @@
+import { DynamoRatchet } from './dynamo-ratchet.js';
+import { DynamoDBDocumentClient, PutCommandOutput, QueryCommandInput, ScanCommandInput } from '@aws-sdk/lib-dynamodb';
+import { beforeEach, describe, expect, test } from 'vitest';
+
+import { mockClient } from 'aws-sdk-client-mock';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { LoggerLevelName } from '@bitblit/ratchet-common/logger/logger-level-name';
+
+let mockDynamo;
+
+describe('#dynamoRatchet', function () {
+  mockDynamo = mockClient(DynamoDBDocumentClient);
+
+  beforeEach(() => {
+    mockDynamo.reset();
+  });
+
+  test.skip('should handle ProvisionedThroughputExceeded exceptions', async () => {
+    const dr: DynamoRatchet = new DynamoRatchet(mockDynamo);
+    let row: number = 0;
+    const scan: ScanCommandInput = {
+      TableName: 'cache-device-data',
+      ProjectionExpression: 'compositeKey',
+    };
+
+    Logger.info('Starting scan');
+    const cnt: number = await dr.fullyExecuteProcessOverScan<any>(scan, async (ob) => {
+      if (row % 100 === 0) {
+        Logger.info('Row : %d : %j', row, ob);
+      }
+      row++;
+    });
+
+    Logger.info('Count was : %d', cnt);
+  }, 300_000_000);
+
+  test.skip('should only write if a field is null', async () => {
+    const dr: DynamoRatchet = new DynamoRatchet(mockDynamo);
+    const tableName: string = 'some-table';
+
+    const test1: any = { lockingKey: 'aaa', xx: null };
+    await dr.simplePut(tableName, test1);
+
+    const test2: any = { lockingKey: 'aaa', xx: 5 };
+    const val1: boolean = await dr.simplePutOnlyIfFieldIsNullOrUndefined(tableName, test2, 'xx');
+
+    expect(val1).toBeTruthy();
+
+    const test3: any = { lockingKey: 'aaa', xx: 7 };
+    const val2: boolean = await dr.simplePutOnlyIfFieldIsNullOrUndefined(tableName, test3, 'xx');
+
+    expect(val2).toBeFalsy();
+  });
+
+  test.skip('should fetch items from a key-only index query', async () => {
+    const dr: DynamoRatchet = new DynamoRatchet(mockDynamo);
+    const tableName: string = 'some-table';
+    const limit: number = 200;
+
+    const qry: QueryCommandInput = {
+      TableName: tableName,
+      IndexName: 'purchaseId-notBeforeEpochMS-index',
+      KeyConditionExpression: 'purchaseId = :purchaseId',
+      Limit: limit,
+      ExpressionAttributeValues: {
+        ':purchaseId': 'some-purchase',
+      },
+    };
+
+    const res: any[] = await dr.fetchFullObjectsMatchingKeysOnlyIndexQuery(qry, ['key1', 'key2']);
+
+    Logger.info('Got : %s', res);
+    expect(res).toBeTruthy();
+  });
+
+  test.skip('should increment the counter and return the new value', async () => {
+    const dr: DynamoRatchet = new DynamoRatchet(mockDynamo);
+    const tableName: string = 'xxx';
+    const res: number = await dr.atomicCounter(tableName, { groupId: 'global', unitId: 'sequence' }, 'lastValue', 1);
+    Logger.info('Got : %s', res);
+    expect(res).toBeTruthy();
+  });
+
+  test.skip('should stop after the soft limit', async () => {
+    const dr: DynamoRatchet = new DynamoRatchet(mockDynamo);
+
+    const now: number = new Date().getTime();
+    const _nowSec: number = Math.floor(now / 1000);
+    const curHash: string = 'someHash';
+
+    const qry: QueryCommandInput = {
+      TableName: 'some-table',
+      KeyConditionExpression: 'hashVal = :hashVal',
+      ExpressionAttributeValues: {
+        ':hashVal': curHash,
+      },
+    };
+
+    const res: any[] = await dr.fullyExecuteQuery<any>(qry, null, 150);
+
+    Logger.info('Got : %s', res);
+    expect(res).toBeTruthy();
+  }, 300_000);
+
+  test.skip('should run an insert / read test for slowdown', async () => {
+    const dr: DynamoRatchet = new DynamoRatchet(mockDynamo);
+
+    Logger.setLevel(LoggerLevelName.debug);
+    const now: number = new Date().getTime();
+    const _nowSec: number = Math.floor(now / 1000);
+
+    const nums: any[] = [];
+    for (let i = 0; i < 300; i++) {
+      const toWrite: any = {
+        id: 'CW_DDB_TEST',
+        data: 'SOME DATA',
+        numData: i,
+      };
+      nums.push(toWrite);
+    }
+    Logger.info('About to start');
+    const writeProms: Promise<any>[] = nums.map((n) => dr.simplePut('some-table', n));
+    const writeOut: any[] = await Promise.all(writeProms);
+    Logger.info('Write : %j', writeOut);
+
+    const readProms: Promise<any>[] = [];
+    Logger.info('Start : %d', new Date().getTime());
+    for (let i = 0; i < 10000; i++) {
+      readProms.push(dr.simpleGet('some-table', { id: 'CW_DDB_TEST' }));
+    }
+    Logger.info('Mid : %d', new Date().getTime());
+    const readOut: any[] = await Promise.all(readProms);
+    Logger.info('Read : %d : %j', new Date().getTime(), readOut);
+  });
+
+  test.skip('should run a collision test', async () => {
+    const dr: DynamoRatchet = new DynamoRatchet(mockDynamo);
+
+    Logger.setLevel(LoggerLevelName.debug);
+
+    const val: any = {
+      k1: 'abc',
+      k2: 1,
+    };
+
+    for (let i = 0; i < 5; i++) {
+      const rval: PutCommandOutput = await dr.simplePutWithCollisionAvoidance(
+        'cwtest',
+        val,
+        ['k1', 'k2'],
+        (v) => {
+          v.k2++;
+          return v;
+        },
+        null,
+        3,
+      );
+      Logger.info('output was : %j', rval);
+    }
+  });
+
+  test.skip('should do a simple get with counter decrement', async () => {
+    const dr: DynamoRatchet = new DynamoRatchet(mockDynamo);
+
+    Logger.setLevel(LoggerLevelName.debug);
+
+    const v: any = await dr.simpleGetWithCounterDecrement<any>('cwtest', { k1: 'abc', k2: 11 }, 'counter', true);
+    Logger.info('Got : %j', v);
+  });
+
+  test.skip('should do a full query', async () => {
+    const dr: DynamoRatchet = new DynamoRatchet(mockDynamo);
+
+    Logger.setLevel(LoggerLevelName.debug);
+
+    const input: QueryCommandInput = {
+      TableName: 'some-table',
+      KeyConditionExpression: 'groupId = :g',
+      ExpressionAttributeValues: {
+        ':g': 'NeonBatch',
+      },
+    };
+
+    const res: any[] = await dr.fullyExecuteQuery<any>(input);
+    Logger.info('Got : %j', res);
+  });
+
+  test.skip('should do a process over full query', async () => {
+    const dr: DynamoRatchet = new DynamoRatchet(mockDynamo);
+
+    Logger.setLevel(LoggerLevelName.debug);
+
+    const input: QueryCommandInput = {
+      TableName: 'some-table',
+      KeyConditionExpression: 'groupId = :g',
+      ExpressionAttributeValues: {
+        ':g': 'NeonBatch',
+      },
+    };
+
+    const res: number = await dr.fullyExecuteProcessOverQuery(input, async (v) => {
+      Logger.info('Proc %j', v);
+    });
+    Logger.info('Got : %j', res);
+  });
+});