@bitblit/ratchet-aws 6.0.146-alpha → 6.0.148-alpha

package/src/cloudwatch/cloud-watch-logs-ratchet.spec.ts

@@ -0,0 +1,123 @@
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { CloudWatchLogsRatchet } from './cloud-watch-logs-ratchet.js';
+import {
+  CloudWatchLogsClient,
+  DeleteLogGroupCommand,
+  DeleteLogStreamCommand,
+  DescribeLogGroupsCommand,
+  DescribeLogStreamsCommand,
+  GetQueryResultsCommand,
+  GetQueryResultsCommandOutput,
+  LogGroup,
+  LogStream,
+  StartQueryCommand,
+  StartQueryCommandInput,
+} from '@aws-sdk/client-cloudwatch-logs';
+import { mockClient } from 'aws-sdk-client-mock';
+import { beforeEach, describe, expect, test } from 'vitest';
+
+let mockCW;
+
+describe('#cloudWatchLogsRatchet', function () {
+  mockCW = mockClient(CloudWatchLogsClient);
+
+  beforeEach(() => {
+    mockCW.reset();
+  });
+
+  test('should find the stream with the oldest timestamp in a group', async () => {
+    mockCW.on(DescribeLogStreamsCommand).resolves({
+      logStreams: [
+        { logStreamName: '1', firstEventTimestamp: 100 },
+        { logStreamName: '2', firstEventTimestamp: 200 },
+      ],
+    } as never);
+
+    const cw: CloudWatchLogsRatchet = new CloudWatchLogsRatchet(mockCW);
+    const res: LogStream = await cw.findStreamWithOldestEventInGroup('test');
+    expect(res).toBeTruthy();
+    expect(res.logStreamName).toEqual('1');
+  });
+
+  test('should find the oldest timestamp in group', async () => {
+    mockCW.on(DescribeLogStreamsCommand).resolves({
+      logStreams: [
+        { logStreamName: '1', firstEventTimestamp: 100 },
+        { logStreamName: '2', firstEventTimestamp: 200 },
+      ],
+    } as never);
+
+    const cw: CloudWatchLogsRatchet = new CloudWatchLogsRatchet(mockCW);
+    const res: number = await cw.findOldestEventTimestampInGroup('test');
+    expect(res).toEqual(100);
+  });
+
+  test('should remove log groups', async () => {
+    mockCW.on(DeleteLogGroupCommand).resolves(null);
+
+    const cw: CloudWatchLogsRatchet = new CloudWatchLogsRatchet(mockCW);
+    const res: boolean[] = await cw.removeLogGroups([{ logGroupName: '1' }, { logGroupName: '2' }]);
+    expect(res.length).toEqual(2);
+  });
+
+  test('should remove log groups with prefix', async () => {
+    mockCW.on(DescribeLogGroupsCommand).resolves({
+      logGroups: [{ logGroupName: 'pre1' }, { logGroupName: 'pre2' }],
+    } as never);
+
+    mockCW.on(DeleteLogGroupCommand).resolves(null);
+
+    const cw: CloudWatchLogsRatchet = new CloudWatchLogsRatchet(mockCW);
+    const res: boolean[] = await cw.removeLogGroupsWithPrefix('pre');
+    expect(res.length).toEqual(2);
+  });
+
+  test('should remove empty or old log streams', async () => {
+    mockCW.on(DescribeLogStreamsCommand).resolves({
+      logStreams: [{ logStreamName: '1' }, { logStreamName: '2' }],
+    } as never);
+
+    mockCW.on(DeleteLogStreamCommand).resolves(null);
+
+    const cw: CloudWatchLogsRatchet = new CloudWatchLogsRatchet(mockCW);
+    const res: LogStream[] = await cw.removeEmptyOrOldLogStreams('test', 1000);
+    expect(res).toBeTruthy();
+    expect(res.length).toEqual(2);
+  });
+
+  test('should find all matching groups', async () => {
+    mockCW.on(DescribeLogGroupsCommand).resolves({ logGroups: [{ logGroupName: '1' }, { logGroupName: '2' }] } as never);
+
+    const cw: CloudWatchLogsRatchet = new CloudWatchLogsRatchet(mockCW);
+    const prefix: string = '/';
+
+    const res: LogGroup[] = await cw.findLogGroups(prefix);
+    expect(res).toBeTruthy();
+    expect(res.length).toEqual(2);
+  });
+
+  test('should execute an insights query', async () => {
+    mockCW.on(StartQueryCommand).resolves({ queryId: 'test' });
+
+    mockCW.on(GetQueryResultsCommand).resolves({});
+
+    const cw: CloudWatchLogsRatchet = new CloudWatchLogsRatchet(mockCW);
+
+    const logGroups: string[] = ['/someGroup'];
+
+    const now: number = Math.floor(new Date().getTime() / 1000);
+
+    const req: StartQueryCommandInput = {
+      endTime: now,
+      limit: 200,
+      logGroupNames: logGroups,
+      queryString: 'fields @timestamp, @message | sort @timestamp desc',
+      startTime: now - 60 * 60 * 24, // 1 day ago
+    };
+    const res: GetQueryResultsCommandOutput = await cw.fullyExecuteInsightsQuery(req);
+
+    expect(res).toBeTruthy();
+
+    Logger.info('Got : %j', res);
+  });
+});

package/src/cloudwatch/cloud-watch-logs-ratchet.ts

@@ -0,0 +1,232 @@
+import {
+  CloudWatchLogsClient,
+  DeleteLogGroupCommand,
+  DeleteLogGroupCommandInput,
+  DeleteLogStreamCommand,
+  DescribeLogGroupsCommand,
+  DescribeLogGroupsCommandInput,
+  DescribeLogGroupsCommandOutput,
+  DescribeLogStreamsCommand,
+  DescribeLogStreamsCommandOutput,
+  GetQueryResultsCommand,
+  GetQueryResultsCommandOutput,
+  LogGroup,
+  LogStream,
+  OrderBy,
+  StartQueryCommand,
+  StartQueryCommandInput,
+  StartQueryCommandOutput,
+  StopQueryCommand,
+  StopQueryCommandOutput,
+} from '@aws-sdk/client-cloudwatch-logs';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { PromiseRatchet } from '@bitblit/ratchet-common/lang/promise-ratchet';
+import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
+import { StringRatchet } from '@bitblit/ratchet-common/lang/string-ratchet';
+
+export class CloudWatchLogsRatchet {
+  private static readonly MAX_DELETE_RETRIES = 5;
+  private cwLogs: CloudWatchLogsClient;
+
+  constructor(cloudwatchLogs: CloudWatchLogsClient = null) {
+    this.cwLogs = cloudwatchLogs ? cloudwatchLogs : new CloudWatchLogsClient({ region: 'us-east-1' });
+  }
+
+  public get cloudWatchLogsClient(): CloudWatchLogsClient {
+    return this.cwLogs;
+  }
+
+  public async removeEmptyOrOldLogStreams(
+    logGroupName: string,
+    maxToRemove = 1000,
+    oldestEventEpochMS: number = null,
+  ): Promise<LogStream[]> {
+    Logger.info('Removing empty streams from %s, oldest event epoch MS : %d', logGroupName, oldestEventEpochMS);
+    const streamSearchParams: any = {
+      logGroupName: logGroupName,
+      orderBy: OrderBy.LastEventTime,
+    };
+
+    const oldestEventTester: number = oldestEventEpochMS || 1;
+    let totalStreams = 0;
+    const removedStreams: LogStream[] = [];
+    const failedRemovedStreams: LogStream[] = [];
+    let waitPerDescribe = 10;
+
+    do {
+      Logger.debug('Executing search for streams');
+      try {
+        const streams: DescribeLogStreamsCommandOutput = await this.cwLogs.send(new DescribeLogStreamsCommand(streamSearchParams));
+        totalStreams += streams.logStreams.length;
+
+        Logger.debug('Found %d streams (%d so far, %d to delete)', streams.logStreams.length, totalStreams, removedStreams.length);
+
+        for (let i = 0; i < streams.logStreams.length && removedStreams.length < maxToRemove; i++) {
+          const st: LogStream = streams.logStreams[i];
+          if (!st.firstEventTimestamp) {
+            removedStreams.push(st);
+          } else if (st.lastEventTimestamp < oldestEventTester) {
+            removedStreams.push(st);
+          }
+        }
+
+        streamSearchParams['nextToken'] = streams.nextToken;
+      } catch (err) {
+        const oldWait: number = waitPerDescribe;
+        waitPerDescribe = Math.min(1000, waitPerDescribe * 1.5);
+        Logger.info('Caught while describing %s, increasing wait between deletes (was %d, now %d)', err, oldWait, waitPerDescribe);
+      }
+    } while (!!streamSearchParams['nextToken'] && removedStreams.length < maxToRemove);
+
+    Logger.info('Found %d streams to delete', removedStreams.length);
+    let waitPer = 10;
+
+    for (const rStream of removedStreams) {
+      //for (let i = 0; i < removedStreams.length; i++) {
+      const delParams = {
+        logGroupName: logGroupName,
+        logStreamName: rStream.logStreamName,
+      };
+
+      const type: string = rStream.storedBytes === 0 ? 'empty' : 'old';
+      Logger.info('Removing %s stream %s', type, rStream.logStreamName);
+      let removed = false;
+      let retry = 0;
+      while (!removed && retry < CloudWatchLogsRatchet.MAX_DELETE_RETRIES) {
+        try {
+          await this.cwLogs.send(new DeleteLogStreamCommand(delParams));
+          removed = true;
+          await PromiseRatchet.wait(waitPer);
+        } catch (err) {
+          retry++;
+          const oldWait: number = waitPer;
+          waitPer = Math.min(1000, waitPer * 1.5);
+          Logger.info(
+            'Caught %s, increasing wait between deletes and retrying (wait was %d, now %d) (Retry %d of %d)',
+            err,
+            oldWait,
+            waitPer,
+            retry,
+            CloudWatchLogsRatchet.MAX_DELETE_RETRIES,
+          );
+        }
+      }
+      if (!removed) {
+        // Ran out of retries
+        failedRemovedStreams.push(rStream);
+      }
+    }
+
+    Logger.warn('Failed to remove streams : %j', failedRemovedStreams);
+    return removedStreams;
+  }
+
+  public async findOldestEventTimestampInGroup(logGroupName: string): Promise<number> {
+    const stream: LogStream = await this.findStreamWithOldestEventInGroup(logGroupName);
+    return stream ? stream.firstEventTimestamp : null;
+  }
+
+  public async findStreamWithOldestEventInGroup(logGroupName: string): Promise<LogStream> {
+    Logger.info('Finding oldest event in : %s', logGroupName);
+    let rval: LogStream = null;
+    try {
+      const streamSearchParams = {
+        logGroupName: logGroupName,
+        orderBy: OrderBy.LastEventTime,
+      };
+
+      let totalStreams = 0;
+      do {
+        Logger.debug('Executing search for streams');
+        const streams: DescribeLogStreamsCommandOutput = await this.cwLogs.send(new DescribeLogStreamsCommand(streamSearchParams));
+        totalStreams += streams.logStreams.length;
+
+        Logger.debug('Found %d streams (%d so far)', streams.logStreams.length, totalStreams);
+
+        streams.logStreams.forEach((s) => {
+          if (s.firstEventTimestamp && (rval === null || s.firstEventTimestamp < rval.firstEventTimestamp)) {
+            rval = s;
+          }
+        });
+
+        streamSearchParams['nextToken'] = streams.nextToken;
+      } while (streamSearchParams['nextToken']);
+    } catch (err) {
+      Logger.error('Error attempting to find oldest event in group : %s : %s', logGroupName, err, err);
+    }
+    return rval;
+  }
+
+  public async findLogGroups(prefix: string): Promise<LogGroup[]> {
+    RequireRatchet.notNullOrUndefined(prefix);
+
+    const params: DescribeLogGroupsCommandInput = {
+      logGroupNamePrefix: prefix,
+    };
+    let rval: LogGroup[] = [];
+
+    do {
+      Logger.info('%d found, pulling log groups : %j', rval.length, params);
+      const res: DescribeLogGroupsCommandOutput = await this.cwLogs.send(new DescribeLogGroupsCommand(params));
+      rval = rval.concat(res.logGroups);
+      params.nextToken = res.nextToken;
+    } while (params.nextToken);
+
+    return rval;
+  }
+
+  public async removeLogGroups(groups: LogGroup[]): Promise<boolean[]> {
+    RequireRatchet.notNullOrUndefined(groups);
+    const rval: boolean[] = [];
+
+    for (const dGroup of groups) {
+      try {
+        Logger.info('Deleting %j', dGroup);
+        const req: DeleteLogGroupCommandInput = {
+          logGroupName: dGroup.logGroupName,
+        };
+        await this.cwLogs.send(new DeleteLogGroupCommand(req));
+        rval.push(true);
+      } catch (err) {
+        Logger.error('Failure to delete %j : %s', dGroup, err);
+        rval.push(false);
+      }
+    }
+
+    return rval;
+  }
+
+  public async removeLogGroupsWithPrefix(prefix: string): Promise<boolean[]> {
+    RequireRatchet.notNullOrUndefined(prefix);
+    RequireRatchet.true(StringRatchet.trimToEmpty(prefix).length > 0); // Don't allow nuke all here
+
+    Logger.info('Removing log groups with prefix %s', prefix);
+    const groups: LogGroup[] = await this.findLogGroups(prefix);
+    return await this.removeLogGroups(groups);
+  }
+
+  public async fullyExecuteInsightsQuery(sqr: StartQueryCommandInput): Promise<GetQueryResultsCommandOutput> {
+    RequireRatchet.notNullOrUndefined(sqr);
+    Logger.debug('Starting insights query : %j', sqr);
+    const resp: StartQueryCommandOutput = await this.cwLogs.send(new StartQueryCommand(sqr));
+    Logger.debug('Got query id %j', resp);
+
+    let rval: GetQueryResultsCommandOutput = null;
+    let delayMS: number = 100;
+    while (!rval || ['Running', 'Scheduled'].includes(rval.status)) {
+      rval = await this.cwLogs.send(new GetQueryResultsCommand({ queryId: resp.queryId }));
+      await PromiseRatchet.wait(delayMS);
+      delayMS *= 2;
+      Logger.info('Got : %j', rval);
+    }
+    return rval;
+  }
+
+  public async abortInsightsQuery(queryId: string): Promise<StopQueryCommandOutput> {
+    let rval: StopQueryCommandOutput = null;
+    if (queryId) {
+      rval = await this.cwLogs.send(new StopQueryCommand({ queryId: queryId }));
+    }
+    return rval;
+  }
+}

package/src/cloudwatch/cloud-watch-metrics-ratchet.spec.ts

@@ -0,0 +1,30 @@
+import { CloudWatchMetricsRatchet } from './cloud-watch-metrics-ratchet.js';
+
+import { CloudWatchClient, PutMetricDataCommand, StandardUnit } from '@aws-sdk/client-cloudwatch';
+import { mockClient } from 'aws-sdk-client-mock';
+import { beforeEach, describe, expect, test } from 'vitest';
+import { KeyValue } from '@bitblit/ratchet-common/lang/key-value';
+
+let mockCW;
+
+describe('#cloudWatchMetricsRatchet', function () {
+  mockCW = mockClient(CloudWatchClient);
+  beforeEach(() => {
+    mockCW.reset();
+    mockCW.reset();
+  });
+
+  test('should log a cloudwatch metric', async () => {
+    mockCW.on(PutMetricDataCommand).resolves({ ok: true } as never);
+
+    const cw: CloudWatchMetricsRatchet = new CloudWatchMetricsRatchet(mockCW);
+    const dims: KeyValue<any>[] = [
+      { key: 'server', value: 'prod' } as KeyValue<any>,
+      { key: 'stage', value: 'v0' },
+      { key: 'version', value: '20190529-01' },
+    ];
+    const res: any = await cw.writeSingleMetric('Ratchet/TestMetric01', 'MyMetric01', dims, StandardUnit.Count, 2, new Date(), false);
+
+    expect(res).toBeTruthy();
+  });
+});

package/src/cloudwatch/cloud-watch-metrics-ratchet.ts

@@ -0,0 +1,98 @@
+/*
+    Service for interacting with cloudwatch
+*/
+
+import {
+  CloudWatchClient,
+  PutMetricDataCommand,
+  PutMetricDataCommandInput,
+  PutMetricDataCommandOutput,
+  StandardUnit,
+} from '@aws-sdk/client-cloudwatch';
+import { DynamoCountResult } from '../model/dynamo-count-result.js';
+import { CloudWatchMetricsMinuteLevelDynamoCountRequest } from '../model/cloud-watch-metrics-minute-level-dynamo-count-request.js';
+import { DateTime } from 'luxon';
+import { KeyValue } from '@bitblit/ratchet-common/lang/key-value';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+
+export class CloudWatchMetricsRatchet {
+  private cw: CloudWatchClient;
+
+  constructor(cloudWatch: CloudWatchClient = null) {
+    this.cw = cloudWatch ? cloudWatch : new CloudWatchClient({ region: 'us-east-1', apiVersion: '2010-08-01' });
+  }
+
+  public get cloudWatchClient(): CloudWatchClient {
+    return this.cw;
+  }
+
+  public async writeSingleMetric(
+    namespace: string,
+    metric: string,
+    dims: KeyValue<any>[],
+    unit: StandardUnit = StandardUnit.None,
+    value: number,
+    timestampDate: Date = new Date(),
+    highResolution = false,
+  ): Promise<any> {
+    const cwDims: any[] = [];
+    if (!!dims && dims.length > 0) {
+      dims.forEach((d) => {
+        cwDims.push({ Name: d.key, Value: d.value });
+      });
+    }
+    const storageResolution: number = highResolution ? 1 : 60;
+
+    const metricData: PutMetricDataCommandInput = {
+      Namespace: namespace,
+      MetricData: [
+        {
+          MetricName: metric,
+          Dimensions: cwDims,
+          Unit: unit,
+          Value: value,
+          Timestamp: timestampDate,
+          StorageResolution: storageResolution,
+        },
+      ],
+    };
+    Logger.silly('Writing metric to cw : %j', metricData);
+
+    const result: PutMetricDataCommandOutput = await this.cw.send(new PutMetricDataCommand(metricData));
+    Logger.silly('Result: %j', result);
+    return result;
+  }
+
+  public async writeDynamoCountAsMinuteLevelMetric(req: CloudWatchMetricsMinuteLevelDynamoCountRequest): Promise<number> {
+    Logger.info('Publishing %s / %s metric for %s UTC', req.namespace, req.metric, req.minuteUTC);
+
+    if (!!req.scan && !!req.query) {
+      throw new Error('Must send query or scan, but not both');
+    }
+    if (!req.scan && !req.query) {
+      throw new Error('You must specify either a scan or a query');
+    }
+
+    const cnt: DynamoCountResult = req.query
+      ? await req.dynamoRatchet.fullyExecuteQueryCount(req.query)
+      : await req.dynamoRatchet.fullyExecuteScanCount(req.scan);
+
+    Logger.debug('%s / %s for %s are %j', req.namespace, req.metric, req.minuteUTC, cnt);
+
+    const parseDateString: string = req.minuteUTC.split(' ').join('T') + ':00Z';
+    const parseDate: Date = DateTime.fromISO(parseDateString).toJSDate();
+
+    const metricRes: any = await this.writeSingleMetric(
+      req.namespace,
+      req.metric,
+      req.dims,
+      StandardUnit.Count,
+      cnt.count,
+      parseDate,
+      false,
+    );
+    Logger.debug('Metrics response: %j', metricRes);
+
+    return cnt.count;
+  }
+}

package/src/dao/example-prototype-dao-item.ts

@@ -0,0 +1,8 @@
+export interface ExamplePrototypeDaoItem {
+  fieldA: number;
+  fieldB: string;
+  type: string;
+  createdEpochMS?: number;
+  updatedEpochMS?: number;
+  guid?: string;
+}

package/src/dao/memory-prototype-dao-provider.ts

@@ -0,0 +1,16 @@
+// Memory version for testing
+import { PrototypeDaoProvider } from './prototype-dao-provider.js';
+import { PrototypeDaoDb } from './prototype-dao-db.js';
+
+export class MemoryPrototypeDaoProvider implements PrototypeDaoProvider<any> {
+  private _db: PrototypeDaoDb<any> = null;
+
+  public async loadDatabase(): Promise<PrototypeDaoDb<any>> {
+    return this._db;
+  }
+
+  public async storeDatabase(db: PrototypeDaoDb<any>): Promise<boolean> {
+    this._db = db;
+    return true;
+  }
+}

package/src/dao/prototype-dao-config.ts

@@ -0,0 +1,8 @@
+export interface PrototypeDaoConfig {
+  guidCreateFunction: () => string;
+  guidFieldName: string;
+  createdEpochMSFieldName?: string;
+  updatedEpochMSFieldName?: string;
+  createdUtcTimestampFieldName?: string;
+  updatedUtcTimestampFieldName?: string;
+}

package/src/dao/prototype-dao-db.ts

@@ -0,0 +1,4 @@
+export interface PrototypeDaoDb<T> {
+  items: T[];
+  lastModifiedEpochMS: number;
+}

package/src/dao/prototype-dao-provider.ts

@@ -0,0 +1,6 @@
+import { PrototypeDaoDb } from './prototype-dao-db.js';
+
+export interface PrototypeDaoProvider<T> {
+  storeDatabase(db: PrototypeDaoDb<T>): Promise<boolean>;
+  loadDatabase(): Promise<PrototypeDaoDb<T>>;
+}

package/src/dao/prototype-dao.spec.ts

@@ -0,0 +1,33 @@
+import { PrototypeDao } from './prototype-dao.js';
+import { describe, expect, test } from 'vitest';
+import { MemoryPrototypeDaoProvider } from './memory-prototype-dao-provider.js';
+import { ExamplePrototypeDaoItem } from './example-prototype-dao-item.js';
+
+describe('#PrototypeDao', () => {
+  test.skip('Should save/load files', async () => {
+    const svc: PrototypeDao<ExamplePrototypeDaoItem> = new PrototypeDao<ExamplePrototypeDaoItem>(new MemoryPrototypeDaoProvider());
+    await svc.resetDatabase();
+
+    await svc.store({ fieldA: 1, fieldB: 'test1', type: 'a' });
+    await svc.store({ fieldA: 2, fieldB: 'test2', type: 'a' });
+    await svc.store({ fieldA: 2, fieldB: 'test3', type: 'b' });
+    await svc.store({ guid: 'forceGuid', fieldA: 4, fieldB: 'test4', type: 'c' });
+
+    const test1: ExamplePrototypeDaoItem[] = await svc.fetchAll();
+    expect(test1.length).toBe(4);
+
+    const test2: ExamplePrototypeDaoItem = await svc.fetchById('forceGuid');
+    expect(test2).not.toBeNull();
+    expect(test2.createdEpochMS).not.toBeNull();
+    expect(test2.updatedEpochMS).not.toBeNull();
+
+    const test3: ExamplePrototypeDaoItem[] = await svc.searchByField('fieldA', 4);
+    expect(test3.length).toBe(1);
+    expect(test3[0].guid).toEqual('forceGuid');
+
+    const test4: ExamplePrototypeDaoItem[] = await svc.searchByFieldMap({ type: 'a', fieldA: 2 });
+    expect(test4.length).toBe(1);
+
+    await svc.resetDatabase();
+  }, 300_000);
+});

package/src/dao/prototype-dao.ts

@@ -0,0 +1,110 @@
+import { PrototypeDaoProvider } from './prototype-dao-provider.js';
+import { PrototypeDaoDb } from './prototype-dao-db.js';
+import { PrototypeDaoConfig } from './prototype-dao-config.js';
+import { DateTime } from 'luxon';
+import { StringRatchet } from '@bitblit/ratchet-common/lang/string-ratchet';
+import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
+import { MapRatchet } from '@bitblit/ratchet-common/lang/map-ratchet';
+
+/*
+  PrototypeDao makes it quick to stand up a simple data access object
+  key/value store backed by something simple (usually a file in S3).
+
+  It is slow and not very powerful, but is meant to be stood up rapidly, and then
+  replaced once you have something useful to do the Dao's job
+
+  Note this thing pulls the WHOLE DB for every single operation.  You really don't wanna use it
+  for anything like serious workloads
+
+ */
+export class PrototypeDao<T> {
+  public static defaultDaoConfig(): PrototypeDaoConfig {
+    return {
+      guidCreateFunction: StringRatchet.createType4Guid,
+      guidFieldName: 'guid',
+      createdEpochMSFieldName: 'createdEpochMS',
+      updatedEpochMSFieldName: 'updatedEpochMS',
+      createdUtcTimestampFieldName: null,
+      updatedUtcTimestampFieldName: null,
+    };
+  }
+
+  constructor(
+    private provider: PrototypeDaoProvider<T>,
+    private cfg: PrototypeDaoConfig = PrototypeDao.defaultDaoConfig(),
+  ) {
+    RequireRatchet.notNullOrUndefined(provider, 'provider');
+    RequireRatchet.notNullOrUndefined(cfg, 'cfg');
+    RequireRatchet.notNullOrUndefined(cfg.guidCreateFunction, 'cfg.guidCreateFunction');
+    RequireRatchet.notNullOrUndefined(cfg.guidFieldName, 'cfg.guidFieldName');
+  }
+
+  public async fetchAll(): Promise<T[]> {
+    const db: PrototypeDaoDb<T> = await this.provider.loadDatabase();
+    return db.items || [];
+  }
+
+  public async resetDatabase(): Promise<void> {
+    await this.provider.storeDatabase({ items: [], lastModifiedEpochMS: Date.now() });
+  }
+
+  public async removeItems(guids: string[]): Promise<T[]> {
+    let old: T[] = await this.fetchAll();
+    if (guids) {
+      old = old.filter((t) => !guids.includes(t[this.cfg.guidFieldName]));
+      await this.provider.storeDatabase({ items: old, lastModifiedEpochMS: Date.now() });
+    }
+    return old;
+  }
+
+  public async store(value: T): Promise<T> {
+    let old: T[] = await this.fetchAll();
+    if (value) {
+      value[this.cfg.guidFieldName] = value[this.cfg.guidFieldName] || this.cfg.guidCreateFunction();
+      if (this.cfg.createdEpochMSFieldName) {
+        value[this.cfg.createdEpochMSFieldName] = value[this.cfg.createdEpochMSFieldName] || Date.now();
+      }
+      if (this.cfg.createdUtcTimestampFieldName) {
+        value[this.cfg.createdUtcTimestampFieldName] = value[this.cfg.createdUtcTimestampFieldName] || DateTime.utc().toISO();
+      }
+      if (this.cfg.updatedEpochMSFieldName) {
+        value[this.cfg.updatedEpochMSFieldName] = Date.now();
+      }
+      if (this.cfg.updatedUtcTimestampFieldName) {
+        value[this.cfg.updatedUtcTimestampFieldName] = DateTime.utc().toISO();
+      }
+      old = old.filter((t) => t[this.cfg.guidFieldName] !== value[this.cfg.guidFieldName]);
+      old.push(value);
+      await this.provider.storeDatabase({ items: old, lastModifiedEpochMS: Date.now() });
+    }
+    return value;
+  }
+
+  public async fetchById(guid: string): Promise<T> {
+    const old: T[] = await this.fetchAll();
+    return old.find((t) => t[this.cfg.guidFieldName] === guid);
+  }
+
+  public async searchByField<R>(fieldDotPath: string, fieldValue: R): Promise<T[]> {
+    RequireRatchet.notNullOrUndefined(fieldDotPath, 'fieldDotPath');
+    RequireRatchet.notNullOrUndefined(fieldValue, 'fieldValue');
+    const map: Record<string, any> = {};
+    map[fieldDotPath] = fieldValue;
+    return this.searchByFieldMap(map);
+  }
+
+  public async searchByFieldMap(input: Record<string, any>): Promise<T[]> {
+    RequireRatchet.notNullOrUndefined(input, 'input');
+    let old: T[] = await this.fetchAll();
+    old = old.filter((t) => {
+      let matchAll: boolean = true;
+      Object.keys(input).forEach((k) => {
+        const val: any = MapRatchet.findValueDotPath(t, k);
+        const fieldValue: any = input[k];
+        matchAll = matchAll && val === fieldValue;
+      });
+      return matchAll;
+    });
+    return old;
+  }
+}