@bitblit/ratchet-aws 6.0.145-alpha → 6.0.147-alpha

package/src/s3/impl/s3-prototype-dao-provider.ts

@@ -0,0 +1,37 @@
+import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
+import { PutObjectOutput } from '@aws-sdk/client-s3';
+import { PrototypeDaoProvider } from '../../dao/prototype-dao-provider.js';
+import { PrototypeDaoDb } from '../../dao/prototype-dao-db.js';
+import { S3CacheRatchetLike } from '../s3-cache-ratchet-like.js';
+
+/* An implementation that puts all the values in a single JSON file in S3
+  This won't scale well at all for any kind of serious load, but is the easiest
+  solution for a very low traffic website since it doesn't require setting up tables,
+  provisioning, etc
+ */
+export class S3PrototypeDaoProvider<T> implements PrototypeDaoProvider<T> {
+  constructor(
+    private s3CacheRatchet: S3CacheRatchetLike,
+    private keyName: string,
+  ) {
+    RequireRatchet.notNullOrUndefined(s3CacheRatchet, 's3CacheRatchet');
+    RequireRatchet.notNullUndefinedOrOnlyWhitespaceString(s3CacheRatchet.getDefaultBucket(), 's3CacheRatchet.defaultBucket');
+    RequireRatchet.notNullUndefinedOrOnlyWhitespaceString(keyName, 'keyName');
+  }
+
+  public async storeDatabase(inDb: PrototypeDaoDb<T>): Promise<boolean> {
+    const toSave: PrototypeDaoDb<T> = inDb || { items: [], lastModifiedEpochMS: null };
+    toSave.lastModifiedEpochMS = Date.now();
+    const put: PutObjectOutput = await this.s3CacheRatchet.writeObjectToCacheFile(this.keyName, toSave);
+    const rval: boolean = !!put;
+    return rval;
+  }
+
+  public async loadDatabase(): Promise<PrototypeDaoDb<T>> {
+    const rval: PrototypeDaoDb<T> = (await this.s3CacheRatchet.fetchCacheFileAsObject<PrototypeDaoDb<any>>(this.keyName)) || {
+      items: [],
+      lastModifiedEpochMS: Date.now(),
+    };
+    return rval;
+  }
+}

package/src/s3/impl/s3-remote-file-tracking-provider-options.ts

@@ -0,0 +1,6 @@
+import { S3CacheRatchetLike } from '../s3-cache-ratchet-like.js';
+
+// For future configuration
+export interface S3RemoteFileTrackingProviderOptions {
+  s3CacheRatchet: S3CacheRatchetLike;
+}

package/src/s3/impl/s3-remote-file-tracking-provider.spec.ts

@@ -0,0 +1,67 @@
+import { beforeEach, describe, expect, test } from 'vitest';
+import { S3RemoteFileTrackingProvider } from './s3-remote-file-tracking-provider.js';
+import { mock, MockProxy } from 'vitest-mock-extended';
+import { S3CacheRatchetLike } from '../s3-cache-ratchet-like.js';
+import { S3CacheRatchet } from '../s3-cache-ratchet.js';
+import { S3Client } from '@aws-sdk/client-s3';
+import { S3RemoteFileTrackingProviderOptions } from './s3-remote-file-tracking-provider-options.js';
+import { AwsCredentialsRatchet } from '../../iam/aws-credentials-ratchet.js';
+import { RemoteFileTracker } from '@bitblit/ratchet-common/network/remote-file-tracker/remote-file-tracker';
+import {
+  RemoteStatusDataAndContent
+} from "@bitblit/ratchet-common/lib/network/remote-file-tracker/remote-status-data-and-content";
+
+let mockS3Ratchet: MockProxy<S3CacheRatchetLike>;
+
+describe('#S3RemoteFileTrackingProvider', () => {
+  beforeEach(() => {
+    mockS3Ratchet = mock<S3CacheRatchetLike>();
+    mockS3Ratchet.getDefaultBucket.mockReturnValue('TEST-BUCKET');
+  });
+
+  test.skip('Should save/load files', async () => {
+    AwsCredentialsRatchet.applySetProfileEnvironmentalVariable('erigir');
+    const ratchet: S3CacheRatchetLike = new S3CacheRatchet(new S3Client({ region: 'us-east-1' }), 'erigir-backup');
+    // setup initial state
+    await ratchet.writeStringToCacheFile('test.txt', 'This is a test ' + new Date().toISOString());
+
+    const testOpts: S3RemoteFileTrackingProviderOptions = {
+      //s3CacheRatchet: mockS3Ratchet
+      s3CacheRatchet: ratchet,
+    };
+
+    const svc: S3RemoteFileTrackingProvider = new S3RemoteFileTrackingProvider(testOpts);
+    const obj = new RemoteFileTracker<string>({
+      key: 'test.txt',
+      provider: svc,
+    });
+
+    expect(obj.remoteStatusData).toBeNull;
+    await obj.sync();
+    expect(obj.remoteStatusData).not.toBeNull;
+
+    const changed1: boolean = await obj.modifiedSinceLastSync();
+    expect(changed1).toEqual(false);
+
+    await ratchet.writeStringToCacheFile('test.txt', 'This is a test ' + new Date().toISOString());
+
+    const changed2: boolean = await obj.modifiedSinceLastSync();
+    expect(changed2).toEqual(true);
+
+    const raw: RemoteStatusDataAndContent<string> = await obj.pullRemoteData();
+    expect(raw).not.toBeNull;
+    const data: string = await RemoteFileTracker.dataAsString(raw);
+    expect(data).not.toBeNull;
+
+    expect(data.startsWith('This is a test')).toBeTruthy;
+
+    const pushRes: any = await obj.pushStringToRemote('Local-Test', { force: false, backup: true });
+    expect(pushRes).not.toBeNull;
+
+    const raw2: RemoteStatusDataAndContent<string> = await obj.pullRemoteData();
+    expect(raw2).not.toBeNull;
+    const data2: string = await RemoteFileTracker.dataAsString(raw2);
+    expect(data2).not.toBeNull;
+    expect(data2).toEqual('Local-Test');
+  }, 300_000);
+});

package/src/s3/impl/s3-remote-file-tracking-provider.ts

@@ -0,0 +1,157 @@
+import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { StopWatch } from '@bitblit/ratchet-common/lang/stop-watch';
+import {
+  CompleteMultipartUploadCommandOutput,
+  GetObjectCommandInput,
+  GetObjectCommandOutput,
+  HeadObjectCommandOutput,
+} from '@aws-sdk/client-s3';
+import { DateTime } from 'luxon';
+import { RemoteFileTrackingProvider } from '@bitblit/ratchet-common/network/remote-file-tracker/remote-file-tracking-provider';
+import { StringRatchet } from '@bitblit/ratchet-common/lang/string-ratchet';
+import { RemoteStatusData } from '@bitblit/ratchet-common/network/remote-file-tracker/remote-status-data';
+import { S3RemoteFileTrackingProviderOptions } from './s3-remote-file-tracking-provider-options.js';
+import { RemoteStatusDataAndContent } from '@bitblit/ratchet-common/network/remote-file-tracker/remote-status-data-and-content';
+import { FileTransferResult } from '@bitblit/ratchet-common/network/remote-file-tracker/file-transfer-result';
+import { BackupResult } from '@bitblit/ratchet-common/network/remote-file-tracker/backup-result';
+import { RemoteFileTrackerPushOptions } from '@bitblit/ratchet-common/network/remote-file-tracker/remote-file-tracker-push-options';
+import { RemoteFileTracker } from '@bitblit/ratchet-common/network/remote-file-tracker/remote-file-tracker';
+import { ErrorRatchet } from '@bitblit/ratchet-common/lang/error-ratchet';
+import { FileTransferResultType } from '@bitblit/ratchet-common/network/remote-file-tracker/file-transfer-result-type';
+
+// Keeps a local file up-to-date with a file on S3
+export class S3RemoteFileTrackingProvider implements RemoteFileTrackingProvider<string> {
+  constructor(private opts: S3RemoteFileTrackingProviderOptions) {
+    RequireRatchet.notNullOrUndefined(opts, 'opts');
+    RequireRatchet.notNullOrUndefined(opts.s3CacheRatchet, 'opts.s3CacheRatchet');
+    RequireRatchet.notNullOrUndefined(opts.s3CacheRatchet.getDefaultBucket(), 'opts.s3CacheRatchet must have default bucket set');
+  }
+
+  public async readRemoteStatus(key: string): Promise<RemoteStatusData<string>> {
+    let rval: RemoteStatusData<string> = null;
+    if (StringRatchet.trimToNull(key)) {
+      const meta: HeadObjectCommandOutput = await this.opts.s3CacheRatchet.fetchMetaForCacheFile(key);
+      if (meta) {
+        rval = {
+          key: key,
+          statusTakenEpochMs: Date.now(),
+          remoteSizeInBytes: meta.ContentLength,
+          remoteLastUpdatedEpochMs: meta.LastModified.getTime(),
+          remoteHash: meta.ETag,
+        };
+      }
+    }
+    return rval;
+  }
+
+  public async pullRemoteData(key: string, ifNewerThan?: RemoteStatusData<string>): Promise<RemoteStatusDataAndContent<string>> {
+    let rval: RemoteStatusDataAndContent<string> = null;
+    const sw: StopWatch = new StopWatch();
+
+    const req: GetObjectCommandInput = {
+      Bucket: this.opts.s3CacheRatchet.getDefaultBucket(),
+      Key: key,
+      IfModifiedSince: ifNewerThan?.remoteLastUpdatedEpochMs ? new Date(ifNewerThan.remoteLastUpdatedEpochMs) : null,
+    };
+
+    const output: GetObjectCommandOutput = await this.opts.s3CacheRatchet.fetchCacheFilePassThru(req);
+    if (output) {
+      rval = {
+        status: {
+          key: key,
+          statusTakenEpochMs: Date.now(),
+          remoteSizeInBytes: output.ContentLength,
+          remoteLastUpdatedEpochMs: output.LastModified.getTime(),
+          remoteHash: output.ETag,
+        },
+        content: output.Body.transformToWebStream(),
+      };
+      Logger.info('Fetched remote to local, %d bytes in %s : %s', output.ContentLength, sw.dump(), key);
+    } else {
+      Logger.info('Did not pull %s - not modified', key);
+    }
+
+    return rval;
+  }
+
+  public async sendDataToRemote(
+    src: ReadableStream,
+    key: string,
+    opts: RemoteFileTrackerPushOptions,
+    checkStatus: RemoteStatusData<string>,
+  ): Promise<FileTransferResult> {
+    RequireRatchet.notNullOrUndefined(src, 'src');
+    RequireRatchet.notNullOrUndefined(key, 'key');
+    RequireRatchet.notNullOrUndefined(opts, 'opts');
+    const rval: FileTransferResult = {
+      type: null,
+      error: null,
+      bytesTransferred: null,
+      backupResult: BackupResult.NotRequested,
+    };
+
+    const sw: StopWatch = new StopWatch();
+    Logger.info('Sending local data to remote : %s : %j : %j', key, opts, checkStatus);
+
+    // Check it was not modified
+    // TODO: should be able to wrap this into the PUT request probably?
+    if (!opts.force && checkStatus?.remoteLastUpdatedEpochMs) {
+      sw.start('statusCheck');
+      const current: RemoteStatusData<string> = await this.readRemoteStatus(key);
+      if (!RemoteFileTracker.statusMatch(checkStatus, current)) {
+        rval.type = FileTransferResultType.Error;
+        rval.error =
+          'CheckStatus did not match, was ' +
+          JSON.stringify(checkStatus) +
+          ' but current is ' +
+          JSON.stringify(current) +
+          ' and force not specified';
+        rval.bytesTransferred = 0;
+        return rval; // TODO: Refactor for single exit point
+      }
+      sw.stop('statusCheck');
+      Logger.info('Performed status check in %s', sw.dump('statusCheck'));
+    }
+
+    // Backup if requested
+    if (opts.backup) {
+      sw.start('backup');
+      rval.backupResult = await this.backupRemote(key);
+      sw.stop('backup');
+      Logger.info('Performed backup in %s', sw.dump('backup'));
+    }
+
+    // Write the file
+    try {
+      sw.start('send');
+      const out: CompleteMultipartUploadCommandOutput = await this.opts.s3CacheRatchet.writeStreamToCacheFile(key, src);
+      sw.stop('send');
+      Logger.info('Sent to remote in %s : %j', sw.dump('send'), out);
+    } catch (err) {
+      Logger.error('Failed to write %s - %s', key, err, err);
+      rval.type = FileTransferResultType.Error;
+      rval.error = ErrorRatchet.safeStringifyErr(err);
+      rval.bytesTransferred = 0;
+    }
+    Logger.info('Overall timing : %s', sw.dump());
+    return rval;
+  }
+
+  public async backupRemote(key: string): Promise<BackupResult> {
+    let rval: BackupResult = null;
+    try {
+      const lastSlash: number = key.lastIndexOf('/');
+      const datePart: string = '/backup/' + DateTime.now().toFormat('yyyy/MM/dd/HH/mm/ss') + '/';
+      const newPath: string = lastSlash > -1 ? key.substring(0, lastSlash) + datePart + key.substring(lastSlash + 1) : datePart + key;
+
+      Logger.info('Backing up path %s to %s', key, newPath);
+      await this.opts.s3CacheRatchet.copyFile(key, newPath);
+      rval = BackupResult.Success;
+    } catch (err) {
+      Logger.error('Failed to backup %s : %s', key, err, err);
+      rval = BackupResult.Error;
+    }
+    return rval;
+  }
+}

package/src/s3/impl/s3-storage-provider.spec.ts

@@ -0,0 +1,32 @@
+import { S3StorageProvider } from './s3-storage-provider.js';
+import { S3Client } from '@aws-sdk/client-s3';
+import { describe, expect, test } from 'vitest';
+import { S3CacheRatchet } from '../s3-cache-ratchet.js';
+import { SimpleCache } from '../../cache/simple-cache.js';
+import { SimpleCacheObjectWrapper } from '../../cache/simple-cache-object-wrapper.js';
+
+describe('#S3StorageProvider', function () {
+  test.skip('should read/write/delete with an S3 handler', async () => {
+    const s3: S3Client = new S3Client({ region: 'us-east-1' });
+    const cache: S3CacheRatchet = new S3CacheRatchet(s3, 'test-bucket');
+    const s3StorageProvider: S3StorageProvider = new S3StorageProvider(cache, 'test-cache');
+
+    const simpleCache: SimpleCache = new SimpleCache(s3StorageProvider, 2000000);
+
+    await simpleCache.removeFromCache('test1'); // Make sure clear
+
+    const test1a: SimpleCacheObjectWrapper<any> = await simpleCache.fetchWrapper<any>('test1', () => Promise.resolve({ x: 1 }));
+    expect(test1a).not.toBeNull();
+    expect(test1a.generated).toBeTruthy();
+    expect(test1a.value).not.toBeNull();
+    expect(test1a.value['x']).toEqual(1);
+
+    const test1b: SimpleCacheObjectWrapper<any> = await simpleCache.fetchWrapper<any>('test1', () => Promise.resolve({ x: 1 }));
+    expect(test1b).not.toBeNull();
+    expect(test1b.generated).toBeFalsy();
+    expect(test1b.value).not.toBeNull();
+    expect(test1b.value['x']).toEqual(1);
+
+    await simpleCache.removeFromCache('test1'); // Make sure clear
+  }, 60_000);
+});

package/src/s3/impl/s3-storage-provider.ts

@@ -0,0 +1,60 @@
+/*
+  Objects implementing this interface can store and retrieve objects in a cache, using a read-thru
+  approach.
+*/
+
+import { SimpleCacheObjectWrapper } from '../../cache/simple-cache-object-wrapper.js';
+import { SimpleCacheStorageProvider } from '../../cache/simple-cache-storage-provider.js';
+import { PutObjectOutput } from '@aws-sdk/client-s3';
+import { S3CacheRatchetLike } from '../s3-cache-ratchet-like.js';
+import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
+import { StringRatchet } from '@bitblit/ratchet-common/lang/string-ratchet';
+
+export class S3StorageProvider implements SimpleCacheStorageProvider {
+  constructor(
+    private s3CacheRatchet: S3CacheRatchetLike,
+    private prefix: string,
+  ) {
+    RequireRatchet.notNullOrUndefined(this.s3CacheRatchet, 's3CacheRatchet');
+    RequireRatchet.notNullOrUndefined(this.s3CacheRatchet.getDefaultBucket(), 's3CacheRatchet.defaultBucket');
+  }
+
+  public keyToPath(cacheKey: string): string {
+    let rval: string = StringRatchet.trimToEmpty(this.prefix);
+    if (rval.length > 0 && !rval.endsWith('/')) {
+      rval += '/';
+    }
+    rval += cacheKey;
+    return rval;
+  }
+
+  public async readFromCache<T>(cacheKey: string): Promise<SimpleCacheObjectWrapper<T>> {
+    const rval: SimpleCacheObjectWrapper<T> = await this.s3CacheRatchet.fetchCacheFileAsObject<SimpleCacheObjectWrapper<T>>(
+      this.keyToPath(cacheKey),
+    );
+    return rval;
+  }
+
+  public async storeInCache<T>(value: SimpleCacheObjectWrapper<T>): Promise<boolean> {
+    RequireRatchet.notNullOrUndefined(value, 'value');
+    RequireRatchet.notNullOrUndefined(value.cacheKey, 'value.cacheKey');
+    const tmp: PutObjectOutput = await this.s3CacheRatchet.writeObjectToCacheFile(this.keyToPath(value.cacheKey), value);
+    return !!tmp;
+  }
+
+  public async removeFromCache(cacheKey: string): Promise<void> {
+    await this.s3CacheRatchet.removeCacheFile(this.keyToPath(cacheKey));
+  }
+
+  public async clearCache(): Promise<number> {
+    const keys: string[] = await this.s3CacheRatchet.directChildrenOfPrefix(this.keyToPath(''));
+    const _removed: any[] = await Promise.all(keys.map((k) => this.removeFromCache(k)));
+    return keys.length;
+  }
+
+  public async readAll(): Promise<SimpleCacheObjectWrapper<any>[]> {
+    const keys: string[] = await this.s3CacheRatchet.directChildrenOfPrefix(this.keyToPath(''));
+    const rval: SimpleCacheObjectWrapper<any>[] = await Promise.all(keys.map((k) => this.readFromCache(k)));
+    return rval;
+  }
+}

package/src/s3/s3-cache-ratchet-like.ts

@@ -0,0 +1,64 @@
+/*
+  Objects implementing S3CacheRatchetLike wrap S3 with an ability to store and retrieve objects cached as json files
+*/
+
+import { Readable } from 'stream';
+import {
+  CompleteMultipartUploadCommandOutput,
+  CopyObjectCommandOutput,
+  DeleteObjectCommandOutput,
+  GetObjectCommandInput,
+  GetObjectCommandOutput,
+  HeadObjectCommandOutput,
+  PutObjectCommandInput,
+  S3Client,
+} from '@aws-sdk/client-s3';
+
+export interface S3CacheRatchetLike {
+  getDefaultBucket(): string;
+  getS3Client(): S3Client;
+  fileExists(key: string, bucket?: string): Promise<boolean>;
+
+  fetchCacheFilePassThru(req: GetObjectCommandInput): Promise<GetObjectCommandOutput>;
+
+  fetchCacheFileAsS3GetObjectCommandOutput(key: string, bucket?: string): Promise<GetObjectCommandOutput>;
+
+  fetchCacheFileAsReadableStream(key: string, bucket?: string): Promise<ReadableStream>;
+
+  fetchCacheFileAsBuffer(key: string, bucket?: string): Promise<Buffer>;
+
+  fetchCacheFileAsString(key: string, bucket?: string): Promise<string>;
+
+  fetchCacheFileAsObject<T>(key: string, bucket?: string): Promise<T>;
+
+  removeCacheFile(key: string, bucket?: string): Promise<DeleteObjectCommandOutput>;
+  writeObjectToCacheFile(
+    key: string,
+    dataObject: any,
+    template?: PutObjectCommandInput,
+    bucket?: string,
+  ): Promise<CompleteMultipartUploadCommandOutput>;
+
+  writeStringToCacheFile(
+    key: string,
+    dataString: string,
+    template?: PutObjectCommandInput,
+    bucket?: string,
+  ): Promise<CompleteMultipartUploadCommandOutput>;
+
+  writeStreamToCacheFile(
+    key: string,
+    data: ReadableStream | Readable,
+    template?: PutObjectCommandInput,
+    bucket?: string,
+  ): Promise<CompleteMultipartUploadCommandOutput>;
+
+  synchronize(srcPrefix: string, targetPrefix: string, targetRatchet?: S3CacheRatchetLike, recurseSubFolders?: boolean): Promise<string[]>;
+  preSignedDownloadUrlForCacheFile(key: string, expirationSeconds?: number, bucket?: string): Promise<string>;
+  fetchMetaForCacheFile(key: string, bucket?: string): Promise<HeadObjectCommandOutput>;
+  cacheFileAgeInSeconds(key: string, bucket?: string): Promise<number>;
+  copyFile(srcKey: string, dstKey: string, srcBucket?: string, dstBucket?: string): Promise<CopyObjectCommandOutput>;
+  quietCopyFile(srcKey: string, dstKey: string, srcBucket?: string, dstBucket?: string): Promise<boolean>;
+  directChildrenOfPrefix(prefix: string, expandFiles?: boolean, bucket?: string, maxToReturn?: number): Promise<string[]>;
+  allSubFoldersOfPrefix(prefix: string, bucket?: string): Promise<string[]>;
+}

package/src/s3/s3-cache-ratchet.spec.ts

@@ -0,0 +1,150 @@
+import {
+  CopyObjectCommand,
+  CreateMultipartUploadCommand,
+  GetObjectCommand,
+  HeadObjectCommand,
+  PutObjectCommand,
+  PutObjectCommandInput,
+  PutObjectCommandOutput,
+  S3Client,
+  UploadPartCommand,
+} from '@aws-sdk/client-s3';
+import { S3CacheRatchet } from './s3-cache-ratchet.js';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { StringRatchet } from '@bitblit/ratchet-common/lang/string-ratchet';
+import { WebStreamRatchet } from '@bitblit/ratchet-common/lang/web-stream-ratchet';
+import { mockClient } from 'aws-sdk-client-mock';
+import { beforeEach, describe, expect, test, vi } from 'vitest';
+
+vi.mock('@aws-sdk/s3-request-presigner', async (importOriginal) => {
+  const mod = await importOriginal<typeof import('@aws-sdk/s3-request-presigner')>();
+  return {
+    ...mod,
+    getSignedUrl: vi.fn(() => Promise.resolve('https://test.link/test.jpg')),
+  };
+});
+
+let mockS3 = null;
+let mockS3OtherAccount = null;
+
+describe('#fileExists', function () {
+  // I'd rather do this above but then typescript screams it does not implement S3Client
+  mockS3 = mockClient(S3Client);
+  mockS3OtherAccount = mockClient(S3Client);
+
+  beforeEach(() => {
+    mockS3.reset();
+    mockS3OtherAccount.reset();
+  });
+
+  test('should return false for files that do not exist', async () => {
+    mockS3.on(HeadObjectCommand).rejects({ statusCode: 404, $metadata: null });
+    const cache: S3CacheRatchet = new S3CacheRatchet(mockS3, 'test-bucket');
+    const out: boolean = await cache.fileExists('test-missing-file');
+
+    expect(out).toEqual(false);
+  });
+
+  // TODO: CAW 2023-02-28 : Make me work after the stub gets fixed for this
+  test.skip('should create a expiring link', async () => {
+    //mockS3.getSignedUrl.mockReturnValue('https://test.link/test.jpg');
+
+    const cache: S3CacheRatchet = new S3CacheRatchet(mockS3, 'test-bucket');
+    const out: string = await cache.preSignedDownloadUrlForCacheFile('test.jpg', 300);
+
+    expect(out).toEqual('https://test.link/test.jpg');
+  });
+
+  test('should copy an object', async () => {
+    mockS3.on(CopyObjectCommand).resolves({});
+    const cache: S3CacheRatchet = new S3CacheRatchet(mockS3, 'test-bucket');
+    const out: boolean = await cache.quietCopyFile('test.png', 'test2.png');
+
+    expect(out).toBeTruthy();
+  });
+
+  // TODO: CAW 2023-02-28 : Make me work after the stub gets fixed for this
+  test.skip('should copy a file to s3', async () => {
+    //mockS3['config'] = {};
+    //mockS3.reset();
+    // This mocks for uploading small files
+    mockS3.on(PutObjectCommand).resolves({});
+    // These mock for the multipart upload command
+    mockS3.on(CreateMultipartUploadCommand).resolves({ UploadId: '1' });
+    mockS3.on(UploadPartCommand).resolves({ ETag: '1' });
+
+    const cache: S3CacheRatchet = new S3CacheRatchet(mockS3, 'test-bucket');
+    const stream: ReadableStream = WebStreamRatchet.stringToWebReadableStream(StringRatchet.createRandomHexString(1024 * 1024 * 6));
+
+    const out: PutObjectCommandOutput = await cache.writeStreamToCacheFile('s3-cache-ratchet.spec.ts', stream, {
+      ContentType: 'text/typescript',
+    } as PutObjectCommandInput);
+
+    Logger.info('Calls: %j', mockS3.calls());
+    expect(out).toBeTruthy();
+  });
+
+  test('should pull a file as a string', async () => {
+    // Need to re-call this multiple times to get the stream reset
+    async function createNew(): Promise<any> {
+      return {
+        Body: {
+          transformToByteArray: async () =>
+            Promise.resolve(StringRatchet.stringToUint8Array(JSON.stringify({ test: StringRatchet.createRandomHexString(128) }))),
+          transformToString: async () => Promise.resolve(JSON.stringify({ test: StringRatchet.createRandomHexString(128) })),
+        },
+        $metadata: null,
+      };
+    }
+
+    mockS3.on(GetObjectCommand).resolves(createNew());
+
+    const cache: S3CacheRatchet = new S3CacheRatchet(mockS3, 'test-bucket');
+    const fileName: string = 'test-file.json';
+
+    const outBuf: Buffer = await cache.fetchCacheFileAsBuffer(fileName);
+    expect(outBuf).toBeTruthy();
+    expect(outBuf.length).toBeGreaterThan(100);
+    mockS3.reset();
+    mockS3.on(GetObjectCommand).resolves(createNew());
+
+    const outString: string = await cache.fetchCacheFileAsString(fileName);
+    expect(outString).toBeTruthy();
+    expect(outString.length).toBeGreaterThan(100);
+    mockS3.reset();
+    mockS3.on(GetObjectCommand).resolves(createNew());
+
+    const outObject: any = await cache.fetchCacheFileAsObject(fileName);
+    expect(outObject).toBeTruthy();
+    expect(outObject['test']).toBeTruthy();
+  });
+
+  //---
+
+  test.skip('should sync 2 folders', async () => {
+    const cache1: S3CacheRatchet = new S3CacheRatchet(mockS3, 'test1');
+    const cache2: S3CacheRatchet = new S3CacheRatchet(mockS3, 'test2');
+    const out: string[] = await cache1.synchronize('src/', 'dst/', cache2);
+
+    expect(out).not.toBeNull();
+  }, 60_000);
+
+  test.skip('should list direct children past 1000', async () => {
+    const s3: S3Client = new S3Client({ region: 'us-east-1' });
+    const cache: S3CacheRatchet = new S3CacheRatchet(s3, 'test-bucket');
+    const out: string[] = await cache.directChildrenOfPrefix('test/aws/test-path-with-lots-of-childen/');
+    expect(out).toBeTruthy();
+    expect(out.length).toBeGreaterThan(1000);
+
+    Logger.info('Got: %s', out);
+    expect(out).toBeTruthy();
+  });
+
+  test.skip('should sync cross-account', async () => {
+    const cache1: S3CacheRatchet = new S3CacheRatchet(mockS3, 'bucket1');
+    const cache2: S3CacheRatchet = new S3CacheRatchet(mockS3OtherAccount, 'bucket2');
+
+    const res: any = await cache1.synchronize('test1/', 'test2/', cache2, true);
+    expect(res).not.toBeNull();
+  }, 50_000);
+});