@bitblit/ratchet-aws 6.0.146-alpha → 6.0.148-alpha

package/src/s3/s3-cache-ratchet.ts

@@ -0,0 +1,476 @@
+import {
+  CompleteMultipartUploadCommandOutput,
+  CopyObjectCommand,
+  CopyObjectCommandInput,
+  CopyObjectCommandOutput,
+  DeleteObjectCommand,
+  DeleteObjectCommandInput,
+  DeleteObjectCommandOutput,
+  GetObjectCommand,
+  GetObjectCommandInput,
+  GetObjectCommandOutput,
+  HeadObjectCommand,
+  HeadObjectCommandOutput,
+  ListObjectsCommand,
+  ListObjectsCommandInput,
+  ListObjectsCommandOutput,
+  ListObjectsV2CommandInput,
+  ListObjectsV2CommandOutput,
+  NoSuchKey,
+  NotFound, PutObjectCommand,
+  PutObjectCommandInput,
+  PutObjectCommandOutput,
+  S3Client
+} from "@aws-sdk/client-s3";
+import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { StringRatchet } from '@bitblit/ratchet-common/lang/string-ratchet';
+import { StopWatch } from '@bitblit/ratchet-common/lang/stop-watch';
+import { WebStreamRatchet } from '@bitblit/ratchet-common/lang/web-stream-ratchet';
+
+import { StreamingBlobPayloadInputTypes } from '@smithy/types';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
+import { Progress, Upload } from '@aws-sdk/lib-storage';
+import { Readable } from 'stream';
+import { S3CacheRatchetLike } from './s3-cache-ratchet-like.js';
+import { ExpandedFileChildren } from './expanded-file-children.js';
+
+export class S3CacheRatchet implements S3CacheRatchetLike {
+  constructor(
+    private s3: S3Client,
+    private defaultBucket: string = null,
+  ) {
+    RequireRatchet.notNullOrUndefined(this.s3, 's3');
+  }
+
+  public get s3Client(): S3Client {
+    return this.s3;
+  }
+
+  public static applyCacheControlMaxAge(input: PutObjectCommandInput, seconds: number): PutObjectCommandInput {
+    if (input && seconds) {
+      input.CacheControl = 'max-age=' + seconds;
+    }
+    return input;
+  }
+
+  public static applyUserMetaData(input: PutObjectCommandInput, key: string, value: string): PutObjectCommandInput {
+    if (input && StringRatchet.trimToNull(key) && StringRatchet.trimToNull(value)) {
+      input.Metadata = input.Metadata || {};
+      input.Metadata[key] = value;
+    }
+    return input;
+  }
+
+  public getDefaultBucket(): string {
+    return this.defaultBucket;
+  }
+
+  public getS3Client(): S3Client {
+    return this.s3;
+  }
+
+  public async fileExists(key: string, bucket: string = null): Promise<boolean> {
+    try {
+      const head: HeadObjectCommandOutput = await this.fetchMetaForCacheFile(key, this.bucketVal(bucket));
+      return !!head;
+    } catch (err) {
+      Logger.silly('Error calling file exists (as expected) %s', err);
+      return false;
+    }
+  }
+
+  public async fetchCacheFilePassThru(req: GetObjectCommandInput): Promise<GetObjectCommandOutput> {
+    let rval: GetObjectCommandOutput = null;
+    try {
+      rval = await this.s3.send(new GetObjectCommand(req));
+    } catch (err) {
+      if (err instanceof NoSuchKey) {
+        Logger.debug('Key %s not found - returning null', req.Key);
+        rval = null;
+      } else {
+        // Rethrow everything else
+        throw err;
+      }
+    }
+    return rval;
+  }
+
+  public async fetchCacheFileAsS3GetObjectCommandOutput(key: string, bucket: string = null): Promise<GetObjectCommandOutput> {
+    let rval: GetObjectCommandOutput = null;
+    try {
+      const params: GetObjectCommandInput = {
+        Bucket: this.bucketVal(bucket),
+        Key: key,
+      };
+      rval = await this.s3.send(new GetObjectCommand(params));
+    } catch (err) {
+      if (err instanceof NoSuchKey) {
+        Logger.debug('Key %s not found - returning null', key);
+        rval = null;
+      } else {
+        // Rethrow everything else
+        throw err;
+      }
+    }
+    return rval;
+  }
+
+  public async fetchCacheFileAsReadableStream(key: string, bucket: string = null): Promise<ReadableStream> {
+    const out: GetObjectCommandOutput = await this.fetchCacheFileAsS3GetObjectCommandOutput(key, bucket);
+
+    return out.Body.transformToWebStream();
+  }
+
+  public async fetchCacheFileAsBuffer(key: string, bucket: string = null): Promise<Buffer> {
+    let rval: Buffer = null;
+    const out: GetObjectCommandOutput = await this.fetchCacheFileAsS3GetObjectCommandOutput(key, bucket);
+    if (out?.Body) {
+      const tmp: Uint8Array = await out.Body.transformToByteArray();
+      rval = Buffer.from(tmp);
+    }
+    return rval;
+  }
+
+  public async fetchCacheFileAsString(key: string, bucket: string = null): Promise<string> {
+    let rval: string = null;
+    const out: GetObjectCommandOutput = await this.fetchCacheFileAsS3GetObjectCommandOutput(key, bucket);
+    if (out?.Body) {
+      rval = await out.Body.transformToString();
+    }
+    return rval;
+  }
+
+  public async fetchCacheFileAsObject<T>(key: string, bucket: string = null): Promise<T> {
+    const value: string = await this.fetchCacheFileAsString(key, bucket);
+    return value ? (JSON.parse(value) as T) : null;
+  }
+
+  public async removeCacheFile(key: string, bucket: string = null): Promise<DeleteObjectCommandOutput> {
+    let rval: DeleteObjectCommandOutput = null;
+    const params: DeleteObjectCommandInput = {
+      Bucket: this.bucketVal(bucket),
+      Key: key,
+    };
+    try {
+      rval = await this.s3.send(new DeleteObjectCommand(params));
+    } catch (err) {
+      if (err && err instanceof NotFound) {
+        Logger.info('Swallowing 404 deleting missing object %s %s', bucket, key);
+        rval = null;
+      } else {
+        throw err;
+      }
+    }
+    return rval;
+  }
+
+  // Given new board data, write it to the S3 file and set the refresh flag appropriately
+  public async writeObjectToCacheFile(
+    key: string,
+    dataObject: any,
+    template?: Partial<PutObjectCommandInput>,
+    bucket?: string,
+  ): Promise<CompleteMultipartUploadCommandOutput> {
+    const json = JSON.stringify(dataObject);
+    return this.writeStringToCacheFile(key, json, template, bucket);
+  }
+
+  // Given new board data, write it to the S3 file and set the refresh flag appropriately
+  public async writeStringToCacheFile(
+    key: string,
+    dataString: string,
+    template?: Partial<PutObjectCommandInput>,
+    bucket?: string,
+  ): Promise<CompleteMultipartUploadCommandOutput> {
+    const stream: ReadableStream = WebStreamRatchet.stringToWebReadableStream(dataString);
+    return this.writeStreamToCacheFile(key, stream, template, bucket);
+  }
+
+  public async writeStreamToCacheFile(
+    key: string,
+    data: ReadableStream | Readable, //StreamingBlobPayloadInputTypes
+    template?: Partial<PutObjectCommandInput>,
+    bucket?: string,
+    progressFn: (progress: Progress) => void = (progress) => {
+      Logger.debug('Uploading : %s', progress);
+    },
+  ): Promise<CompleteMultipartUploadCommandOutput> {
+    const params: PutObjectCommandInput = Object.assign({}, template || {}, {
+      Bucket: this.bucketVal(bucket),
+      Key: key,
+      Body: data as StreamingBlobPayloadInputTypes,
+    });
+
+    const upload: Upload = new Upload({
+      client: this.s3,
+      params: params,
+      tags: [],
+      queueSize: 4,
+      partSize: 1024 * 1024 * 5,
+      leavePartsOnError: false,
+    });
+
+    if (progressFn) {
+      upload.on('httpUploadProgress', progressFn);
+    }
+    const result: CompleteMultipartUploadCommandOutput = await upload.done();
+
+    return result;
+  }
+
+  public async synchronize(
+    srcPrefix: string,
+    targetPrefix: string,
+    targetRatchet: S3CacheRatchetLike = this,
+    recurseSubFolders: boolean = false,
+  ): Promise<string[]> {
+    RequireRatchet.notNullOrUndefined(srcPrefix, 'srcPrefix');
+    RequireRatchet.notNullOrUndefined(targetPrefix, 'targetPrefix');
+    RequireRatchet.true(srcPrefix.endsWith('/'), 'srcPrefix must end in /');
+    RequireRatchet.true(targetPrefix.endsWith('/'), 'targetPrefix must end in /');
+    let rval: string[] = [];
+    const sourceFiles: string[] = await this.directChildrenOfPrefix(srcPrefix);
+    const targetFiles: string[] = await targetRatchet.directChildrenOfPrefix(targetPrefix);
+    const sw: StopWatch = new StopWatch();
+
+    for (let i = 0; i < sourceFiles.length; i++) {
+      const sourceFile: string = sourceFiles[i];
+      Logger.info('Processing %s : %s', sourceFile, sw.dumpExpected(i / sourceFiles.length));
+      if (sourceFile.endsWith('/')) {
+        if (recurseSubFolders) {
+          Logger.info('%s is a subfolder - recursing');
+          const subs: string[] = await this.synchronize(
+            srcPrefix + sourceFile,
+            targetPrefix + sourceFile,
+            targetRatchet,
+            recurseSubFolders,
+          );
+          Logger.info('Got %d back from %s', subs.length, sourceFile);
+          rval = rval.concat(subs);
+        } else {
+          Logger.info('%s is a subfolder and recurse not specified - skipping', sourceFile);
+        }
+      } else {
+        let shouldCopy: boolean = true;
+        const srcMeta: HeadObjectCommandOutput = await this.fetchMetaForCacheFile(srcPrefix + sourceFile);
+        if (targetFiles.includes(sourceFile)) {
+          const targetMeta: HeadObjectCommandOutput = await targetRatchet.fetchMetaForCacheFile(targetPrefix + sourceFile);
+          if (srcMeta.ETag === targetMeta.ETag) {
+            Logger.debug('Skipping - identical');
+            shouldCopy = false;
+          }
+        }
+        if (shouldCopy) {
+          Logger.debug('Copying...');
+          const srcStream: ReadableStream = await this.fetchCacheFileAsReadableStream(srcPrefix + sourceFile);
+          try {
+            const written: PutObjectCommandOutput = await targetRatchet.writeStreamToCacheFile(
+              targetPrefix + sourceFile,
+              srcStream,
+              srcMeta as unknown as PutObjectCommandInput, // Carry forward any metadata
+              undefined,
+            );
+            Logger.silly('Write result : %j', written);
+            rval.push(sourceFile);
+          } catch (err) {
+            Logger.error('Failed to sync : %s : %s', sourceFile, err);
+          }
+        }
+      }
+    }
+
+    Logger.info('Found %d files, copied %d', sourceFiles.length, rval.length);
+    sw.log();
+    return rval;
+  }
+
+  public async fetchMetaForCacheFile(key: string, bucket: string = null): Promise<HeadObjectCommandOutput> {
+    let rval: HeadObjectCommandOutput = null;
+    try {
+      rval = await this.s3.send(
+        new HeadObjectCommand({
+          Bucket: this.bucketVal(bucket),
+          Key: key,
+        }),
+      );
+    } catch (err) {
+      if (err && err instanceof NotFound) {
+        Logger.info('Cache file %s %s not found returning null', this.bucketVal(bucket), key);
+        rval = null;
+      } else {
+        Logger.error('Unrecognized error, rethrowing : %s', err, err);
+        throw err;
+      }
+    }
+    return rval;
+  }
+
+  public async cacheFileAgeInSeconds(key: string, bucket: string = null): Promise<number> {
+    try {
+      const res: HeadObjectCommandOutput = await this.fetchMetaForCacheFile(key, bucket);
+      if (res && res.LastModified) {
+        return Math.floor((new Date().getTime() - res.LastModified.getTime()) / 1000);
+      } else {
+        Logger.warn('Cache file %s %s had no last modified returning null', this.bucketVal(bucket), key);
+        return null;
+      }
+    } catch (err) {
+      if (err && err instanceof NotFound) {
+        Logger.warn('Cache file %s %s not found returning null', this.bucketVal(bucket), key);
+        return null;
+      } else {
+        throw err;
+      }
+    }
+  }
+
+  // Shortcut to copy/delete
+  public async renameFile(
+    srcKey: string,
+    dstKey: string,
+    srcBucket: string = null,
+    dstBucket: string = null,
+  ): Promise<CopyObjectCommandOutput> {
+    Logger.info('Rename %s to %s (%s/%s)', srcKey, dstKey, srcBucket, dstBucket);
+    const output: CopyObjectCommandOutput = await this.copyFile(srcKey, dstKey, srcBucket, dstBucket);
+    if (output) {
+      await this.removeCacheFile(srcKey, srcBucket);
+    }
+    return output;
+  }
+
+  public async copyFile(
+    srcKey: string,
+    dstKey: string,
+    srcBucket: string = null,
+    dstBucket: string = null,
+    template?: Partial<CopyObjectCommandInput>
+  ): Promise<CopyObjectCommandOutput> {
+    const params: CopyObjectCommandInput = Object.assign({MetadataDirective:'COPY'}, template ?? {}) as CopyObjectCommandInput;
+    params.CopySource= '/' + this.bucketVal(srcBucket) + '/' + srcKey;
+    params.Bucket= this.bucketVal(dstBucket);
+    params.Key= dstKey;
+    Logger.info('Performing copy with %j', params);
+    const rval: CopyObjectCommandOutput = await this.s3.send(new CopyObjectCommand(params));
+    return rval;
+  }
+
+  public async quietCopyFile(srcKey: string, dstKey: string, srcBucket: string = null, dstBucket: string = null): Promise<boolean> {
+    let rval: boolean = false;
+    try {
+      const _tmp: CopyObjectCommandOutput = await this.copyFile(srcKey, dstKey, srcBucket, dstBucket);
+      rval = true;
+    } catch (err) {
+      Logger.silly('Failed to copy file in S3 : %s', err);
+    }
+    return rval;
+  }
+
+  public async preSignedDownloadUrlForCacheFile(key: string, expirationSeconds = 3600, bucket: string = null): Promise<string> {
+    const getCommand: GetObjectCommandInput = {
+      Bucket: this.bucketVal(bucket),
+      Key: key,
+    };
+    const link: string = await getSignedUrl(this.s3, new GetObjectCommand(getCommand), { expiresIn: expirationSeconds });
+    return link;
+  }
+
+  public async preSignedUploadUrlForCacheFile(key: string, expirationSeconds = 3600, bucket: string = null): Promise<string> {
+    const putCommand: PutObjectCommandInput = {
+      Bucket: this.bucketVal(bucket),
+      Key: key,
+    };
+    const link: string = await getSignedUrl(this.s3, new PutObjectCommand(putCommand), { expiresIn: expirationSeconds });
+    return link;
+  }
+
+  public async directChildrenOfPrefix(
+    prefix: string,
+    expandFiles = false,
+    bucket: string = null,
+    maxToReturn: number = null,
+  ): Promise<string[]> {
+    const returnValue: any[] = [];
+
+    const params: ListObjectsCommandInput = {
+      Bucket: this.bucketVal(bucket),
+      Prefix: prefix,
+      Delimiter: '/',
+    };
+
+    let response: ListObjectsCommandOutput = null;
+    do {
+      response = await this.s3.send(new ListObjectsCommand(params));
+
+      const prefixLength = prefix.length;
+      // Process directories
+      if (response['CommonPrefixes']) {
+        response['CommonPrefixes'].forEach((cp) => {
+          if (!maxToReturn || returnValue.length < maxToReturn) {
+            const value = cp['Prefix'].substring(prefixLength);
+            returnValue.push(value);
+          }
+        });
+      }
+
+      // Process files
+      if (response['Contents']) {
+        await Promise.all(
+          response['Contents'].map(async (cp) => {
+            if (!maxToReturn || returnValue.length < maxToReturn) {
+              if (expandFiles) {
+                const expanded: ExpandedFileChildren = {
+                  link: await this.preSignedDownloadUrlForCacheFile(cp['Key'], 3600, bucket),
+                  name: cp['Key'].substring(prefixLength),
+                  size: cp['Size'],
+                };
+                returnValue.push(expanded);
+              } else {
+                returnValue.push(cp['Key'].substring(prefixLength));
+              }
+            }
+          }),
+        );
+      }
+      params.Marker = response.NextMarker;
+    } while (params.Marker && (!maxToReturn || returnValue.length < maxToReturn));
+
+    return returnValue;
+  }
+
+  public async allSubFoldersOfPrefix(prefix: string, bucket: string = null): Promise<string[]> {
+    const returnValue: string[] = [prefix];
+    let idx: number = 0;
+
+    while (idx < returnValue.length) {
+      const next: string = returnValue[idx++];
+      Logger.debug('Pulling %s (%d remaining)', next, returnValue.length - idx);
+      const req: ListObjectsV2CommandInput = {
+        Bucket: this.bucketVal(bucket),
+        Prefix: next,
+        Delimiter: '/',
+      };
+      let resp: ListObjectsV2CommandOutput = null;
+
+      do {
+        req.ContinuationToken = resp ? resp.NextContinuationToken : null;
+        resp = await this.s3.send(new ListObjectsCommand(req));
+        resp.CommonPrefixes.forEach((p) => {
+          returnValue.push(p.Prefix);
+        });
+        Logger.debug('g:%j', resp);
+      } while (resp.NextContinuationToken);
+    }
+
+    return returnValue;
+  }
+
+  private bucketVal(explicitBucket: string): string {
+    const rval: string = explicitBucket ? explicitBucket : this.defaultBucket;
+    if (!rval) {
+      throw 'You must set either the default bucket or pass it explicitly';
+    }
+    return rval;
+  }
+}

package/src/s3/s3-location-sync-ratchet.ts

@@ -0,0 +1,207 @@
+import {
+  CopyObjectCommand,
+  CopyObjectCommandInput,
+  GetObjectCommand,
+  GetObjectOutput,
+  ListObjectsV2Command,
+  ListObjectsV2CommandInput,
+  ListObjectsV2CommandOutput,
+  PutObjectCommandInput,
+  S3Client,
+} from '@aws-sdk/client-s3';
+import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { PromiseRatchet } from '@bitblit/ratchet-common/lang/promise-ratchet';
+import { Upload } from '@aws-sdk/lib-storage';
+
+export interface S3LocationSyncRatchetConfig {
+  srcS3: S3Client;
+  srcBucket: string;
+  srcPrefix: string;
+  dstS3: S3Client;
+  dstBucket: string;
+  dstPrefix: string;
+  maxNumThreads?: number;
+  maxRetries?: number;
+}
+
+interface BucketCmpResult {
+  needCopy: S3Object[];
+  existed: S3Object[];
+  diff: S3Object[];
+}
+
+interface S3Object {
+  Key: string;
+  ETag: string;
+  Size: number;
+  LastModified: Date;
+}
+
+export class S3LocationSyncRatchet {
+  private config: S3LocationSyncRatchetConfig;
+
+  constructor(config: S3LocationSyncRatchetConfig) {
+    RequireRatchet.notNullOrUndefined(config, 'config');
+
+    this.config = config;
+
+    if (!this.config.maxNumThreads) {
+      this.config.maxNumThreads = 15;
+    }
+
+    if (!this.config.maxRetries) {
+      this.config.maxRetries = 5;
+    }
+  }
+
+  public updateSrcPrefix(prefix: string): void {
+    this.config.srcPrefix = prefix;
+  }
+
+  public updateDstPrefix(prefix: string): void {
+    this.config.dstPrefix = prefix;
+  }
+
+  public async copyObject(key: string, size: number, express = false): Promise<void> {
+    const dstKey = key.replace(this.config.srcPrefix, this.config.dstPrefix);
+    let completedCopying = false;
+    let retries = 0;
+    while (!completedCopying && retries < this.config.maxRetries) {
+      Logger.debug(`${retries > 0 ? `Retry ${retries} ` : ''}${express ? 'Express' : 'Slow'} copying
+                [${[this.config.srcBucket, key].join('/')} ---> ${[this.config.dstBucket, dstKey].join('/')}]`);
+      try {
+        if (express) {
+          const params: CopyObjectCommandInput = {
+            CopySource: encodeURIComponent([this.config.srcBucket, key].join('/')),
+            Bucket: this.config.dstBucket,
+            Key: dstKey,
+            MetadataDirective: 'COPY',
+          };
+          await this.config.dstS3.send(new CopyObjectCommand(params));
+        } else {
+          const fetched: GetObjectOutput = await this.config.srcS3.send(new GetObjectCommand({ Bucket: this.config.srcBucket, Key: key }));
+          const params: PutObjectCommandInput = {
+            Bucket: this.config.dstBucket,
+            Key: dstKey,
+            Body: fetched.Body,
+            ContentLength: size,
+          };
+
+          const upload: Upload = new Upload({
+            client: this.config.dstS3,
+            params: params,
+            tags: [],
+            queueSize: 4,
+            partSize: 1024 * 1024 * 5,
+            leavePartsOnError: false,
+          });
+
+          upload.on('httpUploadProgress', (progress) => {
+            Logger.debug('Uploading : %s', progress);
+          });
+          await upload.done();
+        }
+
+        completedCopying = true;
+      } catch (err) {
+        Logger.warn(
+          `Can't ${express ? 'express' : 'slow'} copy
+                  [${[this.config.srcBucket, key].join('/')} ---> ${[this.config.dstBucket, dstKey].join('/')}]: %j`,
+          err,
+        );
+        retries++;
+      }
+    }
+
+    Logger.debug(`Finished ${express ? 'express' : 'slow'} copying
+                  [${[this.config.srcBucket, key].join('/')} ---> ${[this.config.dstBucket, dstKey].join('/')}]`);
+  }
+
+  public async listObjects(bucket: string, prefix: string, s3: S3Client): Promise<any> {
+    Logger.info(`Scanning bucket [${[bucket, prefix].join('/')}]`);
+
+    const params: ListObjectsV2CommandInput = {
+      Bucket: bucket,
+      Prefix: prefix,
+    };
+
+    let more = true;
+    const rval = {};
+
+    while (more) {
+      const response: ListObjectsV2CommandOutput = await s3.send(new ListObjectsV2Command(params));
+      more = response.IsTruncated;
+      response.Contents.forEach((obj) => {
+        rval[obj.Key] = { Key: obj.Key, LastModified: obj.LastModified, ETag: obj.ETag, Size: obj.Size };
+      });
+
+      if (more) {
+        params.ContinuationToken = response.NextContinuationToken;
+      }
+    }
+    return rval;
+  }
+
+  public async startSyncing(): Promise<boolean> {
+    Logger.info(`Syncing [${this.config.srcBucket}/${this.config.srcPrefix}
+                  ---> ${this.config.dstBucket}/${this.config.dstPrefix}]`);
+    const cp = async (obj: S3Object) => {
+      await this.copyObject(obj.Key, obj.Size);
+    };
+
+    let cmpResult: BucketCmpResult = await this.compareSrcAndDst();
+
+    if (cmpResult.needCopy.length > 0 || cmpResult.diff.length > 0) {
+      await PromiseRatchet.runBoundedParallelSingleParam<any>(cp, cmpResult.needCopy, this, this.config.maxNumThreads);
+      await PromiseRatchet.runBoundedParallelSingleParam<any>(cp, cmpResult.diff, this, this.config.maxNumThreads);
+
+      Logger.info('Verifying...');
+      cmpResult = await this.compareSrcAndDst();
+      Logger.debug('Compare result %j', cmpResult);
+    }
+    return cmpResult.needCopy.length === 0 && cmpResult.diff.length === 0;
+  }
+
+  private async compareSrcAndDst(): Promise<BucketCmpResult> {
+    const getSrc: Promise<any> = this.listObjects(this.config.srcBucket, this.config.srcPrefix, this.config.srcS3);
+    const getDst: Promise<any> = this.listObjects(this.config.dstBucket, this.config.dstPrefix, this.config.dstS3);
+
+    const srcObjs = await getSrc;
+    const dstObjs = await getDst;
+
+    const rval: BucketCmpResult = {
+      needCopy: [],
+      existed: [],
+      diff: [],
+    };
+
+    await PromiseRatchet.runBoundedParallelSingleParam(
+      async (key) => {
+        const sObj: any = srcObjs[key];
+        const dstKey: string = key.replace(this.config.srcPrefix, this.config.dstPrefix);
+        const dObj: any = dstKey in dstObjs ? dstObjs[dstKey] : undefined;
+        if (!dObj) {
+          rval.needCopy.push(sObj);
+          return;
+        }
+
+        if (sObj.Size !== dObj.Size) {
+          rval.diff.push(sObj);
+          return;
+        }
+
+        if (sObj.LastModified.getTime() <= dObj.LastModified.getTime()) {
+          rval.existed.push(sObj);
+          return;
+        }
+
+        rval.diff.push(sObj);
+      },
+      Object.keys(srcObjs),
+      this,
+      this.config.maxNumThreads,
+    );
+    return rval;
+  }
+}

package/src/s3/s3-ratchet.spec.ts

@@ -0,0 +1,26 @@
+import { S3Client } from '@aws-sdk/client-s3';
+import { S3Ratchet } from './s3-ratchet.js';
+import { mockClient } from 'aws-sdk-client-mock';
+import { beforeEach, describe, expect, test } from 'vitest';
+
+let mockS3;
+
+describe('#S3Ratchet', function () {
+  mockS3 = mockClient(S3Client);
+  beforeEach(() => {
+    mockS3.reset();
+  });
+
+  test('should checkS3UrlForValidity', async () => {
+    expect(S3Ratchet.checkS3UrlForValidity('s3://test/out/b.txt')).toBeTruthy();
+    expect(S3Ratchet.checkS3UrlForValidity('http://test/out/b.txt')).toBeFalsy();
+  });
+
+  test('should extractBucketFromURL', async () => {
+    expect(S3Ratchet.extractBucketFromURL('s3://test/out/b.txt')).toEqual('test');
+  });
+
+  test('should extractKeyFromURL', async () => {
+    expect(S3Ratchet.extractKeyFromURL('s3://test/out/b.txt')).toEqual('out/b.txt');
+  });
+});