@bitblit/ratchet-aws 5.0.506-alpha → 5.0.508-alpha

package/lib/s3/impl/s3-remote-file-tracking-provider-options.d.ts

@@ -0,0 +1,4 @@
+import { S3CacheRatchetLike } from "../s3-cache-ratchet-like.js";
+export interface S3RemoteFileTrackingProviderOptions {
+    s3CacheRatchet: S3CacheRatchetLike;
+}

package/lib/s3/impl/s3-remote-file-tracking-provider-options.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=s3-remote-file-tracking-provider-options.js.map

package/lib/s3/impl/s3-remote-file-tracking-provider-options.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"s3-remote-file-tracking-provider-options.js","sourceRoot":"","sources":["../../../src/s3/impl/s3-remote-file-tracking-provider-options.ts"],"names":[],"mappings":""}

package/lib/s3/impl/s3-remote-file-tracking-provider.d.ts

@@ -0,0 +1,15 @@
+import { RemoteFileTrackingProvider } from "@bitblit/ratchet-common/network/remote-file-tracker/remote-file-tracking-provider";
+import { RemoteStatusData } from "@bitblit/ratchet-common/network/remote-file-tracker/remote-status-data";
+import { S3RemoteFileTrackingProviderOptions } from "./s3-remote-file-tracking-provider-options.js";
+import { RemoteStatusDataAndContent } from "@bitblit/ratchet-common/network/remote-file-tracker/remote-status-data-and-content";
+import { FileTransferResult } from "@bitblit/ratchet-common/network/remote-file-tracker/file-transfer-result";
+import { BackupResult } from "@bitblit/ratchet-common/network/remote-file-tracker/backup-result";
+import { RemoteFileTrackerPushOptions } from "@bitblit/ratchet-common/network/remote-file-tracker/remote-file-tracker-push-options";
+export declare class S3RemoteFileTrackingProvider implements RemoteFileTrackingProvider<string> {
+    private opts;
+    constructor(opts: S3RemoteFileTrackingProviderOptions);
+    readRemoteStatus(key: string): Promise<RemoteStatusData<string>>;
+    pullRemoteData(key: string, ifNewerThan?: RemoteStatusData<string>): Promise<RemoteStatusDataAndContent<string>>;
+    sendDataToRemote(src: ReadableStream, key: string, opts: RemoteFileTrackerPushOptions, checkStatus: RemoteStatusData<string>): Promise<FileTransferResult>;
+    backupRemote(key: string): Promise<BackupResult>;
+}

package/lib/s3/impl/s3-remote-file-tracking-provider.js

@@ -0,0 +1,125 @@
+import { RequireRatchet } from "@bitblit/ratchet-common/lang/require-ratchet";
+import { Logger } from "@bitblit/ratchet-common/logger/logger";
+import { StopWatch } from "@bitblit/ratchet-common/lang/stop-watch";
+import { DateTime } from "luxon";
+import { StringRatchet } from "@bitblit/ratchet-common/lang/string-ratchet";
+import { BackupResult } from "@bitblit/ratchet-common/network/remote-file-tracker/backup-result";
+import { RemoteFileTracker } from "@bitblit/ratchet-common/network/remote-file-tracker/remote-file-tracker";
+import { ErrorRatchet } from "@bitblit/ratchet-common/lang/error-ratchet";
+import { FileTransferResultType } from "@bitblit/ratchet-common/network/remote-file-tracker/file-transfer-result-type";
+export class S3RemoteFileTrackingProvider {
+    opts;
+    constructor(opts) {
+        this.opts = opts;
+        RequireRatchet.notNullOrUndefined(opts, 'opts');
+        RequireRatchet.notNullOrUndefined(opts.s3CacheRatchet, 'opts.s3CacheRatchet');
+        RequireRatchet.notNullOrUndefined(opts.s3CacheRatchet.getDefaultBucket(), 'opts.s3CacheRatchet must have default bucket set');
+    }
+    async readRemoteStatus(key) {
+        let rval = null;
+        if (StringRatchet.trimToNull(key)) {
+            const meta = await this.opts.s3CacheRatchet.fetchMetaForCacheFile(key);
+            if (meta) {
+                rval = {
+                    key: key,
+                    statusTakenEpochMs: Date.now(),
+                    remoteSizeInBytes: meta.ContentLength,
+                    remoteLastUpdatedEpochMs: meta.LastModified.getTime(),
+                    remoteHash: meta.ETag,
+                };
+            }
+        }
+        return rval;
+    }
+    async pullRemoteData(key, ifNewerThan) {
+        let rval = null;
+        const sw = new StopWatch();
+        const req = {
+            Bucket: this.opts.s3CacheRatchet.getDefaultBucket(),
+            Key: key,
+            IfModifiedSince: ifNewerThan?.remoteLastUpdatedEpochMs ? new Date(ifNewerThan.remoteLastUpdatedEpochMs) : null,
+        };
+        const output = await this.opts.s3CacheRatchet.fetchCacheFilePassThru(req);
+        if (output) {
+            rval = {
+                status: {
+                    key: key,
+                    statusTakenEpochMs: Date.now(),
+                    remoteSizeInBytes: output.ContentLength,
+                    remoteLastUpdatedEpochMs: output.LastModified.getTime(),
+                    remoteHash: output.ETag,
+                },
+                content: output.Body.transformToWebStream()
+            };
+            Logger.info('Fetched remote to local, %d bytes in %s : %s', output.ContentLength, sw.dump(), key);
+        }
+        else {
+            Logger.info('Did not pull %s - not modified', key);
+        }
+        return rval;
+    }
+    async sendDataToRemote(src, key, opts, checkStatus) {
+        RequireRatchet.notNullOrUndefined(src, 'src');
+        RequireRatchet.notNullOrUndefined(key, 'key');
+        RequireRatchet.notNullOrUndefined(opts, 'opts');
+        let rval = {
+            type: null,
+            error: null,
+            bytesTransferred: null,
+            backupResult: BackupResult.NotRequested
+        };
+        const sw = new StopWatch();
+        Logger.info('Sending local data to remote : %s : %j : %j', key, opts, checkStatus);
+        if (!opts.force && checkStatus?.remoteLastUpdatedEpochMs) {
+            sw.start('statusCheck');
+            const current = await this.readRemoteStatus(key);
+            if (!RemoteFileTracker.statusMatch(checkStatus, current)) {
+                rval.type = FileTransferResultType.Error;
+                rval.error = 'CheckStatus did not match, was ' + JSON.stringify(checkStatus) + ' but current is ' + JSON.stringify(current) + ' and force not specified';
+                rval.bytesTransferred = 0;
+                return rval;
+            }
+            sw.stop('statusCheck');
+            Logger.info('Performed status check in %s', sw.dump('statusCheck'));
+        }
+        if (opts.backup) {
+            sw.start('backup');
+            rval.backupResult = await this.backupRemote(key);
+            sw.stop('backup');
+            Logger.info('Performed backup in %s', sw.dump('backup'));
+        }
+        try {
+            sw.start('send');
+            const out = await this.opts.s3CacheRatchet.writeStreamToCacheFile(key, src);
+            sw.stop('send');
+            Logger.info('Sent to remote in %s : %j', sw.dump('send'), out);
+        }
+        catch (err) {
+            Logger.error('Failed to write %s - %s', key, err, err);
+            rval.type = FileTransferResultType.Error;
+            rval.error = ErrorRatchet.safeStringifyErr(err);
+            rval.bytesTransferred = 0;
+        }
+        Logger.info('Overall timing : %s', sw.dump());
+        return rval;
+    }
+    async backupRemote(key) {
+        let rval = null;
+        try {
+            const lastSlash = key.lastIndexOf('/');
+            const datePart = '/backup/' + DateTime.now().toFormat('yyyy/MM/dd/HH/mm/ss') + '/';
+            const newPath = lastSlash > -1
+                ? key.substring(0, lastSlash) + datePart + key.substring(lastSlash + 1)
+                : datePart + key;
+            Logger.info('Backing up path %s to %s', key, newPath);
+            await this.opts.s3CacheRatchet.copyFile(key, newPath);
+            rval = BackupResult.Success;
+        }
+        catch (err) {
+            Logger.error('Failed to backup %s : %s', key, err, err);
+            rval = BackupResult.Error;
+        }
+        return rval;
+    }
+}
+//# sourceMappingURL=s3-remote-file-tracking-provider.js.map

package/lib/s3/impl/s3-remote-file-tracking-provider.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"s3-remote-file-tracking-provider.js","sourceRoot":"","sources":["../../../src/s3/impl/s3-remote-file-tracking-provider.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,8CAA8C,CAAC;AAC9E,OAAO,EAAE,MAAM,EAAE,MAAM,uCAAuC,CAAC;AAC/D,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAOpE,OAAO,EAAE,QAAQ,EAAE,MAAM,OAAO,CAAC;AAIjC,OAAO,EAAE,aAAa,EAAE,MAAM,6CAA6C,CAAC;AAO5E,OAAO,EAAE,YAAY,EAAE,MAAM,mEAAmE,CAAC;AAIjG,OAAO,EAAE,iBAAiB,EAAE,MAAM,yEAAyE,CAAC;AAC5G,OAAO,EAAE,YAAY,EAAE,MAAM,4CAA4C,CAAC;AAC1E,OAAO,EAAE,sBAAsB,EAAE,MAAM,+EAA+E,CAAC;AAGvH,MAAM,OAAO,4BAA4B;IAEnB;IAApB,YAAoB,IAAyC;QAAzC,SAAI,GAAJ,IAAI,CAAqC;QAC3D,cAAc,CAAC,kBAAkB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAChD,cAAc,CAAC,kBAAkB,CAAC,IAAI,CAAC,cAAc,EAAE,qBAAqB,CAAC,CAAC;QAC9E,cAAc,CAAC,kBAAkB,CAAC,IAAI,CAAC,cAAc,CAAC,gBAAgB,EAAE,EAAE,kDAAkD,CAAC,CAAC;IAChI,CAAC;IAEM,KAAK,CAAC,gBAAgB,CAAC,GAAW;QACvC,IAAI,IAAI,GAA6B,IAAI,CAAC;QAC1C,IAAI,aAAa,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,GAA4B,MAAM,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;YAChG,IAAI,IAAI,EAAE,CAAC;gBACT,IAAI,GAAG;oBACL,GAAG,EAAE,GAAG;oBACR,kBAAkB,EAAE,IAAI,CAAC,GAAG,EAAE;oBAC9B,iBAAiB,EAAE,IAAI,CAAC,aAAa;oBACrC,wBAAwB,EAAE,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE;oBACrD,UAAU,EAAE,IAAI,CAAC,IAAI;iBACtB,CAAC;YACJ,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,KAAK,CAAC,cAAc,CAAC,GAAW,EAAE,WAAsC;QAC7E,IAAI,IAAI,GAAuC,IAAI,CAAC;QAClD,MAAM,EAAE,GAAc,IAAI,SAAS,EAAE,CAAC;QAEpC,MAAM,GAAG,GAA0B;YACjC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,gBAAgB,EAAE;YACnD,GAAG,EAAE,GAAG;YACR,eAAe,EAAE,WAAW,EAAE,wBAAwB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,wBAAwB,CAAC,CAAA,CAAC,CAAA,IAAI;SAC7G,CAAC;QAEF,MAAM,MAAM,GAA2B,MAAM,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC;QAClG,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,GAAG;gBACL,MAAM,EAAE;oBACN,GAAG,EAAE,GAAG;oBACR,kBAAkB,EAAE,IAAI,CAAC,GAAG,EAAE;oBAC9B,iBAAiB,EAAE,MAAM,CAAC,aAAa;oBACvC,wBAAwB,EAAE,MAAM,CAAC,YAAY,CAAC,OAAO,EAAE;oBACvD,UAAU,EAAE,MAAM,CAAC,IAAI;iBACxB;gBACD,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,oBAAoB,EAAE;aAC5C,CAAA;YACD,MAAM,CAAC,IAAI,CAAC,8CAA8C,EAAE,MAAM,CAAC,aAAa,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC;QACpG,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC;QACrD,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAEI,KAAK,CAAC,gBAAgB,CAAC,GAAmB,EAAE,GAAW,EAAE,IAAkC,EAAE,WAAqC;QACvI,cAAc,CAAC,kBAAkB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC9C,cAAc,CAAC,kBAAkB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC9C,cAAc,CAAC,kBAAkB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAChD,IAAI,IAAI,GAAuB;YAC7B,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,IAAI;YACX,gBAAgB,EAAE,IAAI;YACtB,YAAY,EAAE,YAAY,CAAC,YAAY;SACxC,CAAC;QAEF,MAAM,EAAE,GAAc,IAAI,SAAS,EAAE,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,6CAA6C,EAAE,GAAG,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;QAInF,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,WAAW,EAAE,wBAAwB,EAAE,CAAC;YACzD,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YACxB,MAAM,OAAO,GAA6B,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;YAC3E,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,WAAW,EAAE,OAAO,CAAC,EAAE,CAAC;gBACzD,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC,KAAK,CAAC;gBACzC,IAAI,CAAC,KAAK,GAAG,iCAAiC,GAAC,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,GAAG,kBAAkB,GAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,0BAA0B,CAAC;gBACrJ,IAAI,CAAC,gBAAgB,GAAE,CAAC,CAAC;gBACzB,OAAO,IAAI,CAAC;YACd,CAAC;YACD,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YACvB,MAAM,CAAC,IAAI,CAAC,8BAA8B,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC;QACtE,CAAC;QAGD,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YACnB,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;YACjD,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClB,MAAM,CAAC,IAAI,CAAC,wBAAwB,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC3D,CAAC;QAGD,IAAI,CAAC;YACH,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACjB,MAAM,GAAG,GAAyC,MAAM,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,sBAAsB,CACrG,GAAG,EACH,GAAG,CAAC,CAAC;YACP,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAChB,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC;QACjE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,yBAAyB,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;YACvD,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC,KAAK,CAAC;YACzC,IAAI,CAAC,KAAK,GAAG,YAAY,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;YAChD,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC;QAC5B,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,qBAAqB,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9C,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,KAAK,CAAC,YAAY,CAAC,GAAW;QACnC,IAAI,IAAI,GAAiB,IAAI,CAAC;QAC9B,IAAI,CAAC;YACH,MAAM,SAAS,GAAW,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YAC/C,MAAM,QAAQ,GAAW,UAAU,GAAG,QAAQ,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,qBAAqB,CAAC,GAAG,GAAG,CAAC;YAC3F,MAAM,OAAO,GACX,SAAS,GAAG,CAAC,CAAC;gBACZ,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC;gBACvE,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC;YAErB,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;YACtD,MAAM,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YACtD,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC;QAC9B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;YACxD,IAAI,GAAG,YAAY,CAAC,KAAK,CAAC;QAC5B,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;CAEF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bitblit/ratchet-aws",
-  "version": "5.0.506-alpha",
+  "version": "5.0.508-alpha",
   "description": "Common tools for use with AWS browser and node",
   "sideEffects": false,
   "type": "module",
@@ -51,7 +51,7 @@
   },
   "license": "Apache-2.0",
   "dependencies": {
-    "@bitblit/ratchet-common": "5.0.506-alpha"
+    "@bitblit/ratchet-common": "5.0.508-alpha"
   },
   "optionalDependencies": {
     "@aws-sdk/client-athena": "3.666.0",
@@ -99,7 +99,7 @@
     "@aws-sdk/lib-storage": "^3.666.0",
     "@aws-sdk/s3-request-presigner": "^3.666.0",
     "@aws-sdk/types": "^3.664.0",
-    "@bitblit/ratchet-common": "5.0.506-alpha",
+    "@bitblit/ratchet-common": "5.0.508-alpha",
     "@smithy/abort-controller": "^3.1.5",
     "@smithy/smithy-client": "^3.4.0",
     "@smithy/util-waiter": "^3.1.6"
@@ -176,7 +176,7 @@
     }
   },
   "devDependencies": {
-    "@bitblit/ratchet-node-only": "5.0.506-alpha",
+    "@bitblit/ratchet-node-only": "5.0.508-alpha",
     "@types/aws-lambda": "8.10.145",
     "aws-sdk-client-mock": "4.0.2"
   }