@bitblit/ratchet-aws 4.0.80-alpha

package/dist/es/s3/s3-cache-ratchet.js

@@ -0,0 +1,328 @@
+import { CopyObjectCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListObjectsCommand, NoSuchKey, } from '@aws-sdk/client-s3';
+import { Logger } from '@bitblit/ratchet-common';
+import { RequireRatchet } from '@bitblit/ratchet-common';
+import { StopWatch } from '@bitblit/ratchet-common';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
+import { Upload } from '@aws-sdk/lib-storage';
+import { StringRatchet } from '@bitblit/ratchet-common';
+import { StreamRatchet } from '@bitblit/ratchet-common';
+export class S3CacheRatchet {
+    constructor(s3, defaultBucket = null) {
+        this.s3 = s3;
+        this.defaultBucket = defaultBucket;
+        RequireRatchet.notNullOrUndefined(this.s3, 's3');
+    }
+    static applyCacheControlMaxAge(input, seconds) {
+        if (input && seconds) {
+            input.CacheControl = 'max-age=' + seconds;
+        }
+        return input;
+    }
+    static applyUserMetaData(input, key, value) {
+        if (input && StringRatchet.trimToNull(key) && StringRatchet.trimToNull(value)) {
+            input.Metadata = input.Metadata || {};
+            input.Metadata[key] = value;
+        }
+        return input;
+    }
+    getDefaultBucket() {
+        return this.defaultBucket;
+    }
+    getS3Client() {
+        return this.s3;
+    }
+    async fileExists(key, bucket = null) {
+        try {
+            const head = await this.fetchMetaForCacheFile(key, this.bucketVal(bucket));
+            return !!head;
+        }
+        catch (err) {
+            Logger.silly('Error calling file exists (as expected) %s', err);
+            return false;
+        }
+    }
+    async fetchCacheFileAsS3GetObjectCommandOutput(key, bucket = null) {
+        let rval = null;
+        try {
+            const params = {
+                Bucket: this.bucketVal(bucket),
+                Key: key,
+            };
+            rval = await this.s3.send(new GetObjectCommand(params));
+        }
+        catch (err) {
+            if (err instanceof NoSuchKey) {
+                Logger.debug('Key %s not found - returning null', key);
+                rval = null;
+            }
+            else {
+                throw err;
+            }
+        }
+        return rval;
+    }
+    async fetchCacheFileAsReadableStream(key, bucket = null) {
+        const out = await this.fetchCacheFileAsS3GetObjectCommandOutput(key, bucket);
+        return out.Body.transformToWebStream();
+    }
+    async fetchCacheFileAsBuffer(key, bucket = null) {
+        let rval = null;
+        const out = await this.fetchCacheFileAsS3GetObjectCommandOutput(key, bucket);
+        if (out?.Body) {
+            const tmp = await out.Body.transformToByteArray();
+            rval = Buffer.from(tmp);
+        }
+        return rval;
+    }
+    async fetchCacheFileAsString(key, bucket = null) {
+        let rval = null;
+        const out = await this.fetchCacheFileAsS3GetObjectCommandOutput(key, bucket);
+        if (out?.Body) {
+            rval = await out.Body.transformToString();
+        }
+        return rval;
+    }
+    async fetchCacheFileAsObject(key, bucket = null) {
+        const value = await this.fetchCacheFileAsString(key, bucket);
+        return value ? JSON.parse(value) : null;
+    }
+    async removeCacheFile(key, bucket = null) {
+        let rval = null;
+        const params = {
+            Bucket: this.bucketVal(bucket),
+            Key: key,
+        };
+        try {
+            rval = await this.s3.send(new DeleteObjectCommand(params));
+        }
+        catch (err) {
+            if (err && err['statusCode'] == 404) {
+                Logger.info('Swallowing 404 deleting missing object %s %s', bucket, key);
+                rval = null;
+            }
+            else {
+                throw err;
+            }
+        }
+        return rval;
+    }
+    async writeObjectToCacheFile(key, dataObject, template, bucket) {
+        const json = JSON.stringify(dataObject);
+        return this.writeStringToCacheFile(key, json, template, bucket);
+    }
+    async writeStringToCacheFile(key, dataString, template, bucket) {
+        const stream = StreamRatchet.stringToWebReadableStream(dataString);
+        return this.writeStreamToCacheFile(key, stream, template, bucket);
+    }
+    async writeStreamToCacheFile(key, data, template, bucket) {
+        const params = Object.assign({}, template || {}, {
+            Bucket: this.bucketVal(bucket),
+            Key: key,
+            Body: data,
+        });
+        const upload = new Upload({
+            client: this.s3,
+            params: params,
+            tags: [],
+            queueSize: 4,
+            partSize: 1024 * 1024 * 5,
+            leavePartsOnError: false,
+        });
+        upload.on('httpUploadProgress', (progress) => {
+            Logger.info('Uploading : %s', progress);
+        });
+        const result = await upload.done();
+        return result;
+    }
+    async synchronize(srcPrefix, targetPrefix, targetRatchet = this, recurseSubFolders = false) {
+        RequireRatchet.notNullOrUndefined(srcPrefix, 'srcPrefix');
+        RequireRatchet.notNullOrUndefined(targetPrefix, 'targetPrefix');
+        RequireRatchet.true(srcPrefix.endsWith('/'), 'srcPrefix must end in /');
+        RequireRatchet.true(targetPrefix.endsWith('/'), 'targetPrefix must end in /');
+        let rval = [];
+        const sourceFiles = await this.directChildrenOfPrefix(srcPrefix);
+        const targetFiles = await targetRatchet.directChildrenOfPrefix(targetPrefix);
+        const sw = new StopWatch();
+        for (let i = 0; i < sourceFiles.length; i++) {
+            const sourceFile = sourceFiles[i];
+            Logger.info('Processing %s : %s', sourceFile, sw.dumpExpected(i / sourceFiles.length));
+            if (sourceFile.endsWith('/')) {
+                if (recurseSubFolders) {
+                    Logger.info('%s is a subfolder - recursing');
+                    const subs = await this.synchronize(srcPrefix + sourceFile, targetPrefix + sourceFile, targetRatchet, recurseSubFolders);
+                    Logger.info('Got %d back from %s', subs.length, sourceFile);
+                    rval = rval.concat(subs);
+                }
+                else {
+                    Logger.info('%s is a subfolder and recurse not specified - skipping', sourceFile);
+                }
+            }
+            else {
+                let shouldCopy = true;
+                const srcMeta = await this.fetchMetaForCacheFile(srcPrefix + sourceFile);
+                if (targetFiles.includes(sourceFile)) {
+                    const targetMeta = await targetRatchet.fetchMetaForCacheFile(targetPrefix + sourceFile);
+                    if (srcMeta.ETag === targetMeta.ETag) {
+                        Logger.debug('Skipping - identical');
+                        shouldCopy = false;
+                    }
+                }
+                if (shouldCopy) {
+                    Logger.debug('Copying...');
+                    const srcStream = await this.fetchCacheFileAsReadableStream(srcPrefix + sourceFile);
+                    try {
+                        const written = await targetRatchet.writeStreamToCacheFile(targetPrefix + sourceFile, srcStream, srcMeta, undefined);
+                        Logger.silly('Write result : %j', written);
+                        rval.push(sourceFile);
+                    }
+                    catch (err) {
+                        Logger.error('Failed to sync : %s : %s', sourceFile, err);
+                    }
+                }
+            }
+        }
+        Logger.info('Found %d files, copied %d', sourceFiles.length, rval.length);
+        sw.log();
+        return rval;
+    }
+    async fetchMetaForCacheFile(key, bucket = null) {
+        let rval = null;
+        try {
+            rval = await this.s3.send(new HeadObjectCommand({
+                Bucket: this.bucketVal(bucket),
+                Key: key,
+            }));
+        }
+        catch (err) {
+            if (err && err['statusCode'] == 404) {
+                Logger.info('Cache file %s %s not found returning null', this.bucketVal(bucket), key);
+                rval = null;
+            }
+            else {
+                Logger.error('Unrecognized error, rethrowing : %s', err, err);
+                throw err;
+            }
+        }
+        return rval;
+    }
+    async cacheFileAgeInSeconds(key, bucket = null) {
+        try {
+            const res = await this.fetchMetaForCacheFile(key, bucket);
+            if (res && res.LastModified) {
+                return Math.floor((new Date().getTime() - res.LastModified.getTime()) / 1000);
+            }
+            else {
+                Logger.warn('Cache file %s %s had no last modified returning null', this.bucketVal(bucket), key);
+                return null;
+            }
+        }
+        catch (err) {
+            if (err && err['statusCode'] == 404) {
+                Logger.warn('Cache file %s %s not found returning null', this.bucketVal(bucket), key);
+                return null;
+            }
+            else {
+                throw err;
+            }
+        }
+    }
+    async copyFile(srcKey, dstKey, srcBucket = null, dstBucket = null) {
+        const params = {
+            CopySource: '/' + this.bucketVal(srcBucket) + '/' + srcKey,
+            Bucket: this.bucketVal(dstBucket),
+            Key: dstKey,
+            MetadataDirective: 'COPY',
+        };
+        const rval = await this.s3.send(new CopyObjectCommand(params));
+        return rval;
+    }
+    async quietCopyFile(srcKey, dstKey, srcBucket = null, dstBucket = null) {
+        let rval = false;
+        try {
+            const tmp = await this.copyFile(srcKey, dstKey, srcBucket, dstBucket);
+            rval = true;
+        }
+        catch (err) {
+            Logger.silly('Failed to copy file in S3 : %s', err);
+        }
+        return rval;
+    }
+    async preSignedDownloadUrlForCacheFile(key, expirationSeconds = 3600, bucket = null) {
+        const getCommand = {
+            Bucket: this.bucketVal(bucket),
+            Key: key,
+        };
+        const link = await getSignedUrl(this.s3, new GetObjectCommand(getCommand), { expiresIn: expirationSeconds });
+        return link;
+    }
+    async directChildrenOfPrefix(prefix, expandFiles = false, bucket = null, maxToReturn = null) {
+        const returnValue = [];
+        const params = {
+            Bucket: this.bucketVal(bucket),
+            Prefix: prefix,
+            Delimiter: '/',
+        };
+        let response = null;
+        do {
+            response = await this.s3.send(new ListObjectsCommand(params));
+            const prefixLength = prefix.length;
+            if (response['CommonPrefixes']) {
+                response['CommonPrefixes'].forEach((cp) => {
+                    if (!maxToReturn || returnValue.length < maxToReturn) {
+                        const value = cp['Prefix'].substring(prefixLength);
+                        returnValue.push(value);
+                    }
+                });
+            }
+            if (response['Contents']) {
+                await Promise.all(response['Contents'].map(async (cp) => {
+                    if (!maxToReturn || returnValue.length < maxToReturn) {
+                        if (expandFiles) {
+                            const expanded = {
+                                link: await this.preSignedDownloadUrlForCacheFile(cp['Key'], 3600, bucket),
+                                name: cp['Key'].substring(prefixLength),
+                                size: cp['Size'],
+                            };
+                            returnValue.push(expanded);
+                        }
+                        else {
+                            returnValue.push(cp['Key'].substring(prefixLength));
+                        }
+                    }
+                }));
+            }
+            params.Marker = response.NextMarker;
+        } while (params.Marker && (!maxToReturn || returnValue.length < maxToReturn));
+        return returnValue;
+    }
+    async allSubFoldersOfPrefix(prefix, bucket = null) {
+        const returnValue = [prefix];
+        let idx = 0;
+        while (idx < returnValue.length) {
+            const next = returnValue[idx++];
+            Logger.debug('Pulling %s (%d remaining)', next, returnValue.length - idx);
+            const req = {
+                Bucket: this.bucketVal(bucket),
+                Prefix: next,
+                Delimiter: '/',
+            };
+            let resp = null;
+            do {
+                req.ContinuationToken = resp ? resp.NextContinuationToken : null;
+                resp = await this.s3.send(new ListObjectsCommand(req));
+                resp.CommonPrefixes.forEach((p) => {
+                    returnValue.push(p.Prefix);
+                });
+                Logger.debug('g:%j', resp);
+            } while (resp.NextContinuationToken);
+        }
+        return returnValue;
+    }
+    bucketVal(explicitBucket) {
+        const rval = explicitBucket ? explicitBucket : this.defaultBucket;
+        if (!rval) {
+            throw 'You must set either the default bucket or pass it explicitly';
+        }
+        return rval;
+    }
+}

package/dist/es/s3/s3-cache-to-local-disk-ratchet.js

@@ -0,0 +1,100 @@
+import crypto from 'crypto';
+import fs from 'fs';
+import path from 'path';
+import { Logger, RequireRatchet, StringRatchet } from '@bitblit/ratchet-common';
+export class S3CacheToLocalDiskRatchet {
+    constructor(s3, tmpFolder, cacheTimeoutSeconds = S3CacheToLocalDiskRatchet.DEFAULT_CACHE_TIMEOUT_SEC) {
+        this.s3 = s3;
+        this.tmpFolder = tmpFolder;
+        this.cacheTimeoutSeconds = cacheTimeoutSeconds;
+        this.currentlyLoading = new Map();
+        RequireRatchet.notNullOrUndefined(s3, 's3');
+        RequireRatchet.notNullOrUndefined(StringRatchet.trimToNull(tmpFolder));
+        RequireRatchet.true(fs.existsSync(tmpFolder), 'folder must exist : ' + tmpFolder);
+    }
+    async getFileString(key) {
+        const buf = await this.getFileBuffer(key);
+        return buf ? buf.toString() : null;
+    }
+    keyToLocalCachePath(key) {
+        const cachedHash = this.generateCacheHash(this.s3.getDefaultBucket() + '/' + key);
+        const rval = path.join(this.tmpFolder, cachedHash);
+        return rval;
+    }
+    removeCacheFileForKey(key) {
+        const localCachePath = this.keyToLocalCachePath(key);
+        Logger.info('Removing cache file for %s : %s', key, localCachePath);
+        if (fs.existsSync(localCachePath)) {
+            fs.unlinkSync(localCachePath);
+        }
+        else {
+            Logger.debug('Skipping delete for %s - does not exist', localCachePath);
+        }
+    }
+    async getFileBuffer(key) {
+        const localCachePath = this.keyToLocalCachePath(key);
+        let rval = null;
+        rval = this.getCacheFileAsBuffer(localCachePath);
+        if (!rval) {
+            Logger.info('No cache. Downloading File s3://%s/%s to %s', this.s3.getDefaultBucket(), key, localCachePath);
+            try {
+                let prom = this.currentlyLoading.get(key);
+                if (prom) {
+                    Logger.info('Already running - wait for that');
+                }
+                else {
+                    Logger.info('Not running - start');
+                    prom = this.updateLocalCacheFile(key, localCachePath);
+                    this.currentlyLoading.set(key, prom);
+                }
+                rval = await prom;
+                this.currentlyLoading.delete(key);
+            }
+            catch (err) {
+                Logger.warn('File %s/%s does not exist. Err code: %s', this.s3.getDefaultBucket(), key, err);
+            }
+        }
+        else {
+            Logger.info('Found cache file for s3://%s/%s. Local path %s', this.s3.getDefaultBucket(), key, localCachePath);
+        }
+        return rval;
+    }
+    async updateLocalCacheFile(key, localCachePath) {
+        const rval = await this.s3.fetchCacheFileAsBuffer(key);
+        if (rval && rval.length > 0) {
+            Logger.info('Saving %d bytes to disk for cache', rval.length);
+            fs.writeFileSync(localCachePath, rval);
+        }
+        return rval;
+    }
+    getCacheFileAsString(filePath) {
+        const buf = this.getCacheFileAsBuffer(filePath);
+        return buf ? buf.toString() : null;
+    }
+    getCacheFileAsBuffer(filePath) {
+        if (!fs.existsSync(filePath)) {
+            return null;
+        }
+        try {
+            const stats = fs.statSync(filePath);
+            const now = new Date().getTime();
+            const duration = (now - stats.ctimeMs) / 1000;
+            if (duration >= this.cacheTimeoutSeconds) {
+                return null;
+            }
+            else {
+                const rval = fs.readFileSync(filePath);
+                return rval;
+            }
+        }
+        catch (err) {
+            Logger.warn('Error getting s3 cache file %s', err);
+        }
+        return null;
+    }
+    generateCacheHash(hashVal) {
+        const rval = crypto.createHash('md5').update(hashVal).digest('hex');
+        return rval;
+    }
+}
+S3CacheToLocalDiskRatchet.DEFAULT_CACHE_TIMEOUT_SEC = 7 * 24 * 3600;

package/dist/es/s3/s3-location-sync-ratchet.js

@@ -0,0 +1,137 @@
+import { CopyObjectCommand, GetObjectCommand, ListObjectsV2Command, } from '@aws-sdk/client-s3';
+import _ from 'lodash';
+import { Logger, PromiseRatchet, RequireRatchet } from '@bitblit/ratchet-common';
+import { Upload } from '@aws-sdk/lib-storage';
+export class S3LocationSyncRatchet {
+    constructor(config) {
+        RequireRatchet.notNullOrUndefined(config, 'config');
+        this.config = config;
+        if (!this.config.maxNumThreads) {
+            this.config.maxNumThreads = 15;
+        }
+        if (!this.config.maxRetries) {
+            this.config.maxRetries = 5;
+        }
+    }
+    updateSrcPrefix(prefix) {
+        this.config.srcPrefix = prefix;
+    }
+    updateDstPrefix(prefix) {
+        this.config.dstPrefix = prefix;
+    }
+    async copyObject(key, size, express = false) {
+        const dstKey = key.replace(this.config.srcPrefix, this.config.dstPrefix);
+        let completedCopying = false;
+        let retries = 0;
+        while (!completedCopying && retries < this.config.maxRetries) {
+            Logger.debug(`${retries > 0 ? `Retry ${retries} ` : ''}${express ? 'Express' : 'Slow'} copying
+                [${[this.config.srcBucket, key].join('/')} ---> ${[this.config.dstBucket, dstKey].join('/')}]`);
+            try {
+                if (express) {
+                    const params = {
+                        CopySource: encodeURIComponent([this.config.srcBucket, key].join('/')),
+                        Bucket: this.config.dstBucket,
+                        Key: dstKey,
+                        MetadataDirective: 'COPY',
+                    };
+                    await this.config.dstS3.send(new CopyObjectCommand(params));
+                }
+                else {
+                    const fetched = await this.config.srcS3.send(new GetObjectCommand({ Bucket: this.config.srcBucket, Key: key }));
+                    const params = {
+                        Bucket: this.config.dstBucket,
+                        Key: dstKey,
+                        Body: fetched.Body,
+                        ContentLength: size,
+                    };
+                    const upload = new Upload({
+                        client: this.config.dstS3,
+                        params: params,
+                        tags: [],
+                        queueSize: 4,
+                        partSize: 1024 * 1024 * 5,
+                        leavePartsOnError: false,
+                    });
+                    upload.on('httpUploadProgress', (progress) => {
+                        Logger.info('Uploading : %s', progress);
+                    });
+                    await upload.done();
+                }
+                completedCopying = true;
+            }
+            catch (err) {
+                Logger.warn(`Can't ${express ? 'express' : 'slow'} copy
+                  [${[this.config.srcBucket, key].join('/')} ---> ${[this.config.dstBucket, dstKey].join('/')}]: %j`, err);
+                retries++;
+            }
+        }
+        Logger.debug(`Finished ${express ? 'express' : 'slow'} copying
+                  [${[this.config.srcBucket, key].join('/')} ---> ${[this.config.dstBucket, dstKey].join('/')}]`);
+    }
+    async listObjects(bucket, prefix, s3) {
+        Logger.info(`Scanning bucket [${[bucket, prefix].join('/')}]`);
+        const params = {
+            Bucket: bucket,
+            Prefix: prefix,
+        };
+        let more = true;
+        const rval = {};
+        while (more) {
+            const response = await s3.send(new ListObjectsV2Command(params));
+            more = response.IsTruncated;
+            _.each(response.Contents, (obj) => {
+                rval[obj.Key] = { Key: obj.Key, LastModified: obj.LastModified, ETag: obj.ETag, Size: obj.Size };
+            });
+            if (more) {
+                params.ContinuationToken = response.NextContinuationToken;
+            }
+        }
+        return rval;
+    }
+    async startSyncing() {
+        Logger.info(`Syncing [${this.config.srcBucket}/${this.config.srcPrefix}
+                  ---> ${this.config.dstBucket}/${this.config.dstPrefix}]`);
+        const cp = async (obj) => {
+            await this.copyObject(obj.Key, obj.Size);
+        };
+        let cmpResult = await this.compareSrcAndDst();
+        if (cmpResult.needCopy.length > 0 || cmpResult.diff.length > 0) {
+            await PromiseRatchet.runBoundedParallelSingleParam(cp, cmpResult.needCopy, this, this.config.maxNumThreads);
+            await PromiseRatchet.runBoundedParallelSingleParam(cp, cmpResult.diff, this, this.config.maxNumThreads);
+            Logger.info('Verifying...');
+            cmpResult = await this.compareSrcAndDst();
+            Logger.debug('Compare result %j', cmpResult);
+        }
+        return cmpResult.needCopy.length === 0 && cmpResult.diff.length === 0;
+    }
+    async compareSrcAndDst() {
+        const getSrc = this.listObjects(this.config.srcBucket, this.config.srcPrefix, this.config.srcS3);
+        const getDst = this.listObjects(this.config.dstBucket, this.config.dstPrefix, this.config.dstS3);
+        const srcObjs = await getSrc;
+        const dstObjs = await getDst;
+        const rval = {
+            needCopy: [],
+            existed: [],
+            diff: [],
+        };
+        await PromiseRatchet.runBoundedParallelSingleParam((key) => {
+            const sObj = srcObjs[key];
+            const dstKey = key.replace(this.config.srcPrefix, this.config.dstPrefix);
+            const dObj = dstObjs.hasOwnProperty(dstKey) ? dstObjs[dstKey] : undefined;
+            if (!dObj) {
+                rval.needCopy.push(sObj);
+                return;
+            }
+            if (sObj.Size !== dObj.Size) {
+                rval.diff.push(sObj);
+                return;
+            }
+            if (sObj.LastModified.getTime() <= dObj.LastModified.getTime()) {
+                rval.existed.push(sObj);
+                return;
+            }
+            rval.diff.push(sObj);
+        }, Object.keys(srcObjs), this, this.config.maxNumThreads);
+        return rval;
+    }
+}

package/dist/es/s3/s3-ratchet.js

@@ -0,0 +1,22 @@
+import { RequireRatchet } from '@bitblit/ratchet-common';
+export class S3Ratchet {
+    static checkS3UrlForValidity(value) {
+        let rval = false;
+        if (value) {
+            rval = value.startsWith('s3://') && value.trim().length > 5;
+        }
+        return rval;
+    }
+    static extractBucketFromURL(value) {
+        RequireRatchet.true(S3Ratchet.checkS3UrlForValidity(value), 'invalid s3 url');
+        const idx1 = value.indexOf('/', 5);
+        const rval = idx1 > 0 ? value.substring(5, idx1) : value.substring(5);
+        return rval;
+    }
+    static extractKeyFromURL(value) {
+        RequireRatchet.true(S3Ratchet.checkS3UrlForValidity(value), 'invalid s3 url');
+        const idx1 = value.indexOf('/', 5);
+        const rval = idx1 > 0 ? value.substring(idx1 + 1) : null;
+        return rval;
+    }
+}

package/dist/es/ses/email-attachment.js

@@ -0,0 +1 @@
+export {};

package/dist/es/ses/mailer-config.js

@@ -0,0 +1 @@
+export {};

package/dist/es/ses/mailer-like.js

@@ -0,0 +1 @@
+export {};