@bitblit/ratchet-aws 4.0.80-alpha

package/dist/cjs/daemon/daemon-process-state.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/cjs/daemon/daemon-util.js

@@ -0,0 +1,152 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DaemonUtil = void 0;
+const ratchet_common_1 = require("@bitblit/ratchet-common");
+const client_s3_1 = require("@aws-sdk/client-s3");
+const lib_storage_1 = require("@aws-sdk/lib-storage");
+class DaemonUtil {
+    static async start(cache, id, s3Key, options) {
+        try {
+            options.meta = options.meta || {};
+            ratchet_common_1.Logger.info('Starting daemon, key: %s, options: %j', s3Key, options);
+            const now = new Date().getTime();
+            const newState = {
+                id: id,
+                title: options.title,
+                lastUpdatedEpochMS: now,
+                lastUpdatedMessage: 'Created',
+                targetFileName: options.targetFileName,
+                startedEpochMS: now,
+                completedEpochMS: null,
+                meta: options.meta,
+                error: null,
+                link: null,
+                contentType: options.contentType,
+            };
+            const rval = await DaemonUtil.writeState(cache, s3Key, newState, DaemonUtil.DEFAULT_CONTENT);
+            return rval;
+        }
+        catch (err) {
+            ratchet_common_1.Logger.error('Error while trying to start a daemon: %j %s', options, err);
+            throw err;
+        }
+    }
+    static async writeState(cache, s3Key, newState, contents) {
+        try {
+            const s3meta = {};
+            newState.lastUpdatedEpochMS = new Date().getTime();
+            s3meta[DaemonUtil.DAEMON_METADATA_KEY] = JSON.stringify(newState);
+            const params = {
+                Bucket: cache.getDefaultBucket(),
+                Key: s3Key,
+                ContentType: newState.contentType,
+                Metadata: s3meta,
+                Body: new Blob([contents]),
+            };
+            if (newState.targetFileName) {
+                params.ContentDisposition = 'attachment;filename="' + newState.targetFileName + '"';
+            }
+            const written = await cache.getS3Client().send(new client_s3_1.PutObjectCommand(params));
+            ratchet_common_1.Logger.silly('Daemon wrote : %s', written);
+            return DaemonUtil.stat(cache, s3Key);
+        }
+        catch (err) {
+            ratchet_common_1.Logger.error('Error while trying to write a daemon stat: %j %s', newState, err);
+            throw err;
+        }
+    }
+    static async streamDataAndFinish(cache, s3Key, data) {
+        ratchet_common_1.Logger.debug('Streaming data to %s', s3Key);
+        const inStat = await DaemonUtil.updateMessage(cache, s3Key, 'Streaming data');
+        inStat.completedEpochMS = new Date().getTime();
+        inStat.lastUpdatedMessage = 'Complete';
+        const s3meta = {};
+        s3meta[DaemonUtil.DAEMON_METADATA_KEY] = JSON.stringify(inStat);
+        const params = {
+            Bucket: cache.getDefaultBucket(),
+            Key: s3Key,
+            ContentType: inStat.contentType,
+            Metadata: s3meta,
+            Body: data,
+        };
+        const upload = new lib_storage_1.Upload({
+            client: cache.getS3Client(),
+            params: params,
+            tags: [],
+            queueSize: 4,
+            partSize: 1024 * 1024 * 5,
+            leavePartsOnError: false,
+        });
+        upload.on('httpUploadProgress', (progress) => {
+            ratchet_common_1.Logger.info('Uploading : %s', progress);
+        });
+        const written = await upload.done();
+        ratchet_common_1.Logger.silly('Daemon wrote : %s', written);
+        return DaemonUtil.stat(cache, s3Key);
+    }
+    static async updateMessage(cache, s3Key, newMessage) {
+        try {
+            const inStat = await DaemonUtil.stat(cache, s3Key);
+            inStat.lastUpdatedMessage = newMessage;
+            return DaemonUtil.writeState(cache, s3Key, inStat, DaemonUtil.DEFAULT_CONTENT);
+        }
+        catch (err) {
+            ratchet_common_1.Logger.error('Error while trying to update a daemon message: %j %s', s3Key, err);
+            throw err;
+        }
+    }
+    static async stat(s3Cache, path) {
+        try {
+            ratchet_common_1.Logger.debug('Daemon stat for path %s / %s', s3Cache.getDefaultBucket(), path);
+            let stat = null;
+            const meta = await s3Cache.fetchMetaForCacheFile(path);
+            ratchet_common_1.Logger.debug('Daemon: Meta is %j', meta);
+            const metaString = meta && meta.Metadata ? meta.Metadata[DaemonUtil.DAEMON_METADATA_KEY] : null;
+            if (metaString) {
+                stat = JSON.parse(metaString);
+                if (stat.completedEpochMS && !stat.error) {
+                    stat.link = await s3Cache.preSignedDownloadUrlForCacheFile(path);
+                }
+            }
+            else {
+                ratchet_common_1.Logger.warn('No metadata found! (Head was %j)', meta);
+            }
+            return stat;
+        }
+        catch (err) {
+            ratchet_common_1.Logger.error('Error while trying to fetch a daemon state: %j %s', path, err);
+            throw err;
+        }
+    }
+    static async abort(s3Cache, path) {
+        return DaemonUtil.error(s3Cache, path, 'Aborted');
+    }
+    static async error(s3Cache, path, error) {
+        try {
+            const inStat = await DaemonUtil.stat(s3Cache, path);
+            inStat.error = error;
+            inStat.completedEpochMS = new Date().getTime();
+            return DaemonUtil.writeState(s3Cache, path, inStat, DaemonUtil.DEFAULT_CONTENT);
+        }
+        catch (err) {
+            ratchet_common_1.Logger.error('Error while trying to write a daemon error: %j %s', path, err);
+            throw err;
+        }
+    }
+    static async finalize(s3Cache, path, contents) {
+        try {
+            ratchet_common_1.Logger.info('Finalizing daemon %s with %d bytes', path, contents.length);
+            const inStat = await DaemonUtil.stat(s3Cache, path);
+            inStat.completedEpochMS = new Date().getTime();
+            inStat.lastUpdatedMessage = 'Complete';
+            return DaemonUtil.writeState(s3Cache, path, inStat, contents);
+        }
+        catch (err) {
+            ratchet_common_1.Logger.error('Error while trying to finalize a daemon: %j %s', path, err);
+            throw err;
+        }
+    }
+}
+exports.DaemonUtil = DaemonUtil;
+DaemonUtil.DEFAULT_CONTENT = Buffer.from('DAEMON_PLACEHOLDER');
+DaemonUtil.DAEMON_METADATA_KEY = 'daemon_meta';

package/dist/cjs/daemon/daemon.js

@@ -0,0 +1,126 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Daemon = void 0;
+const ratchet_common_1 = require("@bitblit/ratchet-common");
+const s3_cache_ratchet_1 = require("../s3/s3-cache-ratchet");
+const ratchet_common_2 = require("@bitblit/ratchet-common");
+const daemon_util_1 = require("./daemon-util");
+const ratchet_common_3 = require("@bitblit/ratchet-common");
+class Daemon {
+    constructor(s3, bucket, prefix = '', _defaultGroup = Daemon.DEFAULT_DEFAULT_GROUP, jwtRatchet) {
+        this.s3 = s3;
+        this.bucket = bucket;
+        this.prefix = prefix;
+        this._defaultGroup = _defaultGroup;
+        this.jwtRatchet = jwtRatchet;
+        this.cache = new s3_cache_ratchet_1.S3CacheRatchet(this.s3, this.bucket);
+    }
+    get defaultGroup() {
+        return this._defaultGroup;
+    }
+    async keyToPublicToken(key, expirationSeconds) {
+        ratchet_common_3.RequireRatchet.notNullOrUndefined(this.jwtRatchet, 'You must set jwtRatchet if you wish to use public tokens');
+        ratchet_common_3.RequireRatchet.notNullOrUndefined(key, 'key');
+        ratchet_common_3.RequireRatchet.true(expirationSeconds > 0, 'Expiration seconds must be larger than 0');
+        const token = { daemonKey: key };
+        const publicToken = await this.jwtRatchet.createTokenString(token, expirationSeconds);
+        return publicToken;
+    }
+    keyToPath(key) {
+        return Buffer.from(key, 'base64').toString();
+    }
+    pathToKey(path) {
+        return Buffer.from(path).toString('base64');
+    }
+    generatePath(group = this._defaultGroup) {
+        return this.generatePrefix(group) + ratchet_common_2.StringRatchet.createType4Guid();
+    }
+    generatePrefix(group = this._defaultGroup) {
+        return this.prefix + group + '/';
+    }
+    async start(options) {
+        options.group = options.group || this._defaultGroup;
+        const path = this.generatePath(options.group);
+        const key = this.pathToKey(path);
+        return daemon_util_1.DaemonUtil.start(this.cache, key, path, options);
+    }
+    async writeState(newState, contents) {
+        const key = this.keyToPath(newState.id);
+        return daemon_util_1.DaemonUtil.writeState(this.cache, key, newState, contents);
+    }
+    async clean(group = this._defaultGroup, olderThanSeconds = 60 * 60 * 24 * 7) {
+        try {
+            ratchet_common_1.Logger.info('Daemon removing items older than %d seconds from group %s', olderThanSeconds, group);
+            const original = await this.list(group);
+            const now = new Date().getTime();
+            const removed = [];
+            for (let i = 0; i < original.length; i++) {
+                const test = original[i];
+                const ageSeconds = (now - test.startedEpochMS) / 1000;
+                if (ageSeconds > olderThanSeconds) {
+                    const remove = await this.cache.removeCacheFile(this.keyToPath(test.id));
+                    removed.push(test);
+                }
+            }
+            ratchet_common_1.Logger.debug('Removed %d of %d items', removed.length, original.length);
+            return removed;
+        }
+        catch (err) {
+            ratchet_common_1.Logger.error('Error while trying to clean a daemon: %j %s', group, err);
+            throw err;
+        }
+    }
+    async listKeys(group = this._defaultGroup) {
+        try {
+            const prefix = this.generatePrefix(group);
+            ratchet_common_1.Logger.info('Fetching children of %s', prefix);
+            const rval = await this.cache.directChildrenOfPrefix(prefix);
+            ratchet_common_1.Logger.debug('Found : %j', rval);
+            return rval;
+        }
+        catch (err) {
+            ratchet_common_1.Logger.error('Error while trying to list daemon keys: %j %s', group, err);
+            throw err;
+        }
+    }
+    async list(group = this._defaultGroup) {
+        try {
+            const prefix = this.generatePrefix(group);
+            ratchet_common_1.Logger.info('Fetching children of %s', prefix);
+            const keys = await this.listKeys(group);
+            const proms = keys.map((k) => this.stat(this.pathToKey(this.generatePrefix(group) + k)));
+            const rval = await Promise.all(proms);
+            return rval;
+        }
+        catch (err) {
+            ratchet_common_1.Logger.error('Error while trying to list daemon states: %j %s', group, err);
+            throw err;
+        }
+    }
+    async updateMessage(id, newMessage) {
+        const itemPath = this.keyToPath(id);
+        return daemon_util_1.DaemonUtil.updateMessage(this.cache, itemPath, newMessage);
+    }
+    async stat(key) {
+        const itemPath = this.keyToPath(key);
+        return daemon_util_1.DaemonUtil.stat(this.cache, itemPath);
+    }
+    async statFromPublicToken(publicToken) {
+        ratchet_common_3.RequireRatchet.notNullOrUndefined(this.jwtRatchet, 'You must set jwtRatchet if you wish to use public tokens');
+        ratchet_common_3.RequireRatchet.notNullOrUndefined(publicToken, 'publicToken');
+        const parsed = await this.jwtRatchet.decodeToken(publicToken);
+        const key = parsed === null || parsed === void 0 ? void 0 : parsed.daemonKey;
+        return key ? this.stat(key) : null;
+    }
+    async abort(id) {
+        return daemon_util_1.DaemonUtil.abort(this.cache, this.keyToPath(id));
+    }
+    async error(id, error) {
+        return daemon_util_1.DaemonUtil.error(this.cache, this.keyToPath(id), error);
+    }
+    async finalize(id, contents) {
+        return daemon_util_1.DaemonUtil.finalize(this.cache, this.keyToPath(id), contents);
+    }
+}
+exports.Daemon = Daemon;
+Daemon.DEFAULT_DEFAULT_GROUP = 'DEFAULT';

package/dist/cjs/dao/prototype-dao-config.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/cjs/dao/prototype-dao-db.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/cjs/dao/prototype-dao-provider.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/cjs/dao/prototype-dao.js

@@ -0,0 +1,88 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PrototypeDao = void 0;
+const ratchet_common_1 = require("@bitblit/ratchet-common");
+const luxon_1 = require("luxon");
+class PrototypeDao {
+    static defaultDaoConfig() {
+        return {
+            guidCreateFunction: ratchet_common_1.StringRatchet.createType4Guid,
+            guidFieldName: 'guid',
+            createdEpochMSFieldName: 'createdEpochMS',
+            updatedEpochMSFieldName: 'updatedEpochMS',
+            createdUtcTimestampFieldName: null,
+            updatedUtcTimestampFieldName: null,
+        };
+    }
+    constructor(provider, cfg = PrototypeDao.defaultDaoConfig()) {
+        this.provider = provider;
+        this.cfg = cfg;
+        ratchet_common_1.RequireRatchet.notNullOrUndefined(provider, 'provider');
+        ratchet_common_1.RequireRatchet.notNullOrUndefined(cfg, 'cfg');
+        ratchet_common_1.RequireRatchet.notNullOrUndefined(cfg.guidCreateFunction, 'cfg.guidCreateFunction');
+        ratchet_common_1.RequireRatchet.notNullOrUndefined(cfg.guidFieldName, 'cfg.guidFieldName');
+    }
+    async fetchAll() {
+        const db = await this.provider.loadDatabase();
+        return db.items || [];
+    }
+    async resetDatabase() {
+        await this.provider.storeDatabase({ items: [], lastModifiedEpochMS: Date.now() });
+    }
+    async removeItems(guids) {
+        let old = await this.fetchAll();
+        if (guids) {
+            old = old.filter((t) => !guids.includes(t[this.cfg.guidFieldName]));
+            await this.provider.storeDatabase({ items: old, lastModifiedEpochMS: Date.now() });
+        }
+        return old;
+    }
+    async store(value) {
+        let old = await this.fetchAll();
+        if (value) {
+            value[this.cfg.guidFieldName] = value[this.cfg.guidFieldName] || this.cfg.guidCreateFunction();
+            if (this.cfg.createdEpochMSFieldName) {
+                value[this.cfg.createdEpochMSFieldName] = value[this.cfg.createdEpochMSFieldName] || Date.now();
+            }
+            if (this.cfg.createdUtcTimestampFieldName) {
+                value[this.cfg.createdUtcTimestampFieldName] = value[this.cfg.createdUtcTimestampFieldName] || luxon_1.DateTime.utc().toISO();
+            }
+            if (this.cfg.updatedEpochMSFieldName) {
+                value[this.cfg.updatedEpochMSFieldName] = Date.now();
+            }
+            if (this.cfg.updatedUtcTimestampFieldName) {
+                value[this.cfg.updatedUtcTimestampFieldName] = luxon_1.DateTime.utc().toISO();
+            }
+            old = old.filter((t) => t[this.cfg.guidFieldName] !== value[this.cfg.guidFieldName]);
+            old.push(value);
+            await this.provider.storeDatabase({ items: old, lastModifiedEpochMS: Date.now() });
+        }
+        return value;
+    }
+    async fetchById(guid) {
+        const old = await this.fetchAll();
+        return old.find((t) => t[this.cfg.guidFieldName] === guid);
+    }
+    async searchByField(fieldDotPath, fieldValue) {
+        ratchet_common_1.RequireRatchet.notNullOrUndefined(fieldDotPath, 'fieldDotPath');
+        ratchet_common_1.RequireRatchet.notNullOrUndefined(fieldValue, 'fieldValue');
+        const map = {};
+        map[fieldDotPath] = fieldValue;
+        return this.searchByFieldMap(map);
+    }
+    async searchByFieldMap(input) {
+        ratchet_common_1.RequireRatchet.notNullOrUndefined(input, 'input');
+        let old = await this.fetchAll();
+        old = old.filter((t) => {
+            let matchAll = true;
+            Object.keys(input).forEach((k) => {
+                const val = ratchet_common_1.MapRatchet.findValueDotPath(t, k);
+                const fieldValue = input[k];
+                matchAll = matchAll && val === fieldValue;
+            });
+            return matchAll;
+        });
+        return old;
+    }
+}
+exports.PrototypeDao = PrototypeDao;

package/dist/cjs/dao/s3-prototype-dao-provider.js

@@ -0,0 +1,28 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.S3PrototypeDaoProvider = void 0;
+const ratchet_common_1 = require("@bitblit/ratchet-common");
+class S3PrototypeDaoProvider {
+    constructor(s3CacheRatchet, keyName) {
+        this.s3CacheRatchet = s3CacheRatchet;
+        this.keyName = keyName;
+        ratchet_common_1.RequireRatchet.notNullOrUndefined(s3CacheRatchet, 's3CacheRatchet');
+        ratchet_common_1.RequireRatchet.notNullUndefinedOrOnlyWhitespaceString(s3CacheRatchet.getDefaultBucket(), 's3CacheRatchet.defaultBucket');
+        ratchet_common_1.RequireRatchet.notNullUndefinedOrOnlyWhitespaceString(keyName, 'keyName');
+    }
+    async storeDatabase(inDb) {
+        const toSave = inDb || { items: [], lastModifiedEpochMS: null };
+        toSave.lastModifiedEpochMS = Date.now();
+        const put = await this.s3CacheRatchet.writeObjectToCacheFile(this.keyName, toSave);
+        const rval = !!put;
+        return rval;
+    }
+    async loadDatabase() {
+        const rval = (await this.s3CacheRatchet.fetchCacheFileAsObject(this.keyName)) || {
+            items: [],
+            lastModifiedEpochMS: Date.now(),
+        };
+        return rval;
+    }
+}
+exports.S3PrototypeDaoProvider = S3PrototypeDaoProvider;

package/dist/cjs/dao/s3-simple-dao.js

@@ -0,0 +1,78 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.S3SimpleDao = void 0;
+const ratchet_common_1 = require("@bitblit/ratchet-common");
+const ratchet_common_2 = require("@bitblit/ratchet-common");
+class S3SimpleDao {
+    constructor(cache, prefix) {
+        this.cache = cache;
+        this.prefix = prefix;
+        if (!cache) {
+            throw new Error('cache object may not be null');
+        }
+        if (!cache.getDefaultBucket()) {
+            throw new Error('Supplied cache must have default bucket set');
+        }
+    }
+    buildPathPrefix(path) {
+        let rval = '';
+        if (this.prefix) {
+            rval += this.prefix;
+        }
+        if (path) {
+            rval += path;
+        }
+        return rval;
+    }
+    buildFullPath(id, path) {
+        let rval = this.buildPathPrefix(path);
+        if (rval.length > 0) {
+            rval += '/';
+        }
+        rval += id + '.json';
+        return rval;
+    }
+    async exists(id, path) {
+        const fullPath = this.buildFullPath(id, path);
+        ratchet_common_1.Logger.debug('Check file existence : %s', fullPath);
+        return this.cache.fileExists(fullPath);
+    }
+    async fetch(id, path) {
+        const fullPath = this.buildFullPath(id, path);
+        ratchet_common_1.Logger.debug('Fetching : %s', fullPath);
+        const rval = (await this.cache.fetchCacheFileAsObject(fullPath));
+        rval.id = id;
+        rval.path = path;
+        return rval;
+    }
+    async store(item, path) {
+        item.id = item.id || ratchet_common_2.StringRatchet.createType4Guid();
+        item.lastModifiedEpochMS = new Date().getTime();
+        const fullPath = this.buildFullPath(item.id, path);
+        ratchet_common_1.Logger.debug('Storing : %s', fullPath);
+        const stored = await this.cache.writeObjectToCacheFile(fullPath, item);
+        const read = await this.fetch(item.id, path);
+        return read;
+    }
+    async listItems(path) {
+        const fullPath = this.buildPathPrefix(path);
+        ratchet_common_1.Logger.debug('Listing : %s', fullPath);
+        const rval = await this.cache.directChildrenOfPrefix(fullPath);
+        return rval;
+    }
+    async fetchItemsInPath(path) {
+        const fullPath = this.buildPathPrefix(path);
+        ratchet_common_1.Logger.debug('Full fetch of : %s', fullPath);
+        const items = await this.listItems(path);
+        const promises = items.map((s) => this.fetch(s, path));
+        const rval = await Promise.all(promises);
+        return rval;
+    }
+    async delete(id, path) {
+        const fullPath = this.buildFullPath(id, path);
+        ratchet_common_1.Logger.debug('Deleting : %s', fullPath);
+        const del = await this.cache.removeCacheFile(fullPath);
+        return del != null;
+    }
+}
+exports.S3SimpleDao = S3SimpleDao;

package/dist/cjs/dao/simple-dao-item.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/cjs/dynamodb/dynamo-ratchet-like.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });