@bitblit/ratchet-aws 4.0.80-alpha

package/dist/es/cache/dynamo-db-storage-provider.js

@@ -0,0 +1,107 @@
+import { RequireRatchet } from '@bitblit/ratchet-common';
+export class DynamoDbStorageProvider {
+    constructor(dynamo, opts) {
+        this.dynamo = dynamo;
+        this.opts = opts;
+        RequireRatchet.notNullOrUndefined(this.dynamo, 'dynamo');
+        RequireRatchet.notNullOrUndefined(this.opts, 'opts');
+        RequireRatchet.notNullOrUndefined(this.opts.tableName, 'opts.tableName');
+        RequireRatchet.notNullOrUndefined(this.opts.hashKeyName, 'opts.hashKeyName');
+        RequireRatchet.true(!this.opts.useRangeKeys || (!!this.opts.rangeKeyName && !!this.opts.hashKeyValue), 'invalid range configuration');
+    }
+    static createDefaultOptions() {
+        const rval = {
+            tableName: 'simple-cache',
+            useRangeKeys: false,
+            hashKeyName: 'cache-key',
+            rangeKeyName: null,
+            hashKeyValue: null,
+        };
+        return rval;
+    }
+    createKeyObject(cacheKey) {
+        const keys = {};
+        if (this.opts.useRangeKeys) {
+            keys[this.opts.hashKeyName] = this.opts.hashKeyValue;
+            keys[this.opts.rangeKeyName] = cacheKey;
+        }
+        else {
+            keys[this.opts.hashKeyName] = cacheKey;
+        }
+        return keys;
+    }
+    cleanDynamoFieldsFromObjectInPlace(rval) {
+        if (rval) {
+            delete rval[this.opts.hashKeyName];
+            if (this.opts.rangeKeyName) {
+                delete rval[this.opts.rangeKeyName];
+            }
+            if (this.opts.dynamoExpiresColumnName) {
+                delete rval[this.opts.dynamoExpiresColumnName];
+            }
+        }
+    }
+    extractKeysFromObject(rval) {
+        let keys = null;
+        if (rval) {
+            keys = {};
+            if (this.opts.useRangeKeys) {
+                keys[this.opts.hashKeyName] = this.opts.hashKeyValue;
+                keys[this.opts.rangeKeyName] = rval.cacheKey;
+            }
+            else {
+                keys[this.opts.hashKeyName] = rval.cacheKey;
+            }
+        }
+        return keys;
+    }
+    async readFromCache(cacheKey) {
+        const dKey = this.createKeyObject(cacheKey);
+        const rval = await this.dynamo.simpleGet(this.opts.tableName, dKey);
+        this.cleanDynamoFieldsFromObjectInPlace(rval);
+        return rval;
+    }
+    async storeInCache(value) {
+        RequireRatchet.notNullOrUndefined(value, 'value');
+        RequireRatchet.notNullOrUndefined(value.cacheKey, 'value.cacheKey');
+        const toSave = Object.assign({}, value, this.createKeyObject(value.cacheKey));
+        if (this.opts.dynamoExpiresColumnName && value.expiresEpochMS) {
+            toSave[this.opts.dynamoExpiresColumnName] = Math.floor(value.expiresEpochMS / 1000);
+        }
+        const wrote = await this.dynamo.simplePut(this.opts.tableName, toSave);
+        return !!wrote;
+    }
+    async removeFromCache(cacheKey) {
+        await this.dynamo.simpleDelete(this.opts.tableName, this.createKeyObject(cacheKey));
+    }
+    async clearCache() {
+        const allValues = await this.readAll();
+        const allKeys = allValues.map((a) => this.extractKeysFromObject(a));
+        const rval = await this.dynamo.deleteAllInBatches(this.opts.tableName, allKeys, 25);
+        return rval;
+    }
+    async readAll() {
+        let rval = null;
+        if (this.opts.useRangeKeys) {
+            const qry = {
+                TableName: this.opts.tableName,
+                KeyConditionExpression: '#cacheKey = :cacheKey',
+                ExpressionAttributeNames: {
+                    '#cacheKey': this.opts.hashKeyName,
+                },
+                ExpressionAttributeValues: {
+                    ':cacheKey': this.opts.hashKeyValue,
+                },
+            };
+            rval = await this.dynamo.fullyExecuteQuery(qry);
+        }
+        else {
+            const scan = {
+                TableName: this.opts.tableName,
+            };
+            rval = await this.dynamo.fullyExecuteScan(scan);
+        }
+        rval.forEach((r) => this.cleanDynamoFieldsFromObjectInPlace(r));
+        return rval;
+    }
+}

package/dist/es/cache/s3-storage-provider.js

@@ -0,0 +1,40 @@
+import { RequireRatchet, StringRatchet } from '@bitblit/ratchet-common';
+export class S3StorageProvider {
+    constructor(s3CacheRatchet, prefix) {
+        this.s3CacheRatchet = s3CacheRatchet;
+        this.prefix = prefix;
+        RequireRatchet.notNullOrUndefined(this.s3CacheRatchet, 's3CacheRatchet');
+        RequireRatchet.notNullOrUndefined(this.s3CacheRatchet.getDefaultBucket(), 's3CacheRatchet.defaultBucket');
+    }
+    keyToPath(cacheKey) {
+        let rval = StringRatchet.trimToEmpty(this.prefix);
+        if (rval.length > 0 && !rval.endsWith('/')) {
+            rval += '/';
+        }
+        rval += cacheKey;
+        return rval;
+    }
+    async readFromCache(cacheKey) {
+        const rval = await this.s3CacheRatchet.fetchCacheFileAsObject(this.keyToPath(cacheKey));
+        return rval;
+    }
+    async storeInCache(value) {
+        RequireRatchet.notNullOrUndefined(value, 'value');
+        RequireRatchet.notNullOrUndefined(value.cacheKey, 'value.cacheKey');
+        const tmp = await this.s3CacheRatchet.writeObjectToCacheFile(this.keyToPath(value.cacheKey), value);
+        return !!tmp;
+    }
+    async removeFromCache(cacheKey) {
+        await this.s3CacheRatchet.removeCacheFile(this.keyToPath(cacheKey));
+    }
+    async clearCache() {
+        const keys = await this.s3CacheRatchet.directChildrenOfPrefix(this.keyToPath(''));
+        const removed = await Promise.all(keys.map((k) => this.removeFromCache(k)));
+        return keys.length;
+    }
+    async readAll() {
+        const keys = await this.s3CacheRatchet.directChildrenOfPrefix(this.keyToPath(''));
+        const rval = await Promise.all(keys.map((k) => this.readFromCache(k)));
+        return rval;
+    }
+}

package/dist/es/cache/simple-cache-object-wrapper.js

@@ -0,0 +1 @@
+export {};

package/dist/es/cache/simple-cache-read-options.js

@@ -0,0 +1 @@
+export {};

package/dist/es/cache/simple-cache-storage-provider.js

@@ -0,0 +1 @@
+export {};

package/dist/es/cache/simple-cache.js

@@ -0,0 +1,62 @@
+import { Logger } from '@bitblit/ratchet-common';
+export class SimpleCache {
+    constructor(provider, defaultTimeToLiveMS = 1000 * 60) {
+        this.provider = provider;
+        this.defaultTimeToLiveMS = defaultTimeToLiveMS;
+    }
+    createDefaultReadOptions() {
+        return {
+            maxStalenessMS: null,
+            timeToLiveMS: this.defaultTimeToLiveMS,
+            cacheNullValues: false,
+        };
+    }
+    async fetchWrapper(cacheKey, producer, opts = this.createDefaultReadOptions()) {
+        Logger.silly('Fetching %s', cacheKey);
+        const now = new Date().getTime();
+        let rval = await this.provider.readFromCache(cacheKey);
+        if (rval && rval.expiresEpochMS < now) {
+            Logger.debug('Object found, but expired - removing');
+            rval = null;
+        }
+        if (rval && opts && opts.maxStalenessMS && now - rval.createdEpochMS > opts.maxStalenessMS) {
+            Logger.debug('Object found by too stale - removing');
+            rval = null;
+        }
+        if (!rval) {
+            Logger.debug('%s not found in cache, generating', cacheKey);
+            const tmp = await producer();
+            if (tmp || opts?.cacheNullValues) {
+                Logger.debug('Writing %j to cache');
+                rval = {
+                    cacheKey: cacheKey,
+                    createdEpochMS: now,
+                    expiresEpochMS: opts && opts.timeToLiveMS ? now + opts.timeToLiveMS : null,
+                    value: tmp,
+                    generated: false,
+                };
+                await this.provider.storeInCache(rval);
+                rval.generated = true;
+            }
+        }
+        return rval;
+    }
+    async fetch(cacheKey, producer, opts = null) {
+        const wrapper = await this.fetchWrapper(cacheKey, producer, opts);
+        return wrapper ? wrapper.value : null;
+    }
+    async removeFromCache(cacheKey, returnOldValue) {
+        let rval = null;
+        if (returnOldValue) {
+            rval = await this.fetchWrapper(cacheKey, () => null);
+        }
+        await this.provider.removeFromCache(cacheKey);
+        return rval;
+    }
+    async clearCache() {
+        return this.provider.clearCache();
+    }
+    async readAll() {
+        return this.provider.readAll();
+    }
+}

package/dist/es/cloudwatch/cloud-watch-log-group-ratchet.js

@@ -0,0 +1,69 @@
+import { CloudWatchLogsClient, DescribeLogStreamsCommand, FilterLogEventsCommand, } from '@aws-sdk/client-cloudwatch-logs';
+import { Logger } from '@bitblit/ratchet-common';
+import { StopWatch } from '@bitblit/ratchet-common';
+export class CloudWatchLogGroupRatchet {
+    constructor(logGroup, awsCWLogs = new CloudWatchLogsClient({ region: 'us-east-1' })) {
+        this.logGroup = logGroup;
+        this.awsCWLogs = awsCWLogs;
+    }
+    async readLogStreams(startTimestamp = null, endTimestamp = null) {
+        const params = {
+            logGroupName: this.logGroup,
+            orderBy: 'LastEventTime',
+        };
+        const rval = [];
+        do {
+            Logger.debug('Pulling more log streams (%d found so far)', rval.length);
+            const temp = await this.awsCWLogs.send(new DescribeLogStreamsCommand(params));
+            temp.logStreams.forEach((s) => {
+                if (s.lastEventTimestamp !== null) {
+                    if (!startTimestamp || s.lastEventTimestamp >= startTimestamp) {
+                        if (!endTimestamp || s.firstEventTimestamp <= endTimestamp) {
+                            rval.push(s);
+                        }
+                    }
+                }
+            });
+            params.nextToken = temp.nextToken;
+        } while (!!params.nextToken);
+        Logger.debug('Found %d total, returning', rval.length);
+        return rval;
+    }
+    async readLogStreamNames(startTimestamp = null, endTimestamp = null) {
+        const streams = await this.readLogStreams(startTimestamp, endTimestamp);
+        const rval = streams.map((s) => s.logStreamName);
+        return rval;
+    }
+    async readEvents(filter, startTimestamp = null, endTimestamp = null, sortEvents = true, maxEvents = null) {
+        const sw = new StopWatch();
+        const params = {
+            logGroupName: this.logGroup,
+            endTime: endTimestamp,
+            startTime: startTimestamp,
+        };
+        if (filter) {
+            params.filterPattern = filter;
+        }
+        Logger.debug('Reading log events matching : %j', params);
+        let rval = [];
+        do {
+            Logger.debug('Pulling more log events (%d found so far) : %s', rval.length, sw.dump());
+            const temp = await this.awsCWLogs.send(new FilterLogEventsCommand(params));
+            rval = rval.concat(temp.events);
+            params.nextToken = temp.nextToken;
+        } while (!!params.nextToken && (!maxEvents || rval.length < maxEvents));
+        Logger.debug('Found %d total in %s', rval.length, sw.dump());
+        if (sortEvents) {
+            Logger.debug('Sorting events by timestamp');
+            rval = rval.sort((a, b) => {
+                let rval = a.timestamp - b.timestamp;
+                if (rval === 0) {
+                    rval = a.message.localeCompare(b.message);
+                }
+                return rval;
+            });
+        }
+        sw.log();
+        return rval;
+    }
+}

package/dist/es/cloudwatch/cloud-watch-logs-ratchet.js

@@ -0,0 +1,169 @@
+import { CloudWatchLogsClient, DeleteLogGroupCommand, DeleteLogStreamCommand, DescribeLogGroupsCommand, DescribeLogStreamsCommand, GetQueryResultsCommand, StartQueryCommand, StopQueryCommand, } from '@aws-sdk/client-cloudwatch-logs';
+import { Logger } from '@bitblit/ratchet-common';
+import { PromiseRatchet } from '@bitblit/ratchet-common';
+import { RequireRatchet } from '@bitblit/ratchet-common';
+import { StringRatchet } from '@bitblit/ratchet-common';
+export class CloudWatchLogsRatchet {
+    constructor(cloudwatchLogs = null) {
+        this.cwLogs = cloudwatchLogs ? cloudwatchLogs : new CloudWatchLogsClient({ region: 'us-east-1' });
+    }
+    async removeEmptyOrOldLogStreams(logGroupName, maxToRemove = 1000, oldestEventEpochMS = null) {
+        Logger.info('Removing empty streams from %s, oldest event epoch MS : %d', logGroupName, oldestEventEpochMS);
+        const streamSearchParams = {
+            logGroupName: logGroupName,
+            orderBy: 'LastEventTime',
+        };
+        const oldestEventTester = oldestEventEpochMS || 1;
+        let totalStreams = 0;
+        const removedStreams = [];
+        const failedRemovedStreams = [];
+        let waitPerDescribe = 10;
+        do {
+            Logger.debug('Executing search for streams');
+            try {
+                const streams = await this.cwLogs.send(new DescribeLogStreamsCommand(streamSearchParams));
+                totalStreams += streams.logStreams.length;
+                Logger.debug('Found %d streams (%d so far, %d to delete)', streams.logStreams.length, totalStreams, removedStreams.length);
+                for (let i = 0; i < streams.logStreams.length && removedStreams.length < maxToRemove; i++) {
+                    const st = streams.logStreams[i];
+                    if (!st.firstEventTimestamp) {
+                        removedStreams.push(st);
+                    }
+                    else if (st.lastEventTimestamp < oldestEventTester) {
+                        removedStreams.push(st);
+                    }
+                }
+                streamSearchParams['nextToken'] = streams.nextToken;
+            }
+            catch (err) {
+                const oldWait = waitPerDescribe;
+                waitPerDescribe = Math.min(1000, waitPerDescribe * 1.5);
+                Logger.info('Caught while describing %s, increasing wait between deletes (was %d, now %d)', err, oldWait, waitPerDescribe);
+            }
+        } while (!!streamSearchParams['nextToken'] && removedStreams.length < maxToRemove);
+        Logger.info('Found %d streams to delete', removedStreams.length);
+        let waitPer = 10;
+        for (let i = 0; i < removedStreams.length; i++) {
+            const delParams = {
+                logGroupName: logGroupName,
+                logStreamName: removedStreams[i].logStreamName,
+            };
+            const type = removedStreams[i].storedBytes === 0 ? 'empty' : 'old';
+            Logger.info('Removing %s stream %s', type, removedStreams[i].logStreamName);
+            let removed = false;
+            let retry = 0;
+            while (!removed && retry < CloudWatchLogsRatchet.MAX_DELETE_RETRIES) {
+                try {
+                    await this.cwLogs.send(new DeleteLogStreamCommand(delParams));
+                    removed = true;
+                    await PromiseRatchet.wait(waitPer);
+                }
+                catch (err) {
+                    retry++;
+                    const oldWait = waitPer;
+                    waitPer = Math.min(1000, waitPer * 1.5);
+                    Logger.info('Caught %s, increasing wait between deletes and retrying (wait was %d, now %d) (Retry %d of %d)', err, oldWait, waitPer, retry, CloudWatchLogsRatchet.MAX_DELETE_RETRIES);
+                }
+            }
+            if (!removed) {
+                failedRemovedStreams.push(removedStreams[i]);
+            }
+        }
+        Logger.warn('Failed to remove streams : %j', failedRemovedStreams);
+        return removedStreams;
+    }
+    async findOldestEventTimestampInGroup(logGroupName) {
+        const stream = await this.findStreamWithOldestEventInGroup(logGroupName);
+        return stream ? stream.firstEventTimestamp : null;
+    }
+    async findStreamWithOldestEventInGroup(logGroupName) {
+        Logger.info('Finding oldest event in : %s', logGroupName);
+        let rval = null;
+        try {
+            const streamSearchParams = {
+                logGroupName: logGroupName,
+                orderBy: 'LastEventTime',
+            };
+            let totalStreams = 0;
+            do {
+                Logger.debug('Executing search for streams');
+                const streams = await this.cwLogs.send(new DescribeLogStreamsCommand(streamSearchParams));
+                totalStreams += streams.logStreams.length;
+                Logger.debug('Found %d streams (%d so far)', streams.logStreams.length, totalStreams);
+                streams.logStreams.forEach((s) => {
+                    if (s.firstEventTimestamp && (rval === null || s.firstEventTimestamp < rval.firstEventTimestamp)) {
+                        rval = s;
+                    }
+                });
+                streamSearchParams['nextToken'] = streams.nextToken;
+            } while (!!streamSearchParams['nextToken']);
+        }
+        catch (err) {
+            Logger.error('Error attempting to find oldest event in group : %s : %s', logGroupName, err, err);
+        }
+        return rval;
+    }
+    async findLogGroups(prefix) {
+        RequireRatchet.notNullOrUndefined(prefix);
+        const params = {
+            logGroupNamePrefix: prefix,
+        };
+        let rval = [];
+        do {
+            Logger.info('%d found, pulling log groups : %j', rval.length, params);
+            const res = await this.cwLogs.send(new DescribeLogGroupsCommand(params));
+            rval = rval.concat(res.logGroups);
+            params.nextToken = res.nextToken;
+        } while (!!params.nextToken);
+        return rval;
+    }
+    async removeLogGroups(groups) {
+        RequireRatchet.notNullOrUndefined(groups);
+        const rval = [];
+        for (let i = 0; i < groups.length; i++) {
+            try {
+                Logger.info('Deleting %j', groups[i]);
+                const req = {
+                    logGroupName: groups[i].logGroupName,
+                };
+                await this.cwLogs.send(new DeleteLogGroupCommand(req));
+                rval.push(true);
+            }
+            catch (err) {
+                Logger.error('Failure to delete %j : %s', groups[i], err);
+                rval.push(false);
+            }
+        }
+        return rval;
+    }
+    async removeLogGroupsWithPrefix(prefix) {
+        RequireRatchet.notNullOrUndefined(prefix);
+        RequireRatchet.true(StringRatchet.trimToEmpty(prefix).length > 0);
+        Logger.info('Removing log groups with prefix %s', prefix);
+        const groups = await this.findLogGroups(prefix);
+        return await this.removeLogGroups(groups);
+    }
+    async fullyExecuteInsightsQuery(sqr) {
+        RequireRatchet.notNullOrUndefined(sqr);
+        Logger.debug('Starting insights query : %j', sqr);
+        const resp = await this.cwLogs.send(new StartQueryCommand(sqr));
+        Logger.debug('Got query id %j', resp);
+        let rval = null;
+        let delayMS = 100;
+        while (!rval || ['Running', 'Scheduled'].includes(rval.status)) {
+            rval = await this.cwLogs.send(new GetQueryResultsCommand({ queryId: resp.queryId }));
+            await PromiseRatchet.wait(delayMS);
+            delayMS *= 2;
+            Logger.info('Got : %j', rval);
+        }
+        return rval;
+    }
+    async abortInsightsQuery(queryId) {
+        let rval = null;
+        if (!!queryId) {
+            rval = await this.cwLogs.send(new StopQueryCommand({ queryId: queryId }));
+        }
+        return rval;
+    }
+}
+CloudWatchLogsRatchet.MAX_DELETE_RETRIES = 5;

package/dist/es/cloudwatch/cloud-watch-metrics-ratchet.js

@@ -0,0 +1,53 @@
+import { CloudWatchClient, PutMetricDataCommand } from '@aws-sdk/client-cloudwatch';
+import { Logger } from '@bitblit/ratchet-common';
+import { CloudWatchMetricsUnit } from '../model/cloud-watch-metrics-unit';
+import { DateTime } from 'luxon';
+export class CloudWatchMetricsRatchet {
+    constructor(cloudWatch = null) {
+        this.cw = cloudWatch ? cloudWatch : new CloudWatchClient({ region: 'us-east-1', apiVersion: '2010-08-01' });
+    }
+    async writeSingleMetric(namespace, metric, dims, unit = CloudWatchMetricsUnit.None, value, timestampDate = new Date(), highResolution = false) {
+        const cwDims = [];
+        if (!!dims && dims.length > 0) {
+            dims.forEach((d) => {
+                cwDims.push({ Name: d.key, Value: d.value });
+            });
+        }
+        const storageResolution = highResolution ? 1 : 60;
+        const metricData = {
+            Namespace: namespace,
+            MetricData: [
+                {
+                    MetricName: metric,
+                    Dimensions: cwDims,
+                    Unit: String(unit),
+                    Value: value,
+                    Timestamp: timestampDate,
+                    StorageResolution: storageResolution,
+                },
+            ],
+        };
+        Logger.silly('Writing metric to cw : %j', metricData);
+        const result = await this.cw.send(new PutMetricDataCommand(metricData));
+        Logger.silly('Result: %j', result);
+        return result;
+    }
+    async writeDynamoCountAsMinuteLevelMetric(req) {
+        Logger.info('Publishing %s / %s metric for %s UTC', req.namespace, req.metric, req.minuteUTC);
+        if (!!req.scan && !!req.query) {
+            throw new Error('Must send query or scan, but not both');
+        }
+        if (!req.scan && !req.query) {
+            throw new Error('You must specify either a scan or a query');
+        }
+        const cnt = !!req.query
+            ? await req.dynamoRatchet.fullyExecuteQueryCount(req.query)
+            : await req.dynamoRatchet.fullyExecuteScanCount(req.scan);
+        Logger.debug('%s / %s for %s are %j', req.namespace, req.metric, req.minuteUTC, cnt);
+        const parseDateString = req.minuteUTC.split(' ').join('T') + ':00Z';
+        const parseDate = DateTime.fromISO(parseDateString).toJSDate();
+        const metricRes = await this.writeSingleMetric(req.namespace, req.metric, req.dims, CloudWatchMetricsUnit.Count, cnt.count, parseDate, false);
+        Logger.debug('Metrics response: %j', metricRes);
+        return cnt.count;
+    }
+}

package/dist/es/daemon/daemon-like.js

@@ -0,0 +1 @@
+export {};

package/dist/es/daemon/daemon-process-create-options.js

@@ -0,0 +1 @@
+export {};

package/dist/es/daemon/daemon-process-state-public-token.js

@@ -0,0 +1 @@
+export {};

package/dist/es/daemon/daemon-process-state.js

@@ -0,0 +1 @@
+export {};

package/dist/es/daemon/daemon-util.js

@@ -0,0 +1,148 @@
+import { Logger } from '@bitblit/ratchet-common';
+import { PutObjectCommand, } from '@aws-sdk/client-s3';
+import { Upload } from '@aws-sdk/lib-storage';
+export class DaemonUtil {
+    static async start(cache, id, s3Key, options) {
+        try {
+            options.meta = options.meta || {};
+            Logger.info('Starting daemon, key: %s, options: %j', s3Key, options);
+            const now = new Date().getTime();
+            const newState = {
+                id: id,
+                title: options.title,
+                lastUpdatedEpochMS: now,
+                lastUpdatedMessage: 'Created',
+                targetFileName: options.targetFileName,
+                startedEpochMS: now,
+                completedEpochMS: null,
+                meta: options.meta,
+                error: null,
+                link: null,
+                contentType: options.contentType,
+            };
+            const rval = await DaemonUtil.writeState(cache, s3Key, newState, DaemonUtil.DEFAULT_CONTENT);
+            return rval;
+        }
+        catch (err) {
+            Logger.error('Error while trying to start a daemon: %j %s', options, err);
+            throw err;
+        }
+    }
+    static async writeState(cache, s3Key, newState, contents) {
+        try {
+            const s3meta = {};
+            newState.lastUpdatedEpochMS = new Date().getTime();
+            s3meta[DaemonUtil.DAEMON_METADATA_KEY] = JSON.stringify(newState);
+            const params = {
+                Bucket: cache.getDefaultBucket(),
+                Key: s3Key,
+                ContentType: newState.contentType,
+                Metadata: s3meta,
+                Body: new Blob([contents]),
+            };
+            if (newState.targetFileName) {
+                params.ContentDisposition = 'attachment;filename="' + newState.targetFileName + '"';
+            }
+            const written = await cache.getS3Client().send(new PutObjectCommand(params));
+            Logger.silly('Daemon wrote : %s', written);
+            return DaemonUtil.stat(cache, s3Key);
+        }
+        catch (err) {
+            Logger.error('Error while trying to write a daemon stat: %j %s', newState, err);
+            throw err;
+        }
+    }
+    static async streamDataAndFinish(cache, s3Key, data) {
+        Logger.debug('Streaming data to %s', s3Key);
+        const inStat = await DaemonUtil.updateMessage(cache, s3Key, 'Streaming data');
+        inStat.completedEpochMS = new Date().getTime();
+        inStat.lastUpdatedMessage = 'Complete';
+        const s3meta = {};
+        s3meta[DaemonUtil.DAEMON_METADATA_KEY] = JSON.stringify(inStat);
+        const params = {
+            Bucket: cache.getDefaultBucket(),
+            Key: s3Key,
+            ContentType: inStat.contentType,
+            Metadata: s3meta,
+            Body: data,
+        };
+        const upload = new Upload({
+            client: cache.getS3Client(),
+            params: params,
+            tags: [],
+            queueSize: 4,
+            partSize: 1024 * 1024 * 5,
+            leavePartsOnError: false,
+        });
+        upload.on('httpUploadProgress', (progress) => {
+            Logger.info('Uploading : %s', progress);
+        });
+        const written = await upload.done();
+        Logger.silly('Daemon wrote : %s', written);
+        return DaemonUtil.stat(cache, s3Key);
+    }
+    static async updateMessage(cache, s3Key, newMessage) {
+        try {
+            const inStat = await DaemonUtil.stat(cache, s3Key);
+            inStat.lastUpdatedMessage = newMessage;
+            return DaemonUtil.writeState(cache, s3Key, inStat, DaemonUtil.DEFAULT_CONTENT);
+        }
+        catch (err) {
+            Logger.error('Error while trying to update a daemon message: %j %s', s3Key, err);
+            throw err;
+        }
+    }
+    static async stat(s3Cache, path) {
+        try {
+            Logger.debug('Daemon stat for path %s / %s', s3Cache.getDefaultBucket(), path);
+            let stat = null;
+            const meta = await s3Cache.fetchMetaForCacheFile(path);
+            Logger.debug('Daemon: Meta is %j', meta);
+            const metaString = meta && meta.Metadata ? meta.Metadata[DaemonUtil.DAEMON_METADATA_KEY] : null;
+            if (metaString) {
+                stat = JSON.parse(metaString);
+                if (stat.completedEpochMS && !stat.error) {
+                    stat.link = await s3Cache.preSignedDownloadUrlForCacheFile(path);
+                }
+            }
+            else {
+                Logger.warn('No metadata found! (Head was %j)', meta);
+            }
+            return stat;
+        }
+        catch (err) {
+            Logger.error('Error while trying to fetch a daemon state: %j %s', path, err);
+            throw err;
+        }
+    }
+    static async abort(s3Cache, path) {
+        return DaemonUtil.error(s3Cache, path, 'Aborted');
+    }
+    static async error(s3Cache, path, error) {
+        try {
+            const inStat = await DaemonUtil.stat(s3Cache, path);
+            inStat.error = error;
+            inStat.completedEpochMS = new Date().getTime();
+            return DaemonUtil.writeState(s3Cache, path, inStat, DaemonUtil.DEFAULT_CONTENT);
+        }
+        catch (err) {
+            Logger.error('Error while trying to write a daemon error: %j %s', path, err);
+            throw err;
+        }
+    }
+    static async finalize(s3Cache, path, contents) {
+        try {
+            Logger.info('Finalizing daemon %s with %d bytes', path, contents.length);
+            const inStat = await DaemonUtil.stat(s3Cache, path);
+            inStat.completedEpochMS = new Date().getTime();
+            inStat.lastUpdatedMessage = 'Complete';
+            return DaemonUtil.writeState(s3Cache, path, inStat, contents);
+        }
+        catch (err) {
+            Logger.error('Error while trying to finalize a daemon: %j %s', path, err);
+            throw err;
+        }
+    }
+}
+DaemonUtil.DEFAULT_CONTENT = Buffer.from('DAEMON_PLACEHOLDER');
+DaemonUtil.DAEMON_METADATA_KEY = 'daemon_meta';