@bitblit/ratchet-aws-node-only 4.0.80-alpha

package/dist/cjs/mail/inbound/inbound-email-ratchet.js

@@ -0,0 +1,54 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MultiStream = exports.InboundEmailRatchet = void 0;
+const mailparser_1 = require("mailparser");
+const stream_1 = require("stream");
+const ratchet_common_1 = require("@bitblit/ratchet-common");
+class InboundEmailRatchet {
+    constructor(cache) {
+        this.cache = cache;
+        ratchet_common_1.RequireRatchet.notNullOrUndefined(this.cache, 'cache');
+        ratchet_common_1.RequireRatchet.notNullOrUndefined(this.cache.getDefaultBucket(), 'cache.defaultBucket');
+    }
+    async processEmailFromS3(key) {
+        const rval = false;
+        if (await this.cache.fileExists(key)) {
+            const data = await this.cache.fetchCacheFileAsString(key);
+            return this.processEmailFromBuffer(new Buffer(data));
+        }
+        else {
+            ratchet_common_1.Logger.warn('Cannot process inbound email - no such key : %s', key);
+        }
+        return rval;
+    }
+    async processEmailFromBuffer(buf) {
+        const rval = false;
+        ratchet_common_1.RequireRatchet.notNullOrUndefined(buf, 'buf');
+        ratchet_common_1.Logger.info('Processing inbound email - size %d bytes', buf.length);
+        const message = await (0, mailparser_1.simpleParser)(buf);
+        ratchet_common_1.Logger.info('Found mail from "%s" subject "%s" with %d attachments', message.from.text, message.subject, message.attachments.length);
+        if (!!message &&
+            !!message.subject &&
+            message.subject.toLowerCase().startsWith('reach inventory') &&
+            !!message.attachments &&
+            message.attachments.length === 1 &&
+            message.attachments[0].contentType.toLowerCase() === 'application/zip') {
+        }
+        else {
+            ratchet_common_1.Logger.info('Unrecognized email - not processing');
+        }
+        return rval;
+    }
+}
+exports.InboundEmailRatchet = InboundEmailRatchet;
+class MultiStream extends stream_1.Readable {
+    constructor(object, options = {}) {
+        super(object instanceof Buffer || typeof object === 'string' ? options : { objectMode: true });
+        this._object = object;
+    }
+    _read() {
+        this.push(this._object);
+        this._object = null;
+    }
+}
+exports.MultiStream = MultiStream;

package/dist/es/athena/alb-athena-log-ratchet.js

@@ -0,0 +1,153 @@
+import { readFileSync } from 'fs';
+import path from 'path';
+import { Logger } from '@bitblit/ratchet-common';
+import { RequireRatchet } from '@bitblit/ratchet-common';
+import { StringRatchet } from '@bitblit/ratchet-common';
+import { S3Ratchet } from '@bitblit/ratchet-aws';
+import { CsvRatchet } from '@bitblit/ratchet-node-only';
+export class AlbAthenaLogRatchet {
+    constructor(athena, athenaTableName) {
+        this.athena = athena;
+        this.athenaTableName = athenaTableName;
+        RequireRatchet.notNullOrUndefined(athena, 'athena');
+        RequireRatchet.notNullOrUndefined(StringRatchet.trimToNull(athenaTableName), 'athenaTableName');
+    }
+    async updatePartitions(rootPath, s3, startTimeEpochMS = new Date().getTime() - 1000 * 60 * 60 * 24, endTimeEpochMS = new Date().getTime()) {
+        RequireRatchet.true(S3Ratchet.checkS3UrlForValidity(rootPath), 'root path not valid');
+        RequireRatchet.notNullOrUndefined(s3, 's3');
+        Logger.info('Updating partitions for %s from %s', this.athenaTableName, rootPath);
+        const bucketName = S3Ratchet.extractBucketFromURL(rootPath);
+        const rootKey = S3Ratchet.extractKeyFromURL(rootPath);
+        let current = startTimeEpochMS;
+        const clauses = [];
+        while (current < endTimeEpochMS) {
+            const dateUtcVal = new Date(current).toISOString().substring(0, 10);
+            Logger.info('d:%s', dateUtcVal);
+            const dateParts = dateUtcVal.split('-');
+            clauses.push("PARTITION (date_utc_partition='" +
+                dateUtcVal +
+                "') LOCATION '" +
+                rootPath +
+                '/' +
+                dateParts[0] +
+                '/' +
+                dateParts[1] +
+                '/' +
+                dateParts[2] +
+                "'");
+            current += 1000 * 60 * 60 * 24;
+        }
+        if (clauses.length > 0) {
+            const stmt = 'ALTER TABLE ' + this.athenaTableName + ' ADD IF NOT EXISTS \n' + clauses.join('\n');
+            await this.athena.runQueryToObjects(stmt);
+        }
+        else {
+            Logger.warn('Not updating partitions - no time between time clauses');
+        }
+        return clauses;
+    }
+    async createTable(rootPath, replaceIfPresent = false) {
+        RequireRatchet.true(S3Ratchet.checkS3UrlForValidity(rootPath), 'root path not valid');
+        let rval = false;
+        Logger.info('Creating ALB table %s', this.athenaTableName);
+        if (replaceIfPresent) {
+            Logger.info('Replace if present specified, removed old table');
+            try {
+                await this.athena.runQueryToObjects('drop table ' + this.athenaTableName);
+            }
+            catch (err) {
+                Logger.info('Drop error : %j', err);
+            }
+        }
+        let tableCreateQry = readFileSync(path.join(__dirname, '../static/albAthenaTableCreate.txt')).toString();
+        tableCreateQry = tableCreateQry.split('{{TABLE NAME}}').join(this.athenaTableName);
+        tableCreateQry = tableCreateQry.split('{{ALB_LOG_ROOT}}').join(rootPath);
+        Logger.info('Creating table with %s', tableCreateQry);
+        try {
+            await this.athena.runQueryToObjects(tableCreateQry);
+            rval = true;
+        }
+        catch (err) {
+            Logger.error('Error creating table : %s', err);
+        }
+        return rval;
+    }
+    static async readLogObjectsFromCsvStream(readStream) {
+        return CsvRatchet.streamParse(readStream, (p) => p);
+    }
+    static async readLogObjectsFromFile(fileName) {
+        return CsvRatchet.fileParse(fileName, (p) => p);
+    }
+    async fetchAlbLogRecords(qry) {
+        const tempFile = await this.fetchAlbLogRecordsToFile(qry);
+        return AlbAthenaLogRatchet.readLogObjectsFromFile(tempFile);
+    }
+    async fetchAlbLogRecordsToFile(qry, outputFileName = null) {
+        Logger.info('Querying %s : %j', this.athenaTableName, qry);
+        let qrySt = 'select * from ' + this.athenaTableName + ' where 1=1 ';
+        if (qry.startTimeEpochMS) {
+            if (qry.startTimeEpochMS) {
+                qrySt += " AND time >= '" + new Date(qry.startTimeEpochMS).toISOString() + "'";
+                qrySt += " AND date_utc_partition >='" + new Date(qry.startTimeEpochMS).toISOString().substring(0, 10) + "'";
+            }
+            if (qry.endTimeEpochMS) {
+                qrySt += " AND time < '" + new Date(qry.endTimeEpochMS).toISOString() + "'";
+                qrySt += " AND date_utc_partition <='" + new Date(qry.endTimeEpochMS).toISOString().substring(0, 10) + "'";
+            }
+            if (qry.requestUrlFilter) {
+                qrySt += " AND request_url LIKE '" + qry.requestUrlFilter + "'";
+            }
+            if (qry.limit) {
+                qrySt += ' LIMIT ' + qry.limit;
+            }
+        }
+        const result = await this.athena.runQueryToFile(qrySt, null, outputFileName);
+        return result;
+    }
+}
+AlbAthenaLogRatchet.CREATE_TABLE_STATEMENT = 'CREATE EXTERNAL TABLE IF NOT EXISTS `{{TABLE NAME}}`(\n' +
+    "  `type` string COMMENT '',\n" +
+    "  `time` string COMMENT '',\n" +
+    "  `elb` string COMMENT '',\n" +
+    "  `client_ip` string COMMENT '',\n" +
+    "  `client_port` int COMMENT '',\n" +
+    "  `target_ip` string COMMENT '',\n" +
+    "  `target_port` int COMMENT '',\n" +
+    "  `request_processing_time` double COMMENT '',\n" +
+    "  `target_processing_time` double COMMENT '',\n" +
+    "  `response_processing_time` double COMMENT '',\n" +
+    "  `elb_status_code` string COMMENT '',\n" +
+    "  `target_status_code` string COMMENT '',\n" +
+    "  `received_bytes` bigint COMMENT '',\n" +
+    "  `sent_bytes` bigint COMMENT '',\n" +
+    "  `request_verb` string COMMENT '',\n" +
+    "  `request_url` string COMMENT '',\n" +
+    "  `request_proto` string COMMENT '',\n" +
+    "  `user_agent` string COMMENT '',\n" +
+    "  `ssl_cipher` string COMMENT '',\n" +
+    "  `ssl_protocol` string COMMENT '',\n" +
+    "  `target_group_arn` string COMMENT '',\n" +
+    "  `trace_id` string COMMENT '',\n" +
+    "  `domain_name` string COMMENT '',\n" +
+    "  `chosen_cert_arn` string COMMENT '',\n" +
+    "  `matched_rule_priority` string COMMENT '',\n" +
+    "  `request_creation_time` string COMMENT '',\n" +
+    "  `actions_executed` string COMMENT '',\n" +
+    "  `redirect_url` string COMMENT '',\n" +
+    "  `lambda_error_reason` string COMMENT '',\n" +
+    "  `target_port_list` string COMMENT '',\n" +
+    "  `target_status_code_list` string COMMENT '',\n" +
+    "  `new_field` string COMMENT '')\n" +
+    'PARTITIONED BY (\n' +
+    '  `date_utc_partition` string\n' +
+    ')\n' +
+    'ROW FORMAT SERDE\n' +
+    "  'org.apache.hadoop.hive.serde2.RegexSerDe'\n" +
+    'WITH SERDEPROPERTIES (\n' +
+    '  \'input.regex\'=\'([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \\"([^ ]*) ([^ ]*) (- |[^ ]*)\\" \\"([^\\"]*)\\" ([A-Z0-9-]+) ([A-Za-z0-9.-]*) ([^ ]*) \\"([^\\"]*)\\" \\"([^\\"]*)\\" \\"([^\\"]*)\\" ([-.0-9]*) ([^ ]*) \\"([^\\"]*)\\" \\"([^\\"]*)\\" \\"([^ ]*)\\" \\"([^s]+)\\" \\"([^s]+)\\"(.*)\')\n' +
+    'STORED AS INPUTFORMAT\n' +
+    "  'org.apache.hadoop.mapred.TextInputFormat'\n" +
+    'OUTPUTFORMAT\n' +
+    "  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\n" +
+    'LOCATION\n' +
+    "  '{{ALB_LOG_ROOT}}'\n";

package/dist/es/athena/athena-ratchet.js

@@ -0,0 +1,154 @@
+import { GetNamedQueryCommand, GetQueryExecutionCommand, ListNamedQueriesCommand, StartQueryExecutionCommand, } from '@aws-sdk/client-athena';
+import { StringRatchet } from '@bitblit/ratchet-common';
+import { Logger } from '@bitblit/ratchet-common';
+import { StopWatch } from '@bitblit/ratchet-common';
+import { PromiseRatchet } from '@bitblit/ratchet-common';
+import { GetObjectCommand } from '@aws-sdk/client-s3';
+import tmp from 'tmp';
+import fs from 'fs';
+import { CsvRatchet } from '@bitblit/ratchet-node-only';
+import { RequireRatchet } from '@bitblit/ratchet-common';
+export class AthenaRatchet {
+    constructor(athena, s3, outputLocation) {
+        this.athena = athena;
+        this.s3 = s3;
+        this.outputLocation = outputLocation;
+        RequireRatchet.notNullOrUndefined(athena);
+        RequireRatchet.notNullOrUndefined(s3);
+        RequireRatchet.notNullOrUndefined(outputLocation);
+        RequireRatchet.true(outputLocation.startsWith('s3://'));
+    }
+    static athenaRowsToObject(input) {
+        const colNames = input[0].Data.map((d) => d.VarCharValue);
+        const temp = input.slice(1);
+        const rval = temp.map((t) => {
+            const newItem = {};
+            for (let i = 0; i < t.Data.length; i++) {
+                newItem[colNames[i]] = t.Data[i].VarCharValue;
+            }
+            return newItem;
+        });
+        return rval;
+    }
+    static applyParamsToQuery(query, queryParams) {
+        let rval = query;
+        if (!!rval && !!queryParams) {
+            Object.keys(queryParams).forEach((k) => {
+                const val = StringRatchet.safeString(queryParams[k]);
+                const kk = '{' + k + '}';
+                rval = rval.split(kk).join(val);
+            });
+        }
+        return rval;
+    }
+    async fetchQueryIds() {
+        const params = {
+            NextToken: null,
+        };
+        let rval = [];
+        let next = null;
+        do {
+            next = await this.athena.send(new ListNamedQueriesCommand(params));
+            rval = rval.concat(next.NamedQueryIds);
+            params.NextToken = next.NextToken;
+        } while (!!params.NextToken);
+        return rval;
+    }
+    async listQueries() {
+        const rval = [];
+        const ids = await this.fetchQueryIds();
+        Logger.debug('Finding %d items', ids.length);
+        for (let i = 0; i < ids.length; i++) {
+            const params = {
+                NamedQueryId: ids[i],
+            };
+            const val = await this.athena.send(new GetNamedQueryCommand(params));
+            rval.push(val.NamedQuery);
+        }
+        return rval;
+    }
+    async findQueryByName(name) {
+        const all = await this.listQueries();
+        const rval = all.find((a) => a.Name.toLowerCase() == name.toLowerCase());
+        return rval;
+    }
+    async runQueryToObjects(queryIn, queryParams = {}, pingTimeMS = 2000) {
+        Logger.info('Running query to objects');
+        const outputLoc = await this.runQueryToOutputLocation(queryIn, queryParams, pingTimeMS);
+        Logger.info('Query succeeded, processing file from %s', outputLoc);
+        const bucketName = outputLoc.substring(5, outputLoc.indexOf('/', 5));
+        const obKey = outputLoc.substring(outputLoc.indexOf('/', 5) + 1);
+        const req = {
+            Bucket: bucketName,
+            Key: obKey,
+        };
+        const getFileOut = await this.s3.send(new GetObjectCommand(req));
+        const rval = await CsvRatchet.stringParse(getFileOut.Body.toString(), (p) => {
+            return p;
+        }, { columns: true, skip_empty_lines: true });
+        return rval;
+    }
+    async runQueryToFile(queryIn, queryParams = {}, targetDataFileIn = null, pingTimeMS = 2000) {
+        Logger.info('Running query to file');
+        const outputLoc = await this.runQueryToOutputLocation(queryIn, queryParams, pingTimeMS);
+        Logger.info('Query succeeded, pulling file from %s', outputLoc);
+        const bucketName = outputLoc.substring(5, outputLoc.indexOf('/', 5));
+        const obKey = outputLoc.substring(outputLoc.indexOf('/', 5) + 1);
+        const req = {
+            Bucket: bucketName,
+            Key: obKey,
+        };
+        const targetDataFile = targetDataFileIn || tmp.fileSync({ postfix: '.csv', keep: false }).name;
+        const fileStream = fs.createWriteStream(targetDataFile);
+        const output = await this.s3.send(new GetObjectCommand(req));
+        const readStream = output.Body;
+        readStream.pipe(fileStream);
+        const rval = await PromiseRatchet.resolveOnEvent(readStream, ['finish', 'close'], ['error'], targetDataFile);
+        Logger.silly('Response: %s', rval);
+        return targetDataFile;
+    }
+    async runQueryToOutputLocation(queryIn, queryParams = {}, pingTimeMS = 2000) {
+        let rval = null;
+        const timer = new StopWatch();
+        const query = AthenaRatchet.applyParamsToQuery(queryIn, queryParams);
+        try {
+            Logger.info('Starting query : %s', query);
+            const token = StringRatchet.createType4Guid();
+            const params = {
+                QueryString: query,
+                ResultConfiguration: {
+                    OutputLocation: this.outputLocation,
+                    EncryptionConfiguration: {
+                        EncryptionOption: 'SSE_S3',
+                    },
+                },
+                ClientRequestToken: token,
+                QueryExecutionContext: {
+                    Database: 'default',
+                },
+            };
+            const startToken = await this.athena.send(new StartQueryExecutionCommand(params));
+            const getExecParams = {
+                QueryExecutionId: startToken.QueryExecutionId,
+            };
+            const finalStates = ['FAILED', 'CANCELLED', 'SUCCEEDED'];
+            let curState = await this.athena.send(new GetQueryExecutionCommand(getExecParams));
+            while (finalStates.indexOf(curState.QueryExecution.Status.State) === -1) {
+                await PromiseRatchet.createTimeoutPromise('wait', pingTimeMS);
+                Logger.debug('%s : %s : %s', curState.QueryExecution.Status.State, timer.dump(), query);
+                curState = await this.athena.send(new GetQueryExecutionCommand(getExecParams));
+            }
+            if (curState.QueryExecution.Status.State === 'FAILED') {
+                Logger.warn('Query failed : %s', curState.QueryExecution.Status.StateChangeReason);
+            }
+            else if (curState.QueryExecution.Status.State === 'SUCCEEDED') {
+                rval = curState.QueryExecution.ResultConfiguration.OutputLocation;
+            }
+        }
+        catch (err) {
+            Logger.warn('Failure : %s', err, err);
+        }
+        Logger.info('Query took %s : %s', timer.dump(), query);
+        return rval;
+    }
+}

package/dist/es/build/ratchet-aws-node-only-info.js

@@ -0,0 +1,14 @@
+export class RatchetAwsNodeOnlyInfo {
+    constructor() { }
+    static buildInformation() {
+        const val = {
+            version: '80',
+            hash: '10fcb761c2fa4186df89e527e948f5780a4e14ac',
+            branch: 'alpha-2023-03-12-2',
+            tag: 'alpha-2023-03-12-2',
+            timeBuiltISO: '2023-03-12T20:00:51-0700',
+            notes: '',
+        };
+        return val;
+    }
+}

package/dist/es/cli/dynamo-exporter.js

@@ -0,0 +1,76 @@
+import { StringRatchet } from '@bitblit/ratchet-common';
+import { Logger } from '@bitblit/ratchet-common';
+import { PromiseRatchet } from '@bitblit/ratchet-common';
+import { RequireRatchet } from '@bitblit/ratchet-common';
+import fs from 'fs';
+import readline from 'readline';
+export class DynamoExporter {
+    constructor() { }
+    static async importJsonLFileToTable(dynamo, tableName, filename) {
+        RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');
+        RequireRatchet.notNullOrUndefined(tableName, 'tableName');
+        RequireRatchet.notNullOrUndefined(filename, 'filename');
+        const fileStream = fs.createReadStream(filename);
+        const rl = readline.createInterface({
+            input: fileStream,
+            crlfDelay: Infinity,
+        });
+        let rval = 0;
+        for await (const line of rl) {
+            if (rval % 100 === 0) {
+                Logger.info('Importing line %d', rval);
+            }
+            if (StringRatchet.trimToNull(line)) {
+                const parsed = JSON.parse(line);
+                await dynamo.simplePut(tableName, parsed);
+                rval++;
+            }
+        }
+        return rval;
+    }
+    static async exportScanToJsonLFile(dynamo, scan, filename) {
+        RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');
+        RequireRatchet.notNullOrUndefined(scan, 'scan');
+        RequireRatchet.notNullOrUndefined(filename, 'filename');
+        const ws = fs.createWriteStream(filename);
+        ws.on('end', () => {
+            Logger.debug('Write complete');
+        });
+        const rval = await DynamoExporter.exportScanToJsonLWriteStream(dynamo, scan, ws);
+        await PromiseRatchet.resolveOnEvent(ws, ['finish', 'close'], ['error']);
+        ws.close();
+        return rval;
+    }
+    static async exportQueryToJsonLFile(dynamo, qry, filename) {
+        RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');
+        RequireRatchet.notNullOrUndefined(qry, 'qry');
+        RequireRatchet.notNullOrUndefined(filename, 'filename');
+        const ws = fs.createWriteStream(filename);
+        ws.on('end', () => {
+            Logger.debug('Write complete');
+        });
+        const rval = await DynamoExporter.exportQueryToJsonLWriteStream(dynamo, qry, ws);
+        await PromiseRatchet.resolveOnEvent(ws, ['finish', 'close'], ['error']);
+        ws.close();
+        return rval;
+    }
+    static async exportScanToJsonLWriteStream(dynamo, scan, target) {
+        RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');
+        RequireRatchet.notNullOrUndefined(scan, 'scan');
+        RequireRatchet.notNullOrUndefined(target, 'target');
+        const rval = await dynamo.fullyExecuteProcessOverScan(scan, async (row) => DynamoExporter.writeItemToJsonLStream(row, target, false));
+        return rval;
+    }
+    static async exportQueryToJsonLWriteStream(dynamo, qry, target) {
+        RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');
+        RequireRatchet.notNullOrUndefined(qry, 'qry');
+        RequireRatchet.notNullOrUndefined(target, 'target');
+        const rval = await dynamo.fullyExecuteProcessOverQuery(qry, async (row) => DynamoExporter.writeItemToJsonLStream(row, target, false));
+        return rval;
+    }
+    static writeItemToJsonLStream(item, target, includeNulls = false) {
+        if (!!item || includeNulls) {
+            target.write(JSON.stringify(item) + '\n');
+        }
+    }
+}

package/dist/es/cli/ratchet-cli-handler.js

@@ -0,0 +1,15 @@
+import { AbstractRatchetCliHandler } from '@bitblit/ratchet-node-only';
+import { SiteUploader } from './site-uploader/site-uploader';
+import { StartInstanceAndSsh } from './start-instance-and-ssh';
+import { RatchetAwsNodeOnlyInfo } from '../build/ratchet-aws-node-only-info';
+export class RatchetCliHandler extends AbstractRatchetCliHandler {
+    fetchHandlerMap() {
+        return {
+            'site-uploader': SiteUploader.runFromCliArgs,
+            'start-instance-and-ssh': StartInstanceAndSsh.runFromCliArgs,
+        };
+    }
+    fetchVersionInfo() {
+        return RatchetAwsNodeOnlyInfo.buildInformation();
+    }
+}

package/dist/es/cli/site-uploader/site-uploader.js

@@ -0,0 +1,112 @@
+import fs from 'fs';
+import walk from 'walk';
+import { S3Client } from '@aws-sdk/client-s3';
+import path from 'path';
+import mime from 'mime-types';
+import { Logger } from '@bitblit/ratchet-common';
+import { Upload } from '@aws-sdk/lib-storage';
+export class SiteUploader {
+    constructor(srcDir, bucketName, configFile) {
+        this.s3 = new S3Client({ region: 'us-east-1' });
+        this.srcDir = srcDir;
+        this.bucketName = bucketName;
+        this.config = JSON.parse(fs.readFileSync(configFile).toString('ascii'));
+    }
+    static createFromArgs(args) {
+        if (args && args.length === 3) {
+            const src = args[0];
+            const bucket = args[1];
+            const configFile = args[2];
+            return new SiteUploader(src, bucket, configFile);
+        }
+        else {
+            console.log('Usage : node ratchet-site-uploader {srcDir} {bucket} {configFile} (Found ' + args + ' arguments, need 3)');
+            return null;
+        }
+    }
+    static async runFromCliArgs(args) {
+        const inst = SiteUploader.createFromArgs(args);
+        return inst.runPump();
+    }
+    findMatch(prefix, fileName, config) {
+        let found = null;
+        if (prefix != null && fileName != null && config != null && config.mapping != null) {
+            config.mapping.forEach((entry) => {
+                if (found == null) {
+                    if (entry.prefixMatch == null || prefix.match(entry.prefixMatch)) {
+                        if (entry.fileMatch == null || fileName.match(entry.fileMatch)) {
+                            found = entry;
+                        }
+                    }
+                }
+            });
+        }
+        return found;
+    }
+    findMime(fileName, config) {
+        let found = null;
+        if (config != null && config.customMimeTypeMapping != null) {
+            Object.keys(config.customMimeTypeMapping).forEach((k) => {
+                if (found == null && fileName.endsWith(k)) {
+                    found = config.customMimeTypeMapping[k];
+                }
+            });
+        }
+        if (found == null) {
+            found = mime.lookup(fileName);
+        }
+        if (found == null) {
+            found = 'binary/octet-stream';
+        }
+        return found;
+    }
+    runPump() {
+        return new Promise((resolve, reject) => {
+            Logger.info('Uploading contents of %s to %s using %j as config', this.srcDir, this.bucketName, this.config);
+            const options = {};
+            const walker = walk.walk(this.srcDir, options);
+            walker.on('file', function (root, fileStats, next) {
+                Logger.info('Processing %j', fileStats.name);
+                const prefix = root == this.srcDir ? '' : root.substring(this.srcDir.length + 1) + '/';
+                const proc = this.findMatch(prefix, fileStats.name, this.config);
+                const key = prefix + fileStats.name;
+                Logger.info('Uploading file : %s/%s to key %s with %j', root, fileStats.name, key, proc);
+                const params = proc && proc.putParams ? JSON.parse(JSON.stringify(proc.putParams)) : {};
+                params.Bucket = this.bucketName;
+                params.Key = key;
+                params.Body = fs.readFileSync(path.join(root, fileStats.name));
+                if (!params.ContentType) {
+                    params.ContentType = this.findMime(fileStats.name, this.config);
+                }
+                const upload = new Upload({
+                    client: this.s3,
+                    params: params,
+                    tags: [],
+                    queueSize: 4,
+                    partSize: 1024 * 1024 * 5,
+                    leavePartsOnError: false,
+                });
+                upload.on('httpUploadProgress', (progress) => {
+                    Logger.info('Uploading : %s', progress);
+                });
+                upload
+                    .done()
+                    .then((result) => {
+                    Logger.info('Finished upload of %s: %j', key, result);
+                    next();
+                })
+                    .catch((err) => {
+                    Logger.warn('%s failed to upload : %s : Continuing', key, err);
+                    next();
+                });
+            }.bind(this));
+            walker.on('errors', function (root, nodeStatsArray, next) {
+                next();
+            });
+            walker.on('end', function () {
+                Logger.info('All done');
+                resolve(true);
+            });
+        });
+    }
+}

package/dist/es/cli/start-instance-and-ssh.js

@@ -0,0 +1,62 @@
+import { Logger } from '@bitblit/ratchet-common';
+import { Ec2Ratchet } from '@bitblit/ratchet-aws';
+import { spawnSync } from 'child_process';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+export class StartInstanceAndSsh {
+    constructor(instanceId, publicKeyFile = path.join(os.homedir(), '.ssh', 'id_rsa.pub'), instanceOsUser = 'ec2-user', region = 'us-east-1', availabilityZone = 'us-east-1a') {
+        this.instanceId = instanceId;
+        this.publicKeyFile = publicKeyFile;
+        this.instanceOsUser = instanceOsUser;
+        this.region = region;
+        this.availabilityZone = availabilityZone;
+        this.ec2Ratchet = new Ec2Ratchet(this.region, this.availabilityZone);
+    }
+    static createFromArgs(args) {
+        if (args?.length === 1 || args?.length === 2) {
+            const instanceId = args[0];
+            return new StartInstanceAndSsh(instanceId);
+        }
+        else {
+            Logger.info('Usage : ratchet-start-instance-and-ssh {instanceId} {publicKeyFile} (Found %s arguments, need 1 or 2)', args);
+            return null;
+        }
+    }
+    static async runFromCliArgs(args) {
+        const inst = StartInstanceAndSsh.createFromArgs(args);
+        return inst.run();
+    }
+    async run() {
+        let instance = await this.ec2Ratchet.describeInstance(this.instanceId);
+        if (!!instance) {
+            let launched = false;
+            if (instance.State.Code == 16) {
+                Logger.info('Instance is already running...');
+                launched = true;
+            }
+            else {
+                Logger.info('Instance is not running... starting up : %s', this.instanceId);
+                launched = await this.ec2Ratchet.launchInstance(this.instanceId, 1000 * 30);
+            }
+            if (launched) {
+                Logger.info('Uploading public key...');
+                const publicKeyText = fs.readFileSync(this.publicKeyFile).toString();
+                const publicKeyResponse = await this.ec2Ratchet.sendPublicKeyToEc2Instance(this.instanceId, publicKeyText, this.instanceOsUser);
+                Logger.info('Key response : %j', publicKeyResponse);
+                instance = instance && instance.PublicIpAddress ? instance : await this.ec2Ratchet.describeInstance(this.instanceId);
+                Logger.info('Instance IP address is %s', instance.PublicIpAddress);
+                const ret = spawnSync('ssh', [this.instanceOsUser + '@' + instance.PublicIpAddress], {
+                    stdio: 'inherit',
+                });
+                Logger.info('%j', ret);
+            }
+            else {
+                Logger.info('Instance could not start - check logs');
+            }
+        }
+        else {
+            Logger.info('No such instance found - check your AWS keys? : %s', this.instanceId);
+        }
+    }
+}

package/dist/es/index.js

@@ -0,0 +1,8 @@
+export * from './athena/alb-athena-log-ratchet';
+export * from './athena/athena-ratchet';
+export * from './build/ratchet-aws-node-only-info';
+export * from './cli/dynamo-exporter';
+export * from './cli/ratchet-cli-handler';
+export * from './cli/start-instance-and-ssh';
+export * from './cli/site-uploader/site-uploader';
+export * from './mail/inbound/inbound-email-ratchet';