@bitblit/ratchet-aws-node-only 4.0.80-alpha

package/dist/cjs/athena/alb-athena-log-ratchet.js

@@ -0,0 +1,158 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AlbAthenaLogRatchet = void 0;
+const tslib_1 = require("tslib");
+const fs_1 = require("fs");
+const path_1 = tslib_1.__importDefault(require("path"));
+const ratchet_common_1 = require("@bitblit/ratchet-common");
+const ratchet_common_2 = require("@bitblit/ratchet-common");
+const ratchet_common_3 = require("@bitblit/ratchet-common");
+const ratchet_aws_1 = require("@bitblit/ratchet-aws");
+const ratchet_node_only_1 = require("@bitblit/ratchet-node-only");
+class AlbAthenaLogRatchet {
+    constructor(athena, athenaTableName) {
+        this.athena = athena;
+        this.athenaTableName = athenaTableName;
+        ratchet_common_2.RequireRatchet.notNullOrUndefined(athena, 'athena');
+        ratchet_common_2.RequireRatchet.notNullOrUndefined(ratchet_common_3.StringRatchet.trimToNull(athenaTableName), 'athenaTableName');
+    }
+    async updatePartitions(rootPath, s3, startTimeEpochMS = new Date().getTime() - 1000 * 60 * 60 * 24, endTimeEpochMS = new Date().getTime()) {
+        ratchet_common_2.RequireRatchet.true(ratchet_aws_1.S3Ratchet.checkS3UrlForValidity(rootPath), 'root path not valid');
+        ratchet_common_2.RequireRatchet.notNullOrUndefined(s3, 's3');
+        ratchet_common_1.Logger.info('Updating partitions for %s from %s', this.athenaTableName, rootPath);
+        const bucketName = ratchet_aws_1.S3Ratchet.extractBucketFromURL(rootPath);
+        const rootKey = ratchet_aws_1.S3Ratchet.extractKeyFromURL(rootPath);
+        let current = startTimeEpochMS;
+        const clauses = [];
+        while (current < endTimeEpochMS) {
+            const dateUtcVal = new Date(current).toISOString().substring(0, 10);
+            ratchet_common_1.Logger.info('d:%s', dateUtcVal);
+            const dateParts = dateUtcVal.split('-');
+            clauses.push("PARTITION (date_utc_partition='" +
+                dateUtcVal +
+                "') LOCATION '" +
+                rootPath +
+                '/' +
+                dateParts[0] +
+                '/' +
+                dateParts[1] +
+                '/' +
+                dateParts[2] +
+                "'");
+            current += 1000 * 60 * 60 * 24;
+        }
+        if (clauses.length > 0) {
+            const stmt = 'ALTER TABLE ' + this.athenaTableName + ' ADD IF NOT EXISTS \n' + clauses.join('\n');
+            await this.athena.runQueryToObjects(stmt);
+        }
+        else {
+            ratchet_common_1.Logger.warn('Not updating partitions - no time between time clauses');
+        }
+        return clauses;
+    }
+    async createTable(rootPath, replaceIfPresent = false) {
+        ratchet_common_2.RequireRatchet.true(ratchet_aws_1.S3Ratchet.checkS3UrlForValidity(rootPath), 'root path not valid');
+        let rval = false;
+        ratchet_common_1.Logger.info('Creating ALB table %s', this.athenaTableName);
+        if (replaceIfPresent) {
+            ratchet_common_1.Logger.info('Replace if present specified, removed old table');
+            try {
+                await this.athena.runQueryToObjects('drop table ' + this.athenaTableName);
+            }
+            catch (err) {
+                ratchet_common_1.Logger.info('Drop error : %j', err);
+            }
+        }
+        let tableCreateQry = (0, fs_1.readFileSync)(path_1.default.join(__dirname, '../static/albAthenaTableCreate.txt')).toString();
+        tableCreateQry = tableCreateQry.split('{{TABLE NAME}}').join(this.athenaTableName);
+        tableCreateQry = tableCreateQry.split('{{ALB_LOG_ROOT}}').join(rootPath);
+        ratchet_common_1.Logger.info('Creating table with %s', tableCreateQry);
+        try {
+            await this.athena.runQueryToObjects(tableCreateQry);
+            rval = true;
+        }
+        catch (err) {
+            ratchet_common_1.Logger.error('Error creating table : %s', err);
+        }
+        return rval;
+    }
+    static async readLogObjectsFromCsvStream(readStream) {
+        return ratchet_node_only_1.CsvRatchet.streamParse(readStream, (p) => p);
+    }
+    static async readLogObjectsFromFile(fileName) {
+        return ratchet_node_only_1.CsvRatchet.fileParse(fileName, (p) => p);
+    }
+    async fetchAlbLogRecords(qry) {
+        const tempFile = await this.fetchAlbLogRecordsToFile(qry);
+        return AlbAthenaLogRatchet.readLogObjectsFromFile(tempFile);
+    }
+    async fetchAlbLogRecordsToFile(qry, outputFileName = null) {
+        ratchet_common_1.Logger.info('Querying %s : %j', this.athenaTableName, qry);
+        let qrySt = 'select * from ' + this.athenaTableName + ' where 1=1 ';
+        if (qry.startTimeEpochMS) {
+            if (qry.startTimeEpochMS) {
+                qrySt += " AND time >= '" + new Date(qry.startTimeEpochMS).toISOString() + "'";
+                qrySt += " AND date_utc_partition >='" + new Date(qry.startTimeEpochMS).toISOString().substring(0, 10) + "'";
+            }
+            if (qry.endTimeEpochMS) {
+                qrySt += " AND time < '" + new Date(qry.endTimeEpochMS).toISOString() + "'";
+                qrySt += " AND date_utc_partition <='" + new Date(qry.endTimeEpochMS).toISOString().substring(0, 10) + "'";
+            }
+            if (qry.requestUrlFilter) {
+                qrySt += " AND request_url LIKE '" + qry.requestUrlFilter + "'";
+            }
+            if (qry.limit) {
+                qrySt += ' LIMIT ' + qry.limit;
+            }
+        }
+        const result = await this.athena.runQueryToFile(qrySt, null, outputFileName);
+        return result;
+    }
+}
+exports.AlbAthenaLogRatchet = AlbAthenaLogRatchet;
+AlbAthenaLogRatchet.CREATE_TABLE_STATEMENT = 'CREATE EXTERNAL TABLE IF NOT EXISTS `{{TABLE NAME}}`(\n' +
+    "  `type` string COMMENT '',\n" +
+    "  `time` string COMMENT '',\n" +
+    "  `elb` string COMMENT '',\n" +
+    "  `client_ip` string COMMENT '',\n" +
+    "  `client_port` int COMMENT '',\n" +
+    "  `target_ip` string COMMENT '',\n" +
+    "  `target_port` int COMMENT '',\n" +
+    "  `request_processing_time` double COMMENT '',\n" +
+    "  `target_processing_time` double COMMENT '',\n" +
+    "  `response_processing_time` double COMMENT '',\n" +
+    "  `elb_status_code` string COMMENT '',\n" +
+    "  `target_status_code` string COMMENT '',\n" +
+    "  `received_bytes` bigint COMMENT '',\n" +
+    "  `sent_bytes` bigint COMMENT '',\n" +
+    "  `request_verb` string COMMENT '',\n" +
+    "  `request_url` string COMMENT '',\n" +
+    "  `request_proto` string COMMENT '',\n" +
+    "  `user_agent` string COMMENT '',\n" +
+    "  `ssl_cipher` string COMMENT '',\n" +
+    "  `ssl_protocol` string COMMENT '',\n" +
+    "  `target_group_arn` string COMMENT '',\n" +
+    "  `trace_id` string COMMENT '',\n" +
+    "  `domain_name` string COMMENT '',\n" +
+    "  `chosen_cert_arn` string COMMENT '',\n" +
+    "  `matched_rule_priority` string COMMENT '',\n" +
+    "  `request_creation_time` string COMMENT '',\n" +
+    "  `actions_executed` string COMMENT '',\n" +
+    "  `redirect_url` string COMMENT '',\n" +
+    "  `lambda_error_reason` string COMMENT '',\n" +
+    "  `target_port_list` string COMMENT '',\n" +
+    "  `target_status_code_list` string COMMENT '',\n" +
+    "  `new_field` string COMMENT '')\n" +
+    'PARTITIONED BY (\n' +
+    '  `date_utc_partition` string\n' +
+    ')\n' +
+    'ROW FORMAT SERDE\n' +
+    "  'org.apache.hadoop.hive.serde2.RegexSerDe'\n" +
+    'WITH SERDEPROPERTIES (\n' +
+    '  \'input.regex\'=\'([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \\"([^ ]*) ([^ ]*) (- |[^ ]*)\\" \\"([^\\"]*)\\" ([A-Z0-9-]+) ([A-Za-z0-9.-]*) ([^ ]*) \\"([^\\"]*)\\" \\"([^\\"]*)\\" \\"([^\\"]*)\\" ([-.0-9]*) ([^ ]*) \\"([^\\"]*)\\" \\"([^\\"]*)\\" \\"([^ ]*)\\" \\"([^s]+)\\" \\"([^s]+)\\"(.*)\')\n' +
+    'STORED AS INPUTFORMAT\n' +
+    "  'org.apache.hadoop.mapred.TextInputFormat'\n" +
+    'OUTPUTFORMAT\n' +
+    "  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\n" +
+    'LOCATION\n' +
+    "  '{{ALB_LOG_ROOT}}'\n";

package/dist/cjs/athena/athena-ratchet.js

@@ -0,0 +1,159 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AthenaRatchet = void 0;
+const tslib_1 = require("tslib");
+const client_athena_1 = require("@aws-sdk/client-athena");
+const ratchet_common_1 = require("@bitblit/ratchet-common");
+const ratchet_common_2 = require("@bitblit/ratchet-common");
+const ratchet_common_3 = require("@bitblit/ratchet-common");
+const ratchet_common_4 = require("@bitblit/ratchet-common");
+const client_s3_1 = require("@aws-sdk/client-s3");
+const tmp_1 = tslib_1.__importDefault(require("tmp"));
+const fs_1 = tslib_1.__importDefault(require("fs"));
+const ratchet_node_only_1 = require("@bitblit/ratchet-node-only");
+const ratchet_common_5 = require("@bitblit/ratchet-common");
+class AthenaRatchet {
+    constructor(athena, s3, outputLocation) {
+        this.athena = athena;
+        this.s3 = s3;
+        this.outputLocation = outputLocation;
+        ratchet_common_5.RequireRatchet.notNullOrUndefined(athena);
+        ratchet_common_5.RequireRatchet.notNullOrUndefined(s3);
+        ratchet_common_5.RequireRatchet.notNullOrUndefined(outputLocation);
+        ratchet_common_5.RequireRatchet.true(outputLocation.startsWith('s3://'));
+    }
+    static athenaRowsToObject(input) {
+        const colNames = input[0].Data.map((d) => d.VarCharValue);
+        const temp = input.slice(1);
+        const rval = temp.map((t) => {
+            const newItem = {};
+            for (let i = 0; i < t.Data.length; i++) {
+                newItem[colNames[i]] = t.Data[i].VarCharValue;
+            }
+            return newItem;
+        });
+        return rval;
+    }
+    static applyParamsToQuery(query, queryParams) {
+        let rval = query;
+        if (!!rval && !!queryParams) {
+            Object.keys(queryParams).forEach((k) => {
+                const val = ratchet_common_1.StringRatchet.safeString(queryParams[k]);
+                const kk = '{' + k + '}';
+                rval = rval.split(kk).join(val);
+            });
+        }
+        return rval;
+    }
+    async fetchQueryIds() {
+        const params = {
+            NextToken: null,
+        };
+        let rval = [];
+        let next = null;
+        do {
+            next = await this.athena.send(new client_athena_1.ListNamedQueriesCommand(params));
+            rval = rval.concat(next.NamedQueryIds);
+            params.NextToken = next.NextToken;
+        } while (!!params.NextToken);
+        return rval;
+    }
+    async listQueries() {
+        const rval = [];
+        const ids = await this.fetchQueryIds();
+        ratchet_common_2.Logger.debug('Finding %d items', ids.length);
+        for (let i = 0; i < ids.length; i++) {
+            const params = {
+                NamedQueryId: ids[i],
+            };
+            const val = await this.athena.send(new client_athena_1.GetNamedQueryCommand(params));
+            rval.push(val.NamedQuery);
+        }
+        return rval;
+    }
+    async findQueryByName(name) {
+        const all = await this.listQueries();
+        const rval = all.find((a) => a.Name.toLowerCase() == name.toLowerCase());
+        return rval;
+    }
+    async runQueryToObjects(queryIn, queryParams = {}, pingTimeMS = 2000) {
+        ratchet_common_2.Logger.info('Running query to objects');
+        const outputLoc = await this.runQueryToOutputLocation(queryIn, queryParams, pingTimeMS);
+        ratchet_common_2.Logger.info('Query succeeded, processing file from %s', outputLoc);
+        const bucketName = outputLoc.substring(5, outputLoc.indexOf('/', 5));
+        const obKey = outputLoc.substring(outputLoc.indexOf('/', 5) + 1);
+        const req = {
+            Bucket: bucketName,
+            Key: obKey,
+        };
+        const getFileOut = await this.s3.send(new client_s3_1.GetObjectCommand(req));
+        const rval = await ratchet_node_only_1.CsvRatchet.stringParse(getFileOut.Body.toString(), (p) => {
+            return p;
+        }, { columns: true, skip_empty_lines: true });
+        return rval;
+    }
+    async runQueryToFile(queryIn, queryParams = {}, targetDataFileIn = null, pingTimeMS = 2000) {
+        ratchet_common_2.Logger.info('Running query to file');
+        const outputLoc = await this.runQueryToOutputLocation(queryIn, queryParams, pingTimeMS);
+        ratchet_common_2.Logger.info('Query succeeded, pulling file from %s', outputLoc);
+        const bucketName = outputLoc.substring(5, outputLoc.indexOf('/', 5));
+        const obKey = outputLoc.substring(outputLoc.indexOf('/', 5) + 1);
+        const req = {
+            Bucket: bucketName,
+            Key: obKey,
+        };
+        const targetDataFile = targetDataFileIn || tmp_1.default.fileSync({ postfix: '.csv', keep: false }).name;
+        const fileStream = fs_1.default.createWriteStream(targetDataFile);
+        const output = await this.s3.send(new client_s3_1.GetObjectCommand(req));
+        const readStream = output.Body;
+        readStream.pipe(fileStream);
+        const rval = await ratchet_common_4.PromiseRatchet.resolveOnEvent(readStream, ['finish', 'close'], ['error'], targetDataFile);
+        ratchet_common_2.Logger.silly('Response: %s', rval);
+        return targetDataFile;
+    }
+    async runQueryToOutputLocation(queryIn, queryParams = {}, pingTimeMS = 2000) {
+        let rval = null;
+        const timer = new ratchet_common_3.StopWatch();
+        const query = AthenaRatchet.applyParamsToQuery(queryIn, queryParams);
+        try {
+            ratchet_common_2.Logger.info('Starting query : %s', query);
+            const token = ratchet_common_1.StringRatchet.createType4Guid();
+            const params = {
+                QueryString: query,
+                ResultConfiguration: {
+                    OutputLocation: this.outputLocation,
+                    EncryptionConfiguration: {
+                        EncryptionOption: 'SSE_S3',
+                    },
+                },
+                ClientRequestToken: token,
+                QueryExecutionContext: {
+                    Database: 'default',
+                },
+            };
+            const startToken = await this.athena.send(new client_athena_1.StartQueryExecutionCommand(params));
+            const getExecParams = {
+                QueryExecutionId: startToken.QueryExecutionId,
+            };
+            const finalStates = ['FAILED', 'CANCELLED', 'SUCCEEDED'];
+            let curState = await this.athena.send(new client_athena_1.GetQueryExecutionCommand(getExecParams));
+            while (finalStates.indexOf(curState.QueryExecution.Status.State) === -1) {
+                await ratchet_common_4.PromiseRatchet.createTimeoutPromise('wait', pingTimeMS);
+                ratchet_common_2.Logger.debug('%s : %s : %s', curState.QueryExecution.Status.State, timer.dump(), query);
+                curState = await this.athena.send(new client_athena_1.GetQueryExecutionCommand(getExecParams));
+            }
+            if (curState.QueryExecution.Status.State === 'FAILED') {
+                ratchet_common_2.Logger.warn('Query failed : %s', curState.QueryExecution.Status.StateChangeReason);
+            }
+            else if (curState.QueryExecution.Status.State === 'SUCCEEDED') {
+                rval = curState.QueryExecution.ResultConfiguration.OutputLocation;
+            }
+        }
+        catch (err) {
+            ratchet_common_2.Logger.warn('Failure : %s', err, err);
+        }
+        ratchet_common_2.Logger.info('Query took %s : %s', timer.dump(), query);
+        return rval;
+    }
+}
+exports.AthenaRatchet = AthenaRatchet;

package/dist/cjs/build/ratchet-aws-node-only-info.js

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RatchetAwsNodeOnlyInfo = void 0;
+class RatchetAwsNodeOnlyInfo {
+    constructor() { }
+    static buildInformation() {
+        const val = {
+            version: '80',
+            hash: '10fcb761c2fa4186df89e527e948f5780a4e14ac',
+            branch: 'alpha-2023-03-12-2',
+            tag: 'alpha-2023-03-12-2',
+            timeBuiltISO: '2023-03-12T20:00:51-0700',
+            notes: '',
+        };
+        return val;
+    }
+}
+exports.RatchetAwsNodeOnlyInfo = RatchetAwsNodeOnlyInfo;

package/dist/cjs/cli/dynamo-exporter.js

@@ -0,0 +1,81 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DynamoExporter = void 0;
+const tslib_1 = require("tslib");
+const ratchet_common_1 = require("@bitblit/ratchet-common");
+const ratchet_common_2 = require("@bitblit/ratchet-common");
+const ratchet_common_3 = require("@bitblit/ratchet-common");
+const ratchet_common_4 = require("@bitblit/ratchet-common");
+const fs_1 = tslib_1.__importDefault(require("fs"));
+const readline_1 = tslib_1.__importDefault(require("readline"));
+class DynamoExporter {
+    constructor() { }
+    static async importJsonLFileToTable(dynamo, tableName, filename) {
+        ratchet_common_4.RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');
+        ratchet_common_4.RequireRatchet.notNullOrUndefined(tableName, 'tableName');
+        ratchet_common_4.RequireRatchet.notNullOrUndefined(filename, 'filename');
+        const fileStream = fs_1.default.createReadStream(filename);
+        const rl = readline_1.default.createInterface({
+            input: fileStream,
+            crlfDelay: Infinity,
+        });
+        let rval = 0;
+        for await (const line of rl) {
+            if (rval % 100 === 0) {
+                ratchet_common_2.Logger.info('Importing line %d', rval);
+            }
+            if (ratchet_common_1.StringRatchet.trimToNull(line)) {
+                const parsed = JSON.parse(line);
+                await dynamo.simplePut(tableName, parsed);
+                rval++;
+            }
+        }
+        return rval;
+    }
+    static async exportScanToJsonLFile(dynamo, scan, filename) {
+        ratchet_common_4.RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');
+        ratchet_common_4.RequireRatchet.notNullOrUndefined(scan, 'scan');
+        ratchet_common_4.RequireRatchet.notNullOrUndefined(filename, 'filename');
+        const ws = fs_1.default.createWriteStream(filename);
+        ws.on('end', () => {
+            ratchet_common_2.Logger.debug('Write complete');
+        });
+        const rval = await DynamoExporter.exportScanToJsonLWriteStream(dynamo, scan, ws);
+        await ratchet_common_3.PromiseRatchet.resolveOnEvent(ws, ['finish', 'close'], ['error']);
+        ws.close();
+        return rval;
+    }
+    static async exportQueryToJsonLFile(dynamo, qry, filename) {
+        ratchet_common_4.RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');
+        ratchet_common_4.RequireRatchet.notNullOrUndefined(qry, 'qry');
+        ratchet_common_4.RequireRatchet.notNullOrUndefined(filename, 'filename');
+        const ws = fs_1.default.createWriteStream(filename);
+        ws.on('end', () => {
+            ratchet_common_2.Logger.debug('Write complete');
+        });
+        const rval = await DynamoExporter.exportQueryToJsonLWriteStream(dynamo, qry, ws);
+        await ratchet_common_3.PromiseRatchet.resolveOnEvent(ws, ['finish', 'close'], ['error']);
+        ws.close();
+        return rval;
+    }
+    static async exportScanToJsonLWriteStream(dynamo, scan, target) {
+        ratchet_common_4.RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');
+        ratchet_common_4.RequireRatchet.notNullOrUndefined(scan, 'scan');
+        ratchet_common_4.RequireRatchet.notNullOrUndefined(target, 'target');
+        const rval = await dynamo.fullyExecuteProcessOverScan(scan, async (row) => DynamoExporter.writeItemToJsonLStream(row, target, false));
+        return rval;
+    }
+    static async exportQueryToJsonLWriteStream(dynamo, qry, target) {
+        ratchet_common_4.RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');
+        ratchet_common_4.RequireRatchet.notNullOrUndefined(qry, 'qry');
+        ratchet_common_4.RequireRatchet.notNullOrUndefined(target, 'target');
+        const rval = await dynamo.fullyExecuteProcessOverQuery(qry, async (row) => DynamoExporter.writeItemToJsonLStream(row, target, false));
+        return rval;
+    }
+    static writeItemToJsonLStream(item, target, includeNulls = false) {
+        if (!!item || includeNulls) {
+            target.write(JSON.stringify(item) + '\n');
+        }
+    }
+}
+exports.DynamoExporter = DynamoExporter;

package/dist/cjs/cli/ratchet-cli-handler.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RatchetCliHandler = void 0;
+const ratchet_node_only_1 = require("@bitblit/ratchet-node-only");
+const site_uploader_1 = require("./site-uploader/site-uploader");
+const start_instance_and_ssh_1 = require("./start-instance-and-ssh");
+const ratchet_aws_node_only_info_1 = require("../build/ratchet-aws-node-only-info");
+class RatchetCliHandler extends ratchet_node_only_1.AbstractRatchetCliHandler {
+    fetchHandlerMap() {
+        return {
+            'site-uploader': site_uploader_1.SiteUploader.runFromCliArgs,
+            'start-instance-and-ssh': start_instance_and_ssh_1.StartInstanceAndSsh.runFromCliArgs,
+        };
+    }
+    fetchVersionInfo() {
+        return ratchet_aws_node_only_info_1.RatchetAwsNodeOnlyInfo.buildInformation();
+    }
+}
+exports.RatchetCliHandler = RatchetCliHandler;

package/dist/cjs/cli/site-uploader/site-uploader.js

@@ -0,0 +1,117 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SiteUploader = void 0;
+const tslib_1 = require("tslib");
+const fs_1 = tslib_1.__importDefault(require("fs"));
+const walk_1 = tslib_1.__importDefault(require("walk"));
+const client_s3_1 = require("@aws-sdk/client-s3");
+const path_1 = tslib_1.__importDefault(require("path"));
+const mime_types_1 = tslib_1.__importDefault(require("mime-types"));
+const ratchet_common_1 = require("@bitblit/ratchet-common");
+const lib_storage_1 = require("@aws-sdk/lib-storage");
+class SiteUploader {
+    constructor(srcDir, bucketName, configFile) {
+        this.s3 = new client_s3_1.S3Client({ region: 'us-east-1' });
+        this.srcDir = srcDir;
+        this.bucketName = bucketName;
+        this.config = JSON.parse(fs_1.default.readFileSync(configFile).toString('ascii'));
+    }
+    static createFromArgs(args) {
+        if (args && args.length === 3) {
+            const src = args[0];
+            const bucket = args[1];
+            const configFile = args[2];
+            return new SiteUploader(src, bucket, configFile);
+        }
+        else {
+            console.log('Usage : node ratchet-site-uploader {srcDir} {bucket} {configFile} (Found ' + args + ' arguments, need 3)');
+            return null;
+        }
+    }
+    static async runFromCliArgs(args) {
+        const inst = SiteUploader.createFromArgs(args);
+        return inst.runPump();
+    }
+    findMatch(prefix, fileName, config) {
+        let found = null;
+        if (prefix != null && fileName != null && config != null && config.mapping != null) {
+            config.mapping.forEach((entry) => {
+                if (found == null) {
+                    if (entry.prefixMatch == null || prefix.match(entry.prefixMatch)) {
+                        if (entry.fileMatch == null || fileName.match(entry.fileMatch)) {
+                            found = entry;
+                        }
+                    }
+                }
+            });
+        }
+        return found;
+    }
+    findMime(fileName, config) {
+        let found = null;
+        if (config != null && config.customMimeTypeMapping != null) {
+            Object.keys(config.customMimeTypeMapping).forEach((k) => {
+                if (found == null && fileName.endsWith(k)) {
+                    found = config.customMimeTypeMapping[k];
+                }
+            });
+        }
+        if (found == null) {
+            found = mime_types_1.default.lookup(fileName);
+        }
+        if (found == null) {
+            found = 'binary/octet-stream';
+        }
+        return found;
+    }
+    runPump() {
+        return new Promise((resolve, reject) => {
+            ratchet_common_1.Logger.info('Uploading contents of %s to %s using %j as config', this.srcDir, this.bucketName, this.config);
+            const options = {};
+            const walker = walk_1.default.walk(this.srcDir, options);
+            walker.on('file', function (root, fileStats, next) {
+                ratchet_common_1.Logger.info('Processing %j', fileStats.name);
+                const prefix = root == this.srcDir ? '' : root.substring(this.srcDir.length + 1) + '/';
+                const proc = this.findMatch(prefix, fileStats.name, this.config);
+                const key = prefix + fileStats.name;
+                ratchet_common_1.Logger.info('Uploading file : %s/%s to key %s with %j', root, fileStats.name, key, proc);
+                const params = proc && proc.putParams ? JSON.parse(JSON.stringify(proc.putParams)) : {};
+                params.Bucket = this.bucketName;
+                params.Key = key;
+                params.Body = fs_1.default.readFileSync(path_1.default.join(root, fileStats.name));
+                if (!params.ContentType) {
+                    params.ContentType = this.findMime(fileStats.name, this.config);
+                }
+                const upload = new lib_storage_1.Upload({
+                    client: this.s3,
+                    params: params,
+                    tags: [],
+                    queueSize: 4,
+                    partSize: 1024 * 1024 * 5,
+                    leavePartsOnError: false,
+                });
+                upload.on('httpUploadProgress', (progress) => {
+                    ratchet_common_1.Logger.info('Uploading : %s', progress);
+                });
+                upload
+                    .done()
+                    .then((result) => {
+                    ratchet_common_1.Logger.info('Finished upload of %s: %j', key, result);
+                    next();
+                })
+                    .catch((err) => {
+                    ratchet_common_1.Logger.warn('%s failed to upload : %s : Continuing', key, err);
+                    next();
+                });
+            }.bind(this));
+            walker.on('errors', function (root, nodeStatsArray, next) {
+                next();
+            });
+            walker.on('end', function () {
+                ratchet_common_1.Logger.info('All done');
+                resolve(true);
+            });
+        });
+    }
+}
+exports.SiteUploader = SiteUploader;

package/dist/cjs/cli/start-instance-and-ssh.js

@@ -0,0 +1,67 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StartInstanceAndSsh = void 0;
+const tslib_1 = require("tslib");
+const ratchet_common_1 = require("@bitblit/ratchet-common");
+const ratchet_aws_1 = require("@bitblit/ratchet-aws");
+const child_process_1 = require("child_process");
+const fs_1 = tslib_1.__importDefault(require("fs"));
+const os_1 = tslib_1.__importDefault(require("os"));
+const path_1 = tslib_1.__importDefault(require("path"));
+class StartInstanceAndSsh {
+    constructor(instanceId, publicKeyFile = path_1.default.join(os_1.default.homedir(), '.ssh', 'id_rsa.pub'), instanceOsUser = 'ec2-user', region = 'us-east-1', availabilityZone = 'us-east-1a') {
+        this.instanceId = instanceId;
+        this.publicKeyFile = publicKeyFile;
+        this.instanceOsUser = instanceOsUser;
+        this.region = region;
+        this.availabilityZone = availabilityZone;
+        this.ec2Ratchet = new ratchet_aws_1.Ec2Ratchet(this.region, this.availabilityZone);
+    }
+    static createFromArgs(args) {
+        if ((args === null || args === void 0 ? void 0 : args.length) === 1 || (args === null || args === void 0 ? void 0 : args.length) === 2) {
+            const instanceId = args[0];
+            return new StartInstanceAndSsh(instanceId);
+        }
+        else {
+            ratchet_common_1.Logger.info('Usage : ratchet-start-instance-and-ssh {instanceId} {publicKeyFile} (Found %s arguments, need 1 or 2)', args);
+            return null;
+        }
+    }
+    static async runFromCliArgs(args) {
+        const inst = StartInstanceAndSsh.createFromArgs(args);
+        return inst.run();
+    }
+    async run() {
+        let instance = await this.ec2Ratchet.describeInstance(this.instanceId);
+        if (!!instance) {
+            let launched = false;
+            if (instance.State.Code == 16) {
+                ratchet_common_1.Logger.info('Instance is already running...');
+                launched = true;
+            }
+            else {
+                ratchet_common_1.Logger.info('Instance is not running... starting up : %s', this.instanceId);
+                launched = await this.ec2Ratchet.launchInstance(this.instanceId, 1000 * 30);
+            }
+            if (launched) {
+                ratchet_common_1.Logger.info('Uploading public key...');
+                const publicKeyText = fs_1.default.readFileSync(this.publicKeyFile).toString();
+                const publicKeyResponse = await this.ec2Ratchet.sendPublicKeyToEc2Instance(this.instanceId, publicKeyText, this.instanceOsUser);
+                ratchet_common_1.Logger.info('Key response : %j', publicKeyResponse);
+                instance = instance && instance.PublicIpAddress ? instance : await this.ec2Ratchet.describeInstance(this.instanceId);
+                ratchet_common_1.Logger.info('Instance IP address is %s', instance.PublicIpAddress);
+                const ret = (0, child_process_1.spawnSync)('ssh', [this.instanceOsUser + '@' + instance.PublicIpAddress], {
+                    stdio: 'inherit',
+                });
+                ratchet_common_1.Logger.info('%j', ret);
+            }
+            else {
+                ratchet_common_1.Logger.info('Instance could not start - check logs');
+            }
+        }
+        else {
+            ratchet_common_1.Logger.info('No such instance found - check your AWS keys? : %s', this.instanceId);
+        }
+    }
+}
+exports.StartInstanceAndSsh = StartInstanceAndSsh;

package/dist/cjs/index.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const tslib_1 = require("tslib");
+tslib_1.__exportStar(require("./athena/alb-athena-log-ratchet"), exports);
+tslib_1.__exportStar(require("./athena/athena-ratchet"), exports);
+tslib_1.__exportStar(require("./build/ratchet-aws-node-only-info"), exports);
+tslib_1.__exportStar(require("./cli/dynamo-exporter"), exports);
+tslib_1.__exportStar(require("./cli/ratchet-cli-handler"), exports);
+tslib_1.__exportStar(require("./cli/start-instance-and-ssh"), exports);
+tslib_1.__exportStar(require("./cli/site-uploader/site-uploader"), exports);
+tslib_1.__exportStar(require("./mail/inbound/inbound-email-ratchet"), exports);