@bitblit/ratchet-aws-node-only 4.0.419-alpha → 4.0.421-alpha

package/lib/athena/alb-athena-log-ratchet.d.ts

@@ -0,0 +1,56 @@
+import { ReadStream } from 'fs';
+import { AthenaRatchet } from './athena-ratchet.js';
+import { S3Client } from '@aws-sdk/client-s3';
+export declare class AlbAthenaLogRatchet {
+    private athena;
+    private athenaTableName;
+    constructor(athena: AthenaRatchet, athenaTableName: string);
+    updatePartitions(rootPath: string, s3: S3Client, startTimeEpochMS?: number, endTimeEpochMS?: number): Promise<string[]>;
+    createTable(rootPath: string, replaceIfPresent?: boolean): Promise<boolean>;
+    static readLogObjectsFromCsvStream(readStream: ReadStream): Promise<AlbLogRecord[]>;
+    static readLogObjectsFromFile(fileName: string): Promise<AlbLogRecord[]>;
+    fetchAlbLogRecords(qry: AlbLogRecordQuery): Promise<AlbLogRecord[]>;
+    fetchAlbLogRecordsToFile(qry: AlbLogRecordQuery, outputFileName?: string): Promise<string>;
+    static readonly CREATE_TABLE_STATEMENT: string;
+}
+export interface AlbLogRecordQuery {
+    startTimeEpochMS?: number;
+    endTimeEpochMS?: number;
+    requestUrlFilter?: string;
+    limit?: number;
+}
+export interface AlbLogRecord {
+    type: string;
+    time: string;
+    elb: string;
+    client_ip: string;
+    client_port: string;
+    target_ip: string;
+    target_port: string;
+    request_processing_time: string;
+    target_processing_time: string;
+    response_processing_time: string;
+    elb_status_code: string;
+    target_status_code: string;
+    received_bytes: string;
+    sent_bytes: string;
+    request_verb: string;
+    request_url: string;
+    request_proto: string;
+    user_agent: string;
+    ssl_cipher: string;
+    ssl_protocol: string;
+    target_group_arn: string;
+    trace_id: string;
+    domain_name: string;
+    chosen_cert_arn: string;
+    matched_rule_priority: string;
+    request_creation_time: string;
+    actions_executed: string;
+    redirect_url: string;
+    lambda_error_reason: string;
+    target_port_list: string;
+    target_status_code_list: string;
+    new_field: string;
+    date_utc_partition: string;
+}

package/lib/athena/alb-athena-log-ratchet.js

@@ -0,0 +1,156 @@
+import { readFileSync } from 'fs';
+import path from 'path';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
+import { StringRatchet } from '@bitblit/ratchet-common/lang/string-ratchet';
+import { EsmRatchet } from '@bitblit/ratchet-common/lang/esm-ratchet';
+import { CsvRatchet } from '@bitblit/ratchet-node-only/csv/csv-ratchet';
+import { S3Ratchet } from '@bitblit/ratchet-aws/s3/s3-ratchet';
+export class AlbAthenaLogRatchet {
+    athena;
+    athenaTableName;
+    constructor(athena, athenaTableName) {
+        this.athena = athena;
+        this.athenaTableName = athenaTableName;
+        RequireRatchet.notNullOrUndefined(athena, 'athena');
+        RequireRatchet.notNullOrUndefined(StringRatchet.trimToNull(athenaTableName), 'athenaTableName');
+    }
+    async updatePartitions(rootPath, s3, startTimeEpochMS = new Date().getTime() - 1000 * 60 * 60 * 24, endTimeEpochMS = new Date().getTime()) {
+        RequireRatchet.true(S3Ratchet.checkS3UrlForValidity(rootPath), 'root path not valid');
+        RequireRatchet.notNullOrUndefined(s3, 's3');
+        Logger.info('Updating partitions for %s from %s', this.athenaTableName, rootPath);
+        const bucketName = S3Ratchet.extractBucketFromURL(rootPath);
+        const rootKey = S3Ratchet.extractKeyFromURL(rootPath);
+        let current = startTimeEpochMS;
+        const clauses = [];
+        while (current < endTimeEpochMS) {
+            const dateUtcVal = new Date(current).toISOString().substring(0, 10);
+            Logger.info('d:%s', dateUtcVal);
+            const dateParts = dateUtcVal.split('-');
+            clauses.push("PARTITION (date_utc_partition='" +
+                dateUtcVal +
+                "') LOCATION '" +
+                rootPath +
+                '/' +
+                dateParts[0] +
+                '/' +
+                dateParts[1] +
+                '/' +
+                dateParts[2] +
+                "'");
+            current += 1000 * 60 * 60 * 24;
+        }
+        if (clauses.length > 0) {
+            const stmt = 'ALTER TABLE ' + this.athenaTableName + ' ADD IF NOT EXISTS \n' + clauses.join('\n');
+            await this.athena.runQueryToObjects(stmt);
+        }
+        else {
+            Logger.warn('Not updating partitions - no time between time clauses');
+        }
+        return clauses;
+    }
+    async createTable(rootPath, replaceIfPresent = false) {
+        RequireRatchet.true(S3Ratchet.checkS3UrlForValidity(rootPath), 'root path not valid');
+        let rval = false;
+        Logger.info('Creating ALB table %s', this.athenaTableName);
+        if (replaceIfPresent) {
+            Logger.info('Replace if present specified, removed old table');
+            try {
+                await this.athena.runQueryToObjects('drop table ' + this.athenaTableName);
+            }
+            catch (err) {
+                Logger.info('Drop error : %j', err);
+            }
+        }
+        let tableCreateQry = readFileSync(path.join(EsmRatchet.fetchDirName(import.meta.url), '../static/albAthenaTableCreate.txt')).toString();
+        tableCreateQry = StringRatchet.simpleTemplateFill(tableCreateQry, { TABLE_NAME: this.athenaTableName, ALB_LOG_ROOT: rootPath }, true);
+        Logger.info('Creating table with %s', tableCreateQry);
+        try {
+            await this.athena.runQueryToObjects(tableCreateQry);
+            rval = true;
+        }
+        catch (err) {
+            Logger.error('Error creating table : %s', err);
+        }
+        return rval;
+    }
+    static async readLogObjectsFromCsvStream(readStream) {
+        return CsvRatchet.streamParse(readStream, (p) => p);
+    }
+    static async readLogObjectsFromFile(fileName) {
+        return CsvRatchet.fileParse(fileName, (p) => p);
+    }
+    async fetchAlbLogRecords(qry) {
+        const tempFile = await this.fetchAlbLogRecordsToFile(qry);
+        return AlbAthenaLogRatchet.readLogObjectsFromFile(tempFile);
+    }
+    async fetchAlbLogRecordsToFile(qry, outputFileName = null) {
+        Logger.info('Querying %s : %j', this.athenaTableName, qry);
+        let qrySt = 'select * from ' + this.athenaTableName + ' where 1=1 ';
+        if (qry.startTimeEpochMS) {
+            if (qry.startTimeEpochMS) {
+                qrySt += " AND time >= '" + new Date(qry.startTimeEpochMS).toISOString() + "'";
+                qrySt += " AND date_utc_partition >='" + new Date(qry.startTimeEpochMS).toISOString().substring(0, 10) + "'";
+            }
+            if (qry.endTimeEpochMS) {
+                qrySt += " AND time < '" + new Date(qry.endTimeEpochMS).toISOString() + "'";
+                qrySt += " AND date_utc_partition <='" + new Date(qry.endTimeEpochMS).toISOString().substring(0, 10) + "'";
+            }
+            if (qry.requestUrlFilter) {
+                qrySt += " AND request_url LIKE '" + qry.requestUrlFilter + "'";
+            }
+            if (qry.limit) {
+                qrySt += ' LIMIT ' + qry.limit;
+            }
+        }
+        const result = await this.athena.runQueryToFile(qrySt, null, outputFileName);
+        return result;
+    }
+    static CREATE_TABLE_STATEMENT = 'CREATE EXTERNAL TABLE IF NOT EXISTS `{{TABLE NAME}}`(\n' +
+        "  `type` string COMMENT '',\n" +
+        "  `time` string COMMENT '',\n" +
+        "  `elb` string COMMENT '',\n" +
+        "  `client_ip` string COMMENT '',\n" +
+        "  `client_port` int COMMENT '',\n" +
+        "  `target_ip` string COMMENT '',\n" +
+        "  `target_port` int COMMENT '',\n" +
+        "  `request_processing_time` double COMMENT '',\n" +
+        "  `target_processing_time` double COMMENT '',\n" +
+        "  `response_processing_time` double COMMENT '',\n" +
+        "  `elb_status_code` string COMMENT '',\n" +
+        "  `target_status_code` string COMMENT '',\n" +
+        "  `received_bytes` bigint COMMENT '',\n" +
+        "  `sent_bytes` bigint COMMENT '',\n" +
+        "  `request_verb` string COMMENT '',\n" +
+        "  `request_url` string COMMENT '',\n" +
+        "  `request_proto` string COMMENT '',\n" +
+        "  `user_agent` string COMMENT '',\n" +
+        "  `ssl_cipher` string COMMENT '',\n" +
+        "  `ssl_protocol` string COMMENT '',\n" +
+        "  `target_group_arn` string COMMENT '',\n" +
+        "  `trace_id` string COMMENT '',\n" +
+        "  `domain_name` string COMMENT '',\n" +
+        "  `chosen_cert_arn` string COMMENT '',\n" +
+        "  `matched_rule_priority` string COMMENT '',\n" +
+        "  `request_creation_time` string COMMENT '',\n" +
+        "  `actions_executed` string COMMENT '',\n" +
+        "  `redirect_url` string COMMENT '',\n" +
+        "  `lambda_error_reason` string COMMENT '',\n" +
+        "  `target_port_list` string COMMENT '',\n" +
+        "  `target_status_code_list` string COMMENT '',\n" +
+        "  `new_field` string COMMENT '')\n" +
+        'PARTITIONED BY (\n' +
+        '  `date_utc_partition` string\n' +
+        ')\n' +
+        'ROW FORMAT SERDE\n' +
+        "  'org.apache.hadoop.hive.serde2.RegexSerDe'\n" +
+        'WITH SERDEPROPERTIES (\n' +
+        '  \'input.regex\'=\'([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \\"([^ ]*) ([^ ]*) (- |[^ ]*)\\" \\"([^\\"]*)\\" ([A-Z0-9-]+) ([A-Za-z0-9.-]*) ([^ ]*) \\"([^\\"]*)\\" \\"([^\\"]*)\\" \\"([^\\"]*)\\" ([-.0-9]*) ([^ ]*) \\"([^\\"]*)\\" \\"([^\\"]*)\\" \\"([^ ]*)\\" \\"([^s]+)\\" \\"([^s]+)\\"(.*)\')\n' +
+        'STORED AS INPUTFORMAT\n' +
+        "  'org.apache.hadoop.mapred.TextInputFormat'\n" +
+        'OUTPUTFORMAT\n' +
+        "  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\n" +
+        'LOCATION\n' +
+        "  '{{ALB_LOG_ROOT}}'\n";
+}
+//# sourceMappingURL=alb-athena-log-ratchet.js.map

package/lib/athena/alb-athena-log-ratchet.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"alb-athena-log-ratchet.js","sourceRoot":"","sources":["../../src/athena/alb-athena-log-ratchet.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAc,MAAM,IAAI,CAAC;AAC9C,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,EAAE,MAAM,EAAE,MAAM,uCAAuC,CAAC;AAC/D,OAAO,EAAE,cAAc,EAAE,MAAM,8CAA8C,CAAC;AAC9E,OAAO,EAAE,aAAa,EAAE,MAAM,6CAA6C,CAAC;AAC5E,OAAO,EAAE,UAAU,EAAE,MAAM,0CAA0C,CAAC;AACtE,OAAO,EAAE,UAAU,EAAE,MAAM,4CAA4C,CAAC;AAExE,OAAO,EAAE,SAAS,EAAE,MAAM,oCAAoC,CAAC;AAI/D,MAAM,OAAO,mBAAmB;IAEpB;IACA;IAFV,YACU,MAAqB,EACrB,eAAuB;QADvB,WAAM,GAAN,MAAM,CAAe;QACrB,oBAAe,GAAf,eAAe,CAAQ;QAE/B,cAAc,CAAC,kBAAkB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACpD,cAAc,CAAC,kBAAkB,CAAC,aAAa,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,iBAAiB,CAAC,CAAC;IAClG,CAAC;IAEM,KAAK,CAAC,gBAAgB,CAC3B,QAAgB,EAChB,EAAY,EACZ,mBAA2B,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EACrE,iBAAyB,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE;QAE7C,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,qBAAqB,CAAC,QAAQ,CAAC,EAAE,qBAAqB,CAAC,CAAC;QACtF,cAAc,CAAC,kBAAkB,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAC5C,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,IAAI,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC;QAClF,MAAM,UAAU,GAAW,SAAS,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC;QACpE,MAAM,OAAO,GAAW,SAAS,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAE9D,IAAI,OAAO,GAAW,gBAAgB,CAAC;QACvC,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,OAAO,OAAO,GAAG,cAAc,EAAE,CAAC;YAChC,MAAM,UAAU,GAAW,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC5E,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;YAChC,MAAM,SAAS,GAAa,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAClD,OAAO,CAAC,IAAI,CACV,iCAAiC;gBAC/B,UAAU;gBACV,eAAe;gBACf,QAAQ;gBACR,GAAG;gBACH,SAAS,CAAC,CAAC,CAAC;gBACZ,GAAG;gBACH,SAAS,CAAC,CAAC,CAAC;gBACZ,GAAG;gBACH,SAAS,CAAC,CAAC,CAAC;gBACZ,GAAG,CACN,CAAC;YACF,OAAO,IAAI,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;QACjC,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,GAAW,cAAc,GAAG,IAAI,CAAC,eAAe,GAAG,uBAAuB,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1G,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAM,IAAI,CAAC,CAAC;QACjD,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,wDAAwD,CAAC,CAAC;QACxE,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAEM,KAAK,CAAC,WAAW,CAAC,QAAgB,EAAE,mBAA4B,KAAK;QAC1E,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,qBAAqB,CAAC,QAAQ,CAAC,EAAE,qBAAqB,CAAC,CAAC;QACtF,IAAI,IAAI,GAAY,KAAK,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,uBAAuB,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;QAC3D,IAAI,gBAAgB,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;YAC/D,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAM,aAAa,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC;YACjF,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,CAAC,IAAI,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QAED,IAAI,cAAc,GAAW,YAAY,CACvC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,oCAAoC,CAAC,CAC1F,CAAC,QAAQ,EAAE,CAAC;QACb,cAAc,GAAG,aAAa,CAAC,kBAAkB,CAAC,cAAc,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,eAAe,EAAE,YAAY,EAAE,QAAQ,EAAE,EAAE,IAAI,CAAC,CAAC;QACtI,MAAM,CAAC,IAAI,CAAC,wBAAwB,EAAE,cAAc,CAAC,CAAC;QAEtD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAM,cAAc,CAAC,CAAC;YACzD,IAAI,GAAG,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,GAAG,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,2BAA2B,CAAC,UAAsB;QACpE,OAAO,UAAU,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IACtD,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,QAAgB;QACzD,OAAO,UAAU,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC;IAEM,KAAK,CAAC,kBAAkB,CAAC,GAAsB;QACpD,MAAM,QAAQ,GAAW,MAAM,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,CAAC;QAClE,OAAO,mBAAmB,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;IAC9D,CAAC;IAEM,KAAK,CAAC,wBAAwB,CAAC,GAAsB,EAAE,iBAAyB,IAAI;QACzF,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,IAAI,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;QAE3D,IAAI,KAAK,GAAW,gBAAgB,GAAG,IAAI,CAAC,eAAe,GAAG,aAAa,CAAC;QAC5E,IAAI,GAAG,CAAC,gBAAgB,EAAE,CAAC;YAEzB,IAAI,GAAG,CAAC,gBAAgB,EAAE,CAAC;gBACzB,KAAK,IAAI,gBAAgB,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,WAAW,EAAE,GAAG,GAAG,CAAC;gBAC/E,KAAK,IAAI,6BAA6B,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC;YAC/G,CAAC;YACD,IAAI,GAAG,CAAC,cAAc,EAAE,CAAC;gBACvB,KAAK,IAAI,eAAe,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,WAAW,EAAE,GAAG,GAAG,CAAC;gBAC5E,KAAK,IAAI,6BAA6B,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC;YAC7G,CAAC;YACD,IAAI,GAAG,CAAC,gBAAgB,EAAE,CAAC;gBACzB,KAAK,IAAI,yBAAyB,GAAG,GAAG,CAAC,gBAAgB,GAAG,GAAG,CAAC;YAClE,CAAC;YACD,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;gBACd,KAAK,IAAI,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC;YACjC,CAAC;QACH,CAAC;QAED,MAAM,MAAM,GAAW,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;QACrF,OAAO,MAAM,CAAC;IAChB,CAAC;IAEM,MAAM,CAAU,sBAAsB,GAC3C,yDAAyD;QACzD,+BAA+B;QAC/B,+BAA+B;QAC/B,8BAA8B;QAC9B,oCAAoC;QACpC,mCAAmC;QACnC,oCAAoC;QACpC,mCAAmC;QACnC,kDAAkD;QAClD,iDAAiD;QACjD,mDAAmD;QACnD,0CAA0C;QAC1C,6CAA6C;QAC7C,yCAAyC;QACzC,qCAAqC;QACrC,uCAAuC;QACvC,sCAAsC;QACtC,wCAAwC;QACxC,qCAAqC;QACrC,qCAAqC;QACrC,uCAAuC;QACvC,2CAA2C;QAC3C,mCAAmC;QACnC,sCAAsC;QACtC,0CAA0C;QAC1C,gDAAgD;QAChD,gDAAgD;QAChD,2CAA2C;QAC3C,uCAAuC;QACvC,8CAA8C;QAC9C,2CAA2C;QAC3C,kDAAkD;QAClD,oCAAoC;QACpC,oBAAoB;QACpB,iCAAiC;QACjC,KAAK;QACL,oBAAoB;QACpB,gDAAgD;QAChD,0BAA0B;QAC1B,2YAA2Y;QAC3Y,yBAAyB;QACzB,gDAAgD;QAChD,gBAAgB;QAChB,kEAAkE;QAClE,YAAY;QACZ,wBAAwB,CAAC"}

package/lib/athena/athena-ratchet.d.ts

@@ -0,0 +1,15 @@
+import { AthenaClient, NamedQuery, Row } from '@aws-sdk/client-athena';
+import { S3Client } from '@aws-sdk/client-s3';
+export declare class AthenaRatchet {
+    private athena;
+    private s3;
+    private outputLocation;
+    constructor(athena: AthenaClient, s3: S3Client, outputLocation: string);
+    static athenaRowsToObject<T>(input: Row[]): T[];
+    fetchQueryIds(): Promise<string[]>;
+    listQueries(): Promise<NamedQuery[]>;
+    findQueryByName(name: string): Promise<NamedQuery>;
+    runQueryToObjects<T>(queryIn: string, queryParams?: any, pingTimeMS?: number): Promise<T[]>;
+    runQueryToFile(queryIn: string, queryParams?: any, targetDataFileIn?: string, pingTimeMS?: number): Promise<string>;
+    private runQueryToOutputLocation;
+}

package/lib/athena/athena-ratchet.js

@@ -0,0 +1,148 @@
+import { GetNamedQueryCommand, GetQueryExecutionCommand, ListNamedQueriesCommand, StartQueryExecutionCommand, } from '@aws-sdk/client-athena';
+import { GetObjectCommand } from '@aws-sdk/client-s3';
+import tmp from 'tmp';
+import fs from 'fs';
+import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { CsvRatchet } from '@bitblit/ratchet-node-only/csv/csv-ratchet';
+import { PromiseRatchet } from '@bitblit/ratchet-common/lang/promise-ratchet';
+import { StopWatch } from '@bitblit/ratchet-common/lang/stop-watch';
+import { StringRatchet } from '@bitblit/ratchet-common/lang/string-ratchet';
+export class AthenaRatchet {
+    athena;
+    s3;
+    outputLocation;
+    constructor(athena, s3, outputLocation) {
+        this.athena = athena;
+        this.s3 = s3;
+        this.outputLocation = outputLocation;
+        RequireRatchet.notNullOrUndefined(athena);
+        RequireRatchet.notNullOrUndefined(s3);
+        RequireRatchet.notNullOrUndefined(outputLocation);
+        RequireRatchet.true(outputLocation.startsWith('s3://'));
+    }
+    static athenaRowsToObject(input) {
+        const colNames = input[0].Data.map((d) => d.VarCharValue);
+        const temp = input.slice(1);
+        const rval = temp.map((t) => {
+            const newItem = {};
+            for (let i = 0; i < t.Data.length; i++) {
+                newItem[colNames[i]] = t.Data[i].VarCharValue;
+            }
+            return newItem;
+        });
+        return rval;
+    }
+    async fetchQueryIds() {
+        const params = {
+            NextToken: null,
+        };
+        let rval = [];
+        let next = null;
+        do {
+            next = await this.athena.send(new ListNamedQueriesCommand(params));
+            rval = rval.concat(next.NamedQueryIds);
+            params.NextToken = next.NextToken;
+        } while (params.NextToken);
+        return rval;
+    }
+    async listQueries() {
+        const rval = [];
+        const ids = await this.fetchQueryIds();
+        Logger.debug('Finding %d items', ids.length);
+        for (let i = 0; i < ids.length; i++) {
+            const params = {
+                NamedQueryId: ids[i],
+            };
+            const val = await this.athena.send(new GetNamedQueryCommand(params));
+            rval.push(val.NamedQuery);
+        }
+        return rval;
+    }
+    async findQueryByName(name) {
+        const all = await this.listQueries();
+        const rval = all.find((a) => a.Name.toLowerCase() == name.toLowerCase());
+        return rval;
+    }
+    async runQueryToObjects(queryIn, queryParams = {}, pingTimeMS = 2000) {
+        Logger.info('Running query to objects');
+        const outputLoc = await this.runQueryToOutputLocation(queryIn, queryParams, pingTimeMS);
+        Logger.info('Query succeeded, processing file from %s', outputLoc);
+        const bucketName = outputLoc.substring(5, outputLoc.indexOf('/', 5));
+        const obKey = outputLoc.substring(outputLoc.indexOf('/', 5) + 1);
+        const req = {
+            Bucket: bucketName,
+            Key: obKey,
+        };
+        const getFileOut = await this.s3.send(new GetObjectCommand(req));
+        const bodyAsString = await getFileOut.Body.transformToString();
+        const rval = await CsvRatchet.stringParse(bodyAsString, (p) => {
+            return p;
+        }, { columns: true, skip_empty_lines: true });
+        return rval;
+    }
+    async runQueryToFile(queryIn, queryParams = {}, targetDataFileIn = null, pingTimeMS = 2000) {
+        Logger.info('Running query to file');
+        const outputLoc = await this.runQueryToOutputLocation(queryIn, queryParams, pingTimeMS);
+        Logger.info('Query succeeded, pulling file from %s', outputLoc);
+        const bucketName = outputLoc.substring(5, outputLoc.indexOf('/', 5));
+        const obKey = outputLoc.substring(outputLoc.indexOf('/', 5) + 1);
+        const req = {
+            Bucket: bucketName,
+            Key: obKey,
+        };
+        const targetDataFile = targetDataFileIn || tmp.fileSync({ postfix: '.csv', keep: false }).name;
+        const fileStream = fs.createWriteStream(targetDataFile);
+        const output = await this.s3.send(new GetObjectCommand(req));
+        const readStream = output.Body;
+        readStream.pipe(fileStream);
+        const rval = await PromiseRatchet.resolveOnEvent(readStream, ['finish', 'close'], ['error'], targetDataFile);
+        Logger.silly('Response: %s', rval);
+        return targetDataFile;
+    }
+    async runQueryToOutputLocation(queryIn, queryParams = {}, pingTimeMS = 2000) {
+        let rval = null;
+        const timer = new StopWatch();
+        const query = StringRatchet.simpleTemplateFill(queryIn, queryParams, true, '{', '}');
+        try {
+            Logger.info('Starting query : %s', query);
+            const token = StringRatchet.createType4Guid();
+            const params = {
+                QueryString: query,
+                ResultConfiguration: {
+                    OutputLocation: this.outputLocation,
+                    EncryptionConfiguration: {
+                        EncryptionOption: 'SSE_S3',
+                    },
+                },
+                ClientRequestToken: token,
+                QueryExecutionContext: {
+                    Database: 'default',
+                },
+            };
+            const startToken = await this.athena.send(new StartQueryExecutionCommand(params));
+            const getExecParams = {
+                QueryExecutionId: startToken.QueryExecutionId,
+            };
+            const finalStates = ['FAILED', 'CANCELLED', 'SUCCEEDED'];
+            let curState = await this.athena.send(new GetQueryExecutionCommand(getExecParams));
+            while (finalStates.indexOf(curState.QueryExecution.Status.State) === -1) {
+                await PromiseRatchet.createTimeoutPromise('wait', pingTimeMS);
+                Logger.debug('%s : %s : %s', curState.QueryExecution.Status.State, timer.dump(), query);
+                curState = await this.athena.send(new GetQueryExecutionCommand(getExecParams));
+            }
+            if (curState.QueryExecution.Status.State === 'FAILED') {
+                Logger.warn('Query failed : %s', curState.QueryExecution.Status.StateChangeReason);
+            }
+            else if (curState.QueryExecution.Status.State === 'SUCCEEDED') {
+                rval = curState.QueryExecution.ResultConfiguration.OutputLocation;
+            }
+        }
+        catch (err) {
+            Logger.warn('Failure : %s', err, err);
+        }
+        Logger.info('Query took %s : %s', timer.dump(), query);
+        return rval;
+    }
+}
+//# sourceMappingURL=athena-ratchet.js.map

package/lib/athena/athena-ratchet.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"athena-ratchet.js","sourceRoot":"","sources":["../../src/athena/athena-ratchet.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,oBAAoB,EAEpB,wBAAwB,EAExB,uBAAuB,EAIvB,0BAA0B,GAG3B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,gBAAgB,EAAsD,MAAM,oBAAoB,CAAC;AAC1G,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,EAAmB,MAAM,IAAI,CAAC;AAErC,OAAO,EAAE,cAAc,EAAE,MAAM,8CAA8C,CAAC;AAC9E,OAAO,EAAE,MAAM,EAAE,MAAM,uCAAuC,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,4CAA4C,CAAC;AACxE,OAAO,EAAE,cAAc,EAAE,MAAM,8CAA8C,CAAC;AAC9E,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,6CAA6C,CAAC;AAE5E,MAAM,OAAO,aAAa;IAEd;IACA;IACA;IAHV,YACU,MAAoB,EACpB,EAAY,EACZ,cAAsB;QAFtB,WAAM,GAAN,MAAM,CAAc;QACpB,OAAE,GAAF,EAAE,CAAU;QACZ,mBAAc,GAAd,cAAc,CAAQ;QAE9B,cAAc,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAC1C,cAAc,CAAC,kBAAkB,CAAC,EAAE,CAAC,CAAC;QACtC,cAAc,CAAC,kBAAkB,CAAC,cAAc,CAAC,CAAC;QAClD,cAAc,CAAC,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1D,CAAC;IAEM,MAAM,CAAC,kBAAkB,CAAI,KAAY;QAC9C,MAAM,QAAQ,GAAa,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;QACpE,MAAM,IAAI,GAAU,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACnC,MAAM,IAAI,GAAQ,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YAC/B,MAAM,OAAO,GAAM,EAAO,CAAC;YAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC;YAChD,CAAC;YACD,OAAO,OAAO,CAAC;QACjB,CAAC,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,KAAK,CAAC,aAAa;QACxB,MAAM,MAAM,GAAG;YAEb,SAAS,EAAE,IAAI;SAChB,CAAC;QAEF,IAAI,IAAI,GAAa,EAAE,CAAC;QACxB,IAAI,IAAI,GAAkC,IAAI,CAAC;QAE/C,GAAG,CAAC;YACF,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,uBAAuB,CAAC,MAAM,CAAC,CAAC,CAAC;YACnE,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YACvC,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QACpC,CAAC,QAAQ,MAAM,CAAC,SAAS,EAAE;QAE3B,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,KAAK,CAAC,WAAW;QACtB,MAAM,IAAI,GAAiB,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAa,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QACjD,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,MAAM,MAAM,GAAG;gBACb,YAAY,EAAE,GAAG,CAAC,CAAC,CAAC;aACrB,CAAC;YACF,MAAM,GAAG,GAA+B,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC;YACjG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC5B,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,IAAY;QACvC,MAAM,GAAG,GAAiB,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACnD,MAAM,IAAI,GAAe,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QACrF,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,KAAK,CAAC,iBAAiB,CAAI,OAAe,EAAE,cAAmB,EAAE,EAAE,UAAU,GAAG,IAAI;QACzF,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,OAAO,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;QACxF,MAAM,CAAC,IAAI,CAAC,0CAA0C,EAAE,SAAS,CAAC,CAAC;QAEnE,MAAM,UAAU,GAAW,SAAS,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7E,MAAM,KAAK,GAAW,SAAS,CAAC,SAAS,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAEzE,MAAM,GAAG,GAAqB;YAC5B,MAAM,EAAE,UAAU;YAClB,GAAG,EAAE,KAAK;SACX,CAAC;QACF,MAAM,UAAU,GAA2B,MAAM,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC;QAEzF,MAAM,YAAY,GAAW,MAAM,UAAU,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAEvE,MAAM,IAAI,GAAQ,MAAM,UAAU,CAAC,WAAW,CAC5C,YAAY,EACZ,CAAC,CAAC,EAAE,EAAE;YACJ,OAAO,CAAC,CAAC;QACX,CAAC,EACD,EAAE,OAAO,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAC1C,CAAC;QAEF,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,KAAK,CAAC,cAAc,CAAC,OAAe,EAAE,cAAmB,EAAE,EAAE,mBAA2B,IAAI,EAAE,UAAU,GAAG,IAAI;QACpH,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACrC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,OAAO,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;QACxF,MAAM,CAAC,IAAI,CAAC,uCAAuC,EAAE,SAAS,CAAC,CAAC;QAEhE,MAAM,UAAU,GAAW,SAAS,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7E,MAAM,KAAK,GAAW,SAAS,CAAC,SAAS,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAEzE,MAAM,GAAG,GAAqB;YAC5B,MAAM,EAAE,UAAU;YAClB,GAAG,EAAE,KAAK;SACX,CAAC;QAEF,MAAM,cAAc,GAAW,gBAAgB,IAAI,GAAG,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC;QACvG,MAAM,UAAU,GAAgB,EAAE,CAAC,iBAAiB,CAAC,cAAc,CAAC,CAAC;QACrE,MAAM,MAAM,GAA2B,MAAM,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC;QAErF,MAAM,UAAU,GAAa,MAAM,CAAC,IAAgB,CAAC;QACrD,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAE5B,MAAM,IAAI,GAAW,MAAM,cAAc,CAAC,cAAc,CAAS,UAAU,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,cAAc,CAAC,CAAC;QAC7H,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;QACnC,OAAO,cAAc,CAAC;IACxB,CAAC;IAEO,KAAK,CAAC,wBAAwB,CAAC,OAAe,EAAE,cAAmB,EAAE,EAAE,UAAU,GAAG,IAAI;QAC9F,IAAI,IAAI,GAAW,IAAI,CAAC;QACxB,MAAM,KAAK,GAAc,IAAI,SAAS,EAAE,CAAC;QACzC,MAAM,KAAK,GAAW,aAAa,CAAC,kBAAkB,CAAC,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;QAE7F,IAAI,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,qBAAqB,EAAE,KAAK,CAAC,CAAC;YAE1C,MAAM,KAAK,GAAW,aAAa,CAAC,eAAe,EAAE,CAAC;YACtD,MAAM,MAAM,GAA6B;gBACvC,WAAW,EAAE,KAAK;gBAClB,mBAAmB,EAAE;oBAEnB,cAAc,EAAE,IAAI,CAAC,cAAc;oBACnC,uBAAuB,EAAE;wBACvB,gBAAgB,EAAE,QAAQ;qBAE3B;iBACF;gBACD,kBAAkB,EAAE,KAAK;gBACzB,qBAAqB,EAAE;oBACrB,QAAQ,EAAE,SAAS;iBACpB;aACF,CAAC;YAEF,MAAM,UAAU,GAAqC,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,0BAA0B,CAAC,MAAM,CAAC,CAAC,CAAC;YAEpH,MAAM,aAAa,GAAQ;gBACzB,gBAAgB,EAAE,UAAU,CAAC,gBAAgB;aAC9C,CAAC;YAEF,MAAM,WAAW,GAAa,CAAC,QAAQ,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC;YACnE,IAAI,QAAQ,GAAmC,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,wBAAwB,CAAC,aAAa,CAAC,CAAC,CAAC;YACnH,OAAO,WAAW,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gBACxE,MAAM,cAAc,CAAC,oBAAoB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;gBAC9D,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,QAAQ,CAAC,cAAc,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC,CAAC;gBACxF,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,wBAAwB,CAAC,aAAa,CAAC,CAAC,CAAC;YACjF,CAAC;YAED,IAAI,QAAQ,CAAC,cAAc,CAAC,MAAM,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;gBACtD,MAAM,CAAC,IAAI,CAAC,mBAAmB,EAAE,QAAQ,CAAC,cAAc,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;YACrF,CAAC;iBAAM,IAAI,QAAQ,CAAC,cAAc,CAAC,MAAM,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;gBAChE,IAAI,GAAG,QAAQ,CAAC,cAAc,CAAC,mBAAmB,CAAC,cAAc,CAAC;YACpE,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,cAAc,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;QACxC,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,oBAAoB,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC,CAAC;QAEvD,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}

package/lib/athena/index.d.ts

@@ -0,0 +1,2 @@
+export * from './alb-athena-log-ratchet.js';
+export * from './athena-ratchet.js';

package/lib/athena/index.js

@@ -0,0 +1,3 @@
+export * from './alb-athena-log-ratchet.js';
+export * from './athena-ratchet.js';
+//# sourceMappingURL=index.js.map

package/lib/athena/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/athena/index.ts"],"names":[],"mappings":"AAIA,cAAc,6BAA6B,CAAC;AAC5C,cAAc,qBAAqB,CAAC"}

package/lib/build/index.d.ts

@@ -0,0 +1 @@
+export * from './ratchet-aws-node-only-info.js';

package/lib/build/index.js

@@ -0,0 +1,2 @@
+export * from './ratchet-aws-node-only-info.js';
+//# sourceMappingURL=index.js.map

package/lib/build/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/build/index.ts"],"names":[],"mappings":"AAIA,cAAc,iCAAiC,CAAC"}

package/lib/build/ratchet-aws-node-only-info.d.ts

@@ -0,0 +1,5 @@
+import { BuildInformation } from '@bitblit/ratchet-common/build/build-information';
+export declare class RatchetAwsNodeOnlyInfo {
+    private constructor();
+    static buildInformation(): BuildInformation;
+}

package/lib/build/ratchet-aws-node-only-info.js

@@ -0,0 +1,15 @@
+export class RatchetAwsNodeOnlyInfo {
+    constructor() { }
+    static buildInformation() {
+        const val = {
+            version: 'LOCAL-SNAPSHOT',
+            hash: 'LOCAL-HASH',
+            branch: 'LOCAL-BRANCH',
+            tag: 'LOCAL-TAG',
+            timeBuiltISO: 'LOCAL-TIME-ISO',
+            notes: 'LOCAL-NOTES',
+        };
+        return val;
+    }
+}
+//# sourceMappingURL=ratchet-aws-node-only-info.js.map

package/lib/build/ratchet-aws-node-only-info.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ratchet-aws-node-only-info.js","sourceRoot":"","sources":["../../src/build/ratchet-aws-node-only-info.ts"],"names":[],"mappings":"AAEA,MAAM,OAAO,sBAAsB;IAGjC,gBAAuB,CAAC;IAEjB,MAAM,CAAC,gBAAgB;QAC5B,MAAM,GAAG,GAAqB;YAC5B,OAAO,EAAE,gBAAgB;YACzB,IAAI,EAAE,YAAY;YAClB,MAAM,EAAE,cAAc;YACtB,GAAG,EAAE,WAAW;YAChB,YAAY,EAAE,gBAAgB;YAC9B,KAAK,EAAE,aAAa;SACrB,CAAC;QACF,OAAO,GAAG,CAAC;IACb,CAAC;CACF"}

package/lib/cli/dynamo-exporter.d.ts

@@ -0,0 +1,12 @@
+import { WriteStream } from 'fs';
+import { DynamoRatchet } from '@bitblit/ratchet-aws/dynamodb/dynamo-ratchet';
+import { QueryCommandInput, ScanCommandInput } from '@aws-sdk/lib-dynamodb';
+export declare class DynamoExporter {
+    private constructor();
+    static importJsonLFileToTable(dynamo: DynamoRatchet, tableName: string, filename: string): Promise<number>;
+    static exportScanToJsonLFile(dynamo: DynamoRatchet, scan: ScanCommandInput, filename: string): Promise<number>;
+    static exportQueryToJsonLFile(dynamo: DynamoRatchet, qry: QueryCommandInput, filename: string): Promise<number>;
+    static exportScanToJsonLWriteStream(dynamo: DynamoRatchet, scan: ScanCommandInput, target: WriteStream): Promise<number>;
+    static exportQueryToJsonLWriteStream(dynamo: DynamoRatchet, qry: QueryCommandInput, target: WriteStream): Promise<number>;
+    static writeItemToJsonLStream(item: any, target: WriteStream, includeNulls?: boolean): void;
+}

package/lib/cli/dynamo-exporter.js

@@ -0,0 +1,77 @@
+import { StringRatchet } from '@bitblit/ratchet-common/lang/string-ratchet';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { PromiseRatchet } from '@bitblit/ratchet-common/lang/promise-ratchet';
+import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
+import fs from 'fs';
+import readline from 'readline';
+export class DynamoExporter {
+    constructor() { }
+    static async importJsonLFileToTable(dynamo, tableName, filename) {
+        RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');
+        RequireRatchet.notNullOrUndefined(tableName, 'tableName');
+        RequireRatchet.notNullOrUndefined(filename, 'filename');
+        const fileStream = fs.createReadStream(filename);
+        const rl = readline.createInterface({
+            input: fileStream,
+            crlfDelay: Infinity,
+        });
+        let rval = 0;
+        for await (const line of rl) {
+            if (rval % 100 === 0) {
+                Logger.info('Importing line %d', rval);
+            }
+            if (StringRatchet.trimToNull(line)) {
+                const parsed = JSON.parse(line);
+                await dynamo.simplePut(tableName, parsed);
+                rval++;
+            }
+        }
+        return rval;
+    }
+    static async exportScanToJsonLFile(dynamo, scan, filename) {
+        RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');
+        RequireRatchet.notNullOrUndefined(scan, 'scan');
+        RequireRatchet.notNullOrUndefined(filename, 'filename');
+        const ws = fs.createWriteStream(filename);
+        ws.on('end', () => {
+            Logger.debug('Write complete');
+        });
+        const rval = await DynamoExporter.exportScanToJsonLWriteStream(dynamo, scan, ws);
+        await PromiseRatchet.resolveOnEvent(ws, ['finish', 'close'], ['error']);
+        ws.close();
+        return rval;
+    }
+    static async exportQueryToJsonLFile(dynamo, qry, filename) {
+        RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');
+        RequireRatchet.notNullOrUndefined(qry, 'qry');
+        RequireRatchet.notNullOrUndefined(filename, 'filename');
+        const ws = fs.createWriteStream(filename);
+        ws.on('end', () => {
+            Logger.debug('Write complete');
+        });
+        const rval = await DynamoExporter.exportQueryToJsonLWriteStream(dynamo, qry, ws);
+        await PromiseRatchet.resolveOnEvent(ws, ['finish', 'close'], ['error']);
+        ws.close();
+        return rval;
+    }
+    static async exportScanToJsonLWriteStream(dynamo, scan, target) {
+        RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');
+        RequireRatchet.notNullOrUndefined(scan, 'scan');
+        RequireRatchet.notNullOrUndefined(target, 'target');
+        const rval = await dynamo.fullyExecuteProcessOverScan(scan, async (row) => DynamoExporter.writeItemToJsonLStream(row, target, false));
+        return rval;
+    }
+    static async exportQueryToJsonLWriteStream(dynamo, qry, target) {
+        RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');
+        RequireRatchet.notNullOrUndefined(qry, 'qry');
+        RequireRatchet.notNullOrUndefined(target, 'target');
+        const rval = await dynamo.fullyExecuteProcessOverQuery(qry, async (row) => DynamoExporter.writeItemToJsonLStream(row, target, false));
+        return rval;
+    }
+    static writeItemToJsonLStream(item, target, includeNulls = false) {
+        if (!!item || includeNulls) {
+            target.write(JSON.stringify(item) + '\n');
+        }
+    }
+}
+//# sourceMappingURL=dynamo-exporter.js.map

package/lib/cli/dynamo-exporter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dynamo-exporter.js","sourceRoot":"","sources":["../../src/cli/dynamo-exporter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,6CAA6C,CAAC;AAC5E,OAAO,EAAE,MAAM,EAAE,MAAM,uCAAuC,CAAC;AAC/D,OAAO,EAAE,cAAc,EAAE,MAAM,8CAA8C,CAAC;AAC9E,OAAO,EAAE,cAAc,EAAE,MAAM,8CAA8C,CAAC;AAC9E,OAAO,EAAmB,MAAM,IAAI,CAAC;AAGrC,OAAO,QAAQ,MAAM,UAAU,CAAC;AAGhC,MAAM,OAAO,cAAc;IAGzB,gBAAuB,CAAC;IAEjB,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,MAAqB,EAAE,SAAiB,EAAE,QAAgB;QACnG,cAAc,CAAC,kBAAkB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACpD,cAAc,CAAC,kBAAkB,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;QAC1D,cAAc,CAAC,kBAAkB,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAExD,MAAM,UAAU,GAAG,EAAE,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAEjD,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;YAClC,KAAK,EAAE,UAAU;YACjB,SAAS,EAAE,QAAQ;SACpB,CAAC,CAAC;QAIH,IAAI,IAAI,GAAW,CAAC,CAAC;QACrB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;YAC5B,IAAI,IAAI,GAAG,GAAG,KAAK,CAAC,EAAE,CAAC;gBACrB,MAAM,CAAC,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC;YACzC,CAAC;YACD,IAAI,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnC,MAAM,MAAM,GAAQ,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACrC,MAAM,MAAM,CAAC,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;gBAC1C,IAAI,EAAE,CAAC;YACT,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC,MAAqB,EAAE,IAAsB,EAAE,QAAgB;QACvG,cAAc,CAAC,kBAAkB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACpD,cAAc,CAAC,kBAAkB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAChD,cAAc,CAAC,kBAAkB,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACxD,MAAM,EAAE,GAAgB,EAAE,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QACvD,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YAChB,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;QAEH,MAAM,IAAI,GAAW,MAAM,cAAc,CAAC,4BAA4B,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;QAEzF,MAAM,cAAc,CAAC,cAAc,CAAC,EAAE,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;QACxE,EAAE,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,MAAqB,EAAE,GAAsB,EAAE,QAAgB;QACxG,cAAc,CAAC,kBAAkB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACpD,cAAc,CAAC,kBAAkB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC9C,cAAc,CAAC,kBAAkB,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAExD,MAAM,EAAE,GAAgB,EAAE,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QACvD,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YAChB,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;QAEH,MAAM,IAAI,GAAW,MAAM,cAAc,CAAC,6BAA6B,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;QAEzF,MAAM,cAAc,CAAC,cAAc,CAAC,EAAE,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;QACxE,EAAE,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,4BAA4B,CAAC,MAAqB,EAAE,IAAsB,EAAE,MAAmB;QACjH,cAAc,CAAC,kBAAkB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACpD,cAAc,CAAC,kBAAkB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAChD,cAAc,CAAC,kBAAkB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAEpD,MAAM,IAAI,GAAW,MAAM,MAAM,CAAC,2BAA2B,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAChF,cAAc,CAAC,sBAAsB,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,CAC1D,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,MAAqB,EAAE,GAAsB,EAAE,MAAmB;QAClH,cAAc,CAAC,kBAAkB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACpD,cAAc,CAAC,kBAAkB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC9C,cAAc,CAAC,kBAAkB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAEpD,MAAM,IAAI,GAAW,MAAM,MAAM,CAAC,4BAA4B,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAChF,cAAc,CAAC,sBAAsB,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,CAC1D,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;IACM,MAAM,CAAC,sBAAsB,CAAC,IAAS,EAAE,MAAmB,EAAE,eAAwB,KAAK;QAChG,IAAI,CAAC,CAAC,IAAI,IAAI,YAAY,EAAE,CAAC;YAC3B,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;CACF"}

package/lib/cli/index.d.ts

@@ -0,0 +1,4 @@
+export * from './dynamo-exporter.js';
+export * from './ratchet-cli-handler.js';
+export * from './start-instance-and-ssh.js';
+export * from './site-uploader/site-uploader.js';

package/lib/cli/index.js

@@ -0,0 +1,5 @@
+export * from './dynamo-exporter.js';
+export * from './ratchet-cli-handler.js';
+export * from './start-instance-and-ssh.js';
+export * from './site-uploader/site-uploader.js';
+//# sourceMappingURL=index.js.map

package/lib/cli/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/cli/index.ts"],"names":[],"mappings":"AAIA,cAAc,sBAAsB,CAAC;AACrC,cAAc,0BAA0B,CAAC;AACzC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,kCAAkC,CAAC"}

package/lib/cli/ratchet-cli-handler.d.ts

@@ -0,0 +1,6 @@
+import { AbstractRatchetCliHandler } from '@bitblit/ratchet-node-only/cli/abstract-ratchet-cli-handler';
+import { BuildInformation } from '@bitblit/ratchet-common/build/build-information';
+export declare class RatchetCliHandler extends AbstractRatchetCliHandler {
+    fetchHandlerMap(): Record<string, any>;
+    fetchVersionInfo(): BuildInformation;
+}

package/lib/cli/ratchet-cli-handler.js

@@ -0,0 +1,16 @@
+import { AbstractRatchetCliHandler } from '@bitblit/ratchet-node-only/cli/abstract-ratchet-cli-handler';
+import { SiteUploader } from './site-uploader/site-uploader.js';
+import { StartInstanceAndSsh } from './start-instance-and-ssh.js';
+import { RatchetAwsNodeOnlyInfo } from '../build/ratchet-aws-node-only-info.js';
+export class RatchetCliHandler extends AbstractRatchetCliHandler {
+    fetchHandlerMap() {
+        return {
+            'site-uploader': SiteUploader.runFromCliArgs,
+            'start-instance-and-ssh': StartInstanceAndSsh.runFromCliArgs,
+        };
+    }
+    fetchVersionInfo() {
+        return RatchetAwsNodeOnlyInfo.buildInformation();
+    }
+}
+//# sourceMappingURL=ratchet-cli-handler.js.map

package/lib/cli/ratchet-cli-handler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ratchet-cli-handler.js","sourceRoot":"","sources":["../../src/cli/ratchet-cli-handler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,6DAA6D,CAAC;AACxG,OAAO,EAAE,YAAY,EAAE,MAAM,kCAAkC,CAAC;AAChE,OAAO,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AAElE,OAAO,EAAE,sBAAsB,EAAE,MAAM,wCAAwC,CAAC;AAEhF,MAAM,OAAO,iBAAkB,SAAQ,yBAAyB;IAC9D,eAAe;QACb,OAAO;YACL,eAAe,EAAE,YAAY,CAAC,cAAc;YAC5C,wBAAwB,EAAE,mBAAmB,CAAC,cAAc;SAC7D,CAAC;IACJ,CAAC;IAED,gBAAgB;QACd,OAAO,sBAAsB,CAAC,gBAAgB,EAAE,CAAC;IACnD,CAAC;CACF"}

package/lib/cli/site-uploader/site-uploader.d.ts

@@ -0,0 +1,12 @@
+export declare class SiteUploader {
+    private srcDir;
+    private bucketName;
+    private config;
+    private readonly s3;
+    constructor(srcDir: string, bucketName: string, configFile: string);
+    static createFromArgs(args: string[]): SiteUploader;
+    static runFromCliArgs(args: string[]): Promise<void>;
+    findMatch(prefix: string, fileName: string, config: any): any;
+    findMime(fileName: string, config: any): string;
+    runPump(): Promise<any>;
+}