@bitblit/ratchet-aws-node-only 4.0.116-alpha → 4.0.119-alpha

package/lib/mail/inbound/inbound-email-ratchet.d.ts

@@ -1,4 +1,4 @@
-/// <reference types="node" resolution-mode="require"/>
+/// <reference types="node" />
 import { S3CacheRatchet } from '@bitblit/ratchet-aws';
 import { ParsedEmailProcessor } from './parsed-email-processor.js';
 export declare class InboundEmailRatchet {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bitblit/ratchet-aws-node-only",
-  "version": "4.0.116-alpha",
+  "version": "4.0.119-alpha",
   "description": "Common tools for use with AWS (Node only)",
   "note-on-side-effects": "Technically the entries in 'bin' below might be side effects, but they are called explicitly",
   "sideEffects": false,
@@ -8,11 +8,16 @@
     "ratchet-aws-cli": "./bin/cli.js"
   },
   "type": "module",
-  "module": "index.js",
   "files": [
     "lib/*",
     "bin/*"
   ],
+  "exports": {
+    ".": {
+      "types": "./lib/index.d.ts",
+      "import": "./lib/index.mjs"
+    }
+  },
   "contributors": [
     "Christopher Weiss <bitblit@gmail.com>",
     "William Weiss <npm@codification.org>",
@@ -39,9 +44,9 @@
     "docs": "typedoc",
     "lint": "eslint src/**/*.ts",
     "lint-fix": "eslint --fix src/**/*.ts",
-    "__generate-barrels": "barrelsby -q --delete -d src -e .*\\.spec\\.ts",
-    "__build": "yarn run generate-barrels && tsc",
-    "build": "tsc"
+    "generate-barrels": "barrelsby -q --delete -d src -e .*\\.spec\\.ts && sed -i 's/\\x27;/.js\\x27;/' src/index.ts",
+    "build": "yarn clean && yarn generate-barrels && rollup -c rollup.config.js",
+    "force-build": "tsc --build --force"
   },
   "repository": {
     "type": "git",
@@ -60,35 +65,16 @@
   },
   "license": "Apache-2.0",
   "dependencies": {
-    "@aws-sdk/client-athena": "3.328.0",
-    "@aws-sdk/client-ec2": "3.328.0",
-    "@aws-sdk/client-ec2-instance-connect": "3.328.0",
-    "@aws-sdk/client-s3": "3.328.0",
-    "@aws-sdk/lib-storage": "3.328.0",
-    "@aws-sdk/smithy-client": "3.325.0",
-    "@aws-sdk/types": "3.310.0",
-    "@bitblit/ratchet-aws": "4.0.116-alpha",
-    "@bitblit/ratchet-common": "4.0.116-alpha",
+    "@aws-sdk/client-athena": "3.332.0",
+    "@bitblit/ratchet-aws": "4.0.119-alpha",
+    "@bitblit/ratchet-common": "4.0.119-alpha",
     "mailparser": "3.6.4",
     "mime-types": "2.1.35",
     "tmp": "0.2.1",
-    "unzipper": "0.10.11",
+    "unzipper": "0.10.14",
     "walk": "2.3.15"
   },
   "peerDependencies": {
-    "@aws-sdk/client-athena": "^3.328.0",
-    "@aws-sdk/client-ec2": "^3.328.0",
-    "@aws-sdk/client-ec2-instance-connect": "^3.328.0",
-    "@aws-sdk/client-s3": "^3.328.0",
-    "@aws-sdk/lib-storage": "^3.328.0",
-    "@aws-sdk/smithy-client": "^3.328.0",
-    "@aws-sdk/types": "^3.310.0",
-    "@bitblit/ratchet-aws": "4.0.116-alpha",
-    "@bitblit/ratchet-common": "4.0.116-alpha",
-    "mailparser": "^3.6.4",
-    "mime-types": "^2.1.35",
-    "tmp": "^0.2.1",
-    "walk": "^2.3.15"
   },
   "devDependencies": {}
 }

package/lib/athena/alb-athena-log-ratchet.js

@@ -1,158 +0,0 @@
-import { readFileSync } from 'fs';
-import path from 'path';
-import { Logger } from '@bitblit/ratchet-common/lib/logger/logger.js';
-import { RequireRatchet } from '@bitblit/ratchet-common/lib/lang/require-ratchet.js';
-import { StringRatchet } from '@bitblit/ratchet-common/lib/lang/string-ratchet.js';
-import { EsmRatchet } from '@bitblit/ratchet-common/lib/lang/esm-ratchet.js';
-import { CsvRatchet } from '@bitblit/ratchet-node-only/lib/csv/csv-ratchet.js';
-import { S3Ratchet } from '@bitblit/ratchet-aws';
-class AlbAthenaLogRatchet {
-    athena;
-    athenaTableName;
-    constructor(athena, athenaTableName) {
-        this.athena = athena;
-        this.athenaTableName = athenaTableName;
-        RequireRatchet.notNullOrUndefined(athena, 'athena');
-        RequireRatchet.notNullOrUndefined(StringRatchet.trimToNull(athenaTableName), 'athenaTableName');
-    }
-    async updatePartitions(rootPath, s3, startTimeEpochMS = new Date().getTime() - 1000 * 60 * 60 * 24, endTimeEpochMS = new Date().getTime()) {
-        RequireRatchet.true(S3Ratchet.checkS3UrlForValidity(rootPath), 'root path not valid');
-        RequireRatchet.notNullOrUndefined(s3, 's3');
-        Logger.info('Updating partitions for %s from %s', this.athenaTableName, rootPath);
-        const bucketName = S3Ratchet.extractBucketFromURL(rootPath);
-        const rootKey = S3Ratchet.extractKeyFromURL(rootPath);
-        let current = startTimeEpochMS;
-        const clauses = [];
-        while (current < endTimeEpochMS) {
-            const dateUtcVal = new Date(current).toISOString().substring(0, 10);
-            Logger.info('d:%s', dateUtcVal);
-            const dateParts = dateUtcVal.split('-');
-            clauses.push("PARTITION (date_utc_partition='" +
-                dateUtcVal +
-                "') LOCATION '" +
-                rootPath +
-                '/' +
-                dateParts[0] +
-                '/' +
-                dateParts[1] +
-                '/' +
-                dateParts[2] +
-                "'");
-            current += 1000 * 60 * 60 * 24;
-        }
-        if (clauses.length > 0) {
-            const stmt = 'ALTER TABLE ' + this.athenaTableName + ' ADD IF NOT EXISTS \n' + clauses.join('\n');
-            await this.athena.runQueryToObjects(stmt);
-        }
-        else {
-            Logger.warn('Not updating partitions - no time between time clauses');
-        }
-        return clauses;
-    }
-    async createTable(rootPath, replaceIfPresent = false) {
-        RequireRatchet.true(S3Ratchet.checkS3UrlForValidity(rootPath), 'root path not valid');
-        let rval = false;
-        Logger.info('Creating ALB table %s', this.athenaTableName);
-        if (replaceIfPresent) {
-            Logger.info('Replace if present specified, removed old table');
-            try {
-                await this.athena.runQueryToObjects('drop table ' + this.athenaTableName);
-            }
-            catch (err) {
-                Logger.info('Drop error : %j', err);
-            }
-        }
-        let tableCreateQry = readFileSync(path.join(EsmRatchet.fetchDirName(import.meta.url), '../static/albAthenaTableCreate.txt')).toString();
-        tableCreateQry = tableCreateQry.split('{{TABLE NAME}}').join(this.athenaTableName);
-        tableCreateQry = tableCreateQry.split('{{ALB_LOG_ROOT}}').join(rootPath);
-        Logger.info('Creating table with %s', tableCreateQry);
-        try {
-            await this.athena.runQueryToObjects(tableCreateQry);
-            rval = true;
-        }
-        catch (err) {
-            Logger.error('Error creating table : %s', err);
-        }
-        return rval;
-    }
-    static async readLogObjectsFromCsvStream(readStream) {
-        return CsvRatchet.streamParse(readStream, (p) => p);
-    }
-    static async readLogObjectsFromFile(fileName) {
-        return CsvRatchet.fileParse(fileName, (p) => p);
-    }
-    async fetchAlbLogRecords(qry) {
-        const tempFile = await this.fetchAlbLogRecordsToFile(qry);
-        return AlbAthenaLogRatchet.readLogObjectsFromFile(tempFile);
-    }
-    async fetchAlbLogRecordsToFile(qry, outputFileName = null) {
-        Logger.info('Querying %s : %j', this.athenaTableName, qry);
-        let qrySt = 'select * from ' + this.athenaTableName + ' where 1=1 ';
-        if (qry.startTimeEpochMS) {
-            if (qry.startTimeEpochMS) {
-                qrySt += " AND time >= '" + new Date(qry.startTimeEpochMS).toISOString() + "'";
-                qrySt += " AND date_utc_partition >='" + new Date(qry.startTimeEpochMS).toISOString().substring(0, 10) + "'";
-            }
-            if (qry.endTimeEpochMS) {
-                qrySt += " AND time < '" + new Date(qry.endTimeEpochMS).toISOString() + "'";
-                qrySt += " AND date_utc_partition <='" + new Date(qry.endTimeEpochMS).toISOString().substring(0, 10) + "'";
-            }
-            if (qry.requestUrlFilter) {
-                qrySt += " AND request_url LIKE '" + qry.requestUrlFilter + "'";
-            }
-            if (qry.limit) {
-                qrySt += ' LIMIT ' + qry.limit;
-            }
-        }
-        const result = await this.athena.runQueryToFile(qrySt, null, outputFileName);
-        return result;
-    }
-    static CREATE_TABLE_STATEMENT = 'CREATE EXTERNAL TABLE IF NOT EXISTS `{{TABLE NAME}}`(\n' +
-        "  `type` string COMMENT '',\n" +
-        "  `time` string COMMENT '',\n" +
-        "  `elb` string COMMENT '',\n" +
-        "  `client_ip` string COMMENT '',\n" +
-        "  `client_port` int COMMENT '',\n" +
-        "  `target_ip` string COMMENT '',\n" +
-        "  `target_port` int COMMENT '',\n" +
-        "  `request_processing_time` double COMMENT '',\n" +
-        "  `target_processing_time` double COMMENT '',\n" +
-        "  `response_processing_time` double COMMENT '',\n" +
-        "  `elb_status_code` string COMMENT '',\n" +
-        "  `target_status_code` string COMMENT '',\n" +
-        "  `received_bytes` bigint COMMENT '',\n" +
-        "  `sent_bytes` bigint COMMENT '',\n" +
-        "  `request_verb` string COMMENT '',\n" +
-        "  `request_url` string COMMENT '',\n" +
-        "  `request_proto` string COMMENT '',\n" +
-        "  `user_agent` string COMMENT '',\n" +
-        "  `ssl_cipher` string COMMENT '',\n" +
-        "  `ssl_protocol` string COMMENT '',\n" +
-        "  `target_group_arn` string COMMENT '',\n" +
-        "  `trace_id` string COMMENT '',\n" +
-        "  `domain_name` string COMMENT '',\n" +
-        "  `chosen_cert_arn` string COMMENT '',\n" +
-        "  `matched_rule_priority` string COMMENT '',\n" +
-        "  `request_creation_time` string COMMENT '',\n" +
-        "  `actions_executed` string COMMENT '',\n" +
-        "  `redirect_url` string COMMENT '',\n" +
-        "  `lambda_error_reason` string COMMENT '',\n" +
-        "  `target_port_list` string COMMENT '',\n" +
-        "  `target_status_code_list` string COMMENT '',\n" +
-        "  `new_field` string COMMENT '')\n" +
-        'PARTITIONED BY (\n' +
-        '  `date_utc_partition` string\n' +
-        ')\n' +
-        'ROW FORMAT SERDE\n' +
-        "  'org.apache.hadoop.hive.serde2.RegexSerDe'\n" +
-        'WITH SERDEPROPERTIES (\n' +
-        '  \'input.regex\'=\'([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \\"([^ ]*) ([^ ]*) (- |[^ ]*)\\" \\"([^\\"]*)\\" ([A-Z0-9-]+) ([A-Za-z0-9.-]*) ([^ ]*) \\"([^\\"]*)\\" \\"([^\\"]*)\\" \\"([^\\"]*)\\" ([-.0-9]*) ([^ ]*) \\"([^\\"]*)\\" \\"([^\\"]*)\\" \\"([^ ]*)\\" \\"([^s]+)\\" \\"([^s]+)\\"(.*)\')\n' +
-        'STORED AS INPUTFORMAT\n' +
-        "  'org.apache.hadoop.mapred.TextInputFormat'\n" +
-        'OUTPUTFORMAT\n' +
-        "  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\n" +
-        'LOCATION\n' +
-        "  '{{ALB_LOG_ROOT}}'\n";
-}
-export { AlbAthenaLogRatchet };
-//# sourceMappingURL=alb-athena-log-ratchet.js.map

package/lib/athena/alb-athena-log-ratchet.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"alb-athena-log-ratchet.js","sourceRoot":"","sources":["../../src/athena/alb-athena-log-ratchet.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAc,MAAM,IAAI,CAAC;AAC9C,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,EAAE,MAAM,EAAE,MAAM,8CAA8C,CAAC;AACtE,OAAO,EAAE,cAAc,EAAE,MAAM,qDAAqD,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,MAAM,oDAAoD,CAAC;AACnF,OAAO,EAAE,UAAU,EAAE,MAAM,iDAAiD,CAAC;AAC7E,OAAO,EAAE,UAAU,EAAE,MAAM,mDAAmD,CAAC;AAE/E,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AAIjD,MAAa,mBAAmB;IACV;IAA+B;IAAnD,YAAoB,MAAqB,EAAU,eAAuB;QAAtD,WAAM,GAAN,MAAM,CAAe;QAAU,oBAAe,GAAf,eAAe,CAAQ;QACxE,cAAc,CAAC,kBAAkB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACpD,cAAc,CAAC,kBAAkB,CAAC,aAAa,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,iBAAiB,CAAC,CAAC;IAClG,CAAC;IAEM,KAAK,CAAC,gBAAgB,CAC3B,QAAgB,EAChB,EAAY,EACZ,mBAA2B,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EACrE,iBAAyB,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE;QAE7C,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,qBAAqB,CAAC,QAAQ,CAAC,EAAE,qBAAqB,CAAC,CAAC;QACtF,cAAc,CAAC,kBAAkB,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAC5C,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,IAAI,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC;QAClF,MAAM,UAAU,GAAW,SAAS,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC;QACpE,MAAM,OAAO,GAAW,SAAS,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAE9D,IAAI,OAAO,GAAW,gBAAgB,CAAC;QACvC,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,OAAO,OAAO,GAAG,cAAc,EAAE;YAC/B,MAAM,UAAU,GAAW,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC5E,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;YAChC,MAAM,SAAS,GAAa,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAClD,OAAO,CAAC,IAAI,CACV,iCAAiC;gBAC/B,UAAU;gBACV,eAAe;gBACf,QAAQ;gBACR,GAAG;gBACH,SAAS,CAAC,CAAC,CAAC;gBACZ,GAAG;gBACH,SAAS,CAAC,CAAC,CAAC;gBACZ,GAAG;gBACH,SAAS,CAAC,CAAC,CAAC;gBACZ,GAAG,CACN,CAAC;YACF,OAAO,IAAI,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;SAChC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;YACtB,MAAM,IAAI,GAAW,cAAc,GAAG,IAAI,CAAC,eAAe,GAAG,uBAAuB,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1G,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAM,IAAI,CAAC,CAAC;SAChD;aAAM;YACL,MAAM,CAAC,IAAI,CAAC,wDAAwD,CAAC,CAAC;SACvE;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAEM,KAAK,CAAC,WAAW,CAAC,QAAgB,EAAE,mBAA4B,KAAK;QAC1E,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,qBAAqB,CAAC,QAAQ,CAAC,EAAE,qBAAqB,CAAC,CAAC;QACtF,IAAI,IAAI,GAAY,KAAK,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,uBAAuB,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;QAC3D,IAAI,gBAAgB,EAAE;YACpB,MAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;YAC/D,IAAI;gBACF,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAM,aAAa,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC;aAChF;YAAC,OAAO,GAAG,EAAE;gBACZ,MAAM,CAAC,IAAI,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;aACrC;SACF;QAED,IAAI,cAAc,GAAW,YAAY,CACvC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,oCAAoC,CAAC,CAC1F,CAAC,QAAQ,EAAE,CAAC;QACb,cAAc,GAAG,cAAc,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACnF,cAAc,GAAG,cAAc,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzE,MAAM,CAAC,IAAI,CAAC,wBAAwB,EAAE,cAAc,CAAC,CAAC;QAEtD,IAAI;YACF,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAM,cAAc,CAAC,CAAC;YACzD,IAAI,GAAG,IAAI,CAAC;SACb;QAAC,OAAO,GAAG,EAAE;YACZ,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,GAAG,CAAC,CAAC;SAChD;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,2BAA2B,CAAC,UAAsB;QACpE,OAAO,UAAU,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IACtD,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,QAAgB;QACzD,OAAO,UAAU,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC;IAEM,KAAK,CAAC,kBAAkB,CAAC,GAAsB;QACpD,MAAM,QAAQ,GAAW,MAAM,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,CAAC;QAClE,OAAO,mBAAmB,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;IAC9D,CAAC;IAEM,KAAK,CAAC,wBAAwB,CAAC,GAAsB,EAAE,iBAAyB,IAAI;QACzF,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,IAAI,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;QAE3D,IAAI,KAAK,GAAW,gBAAgB,GAAG,IAAI,CAAC,eAAe,GAAG,aAAa,CAAC;QAC5E,IAAI,GAAG,CAAC,gBAAgB,EAAE;YAExB,IAAI,GAAG,CAAC,gBAAgB,EAAE;gBACxB,KAAK,IAAI,gBAAgB,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,WAAW,EAAE,GAAG,GAAG,CAAC;gBAC/E,KAAK,IAAI,6BAA6B,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC;aAC9G;YACD,IAAI,GAAG,CAAC,cAAc,EAAE;gBACtB,KAAK,IAAI,eAAe,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,WAAW,EAAE,GAAG,GAAG,CAAC;gBAC5E,KAAK,IAAI,6BAA6B,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC;aAC5G;YACD,IAAI,GAAG,CAAC,gBAAgB,EAAE;gBACxB,KAAK,IAAI,yBAAyB,GAAG,GAAG,CAAC,gBAAgB,GAAG,GAAG,CAAC;aACjE;YACD,IAAI,GAAG,CAAC,KAAK,EAAE;gBACb,KAAK,IAAI,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC;aAChC;SACF;QAED,MAAM,MAAM,GAAW,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;QACrF,OAAO,MAAM,CAAC;IAChB,CAAC;IAEM,MAAM,CAAU,sBAAsB,GAC3C,yDAAyD;QACzD,+BAA+B;QAC/B,+BAA+B;QAC/B,8BAA8B;QAC9B,oCAAoC;QACpC,mCAAmC;QACnC,oCAAoC;QACpC,mCAAmC;QACnC,kDAAkD;QAClD,iDAAiD;QACjD,mDAAmD;QACnD,0CAA0C;QAC1C,6CAA6C;QAC7C,yCAAyC;QACzC,qCAAqC;QACrC,uCAAuC;QACvC,sCAAsC;QACtC,wCAAwC;QACxC,qCAAqC;QACrC,qCAAqC;QACrC,uCAAuC;QACvC,2CAA2C;QAC3C,mCAAmC;QACnC,sCAAsC;QACtC,0CAA0C;QAC1C,gDAAgD;QAChD,gDAAgD;QAChD,2CAA2C;QAC3C,uCAAuC;QACvC,8CAA8C;QAC9C,2CAA2C;QAC3C,kDAAkD;QAClD,oCAAoC;QACpC,oBAAoB;QACpB,iCAAiC;QACjC,KAAK;QACL,oBAAoB;QACpB,gDAAgD;QAChD,0BAA0B;QAC1B,2YAA2Y;QAC3Y,yBAAyB;QACzB,gDAAgD;QAChD,gBAAgB;QAChB,kEAAkE;QAClE,YAAY;QACZ,wBAAwB,CAAC;;SAlKhB,mBAAmB"}

package/lib/athena/alb-athena-log-ratchet.spec.js

@@ -1,29 +0,0 @@
-import { AthenaRatchet } from './athena-ratchet.js';
-import { Logger } from '@bitblit/ratchet-common/lib/logger/logger.js';
-import { LoggerLevelName } from '@bitblit/ratchet-common/lib/logger/logger-level-name.js';
-import { TimeZoneRatchet } from '@bitblit/ratchet-common/lib/lang/time-zone-ratchet.js';
-import { AlbAthenaLogRatchet } from './alb-athena-log-ratchet.js';
-import { S3Client } from '@aws-sdk/client-s3';
-import { AthenaClient } from '@aws-sdk/client-athena';
-describe('#AlbAthenaLogRatchet', function () {
-    it('Placeholder', async () => {
-        expect(2).toEqual(2);
-    });
-    xit('should test a query', async () => {
-        Logger.setLevel(LoggerLevelName.debug);
-        const athena = new AthenaClient({ region: 'us-east-1' });
-        const s3 = new S3Client({ region: 'us-east-1' });
-        const outputDir = 's3://alb-log-bucket/temp';
-        const athRatchet = new AthenaRatchet(athena, s3, outputDir);
-        const srv = new AlbAthenaLogRatchet(athRatchet, 'alb_logs.log_table');
-        const qry = {
-            startTimeEpochMS: TimeZoneRatchet.PACIFIC.startOfTodayEpochMS(),
-            endTimeEpochMS: TimeZoneRatchet.PACIFIC.startOfTodayEpochMS() + 1000 * 60 * 10,
-            limit: 10,
-        };
-        const result = await srv.fetchAlbLogRecords(qry);
-        expect(result).toBeTruthy();
-        Logger.info('Got objects : %j', result);
-    });
-});
-//# sourceMappingURL=alb-athena-log-ratchet.spec.js.map

package/lib/athena/alb-athena-log-ratchet.spec.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"alb-athena-log-ratchet.spec.js","sourceRoot":"","sources":["../../src/athena/alb-athena-log-ratchet.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,8CAA8C,CAAC;AACtE,OAAO,EAAE,eAAe,EAAE,MAAM,yDAAyD,CAAC;AAC1F,OAAO,EAAE,eAAe,EAAE,MAAM,uDAAuD,CAAC;AAExF,OAAO,EAAE,mBAAmB,EAAmC,MAAM,6BAA6B,CAAC;AACnG,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEtD,QAAQ,CAAC,sBAAsB,EAAE;IAC/B,EAAE,CAAC,aAAa,EAAE,KAAK,IAAI,EAAE;QAC3B,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,GAAG,CAAC,qBAAqB,EAAE,KAAK,IAAI,EAAE;QACpC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QACvC,MAAM,MAAM,GAAiB,IAAI,YAAY,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;QACvE,MAAM,EAAE,GAAa,IAAI,QAAQ,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;QAE3D,MAAM,SAAS,GAAW,0BAA0B,CAAC;QACrD,MAAM,UAAU,GAAkB,IAAI,aAAa,CAAC,MAAM,EAAE,EAAE,EAAE,SAAS,CAAC,CAAC;QAC3E,MAAM,GAAG,GAAwB,IAAI,mBAAmB,CAAC,UAAU,EAAE,oBAAoB,CAAC,CAAC;QAE3F,MAAM,GAAG,GAAsB;YAC7B,gBAAgB,EAAE,eAAe,CAAC,OAAO,CAAC,mBAAmB,EAAE;YAC/D,cAAc,EAAE,eAAe,CAAC,OAAO,CAAC,mBAAmB,EAAE,GAAG,IAAI,GAAG,EAAE,GAAG,EAAE;YAC9E,KAAK,EAAE,EAAE;SACV,CAAC;QAEF,MAAM,MAAM,GAAmB,MAAM,GAAG,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;QAEjE,MAAM,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/lib/athena/athena-ratchet.js

@@ -1,158 +0,0 @@
-import { GetNamedQueryCommand, GetQueryExecutionCommand, ListNamedQueriesCommand, StartQueryExecutionCommand, } from '@aws-sdk/client-athena';
-import { StringRatchet } from '@bitblit/ratchet-common/lib/lang/string-ratchet.js';
-import { Logger } from '@bitblit/ratchet-common/lib/logger/logger.js';
-import { StopWatch } from '@bitblit/ratchet-common/lib/lang/stop-watch.js';
-import { PromiseRatchet } from '@bitblit/ratchet-common/lib/lang/promise-ratchet.js';
-import { GetObjectCommand } from '@aws-sdk/client-s3';
-import tmp from 'tmp';
-import fs from 'fs';
-import { CsvRatchet } from '@bitblit/ratchet-node-only/lib/csv/csv-ratchet.js';
-import { RequireRatchet } from '@bitblit/ratchet-common/lib/lang/require-ratchet.js';
-export class AthenaRatchet {
-    athena;
-    s3;
-    outputLocation;
-    constructor(athena, s3, outputLocation) {
-        this.athena = athena;
-        this.s3 = s3;
-        this.outputLocation = outputLocation;
-        RequireRatchet.notNullOrUndefined(athena);
-        RequireRatchet.notNullOrUndefined(s3);
-        RequireRatchet.notNullOrUndefined(outputLocation);
-        RequireRatchet.true(outputLocation.startsWith('s3://'));
-    }
-    static athenaRowsToObject(input) {
-        const colNames = input[0].Data.map((d) => d.VarCharValue);
-        const temp = input.slice(1);
-        const rval = temp.map((t) => {
-            const newItem = {};
-            for (let i = 0; i < t.Data.length; i++) {
-                newItem[colNames[i]] = t.Data[i].VarCharValue;
-            }
-            return newItem;
-        });
-        return rval;
-    }
-    static applyParamsToQuery(query, queryParams) {
-        let rval = query;
-        if (!!rval && !!queryParams) {
-            Object.keys(queryParams).forEach((k) => {
-                const val = StringRatchet.safeString(queryParams[k]);
-                const kk = '{' + k + '}';
-                rval = rval.split(kk).join(val);
-            });
-        }
-        return rval;
-    }
-    async fetchQueryIds() {
-        const params = {
-            NextToken: null,
-        };
-        let rval = [];
-        let next = null;
-        do {
-            next = await this.athena.send(new ListNamedQueriesCommand(params));
-            rval = rval.concat(next.NamedQueryIds);
-            params.NextToken = next.NextToken;
-        } while (!!params.NextToken);
-        return rval;
-    }
-    async listQueries() {
-        const rval = [];
-        const ids = await this.fetchQueryIds();
-        Logger.debug('Finding %d items', ids.length);
-        for (let i = 0; i < ids.length; i++) {
-            const params = {
-                NamedQueryId: ids[i],
-            };
-            const val = await this.athena.send(new GetNamedQueryCommand(params));
-            rval.push(val.NamedQuery);
-        }
-        return rval;
-    }
-    async findQueryByName(name) {
-        const all = await this.listQueries();
-        const rval = all.find((a) => a.Name.toLowerCase() == name.toLowerCase());
-        return rval;
-    }
-    async runQueryToObjects(queryIn, queryParams = {}, pingTimeMS = 2000) {
-        Logger.info('Running query to objects');
-        const outputLoc = await this.runQueryToOutputLocation(queryIn, queryParams, pingTimeMS);
-        Logger.info('Query succeeded, processing file from %s', outputLoc);
-        const bucketName = outputLoc.substring(5, outputLoc.indexOf('/', 5));
-        const obKey = outputLoc.substring(outputLoc.indexOf('/', 5) + 1);
-        const req = {
-            Bucket: bucketName,
-            Key: obKey,
-        };
-        const getFileOut = await this.s3.send(new GetObjectCommand(req));
-        const rval = await CsvRatchet.stringParse(getFileOut.Body.toString(), (p) => {
-            return p;
-        }, { columns: true, skip_empty_lines: true });
-        return rval;
-    }
-    async runQueryToFile(queryIn, queryParams = {}, targetDataFileIn = null, pingTimeMS = 2000) {
-        Logger.info('Running query to file');
-        const outputLoc = await this.runQueryToOutputLocation(queryIn, queryParams, pingTimeMS);
-        Logger.info('Query succeeded, pulling file from %s', outputLoc);
-        const bucketName = outputLoc.substring(5, outputLoc.indexOf('/', 5));
-        const obKey = outputLoc.substring(outputLoc.indexOf('/', 5) + 1);
-        const req = {
-            Bucket: bucketName,
-            Key: obKey,
-        };
-        const targetDataFile = targetDataFileIn || tmp.fileSync({ postfix: '.csv', keep: false }).name;
-        const fileStream = fs.createWriteStream(targetDataFile);
-        const output = await this.s3.send(new GetObjectCommand(req));
-        const readStream = output.Body;
-        readStream.pipe(fileStream);
-        const rval = await PromiseRatchet.resolveOnEvent(readStream, ['finish', 'close'], ['error'], targetDataFile);
-        Logger.silly('Response: %s', rval);
-        return targetDataFile;
-    }
-    async runQueryToOutputLocation(queryIn, queryParams = {}, pingTimeMS = 2000) {
-        let rval = null;
-        const timer = new StopWatch();
-        const query = AthenaRatchet.applyParamsToQuery(queryIn, queryParams);
-        try {
-            Logger.info('Starting query : %s', query);
-            const token = StringRatchet.createType4Guid();
-            const params = {
-                QueryString: query,
-                ResultConfiguration: {
-                    OutputLocation: this.outputLocation,
-                    EncryptionConfiguration: {
-                        EncryptionOption: 'SSE_S3',
-                    },
-                },
-                ClientRequestToken: token,
-                QueryExecutionContext: {
-                    Database: 'default',
-                },
-            };
-            const startToken = await this.athena.send(new StartQueryExecutionCommand(params));
-            const getExecParams = {
-                QueryExecutionId: startToken.QueryExecutionId,
-            };
-            const finalStates = ['FAILED', 'CANCELLED', 'SUCCEEDED'];
-            let curState = await this.athena.send(new GetQueryExecutionCommand(getExecParams));
-            while (finalStates.indexOf(curState.QueryExecution.Status.State) === -1) {
-                await PromiseRatchet.createTimeoutPromise('wait', pingTimeMS);
-                Logger.debug('%s : %s : %s', curState.QueryExecution.Status.State, timer.dump(), query);
-                curState = await this.athena.send(new GetQueryExecutionCommand(getExecParams));
-            }
-            if (curState.QueryExecution.Status.State === 'FAILED') {
-                Logger.warn('Query failed : %s', curState.QueryExecution.Status.StateChangeReason);
-            }
-            else if (curState.QueryExecution.Status.State === 'SUCCEEDED') {
-                rval = curState.QueryExecution.ResultConfiguration.OutputLocation;
-            }
-        }
-        catch (err) {
-            Logger.warn('Failure : %s', err, err);
-        }
-        Logger.info('Query took %s : %s', timer.dump(), query);
-        return rval;
-    }
-}
-//# sourceMappingURL=athena-ratchet.js.map

package/lib/athena/athena-ratchet.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"athena-ratchet.js","sourceRoot":"","sources":["../../src/athena/athena-ratchet.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,oBAAoB,EAEpB,wBAAwB,EAExB,uBAAuB,EAIvB,0BAA0B,GAG3B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,aAAa,EAAE,MAAM,oDAAoD,CAAC;AACnF,OAAO,EAAE,MAAM,EAAE,MAAM,8CAA8C,CAAC;AACtE,OAAO,EAAE,SAAS,EAAE,MAAM,gDAAgD,CAAC;AAC3E,OAAO,EAAE,cAAc,EAAE,MAAM,qDAAqD,CAAC;AACrF,OAAO,EAAE,gBAAgB,EAAsD,MAAM,oBAAoB,CAAC;AAC1G,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,EAAmB,MAAM,IAAI,CAAC;AACrC,OAAO,EAAE,UAAU,EAAE,MAAM,mDAAmD,CAAC;AAC/E,OAAO,EAAE,cAAc,EAAE,MAAM,qDAAqD,CAAC;AAGrF,MAAM,OAAO,aAAa;IACJ;IAA8B;IAAsB;IAAxE,YAAoB,MAAoB,EAAU,EAAY,EAAU,cAAsB;QAA1E,WAAM,GAAN,MAAM,CAAc;QAAU,OAAE,GAAF,EAAE,CAAU;QAAU,mBAAc,GAAd,cAAc,CAAQ;QAC5F,cAAc,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAC1C,cAAc,CAAC,kBAAkB,CAAC,EAAE,CAAC,CAAC;QACtC,cAAc,CAAC,kBAAkB,CAAC,cAAc,CAAC,CAAC;QAClD,cAAc,CAAC,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1D,CAAC;IAEM,MAAM,CAAC,kBAAkB,CAAI,KAAY;QAC9C,MAAM,QAAQ,GAAa,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;QACpE,MAAM,IAAI,GAAU,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACnC,MAAM,IAAI,GAAQ,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YAC/B,MAAM,OAAO,GAAM,EAAO,CAAC;YAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACtC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC;aAC/C;YACD,OAAO,OAAO,CAAC;QACjB,CAAC,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,MAAM,CAAC,kBAAkB,CAAI,KAAa,EAAE,WAAc;QAC/D,IAAI,IAAI,GAAW,KAAK,CAAC;QACzB,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,WAAW,EAAE;YAC3B,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;gBACrC,MAAM,GAAG,GAAW,aAAa,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC7D,MAAM,EAAE,GAAW,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;gBACjC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAClC,CAAC,CAAC,CAAC;SACJ;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,KAAK,CAAC,aAAa;QACxB,MAAM,MAAM,GAAG;YAEb,SAAS,EAAE,IAAI;SAChB,CAAC;QAEF,IAAI,IAAI,GAAa,EAAE,CAAC;QACxB,IAAI,IAAI,GAAkC,IAAI,CAAC;QAE/C,GAAG;YACD,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,uBAAuB,CAAC,MAAM,CAAC,CAAC,CAAC;YACnE,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YACvC,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;SACnC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,EAAE;QAE7B,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,KAAK,CAAC,WAAW;QACtB,MAAM,IAAI,GAAiB,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAa,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QACjD,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACnC,MAAM,MAAM,GAAG;gBACb,YAAY,EAAE,GAAG,CAAC,CAAC,CAAC;aACrB,CAAC;YACF,MAAM,GAAG,GAA+B,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC;YACjG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;SAC3B;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,IAAY;QACvC,MAAM,GAAG,GAAiB,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACnD,MAAM,IAAI,GAAe,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QACrF,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,KAAK,CAAC,iBAAiB,CAAI,OAAe,EAAE,cAAmB,EAAE,EAAE,UAAU,GAAG,IAAI;QACzF,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,OAAO,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;QACxF,MAAM,CAAC,IAAI,CAAC,0CAA0C,EAAE,SAAS,CAAC,CAAC;QAEnE,MAAM,UAAU,GAAW,SAAS,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7E,MAAM,KAAK,GAAW,SAAS,CAAC,SAAS,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAEzE,MAAM,GAAG,GAAqB;YAC5B,MAAM,EAAE,UAAU;YAClB,GAAG,EAAE,KAAK;SACX,CAAC;QACF,MAAM,UAAU,GAA2B,MAAM,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC;QAEzF,MAAM,IAAI,GAAQ,MAAM,UAAU,CAAC,WAAW,CAC5C,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,EAC1B,CAAC,CAAC,EAAE,EAAE;YACJ,OAAO,CAAC,CAAC;QACX,CAAC,EACD,EAAE,OAAO,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAC1C,CAAC;QAEF,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,KAAK,CAAC,cAAc,CAAC,OAAe,EAAE,cAAmB,EAAE,EAAE,mBAA2B,IAAI,EAAE,UAAU,GAAG,IAAI;QACpH,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACrC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,OAAO,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;QACxF,MAAM,CAAC,IAAI,CAAC,uCAAuC,EAAE,SAAS,CAAC,CAAC;QAEhE,MAAM,UAAU,GAAW,SAAS,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7E,MAAM,KAAK,GAAW,SAAS,CAAC,SAAS,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAEzE,MAAM,GAAG,GAAqB;YAC5B,MAAM,EAAE,UAAU;YAClB,GAAG,EAAE,KAAK;SACX,CAAC;QAEF,MAAM,cAAc,GAAW,gBAAgB,IAAI,GAAG,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC;QACvG,MAAM,UAAU,GAAgB,EAAE,CAAC,iBAAiB,CAAC,cAAc,CAAC,CAAC;QACrE,MAAM,MAAM,GAA2B,MAAM,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC;QAErF,MAAM,UAAU,GAAa,MAAM,CAAC,IAAgB,CAAC;QACrD,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAE5B,MAAM,IAAI,GAAW,MAAM,cAAc,CAAC,cAAc,CAAS,UAAU,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,cAAc,CAAC,CAAC;QAC7H,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;QACnC,OAAO,cAAc,CAAC;IACxB,CAAC;IAEO,KAAK,CAAC,wBAAwB,CAAC,OAAe,EAAE,cAAmB,EAAE,EAAE,UAAU,GAAG,IAAI;QAC9F,IAAI,IAAI,GAAW,IAAI,CAAC;QACxB,MAAM,KAAK,GAAc,IAAI,SAAS,EAAE,CAAC;QACzC,MAAM,KAAK,GAAW,aAAa,CAAC,kBAAkB,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QAE7E,IAAI;YACF,MAAM,CAAC,IAAI,CAAC,qBAAqB,EAAE,KAAK,CAAC,CAAC;YAE1C,MAAM,KAAK,GAAW,aAAa,CAAC,eAAe,EAAE,CAAC;YACtD,MAAM,MAAM,GAA6B;gBACvC,WAAW,EAAE,KAAK;gBAClB,mBAAmB,EAAE;oBAEnB,cAAc,EAAE,IAAI,CAAC,cAAc;oBACnC,uBAAuB,EAAE;wBACvB,gBAAgB,EAAE,QAAQ;qBAE3B;iBACF;gBACD,kBAAkB,EAAE,KAAK;gBACzB,qBAAqB,EAAE;oBACrB,QAAQ,EAAE,SAAS;iBACpB;aACF,CAAC;YAEF,MAAM,UAAU,GAAqC,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,0BAA0B,CAAC,MAAM,CAAC,CAAC,CAAC;YAEpH,MAAM,aAAa,GAAQ;gBACzB,gBAAgB,EAAE,UAAU,CAAC,gBAAgB;aAC9C,CAAC;YAEF,MAAM,WAAW,GAAa,CAAC,QAAQ,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC;YACnE,IAAI,QAAQ,GAAmC,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,wBAAwB,CAAC,aAAa,CAAC,CAAC,CAAC;YACnH,OAAO,WAAW,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE;gBACvE,MAAM,cAAc,CAAC,oBAAoB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;gBAC9D,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,QAAQ,CAAC,cAAc,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC,CAAC;gBACxF,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,wBAAwB,CAAC,aAAa,CAAC,CAAC,CAAC;aAChF;YAED,IAAI,QAAQ,CAAC,cAAc,CAAC,MAAM,CAAC,KAAK,KAAK,QAAQ,EAAE;gBACrD,MAAM,CAAC,IAAI,CAAC,mBAAmB,EAAE,QAAQ,CAAC,cAAc,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;aACpF;iBAAM,IAAI,QAAQ,CAAC,cAAc,CAAC,MAAM,CAAC,KAAK,KAAK,WAAW,EAAE;gBAC/D,IAAI,GAAG,QAAQ,CAAC,cAAc,CAAC,mBAAmB,CAAC,cAAc,CAAC;aACnE;SACF;QAAC,OAAO,GAAG,EAAE;YACZ,MAAM,CAAC,IAAI,CAAC,cAAc,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;SACvC;QACD,MAAM,CAAC,IAAI,CAAC,oBAAoB,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC,CAAC;QAEvD,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}

package/lib/athena/athena-ratchet.spec.js

@@ -1,28 +0,0 @@
-import { AthenaRatchet } from './athena-ratchet.js';
-import { Logger } from '@bitblit/ratchet-common/lib/logger/logger.js';
-import { AthenaClient, GetQueryExecutionCommand, StartQueryExecutionCommand, } from '@aws-sdk/client-athena';
-import { S3Client } from '@aws-sdk/client-s3';
-import { mockClient } from 'aws-sdk-client-mock';
-let mockAthena;
-let mockS3;
-describe('#AthenaRatchet', function () {
-    mockAthena = mockClient(AthenaClient);
-    mockS3 = mockClient(S3Client);
-    beforeEach(() => {
-        mockAthena.reset();
-        mockS3.reset();
-    });
-    xit('should test a query', async () => {
-        const outputDir = 's3://your-bucket/your-prefix';
-        const qry = 'select * from test limit 20000';
-        mockAthena.on(StartQueryExecutionCommand).resolves({ jobName: 'b' });
-        mockAthena.on(GetQueryExecutionCommand).resolves({
-            QueryExecution: { ResultConfiguration: { OutputLocation: 'test' }, Status: { State: 'SUCCEEDED' } },
-        });
-        const ratchet = new AthenaRatchet(mockAthena, mockS3, outputDir);
-        const result = await ratchet.runQueryToObjects(qry);
-        expect(result).toBeTruthy();
-        Logger.info('Got objects : %j', result);
-    });
-});
-//# sourceMappingURL=athena-ratchet.spec.js.map

package/lib/athena/athena-ratchet.spec.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"athena-ratchet.spec.js","sourceRoot":"","sources":["../../src/athena/athena-ratchet.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,8CAA8C,CAAC;AACtE,OAAO,EACL,YAAY,EACZ,wBAAwB,EAExB,0BAA0B,GAE3B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,IAAI,UAAU,CAAC;AACf,IAAI,MAAM,CAAC;AAEX,QAAQ,CAAC,gBAAgB,EAAE;IACzB,UAAU,GAAG,UAAU,CAAC,YAAY,CAAC,CAAC;IACtC,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAE9B,UAAU,CAAC,GAAG,EAAE;QACd,UAAU,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC,CAAC,CAAC;IAEH,GAAG,CAAC,qBAAqB,EAAE,KAAK,IAAI,EAAE;QACpC,MAAM,SAAS,GAAW,8BAA8B,CAAC;QACzD,MAAM,GAAG,GAAW,gCAAgC,CAAC;QAErD,UAAU,CAAC,EAAE,CAAC,0BAA0B,CAAC,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,GAAG,EAA+B,CAAC,CAAC;QAClG,UAAU,CAAC,EAAE,CAAC,wBAAwB,CAAC,CAAC,QAAQ,CAAC;YAC/C,cAAc,EAAE,EAAE,mBAAmB,EAAE,EAAE,cAAc,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE;SACzE,CAAC,CAAC;QAE9B,MAAM,OAAO,GAAkB,IAAI,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QAEhF,MAAM,MAAM,GAAU,MAAM,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC;QAG3D,MAAM,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/lib/build/ratchet-aws-node-only-info.js

@@ -1,15 +0,0 @@
-export class RatchetAwsNodeOnlyInfo {
-    constructor() { }
-    static buildInformation() {
-        const val = {
-            version: 'LOCAL-SNAPSHOT',
-            hash: 'LOCAL-HASH',
-            branch: 'LOCAL-BRANCH',
-            tag: 'LOCAL-TAG',
-            timeBuiltISO: 'LOCAL-TIME-ISO',
-            notes: 'LOCAL-NOTES',
-        };
-        return val;
-    }
-}
-//# sourceMappingURL=ratchet-aws-node-only-info.js.map

package/lib/build/ratchet-aws-node-only-info.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"ratchet-aws-node-only-info.js","sourceRoot":"","sources":["../../src/build/ratchet-aws-node-only-info.ts"],"names":[],"mappings":"AAEA,MAAM,OAAO,sBAAsB;IAGjC,gBAAuB,CAAC;IAEjB,MAAM,CAAC,gBAAgB;QAC5B,MAAM,GAAG,GAAqB;YAC5B,OAAO,EAAE,gBAAgB;YACzB,IAAI,EAAE,YAAY;YAClB,MAAM,EAAE,cAAc;YACtB,GAAG,EAAE,WAAW;YAChB,YAAY,EAAE,gBAAgB;YAC9B,KAAK,EAAE,aAAa;SACrB,CAAC;QACF,OAAO,GAAG,CAAC;IACb,CAAC;CACF"}

package/lib/cli/dynamo-exporter.js

@@ -1,77 +0,0 @@
-import { StringRatchet } from '@bitblit/ratchet-common/lib/lang/string-ratchet.js';
-import { Logger } from '@bitblit/ratchet-common/lib/logger/logger.js';
-import { PromiseRatchet } from '@bitblit/ratchet-common/lib/lang/promise-ratchet.js';
-import { RequireRatchet } from '@bitblit/ratchet-common/lib/lang/require-ratchet.js';
-import fs from 'fs';
-import readline from 'readline';
-export class DynamoExporter {
-    constructor() { }
-    static async importJsonLFileToTable(dynamo, tableName, filename) {
-        RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');
-        RequireRatchet.notNullOrUndefined(tableName, 'tableName');
-        RequireRatchet.notNullOrUndefined(filename, 'filename');
-        const fileStream = fs.createReadStream(filename);
-        const rl = readline.createInterface({
-            input: fileStream,
-            crlfDelay: Infinity,
-        });
-        let rval = 0;
-        for await (const line of rl) {
-            if (rval % 100 === 0) {
-                Logger.info('Importing line %d', rval);
-            }
-            if (StringRatchet.trimToNull(line)) {
-                const parsed = JSON.parse(line);
-                await dynamo.simplePut(tableName, parsed);
-                rval++;
-            }
-        }
-        return rval;
-    }
-    static async exportScanToJsonLFile(dynamo, scan, filename) {
-        RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');
-        RequireRatchet.notNullOrUndefined(scan, 'scan');
-        RequireRatchet.notNullOrUndefined(filename, 'filename');
-        const ws = fs.createWriteStream(filename);
-        ws.on('end', () => {
-            Logger.debug('Write complete');
-        });
-        const rval = await DynamoExporter.exportScanToJsonLWriteStream(dynamo, scan, ws);
-        await PromiseRatchet.resolveOnEvent(ws, ['finish', 'close'], ['error']);
-        ws.close();
-        return rval;
-    }
-    static async exportQueryToJsonLFile(dynamo, qry, filename) {
-        RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');
-        RequireRatchet.notNullOrUndefined(qry, 'qry');
-        RequireRatchet.notNullOrUndefined(filename, 'filename');
-        const ws = fs.createWriteStream(filename);
-        ws.on('end', () => {
-            Logger.debug('Write complete');
-        });
-        const rval = await DynamoExporter.exportQueryToJsonLWriteStream(dynamo, qry, ws);
-        await PromiseRatchet.resolveOnEvent(ws, ['finish', 'close'], ['error']);
-        ws.close();
-        return rval;
-    }
-    static async exportScanToJsonLWriteStream(dynamo, scan, target) {
-        RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');
-        RequireRatchet.notNullOrUndefined(scan, 'scan');
-        RequireRatchet.notNullOrUndefined(target, 'target');
-        const rval = await dynamo.fullyExecuteProcessOverScan(scan, async (row) => DynamoExporter.writeItemToJsonLStream(row, target, false));
-        return rval;
-    }
-    static async exportQueryToJsonLWriteStream(dynamo, qry, target) {
-        RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');
-        RequireRatchet.notNullOrUndefined(qry, 'qry');
-        RequireRatchet.notNullOrUndefined(target, 'target');
-        const rval = await dynamo.fullyExecuteProcessOverQuery(qry, async (row) => DynamoExporter.writeItemToJsonLStream(row, target, false));
-        return rval;
-    }
-    static writeItemToJsonLStream(item, target, includeNulls = false) {
-        if (!!item || includeNulls) {
-            target.write(JSON.stringify(item) + '\n');
-        }
-    }
-}
-//# sourceMappingURL=dynamo-exporter.js.map

package/lib/cli/dynamo-exporter.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"dynamo-exporter.js","sourceRoot":"","sources":["../../src/cli/dynamo-exporter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,oDAAoD,CAAC;AACnF,OAAO,EAAE,MAAM,EAAE,MAAM,8CAA8C,CAAC;AACtE,OAAO,EAAE,cAAc,EAAE,MAAM,qDAAqD,CAAC;AACrF,OAAO,EAAE,cAAc,EAAE,MAAM,qDAAqD,CAAC;AACrF,OAAO,EAAmB,MAAM,IAAI,CAAC;AAErC,OAAO,QAAQ,MAAM,UAAU,CAAC;AAGhC,MAAM,OAAO,cAAc;IAGzB,gBAAuB,CAAC;IAEjB,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,MAAqB,EAAE,SAAiB,EAAE,QAAgB;QACnG,cAAc,CAAC,kBAAkB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACpD,cAAc,CAAC,kBAAkB,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;QAC1D,cAAc,CAAC,kBAAkB,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAExD,MAAM,UAAU,GAAG,EAAE,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAEjD,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;YAClC,KAAK,EAAE,UAAU;YACjB,SAAS,EAAE,QAAQ;SACpB,CAAC,CAAC;QAIH,IAAI,IAAI,GAAW,CAAC,CAAC;QACrB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE;YAC3B,IAAI,IAAI,GAAG,GAAG,KAAK,CAAC,EAAE;gBACpB,MAAM,CAAC,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC;aACxC;YACD,IAAI,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE;gBAClC,MAAM,MAAM,GAAQ,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACrC,MAAM,MAAM,CAAC,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;gBAC1C,IAAI,EAAE,CAAC;aACR;SACF;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC,MAAqB,EAAE,IAAsB,EAAE,QAAgB;QACvG,cAAc,CAAC,kBAAkB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACpD,cAAc,CAAC,kBAAkB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAChD,cAAc,CAAC,kBAAkB,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACxD,MAAM,EAAE,GAAgB,EAAE,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QACvD,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YAChB,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;QAEH,MAAM,IAAI,GAAW,MAAM,cAAc,CAAC,4BAA4B,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;QAEzF,MAAM,cAAc,CAAC,cAAc,CAAC,EAAE,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;QACxE,EAAE,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,MAAqB,EAAE,GAAsB,EAAE,QAAgB;QACxG,cAAc,CAAC,kBAAkB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACpD,cAAc,CAAC,kBAAkB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC9C,cAAc,CAAC,kBAAkB,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAExD,MAAM,EAAE,GAAgB,EAAE,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QACvD,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YAChB,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;QAEH,MAAM,IAAI,GAAW,MAAM,cAAc,CAAC,6BAA6B,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;QAEzF,MAAM,cAAc,CAAC,cAAc,CAAC,EAAE,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;QACxE,EAAE,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,4BAA4B,CAAC,MAAqB,EAAE,IAAsB,EAAE,MAAmB;QACjH,cAAc,CAAC,kBAAkB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACpD,cAAc,CAAC,kBAAkB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAChD,cAAc,CAAC,kBAAkB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAEpD,MAAM,IAAI,GAAW,MAAM,MAAM,CAAC,2BAA2B,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAChF,cAAc,CAAC,sBAAsB,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,CAC1D,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,MAAqB,EAAE,GAAsB,EAAE,MAAmB;QAClH,cAAc,CAAC,kBAAkB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACpD,cAAc,CAAC,kBAAkB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC9C,cAAc,CAAC,kBAAkB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAEpD,MAAM,IAAI,GAAW,MAAM,MAAM,CAAC,4BAA4B,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAChF,cAAc,CAAC,sBAAsB,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,CAC1D,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;IACM,MAAM,CAAC,sBAAsB,CAAC,IAAS,EAAE,MAAmB,EAAE,eAAwB,KAAK;QAChG,IAAI,CAAC,CAAC,IAAI,IAAI,YAAY,EAAE;YAC1B,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;SAC3C;IACH,CAAC;CACF"}

package/lib/cli/ratchet-cli-handler.js

@@ -1,16 +0,0 @@
-import { AbstractRatchetCliHandler } from '@bitblit/ratchet-node-only/lib/cli/abstract-ratchet-cli-handler.js';
-import { SiteUploader } from './site-uploader/site-uploader.js';
-import { StartInstanceAndSsh } from './start-instance-and-ssh.js';
-import { RatchetAwsNodeOnlyInfo } from '../build/ratchet-aws-node-only-info.js';
-export class RatchetCliHandler extends AbstractRatchetCliHandler {
-    fetchHandlerMap() {
-        return {
-            'site-uploader': SiteUploader.runFromCliArgs,
-            'start-instance-and-ssh': StartInstanceAndSsh.runFromCliArgs,
-        };
-    }
-    fetchVersionInfo() {
-        return RatchetAwsNodeOnlyInfo.buildInformation();
-    }
-}
-//# sourceMappingURL=ratchet-cli-handler.js.map

package/lib/cli/ratchet-cli-handler.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"ratchet-cli-handler.js","sourceRoot":"","sources":["../../src/cli/ratchet-cli-handler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,oEAAoE,CAAC;AAC/G,OAAO,EAAE,YAAY,EAAE,MAAM,kCAAkC,CAAC;AAChE,OAAO,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AAElE,OAAO,EAAE,sBAAsB,EAAE,MAAM,wCAAwC,CAAC;AAEhF,MAAM,OAAO,iBAAkB,SAAQ,yBAAyB;IAC9D,eAAe;QACb,OAAO;YACL,eAAe,EAAE,YAAY,CAAC,cAAc;YAC5C,wBAAwB,EAAE,mBAAmB,CAAC,cAAc;SAC7D,CAAC;IACJ,CAAC;IAED,gBAAgB;QACd,OAAO,sBAAsB,CAAC,gBAAgB,EAAE,CAAC;IACnD,CAAC;CACF"}