npm - @bitblit/ratchet-aws-node-only - Versions diffs - 4.0.226-alpha → 4.0.232-alpha - Mend

@bitblit/ratchet-aws-node-only 4.0.226-alpha → 4.0.232-alpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (3) hide show

package/lib/index.mjs CHANGED Viewed

@@ -1,2 +1,2 @@
-import e,{readFileSync as t}from"fs";import n from"path";import{RequireRatchet as i,StringRatchet as s,Logger as r,EsmRatchet as a,PromiseRatchet as o,StopWatch as c,MultiStream as l}from"@bitblit/ratchet-common";import{CsvRatchet as u,AbstractRatchetCliHandler as h}from"@bitblit/ratchet-node-only";import{S3Ratchet as d,Ec2Ratchet as f}from"@bitblit/ratchet-aws";import{ListNamedQueriesCommand as m,GetNamedQueryCommand as p,StartQueryExecutionCommand as g,GetQueryExecutionCommand as y}from"@aws-sdk/client-athena";import{GetObjectCommand as T,S3Client as O}from"@aws-sdk/client-s3";import E from"tmp";import N from"readline";import S from"walk";import b from"mime-types";import{Upload as w}from"@aws-sdk/lib-storage";import{spawnSync as M}from"child_process";import C from"os";import F from"unzipper";import{DateTime as L}from"luxon";import{simpleParser as _}from"mailparser";import A from"crypto";class I{athena;athenaTableName;constructor(e,t){this.athena=e,this.athenaTableName=t,i.notNullOrUndefined(e,"athena"),i.notNullOrUndefined(s.trimToNull(t),"athenaTableName")}async updatePartitions(e,t,n=(new Date).getTime()-864e5,s=(new Date).getTime()){i.true(d.checkS3UrlForValidity(e),"root path not valid"),i.notNullOrUndefined(t,"s3"),r.info("Updating partitions for %s from %s",this.athenaTableName,e),d.extractBucketFromURL(e),d.extractKeyFromURL(e);let a=n;const o=[];for(;a<s;){const t=new Date(a).toISOString().substring(0,10);r.info("d:%s",t);const n=t.split("-");o.push("PARTITION (date_utc_partition='"+t+"') LOCATION '"+e+"/"+n[0]+"/"+n[1]+"/"+n[2]+"'"),a+=864e5}if(o.length>0){const e="ALTER TABLE "+this.athenaTableName+" ADD IF NOT EXISTS \n"+o.join("\n");await this.athena.runQueryToObjects(e)}else r.warn("Not updating partitions - no time between time clauses");return o}async createTable(e,s=!1){i.true(d.checkS3UrlForValidity(e),"root path not valid");let o=!1;if(r.info("Creating ALB table %s",this.athenaTableName),s){r.info("Replace if present specified, removed old table");try{await this.athena.runQueryToObjects("drop table "+this.athenaTableName)}catch(e){r.info("Drop error : %j",e)}}let c=t(n.join(a.fetchDirName(import.meta.url),"../static/albAthenaTableCreate.txt")).toString();c=c.split("{{TABLE NAME}}").join(this.athenaTableName),c=c.split("{{ALB_LOG_ROOT}}").join(e),r.info("Creating table with %s",c);try{await this.athena.runQueryToObjects(c),o=!0}catch(e){r.error("Error creating table : %s",e)}return o}static async readLogObjectsFromCsvStream(e){return u.streamParse(e,(e=>e))}static async readLogObjectsFromFile(e){return u.fileParse(e,(e=>e))}async fetchAlbLogRecords(e){const t=await this.fetchAlbLogRecordsToFile(e);return I.readLogObjectsFromFile(t)}async fetchAlbLogRecordsToFile(e,t=null){r.info("Querying %s : %j",this.athenaTableName,e);let n="select * from "+this.athenaTableName+" where 1=1 ";e.startTimeEpochMS&&(e.startTimeEpochMS&&(n+=" AND time >= '"+new Date(e.startTimeEpochMS).toISOString()+"'",n+=" AND date_utc_partition >='"+new Date(e.startTimeEpochMS).toISOString().substring(0,10)+"'"),e.endTimeEpochMS&&(n+=" AND time < '"+new Date(e.endTimeEpochMS).toISOString()+"'",n+=" AND date_utc_partition <='"+new Date(e.endTimeEpochMS).toISOString().substring(0,10)+"'"),e.requestUrlFilter&&(n+=" AND request_url LIKE '"+e.requestUrlFilter+"'"),e.limit&&(n+=" LIMIT "+e.limit));return await this.athena.runQueryToFile(n,null,t)}static CREATE_TABLE_STATEMENT="CREATE EXTERNAL TABLE IF NOT EXISTS `{{TABLE NAME}}`(\n  `type` string COMMENT '',\n  `time` string COMMENT '',\n  `elb` string COMMENT '',\n  `client_ip` string COMMENT '',\n  `client_port` int COMMENT '',\n  `target_ip` string COMMENT '',\n  `target_port` int COMMENT '',\n  `request_processing_time` double COMMENT '',\n  `target_processing_time` double COMMENT '',\n  `response_processing_time` double COMMENT '',\n  `elb_status_code` string COMMENT '',\n  `target_status_code` string COMMENT '',\n  `received_bytes` bigint COMMENT '',\n  `sent_bytes` bigint COMMENT '',\n  `request_verb` string COMMENT '',\n  `request_url` string COMMENT '',\n  `request_proto` string COMMENT '',\n  `user_agent` string COMMENT '',\n  `ssl_cipher` string COMMENT '',\n  `ssl_protocol` string COMMENT '',\n  `target_group_arn` string COMMENT '',\n  `trace_id` string COMMENT '',\n  `domain_name` string COMMENT '',\n  `chosen_cert_arn` string COMMENT '',\n  `matched_rule_priority` string COMMENT '',\n  `request_creation_time` string COMMENT '',\n  `actions_executed` string COMMENT '',\n  `redirect_url` string COMMENT '',\n  `lambda_error_reason` string COMMENT '',\n  `target_port_list` string COMMENT '',\n  `target_status_code_list` string COMMENT '',\n  `new_field` string COMMENT '')\nPARTITIONED BY (\n  `date_utc_partition` string\n)\nROW FORMAT SERDE\n  'org.apache.hadoop.hive.serde2.RegexSerDe'\nWITH SERDEPROPERTIES (\n  'input.regex'='([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \\\"([^ ]*) ([^ ]*) (- |[^ ]*)\\\" \\\"([^\\\"]*)\\\" ([A-Z0-9-]+) ([A-Za-z0-9.-]*) ([^ ]*) \\\"([^\\\"]*)\\\" \\\"([^\\\"]*)\\\" \\\"([^\\\"]*)\\\" ([-.0-9]*) ([^ ]*) \\\"([^\\\"]*)\\\" \\\"([^\\\"]*)\\\" \\\"([^ ]*)\\\" \\\"([^s]+)\\\" \\\"([^s]+)\\\"(.*)')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  '{{ALB_LOG_ROOT}}'\n"}class U{athena;s3;outputLocation;constructor(e,t,n){this.athena=e,this.s3=t,this.outputLocation=n,i.notNullOrUndefined(e),i.notNullOrUndefined(t),i.notNullOrUndefined(n),i.true(n.startsWith("s3://"))}static athenaRowsToObject(e){const t=e[0].Data.map((e=>e.VarCharValue));return e.slice(1).map((e=>{const n={};for(let i=0;i<e.Data.length;i++)n[t[i]]=e.Data[i].VarCharValue;return n}))}static applyParamsToQuery(e,t){let n=e;return n&&t&&Object.keys(t).forEach((e=>{const i=s.safeString(t[e]),r="{"+e+"}";n=n.split(r).join(i)})),n}async fetchQueryIds(){const e={NextToken:null};let t=[],n=null;do{n=await this.athena.send(new m(e)),t=t.concat(n.NamedQueryIds),e.NextToken=n.NextToken}while(e.NextToken);return t}async listQueries(){const e=[],t=await this.fetchQueryIds();r.debug("Finding %d items",t.length);for(let n=0;n<t.length;n++){const i={NamedQueryId:t[n]},s=await this.athena.send(new p(i));e.push(s.NamedQuery)}return e}async findQueryByName(e){return(await this.listQueries()).find((t=>t.Name.toLowerCase()==e.toLowerCase()))}async runQueryToObjects(e,t={},n=2e3){r.info("Running query to objects");const i=await this.runQueryToOutputLocation(e,t,n);r.info("Query succeeded, processing file from %s",i);const s={Bucket:i.substring(5,i.indexOf("/",5)),Key:i.substring(i.indexOf("/",5)+1)},a=await this.s3.send(new T(s));return await u.stringParse(a.Body.toString(),(e=>e),{columns:!0,skip_empty_lines:!0})}async runQueryToFile(t,n={},i=null,s=2e3){r.info("Running query to file");const a=await this.runQueryToOutputLocation(t,n,s);r.info("Query succeeded, pulling file from %s",a);const c={Bucket:a.substring(5,a.indexOf("/",5)),Key:a.substring(a.indexOf("/",5)+1)},l=i||E.fileSync({postfix:".csv",keep:!1}).name,u=e.createWriteStream(l),h=(await this.s3.send(new T(c))).Body;h.pipe(u);const d=await o.resolveOnEvent(h,["finish","close"],["error"],l);return r.silly("Response: %s",d),l}async runQueryToOutputLocation(e,t={},n=2e3){let i=null;const a=new c,l=U.applyParamsToQuery(e,t);try{r.info("Starting query : %s",l);const e=s.createType4Guid(),t={QueryString:l,ResultConfiguration:{OutputLocation:this.outputLocation,EncryptionConfiguration:{EncryptionOption:"SSE_S3"}},ClientRequestToken:e,QueryExecutionContext:{Database:"default"}},c={QueryExecutionId:(await this.athena.send(new g(t))).QueryExecutionId},u=["FAILED","CANCELLED","SUCCEEDED"];let h=await this.athena.send(new y(c));for(;-1===u.indexOf(h.QueryExecution.Status.State);)await o.createTimeoutPromise("wait",n),r.debug("%s : %s : %s",h.QueryExecution.Status.State,a.dump(),l),h=await this.athena.send(new y(c));"FAILED"===h.QueryExecution.Status.State?r.warn("Query failed : %s",h.QueryExecution.Status.StateChangeReason):"SUCCEEDED"===h.QueryExecution.Status.State&&(i=h.QueryExecution.ResultConfiguration.OutputLocation)}catch(e){r.warn("Failure : %s",e,e)}return r.info("Query took %s : %s",a.dump(),l),i}}class x{constructor(){}static buildInformation(){return{version:"226",hash:"3756dbef763df895de882697478a5d8237fb4544",branch:"alpha-2023-09-22-1",tag:"alpha-2023-09-22-1",timeBuiltISO:"2023-09-22T17:32:41-0700",notes:"No notes"}}}class D{constructor(){}static async importJsonLFileToTable(t,n,a){i.notNullOrUndefined(t,"dynamo"),i.notNullOrUndefined(n,"tableName"),i.notNullOrUndefined(a,"filename");const o=e.createReadStream(a),c=N.createInterface({input:o,crlfDelay:1/0});let l=0;for await(const e of c)if(l%100==0&&r.info("Importing line %d",l),s.trimToNull(e)){const i=JSON.parse(e);await t.simplePut(n,i),l++}return l}static async exportScanToJsonLFile(t,n,s){i.notNullOrUndefined(t,"dynamo"),i.notNullOrUndefined(n,"scan"),i.notNullOrUndefined(s,"filename");const a=e.createWriteStream(s);a.on("end",(()=>{r.debug("Write complete")}));const c=await D.exportScanToJsonLWriteStream(t,n,a);return await o.resolveOnEvent(a,["finish","close"],["error"]),a.close(),c}static async exportQueryToJsonLFile(t,n,s){i.notNullOrUndefined(t,"dynamo"),i.notNullOrUndefined(n,"qry"),i.notNullOrUndefined(s,"filename");const a=e.createWriteStream(s);a.on("end",(()=>{r.debug("Write complete")}));const c=await D.exportQueryToJsonLWriteStream(t,n,a);return await o.resolveOnEvent(a,["finish","close"],["error"]),a.close(),c}static async exportScanToJsonLWriteStream(e,t,n){i.notNullOrUndefined(e,"dynamo"),i.notNullOrUndefined(t,"scan"),i.notNullOrUndefined(n,"target");return await e.fullyExecuteProcessOverScan(t,(async e=>D.writeItemToJsonLStream(e,n,!1)))}static async exportQueryToJsonLWriteStream(e,t,n){i.notNullOrUndefined(e,"dynamo"),i.notNullOrUndefined(t,"qry"),i.notNullOrUndefined(n,"target");return await e.fullyExecuteProcessOverQuery(t,(async e=>D.writeItemToJsonLStream(e,n,!1)))}static writeItemToJsonLStream(e,t,n=!1){(e||n)&&t.write(JSON.stringify(e)+"\n")}}class k{srcDir;bucketName;config;s3=new O({region:"us-east-1"});constructor(t,n,i){this.srcDir=t,this.bucketName=n,this.config=JSON.parse(e.readFileSync(i).toString("ascii"))}static createFromArgs(e){if(e&&3===e.length){const t=e[0],n=e[1],i=e[2];return new k(t,n,i)}return console.log("Usage : node ratchet-site-uploader {srcDir} {bucket} {configFile} (Found "+e+" arguments, need 3)"),null}static async runFromCliArgs(e){return k.createFromArgs(e).runPump()}findMatch(e,t,n){let i=null;return null!=e&&null!=t&&null!=n&&null!=n.mapping&&n.mapping.forEach((n=>{null==i&&(null==n.prefixMatch||e.match(n.prefixMatch))&&(null==n.fileMatch||t.match(n.fileMatch))&&(i=n)})),i}findMime(e,t){let n=null;return null!=t&&null!=t.customMimeTypeMapping&&Object.keys(t.customMimeTypeMapping).forEach((i=>{null==n&&e.endsWith(i)&&(n=t.customMimeTypeMapping[i])})),null==n&&(n=b.lookup(e)),null==n&&(n="binary/octet-stream"),n}runPump(){return new Promise(((t,i)=>{r.info("Uploading contents of %s to %s using %j as config",this.srcDir,this.bucketName,this.config);const s=S.walk(this.srcDir,{});s.on("file",function(t,i,s){r.info("Processing %j",i.name);const a=t==this.srcDir?"":t.substring(this.srcDir.length+1)+"/",o=this.findMatch(a,i.name,this.config),c=a+i.name;r.info("Uploading file : %s/%s to key %s with %j",t,i.name,c,o);const l=o&&o.putParams?JSON.parse(JSON.stringify(o.putParams)):{};l.Bucket=this.bucketName,l.Key=c,l.Body=e.readFileSync(n.join(t,i.name)),l.ContentType||(l.ContentType=this.findMime(i.name,this.config));const u=new w({client:this.s3,params:l,tags:[],queueSize:4,partSize:5242880,leavePartsOnError:!1});u.on("httpUploadProgress",(e=>{r.info("Uploading : %s",e)})),u.done().then((e=>{r.info("Finished upload of %s: %j",c,e),s()})).catch((e=>{r.warn("%s failed to upload : %s : Continuing",c,e),s()}))}.bind(this)),s.on("errors",(function(e,t,n){n()})),s.on("end",(function(){r.info("All done"),t(!0)}))}))}}class R{instanceId;publicKeyFile;instanceOsUser;region;availabilityZone;ec2Ratchet;constructor(e,t=n.join(C.homedir(),".ssh","id_rsa.pub"),i="ec2-user",s="us-east-1",r="us-east-1a"){this.instanceId=e,this.publicKeyFile=t,this.instanceOsUser=i,this.region=s,this.availabilityZone=r,this.ec2Ratchet=new f(this.region,this.availabilityZone)}static createFromArgs(e){if(1===e?.length||2===e?.length){const t=e[0];return new R(t)}return r.info("Usage : ratchet-start-instance-and-ssh {instanceId} {publicKeyFile} (Found %s arguments, need 1 or 2)",e),null}static async runFromCliArgs(e){return R.createFromArgs(e).run()}async run(){let t=await this.ec2Ratchet.describeInstance(this.instanceId);if(t){let n=!1;if(16==t.State.Code?(r.info("Instance is already running..."),n=!0):(r.info("Instance is not running... starting up : %s",this.instanceId),n=await this.ec2Ratchet.launchInstance(this.instanceId,3e4)),n){r.info("Uploading public key...");const n=e.readFileSync(this.publicKeyFile).toString(),i=await this.ec2Ratchet.sendPublicKeyToEc2Instance(this.instanceId,n,this.instanceOsUser);r.info("Key response : %j",i),t=t&&t.PublicIpAddress?t:await this.ec2Ratchet.describeInstance(this.instanceId),r.info("Instance IP address is %s",t.PublicIpAddress);const s=M("ssh",[this.instanceOsUser+"@"+t.PublicIpAddress],{stdio:"inherit"});r.info("%j",s)}else r.info("Instance could not start - check logs")}else r.info("No such instance found - check your AWS keys? : %s",this.instanceId)}}class P extends h{fetchHandlerMap(){return{"site-uploader":k.runFromCliArgs,"start-instance-and-ssh":R.runFromCliArgs}}fetchVersionInfo(){return x.buildInformation()}}class Q{canProcess(e){return!0}async processEmail(e){const t=[];try{i.notNullOrUndefined(e,"msg"),r.info("Processing Broadsign reach inbound inventory email");const n=e.attachments[0].content;r.info("Unzipping attachment");const s=new l(n);let a=null;const o=s.pipe(F.Parse()).on("entry",(async e=>{e.path.toLowerCase().endsWith("csv")?a=await e.buffer():(r.info("Pass: %s",e.path),e.autodrain())})).promise();await o;const c=await u.stringParse(a.toString(),(e=>e),{columns:!1,skip_empty_lines:!0});if(c.length>1){const e="drop table if exists sample";let n="create table sample (pump_date varchar(255),";const i=c[0];let s="insert into sample (pump_date,",a="?,";for(let e=0;e<i.length;e++){e>0&&(n+=", ",s+=", ",a+=", ");const t=i[e].toLowerCase().split(" ").join("_");s+=t,a+="?",n+=t+" varchar(255)","id"===t?n+=" primary key":"device_id"===t&&(n+=" unique")}n+=")",s+=") values ",r.info("Recreating table");t.push({statement:e}),t.push({statement:n});const o=L.utc().toISO();let l=s,u=[];for(let e=1;e<c.length;e++)l>s&&(l+=","),l+="("+a+")",u=u.concat(o,c[e]),e%25!=0&&e!==c.length-1||(t.push({statement:l,params:u}),l=s,u=[],r.info("Inserted %d of %d rows",e,c.length));r.info("Finished insertion of %d rows",c.length)}}catch(e){r.error("Failure: %s : %j",e,t,e)}return t}}class B{cache;processors;constructor(e,t){this.cache=e,this.processors=t,i.notNullOrUndefined(this.cache,"cache"),i.notNullOrUndefined(this.cache.getDefaultBucket(),"cache.defaultBucket")}async processEmailFromS3(e){if(await this.cache.fileExists(e)){const t=await this.cache.fetchCacheFileAsString(e);return this.processEmailFromBuffer(new Buffer(t))}return r.warn("Cannot process inbound email - no such key : %s",e),!1}async processEmailFromBuffer(e){i.notNullOrUndefined(e,"buf"),r.info("Processing inbound email - size %d bytes",e.length);const t=await _(e);r.info('Found mail from "%s" subject "%s" with %d attachments',t?.from?.text,t?.subject,t?.attachments?.length);let n=!1;for(let e=0;e<this.processors.length&&!n;e++)if(this.processors[e].canProcess(t)){r.info("Processing message with processor %d",e);const i=await this.processors[e].processEmail(t);r.info("Result was : %j",i),n=!0}return n}}class j{canProcess(e){return!0}async processEmail(e){return e.body}}class v{s3;tmpFolder;cacheTimeoutSeconds;static DEFAULT_CACHE_TIMEOUT_SEC=604800;currentlyLoading=new Map;constructor(t,n,r=v.DEFAULT_CACHE_TIMEOUT_SEC){this.s3=t,this.tmpFolder=n,this.cacheTimeoutSeconds=r,i.notNullOrUndefined(t,"s3"),i.notNullOrUndefined(s.trimToNull(n)),i.true(e.existsSync(n),"folder must exist : "+n)}async getFileString(e){const t=await this.getFileBuffer(e);return t?t.toString():null}keyToLocalCachePath(e){const t=this.generateCacheHash(this.s3.getDefaultBucket()+"/"+e);return n.join(this.tmpFolder,t)}removeCacheFileForKey(t){const n=this.keyToLocalCachePath(t);r.info("Removing cache file for %s : %s",t,n),e.existsSync(n)?e.unlinkSync(n):r.debug("Skipping delete for %s - does not exist",n)}async getFileBuffer(e){const t=this.keyToLocalCachePath(e);let n=null;if(n=this.getCacheFileAsBuffer(t),n)r.info("Found cache file for s3://%s/%s. Local path %s",this.s3.getDefaultBucket(),e,t);else{r.info("No cache. Downloading File s3://%s/%s to %s",this.s3.getDefaultBucket(),e,t);try{let i=this.currentlyLoading.get(e);i?r.info("Already running - wait for that"):(r.info("Not running - start"),i=this.updateLocalCacheFile(e,t),this.currentlyLoading.set(e,i)),n=await i,this.currentlyLoading.delete(e)}catch(t){r.warn("File %s/%s does not exist. Err code: %s",this.s3.getDefaultBucket(),e,t)}}return n}async updateLocalCacheFile(t,n){const i=await this.s3.fetchCacheFileAsBuffer(t);return i&&i.length>0&&(r.info("Saving %d bytes to disk for cache",i.length),e.writeFileSync(n,i)),i}getCacheFileAsString(e){const t=this.getCacheFileAsBuffer(e);return t?t.toString():null}getCacheFileAsBuffer(t){if(!e.existsSync(t))return null;try{const n=e.statSync(t),i=(new Date).getTime();if((i-n.ctimeMs)/1e3>=this.cacheTimeoutSeconds)return null;return e.readFileSync(t)}catch(e){r.warn("Error getting s3 cache file %s",e)}return null}generateCacheHash(e){return A.createHash("md5").update(e).digest("hex")}}export{I as AlbAthenaLogRatchet,U as AthenaRatchet,D as DynamoExporter,Q as EmailToDbInsertProcessor,B as InboundEmailRatchet,x as RatchetAwsNodeOnlyInfo,P as RatchetCliHandler,v as S3CacheToLocalDiskRatchet,j as SampleEmailProcessor,k as SiteUploader,R as StartInstanceAndSsh};
+import e,{readFileSync as t}from"fs";import n from"path";import{RequireRatchet as i,StringRatchet as s,Logger as r,EsmRatchet as a,PromiseRatchet as o,StopWatch as c,MultiStream as l}from"@bitblit/ratchet-common";import{CsvRatchet as u,AbstractRatchetCliHandler as h}from"@bitblit/ratchet-node-only";import{S3Ratchet as d,Ec2Ratchet as f}from"@bitblit/ratchet-aws";import{ListNamedQueriesCommand as m,GetNamedQueryCommand as p,StartQueryExecutionCommand as g,GetQueryExecutionCommand as y}from"@aws-sdk/client-athena";import{GetObjectCommand as T,S3Client as O}from"@aws-sdk/client-s3";import E from"tmp";import N from"readline";import S from"walk";import b from"mime-types";import{Upload as w}from"@aws-sdk/lib-storage";import{spawnSync as M}from"child_process";import C from"os";import F from"unzipper";import{DateTime as L}from"luxon";import{simpleParser as _}from"mailparser";import A from"crypto";class I{athena;athenaTableName;constructor(e,t){this.athena=e,this.athenaTableName=t,i.notNullOrUndefined(e,"athena"),i.notNullOrUndefined(s.trimToNull(t),"athenaTableName")}async updatePartitions(e,t,n=(new Date).getTime()-864e5,s=(new Date).getTime()){i.true(d.checkS3UrlForValidity(e),"root path not valid"),i.notNullOrUndefined(t,"s3"),r.info("Updating partitions for %s from %s",this.athenaTableName,e),d.extractBucketFromURL(e),d.extractKeyFromURL(e);let a=n;const o=[];for(;a<s;){const t=new Date(a).toISOString().substring(0,10);r.info("d:%s",t);const n=t.split("-");o.push("PARTITION (date_utc_partition='"+t+"') LOCATION '"+e+"/"+n[0]+"/"+n[1]+"/"+n[2]+"'"),a+=864e5}if(o.length>0){const e="ALTER TABLE "+this.athenaTableName+" ADD IF NOT EXISTS \n"+o.join("\n");await this.athena.runQueryToObjects(e)}else r.warn("Not updating partitions - no time between time clauses");return o}async createTable(e,s=!1){i.true(d.checkS3UrlForValidity(e),"root path not valid");let o=!1;if(r.info("Creating ALB table %s",this.athenaTableName),s){r.info("Replace if present specified, removed old table");try{await this.athena.runQueryToObjects("drop table "+this.athenaTableName)}catch(e){r.info("Drop error : %j",e)}}let c=t(n.join(a.fetchDirName(import.meta.url),"../static/albAthenaTableCreate.txt")).toString();c=c.split("{{TABLE NAME}}").join(this.athenaTableName),c=c.split("{{ALB_LOG_ROOT}}").join(e),r.info("Creating table with %s",c);try{await this.athena.runQueryToObjects(c),o=!0}catch(e){r.error("Error creating table : %s",e)}return o}static async readLogObjectsFromCsvStream(e){return u.streamParse(e,(e=>e))}static async readLogObjectsFromFile(e){return u.fileParse(e,(e=>e))}async fetchAlbLogRecords(e){const t=await this.fetchAlbLogRecordsToFile(e);return I.readLogObjectsFromFile(t)}async fetchAlbLogRecordsToFile(e,t=null){r.info("Querying %s : %j",this.athenaTableName,e);let n="select * from "+this.athenaTableName+" where 1=1 ";e.startTimeEpochMS&&(e.startTimeEpochMS&&(n+=" AND time >= '"+new Date(e.startTimeEpochMS).toISOString()+"'",n+=" AND date_utc_partition >='"+new Date(e.startTimeEpochMS).toISOString().substring(0,10)+"'"),e.endTimeEpochMS&&(n+=" AND time < '"+new Date(e.endTimeEpochMS).toISOString()+"'",n+=" AND date_utc_partition <='"+new Date(e.endTimeEpochMS).toISOString().substring(0,10)+"'"),e.requestUrlFilter&&(n+=" AND request_url LIKE '"+e.requestUrlFilter+"'"),e.limit&&(n+=" LIMIT "+e.limit));return await this.athena.runQueryToFile(n,null,t)}static CREATE_TABLE_STATEMENT="CREATE EXTERNAL TABLE IF NOT EXISTS `{{TABLE NAME}}`(\n  `type` string COMMENT '',\n  `time` string COMMENT '',\n  `elb` string COMMENT '',\n  `client_ip` string COMMENT '',\n  `client_port` int COMMENT '',\n  `target_ip` string COMMENT '',\n  `target_port` int COMMENT '',\n  `request_processing_time` double COMMENT '',\n  `target_processing_time` double COMMENT '',\n  `response_processing_time` double COMMENT '',\n  `elb_status_code` string COMMENT '',\n  `target_status_code` string COMMENT '',\n  `received_bytes` bigint COMMENT '',\n  `sent_bytes` bigint COMMENT '',\n  `request_verb` string COMMENT '',\n  `request_url` string COMMENT '',\n  `request_proto` string COMMENT '',\n  `user_agent` string COMMENT '',\n  `ssl_cipher` string COMMENT '',\n  `ssl_protocol` string COMMENT '',\n  `target_group_arn` string COMMENT '',\n  `trace_id` string COMMENT '',\n  `domain_name` string COMMENT '',\n  `chosen_cert_arn` string COMMENT '',\n  `matched_rule_priority` string COMMENT '',\n  `request_creation_time` string COMMENT '',\n  `actions_executed` string COMMENT '',\n  `redirect_url` string COMMENT '',\n  `lambda_error_reason` string COMMENT '',\n  `target_port_list` string COMMENT '',\n  `target_status_code_list` string COMMENT '',\n  `new_field` string COMMENT '')\nPARTITIONED BY (\n  `date_utc_partition` string\n)\nROW FORMAT SERDE\n  'org.apache.hadoop.hive.serde2.RegexSerDe'\nWITH SERDEPROPERTIES (\n  'input.regex'='([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \\\"([^ ]*) ([^ ]*) (- |[^ ]*)\\\" \\\"([^\\\"]*)\\\" ([A-Z0-9-]+) ([A-Za-z0-9.-]*) ([^ ]*) \\\"([^\\\"]*)\\\" \\\"([^\\\"]*)\\\" \\\"([^\\\"]*)\\\" ([-.0-9]*) ([^ ]*) \\\"([^\\\"]*)\\\" \\\"([^\\\"]*)\\\" \\\"([^ ]*)\\\" \\\"([^s]+)\\\" \\\"([^s]+)\\\"(.*)')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  '{{ALB_LOG_ROOT}}'\n"}class U{athena;s3;outputLocation;constructor(e,t,n){this.athena=e,this.s3=t,this.outputLocation=n,i.notNullOrUndefined(e),i.notNullOrUndefined(t),i.notNullOrUndefined(n),i.true(n.startsWith("s3://"))}static athenaRowsToObject(e){const t=e[0].Data.map((e=>e.VarCharValue));return e.slice(1).map((e=>{const n={};for(let i=0;i<e.Data.length;i++)n[t[i]]=e.Data[i].VarCharValue;return n}))}static applyParamsToQuery(e,t){let n=e;return n&&t&&Object.keys(t).forEach((e=>{const i=s.safeString(t[e]),r="{"+e+"}";n=n.split(r).join(i)})),n}async fetchQueryIds(){const e={NextToken:null};let t=[],n=null;do{n=await this.athena.send(new m(e)),t=t.concat(n.NamedQueryIds),e.NextToken=n.NextToken}while(e.NextToken);return t}async listQueries(){const e=[],t=await this.fetchQueryIds();r.debug("Finding %d items",t.length);for(let n=0;n<t.length;n++){const i={NamedQueryId:t[n]},s=await this.athena.send(new p(i));e.push(s.NamedQuery)}return e}async findQueryByName(e){return(await this.listQueries()).find((t=>t.Name.toLowerCase()==e.toLowerCase()))}async runQueryToObjects(e,t={},n=2e3){r.info("Running query to objects");const i=await this.runQueryToOutputLocation(e,t,n);r.info("Query succeeded, processing file from %s",i);const s={Bucket:i.substring(5,i.indexOf("/",5)),Key:i.substring(i.indexOf("/",5)+1)},a=await this.s3.send(new T(s));return await u.stringParse(a.Body.toString(),(e=>e),{columns:!0,skip_empty_lines:!0})}async runQueryToFile(t,n={},i=null,s=2e3){r.info("Running query to file");const a=await this.runQueryToOutputLocation(t,n,s);r.info("Query succeeded, pulling file from %s",a);const c={Bucket:a.substring(5,a.indexOf("/",5)),Key:a.substring(a.indexOf("/",5)+1)},l=i||E.fileSync({postfix:".csv",keep:!1}).name,u=e.createWriteStream(l),h=(await this.s3.send(new T(c))).Body;h.pipe(u);const d=await o.resolveOnEvent(h,["finish","close"],["error"],l);return r.silly("Response: %s",d),l}async runQueryToOutputLocation(e,t={},n=2e3){let i=null;const a=new c,l=U.applyParamsToQuery(e,t);try{r.info("Starting query : %s",l);const e=s.createType4Guid(),t={QueryString:l,ResultConfiguration:{OutputLocation:this.outputLocation,EncryptionConfiguration:{EncryptionOption:"SSE_S3"}},ClientRequestToken:e,QueryExecutionContext:{Database:"default"}},c={QueryExecutionId:(await this.athena.send(new g(t))).QueryExecutionId},u=["FAILED","CANCELLED","SUCCEEDED"];let h=await this.athena.send(new y(c));for(;-1===u.indexOf(h.QueryExecution.Status.State);)await o.createTimeoutPromise("wait",n),r.debug("%s : %s : %s",h.QueryExecution.Status.State,a.dump(),l),h=await this.athena.send(new y(c));"FAILED"===h.QueryExecution.Status.State?r.warn("Query failed : %s",h.QueryExecution.Status.StateChangeReason):"SUCCEEDED"===h.QueryExecution.Status.State&&(i=h.QueryExecution.ResultConfiguration.OutputLocation)}catch(e){r.warn("Failure : %s",e,e)}return r.info("Query took %s : %s",a.dump(),l),i}}class x{constructor(){}static buildInformation(){return{version:"232",hash:"bcce209de9712e649cb0c1ea207f3297337b39e3",branch:"alpha-2023-09-25-6",tag:"alpha-2023-09-25-6",timeBuiltISO:"2023-09-25T20:00:31-0700",notes:"No notes"}}}class D{constructor(){}static async importJsonLFileToTable(t,n,a){i.notNullOrUndefined(t,"dynamo"),i.notNullOrUndefined(n,"tableName"),i.notNullOrUndefined(a,"filename");const o=e.createReadStream(a),c=N.createInterface({input:o,crlfDelay:1/0});let l=0;for await(const e of c)if(l%100==0&&r.info("Importing line %d",l),s.trimToNull(e)){const i=JSON.parse(e);await t.simplePut(n,i),l++}return l}static async exportScanToJsonLFile(t,n,s){i.notNullOrUndefined(t,"dynamo"),i.notNullOrUndefined(n,"scan"),i.notNullOrUndefined(s,"filename");const a=e.createWriteStream(s);a.on("end",(()=>{r.debug("Write complete")}));const c=await D.exportScanToJsonLWriteStream(t,n,a);return await o.resolveOnEvent(a,["finish","close"],["error"]),a.close(),c}static async exportQueryToJsonLFile(t,n,s){i.notNullOrUndefined(t,"dynamo"),i.notNullOrUndefined(n,"qry"),i.notNullOrUndefined(s,"filename");const a=e.createWriteStream(s);a.on("end",(()=>{r.debug("Write complete")}));const c=await D.exportQueryToJsonLWriteStream(t,n,a);return await o.resolveOnEvent(a,["finish","close"],["error"]),a.close(),c}static async exportScanToJsonLWriteStream(e,t,n){i.notNullOrUndefined(e,"dynamo"),i.notNullOrUndefined(t,"scan"),i.notNullOrUndefined(n,"target");return await e.fullyExecuteProcessOverScan(t,(async e=>D.writeItemToJsonLStream(e,n,!1)))}static async exportQueryToJsonLWriteStream(e,t,n){i.notNullOrUndefined(e,"dynamo"),i.notNullOrUndefined(t,"qry"),i.notNullOrUndefined(n,"target");return await e.fullyExecuteProcessOverQuery(t,(async e=>D.writeItemToJsonLStream(e,n,!1)))}static writeItemToJsonLStream(e,t,n=!1){(e||n)&&t.write(JSON.stringify(e)+"\n")}}class k{srcDir;bucketName;config;s3=new O({region:"us-east-1"});constructor(t,n,i){this.srcDir=t,this.bucketName=n,this.config=JSON.parse(e.readFileSync(i).toString("ascii"))}static createFromArgs(e){if(e&&3===e.length){const t=e[0],n=e[1],i=e[2];return new k(t,n,i)}return console.log("Usage : node ratchet-site-uploader {srcDir} {bucket} {configFile} (Found "+e+" arguments, need 3)"),null}static async runFromCliArgs(e){return k.createFromArgs(e).runPump()}findMatch(e,t,n){let i=null;return null!=e&&null!=t&&null!=n&&null!=n.mapping&&n.mapping.forEach((n=>{null==i&&(null==n.prefixMatch||e.match(n.prefixMatch))&&(null==n.fileMatch||t.match(n.fileMatch))&&(i=n)})),i}findMime(e,t){let n=null;return null!=t&&null!=t.customMimeTypeMapping&&Object.keys(t.customMimeTypeMapping).forEach((i=>{null==n&&e.endsWith(i)&&(n=t.customMimeTypeMapping[i])})),null==n&&(n=b.lookup(e)),null==n&&(n="binary/octet-stream"),n}runPump(){return new Promise(((t,i)=>{r.info("Uploading contents of %s to %s using %j as config",this.srcDir,this.bucketName,this.config);const s=S.walk(this.srcDir,{});s.on("file",function(t,i,s){r.info("Processing %j",i.name);const a=t==this.srcDir?"":t.substring(this.srcDir.length+1)+"/",o=this.findMatch(a,i.name,this.config),c=a+i.name;r.info("Uploading file : %s/%s to key %s with %j",t,i.name,c,o);const l=o&&o.putParams?JSON.parse(JSON.stringify(o.putParams)):{};l.Bucket=this.bucketName,l.Key=c,l.Body=e.readFileSync(n.join(t,i.name)),l.ContentType||(l.ContentType=this.findMime(i.name,this.config));const u=new w({client:this.s3,params:l,tags:[],queueSize:4,partSize:5242880,leavePartsOnError:!1});u.on("httpUploadProgress",(e=>{r.info("Uploading : %s",e)})),u.done().then((e=>{r.info("Finished upload of %s: %j",c,e),s()})).catch((e=>{r.warn("%s failed to upload : %s : Continuing",c,e),s()}))}.bind(this)),s.on("errors",(function(e,t,n){n()})),s.on("end",(function(){r.info("All done"),t(!0)}))}))}}class R{instanceId;publicKeyFile;instanceOsUser;region;availabilityZone;ec2Ratchet;constructor(e,t=n.join(C.homedir(),".ssh","id_rsa.pub"),i="ec2-user",s="us-east-1",r="us-east-1a"){this.instanceId=e,this.publicKeyFile=t,this.instanceOsUser=i,this.region=s,this.availabilityZone=r,this.ec2Ratchet=new f(this.region,this.availabilityZone)}static createFromArgs(e){if(1===e?.length||2===e?.length){const t=e[0];return new R(t)}return r.info("Usage : ratchet-start-instance-and-ssh {instanceId} {publicKeyFile} (Found %s arguments, need 1 or 2)",e),null}static async runFromCliArgs(e){return R.createFromArgs(e).run()}async run(){let t=await this.ec2Ratchet.describeInstance(this.instanceId);if(t){let n=!1;if(16==t.State.Code?(r.info("Instance is already running..."),n=!0):(r.info("Instance is not running... starting up : %s",this.instanceId),n=await this.ec2Ratchet.launchInstance(this.instanceId,3e4)),n){r.info("Uploading public key...");const n=e.readFileSync(this.publicKeyFile).toString(),i=await this.ec2Ratchet.sendPublicKeyToEc2Instance(this.instanceId,n,this.instanceOsUser);r.info("Key response : %j",i),t=t&&t.PublicIpAddress?t:await this.ec2Ratchet.describeInstance(this.instanceId),r.info("Instance IP address is %s",t.PublicIpAddress);const s=M("ssh",[this.instanceOsUser+"@"+t.PublicIpAddress],{stdio:"inherit"});r.info("%j",s)}else r.info("Instance could not start - check logs")}else r.info("No such instance found - check your AWS keys? : %s",this.instanceId)}}class P extends h{fetchHandlerMap(){return{"site-uploader":k.runFromCliArgs,"start-instance-and-ssh":R.runFromCliArgs}}fetchVersionInfo(){return x.buildInformation()}}class Q{canProcess(e){return!0}async processEmail(e){const t=[];try{i.notNullOrUndefined(e,"msg"),r.info("Processing Broadsign reach inbound inventory email");const n=e.attachments[0].content;r.info("Unzipping attachment");const s=new l(n);let a=null;const o=s.pipe(F.Parse()).on("entry",(async e=>{e.path.toLowerCase().endsWith("csv")?a=await e.buffer():(r.info("Pass: %s",e.path),e.autodrain())})).promise();await o;const c=await u.stringParse(a.toString(),(e=>e),{columns:!1,skip_empty_lines:!0});if(c.length>1){const e="drop table if exists sample";let n="create table sample (pump_date varchar(255),";const i=c[0];let s="insert into sample (pump_date,",a="?,";for(let e=0;e<i.length;e++){e>0&&(n+=", ",s+=", ",a+=", ");const t=i[e].toLowerCase().split(" ").join("_");s+=t,a+="?",n+=t+" varchar(255)","id"===t?n+=" primary key":"device_id"===t&&(n+=" unique")}n+=")",s+=") values ",r.info("Recreating table");t.push({statement:e}),t.push({statement:n});const o=L.utc().toISO();let l=s,u=[];for(let e=1;e<c.length;e++)l>s&&(l+=","),l+="("+a+")",u=u.concat(o,c[e]),e%25!=0&&e!==c.length-1||(t.push({statement:l,params:u}),l=s,u=[],r.info("Inserted %d of %d rows",e,c.length));r.info("Finished insertion of %d rows",c.length)}}catch(e){r.error("Failure: %s : %j",e,t,e)}return t}}class B{cache;processors;constructor(e,t){this.cache=e,this.processors=t,i.notNullOrUndefined(this.cache,"cache"),i.notNullOrUndefined(this.cache.getDefaultBucket(),"cache.defaultBucket")}async processEmailFromS3(e){if(await this.cache.fileExists(e)){const t=await this.cache.fetchCacheFileAsString(e);return this.processEmailFromBuffer(new Buffer(t))}return r.warn("Cannot process inbound email - no such key : %s",e),!1}async processEmailFromBuffer(e){i.notNullOrUndefined(e,"buf"),r.info("Processing inbound email - size %d bytes",e.length);const t=await _(e);r.info('Found mail from "%s" subject "%s" with %d attachments',t?.from?.text,t?.subject,t?.attachments?.length);let n=!1;for(let e=0;e<this.processors.length&&!n;e++)if(this.processors[e].canProcess(t)){r.info("Processing message with processor %d",e);const i=await this.processors[e].processEmail(t);r.info("Result was : %j",i),n=!0}return n}}class j{canProcess(e){return!0}async processEmail(e){return e.body}}class v{s3;tmpFolder;cacheTimeoutSeconds;static DEFAULT_CACHE_TIMEOUT_SEC=604800;currentlyLoading=new Map;constructor(t,n,r=v.DEFAULT_CACHE_TIMEOUT_SEC){this.s3=t,this.tmpFolder=n,this.cacheTimeoutSeconds=r,i.notNullOrUndefined(t,"s3"),i.notNullOrUndefined(s.trimToNull(n)),i.true(e.existsSync(n),"folder must exist : "+n)}async getFileString(e){const t=await this.getFileBuffer(e);return t?t.toString():null}keyToLocalCachePath(e){const t=this.generateCacheHash(this.s3.getDefaultBucket()+"/"+e);return n.join(this.tmpFolder,t)}removeCacheFileForKey(t){const n=this.keyToLocalCachePath(t);r.info("Removing cache file for %s : %s",t,n),e.existsSync(n)?e.unlinkSync(n):r.debug("Skipping delete for %s - does not exist",n)}async getFileBuffer(e){const t=this.keyToLocalCachePath(e);let n=null;if(n=this.getCacheFileAsBuffer(t),n)r.info("Found cache file for s3://%s/%s. Local path %s",this.s3.getDefaultBucket(),e,t);else{r.info("No cache. Downloading File s3://%s/%s to %s",this.s3.getDefaultBucket(),e,t);try{let i=this.currentlyLoading.get(e);i?r.info("Already running - wait for that"):(r.info("Not running - start"),i=this.updateLocalCacheFile(e,t),this.currentlyLoading.set(e,i)),n=await i,this.currentlyLoading.delete(e)}catch(t){r.warn("File %s/%s does not exist. Err code: %s",this.s3.getDefaultBucket(),e,t)}}return n}async updateLocalCacheFile(t,n){const i=await this.s3.fetchCacheFileAsBuffer(t);return i&&i.length>0&&(r.info("Saving %d bytes to disk for cache",i.length),e.writeFileSync(n,i)),i}getCacheFileAsString(e){const t=this.getCacheFileAsBuffer(e);return t?t.toString():null}getCacheFileAsBuffer(t){if(!e.existsSync(t))return null;try{const n=e.statSync(t),i=(new Date).getTime();if((i-n.ctimeMs)/1e3>=this.cacheTimeoutSeconds)return null;return e.readFileSync(t)}catch(e){r.warn("Error getting s3 cache file %s",e)}return null}generateCacheHash(e){return A.createHash("md5").update(e).digest("hex")}}export{I as AlbAthenaLogRatchet,U as AthenaRatchet,D as DynamoExporter,Q as EmailToDbInsertProcessor,B as InboundEmailRatchet,x as RatchetAwsNodeOnlyInfo,P as RatchetCliHandler,v as S3CacheToLocalDiskRatchet,j as SampleEmailProcessor,k as SiteUploader,R as StartInstanceAndSsh};
 //# sourceMappingURL=index.mjs.map

package/lib/index.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"index.mjs","sources":["../src/athena/alb-athena-log-ratchet.ts","../src/athena/athena-ratchet.ts","../src/build/ratchet-aws-node-only-info.ts","../src/cli/dynamo-exporter.ts","../src/cli/site-uploader/site-uploader.ts","../src/cli/start-instance-and-ssh.ts","../src/cli/ratchet-cli-handler.ts","../src/mail/inbound/email-to-db-insert-processor.ts","../src/mail/inbound/inbound-email-ratchet.ts","../src/mail/inbound/sample-email-processor.ts","../src/s3/s3-cache-to-local-disk-ratchet.ts"],"sourcesContent":["import { readFileSync } from 'fs';\nimport path from 'path';\nimport { Logger } from '@bitblit/ratchet-common';\nimport { RequireRatchet } from '@bitblit/ratchet-common';\nimport { StringRatchet } from '@bitblit/ratchet-common';\nimport { EsmRatchet } from '@bitblit/ratchet-common';\nimport { CsvRatchet } from '@bitblit/ratchet-node-only';\nimport { S3Ratchet } from '@bitblit/ratchet-aws';\nexport class AlbAthenaLogRatchet {\n athena;\n athenaTableName;\n constructor(athena, athenaTableName) {\n this.athena = athena;\n this.athenaTableName = athenaTableName;\n RequireRatchet.notNullOrUndefined(athena, 'athena');\n RequireRatchet.notNullOrUndefined(StringRatchet.trimToNull(athenaTableName), 'athenaTableName');\n }\n async updatePartitions(rootPath, s3, startTimeEpochMS = new Date().getTime() - 1000 * 60 * 60 * 24, endTimeEpochMS = new Date().getTime()) {\n RequireRatchet.true(S3Ratchet.checkS3UrlForValidity(rootPath), 'root path not valid');\n RequireRatchet.notNullOrUndefined(s3, 's3');\n Logger.info('Updating partitions for %s from %s', this.athenaTableName, rootPath);\n const bucketName = S3Ratchet.extractBucketFromURL(rootPath);\n const rootKey = S3Ratchet.extractKeyFromURL(rootPath);\n let current = startTimeEpochMS;\n const clauses = [];\n while (current < endTimeEpochMS) {\n const dateUtcVal = new Date(current).toISOString().substring(0, 10);\n Logger.info('d:%s', dateUtcVal);\n const dateParts = dateUtcVal.split('-');\n clauses.push(\"PARTITION (date_utc_partition='\" +\n dateUtcVal +\n \"') LOCATION '\" +\n rootPath +\n '/' +\n dateParts[0] +\n '/' +\n dateParts[1] +\n '/' +\n dateParts[2] +\n \"'\");\n current += 1000 * 60 * 60 * 24;\n }\n if (clauses.length > 0) {\n const stmt = 'ALTER TABLE ' + this.athenaTableName + ' ADD IF NOT EXISTS \\n' + clauses.join('\\n');\n await this.athena.runQueryToObjects(stmt);\n }\n else {\n Logger.warn('Not updating partitions - no time between time clauses');\n }\n return clauses;\n }\n async createTable(rootPath, replaceIfPresent = false) {\n RequireRatchet.true(S3Ratchet.checkS3UrlForValidity(rootPath), 'root path not valid');\n let rval = false;\n Logger.info('Creating ALB table %s', this.athenaTableName);\n if (replaceIfPresent) {\n Logger.info('Replace if present specified, removed old table');\n try {\n await this.athena.runQueryToObjects('drop table ' + this.athenaTableName);\n }\n catch (err) {\n Logger.info('Drop error : %j', err);\n }\n }\n let tableCreateQry = readFileSync(path.join(EsmRatchet.fetchDirName(import.meta.url), '../static/albAthenaTableCreate.txt')).toString();\n tableCreateQry = tableCreateQry.split('{{TABLE NAME}}').join(this.athenaTableName);\n tableCreateQry = tableCreateQry.split('{{ALB_LOG_ROOT}}').join(rootPath);\n Logger.info('Creating table with %s', tableCreateQry);\n try {\n await this.athena.runQueryToObjects(tableCreateQry);\n rval = true;\n }\n catch (err) {\n Logger.error('Error creating table : %s', err);\n }\n return rval;\n }\n static async readLogObjectsFromCsvStream(readStream) {\n return CsvRatchet.streamParse(readStream, (p) => p);\n }\n static async readLogObjectsFromFile(fileName) {\n return CsvRatchet.fileParse(fileName, (p) => p);\n }\n async fetchAlbLogRecords(qry) {\n const tempFile = await this.fetchAlbLogRecordsToFile(qry);\n return AlbAthenaLogRatchet.readLogObjectsFromFile(tempFile);\n }\n async fetchAlbLogRecordsToFile(qry, outputFileName = null) {\n Logger.info('Querying %s : %j', this.athenaTableName, qry);\n let qrySt = 'select * from ' + this.athenaTableName + ' where 1=1 ';\n if (qry.startTimeEpochMS) {\n if (qry.startTimeEpochMS) {\n qrySt += \" AND time >= '\" + new Date(qry.startTimeEpochMS).toISOString() + \"'\";\n qrySt += \" AND date_utc_partition >='\" + new Date(qry.startTimeEpochMS).toISOString().substring(0, 10) + \"'\";\n }\n if (qry.endTimeEpochMS) {\n qrySt += \" AND time < '\" + new Date(qry.endTimeEpochMS).toISOString() + \"'\";\n qrySt += \" AND date_utc_partition <='\" + new Date(qry.endTimeEpochMS).toISOString().substring(0, 10) + \"'\";\n }\n if (qry.requestUrlFilter) {\n qrySt += \" AND request_url LIKE '\" + qry.requestUrlFilter + \"'\";\n }\n if (qry.limit) {\n qrySt += ' LIMIT ' + qry.limit;\n }\n }\n const result = await this.athena.runQueryToFile(qrySt, null, outputFileName);\n return result;\n }\n static CREATE_TABLE_STATEMENT = 'CREATE EXTERNAL TABLE IF NOT EXISTS `{{TABLE NAME}}`(\\n' +\n \" `type` string COMMENT '',\\n\" +\n \" `time` string COMMENT '',\\n\" +\n \" `elb` string COMMENT '',\\n\" +\n \" `client_ip` string COMMENT '',\\n\" +\n \" `client_port` int COMMENT '',\\n\" +\n \" `target_ip` string COMMENT '',\\n\" +\n \" `target_port` int COMMENT '',\\n\" +\n \" `request_processing_time` double COMMENT '',\\n\" +\n \" `target_processing_time` double COMMENT '',\\n\" +\n \" `response_processing_time` double COMMENT '',\\n\" +\n \" `elb_status_code` string COMMENT '',\\n\" +\n \" `target_status_code` string COMMENT '',\\n\" +\n \" `received_bytes` bigint COMMENT '',\\n\" +\n \" `sent_bytes` bigint COMMENT '',\\n\" +\n \" `request_verb` string COMMENT '',\\n\" +\n \" `request_url` string COMMENT '',\\n\" +\n \" `request_proto` string COMMENT '',\\n\" +\n \" `user_agent` string COMMENT '',\\n\" +\n \" `ssl_cipher` string COMMENT '',\\n\" +\n \" `ssl_protocol` string COMMENT '',\\n\" +\n \" `target_group_arn` string COMMENT '',\\n\" +\n \" `trace_id` string COMMENT '',\\n\" +\n \" `domain_name` string COMMENT '',\\n\" +\n \" `chosen_cert_arn` string COMMENT '',\\n\" +\n \" `matched_rule_priority` string COMMENT '',\\n\" +\n \" `request_creation_time` string COMMENT '',\\n\" +\n \" `actions_executed` string COMMENT '',\\n\" +\n \" `redirect_url` string COMMENT '',\\n\" +\n \" `lambda_error_reason` string COMMENT '',\\n\" +\n \" `target_port_list` string COMMENT '',\\n\" +\n \" `target_status_code_list` string COMMENT '',\\n\" +\n \" `new_field` string COMMENT '')\\n\" +\n 'PARTITIONED BY (\\n' +\n ' `date_utc_partition` string\\n' +\n ')\\n' +\n 'ROW FORMAT SERDE\\n' +\n \" 'org.apache.hadoop.hive.serde2.RegexSerDe'\\n\" +\n 'WITH SERDEPROPERTIES (\\n' +\n ' \\'input.regex\\'=\\'([^ ]) ([^ ]) ([^ ]) ([^ ]):([0-9]) ([^ ])[:-]([0-9]) ([-.0-9]) ([-.0-9]) ([-.0-9]) (\|[-0-9]) (-\|[-0-9]) ([-0-9]) ([-0-9]) \\\\\"([^ ]) ([^ ]) (- \|[^ ])\\\\\" \\\\\"([^\\\\\"])\\\\\" ([A-Z0-9-]+) ([A-Za-z0-9.-]) ([^ ]) \\\\\"([^\\\\\"])\\\\\" \\\\\"([^\\\\\"])\\\\\" \\\\\"([^\\\\\"])\\\\\" ([-.0-9]) ([^ ]) \\\\\"([^\\\\\"])\\\\\" \\\\\"([^\\\\\"])\\\\\" \\\\\"([^ ])\\\\\" \\\\\"([^s]+)\\\\\" \\\\\"([^s]+)\\\\\"(.)\\')\\n' +\n 'STORED AS INPUTFORMAT\\n' +\n \" 'org.apache.hadoop.mapred.TextInputFormat'\\n\" +\n 'OUTPUTFORMAT\\n' +\n \" 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\\n\" +\n 'LOCATION\\n' +\n \" '{{ALB_LOG_ROOT}}'\\n\";\n}\n//# sourceMappingURL=alb-athena-log-ratchet.js.map","import { GetNamedQueryCommand, GetQueryExecutionCommand, ListNamedQueriesCommand, StartQueryExecutionCommand, } from '@aws-sdk/client-athena';\nimport { StringRatchet } from '@bitblit/ratchet-common';\nimport { Logger } from '@bitblit/ratchet-common';\nimport { StopWatch } from '@bitblit/ratchet-common';\nimport { PromiseRatchet } from '@bitblit/ratchet-common';\nimport { GetObjectCommand } from '@aws-sdk/client-s3';\nimport tmp from 'tmp';\nimport fs from 'fs';\nimport { CsvRatchet } from '@bitblit/ratchet-node-only';\nimport { RequireRatchet } from '@bitblit/ratchet-common';\nexport class AthenaRatchet {\n athena;\n s3;\n outputLocation;\n constructor(athena, s3, outputLocation) {\n this.athena = athena;\n this.s3 = s3;\n this.outputLocation = outputLocation;\n RequireRatchet.notNullOrUndefined(athena);\n RequireRatchet.notNullOrUndefined(s3);\n RequireRatchet.notNullOrUndefined(outputLocation);\n RequireRatchet.true(outputLocation.startsWith('s3://'));\n }\n static athenaRowsToObject(input) {\n const colNames = input[0].Data.map((d) => d.VarCharValue);\n const temp = input.slice(1);\n const rval = temp.map((t) => {\n const newItem = {};\n for (let i = 0; i < t.Data.length; i++) {\n newItem[colNames[i]] = t.Data[i].VarCharValue;\n }\n return newItem;\n });\n return rval;\n }\n static applyParamsToQuery(query, queryParams) {\n let rval = query;\n if (!!rval && !!queryParams) {\n Object.keys(queryParams).forEach((k) => {\n const val = StringRatchet.safeString(queryParams[k]);\n const kk = '{' + k + '}';\n rval = rval.split(kk).join(val);\n });\n }\n return rval;\n }\n async fetchQueryIds() {\n const params = {\n NextToken: null,\n };\n let rval = [];\n let next = null;\n do {\n next = await this.athena.send(new ListNamedQueriesCommand(params));\n rval = rval.concat(next.NamedQueryIds);\n params.NextToken = next.NextToken;\n } while (!!params.NextToken);\n return rval;\n }\n async listQueries() {\n const rval = [];\n const ids = await this.fetchQueryIds();\n Logger.debug('Finding %d items', ids.length);\n for (let i = 0; i < ids.length; i++) {\n const params = {\n NamedQueryId: ids[i],\n };\n const val = await this.athena.send(new GetNamedQueryCommand(params));\n rval.push(val.NamedQuery);\n }\n return rval;\n }\n async findQueryByName(name) {\n const all = await this.listQueries();\n const rval = all.find((a) => a.Name.toLowerCase() == name.toLowerCase());\n return rval;\n }\n async runQueryToObjects(queryIn, queryParams = {}, pingTimeMS = 2000) {\n Logger.info('Running query to objects');\n const outputLoc = await this.runQueryToOutputLocation(queryIn, queryParams, pingTimeMS);\n Logger.info('Query succeeded, processing file from %s', outputLoc);\n const bucketName = outputLoc.substring(5, outputLoc.indexOf('/', 5));\n const obKey = outputLoc.substring(outputLoc.indexOf('/', 5) + 1);\n const req = {\n Bucket: bucketName,\n Key: obKey,\n };\n const getFileOut = await this.s3.send(new GetObjectCommand(req));\n const rval = await CsvRatchet.stringParse(getFileOut.Body.toString(), (p) => {\n return p;\n }, { columns: true, skip_empty_lines: true });\n return rval;\n }\n async runQueryToFile(queryIn, queryParams = {}, targetDataFileIn = null, pingTimeMS = 2000) {\n Logger.info('Running query to file');\n const outputLoc = await this.runQueryToOutputLocation(queryIn, queryParams, pingTimeMS);\n Logger.info('Query succeeded, pulling file from %s', outputLoc);\n const bucketName = outputLoc.substring(5, outputLoc.indexOf('/', 5));\n const obKey = outputLoc.substring(outputLoc.indexOf('/', 5) + 1);\n const req = {\n Bucket: bucketName,\n Key: obKey,\n };\n const targetDataFile = targetDataFileIn \|\| tmp.fileSync({ postfix: '.csv', keep: false }).name;\n const fileStream = fs.createWriteStream(targetDataFile);\n const output = await this.s3.send(new GetObjectCommand(req));\n const readStream = output.Body;\n readStream.pipe(fileStream);\n const rval = await PromiseRatchet.resolveOnEvent(readStream, ['finish', 'close'], ['error'], targetDataFile);\n Logger.silly('Response: %s', rval);\n return targetDataFile;\n }\n async runQueryToOutputLocation(queryIn, queryParams = {}, pingTimeMS = 2000) {\n let rval = null;\n const timer = new StopWatch();\n const query = AthenaRatchet.applyParamsToQuery(queryIn, queryParams);\n try {\n Logger.info('Starting query : %s', query);\n const token = StringRatchet.createType4Guid();\n const params = {\n QueryString: query,\n ResultConfiguration: {\n OutputLocation: this.outputLocation,\n EncryptionConfiguration: {\n EncryptionOption: 'SSE_S3',\n },\n },\n ClientRequestToken: token,\n QueryExecutionContext: {\n Database: 'default',\n },\n };\n const startToken = await this.athena.send(new StartQueryExecutionCommand(params));\n const getExecParams = {\n QueryExecutionId: startToken.QueryExecutionId,\n };\n const finalStates = ['FAILED', 'CANCELLED', 'SUCCEEDED'];\n let curState = await this.athena.send(new GetQueryExecutionCommand(getExecParams));\n while (finalStates.indexOf(curState.QueryExecution.Status.State) === -1) {\n await PromiseRatchet.createTimeoutPromise('wait', pingTimeMS);\n Logger.debug('%s : %s : %s', curState.QueryExecution.Status.State, timer.dump(), query);\n curState = await this.athena.send(new GetQueryExecutionCommand(getExecParams));\n }\n if (curState.QueryExecution.Status.State === 'FAILED') {\n Logger.warn('Query failed : %s', curState.QueryExecution.Status.StateChangeReason);\n }\n else if (curState.QueryExecution.Status.State === 'SUCCEEDED') {\n rval = curState.QueryExecution.ResultConfiguration.OutputLocation;\n }\n }\n catch (err) {\n Logger.warn('Failure : %s', err, err);\n }\n Logger.info('Query took %s : %s', timer.dump(), query);\n return rval;\n }\n}\n//# sourceMappingURL=athena-ratchet.js.map","export class RatchetAwsNodeOnlyInfo {\n constructor() { }\n static buildInformation() {\n const val = {\n version: 'LOCAL-SNAPSHOT',\n hash: 'LOCAL-HASH',\n branch: 'LOCAL-BRANCH',\n tag: 'LOCAL-TAG',\n timeBuiltISO: 'LOCAL-TIME-ISO',\n notes: 'LOCAL-NOTES',\n };\n return val;\n }\n}\n//# sourceMappingURL=ratchet-aws-node-only-info.js.map","import { StringRatchet } from '@bitblit/ratchet-common';\nimport { Logger } from '@bitblit/ratchet-common';\nimport { PromiseRatchet } from '@bitblit/ratchet-common';\nimport { RequireRatchet } from '@bitblit/ratchet-common';\nimport fs from 'fs';\nimport readline from 'readline';\nexport class DynamoExporter {\n constructor() { }\n static async importJsonLFileToTable(dynamo, tableName, filename) {\n RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');\n RequireRatchet.notNullOrUndefined(tableName, 'tableName');\n RequireRatchet.notNullOrUndefined(filename, 'filename');\n const fileStream = fs.createReadStream(filename);\n const rl = readline.createInterface({\n input: fileStream,\n crlfDelay: Infinity,\n });\n let rval = 0;\n for await (const line of rl) {\n if (rval % 100 === 0) {\n Logger.info('Importing line %d', rval);\n }\n if (StringRatchet.trimToNull(line)) {\n const parsed = JSON.parse(line);\n await dynamo.simplePut(tableName, parsed);\n rval++;\n }\n }\n return rval;\n }\n static async exportScanToJsonLFile(dynamo, scan, filename) {\n RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');\n RequireRatchet.notNullOrUndefined(scan, 'scan');\n RequireRatchet.notNullOrUndefined(filename, 'filename');\n const ws = fs.createWriteStream(filename);\n ws.on('end', () => {\n Logger.debug('Write complete');\n });\n const rval = await DynamoExporter.exportScanToJsonLWriteStream(dynamo, scan, ws);\n await PromiseRatchet.resolveOnEvent(ws, ['finish', 'close'], ['error']);\n ws.close();\n return rval;\n }\n static async exportQueryToJsonLFile(dynamo, qry, filename) {\n RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');\n RequireRatchet.notNullOrUndefined(qry, 'qry');\n RequireRatchet.notNullOrUndefined(filename, 'filename');\n const ws = fs.createWriteStream(filename);\n ws.on('end', () => {\n Logger.debug('Write complete');\n });\n const rval = await DynamoExporter.exportQueryToJsonLWriteStream(dynamo, qry, ws);\n await PromiseRatchet.resolveOnEvent(ws, ['finish', 'close'], ['error']);\n ws.close();\n return rval;\n }\n static async exportScanToJsonLWriteStream(dynamo, scan, target) {\n RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');\n RequireRatchet.notNullOrUndefined(scan, 'scan');\n RequireRatchet.notNullOrUndefined(target, 'target');\n const rval = await dynamo.fullyExecuteProcessOverScan(scan, async (row) => DynamoExporter.writeItemToJsonLStream(row, target, false));\n return rval;\n }\n static async exportQueryToJsonLWriteStream(dynamo, qry, target) {\n RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');\n RequireRatchet.notNullOrUndefined(qry, 'qry');\n RequireRatchet.notNullOrUndefined(target, 'target');\n const rval = await dynamo.fullyExecuteProcessOverQuery(qry, async (row) => DynamoExporter.writeItemToJsonLStream(row, target, false));\n return rval;\n }\n static writeItemToJsonLStream(item, target, includeNulls = false) {\n if (!!item \|\| includeNulls) {\n target.write(JSON.stringify(item) + '\\n');\n }\n }\n}\n//# sourceMappingURL=dynamo-exporter.js.map","import fs from 'fs';\nimport walk from 'walk';\nimport { S3Client } from '@aws-sdk/client-s3';\nimport path from 'path';\nimport mime from 'mime-types';\nimport { Logger } from '@bitblit/ratchet-common';\nimport { Upload } from '@aws-sdk/lib-storage';\nexport class SiteUploader {\n srcDir;\n bucketName;\n config;\n s3 = new S3Client({ region: 'us-east-1' });\n constructor(srcDir, bucketName, configFile) {\n this.srcDir = srcDir;\n this.bucketName = bucketName;\n this.config = JSON.parse(fs.readFileSync(configFile).toString('ascii'));\n }\n static createFromArgs(args) {\n if (args && args.length === 3) {\n const src = args[0];\n const bucket = args[1];\n const configFile = args[2];\n return new SiteUploader(src, bucket, configFile);\n }\n else {\n console.log('Usage : node ratchet-site-uploader {srcDir} {bucket} {configFile} (Found ' + args + ' arguments, need 3)');\n return null;\n }\n }\n static async runFromCliArgs(args) {\n const inst = SiteUploader.createFromArgs(args);\n return inst.runPump();\n }\n findMatch(prefix, fileName, config) {\n let found = null;\n if (prefix != null && fileName != null && config != null && config.mapping != null) {\n config.mapping.forEach((entry) => {\n if (found == null) {\n if (entry.prefixMatch == null \|\| prefix.match(entry.prefixMatch)) {\n if (entry.fileMatch == null \|\| fileName.match(entry.fileMatch)) {\n found = entry;\n }\n }\n }\n });\n }\n return found;\n }\n findMime(fileName, config) {\n let found = null;\n if (config != null && config.customMimeTypeMapping != null) {\n Object.keys(config.customMimeTypeMapping).forEach((k) => {\n if (found == null && fileName.endsWith(k)) {\n found = config.customMimeTypeMapping[k];\n }\n });\n }\n if (found == null) {\n found = mime.lookup(fileName);\n }\n if (found == null) {\n found = 'binary/octet-stream';\n }\n return found;\n }\n runPump() {\n return new Promise((resolve, reject) => {\n Logger.info('Uploading contents of %s to %s using %j as config', this.srcDir, this.bucketName, this.config);\n const options = {};\n const walker = walk.walk(this.srcDir, options);\n walker.on('file', function (root, fileStats, next) {\n Logger.info('Processing %j', fileStats.name);\n const prefix = root == this.srcDir ? '' : root.substring(this.srcDir.length + 1) + '/';\n const proc = this.findMatch(prefix, fileStats.name, this.config);\n const key = prefix + fileStats.name;\n Logger.info('Uploading file : %s/%s to key %s with %j', root, fileStats.name, key, proc);\n const params = proc && proc.putParams ? JSON.parse(JSON.stringify(proc.putParams)) : {};\n params.Bucket = this.bucketName;\n params.Key = key;\n params.Body = fs.readFileSync(path.join(root, fileStats.name));\n if (!params.ContentType) {\n params.ContentType = this.findMime(fileStats.name, this.config);\n }\n const upload = new Upload({\n client: this.s3,\n params: params,\n tags: [],\n queueSize: 4,\n partSize: 1024 1024 * 5,\n leavePartsOnError: false,\n });\n upload.on('httpUploadProgress', (progress) => {\n Logger.info('Uploading : %s', progress);\n });\n upload\n .done()\n .then((result) => {\n Logger.info('Finished upload of %s: %j', key, result);\n next();\n })\n .catch((err) => {\n Logger.warn('%s failed to upload : %s : Continuing', key, err);\n next();\n });\n }.bind(this));\n walker.on('errors', function (root, nodeStatsArray, next) {\n next();\n });\n walker.on('end', function () {\n Logger.info('All done');\n resolve(true);\n });\n });\n }\n}\n//# sourceMappingURL=site-uploader.js.map","import { Logger } from '@bitblit/ratchet-common';\nimport { Ec2Ratchet } from '@bitblit/ratchet-aws';\nimport { spawnSync } from 'child_process';\nimport fs from 'fs';\nimport os from 'os';\nimport path from 'path';\nexport class StartInstanceAndSsh {\n instanceId;\n publicKeyFile;\n instanceOsUser;\n region;\n availabilityZone;\n ec2Ratchet;\n constructor(instanceId, publicKeyFile = path.join(os.homedir(), '.ssh', 'id_rsa.pub'), instanceOsUser = 'ec2-user', region = 'us-east-1', availabilityZone = 'us-east-1a') {\n this.instanceId = instanceId;\n this.publicKeyFile = publicKeyFile;\n this.instanceOsUser = instanceOsUser;\n this.region = region;\n this.availabilityZone = availabilityZone;\n this.ec2Ratchet = new Ec2Ratchet(this.region, this.availabilityZone);\n }\n static createFromArgs(args) {\n if (args?.length === 1 \|\| args?.length === 2) {\n const instanceId = args[0];\n return new StartInstanceAndSsh(instanceId);\n }\n else {\n Logger.info('Usage : ratchet-start-instance-and-ssh {instanceId} {publicKeyFile} (Found %s arguments, need 1 or 2)', args);\n return null;\n }\n }\n static async runFromCliArgs(args) {\n const inst = StartInstanceAndSsh.createFromArgs(args);\n return inst.run();\n }\n async run() {\n let instance = await this.ec2Ratchet.describeInstance(this.instanceId);\n if (!!instance) {\n let launched = false;\n if (instance.State.Code == 16) {\n Logger.info('Instance is already running...');\n launched = true;\n }\n else {\n Logger.info('Instance is not running... starting up : %s', this.instanceId);\n launched = await this.ec2Ratchet.launchInstance(this.instanceId, 1000 * 30);\n }\n if (launched) {\n Logger.info('Uploading public key...');\n const publicKeyText = fs.readFileSync(this.publicKeyFile).toString();\n const publicKeyResponse = await this.ec2Ratchet.sendPublicKeyToEc2Instance(this.instanceId, publicKeyText, this.instanceOsUser);\n Logger.info('Key response : %j', publicKeyResponse);\n instance = instance && instance.PublicIpAddress ? instance : await this.ec2Ratchet.describeInstance(this.instanceId);\n Logger.info('Instance IP address is %s', instance.PublicIpAddress);\n const ret = spawnSync('ssh', [this.instanceOsUser + '@' + instance.PublicIpAddress], {\n stdio: 'inherit',\n });\n Logger.info('%j', ret);\n }\n else {\n Logger.info('Instance could not start - check logs');\n }\n }\n else {\n Logger.info('No such instance found - check your AWS keys? : %s', this.instanceId);\n }\n }\n}\n//# sourceMappingURL=start-instance-and-ssh.js.map","import { AbstractRatchetCliHandler } from '@bitblit/ratchet-node-only';\nimport { SiteUploader } from './site-uploader/site-uploader.js';\nimport { StartInstanceAndSsh } from './start-instance-and-ssh.js';\nimport { RatchetAwsNodeOnlyInfo } from '../build/ratchet-aws-node-only-info.js';\nexport class RatchetCliHandler extends AbstractRatchetCliHandler {\n fetchHandlerMap() {\n return {\n 'site-uploader': SiteUploader.runFromCliArgs,\n 'start-instance-and-ssh': StartInstanceAndSsh.runFromCliArgs,\n };\n }\n fetchVersionInfo() {\n return RatchetAwsNodeOnlyInfo.buildInformation();\n }\n}\n//# sourceMappingURL=ratchet-cli-handler.js.map","import { RequireRatchet } from '@bitblit/ratchet-common';\nimport { Logger } from '@bitblit/ratchet-common';\nimport { MultiStream } from '@bitblit/ratchet-common';\nimport unzipper from 'unzipper';\nimport { CsvRatchet } from '@bitblit/ratchet-node-only';\nimport { DateTime } from 'luxon';\nexport class EmailToDbInsertProcessor {\n canProcess(mail) {\n return true;\n }\n async processEmail(msg) {\n const rval = [];\n try {\n RequireRatchet.notNullOrUndefined(msg, 'msg');\n Logger.info('Processing Broadsign reach inbound inventory email');\n const data = msg.attachments[0].content;\n Logger.info('Unzipping attachment');\n const rs = new MultiStream(data);\n let wBuf = null;\n const prom = rs\n .pipe(unzipper.Parse())\n .on('entry', async (entry) => {\n if (entry.path.toLowerCase().endsWith('csv')) {\n wBuf = await entry.buffer();\n }\n else {\n Logger.info('Pass: %s', entry.path);\n entry.autodrain();\n }\n })\n .promise();\n await prom;\n const csvParsed = await CsvRatchet.stringParse(wBuf.toString(), (o) => o, {\n columns: false,\n skip_empty_lines: true,\n });\n if (csvParsed.length > 1) {\n const dropTable = 'drop table if exists sample';\n let createTable = 'create table sample (pump_date varchar(255),';\n const colNames = csvParsed[0];\n let insertPrefix = 'insert into sample (pump_date,';\n let insertQ = '?,';\n for (let i = 0; i < colNames.length; i++) {\n if (i > 0) {\n createTable += ', ';\n insertPrefix += ', ';\n insertQ += ', ';\n }\n const kOut = colNames[i].toLowerCase().split(' ').join('_');\n insertPrefix += kOut;\n insertQ += '?';\n createTable += kOut + ' varchar(255)';\n if (kOut === 'id') {\n createTable += ' primary key';\n }\n else if (kOut === 'device_id') {\n createTable += ' unique';\n }\n }\n createTable += ')';\n insertPrefix += ') values ';\n Logger.info('Recreating table');\n const dropRes = rval.push({ statement: dropTable });\n const createRes = rval.push({ statement: createTable });\n const pumpDate = DateTime.utc().toISO();\n let insertStmt = insertPrefix;\n let insertParams = [];\n for (let i = 1; i < csvParsed.length; i++) {\n if (insertStmt > insertPrefix) {\n insertStmt += ',';\n }\n insertStmt += '(' + insertQ + ')';\n insertParams = insertParams.concat(pumpDate, csvParsed[i]);\n if (i % 25 === 0 \|\| i === csvParsed.length - 1) {\n rval.push({ statement: insertStmt, params: insertParams });\n insertStmt = insertPrefix;\n insertParams = [];\n Logger.info('Inserted %d of %d rows', i, csvParsed.length);\n }\n }\n Logger.info('Finished insertion of %d rows', csvParsed.length);\n }\n }\n catch (err) {\n Logger.error('Failure: %s : %j', err, rval, err);\n }\n return rval;\n }\n}\n//# sourceMappingURL=email-to-db-insert-processor.js.map","import { simpleParser } from 'mailparser';\nimport { Logger } from '@bitblit/ratchet-common';\nimport { RequireRatchet } from '@bitblit/ratchet-common';\nexport class InboundEmailRatchet {\n cache;\n processors;\n constructor(cache, processors) {\n this.cache = cache;\n this.processors = processors;\n RequireRatchet.notNullOrUndefined(this.cache, 'cache');\n RequireRatchet.notNullOrUndefined(this.cache.getDefaultBucket(), 'cache.defaultBucket');\n }\n async processEmailFromS3(key) {\n const rval = false;\n if (await this.cache.fileExists(key)) {\n const data = await this.cache.fetchCacheFileAsString(key);\n return this.processEmailFromBuffer(new Buffer(data));\n }\n else {\n Logger.warn('Cannot process inbound email - no such key : %s', key);\n }\n return rval;\n }\n async processEmailFromBuffer(buf) {\n const rval = false;\n RequireRatchet.notNullOrUndefined(buf, 'buf');\n Logger.info('Processing inbound email - size %d bytes', buf.length);\n const message = await simpleParser(buf);\n Logger.info('Found mail from \"%s\" subject \"%s\" with %d attachments', message?.from?.text, message?.subject, message?.attachments?.length);\n let procd = false;\n for (let i = 0; i < this.processors.length && !procd; i++) {\n if (this.processors[i].canProcess(message)) {\n Logger.info('Processing message with processor %d', i);\n const result = await this.processors[i].processEmail(message);\n Logger.info('Result was : %j', result);\n procd = true;\n }\n }\n return procd;\n }\n}\n//# sourceMappingURL=inbound-email-ratchet.js.map","export class SampleEmailProcessor {\n canProcess(mail) {\n return true;\n }\n async processEmail(msg) {\n return msg.body;\n }\n}\n//# sourceMappingURL=sample-email-processor.js.map","import crypto from 'crypto';\nimport fs from 'fs';\nimport path from 'path';\nimport { Logger, RequireRatchet, StringRatchet } from '@bitblit/ratchet-common';\nexport class S3CacheToLocalDiskRatchet {\n s3;\n tmpFolder;\n cacheTimeoutSeconds;\n static DEFAULT_CACHE_TIMEOUT_SEC = 7 * 24 * 3600;\n currentlyLoading = new Map();\n constructor(s3, tmpFolder, cacheTimeoutSeconds = S3CacheToLocalDiskRatchet.DEFAULT_CACHE_TIMEOUT_SEC) {\n this.s3 = s3;\n this.tmpFolder = tmpFolder;\n this.cacheTimeoutSeconds = cacheTimeoutSeconds;\n RequireRatchet.notNullOrUndefined(s3, 's3');\n RequireRatchet.notNullOrUndefined(StringRatchet.trimToNull(tmpFolder));\n RequireRatchet.true(fs.existsSync(tmpFolder), 'folder must exist : ' + tmpFolder);\n }\n async getFileString(key) {\n const buf = await this.getFileBuffer(key);\n return buf ? buf.toString() : null;\n }\n keyToLocalCachePath(key) {\n const cachedHash = this.generateCacheHash(this.s3.getDefaultBucket() + '/' + key);\n const rval = path.join(this.tmpFolder, cachedHash);\n return rval;\n }\n removeCacheFileForKey(key) {\n const localCachePath = this.keyToLocalCachePath(key);\n Logger.info('Removing cache file for %s : %s', key, localCachePath);\n if (fs.existsSync(localCachePath)) {\n fs.unlinkSync(localCachePath);\n }\n else {\n Logger.debug('Skipping delete for %s - does not exist', localCachePath);\n }\n }\n async getFileBuffer(key) {\n const localCachePath = this.keyToLocalCachePath(key);\n let rval = null;\n rval = this.getCacheFileAsBuffer(localCachePath);\n if (!rval) {\n Logger.info('No cache. Downloading File s3://%s/%s to %s', this.s3.getDefaultBucket(), key, localCachePath);\n try {\n let prom = this.currentlyLoading.get(key);\n if (prom) {\n Logger.info('Already running - wait for that');\n }\n else {\n Logger.info('Not running - start');\n prom = this.updateLocalCacheFile(key, localCachePath);\n this.currentlyLoading.set(key, prom);\n }\n rval = await prom;\n this.currentlyLoading.delete(key);\n }\n catch (err) {\n Logger.warn('File %s/%s does not exist. Err code: %s', this.s3.getDefaultBucket(), key, err);\n }\n }\n else {\n Logger.info('Found cache file for s3://%s/%s. Local path %s', this.s3.getDefaultBucket(), key, localCachePath);\n }\n return rval;\n }\n async updateLocalCacheFile(key, localCachePath) {\n const rval = await this.s3.fetchCacheFileAsBuffer(key);\n if (rval && rval.length > 0) {\n Logger.info('Saving %d bytes to disk for cache', rval.length);\n fs.writeFileSync(localCachePath, rval);\n }\n return rval;\n }\n getCacheFileAsString(filePath) {\n const buf = this.getCacheFileAsBuffer(filePath);\n return buf ? buf.toString() : null;\n }\n getCacheFileAsBuffer(filePath) {\n if (!fs.existsSync(filePath)) {\n return null;\n }\n try {\n const stats = fs.statSync(filePath);\n const now = new Date().getTime();\n const duration = (now - stats.ctimeMs) / 1000;\n if (duration >= this.cacheTimeoutSeconds) {\n return null;\n }\n else {\n const rval = fs.readFileSync(filePath);\n return rval;\n }\n }\n catch (err) {\n Logger.warn('Error getting s3 cache file %s', err);\n }\n return null;\n }\n generateCacheHash(hashVal) {\n const rval = crypto.createHash('md5').update(hashVal).digest('hex');\n return rval;\n }\n}\n//# sourceMappingURL=s3-cache-to-local-disk-ratchet.js.map"],"names":["AlbAthenaLogRatchet","athena","athenaTableName","constructor","this","RequireRatchet","notNullOrUndefined","StringRatchet","trimToNull","updatePartitions","rootPath","s3","startTimeEpochMS","Date","getTime","endTimeEpochMS","true","S3Ratchet","checkS3UrlForValidity","Logger","info","extractBucketFromURL","extractKeyFromURL","current","clauses","dateUtcVal","toISOString","substring","dateParts","split","push","length","stmt","join","runQueryToObjects","warn","createTable","replaceIfPresent","rval","err","tableCreateQry","readFileSync","path","EsmRatchet","fetchDirName","url","toString","error","readLogObjectsFromCsvStream","readStream","CsvRatchet","streamParse","p","readLogObjectsFromFile","fileName","fileParse","fetchAlbLogRecords","qry","tempFile","fetchAlbLogRecordsToFile","outputFileName","qrySt","requestUrlFilter","limit","runQueryToFile","static","AthenaRatchet","outputLocation","startsWith","athenaRowsToObject","input","colNames","Data","map","d","VarCharValue","slice","t","newItem","i","applyParamsToQuery","query","queryParams","Object","keys","forEach","k","val","safeString","kk","fetchQueryIds","params","NextToken","next","send","ListNamedQueriesCommand","concat","NamedQueryIds","listQueries","ids","debug","NamedQueryId","GetNamedQueryCommand","NamedQuery","findQueryByName","name","find","a","Name","toLowerCase","queryIn","pingTimeMS","outputLoc","runQueryToOutputLocation","req","Bucket","indexOf","Key","getFileOut","GetObjectCommand","stringParse","Body","columns","skip_empty_lines","targetDataFileIn","targetDataFile","tmp","fileSync","postfix","keep","fileStream","fs","createWriteStream","pipe","PromiseRatchet","resolveOnEvent","silly","timer","StopWatch","token","createType4Guid","QueryString","ResultConfiguration","OutputLocation","EncryptionConfiguration","EncryptionOption","ClientRequestToken","QueryExecutionContext","Database","getExecParams","QueryExecutionId","StartQueryExecutionCommand","finalStates","curState","GetQueryExecutionCommand","QueryExecution","Status","State","createTimeoutPromise","dump","StateChangeReason","RatchetAwsNodeOnlyInfo","buildInformation","version","hash","branch","tag","timeBuiltISO","notes","DynamoExporter","importJsonLFileToTable","dynamo","tableName","filename","createReadStream","rl","readline","createInterface","crlfDelay","Infinity","line","parsed","JSON","parse","simplePut","exportScanToJsonLFile","scan","ws","on","exportScanToJsonLWriteStream","close","exportQueryToJsonLFile","exportQueryToJsonLWriteStream","target","fullyExecuteProcessOverScan","async","row","writeItemToJsonLStream","fullyExecuteProcessOverQuery","item","includeNulls","write","stringify","SiteUploader","srcDir","bucketName","config","S3Client","region","configFile","createFromArgs","args","src","bucket","console","log","runFromCliArgs","runPump","findMatch","prefix","found","mapping","entry","prefixMatch","match","fileMatch","findMime","customMimeTypeMapping","endsWith","mime","lookup","Promise","resolve","reject","walker","walk","root","fileStats","proc","key","putParams","ContentType","upload","Upload","client","tags","queueSize","partSize","leavePartsOnError","progress","done","then","result","catch","bind","nodeStatsArray","StartInstanceAndSsh","instanceId","publicKeyFile","instanceOsUser","availabilityZone","ec2Ratchet","os","homedir","Ec2Ratchet","run","instance","describeInstance","launched","Code","launchInstance","publicKeyText","publicKeyResponse","sendPublicKeyToEc2Instance","PublicIpAddress","ret","spawnSync","stdio","RatchetCliHandler","AbstractRatchetCliHandler","fetchHandlerMap","fetchVersionInfo","EmailToDbInsertProcessor","canProcess","mail","processEmail","msg","data","attachments","content","rs","MultiStream","wBuf","prom","unzipper","Parse","buffer","autodrain","promise","csvParsed","o","dropTable","insertPrefix","insertQ","kOut","statement","pumpDate","DateTime","utc","toISO","insertStmt","insertParams","InboundEmailRatchet","cache","processors","getDefaultBucket","processEmailFromS3","fileExists","fetchCacheFileAsString","processEmailFromBuffer","Buffer","buf","message","simpleParser","from","text","subject","procd","SampleEmailProcessor","body","S3CacheToLocalDiskRatchet","tmpFolder","cacheTimeoutSeconds","currentlyLoading","Map","DEFAULT_CACHE_TIMEOUT_SEC","existsSync","getFileString","getFileBuffer","keyToLocalCachePath","cachedHash","generateCacheHash","removeCacheFileForKey","localCachePath","unlinkSync","getCacheFileAsBuffer","get","updateLocalCacheFile","set","delete","fetchCacheFileAsBuffer","writeFileSync","getCacheFileAsString","filePath","stats","statSync","now","ctimeMs","hashVal","crypto","createHash","update","digest"],"mappings":"s4BAQO,MAAMA,EACTC,OACAC,gBACA,WAAAC,CAAYF,EAAQC,GAChBE,KAAKH,OAASA,EACdG,KAAKF,gBAAkBA,EACvBG,EAAeC,mBAAmBL,EAAQ,UAC1CI,EAAeC,mBAAmBC,EAAcC,WAAWN,GAAkB,kBAChF,CACD,sBAAMO,CAAiBC,EAAUC,EAAIC,GAAmB,IAAIC,MAAOC,UAAY,MAAqBC,GAAiB,IAAIF,MAAOC,WAC5HT,EAAeW,KAAKC,EAAUC,sBAAsBR,GAAW,uBAC/DL,EAAeC,mBAAmBK,EAAI,MACtCQ,EAAOC,KAAK,qCAAsChB,KAAKF,gBAAiBQ,GACrDO,EAAUI,qBAAqBX,GAClCO,EAAUK,kBAAkBZ,GAC5C,IAAIa,EAAUX,EACd,MAAMY,EAAU,GAChB,KAAOD,EAAUR,GAAgB,CAC7B,MAAMU,EAAa,IAAIZ,KAAKU,GAASG,cAAcC,UAAU,EAAG,IAChER,EAAOC,KAAK,OAAQK,GACpB,MAAMG,EAAYH,EAAWI,MAAM,KACnCL,EAAQM,KAAK,kCACTL,EACA,gBACAf,EACA,IACAkB,EAAU,GACV,IACAA,EAAU,GACV,IACAA,EAAU,GACV,KACJL,GAAW,KACd,CACD,GAAIC,EAAQO,OAAS,EAAG,CACpB,MAAMC,EAAO,eAAiB5B,KAAKF,gBAAkB,wBAA0BsB,EAAQS,KAAK,YACtF7B,KAAKH,OAAOiC,kBAAkBF,EACvC,MAEGb,EAAOgB,KAAK,0DAEhB,OAAOX,CACV,CACD,iBAAMY,CAAY1B,EAAU2B,GAAmB,GAC3ChC,EAAeW,KAAKC,EAAUC,sBAAsBR,GAAW,uBAC/D,IAAI4B,GAAO,EAEX,GADAnB,EAAOC,KAAK,wBAAyBhB,KAAKF,iBACtCmC,EAAkB,CAClBlB,EAAOC,KAAK,mDACZ,UACUhB,KAAKH,OAAOiC,kBAAkB,cAAgB9B,KAAKF,gBAC5D,CACD,MAAOqC,GACHpB,EAAOC,KAAK,kBAAmBmB,EAClC,CACJ,CACD,IAAIC,EAAiBC,EAAaC,EAAKT,KAAKU,EAAWC,yBAAyBC,KAAM,uCAAuCC,WAC7HN,EAAiBA,EAAeX,MAAM,kBAAkBI,KAAK7B,KAAKF,iBAClEsC,EAAiBA,EAAeX,MAAM,oBAAoBI,KAAKvB,GAC/DS,EAAOC,KAAK,yBAA0BoB,GACtC,UACUpC,KAAKH,OAAOiC,kBAAkBM,GACpCF,GAAO,CACV,CACD,MAAOC,GACHpB,EAAO4B,MAAM,4BAA6BR,EAC7C,CACD,OAAOD,CACV,CACD,wCAAaU,CAA4BC,GACrC,OAAOC,EAAWC,YAAYF,GAAaG,GAAMA,GACpD,CACD,mCAAaC,CAAuBC,GAChC,OAAOJ,EAAWK,UAAUD,GAAWF,GAAMA,GAChD,CACD,wBAAMI,CAAmBC,GACrB,MAAMC,QAAiBtD,KAAKuD,yBAAyBF,GACrD,OAAOzD,EAAoBqD,uBAAuBK,EACrD,CACD,8BAAMC,CAAyBF,EAAKG,EAAiB,MACjDzC,EAAOC,KAAK,mBAAoBhB,KAAKF,gBAAiBuD,GACtD,IAAII,EAAQ,iBAAmBzD,KAAKF,gBAAkB,cAClDuD,EAAI7C,mBACA6C,EAAI7C,mBACJiD,GAAS,iBAAmB,IAAIhD,KAAK4C,EAAI7C,kBAAkBc,cAAgB,IAC3EmC,GAAS,8BAAgC,IAAIhD,KAAK4C,EAAI7C,kBAAkBc,cAAcC,UAAU,EAAG,IAAM,KAEzG8B,EAAI1C,iBACJ8C,GAAS,gBAAkB,IAAIhD,KAAK4C,EAAI1C,gBAAgBW,cAAgB,IACxEmC,GAAS,8BAAgC,IAAIhD,KAAK4C,EAAI1C,gBAAgBW,cAAcC,UAAU,EAAG,IAAM,KAEvG8B,EAAIK,mBACJD,GAAS,0BAA4BJ,EAAIK,iBAAmB,KAE5DL,EAAIM,QACJF,GAAS,UAAYJ,EAAIM,QAIjC,aADqB3D,KAAKH,OAAO+D,eAAeH,EAAO,KAAMD,EAEhE,CACDK,8BAAgC,k+DCnG7B,MAAMC,EACTjE,OACAU,GACAwD,eACA,WAAAhE,CAAYF,EAAQU,EAAIwD,GACpB/D,KAAKH,OAASA,EACdG,KAAKO,GAAKA,EACVP,KAAK+D,eAAiBA,EACtB9D,EAAeC,mBAAmBL,GAClCI,EAAeC,mBAAmBK,GAClCN,EAAeC,mBAAmB6D,GAClC9D,EAAeW,KAAKmD,EAAeC,WAAW,SACjD,CACD,yBAAOC,CAAmBC,GACtB,MAAMC,EAAWD,EAAM,GAAGE,KAAKC,KAAKC,GAAMA,EAAEC,eAS5C,OARaL,EAAMM,MAAM,GACPH,KAAKI,IACnB,MAAMC,EAAU,CAAA,EAChB,IAAK,IAAIC,EAAI,EAAGA,EAAIF,EAAEL,KAAKzC,OAAQgD,IAC/BD,EAAQP,EAASQ,IAAMF,EAAEL,KAAKO,GAAGJ,aAErC,OAAOG,CAAO,GAGrB,CACD,yBAAOE,CAAmBC,EAAOC,GAC7B,IAAI5C,EAAO2C,EAQX,OAPM3C,GAAU4C,GACZC,OAAOC,KAAKF,GAAaG,SAASC,IAC9B,MAAMC,EAAMhF,EAAciF,WAAWN,EAAYI,IAC3CG,EAAK,IAAMH,EAAI,IACrBhD,EAAOA,EAAKT,MAAM4D,GAAIxD,KAAKsD,EAAI,IAGhCjD,CACV,CACD,mBAAMoD,GACF,MAAMC,EAAS,CACXC,UAAW,MAEf,IAAItD,EAAO,GACPuD,EAAO,KACX,GACIA,QAAazF,KAAKH,OAAO6F,KAAK,IAAIC,EAAwBJ,IAC1DrD,EAAOA,EAAK0D,OAAOH,EAAKI,eACxBN,EAAOC,UAAYC,EAAKD,gBACjBD,EAAOC,WAClB,OAAOtD,CACV,CACD,iBAAM4D,GACF,MAAM5D,EAAO,GACP6D,QAAY/F,KAAKsF,gBACvBvE,EAAOiF,MAAM,mBAAoBD,EAAIpE,QACrC,IAAK,IAAIgD,EAAI,EAAGA,EAAIoB,EAAIpE,OAAQgD,IAAK,CACjC,MAAMY,EAAS,CACXU,aAAcF,EAAIpB,IAEhBQ,QAAYnF,KAAKH,OAAO6F,KAAK,IAAIQ,EAAqBX,IAC5DrD,EAAKR,KAAKyD,EAAIgB,WACjB,CACD,OAAOjE,CACV,CACD,qBAAMkE,CAAgBC,GAGlB,aAFkBrG,KAAK8F,eACNQ,MAAMC,GAAMA,EAAEC,KAAKC,eAAiBJ,EAAKI,eAE7D,CACD,uBAAM3E,CAAkB4E,EAAS5B,EAAc,CAAA,EAAI6B,EAAa,KAC5D5F,EAAOC,KAAK,4BACZ,MAAM4F,QAAkB5G,KAAK6G,yBAAyBH,EAAS5B,EAAa6B,GAC5E5F,EAAOC,KAAK,2CAA4C4F,GACxD,MAEME,EAAM,CACRC,OAHeH,EAAUrF,UAAU,EAAGqF,EAAUI,QAAQ,IAAK,IAI7DC,IAHUL,EAAUrF,UAAUqF,EAAUI,QAAQ,IAAK,GAAK,IAKxDE,QAAmBlH,KAAKO,GAAGmF,KAAK,IAAIyB,EAAiBL,IAI3D,aAHmBhE,EAAWsE,YAAYF,EAAWG,KAAK3E,YAAaM,GAC5DA,GACR,CAAEsE,SAAS,EAAMC,kBAAkB,GAEzC,CACD,oBAAM3D,CAAe8C,EAAS5B,EAAc,CAAA,EAAI0C,EAAmB,KAAMb,EAAa,KAClF5F,EAAOC,KAAK,yBACZ,MAAM4F,QAAkB5G,KAAK6G,yBAAyBH,EAAS5B,EAAa6B,GAC5E5F,EAAOC,KAAK,wCAAyC4F,GACrD,MAEME,EAAM,CACRC,OAHeH,EAAUrF,UAAU,EAAGqF,EAAUI,QAAQ,IAAK,IAI7DC,IAHUL,EAAUrF,UAAUqF,EAAUI,QAAQ,IAAK,GAAK,IAKxDS,EAAiBD,GAAoBE,EAAIC,SAAS,CAAEC,QAAS,OAAQC,MAAM,IAASxB,KACpFyB,EAAaC,EAAGC,kBAAkBP,GAElC5E,SADe7C,KAAKO,GAAGmF,KAAK,IAAIyB,EAAiBL,KAC7BO,KAC1BxE,EAAWoF,KAAKH,GAChB,MAAM5F,QAAagG,EAAeC,eAAetF,EAAY,CAAC,SAAU,SAAU,CAAC,SAAU4E,GAE7F,OADA1G,EAAOqH,MAAM,eAAgBlG,GACtBuF,CACV,CACD,8BAAMZ,CAAyBH,EAAS5B,EAAc,CAAA,EAAI6B,EAAa,KACnE,IAAIzE,EAAO,KACX,MAAMmG,EAAQ,IAAIC,EACZzD,EAAQf,EAAcc,mBAAmB8B,EAAS5B,GACxD,IACI/D,EAAOC,KAAK,sBAAuB6D,GACnC,MAAM0D,EAAQpI,EAAcqI,kBACtBjD,EAAS,CACXkD,YAAa5D,EACb6D,oBAAqB,CACjBC,eAAgB3I,KAAK+D,eACrB6E,wBAAyB,CACrBC,iBAAkB,WAG1BC,mBAAoBP,EACpBQ,sBAAuB,CACnBC,SAAU,YAIZC,EAAgB,CAClBC,wBAFqBlJ,KAAKH,OAAO6F,KAAK,IAAIyD,EAA2B5D,KAExC2D,kBAE3BE,EAAc,CAAC,SAAU,YAAa,aAC5C,IAAIC,QAAiBrJ,KAAKH,OAAO6F,KAAK,IAAI4D,EAAyBL,IACnE,MAAsE,IAA/DG,EAAYpC,QAAQqC,EAASE,eAAeC,OAAOC,cAChDvB,EAAewB,qBAAqB,OAAQ/C,GAClD5F,EAAOiF,MAAM,eAAgBqD,EAASE,eAAeC,OAAOC,MAAOpB,EAAMsB,OAAQ9E,GACjFwE,QAAiBrJ,KAAKH,OAAO6F,KAAK,IAAI4D,EAAyBL,IAEtB,WAAzCI,EAASE,eAAeC,OAAOC,MAC/B1I,EAAOgB,KAAK,oBAAqBsH,EAASE,eAAeC,OAAOI,mBAElB,cAAzCP,EAASE,eAAeC,OAAOC,QACpCvH,EAAOmH,EAASE,eAAeb,oBAAoBC,eAE1D,CACD,MAAOxG,GACHpB,EAAOgB,KAAK,eAAgBI,EAAKA,EACpC,CAED,OADApB,EAAOC,KAAK,qBAAsBqH,EAAMsB,OAAQ9E,GACzC3C,CACV,EC3JE,MAAM2H,EACT,WAAA9J,GAAiB,CACjB,uBAAO+J,GASH,MARY,CACRC,QAAS,iBACTC,KAAM,aACNC,OAAQ,eACRC,IAAK,YACLC,aAAc,iBACdC,MAAO,cAGd,ECNE,MAAMC,EACT,WAAAtK,GAAiB,CACjB,mCAAauK,CAAuBC,EAAQC,EAAWC,GACnDxK,EAAeC,mBAAmBqK,EAAQ,UAC1CtK,EAAeC,mBAAmBsK,EAAW,aAC7CvK,EAAeC,mBAAmBuK,EAAU,YAC5C,MAAM3C,EAAaC,EAAG2C,iBAAiBD,GACjCE,EAAKC,EAASC,gBAAgB,CAChC3G,MAAO4D,EACPgD,UAAWC,MAEf,IAAI7I,EAAO,EACX,UAAW,MAAM8I,KAAQL,EAIrB,GAHIzI,EAAO,KAAQ,GACfnB,EAAOC,KAAK,oBAAqBkB,GAEjC/B,EAAcC,WAAW4K,GAAO,CAChC,MAAMC,EAASC,KAAKC,MAAMH,SACpBT,EAAOa,UAAUZ,EAAWS,GAClC/I,GACH,CAEL,OAAOA,CACV,CACD,kCAAamJ,CAAsBd,EAAQe,EAAMb,GAC7CxK,EAAeC,mBAAmBqK,EAAQ,UAC1CtK,EAAeC,mBAAmBoL,EAAM,QACxCrL,EAAeC,mBAAmBuK,EAAU,YAC5C,MAAMc,EAAKxD,EAAGC,kBAAkByC,GAChCc,EAAGC,GAAG,OAAO,KACTzK,EAAOiF,MAAM,iBAAiB,IAElC,MAAM9D,QAAamI,EAAeoB,6BAA6BlB,EAAQe,EAAMC,GAG7E,aAFMrD,EAAeC,eAAeoD,EAAI,CAAC,SAAU,SAAU,CAAC,UAC9DA,EAAGG,QACIxJ,CACV,CACD,mCAAayJ,CAAuBpB,EAAQlH,EAAKoH,GAC7CxK,EAAeC,mBAAmBqK,EAAQ,UAC1CtK,EAAeC,mBAAmBmD,EAAK,OACvCpD,EAAeC,mBAAmBuK,EAAU,YAC5C,MAAMc,EAAKxD,EAAGC,kBAAkByC,GAChCc,EAAGC,GAAG,OAAO,KACTzK,EAAOiF,MAAM,iBAAiB,IAElC,MAAM9D,QAAamI,EAAeuB,8BAA8BrB,EAAQlH,EAAKkI,GAG7E,aAFMrD,EAAeC,eAAeoD,EAAI,CAAC,SAAU,SAAU,CAAC,UAC9DA,EAAGG,QACIxJ,CACV,CACD,yCAAauJ,CAA6BlB,EAAQe,EAAMO,GACpD5L,EAAeC,mBAAmBqK,EAAQ,UAC1CtK,EAAeC,mBAAmBoL,EAAM,QACxCrL,EAAeC,mBAAmB2L,EAAQ,UAE1C,aADmBtB,EAAOuB,4BAA4BR,GAAMS,MAAOC,GAAQ3B,EAAe4B,uBAAuBD,EAAKH,GAAQ,IAEjI,CACD,0CAAaD,CAA8BrB,EAAQlH,EAAKwI,GACpD5L,EAAeC,mBAAmBqK,EAAQ,UAC1CtK,EAAeC,mBAAmBmD,EAAK,OACvCpD,EAAeC,mBAAmB2L,EAAQ,UAE1C,aADmBtB,EAAO2B,6BAA6B7I,GAAK0I,MAAOC,GAAQ3B,EAAe4B,uBAAuBD,EAAKH,GAAQ,IAEjI,CACD,6BAAOI,CAAuBE,EAAMN,EAAQO,GAAe,IACjDD,GAAQC,IACVP,EAAOQ,MAAMnB,KAAKoB,UAAUH,GAAQ,KAE3C,ECnEE,MAAMI,EACTC,OACAC,WACAC,OACAnM,GAAK,IAAIoM,EAAS,CAAEC,OAAQ,cAC5B,WAAA7M,CAAYyM,EAAQC,EAAYI,GAC5B7M,KAAKwM,OAASA,EACdxM,KAAKyM,WAAaA,EAClBzM,KAAK0M,OAASxB,KAAKC,MAAMpD,EAAG1F,aAAawK,GAAYnK,SAAS,SACjE,CACD,qBAAOoK,CAAeC,GAClB,GAAIA,GAAwB,IAAhBA,EAAKpL,OAAc,CAC3B,MAAMqL,EAAMD,EAAK,GACXE,EAASF,EAAK,GACdF,EAAaE,EAAK,GACxB,OAAO,IAAIR,EAAaS,EAAKC,EAAQJ,EACxC,CAGG,OADAK,QAAQC,IAAI,4EAA8EJ,EAAO,uBAC1F,IAEd,CACD,2BAAaK,CAAeL,GAExB,OADaR,EAAaO,eAAeC,GAC7BM,SACf,CACD,SAAAC,CAAUC,EAAQrK,EAAUwJ,GACxB,IAAIc,EAAQ,KAYZ,OAXc,MAAVD,GAA8B,MAAZrK,GAA8B,MAAVwJ,GAAoC,MAAlBA,EAAOe,SAC/Df,EAAOe,QAAQxI,SAASyI,IACP,MAATF,IACyB,MAArBE,EAAMC,aAAuBJ,EAAOK,MAAMF,EAAMC,gBACzB,MAAnBD,EAAMG,WAAqB3K,EAAS0K,MAAMF,EAAMG,cAChDL,EAAQE,EAGnB,IAGFF,CACV,CACD,QAAAM,CAAS5K,EAAUwJ,GACf,IAAIc,EAAQ,KAcZ,OAbc,MAAVd,GAAkD,MAAhCA,EAAOqB,uBACzBhJ,OAAOC,KAAK0H,EAAOqB,uBAAuB9I,SAASC,IAClC,MAATsI,GAAiBtK,EAAS8K,SAAS9I,KACnCsI,EAAQd,EAAOqB,sBAAsB7I,GACxC,IAGI,MAATsI,IACAA,EAAQS,EAAKC,OAAOhL,IAEX,MAATsK,IACAA,EAAQ,uBAELA,CACV,CACD,OAAAH,GACI,OAAO,IAAIc,SAAQ,CAACC,EAASC,KACzBtN,EAAOC,KAAK,oDAAqDhB,KAAKwM,OAAQxM,KAAKyM,WAAYzM,KAAK0M,QACpG,MACM4B,EAASC,EAAKA,KAAKvO,KAAKwM,OADd,CAAA,GAEhB8B,EAAO9C,GAAG,OAAQ,SAAUgD,EAAMC,EAAWhJ,GACzC1E,EAAOC,KAAK,gBAAiByN,EAAUpI,MACvC,MAAMkH,EAASiB,GAAQxO,KAAKwM,OAAS,GAAKgC,EAAKjN,UAAUvB,KAAKwM,OAAO7K,OAAS,GAAK,IAC7E+M,EAAO1O,KAAKsN,UAAUC,EAAQkB,EAAUpI,KAAMrG,KAAK0M,QACnDiC,EAAMpB,EAASkB,EAAUpI,KAC/BtF,EAAOC,KAAK,2CAA4CwN,EAAMC,EAAUpI,KAAMsI,EAAKD,GACnF,MAAMnJ,EAASmJ,GAAQA,EAAKE,UAAY1D,KAAKC,MAAMD,KAAKoB,UAAUoC,EAAKE,YAAc,CAAA,EACrFrJ,EAAOwB,OAAS/G,KAAKyM,WACrBlH,EAAO0B,IAAM0H,EACbpJ,EAAO8B,KAAOU,EAAG1F,aAAaC,EAAKT,KAAK2M,EAAMC,EAAUpI,OACnDd,EAAOsJ,cACRtJ,EAAOsJ,YAAc7O,KAAK8N,SAASW,EAAUpI,KAAMrG,KAAK0M,SAE5D,MAAMoC,EAAS,IAAIC,EAAO,CACtBC,OAAQhP,KAAKO,GACbgF,OAAQA,EACR0J,KAAM,GACNC,UAAW,EACXC,SAAU,QACVC,mBAAmB,IAEvBN,EAAOtD,GAAG,sBAAuB6D,IAC7BtO,EAAOC,KAAK,iBAAkBqO,EAAS,IAE3CP,EACKQ,OACAC,MAAMC,IACPzO,EAAOC,KAAK,4BAA6B2N,EAAKa,GAC9C/J,GAAM,IAELgK,OAAOtN,IACRpB,EAAOgB,KAAK,wCAAyC4M,EAAKxM,GAC1DsD,GAAM,GAE1B,EAAciK,KAAK1P,OACPsO,EAAO9C,GAAG,UAAU,SAAUgD,EAAMmB,EAAgBlK,GAChDA,GAChB,IACY6I,EAAO9C,GAAG,OAAO,WACbzK,EAAOC,KAAK,YACZoN,GAAQ,EACxB,GAAc,GAET,EC3GE,MAAMwB,EACTC,WACAC,cACAC,eACAnD,OACAoD,iBACAC,WACA,WAAAlQ,CAAY8P,EAAYC,EAAgBxN,EAAKT,KAAKqO,EAAGC,UAAW,OAAQ,cAAeJ,EAAiB,WAAYnD,EAAS,YAAaoD,EAAmB,cACzJhQ,KAAK6P,WAAaA,EAClB7P,KAAK8P,cAAgBA,EACrB9P,KAAK+P,eAAiBA,EACtB/P,KAAK4M,OAASA,EACd5M,KAAKgQ,iBAAmBA,EACxBhQ,KAAKiQ,WAAa,IAAIG,EAAWpQ,KAAK4M,OAAQ5M,KAAKgQ,iBACtD,CACD,qBAAOlD,CAAeC,GAClB,GAAqB,IAAjBA,GAAMpL,QAAiC,IAAjBoL,GAAMpL,OAAc,CAC1C,MAAMkO,EAAa9C,EAAK,GACxB,OAAO,IAAI6C,EAAoBC,EAClC,CAGG,OADA9O,EAAOC,KAAK,wGAAyG+L,GAC9G,IAEd,CACD,2BAAaK,CAAeL,GAExB,OADa6C,EAAoB9C,eAAeC,GACpCsD,KACf,CACD,SAAMA,GACF,IAAIC,QAAiBtQ,KAAKiQ,WAAWM,iBAAiBvQ,KAAK6P,YAC3D,GAAMS,EAAU,CACZ,IAAIE,GAAW,EASf,GAR2B,IAAvBF,EAAS7G,MAAMgH,MACf1P,EAAOC,KAAK,kCACZwP,GAAW,IAGXzP,EAAOC,KAAK,8CAA+ChB,KAAK6P,YAChEW,QAAiBxQ,KAAKiQ,WAAWS,eAAe1Q,KAAK6P,WAAY,MAEjEW,EAAU,CACVzP,EAAOC,KAAK,2BACZ,MAAM2P,EAAgB5I,EAAG1F,aAAarC,KAAK8P,eAAepN,WACpDkO,QAA0B5Q,KAAKiQ,WAAWY,2BAA2B7Q,KAAK6P,WAAYc,EAAe3Q,KAAK+P,gBAChHhP,EAAOC,KAAK,oBAAqB4P,GACjCN,EAAWA,GAAYA,EAASQ,gBAAkBR,QAAiBtQ,KAAKiQ,WAAWM,iBAAiBvQ,KAAK6P,YACzG9O,EAAOC,KAAK,4BAA6BsP,EAASQ,iBAClD,MAAMC,EAAMC,EAAU,MAAO,CAAChR,KAAK+P,eAAiB,IAAMO,EAASQ,iBAAkB,CACjFG,MAAO,YAEXlQ,EAAOC,KAAK,KAAM+P,EACrB,MAEGhQ,EAAOC,KAAK,wCAEnB,MAEGD,EAAOC,KAAK,qDAAsDhB,KAAK6P,WAE9E,EC9DE,MAAMqB,UAA0BC,EACnC,eAAAC,GACI,MAAO,CACH,gBAAiB7E,EAAaa,eAC9B,yBAA0BwC,EAAoBxC,eAErD,CACD,gBAAAiE,GACI,OAAOxH,EAAuBC,kBACjC,ECPE,MAAMwH,EACT,UAAAC,CAAWC,GACP,OAAO,CACV,CACD,kBAAMC,CAAaC,GACf,MAAMxP,EAAO,GACb,IACIjC,EAAeC,mBAAmBwR,EAAK,OACvC3Q,EAAOC,KAAK,sDACZ,MAAM2Q,EAAOD,EAAIE,YAAY,GAAGC,QAChC9Q,EAAOC,KAAK,wBACZ,MAAM8Q,EAAK,IAAIC,EAAYJ,GAC3B,IAAIK,EAAO,KACX,MAAMC,EAAOH,EACR7J,KAAKiK,EAASC,SACd3G,GAAG,SAASO,MAAO2B,IAChBA,EAAMpL,KAAKmE,cAAcuH,SAAS,OAClCgE,QAAatE,EAAM0E,UAGnBrR,EAAOC,KAAK,WAAY0M,EAAMpL,MAC9BoL,EAAM2E,YACT,IAEAC,gBACCL,EACN,MAAMM,QAAkBzP,EAAWsE,YAAY4K,EAAKtP,YAAa8P,GAAMA,GAAG,CACtElL,SAAS,EACTC,kBAAkB,IAEtB,GAAIgL,EAAU5Q,OAAS,EAAG,CACtB,MAAM8Q,EAAY,8BAClB,IAAIzQ,EAAc,+CAClB,MAAMmC,EAAWoO,EAAU,GAC3B,IAAIG,EAAe,iCACfC,EAAU,KACd,IAAK,IAAIhO,EAAI,EAAGA,EAAIR,EAASxC,OAAQgD,IAAK,CAClCA,EAAI,IACJ3C,GAAe,KACf0Q,GAAgB,KAChBC,GAAW,MAEf,MAAMC,EAAOzO,EAASQ,GAAG8B,cAAchF,MAAM,KAAKI,KAAK,KACvD6Q,GAAgBE,EAChBD,GAAW,IACX3Q,GAAe4Q,EAAO,gBACT,OAATA,EACA5Q,GAAe,eAED,cAAT4Q,IACL5Q,GAAe,UAEtB,CACDA,GAAe,IACf0Q,GAAgB,YAChB3R,EAAOC,KAAK,oBACIkB,EAAKR,KAAK,CAAEmR,UAAWJ,IACrBvQ,EAAKR,KAAK,CAAEmR,UAAW7Q,IADzC,MAEM8Q,EAAWC,EAASC,MAAMC,QAChC,IAAIC,EAAaR,EACbS,EAAe,GACnB,IAAK,IAAIxO,EAAI,EAAGA,EAAI4N,EAAU5Q,OAAQgD,IAC9BuO,EAAaR,IACbQ,GAAc,KAElBA,GAAc,IAAMP,EAAU,IAC9BQ,EAAeA,EAAavN,OAAOkN,EAAUP,EAAU5N,IACnDA,EAAI,IAAO,GAAKA,IAAM4N,EAAU5Q,OAAS,IACzCO,EAAKR,KAAK,CAAEmR,UAAWK,EAAY3N,OAAQ4N,IAC3CD,EAAaR,EACbS,EAAe,GACfpS,EAAOC,KAAK,yBAA0B2D,EAAG4N,EAAU5Q,SAG3DZ,EAAOC,KAAK,gCAAiCuR,EAAU5Q,OAC1D,CACJ,CACD,MAAOQ,GACHpB,EAAO4B,MAAM,mBAAoBR,EAAKD,EAAMC,EAC/C,CACD,OAAOD,CACV,ECpFE,MAAMkR,EACTC,MACAC,WACA,WAAAvT,CAAYsT,EAAOC,GACftT,KAAKqT,MAAQA,EACbrT,KAAKsT,WAAaA,EAClBrT,EAAeC,mBAAmBF,KAAKqT,MAAO,SAC9CpT,EAAeC,mBAAmBF,KAAKqT,MAAME,mBAAoB,sBACpE,CACD,wBAAMC,CAAmB7E,GAErB,SAAU3O,KAAKqT,MAAMI,WAAW9E,GAAM,CAClC,MAAMgD,QAAa3R,KAAKqT,MAAMK,uBAAuB/E,GACrD,OAAO3O,KAAK2T,uBAAuB,IAAIC,OAAOjC,GACjD,CAID,OAFI5Q,EAAOgB,KAAK,kDAAmD4M,IANtD,CAShB,CACD,4BAAMgF,CAAuBE,GAEzB5T,EAAeC,mBAAmB2T,EAAK,OACvC9S,EAAOC,KAAK,2CAA4C6S,EAAIlS,QAC5D,MAAMmS,QAAgBC,EAAaF,GACnC9S,EAAOC,KAAK,wDAAyD8S,GAASE,MAAMC,KAAMH,GAASI,QAASJ,GAASlC,aAAajQ,QAClI,IAAIwS,GAAQ,EACZ,IAAK,IAAIxP,EAAI,EAAGA,EAAI3E,KAAKsT,WAAW3R,SAAWwS,EAAOxP,IAClD,GAAI3E,KAAKsT,WAAW3O,GAAG4M,WAAWuC,GAAU,CACxC/S,EAAOC,KAAK,uCAAwC2D,GACpD,MAAM6K,QAAexP,KAAKsT,WAAW3O,GAAG8M,aAAaqC,GACrD/S,EAAOC,KAAK,kBAAmBwO,GAC/B2E,GAAQ,CACX,CAEL,OAAOA,CACV,ECvCE,MAAMC,EACT,UAAA7C,CAAWC,GACP,OAAO,CACV,CACD,kBAAMC,CAAaC,GACf,OAAOA,EAAI2C,IACd,ECFE,MAAMC,EACT/T,GACAgU,UACAC,oBACA3Q,iCAAmC,OACnC4Q,iBAAmB,IAAIC,IACvB,WAAA3U,CAAYQ,EAAIgU,EAAWC,EAAsBF,EAA0BK,2BACvE3U,KAAKO,GAAKA,EACVP,KAAKuU,UAAYA,EACjBvU,KAAKwU,oBAAsBA,EAC3BvU,EAAeC,mBAAmBK,EAAI,MACtCN,EAAeC,mBAAmBC,EAAcC,WAAWmU,IAC3DtU,EAAeW,KAAKmH,EAAG6M,WAAWL,GAAY,uBAAyBA,EAC1E,CACD,mBAAMM,CAAclG,GAChB,MAAMkF,QAAY7T,KAAK8U,cAAcnG,GACrC,OAAOkF,EAAMA,EAAInR,WAAa,IACjC,CACD,mBAAAqS,CAAoBpG,GAChB,MAAMqG,EAAahV,KAAKiV,kBAAkBjV,KAAKO,GAAGgT,mBAAqB,IAAM5E,GAE7E,OADarM,EAAKT,KAAK7B,KAAKuU,UAAWS,EAE1C,CACD,qBAAAE,CAAsBvG,GAClB,MAAMwG,EAAiBnV,KAAK+U,oBAAoBpG,GAChD5N,EAAOC,KAAK,kCAAmC2N,EAAKwG,GAChDpN,EAAG6M,WAAWO,GACdpN,EAAGqN,WAAWD,GAGdpU,EAAOiF,MAAM,0CAA2CmP,EAE/D,CACD,mBAAML,CAAcnG,GAChB,MAAMwG,EAAiBnV,KAAK+U,oBAAoBpG,GAChD,IAAIzM,EAAO,KAEX,GADAA,EAAOlC,KAAKqV,qBAAqBF,GAC5BjT,EAoBDnB,EAAOC,KAAK,iDAAkDhB,KAAKO,GAAGgT,mBAAoB5E,EAAKwG,OApBxF,CACPpU,EAAOC,KAAK,8CAA+ChB,KAAKO,GAAGgT,mBAAoB5E,EAAKwG,GAC5F,IACI,IAAIlD,EAAOjS,KAAKyU,iBAAiBa,IAAI3G,GACjCsD,EACAlR,EAAOC,KAAK,oCAGZD,EAAOC,KAAK,uBACZiR,EAAOjS,KAAKuV,qBAAqB5G,EAAKwG,GACtCnV,KAAKyU,iBAAiBe,IAAI7G,EAAKsD,IAEnC/P,QAAa+P,EACbjS,KAAKyU,iBAAiBgB,OAAO9G,EAChC,CACD,MAAOxM,GACHpB,EAAOgB,KAAK,0CAA2C/B,KAAKO,GAAGgT,mBAAoB5E,EAAKxM,EAC3F,CACJ,CAID,OAAOD,CACV,CACD,0BAAMqT,CAAqB5G,EAAKwG,GAC5B,MAAMjT,QAAalC,KAAKO,GAAGmV,uBAAuB/G,GAKlD,OAJIzM,GAAQA,EAAKP,OAAS,IACtBZ,EAAOC,KAAK,oCAAqCkB,EAAKP,QACtDoG,EAAG4N,cAAcR,EAAgBjT,IAE9BA,CACV,CACD,oBAAA0T,CAAqBC,GACjB,MAAMhC,EAAM7T,KAAKqV,qBAAqBQ,GACtC,OAAOhC,EAAMA,EAAInR,WAAa,IACjC,CACD,oBAAA2S,CAAqBQ,GACjB,IAAK9N,EAAG6M,WAAWiB,GACf,OAAO,KAEX,IACI,MAAMC,EAAQ/N,EAAGgO,SAASF,GACpBG,GAAM,IAAIvV,MAAOC,UAEvB,IADkBsV,EAAMF,EAAMG,SAAW,KACzBjW,KAAKwU,oBACjB,OAAO,KAIP,OADazM,EAAG1F,aAAawT,EAGpC,CACD,MAAO1T,GACHpB,EAAOgB,KAAK,iCAAkCI,EACjD,CACD,OAAO,IACV,CACD,iBAAA8S,CAAkBiB,GAEd,OADaC,EAAOC,WAAW,OAAOC,OAAOH,GAASI,OAAO,MAEhE"}
1	+ {"version":3,"file":"index.mjs","sources":["../src/athena/alb-athena-log-ratchet.ts","../src/athena/athena-ratchet.ts","../src/build/ratchet-aws-node-only-info.ts","../src/cli/dynamo-exporter.ts","../src/cli/site-uploader/site-uploader.ts","../src/cli/start-instance-and-ssh.ts","../src/cli/ratchet-cli-handler.ts","../src/mail/inbound/email-to-db-insert-processor.ts","../src/mail/inbound/inbound-email-ratchet.ts","../src/mail/inbound/sample-email-processor.ts","../src/s3/s3-cache-to-local-disk-ratchet.ts"],"sourcesContent":[null,null,null,null,null,null,null,null,null,null,null],"names":["AlbAthenaLogRatchet","athena","athenaTableName","constructor","this","RequireRatchet","notNullOrUndefined","StringRatchet","trimToNull","updatePartitions","rootPath","s3","startTimeEpochMS","Date","getTime","endTimeEpochMS","true","S3Ratchet","checkS3UrlForValidity","Logger","info","extractBucketFromURL","extractKeyFromURL","current","clauses","dateUtcVal","toISOString","substring","dateParts","split","push","length","stmt","join","runQueryToObjects","warn","createTable","replaceIfPresent","rval","err","tableCreateQry","readFileSync","path","EsmRatchet","fetchDirName","url","toString","error","readLogObjectsFromCsvStream","readStream","CsvRatchet","streamParse","p","readLogObjectsFromFile","fileName","fileParse","fetchAlbLogRecords","qry","tempFile","fetchAlbLogRecordsToFile","outputFileName","qrySt","requestUrlFilter","limit","runQueryToFile","static","AthenaRatchet","outputLocation","startsWith","athenaRowsToObject","input","colNames","Data","map","d","VarCharValue","slice","t","newItem","i","applyParamsToQuery","query","queryParams","Object","keys","forEach","k","val","safeString","kk","fetchQueryIds","params","NextToken","next","send","ListNamedQueriesCommand","concat","NamedQueryIds","listQueries","ids","debug","NamedQueryId","GetNamedQueryCommand","NamedQuery","findQueryByName","name","find","a","Name","toLowerCase","queryIn","pingTimeMS","outputLoc","runQueryToOutputLocation","req","Bucket","indexOf","Key","getFileOut","GetObjectCommand","stringParse","Body","columns","skip_empty_lines","targetDataFileIn","targetDataFile","tmp","fileSync","postfix","keep","fileStream","fs","createWriteStream","pipe","PromiseRatchet","resolveOnEvent","silly","timer","StopWatch","token","createType4Guid","QueryString","ResultConfiguration","OutputLocation","EncryptionConfiguration","EncryptionOption","ClientRequestToken","QueryExecutionContext","Database","getExecParams","QueryExecutionId","StartQueryExecutionCommand","finalStates","curState","GetQueryExecutionCommand","QueryExecution","Status","State","createTimeoutPromise","dump","StateChangeReason","RatchetAwsNodeOnlyInfo","buildInformation","version","hash","branch","tag","timeBuiltISO","notes","DynamoExporter","importJsonLFileToTable","dynamo","tableName","filename","createReadStream","rl","readline","createInterface","crlfDelay","Infinity","line","parsed","JSON","parse","simplePut","exportScanToJsonLFile","scan","ws","on","exportScanToJsonLWriteStream","close","exportQueryToJsonLFile","exportQueryToJsonLWriteStream","target","fullyExecuteProcessOverScan","async","row","writeItemToJsonLStream","fullyExecuteProcessOverQuery","item","includeNulls","write","stringify","SiteUploader","srcDir","bucketName","config","S3Client","region","configFile","createFromArgs","args","src","bucket","console","log","runFromCliArgs","runPump","findMatch","prefix","found","mapping","entry","prefixMatch","match","fileMatch","findMime","customMimeTypeMapping","endsWith","mime","lookup","Promise","resolve","reject","walker","walk","root","fileStats","proc","key","putParams","ContentType","upload","Upload","client","tags","queueSize","partSize","leavePartsOnError","progress","done","then","result","catch","bind","nodeStatsArray","StartInstanceAndSsh","instanceId","publicKeyFile","instanceOsUser","availabilityZone","ec2Ratchet","os","homedir","Ec2Ratchet","run","instance","describeInstance","launched","Code","launchInstance","publicKeyText","publicKeyResponse","sendPublicKeyToEc2Instance","PublicIpAddress","ret","spawnSync","stdio","RatchetCliHandler","AbstractRatchetCliHandler","fetchHandlerMap","fetchVersionInfo","EmailToDbInsertProcessor","canProcess","mail","processEmail","msg","data","attachments","content","rs","MultiStream","wBuf","prom","unzipper","Parse","buffer","autodrain","promise","csvParsed","o","dropTable","insertPrefix","insertQ","kOut","statement","pumpDate","DateTime","utc","toISO","insertStmt","insertParams","InboundEmailRatchet","cache","processors","getDefaultBucket","processEmailFromS3","fileExists","fetchCacheFileAsString","processEmailFromBuffer","Buffer","buf","message","simpleParser","from","text","subject","procd","SampleEmailProcessor","body","S3CacheToLocalDiskRatchet","tmpFolder","cacheTimeoutSeconds","currentlyLoading","Map","DEFAULT_CACHE_TIMEOUT_SEC","existsSync","getFileString","getFileBuffer","keyToLocalCachePath","cachedHash","generateCacheHash","removeCacheFileForKey","localCachePath","unlinkSync","getCacheFileAsBuffer","get","updateLocalCacheFile","set","delete","fetchCacheFileAsBuffer","writeFileSync","getCacheFileAsString","filePath","stats","statSync","now","ctimeMs","hashVal","crypto","createHash","update","digest"],"mappings":"44BAaaA,EACSC,OAA+BC,gBAAnD,WAAAC,CAAoBF,EAA+BC,GAA/BE,KAAMH,OAANA,EAA+BG,KAAeF,gBAAfA,EACjDG,EAAeC,mBAAmBL,EAAQ,UAC1CI,EAAeC,mBAAmBC,EAAcC,WAAWN,GAAkB,kBAC9E,CAEM,sBAAMO,CACXC,EACAC,EACAC,GAA2B,IAAIC,MAAOC,UAAY,MAClDC,GAAyB,IAAIF,MAAOC,WAEpCT,EAAeW,KAAKC,EAAUC,sBAAsBR,GAAW,uBAC/DL,EAAeC,mBAAmBK,EAAI,MACtCQ,EAAOC,KAAK,qCAAsChB,KAAKF,gBAAiBQ,GAC7CO,EAAUI,qBAAqBX,GAClCO,EAAUK,kBAAkBZ,GAEpD,IAAIa,EAAkBX,EACtB,MAAMY,EAAoB,GAC1B,KAAOD,EAAUR,GAAgB,CAC/B,MAAMU,EAAqB,IAAIZ,KAAKU,GAASG,cAAcC,UAAU,EAAG,IACxER,EAAOC,KAAK,OAAQK,GACpB,MAAMG,EAAsBH,EAAWI,MAAM,KAC7CL,EAAQM,KACN,kCACEL,EACA,gBACAf,EACA,IACAkB,EAAU,GACV,IACAA,EAAU,GACV,IACAA,EAAU,GACV,KAEJL,GAAW,KACZ,CACD,GAAIC,EAAQO,OAAS,EAAG,CACtB,MAAMC,EAAe,eAAiB5B,KAAKF,gBAAkB,wBAA0BsB,EAAQS,KAAK,YAC9F7B,KAAKH,OAAOiC,kBAAuBF,EAC1C,MACCb,EAAOgB,KAAK,0DAEd,OAAOX,CACR,CAEM,iBAAMY,CAAY1B,EAAkB2B,GAA4B,GACrEhC,EAAeW,KAAKC,EAAUC,sBAAsBR,GAAW,uBAC/D,IAAI4B,GAAgB,EAEpB,GADAnB,EAAOC,KAAK,wBAAyBhB,KAAKF,iBACtCmC,EAAkB,CACpBlB,EAAOC,KAAK,mDACZ,UACQhB,KAAKH,OAAOiC,kBAAuB,cAAgB9B,KAAKF,gBAC/D,CAAC,MAAOqC,GACPpB,EAAOC,KAAK,kBAAmBmB,EAChC,CACF,CAED,IAAIC,EAAyBC,EAC3BC,EAAKT,KAAKU,EAAWC,yBAAyBC,KAAM,uCACpDC,WACFN,EAAiBA,EAAeX,MAAM,kBAAkBI,KAAK7B,KAAKF,iBAClEsC,EAAiBA,EAAeX,MAAM,oBAAoBI,KAAKvB,GAC/DS,EAAOC,KAAK,yBAA0BoB,GAEtC,UACQpC,KAAKH,OAAOiC,kBAAuBM,GACzCF,GAAO,CACR,CAAC,MAAOC,GACPpB,EAAO4B,MAAM,4BAA6BR,EAC3C,CACD,OAAOD,CACR,CAEM,wCAAaU,CAA4BC,GAC9C,OAAOC,EAAWC,YAAYF,GAAaG,GAAMA,GAClD,CAEM,mCAAaC,CAAuBC,GACzC,OAAOJ,EAAWK,UAAUD,GAAWF,GAAMA,GAC9C,CAEM,wBAAMI,CAAmBC,GAC9B,MAAMC,QAAyBtD,KAAKuD,yBAAyBF,GAC7D,OAAOzD,EAAoBqD,uBAAuBK,EACnD,CAEM,8BAAMC,CAAyBF,EAAwBG,EAAyB,MACrFzC,EAAOC,KAAK,mBAAoBhB,KAAKF,gBAAiBuD,GAEtD,IAAII,EAAgB,iBAAmBzD,KAAKF,gBAAkB,cAC1DuD,EAAI7C,mBAEF6C,EAAI7C,mBACNiD,GAAS,iBAAmB,IAAIhD,KAAK4C,EAAI7C,kBAAkBc,cAAgB,IAC3EmC,GAAS,8BAAgC,IAAIhD,KAAK4C,EAAI7C,kBAAkBc,cAAcC,UAAU,EAAG,IAAM,KAEvG8B,EAAI1C,iBACN8C,GAAS,gBAAkB,IAAIhD,KAAK4C,EAAI1C,gBAAgBW,cAAgB,IACxEmC,GAAS,8BAAgC,IAAIhD,KAAK4C,EAAI1C,gBAAgBW,cAAcC,UAAU,EAAG,IAAM,KAErG8B,EAAIK,mBACND,GAAS,0BAA4BJ,EAAIK,iBAAmB,KAE1DL,EAAIM,QACNF,GAAS,UAAYJ,EAAIM,QAK7B,aAD6B3D,KAAKH,OAAO+D,eAAeH,EAAO,KAAMD,EAEtE,CAEMK,8BACL,w+DCzGSC,EACSjE,OAA8BU,GAAsBwD,eAAxE,WAAAhE,CAAoBF,EAA8BU,EAAsBwD,GAApD/D,KAAMH,OAANA,EAA8BG,KAAEO,GAAFA,EAAsBP,KAAc+D,eAAdA,EACtE9D,EAAeC,mBAAmBL,GAClCI,EAAeC,mBAAmBK,GAClCN,EAAeC,mBAAmB6D,GAClC9D,EAAeW,KAAKmD,EAAeC,WAAW,SAC/C,CAEM,yBAAOC,CAAsBC,GAClC,MAAMC,EAAqBD,EAAM,GAAGE,KAAKC,KAAKC,GAAMA,EAAEC,eAStD,OARoBL,EAAMM,MAAM,GACTH,KAAKI,IAC1B,MAAMC,EAAa,CAAA,EACnB,IAAK,IAAIC,EAAI,EAAGA,EAAIF,EAAEL,KAAKzC,OAAQgD,IACjCD,EAAQP,EAASQ,IAAMF,EAAEL,KAAKO,GAAGJ,aAEnC,OAAOG,CAAO,GAGjB,CAEM,yBAAOE,CAAsBC,EAAeC,GACjD,IAAI5C,EAAe2C,EAQnB,OAPM3C,GAAU4C,GACdC,OAAOC,KAAKF,GAAaG,SAASC,IAChC,MAAMC,EAAchF,EAAciF,WAAWN,EAAYI,IACnDG,EAAa,IAAMH,EAAI,IAC7BhD,EAAOA,EAAKT,MAAM4D,GAAIxD,KAAKsD,EAAI,IAG5BjD,CACR,CAEM,mBAAMoD,GACX,MAAMC,EAAS,CAEbC,UAAW,MAGb,IAAItD,EAAiB,GACjBuD,EAAsC,KAE1C,GACEA,QAAazF,KAAKH,OAAO6F,KAAK,IAAIC,EAAwBJ,IAC1DrD,EAAOA,EAAK0D,OAAOH,EAAKI,eACxBN,EAAOC,UAAYC,EAAKD,gBACfD,EAAOC,WAElB,OAAOtD,CACR,CAEM,iBAAM4D,GACX,MAAM5D,EAAqB,GACrB6D,QAAsB/F,KAAKsF,gBACjCvE,EAAOiF,MAAM,mBAAoBD,EAAIpE,QACrC,IAAK,IAAIgD,EAAI,EAAGA,EAAIoB,EAAIpE,OAAQgD,IAAK,CACnC,MAAMY,EAAS,CACbU,aAAcF,EAAIpB,IAEdQ,QAAwCnF,KAAKH,OAAO6F,KAAK,IAAIQ,EAAqBX,IACxFrD,EAAKR,KAAKyD,EAAIgB,WACf,CAED,OAAOjE,CACR,CAEM,qBAAMkE,CAAgBC,GAG3B,aAFgCrG,KAAK8F,eACRQ,MAAMC,GAAMA,EAAEC,KAAKC,eAAiBJ,EAAKI,eAEvE,CAEM,uBAAM3E,CAAqB4E,EAAiB5B,EAAmB,CAAA,EAAI6B,EAAa,KACrF5F,EAAOC,KAAK,4BACZ,MAAM4F,QAAkB5G,KAAK6G,yBAAyBH,EAAS5B,EAAa6B,GAC5E5F,EAAOC,KAAK,2CAA4C4F,GAExD,MAGME,EAAwB,CAC5BC,OAJyBH,EAAUrF,UAAU,EAAGqF,EAAUI,QAAQ,IAAK,IAKvEC,IAJoBL,EAAUrF,UAAUqF,EAAUI,QAAQ,IAAK,GAAK,IAMhEE,QAA2ClH,KAAKO,GAAGmF,KAAK,IAAIyB,EAAiBL,IAUnF,aARwBhE,EAAWsE,YACjCF,EAAWG,KAAK3E,YACfM,GACQA,GAET,CAAEsE,SAAS,EAAMC,kBAAkB,GAItC,CAEM,oBAAM3D,CAAe8C,EAAiB5B,EAAmB,CAAA,EAAI0C,EAA2B,KAAMb,EAAa,KAChH5F,EAAOC,KAAK,yBACZ,MAAM4F,QAAkB5G,KAAK6G,yBAAyBH,EAAS5B,EAAa6B,GAC5E5F,EAAOC,KAAK,wCAAyC4F,GAErD,MAGME,EAAwB,CAC5BC,OAJyBH,EAAUrF,UAAU,EAAGqF,EAAUI,QAAQ,IAAK,IAKvEC,IAJoBL,EAAUrF,UAAUqF,EAAUI,QAAQ,IAAK,GAAK,IAOhES,EAAyBD,GAAoBE,EAAIC,SAAS,CAAEC,QAAS,OAAQC,MAAM,IAASxB,KAC5FyB,EAA0BC,EAAGC,kBAAkBP,GAG/C5E,SAFuC7C,KAAKO,GAAGmF,KAAK,IAAIyB,EAAiBL,KAE3CO,KACpCxE,EAAWoF,KAAKH,GAEhB,MAAM5F,QAAqBgG,EAAeC,eAAuBtF,EAAY,CAAC,SAAU,SAAU,CAAC,SAAU4E,GAE7G,OADA1G,EAAOqH,MAAM,eAAgBlG,GACtBuF,CACR,CAEO,8BAAMZ,CAAyBH,EAAiB5B,EAAmB,CAAA,EAAI6B,EAAa,KAC1F,IAAIzE,EAAe,KACnB,MAAMmG,EAAmB,IAAIC,EACvBzD,EAAgBf,EAAcc,mBAAmB8B,EAAS5B,GAEhE,IACE/D,EAAOC,KAAK,sBAAuB6D,GAEnC,MAAM0D,EAAgBpI,EAAcqI,kBAC9BjD,EAAmC,CACvCkD,YAAa5D,EACb6D,oBAAqB,CAEnBC,eAAgB3I,KAAK+D,eACrB6E,wBAAyB,CACvBC,iBAAkB,WAItBC,mBAAoBP,EACpBQ,sBAAuB,CACrBC,SAAU,YAMRC,EAAqB,CACzBC,wBAHyDlJ,KAAKH,OAAO6F,KAAK,IAAIyD,EAA2B5D,KAG5E2D,kBAGzBE,EAAwB,CAAC,SAAU,YAAa,aACtD,IAAIC,QAAiDrJ,KAAKH,OAAO6F,KAAK,IAAI4D,EAAyBL,IACnG,MAAsE,IAA/DG,EAAYpC,QAAQqC,EAASE,eAAeC,OAAOC,cAClDvB,EAAewB,qBAAqB,OAAQ/C,GAClD5F,EAAOiF,MAAM,eAAgBqD,EAASE,eAAeC,OAAOC,MAAOpB,EAAMsB,OAAQ9E,GACjFwE,QAAiBrJ,KAAKH,OAAO6F,KAAK,IAAI4D,EAAyBL,IAGpB,WAAzCI,EAASE,eAAeC,OAAOC,MACjC1I,EAAOgB,KAAK,oBAAqBsH,EAASE,eAAeC,OAAOI,mBACd,cAAzCP,EAASE,eAAeC,OAAOC,QACxCvH,EAAOmH,EAASE,eAAeb,oBAAoBC,eAEtD,CAAC,MAAOxG,GACPpB,EAAOgB,KAAK,eAAgBI,EAAKA,EAClC,CAGD,OAFApB,EAAOC,KAAK,qBAAsBqH,EAAMsB,OAAQ9E,GAEzC3C,CACR,QCnMU2H,EAGX,WAAA9J,GAAwB,CAEjB,uBAAO+J,GASZ,MAR8B,CAC5BC,QAAS,iBACTC,KAAM,aACNC,OAAQ,eACRC,IAAK,YACLC,aAAc,iBACdC,MAAO,cAGV,QCRUC,EAGX,WAAAtK,GAAwB,CAEjB,mCAAauK,CAAuBC,EAAuBC,EAAmBC,GACnFxK,EAAeC,mBAAmBqK,EAAQ,UAC1CtK,EAAeC,mBAAmBsK,EAAW,aAC7CvK,EAAeC,mBAAmBuK,EAAU,YAE5C,MAAM3C,EAAaC,EAAG2C,iBAAiBD,GAEjCE,EAAKC,EAASC,gBAAgB,CAClC3G,MAAO4D,EACPgD,UAAWC,MAKb,IAAI7I,EAAe,EACnB,UAAW,MAAM8I,KAAQL,EAIvB,GAHIzI,EAAO,KAAQ,GACjBnB,EAAOC,KAAK,oBAAqBkB,GAE/B/B,EAAcC,WAAW4K,GAAO,CAClC,MAAMC,EAAcC,KAAKC,MAAMH,SACzBT,EAAOa,UAAUZ,EAAWS,GAClC/I,GACD,CAGH,OAAOA,CACR,CAEM,kCAAamJ,CAAsBd,EAAuBe,EAAwBb,GACvFxK,EAAeC,mBAAmBqK,EAAQ,UAC1CtK,EAAeC,mBAAmBoL,EAAM,QACxCrL,EAAeC,mBAAmBuK,EAAU,YAC5C,MAAMc,EAAkBxD,EAAGC,kBAAkByC,GAC7Cc,EAAGC,GAAG,OAAO,KACXzK,EAAOiF,MAAM,iBAAiB,IAGhC,MAAM9D,QAAqBmI,EAAeoB,6BAA6BlB,EAAQe,EAAMC,GAIrF,aAFMrD,EAAeC,eAAeoD,EAAI,CAAC,SAAU,SAAU,CAAC,UAC9DA,EAAGG,QACIxJ,CACR,CAEM,mCAAayJ,CAAuBpB,EAAuBlH,EAAwBoH,GACxFxK,EAAeC,mBAAmBqK,EAAQ,UAC1CtK,EAAeC,mBAAmBmD,EAAK,OACvCpD,EAAeC,mBAAmBuK,EAAU,YAE5C,MAAMc,EAAkBxD,EAAGC,kBAAkByC,GAC7Cc,EAAGC,GAAG,OAAO,KACXzK,EAAOiF,MAAM,iBAAiB,IAGhC,MAAM9D,QAAqBmI,EAAeuB,8BAA8BrB,EAAQlH,EAAKkI,GAIrF,aAFMrD,EAAeC,eAAeoD,EAAI,CAAC,SAAU,SAAU,CAAC,UAC9DA,EAAGG,QACIxJ,CACR,CAEM,yCAAauJ,CAA6BlB,EAAuBe,EAAwBO,GAC9F5L,EAAeC,mBAAmBqK,EAAQ,UAC1CtK,EAAeC,mBAAmBoL,EAAM,QACxCrL,EAAeC,mBAAmB2L,EAAQ,UAK1C,aAH2BtB,EAAOuB,4BAA4BR,GAAMS,MAAOC,GACzE3B,EAAe4B,uBAAuBD,EAAKH,GAAQ,IAGtD,CAEM,0CAAaD,CAA8BrB,EAAuBlH,EAAwBwI,GAC/F5L,EAAeC,mBAAmBqK,EAAQ,UAC1CtK,EAAeC,mBAAmBmD,EAAK,OACvCpD,EAAeC,mBAAmB2L,EAAQ,UAK1C,aAH2BtB,EAAO2B,6BAA6B7I,GAAK0I,MAAOC,GACzE3B,EAAe4B,uBAAuBD,EAAKH,GAAQ,IAGtD,CACM,6BAAOI,CAAuBE,EAAWN,EAAqBO,GAAwB,IACrFD,GAAQC,IACZP,EAAOQ,MAAMnB,KAAKoB,UAAUH,GAAQ,KAEvC,QC7FUI,EACHC,OACAC,WACAC,OACSnM,GAAe,IAAIoM,EAAS,CAAEC,OAAQ,cAEvD,WAAA7M,CAAYyM,EAAgBC,EAAoBI,GAC9C7M,KAAKwM,OAASA,EACdxM,KAAKyM,WAAaA,EAClBzM,KAAK0M,OAASxB,KAAKC,MAAMpD,EAAG1F,aAAawK,GAAYnK,SAAS,SAC/D,CAEM,qBAAOoK,CAAeC,GAC3B,GAAIA,GAAwB,IAAhBA,EAAKpL,OAAc,CAC7B,MAAMqL,EAAMD,EAAK,GACXE,EAASF,EAAK,GACdF,EAAaE,EAAK,GAExB,OAAO,IAAIR,EAAaS,EAAKC,EAAQJ,EACtC,CAEC,OADAK,QAAQC,IAAI,4EAA8EJ,EAAO,uBAC1F,IAEV,CAEM,2BAAaK,CAAeL,GAEjC,OAD2BR,EAAaO,eAAeC,GAC3CM,SACb,CAGD,SAAAC,CAAUC,EAAgBrK,EAAkBwJ,GAC1C,IAAIc,EAAQ,KAcZ,OAZc,MAAVD,GAA8B,MAAZrK,GAA8B,MAAVwJ,GAAoC,MAAlBA,EAAOe,SACjEf,EAAOe,QAAQxI,SAASyI,IACT,MAATF,IACuB,MAArBE,EAAMC,aAAuBJ,EAAOK,MAAMF,EAAMC,gBAC3B,MAAnBD,EAAMG,WAAqB3K,EAAS0K,MAAMF,EAAMG,cAClDL,EAAQE,EAGb,IAIEF,CACR,CAGD,QAAAM,CAAS5K,EAAkBwJ,GACzB,IAAIc,EAAQ,KAkBZ,OAhBc,MAAVd,GAAkD,MAAhCA,EAAOqB,uBAC3BhJ,OAAOC,KAAK0H,EAAOqB,uBAAuB9I,SAASC,IACpC,MAATsI,GAAiBtK,EAAS8K,SAAS9I,KACrCsI,EAAQd,EAAOqB,sBAAsB7I,GACtC,IAIQ,MAATsI,IACFA,EAAQS,EAAKC,OAAOhL,IAGT,MAATsK,IACFA,EAAQ,uBAGHA,CACR,CAED,OAAAH,GAEE,OAAO,IAAIc,SAAa,CAACC,EAASC,KAChCtN,EAAOC,KAAK,oDAAqDhB,KAAKwM,OAAQxM,KAAKyM,WAAYzM,KAAK0M,QAGpG,MACM4B,EAASC,EAAKA,KAAKvO,KAAKwM,OADd,CAAA,GAGhB8B,EAAO9C,GACL,OACA,SAAUgD,EAAMC,EAAWhJ,GACzB1E,EAAOC,KAAK,gBAAiByN,EAAUpI,MACvC,MAAMkH,EAAiBiB,GAAQxO,KAAKwM,OAAS,GAAKgC,EAAKjN,UAAUvB,KAAKwM,OAAO7K,OAAS,GAAK,IAErF+M,EAAY1O,KAAKsN,UAAUC,EAAQkB,EAAUpI,KAAMrG,KAAK0M,QACxDiC,EAAcpB,EAASkB,EAAUpI,KACvCtF,EAAOC,KAAK,2CAA4CwN,EAAMC,EAAUpI,KAAMsI,EAAKD,GAEnF,MAAMnJ,EAAcmJ,GAAQA,EAAKE,UAAY1D,KAAKC,MAAMD,KAAKoB,UAAUoC,EAAKE,YAAc,CAAA,EAE1FrJ,EAAOwB,OAAS/G,KAAKyM,WACrBlH,EAAO0B,IAAM0H,EACbpJ,EAAO8B,KAAOU,EAAG1F,aAAaC,EAAKT,KAAK2M,EAAMC,EAAUpI,OAEnDd,EAAOsJ,cACVtJ,EAAOsJ,YAAc7O,KAAK8N,SAASW,EAAUpI,KAAMrG,KAAK0M,SAG1D,MAAMoC,EAAiB,IAAIC,EAAO,CAChCC,OAAQhP,KAAKO,GACbgF,OAAQA,EACR0J,KAAM,GACNC,UAAW,EACXC,SAAU,QACVC,mBAAmB,IAGrBN,EAAOtD,GAAG,sBAAuB6D,IAC/BtO,EAAOC,KAAK,iBAAkBqO,EAAS,IAEzCP,EACGQ,OACAC,MAAMC,IACLzO,EAAOC,KAAK,4BAA6B2N,EAAKa,GAC9C/J,GAAM,IAEPgK,OAAOtN,IACNpB,EAAOgB,KAAK,wCAAyC4M,EAAKxM,GAC1DsD,GAAM,GAEZ,EAAEiK,KAAK1P,OAGTsO,EAAO9C,GAAG,UAAU,SAAUgD,EAAMmB,EAAgBlK,GAClDA,GACF,IAEA6I,EAAO9C,GAAG,OAAO,WACfzK,EAAOC,KAAK,YACZoN,GAAQ,EACV,GAAE,GAEL,QCtIUwB,EACHC,WACAC,cACAC,eACAnD,OACAoD,iBACAC,WAER,WAAAlQ,CACE8P,EACAC,EAAwBxN,EAAKT,KAAKqO,EAAGC,UAAW,OAAQ,cACxDJ,EAAyB,WACzBnD,EAAiB,YACjBoD,EAA2B,cAE3BhQ,KAAK6P,WAAaA,EAClB7P,KAAK8P,cAAgBA,EACrB9P,KAAK+P,eAAiBA,EACtB/P,KAAK4M,OAASA,EACd5M,KAAKgQ,iBAAmBA,EAExBhQ,KAAKiQ,WAAa,IAAIG,EAAWpQ,KAAK4M,OAAQ5M,KAAKgQ,iBACpD,CAEM,qBAAOlD,CAAeC,GAC3B,GAAqB,IAAjBA,GAAMpL,QAAiC,IAAjBoL,GAAMpL,OAAc,CAC5C,MAAMkO,EAAa9C,EAAK,GAGxB,OAAO,IAAI6C,EAAoBC,EAChC,CAEC,OADA9O,EAAOC,KAAK,wGAAyG+L,GAC9G,IAEV,CAEM,2BAAaK,CAAeL,GAEjC,OADkC6C,EAAoB9C,eAAeC,GACzDsD,KACb,CAEM,SAAMA,GAEX,IAAIC,QAA2BtQ,KAAKiQ,WAAWM,iBAAiBvQ,KAAK6P,YACrE,GAAMS,EAAU,CACd,IAAIE,GAAoB,EASxB,GAR2B,IAAvBF,EAAS7G,MAAMgH,MACjB1P,EAAOC,KAAK,kCACZwP,GAAW,IAEXzP,EAAOC,KAAK,8CAA+ChB,KAAK6P,YAChEW,QAAiBxQ,KAAKiQ,WAAWS,eAAe1Q,KAAK6P,WAAY,MAG/DW,EAAU,CACZzP,EAAOC,KAAK,2BACZ,MAAM2P,EAAwB5I,EAAG1F,aAAarC,KAAK8P,eAAepN,WAC5DkO,QAAoD5Q,KAAKiQ,WAAWY,2BACxE7Q,KAAK6P,WACLc,EACA3Q,KAAK+P,gBAEPhP,EAAOC,KAAK,oBAAqB4P,GAEjCN,EAAWA,GAAYA,EAASQ,gBAAkBR,QAAiBtQ,KAAKiQ,WAAWM,iBAAiBvQ,KAAK6P,YACzG9O,EAAOC,KAAK,4BAA6BsP,EAASQ,iBAClD,MAAMC,EAAgCC,EAAU,MAAO,CAAChR,KAAK+P,eAAiB,IAAMO,EAASQ,iBAAkB,CAC7GG,MAAO,YAGTlQ,EAAOC,KAAK,KAAM+P,EACnB,MACChQ,EAAOC,KAAK,wCAEf,MACCD,EAAOC,KAAK,qDAAsDhB,KAAK6P,WAG1E,ECjFG,MAAOqB,UAA0BC,EACrC,eAAAC,GACE,MAAO,CACL,gBAAiB7E,EAAaa,eAC9B,yBAA0BwC,EAAoBxC,eAEjD,CAED,gBAAAiE,GACE,OAAOxH,EAAuBC,kBAC/B,QCRUwH,EACJ,UAAAC,CAAWC,GAChB,OAAO,CACR,CAEM,kBAAMC,CAAaC,GACxB,MAAMxP,EAA6B,GACnC,IACEjC,EAAeC,mBAAmBwR,EAAK,OACvC3Q,EAAOC,KAAK,sDACZ,MAAM2Q,EAAeD,EAAIE,YAAY,GAAGC,QAExC9Q,EAAOC,KAAK,wBACZ,MAAM8Q,EAAkB,IAAIC,EAAYJ,GACxC,IAAIK,EAAe,KACnB,MAAMC,EAAqBH,EACxB7J,KAAKiK,EAASC,SACd3G,GAAG,SAASO,MAAO2B,IACdA,EAAMpL,KAAKmE,cAAcuH,SAAS,OACpCgE,QAAatE,EAAM0E,UAEnBrR,EAAOC,KAAK,WAAY0M,EAAMpL,MAC9BoL,EAAM2E,YACP,IAEFC,gBACGL,EACN,MAAMM,QAAyBzP,EAAWsE,YAAY4K,EAAKtP,YAAa8P,GAAMA,GAAG,CAC/ElL,SAAS,EACTC,kBAAkB,IAGpB,GAAIgL,EAAU5Q,OAAS,EAAG,CACxB,MAAM8Q,EAAoB,8BAC1B,IAAIzQ,EAAsB,+CAC1B,MAAMmC,EAAqBoO,EAAU,GAErC,IAAIG,EAAuB,iCACvBC,EAAkB,KAEtB,IAAK,IAAIhO,EAAI,EAAGA,EAAIR,EAASxC,OAAQgD,IAAK,CACpCA,EAAI,IACN3C,GAAe,KACf0Q,GAAgB,KAChBC,GAAW,MAEb,MAAMC,EAAezO,EAASQ,GAAG8B,cAAchF,MAAM,KAAKI,KAAK,KAC/D6Q,GAAgBE,EAChBD,GAAW,IACX3Q,GAAe4Q,EAAO,gBACT,OAATA,EACF5Q,GAAe,eACG,cAAT4Q,IACT5Q,GAAe,UAElB,CACDA,GAAe,IACf0Q,GAAgB,YAGhB3R,EAAOC,KAAK,oBACSkB,EAAKR,KAAK,CAAEmR,UAAWJ,IACrBvQ,EAAKR,KAAK,CAAEmR,UAAW7Q,IAD9C,MAEM8Q,EAAmBC,EAASC,MAAMC,QACxC,IAAIC,EAAqBR,EACrBS,EAAsB,GAE1B,IAAK,IAAIxO,EAAI,EAAGA,EAAI4N,EAAU5Q,OAAQgD,IAChCuO,EAAaR,IACfQ,GAAc,KAEhBA,GAAc,IAAMP,EAAU,IAC9BQ,EAAeA,EAAavN,OAAOkN,EAAUP,EAAU5N,IAEnDA,EAAI,IAAO,GAAKA,IAAM4N,EAAU5Q,OAAS,IAC3CO,EAAKR,KAAK,CAAEmR,UAAWK,EAAY3N,OAAQ4N,IAC3CD,EAAaR,EACbS,EAAe,GACfpS,EAAOC,KAAK,yBAA0B2D,EAAG4N,EAAU5Q,SAIvDZ,EAAOC,KAAK,gCAAiCuR,EAAU5Q,OACxD,CACF,CAAC,MAAOQ,GACPpB,EAAO4B,MAAM,mBAAoBR,EAAKD,EAAMC,EAC7C,CAED,OAAOD,CACR,QCxFUkR,EACSC,MAA+BC,WAAnD,WAAAvT,CAAoBsT,EAA+BC,GAA/BtT,KAAKqT,MAALA,EAA+BrT,KAAUsT,WAAVA,EACjDrT,EAAeC,mBAAmBF,KAAKqT,MAAO,SAC9CpT,EAAeC,mBAAmBF,KAAKqT,MAAME,mBAAoB,sBAClE,CAEM,wBAAMC,CAAmB7E,GAE9B,SAAU3O,KAAKqT,MAAMI,WAAW9E,GAAM,CACpC,MAAMgD,QAAqB3R,KAAKqT,MAAMK,uBAAuB/E,GAC7D,OAAO3O,KAAK2T,uBAAuB,IAAIC,OAAOjC,GAC/C,CAID,OAHE5Q,EAAOgB,KAAK,kDAAmD4M,IAL3C,CASvB,CAEM,4BAAMgF,CAAuBE,GAElC5T,EAAeC,mBAAmB2T,EAAK,OACvC9S,EAAOC,KAAK,2CAA4C6S,EAAIlS,QAE5D,MAAMmS,QAA4BC,EAAaF,GAC/C9S,EAAOC,KACL,wDACA8S,GAASE,MAAMC,KACfH,GAASI,QACTJ,GAASlC,aAAajQ,QAGxB,IAAIwS,GAAiB,EACrB,IAAK,IAAIxP,EAAI,EAAGA,EAAI3E,KAAKsT,WAAW3R,SAAWwS,EAAOxP,IACpD,GAAI3E,KAAKsT,WAAW3O,GAAG4M,WAAWuC,GAAU,CAC1C/S,EAAOC,KAAK,uCAAwC2D,GACpD,MAAM6K,QAAoBxP,KAAKsT,WAAW3O,GAAG8M,aAAaqC,GAC1D/S,EAAOC,KAAK,kBAAmBwO,GAC/B2E,GAAQ,CACT,CAGH,OAAOA,CACR,QChDUC,EACJ,UAAA7C,CAAWC,GAChB,OAAO,CACR,CAEM,kBAAMC,CAAaC,GACxB,OAAOA,EAAI2C,IACZ,QCDUC,EAMD/T,GACAgU,UACAC,oBAPF3Q,iCAA4C,OAE5C4Q,iBAAiD,IAAIC,IAE7D,WAAA3U,CACUQ,EACAgU,EACAC,EAA8BF,EAA0BK,2BAFxD3U,KAAEO,GAAFA,EACAP,KAASuU,UAATA,EACAvU,KAAmBwU,oBAAnBA,EAERvU,EAAeC,mBAAmBK,EAAI,MACtCN,EAAeC,mBAAmBC,EAAcC,WAAWmU,IAC3DtU,EAAeW,KAAKmH,EAAG6M,WAAWL,GAAY,uBAAyBA,EACxE,CAEM,mBAAMM,CAAclG,GACzB,MAAMkF,QAAoB7T,KAAK8U,cAAcnG,GAC7C,OAAOkF,EAAMA,EAAInR,WAAa,IAC/B,CAEO,mBAAAqS,CAAoBpG,GAC1B,MAAMqG,EAAqBhV,KAAKiV,kBAAkBjV,KAAKO,GAAGgT,mBAAqB,IAAM5E,GAErF,OADqBrM,EAAKT,KAAK7B,KAAKuU,UAAWS,EAEhD,CAEM,qBAAAE,CAAsBvG,GAC3B,MAAMwG,EAAiBnV,KAAK+U,oBAAoBpG,GAChD5N,EAAOC,KAAK,kCAAmC2N,EAAKwG,GAChDpN,EAAG6M,WAAWO,GAChBpN,EAAGqN,WAAWD,GAEdpU,EAAOiF,MAAM,0CAA2CmP,EAE3D,CAEM,mBAAML,CAAcnG,GACzB,MAAMwG,EAAyBnV,KAAK+U,oBAAoBpG,GAExD,IAAIzM,EAAe,KAGnB,GAFAA,EAAOlC,KAAKqV,qBAAqBF,GAE5BjT,EAiBHnB,EAAOC,KAAK,iDAAkDhB,KAAKO,GAAGgT,mBAAoB5E,EAAKwG,OAjBtF,CACTpU,EAAOC,KAAK,8CAA+ChB,KAAKO,GAAGgT,mBAAoB5E,EAAKwG,GAC5F,IACE,IAAIlD,EAAwBjS,KAAKyU,iBAAiBa,IAAI3G,GAClDsD,EACFlR,EAAOC,KAAK,oCAEZD,EAAOC,KAAK,uBACZiR,EAAOjS,KAAKuV,qBAAqB5G,EAAKwG,GACtCnV,KAAKyU,iBAAiBe,IAAI7G,EAAKsD,IAEjC/P,QAAa+P,EACbjS,KAAKyU,iBAAiBgB,OAAO9G,EAC9B,CAAC,MAAOxM,GACPpB,EAAOgB,KAAK,0CAA2C/B,KAAKO,GAAGgT,mBAAoB5E,EAAKxM,EACzF,CACF,CAGD,OAAOD,CACR,CAEO,0BAAMqT,CAAqB5G,EAAawG,GAC9C,MAAMjT,QAAqBlC,KAAKO,GAAGmV,uBAAuB/G,GAK1D,OAJIzM,GAAQA,EAAKP,OAAS,IACxBZ,EAAOC,KAAK,oCAAqCkB,EAAKP,QACtDoG,EAAG4N,cAAcR,EAAgBjT,IAE5BA,CACR,CAEM,oBAAA0T,CAAqBC,GAC1B,MAAMhC,EAAc7T,KAAKqV,qBAAqBQ,GAC9C,OAAOhC,EAAMA,EAAInR,WAAa,IAC/B,CAEM,oBAAA2S,CAAqBQ,GAC1B,IAAK9N,EAAG6M,WAAWiB,GACjB,OAAO,KAGT,IACE,MAAMC,EAAQ/N,EAAGgO,SAASF,GACpBG,GAAM,IAAIvV,MAAOC,UAGvB,IAD0BsV,EAAMF,EAAMG,SAAW,KACjCjW,KAAKwU,oBACnB,OAAO,KAGP,OADqBzM,EAAG1F,aAAawT,EAGxC,CAAC,MAAO1T,GACPpB,EAAOgB,KAAK,iCAAkCI,EAC/C,CAED,OAAO,IACR,CAEO,iBAAA8S,CAAkBiB,GAGxB,OAFqBC,EAAOC,WAAW,OAAOC,OAAOH,GAASI,OAAO,MAGtE"}

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@bitblit/ratchet-aws-node-only",
-  "version": "4.0.226-alpha",
+  "version": "4.0.232-alpha",
   "description": "Common tools for use with AWS (Node only)",
   "note-on-side-effects": "Technically the entries in 'bin' below might be side effects, but they are called explicitly",
   "sideEffects": false,
@@ -56,10 +56,10 @@
   },
   "license": "Apache-2.0",
   "dependencies": {
-    "@aws-sdk/client-athena": "3.405.0",
-    "@aws-sdk/types": "3.398.0",
-    "@bitblit/ratchet-aws": "4.0.226-alpha",
-    "@bitblit/ratchet-common": "4.0.226-alpha",
+    "@aws-sdk/client-athena": "3.418.0",
+    "@aws-sdk/types": "3.418.0",
+    "@bitblit/ratchet-aws": "4.0.232-alpha",
+    "@bitblit/ratchet-common": "4.0.232-alpha",
     "@smithy/abort-controller": "2.0.5",
     "@smithy/smithy-client": "2.0.5",
     "@smithy/util-waiter": "2.0.5",