@bitblit/ratchet-aws-node-only 4.0.366-alpha → 4.0.367-alpha

package/lib/index.mjs

@@ -1,2 +1,2 @@
-import e,{readFileSync as t}from"fs";import n from"path";import{RequireRatchet as s,StringRatchet as i,Logger as r,EsmRatchet as a,PromiseRatchet as o,StopWatch as c}from"@bitblit/ratchet-common";import{CsvRatchet as l,AbstractRatchetCliHandler as u,MultiStream as h}from"@bitblit/ratchet-node-only";import{S3Ratchet as d,Ec2Ratchet as f}from"@bitblit/ratchet-aws";import{ListNamedQueriesCommand as m,GetNamedQueryCommand as p,StartQueryExecutionCommand as g,GetQueryExecutionCommand as y}from"@aws-sdk/client-athena";import{GetObjectCommand as T,S3Client as O}from"@aws-sdk/client-s3";import N from"tmp";import E from"readline";import S from"walk";import b from"mime-types";import{Upload as w}from"@aws-sdk/lib-storage";import{spawnSync as M}from"child_process";import C from"os";import F from"unzipper";import{DateTime as A}from"luxon";import{simpleParser as L}from"mailparser";import _ from"crypto";class I{athena;athenaTableName;constructor(e,t){this.athena=e,this.athenaTableName=t,s.notNullOrUndefined(e,"athena"),s.notNullOrUndefined(i.trimToNull(t),"athenaTableName")}async updatePartitions(e,t,n=(new Date).getTime()-864e5,i=(new Date).getTime()){s.true(d.checkS3UrlForValidity(e),"root path not valid"),s.notNullOrUndefined(t,"s3"),r.info("Updating partitions for %s from %s",this.athenaTableName,e),d.extractBucketFromURL(e),d.extractKeyFromURL(e);let a=n;const o=[];for(;a<i;){const t=new Date(a).toISOString().substring(0,10);r.info("d:%s",t);const n=t.split("-");o.push("PARTITION (date_utc_partition='"+t+"') LOCATION '"+e+"/"+n[0]+"/"+n[1]+"/"+n[2]+"'"),a+=864e5}if(o.length>0){const e="ALTER TABLE "+this.athenaTableName+" ADD IF NOT EXISTS \n"+o.join("\n");await this.athena.runQueryToObjects(e)}else r.warn("Not updating partitions - no time between time clauses");return o}async createTable(e,i=!1){s.true(d.checkS3UrlForValidity(e),"root path not valid");let o=!1;if(r.info("Creating ALB table %s",this.athenaTableName),i){r.info("Replace if present specified, removed old table");try{await this.athena.runQueryToObjects("drop table "+this.athenaTableName)}catch(e){r.info("Drop error : %j",e)}}let c=t(n.join(a.fetchDirName(import.meta.url),"../static/albAthenaTableCreate.txt")).toString();c=c.split("{{TABLE NAME}}").join(this.athenaTableName),c=c.split("{{ALB_LOG_ROOT}}").join(e),r.info("Creating table with %s",c);try{await this.athena.runQueryToObjects(c),o=!0}catch(e){r.error("Error creating table : %s",e)}return o}static async readLogObjectsFromCsvStream(e){return l.streamParse(e,(e=>e))}static async readLogObjectsFromFile(e){return l.fileParse(e,(e=>e))}async fetchAlbLogRecords(e){const t=await this.fetchAlbLogRecordsToFile(e);return I.readLogObjectsFromFile(t)}async fetchAlbLogRecordsToFile(e,t=null){r.info("Querying %s : %j",this.athenaTableName,e);let n="select * from "+this.athenaTableName+" where 1=1 ";e.startTimeEpochMS&&(e.startTimeEpochMS&&(n+=" AND time >= '"+new Date(e.startTimeEpochMS).toISOString()+"'",n+=" AND date_utc_partition >='"+new Date(e.startTimeEpochMS).toISOString().substring(0,10)+"'"),e.endTimeEpochMS&&(n+=" AND time < '"+new Date(e.endTimeEpochMS).toISOString()+"'",n+=" AND date_utc_partition <='"+new Date(e.endTimeEpochMS).toISOString().substring(0,10)+"'"),e.requestUrlFilter&&(n+=" AND request_url LIKE '"+e.requestUrlFilter+"'"),e.limit&&(n+=" LIMIT "+e.limit));return await this.athena.runQueryToFile(n,null,t)}static CREATE_TABLE_STATEMENT="CREATE EXTERNAL TABLE IF NOT EXISTS `{{TABLE NAME}}`(\n  `type` string COMMENT '',\n  `time` string COMMENT '',\n  `elb` string COMMENT '',\n  `client_ip` string COMMENT '',\n  `client_port` int COMMENT '',\n  `target_ip` string COMMENT '',\n  `target_port` int COMMENT '',\n  `request_processing_time` double COMMENT '',\n  `target_processing_time` double COMMENT '',\n  `response_processing_time` double COMMENT '',\n  `elb_status_code` string COMMENT '',\n  `target_status_code` string COMMENT '',\n  `received_bytes` bigint COMMENT '',\n  `sent_bytes` bigint COMMENT '',\n  `request_verb` string COMMENT '',\n  `request_url` string COMMENT '',\n  `request_proto` string COMMENT '',\n  `user_agent` string COMMENT '',\n  `ssl_cipher` string COMMENT '',\n  `ssl_protocol` string COMMENT '',\n  `target_group_arn` string COMMENT '',\n  `trace_id` string COMMENT '',\n  `domain_name` string COMMENT '',\n  `chosen_cert_arn` string COMMENT '',\n  `matched_rule_priority` string COMMENT '',\n  `request_creation_time` string COMMENT '',\n  `actions_executed` string COMMENT '',\n  `redirect_url` string COMMENT '',\n  `lambda_error_reason` string COMMENT '',\n  `target_port_list` string COMMENT '',\n  `target_status_code_list` string COMMENT '',\n  `new_field` string COMMENT '')\nPARTITIONED BY (\n  `date_utc_partition` string\n)\nROW FORMAT SERDE\n  'org.apache.hadoop.hive.serde2.RegexSerDe'\nWITH SERDEPROPERTIES (\n  'input.regex'='([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \\\"([^ ]*) ([^ ]*) (- |[^ ]*)\\\" \\\"([^\\\"]*)\\\" ([A-Z0-9-]+) ([A-Za-z0-9.-]*) ([^ ]*) \\\"([^\\\"]*)\\\" \\\"([^\\\"]*)\\\" \\\"([^\\\"]*)\\\" ([-.0-9]*) ([^ ]*) \\\"([^\\\"]*)\\\" \\\"([^\\\"]*)\\\" \\\"([^ ]*)\\\" \\\"([^s]+)\\\" \\\"([^s]+)\\\"(.*)')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  '{{ALB_LOG_ROOT}}'\n"}class U{athena;s3;outputLocation;constructor(e,t,n){this.athena=e,this.s3=t,this.outputLocation=n,s.notNullOrUndefined(e),s.notNullOrUndefined(t),s.notNullOrUndefined(n),s.true(n.startsWith("s3://"))}static athenaRowsToObject(e){const t=e[0].Data.map((e=>e.VarCharValue));return e.slice(1).map((e=>{const n={};for(let s=0;s<e.Data.length;s++)n[t[s]]=e.Data[s].VarCharValue;return n}))}static applyParamsToQuery(e,t){let n=e;return n&&t&&Object.keys(t).forEach((e=>{const s=i.safeString(t[e]),r="{"+e+"}";n=n.split(r).join(s)})),n}async fetchQueryIds(){const e={NextToken:null};let t=[],n=null;do{n=await this.athena.send(new m(e)),t=t.concat(n.NamedQueryIds),e.NextToken=n.NextToken}while(e.NextToken);return t}async listQueries(){const e=[],t=await this.fetchQueryIds();r.debug("Finding %d items",t.length);for(let n=0;n<t.length;n++){const s={NamedQueryId:t[n]},i=await this.athena.send(new p(s));e.push(i.NamedQuery)}return e}async findQueryByName(e){return(await this.listQueries()).find((t=>t.Name.toLowerCase()==e.toLowerCase()))}async runQueryToObjects(e,t={},n=2e3){r.info("Running query to objects");const s=await this.runQueryToOutputLocation(e,t,n);r.info("Query succeeded, processing file from %s",s);const i={Bucket:s.substring(5,s.indexOf("/",5)),Key:s.substring(s.indexOf("/",5)+1)},a=await this.s3.send(new T(i)),o=await a.Body.transformToString();return await l.stringParse(o,(e=>e),{columns:!0,skip_empty_lines:!0})}async runQueryToFile(t,n={},s=null,i=2e3){r.info("Running query to file");const a=await this.runQueryToOutputLocation(t,n,i);r.info("Query succeeded, pulling file from %s",a);const c={Bucket:a.substring(5,a.indexOf("/",5)),Key:a.substring(a.indexOf("/",5)+1)},l=s||N.fileSync({postfix:".csv",keep:!1}).name,u=e.createWriteStream(l),h=(await this.s3.send(new T(c))).Body;h.pipe(u);const d=await o.resolveOnEvent(h,["finish","close"],["error"],l);return r.silly("Response: %s",d),l}async runQueryToOutputLocation(e,t={},n=2e3){let s=null;const a=new c,l=U.applyParamsToQuery(e,t);try{r.info("Starting query : %s",l);const e=i.createType4Guid(),t={QueryString:l,ResultConfiguration:{OutputLocation:this.outputLocation,EncryptionConfiguration:{EncryptionOption:"SSE_S3"}},ClientRequestToken:e,QueryExecutionContext:{Database:"default"}},c={QueryExecutionId:(await this.athena.send(new g(t))).QueryExecutionId},u=["FAILED","CANCELLED","SUCCEEDED"];let h=await this.athena.send(new y(c));for(;-1===u.indexOf(h.QueryExecution.Status.State);)await o.createTimeoutPromise("wait",n),r.debug("%s : %s : %s",h.QueryExecution.Status.State,a.dump(),l),h=await this.athena.send(new y(c));"FAILED"===h.QueryExecution.Status.State?r.warn("Query failed : %s",h.QueryExecution.Status.StateChangeReason):"SUCCEEDED"===h.QueryExecution.Status.State&&(s=h.QueryExecution.ResultConfiguration.OutputLocation)}catch(e){r.warn("Failure : %s",e,e)}return r.info("Query took %s : %s",a.dump(),l),s}}class x{constructor(){}static buildInformation(){return{version:"366",hash:"6a2e3ab20a9aa553758702097b3be8d8174a2a2a",branch:"alpha-2024-06-19-1",tag:"alpha-2024-06-19-1",timeBuiltISO:"2024-06-19T12:19:55-0700",notes:"No notes"}}}class k{constructor(){}static async importJsonLFileToTable(t,n,a){s.notNullOrUndefined(t,"dynamo"),s.notNullOrUndefined(n,"tableName"),s.notNullOrUndefined(a,"filename");const o=e.createReadStream(a),c=E.createInterface({input:o,crlfDelay:1/0});let l=0;for await(const e of c)if(l%100==0&&r.info("Importing line %d",l),i.trimToNull(e)){const s=JSON.parse(e);await t.simplePut(n,s),l++}return l}static async exportScanToJsonLFile(t,n,i){s.notNullOrUndefined(t,"dynamo"),s.notNullOrUndefined(n,"scan"),s.notNullOrUndefined(i,"filename");const a=e.createWriteStream(i);a.on("end",(()=>{r.debug("Write complete")}));const c=await k.exportScanToJsonLWriteStream(t,n,a);return await o.resolveOnEvent(a,["finish","close"],["error"]),a.close(),c}static async exportQueryToJsonLFile(t,n,i){s.notNullOrUndefined(t,"dynamo"),s.notNullOrUndefined(n,"qry"),s.notNullOrUndefined(i,"filename");const a=e.createWriteStream(i);a.on("end",(()=>{r.debug("Write complete")}));const c=await k.exportQueryToJsonLWriteStream(t,n,a);return await o.resolveOnEvent(a,["finish","close"],["error"]),a.close(),c}static async exportScanToJsonLWriteStream(e,t,n){s.notNullOrUndefined(e,"dynamo"),s.notNullOrUndefined(t,"scan"),s.notNullOrUndefined(n,"target");return await e.fullyExecuteProcessOverScan(t,(async e=>k.writeItemToJsonLStream(e,n,!1)))}static async exportQueryToJsonLWriteStream(e,t,n){s.notNullOrUndefined(e,"dynamo"),s.notNullOrUndefined(t,"qry"),s.notNullOrUndefined(n,"target");return await e.fullyExecuteProcessOverQuery(t,(async e=>k.writeItemToJsonLStream(e,n,!1)))}static writeItemToJsonLStream(e,t,n=!1){(e||n)&&t.write(JSON.stringify(e)+"\n")}}class D{srcDir;bucketName;config;s3=new O({region:"us-east-1"});constructor(t,n,s){this.srcDir=t,this.bucketName=n,this.config=JSON.parse(e.readFileSync(s).toString("ascii"))}static createFromArgs(e){if(e&&3===e.length){const t=e[0],n=e[1],s=e[2];return new D(t,n,s)}return console.log("Usage : node ratchet-site-uploader {srcDir} {bucket} {configFile} (Found "+e+" arguments, need 3)"),null}static async runFromCliArgs(e){return D.createFromArgs(e).runPump()}findMatch(e,t,n){let s=null;return null!=e&&null!=t&&null!=n&&null!=n.mapping&&n.mapping.forEach((n=>{null==s&&(null==n.prefixMatch||e.match(n.prefixMatch))&&(null==n.fileMatch||t.match(n.fileMatch))&&(s=n)})),s}findMime(e,t){let n=null;return null!=t&&null!=t.customMimeTypeMapping&&Object.keys(t.customMimeTypeMapping).forEach((s=>{null==n&&e.endsWith(s)&&(n=t.customMimeTypeMapping[s])})),null==n&&(n=b.lookup(e)),null==n&&(n="binary/octet-stream"),n}runPump(){return new Promise(((t,s)=>{r.info("Uploading contents of %s to %s using %j as config",this.srcDir,this.bucketName,this.config);const i=S.walk(this.srcDir,{});i.on("file",function(t,s,i){r.info("Processing %j",s.name);const a=t==this.srcDir?"":t.substring(this.srcDir.length+1)+"/",o=this.findMatch(a,s.name,this.config),c=a+s.name;r.info("Uploading file : %s/%s to key %s with %j",t,s.name,c,o);const l=o&&o.putParams?JSON.parse(JSON.stringify(o.putParams)):{};l.Bucket=this.bucketName,l.Key=c,l.Body=e.readFileSync(n.join(t,s.name)),l.ContentType||(l.ContentType=this.findMime(s.name,this.config));const u=new w({client:this.s3,params:l,tags:[],queueSize:4,partSize:5242880,leavePartsOnError:!1});u.on("httpUploadProgress",(e=>{r.debug("Uploading : %s",e)})),u.done().then((e=>{r.info("Finished upload of %s: %j",c,e),i()})).catch((e=>{r.warn("%s failed to upload : %s : Continuing",c,e),i()}))}.bind(this)),i.on("errors",(function(e,t,n){n()})),i.on("end",(function(){r.info("All done"),t(!0)}))}))}}class P{ec2Ratchet;constructor(e){this.ec2Ratchet=e}async startInstanceAndUploadPublicKeyFile(t,n,i="ec2-user"){s.notNullUndefinedOrOnlyWhitespaceString(t,"instanceId"),s.notNullUndefinedOrOnlyWhitespaceString(n,"filePath"),s.true(e.existsSync(n),"File does not exist"),r.info("Starting instance %s and uploading contents of public key file %s",t,n);const a=e.readFileSync(n).toString();return this.startInstanceAndUploadPublicKey(t,a,i)}async startInstanceAndUploadPublicKey(e,t,n="ec2-user"){r.info("Starting instance %s, public key length %d, user %s",e,t.length,n);let s=await this.ec2Ratchet.describeInstance(e);if(s){let i=!1;if(16==s.State.Code?(r.info("Instance is already running..."),i=!0):(r.info("Instance is not running... starting up : %s",e),i=await this.ec2Ratchet.launchInstance(e,3e4)),i){r.info("Uploading public key...");const i=await this.ec2Ratchet.sendPublicKeyToEc2Instance(e,t,n);r.info("Key response : %j",i),s=s&&s.PublicIpAddress?s:await this.ec2Ratchet.describeInstance(e),r.info("Instance IP address is %s",s.PublicIpAddress)}else r.info("Instance could not start - check logs")}else r.info("No such instance found - check your AWS keys? : %s",e);return s}}class R{instanceId;publicKeyFile;instanceOsUser;region;availabilityZone;ec2Ratchet;instanceUtil;constructor(e,t=n.join(C.homedir(),".ssh","id_rsa.pub"),s="ec2-user",i="us-east-1",r="us-east-1a"){this.instanceId=e,this.publicKeyFile=t,this.instanceOsUser=s,this.region=i,this.availabilityZone=r,this.ec2Ratchet=new f(this.region,this.availabilityZone),this.instanceUtil=new P(this.ec2Ratchet)}static createFromArgs(e){if(1===e?.length||2===e?.length){const t=e[0];return new R(t)}return r.info("Usage : ratchet-start-instance-and-ssh {instanceId} {publicKeyFile} (Found %s arguments, need 1 or 2)",e),null}static async runFromCliArgs(e){return R.createFromArgs(e).run()}async run(){let e=await this.instanceUtil.startInstanceAndUploadPublicKeyFile(this.instanceId,this.publicKeyFile,this.instanceOsUser);if(e){r.info("Instance IP address is %s",e.PublicIpAddress);const t=M("ssh",[this.instanceOsUser+"@"+e.PublicIpAddress],{stdio:"inherit"});r.info("%j",t)}else r.info("No such instance found - check your AWS keys? : %s",this.instanceId)}}class Q extends u{fetchHandlerMap(){return{"site-uploader":D.runFromCliArgs,"start-instance-and-ssh":R.runFromCliArgs}}fetchVersionInfo(){return x.buildInformation()}}class B{canProcess(e){return!0}async processEmail(e){const t=[];try{s.notNullOrUndefined(e,"msg"),r.info("Processing Broadsign reach inbound inventory email");const n=e.attachments[0].content;r.info("Unzipping attachment");const i=new h(n);let a=null;const o=i.pipe(F.Parse()).on("entry",(async e=>{e.path.toLowerCase().endsWith("csv")?a=await e.buffer():(r.info("Pass: %s",e.path),e.autodrain())})).promise();await o;const c=await l.stringParse(a.toString(),(e=>e),{columns:!1,skip_empty_lines:!0});if(c.length>1){const e="drop table if exists sample";let n="create table sample (pump_date varchar(255),";const s=c[0];let i="insert into sample (pump_date,",a="?,";for(let e=0;e<s.length;e++){e>0&&(n+=", ",i+=", ",a+=", ");const t=s[e].toLowerCase().split(" ").join("_");i+=t,a+="?",n+=t+" varchar(255)","id"===t?n+=" primary key":"device_id"===t&&(n+=" unique")}n+=")",i+=") values ",r.info("Recreating table");t.push({statement:e}),t.push({statement:n});const o=A.utc().toISO();let l=i,u=[];for(let e=1;e<c.length;e++)l>i&&(l+=","),l+="("+a+")",u=u.concat(o,c[e]),e%25!=0&&e!==c.length-1||(t.push({statement:l,params:u}),l=i,u=[],r.info("Inserted %d of %d rows",e,c.length));r.info("Finished insertion of %d rows",c.length)}}catch(e){r.error("Failure: %s : %j",e,t,e)}return t}}class j{cache;processors;constructor(e,t){this.cache=e,this.processors=t,s.notNullOrUndefined(this.cache,"cache"),s.notNullOrUndefined(this.cache.getDefaultBucket(),"cache.defaultBucket")}async processEmailFromS3(e){if(await this.cache.fileExists(e)){const t=await this.cache.fetchCacheFileAsString(e);return this.processEmailFromBuffer(new Buffer(t))}return r.warn("Cannot process inbound email - no such key : %s",e),!1}async processEmailFromBuffer(e){s.notNullOrUndefined(e,"buf"),r.info("Processing inbound email - size %d bytes",e.length);const t=await L(e);r.info('Found mail from "%s" subject "%s" with %d attachments',t?.from?.text,t?.subject,t?.attachments?.length);let n=!1;for(let e=0;e<this.processors.length&&!n;e++)if(this.processors[e].canProcess(t)){r.info("Processing message with processor %d",e);const s=await this.processors[e].processEmail(t);r.info("Result was : %j",s),n=!0}return n}}class v{canProcess(e){return!0}async processEmail(e){return e.body}}class W{s3;tmpFolder;cacheTimeoutSeconds;static DEFAULT_CACHE_TIMEOUT_SEC=604800;currentlyLoading=new Map;constructor(t,n,r=W.DEFAULT_CACHE_TIMEOUT_SEC){this.s3=t,this.tmpFolder=n,this.cacheTimeoutSeconds=r,s.notNullOrUndefined(t,"s3"),s.notNullOrUndefined(i.trimToNull(n)),s.true(e.existsSync(n),"folder must exist : "+n)}async getFileString(e){const t=await this.getFileBuffer(e);return t?t.toString():null}keyToLocalCachePath(e){const t=this.generateCacheHash(this.s3.getDefaultBucket()+"/"+e);return n.join(this.tmpFolder,t)}removeCacheFileForKey(t){const n=this.keyToLocalCachePath(t);r.info("Removing cache file for %s : %s",t,n),e.existsSync(n)?e.unlinkSync(n):r.debug("Skipping delete for %s - does not exist",n)}async getFileBuffer(e){const t=this.keyToLocalCachePath(e);let n=null;if(n=this.getCacheFileAsBuffer(t),n)r.info("Found cache file for s3://%s/%s. Local path %s",this.s3.getDefaultBucket(),e,t);else{r.info("No cache. Downloading File s3://%s/%s to %s",this.s3.getDefaultBucket(),e,t);try{let s=this.currentlyLoading.get(e);s?r.info("Already running - wait for that"):(r.info("Not running - start"),s=this.updateLocalCacheFile(e,t),this.currentlyLoading.set(e,s)),n=await s,this.currentlyLoading.delete(e)}catch(t){r.warn("File %s/%s does not exist. Err code: %s",this.s3.getDefaultBucket(),e,t)}}return n}async updateLocalCacheFile(t,n){const s=await this.s3.fetchCacheFileAsBuffer(t);return s&&s.length>0&&(r.info("Saving %d bytes to disk for cache",s.length),e.writeFileSync(n,s)),s}getCacheFileAsString(e){const t=this.getCacheFileAsBuffer(e);return t?t.toString():null}getCacheFileAsBuffer(t){if(!e.existsSync(t))return null;try{const n=e.statSync(t),s=(new Date).getTime();if((s-n.ctimeMs)/1e3>=this.cacheTimeoutSeconds)return null;return e.readFileSync(t)}catch(e){r.warn("Error getting s3 cache file %s",e)}return null}generateCacheHash(e){return _.createHash("md5").update(e).digest("hex")}}export{I as AlbAthenaLogRatchet,U as AthenaRatchet,k as DynamoExporter,P as Ec2InstanceUtil,B as EmailToDbInsertProcessor,j as InboundEmailRatchet,x as RatchetAwsNodeOnlyInfo,Q as RatchetCliHandler,W as S3CacheToLocalDiskRatchet,v as SampleEmailProcessor,D as SiteUploader,R as StartInstanceAndSsh};
+import e,{readFileSync as t}from"fs";import n from"path";import{RequireRatchet as s,StringRatchet as i,Logger as r,EsmRatchet as a,PromiseRatchet as o,StopWatch as c}from"@bitblit/ratchet-common";import{CsvRatchet as l,AbstractRatchetCliHandler as u,MultiStream as h}from"@bitblit/ratchet-node-only";import{S3Ratchet as d,Ec2Ratchet as f}from"@bitblit/ratchet-aws";import{ListNamedQueriesCommand as m,GetNamedQueryCommand as p,StartQueryExecutionCommand as g,GetQueryExecutionCommand as y}from"@aws-sdk/client-athena";import{GetObjectCommand as T,S3Client as O}from"@aws-sdk/client-s3";import N from"tmp";import E from"readline";import S from"walk";import b from"mime-types";import{Upload as w}from"@aws-sdk/lib-storage";import{spawnSync as M}from"child_process";import C from"os";import F from"unzipper";import{DateTime as A}from"luxon";import{simpleParser as L}from"mailparser";import _ from"crypto";class I{athena;athenaTableName;constructor(e,t){this.athena=e,this.athenaTableName=t,s.notNullOrUndefined(e,"athena"),s.notNullOrUndefined(i.trimToNull(t),"athenaTableName")}async updatePartitions(e,t,n=(new Date).getTime()-864e5,i=(new Date).getTime()){s.true(d.checkS3UrlForValidity(e),"root path not valid"),s.notNullOrUndefined(t,"s3"),r.info("Updating partitions for %s from %s",this.athenaTableName,e),d.extractBucketFromURL(e),d.extractKeyFromURL(e);let a=n;const o=[];for(;a<i;){const t=new Date(a).toISOString().substring(0,10);r.info("d:%s",t);const n=t.split("-");o.push("PARTITION (date_utc_partition='"+t+"') LOCATION '"+e+"/"+n[0]+"/"+n[1]+"/"+n[2]+"'"),a+=864e5}if(o.length>0){const e="ALTER TABLE "+this.athenaTableName+" ADD IF NOT EXISTS \n"+o.join("\n");await this.athena.runQueryToObjects(e)}else r.warn("Not updating partitions - no time between time clauses");return o}async createTable(e,i=!1){s.true(d.checkS3UrlForValidity(e),"root path not valid");let o=!1;if(r.info("Creating ALB table %s",this.athenaTableName),i){r.info("Replace if present specified, removed old table");try{await this.athena.runQueryToObjects("drop table "+this.athenaTableName)}catch(e){r.info("Drop error : %j",e)}}let c=t(n.join(a.fetchDirName(import.meta.url),"../static/albAthenaTableCreate.txt")).toString();c=c.split("{{TABLE NAME}}").join(this.athenaTableName),c=c.split("{{ALB_LOG_ROOT}}").join(e),r.info("Creating table with %s",c);try{await this.athena.runQueryToObjects(c),o=!0}catch(e){r.error("Error creating table : %s",e)}return o}static async readLogObjectsFromCsvStream(e){return l.streamParse(e,(e=>e))}static async readLogObjectsFromFile(e){return l.fileParse(e,(e=>e))}async fetchAlbLogRecords(e){const t=await this.fetchAlbLogRecordsToFile(e);return I.readLogObjectsFromFile(t)}async fetchAlbLogRecordsToFile(e,t=null){r.info("Querying %s : %j",this.athenaTableName,e);let n="select * from "+this.athenaTableName+" where 1=1 ";e.startTimeEpochMS&&(e.startTimeEpochMS&&(n+=" AND time >= '"+new Date(e.startTimeEpochMS).toISOString()+"'",n+=" AND date_utc_partition >='"+new Date(e.startTimeEpochMS).toISOString().substring(0,10)+"'"),e.endTimeEpochMS&&(n+=" AND time < '"+new Date(e.endTimeEpochMS).toISOString()+"'",n+=" AND date_utc_partition <='"+new Date(e.endTimeEpochMS).toISOString().substring(0,10)+"'"),e.requestUrlFilter&&(n+=" AND request_url LIKE '"+e.requestUrlFilter+"'"),e.limit&&(n+=" LIMIT "+e.limit));return await this.athena.runQueryToFile(n,null,t)}static CREATE_TABLE_STATEMENT="CREATE EXTERNAL TABLE IF NOT EXISTS `{{TABLE NAME}}`(\n  `type` string COMMENT '',\n  `time` string COMMENT '',\n  `elb` string COMMENT '',\n  `client_ip` string COMMENT '',\n  `client_port` int COMMENT '',\n  `target_ip` string COMMENT '',\n  `target_port` int COMMENT '',\n  `request_processing_time` double COMMENT '',\n  `target_processing_time` double COMMENT '',\n  `response_processing_time` double COMMENT '',\n  `elb_status_code` string COMMENT '',\n  `target_status_code` string COMMENT '',\n  `received_bytes` bigint COMMENT '',\n  `sent_bytes` bigint COMMENT '',\n  `request_verb` string COMMENT '',\n  `request_url` string COMMENT '',\n  `request_proto` string COMMENT '',\n  `user_agent` string COMMENT '',\n  `ssl_cipher` string COMMENT '',\n  `ssl_protocol` string COMMENT '',\n  `target_group_arn` string COMMENT '',\n  `trace_id` string COMMENT '',\n  `domain_name` string COMMENT '',\n  `chosen_cert_arn` string COMMENT '',\n  `matched_rule_priority` string COMMENT '',\n  `request_creation_time` string COMMENT '',\n  `actions_executed` string COMMENT '',\n  `redirect_url` string COMMENT '',\n  `lambda_error_reason` string COMMENT '',\n  `target_port_list` string COMMENT '',\n  `target_status_code_list` string COMMENT '',\n  `new_field` string COMMENT '')\nPARTITIONED BY (\n  `date_utc_partition` string\n)\nROW FORMAT SERDE\n  'org.apache.hadoop.hive.serde2.RegexSerDe'\nWITH SERDEPROPERTIES (\n  'input.regex'='([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \\\"([^ ]*) ([^ ]*) (- |[^ ]*)\\\" \\\"([^\\\"]*)\\\" ([A-Z0-9-]+) ([A-Za-z0-9.-]*) ([^ ]*) \\\"([^\\\"]*)\\\" \\\"([^\\\"]*)\\\" \\\"([^\\\"]*)\\\" ([-.0-9]*) ([^ ]*) \\\"([^\\\"]*)\\\" \\\"([^\\\"]*)\\\" \\\"([^ ]*)\\\" \\\"([^s]+)\\\" \\\"([^s]+)\\\"(.*)')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  '{{ALB_LOG_ROOT}}'\n"}class U{athena;s3;outputLocation;constructor(e,t,n){this.athena=e,this.s3=t,this.outputLocation=n,s.notNullOrUndefined(e),s.notNullOrUndefined(t),s.notNullOrUndefined(n),s.true(n.startsWith("s3://"))}static athenaRowsToObject(e){const t=e[0].Data.map((e=>e.VarCharValue));return e.slice(1).map((e=>{const n={};for(let s=0;s<e.Data.length;s++)n[t[s]]=e.Data[s].VarCharValue;return n}))}static applyParamsToQuery(e,t){let n=e;return n&&t&&Object.keys(t).forEach((e=>{const s=i.safeString(t[e]),r="{"+e+"}";n=n.split(r).join(s)})),n}async fetchQueryIds(){const e={NextToken:null};let t=[],n=null;do{n=await this.athena.send(new m(e)),t=t.concat(n.NamedQueryIds),e.NextToken=n.NextToken}while(e.NextToken);return t}async listQueries(){const e=[],t=await this.fetchQueryIds();r.debug("Finding %d items",t.length);for(let n=0;n<t.length;n++){const s={NamedQueryId:t[n]},i=await this.athena.send(new p(s));e.push(i.NamedQuery)}return e}async findQueryByName(e){return(await this.listQueries()).find((t=>t.Name.toLowerCase()==e.toLowerCase()))}async runQueryToObjects(e,t={},n=2e3){r.info("Running query to objects");const s=await this.runQueryToOutputLocation(e,t,n);r.info("Query succeeded, processing file from %s",s);const i={Bucket:s.substring(5,s.indexOf("/",5)),Key:s.substring(s.indexOf("/",5)+1)},a=await this.s3.send(new T(i)),o=await a.Body.transformToString();return await l.stringParse(o,(e=>e),{columns:!0,skip_empty_lines:!0})}async runQueryToFile(t,n={},s=null,i=2e3){r.info("Running query to file");const a=await this.runQueryToOutputLocation(t,n,i);r.info("Query succeeded, pulling file from %s",a);const c={Bucket:a.substring(5,a.indexOf("/",5)),Key:a.substring(a.indexOf("/",5)+1)},l=s||N.fileSync({postfix:".csv",keep:!1}).name,u=e.createWriteStream(l),h=(await this.s3.send(new T(c))).Body;h.pipe(u);const d=await o.resolveOnEvent(h,["finish","close"],["error"],l);return r.silly("Response: %s",d),l}async runQueryToOutputLocation(e,t={},n=2e3){let s=null;const a=new c,l=U.applyParamsToQuery(e,t);try{r.info("Starting query : %s",l);const e=i.createType4Guid(),t={QueryString:l,ResultConfiguration:{OutputLocation:this.outputLocation,EncryptionConfiguration:{EncryptionOption:"SSE_S3"}},ClientRequestToken:e,QueryExecutionContext:{Database:"default"}},c={QueryExecutionId:(await this.athena.send(new g(t))).QueryExecutionId},u=["FAILED","CANCELLED","SUCCEEDED"];let h=await this.athena.send(new y(c));for(;-1===u.indexOf(h.QueryExecution.Status.State);)await o.createTimeoutPromise("wait",n),r.debug("%s : %s : %s",h.QueryExecution.Status.State,a.dump(),l),h=await this.athena.send(new y(c));"FAILED"===h.QueryExecution.Status.State?r.warn("Query failed : %s",h.QueryExecution.Status.StateChangeReason):"SUCCEEDED"===h.QueryExecution.Status.State&&(s=h.QueryExecution.ResultConfiguration.OutputLocation)}catch(e){r.warn("Failure : %s",e,e)}return r.info("Query took %s : %s",a.dump(),l),s}}class x{constructor(){}static buildInformation(){return{version:"367",hash:"4af1e13e7622c8aedc5061103c78c68876afe37c",branch:"alpha-2024-06-19-4",tag:"alpha-2024-06-19-4",timeBuiltISO:"2024-06-19T14:05:22-0700",notes:"No notes"}}}class k{constructor(){}static async importJsonLFileToTable(t,n,a){s.notNullOrUndefined(t,"dynamo"),s.notNullOrUndefined(n,"tableName"),s.notNullOrUndefined(a,"filename");const o=e.createReadStream(a),c=E.createInterface({input:o,crlfDelay:1/0});let l=0;for await(const e of c)if(l%100==0&&r.info("Importing line %d",l),i.trimToNull(e)){const s=JSON.parse(e);await t.simplePut(n,s),l++}return l}static async exportScanToJsonLFile(t,n,i){s.notNullOrUndefined(t,"dynamo"),s.notNullOrUndefined(n,"scan"),s.notNullOrUndefined(i,"filename");const a=e.createWriteStream(i);a.on("end",(()=>{r.debug("Write complete")}));const c=await k.exportScanToJsonLWriteStream(t,n,a);return await o.resolveOnEvent(a,["finish","close"],["error"]),a.close(),c}static async exportQueryToJsonLFile(t,n,i){s.notNullOrUndefined(t,"dynamo"),s.notNullOrUndefined(n,"qry"),s.notNullOrUndefined(i,"filename");const a=e.createWriteStream(i);a.on("end",(()=>{r.debug("Write complete")}));const c=await k.exportQueryToJsonLWriteStream(t,n,a);return await o.resolveOnEvent(a,["finish","close"],["error"]),a.close(),c}static async exportScanToJsonLWriteStream(e,t,n){s.notNullOrUndefined(e,"dynamo"),s.notNullOrUndefined(t,"scan"),s.notNullOrUndefined(n,"target");return await e.fullyExecuteProcessOverScan(t,(async e=>k.writeItemToJsonLStream(e,n,!1)))}static async exportQueryToJsonLWriteStream(e,t,n){s.notNullOrUndefined(e,"dynamo"),s.notNullOrUndefined(t,"qry"),s.notNullOrUndefined(n,"target");return await e.fullyExecuteProcessOverQuery(t,(async e=>k.writeItemToJsonLStream(e,n,!1)))}static writeItemToJsonLStream(e,t,n=!1){(e||n)&&t.write(JSON.stringify(e)+"\n")}}class D{srcDir;bucketName;config;s3=new O({region:"us-east-1"});constructor(t,n,s){this.srcDir=t,this.bucketName=n,this.config=JSON.parse(e.readFileSync(s).toString("ascii"))}static createFromArgs(e){if(e&&3===e.length){const t=e[0],n=e[1],s=e[2];return new D(t,n,s)}return console.log("Usage : node ratchet-site-uploader {srcDir} {bucket} {configFile} (Found "+e+" arguments, need 3)"),null}static async runFromCliArgs(e){return D.createFromArgs(e).runPump()}findMatch(e,t,n){let s=null;return null!=e&&null!=t&&null!=n&&null!=n.mapping&&n.mapping.forEach((n=>{null==s&&(null==n.prefixMatch||e.match(n.prefixMatch))&&(null==n.fileMatch||t.match(n.fileMatch))&&(s=n)})),s}findMime(e,t){let n=null;return null!=t&&null!=t.customMimeTypeMapping&&Object.keys(t.customMimeTypeMapping).forEach((s=>{null==n&&e.endsWith(s)&&(n=t.customMimeTypeMapping[s])})),null==n&&(n=b.lookup(e)),null==n&&(n="binary/octet-stream"),n}runPump(){return new Promise(((t,s)=>{r.info("Uploading contents of %s to %s using %j as config",this.srcDir,this.bucketName,this.config);const i=S.walk(this.srcDir,{});i.on("file",function(t,s,i){r.info("Processing %j",s.name);const a=t==this.srcDir?"":t.substring(this.srcDir.length+1)+"/",o=this.findMatch(a,s.name,this.config),c=a+s.name;r.info("Uploading file : %s/%s to key %s with %j",t,s.name,c,o);const l=o&&o.putParams?JSON.parse(JSON.stringify(o.putParams)):{};l.Bucket=this.bucketName,l.Key=c,l.Body=e.readFileSync(n.join(t,s.name)),l.ContentType||(l.ContentType=this.findMime(s.name,this.config));const u=new w({client:this.s3,params:l,tags:[],queueSize:4,partSize:5242880,leavePartsOnError:!1});u.on("httpUploadProgress",(e=>{r.debug("Uploading : %s",e)})),u.done().then((e=>{r.info("Finished upload of %s: %j",c,e),i()})).catch((e=>{r.warn("%s failed to upload : %s : Continuing",c,e),i()}))}.bind(this)),i.on("errors",(function(e,t,n){n()})),i.on("end",(function(){r.info("All done"),t(!0)}))}))}}class P{ec2Ratchet;constructor(e){this.ec2Ratchet=e}async startInstanceAndUploadPublicKeyFile(t,n,i="ec2-user"){s.notNullUndefinedOrOnlyWhitespaceString(t,"instanceId"),s.notNullUndefinedOrOnlyWhitespaceString(n,"filePath"),s.true(e.existsSync(n),"File does not exist"),r.info("Starting instance %s and uploading contents of public key file %s",t,n);const a=e.readFileSync(n).toString();return this.startInstanceAndUploadPublicKey(t,a,i)}async startInstanceAndUploadPublicKey(e,t,n="ec2-user"){r.info("Starting instance %s, public key length %d, user %s",e,t.length,n);let s=await this.ec2Ratchet.describeInstance(e);if(s){let i=!1;if(16==s.State.Code?(r.info("Instance is already running..."),i=!0):(r.info("Instance is not running... starting up : %s",e),i=await this.ec2Ratchet.launchInstance(e,3e4)),i){r.info("Uploading public key...");const i=await this.ec2Ratchet.sendPublicKeyToEc2Instance(e,t,n);r.info("Key response : %j",i),s=s&&s.PublicIpAddress?s:await this.ec2Ratchet.describeInstance(e),r.info("Instance IP address is %s",s.PublicIpAddress)}else r.info("Instance could not start - check logs")}else r.info("No such instance found - check your AWS keys? : %s",e);return s}}class R{instanceId;publicKeyFile;instanceOsUser;region;availabilityZone;ec2Ratchet;instanceUtil;constructor(e,t=n.join(C.homedir(),".ssh","id_rsa.pub"),s="ec2-user",i="us-east-1",r="us-east-1a"){this.instanceId=e,this.publicKeyFile=t,this.instanceOsUser=s,this.region=i,this.availabilityZone=r,this.ec2Ratchet=new f(this.region,this.availabilityZone),this.instanceUtil=new P(this.ec2Ratchet)}static createFromArgs(e){if(1===e?.length||2===e?.length){const t=e[0];return new R(t)}return r.info("Usage : ratchet-start-instance-and-ssh {instanceId} {publicKeyFile} (Found %s arguments, need 1 or 2)",e),null}static async runFromCliArgs(e){return R.createFromArgs(e).run()}async run(){let e=await this.instanceUtil.startInstanceAndUploadPublicKeyFile(this.instanceId,this.publicKeyFile,this.instanceOsUser);if(e){r.info("Instance IP address is %s",e.PublicIpAddress);const t=M("ssh",[this.instanceOsUser+"@"+e.PublicIpAddress],{stdio:"inherit"});r.info("%j",t)}else r.info("No such instance found - check your AWS keys? : %s",this.instanceId)}}class Q extends u{fetchHandlerMap(){return{"site-uploader":D.runFromCliArgs,"start-instance-and-ssh":R.runFromCliArgs}}fetchVersionInfo(){return x.buildInformation()}}class B{canProcess(e){return!0}async processEmail(e){const t=[];try{s.notNullOrUndefined(e,"msg"),r.info("Processing Broadsign reach inbound inventory email");const n=e.attachments[0].content;r.info("Unzipping attachment");const i=new h(n);let a=null;const o=i.pipe(F.Parse()).on("entry",(async e=>{e.path.toLowerCase().endsWith("csv")?a=await e.buffer():(r.info("Pass: %s",e.path),e.autodrain())})).promise();await o;const c=await l.stringParse(a.toString(),(e=>e),{columns:!1,skip_empty_lines:!0});if(c.length>1){const e="drop table if exists sample";let n="create table sample (pump_date varchar(255),";const s=c[0];let i="insert into sample (pump_date,",a="?,";for(let e=0;e<s.length;e++){e>0&&(n+=", ",i+=", ",a+=", ");const t=s[e].toLowerCase().split(" ").join("_");i+=t,a+="?",n+=t+" varchar(255)","id"===t?n+=" primary key":"device_id"===t&&(n+=" unique")}n+=")",i+=") values ",r.info("Recreating table");t.push({statement:e}),t.push({statement:n});const o=A.utc().toISO();let l=i,u=[];for(let e=1;e<c.length;e++)l>i&&(l+=","),l+="("+a+")",u=u.concat(o,c[e]),e%25!=0&&e!==c.length-1||(t.push({statement:l,params:u}),l=i,u=[],r.info("Inserted %d of %d rows",e,c.length));r.info("Finished insertion of %d rows",c.length)}}catch(e){r.error("Failure: %s : %j",e,t,e)}return t}}class j{cache;processors;constructor(e,t){this.cache=e,this.processors=t,s.notNullOrUndefined(this.cache,"cache"),s.notNullOrUndefined(this.cache.getDefaultBucket(),"cache.defaultBucket")}async processEmailFromS3(e){if(await this.cache.fileExists(e)){const t=await this.cache.fetchCacheFileAsString(e);return this.processEmailFromBuffer(new Buffer(t))}return r.warn("Cannot process inbound email - no such key : %s",e),!1}async processEmailFromBuffer(e){s.notNullOrUndefined(e,"buf"),r.info("Processing inbound email - size %d bytes",e.length);const t=await L(e);r.info('Found mail from "%s" subject "%s" with %d attachments',t?.from?.text,t?.subject,t?.attachments?.length);let n=!1;for(let e=0;e<this.processors.length&&!n;e++)if(this.processors[e].canProcess(t)){r.info("Processing message with processor %d",e);const s=await this.processors[e].processEmail(t);r.info("Result was : %j",s),n=!0}return n}}class v{canProcess(e){return!0}async processEmail(e){return e.body}}class W{s3;tmpFolder;cacheTimeoutSeconds;static DEFAULT_CACHE_TIMEOUT_SEC=604800;currentlyLoading=new Map;constructor(t,n,r=W.DEFAULT_CACHE_TIMEOUT_SEC){this.s3=t,this.tmpFolder=n,this.cacheTimeoutSeconds=r,s.notNullOrUndefined(t,"s3"),s.notNullOrUndefined(i.trimToNull(n)),s.true(e.existsSync(n),"folder must exist : "+n)}async getFileString(e){const t=await this.getFileBuffer(e);return t?t.toString():null}keyToLocalCachePath(e){const t=this.generateCacheHash(this.s3.getDefaultBucket()+"/"+e);return n.join(this.tmpFolder,t)}removeCacheFileForKey(t){const n=this.keyToLocalCachePath(t);r.info("Removing cache file for %s : %s",t,n),e.existsSync(n)?e.unlinkSync(n):r.debug("Skipping delete for %s - does not exist",n)}async getFileBuffer(e){const t=this.keyToLocalCachePath(e);let n=null;if(n=this.getCacheFileAsBuffer(t),n)r.info("Found cache file for s3://%s/%s. Local path %s",this.s3.getDefaultBucket(),e,t);else{r.info("No cache. Downloading File s3://%s/%s to %s",this.s3.getDefaultBucket(),e,t);try{let s=this.currentlyLoading.get(e);s?r.info("Already running - wait for that"):(r.info("Not running - start"),s=this.updateLocalCacheFile(e,t),this.currentlyLoading.set(e,s)),n=await s,this.currentlyLoading.delete(e)}catch(t){r.warn("File %s/%s does not exist. Err code: %s",this.s3.getDefaultBucket(),e,t)}}return n}async updateLocalCacheFile(t,n){const s=await this.s3.fetchCacheFileAsBuffer(t);return s&&s.length>0&&(r.info("Saving %d bytes to disk for cache",s.length),e.writeFileSync(n,s)),s}getCacheFileAsString(e){const t=this.getCacheFileAsBuffer(e);return t?t.toString():null}getCacheFileAsBuffer(t){if(!e.existsSync(t))return null;try{const n=e.statSync(t),s=(new Date).getTime();if((s-n.ctimeMs)/1e3>=this.cacheTimeoutSeconds)return null;return e.readFileSync(t)}catch(e){r.warn("Error getting s3 cache file %s",e)}return null}generateCacheHash(e){return _.createHash("md5").update(e).digest("hex")}}export{I as AlbAthenaLogRatchet,U as AthenaRatchet,k as DynamoExporter,P as Ec2InstanceUtil,B as EmailToDbInsertProcessor,j as InboundEmailRatchet,x as RatchetAwsNodeOnlyInfo,Q as RatchetCliHandler,W as S3CacheToLocalDiskRatchet,v as SampleEmailProcessor,D as SiteUploader,R as StartInstanceAndSsh};
 //# sourceMappingURL=index.mjs.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bitblit/ratchet-aws-node-only",
-  "version": "4.0.366-alpha",
+  "version": "4.0.367-alpha",
   "description": "Common tools for use with AWS (Node only)",
   "note-on-side-effects": "Technically the entries in 'bin' below might be side effects, but they are called explicitly",
   "sideEffects": false,
@@ -60,8 +60,8 @@
     "@aws-sdk/client-athena": "3.600.0",
     "@aws-sdk/client-sts": "3.600.0",
     "@aws-sdk/types": "3.598.0",
-    "@bitblit/ratchet-aws": "4.0.366-alpha",
-    "@bitblit/ratchet-common": "4.0.366-alpha",
+    "@bitblit/ratchet-aws": "4.0.367-alpha",
+    "@bitblit/ratchet-common": "4.0.367-alpha",
     "@smithy/abort-controller": "3.1.0",
     "@smithy/smithy-client": "3.1.4",
     "@smithy/util-waiter": "3.1.0",