@bitblit/ratchet-aws-node-only 4.0.115-alpha → 4.0.119-alpha

package/lib/cli/site-uploader/site-uploader.js

@@ -1,115 +0,0 @@
-import fs from 'fs';
-import walk from 'walk';
-import { S3Client } from '@aws-sdk/client-s3';
-import path from 'path';
-import mime from 'mime-types';
-import { Logger } from '@bitblit/ratchet-common/lib/logger/logger.js';
-import { Upload } from '@aws-sdk/lib-storage';
-export class SiteUploader {
-    srcDir;
-    bucketName;
-    config;
-    s3 = new S3Client({ region: 'us-east-1' });
-    constructor(srcDir, bucketName, configFile) {
-        this.srcDir = srcDir;
-        this.bucketName = bucketName;
-        this.config = JSON.parse(fs.readFileSync(configFile).toString('ascii'));
-    }
-    static createFromArgs(args) {
-        if (args && args.length === 3) {
-            const src = args[0];
-            const bucket = args[1];
-            const configFile = args[2];
-            return new SiteUploader(src, bucket, configFile);
-        }
-        else {
-            console.log('Usage : node ratchet-site-uploader {srcDir} {bucket} {configFile} (Found ' + args + ' arguments, need 3)');
-            return null;
-        }
-    }
-    static async runFromCliArgs(args) {
-        const inst = SiteUploader.createFromArgs(args);
-        return inst.runPump();
-    }
-    findMatch(prefix, fileName, config) {
-        let found = null;
-        if (prefix != null && fileName != null && config != null && config.mapping != null) {
-            config.mapping.forEach((entry) => {
-                if (found == null) {
-                    if (entry.prefixMatch == null || prefix.match(entry.prefixMatch)) {
-                        if (entry.fileMatch == null || fileName.match(entry.fileMatch)) {
-                            found = entry;
-                        }
-                    }
-                }
-            });
-        }
-        return found;
-    }
-    findMime(fileName, config) {
-        let found = null;
-        if (config != null && config.customMimeTypeMapping != null) {
-            Object.keys(config.customMimeTypeMapping).forEach((k) => {
-                if (found == null && fileName.endsWith(k)) {
-                    found = config.customMimeTypeMapping[k];
-                }
-            });
-        }
-        if (found == null) {
-            found = mime.lookup(fileName);
-        }
-        if (found == null) {
-            found = 'binary/octet-stream';
-        }
-        return found;
-    }
-    runPump() {
-        return new Promise((resolve, reject) => {
-            Logger.info('Uploading contents of %s to %s using %j as config', this.srcDir, this.bucketName, this.config);
-            const options = {};
-            const walker = walk.walk(this.srcDir, options);
-            walker.on('file', function (root, fileStats, next) {
-                Logger.info('Processing %j', fileStats.name);
-                const prefix = root == this.srcDir ? '' : root.substring(this.srcDir.length + 1) + '/';
-                const proc = this.findMatch(prefix, fileStats.name, this.config);
-                const key = prefix + fileStats.name;
-                Logger.info('Uploading file : %s/%s to key %s with %j', root, fileStats.name, key, proc);
-                const params = proc && proc.putParams ? JSON.parse(JSON.stringify(proc.putParams)) : {};
-                params.Bucket = this.bucketName;
-                params.Key = key;
-                params.Body = fs.readFileSync(path.join(root, fileStats.name));
-                if (!params.ContentType) {
-                    params.ContentType = this.findMime(fileStats.name, this.config);
-                }
-                const upload = new Upload({
-                    client: this.s3,
-                    params: params,
-                    tags: [],
-                    queueSize: 4,
-                    partSize: 1024 * 1024 * 5,
-                    leavePartsOnError: false,
-                });
-                upload.on('httpUploadProgress', (progress) => {
-                    Logger.info('Uploading : %s', progress);
-                });
-                upload
-                    .done()
-                    .then((result) => {
-                    Logger.info('Finished upload of %s: %j', key, result);
-                    next();
-                })
-                    .catch((err) => {
-                    Logger.warn('%s failed to upload : %s : Continuing', key, err);
-                    next();
-                });
-            }.bind(this));
-            walker.on('errors', function (root, nodeStatsArray, next) {
-                next();
-            });
-            walker.on('end', function () {
-                Logger.info('All done');
-                resolve(true);
-            });
-        });
-    }
-}

package/lib/cli/start-instance-and-ssh.js

@@ -1,68 +0,0 @@
-import { Logger } from '@bitblit/ratchet-common/lib/logger/logger.js';
-import { Ec2Ratchet } from '@bitblit/ratchet-aws/lib/ec2/ec2-ratchet.js';
-import { spawnSync } from 'child_process';
-import fs from 'fs';
-import os from 'os';
-import path from 'path';
-export class StartInstanceAndSsh {
-    instanceId;
-    publicKeyFile;
-    instanceOsUser;
-    region;
-    availabilityZone;
-    ec2Ratchet;
-    constructor(instanceId, publicKeyFile = path.join(os.homedir(), '.ssh', 'id_rsa.pub'), instanceOsUser = 'ec2-user', region = 'us-east-1', availabilityZone = 'us-east-1a') {
-        this.instanceId = instanceId;
-        this.publicKeyFile = publicKeyFile;
-        this.instanceOsUser = instanceOsUser;
-        this.region = region;
-        this.availabilityZone = availabilityZone;
-        this.ec2Ratchet = new Ec2Ratchet(this.region, this.availabilityZone);
-    }
-    static createFromArgs(args) {
-        if (args?.length === 1 || args?.length === 2) {
-            const instanceId = args[0];
-            return new StartInstanceAndSsh(instanceId);
-        }
-        else {
-            Logger.info('Usage : ratchet-start-instance-and-ssh {instanceId} {publicKeyFile} (Found %s arguments, need 1 or 2)', args);
-            return null;
-        }
-    }
-    static async runFromCliArgs(args) {
-        const inst = StartInstanceAndSsh.createFromArgs(args);
-        return inst.run();
-    }
-    async run() {
-        let instance = await this.ec2Ratchet.describeInstance(this.instanceId);
-        if (!!instance) {
-            let launched = false;
-            if (instance.State.Code == 16) {
-                Logger.info('Instance is already running...');
-                launched = true;
-            }
-            else {
-                Logger.info('Instance is not running... starting up : %s', this.instanceId);
-                launched = await this.ec2Ratchet.launchInstance(this.instanceId, 1000 * 30);
-            }
-            if (launched) {
-                Logger.info('Uploading public key...');
-                const publicKeyText = fs.readFileSync(this.publicKeyFile).toString();
-                const publicKeyResponse = await this.ec2Ratchet.sendPublicKeyToEc2Instance(this.instanceId, publicKeyText, this.instanceOsUser);
-                Logger.info('Key response : %j', publicKeyResponse);
-                instance = instance && instance.PublicIpAddress ? instance : await this.ec2Ratchet.describeInstance(this.instanceId);
-                Logger.info('Instance IP address is %s', instance.PublicIpAddress);
-                const ret = spawnSync('ssh', [this.instanceOsUser + '@' + instance.PublicIpAddress], {
-                    stdio: 'inherit',
-                });
-                Logger.info('%j', ret);
-            }
-            else {
-                Logger.info('Instance could not start - check logs');
-            }
-        }
-        else {
-            Logger.info('No such instance found - check your AWS keys? : %s', this.instanceId);
-        }
-    }
-}

package/lib/index.js

@@ -1 +0,0 @@
-export {};

package/lib/mail/inbound/email-to-db-insert-processor.js

@@ -1,89 +0,0 @@
-import { RequireRatchet } from '@bitblit/ratchet-common/lib/lang/require-ratchet.js';
-import { Logger } from '@bitblit/ratchet-common/lib/logger/logger.js';
-import { MultiStream } from '@bitblit/ratchet-common/lib/stream/multi-stream.js';
-import unzipper from 'unzipper';
-import { CsvRatchet } from '@bitblit/ratchet-node-only/lib/csv/csv-ratchet.js';
-import { DateTime } from 'luxon';
-export class EmailToDbInsertProcessor {
-    canProcess(mail) {
-        return true;
-    }
-    async processEmail(msg) {
-        const rval = [];
-        try {
-            RequireRatchet.notNullOrUndefined(msg, 'msg');
-            Logger.info('Processing Broadsign reach inbound inventory email');
-            const data = msg.attachments[0].content;
-            Logger.info('Unzipping attachment');
-            const rs = new MultiStream(data);
-            let wBuf = null;
-            const prom = rs
-                .pipe(unzipper.Parse())
-                .on('entry', async (entry) => {
-                if (entry.path.toLowerCase().endsWith('csv')) {
-                    wBuf = await entry.buffer();
-                }
-                else {
-                    Logger.info('Pass: %s', entry.path);
-                    entry.autodrain();
-                }
-            })
-                .promise();
-            await prom;
-            const csvParsed = await CsvRatchet.stringParse(wBuf.toString(), (o) => o, {
-                columns: false,
-                skip_empty_lines: true,
-            });
-            if (csvParsed.length > 1) {
-                const dropTable = 'drop table if exists sample';
-                let createTable = 'create table sample (pump_date varchar(255),';
-                const colNames = csvParsed[0];
-                let insertPrefix = 'insert into sample (pump_date,';
-                let insertQ = '?,';
-                for (let i = 0; i < colNames.length; i++) {
-                    if (i > 0) {
-                        createTable += ', ';
-                        insertPrefix += ', ';
-                        insertQ += ', ';
-                    }
-                    const kOut = colNames[i].toLowerCase().split(' ').join('_');
-                    insertPrefix += kOut;
-                    insertQ += '?';
-                    createTable += kOut + ' varchar(255)';
-                    if (kOut === 'id') {
-                        createTable += ' primary key';
-                    }
-                    else if (kOut === 'device_id') {
-                        createTable += ' unique';
-                    }
-                }
-                createTable += ')';
-                insertPrefix += ') values ';
-                Logger.info('Recreating table');
-                const dropRes = rval.push({ statement: dropTable });
-                const createRes = rval.push({ statement: createTable });
-                const pumpDate = DateTime.utc().toISO();
-                let insertStmt = insertPrefix;
-                let insertParams = [];
-                for (let i = 1; i < csvParsed.length; i++) {
-                    if (insertStmt > insertPrefix) {
-                        insertStmt += ',';
-                    }
-                    insertStmt += '(' + insertQ + ')';
-                    insertParams = insertParams.concat(pumpDate, csvParsed[i]);
-                    if (i % 25 === 0 || i === csvParsed.length - 1) {
-                        rval.push({ statement: insertStmt, params: insertParams });
-                        insertStmt = insertPrefix;
-                        insertParams = [];
-                        Logger.info('Inserted %d of %d rows', i, csvParsed.length);
-                    }
-                }
-                Logger.info('Finished insertion of %d rows', csvParsed.length);
-            }
-        }
-        catch (err) {
-            Logger.error('Failure: %s : %j', err, rval, err);
-        }
-        return rval;
-    }
-}

package/lib/mail/inbound/inbound-email-ratchet.js

@@ -1,41 +0,0 @@
-import { simpleParser } from 'mailparser';
-import { Logger } from '@bitblit/ratchet-common/lib/logger/logger.js';
-import { RequireRatchet } from '@bitblit/ratchet-common/lib/lang/require-ratchet.js';
-export class InboundEmailRatchet {
-    cache;
-    processors;
-    constructor(cache, processors) {
-        this.cache = cache;
-        this.processors = processors;
-        RequireRatchet.notNullOrUndefined(this.cache, 'cache');
-        RequireRatchet.notNullOrUndefined(this.cache.getDefaultBucket(), 'cache.defaultBucket');
-    }
-    async processEmailFromS3(key) {
-        const rval = false;
-        if (await this.cache.fileExists(key)) {
-            const data = await this.cache.fetchCacheFileAsString(key);
-            return this.processEmailFromBuffer(new Buffer(data));
-        }
-        else {
-            Logger.warn('Cannot process inbound email - no such key : %s', key);
-        }
-        return rval;
-    }
-    async processEmailFromBuffer(buf) {
-        const rval = false;
-        RequireRatchet.notNullOrUndefined(buf, 'buf');
-        Logger.info('Processing inbound email - size %d bytes', buf.length);
-        const message = await simpleParser(buf);
-        Logger.info('Found mail from "%s" subject "%s" with %d attachments', message?.from?.text, message?.subject, message?.attachments?.length);
-        let procd = false;
-        for (let i = 0; i < this.processors.length && !procd; i++) {
-            if (this.processors[i].canProcess(message)) {
-                Logger.info('Processing message with processor %d', i);
-                const result = await this.processors[i].processEmail(message);
-                Logger.info('Result was : %j', result);
-                procd = true;
-            }
-        }
-        return procd;
-    }
-}

package/lib/mail/inbound/inbound-email-ratchet.spec.js

@@ -1,18 +0,0 @@
-import { InboundEmailRatchet } from './inbound-email-ratchet.js';
-import { JestRatchet } from '@bitblit/ratchet-jest/lib/jest/jest-ratchet.js';
-import { jest } from '@jest/globals';
-import { SampleEmailProcessor } from './sample-email-processor.js';
-let mockS3CR;
-describe('#inboundEmailService', () => {
-    beforeEach(() => {
-        mockS3CR = JestRatchet.mock(jest.fn);
-    });
-    it('should process an email from S3', async () => {
-        mockS3CR.getDefaultBucket.mockReturnValueOnce('TEST-BUCKET');
-        mockS3CR.fileExists.mockResolvedValueOnce(true);
-        mockS3CR.fetchCacheFileAsString.mockResolvedValue('TEST');
-        const svc = new InboundEmailRatchet(mockS3CR, [new SampleEmailProcessor()]);
-        const res = await svc.processEmailFromS3('some-key');
-        expect(res).not.toBeUndefined();
-    });
-});

package/lib/mail/inbound/parsed-email-processor.js

@@ -1 +0,0 @@
-export {};

package/lib/mail/inbound/sample-email-processor.js

@@ -1,8 +0,0 @@
-export class SampleEmailProcessor {
-    canProcess(mail) {
-        return true;
-    }
-    async processEmail(msg) {
-        return msg.body;
-    }
-}