@stamhoofd/email 2.1.1

package/package.json

@@ -0,0 +1,25 @@
+{
+    "name": "@stamhoofd/email",
+    "version": "2.1.1",
+    "main": "./dist/index.js",
+    "types": "./dist/index.d.ts",
+    "license": "UNLICENCED",
+    "sideEffects": false,
+    "files": [
+        "src"
+    ],
+    "scripts": {
+        "build": "tsc -b && mkdir -p ./dist/migrations && cp ./src/migrations/*.sql ./dist/migrations",
+        "build:full": "rm -rf ./dist && yarn build"
+    },
+    "peerDependencies": {
+        "@simonbackx/simple-errors": "^1.4"
+    },
+    "devDependencies": {
+        "@types/nodemailer": "6.4.14"
+    },
+    "dependencies": {
+        "html-to-text": "^8.1.0",
+        "nodemailer": "6.9.13"
+    }
+}

package/src/classes/Email.test.ts

@@ -0,0 +1,28 @@
+import { Email } from "./Email"
+
+describe("Email", () => {
+    it("should parse e-mail strings correctly", () => {
+        expect(Email.parseEmailStr('"My crazy name" <hallo@stamhoofd.be>')).toEqual(["hallo@stamhoofd.be"])
+        expect(Email.parseEmailStr('hallo@stamhoofd.be')).toEqual(["hallo@stamhoofd.be"])
+        expect(Email.parseEmailStr('hallo@stamhoofd.be ')).toEqual(["hallo@stamhoofd.be"])
+        expect(Email.parseEmailStr('"hallo@stamhoofd.be"')).toEqual(["hallo@stamhoofd.be"])
+        expect(Email.parseEmailStr('<"hallo@stamhoofd.be">')).toEqual(["hallo@stamhoofd.be"])
+        expect(Email.parseEmailStr('Gekke naam <"hallo@stamhoofd.be">')).toEqual(["hallo@stamhoofd.be"])
+        expect(Email.parseEmailStr('<"hallo@stam\\"hoofd.be">')).toEqual([])
+        expect(Email.parseEmailStr('<"hallo@stam,hoofd.be">')).toEqual([])
+        expect(Email.parseEmailStr('<hallo@stam,hoofd.be>')).toEqual([])
+        expect(Email.parseEmailStr('"Hallo <dit is een test>" <hallo@stamhoofd.be>')).toEqual(["hallo@stamhoofd.be"])
+        expect(Email.parseEmailStr('"Hallo 👋🏽" <hallo@stamhoofd.be>')).toEqual(["hallo@stamhoofd.be"])
+        expect(Email.parseEmailStr('Hallo <hallo@stam.hoofd.be>')).toEqual(["hallo@stam.hoofd.be"])
+        expect(Email.parseEmailStr('Hallo <hallo@stam.hoofd.be')).toEqual([])
+
+        expect(Email.parseEmailStr('"Hallo <dit is \\" een test>" <hallo@stamhoofd.be>')).toEqual(["hallo@stamhoofd.be"])
+        expect(Email.parseEmailStr('"Hallo <dit is \\" een test佐>" <hallo@stamhoofd.be>')).toEqual(["hallo@stamhoofd.be"])
+
+        expect(Email.parseEmailStr('"Voornaam \'l Achternaam, en nog iets" <jwz@netscape.com>')).toEqual(["jwz@netscape.com"])
+        expect(Email.parseEmailStr('"Voornaam \'l Achternaam, en nog iets" <jwz@netscape.com>, "My crazy name" <hallo@stamhoofd.be>')).toEqual(["jwz@netscape.com", "hallo@stamhoofd.be"])
+
+        expect(Email.parseEmailStr('hallo@example.com, test@test.com')).toEqual(["hallo@example.com", "test@test.com"])
+        expect(Email.parseEmailStr('hallo@example.com, invalid, test@test.com')).toEqual(["hallo@example.com", "test@test.com"])
+    })
+})

package/src/classes/Email.ts

@@ -0,0 +1,374 @@
+import { DataValidator, Formatter } from '@stamhoofd/utility';
+import nodemailer from "nodemailer"
+import Mail from 'nodemailer/lib/mailer';
+import { EmailAddress } from '../models/EmailAddress';
+import htmlToText from 'html-to-text';
+import { sleep } from '@stamhoofd/utility';
+import { I18n } from "@stamhoofd/backend-i18n"
+
+export type EmailInterfaceRecipient = {
+    name?: string|null;
+    email: string;
+}
+
+export type EmailInterfaceBase = {
+    to: string|EmailInterfaceRecipient[];
+    bcc?: string;
+    replyTo?: string;
+    subject: string;
+    text?: string;
+    html?: string;
+    attachments?: { filename: string; path?: string; href?: string; content?: string|Buffer; contentType?: string }[];
+    retryCount?: number;
+    type?: "transactional" | "broadcast",
+    headers?: Record<string, string>|null
+}
+
+export type EmailInterface = EmailInterfaceBase & {
+    from: string;
+}
+
+/// An email builder is called until it returns undefined. This allows to reduce memory usage for an e-mail with multiple recipients
+export type EmailBuilder = () => EmailInterface | undefined
+
+class EmailStatic {
+    transporter: Mail;
+    transactionalTransporter: Mail;
+    rps = 14
+
+    currentQueue: EmailBuilder[] = []
+    sending = false
+
+    setupIfNeeded() {
+        if (this.transporter) {
+            return;
+        }
+        if (!STAMHOOFD.SMTP_HOST || !STAMHOOFD.SMTP_PORT) {
+            throw new Error("Missing environment variables to send emails");
+            return;
+        }
+
+        // create reusable transporter object using the default SMTP transport
+        this.transporter = nodemailer.createTransport({
+            pool: true,
+            host: STAMHOOFD.SMTP_HOST,
+            port: STAMHOOFD.SMTP_PORT,
+            auth: {
+                user: STAMHOOFD.SMTP_USERNAME, // generated ethereal user
+                pass: STAMHOOFD.SMTP_PASSWORD // generated ethereal password
+            }
+        });
+        
+        // create reusable transporter object using the default SMTP transport
+        this.transactionalTransporter = nodemailer.createTransport({
+            pool: true,
+            host: STAMHOOFD.TRANSACTIONAL_SMTP_HOST,
+            port: STAMHOOFD.TRANSACTIONAL_SMTP_PORT,
+            auth: {
+                user: STAMHOOFD.TRANSACTIONAL_SMTP_USERNAME, // generated ethereal user
+                pass: STAMHOOFD.TRANSACTIONAL_SMTP_PASSWORD // generated ethereal password
+            }
+        });
+
+        // verify connection configuration
+        this.transporter.verify((error) => {
+            if (error) {
+                console.error("SMTP server not working", error);
+            } else {
+                console.log("SMTP server is ready to take our messages");
+            }
+        });
+
+        // verify connection configuration
+        this.transactionalTransporter.verify((error) => {
+            if (error) {
+                console.error("Transactinoal SMTP server not working", error);
+            } else {
+                console.log("Transactinoal SMTP server is ready to take our messages");
+            }
+        });
+    }
+
+    private sendNextIfNeeded() {
+        if (!this.sending) {
+            if (this.currentQueue.length == 0) {
+                console.log("mail queue is empty")
+                return
+            }
+            let next = this.currentQueue[0]()
+
+            while (next === undefined) {
+                this.currentQueue.shift()
+                if (this.currentQueue.length == 0) {
+                    console.log("mail queue is empty")
+                    return
+                }
+                next = this.currentQueue[0]()
+            }
+
+            this.sending = true;
+            this.doSend(next).catch(e => {
+                console.error(e)
+            }).finally(() => {
+                this.sending = false
+                this.sendNextIfNeeded()
+            })
+        }
+    }
+
+    parseTo(to: string|EmailInterfaceRecipient[]): EmailInterfaceRecipient[] {
+        if (typeof to === "string") {
+            return this.parseEmailStr(to).map(email => ({ email }))
+        }
+
+        // Filter invalid email addresses
+        return to.filter(r => DataValidator.isEmailValid(r.email))
+    }
+
+    /**
+     * Get the raw email
+     */
+    parseEmailStr(emailStr: string): string[] {
+        let insideQuote = false
+        let escaped = false
+        let inAddr = false
+        let email = ""
+        let didFindAddr = false
+        let cleanedStr = ""
+
+        const addresses: string[] = []
+
+        function endAddress() {
+            let m: string
+            if (didFindAddr) {
+                m = email.trim()
+            } else {
+                m = cleanedStr.trim()
+            }
+            if (DataValidator.isEmailValid(m)) {
+                addresses.push(m)
+            }
+            didFindAddr = false
+            email = ""
+            inAddr = false
+            insideQuote = false
+            escaped = false
+            cleanedStr = ""
+        }
+
+        // eslint-disable-next-line @typescript-eslint/prefer-for-of
+        for (let index = 0; index < emailStr.length; index++) {
+            const shouldEscape = escaped
+            if (escaped) {
+                escaped = false
+            }
+            const character = emailStr[index];
+            if (insideQuote) {
+                if (character === "\\") {
+                    escaped = true
+                    continue
+                }
+            }
+
+            if (!shouldEscape) {
+                if (character === "\"") {
+                    if (insideQuote) {
+                        insideQuote = false
+                        continue
+                    }
+                    insideQuote = true
+                    continue
+                }
+
+                if (!insideQuote) {
+                    if (character === "<") {
+                        inAddr = true
+                        continue
+                    }
+
+                    if (character === ">") {
+                        inAddr = false
+                        didFindAddr = true
+                        continue
+                    }
+
+                    if (character === ",") {
+                        // End previous address
+                        endAddress()
+                        continue
+                    }
+                }
+            }
+
+            if (inAddr) {
+                email += character
+            }
+            cleanedStr += character
+        }
+
+        endAddress()
+        return addresses
+    }
+
+    private async doSend(data: EmailInterface) {
+        if (STAMHOOFD.environment === 'test') {
+            // Do not send any emails
+            return;
+        }
+
+        // Check if this email is not marked as spam
+        // Filter recipients if bounced or spam
+        let recipients = this.parseTo(data.to)
+        if (recipients.length === 0) {
+            // Invalid string
+            console.warn("Invalid e-mail string: '"+data.to+"'. E-mail skipped")
+            return
+        }
+
+        // Check spam and bounces
+        recipients = await EmailAddress.filterSendTo(recipients)
+
+        if (recipients.length === 0) {
+            // Invalid string
+            console.warn("Filtered all emails due hard bounce or spam '"+data.to+"'. E-mail skipped")
+            return
+        }
+
+        // Filter by environment
+        if (STAMHOOFD.environment === 'staging') {
+            recipients = recipients.filter(mail => !mail.email.includes("geen-email"))
+        }
+        if (STAMHOOFD.environment === 'development') {
+            recipients = recipients.filter(mail => mail.email.endsWith("@stamhoofd.be") || mail.email.endsWith("@bounce-testing.postmarkapp.com"))
+        }
+
+        if (recipients.length === 0) {
+            // Invalid string
+            console.warn("Filtered all emails due to environment filter '"+data.to+"'. E-mail skipped")
+            return
+        }
+    
+        // Rebuild to
+        const to = recipients.map((recipient) => {
+            if (!recipient.name) {
+                return recipient.email
+            }
+            const cleanedName = Formatter.emailSenderName(recipient.name)
+            if (cleanedName.length < 2) {
+                return recipient.email
+            }
+            return '"'+cleanedName+'" <'+recipient.email+'>'
+        }).join(", ")
+
+        this.setupIfNeeded();
+
+        // send mail with defined transport object
+        const mail: any = {
+            from: data.from, // sender address
+            bcc: (STAMHOOFD.environment === "production" || !data.bcc) ? data.bcc : "simon@stamhoofd.be",
+            replyTo: data.replyTo,
+            to,
+            subject: data.subject, // Subject line
+            text: data.text, // plain text body
+        };
+
+        if (data.attachments) {
+            mail.attachments = data.attachments;
+        }
+
+        if (data.headers) {
+            mail.headers = data.headers;
+        }
+
+        if (data.html) {
+            mail.html = data.html;
+
+            if (!data.text) {
+                mail.text = htmlToText.fromString(data.html, {
+                    wordwrap: null,
+                    unorderedListItemPrefix: " - "
+                });
+            }
+        }
+
+        try {
+            if (!data.from.includes('@stamhoofd.be') && !data.from.includes('@stamhoofd.nl')) {
+                // Not supported
+                data.type = 'broadcast'
+            }
+
+            const transporter = (data.type === "transactional") ? this.transactionalTransporter : this.transporter
+
+            
+            const info = await transporter.sendMail(mail);
+            console.log("Message sent:", to, data.subject, info.messageId, data.type);
+        } catch (e) {
+            console.error("Failed to send e-mail:")
+            console.error(e);
+            console.error(mail);
+
+            if (STAMHOOFD.environment === 'development') {
+                return;
+            }
+
+            // Sleep 1 second to give servers some time to fix possible rate limits
+            await sleep(1000);
+
+            // Reschedule twice (at maximum) to fix temporary connection issues
+            data.retryCount = (data.retryCount ?? 0) + 1;
+
+            if (data.retryCount <= 2) {
+                this.send(data);
+            } else {
+                // Email address is not verified.
+                if (!data.from.includes("hallo@stamhoofd.be")) {
+                    this.sendInternal({
+                        to: "hallo@stamhoofd.be",
+                        subject: "E-mail kon niet worden verzonden",
+                        text: "Een e-mail vanaf "+data.from+" kon niet worden verstuurd aan "+mail.to+": \n\n"+e+"\n\n"+(mail.text ?? ""),
+                        type: (data.type === "transactional") ? "broadcast" : "transactional"
+                    }, new I18n("nl", "BE"))
+                }
+            }
+        }
+    }
+
+    getInternalEmailFor(i18n: I18n) {
+        return '"Stamhoofd" <'+ (i18n.$t("shared.emails.general")) +'>'
+    }
+
+    getPersonalEmailFor(i18n: I18n) {
+        return '"Simon Backx" <'+ (i18n.$t("shared.emails.personal")) +'>'
+    }
+
+    /**
+     * Send an internal e-mail (from stamhoofd)
+     */
+    sendInternal(data: EmailInterfaceBase, i18n: I18n) {
+        const mail = Object.assign(data, { from: this.getInternalEmailFor(i18n) })
+        this.send(mail)
+    }
+
+    send(data: EmailInterface) {
+        if (STAMHOOFD.environment === 'test') {
+            // Do not send any emails
+            return;
+        }
+        let didSend = false
+
+        this.schedule(() => {
+            if (didSend) {
+                return undefined
+            }
+            didSend = true;
+            return data
+        })
+    }
+
+    schedule(builder: EmailBuilder) {
+        this.currentQueue.push(builder)
+        this.sendNextIfNeeded()
+    }
+}
+
+export const Email = new EmailStatic();

package/src/index.ts

@@ -0,0 +1,2 @@
+export * from "./models/EmailAddress"
+export * from "./classes/Email"

package/src/migrations/1603433451-email-addresses.sql

@@ -0,0 +1,14 @@
+CREATE TABLE `email_addresses` (
+  `id` varchar(36) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
+  `email` varchar(255) DEFAULT NULL,
+  `organizationId` varchar(36) CHARACTER SET ascii COLLATE ascii_general_ci DEFAULT NULL,
+  `markedAsSpam` tinyint(1) DEFAULT NULL,
+  `hardBounce` tinyint(1) DEFAULT NULL,
+  `unsubscribedMarketing` tinyint(1) DEFAULT NULL,
+  `unsubscribedAll` tinyint(1) DEFAULT NULL,
+  `token` varchar(255) CHARACTER SET ascii COLLATE ascii_general_ci DEFAULT NULL,
+  `createdAt` datetime NOT NULL,
+  `updatedAt` datetime NOT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `email` (`email`,`organizationId`) USING BTREE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;

package/src/models/EmailAddress.ts

@@ -0,0 +1,144 @@
+import { column, Database, Model } from '@simonbackx/simple-database';
+import crypto from "crypto";
+import { v4 as uuidv4 } from "uuid";
+import { QueueHandler } from '@stamhoofd/queues';
+import { EmailInterfaceRecipient } from '../classes/Email';
+
+async function randomBytes(size: number): Promise<Buffer> {
+    return new Promise((resolve, reject) => {
+        crypto.randomBytes(size, (err: Error | null, buf: Buffer) => {
+            if (err) {
+                reject(err);
+                return;
+            }
+            resolve(buf);
+        });
+    });
+}
+
+export class EmailAddress extends Model {
+    static table = "email_addresses";
+
+     @column({
+        primary: true, type: "string", beforeSave(value) {
+            return value ?? uuidv4();
+        }
+    })
+    id!: string;
+
+    @column({ type: "string", nullable: true })
+    organizationId: string | null = null;
+
+    // Columns
+    @column({ type: "string" })
+    email: string;
+
+    @column({ type: "boolean" })
+    markedAsSpam = false;
+
+    @column({ type: "boolean" })
+    hardBounce = false;
+
+    @column({ type: "boolean" })
+    unsubscribedMarketing = false;
+
+    @column({ type: "boolean" })
+    unsubscribedAll = false;
+
+    @column({ type: "string", nullable: true })
+    token: string | null;
+
+    /**
+     * createdAt behaves more like createdAt for Challenge. Since every save is considered to have a new challenge
+     */
+    @column({
+        type: "datetime", beforeSave() {
+            const date = new Date()
+            date.setMilliseconds(0)
+            return date
+        }
+    })
+    createdAt: Date
+
+    @column({
+        type: "datetime", beforeSave() {
+            const date = new Date()
+            date.setMilliseconds(0)
+            return date
+        }
+    })
+    updatedAt: Date
+
+    static async getOrCreate(email: string, organizationId: string | null): Promise<EmailAddress> {
+        // Prevent race conditions when checking the same email address at the same time, and creating a new one
+        return await QueueHandler.schedule("email-address/create-"+email+'-'+organizationId, async () => {
+            const existing = await this.getByEmail(email, organizationId)
+            if (existing) {
+                return existing
+            }
+
+            const n = new EmailAddress()
+            n.organizationId = organizationId
+            n.email = email
+            n.token = (await randomBytes(64)).toString("base64").toUpperCase();
+
+            await n.save()
+
+            return n
+        });
+    }
+
+    // Methods
+    static async getByEmails(emails: string[], organizationId: string | null): Promise<EmailAddress[]> {
+        if (emails.length > 30) {
+            // Normally an organization will never have so much bounces, so we'll request all emails and filter in them
+            const all = await this.where({ organizationId }, { limit: 1000 })
+            return all.filter(e => emails.includes(e.email))
+        }
+
+        if (emails.length == 0) {
+            return []
+        }
+
+        if (organizationId === null) {
+            const [rows] = await Database.select(
+                `SELECT ${this.getDefaultSelect()} FROM ${this.table} WHERE \`email\` IN (?) AND \`organizationId\` is NULL`,
+                [emails]
+            );
+
+            return this.fromRows(rows, this.table);
+        }
+
+        const [rows] = await Database.select(
+            `SELECT ${this.getDefaultSelect()} FROM ${this.table} WHERE \`email\` IN (?) AND \`organizationId\` = ?`,
+            [emails, organizationId]
+        );
+
+        return this.fromRows(rows, this.table);
+    }
+
+    // Methods
+    static async getByEmail(email: string, organizationId: string | null): Promise<EmailAddress | undefined> {
+        return (await this.where({ email, organizationId }, { limit: 1 }))[0]
+    }
+
+    // Methods
+    static async filterSendTo(recipients: EmailInterfaceRecipient[]): Promise<EmailInterfaceRecipient[]> {
+        if (recipients.length == 0) {
+            return []
+        }
+
+        const emails = recipients.map(r => r.email)
+        const [results] = await Database.select(
+            `SELECT email FROM ${this.table} WHERE \`email\` IN (?) AND (\`hardBounce\` = 1 OR \`markedAsSpam\` = 1)`,
+            [emails]
+        );
+
+        const remove = results.map(r => r[this.table]['email'])
+        if (remove.length == 0) {
+            return recipients
+        }
+
+        return recipients.filter(r => !remove.includes(r.email))
+    }
+}