@budibase/worker 3.4.20 → 3.4.22

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "3.4.20",
+  "version": "3.4.22",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -85,11 +85,14 @@
     "@types/jsonwebtoken": "9.0.3",
     "@types/koa__router": "12.0.4",
     "@types/lodash": "4.14.200",
+    "@types/maildev": "^0.0.7",
     "@types/node-fetch": "2.6.4",
+    "@types/nodemailer": "^6.4.17",
     "@types/server-destroy": "1.0.1",
     "@types/supertest": "2.0.14",
     "@types/uuid": "8.3.4",
     "jest": "29.7.0",
+    "maildev": "^2.2.1",
     "nock": "^13.5.4",
     "nodemon": "2.0.15",
     "rimraf": "3.0.2",
@@ -114,5 +117,5 @@
       }
     }
   },
-  "gitHead": "7f0df167ae6af3b8b9e45e0c7fe70dd20216d707"
+  "gitHead": "d13bdcbc41f1c15066ef24c19290452cd5c4dd46"
 }

package/src/api/controllers/global/email.ts

@@ -11,7 +11,6 @@ export async function sendEmail(
   ctx: UserCtx<SendEmailRequest, SendEmailResponse>
 ) {
   let {
-    workspaceId,
     email,
     userId,
     purpose,
@@ -24,13 +23,15 @@ export async function sendEmail(
     invite,
     attachments,
   } = ctx.request.body
-  let user: any
+  let user: User | undefined = undefined
   if (userId) {
     const db = tenancy.getGlobalDB()
-    user = await db.get<User>(userId)
+    user = await db.tryGet<User>(userId)
+    if (!user) {
+      ctx.throw(404, "User not found.")
+    }
   }
   const response = await sendEmailFn(email, purpose, {
-    workspaceId,
     user,
     contents,
     from,

package/src/api/routes/global/tests/email.spec.ts

@@ -1,33 +1,269 @@
-jest.mock("nodemailer")
-import { EmailTemplatePurpose } from "@budibase/types"
-import { TestConfiguration, mocks } from "../../../../tests"
-
-const sendMailMock = mocks.email.mock()
+import { EmailTemplatePurpose, SendEmailRequest } from "@budibase/types"
+import { TestConfiguration } from "../../../../tests"
+import {
+  captureEmail,
+  deleteAllEmail,
+  getAttachments,
+  Mailserver,
+  startMailserver,
+  stopMailserver,
+} from "../../../../tests/mocks/email"
+import { objectStore } from "@budibase/backend-core"
 
 describe("/api/global/email", () => {
   const config = new TestConfiguration()
+  let mailserver: Mailserver
 
   beforeAll(async () => {
     await config.beforeAll()
+    mailserver = await startMailserver(config)
   })
 
   afterAll(async () => {
+    await stopMailserver(mailserver)
     await config.afterAll()
   })
 
-  it("should be able to send an email (with mocking)", async () => {
-    // initially configure settings
-    await config.saveSmtpConfig()
-    await config.saveSettingsConfig()
+  beforeEach(async () => {
+    await deleteAllEmail(mailserver)
+  })
+
+  interface TestCase {
+    req: Partial<SendEmailRequest>
+    expectedStatus?: number
+    expectedContents?: string
+  }
+
+  const testCases: TestCase[] = [
+    {
+      req: {
+        purpose: EmailTemplatePurpose.WELCOME,
+      },
+      expectedContents: `Thanks for getting started with Budibase's Budibase platform.`,
+    },
+    {
+      req: {
+        purpose: EmailTemplatePurpose.INVITATION,
+      },
+      expectedContents: `Use the button below to set up your account and get started:`,
+    },
+    {
+      req: {
+        purpose: EmailTemplatePurpose.PASSWORD_RECOVERY,
+      },
+      expectedContents: `You recently requested to reset your password for your Budibase account in your Budibase platform`,
+    },
+    {
+      req: {
+        purpose: EmailTemplatePurpose.CUSTOM,
+        contents: "Hello, world!",
+      },
+      expectedContents: "Hello, world!",
+    },
+  ]
+
+  it.each(testCases)(
+    "can send $req.purpose emails",
+    async ({ req, expectedContents, expectedStatus }) => {
+      const email = await captureEmail(mailserver, async () => {
+        const res = await config.api.emails.sendEmail(
+          {
+            email: "to@example.com",
+            subject: "Test",
+            userId: config.user!._id,
+            purpose: EmailTemplatePurpose.WELCOME,
+            ...req,
+          },
+          {
+            status: expectedStatus || 200,
+          }
+        )
+        expect(res.message).toBeDefined()
+      })
+
+      expect(email.html).toContain(expectedContents)
+      expect(email.html).not.toContain("Invalid binding")
+    }
+  )
+
+  it("should be able to send an email with an attachment", async () => {
+    let bucket = "testbucket"
+    let filename = "test.txt"
+    await objectStore.upload({
+      bucket,
+      filename,
+      body: Buffer.from("test data"),
+    })
+    let presignedUrl = await objectStore.getPresignedUrl(
+      bucket,
+      filename,
+      60000
+    )
+
+    let attachmentObject = {
+      url: presignedUrl,
+      filename,
+    }
+
+    const email = await captureEmail(mailserver, async () => {
+      const res = await config.api.emails.sendEmail({
+        email: "to@example.com",
+        subject: "Test",
+        userId: config.user!._id,
+        purpose: EmailTemplatePurpose.WELCOME,
+        attachments: [attachmentObject],
+      })
+      expect(res.message).toBeDefined()
+    })
+
+    expect(email.html).toContain(
+      "Thanks for getting started with Budibase's Budibase platform."
+    )
+    expect(email.html).not.toContain("Invalid binding")
+
+    const attachments = await getAttachments(mailserver, email)
+    expect(attachments).toEqual(["test data"])
+  })
+
+  it("should be able to send email without a userId", async () => {
+    const res = await config.api.emails.sendEmail({
+      email: "to@example.com",
+      subject: "Test",
+      purpose: EmailTemplatePurpose.WELCOME,
+    })
+    expect(res.message).toBeDefined()
+  })
 
+  it("should fail to send a password reset email without a userId", async () => {
     const res = await config.api.emails.sendEmail(
-      EmailTemplatePurpose.INVITATION
+      {
+        email: "to@example.com",
+        subject: "Test",
+        purpose: EmailTemplatePurpose.PASSWORD_RECOVERY,
+      },
+      {
+        status: 400,
+      }
     )
+    expect(res.message).toBeDefined()
+  })
+
+  it("can cc people", async () => {
+    const email = await captureEmail(mailserver, async () => {
+      await config.api.emails.sendEmail({
+        email: "to@example.com",
+        cc: "cc@example.com",
+        subject: "Test",
+        purpose: EmailTemplatePurpose.CUSTOM,
+        contents: "Hello, world!",
+      })
+    })
 
-    expect(res.body.message).toBeDefined()
-    expect(sendMailMock).toHaveBeenCalled()
-    const emailCall = sendMailMock.mock.calls[0][0]
-    expect(emailCall.subject).toBe("Hello!")
-    expect(emailCall.html).not.toContain("Invalid binding")
+    expect(email.cc).toEqual([{ address: "cc@example.com", name: "" }])
+  })
+
+  it("can bcc people", async () => {
+    const email = await captureEmail(mailserver, async () => {
+      await config.api.emails.sendEmail({
+        email: "to@example.com",
+        bcc: "bcc@example.com",
+        subject: "Test",
+        purpose: EmailTemplatePurpose.CUSTOM,
+        contents: "Hello, world!",
+      })
+    })
+
+    expect(email.calculatedBcc).toEqual([
+      { address: "bcc@example.com", name: "" },
+    ])
+  })
+
+  it("can change the from address", async () => {
+    const email = await captureEmail(mailserver, async () => {
+      const res = await config.api.emails.sendEmail({
+        email: "to@example.com",
+        from: "from@example.com",
+        subject: "Test",
+        purpose: EmailTemplatePurpose.CUSTOM,
+        contents: "Hello, world!",
+      })
+      expect(res.message).toBeDefined()
+    })
+
+    expect(email.to).toEqual([{ address: "to@example.com", name: "" }])
+    expect(email.from).toEqual([{ address: "from@example.com", name: "" }])
+  })
+
+  it("can send a calendar invite", async () => {
+    const startTime = new Date()
+    const endTime = new Date()
+
+    const email = await captureEmail(mailserver, async () => {
+      await config.api.emails.sendEmail({
+        email: "to@example.com",
+        subject: "Test",
+        purpose: EmailTemplatePurpose.CUSTOM,
+        contents: "Hello, world!",
+        invite: {
+          startTime,
+          endTime,
+          summary: "Summary",
+          location: "Location",
+          url: "http://example.com",
+        },
+      })
+    })
+
+    expect(email.alternatives).toEqual([
+      {
+        charset: "utf-8",
+        contentType: "text/calendar",
+        method: "REQUEST",
+        transferEncoding: "7bit",
+        content: expect.any(String),
+      },
+    ])
+
+    // Reference iCal invite:
+    //   BEGIN:VCALENDAR
+    //   VERSION:2.0
+    //   PRODID:-//sebbo.net//ical-generator//EN
+    //   NAME:Invite
+    //   X-WR-CALNAME:Invite
+    //   BEGIN:VEVENT
+    //   UID:2b5947b7-ec5a-4341-8d70-8d8130183f2a
+    //   SEQUENCE:0
+    //   DTSTAMP:20200101T000000Z
+    //   DTSTART:20200101T000000Z
+    //   DTEND:20200101T000000Z
+    //   SUMMARY:Summary
+    //   LOCATION:Location
+    //   URL;VALUE=URI:http://example.com
+    //   END:VEVENT
+    //   END:VCALENDAR
+    expect(email.alternatives[0].content).toContain("BEGIN:VCALENDAR")
+    expect(email.alternatives[0].content).toContain("BEGIN:VEVENT")
+    expect(email.alternatives[0].content).toContain("UID:")
+    expect(email.alternatives[0].content).toContain("SEQUENCE:0")
+    expect(email.alternatives[0].content).toContain("SUMMARY:Summary")
+    expect(email.alternatives[0].content).toContain("LOCATION:Location")
+    expect(email.alternatives[0].content).toContain(
+      "URL;VALUE=URI:http://example.com"
+    )
+    expect(email.alternatives[0].content).toContain("END:VEVENT")
+    expect(email.alternatives[0].content).toContain("END:VCALENDAR")
+
+    const formatDate = (date: Date) =>
+      date.toISOString().replace(/[-:]/g, "").split(".")[0] + "Z"
+
+    expect(email.alternatives[0].content).toContain(
+      `DTSTAMP:${formatDate(startTime)}`
+    )
+    expect(email.alternatives[0].content).toContain(
+      `DTSTART:${formatDate(startTime)}`
+    )
+    expect(email.alternatives[0].content).toContain(
+      `DTEND:${formatDate(endTime)}`
+    )
   })
 })

package/src/tests/TestConfiguration.ts

@@ -32,6 +32,8 @@ import {
   AuthToken,
   SCIMConfig,
   ConfigType,
+  SMTPConfig,
+  SMTPInnerConfig,
 } from "@budibase/types"
 import API from "./api"
 import jwt, { Secret } from "jsonwebtoken"
@@ -348,9 +350,15 @@ class TestConfiguration {
 
   // CONFIGS - SMTP
 
-  async saveSmtpConfig() {
+  async saveSmtpConfig(config?: SMTPInnerConfig) {
     await this.deleteConfig(Config.SMTP)
-    await this._req(structures.configs.smtp(), null, controllers.config.save)
+
+    let smtpConfig: SMTPConfig = structures.configs.smtp()
+    if (config) {
+      smtpConfig = { type: ConfigType.SMTP, config }
+    }
+
+    await this._req(smtpConfig, null, controllers.config.save)
   }
 
   async saveEtherealSmtpConfig() {

package/src/tests/api/email.ts

@@ -1,19 +1,18 @@
-import { EmailAttachment } from "@budibase/types"
+import { SendEmailRequest, SendEmailResponse } from "@budibase/types"
 import { TestAPI } from "./base"
 
 export class EmailAPI extends TestAPI {
-  sendEmail = (purpose: string, attachments?: EmailAttachment[]) => {
-    return this.request
+  sendEmail = async (
+    req: SendEmailRequest,
+    expectations?: { status?: number }
+  ): Promise<SendEmailResponse> => {
+    const res = await this.request
       .post(`/api/global/email/send`)
-      .send({
-        email: "test@example.com",
-        attachments,
-        purpose,
-        tenantId: this.config.getTenantId(),
-        userId: this.config.user!._id!,
-      })
+      .send(req)
       .set(this.config.defaultHeaders())
       .expect("Content-Type", /json/)
-      .expect(200)
+      .expect(expectations?.status || 200)
+
+    return res.body as SendEmailResponse
   }
 }

package/src/tests/mocks/email.ts

@@ -1,3 +1,10 @@
+import MailDev from "maildev"
+import { promisify } from "util"
+import TestConfiguration from "../TestConfiguration"
+
+/**
+ * @deprecated please use the `MailDev` email server instead of this mock.
+ */
 export function mock() {
   // mock the email system
   const sendMailMock = jest.fn()
@@ -8,3 +15,170 @@ export function mock() {
   })
   return sendMailMock
 }
+
+export type Mailserver = InstanceType<typeof MailDev>
+export type MailserverConfig = ConstructorParameters<typeof MailDev>[0]
+
+export interface Attachment {
+  checksum: string
+  contentId: string
+  contentType: string
+  fileName: string
+  generatedFileName: string
+  length: number
+  transferEncoding: string
+  transformed: boolean
+}
+
+export interface Address {
+  address: string
+  args?: boolean
+  name?: string
+}
+
+export interface Alternative {
+  contentType: string
+  content: string
+  charset: string
+  method: string
+  transferEncoding: string
+}
+
+export interface Envelope {
+  from: Address
+  to: Address[]
+  host: string
+  remoteAddress: string
+}
+
+export interface Email {
+  attachments: Attachment[]
+  alternatives: Alternative[]
+  calculatedBcc: Address[]
+  cc: Address[]
+  date: string
+  envelope: Envelope
+  from: Address[]
+  headers: Record<string, string>
+  html: string
+  id: string
+  messageId: string
+  priority: string
+  read: boolean
+  size: number
+  sizeHuman: string
+  source: string
+  time: Date
+  to: Address[]
+}
+
+export function getUnusedPort(): Promise<number> {
+  return new Promise((resolve, reject) => {
+    const server = require("net").createServer()
+    server.unref()
+    server.on("error", reject)
+    server.listen(0, () => {
+      const port = server.address().port
+      server.close(() => {
+        resolve(port)
+      })
+    })
+  })
+}
+
+export async function captureEmail(
+  mailserver: Mailserver,
+  f: () => Promise<void>
+): Promise<Email> {
+  const timeoutMs = 5000
+  let timeout: ReturnType<typeof setTimeout> | undefined = undefined
+  const cancel = () => {
+    if (timeout) {
+      clearTimeout(timeout)
+      timeout = undefined
+    }
+  }
+  const timeoutPromise = new Promise<never>((_, reject) => {
+    timeout = setTimeout(() => {
+      reject(new Error("Timed out waiting for email"))
+    }, timeoutMs)
+  })
+  const mailPromise = new Promise<Email>(resolve => {
+    // @ts-expect-error - types are wrong
+    mailserver.once("new", email => {
+      resolve(email as Email)
+      cancel()
+    })
+  })
+  const emailPromise = Promise.race([mailPromise, timeoutPromise])
+  try {
+    await f()
+  } finally {
+    cancel()
+  }
+  return await emailPromise
+}
+
+export async function startMailserver(
+  config: TestConfiguration,
+  opts?: MailserverConfig
+): Promise<Mailserver> {
+  if (!opts) {
+    opts = {}
+  }
+  if (!opts.smtp) {
+    opts.smtp = await getUnusedPort()
+  }
+  const mailserver = new MailDev(opts || {})
+  await new Promise((resolve, reject) => {
+    mailserver.listen(err => {
+      if (err) {
+        return reject(err)
+      }
+      resolve(mailserver)
+    })
+  })
+  await config.saveSmtpConfig({
+    host: "localhost",
+    port: opts.smtp,
+    secure: false,
+    from: "test@example.com",
+  })
+  return mailserver
+}
+
+export function deleteAllEmail(mailserver: Mailserver) {
+  return promisify(mailserver.deleteAllEmail).bind(mailserver)()
+}
+
+export function stopMailserver(mailserver: Mailserver) {
+  return promisify(mailserver.close).bind(mailserver)()
+}
+
+export function getAttachment(
+  mailserver: Mailserver,
+  email: Email,
+  attachment: Attachment
+) {
+  return new Promise<string>(resolve => {
+    // @ts-expect-error - types are wrong
+    mailserver.getEmailAttachment(
+      email.id,
+      attachment.generatedFileName,
+      (err: any, _contentType: string, stream: ReadableStream) => {
+        if (err) {
+          throw err
+        }
+        resolve(new Response(stream).text())
+      }
+    )
+  })
+}
+
+export function getAttachments(mailserver: Mailserver, email: Email) {
+  return Promise.all(
+    email.attachments.map(attachment =>
+      getAttachment(mailserver, email, attachment)
+    )
+  )
+}

package/src/utilities/email.ts

@@ -4,16 +4,17 @@ import { getTemplateByPurpose, EmailTemplates } from "../constants/templates"
 import { getSettingsTemplateContext } from "./templates"
 import { processString } from "@budibase/string-templates"
 import {
-  User,
   SendEmailOpts,
   SMTPInnerConfig,
   EmailTemplatePurpose,
+  User,
 } from "@budibase/types"
-import { configs, cache, objectStore } from "@budibase/backend-core"
+import { configs, cache, objectStore, HTTPError } from "@budibase/backend-core"
 import ical from "ical-generator"
 import _ from "lodash"
 
-const nodemailer = require("nodemailer")
+import nodemailer from "nodemailer"
+import SMTPTransport from "nodemailer/lib/smtp-transport"
 
 const TEST_MODE = env.ENABLE_EMAIL_TEST_MODE && env.isDev()
 const TYPE = TemplateType.EMAIL
@@ -26,7 +27,7 @@ const FULL_EMAIL_PURPOSES = [
 ]
 
 function createSMTPTransport(config?: SMTPInnerConfig) {
-  let options: any
+  let options: SMTPTransport.Options
   let secure = config?.secure
   // default it if not specified
   if (secure == null) {
@@ -59,22 +60,6 @@ function createSMTPTransport(config?: SMTPInnerConfig) {
   return nodemailer.createTransport(options)
 }
 
-async function getLinkCode(
-  purpose: EmailTemplatePurpose,
-  email: string,
-  user: User,
-  info: any = null
-) {
-  switch (purpose) {
-    case EmailTemplatePurpose.PASSWORD_RECOVERY:
-      return cache.passwordReset.createCode(user._id!, info)
-    case EmailTemplatePurpose.INVITATION:
-      return cache.invite.createCode(email, info)
-    default:
-      return null
-  }
-}
-
 /**
  * Builds an email using handlebars and the templates found in the system (default or otherwise).
  * @param purpose the purpose of the email being built, e.g. invitation, password reset.
@@ -87,8 +72,8 @@ async function getLinkCode(
 async function buildEmail(
   purpose: EmailTemplatePurpose,
   email: string,
-  context: any,
-  { user, contents }: any = {}
+  context: Record<string, any>,
+  { user, contents }: { user?: User; contents?: string } = {}
 ) {
   // this isn't a full email
   if (FULL_EMAIL_PURPOSES.indexOf(purpose) === -1) {
@@ -106,8 +91,8 @@ async function buildEmail(
     throw "Unable to build email, missing base components"
   }
 
-  let name = user ? user.name : undefined
-  if (user && !name && user.firstName) {
+  let name: string | undefined
+  if (user && user.firstName) {
     name = user.lastName ? `${user.firstName} ${user.lastName}` : user.firstName
   }
   context = {
@@ -158,10 +143,21 @@ export async function sendEmail(
   }
   const transport = createSMTPTransport(config)
   // if there is a link code needed this will retrieve it
-  const code = await getLinkCode(purpose, email, opts.user, opts?.info)
+  let code: string | null = null
+  switch (purpose) {
+    case EmailTemplatePurpose.PASSWORD_RECOVERY:
+      if (!opts.user || !opts.user._id) {
+        throw new HTTPError("User must be provided for password recovery.", 400)
+      }
+      code = await cache.passwordReset.createCode(opts.user._id, opts.info)
+      break
+    case EmailTemplatePurpose.INVITATION:
+      code = await cache.invite.createCode(email, opts.info)
+      break
+  }
   let context = await getSettingsTemplateContext(purpose, code)
 
-  let message: any = {
+  let message: Parameters<typeof transport.sendMail>[0] = {
     from: opts?.from || config?.from,
     html: await buildEmail(purpose, email, context, {
       user: opts?.user,

package/src/api/routes/global/tests/realEmail.spec.ts

@@ -1,108 +0,0 @@
-jest.unmock("node-fetch")
-import { TestConfiguration } from "../../../../tests"
-import { objectStore } from "@budibase/backend-core"
-import { helpers } from "@budibase/shared-core"
-
-import tk from "timekeeper"
-import { EmailAttachment, EmailTemplatePurpose } from "@budibase/types"
-
-const fetch = require("node-fetch")
-
-const nodemailer = require("nodemailer")
-
-// for the real email tests give them a long time to try complete/fail
-jest.setTimeout(30000)
-
-describe("/api/global/email", () => {
-  const config = new TestConfiguration()
-
-  beforeAll(async () => {
-    tk.reset()
-    await config.beforeAll()
-  })
-
-  afterAll(async () => {
-    await config.afterAll()
-  })
-
-  async function sendRealEmail(
-    purpose: string,
-    attachments?: EmailAttachment[]
-  ) {
-    let response, text
-    try {
-      await helpers.withTimeout(20000, () => config.saveEtherealSmtpConfig())
-      await helpers.withTimeout(20000, () => config.saveSettingsConfig())
-      let res
-      if (attachments) {
-        res = await config.api.emails
-          .sendEmail(purpose, attachments)
-          .timeout(20000)
-      } else {
-        res = await config.api.emails.sendEmail(purpose).timeout(20000)
-      }
-      // ethereal hiccup, can't test right now
-      if (res.status >= 300) {
-        return
-      }
-      expect(res.body.message).toBeDefined()
-      const testUrl = nodemailer.getTestMessageUrl(res.body)
-      expect(testUrl).toBeDefined()
-      response = await fetch(testUrl)
-      text = await response.text()
-    } catch (err: any) {
-      // ethereal hiccup, can't test right now
-      if (parseInt(err.status) >= 300 || (err && err.errno === "ETIME")) {
-        return
-      } else {
-        throw err
-      }
-    }
-    let toCheckFor
-    switch (purpose) {
-      case EmailTemplatePurpose.WELCOME:
-        toCheckFor = `Thanks for getting started with Budibase's Budibase platform.`
-        break
-      case EmailTemplatePurpose.INVITATION:
-        toCheckFor = `Use the button below to set up your account and get started:`
-        break
-      case EmailTemplatePurpose.PASSWORD_RECOVERY:
-        toCheckFor = `You recently requested to reset your password for your Budibase account in your Budibase platform`
-        break
-    }
-    expect(text).toContain(toCheckFor)
-  }
-
-  it("should be able to send a welcome email", async () => {
-    await sendRealEmail(EmailTemplatePurpose.WELCOME)
-  })
-
-  it("should be able to send a invitation email", async () => {
-    await sendRealEmail(EmailTemplatePurpose.INVITATION)
-  })
-
-  it("should be able to send a password recovery email", async () => {
-    await sendRealEmail(EmailTemplatePurpose.PASSWORD_RECOVERY)
-  })
-
-  it("should be able to send an email with attachments", async () => {
-    let bucket = "testbucket"
-    let filename = "test.txt"
-    await objectStore.upload({
-      bucket,
-      filename,
-      body: Buffer.from("test data"),
-    })
-    let presignedUrl = await objectStore.getPresignedUrl(
-      bucket,
-      filename,
-      60000
-    )
-
-    let attachmentObject = {
-      url: presignedUrl,
-      filename,
-    }
-    await sendRealEmail(EmailTemplatePurpose.WELCOME, [attachmentObject])
-  })
-})