@budibase/backend-core 3.2.5 → 3.2.7

package/src/cache/docWritethrough.ts

@@ -1,105 +0,0 @@
-import { AnyDocument, Database } from "@budibase/types"
-
-import { JobQueue, Queue, createQueue } from "../queue"
-import * as dbUtils from "../db"
-
-interface ProcessDocMessage {
-  dbName: string
-  docId: string
-  data: Record<string, any>
-}
-
-const PERSIST_MAX_ATTEMPTS = 100
-let processor: DocWritethroughProcessor | undefined
-
-export class DocWritethroughProcessor {
-  private static _queue: Queue
-
-  public static get queue() {
-    if (!DocWritethroughProcessor._queue) {
-      DocWritethroughProcessor._queue = createQueue<ProcessDocMessage>(
-        JobQueue.DOC_WRITETHROUGH_QUEUE,
-        {
-          jobOptions: {
-            attempts: PERSIST_MAX_ATTEMPTS,
-          },
-        }
-      )
-    }
-
-    return DocWritethroughProcessor._queue
-  }
-
-  init() {
-    DocWritethroughProcessor.queue.process(async message => {
-      try {
-        await this.persistToDb(message.data)
-      } catch (err: any) {
-        if (err.status === 409) {
-          // If we get a 409, it means that another job updated it meanwhile. We want to retry it to persist it again.
-          throw new Error(
-            `Conflict persisting message ${message.id}. Attempt ${message.attemptsMade}`
-          )
-        }
-
-        throw err
-      }
-    })
-    return this
-  }
-
-  private async persistToDb({
-    dbName,
-    docId,
-    data,
-  }: {
-    dbName: string
-    docId: string
-    data: Record<string, any>
-  }) {
-    const db = dbUtils.getDB(dbName)
-    let doc: AnyDocument | undefined
-    try {
-      doc = await db.get(docId)
-    } catch {
-      doc = { _id: docId }
-    }
-
-    doc = { ...doc, ...data }
-    await db.put(doc)
-  }
-}
-
-export class DocWritethrough {
-  private db: Database
-  private _docId: string
-
-  constructor(db: Database, docId: string) {
-    this.db = db
-    this._docId = docId
-  }
-
-  get docId() {
-    return this._docId
-  }
-
-  async patch(data: Record<string, any>) {
-    await DocWritethroughProcessor.queue.add({
-      dbName: this.db.name,
-      docId: this.docId,
-      data,
-    })
-  }
-}
-
-export function init(): DocWritethroughProcessor {
-  processor = new DocWritethroughProcessor().init()
-  return processor
-}
-
-export function getProcessor(): DocWritethroughProcessor {
-  if (!processor) {
-    return init()
-  }
-  return processor
-}

package/src/cache/generic.ts

@@ -1,33 +0,0 @@
-import BaseCache from "./base"
-
-const GENERIC = new BaseCache()
-
-export enum CacheKey {
-  CHECKLIST = "checklist",
-  INSTALLATION = "installation",
-  ANALYTICS_ENABLED = "analyticsEnabled",
-  UNIQUE_TENANT_ID = "uniqueTenantId",
-  EVENTS = "events",
-  BACKFILL_METADATA = "backfillMetadata",
-  EVENTS_RATE_LIMIT = "eventsRateLimit",
-}
-
-export enum TTL {
-  ONE_MINUTE = 600,
-  ONE_HOUR = 3600,
-  ONE_DAY = 86400,
-}
-
-export const keys = (...args: Parameters<typeof GENERIC.keys>) =>
-  GENERIC.keys(...args)
-export const get = (...args: Parameters<typeof GENERIC.get>) =>
-  GENERIC.get(...args)
-export const store = (...args: Parameters<typeof GENERIC.store>) =>
-  GENERIC.store(...args)
-export const destroy = (...args: Parameters<typeof GENERIC.delete>) =>
-  GENERIC.delete(...args)
-export const withCache = <T>(
-  ...args: Parameters<typeof GENERIC.withCache<T>>
-) => GENERIC.withCache(...args)
-export const bustCache = (...args: Parameters<typeof GENERIC.bustCache>) =>
-  GENERIC.bustCache(...args)

package/src/cache/index.ts

@@ -1,8 +0,0 @@
-export * as generic from "./generic"
-export * as user from "./user"
-export * as app from "./appMetadata"
-export * as writethrough from "./writethrough"
-export * as invite from "./invite"
-export * as passwordReset from "./passwordReset"
-export * from "./generic"
-export * as docWritethrough from "./docWritethrough"

package/src/cache/invite.ts

@@ -1,86 +0,0 @@
-import * as utils from "../utils"
-import { Duration } from "../utils"
-import env from "../environment"
-import { getTenantId } from "../context"
-import * as redis from "../redis/init"
-
-const TTL_SECONDS = Duration.fromDays(7).toSeconds()
-
-interface Invite {
-  email: string
-  info: any
-}
-
-interface InviteWithCode extends Invite {
-  code: string
-}
-
-/**
- * Given an invite code and invite body, allow the update an existing/valid invite in redis
- * @param code The invite code for an invite in redis
- * @param value The body of the updated user invitation
- */
-export async function updateCode(code: string, value: Invite) {
-  const client = await redis.getInviteClient()
-  await client.store(code, value, TTL_SECONDS)
-}
-
-/**
- * Generates an invitation code and writes it to redis - which can later be checked for user creation.
- * @param email the email address which the code is being sent to (for use later).
- * @param info Information to be carried along with the invitation.
- * @return returns the code that was stored to redis.
- */
-export async function createCode(email: string, info: any): Promise<string> {
-  const code = utils.newid()
-  const client = await redis.getInviteClient()
-  await client.store(code, { email, info }, TTL_SECONDS)
-  return code
-}
-
-/**
- * Checks that the provided invite code is valid - will return the email address of user that was invited.
- * @param code the invite code that was provided as part of the link.
- * @return If the code is valid then an email address will be returned.
- */
-export async function getCode(code: string): Promise<Invite> {
-  const client = await redis.getInviteClient()
-  const value = (await client.get(code)) as Invite | undefined
-  if (!value) {
-    throw "Invitation is not valid or has expired, please request a new one."
-  }
-  return value
-}
-
-export async function deleteCode(code: string) {
-  const client = await redis.getInviteClient()
-  await client.delete(code)
-}
-
-/**
- Get all currently available user invitations for the current tenant.
- **/
-export async function getInviteCodes(): Promise<InviteWithCode[]> {
-  const client = await redis.getInviteClient()
-  const invites: { key: string; value: Invite }[] = await client.scan()
-
-  const results: InviteWithCode[] = invites.map(invite => {
-    return {
-      ...invite.value,
-      code: invite.key,
-    }
-  })
-  if (!env.MULTI_TENANCY) {
-    return results
-  }
-  const tenantId = getTenantId()
-  return results.filter(invite => tenantId === invite.info.tenantId)
-}
-
-export async function getExistingInvites(
-  emails: string[]
-): Promise<InviteWithCode[]> {
-  return (await getInviteCodes()).filter(invite =>
-    emails.includes(invite.email)
-  )
-}

package/src/cache/passwordReset.ts

@@ -1,49 +0,0 @@
-import * as redis from "../redis/init"
-import * as utils from "../utils"
-import { Duration } from "../utils"
-
-const TTL_SECONDS = Duration.fromHours(1).toSeconds()
-
-interface PasswordReset {
-  userId: string
-  info: any
-}
-
-/**
- * Given a user ID this will store a code (that is returned) for an hour in redis.
- * The user can then return this code for resetting their password (through their reset link).
- * @param userId the ID of the user which is to be reset.
- * @param info Info about the user/the reset process.
- * @return returns the code that was stored to redis.
- */
-export async function createCode(userId: string, info: any): Promise<string> {
-  const code = utils.newid()
-  const client = await redis.getPasswordResetClient()
-  await client.store(code, { userId, info }, TTL_SECONDS)
-  return code
-}
-
-/**
- * Given a reset code this will lookup to redis, check if the code is valid.
- * @param code The code provided via the email link.
- * @return returns the user ID if it is found
- */
-export async function getCode(code: string): Promise<PasswordReset> {
-  const client = await redis.getPasswordResetClient()
-  const value = (await client.get(code)) as PasswordReset | undefined
-  if (!value) {
-    throw new Error(
-      "Provided information is not valid, cannot reset password - please try again."
-    )
-  }
-  return value
-}
-
-/**
- * Given a reset code this will invalidate it.
- * @param code The code provided via the email link.
- */
-export async function invalidateCode(code: string): Promise<void> {
-  const client = await redis.getPasswordResetClient()
-  await client.delete(code)
-}

package/src/cache/tests/docWritethrough.spec.ts

@@ -1,296 +0,0 @@
-import tk from "timekeeper"
-
-import _ from "lodash"
-import {
-  DBTestConfiguration,
-  generator,
-  structures,
-  utils,
-} from "../../../tests"
-import { getDB } from "../../db"
-
-import {
-  DocWritethrough,
-  DocWritethroughProcessor,
-  init,
-} from "../docWritethrough"
-
-const initialTime = Date.now()
-
-async function waitForQueueCompletion() {
-  await utils.queue.processMessages(DocWritethroughProcessor.queue)
-}
-
-beforeAll(() => utils.queue.useRealQueues())
-
-describe("docWritethrough", () => {
-  beforeAll(() => {
-    init()
-  })
-
-  const config = new DBTestConfiguration()
-
-  const db = getDB(structures.db.id())
-  let documentId: string
-  let docWritethrough: DocWritethrough
-
-  describe("patch", () => {
-    function generatePatchObject(fieldCount: number) {
-      const keys = generator.unique(() => generator.guid(), fieldCount)
-      return keys.reduce((acc, c) => {
-        acc[c] = generator.word()
-        return acc
-      }, {} as Record<string, any>)
-    }
-
-    beforeEach(async () => {
-      jest.clearAllMocks()
-      documentId = structures.uuid()
-      docWritethrough = new DocWritethrough(db, documentId)
-    })
-
-    it("patching will not persist until the messages are persisted", async () => {
-      await config.doInTenant(async () => {
-        await docWritethrough.patch(generatePatchObject(2))
-        await docWritethrough.patch(generatePatchObject(2))
-
-        expect(await db.exists(documentId)).toBe(false)
-      })
-    })
-
-    it("patching will persist when the messages are persisted", async () => {
-      await config.doInTenant(async () => {
-        const patch1 = generatePatchObject(2)
-        const patch2 = generatePatchObject(2)
-        await docWritethrough.patch(patch1)
-        await docWritethrough.patch(patch2)
-
-        await waitForQueueCompletion()
-
-        // This will not be persisted
-        const patch3 = generatePatchObject(3)
-        await docWritethrough.patch(patch3)
-
-        expect(await db.tryGet(documentId)).toEqual({
-          _id: documentId,
-          ...patch1,
-          ...patch2,
-          _rev: expect.stringMatching(/2-.+/),
-          createdAt: new Date(initialTime).toISOString(),
-          updatedAt: new Date(initialTime).toISOString(),
-        })
-      })
-    })
-
-    it("patching will persist keeping the previous data", async () => {
-      await config.doInTenant(async () => {
-        const patch1 = generatePatchObject(2)
-        const patch2 = generatePatchObject(2)
-        await docWritethrough.patch(patch1)
-        await docWritethrough.patch(patch2)
-
-        await waitForQueueCompletion()
-
-        const patch3 = generatePatchObject(3)
-        await docWritethrough.patch(patch3)
-
-        await waitForQueueCompletion()
-
-        expect(await db.tryGet(documentId)).toEqual(
-          expect.objectContaining({
-            _id: documentId,
-            ...patch1,
-            ...patch2,
-            ...patch3,
-          })
-        )
-      })
-    })
-
-    it("date audit fields are set correctly when persisting", async () => {
-      await config.doInTenant(async () => {
-        const patch1 = generatePatchObject(2)
-        const patch2 = generatePatchObject(2)
-        await docWritethrough.patch(patch1)
-        const date1 = new Date()
-        await waitForQueueCompletion()
-        await docWritethrough.patch(patch2)
-
-        tk.travel(Date.now() + 100)
-        const date2 = new Date()
-        await waitForQueueCompletion()
-
-        expect(date1).not.toEqual(date2)
-        expect(await db.tryGet(documentId)).toEqual(
-          expect.objectContaining({
-            createdAt: date1.toISOString(),
-            updatedAt: date2.toISOString(),
-          })
-        )
-      })
-    })
-
-    it("concurrent patches will override keys", async () => {
-      await config.doInTenant(async () => {
-        const patch1 = generatePatchObject(2)
-        await docWritethrough.patch(patch1)
-        await waitForQueueCompletion()
-        const patch2 = generatePatchObject(1)
-        await docWritethrough.patch(patch2)
-
-        const keyToOverride = _.sample(Object.keys(patch1))!
-        expect(await db.tryGet(documentId)).toEqual(
-          expect.objectContaining({
-            [keyToOverride]: patch1[keyToOverride],
-          })
-        )
-
-        await waitForQueueCompletion()
-
-        const patch3 = {
-          ...generatePatchObject(3),
-          [keyToOverride]: generator.word(),
-        }
-        await docWritethrough.patch(patch3)
-        await waitForQueueCompletion()
-
-        expect(await db.tryGet(documentId)).toEqual(
-          expect.objectContaining({
-            ...patch1,
-            ...patch2,
-            ...patch3,
-          })
-        )
-      })
-    })
-
-    it("concurrent patches to different docWritethrough will not pollute each other", async () => {
-      await config.doInTenant(async () => {
-        const secondDocWritethrough = new DocWritethrough(
-          db,
-          structures.db.id()
-        )
-
-        const doc1Patch = generatePatchObject(2)
-        await docWritethrough.patch(doc1Patch)
-        const doc2Patch = generatePatchObject(1)
-        await secondDocWritethrough.patch(doc2Patch)
-
-        await waitForQueueCompletion()
-
-        const doc1Patch2 = generatePatchObject(3)
-        await docWritethrough.patch(doc1Patch2)
-        const doc2Patch2 = generatePatchObject(3)
-        await secondDocWritethrough.patch(doc2Patch2)
-        await waitForQueueCompletion()
-
-        expect(await db.tryGet(docWritethrough.docId)).toEqual(
-          expect.objectContaining({
-            ...doc1Patch,
-            ...doc1Patch2,
-          })
-        )
-
-        expect(await db.tryGet(secondDocWritethrough.docId)).toEqual(
-          expect.objectContaining({
-            ...doc2Patch,
-            ...doc2Patch2,
-          })
-        )
-      })
-    })
-
-    it("cached values are persisted only once", async () => {
-      await config.doInTenant(async () => {
-        const initialPatch = generatePatchObject(5)
-
-        await docWritethrough.patch(initialPatch)
-        await waitForQueueCompletion()
-
-        expect(await db.tryGet(documentId)).toEqual(
-          expect.objectContaining(initialPatch)
-        )
-
-        await db.remove(await db.get(documentId))
-
-        await waitForQueueCompletion()
-        const extraPatch = generatePatchObject(5)
-        await docWritethrough.patch(extraPatch)
-        await waitForQueueCompletion()
-
-        expect(await db.tryGet(documentId)).toEqual(
-          expect.objectContaining(extraPatch)
-        )
-        expect(await db.tryGet(documentId)).not.toEqual(
-          expect.objectContaining(initialPatch)
-        )
-      })
-    })
-
-    it("concurrent calls will not cause conflicts", async () => {
-      async function parallelPatch(count: number) {
-        const patches = Array.from({ length: count }).map(() =>
-          generatePatchObject(1)
-        )
-        await Promise.all(patches.map(p => docWritethrough.patch(p)))
-
-        return patches.reduce((acc, c) => {
-          acc = { ...acc, ...c }
-          return acc
-        }, {})
-      }
-      const queueMessageSpy = jest.spyOn(DocWritethroughProcessor.queue, "add")
-
-      await config.doInTenant(async () => {
-        let patches = await parallelPatch(5)
-        expect(queueMessageSpy).toHaveBeenCalledTimes(5)
-
-        await waitForQueueCompletion()
-        expect(await db.tryGet(documentId)).toEqual(
-          expect.objectContaining(patches)
-        )
-
-        patches = { ...patches, ...(await parallelPatch(40)) }
-        expect(queueMessageSpy).toHaveBeenCalledTimes(45)
-
-        await waitForQueueCompletion()
-        expect(await db.tryGet(documentId)).toEqual(
-          expect.objectContaining(patches)
-        )
-
-        patches = { ...patches, ...(await parallelPatch(10)) }
-        expect(queueMessageSpy).toHaveBeenCalledTimes(55)
-
-        await waitForQueueCompletion()
-        expect(await db.tryGet(documentId)).toEqual(
-          expect.objectContaining(patches)
-        )
-      })
-    })
-
-    it("patches will execute in order", async () => {
-      let incrementalValue = 0
-      const keyToOverride = generator.word()
-      async function incrementalPatches(count: number) {
-        for (let i = 0; i < count; i++) {
-          await docWritethrough.patch({ [keyToOverride]: ++incrementalValue })
-        }
-      }
-
-      await config.doInTenant(async () => {
-        await incrementalPatches(5)
-
-        await waitForQueueCompletion()
-        expect(await db.tryGet(documentId)).toEqual(
-          expect.objectContaining({ [keyToOverride]: 5 })
-        )
-
-        await incrementalPatches(40)
-        await waitForQueueCompletion()
-        expect(await db.tryGet(documentId)).toEqual(
-          expect.objectContaining({ [keyToOverride]: 45 })
-        )
-      })
-    })
-  })
-})