@budibase/worker 3.27.4 → 3.28.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "3.27.4",
+  "version": "3.28.0",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -107,5 +107,5 @@
       }
     }
   },
-  "gitHead": "d0d77e2d7618885095e00c73cefcfefd8ad54877"
+  "gitHead": "b57235144ae6ed94b1b54d9ea73335106feedc02"
 }

package/src/api/controllers/global/github.ts

@@ -0,0 +1,77 @@
+import fetch from "node-fetch"
+import { type GetGitHubStarsResponse, type UserCtx } from "@budibase/types"
+import { cache } from "@budibase/backend-core"
+
+const CACHE_TTL_MS = 6 * 60 * 60 * 1000
+const FAILURE_TTL_MS = 5 * 60 * 1000
+const GITHUB_TIMEOUT_MS = 5000
+const GITHUB_REPO_URL = "https://api.github.com/repos/budibase/budibase"
+const USER_AGENT = "Budibase"
+
+const CACHE_KEY = "global:github:stars"
+const RETENTION_TTL_SECONDS = 30 * 24 * 60 * 60
+
+interface StarsCacheEnvelope {
+  value: GetGitHubStarsResponse
+  expiresAt: number
+}
+
+export async function getStars(ctx: UserCtx<void, GetGitHubStarsResponse>) {
+  const envelope = (await cache.get(CACHE_KEY, {
+    useTenancy: false,
+  })) as StarsCacheEnvelope | null
+  if (envelope && envelope.expiresAt > Date.now()) {
+    ctx.body = envelope.value
+    return
+  }
+
+  try {
+    const response = await fetch(GITHUB_REPO_URL, {
+      headers: {
+        Accept: "application/vnd.github+json",
+        "User-Agent": USER_AGENT,
+      },
+      timeout: GITHUB_TIMEOUT_MS,
+    })
+
+    if (!response.ok) {
+      throw new Error(`GitHub response: ${response.status}`)
+    }
+
+    const json = (await response.json()) as { stargazers_count?: number }
+    const stars = json.stargazers_count
+
+    if (typeof stars !== "number") {
+      throw new Error("GitHub stars missing")
+    }
+
+    const value: GetGitHubStarsResponse = {
+      stars,
+      fetchedAt: new Date().toISOString(),
+    }
+    const toStore: StarsCacheEnvelope = {
+      value,
+      expiresAt: Date.now() + CACHE_TTL_MS,
+    }
+
+    await cache.store(CACHE_KEY, toStore, RETENTION_TTL_SECONDS, {
+      useTenancy: false,
+    })
+
+    ctx.body = value
+  } catch (err) {
+    console.error("Failed to fetch GitHub stars", err)
+
+    const value = envelope?.value || { stars: null, fetchedAt: null }
+    const toStore: StarsCacheEnvelope = {
+      value,
+      expiresAt: Date.now() + FAILURE_TTL_MS,
+    }
+
+    await cache.store(CACHE_KEY, toStore, RETENTION_TTL_SECONDS, {
+      useTenancy: false,
+    })
+
+    ctx.body = value
+  }
+}

package/src/api/controllers/global/tests/github.spec.ts

@@ -0,0 +1,193 @@
+import { type GetGitHubStarsResponse } from "@budibase/types"
+
+jest.mock("node-fetch", () => jest.fn())
+
+interface CacheOpts {
+  useTenancy?: boolean
+}
+
+interface TestCtx {
+  body: GetGitHubStarsResponse | undefined
+}
+
+describe("GitHub controller", () => {
+  let cacheGet: jest.Mock
+  let cacheStore: jest.Mock
+  let cacheState: Map<string, unknown>
+  let consoleErrorSpy: jest.SpyInstance
+
+  beforeEach(() => {
+    jest.resetModules()
+    jest.clearAllMocks()
+
+    jest.useFakeTimers()
+    jest.setSystemTime(new Date("2020-01-01T00:00:00.000Z"))
+
+    consoleErrorSpy = jest.spyOn(console, "error").mockImplementation(() => {})
+
+    cacheState = new Map()
+    jest.doMock("@budibase/backend-core", () => {
+      cacheGet = jest.fn(async (key: string, opts?: CacheOpts) => {
+        if (opts?.useTenancy !== false) {
+          throw new Error("Expected global cache usage")
+        }
+        return cacheState.get(key) ?? null
+      })
+
+      cacheStore = jest.fn(
+        async (
+          key: string,
+          value: unknown,
+          _ttl?: number,
+          opts?: CacheOpts
+        ) => {
+          if (opts?.useTenancy !== false) {
+            throw new Error("Expected global cache usage")
+          }
+          cacheState.set(key, value)
+        }
+      )
+
+      return {
+        cache: {
+          get: cacheGet,
+          store: cacheStore,
+        },
+      }
+    })
+  })
+
+  afterEach(() => {
+    consoleErrorSpy.mockRestore()
+    jest.useRealTimers()
+  })
+
+  const getFetchMock = () => {
+    return jest.requireMock("node-fetch") as unknown as jest.Mock
+  }
+
+  const makeCtx = () => {
+    const ctx: TestCtx = { body: undefined }
+    return ctx
+  }
+
+  function assertHasBody(
+    ctx: TestCtx
+  ): asserts ctx is { body: GetGitHubStarsResponse } {
+    if (!ctx.body) {
+      throw new Error("Expected ctx.body to be set")
+    }
+  }
+
+  it("adds a request timeout", async () => {
+    const fetchMock = getFetchMock()
+    fetchMock.mockResolvedValue({
+      ok: true,
+      status: 200,
+      json: async () => ({ stargazers_count: 123 }),
+    })
+
+    const { getStars } = require("../github")
+    const ctx = makeCtx()
+    await getStars(ctx)
+
+    expect(fetchMock).toHaveBeenCalledTimes(1)
+    expect(fetchMock.mock.calls[0][1]).toEqual(
+      expect.objectContaining({ timeout: 5000 })
+    )
+    assertHasBody(ctx)
+    expect(ctx.body.stars).toEqual(123)
+
+    expect(cacheGet).toHaveBeenCalledWith("global:github:stars", {
+      useTenancy: false,
+    })
+    expect(cacheStore).toHaveBeenCalledWith(
+      "global:github:stars",
+      expect.any(Object),
+      expect.any(Number),
+      { useTenancy: false }
+    )
+  })
+
+  it("caches successful responses", async () => {
+    const fetchMock = getFetchMock()
+    fetchMock.mockResolvedValue({
+      ok: true,
+      status: 200,
+      json: async () => ({ stargazers_count: 999 }),
+    })
+
+    const { getStars } = require("../github")
+    const ctx1 = makeCtx()
+    await getStars(ctx1)
+    const ctx2 = makeCtx()
+    await getStars(ctx2)
+
+    expect(fetchMock).toHaveBeenCalledTimes(1)
+    expect(ctx2.body).toEqual(ctx1.body)
+  })
+
+  it("serves cached value and sets a short failure TTL on error", async () => {
+    const fetchMock = getFetchMock()
+    fetchMock.mockRejectedValue(new Error("timeout"))
+
+    const { getStars } = require("../github")
+    const ctx1 = makeCtx()
+    await getStars(ctx1)
+    const ctx2 = makeCtx()
+    await getStars(ctx2)
+
+    expect(fetchMock).toHaveBeenCalledTimes(1)
+    expect(ctx1.body).toEqual({ stars: null, fetchedAt: null })
+    expect(ctx2.body).toEqual(ctx1.body)
+  })
+
+  it("returns last known value on refresh failure", async () => {
+    const fetchMock = getFetchMock()
+    fetchMock.mockResolvedValue({
+      ok: true,
+      status: 200,
+      json: async () => ({ stargazers_count: 111 }),
+    })
+
+    const { getStars } = require("../github")
+    const ctx1 = makeCtx()
+    await getStars(ctx1)
+
+    jest.setSystemTime(new Date(Date.now() + 6 * 60 * 60 * 1000 + 1))
+    fetchMock.mockRejectedValue(new Error("github down"))
+
+    const ctx2 = makeCtx()
+    await getStars(ctx2)
+
+    expect(fetchMock).toHaveBeenCalledTimes(2)
+    expect(ctx2.body).toEqual(ctx1.body)
+  })
+
+  it("retries after failure backoff expires", async () => {
+    const fetchMock = getFetchMock()
+    fetchMock.mockRejectedValue(new Error("timeout"))
+
+    const { getStars } = require("../github")
+    const ctx1 = makeCtx()
+    await getStars(ctx1)
+    const ctx2 = makeCtx()
+    await getStars(ctx2)
+
+    expect(fetchMock).toHaveBeenCalledTimes(1)
+
+    jest.setSystemTime(new Date(Date.now() + 5 * 60 * 1000 + 1))
+    fetchMock.mockResolvedValue({
+      ok: true,
+      status: 200,
+      json: async () => ({ stargazers_count: 222 }),
+    })
+
+    const ctx3 = makeCtx()
+    await getStars(ctx3)
+
+    expect(fetchMock).toHaveBeenCalledTimes(2)
+    assertHasBody(ctx3)
+    expect(ctx3.body.stars).toEqual(222)
+  })
+})

package/src/api/controllers/global/users.ts

@@ -268,10 +268,12 @@ export const adminUser = async (
   })
 }
 
-export const countByApp = async (ctx: UserCtx<void, CountUserResponse>) => {
-  const appId = ctx.params.appId
+export const countByWorkspace = async (
+  ctx: UserCtx<void, CountUserResponse>
+) => {
+  const workspaceId = ctx.params.workspaceId
   try {
-    ctx.body = await userSdk.db.countUsersByApp(appId)
+    ctx.body = await userSdk.db.countUsersByWorkspace(workspaceId)
   } catch (err: any) {
     ctx.throw(err.status || 400, err)
   }
@@ -589,11 +591,12 @@ export const inviteMultiple = async (
 export const removeMultipleInvites = async (
   ctx: Ctx<DeleteInviteUsersRequest, DeleteInviteUsersResponse>
 ) => {
+  const tenantId = context.getTenantId()
   const inviteCodesToRemove = ctx.request.body.map(
     (invite: DeleteInviteUserRequest) => invite.code
   )
   for (const code of inviteCodesToRemove) {
-    await cache.invite.deleteCode(code)
+    await cache.invite.deleteCode(code, tenantId)
   }
   ctx.body = {
     message: "User invites successfully removed.",
@@ -602,9 +605,13 @@ export const removeMultipleInvites = async (
 
 export const checkInvite = async (ctx: UserCtx<void, CheckInviteResponse>) => {
   const { code } = ctx.params
+  const tenantId =
+    typeof ctx.request.query.tenantId === "string"
+      ? ctx.request.query.tenantId
+      : undefined
   let invite
   try {
-    invite = await cache.invite.getCode(code)
+    invite = await cache.invite.getCode(code, tenantId)
   } catch (e) {
     console.warn("Error getting invite from code", e)
     ctx.throw(400, "There was a problem with the invite")
@@ -637,7 +644,8 @@ export const addWorkspaceIdToInvite = async (
   const prodWorkspaceId = db.getProdWorkspaceID(workspaceId)
 
   try {
-    const invite = await cache.invite.getCode(code)
+    const tenantId = context.getTenantId()
+    const invite = await cache.invite.getCode(code, tenantId)
     invite.info.apps ??= {}
     invite.info.apps[prodWorkspaceId] = role
 
@@ -660,7 +668,8 @@ export const removeWorkspaceIdFromInvite = async (
   const prodWorkspaceId = db.getProdWorkspaceID(workspaceId)
 
   try {
-    const invite = await cache.invite.getCode(code)
+    const tenantId = context.getTenantId()
+    const invite = await cache.invite.getCode(code, tenantId)
     invite.info.apps ??= {}
     delete invite.info.apps[prodWorkspaceId]
 
@@ -674,7 +683,13 @@ export const removeWorkspaceIdFromInvite = async (
 export const inviteAccept = async (
   ctx: Ctx<AcceptUserInviteRequest, AcceptUserInviteResponse>
 ) => {
-  const { inviteCode, password, firstName, lastName } = ctx.request.body
+  const { inviteCode, password, firstName, lastName, tenantId } =
+    ctx.request.body
+  const queryTenantId =
+    typeof ctx.request.query.tenantId === "string"
+      ? ctx.request.query.tenantId
+      : undefined
+  const resolvedTenantId = tenantId || queryTenantId
   try {
     await locks.doWithLock(
       {
@@ -685,7 +700,10 @@ export const inviteAccept = async (
       },
       async () => {
         // info is an extension of the user object that was stored by global
-        const { email, info } = await cache.invite.getCode(inviteCode)
+        const { email, info } = await cache.invite.getCode(
+          inviteCode,
+          resolvedTenantId
+        )
         const user = await tenancy.doInTenant(info.tenantId, async () => {
           let request: any = {
             firstName,
@@ -715,7 +733,7 @@ export const inviteAccept = async (
           return saved
         })
 
-        await cache.invite.deleteCode(inviteCode)
+        await cache.invite.deleteCode(inviteCode, resolvedTenantId)
 
         // make sure onboarding flow is cleared
         ctx.cookies.set(BpmStatusKey.ONBOARDING, BpmStatusValue.COMPLETED, {

package/src/api/routes/global/github.ts

@@ -0,0 +1,4 @@
+import * as controller from "../../controllers/global/github"
+import { builderRoutes } from "../endpointGroups"
+
+builderRoutes.get("/api/global/github/stars", controller.getStars)

package/src/api/routes/global/users.ts

@@ -51,6 +51,7 @@ function buildInviteAcceptValidation() {
     password: Joi.string().optional(),
     firstName: Joi.string().optional(),
     lastName: Joi.string().optional(),
+    tenantId: Joi.string().optional(),
   }).required().unknown(true))
 }
 
@@ -91,7 +92,7 @@ adminRoutes
 
 builderOrAdminRoutes
   .get("/api/global/users", controller.fetch)
-  .get("/api/global/users/count/:appId", controller.countByApp)
+  .get("/api/global/users/count/:workspaceId", controller.countByWorkspace)
   .get("/api/global/users/invites", controller.getUserInvites)
   .get("/api/global/users/:id", controller.find)
   .post(

package/src/api/routes/index.ts

@@ -7,6 +7,7 @@ import "./global/auth"
 import "./global/configs"
 import "./global/email"
 import "./global/events"
+import "./global/github"
 import "./global/license"
 import "./global/roles"
 import "./global/self"

package/src/tests/api/users.ts

@@ -46,6 +46,7 @@ export class UserAPI extends TestAPI {
         password: "newpassword1",
         inviteCode: code,
         firstName: "Ted",
+        tenantId: this.config.getTenantId(),
       })
       .expect("Content-Type", /json/)
       .expect(200)