@budibase/worker 2.6.23 → 2.6.24-alpha.0

package/.dockerignore

@@ -1,7 +1,4 @@
-node_modules
-npm-debug.log
-Dockerfile
-.dockerignore
-.git
-.gitignore
-
+*
+!/dist/
+!/docker_run.sh
+!/package.json

package/Dockerfile

@@ -7,13 +7,20 @@ LABEL com.centurylinklabs.watchtower.lifecycle.post-check="scripts/watchtower-ho
 
 WORKDIR /app
 
-# copy files and install dependencies
-COPY . ./
 # handle node-gyp
-RUN apk add --no-cache --virtual .gyp python3 make g++ \
-    && yarn && apk del .gyp
+RUN apk add --no-cache --virtual .gyp python3 make g++
 RUN yarn global add pm2
 
+
+COPY package.json .
+RUN yarn install --frozen-lockfile --production=true
+# Remove unneeded data from file system to reduce image size
+RUN apk del .gyp \
+    && yarn cache clean
+
+COPY dist/ dist/
+COPY docker_run.sh .
+
 EXPOSE 4001
 
 # have to add node environment production after install

package/jest.config.ts

@@ -16,15 +16,16 @@ const config: Config.InitialOptions = {
     "@budibase/backend-core/(.*)": "<rootDir>/../backend-core/$1",
     "@budibase/backend-core": "<rootDir>/../backend-core/src",
     "@budibase/types": "<rootDir>/../types/src",
+    "@budibase/shared-core": ["<rootDir>/../shared-core/src"],
   },
 }
 
 // add pro sources if they exist
-if (fs.existsSync("../../../budibase-pro")) {
-  config.moduleNameMapper["@budibase/pro/(.*)"] =
-    "<rootDir>/../../../budibase-pro/packages/pro/$1"
-  config.moduleNameMapper["@budibase/pro"] =
-    "<rootDir>/../../../budibase-pro/packages/pro/src"
+if (fs.existsSync("../pro/packages")) {
+  config.moduleNameMapper!["@budibase/pro/(.*)"] =
+    "<rootDir>/../pro/packages/pro/$1"
+  config.moduleNameMapper!["@budibase/pro"] =
+    "<rootDir>/../pro/packages/pro/src"
 }
 
 export default config

package/nodemon.json

@@ -1,6 +1,10 @@
 {
-  "watch": ["src", "../backend-core", "../../../budibase-pro/packages/pro"],
+  "watch": ["src", "../backend-core", "../pro/packages/pro"],
   "ext": "js,ts,json",
-  "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js", "../backend-core/dist/**/*"],
-  "exec": "ts-node src/index.ts"
+  "ignore": [
+    "src/**/*.spec.ts",
+    "src/**/*.spec.js",
+    "../backend-core/dist/**/*"
+  ],
+  "exec": "yarn build && node dist/index.js"
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "2.6.23",
+  "version": "2.6.24-alpha.0",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -13,15 +13,16 @@
   ],
   "scripts": {
     "prebuild": "rimraf dist/",
-    "build": "tsc -p tsconfig.build.json",
-    "postbuild": "copyfiles -u 1 src/**/*.hbs dist/",
+    "build": "node ../../scripts/build.js",
+    "check:types": "tsc -p tsconfig.build.json --noEmit",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "run:docker": "node dist/index.js",
     "debug": "yarn build && node --expose-gc --inspect=9223 dist/index.js",
     "run:docker:cluster": "pm2-runtime start pm2.config.js",
-    "build:docker": "docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION",
+    "predocker": "yarn build --configuration=production",
+    "build:docker": "yarn predocker && docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION",
     "dev:stack:init": "node ./scripts/dev/manage.js init",
-    "dev:builder": "npm run dev:stack:init && nodemon",
+    "dev:builder": "npm run dev:stack:init && rimraf dist/ && nodemon",
     "dev:built": "yarn run dev:stack:init && yarn run run:docker",
     "test": "bash scripts/test.sh",
     "test:watch": "jest --watch",
@@ -37,10 +38,10 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "^2.6.23",
-    "@budibase/pro": "2.6.22",
-    "@budibase/string-templates": "^2.6.23",
-    "@budibase/types": "^2.6.23",
+    "@budibase/backend-core": "2.6.24-alpha.0",
+    "@budibase/pro": "2.6.24-alpha.0",
+    "@budibase/string-templates": "2.6.24-alpha.0",
+    "@budibase/types": "2.6.24-alpha.0",
     "@koa/router": "8.0.8",
     "@sentry/node": "6.17.7",
     "@techpass/passport-openidconnect": "0.3.2",
@@ -102,5 +103,5 @@
     "typescript": "4.7.3",
     "update-dotenv": "1.1.1"
   },
-  "gitHead": "565bea58fabc8b214d646ea72b24e4c11662af98"
+  "gitHead": "71a9558d12c55e7d9ef87b82995c62497662acf8"
 }

package/src/api/controllers/global/configs.ts

@@ -11,7 +11,6 @@ import {
   tenancy,
 } from "@budibase/backend-core"
 import { checkAnyUserExists } from "../../../utilities/users"
-import { getLicensedConfig } from "../../../utilities/configs"
 import {
   Config,
   ConfigType,
@@ -24,6 +23,7 @@ import {
   isSettingsConfig,
   isSMTPConfig,
   OIDCConfigs,
+  SettingsBrandingConfig,
   SettingsInnerConfig,
   SSOConfig,
   SSOConfigType,
@@ -143,13 +143,29 @@ async function hasActivatedConfig(ssoConfigs?: SSOConfigs) {
   return !!Object.values(ssoConfigs).find(c => c?.activated)
 }
 
-async function verifySettingsConfig(config: SettingsInnerConfig) {
+async function verifySettingsConfig(
+  config: SettingsInnerConfig & SettingsBrandingConfig,
+  existingConfig?: SettingsInnerConfig & SettingsBrandingConfig
+) {
   if (config.isSSOEnforced) {
     const valid = await hasActivatedConfig()
     if (!valid) {
       throw new Error("Cannot enforce SSO without an activated configuration")
     }
   }
+
+  // always preserve file attributes
+  // these should be set via upload instead
+  // only allow for deletion by checking empty string to bypass this behaviour
+
+  if (existingConfig && config.logoUrl !== "") {
+    config.logoUrl = existingConfig.logoUrl
+    config.logoUrlEtag = existingConfig.logoUrlEtag
+  }
+  if (existingConfig && config.faviconUrl !== "") {
+    config.faviconUrl = existingConfig.faviconUrl
+    config.faviconUrlEtag = existingConfig.faviconUrlEtag
+  }
 }
 
 async function verifySSOConfig(type: SSOConfigType, config: SSOConfig) {
@@ -199,7 +215,7 @@ export async function save(ctx: UserCtx<Config>) {
         await email.verifyConfig(config)
         break
       case ConfigType.SETTINGS:
-        await verifySettingsConfig(config)
+        await verifySettingsConfig(config, existingConfig?.config)
         break
       case ConfigType.GOOGLE:
         await verifyGoogleConfig(config)
@@ -321,14 +337,15 @@ export async function publicSettings(
       )
     }
 
-    if (branding.faviconUrl && branding.faviconUrl !== "") {
-      // @ts-ignore
-      config.faviconUrl = objectStore.getGlobalFileUrl(
-        "settings",
-        "faviconUrl",
-        branding.faviconUrl
-      )
-    }
+    // enrich the favicon url - empty url means deleted
+    const faviconUrl =
+      branding.faviconUrl && branding.faviconUrl !== ""
+        ? objectStore.getGlobalFileUrl(
+            "settings",
+            "faviconUrl",
+            branding.faviconUrlEtag
+          )
+        : undefined
 
     // google
     const googleConfig = await configs.getGoogleConfig()
@@ -353,6 +370,7 @@ export async function publicSettings(
       config: {
         ...config,
         ...branding,
+        ...{ faviconUrl },
         google,
         googleDatasourceConfigured,
         oidc,

package/src/api/controllers/global/users.ts

@@ -69,9 +69,11 @@ const bulkCreate = async (users: User[], groupIds: string[]) => {
   return await userSdk.bulkCreate(users, groupIds)
 }
 
-export const bulkUpdate = async (ctx: any) => {
+export const bulkUpdate = async (
+  ctx: Ctx<BulkUserRequest, BulkUserResponse>
+) => {
   const currentUserId = ctx.user._id
-  const input = ctx.request.body as BulkUserRequest
+  const input = ctx.request.body
   let created, deleted
   try {
     if (input.create) {
@@ -83,7 +85,7 @@ export const bulkUpdate = async (ctx: any) => {
   } catch (err: any) {
     ctx.throw(err.status || 400, err?.message || err)
   }
-  ctx.body = { created, deleted } as BulkUserResponse
+  ctx.body = { created, deleted }
 }
 
 const parseBooleanParam = (param: any) => {
@@ -184,15 +186,15 @@ export const destroy = async (ctx: any) => {
   }
 }
 
-export const getAppUsers = async (ctx: any) => {
-  const body = ctx.request.body as SearchUsersRequest
+export const getAppUsers = async (ctx: Ctx<SearchUsersRequest>) => {
+  const body = ctx.request.body
   const users = await userSdk.getUsersByAppAccess(body?.appId)
 
   ctx.body = { data: users }
 }
 
-export const search = async (ctx: any) => {
-  const body = ctx.request.body as SearchUsersRequest
+export const search = async (ctx: Ctx<SearchUsersRequest>) => {
+  const body = ctx.request.body
 
   if (body.paginated === false) {
     await getAppUsers(ctx)
@@ -238,8 +240,8 @@ export const tenantUserLookup = async (ctx: any) => {
 /* 
   Encapsulate the app user onboarding flows here.
 */
-export const onboardUsers = async (ctx: any) => {
-  const request = ctx.request.body as InviteUsersRequest | BulkUserRequest
+export const onboardUsers = async (ctx: Ctx<InviteUsersRequest>) => {
+  const request = ctx.request.body
   const isBulkCreate = "create" in request
 
   const emailConfigured = await isEmailConfigured()
@@ -255,7 +257,7 @@ export const onboardUsers = async (ctx: any) => {
   } else if (emailConfigured) {
     onboardingResponse = await inviteMultiple(ctx)
   } else if (!emailConfigured) {
-    const inviteRequest = ctx.request.body as InviteUsersRequest
+    const inviteRequest = ctx.request.body
 
     let createdPasswords: any = {}
 
@@ -295,10 +297,10 @@ export const onboardUsers = async (ctx: any) => {
   }
 }
 
-export const invite = async (ctx: any) => {
-  const request = ctx.request.body as InviteUserRequest
+export const invite = async (ctx: Ctx<InviteUserRequest>) => {
+  const request = ctx.request.body
 
-  let multiRequest = [request] as InviteUsersRequest
+  let multiRequest = [request]
   const response = await userSdk.invite(multiRequest)
 
   // explicitly throw for single user invite
@@ -318,8 +320,8 @@ export const invite = async (ctx: any) => {
   }
 }
 
-export const inviteMultiple = async (ctx: any) => {
-  const request = ctx.request.body as InviteUsersRequest
+export const inviteMultiple = async (ctx: Ctx<InviteUsersRequest>) => {
+  const request = ctx.request.body
   ctx.body = await userSdk.invite(request)
 }
 
@@ -424,7 +426,6 @@ export const inviteAccept = async (
     if (err.code === ErrorCode.USAGE_LIMIT_EXCEEDED) {
       // explicitly re-throw limit exceeded errors
       ctx.throw(400, err)
-      return
     }
     console.warn("Error inviting user", err)
     ctx.throw(400, "Unable to create new user, invitation invalid.")

package/src/api/routes/global/tests/groups.spec.ts

@@ -1,5 +1,9 @@
 import { events } from "@budibase/backend-core"
+import { generator } from "@budibase/backend-core/tests"
 import { structures, TestConfiguration, mocks } from "../../../../tests"
+import { UserGroup } from "@budibase/types"
+
+mocks.licenses.useGroups()
 
 describe("/api/global/groups", () => {
   const config = new TestConfiguration()
@@ -13,6 +17,7 @@ describe("/api/global/groups", () => {
   })
 
   beforeEach(async () => {
+    jest.resetAllMocks()
     mocks.licenses.useGroups()
   })
 
@@ -24,6 +29,63 @@ describe("/api/global/groups", () => {
       expect(events.group.updated).not.toBeCalled()
       expect(events.group.permissionsEdited).not.toBeCalled()
     })
+
+    it("should not allow undefined names", async () => {
+      const group = { ...structures.groups.UserGroup(), name: undefined } as any
+      const response = await config.api.groups.saveGroup(group, { expect: 400 })
+      expect(JSON.parse(response.text).message).toEqual(
+        'Invalid body - "name" is required'
+      )
+    })
+
+    it("should not allow empty names", async () => {
+      const group = { ...structures.groups.UserGroup(), name: "" }
+      const response = await config.api.groups.saveGroup(group, { expect: 400 })
+      expect(JSON.parse(response.text).message).toEqual(
+        'Invalid body - "name" is not allowed to be empty'
+      )
+    })
+
+    it("should not allow whitespace names", async () => {
+      const group = { ...structures.groups.UserGroup(), name: "   " }
+      const response = await config.api.groups.saveGroup(group, { expect: 400 })
+      expect(JSON.parse(response.text).message).toEqual(
+        'Invalid body - "name" is not allowed to be empty'
+      )
+    })
+
+    it("should trim names", async () => {
+      const group = { ...structures.groups.UserGroup(), name: "   group name " }
+      await config.api.groups.saveGroup(group)
+      expect(events.group.created).toBeCalledWith(
+        expect.objectContaining({ name: "group name" })
+      )
+    })
+
+    describe("name max length", () => {
+      const maxLength = 50
+
+      it(`should allow names shorter than ${maxLength} characters`, async () => {
+        const group = {
+          ...structures.groups.UserGroup(),
+          name: structures.generator.word({ length: maxLength }),
+        }
+        await config.api.groups.saveGroup(group, { expect: 200 })
+      })
+
+      it(`should not allow names longer than ${maxLength} characters`, async () => {
+        const group = {
+          ...structures.groups.UserGroup(),
+          name: structures.generator.word({ length: maxLength + 1 }),
+        }
+        const response = await config.api.groups.saveGroup(group, {
+          expect: 400,
+        })
+        expect(JSON.parse(response.text).message).toEqual(
+          'Invalid body - "name" length must be less than or equal to 50 characters long'
+        )
+      })
+    })
   })
 
   describe("update", () => {
@@ -55,4 +117,118 @@ describe("/api/global/groups", () => {
       })
     })
   })
+
+  describe("find users", () => {
+    describe("without users", () => {
+      let group: UserGroup
+      beforeAll(async () => {
+        group = structures.groups.UserGroup()
+        await config.api.groups.saveGroup(group)
+      })
+
+      it("should return empty", async () => {
+        const result = await config.api.groups.searchUsers(group._id!)
+        expect(result.body).toEqual({
+          users: [],
+          bookmark: undefined,
+          hasNextPage: false,
+        })
+      })
+    })
+
+    describe("existing users", () => {
+      let groupId: string
+      let users: { _id: string; email: string }[] = []
+
+      beforeAll(async () => {
+        groupId = (
+          await config.api.groups.saveGroup(structures.groups.UserGroup())
+        ).body._id
+
+        await Promise.all(
+          Array.from({ length: 30 }).map(async (_, i) => {
+            const email = `user${i}@${generator.domain()}`
+            const user = await config.api.users.saveUser({
+              ...structures.users.user(),
+              email,
+            })
+            users.push({ _id: user.body._id, email })
+          })
+        )
+        users = users.sort((a, b) => a._id.localeCompare(b._id))
+        await config.api.groups.updateGroupUsers(groupId, {
+          add: users.map(u => u._id),
+          remove: [],
+        })
+      })
+
+      describe("pagination", () => {
+        it("should return first page", async () => {
+          const result = await config.api.groups.searchUsers(groupId)
+          expect(result.body).toEqual({
+            users: users.slice(0, 10),
+            bookmark: users[10]._id,
+            hasNextPage: true,
+          })
+        })
+
+        it("given a bookmark, should return skip items", async () => {
+          const result = await config.api.groups.searchUsers(groupId, {
+            bookmark: users[7]._id,
+          })
+          expect(result.body).toEqual({
+            users: users.slice(7, 17),
+            bookmark: users[17]._id,
+            hasNextPage: true,
+          })
+        })
+
+        it("bookmarking the last page, should return last page info", async () => {
+          const result = await config.api.groups.searchUsers(groupId, {
+            bookmark: users[20]._id,
+          })
+          expect(result.body).toEqual({
+            users: users.slice(20),
+            bookmark: undefined,
+            hasNextPage: false,
+          })
+        })
+      })
+
+      describe("search by email", () => {
+        it('should be able to search "starting" by email', async () => {
+          const result = await config.api.groups.searchUsers(groupId, {
+            emailSearch: `user1`,
+          })
+
+          const matchedUsers = users
+            .filter(u => u.email.startsWith("user1"))
+            .sort((a, b) => a.email.localeCompare(b.email))
+
+          expect(result.body).toEqual({
+            users: matchedUsers.slice(0, 10),
+            bookmark: matchedUsers[10].email,
+            hasNextPage: true,
+          })
+        })
+
+        it("should be able to bookmark when searching by email", async () => {
+          const matchedUsers = users
+            .filter(u => u.email.startsWith("user1"))
+            .sort((a, b) => a.email.localeCompare(b.email))
+
+          const result = await config.api.groups.searchUsers(groupId, {
+            emailSearch: `user1`,
+            bookmark: matchedUsers[4].email,
+          })
+
+          expect(result.body).toEqual({
+            users: matchedUsers.slice(4),
+            bookmark: undefined,
+            hasNextPage: false,
+          })
+        })
+      })
+    })
+  })
 })

package/src/tests/api/groups.ts

@@ -7,13 +7,13 @@ export class GroupsAPI extends TestAPI {
     super(config)
   }
 
-  saveGroup = (group: UserGroup) => {
+  saveGroup = (group: UserGroup, { expect } = { expect: 200 }) => {
     return this.request
       .post(`/api/global/groups`)
       .send(group)
       .set(this.config.defaultHeaders())
       .expect("Content-Type", /json/)
-      .expect(200)
+      .expect(expect)
   }
 
   deleteGroup = (id: string, rev: string) => {
@@ -23,4 +23,34 @@ export class GroupsAPI extends TestAPI {
       .expect("Content-Type", /json/)
       .expect(200)
   }
+
+  searchUsers = (
+    id: string,
+    params?: { bookmark?: string; emailSearch?: string }
+  ) => {
+    let url = `/api/global/groups/${id}/users?`
+    if (params?.bookmark) {
+      url += `bookmark=${params.bookmark}&`
+    }
+    if (params?.emailSearch) {
+      url += `emailSearch=${params.emailSearch}&`
+    }
+    return this.request
+      .get(url)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(200)
+  }
+
+  updateGroupUsers = (
+    id: string,
+    body: { add: string[]; remove: string[] }
+  ) => {
+    return this.request
+      .post(`/api/global/groups/${id}/users`)
+      .send(body)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(200)
+  }
 }

package/src/tests/structures/groups.ts

@@ -1,10 +1,25 @@
+import { generator } from "@budibase/backend-core/tests"
+import { db } from "@budibase/backend-core"
+import { UserGroupRoles } from "@budibase/types"
+
 export const UserGroup = () => {
+  const appsCount = generator.integer({ min: 0, max: 3 })
+  const roles = Array.from({ length: appsCount }).reduce(
+    (p: UserGroupRoles, v) => {
+      return {
+        ...p,
+        [db.generateAppID()]: generator.pickone(["ADMIN", "POWER", "BASIC"]),
+      }
+    },
+    {}
+  )
+
   let group = {
     apps: [],
-    color: "var(--spectrum-global-color-blue-600)",
-    icon: "UserGroup",
-    name: "New group",
-    roles: { app_uuid1: "ADMIN", app_uuid2: "POWER" },
+    color: generator.color(),
+    icon: generator.word(),
+    name: generator.word(),
+    roles: roles,
     users: [],
   }
   return group

package/src/utilities/redis.ts

@@ -7,7 +7,7 @@ function getExpirySecondsForDB(db: string) {
       // a hour
       return 3600
     case redis.utils.Databases.INVITATIONS:
-      // a day
+      // a week
       return 604800
   }
 }

package/tsconfig.build.json

@@ -8,13 +8,18 @@
     "esModuleInterop": true,
     "resolveJsonModule": true,
     "incremental": true,
-    "types": [ "node", "jest" ],
+    "types": ["node", "jest"],
     "outDir": "dist",
-    "skipLibCheck": true
+    "skipLibCheck": true,
+    "paths": {
+      "@budibase/types": ["../types/src"],
+      "@budibase/backend-core": ["../backend-core/src"],
+      "@budibase/backend-core/*": ["../backend-core/*"],
+      "@budibase/shared-core": ["../shared-core/src"],
+      "@budibase/pro": ["../pro/packages/pro/src"]
+    }
   },
-  "include": [
-    "src/**/*"
-  ],
+  "include": ["src/**/*"],
   "exclude": [
     "node_modules",
     "dist",
@@ -22,4 +27,4 @@
     "**/*.spec.js",
     "**/*.spec.ts"
   ]
-}
+}

package/tsconfig.json

@@ -4,23 +4,12 @@
     "composite": true,
     "declaration": true,
     "sourceMap": true,
-    "baseUrl": ".",
-    "paths": {
-      "@budibase/types": ["../types/src"],
-      "@budibase/backend-core": ["../backend-core/src"],
-      "@budibase/backend-core/*": ["../backend-core/*"],
-      "@budibase/pro": ["../../../budibase-pro/packages/pro/src"]
-    }
+    "baseUrl": "."
   },
   "ts-node": {
     "require": ["tsconfig-paths/register"],
     "swc": true
   },
-  "references": [
-    { "path": "../types" },
-    { "path": "../backend-core" },
-    { "path": "../../../budibase-pro/packages/pro" }
-  ],
   "include": ["src/**/*"],
   "exclude": ["dist"]
 }