@budibase/worker 2.13.36 → 2.13.38

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "2.13.36",
+  "version": "2.13.38",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -21,7 +21,7 @@
     "debug": "yarn build && node --expose-gc --inspect=9223 dist/index.js",
     "run:docker:cluster": "pm2-runtime start pm2.config.js",
     "dev:stack:init": "node ./scripts/dev/manage.js init",
-    "dev:builder": "npm run dev:stack:init && nodemon",
+    "dev": "npm run dev:stack:init && nodemon",
     "dev:built": "yarn run dev:stack:init && yarn run run:docker",
     "test": "bash scripts/test.sh",
     "test:watch": "jest --watch",
@@ -37,10 +37,10 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "2.13.36",
-    "@budibase/pro": "2.13.36",
-    "@budibase/string-templates": "2.13.36",
-    "@budibase/types": "2.13.36",
+    "@budibase/backend-core": "2.13.38",
+    "@budibase/pro": "2.13.38",
+    "@budibase/string-templates": "2.13.38",
+    "@budibase/types": "2.13.38",
     "@koa/router": "8.0.8",
     "@techpass/passport-openidconnect": "0.3.2",
     "@types/global-agent": "2.1.1",
@@ -75,26 +75,26 @@
     "@swc/jest": "0.2.27",
     "@trendyol/jest-testcontainers": "2.1.1",
     "@types/jest": "29.5.5",
-    "@types/jsonwebtoken": "8.5.1",
+    "@types/jsonwebtoken": "9.0.3",
     "@types/koa": "2.13.4",
     "@types/koa__router": "8.0.8",
     "@types/lodash": "4.14.200",
     "@types/node": "18.17.0",
     "@types/node-fetch": "2.6.4",
     "@types/server-destroy": "1.0.1",
-    "@types/supertest": "2.0.12",
+    "@types/supertest": "2.0.14",
     "@types/uuid": "8.3.4",
-    "jest": "29.6.2",
+    "jest": "29.7.0",
     "nodemon": "2.0.15",
     "rimraf": "3.0.2",
-    "supertest": "6.2.2",
+    "supertest": "6.3.3",
     "timekeeper": "2.2.0",
     "typescript": "5.2.2",
     "update-dotenv": "1.1.1"
   },
   "nx": {
     "targets": {
-      "dev:builder": {
+      "dev": {
         "dependsOn": [
           {
             "comment": "Required for pro usage when submodule not loaded",
@@ -107,5 +107,5 @@
       }
     }
   },
-  "gitHead": "933421818c287b302773026855256fce4845259a"
+  "gitHead": "02e573ced8c766dcac96b29af4eeeab1416e22f4"
 }

package/scripts/dev/manage.js

@@ -1,44 +1,40 @@
 #!/usr/bin/env node
-const path = require("path")
-const fs = require("fs")
+const { parsed: existingConfig } = require("dotenv").config()
+const updateDotEnv = require("update-dotenv")
 
 async function init() {
-  const envFilePath = path.join(process.cwd(), ".env")
-  if (!fs.existsSync(envFilePath)) {
-    const envFileJson = {
-      SELF_HOSTED: 1,
-      PORT: 4002,
-      CLUSTER_PORT: 10000,
-      JWT_SECRET: "testsecret",
-      INTERNAL_API_KEY: "budibase",
-      MINIO_ACCESS_KEY: "budibase",
-      MINIO_SECRET_KEY: "budibase",
-      REDIS_URL: "localhost:6379",
-      REDIS_PASSWORD: "budibase",
-      MINIO_URL: "http://localhost:4004",
-      COUCH_DB_URL: "http://budibase:budibase@localhost:4005",
-      COUCH_DB_USERNAME: "budibase",
-      COUCH_DB_PASSWORD: "budibase",
-      // empty string is false
-      MULTI_TENANCY: "",
-      DISABLE_ACCOUNT_PORTAL: 1,
-      ACCOUNT_PORTAL_URL: "http://localhost:10001",
-      ACCOUNT_PORTAL_API_KEY: "budibase",
-      PLATFORM_URL: "http://localhost:10000",
-      APPS_URL: "http://localhost:4001",
-      SERVICE: "worker-service",
-      DEPLOYMENT_ENVIRONMENT: "development",
-      TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR",
-      ENABLE_EMAIL_TEST_MODE: 1,
-      HTTP_LOGGING: 0,
-      VERSION: "0.0.0+local",
-    }
-    let envFile = ""
-    Object.keys(envFileJson).forEach(key => {
-      envFile += `${key}=${envFileJson[key]}\n`
-    })
-    fs.writeFileSync(envFilePath, envFile)
+  let config = {
+    SELF_HOSTED: "1",
+    PORT: "4002",
+    CLUSTER_PORT: "10000",
+    JWT_SECRET: "testsecret",
+    INTERNAL_API_KEY: "budibase",
+    MINIO_ACCESS_KEY: "budibase",
+    MINIO_SECRET_KEY: "budibase",
+    REDIS_URL: "localhost:6379",
+    REDIS_PASSWORD: "budibase",
+    MINIO_URL: "http://localhost:4004",
+    COUCH_DB_URL: "http://budibase:budibase@localhost:4005",
+    COUCH_DB_USERNAME: "budibase",
+    COUCH_DB_PASSWORD: "budibase",
+    // empty string is false
+    MULTI_TENANCY: "",
+    DISABLE_ACCOUNT_PORTAL: "1",
+    ACCOUNT_PORTAL_URL: "http://localhost:10001",
+    ACCOUNT_PORTAL_API_KEY: "budibase",
+    PLATFORM_URL: "http://localhost:10000",
+    APPS_URL: "http://localhost:4001",
+    SERVICE: "worker-service",
+    DEPLOYMENT_ENVIRONMENT: "development",
+    TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR",
+    ENABLE_EMAIL_TEST_MODE: "1",
+    HTTP_LOGGING: "0",
+    VERSION: "0.0.0+local",
   }
+
+  config = { ...config, ...existingConfig }
+
+  await updateDotEnv(config)
 }
 
 // if more than init required use this to determine the command type

package/src/api/controllers/global/users.ts

@@ -26,6 +26,7 @@ import {
   migrations,
   platform,
   tenancy,
+  db,
 } from "@budibase/backend-core"
 import { checkAnyUserExists } from "../../../utilities/users"
 import { isEmailConfigured } from "../../../utilities/email"
@@ -185,9 +186,27 @@ export const getAppUsers = async (ctx: Ctx<SearchUsersRequest>) => {
 export const search = async (ctx: Ctx<SearchUsersRequest>) => {
   const body = ctx.request.body
 
-  // TODO: for now only one supported search key, string.email
-  if (body?.query && !userSdk.core.isSupportedUserSearch(body.query)) {
-    ctx.throw(501, "Can only search by string.email or equal._id")
+  // TODO: for now only two supported search keys; string.email and equal._id
+  if (body?.query) {
+    // Clean numeric prefixing. This will overwrite duplicate search fields,
+    // but this is fine because we only support a single custom search on
+    // email and id
+    for (let filters of Object.values(body.query)) {
+      if (filters && typeof filters === "object") {
+        for (let [field, value] of Object.entries(filters)) {
+          delete filters[field]
+          const cleanedField = db.removeKeyNumbering(field)
+          if (filters[cleanedField] !== undefined) {
+            ctx.throw(400, "Only 1 filter per field is supported")
+          }
+          filters[cleanedField] = value
+        }
+      }
+    }
+    // Validate we aren't trying to search on any illegal fields
+    if (!userSdk.core.isSupportedUserSearch(body.query)) {
+      ctx.throw(400, "Can only search by string.email or equal._id")
+    }
   }
 
   if (body.paginate === false) {

package/src/api/controllers/system/environment.ts

@@ -1,6 +1,25 @@
 import { Ctx } from "@budibase/types"
 import env from "../../../environment"
 import { env as coreEnv } from "@budibase/backend-core"
+import nodeFetch from "node-fetch"
+
+let sqsAvailable: boolean
+async function isSqsAvailable() {
+  if (sqsAvailable !== undefined) {
+    return sqsAvailable
+  }
+
+  try {
+    await nodeFetch(coreEnv.COUCH_DB_SQL_URL, {
+      timeout: 1000,
+    })
+    sqsAvailable = true
+    return true
+  } catch (e) {
+    sqsAvailable = false
+    return false
+  }
+}
 
 export const fetch = async (ctx: Ctx) => {
   ctx.body = {
@@ -12,4 +31,10 @@ export const fetch = async (ctx: Ctx) => {
     baseUrl: env.PLATFORM_URL,
     isDev: env.isDev() && !env.isTest(),
   }
+
+  if (env.SELF_HOSTED) {
+    ctx.body.infrastructure = {
+      sqs: await isSqsAvailable(),
+    }
+  }
 }

package/src/api/routes/global/tests/users.spec.ts

@@ -590,6 +590,15 @@ describe("/api/global/users", () => {
       expect(response.body.data[0].email).toBe(user.email)
     })
 
+    it("should be able to search by email with numeric prefixing", async () => {
+      const user = await config.createUser()
+      const response = await config.api.users.searchUsers({
+        query: { string: { ["999:email"]: user.email } },
+      })
+      expect(response.body.data.length).toBe(1)
+      expect(response.body.data[0].email).toBe(user.email)
+    })
+
     it("should be able to search by _id", async () => {
       const user = await config.createUser()
       const response = await config.api.users.searchUsers({
@@ -599,13 +608,52 @@ describe("/api/global/users", () => {
       expect(response.body.data[0]._id).toBe(user._id)
     })
 
+    it("should be able to search by _id with numeric prefixing", async () => {
+      const user = await config.createUser()
+      const response = await config.api.users.searchUsers({
+        query: { equal: { ["1:_id"]: user._id } },
+      })
+      expect(response.body.data.length).toBe(1)
+      expect(response.body.data[0]._id).toBe(user._id)
+    })
+
+    it("should throw an error when using multiple filters on the same field", async () => {
+      const user = await config.createUser()
+      await config.api.users.searchUsers(
+        {
+          query: {
+            string: {
+              ["1:email"]: user.email,
+              ["2:email"]: "something else",
+            },
+          },
+        },
+        { status: 400 }
+      )
+    })
+
+    it("should throw an error when using multiple filters on the same field without prefixes", async () => {
+      const user = await config.createUser()
+      await config.api.users.searchUsers(
+        {
+          query: {
+            string: {
+              ["_id"]: user.email,
+              ["999:_id"]: "something else",
+            },
+          },
+        },
+        { status: 400 }
+      )
+    })
+
     it("should throw an error when unimplemented options used", async () => {
       const user = await config.createUser()
       await config.api.users.searchUsers(
         {
           query: { equal: { firstName: user.firstName } },
         },
-        { status: 501 }
+        { status: 400 }
       )
     })
 

package/src/api/routes/system/tests/environment.spec.ts

@@ -1,5 +1,7 @@
 import { TestConfiguration } from "../../../../tests"
 
+jest.unmock("node-fetch")
+
 describe("/api/system/environment", () => {
   const config = new TestConfiguration()
 
@@ -27,5 +29,22 @@ describe("/api/system/environment", () => {
         offlineMode: false,
       })
     })
+
+    it("returns the expected environment for self hosters", async () => {
+      await config.withEnv({ SELF_HOSTED: true }, async () => {
+        const env = await config.api.environment.getEnvironment()
+        expect(env.body).toEqual({
+          cloud: false,
+          disableAccountPortal: 0,
+          isDev: false,
+          multiTenancy: true,
+          baseUrl: "http://localhost:10000",
+          offlineMode: false,
+          infrastructure: {
+            sqs: false,
+          },
+        })
+      })
+    })
   })
 })

package/src/tests/TestConfiguration.ts

@@ -36,6 +36,7 @@ import {
 } from "@budibase/types"
 import API from "./api"
 import jwt, { Secret } from "jsonwebtoken"
+import cloneDeep from "lodash/fp/cloneDeep"
 
 class TestConfiguration {
   server: any
@@ -240,6 +241,34 @@ class TestConfiguration {
     return { message: "Admin user only endpoint.", status: 403 }
   }
 
+  async withEnv(newEnvVars: Partial<typeof env>, f: () => Promise<void>) {
+    let cleanup = this.setEnv(newEnvVars)
+    try {
+      await f()
+    } finally {
+      cleanup()
+    }
+  }
+
+  /*
+   * Sets the environment variables to the given values and returns a function
+   * that can be called to reset the environment variables to their original values.
+   */
+  setEnv(newEnvVars: Partial<typeof env>): () => void {
+    const oldEnv = cloneDeep(env)
+
+    let key: keyof typeof newEnvVars
+    for (key in newEnvVars) {
+      env._set(key, newEnvVars[key])
+    }
+
+    return () => {
+      for (const [key, value] of Object.entries(oldEnv)) {
+        env._set(key, value)
+      }
+    }
+  }
+
   // USERS
 
   async createDefaultUser() {