@budibase/backend-core 2.10.10 → 2.10.12-alpha.0

package/dist/tests/core/utilities/structures/users.js.map

@@ -1 +1 @@
-{"version":3,"file":"users.js","sourceRoot":"","sources":["../../../../../tests/core/utilities/structures/users.ts"],"names":[],"mappings":";;;AAOA,qCAA+B;AAC/B,+BAAmC;AAEnC,mCAAyC;AAAhC,8FAAA,IAAI,OAAA;AAAE,kGAAA,QAAQ,OAAA;AAEhB,MAAM,SAAS,GAAG,CAAC,SAAe,EAAa,EAAE;IACtD,uCACK,IAAA,aAAI,EAAC,SAAS,CAAC,KAClB,KAAK,EAAE;YACL,MAAM,EAAE,IAAI;SACb,EACD,OAAO,EAAE;YACP,MAAM,EAAE,IAAI;SACb,IACF;AACH,CAAC,CAAA;AAVY,QAAA,SAAS,aAUrB;AAEM,MAAM,aAAa,GAAG,CAAC,SAAe,EAAiB,EAAE;IAC9D,uCACK,IAAA,aAAI,EAAC,SAAS,CAAC,KAClB,KAAK,EAAE;YACL,MAAM,EAAE,IAAI;SACb,IACF;AACH,CAAC,CAAA;AAPY,QAAA,aAAa,iBAOzB;AAEM,MAAM,WAAW,GAAG,CAAC,SAAe,EAAe,EAAE;IAC1D,uCACK,IAAA,aAAI,EAAC,SAAS,CAAC,KAClB,OAAO,EAAE;YACP,MAAM,EAAE,IAAI;SACb,IACF;AACH,CAAC,CAAA;AAPY,QAAA,WAAW,eAOvB;AAEM,MAAM,cAAc,GAAG,CAAC,KAAa,EAAE,SAAe,EAAe,EAAE;IAC5E,uCACK,IAAA,aAAI,EAAC,SAAS,CAAC,KAClB,OAAO,EAAE;YACP,IAAI,EAAE,CAAC,KAAK,CAAC;SACd,IACF;AACH,CAAC,CAAA;AAPY,QAAA,cAAc,kBAO1B;AAED,SAAgB,OAAO,CACrB,OAAiD,EAAE;;IAEnD,MAAM,IAAI,GAAG,IAAA,aAAI,EAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC5B,OAAO,IAAI,CAAC,QAAQ,CAAA;IAEpB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;QACjB,IAAI,CAAC,OAAO,GAAG,IAAA,iBAAW,EAAC,IAAI,CAAC,CAAA;KACjC;IAED,uCACK,IAAI,KACP,kBAAkB,EAAE,KAAK,EACzB,MAAM,EAAE,MAAA,IAAI,CAAC,OAAO,0CAAE,MAAM,EAC5B,QAAQ,EAAE,MAAA,IAAI,CAAC,OAAO,0CAAE,QAAS,EACjC,YAAY,EAAE,MAAA,IAAI,CAAC,OAAO,0CAAE,YAAa,EACzC,iBAAiB,EAAE;YACjB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,OAAO,EAAE,IAAI,CAAC,UAAU;SACzB,IACF;AACH,CAAC;AArBD,0BAqBC"}
+{"version":3,"file":"users.js","sourceRoot":"","sources":["../../../../../tests/core/utilities/structures/users.ts"],"names":[],"mappings":";;;AAQA,+BAAmC;AACnC,qCAA+B;AAC/B,2CAAuC;AACvC,wBAA0B;AAC1B,mDAA6D;AAEtD,MAAM,QAAQ,GAAG,GAAG,EAAE;IAC3B,OAAO,GAAG,IAAA,aAAI,GAAE,WAAW,CAAA;AAC7B,CAAC,CAAA;AAFY,QAAA,QAAQ,YAEpB;AAEM,MAAM,IAAI,GAAG,CAAC,SAAyC,EAAQ,EAAE;IACtE,MAAM,MAAM,GAAG,CAAA,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,GAAG,KAAI,IAAA,6BAAoB,GAAE,CAAA;IACvD,uBACE,GAAG,EAAE,MAAM,EACX,MAAM,EACN,KAAK,EAAE,IAAA,gBAAQ,GAAE,EACjB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,EAC5B,SAAS,EAAE,qBAAS,CAAC,KAAK,EAAE,EAC5B,QAAQ,EAAE,qBAAS,CAAC,IAAI,EAAE,EAC1B,UAAU,EAAE,iBAAiB,EAC7B,QAAQ,EAAE,SAAM,CAAC,EAAE,EAAE,IAClB,SAAS,EACb;AACH,CAAC,CAAA;AAdY,QAAA,IAAI,QAchB;AAEM,MAAM,SAAS,GAAG,CAAC,SAAe,EAAa,EAAE;IACtD,uCACK,IAAA,YAAI,EAAC,SAAS,CAAC,KAClB,KAAK,EAAE;YACL,MAAM,EAAE,IAAI;SACb,EACD,OAAO,EAAE;YACP,MAAM,EAAE,IAAI;SACb,IACF;AACH,CAAC,CAAA;AAVY,QAAA,SAAS,aAUrB;AAEM,MAAM,aAAa,GAAG,CAAC,SAAe,EAAiB,EAAE;IAC9D,uCACK,IAAA,YAAI,EAAC,SAAS,CAAC,KAClB,KAAK,EAAE;YACL,MAAM,EAAE,IAAI;SACb,IACF;AACH,CAAC,CAAA;AAPY,QAAA,aAAa,iBAOzB;AAEM,MAAM,WAAW,GAAG,CAAC,SAAe,EAAe,EAAE;IAC1D,uCACK,IAAA,YAAI,EAAC,SAAS,CAAC,KAClB,OAAO,EAAE;YACP,MAAM,EAAE,IAAI;SACb,IACF;AACH,CAAC,CAAA;AAPY,QAAA,WAAW,eAOvB;AAEM,MAAM,cAAc,GAAG,CAAC,KAAa,EAAE,SAAe,EAAe,EAAE;IAC5E,uCACK,IAAA,YAAI,EAAC,SAAS,CAAC,KAClB,OAAO,EAAE;YACP,IAAI,EAAE,CAAC,KAAK,CAAC;SACd,IACF;AACH,CAAC,CAAA;AAPY,QAAA,cAAc,kBAO1B;AAED,SAAgB,OAAO,CACrB,OAAiD,EAAE;;IAEnD,MAAM,IAAI,GAAG,IAAA,YAAI,EAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC5B,OAAO,IAAI,CAAC,QAAQ,CAAA;IAEpB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;QACjB,IAAI,CAAC,OAAO,GAAG,IAAA,iBAAW,EAAC,IAAI,CAAC,CAAA;KACjC;IAED,uCACK,IAAI,KACP,kBAAkB,EAAE,KAAK,EACzB,MAAM,EAAE,MAAA,IAAI,CAAC,OAAO,0CAAE,MAAM,EAC5B,QAAQ,EAAE,MAAA,IAAI,CAAC,OAAO,0CAAE,QAAS,EACjC,YAAY,EAAE,MAAA,IAAI,CAAC,OAAO,0CAAE,YAAa,EACzC,iBAAiB,EAAE;YACjB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,OAAO,EAAE,IAAI,CAAC,UAAU;SACzB,IACF;AACH,CAAC;AArBD,0BAqBC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.10.10",
+  "version": "2.10.12-alpha.0",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/index.js",
   "types": "dist/src/index.d.ts",
@@ -23,8 +23,8 @@
   "dependencies": {
     "@budibase/nano": "10.1.2",
     "@budibase/pouchdb-replication-stream": "1.2.10",
-    "@budibase/shared-core": "2.10.10",
-    "@budibase/types": "2.10.10",
+    "@budibase/shared-core": "2.10.12-alpha.0",
+    "@budibase/types": "2.10.12-alpha.0",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",
     "aws-sdk": "2.1030.0",
@@ -104,5 +104,5 @@
       }
     }
   },
-  "gitHead": "4e2d324324bda5cac4a80d7444755c0617258382"
+  "gitHead": "dc46ef326f5f6d9eecee14c8b1ffdc73b4bdfa54"
 }

package/src/cache/tests/user.spec.ts

@@ -0,0 +1,145 @@
+import { User } from "@budibase/types"
+import { generator, structures } from "../../../tests"
+import { DBTestConfiguration } from "../../../tests/extra"
+import { getUsers } from "../user"
+import { getGlobalDB } from "../../context"
+import _ from "lodash"
+
+import * as redis from "../../redis/init"
+import { UserDB } from "../../users"
+
+const config = new DBTestConfiguration()
+
+describe("user cache", () => {
+  describe("getUsers", () => {
+    const users: User[] = []
+    beforeAll(async () => {
+      const userCount = 10
+      const userIds = generator.arrayOf(() => generator.guid(), {
+        min: userCount,
+        max: userCount,
+      })
+
+      await config.doInTenant(async () => {
+        const db = getGlobalDB()
+        for (const userId of userIds) {
+          const user = structures.users.user({ _id: userId })
+          await db.put(user)
+          users.push(user)
+        }
+      })
+    })
+
+    beforeEach(async () => {
+      jest.clearAllMocks()
+
+      const redisClient = await redis.getUserClient()
+      await redisClient.clear()
+    })
+
+    it("when no user is in cache, all of them are retrieved from db", async () => {
+      const usersToRequest = _.sampleSize(users, 5)
+
+      const userIdsToRequest = usersToRequest.map(x => x._id!)
+
+      jest.spyOn(UserDB, "bulkGet")
+
+      const results = await config.doInTenant(() => getUsers(userIdsToRequest))
+
+      expect(results.users).toHaveLength(5)
+      expect(results).toEqual({
+        users: usersToRequest.map(u => ({
+          ...u,
+          budibaseAccess: true,
+          _rev: expect.any(String),
+        })),
+      })
+
+      expect(UserDB.bulkGet).toBeCalledTimes(1)
+      expect(UserDB.bulkGet).toBeCalledWith(userIdsToRequest)
+    })
+
+    it("on a second all, all of them are retrieved from cache", async () => {
+      const usersToRequest = _.sampleSize(users, 5)
+
+      const userIdsToRequest = usersToRequest.map(x => x._id!)
+
+      jest.spyOn(UserDB, "bulkGet")
+
+      await config.doInTenant(() => getUsers(userIdsToRequest))
+      const resultsFromCache = await config.doInTenant(() =>
+        getUsers(userIdsToRequest)
+      )
+
+      expect(resultsFromCache.users).toHaveLength(5)
+      expect(resultsFromCache).toEqual({
+        users: expect.arrayContaining(
+          usersToRequest.map(u => ({
+            ...u,
+            budibaseAccess: true,
+            _rev: expect.any(String),
+          }))
+        ),
+      })
+
+      expect(UserDB.bulkGet).toBeCalledTimes(1)
+    })
+
+    it("when some users are cached, only the missing ones are retrieved from db", async () => {
+      const usersToRequest = _.sampleSize(users, 5)
+
+      const userIdsToRequest = usersToRequest.map(x => x._id!)
+
+      jest.spyOn(UserDB, "bulkGet")
+
+      await config.doInTenant(() =>
+        getUsers([userIdsToRequest[0], userIdsToRequest[3]])
+      )
+      ;(UserDB.bulkGet as jest.Mock).mockClear()
+
+      const results = await config.doInTenant(() => getUsers(userIdsToRequest))
+
+      expect(results.users).toHaveLength(5)
+      expect(results).toEqual({
+        users: expect.arrayContaining(
+          usersToRequest.map(u => ({
+            ...u,
+            budibaseAccess: true,
+            _rev: expect.any(String),
+          }))
+        ),
+      })
+
+      expect(UserDB.bulkGet).toBeCalledTimes(1)
+      expect(UserDB.bulkGet).toBeCalledWith([
+        userIdsToRequest[1],
+        userIdsToRequest[2],
+        userIdsToRequest[4],
+      ])
+    })
+
+    it("requesting existing and unexisting ids will return found ones", async () => {
+      const usersToRequest = _.sampleSize(users, 3)
+      const missingIds = [generator.guid(), generator.guid()]
+
+      const userIdsToRequest = _.shuffle([
+        ...missingIds,
+        ...usersToRequest.map(x => x._id!),
+      ])
+
+      const results = await config.doInTenant(() => getUsers(userIdsToRequest))
+
+      expect(results.users).toHaveLength(3)
+      expect(results).toEqual({
+        users: expect.arrayContaining(
+          usersToRequest.map(u => ({
+            ...u,
+            budibaseAccess: true,
+            _rev: expect.any(String),
+          }))
+        ),
+        notFoundIds: expect.arrayContaining(missingIds),
+      })
+    })
+  })
+})

package/src/cache/user.ts

@@ -6,6 +6,7 @@ import env from "../environment"
 import * as accounts from "../accounts"
 import { UserDB } from "../users"
 import { sdk } from "@budibase/shared-core"
+import { User } from "@budibase/types"
 
 const EXPIRY_SECONDS = 3600
 
@@ -27,6 +28,35 @@ async function populateFromDB(userId: string, tenantId: string) {
   return user
 }
 
+async function populateUsersFromDB(
+  userIds: string[]
+): Promise<{ users: User[]; notFoundIds?: string[] }> {
+  const getUsersResponse = await UserDB.bulkGet(userIds)
+
+  // Handle missed user ids
+  const notFoundIds = userIds.filter((uid, i) => !getUsersResponse[i])
+
+  const users = getUsersResponse.filter(x => x)
+
+  await Promise.all(
+    users.map(async (user: any) => {
+      user.budibaseAccess = true
+      if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
+        const account = await accounts.getAccount(user.email)
+        if (account) {
+          user.account = account
+          user.accountPortalAccess = true
+        }
+      }
+    })
+  )
+
+  if (notFoundIds.length) {
+    return { users, notFoundIds }
+  }
+  return { users }
+}
+
 /**
  * Get the requested user by id.
  * Use redis cache to first read the user.
@@ -77,6 +107,36 @@ export async function getUser(
   return user
 }
 
+/**
+ * Get the requested users by id.
+ * Use redis cache to first read the users.
+ * If not present fallback to loading the users directly and re-caching.
+ * @param {*} userIds the ids of the user to get
+ * @param {*} tenantId the tenant of the users to get
+ * @returns
+ */
+export async function getUsers(
+  userIds: string[]
+): Promise<{ users: User[]; notFoundIds?: string[] }> {
+  const client = await redis.getUserClient()
+  // try cache
+  let usersFromCache = await client.bulkGet(userIds)
+  const missingUsersFromCache = userIds.filter(uid => !usersFromCache[uid])
+  const users = Object.values(usersFromCache)
+  let notFoundIds
+
+  if (missingUsersFromCache.length) {
+    const usersFromDb = await populateUsersFromDB(missingUsersFromCache)
+
+    notFoundIds = usersFromDb.notFoundIds
+    for (const userToCache of usersFromDb.users) {
+      await client.store(userToCache._id!, userToCache, EXPIRY_SECONDS)
+    }
+    users.push(...usersFromDb.users)
+  }
+  return { users, notFoundIds: notFoundIds }
+}
+
 export async function invalidateUser(userId: string) {
   const client = await redis.getUserClient()
   await client.delete(userId)

package/src/middleware/passport/sso/tests/sso.spec.ts

@@ -102,6 +102,7 @@ describe("sso", () => {
 
           // modified external id to match user format
           ssoUser._id = "us_" + details.userId
+          delete ssoUser.userId
 
           // new sso user won't have a password
           delete ssoUser.password

package/src/redis/redis.ts

@@ -250,7 +250,7 @@ class RedisWrapper {
     const prefixedKeys = keys.map(key => addDbPrefix(db, key))
     let response = await this.getClient().mget(prefixedKeys)
     if (Array.isArray(response)) {
-      let final: any = {}
+      let final: Record<string, any> = {}
       let count = 0
       for (let result of response) {
         if (result) {

package/tests/core/utilities/structures/sso.ts

@@ -13,8 +13,7 @@ import {
 } from "@budibase/types"
 import { generator } from "./generator"
 import { email, uuid } from "./common"
-import * as shared from "./shared"
-import { user } from "./shared"
+import * as users from "./users"
 import sample from "lodash/sample"
 
 export function OAuth(): OAuth2 {
@@ -26,7 +25,7 @@ export function OAuth(): OAuth2 {
 
 export function authDetails(userDoc?: User): SSOAuthDetails {
   if (!userDoc) {
-    userDoc = user()
+    userDoc = users.user()
   }
 
   const userId = userDoc._id || uuid()
@@ -52,7 +51,7 @@ export function providerType(): SSOProviderType {
 
 export function ssoProfile(user?: User): SSOProfile {
   if (!user) {
-    user = shared.user()
+    user = users.user()
   }
   return {
     id: user._id!,

package/tests/core/utilities/structures/users.ts

@@ -4,11 +4,33 @@ import {
   BuilderUser,
   SSOAuthDetails,
   SSOUser,
+  User,
 } from "@budibase/types"
-import { user } from "./shared"
 import { authDetails } from "./sso"
+import { uuid } from "./common"
+import { generator } from "./generator"
+import { tenant } from "."
+import { generateGlobalUserID } from "../../../../src/docIds"
 
-export { user, newEmail } from "./shared"
+export const newEmail = () => {
+  return `${uuid()}@test.com`
+}
+
+export const user = (userProps?: Partial<Omit<User, "userId">>): User => {
+  const userId = userProps?._id || generateGlobalUserID()
+  return {
+    _id: userId,
+    userId,
+    email: newEmail(),
+    password: "test",
+    roles: { app_test: "admin" },
+    firstName: generator.first(),
+    lastName: generator.last(),
+    pictureUrl: "http://test.com",
+    tenantId: tenant.id(),
+    ...userProps,
+  }
+}
 
 export const adminUser = (userProps?: any): AdminUser => {
   return {

package/dist/tests/core/utilities/structures/shared.d.ts

@@ -1,3 +0,0 @@
-import { User } from "@budibase/types";
-export declare const newEmail: () => string;
-export declare const user: (userProps?: any) => User;

package/dist/tests/core/utilities/structures/shared.js

@@ -1,14 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.user = exports.newEmail = void 0;
-const generator_1 = require("./generator");
-const common_1 = require("./common");
-const newEmail = () => {
-    return `${(0, common_1.uuid)()}@test.com`;
-};
-exports.newEmail = newEmail;
-const user = (userProps) => {
-    return Object.assign({ email: (0, exports.newEmail)(), password: "test", roles: { app_test: "admin" }, firstName: generator_1.generator.first(), lastName: generator_1.generator.last(), pictureUrl: "http://test.com" }, userProps);
-};
-exports.user = user;
-//# sourceMappingURL=shared.js.map

package/dist/tests/core/utilities/structures/shared.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../../../../tests/core/utilities/structures/shared.ts"],"names":[],"mappings":";;;AACA,2CAAuC;AACvC,qCAA+B;AAExB,MAAM,QAAQ,GAAG,GAAG,EAAE;IAC3B,OAAO,GAAG,IAAA,aAAI,GAAE,WAAW,CAAA;AAC7B,CAAC,CAAA;AAFY,QAAA,QAAQ,YAEpB;AAEM,MAAM,IAAI,GAAG,CAAC,SAAe,EAAQ,EAAE;IAC5C,uBACE,KAAK,EAAE,IAAA,gBAAQ,GAAE,EACjB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,EAC5B,SAAS,EAAE,qBAAS,CAAC,KAAK,EAAE,EAC5B,QAAQ,EAAE,qBAAS,CAAC,IAAI,EAAE,EAC1B,UAAU,EAAE,iBAAiB,IAC1B,SAAS,EACb;AACH,CAAC,CAAA;AAVY,QAAA,IAAI,QAUhB"}

package/tests/core/utilities/structures/shared.ts

@@ -1,19 +0,0 @@
-import { User } from "@budibase/types"
-import { generator } from "./generator"
-import { uuid } from "./common"
-
-export const newEmail = () => {
-  return `${uuid()}@test.com`
-}
-
-export const user = (userProps?: any): User => {
-  return {
-    email: newEmail(),
-    password: "test",
-    roles: { app_test: "admin" },
-    firstName: generator.first(),
-    lastName: generator.last(),
-    pictureUrl: "http://test.com",
-    ...userProps,
-  }
-}