@budibase/server 2.1.43-alpha.2 → 2.1.43

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.1.43-alpha.2",
+  "version": "2.1.43",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -36,6 +36,40 @@
     "env:account:enable": "node scripts/account.js enable",
     "env:account:disable": "node scripts/account.js disable"
   },
+  "jest": {
+    "preset": "ts-jest",
+    "testEnvironment": "node",
+    "moduleNameMapper": {
+      "@budibase/backend-core/(.*)": "<rootDir>/../backend-core/$1",
+      "@budibase/backend-core": "<rootDir>/../backend-core/src",
+      "@budibase/types": "<rootDir>/../types/src"
+    },
+    "setupFiles": [
+      "./scripts/jestSetup.js"
+    ],
+    "collectCoverageFrom": [
+      "src/**/*.js",
+      "!**/node_modules/**",
+      "!src/db/views/*.js",
+      "!src/api/controllers/deploy/**/*.js",
+      "!src/*.js",
+      "!src/api/controllers/static/**/*",
+      "!src/db/dynamoClient.js",
+      "!src/utilities/usageQuota.js",
+      "!src/api/routes/tests/**/*",
+      "!src/db/tests/**/*",
+      "!src/tests/**/*",
+      "!src/automations/tests/**/*",
+      "!src/utilities/fileProcessor.js",
+      "!src/utilities/fileSystem/**/*",
+      "!src/utilities/redis.js"
+    ],
+    "coverageReporters": [
+      "lcov",
+      "json",
+      "clover"
+    ]
+  },
   "keywords": [
     "budibase"
   ],
@@ -43,11 +77,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.1.43-alpha.2",
-    "@budibase/client": "2.1.43-alpha.2",
-    "@budibase/pro": "2.1.43-alpha.1",
-    "@budibase/string-templates": "2.1.43-alpha.2",
-    "@budibase/types": "2.1.43-alpha.2",
+    "@budibase/backend-core": "^2.1.43",
+    "@budibase/client": "^2.1.43",
+    "@budibase/pro": "2.1.42",
+    "@budibase/string-templates": "^2.1.43",
+    "@budibase/types": "^2.1.43",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -137,7 +171,7 @@
     "@types/oracledb": "5.2.2",
     "@types/pouchdb": "6.4.0",
     "@types/redis": "4.0.11",
-    "@typescript-eslint/parser": "5.45.0",
+    "@typescript-eslint/parser": "5.12.0",
     "apidoc": "0.50.4",
     "babel-jest": "27.5.1",
     "copyfiles": "2.4.1",
@@ -145,7 +179,7 @@
     "eslint": "6.8.0",
     "ioredis-mock": "7.2.0",
     "is-wsl": "2.2.0",
-    "jest": "28.1.1",
+    "jest": "27.5.1",
     "jest-openapi": "0.14.2",
     "nodemon": "2.0.15",
     "openapi-types": "9.3.1",
@@ -156,14 +190,14 @@
     "supertest": "4.0.2",
     "swagger-jsdoc": "6.1.0",
     "timekeeper": "2.2.0",
-    "ts-jest": "28.0.4",
-    "ts-node": "10.8.1",
+    "ts-jest": "27.1.3",
+    "ts-node": "10.5.0",
     "tsconfig-paths": "4.0.0",
-    "typescript": "4.7.3",
+    "typescript": "4.6.2",
     "update-dotenv": "1.1.1"
   },
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "9967b1962b7ddcbe4fac08fd06b1967e3ce94e01"
+  "gitHead": "462d3be32347f58d90ea2fa98bd1e42faa3b0d9b"
 }

package/scripts/exportAppTemplate.js

@@ -3,7 +3,7 @@ const yargs = require("yargs")
 const fs = require("fs")
 const { join } = require("path")
 require("../src/db").init()
-const { db: dbCore } = require("@budibase/backend-core")
+const { doWithDB } = require("@budibase/backend-core/db")
 // load environment
 const env = require("../src/environment")
 const {
@@ -48,7 +48,7 @@ yargs
       const writeStream = fs.createWriteStream(join(exportPath, "dump.text"))
       // perform couch dump
 
-      await dbCore.doWithDB(appId, async db => {
+      await doWithDB(appId, async db => {
         return db.dump(writeStream, {
           filter: doc =>
             !(

package/{src/tests/jestSetup.ts → scripts/jestSetup.js}

@@ -1,5 +1,5 @@
 const { tmpdir } = require("os")
-import env from "../environment"
+const env = require("../src/environment")
 
 env._set("SELF_HOSTED", "1")
 env._set("NODE_ENV", "jest")
@@ -8,11 +8,8 @@ env._set("CLIENT_ID", "test-client-id")
 env._set("BUDIBASE_DIR", tmpdir("budibase-unittests"))
 env._set("LOG_LEVEL", "silent")
 env._set("PORT", 0)
-env._set("MINIO_URL", "http://localhost")
-env._set("MINIO_ACCESS_KEY", "test")
-env._set("MINIO_SECRET_KEY", "test")
 
-import { mocks } from "@budibase/backend-core/tests"
+const { mocks } = require("@budibase/backend-core/tests")
 
 // mock all dates to 2020-01-01T00:00:00.000Z
 // use tk.reset() to use real dates in individual tests
@@ -20,9 +17,3 @@ const tk = require("timekeeper")
 tk.freeze(mocks.date.MOCK_DATE)
 
 global.console.log = jest.fn() // console.log are ignored in tests
-
-if (!process.env.CI) {
-  // set a longer timeout in dev for debugging
-  // 100 seconds
-  jest.setTimeout(100000)
-}

package/scripts/localdomain.js

@@ -2,36 +2,6 @@
 const updateDotEnv = require("update-dotenv")
 
 const arg = process.argv.slice(2)[0]
-const isEnable = arg === "enable"
-
-let domain = process.argv.slice(2)[1]
-if (!domain) {
-  domain = "local.com"
-}
-
-const getAccountPortalUrl = () => {
-  if (isEnable) {
-    return `http://account.${domain}:10001`
-  } else {
-    return `http://localhost:10001`
-  }
-}
-
-const getBudibaseUrl = () => {
-  if (isEnable) {
-    return `http://${domain}:10000`
-  } else {
-    return `http://localhost:10000`
-  }
-}
-
-const getCookieDomain = () => {
-  if (isEnable) {
-    return `.${domain}`
-  } else {
-    return ""
-  }
-}
 
 /**
  * For testing multi tenancy sub domains locally.
@@ -46,7 +16,9 @@ const getCookieDomain = () => {
  * 127.0.0.1   t2.local.com
  */
 updateDotEnv({
-  ACCOUNT_PORTAL_URL: getAccountPortalUrl(),
-  COOKIE_DOMAIN: getCookieDomain(),
-  PLATFORM_URL: getBudibaseUrl(),
-}).then(() => console.log("Updated server!"))
+  ACCOUNT_PORTAL_URL:
+    arg === "enable"
+      ? "http://account.local.com:10001"
+      : "http://localhost:10001",
+  COOKIE_DOMAIN: arg === "enable" ? ".local.com" : "",
+}).then(() => console.log("Updated worker!"))

package/src/api/controllers/{apikeys.ts → apikeys.js}

@@ -1,10 +1,10 @@
-import { db as dbCore, tenancy } from "@budibase/backend-core"
-import { BBContext, Document } from "@budibase/types"
+const { StaticDatabases } = require("@budibase/backend-core/db")
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
 
-const KEYS_DOC = dbCore.StaticDatabases.GLOBAL.docs.apiKeys
+const KEYS_DOC = StaticDatabases.GLOBAL.docs.apiKeys
 
 async function getBuilderMainDoc() {
-  const db = tenancy.getGlobalDB()
+  const db = getGlobalDB()
   try {
     return await db.get(KEYS_DOC)
   } catch (err) {
@@ -15,24 +15,24 @@ async function getBuilderMainDoc() {
   }
 }
 
-async function setBuilderMainDoc(doc: Document) {
+async function setBuilderMainDoc(doc) {
   // make sure to override the ID
   doc._id = KEYS_DOC
-  const db = tenancy.getGlobalDB()
+  const db = getGlobalDB()
   return db.put(doc)
 }
 
-export async function fetch(ctx: BBContext) {
+exports.fetch = async function (ctx) {
   try {
     const mainDoc = await getBuilderMainDoc()
     ctx.body = mainDoc.apiKeys ? mainDoc.apiKeys : {}
-  } catch (err: any) {
+  } catch (err) {
     /* istanbul ignore next */
     ctx.throw(400, err)
   }
 }
 
-export async function update(ctx: BBContext) {
+exports.update = async function (ctx) {
   const key = ctx.params.key
   const value = ctx.request.body.value
 
@@ -47,7 +47,7 @@ export async function update(ctx: BBContext) {
       _id: resp.id,
       _rev: resp.rev,
     }
-  } catch (err: any) {
+  } catch (err) {
     /* istanbul ignore next */
     ctx.throw(400, err)
   }

package/src/api/controllers/application.ts

@@ -14,36 +14,35 @@ import {
   DocumentType,
   AppStatus,
 } from "../../db/utils"
-import {
-  db as dbCore,
-  roles,
-  cache,
-  tenancy,
-  context,
-  errors,
-  events,
-  migrations,
-} from "@budibase/backend-core"
+const {
+  BUILTIN_ROLE_IDS,
+  AccessController,
+} = require("@budibase/backend-core/roles")
+const { CacheKeys, bustCache } = require("@budibase/backend-core/cache")
+const {
+  getAllApps,
+  isDevAppID,
+  getProdAppID,
+  Replication,
+} = require("@budibase/backend-core/db")
 import { USERS_TABLE_SCHEMA } from "../../constants"
-import { buildDefaultDocs } from "../../db/defaultData/datasource_bb_default"
-
 import { removeAppFromUserRoles } from "../../utilities/workerRequests"
-import {
-  clientLibraryPath,
-  stringToReadStream,
-  isQsTrue,
-} from "../../utilities"
+import { clientLibraryPath, stringToReadStream } from "../../utilities"
 import { getLocksById } from "../../utilities/redis"
 import {
   updateClientLibrary,
   backupClientLibrary,
   revertClientLibrary,
 } from "../../utilities/fileSystem/clientLibrary"
+const { getTenantId, isMultiTenant } = require("@budibase/backend-core/tenancy")
 import { syncGlobalUsers } from "./user"
+const { app: appCache } = require("@budibase/backend-core/cache")
 import { cleanupAutomations } from "../../automations/utils"
+import { context } from "@budibase/backend-core"
 import { checkAppMetadata } from "../../automations/logging"
 import { getUniqueRows } from "../../utilities/usageQuota/rows"
 import { quotas, groups } from "@budibase/pro"
+import { errors, events, migrations } from "@budibase/backend-core"
 import { App, Layout, Screen, MigrationType } from "@budibase/types"
 import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts"
 import { enrichPluginURLs } from "../../utilities/plugins"
@@ -76,7 +75,7 @@ async function getScreens() {
 
 function getUserRoleId(ctx: any) {
   return !ctx.user.role || !ctx.user.role._id
-    ? roles.BUILTIN_ROLE_IDS.PUBLIC
+    ? BUILTIN_ROLE_IDS.PUBLIC
     : ctx.user.role._id
 }
 
@@ -123,8 +122,8 @@ const checkAppName = (
   }
 }
 
-async function createInstance(template: any, includeSampleData: boolean) {
-  const tenantId = tenancy.isMultiTenant() ? tenancy.getTenantId() : null
+async function createInstance(template: any) {
+  const tenantId = isMultiTenant() ? getTenantId() : null
   const baseAppId = generateAppID(tenantId)
   const appId = generateDevAppID(baseAppId)
   await context.updateAppId(appId)
@@ -155,27 +154,15 @@ async function createInstance(template: any, includeSampleData: boolean) {
   } else {
     // create the users table
     await db.put(USERS_TABLE_SCHEMA)
-
-    if (includeSampleData) {
-      // create ootb stock db
-      await addDefaultTables(db)
-    }
   }
 
   return { _id: appId }
 }
 
-const addDefaultTables = async (db: any) => {
-  const defaultDbDocs = buildDefaultDocs()
-
-  // add in the default db data docs - tables, datasource, rows and links
-  await db.bulkDocs([...defaultDbDocs])
-}
-
 export const fetch = async (ctx: any) => {
   const dev = ctx.query && ctx.query.status === AppStatus.DEV
   const all = ctx.query && ctx.query.status === AppStatus.ALL
-  const apps = (await dbCore.getAllApps({ dev, all })) as App[]
+  const apps = await getAllApps({ dev, all })
 
   const appIds = apps
     .filter((app: any) => app.status === "development")
@@ -200,7 +187,7 @@ export const fetch = async (ctx: any) => {
 export const fetchAppDefinition = async (ctx: any) => {
   const layouts = await getLayouts()
   const userRoleId = getUserRoleId(ctx)
-  const accessController = new roles.AccessController()
+  const accessController = new AccessController()
   const screens = await accessController.checkScreensAccess(
     await getScreens(),
     userRoleId
@@ -224,7 +211,7 @@ export const fetchAppPackage = async (ctx: any) => {
   // Only filter screens if the user is not a builder
   if (!(ctx.user.builder && ctx.user.builder.global)) {
     const userRoleId = getUserRoleId(ctx)
-    const accessController = new roles.AccessController()
+    const accessController = new AccessController()
     screens = await accessController.checkScreensAccess(screens, userRoleId)
   }
 
@@ -237,7 +224,7 @@ export const fetchAppPackage = async (ctx: any) => {
 }
 
 const performAppCreate = async (ctx: any) => {
-  const apps = await dbCore.getAllApps({ dev: true })
+  const apps = await getAllApps({ dev: true })
   const name = ctx.request.body.name
   checkAppName(ctx, apps, name)
   const url = getAppUrl(ctx)
@@ -252,8 +239,7 @@ const performAppCreate = async (ctx: any) => {
   if (ctx.request.files && ctx.request.files.templateFile) {
     instanceConfig.file = ctx.request.files.templateFile
   }
-  const includeSampleData = isQsTrue(ctx.request.body.sampleData)
-  const instance = await createInstance(instanceConfig, includeSampleData)
+  const instance = await createInstance(instanceConfig)
   const appId = instance._id
   const db = context.getAppDB()
 
@@ -268,7 +254,7 @@ const performAppCreate = async (ctx: any) => {
     url: url,
     template: templateKey,
     instance,
-    tenantId: tenancy.getTenantId(),
+    tenantId: getTenantId(),
     updatedAt: new Date().toISOString(),
     createdAt: new Date().toISOString(),
     status: AppStatus.DEV,
@@ -327,7 +313,7 @@ const performAppCreate = async (ctx: any) => {
     await createApp(appId)
   }
 
-  await cache.app.invalidateAppMetadata(appId, newApplication)
+  await appCache.invalidateAppMetadata(appId, newApplication)
   return newApplication
 }
 
@@ -357,7 +343,7 @@ const creationEvents = async (request: any, app: App) => {
 }
 
 const appPostCreate = async (ctx: any, app: App) => {
-  const tenantId = tenancy.getTenantId()
+  const tenantId = getTenantId()
   await migrations.backPopulateMigrations({
     type: MigrationType.APP,
     tenantId,
@@ -370,9 +356,7 @@ const appPostCreate = async (ctx: any, app: App) => {
     const rowCount = rows ? rows.length : 0
     if (rowCount) {
       try {
-        await context.doInAppContext(app.appId, () => {
-          return quotas.addRows(rowCount)
-        })
+        await quotas.addRows(rowCount)
       } catch (err: any) {
         if (err.code && err.code === errors.codes.USAGE_LIMIT_EXCEEDED) {
           // this import resulted in row usage exceeding the quota
@@ -390,7 +374,7 @@ const appPostCreate = async (ctx: any, app: App) => {
 export const create = async (ctx: any) => {
   const newApplication = await quotas.addApp(() => performAppCreate(ctx))
   await appPostCreate(ctx, newApplication)
-  await cache.bustCache(cache.CacheKey.CHECKLIST)
+  await bustCache(CacheKeys.CHECKLIST)
   ctx.body = newApplication
   ctx.status = 200
 }
@@ -398,7 +382,7 @@ export const create = async (ctx: any) => {
 // This endpoint currently operates as a PATCH rather than a PUT
 // Thus name and url fields are handled only if present
 export const update = async (ctx: any) => {
-  const apps = await dbCore.getAllApps({ dev: true })
+  const apps = await getAllApps({ dev: true })
   // validation
   const name = ctx.request.body.name
   if (name) {
@@ -471,7 +455,7 @@ const destroyApp = async (ctx: any) => {
   let isUnpublish = ctx.query && ctx.query.unpublish
 
   if (isUnpublish) {
-    appId = dbCore.getProdAppID(appId)
+    appId = getProdAppID(appId)
   }
 
   const db = isUnpublish ? context.getProdAppDB() : context.getAppDB()
@@ -497,7 +481,7 @@ const destroyApp = async (ctx: any) => {
   else {
     await removeAppFromUserRoles(ctx, appId)
   }
-  await cache.app.invalidateAppMetadata(appId)
+  await appCache.invalidateAppMetadata(appId)
   return result
 }
 
@@ -533,17 +517,19 @@ export const sync = async (ctx: any, next: any) => {
   }
 
   const appId = ctx.params.appId
-  if (!dbCore.isDevAppID(appId)) {
+  if (!isDevAppID(appId)) {
     ctx.throw(400, "This action cannot be performed for production apps")
   }
 
   // replicate prod to dev
-  const prodAppId = dbCore.getProdAppID(appId)
+  const prodAppId = getProdAppID(appId)
 
-  // specific case, want to make sure setup is skipped
-  const prodDb = context.getProdAppDB({ skip_setup: true })
-  const exists = await prodDb.exists()
-  if (!exists) {
+  try {
+    // specific case, want to make sure setup is skipped
+    const prodDb = context.getProdAppDB({ skip_setup: true })
+    const info = await prodDb.info()
+    if (info.error) throw info.error
+  } catch (err) {
     // the database doesn't exist. Don't replicate
     ctx.status = 200
     ctx.body = {
@@ -552,7 +538,7 @@ export const sync = async (ctx: any, next: any) => {
     return next()
   }
 
-  const replication = new dbCore.Replication({
+  const replication = new Replication({
     source: prodAppId,
     target: appId,
   })
@@ -593,7 +579,7 @@ export const updateAppPackage = async (appPackage: any, appId: any) => {
 
     await db.put(newAppPackage)
     // remove any cached metadata, so that it will be updated
-    await cache.app.invalidateAppMetadata(appId)
+    await appCache.invalidateAppMetadata(appId)
     return newAppPackage
   })
 }

package/src/api/controllers/auth.ts

@@ -3,7 +3,6 @@ import { InternalTables } from "../../db/utils"
 import { getFullUser } from "../../utilities/users"
 import { roles, context } from "@budibase/backend-core"
 import { groups } from "@budibase/pro"
-import { ContextUser, User, Row } from "@budibase/types"
 
 const PUBLIC_ROLE = roles.BUILTIN_ROLE_IDS.PUBLIC
 
@@ -25,7 +24,7 @@ export async function fetchSelf(ctx: any) {
   }
 
   const appId = context.getAppId()
-  const user: ContextUser = await getFullUser(ctx, userId)
+  const user = await getFullUser(ctx, userId)
   // this shouldn't be returned by the app self
   delete user.roles
   // forward the csrf token from the session
@@ -35,7 +34,7 @@ export async function fetchSelf(ctx: any) {
     const db = context.getAppDB()
     // check for group permissions
     if (!user.roleId || user.roleId === PUBLIC_ROLE) {
-      const groupRoleId = await groups.getGroupRoleId(user as User, appId)
+      const groupRoleId = await groups.getGroupRoleId(user, appId)
       user.roleId = groupRoleId || user.roleId
     }
     // remove the full roles structure
@@ -43,7 +42,7 @@ export async function fetchSelf(ctx: any) {
     try {
       const userTable = await db.get(InternalTables.USER_METADATA)
       // specifically needs to make sure is enriched
-      ctx.body = await outputProcessing(userTable, user as Row)
+      ctx.body = await outputProcessing(userTable, user)
     } catch (err: any) {
       let response
       // user didn't exist in app, don't pretend they do