@budibase/server 2.1.38 → 2.1.40-alpha.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.1.38",
+  "version": "2.1.40-alpha.0",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -36,40 +36,6 @@
     "env:account:enable": "node scripts/account.js enable",
     "env:account:disable": "node scripts/account.js disable"
   },
-  "jest": {
-    "preset": "ts-jest",
-    "testEnvironment": "node",
-    "moduleNameMapper": {
-      "@budibase/backend-core/(.*)": "<rootDir>/../backend-core/$1",
-      "@budibase/backend-core": "<rootDir>/../backend-core/src",
-      "@budibase/types": "<rootDir>/../types/src"
-    },
-    "setupFiles": [
-      "./scripts/jestSetup.js"
-    ],
-    "collectCoverageFrom": [
-      "src/**/*.js",
-      "!**/node_modules/**",
-      "!src/db/views/*.js",
-      "!src/api/controllers/deploy/**/*.js",
-      "!src/*.js",
-      "!src/api/controllers/static/**/*",
-      "!src/db/dynamoClient.js",
-      "!src/utilities/usageQuota.js",
-      "!src/api/routes/tests/**/*",
-      "!src/db/tests/**/*",
-      "!src/tests/**/*",
-      "!src/automations/tests/**/*",
-      "!src/utilities/fileProcessor.js",
-      "!src/utilities/fileSystem/**/*",
-      "!src/utilities/redis.js"
-    ],
-    "coverageReporters": [
-      "lcov",
-      "json",
-      "clover"
-    ]
-  },
   "keywords": [
     "budibase"
   ],
@@ -77,11 +43,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "^2.1.38",
-    "@budibase/client": "^2.1.38",
-    "@budibase/pro": "2.1.37",
-    "@budibase/string-templates": "^2.1.38",
-    "@budibase/types": "^2.1.38",
+    "@budibase/backend-core": "2.1.40-alpha.0",
+    "@budibase/client": "2.1.40-alpha.0",
+    "@budibase/pro": "2.1.39",
+    "@budibase/string-templates": "2.1.40-alpha.0",
+    "@budibase/types": "2.1.40-alpha.0",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -179,7 +145,7 @@
     "eslint": "6.8.0",
     "ioredis-mock": "7.2.0",
     "is-wsl": "2.2.0",
-    "jest": "27.5.1",
+    "jest": "28.1.1",
     "jest-openapi": "0.14.2",
     "nodemon": "2.0.15",
     "openapi-types": "9.3.1",
@@ -190,14 +156,14 @@
     "supertest": "4.0.2",
     "swagger-jsdoc": "6.1.0",
     "timekeeper": "2.2.0",
-    "ts-jest": "27.1.3",
-    "ts-node": "10.5.0",
+    "ts-jest": "28.0.4",
+    "ts-node": "10.8.1",
     "tsconfig-paths": "4.0.0",
-    "typescript": "4.6.2",
+    "typescript": "4.7.3",
     "update-dotenv": "1.1.1"
   },
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "98eceef539c21b0c72c73aa9ec313e38077dc08f"
+  "gitHead": "c1e2c04c89df9f5ce6d5b25d7f6726a9473d9a34"
 }

package/scripts/exportAppTemplate.js

@@ -3,7 +3,7 @@ const yargs = require("yargs")
 const fs = require("fs")
 const { join } = require("path")
 require("../src/db").init()
-const { doWithDB } = require("@budibase/backend-core/db")
+const { db: dbCore } = require("@budibase/backend-core")
 // load environment
 const env = require("../src/environment")
 const {
@@ -48,7 +48,7 @@ yargs
       const writeStream = fs.createWriteStream(join(exportPath, "dump.text"))
       // perform couch dump
 
-      await doWithDB(appId, async db => {
+      await dbCore.doWithDB(appId, async db => {
         return db.dump(writeStream, {
           filter: doc =>
             !(

package/scripts/localdomain.js

@@ -2,6 +2,36 @@
 const updateDotEnv = require("update-dotenv")
 
 const arg = process.argv.slice(2)[0]
+const isEnable = arg === "enable"
+
+let domain = process.argv.slice(2)[1]
+if (!domain) {
+  domain = "local.com"
+}
+
+const getAccountPortalUrl = () => {
+  if (isEnable) {
+    return `http://account.${domain}:10001`
+  } else {
+    return `http://localhost:10001`
+  }
+}
+
+const getBudibaseUrl = () => {
+  if (isEnable) {
+    return `http://${domain}:10000`
+  } else {
+    return `http://localhost:10000`
+  }
+}
+
+const getCookieDomain = () => {
+  if (isEnable) {
+    return `.${domain}`
+  } else {
+    return ""
+  }
+}
 
 /**
  * For testing multi tenancy sub domains locally.
@@ -16,9 +46,7 @@ const arg = process.argv.slice(2)[0]
  * 127.0.0.1   t2.local.com
  */
 updateDotEnv({
-  ACCOUNT_PORTAL_URL:
-    arg === "enable"
-      ? "http://account.local.com:10001"
-      : "http://localhost:10001",
-  COOKIE_DOMAIN: arg === "enable" ? ".local.com" : "",
-}).then(() => console.log("Updated worker!"))
+  ACCOUNT_PORTAL_URL: getAccountPortalUrl(),
+  COOKIE_DOMAIN: getCookieDomain(),
+  PLATFORM_URL: getBudibaseUrl(),
+}).then(() => console.log("Updated server!"))

package/src/api/controllers/{apikeys.js → apikeys.ts}

@@ -1,10 +1,10 @@
-const { StaticDatabases } = require("@budibase/backend-core/db")
-const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+import { db as dbCore, tenancy } from "@budibase/backend-core"
+import { BBContext, Document } from "@budibase/types"
 
-const KEYS_DOC = StaticDatabases.GLOBAL.docs.apiKeys
+const KEYS_DOC = dbCore.StaticDatabases.GLOBAL.docs.apiKeys
 
 async function getBuilderMainDoc() {
-  const db = getGlobalDB()
+  const db = tenancy.getGlobalDB()
   try {
     return await db.get(KEYS_DOC)
   } catch (err) {
@@ -15,24 +15,24 @@ async function getBuilderMainDoc() {
   }
 }
 
-async function setBuilderMainDoc(doc) {
+async function setBuilderMainDoc(doc: Document) {
   // make sure to override the ID
   doc._id = KEYS_DOC
-  const db = getGlobalDB()
+  const db = tenancy.getGlobalDB()
   return db.put(doc)
 }
 
-exports.fetch = async function (ctx) {
+export async function fetch(ctx: BBContext) {
   try {
     const mainDoc = await getBuilderMainDoc()
     ctx.body = mainDoc.apiKeys ? mainDoc.apiKeys : {}
-  } catch (err) {
+  } catch (err: any) {
     /* istanbul ignore next */
     ctx.throw(400, err)
   }
 }
 
-exports.update = async function (ctx) {
+export async function update(ctx: BBContext) {
   const key = ctx.params.key
   const value = ctx.request.body.value
 
@@ -47,7 +47,7 @@ exports.update = async function (ctx) {
       _id: resp.id,
       _rev: resp.rev,
     }
-  } catch (err) {
+  } catch (err: any) {
     /* istanbul ignore next */
     ctx.throw(400, err)
   }

package/src/api/controllers/application.ts

@@ -14,35 +14,36 @@ import {
   DocumentType,
   AppStatus,
 } from "../../db/utils"
-const {
-  BUILTIN_ROLE_IDS,
-  AccessController,
-} = require("@budibase/backend-core/roles")
-const { CacheKeys, bustCache } = require("@budibase/backend-core/cache")
-const {
-  getAllApps,
-  isDevAppID,
-  getProdAppID,
-  Replication,
-} = require("@budibase/backend-core/db")
+import {
+  db as dbCore,
+  roles,
+  cache,
+  tenancy,
+  context,
+  errors,
+  events,
+  migrations,
+} from "@budibase/backend-core"
 import { USERS_TABLE_SCHEMA } from "../../constants"
+import { buildDefaultDocs } from "../../db/defaultData/datasource_bb_default"
+
 import { removeAppFromUserRoles } from "../../utilities/workerRequests"
-import { clientLibraryPath, stringToReadStream } from "../../utilities"
+import {
+  clientLibraryPath,
+  stringToReadStream,
+  isQsTrue,
+} from "../../utilities"
 import { getLocksById } from "../../utilities/redis"
 import {
   updateClientLibrary,
   backupClientLibrary,
   revertClientLibrary,
 } from "../../utilities/fileSystem/clientLibrary"
-const { getTenantId, isMultiTenant } = require("@budibase/backend-core/tenancy")
 import { syncGlobalUsers } from "./user"
-const { app: appCache } = require("@budibase/backend-core/cache")
 import { cleanupAutomations } from "../../automations/utils"
-import { context } from "@budibase/backend-core"
 import { checkAppMetadata } from "../../automations/logging"
 import { getUniqueRows } from "../../utilities/usageQuota/rows"
 import { quotas, groups } from "@budibase/pro"
-import { errors, events, migrations } from "@budibase/backend-core"
 import { App, Layout, Screen, MigrationType } from "@budibase/types"
 import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts"
 import { enrichPluginURLs } from "../../utilities/plugins"
@@ -75,7 +76,7 @@ async function getScreens() {
 
 function getUserRoleId(ctx: any) {
   return !ctx.user.role || !ctx.user.role._id
-    ? BUILTIN_ROLE_IDS.PUBLIC
+    ? roles.BUILTIN_ROLE_IDS.PUBLIC
     : ctx.user.role._id
 }
 
@@ -122,8 +123,8 @@ const checkAppName = (
   }
 }
 
-async function createInstance(template: any) {
-  const tenantId = isMultiTenant() ? getTenantId() : null
+async function createInstance(template: any, includeSampleData: boolean) {
+  const tenantId = tenancy.isMultiTenant() ? tenancy.getTenantId() : null
   const baseAppId = generateAppID(tenantId)
   const appId = generateDevAppID(baseAppId)
   await context.updateAppId(appId)
@@ -154,15 +155,27 @@ async function createInstance(template: any) {
   } else {
     // create the users table
     await db.put(USERS_TABLE_SCHEMA)
+
+    if (includeSampleData) {
+      // create ootb stock db
+      await addDefaultTables(db)
+    }
   }
 
   return { _id: appId }
 }
 
+const addDefaultTables = async (db: any) => {
+  const defaultDbDocs = buildDefaultDocs()
+
+  // add in the default db data docs - tables, datasource, rows and links
+  await db.bulkDocs([...defaultDbDocs])
+}
+
 export const fetch = async (ctx: any) => {
   const dev = ctx.query && ctx.query.status === AppStatus.DEV
   const all = ctx.query && ctx.query.status === AppStatus.ALL
-  const apps = await getAllApps({ dev, all })
+  const apps = (await dbCore.getAllApps({ dev, all })) as App[]
 
   const appIds = apps
     .filter((app: any) => app.status === "development")
@@ -187,7 +200,7 @@ export const fetch = async (ctx: any) => {
 export const fetchAppDefinition = async (ctx: any) => {
   const layouts = await getLayouts()
   const userRoleId = getUserRoleId(ctx)
-  const accessController = new AccessController()
+  const accessController = new roles.AccessController()
   const screens = await accessController.checkScreensAccess(
     await getScreens(),
     userRoleId
@@ -211,7 +224,7 @@ export const fetchAppPackage = async (ctx: any) => {
   // Only filter screens if the user is not a builder
   if (!(ctx.user.builder && ctx.user.builder.global)) {
     const userRoleId = getUserRoleId(ctx)
-    const accessController = new AccessController()
+    const accessController = new roles.AccessController()
     screens = await accessController.checkScreensAccess(screens, userRoleId)
   }
 
@@ -224,7 +237,7 @@ export const fetchAppPackage = async (ctx: any) => {
 }
 
 const performAppCreate = async (ctx: any) => {
-  const apps = await getAllApps({ dev: true })
+  const apps = await dbCore.getAllApps({ dev: true })
   const name = ctx.request.body.name
   checkAppName(ctx, apps, name)
   const url = getAppUrl(ctx)
@@ -239,7 +252,8 @@ const performAppCreate = async (ctx: any) => {
   if (ctx.request.files && ctx.request.files.templateFile) {
     instanceConfig.file = ctx.request.files.templateFile
   }
-  const instance = await createInstance(instanceConfig)
+  const includeSampleData = isQsTrue(ctx.request.body.sampleData)
+  const instance = await createInstance(instanceConfig, includeSampleData)
   const appId = instance._id
   const db = context.getAppDB()
 
@@ -254,7 +268,7 @@ const performAppCreate = async (ctx: any) => {
     url: url,
     template: templateKey,
     instance,
-    tenantId: getTenantId(),
+    tenantId: tenancy.getTenantId(),
     updatedAt: new Date().toISOString(),
     createdAt: new Date().toISOString(),
     status: AppStatus.DEV,
@@ -313,7 +327,7 @@ const performAppCreate = async (ctx: any) => {
     await createApp(appId)
   }
 
-  await appCache.invalidateAppMetadata(appId, newApplication)
+  await cache.app.invalidateAppMetadata(appId, newApplication)
   return newApplication
 }
 
@@ -343,7 +357,7 @@ const creationEvents = async (request: any, app: App) => {
 }
 
 const appPostCreate = async (ctx: any, app: App) => {
-  const tenantId = getTenantId()
+  const tenantId = tenancy.getTenantId()
   await migrations.backPopulateMigrations({
     type: MigrationType.APP,
     tenantId,
@@ -356,7 +370,9 @@ const appPostCreate = async (ctx: any, app: App) => {
     const rowCount = rows ? rows.length : 0
     if (rowCount) {
       try {
-        await quotas.addRows(rowCount)
+        await context.doInAppContext(app.appId, () => {
+          return quotas.addRows(rowCount)
+        })
       } catch (err: any) {
         if (err.code && err.code === errors.codes.USAGE_LIMIT_EXCEEDED) {
           // this import resulted in row usage exceeding the quota
@@ -374,7 +390,7 @@ const appPostCreate = async (ctx: any, app: App) => {
 export const create = async (ctx: any) => {
   const newApplication = await quotas.addApp(() => performAppCreate(ctx))
   await appPostCreate(ctx, newApplication)
-  await bustCache(CacheKeys.CHECKLIST)
+  await cache.bustCache(cache.CacheKey.CHECKLIST)
   ctx.body = newApplication
   ctx.status = 200
 }
@@ -382,7 +398,7 @@ export const create = async (ctx: any) => {
 // This endpoint currently operates as a PATCH rather than a PUT
 // Thus name and url fields are handled only if present
 export const update = async (ctx: any) => {
-  const apps = await getAllApps({ dev: true })
+  const apps = await dbCore.getAllApps({ dev: true })
   // validation
   const name = ctx.request.body.name
   if (name) {
@@ -455,7 +471,7 @@ const destroyApp = async (ctx: any) => {
   let isUnpublish = ctx.query && ctx.query.unpublish
 
   if (isUnpublish) {
-    appId = getProdAppID(appId)
+    appId = dbCore.getProdAppID(appId)
   }
 
   const db = isUnpublish ? context.getProdAppDB() : context.getAppDB()
@@ -481,7 +497,7 @@ const destroyApp = async (ctx: any) => {
   else {
     await removeAppFromUserRoles(ctx, appId)
   }
-  await appCache.invalidateAppMetadata(appId)
+  await cache.app.invalidateAppMetadata(appId)
   return result
 }
 
@@ -517,19 +533,17 @@ export const sync = async (ctx: any, next: any) => {
   }
 
   const appId = ctx.params.appId
-  if (!isDevAppID(appId)) {
+  if (!dbCore.isDevAppID(appId)) {
     ctx.throw(400, "This action cannot be performed for production apps")
   }
 
   // replicate prod to dev
-  const prodAppId = getProdAppID(appId)
+  const prodAppId = dbCore.getProdAppID(appId)
 
-  try {
-    // specific case, want to make sure setup is skipped
-    const prodDb = context.getProdAppDB({ skip_setup: true })
-    const info = await prodDb.info()
-    if (info.error) throw info.error
-  } catch (err) {
+  // specific case, want to make sure setup is skipped
+  const prodDb = context.getProdAppDB({ skip_setup: true })
+  const exists = await prodDb.exists()
+  if (!exists) {
     // the database doesn't exist. Don't replicate
     ctx.status = 200
     ctx.body = {
@@ -538,7 +552,7 @@ export const sync = async (ctx: any, next: any) => {
     return next()
   }
 
-  const replication = new Replication({
+  const replication = new dbCore.Replication({
     source: prodAppId,
     target: appId,
   })
@@ -579,7 +593,7 @@ export const updateAppPackage = async (appPackage: any, appId: any) => {
 
     await db.put(newAppPackage)
     // remove any cached metadata, so that it will be updated
-    await appCache.invalidateAppMetadata(appId)
+    await cache.app.invalidateAppMetadata(appId)
     return newAppPackage
   })
 }

package/src/api/controllers/auth.ts

@@ -3,6 +3,7 @@ import { InternalTables } from "../../db/utils"
 import { getFullUser } from "../../utilities/users"
 import { roles, context } from "@budibase/backend-core"
 import { groups } from "@budibase/pro"
+import { ContextUser, User, Row } from "@budibase/types"
 
 const PUBLIC_ROLE = roles.BUILTIN_ROLE_IDS.PUBLIC
 
@@ -24,7 +25,7 @@ export async function fetchSelf(ctx: any) {
   }
 
   const appId = context.getAppId()
-  const user = await getFullUser(ctx, userId)
+  const user: ContextUser = await getFullUser(ctx, userId)
   // this shouldn't be returned by the app self
   delete user.roles
   // forward the csrf token from the session
@@ -34,7 +35,7 @@ export async function fetchSelf(ctx: any) {
     const db = context.getAppDB()
     // check for group permissions
     if (!user.roleId || user.roleId === PUBLIC_ROLE) {
-      const groupRoleId = await groups.getGroupRoleId(user, appId)
+      const groupRoleId = await groups.getGroupRoleId(user as User, appId)
       user.roleId = groupRoleId || user.roleId
     }
     // remove the full roles structure
@@ -42,7 +43,7 @@ export async function fetchSelf(ctx: any) {
     try {
       const userTable = await db.get(InternalTables.USER_METADATA)
       // specifically needs to make sure is enriched
-      ctx.body = await outputProcessing(userTable, user)
+      ctx.body = await outputProcessing(userTable, user as Row)
     } catch (err: any) {
       let response
       // user didn't exist in app, don't pretend they do