@budibase/backend-core 2.8.31-alpha.0 → 2.8.31

package/jest.config.ts

@@ -1,6 +1,8 @@
 import { Config } from "@jest/types"
+const preset = require("ts-jest/jest-preset")
 
 const baseConfig: Config.InitialProjectOptions = {
+  ...preset,
   preset: "@trendyol/jest-testcontainers",
   setupFiles: ["./tests/jestEnv.ts"],
   setupFilesAfterEnv: ["./tests/jestSetup.ts"],
@@ -9,7 +11,6 @@ const baseConfig: Config.InitialProjectOptions = {
   },
   moduleNameMapper: {
     "@budibase/types": "<rootDir>/../types/src",
-    "@budibase/shared-core": ["<rootDir>/../shared-core/src"],
   },
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.8.31-alpha.0",
+  "version": "2.8.31",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -16,15 +16,14 @@
     "prepack": "cp package.json dist",
     "build": "tsc -p tsconfig.build.json",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
-    "check:types": "tsc -p tsconfig.json --noEmit --paths null",
     "test": "bash scripts/test.sh",
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
     "@budibase/nano": "10.1.2",
     "@budibase/pouchdb-replication-stream": "1.2.10",
-    "@budibase/shared-core": "2.8.31-alpha.0",
-    "@budibase/types": "2.8.31-alpha.0",
+    "@budibase/types": "2.8.31",
+    "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",
     "aws-sdk": "2.1030.0",
@@ -52,20 +51,18 @@
     "pouchdb": "7.3.0",
     "pouchdb-find": "7.2.2",
     "redlock": "4.2.0",
-    "rotating-file-stream": "3.1.0",
     "sanitize-s3-objectkey": "0.0.1",
     "semver": "7.3.7",
     "tar-fs": "2.1.1",
     "uuid": "8.3.2"
   },
   "devDependencies": {
-    "@jest/test-sequencer": "29.6.2",
-    "@shopify/jest-koa-mocks": "5.1.1",
-    "@swc/core": "1.3.71",
-    "@swc/jest": "0.2.27",
+    "@jest/test-sequencer": "29.5.0",
+    "@swc/core": "^1.3.25",
+    "@swc/jest": "^0.2.24",
     "@trendyol/jest-testcontainers": "^2.1.1",
     "@types/chance": "1.1.3",
-    "@types/jest": "29.5.3",
+    "@types/jest": "29.5.0",
     "@types/koa": "2.13.4",
     "@types/lodash": "4.14.180",
     "@types/node": "14.18.20",
@@ -77,17 +74,32 @@
     "@types/uuid": "8.3.4",
     "chance": "1.1.8",
     "ioredis-mock": "8.7.0",
-    "jest": "29.6.2",
-    "jest-environment-node": "29.6.2",
-    "jest-serial-runner": "1.2.1",
+    "jest": "29.5.0",
+    "jest-environment-node": "29.5.0",
+    "jest-serial-runner": "^1.2.1",
     "koa": "2.13.4",
     "nodemon": "2.0.16",
     "pino-pretty": "10.0.0",
     "pouchdb-adapter-memory": "7.2.2",
     "timekeeper": "2.2.0",
+    "ts-jest": "29.0.5",
     "ts-node": "10.8.1",
     "tsconfig-paths": "4.0.0",
     "typescript": "4.7.3"
   },
-  "gitHead": "38e2c52eb34c0fae42da6e7f346996c379cd281f"
+  "nx": {
+    "targets": {
+      "build": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/types"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
+  },
+  "gitHead": "4613b17c1e76328bf279998c821a17195fbaab37"
 }

package/scripts/test.sh

@@ -8,6 +8,6 @@ then
   jest --coverage --runInBand --forceExit
 else
   # --maxWorkers performs better in development
-  echo "jest --coverage --detectOpenHandles"
-  jest --coverage --detectOpenHandles
+  echo "jest --coverage --forceExit"
+  jest --coverage --forceExit
 fi

package/src/auth/auth.ts

@@ -159,7 +159,7 @@ export async function updateUserOAuth(userId: string, oAuthConfig: any) {
 
   try {
     const db = getGlobalDB()
-    const dbUser = await db.get<any>(userId)
+    const dbUser = await db.get(userId)
 
     //Do not overwrite the refresh token if a valid one is not provided.
     if (typeof details.refreshToken !== "string") {

package/src/cache/appMetadata.ts

@@ -2,14 +2,9 @@ import { getAppClient } from "../redis/init"
 import { doWithDB, DocumentType } from "../db"
 import { Database, App } from "@budibase/types"
 
-export enum AppState {
-  INVALID = "invalid",
+const AppState = {
+  INVALID: "invalid",
 }
-
-export interface DeletedApp {
-  state: AppState
-}
-
 const EXPIRY_SECONDS = 3600
 
 /**
@@ -36,7 +31,7 @@ function isInvalid(metadata?: { state: string }) {
  * @param {string} appId the id of the app to get metadata from.
  * @returns {object} the app metadata.
  */
-export async function getAppMetadata(appId: string): Promise<App | DeletedApp> {
+export async function getAppMetadata(appId: string) {
   const client = await getAppClient()
   // try cache
   let metadata = await client.get(appId)
@@ -66,8 +61,11 @@ export async function getAppMetadata(appId: string): Promise<App | DeletedApp> {
     }
     await client.store(appId, metadata, expiry)
   }
-
-  return metadata
+  // we've stored in the cache an object to tell us that it is currently invalid
+  if (isInvalid(metadata)) {
+    throw { status: 404, message: "No app metadata found" }
+  }
+  return metadata as App
 }
 
 /**

package/src/cache/tests/writethrough.spec.ts

@@ -36,7 +36,7 @@ describe("writethrough", () => {
           _id: docId,
           value: 1,
         })
-        const output = await db.get<any>(response.id)
+        const output = await db.get(response.id)
         current = output
         expect(output.value).toBe(1)
       })
@@ -45,7 +45,7 @@ describe("writethrough", () => {
     it("second put shouldn't update DB", async () => {
       await config.doInTenant(async () => {
         const response = await writethrough.put({ ...current, value: 2 })
-        const output = await db.get<any>(response.id)
+        const output = await db.get(response.id)
         expect(current._rev).toBe(output._rev)
         expect(output.value).toBe(1)
       })
@@ -55,7 +55,7 @@ describe("writethrough", () => {
       await config.doInTenant(async () => {
         tk.freeze(Date.now() + DELAY + 1)
         const response = await writethrough.put({ ...current, value: 3 })
-        const output = await db.get<any>(response.id)
+        const output = await db.get(response.id)
         expect(response.rev).not.toBe(current._rev)
         expect(output.value).toBe(3)
 
@@ -79,7 +79,7 @@ describe("writethrough", () => {
           expect.arrayContaining([current._rev, current._rev, newRev])
         )
 
-        const output = await db.get<any>(current._id)
+        const output = await db.get(current._id)
         expect(output.value).toBe(4)
         expect(output._rev).toBe(newRev)
 
@@ -107,7 +107,7 @@ describe("writethrough", () => {
         })
         expect(res.ok).toBe(true)
 
-        const output = await db.get<any>(id)
+        const output = await db.get(id)
         expect(output.value).toBe(3)
         expect(output._rev).toBe(res.rev)
       })
@@ -130,8 +130,8 @@ describe("writethrough", () => {
         const resp2 = await writethrough2.put({ _id: "db1", value: "second" })
         expect(resp1.rev).toBeDefined()
         expect(resp2.rev).toBeDefined()
-        expect((await db.get<any>("db1")).value).toBe("first")
-        expect((await db2.get<any>("db1")).value).toBe("second")
+        expect((await db.get("db1")).value).toBe("first")
+        expect((await db2.get("db1")).value).toBe("second")
       })
     })
   })

package/src/cache/user.ts

@@ -12,7 +12,7 @@ const EXPIRY_SECONDS = 3600
  */
 async function populateFromDB(userId: string, tenantId: string) {
   const db = tenancy.getTenantDB(tenantId)
-  const user = await db.get<any>(userId)
+  const user = await db.get(userId)
   user.budibaseAccess = true
   if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
     const account = await accounts.getAccount(user.email)

package/src/constants/db.ts

@@ -1,5 +1,5 @@
-import { prefixed, DocumentType } from "@budibase/types"
-export { SEPARATOR, UNICODE_MAX, DocumentType } from "@budibase/types"
+export const SEPARATOR = "_"
+export const UNICODE_MAX = "\ufff0"
 
 /**
  * Can be used to create a few different forms of querying a view.
@@ -14,6 +14,8 @@ export enum ViewName {
   USER_BY_APP = "by_app",
   USER_BY_EMAIL = "by_email2",
   BY_API_KEY = "by_api_key",
+  /** @deprecated - could be deleted */
+  USER_BY_BUILDERS = "by_builders",
   LINK = "by_link",
   ROUTING = "screen_routes",
   AUTOMATION_LOGS = "automation_logs",
@@ -34,6 +36,42 @@ export enum InternalTable {
   USER_METADATA = "ta_users",
 }
 
+export enum DocumentType {
+  USER = "us",
+  GROUP = "gr",
+  WORKSPACE = "workspace",
+  CONFIG = "config",
+  TEMPLATE = "template",
+  APP = "app",
+  DEV = "dev",
+  APP_DEV = "app_dev",
+  APP_METADATA = "app_metadata",
+  ROLE = "role",
+  MIGRATIONS = "migrations",
+  DEV_INFO = "devinfo",
+  AUTOMATION_LOG = "log_au",
+  ACCOUNT_METADATA = "acc_metadata",
+  PLUGIN = "plg",
+  DATASOURCE = "datasource",
+  DATASOURCE_PLUS = "datasource_plus",
+  APP_BACKUP = "backup",
+  TABLE = "ta",
+  ROW = "ro",
+  AUTOMATION = "au",
+  LINK = "li",
+  WEBHOOK = "wh",
+  INSTANCE = "inst",
+  LAYOUT = "layout",
+  SCREEN = "screen",
+  QUERY = "query",
+  DEPLOYMENTS = "deployments",
+  METADATA = "metadata",
+  MEM_VIEW = "view",
+  USER_FLAG = "flag",
+  AUTOMATION_METADATA = "meta_au",
+  AUDIT_LOG = "al",
+}
+
 export const StaticDatabases = {
   GLOBAL: {
     name: "global-db",
@@ -57,7 +95,7 @@ export const StaticDatabases = {
   },
 }
 
-export const APP_PREFIX = prefixed(DocumentType.APP)
-export const APP_DEV = prefixed(DocumentType.APP_DEV)
+export const APP_PREFIX = DocumentType.APP + SEPARATOR
+export const APP_DEV = DocumentType.APP_DEV + SEPARATOR
 export const APP_DEV_PREFIX = APP_DEV
 export const BUDIBASE_DATASOURCE_TYPE = "budibase"

package/src/constants/misc.ts

@@ -20,8 +20,6 @@ export enum Header {
   TYPE = "x-budibase-type",
   PREVIEW_ROLE = "x-budibase-role",
   TENANT_ID = "x-budibase-tenant-id",
-  VERIFICATION_CODE = "x-budibase-verification-code",
-  RETURN_VERIFICATION_CODE = "x-budibase-return-verification-code",
   TOKEN = "x-budibase-token",
   CSRF_TOKEN = "x-csrf-token",
   CORRELATION_ID = "x-budibase-correlation-id",

package/src/db/couch/index.ts

@@ -2,4 +2,3 @@ export * from "./connections"
 export * from "./DatabaseImpl"
 export * from "./utils"
 export { init, getPouch, getPouchDB, closePouchDB } from "./pouchDB"
-export * from "../constants"

package/src/db/searchIndexes/searchIndexes.ts

@@ -5,7 +5,7 @@ export async function createUserIndex() {
   const db = getGlobalDB()
   let designDoc
   try {
-    designDoc = await db.get<any>("_design/database")
+    designDoc = await db.get("_design/database")
   } catch (err: any) {
     if (err.status === 404) {
       designDoc = { _id: "_design/database" }

package/src/db/utils.ts

@@ -2,7 +2,7 @@ import env from "../environment"
 import { DEFAULT_TENANT_ID, SEPARATOR, DocumentType } from "../constants"
 import { getTenantId, getGlobalDBName } from "../context"
 import { doWithDB, directCouchAllDbs } from "./db"
-import { AppState, DeletedApp, getAppMetadata } from "../cache/appMetadata"
+import { getAppMetadata } from "../cache/appMetadata"
 import { isDevApp, isDevAppID, getProdAppID } from "../docIds/conversions"
 import { App, Database } from "@budibase/types"
 import { getStartEndKeyURL } from "../docIds"
@@ -101,9 +101,7 @@ export async function getAllApps({
     const response = await Promise.allSettled(appPromises)
     const apps = response
       .filter(
-        (result: any) =>
-          result.status === "fulfilled" &&
-          result.value?.state !== AppState.INVALID
+        (result: any) => result.status === "fulfilled" && result.value != null
       )
       .map(({ value }: any) => value)
     if (!all) {
@@ -128,11 +126,7 @@ export async function getAppsByIDs(appIds: string[]) {
   )
   // have to list the apps which exist, some may have been deleted
   return settled
-    .filter(
-      promise =>
-        promise.status === "fulfilled" &&
-        (promise.value as DeletedApp).state !== AppState.INVALID
-    )
+    .filter(promise => promise.status === "fulfilled")
     .map(promise => (promise as PromiseFulfilledResult<App>).value)
 }
 

package/src/db/views.ts

@@ -105,6 +105,16 @@ export const createApiKeyView = async () => {
   await createView(db, viewJs, ViewName.BY_API_KEY)
 }
 
+export const createUserBuildersView = async () => {
+  const db = getGlobalDB()
+  const viewJs = `function(doc) {
+    if (doc.builder && doc.builder.global === true) {
+      emit(doc._id, doc._id)
+    }
+  }`
+  await createView(db, viewJs, ViewName.USER_BY_BUILDERS)
+}
+
 export interface QueryViewOptions {
   arrayResponse?: boolean
 }
@@ -213,6 +223,7 @@ export const queryPlatformView = async <T>(
 const CreateFuncByName: any = {
   [ViewName.USER_BY_EMAIL]: createNewUserEmailView,
   [ViewName.BY_API_KEY]: createApiKeyView,
+  [ViewName.USER_BY_BUILDERS]: createUserBuildersView,
   [ViewName.USER_BY_APP]: createUserAppView,
 }
 

package/src/environment.ts

@@ -1,5 +1,4 @@
 import { existsSync, readFileSync } from "fs"
-import { ServiceType } from "@budibase/types"
 
 function isTest() {
   return isCypress() || isJest()
@@ -48,10 +47,7 @@ function httpLogging() {
   return process.env.HTTP_LOGGING
 }
 
-function getPackageJsonFields(): {
-  VERSION: string
-  SERVICE_NAME: string
-} {
+function findVersion() {
   function findFileInAncestors(
     fileName: string,
     currentDir: string
@@ -73,31 +69,17 @@ function getPackageJsonFields(): {
   try {
     const packageJsonFile = findFileInAncestors("package.json", process.cwd())
     const content = readFileSync(packageJsonFile!, "utf-8")
-    const parsedContent = JSON.parse(content)
-    return {
-      VERSION: parsedContent.version,
-      SERVICE_NAME: parsedContent.name,
-    }
+    return JSON.parse(content).version
   } catch {
     // throwing an error here is confusing/causes backend-core to be hard to import
-    return { VERSION: "", SERVICE_NAME: "" }
+    return undefined
   }
 }
 
-function isWorker() {
-  return environment.SERVICE_TYPE === ServiceType.WORKER
-}
-
-function isApps() {
-  return environment.SERVICE_TYPE === ServiceType.APPS
-}
-
 const environment = {
   isTest,
   isJest,
   isDev,
-  isWorker,
-  isApps,
   isProd: () => {
     return !isDev()
   },
@@ -164,7 +146,6 @@ const environment = {
   SMTP_FROM_ADDRESS: process.env.SMTP_FROM_ADDRESS,
   DISABLE_JWT_WARNING: process.env.DISABLE_JWT_WARNING,
   BLACKLIST_IPS: process.env.BLACKLIST_IPS,
-  SERVICE_TYPE: "unknown",
   /**
    * Enable to allow an admin user to login using a password.
    * This can be useful to prevent lockout when configuring SSO.
@@ -173,7 +154,7 @@ const environment = {
   ENABLE_SSO_MAINTENANCE_MODE: selfHosted
     ? process.env.ENABLE_SSO_MAINTENANCE_MODE
     : false,
-  ...getPackageJsonFields(),
+  VERSION: findVersion(),
   DISABLE_PINO_LOGGER: process.env.DISABLE_PINO_LOGGER,
   OFFLINE_MODE: process.env.OFFLINE_MODE,
   _set(key: any, value: any) {
@@ -181,7 +162,6 @@ const environment = {
     // @ts-ignore
     environment[key] = value
   },
-  ROLLING_LOG_MAX_SIZE: process.env.ROLLING_LOG_MAX_SIZE || "10M",
 }
 
 // clean up any environment variable edge cases

package/src/events/identification.ts

@@ -21,7 +21,6 @@ import { processors } from "./processors"
 import { newid } from "../utils"
 import * as installation from "../installation"
 import * as configs from "../configs"
-import * as users from "../users"
 import { withCache, TTL, CacheKey } from "../cache/generic"
 
 /**
@@ -165,8 +164,8 @@ const identifyUser = async (
   const id = user._id as string
   const tenantId = await getEventTenantId(user.tenantId)
   const type = IdentityType.USER
-  let builder = users.hasBuilderPermissions(user)
-  let admin = users.hasAdminPermissions(user)
+  let builder = user.builder?.global || false
+  let admin = user.admin?.global || false
   let providerType
   if (isSSOUser(user)) {
     providerType = user.providerType

package/src/logging/index.ts

@@ -1,4 +1,6 @@
 export * as correlation from "./correlation/correlation"
 export { logger } from "./pino/logger"
 export * from "./alerts"
-export * as system from "./system"
+
+// turn off or on context logging i.e. tenantId, appId etc
+export let LOG_CONTEXT = true

package/src/logging/pino/logger.ts

@@ -1,60 +1,37 @@
-import pino, { LoggerOptions } from "pino"
-import pinoPretty from "pino-pretty"
-
-import { IdentityType } from "@budibase/types"
 import env from "../../environment"
+import pino, { LoggerOptions } from "pino"
 import * as context from "../../context"
 import * as correlation from "../correlation"
-
-import { localFileDestination } from "../system"
+import { IdentityType } from "@budibase/types"
+import { LOG_CONTEXT } from "../index"
 
 // LOGGER
 
 let pinoInstance: pino.Logger | undefined
 if (!env.DISABLE_PINO_LOGGER) {
-  const level = env.LOG_LEVEL
   const pinoOptions: LoggerOptions = {
-    level,
+    level: env.LOG_LEVEL,
     formatters: {
-      level: level => {
-        return { level: level.toUpperCase() }
+      level: label => {
+        return { level: label.toUpperCase() }
       },
       bindings: () => {
-        if (env.SELF_HOSTED) {
-          // "service" is being injected in datadog using the pod names,
-          // so we should leave it blank to allow the default behaviour if it's not running self-hosted
-          return {
-            service: env.SERVICE_NAME,
-          }
-        } else {
-          return {}
-        }
+        return {}
       },
     },
     timestamp: () => `,"timestamp":"${new Date(Date.now()).toISOString()}"`,
   }
 
-  const destinations: pino.StreamEntry[] = []
-
-  destinations.push(
-    env.isDev()
-      ? {
-          stream: pinoPretty({ singleLine: true }),
-          level: level as pino.Level,
-        }
-      : { stream: process.stdout, level: level as pino.Level }
-  )
-
-  if (env.SELF_HOSTED) {
-    destinations.push({
-      stream: localFileDestination(),
-      level: level as pino.Level,
-    })
+  if (env.isDev()) {
+    pinoOptions.transport = {
+      target: "pino-pretty",
+      options: {
+        singleLine: true,
+      },
+    }
   }
 
-  pinoInstance = destinations.length
-    ? pino(pinoOptions, pino.multistream(destinations))
-    : pino(pinoOptions)
+  pinoInstance = pino(pinoOptions)
 
   // CONSOLE OVERRIDES
 
@@ -106,13 +83,15 @@ if (!env.DISABLE_PINO_LOGGER) {
 
     let contextObject = {}
 
-    contextObject = {
-      tenantId: getTenantId(),
-      appId: getAppId(),
-      automationId: getAutomationId(),
-      identityId: identity?._id,
-      identityType: identity?.type,
-      correlationId: correlation.getId(),
+    if (LOG_CONTEXT) {
+      contextObject = {
+        tenantId: getTenantId(),
+        appId: getAppId(),
+        automationId: getAutomationId(),
+        identityId: identity?._id,
+        identityType: identity?.type,
+        correlationId: correlation.getId(),
+      }
     }
 
     const mergingObject: any = {

package/src/middleware/adminOnly.ts

@@ -1,8 +1,10 @@
-import { UserCtx } from "@budibase/types"
-import { isAdmin } from "../users"
+import { BBContext } from "@budibase/types"
 
-export default async (ctx: UserCtx, next: any) => {
-  if (!ctx.internal && !isAdmin(ctx.user)) {
+export default async (ctx: BBContext, next: any) => {
+  if (
+    !ctx.internal &&
+    (!ctx.user || !ctx.user.admin || !ctx.user.admin.global)
+  ) {
     ctx.throw(403, "Admin user only endpoint.")
   }
   return next()

package/src/middleware/builderOnly.ts

@@ -1,19 +1,10 @@
-import { UserCtx } from "@budibase/types"
-import { isBuilder, hasBuilderPermissions } from "../users"
-import { getAppId } from "../context"
-import env from "../environment"
+import { BBContext } from "@budibase/types"
 
-export default async (ctx: UserCtx, next: any) => {
-  const appId = getAppId()
-  const builderFn = env.isWorker()
-    ? hasBuilderPermissions
-    : env.isApps()
-    ? isBuilder
-    : undefined
-  if (!builderFn) {
-    throw new Error("Service name unknown - middleware inactive.")
-  }
-  if (!ctx.internal && !builderFn(ctx.user, appId)) {
+export default async (ctx: BBContext, next: any) => {
+  if (
+    !ctx.internal &&
+    (!ctx.user || !ctx.user.builder || !ctx.user.builder.global)
+  ) {
     ctx.throw(403, "Builder user only endpoint.")
   }
   return next()

package/src/middleware/builderOrAdmin.ts

@@ -1,20 +1,12 @@
-import { UserCtx } from "@budibase/types"
-import { isBuilder, isAdmin, hasBuilderPermissions } from "../users"
-import { getAppId } from "../context"
-import env from "../environment"
+import { BBContext } from "@budibase/types"
 
-export default async (ctx: UserCtx, next: any) => {
-  const appId = getAppId()
-  const builderFn = env.isWorker()
-    ? hasBuilderPermissions
-    : env.isApps()
-    ? isBuilder
-    : undefined
-  if (!builderFn) {
-    throw new Error("Service name unknown - middleware inactive.")
-  }
-  if (!ctx.internal && !builderFn(ctx.user, appId) && !isAdmin(ctx.user)) {
-    ctx.throw(403, "Admin/Builder user only endpoint.")
+export default async (ctx: BBContext, next: any) => {
+  if (
+    !ctx.internal &&
+    (!ctx.user || !ctx.user.builder || !ctx.user.builder.global) &&
+    (!ctx.user || !ctx.user.admin || !ctx.user.admin.global)
+  ) {
+    ctx.throw(403, "Builder user only endpoint.")
   }
   return next()
 }