@budibase/backend-core 2.8.30 → 2.8.31-alpha.0

package/jest.config.ts

@@ -1,8 +1,6 @@
 import { Config } from "@jest/types"
-const preset = require("ts-jest/jest-preset")
 
 const baseConfig: Config.InitialProjectOptions = {
-  ...preset,
   preset: "@trendyol/jest-testcontainers",
   setupFiles: ["./tests/jestEnv.ts"],
   setupFilesAfterEnv: ["./tests/jestSetup.ts"],
@@ -11,6 +9,7 @@ const baseConfig: Config.InitialProjectOptions = {
   },
   moduleNameMapper: {
     "@budibase/types": "<rootDir>/../types/src",
+    "@budibase/shared-core": ["<rootDir>/../shared-core/src"],
   },
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.8.30",
+  "version": "2.8.31-alpha.0",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -16,14 +16,15 @@
     "prepack": "cp package.json dist",
     "build": "tsc -p tsconfig.build.json",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
+    "check:types": "tsc -p tsconfig.json --noEmit --paths null",
     "test": "bash scripts/test.sh",
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
     "@budibase/nano": "10.1.2",
     "@budibase/pouchdb-replication-stream": "1.2.10",
-    "@budibase/types": "2.8.30",
-    "@shopify/jest-koa-mocks": "5.0.1",
+    "@budibase/shared-core": "2.8.31-alpha.0",
+    "@budibase/types": "2.8.31-alpha.0",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",
     "aws-sdk": "2.1030.0",
@@ -51,18 +52,20 @@
     "pouchdb": "7.3.0",
     "pouchdb-find": "7.2.2",
     "redlock": "4.2.0",
+    "rotating-file-stream": "3.1.0",
     "sanitize-s3-objectkey": "0.0.1",
     "semver": "7.3.7",
     "tar-fs": "2.1.1",
     "uuid": "8.3.2"
   },
   "devDependencies": {
-    "@jest/test-sequencer": "29.5.0",
-    "@swc/core": "^1.3.25",
-    "@swc/jest": "^0.2.24",
+    "@jest/test-sequencer": "29.6.2",
+    "@shopify/jest-koa-mocks": "5.1.1",
+    "@swc/core": "1.3.71",
+    "@swc/jest": "0.2.27",
     "@trendyol/jest-testcontainers": "^2.1.1",
     "@types/chance": "1.1.3",
-    "@types/jest": "29.5.0",
+    "@types/jest": "29.5.3",
     "@types/koa": "2.13.4",
     "@types/lodash": "4.14.180",
     "@types/node": "14.18.20",
@@ -74,32 +77,17 @@
     "@types/uuid": "8.3.4",
     "chance": "1.1.8",
     "ioredis-mock": "8.7.0",
-    "jest": "29.5.0",
-    "jest-environment-node": "29.5.0",
-    "jest-serial-runner": "^1.2.1",
+    "jest": "29.6.2",
+    "jest-environment-node": "29.6.2",
+    "jest-serial-runner": "1.2.1",
     "koa": "2.13.4",
     "nodemon": "2.0.16",
     "pino-pretty": "10.0.0",
     "pouchdb-adapter-memory": "7.2.2",
     "timekeeper": "2.2.0",
-    "ts-jest": "29.0.5",
     "ts-node": "10.8.1",
     "tsconfig-paths": "4.0.0",
     "typescript": "4.7.3"
   },
-  "nx": {
-    "targets": {
-      "build": {
-        "dependsOn": [
-          {
-            "projects": [
-              "@budibase/types"
-            ],
-            "target": "build"
-          }
-        ]
-      }
-    }
-  },
-  "gitHead": "0bb17a32b1120c5b2526983870dbec4ee8cd6086"
+  "gitHead": "38e2c52eb34c0fae42da6e7f346996c379cd281f"
 }

package/scripts/test.sh

@@ -8,6 +8,6 @@ then
   jest --coverage --runInBand --forceExit
 else
   # --maxWorkers performs better in development
-  echo "jest --coverage --forceExit"
-  jest --coverage --forceExit
+  echo "jest --coverage --detectOpenHandles"
+  jest --coverage --detectOpenHandles
 fi

package/src/auth/auth.ts

@@ -159,7 +159,7 @@ export async function updateUserOAuth(userId: string, oAuthConfig: any) {
 
   try {
     const db = getGlobalDB()
-    const dbUser = await db.get(userId)
+    const dbUser = await db.get<any>(userId)
 
     //Do not overwrite the refresh token if a valid one is not provided.
     if (typeof details.refreshToken !== "string") {

package/src/cache/appMetadata.ts

@@ -2,9 +2,14 @@ import { getAppClient } from "../redis/init"
 import { doWithDB, DocumentType } from "../db"
 import { Database, App } from "@budibase/types"
 
-const AppState = {
-  INVALID: "invalid",
+export enum AppState {
+  INVALID = "invalid",
 }
+
+export interface DeletedApp {
+  state: AppState
+}
+
 const EXPIRY_SECONDS = 3600
 
 /**
@@ -31,7 +36,7 @@ function isInvalid(metadata?: { state: string }) {
  * @param {string} appId the id of the app to get metadata from.
  * @returns {object} the app metadata.
  */
-export async function getAppMetadata(appId: string) {
+export async function getAppMetadata(appId: string): Promise<App | DeletedApp> {
   const client = await getAppClient()
   // try cache
   let metadata = await client.get(appId)
@@ -61,11 +66,8 @@ export async function getAppMetadata(appId: string) {
     }
     await client.store(appId, metadata, expiry)
   }
-  // we've stored in the cache an object to tell us that it is currently invalid
-  if (isInvalid(metadata)) {
-    throw { status: 404, message: "No app metadata found" }
-  }
-  return metadata as App
+
+  return metadata
 }
 
 /**

package/src/cache/tests/writethrough.spec.ts

@@ -36,7 +36,7 @@ describe("writethrough", () => {
           _id: docId,
           value: 1,
         })
-        const output = await db.get(response.id)
+        const output = await db.get<any>(response.id)
         current = output
         expect(output.value).toBe(1)
       })
@@ -45,7 +45,7 @@ describe("writethrough", () => {
     it("second put shouldn't update DB", async () => {
       await config.doInTenant(async () => {
         const response = await writethrough.put({ ...current, value: 2 })
-        const output = await db.get(response.id)
+        const output = await db.get<any>(response.id)
         expect(current._rev).toBe(output._rev)
         expect(output.value).toBe(1)
       })
@@ -55,7 +55,7 @@ describe("writethrough", () => {
       await config.doInTenant(async () => {
         tk.freeze(Date.now() + DELAY + 1)
         const response = await writethrough.put({ ...current, value: 3 })
-        const output = await db.get(response.id)
+        const output = await db.get<any>(response.id)
         expect(response.rev).not.toBe(current._rev)
         expect(output.value).toBe(3)
 
@@ -79,7 +79,7 @@ describe("writethrough", () => {
           expect.arrayContaining([current._rev, current._rev, newRev])
         )
 
-        const output = await db.get(current._id)
+        const output = await db.get<any>(current._id)
         expect(output.value).toBe(4)
         expect(output._rev).toBe(newRev)
 
@@ -107,7 +107,7 @@ describe("writethrough", () => {
         })
         expect(res.ok).toBe(true)
 
-        const output = await db.get(id)
+        const output = await db.get<any>(id)
         expect(output.value).toBe(3)
         expect(output._rev).toBe(res.rev)
       })
@@ -130,8 +130,8 @@ describe("writethrough", () => {
         const resp2 = await writethrough2.put({ _id: "db1", value: "second" })
         expect(resp1.rev).toBeDefined()
         expect(resp2.rev).toBeDefined()
-        expect((await db.get("db1")).value).toBe("first")
-        expect((await db2.get("db1")).value).toBe("second")
+        expect((await db.get<any>("db1")).value).toBe("first")
+        expect((await db2.get<any>("db1")).value).toBe("second")
       })
     })
   })

package/src/cache/user.ts

@@ -12,7 +12,7 @@ const EXPIRY_SECONDS = 3600
  */
 async function populateFromDB(userId: string, tenantId: string) {
   const db = tenancy.getTenantDB(tenantId)
-  const user = await db.get(userId)
+  const user = await db.get<any>(userId)
   user.budibaseAccess = true
   if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
     const account = await accounts.getAccount(user.email)

package/src/constants/db.ts

@@ -1,5 +1,5 @@
-export const SEPARATOR = "_"
-export const UNICODE_MAX = "\ufff0"
+import { prefixed, DocumentType } from "@budibase/types"
+export { SEPARATOR, UNICODE_MAX, DocumentType } from "@budibase/types"
 
 /**
  * Can be used to create a few different forms of querying a view.
@@ -14,8 +14,6 @@ export enum ViewName {
   USER_BY_APP = "by_app",
   USER_BY_EMAIL = "by_email2",
   BY_API_KEY = "by_api_key",
-  /** @deprecated - could be deleted */
-  USER_BY_BUILDERS = "by_builders",
   LINK = "by_link",
   ROUTING = "screen_routes",
   AUTOMATION_LOGS = "automation_logs",
@@ -36,42 +34,6 @@ export enum InternalTable {
   USER_METADATA = "ta_users",
 }
 
-export enum DocumentType {
-  USER = "us",
-  GROUP = "gr",
-  WORKSPACE = "workspace",
-  CONFIG = "config",
-  TEMPLATE = "template",
-  APP = "app",
-  DEV = "dev",
-  APP_DEV = "app_dev",
-  APP_METADATA = "app_metadata",
-  ROLE = "role",
-  MIGRATIONS = "migrations",
-  DEV_INFO = "devinfo",
-  AUTOMATION_LOG = "log_au",
-  ACCOUNT_METADATA = "acc_metadata",
-  PLUGIN = "plg",
-  DATASOURCE = "datasource",
-  DATASOURCE_PLUS = "datasource_plus",
-  APP_BACKUP = "backup",
-  TABLE = "ta",
-  ROW = "ro",
-  AUTOMATION = "au",
-  LINK = "li",
-  WEBHOOK = "wh",
-  INSTANCE = "inst",
-  LAYOUT = "layout",
-  SCREEN = "screen",
-  QUERY = "query",
-  DEPLOYMENTS = "deployments",
-  METADATA = "metadata",
-  MEM_VIEW = "view",
-  USER_FLAG = "flag",
-  AUTOMATION_METADATA = "meta_au",
-  AUDIT_LOG = "al",
-}
-
 export const StaticDatabases = {
   GLOBAL: {
     name: "global-db",
@@ -95,7 +57,7 @@ export const StaticDatabases = {
   },
 }
 
-export const APP_PREFIX = DocumentType.APP + SEPARATOR
-export const APP_DEV = DocumentType.APP_DEV + SEPARATOR
+export const APP_PREFIX = prefixed(DocumentType.APP)
+export const APP_DEV = prefixed(DocumentType.APP_DEV)
 export const APP_DEV_PREFIX = APP_DEV
 export const BUDIBASE_DATASOURCE_TYPE = "budibase"

package/src/constants/misc.ts

@@ -20,6 +20,8 @@ export enum Header {
   TYPE = "x-budibase-type",
   PREVIEW_ROLE = "x-budibase-role",
   TENANT_ID = "x-budibase-tenant-id",
+  VERIFICATION_CODE = "x-budibase-verification-code",
+  RETURN_VERIFICATION_CODE = "x-budibase-return-verification-code",
   TOKEN = "x-budibase-token",
   CSRF_TOKEN = "x-csrf-token",
   CORRELATION_ID = "x-budibase-correlation-id",

package/src/db/constants.ts

@@ -0,0 +1,10 @@
+export const CONSTANT_INTERNAL_ROW_COLS = [
+  "_id",
+  "_rev",
+  "type",
+  "createdAt",
+  "updatedAt",
+  "tableId",
+] as const
+
+export const CONSTANT_EXTERNAL_ROW_COLS = ["_id", "_rev", "tableId"] as const

package/src/db/couch/index.ts

@@ -2,3 +2,4 @@ export * from "./connections"
 export * from "./DatabaseImpl"
 export * from "./utils"
 export { init, getPouch, getPouchDB, closePouchDB } from "./pouchDB"
+export * from "../constants"

package/src/db/searchIndexes/searchIndexes.ts

@@ -5,7 +5,7 @@ export async function createUserIndex() {
   const db = getGlobalDB()
   let designDoc
   try {
-    designDoc = await db.get("_design/database")
+    designDoc = await db.get<any>("_design/database")
   } catch (err: any) {
     if (err.status === 404) {
       designDoc = { _id: "_design/database" }

package/src/db/utils.ts

@@ -2,7 +2,7 @@ import env from "../environment"
 import { DEFAULT_TENANT_ID, SEPARATOR, DocumentType } from "../constants"
 import { getTenantId, getGlobalDBName } from "../context"
 import { doWithDB, directCouchAllDbs } from "./db"
-import { getAppMetadata } from "../cache/appMetadata"
+import { AppState, DeletedApp, getAppMetadata } from "../cache/appMetadata"
 import { isDevApp, isDevAppID, getProdAppID } from "../docIds/conversions"
 import { App, Database } from "@budibase/types"
 import { getStartEndKeyURL } from "../docIds"
@@ -101,7 +101,9 @@ export async function getAllApps({
     const response = await Promise.allSettled(appPromises)
     const apps = response
       .filter(
-        (result: any) => result.status === "fulfilled" && result.value != null
+        (result: any) =>
+          result.status === "fulfilled" &&
+          result.value?.state !== AppState.INVALID
       )
       .map(({ value }: any) => value)
     if (!all) {
@@ -126,7 +128,11 @@ export async function getAppsByIDs(appIds: string[]) {
   )
   // have to list the apps which exist, some may have been deleted
   return settled
-    .filter(promise => promise.status === "fulfilled")
+    .filter(
+      promise =>
+        promise.status === "fulfilled" &&
+        (promise.value as DeletedApp).state !== AppState.INVALID
+    )
     .map(promise => (promise as PromiseFulfilledResult<App>).value)
 }
 

package/src/db/views.ts

@@ -105,16 +105,6 @@ export const createApiKeyView = async () => {
   await createView(db, viewJs, ViewName.BY_API_KEY)
 }
 
-export const createUserBuildersView = async () => {
-  const db = getGlobalDB()
-  const viewJs = `function(doc) {
-    if (doc.builder && doc.builder.global === true) {
-      emit(doc._id, doc._id)
-    }
-  }`
-  await createView(db, viewJs, ViewName.USER_BY_BUILDERS)
-}
-
 export interface QueryViewOptions {
   arrayResponse?: boolean
 }
@@ -223,7 +213,6 @@ export const queryPlatformView = async <T>(
 const CreateFuncByName: any = {
   [ViewName.USER_BY_EMAIL]: createNewUserEmailView,
   [ViewName.BY_API_KEY]: createApiKeyView,
-  [ViewName.USER_BY_BUILDERS]: createUserBuildersView,
   [ViewName.USER_BY_APP]: createUserAppView,
 }
 

package/src/environment.ts

@@ -1,4 +1,5 @@
 import { existsSync, readFileSync } from "fs"
+import { ServiceType } from "@budibase/types"
 
 function isTest() {
   return isCypress() || isJest()
@@ -47,7 +48,10 @@ function httpLogging() {
   return process.env.HTTP_LOGGING
 }
 
-function findVersion() {
+function getPackageJsonFields(): {
+  VERSION: string
+  SERVICE_NAME: string
+} {
   function findFileInAncestors(
     fileName: string,
     currentDir: string
@@ -69,17 +73,31 @@ function findVersion() {
   try {
     const packageJsonFile = findFileInAncestors("package.json", process.cwd())
     const content = readFileSync(packageJsonFile!, "utf-8")
-    return JSON.parse(content).version
+    const parsedContent = JSON.parse(content)
+    return {
+      VERSION: parsedContent.version,
+      SERVICE_NAME: parsedContent.name,
+    }
   } catch {
     // throwing an error here is confusing/causes backend-core to be hard to import
-    return undefined
+    return { VERSION: "", SERVICE_NAME: "" }
   }
 }
 
+function isWorker() {
+  return environment.SERVICE_TYPE === ServiceType.WORKER
+}
+
+function isApps() {
+  return environment.SERVICE_TYPE === ServiceType.APPS
+}
+
 const environment = {
   isTest,
   isJest,
   isDev,
+  isWorker,
+  isApps,
   isProd: () => {
     return !isDev()
   },
@@ -146,6 +164,7 @@ const environment = {
   SMTP_FROM_ADDRESS: process.env.SMTP_FROM_ADDRESS,
   DISABLE_JWT_WARNING: process.env.DISABLE_JWT_WARNING,
   BLACKLIST_IPS: process.env.BLACKLIST_IPS,
+  SERVICE_TYPE: "unknown",
   /**
    * Enable to allow an admin user to login using a password.
    * This can be useful to prevent lockout when configuring SSO.
@@ -154,7 +173,7 @@ const environment = {
   ENABLE_SSO_MAINTENANCE_MODE: selfHosted
     ? process.env.ENABLE_SSO_MAINTENANCE_MODE
     : false,
-  VERSION: findVersion(),
+  ...getPackageJsonFields(),
   DISABLE_PINO_LOGGER: process.env.DISABLE_PINO_LOGGER,
   OFFLINE_MODE: process.env.OFFLINE_MODE,
   _set(key: any, value: any) {
@@ -162,6 +181,7 @@ const environment = {
     // @ts-ignore
     environment[key] = value
   },
+  ROLLING_LOG_MAX_SIZE: process.env.ROLLING_LOG_MAX_SIZE || "10M",
 }
 
 // clean up any environment variable edge cases

package/src/events/identification.ts

@@ -21,6 +21,7 @@ import { processors } from "./processors"
 import { newid } from "../utils"
 import * as installation from "../installation"
 import * as configs from "../configs"
+import * as users from "../users"
 import { withCache, TTL, CacheKey } from "../cache/generic"
 
 /**
@@ -164,8 +165,8 @@ const identifyUser = async (
   const id = user._id as string
   const tenantId = await getEventTenantId(user.tenantId)
   const type = IdentityType.USER
-  let builder = user.builder?.global || false
-  let admin = user.admin?.global || false
+  let builder = users.hasBuilderPermissions(user)
+  let admin = users.hasAdminPermissions(user)
   let providerType
   if (isSSOUser(user)) {
     providerType = user.providerType

package/src/logging/index.ts

@@ -1,6 +1,4 @@
 export * as correlation from "./correlation/correlation"
 export { logger } from "./pino/logger"
 export * from "./alerts"
-
-// turn off or on context logging i.e. tenantId, appId etc
-export let LOG_CONTEXT = true
+export * as system from "./system"

package/src/logging/pino/logger.ts

@@ -1,37 +1,60 @@
-import env from "../../environment"
 import pino, { LoggerOptions } from "pino"
+import pinoPretty from "pino-pretty"
+
+import { IdentityType } from "@budibase/types"
+import env from "../../environment"
 import * as context from "../../context"
 import * as correlation from "../correlation"
-import { IdentityType } from "@budibase/types"
-import { LOG_CONTEXT } from "../index"
+
+import { localFileDestination } from "../system"
 
 // LOGGER
 
 let pinoInstance: pino.Logger | undefined
 if (!env.DISABLE_PINO_LOGGER) {
+  const level = env.LOG_LEVEL
   const pinoOptions: LoggerOptions = {
-    level: env.LOG_LEVEL,
+    level,
     formatters: {
-      level: label => {
-        return { level: label.toUpperCase() }
+      level: level => {
+        return { level: level.toUpperCase() }
       },
       bindings: () => {
-        return {}
+        if (env.SELF_HOSTED) {
+          // "service" is being injected in datadog using the pod names,
+          // so we should leave it blank to allow the default behaviour if it's not running self-hosted
+          return {
+            service: env.SERVICE_NAME,
+          }
+        } else {
+          return {}
+        }
       },
     },
     timestamp: () => `,"timestamp":"${new Date(Date.now()).toISOString()}"`,
   }
 
-  if (env.isDev()) {
-    pinoOptions.transport = {
-      target: "pino-pretty",
-      options: {
-        singleLine: true,
-      },
-    }
+  const destinations: pino.StreamEntry[] = []
+
+  destinations.push(
+    env.isDev()
+      ? {
+          stream: pinoPretty({ singleLine: true }),
+          level: level as pino.Level,
+        }
+      : { stream: process.stdout, level: level as pino.Level }
+  )
+
+  if (env.SELF_HOSTED) {
+    destinations.push({
+      stream: localFileDestination(),
+      level: level as pino.Level,
+    })
   }
 
-  pinoInstance = pino(pinoOptions)
+  pinoInstance = destinations.length
+    ? pino(pinoOptions, pino.multistream(destinations))
+    : pino(pinoOptions)
 
   // CONSOLE OVERRIDES
 
@@ -83,15 +106,13 @@ if (!env.DISABLE_PINO_LOGGER) {
 
     let contextObject = {}
 
-    if (LOG_CONTEXT) {
-      contextObject = {
-        tenantId: getTenantId(),
-        appId: getAppId(),
-        automationId: getAutomationId(),
-        identityId: identity?._id,
-        identityType: identity?.type,
-        correlationId: correlation.getId(),
-      }
+    contextObject = {
+      tenantId: getTenantId(),
+      appId: getAppId(),
+      automationId: getAutomationId(),
+      identityId: identity?._id,
+      identityType: identity?.type,
+      correlationId: correlation.getId(),
     }
 
     const mergingObject: any = {