@budibase/backend-core 2.8.21 → 2.8.22-alpha.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.8.21",
+  "version": "2.8.22-alpha.1",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -22,7 +22,7 @@
   "dependencies": {
     "@budibase/nano": "10.1.2",
     "@budibase/pouchdb-replication-stream": "1.2.10",
-    "@budibase/types": "2.8.21",
+    "@budibase/types": "2.8.22-alpha.1",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",
@@ -51,6 +51,7 @@
     "pouchdb": "7.3.0",
     "pouchdb-find": "7.2.2",
     "redlock": "4.2.0",
+    "rotating-file-stream": "3.1.0",
     "sanitize-s3-objectkey": "0.0.1",
     "semver": "7.3.7",
     "tar-fs": "2.1.1",
@@ -101,5 +102,5 @@
       }
     }
   },
-  "gitHead": "26c531ec81a0bdfb5805f2c8a85d6bbcc5ac7d81"
+  "gitHead": "efcef8d6f8ce0744d1f80f8161a41f55ef6c8ea4"
 }

package/src/auth/auth.ts

@@ -159,7 +159,7 @@ export async function updateUserOAuth(userId: string, oAuthConfig: any) {
 
   try {
     const db = getGlobalDB()
-    const dbUser = await db.get(userId)
+    const dbUser = await db.get<any>(userId)
 
     //Do not overwrite the refresh token if a valid one is not provided.
     if (typeof details.refreshToken !== "string") {

package/src/cache/appMetadata.ts

@@ -2,9 +2,14 @@ import { getAppClient } from "../redis/init"
 import { doWithDB, DocumentType } from "../db"
 import { Database, App } from "@budibase/types"
 
-const AppState = {
-  INVALID: "invalid",
+export enum AppState {
+  INVALID = "invalid",
 }
+
+export interface DeletedApp {
+  state: AppState
+}
+
 const EXPIRY_SECONDS = 3600
 
 /**
@@ -31,7 +36,7 @@ function isInvalid(metadata?: { state: string }) {
  * @param {string} appId the id of the app to get metadata from.
  * @returns {object} the app metadata.
  */
-export async function getAppMetadata(appId: string) {
+export async function getAppMetadata(appId: string): Promise<App | DeletedApp> {
   const client = await getAppClient()
   // try cache
   let metadata = await client.get(appId)
@@ -61,11 +66,8 @@ export async function getAppMetadata(appId: string) {
     }
     await client.store(appId, metadata, expiry)
   }
-  // we've stored in the cache an object to tell us that it is currently invalid
-  if (isInvalid(metadata)) {
-    throw { status: 404, message: "No app metadata found" }
-  }
-  return metadata as App
+
+  return metadata
 }
 
 /**

package/src/cache/user.ts

@@ -12,7 +12,7 @@ const EXPIRY_SECONDS = 3600
  */
 async function populateFromDB(userId: string, tenantId: string) {
   const db = tenancy.getTenantDB(tenantId)
-  const user = await db.get(userId)
+  const user = await db.get<any>(userId)
   user.budibaseAccess = true
   if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
     const account = await accounts.getAccount(user.email)

package/src/constants/misc.ts

@@ -20,6 +20,8 @@ export enum Header {
   TYPE = "x-budibase-type",
   PREVIEW_ROLE = "x-budibase-role",
   TENANT_ID = "x-budibase-tenant-id",
+  VERIFICATION_CODE = "x-budibase-verification-code",
+  RETURN_VERIFICATION_CODE = "x-budibase-return-verification-code",
   TOKEN = "x-budibase-token",
   CSRF_TOKEN = "x-csrf-token",
   CORRELATION_ID = "x-budibase-correlation-id",

package/src/db/searchIndexes/searchIndexes.ts

@@ -5,7 +5,7 @@ export async function createUserIndex() {
   const db = getGlobalDB()
   let designDoc
   try {
-    designDoc = await db.get("_design/database")
+    designDoc = await db.get<any>("_design/database")
   } catch (err: any) {
     if (err.status === 404) {
       designDoc = { _id: "_design/database" }

package/src/db/utils.ts

@@ -2,7 +2,7 @@ import env from "../environment"
 import { DEFAULT_TENANT_ID, SEPARATOR, DocumentType } from "../constants"
 import { getTenantId, getGlobalDBName } from "../context"
 import { doWithDB, directCouchAllDbs } from "./db"
-import { getAppMetadata } from "../cache/appMetadata"
+import { AppState, DeletedApp, getAppMetadata } from "../cache/appMetadata"
 import { isDevApp, isDevAppID, getProdAppID } from "../docIds/conversions"
 import { App, Database } from "@budibase/types"
 import { getStartEndKeyURL } from "../docIds"
@@ -101,7 +101,9 @@ export async function getAllApps({
     const response = await Promise.allSettled(appPromises)
     const apps = response
       .filter(
-        (result: any) => result.status === "fulfilled" && result.value != null
+        (result: any) =>
+          result.status === "fulfilled" &&
+          result.value?.state !== AppState.INVALID
       )
       .map(({ value }: any) => value)
     if (!all) {
@@ -126,7 +128,11 @@ export async function getAppsByIDs(appIds: string[]) {
   )
   // have to list the apps which exist, some may have been deleted
   return settled
-    .filter(promise => promise.status === "fulfilled")
+    .filter(
+      promise =>
+        promise.status === "fulfilled" &&
+        (promise.value as DeletedApp).state !== AppState.INVALID
+    )
     .map(promise => (promise as PromiseFulfilledResult<App>).value)
 }
 

package/src/environment.ts

@@ -47,7 +47,10 @@ function httpLogging() {
   return process.env.HTTP_LOGGING
 }
 
-function findVersion() {
+function getPackageJsonFields(): {
+  VERSION: string
+  SERVICE_NAME: string
+} {
   function findFileInAncestors(
     fileName: string,
     currentDir: string
@@ -69,10 +72,14 @@ function findVersion() {
   try {
     const packageJsonFile = findFileInAncestors("package.json", process.cwd())
     const content = readFileSync(packageJsonFile!, "utf-8")
-    return JSON.parse(content).version
+    const parsedContent = JSON.parse(content)
+    return {
+      VERSION: parsedContent.version,
+      SERVICE_NAME: parsedContent.name,
+    }
   } catch {
     // throwing an error here is confusing/causes backend-core to be hard to import
-    return undefined
+    return { VERSION: "", SERVICE_NAME: "" }
   }
 }
 
@@ -154,13 +161,14 @@ const environment = {
   ENABLE_SSO_MAINTENANCE_MODE: selfHosted
     ? process.env.ENABLE_SSO_MAINTENANCE_MODE
     : false,
-  VERSION: findVersion(),
+  ...getPackageJsonFields(),
   DISABLE_PINO_LOGGER: process.env.DISABLE_PINO_LOGGER,
   _set(key: any, value: any) {
     process.env[key] = value
     // @ts-ignore
     environment[key] = value
   },
+  ROLLING_LOG_MAX_SIZE: process.env.ROLLING_LOG_MAX_SIZE || "10M",
 }
 
 // clean up any environment variable edge cases

package/src/logging/index.ts

@@ -1,6 +1,4 @@
 export * as correlation from "./correlation/correlation"
 export { logger } from "./pino/logger"
 export * from "./alerts"
-
-// turn off or on context logging i.e. tenantId, appId etc
-export let LOG_CONTEXT = true
+export * as system from "./system"

package/src/logging/pino/logger.ts

@@ -1,37 +1,60 @@
-import env from "../../environment"
 import pino, { LoggerOptions } from "pino"
+import pinoPretty from "pino-pretty"
+
+import { IdentityType } from "@budibase/types"
+import env from "../../environment"
 import * as context from "../../context"
 import * as correlation from "../correlation"
-import { IdentityType } from "@budibase/types"
-import { LOG_CONTEXT } from "../index"
+
+import { localFileDestination } from "../system"
 
 // LOGGER
 
 let pinoInstance: pino.Logger | undefined
 if (!env.DISABLE_PINO_LOGGER) {
+  const level = env.LOG_LEVEL
   const pinoOptions: LoggerOptions = {
-    level: env.LOG_LEVEL,
+    level,
     formatters: {
-      level: label => {
-        return { level: label.toUpperCase() }
+      level: level => {
+        return { level: level.toUpperCase() }
       },
       bindings: () => {
-        return {}
+        if (env.SELF_HOSTED) {
+          // "service" is being injected in datadog using the pod names,
+          // so we should leave it blank to allow the default behaviour if it's not running self-hosted
+          return {
+            service: env.SERVICE_NAME,
+          }
+        } else {
+          return {}
+        }
       },
     },
     timestamp: () => `,"timestamp":"${new Date(Date.now()).toISOString()}"`,
   }
 
-  if (env.isDev()) {
-    pinoOptions.transport = {
-      target: "pino-pretty",
-      options: {
-        singleLine: true,
-      },
-    }
+  const destinations: pino.StreamEntry[] = []
+
+  destinations.push(
+    env.isDev()
+      ? {
+          stream: pinoPretty({ singleLine: true }),
+          level: level as pino.Level,
+        }
+      : { stream: process.stdout, level: level as pino.Level }
+  )
+
+  if (env.SELF_HOSTED) {
+    destinations.push({
+      stream: localFileDestination(),
+      level: level as pino.Level,
+    })
   }
 
-  pinoInstance = pino(pinoOptions)
+  pinoInstance = destinations.length
+    ? pino(pinoOptions, pino.multistream(destinations))
+    : pino(pinoOptions)
 
   // CONSOLE OVERRIDES
 
@@ -83,15 +106,13 @@ if (!env.DISABLE_PINO_LOGGER) {
 
     let contextObject = {}
 
-    if (LOG_CONTEXT) {
-      contextObject = {
-        tenantId: getTenantId(),
-        appId: getAppId(),
-        automationId: getAutomationId(),
-        identityId: identity?._id,
-        identityType: identity?.type,
-        correlationId: correlation.getId(),
-      }
+    contextObject = {
+      tenantId: getTenantId(),
+      appId: getAppId(),
+      automationId: getAutomationId(),
+      identityId: identity?._id,
+      identityType: identity?.type,
+      correlationId: correlation.getId(),
     }
 
     const mergingObject: any = {

package/src/logging/system.ts

@@ -0,0 +1,81 @@
+import fs from "fs"
+import path from "path"
+import * as rfs from "rotating-file-stream"
+
+import env from "../environment"
+import { budibaseTempDir } from "../objectStore"
+
+const logsFileName = `budibase.log`
+const budibaseLogsHistoryFileName = "budibase-logs-history.txt"
+
+const logsPath = path.join(budibaseTempDir(), "systemlogs")
+
+function getFullPath(fileName: string) {
+  return path.join(logsPath, fileName)
+}
+
+export function getSingleFileMaxSizeInfo(totalMaxSize: string) {
+  const regex = /(\d+)([A-Za-z])/
+  const match = totalMaxSize?.match(regex)
+  if (!match) {
+    console.warn(`totalMaxSize does not have a valid value`, {
+      totalMaxSize,
+    })
+    return undefined
+  }
+
+  const size = +match[1]
+  const unit = match[2]
+  if (size === 1) {
+    switch (unit) {
+      case "B":
+        return { size: `${size}B`, totalHistoryFiles: 1 }
+      case "K":
+        return { size: `${(size * 1000) / 2}B`, totalHistoryFiles: 1 }
+      case "M":
+        return { size: `${(size * 1000) / 2}K`, totalHistoryFiles: 1 }
+      case "G":
+        return { size: `${(size * 1000) / 2}M`, totalHistoryFiles: 1 }
+      default:
+        return undefined
+    }
+  }
+
+  if (size % 2 === 0) {
+    return { size: `${size / 2}${unit}`, totalHistoryFiles: 1 }
+  }
+
+  return { size: `1${unit}`, totalHistoryFiles: size - 1 }
+}
+
+export function localFileDestination() {
+  const fileInfo = getSingleFileMaxSizeInfo(env.ROLLING_LOG_MAX_SIZE)
+  const outFile = rfs.createStream(logsFileName, {
+    // As we have a rolling size, we want to half the max size
+    size: fileInfo?.size,
+    path: logsPath,
+    maxFiles: fileInfo?.totalHistoryFiles || 1,
+    immutable: true,
+    history: budibaseLogsHistoryFileName,
+    initialRotation: false,
+  })
+
+  return outFile
+}
+
+export function getLogReadStream() {
+  const streams = []
+  const historyFile = getFullPath(budibaseLogsHistoryFileName)
+  if (fs.existsSync(historyFile)) {
+    const fileContent = fs.readFileSync(historyFile, "utf-8")
+    const historyFiles = fileContent.split("\n")
+    for (const historyFile of historyFiles.filter(x => x)) {
+      streams.push(fs.readFileSync(historyFile))
+    }
+  }
+
+  streams.push(fs.readFileSync(getFullPath(logsFileName)))
+
+  const combinedContent = Buffer.concat(streams)
+  return combinedContent
+}

package/src/logging/tests/system.spec.ts

@@ -0,0 +1,61 @@
+import { getSingleFileMaxSizeInfo } from "../system"
+
+describe("system", () => {
+  describe("getSingleFileMaxSizeInfo", () => {
+    it.each([
+      ["100B", "50B"],
+      ["200K", "100K"],
+      ["20M", "10M"],
+      ["4G", "2G"],
+    ])(
+      "Halving even number (%s) returns halved size and 1 history file (%s)",
+      (totalValue, expectedMaxSize) => {
+        const result = getSingleFileMaxSizeInfo(totalValue)
+        expect(result).toEqual({
+          size: expectedMaxSize,
+          totalHistoryFiles: 1,
+        })
+      }
+    )
+
+    it.each([
+      ["5B", "1B", 4],
+      ["17K", "1K", 16],
+      ["21M", "1M", 20],
+      ["3G", "1G", 2],
+    ])(
+      "Halving an odd number (%s) returns as many files as size (-1) (%s)",
+      (totalValue, expectedMaxSize, totalHistoryFiles) => {
+        const result = getSingleFileMaxSizeInfo(totalValue)
+        expect(result).toEqual({
+          size: expectedMaxSize,
+          totalHistoryFiles,
+        })
+      }
+    )
+
+    it.each([
+      ["1B", "1B"],
+      ["1K", "500B"],
+      ["1M", "500K"],
+      ["1G", "500M"],
+    ])(
+      "Halving '%s' returns halved unit (%s)",
+      (totalValue, expectedMaxSize) => {
+        const result = getSingleFileMaxSizeInfo(totalValue)
+        expect(result).toEqual({
+          size: expectedMaxSize,
+          totalHistoryFiles: 1,
+        })
+      }
+    )
+
+    it.each([[undefined], [""], ["50"], ["wrongvalue"]])(
+      "Halving wrongly formatted value ('%s') returns undefined",
+      totalValue => {
+        const result = getSingleFileMaxSizeInfo(totalValue!)
+        expect(result).toBeUndefined()
+      }
+    )
+  })
+})

package/src/users.ts

@@ -67,9 +67,9 @@ export const bulkUpdateGlobalUsers = async (users: User[]) => {
 
 export async function getById(id: string, opts?: GetOpts): Promise<User> {
   const db = context.getGlobalDB()
-  let user = await db.get(id)
+  let user = await db.get<User>(id)
   if (opts?.cleanup) {
-    user = removeUserPassword(user)
+    user = removeUserPassword(user) as User
   }
   return user
 }