@budibase/backend-core 2.9.40-alpha.6 → 2.10.1

package/src/logging/correlation/middleware.ts

@@ -0,0 +1,17 @@
+import { Header } from "../../constants"
+import { v4 as uuid } from "uuid"
+const correlator = require("correlation-id")
+
+const correlation = (ctx: any, next: any) => {
+  // use the provided correlation id header if present
+  let correlationId = ctx.headers[Header.CORRELATION_ID]
+  if (!correlationId) {
+    correlationId = uuid()
+  }
+
+  return correlator.withId(correlationId, () => {
+    return next()
+  })
+}
+
+export default correlation

package/src/logging/index.ts

@@ -0,0 +1,4 @@
+export * as correlation from "./correlation/correlation"
+export { logger } from "./pino/logger"
+export * from "./alerts"
+export * as system from "./system"

package/src/logging/pino/logger.ts

@@ -0,0 +1,232 @@
+import pino, { LoggerOptions } from "pino"
+import pinoPretty from "pino-pretty"
+
+import { IdentityType } from "@budibase/types"
+import env from "../../environment"
+import * as context from "../../context"
+import * as correlation from "../correlation"
+
+import { localFileDestination } from "../system"
+
+// LOGGER
+
+let pinoInstance: pino.Logger | undefined
+if (!env.DISABLE_PINO_LOGGER) {
+  const level = env.LOG_LEVEL
+  const pinoOptions: LoggerOptions = {
+    level,
+    formatters: {
+      level: level => {
+        return { level: level.toUpperCase() }
+      },
+      bindings: () => {
+        if (env.SELF_HOSTED) {
+          // "service" is being injected in datadog using the pod names,
+          // so we should leave it blank to allow the default behaviour if it's not running self-hosted
+          return {
+            service: env.SERVICE_NAME,
+          }
+        } else {
+          return {}
+        }
+      },
+    },
+    timestamp: () => `,"timestamp":"${new Date(Date.now()).toISOString()}"`,
+  }
+
+  const destinations: pino.StreamEntry[] = []
+
+  destinations.push(
+    env.isDev()
+      ? {
+          stream: pinoPretty({ singleLine: true }),
+          level: level as pino.Level,
+        }
+      : { stream: process.stdout, level: level as pino.Level }
+  )
+
+  if (env.SELF_HOSTED) {
+    destinations.push({
+      stream: localFileDestination(),
+      level: level as pino.Level,
+    })
+  }
+
+  pinoInstance = destinations.length
+    ? pino(pinoOptions, pino.multistream(destinations))
+    : pino(pinoOptions)
+
+  // CONSOLE OVERRIDES
+
+  interface MergingObject {
+    objects?: any[]
+    tenantId?: string
+    appId?: string
+    automationId?: string
+    identityId?: string
+    identityType?: IdentityType
+    correlationId?: string
+    err?: Error
+  }
+
+  function isPlainObject(obj: any) {
+    return typeof obj === "object" && obj !== null && !(obj instanceof Error)
+  }
+
+  function isError(obj: any) {
+    return obj instanceof Error
+  }
+
+  function isMessage(obj: any) {
+    return typeof obj === "string"
+  }
+
+  /**
+   * Backwards compatibility between console logging statements
+   * and pino logging requirements.
+   */
+  function getLogParams(args: any[]): [MergingObject, string] {
+    let error = undefined
+    let objects: any[] = []
+    let message = ""
+
+    args.forEach(arg => {
+      if (isMessage(arg)) {
+        message = `${message} ${arg}`.trimStart()
+      }
+      if (isPlainObject(arg)) {
+        objects.push(arg)
+      }
+      if (isError(arg)) {
+        error = arg
+      }
+    })
+
+    const identity = getIdentity()
+
+    let contextObject = {}
+
+    contextObject = {
+      tenantId: getTenantId(),
+      appId: getAppId(),
+      automationId: getAutomationId(),
+      identityId: identity?._id,
+      identityType: identity?.type,
+      correlationId: correlation.getId(),
+    }
+
+    const mergingObject: any = {
+      err: error,
+      pid: process.pid,
+      ...contextObject,
+    }
+
+    if (objects.length) {
+      // init generic data object for params supplied that don't have a
+      // '_logKey' field. This prints an object using argument index as the key
+      // e.g. { 0: {}, 1: {} }
+      const data: any = {}
+      let dataIndex = 0
+
+      for (let i = 0; i < objects.length; i++) {
+        const object = objects[i]
+        // the object has specified a log key
+        // use this instead of generic key
+        const logKey = object._logKey
+        if (logKey) {
+          delete object._logKey
+          mergingObject[logKey] = object
+        } else {
+          data[dataIndex] = object
+          dataIndex++
+        }
+      }
+
+      if (Object.keys(data).length) {
+        mergingObject.data = data
+      }
+    }
+
+    return [mergingObject, message]
+  }
+
+  console.log = (...arg: any[]) => {
+    const [obj, msg] = getLogParams(arg)
+    pinoInstance?.info(obj, msg)
+  }
+  console.info = (...arg: any[]) => {
+    const [obj, msg] = getLogParams(arg)
+    pinoInstance?.info(obj, msg)
+  }
+  console.warn = (...arg: any[]) => {
+    const [obj, msg] = getLogParams(arg)
+    pinoInstance?.warn(obj, msg)
+  }
+  console.error = (...arg: any[]) => {
+    const [obj, msg] = getLogParams(arg)
+    pinoInstance?.error(obj, msg)
+  }
+
+  /**
+   * custom trace impl - this resembles the node trace behaviour rather
+   * than traditional trace logging
+   * @param arg
+   */
+  console.trace = (...arg: any[]) => {
+    const [obj, msg] = getLogParams(arg)
+    if (!obj.err) {
+      // to get stack trace
+      obj.err = new Error()
+    }
+    pinoInstance?.trace(obj, msg)
+  }
+
+  console.debug = (...arg: any) => {
+    const [obj, msg] = getLogParams(arg)
+    pinoInstance?.debug(obj, msg)
+  }
+
+  // CONTEXT
+
+  const getTenantId = () => {
+    let tenantId
+    try {
+      tenantId = context.getTenantId()
+    } catch (e: any) {
+      // do nothing
+    }
+    return tenantId
+  }
+
+  const getAppId = () => {
+    let appId
+    try {
+      appId = context.getAppId()
+    } catch (e) {
+      // do nothing
+    }
+    return appId
+  }
+
+  const getAutomationId = () => {
+    let appId
+    try {
+      appId = context.getAutomationId()
+    } catch (e) {
+      // do nothing
+    }
+    return appId
+  }
+
+  const getIdentity = () => {
+    let identity
+    try {
+      identity = context.getIdentity()
+    } catch (e) {
+      // do nothing
+    }
+    return identity
+  }
+}
+
+export const logger = pinoInstance

package/src/logging/pino/middleware.ts

@@ -0,0 +1,45 @@
+import env from "../../environment"
+import { logger } from "./logger"
+import { IncomingMessage } from "http"
+const pino = require("koa-pino-logger")
+import { Options } from "pino-http"
+import { Ctx } from "@budibase/types"
+const correlator = require("correlation-id")
+
+export function pinoSettings(): Options {
+  return {
+    logger,
+    genReqId: correlator.getId,
+    autoLogging: {
+      ignore: (req: IncomingMessage) => !!req.url?.includes("/health"),
+    },
+    serializers: {
+      req: req => {
+        return {
+          method: req.method,
+          url: req.url,
+          correlationId: req.id,
+        }
+      },
+      res: res => {
+        return {
+          status: res.statusCode,
+        }
+      },
+    },
+  }
+}
+
+function getMiddleware() {
+  if (env.HTTP_LOGGING) {
+    return pino(pinoSettings())
+  } else {
+    return (ctx: Ctx, next: any) => {
+      return next()
+    }
+  }
+}
+
+const pinoMiddleware = getMiddleware()
+
+export default pinoMiddleware

package/src/logging/system.ts

@@ -0,0 +1,81 @@
+import fs from "fs"
+import path from "path"
+import * as rfs from "rotating-file-stream"
+
+import env from "../environment"
+import { budibaseTempDir } from "../objectStore"
+
+const logsFileName = `budibase.log`
+const budibaseLogsHistoryFileName = "budibase-logs-history.txt"
+
+const logsPath = path.join(budibaseTempDir(), "systemlogs")
+
+function getFullPath(fileName: string) {
+  return path.join(logsPath, fileName)
+}
+
+export function getSingleFileMaxSizeInfo(totalMaxSize: string) {
+  const regex = /(\d+)([A-Za-z])/
+  const match = totalMaxSize?.match(regex)
+  if (!match) {
+    console.warn(`totalMaxSize does not have a valid value`, {
+      totalMaxSize,
+    })
+    return undefined
+  }
+
+  const size = +match[1]
+  const unit = match[2]
+  if (size === 1) {
+    switch (unit) {
+      case "B":
+        return { size: `${size}B`, totalHistoryFiles: 1 }
+      case "K":
+        return { size: `${(size * 1000) / 2}B`, totalHistoryFiles: 1 }
+      case "M":
+        return { size: `${(size * 1000) / 2}K`, totalHistoryFiles: 1 }
+      case "G":
+        return { size: `${(size * 1000) / 2}M`, totalHistoryFiles: 1 }
+      default:
+        return undefined
+    }
+  }
+
+  if (size % 2 === 0) {
+    return { size: `${size / 2}${unit}`, totalHistoryFiles: 1 }
+  }
+
+  return { size: `1${unit}`, totalHistoryFiles: size - 1 }
+}
+
+export function localFileDestination() {
+  const fileInfo = getSingleFileMaxSizeInfo(env.ROLLING_LOG_MAX_SIZE)
+  const outFile = rfs.createStream(logsFileName, {
+    // As we have a rolling size, we want to half the max size
+    size: fileInfo?.size,
+    path: logsPath,
+    maxFiles: fileInfo?.totalHistoryFiles || 1,
+    immutable: true,
+    history: budibaseLogsHistoryFileName,
+    initialRotation: false,
+  })
+
+  return outFile
+}
+
+export function getLogReadStream() {
+  const streams = []
+  const historyFile = getFullPath(budibaseLogsHistoryFileName)
+  if (fs.existsSync(historyFile)) {
+    const fileContent = fs.readFileSync(historyFile, "utf-8")
+    const historyFiles = fileContent.split("\n")
+    for (const historyFile of historyFiles.filter(x => x)) {
+      streams.push(fs.readFileSync(historyFile))
+    }
+  }
+
+  streams.push(fs.readFileSync(getFullPath(logsFileName)))
+
+  const combinedContent = Buffer.concat(streams)
+  return combinedContent
+}

package/src/logging/tests/system.spec.ts

@@ -0,0 +1,61 @@
+import { getSingleFileMaxSizeInfo } from "../system"
+
+describe("system", () => {
+  describe("getSingleFileMaxSizeInfo", () => {
+    it.each([
+      ["100B", "50B"],
+      ["200K", "100K"],
+      ["20M", "10M"],
+      ["4G", "2G"],
+    ])(
+      "Halving even number (%s) returns halved size and 1 history file (%s)",
+      (totalValue, expectedMaxSize) => {
+        const result = getSingleFileMaxSizeInfo(totalValue)
+        expect(result).toEqual({
+          size: expectedMaxSize,
+          totalHistoryFiles: 1,
+        })
+      }
+    )
+
+    it.each([
+      ["5B", "1B", 4],
+      ["17K", "1K", 16],
+      ["21M", "1M", 20],
+      ["3G", "1G", 2],
+    ])(
+      "Halving an odd number (%s) returns as many files as size (-1) (%s)",
+      (totalValue, expectedMaxSize, totalHistoryFiles) => {
+        const result = getSingleFileMaxSizeInfo(totalValue)
+        expect(result).toEqual({
+          size: expectedMaxSize,
+          totalHistoryFiles,
+        })
+      }
+    )
+
+    it.each([
+      ["1B", "1B"],
+      ["1K", "500B"],
+      ["1M", "500K"],
+      ["1G", "500M"],
+    ])(
+      "Halving '%s' returns halved unit (%s)",
+      (totalValue, expectedMaxSize) => {
+        const result = getSingleFileMaxSizeInfo(totalValue)
+        expect(result).toEqual({
+          size: expectedMaxSize,
+          totalHistoryFiles: 1,
+        })
+      }
+    )
+
+    it.each([[undefined], [""], ["50"], ["wrongvalue"]])(
+      "Halving wrongly formatted value ('%s') returns undefined",
+      totalValue => {
+        const result = getSingleFileMaxSizeInfo(totalValue!)
+        expect(result).toBeUndefined()
+      }
+    )
+  })
+})

package/src/middleware/adminOnly.ts

@@ -0,0 +1,9 @@
+import { UserCtx } from "@budibase/types"
+import { isAdmin } from "../users"
+
+export default async (ctx: UserCtx, next: any) => {
+  if (!ctx.internal && !isAdmin(ctx.user)) {
+    ctx.throw(403, "Admin user only endpoint.")
+  }
+  return next()
+}

package/src/middleware/auditLog.ts

@@ -0,0 +1,6 @@
+import { BBContext } from "@budibase/types"
+
+export default async (ctx: BBContext | any, next: any) => {
+  // Placeholder for audit log middleware
+  return next()
+}

package/src/middleware/authenticated.ts

@@ -0,0 +1,193 @@
+import { Cookie, Header } from "../constants"
+import {
+  getCookie,
+  clearCookie,
+  openJwt,
+  isValidInternalAPIKey,
+} from "../utils"
+import { getUser } from "../cache/user"
+import { getSession, updateSessionTTL } from "../security/sessions"
+import { buildMatcherRegex, matches } from "./matchers"
+import { SEPARATOR, queryGlobalView, ViewName } from "../db"
+import { getGlobalDB, doInTenant } from "../context"
+import { decrypt } from "../security/encryption"
+import * as identity from "../context/identity"
+import env from "../environment"
+import { Ctx, EndpointMatcher } from "@budibase/types"
+import { InvalidAPIKeyError, ErrorCode } from "../errors"
+
+const ONE_MINUTE = env.SESSION_UPDATE_PERIOD
+  ? parseInt(env.SESSION_UPDATE_PERIOD)
+  : 60 * 1000
+
+interface FinaliseOpts {
+  authenticated?: boolean
+  internal?: boolean
+  publicEndpoint?: boolean
+  version?: string
+  user?: any
+}
+
+function timeMinusOneMinute() {
+  return new Date(Date.now() - ONE_MINUTE).toISOString()
+}
+
+function finalise(ctx: any, opts: FinaliseOpts = {}) {
+  ctx.publicEndpoint = opts.publicEndpoint || false
+  ctx.isAuthenticated = opts.authenticated || false
+  ctx.user = opts.user
+  ctx.internal = opts.internal || false
+  ctx.version = opts.version
+}
+
+async function checkApiKey(apiKey: string, populateUser?: Function) {
+  // check both the primary and the fallback internal api keys
+  // this allows for rotation
+  if (isValidInternalAPIKey(apiKey)) {
+    return { valid: true, user: undefined }
+  }
+  const decrypted = decrypt(apiKey)
+  const tenantId = decrypted.split(SEPARATOR)[0]
+  return doInTenant(tenantId, async () => {
+    let userId
+    try {
+      const db = getGlobalDB()
+      // api key is encrypted in the database
+      userId = (await queryGlobalView(
+        ViewName.BY_API_KEY,
+        {
+          key: apiKey,
+        },
+        db
+      )) as string
+    } catch (err) {
+      userId = undefined
+    }
+    if (userId) {
+      return {
+        valid: true,
+        user: await getUser(userId, tenantId, populateUser),
+      }
+    } else {
+      throw new InvalidAPIKeyError()
+    }
+  })
+}
+
+/**
+ * This middleware is tenancy aware, so that it does not depend on other middlewares being used.
+ * The tenancy modules should not be used here and it should be assumed that the tenancy context
+ * has not yet been populated.
+ */
+export default function (
+  noAuthPatterns: EndpointMatcher[] = [],
+  opts: { publicAllowed?: boolean; populateUser?: Function } = {
+    publicAllowed: false,
+  }
+) {
+  const noAuthOptions = noAuthPatterns ? buildMatcherRegex(noAuthPatterns) : []
+  return async (ctx: Ctx | any, next: any) => {
+    let publicEndpoint = false
+    const version = ctx.request.headers[Header.API_VER]
+    // the path is not authenticated
+    const found = matches(ctx, noAuthOptions)
+    if (found) {
+      publicEndpoint = true
+    }
+    try {
+      // check the actual user is authenticated first, try header or cookie
+      let headerToken = ctx.request.headers[Header.TOKEN]
+
+      const authCookie = getCookie(ctx, Cookie.Auth) || openJwt(headerToken)
+      let apiKey = ctx.request.headers[Header.API_KEY]
+
+      if (!apiKey && ctx.request.headers[Header.AUTHORIZATION]) {
+        apiKey = ctx.request.headers[Header.AUTHORIZATION].split(" ")[1]
+      }
+
+      const tenantId = ctx.request.headers[Header.TENANT_ID]
+      let authenticated = false,
+        user = null,
+        internal = false
+      if (authCookie && !apiKey) {
+        const sessionId = authCookie.sessionId
+        const userId = authCookie.userId
+        let session
+        try {
+          // getting session handles error checking (if session exists etc)
+          session = await getSession(userId, sessionId)
+          if (opts && opts.populateUser) {
+            user = await getUser(
+              userId,
+              session.tenantId,
+              opts.populateUser(ctx)
+            )
+          } else {
+            user = await getUser(userId, session.tenantId)
+          }
+          user.csrfToken = session.csrfToken
+
+          if (session?.lastAccessedAt < timeMinusOneMinute()) {
+            // make sure we denote that the session is still in use
+            await updateSessionTTL(session)
+          }
+          authenticated = true
+        } catch (err: any) {
+          authenticated = false
+          console.error(`Auth Error: ${err.message}`)
+          console.error(err)
+          // remove the cookie as the user does not exist anymore
+          clearCookie(ctx, Cookie.Auth)
+        }
+      }
+      // this is an internal request, no user made it
+      if (!authenticated && apiKey) {
+        const populateUser = opts.populateUser ? opts.populateUser(ctx) : null
+        const { valid, user: foundUser } = await checkApiKey(
+          apiKey,
+          populateUser
+        )
+        if (valid && foundUser) {
+          authenticated = true
+          user = foundUser
+        } else if (valid) {
+          authenticated = true
+          internal = true
+        }
+      }
+      if (!user && tenantId) {
+        user = { tenantId }
+      } else if (user) {
+        delete user.password
+      }
+      // be explicit
+      if (!authenticated) {
+        authenticated = false
+      }
+      // isAuthenticated is a function, so use a variable to be able to check authed state
+      finalise(ctx, { authenticated, user, internal, version, publicEndpoint })
+
+      if (user && user.email) {
+        return identity.doInUserContext(user, ctx, next)
+      } else {
+        return next()
+      }
+    } catch (err: any) {
+      console.error(`Auth Error: ${err.message}`)
+      console.error(err)
+      // invalid token, clear the cookie
+      if (err?.name === "JsonWebTokenError") {
+        clearCookie(ctx, Cookie.Auth)
+      } else if (err?.code === ErrorCode.INVALID_API_KEY) {
+        ctx.throw(403, err.message)
+      }
+      // allow configuring for public access
+      if ((opts && opts.publicAllowed) || publicEndpoint) {
+        finalise(ctx, { authenticated: false, version, publicEndpoint })
+        return next()
+      } else {
+        ctx.throw(err.status || 403, err)
+      }
+    }
+  }
+}

package/src/middleware/builderOnly.ts

@@ -0,0 +1,21 @@
+import { UserCtx } from "@budibase/types"
+import { isBuilder, hasBuilderPermissions } from "../users"
+import { getAppId } from "../context"
+import env from "../environment"
+
+export default async (ctx: UserCtx, next: any) => {
+  const appId = getAppId()
+  const builderFn =
+    env.isWorker() || !appId
+      ? hasBuilderPermissions
+      : env.isApps()
+      ? isBuilder
+      : undefined
+  if (!builderFn) {
+    throw new Error("Service name unknown - middleware inactive.")
+  }
+  if (!ctx.internal && !builderFn(ctx.user, appId)) {
+    ctx.throw(403, "Builder user only endpoint.")
+  }
+  return next()
+}

package/src/middleware/builderOrAdmin.ts

@@ -0,0 +1,21 @@
+import { UserCtx } from "@budibase/types"
+import { isBuilder, isAdmin, hasBuilderPermissions } from "../users"
+import { getAppId } from "../context"
+import env from "../environment"
+
+export default async (ctx: UserCtx, next: any) => {
+  const appId = getAppId()
+  const builderFn =
+    env.isWorker() || !appId
+      ? hasBuilderPermissions
+      : env.isApps()
+      ? isBuilder
+      : undefined
+  if (!builderFn) {
+    throw new Error("Service name unknown - middleware inactive.")
+  }
+  if (!ctx.internal && !builderFn(ctx.user, appId) && !isAdmin(ctx.user)) {
+    ctx.throw(403, "Admin/Builder user only endpoint.")
+  }
+  return next()
+}