@budibase/server 0.0.32 → 0.0.999-alpha.30

package/src/integrations/utils.ts

@@ -0,0 +1,330 @@
+import { SourceName, SqlQuery, Datasource, Table } from "@budibase/types"
+import { DocumentType, SEPARATOR } from "../db/utils"
+import { FieldTypes, BuildSchemaErrors, InvalidColumns } from "../constants"
+
+const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
+const ROW_ID_REGEX = /^\[.*]$/g
+const ENCODED_SPACE = encodeURIComponent(" ")
+
+const SQL_NUMBER_TYPE_MAP = {
+  integer: FieldTypes.NUMBER,
+  int: FieldTypes.NUMBER,
+  decimal: FieldTypes.NUMBER,
+  smallint: FieldTypes.NUMBER,
+  real: FieldTypes.NUMBER,
+  float: FieldTypes.NUMBER,
+  numeric: FieldTypes.NUMBER,
+  mediumint: FieldTypes.NUMBER,
+  dec: FieldTypes.NUMBER,
+  double: FieldTypes.NUMBER,
+  fixed: FieldTypes.NUMBER,
+  "double precision": FieldTypes.NUMBER,
+  number: FieldTypes.NUMBER,
+  binary_float: FieldTypes.NUMBER,
+  binary_double: FieldTypes.NUMBER,
+  money: FieldTypes.NUMBER,
+  smallmoney: FieldTypes.NUMBER,
+}
+
+const SQL_DATE_TYPE_MAP = {
+  timestamp: FieldTypes.DATETIME,
+  time: FieldTypes.DATETIME,
+  datetime: FieldTypes.DATETIME,
+  smalldatetime: FieldTypes.DATETIME,
+  date: FieldTypes.DATETIME,
+}
+
+const SQL_DATE_ONLY_TYPES = ["date"]
+const SQL_TIME_ONLY_TYPES = ["time"]
+
+const SQL_STRING_TYPE_MAP = {
+  varchar: FieldTypes.STRING,
+  char: FieldTypes.STRING,
+  nchar: FieldTypes.STRING,
+  nvarchar: FieldTypes.STRING,
+  ntext: FieldTypes.STRING,
+  enum: FieldTypes.STRING,
+  blob: FieldTypes.STRING,
+  long: FieldTypes.STRING,
+  text: FieldTypes.STRING,
+  bigint: FieldTypes.STRING,
+}
+
+const SQL_BOOLEAN_TYPE_MAP = {
+  boolean: FieldTypes.BOOLEAN,
+  bit: FieldTypes.BOOLEAN,
+  tinyint: FieldTypes.BOOLEAN,
+}
+
+const SQL_MISC_TYPE_MAP = {
+  json: FieldTypes.JSON,
+}
+
+const SQL_TYPE_MAP = {
+  ...SQL_NUMBER_TYPE_MAP,
+  ...SQL_DATE_TYPE_MAP,
+  ...SQL_STRING_TYPE_MAP,
+  ...SQL_BOOLEAN_TYPE_MAP,
+  ...SQL_MISC_TYPE_MAP,
+}
+
+export enum SqlClient {
+  MS_SQL = "mssql",
+  POSTGRES = "pg",
+  MY_SQL = "mysql2",
+  ORACLE = "oracledb",
+}
+
+export function isExternalTable(tableId: string) {
+  return tableId.includes(DocumentType.DATASOURCE)
+}
+
+export function buildExternalTableId(datasourceId: string, tableName: string) {
+  // encode spaces
+  if (tableName.includes(" ")) {
+    tableName = encodeURIComponent(tableName)
+  }
+  return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}`
+}
+
+export function breakExternalTableId(tableId: string | undefined) {
+  if (!tableId) {
+    return {}
+  }
+  const parts = tableId.split(DOUBLE_SEPARATOR)
+  let datasourceId = parts.shift()
+  // if they need joined
+  let tableName = parts.join(DOUBLE_SEPARATOR)
+  // if contains encoded spaces, decode it
+  if (tableName.includes(ENCODED_SPACE)) {
+    tableName = decodeURIComponent(tableName)
+  }
+  return { datasourceId, tableName }
+}
+
+export function generateRowIdField(keyProps: any[] = []) {
+  if (!Array.isArray(keyProps)) {
+    keyProps = [keyProps]
+  }
+  // this conserves order and types
+  // we have to swap the double quotes to single quotes for use in HBS statements
+  // when using the literal helper the double quotes can break things
+  return encodeURIComponent(JSON.stringify(keyProps).replace(/"/g, "'"))
+}
+
+export function isRowId(field: any) {
+  return (
+    Array.isArray(field) ||
+    (typeof field === "string" && field.match(ROW_ID_REGEX) != null)
+  )
+}
+
+export function convertRowId(field: any) {
+  if (Array.isArray(field)) {
+    return field[0]
+  }
+  if (typeof field === "string" && field.match(ROW_ID_REGEX) != null) {
+    return field.substring(1, field.length - 1)
+  }
+  return field
+}
+
+// should always return an array
+export function breakRowIdField(_id: string | { _id: string }): any[] {
+  if (!_id) {
+    return []
+  }
+  // have to replace on the way back as we swapped out the double quotes
+  // when encoding, but JSON can't handle the single quotes
+  const id = typeof _id === "string" ? _id : _id._id
+  const decoded: string = decodeURIComponent(id).replace(/'/g, '"')
+  try {
+    const parsed = JSON.parse(decoded)
+    return Array.isArray(parsed) ? parsed : [parsed]
+  } catch (err) {
+    // wasn't json - likely was handlebars for a many to many
+    return [_id]
+  }
+}
+
+export function convertSqlType(type: string) {
+  let foundType = FieldTypes.STRING
+  const lcType = type.toLowerCase()
+  let matchingTypes = []
+  for (let [external, internal] of Object.entries(SQL_TYPE_MAP)) {
+    if (lcType.includes(external)) {
+      matchingTypes.push({ external, internal })
+    }
+  }
+  //Set the foundType based the longest match
+  if (matchingTypes.length > 0) {
+    foundType = matchingTypes.reduce((acc, val) => {
+      return acc.external.length >= val.external.length ? acc : val
+    }).internal
+  }
+  const schema: any = { type: foundType }
+  if (foundType === FieldTypes.DATETIME) {
+    schema.dateOnly = SQL_DATE_ONLY_TYPES.includes(lcType)
+    schema.timeOnly = SQL_TIME_ONLY_TYPES.includes(lcType)
+  }
+  return schema
+}
+
+export function getSqlQuery(query: SqlQuery | string): SqlQuery {
+  if (typeof query === "string") {
+    return { sql: query }
+  } else {
+    return query
+  }
+}
+
+export function isSQL(datasource: Datasource): boolean {
+  if (!datasource || !datasource.source) {
+    return false
+  }
+  const SQL = [
+    SourceName.POSTGRES,
+    SourceName.SQL_SERVER,
+    SourceName.MYSQL,
+    SourceName.ORACLE,
+  ]
+  return SQL.indexOf(datasource.source) !== -1
+}
+
+export function isIsoDateString(str: string) {
+  if (!/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/.test(str)) {
+    return false
+  }
+  let d = new Date(str)
+  return d.toISOString() === str
+}
+
+/**
+ * This function will determine whether a column is a relationship and whether it
+ * is currently valid. The reason for the validity check is that tables can be deleted
+ * outside of Budibase control and if this is the case it will break Budibase relationships.
+ * The tableIds is a list passed down from the main finalise tables function, which is
+ * based on the tables that have just been fetched. This will only really be used on subsequent
+ * fetches to the first one - if the user is periodically refreshing Budibase knowledge of tables.
+ * @param column The column to check, to see if it is a valid relationship.
+ * @param tableIds The IDs of the tables which currently exist.
+ */
+export function shouldCopyRelationship(
+  column: { type: string; tableId?: string },
+  tableIds: string[]
+) {
+  return (
+    column.type === FieldTypes.LINK &&
+    column.tableId &&
+    tableIds.includes(column.tableId)
+  )
+}
+
+/**
+ * Similar function to the shouldCopyRelationship function, but instead this looks for options and boolean
+ * types. It is possible to switch a string -> options and a number -> boolean (and vice versus) need to make
+ * sure that these get copied over when tables are fetched. Also checks whether they are still valid, if a
+ * column has changed type in the external database then copying it over may not be possible.
+ * @param column The column to check for options or boolean type.
+ * @param fetchedColumn The fetched column to check for the type in the external database.
+ */
+export function shouldCopySpecialColumn(
+  column: { type: string },
+  fetchedColumn: { type: string } | undefined
+) {
+  const isFormula = column.type === FieldTypes.FORMULA
+  const specialTypes = [
+    FieldTypes.OPTIONS,
+    FieldTypes.LONGFORM,
+    FieldTypes.ARRAY,
+    FieldTypes.FORMULA,
+  ]
+  // column has been deleted, remove - formulas will never exist, always copy
+  if (!isFormula && column && !fetchedColumn) {
+    return false
+  }
+  const fetchedIsNumber =
+    !fetchedColumn || fetchedColumn.type === FieldTypes.NUMBER
+  return (
+    specialTypes.indexOf(column.type as FieldTypes) !== -1 ||
+    (fetchedIsNumber && column.type === FieldTypes.BOOLEAN)
+  )
+}
+
+/**
+ * Looks for columns which need to be copied over into the new table definitions, like relationships
+ * and options types.
+ * @param tableName The name of the table which is being checked.
+ * @param table The specific table which is being checked.
+ * @param entities All the tables that existed before - the old table definitions.
+ * @param tableIds The IDs of the tables which exist now, to check if anything has been removed.
+ */
+function copyExistingPropsOver(
+  tableName: string,
+  table: Table,
+  entities: { [key: string]: any },
+  tableIds: [string]
+) {
+  if (entities && entities[tableName]) {
+    if (entities[tableName]?.primaryDisplay) {
+      table.primaryDisplay = entities[tableName].primaryDisplay
+    }
+    if (entities[tableName]?.created) {
+      table.created = entities[tableName]?.created
+    }
+    const existingTableSchema = entities[tableName].schema
+    for (let key in existingTableSchema) {
+      if (!existingTableSchema.hasOwnProperty(key)) {
+        continue
+      }
+      const column = existingTableSchema[key]
+      if (
+        shouldCopyRelationship(column, tableIds) ||
+        shouldCopySpecialColumn(column, table.schema[key])
+      ) {
+        table.schema[key] = existingTableSchema[key]
+      }
+    }
+  }
+  return table
+}
+
+/**
+ * Look through the final table definitions to see if anything needs to be
+ * copied over from the old and if any errors have occurred mark them so
+ * that the user can be made aware.
+ * @param tables The list of tables that have been retrieved from the external database.
+ * @param entities The old list of tables, if there was any to look for definitions in.
+ */
+export function finaliseExternalTables(
+  tables: { [key: string]: any },
+  entities: { [key: string]: any }
+) {
+  const invalidColumns = Object.values(InvalidColumns)
+  let finalTables: { [key: string]: any } = {}
+  const errors: { [key: string]: string } = {}
+  // @ts-ignore
+  const tableIds: [string] = Object.values(tables).map(table => table._id)
+  for (let [name, table] of Object.entries(tables)) {
+    const schemaFields = Object.keys(table.schema)
+    // make sure every table has a key
+    if (table.primary == null || table.primary.length === 0) {
+      errors[name] = BuildSchemaErrors.NO_KEY
+      continue
+    } else if (
+      schemaFields.find(field =>
+        invalidColumns.includes(field as InvalidColumns)
+      )
+    ) {
+      errors[name] = BuildSchemaErrors.INVALID_COLUMN
+      continue
+    }
+    // make sure all previous props have been added back
+    finalTables[name] = copyExistingPropsOver(name, table, entities, tableIds)
+  }
+  // sort the tables by name
+  finalTables = Object.entries(finalTables)
+    .sort(([a], [b]) => a.localeCompare(b))
+    .reduce((r, [k, v]) => ({ ...r, [k]: v }), {})
+  return { tables: finalTables, errors }
+}

package/src/middleware/appInfo.ts

@@ -0,0 +1,20 @@
+import { isDevAppID, isProdAppID } from "../db/utils"
+import { BBContext } from "@budibase/types"
+
+export enum AppType {
+  DEV = "dev",
+  PROD = "prod",
+}
+
+export function middleware({ appType }: { appType?: AppType } = {}) {
+  return (ctx: BBContext, next: any) => {
+    const appId = ctx.appId
+    if (appType === AppType.DEV && appId && !isDevAppID(appId)) {
+      ctx.throw(400, "Only apps in development support this endpoint")
+    }
+    if (appType === AppType.PROD && appId && !isProdAppID(appId)) {
+      ctx.throw(400, "Only apps in production support this endpoint")
+    }
+    return next()
+  }
+}

package/src/middleware/authorized.ts

@@ -0,0 +1,131 @@
+import { roles, permissions, auth, context } from "@budibase/backend-core"
+import { Role } from "@budibase/types"
+import builderMiddleware from "./builder"
+import { isWebhookEndpoint } from "./utils"
+
+function hasResource(ctx: any) {
+  return ctx.resourceId != null
+}
+
+const csrf = auth.buildCsrfMiddleware()
+
+/**
+ * Apply authorization to the requested resource:
+ * - If this is a builder resource the user must be a builder.
+ * - Builders can access all resources.
+ * - Otherwise the user must have the required role.
+ */
+const checkAuthorized = async (
+  ctx: any,
+  resourceRoles: any,
+  permType: any,
+  permLevel: any
+) => {
+  // check if this is a builder api and the user is not a builder
+  const isBuilder = ctx.user && ctx.user.builder && ctx.user.builder.global
+  const isBuilderApi = permType === permissions.PermissionType.BUILDER
+  if (isBuilderApi && !isBuilder) {
+    return ctx.throw(403, "Not Authorized")
+  }
+
+  // check for resource authorization
+  if (!isBuilder) {
+    await checkAuthorizedResource(ctx, resourceRoles, permType, permLevel)
+  }
+}
+
+const checkAuthorizedResource = async (
+  ctx: any,
+  resourceRoles: any,
+  permType: any,
+  permLevel: any
+) => {
+  // get the user's roles
+  const roleId = ctx.roleId || roles.BUILTIN_ROLE_IDS.PUBLIC
+  const userRoles = (await roles.getUserRoleHierarchy(roleId, {
+    idOnly: false,
+  })) as Role[]
+  const permError = "User does not have permission"
+  // check if the user has the required role
+  if (resourceRoles.length > 0) {
+    // deny access if the user doesn't have the required resource role
+    const found = userRoles.find(
+      (role: any) => resourceRoles.indexOf(role._id) !== -1
+    )
+    if (!found) {
+      ctx.throw(403, permError)
+    }
+    // fallback to the base permissions when no resource roles are found
+  } else if (
+    !permissions.doesHaveBasePermission(permType, permLevel, userRoles)
+  ) {
+    ctx.throw(403, permError)
+  }
+}
+
+export default (
+    permType: any,
+    permLevel: any = null,
+    opts = { schema: false }
+  ) =>
+  async (ctx: any, next: any) => {
+    // webhooks don't need authentication, each webhook unique
+    // also internal requests (between services) don't need authorized
+    if (isWebhookEndpoint(ctx) || ctx.internal) {
+      return next()
+    }
+
+    if (!ctx.user) {
+      return ctx.throw(403, "No user info found")
+    }
+
+    // get the resource roles
+    let resourceRoles: any = []
+    let otherLevelRoles: any = []
+    const otherLevel =
+      permLevel === permissions.PermissionLevel.READ
+        ? permissions.PermissionLevel.WRITE
+        : permissions.PermissionLevel.READ
+    const appId = context.getAppId()
+    if (appId && hasResource(ctx)) {
+      resourceRoles = await roles.getRequiredResourceRole(permLevel, ctx)
+      if (opts && opts.schema) {
+        otherLevelRoles = await roles.getRequiredResourceRole(otherLevel, ctx)
+      }
+    }
+
+    // if the resource is public, proceed
+    if (
+      resourceRoles.includes(roles.BUILTIN_ROLE_IDS.PUBLIC) ||
+      (otherLevelRoles &&
+        otherLevelRoles.includes(roles.BUILTIN_ROLE_IDS.PUBLIC))
+    ) {
+      return next()
+    }
+
+    // check authenticated
+    if (!ctx.isAuthenticated) {
+      return ctx.throw(403, "Session not authenticated")
+    }
+
+    // check general builder stuff, this middleware is a good way
+    // to find API endpoints which are builder focused
+    if (permType === permissions.PermissionType.BUILDER) {
+      await builderMiddleware(ctx)
+    }
+
+    try {
+      // check authorized
+      await checkAuthorized(ctx, resourceRoles, permType, permLevel)
+    } catch (err) {
+      // this is a schema, check if
+      if (opts && opts.schema && permLevel) {
+        await checkAuthorized(ctx, otherLevelRoles, permType, otherLevel)
+      } else {
+        throw err
+      }
+    }
+
+    // csrf protection
+    return csrf(ctx, next)
+  }

package/src/middleware/builder.ts

@@ -0,0 +1,97 @@
+import {
+  APP_DEV_PREFIX,
+  DocumentType,
+  getGlobalIDFromUserMetadataID,
+} from "../db/utils"
+import {
+  doesUserHaveLock,
+  updateLock,
+  checkDebounce,
+  setDebounce,
+} from "../utilities/redis"
+import { db as dbCore, cache, permissions } from "@budibase/backend-core"
+import { BBContext, Database } from "@budibase/types"
+
+const DEBOUNCE_TIME_SEC = 30
+
+/************************************************** *
+ * This middleware has been broken out of the       *
+ * "authorized" middleware as it had nothing to do  *
+ * with authorization, but requires the perms       *
+ * imparted by it. This middleware shouldn't        *
+ * be called directly, it should always be called   *
+ * through the authorized middleware                *
+ ****************************************************/
+
+async function checkDevAppLocks(ctx: BBContext) {
+  const appId = ctx.appId
+
+  // if any public usage, don't proceed
+  if (!ctx.user?._id && !ctx.user?.userId) {
+    return
+  }
+
+  // not a development app, don't need to do anything
+  if (!appId || !appId.startsWith(APP_DEV_PREFIX)) {
+    return
+  }
+  if (!(await doesUserHaveLock(appId, ctx.user))) {
+    ctx.throw(400, "User does not hold app lock.")
+  }
+
+  // they do have lock, update it
+  await updateLock(appId, ctx.user)
+}
+
+async function updateAppUpdatedAt(ctx: BBContext) {
+  const appId = ctx.appId
+  // if debouncing skip this update
+  // get methods also aren't updating
+  if (ctx.method === "GET" || (await checkDebounce(appId))) {
+    return
+  }
+  await dbCore.doWithDB(appId, async (db: Database) => {
+    const metadata = await db.get(DocumentType.APP_METADATA)
+    metadata.updatedAt = new Date().toISOString()
+
+    metadata.updatedBy = getGlobalIDFromUserMetadataID(ctx.user?.userId!)
+
+    const response = await db.put(metadata)
+    metadata._rev = response.rev
+    await cache.app.invalidateAppMetadata(appId, metadata)
+    // set a new debounce record with a short TTL
+    await setDebounce(appId, DEBOUNCE_TIME_SEC)
+  })
+}
+
+export default async function builder(ctx: BBContext) {
+  const appId = ctx.appId
+  // this only functions within an app context
+  if (!appId) {
+    return
+  }
+
+  // check authenticated
+  if (!ctx.isAuthenticated) {
+    return ctx.throw(403, "Session not authenticated")
+  }
+
+  const referer = ctx.headers["referer"]
+
+  const overviewPath = "/builder/portal/overview/"
+  const overviewContext = !referer ? false : referer.includes(overviewPath)
+  if (overviewContext) {
+    return
+  }
+
+  const hasAppId = !referer ? false : referer.includes(appId)
+  const editingApp = referer ? hasAppId : false
+  // check this is a builder call and editing
+  if (!editingApp) {
+    return
+  }
+  // check locks
+  await checkDevAppLocks(ctx)
+  // set updated at time on app
+  await updateAppUpdatedAt(ctx)
+}

package/src/middleware/currentapp.ts

@@ -0,0 +1,112 @@
+import {
+  utils,
+  constants,
+  roles,
+  tenancy,
+  context,
+} from "@budibase/backend-core"
+import { generateUserMetadataID, isDevAppID } from "../db/utils"
+import { getCachedSelf } from "../utilities/global"
+import env from "../environment"
+import { isWebhookEndpoint } from "./utils"
+import { UserCtx } from "@budibase/types"
+
+export default async (ctx: UserCtx, next: any) => {
+  // try to get the appID from the request
+  let requestAppId = await utils.getAppIdFromCtx(ctx)
+  if (!requestAppId) {
+    return next()
+  }
+
+  // deny access to application preview
+  if (!env.isTest()) {
+    if (
+      isDevAppID(requestAppId) &&
+      !isWebhookEndpoint(ctx) &&
+      (!ctx.user || !ctx.user.builder || !ctx.user.builder.global)
+    ) {
+      return ctx.redirect("/")
+    }
+  }
+
+  let appId: string | undefined,
+    roleId = roles.BUILTIN_ROLE_IDS.PUBLIC
+  if (!ctx.user?._id) {
+    // not logged in, try to set a cookie for public apps
+    appId = requestAppId
+  } else if (requestAppId != null) {
+    // Different App ID means cookie needs reset, or if the same public user has logged in
+    const globalUser = await getCachedSelf(ctx, requestAppId)
+    appId = requestAppId
+    // retrieving global user gets the right role
+    roleId = globalUser.roleId || roleId
+
+    // Allow builders to specify their role via a header
+    const isBuilder =
+      globalUser && globalUser.builder && globalUser.builder.global
+    const isDevApp = appId && isDevAppID(appId)
+    const roleHeader =
+      ctx.request &&
+      (ctx.request.headers[constants.Header.PREVIEW_ROLE] as string)
+    if (isBuilder && isDevApp && roleHeader) {
+      // Ensure the role is valid by ensuring a definition exists
+      try {
+        if (roleHeader) {
+          await roles.getRole(roleHeader)
+          roleId = roleHeader
+
+          // Delete admin and builder flags so that the specified role is honoured
+          delete ctx.user.builder
+          delete ctx.user.admin
+        }
+      } catch (error) {
+        // Swallow error and do nothing
+      }
+    }
+  }
+
+  // nothing more to do
+  if (!appId) {
+    return next()
+  }
+
+  return context.doInAppContext(appId, async () => {
+    let skipCookie = false
+    // if the user not in the right tenant then make sure they have no permissions
+    // need to judge this only based on the request app ID,
+    if (
+      env.MULTI_TENANCY &&
+      ctx.user?._id &&
+      requestAppId &&
+      !tenancy.isUserInAppTenant(requestAppId, ctx.user)
+    ) {
+      // don't error, simply remove the users rights (they are a public user)
+      delete ctx.user.builder
+      delete ctx.user.admin
+      delete ctx.user.roles
+      ctx.isAuthenticated = false
+      roleId = roles.BUILTIN_ROLE_IDS.PUBLIC
+      skipCookie = true
+    }
+
+    ctx.appId = appId
+    if (roleId) {
+      ctx.roleId = roleId
+      const globalId = ctx.user ? ctx.user._id : undefined
+      const userId = ctx.user
+        ? generateUserMetadataID(ctx.user._id!)
+        : undefined
+      ctx.user = {
+        ...ctx.user!,
+        // override userID with metadata one
+        _id: userId,
+        userId,
+        globalId,
+        roleId,
+        role: await roles.getRole(roleId),
+      }
+    }
+
+    return next()
+  })
+}