@budibase/backend-core 3.2.4 → 3.2.6

package/src/sql/sqlTable.ts

@@ -1,319 +0,0 @@
-import { Knex, knex } from "knex"
-import {
-  FieldType,
-  NumberFieldMetadata,
-  Operation,
-  QueryJson,
-  RelationshipType,
-  RenameColumn,
-  SqlQuery,
-  Table,
-  TableSourceType,
-  SqlClient,
-} from "@budibase/types"
-import { breakExternalTableId, getNativeSql } from "./utils"
-import { helpers, utils } from "@budibase/shared-core"
-import SchemaBuilder = Knex.SchemaBuilder
-import CreateTableBuilder = Knex.CreateTableBuilder
-
-function isIgnoredType(type: FieldType) {
-  const ignored = [FieldType.LINK, FieldType.FORMULA, FieldType.AI]
-  return ignored.indexOf(type) !== -1
-}
-
-function generateSchema(
-  schema: CreateTableBuilder,
-  table: Table,
-  tables: Record<string, Table>,
-  oldTable: null | Table = null,
-  renamed?: RenameColumn
-) {
-  let primaryKeys = table && table.primary ? table.primary : []
-  const columns = Object.values(table.schema)
-  // all columns in a junction table will be meta
-  let metaCols = columns.filter(col => (col as NumberFieldMetadata).meta)
-  let isJunction = metaCols.length === columns.length
-  let columnTypeSet: string[] = []
-
-  // can't change primary once its set for now
-  if (!oldTable) {
-    // junction tables are special - we have an expected format
-    if (isJunction) {
-      schema.primary(metaCols.map(col => col.name))
-    } else if (primaryKeys.length === 1) {
-      schema.increments(primaryKeys[0]).primary()
-      // note that we've set its type
-      columnTypeSet.push(primaryKeys[0])
-    } else {
-      schema.primary(primaryKeys)
-    }
-  }
-
-  // check if any columns need added
-  const foreignKeys = Object.values(table.schema).map(
-    col => (col as any).foreignKey
-  )
-  for (let [key, column] of Object.entries(table.schema)) {
-    // skip things that are already correct
-    const oldColumn = oldTable ? oldTable.schema[key] : null
-    if (
-      (oldColumn && oldColumn.type) ||
-      columnTypeSet.includes(key) ||
-      renamed?.updated === key
-    ) {
-      continue
-    }
-    const columnType = column.type
-    switch (columnType) {
-      case FieldType.STRING:
-      case FieldType.OPTIONS:
-      case FieldType.LONGFORM:
-      case FieldType.BARCODEQR:
-      case FieldType.BB_REFERENCE_SINGLE:
-        // primary key strings have to have a length in some DBs
-        if (primaryKeys.includes(key)) {
-          schema.string(key, 255)
-        } else {
-          schema.text(key)
-        }
-        break
-      case FieldType.NUMBER:
-        // if meta is specified then this is a junction table entry
-        if (column.meta && column.meta.toKey && column.meta.toTable) {
-          const { toKey, toTable } = column.meta
-          schema.integer(key).unsigned()
-          schema.foreign(key).references(`${toTable}.${toKey}`)
-        } else if (foreignKeys.indexOf(key) === -1) {
-          schema.float(key)
-        }
-        break
-      case FieldType.BIGINT:
-        schema.bigint(key)
-        break
-      case FieldType.BOOLEAN:
-        schema.boolean(key)
-        break
-      case FieldType.DATETIME:
-        if (!column.timeOnly) {
-          schema.datetime(key, {
-            useTz: !column.ignoreTimezones,
-          })
-        } else {
-          schema.time(key)
-        }
-        break
-      case FieldType.ARRAY:
-      case FieldType.BB_REFERENCE:
-        if (helpers.schema.isDeprecatedSingleUserColumn(column)) {
-          // This is still required for unit testing, in order to create "deprecated" schemas
-          schema.text(key)
-        } else {
-          schema.json(key)
-        }
-        break
-      case FieldType.LINK:
-        // this side of the relationship doesn't need any SQL work
-        if (
-          column.relationshipType !== RelationshipType.MANY_TO_ONE &&
-          column.relationshipType !== RelationshipType.MANY_TO_MANY
-        ) {
-          if (!column.foreignKey || !column.tableId) {
-            throw new Error("Invalid relationship schema")
-          }
-          const { tableName } = breakExternalTableId(column.tableId)
-          // @ts-ignore
-          const relatedTable = tables[tableName]
-          if (!relatedTable || !relatedTable.primary) {
-            throw new Error(
-              "Referenced table doesn't exist or has no primary keys"
-            )
-          }
-          const relatedPrimary = relatedTable.primary[0]
-          const externalType = relatedTable.schema[relatedPrimary].externalType
-          if (externalType) {
-            schema.specificType(column.foreignKey, externalType)
-          } else {
-            schema.integer(column.foreignKey).unsigned()
-          }
-
-          schema
-            .foreign(column.foreignKey)
-            .references(`${tableName}.${relatedPrimary}`)
-        }
-        break
-      case FieldType.FORMULA:
-        // This is allowed, but nothing to do on the external datasource
-        break
-      case FieldType.AI:
-        // This is allowed, but nothing to do on the external datasource
-        break
-      case FieldType.ATTACHMENTS:
-      case FieldType.ATTACHMENT_SINGLE:
-      case FieldType.SIGNATURE_SINGLE:
-      case FieldType.AUTO:
-      case FieldType.JSON:
-      case FieldType.INTERNAL:
-        throw `${column.type} is not a valid SQL type`
-
-      default:
-        utils.unreachable(columnType)
-    }
-  }
-
-  const oldType = renamed ? oldTable?.schema[renamed.old].type : undefined
-  if (renamed && oldType && !isIgnoredType(oldType)) {
-    schema.renameColumn(renamed.old, renamed.updated)
-  }
-
-  // need to check if any columns have been deleted
-  if (oldTable) {
-    const deletedColumns = Object.entries(oldTable.schema).filter(
-      ([key, column]) =>
-        !isIgnoredType(column.type) && table.schema[key] == null
-    )
-    deletedColumns.forEach(([key, column]) => {
-      if (renamed?.old === key || isIgnoredType(column.type)) {
-        return
-      }
-      if (oldTable.constrained && oldTable.constrained.indexOf(key) !== -1) {
-        schema.dropForeign(key)
-      }
-      schema.dropColumn(key)
-    })
-  }
-
-  return schema
-}
-
-function buildCreateTable(
-  knex: SchemaBuilder,
-  table: Table,
-  tables: Record<string, Table>
-): SchemaBuilder {
-  return knex.createTable(table.name, schema => {
-    generateSchema(schema, table, tables)
-  })
-}
-
-function buildUpdateTable(
-  knex: SchemaBuilder,
-  table: Table,
-  tables: Record<string, Table>,
-  oldTable: Table,
-  renamed: RenameColumn
-): SchemaBuilder {
-  return knex.alterTable(table.name, schema => {
-    generateSchema(schema, table, tables, oldTable, renamed)
-  })
-}
-
-function buildDeleteTable(knex: SchemaBuilder, table: Table): SchemaBuilder {
-  return knex.dropTable(table.name)
-}
-
-class SqlTableQueryBuilder {
-  private readonly sqlClient: SqlClient
-  private extendedSqlClient: SqlClient | undefined
-
-  // pass through client to get flavour of SQL
-  constructor(client: SqlClient) {
-    this.sqlClient = client
-  }
-
-  getBaseSqlClient(): SqlClient {
-    return this.sqlClient
-  }
-
-  getSqlClient(): SqlClient {
-    return this.extendedSqlClient || this.sqlClient
-  }
-
-  // if working in a database like MySQL with many variants (MariaDB)
-  // we can set another client which overrides the base one
-  setExtendedSqlClient(client: SqlClient) {
-    this.extendedSqlClient = client
-  }
-
-  /**
-   * @param json the input JSON structure from which an SQL query will be built.
-   * @return the operation that was found in the JSON.
-   */
-  _operation(json: QueryJson): Operation {
-    return json.endpoint.operation
-  }
-
-  _tableQuery(json: QueryJson): SqlQuery | SqlQuery[] {
-    let client = knex({ client: this.sqlClient }).schema
-    let schemaName = json?.endpoint?.schema
-    if (schemaName) {
-      client = client.withSchema(schemaName)
-    }
-
-    let query: Knex.SchemaBuilder
-    if (!json.table || !json.meta || !json.meta.tables) {
-      throw new Error("Cannot execute without table being specified")
-    }
-    if (json.table.sourceType === TableSourceType.INTERNAL) {
-      throw new Error("Cannot perform table actions for SQS.")
-    }
-
-    switch (this._operation(json)) {
-      case Operation.CREATE_TABLE:
-        query = buildCreateTable(client, json.table, json.meta.tables)
-        break
-      case Operation.UPDATE_TABLE:
-        if (!json.meta || !json.meta.table) {
-          throw new Error("Must specify old table for update")
-        }
-        // renameColumn does not work for MySQL, so return a raw query
-        if (this.sqlClient === SqlClient.MY_SQL && json.meta.renamed) {
-          const updatedColumn = json.meta.renamed.updated
-          const tableName = schemaName
-            ? `\`${schemaName}\`.\`${json.table.name}\``
-            : `\`${json.table.name}\``
-          return {
-            sql: `alter table ${tableName} rename column \`${json.meta.renamed.old}\` to \`${updatedColumn}\`;`,
-            bindings: [],
-          }
-        }
-
-        query = buildUpdateTable(
-          client,
-          json.table,
-          json.meta.tables,
-          json.meta.table,
-          json.meta.renamed!
-        )
-
-        // renameColumn for SQL Server returns a parameterised `sp_rename` query,
-        // which is not supported by SQL Server and gives a syntax error.
-        if (this.sqlClient === SqlClient.MS_SQL && json.meta.renamed) {
-          const oldColumn = json.meta.renamed.old
-          const updatedColumn = json.meta.renamed.updated
-          const tableName = schemaName
-            ? `${schemaName}.${json.table.name}`
-            : `${json.table.name}`
-          const sql = getNativeSql(query)
-          if (Array.isArray(sql)) {
-            for (const query of sql) {
-              if (query.sql.startsWith("exec sp_rename")) {
-                query.sql = `exec sp_rename '${tableName}.${oldColumn}', '${updatedColumn}', 'COLUMN'`
-                query.bindings = []
-              }
-            }
-          }
-
-          return sql
-        }
-        break
-      case Operation.DELETE_TABLE:
-        query = buildDeleteTable(client, json.table)
-        break
-      default:
-        throw new Error("Table operation is of unknown type")
-    }
-    return getNativeSql(query)
-  }
-}
-
-export default SqlTableQueryBuilder

package/src/sql/utils.ts

@@ -1,193 +0,0 @@
-import {
-  DocumentType,
-  ManyToManyRelationshipJson,
-  RelationshipsJson,
-  SqlQuery,
-  Table,
-  TableSourceType,
-} from "@budibase/types"
-import { DEFAULT_BB_DATASOURCE_ID } from "../constants"
-import { Knex } from "knex"
-import { SEPARATOR } from "../db"
-import environment from "../environment"
-
-const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
-const ROW_ID_REGEX = /^\[.*]$/g
-const ENCODED_SPACE = encodeURIComponent(" ")
-const ISO_DATE_REGEX = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/
-const TIME_REGEX = /^(?:\d{2}:)?(?:\d{2}:)(?:\d{2})$/
-
-export function isExternalTableID(tableId: string) {
-  return tableId.startsWith(DocumentType.DATASOURCE + SEPARATOR)
-}
-
-export function isInternalTableID(tableId: string) {
-  return !isExternalTableID(tableId)
-}
-
-export function getNativeSql(
-  query: Knex.SchemaBuilder | Knex.QueryBuilder
-): SqlQuery | SqlQuery[] {
-  let sql = query.toSQL()
-
-  if (Array.isArray(sql)) {
-    return sql as SqlQuery[]
-  }
-  let native: Knex.SqlNative | undefined
-  if (sql.toNative) {
-    native = sql.toNative()
-  }
-  return {
-    sql: native?.sql || sql.sql,
-    bindings: native?.bindings || sql.bindings,
-  } as SqlQuery
-}
-
-export function isExternalTable(table: Table) {
-  if (
-    table?.sourceId &&
-    table.sourceId.includes(DocumentType.DATASOURCE + SEPARATOR) &&
-    table?.sourceId !== DEFAULT_BB_DATASOURCE_ID
-  ) {
-    return true
-  } else if (table?.sourceType === TableSourceType.EXTERNAL) {
-    return true
-  } else if (table?._id && isExternalTableID(table._id)) {
-    return true
-  }
-  return false
-}
-
-export function buildExternalTableId(datasourceId: string, tableName: string) {
-  // encode spaces
-  if (tableName.includes(" ")) {
-    tableName = encodeURIComponent(tableName)
-  }
-  return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}`
-}
-
-export function breakExternalTableId(tableId: string) {
-  const parts = tableId.split(DOUBLE_SEPARATOR)
-  let datasourceId = parts.shift()
-  // if they need joined
-  let tableName = parts.join(DOUBLE_SEPARATOR)
-  // if contains encoded spaces, decode it
-  if (tableName.includes(ENCODED_SPACE)) {
-    tableName = decodeURIComponent(tableName)
-  }
-  if (!datasourceId || !tableName) {
-    throw new Error("Unable to get datasource/table name from table ID")
-  }
-  return { datasourceId, tableName }
-}
-
-export function generateRowIdField(keyProps: any[] = []) {
-  if (!Array.isArray(keyProps)) {
-    keyProps = [keyProps]
-  }
-  for (let index in keyProps) {
-    if (keyProps[index] instanceof Buffer) {
-      keyProps[index] = keyProps[index].toString()
-    }
-  }
-  // this conserves order and types
-  // we have to swap the double quotes to single quotes for use in HBS statements
-  // when using the literal helper the double quotes can break things
-  return encodeURIComponent(JSON.stringify(keyProps).replace(/"/g, "'"))
-}
-
-export function isRowId(field: any) {
-  return (
-    Array.isArray(field) ||
-    (typeof field === "string" && field.match(ROW_ID_REGEX) != null)
-  )
-}
-
-export function convertRowId(field: any) {
-  if (Array.isArray(field)) {
-    return field[0]
-  }
-  if (typeof field === "string" && field.match(ROW_ID_REGEX) != null) {
-    return field.substring(1, field.length - 1)
-  }
-  return field
-}
-
-// should always return an array
-export function breakRowIdField(_id: string | { _id: string }): any[] {
-  if (!_id) {
-    return []
-  }
-  // have to replace on the way back as we swapped out the double quotes
-  // when encoding, but JSON can't handle the single quotes
-  const id = typeof _id === "string" ? _id : _id._id
-  const decoded: string = decodeURIComponent(id).replace(/'/g, '"')
-  try {
-    const parsed = JSON.parse(decoded)
-    return Array.isArray(parsed) ? parsed : [parsed]
-  } catch (err) {
-    // wasn't json - likely was handlebars for a many to many
-    return [_id]
-  }
-}
-
-export function isInvalidISODateString(str: string) {
-  const trimmedValue = str.trim()
-  if (!ISO_DATE_REGEX.test(trimmedValue)) {
-    return false
-  }
-  let d = new Date(trimmedValue)
-  return isNaN(d.getTime())
-}
-
-export function isValidISODateString(str: string) {
-  const trimmedValue = str.trim()
-  if (!ISO_DATE_REGEX.test(trimmedValue)) {
-    return false
-  }
-  let d = new Date(trimmedValue)
-  if (isNaN(d.getTime())) {
-    return false
-  }
-  return d.toISOString() === trimmedValue
-}
-
-export function isValidFilter(value: any) {
-  return value != null && value !== ""
-}
-
-export function isValidTime(value: string) {
-  return TIME_REGEX.test(value)
-}
-
-export function sqlLog(client: string, query: string, values?: any[]) {
-  if (!environment.SQL_LOGGING_ENABLE) {
-    return
-  }
-  let string = `[SQL] [${client.toUpperCase()}] query="${query}"`
-  if (values) {
-    string += ` values="${values.join(", ")}"`
-  }
-  console.log(string)
-}
-
-function isValidManyToManyRelationship(
-  relationship: RelationshipsJson
-): relationship is ManyToManyRelationshipJson {
-  return (
-    !!relationship.through &&
-    !!relationship.fromPrimary &&
-    !!relationship.from &&
-    !!relationship.toPrimary &&
-    !!relationship.to
-  )
-}
-
-export function validateManyToMany(
-  relationship: RelationshipsJson
-): ManyToManyRelationshipJson | undefined {
-  if (isValidManyToManyRelationship(relationship)) {
-    return relationship
-  }
-  return undefined
-}

package/src/tenancy/db.ts

@@ -1,6 +0,0 @@
-import { getDB } from "../db/db"
-import { getGlobalDBName } from "../context"
-
-export function getTenantDB(tenantId: string) {
-  return getDB(getGlobalDBName(tenantId))
-}

package/src/tenancy/index.ts

@@ -1,2 +0,0 @@
-export * from "./db"
-export * from "./tenancy"

package/src/tenancy/tenancy.ts

@@ -1,148 +0,0 @@
-import {
-  DEFAULT_TENANT_ID,
-  getTenantId,
-  getTenantIDFromAppID,
-  isMultiTenant,
-  getPlatformURL,
-} from "../context"
-import {
-  BBContext,
-  TenantResolutionStrategy,
-  GetTenantIdOptions,
-} from "@budibase/types"
-import { Header } from "../constants"
-
-export function addTenantToUrl(url: string) {
-  const tenantId = getTenantId()
-
-  if (isMultiTenant()) {
-    const char = url.indexOf("?") === -1 ? "?" : "&"
-    url += `${char}tenantId=${tenantId}`
-  }
-
-  return url
-}
-
-export const isUserInAppTenant = (appId: string, user?: any) => {
-  let userTenantId
-  if (user) {
-    userTenantId = user.tenantId || DEFAULT_TENANT_ID
-  } else {
-    userTenantId = getTenantId()
-  }
-  const tenantId = getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID
-  return tenantId === userTenantId
-}
-
-const ALL_STRATEGIES = Object.values(TenantResolutionStrategy)
-
-export const getTenantIDFromCtx = (
-  ctx: BBContext,
-  opts: GetTenantIdOptions
-): string | undefined => {
-  // exit early if not multi-tenant
-  if (!isMultiTenant()) {
-    return DEFAULT_TENANT_ID
-  }
-
-  // opt defaults
-  if (opts.allowNoTenant === undefined) {
-    opts.allowNoTenant = false
-  }
-  if (!opts.includeStrategies) {
-    opts.includeStrategies = ALL_STRATEGIES
-  }
-  if (!opts.excludeStrategies) {
-    opts.excludeStrategies = []
-  }
-
-  const isAllowed = (strategy: TenantResolutionStrategy) => {
-    // excluded takes precedence
-    if (opts.excludeStrategies?.includes(strategy)) {
-      return false
-    }
-    if (opts.includeStrategies?.includes(strategy)) {
-      return true
-    }
-  }
-
-  // always use user first
-  if (isAllowed(TenantResolutionStrategy.USER)) {
-    const userTenantId = ctx.user?.tenantId
-    if (userTenantId) {
-      return userTenantId
-    }
-  }
-
-  // header
-  if (isAllowed(TenantResolutionStrategy.HEADER)) {
-    const headerTenantId = ctx.request.headers[Header.TENANT_ID]
-    if (headerTenantId) {
-      return headerTenantId as string
-    }
-  }
-
-  // query param
-  if (isAllowed(TenantResolutionStrategy.QUERY)) {
-    const queryTenantId = ctx.request.query.tenantId
-    if (queryTenantId) {
-      return queryTenantId as string
-    }
-  }
-
-  // subdomain
-  if (isAllowed(TenantResolutionStrategy.SUBDOMAIN)) {
-    // e.g. budibase.app or local.com:10000
-    let platformHost
-    try {
-      platformHost = new URL(getPlatformURL()).host.split(":")[0]
-    } catch (err: any) {
-      // if invalid URL, just don't try to process subdomain
-      if (err.code !== "ERR_INVALID_URL") {
-        throw err
-      }
-    }
-    // e.g. tenant.budibase.app or tenant.local.com
-    const requestHost = ctx.host
-    // parse the tenant id from the difference
-    if (platformHost && requestHost.includes(platformHost)) {
-      const tenantId = requestHost.substring(
-        0,
-        requestHost.indexOf(`.${platformHost}`)
-      )
-      if (tenantId) {
-        return tenantId
-      }
-    }
-  }
-
-  // path
-  if (isAllowed(TenantResolutionStrategy.PATH)) {
-    // params - have to parse manually due to koa-router not run yet
-    const match = ctx.matched.find(
-      (m: any) => !!m.paramNames.find((p: any) => p.name === "tenantId")
-    )
-
-    // get the raw path url - without any query params
-    const ctxUrl = ctx.originalUrl
-    let url
-    if (ctxUrl.includes("?")) {
-      url = ctxUrl.split("?")[0]
-    } else {
-      url = ctxUrl
-    }
-
-    if (match) {
-      const params = match.params(url, match.captures(url), {})
-      if (params.tenantId) {
-        return params.tenantId
-      }
-    }
-  }
-
-  if (!opts.allowNoTenant) {
-    ctx.throw(403, "Tenant id not set")
-  }
-
-  return undefined
-}