@budibase/server 0.0.32 → 0.0.999-alpha.30

package/src/utilities/routing/index.ts

@@ -0,0 +1,32 @@
+import { createRoutingView } from "../../db/views/staticViews"
+import { ViewName, getQueryIndex, UNICODE_MAX } from "../../db/utils"
+import { context } from "@budibase/backend-core"
+import { ScreenRouting } from "@budibase/types"
+
+type ScreenRoutesView = {
+  id: string
+  routing: ScreenRouting
+}
+
+export async function getRoutingInfo(): Promise<ScreenRoutesView[]> {
+  const db = context.getAppDB()
+  try {
+    const allRouting = await db.query<ScreenRoutesView>(
+      getQueryIndex(ViewName.ROUTING),
+      {
+        startkey: "",
+        endkey: UNICODE_MAX,
+      }
+    )
+    return allRouting.rows.map(row => row.value as ScreenRoutesView)
+  } catch (err: any) {
+    // check if the view doesn't exist, it should for all new instances
+    /* istanbul ignore next */
+    if (err != null && err.name === "not_found") {
+      await createRoutingView()
+      return getRoutingInfo()
+    } else {
+      throw err
+    }
+  }
+}

package/src/utilities/rowProcessor/index.ts

@@ -0,0 +1,279 @@
+import * as linkRows from "../../db/linkedRows"
+import { FieldTypes, AutoFieldSubTypes } from "../../constants"
+import { processFormulas, fixAutoColumnSubType } from "./utils"
+import { ObjectStoreBuckets } from "../../constants"
+import { context, db as dbCore, objectStore } from "@budibase/backend-core"
+import { InternalTables } from "../../db/utils"
+import { TYPE_TRANSFORM_MAP } from "./map"
+import { Row, RowAttachment, Table, ContextUser } from "@budibase/types"
+const { cloneDeep } = require("lodash/fp")
+export * from "./utils"
+
+type AutoColumnProcessingOpts = {
+  reprocessing?: boolean
+  noAutoRelationships?: boolean
+}
+
+const BASE_AUTO_ID = 1
+
+/**
+ * Given the old state of the row and the new one after an update, this will
+ * find the keys that have been removed in the updated row.
+ */
+function getRemovedAttachmentKeys(
+  oldRow: Row,
+  row: Row,
+  attachmentKey: string
+) {
+  if (!oldRow[attachmentKey]) {
+    return []
+  }
+  const oldKeys = oldRow[attachmentKey].map((attachment: any) => attachment.key)
+  // no attachments in new row, all removed
+  if (!row[attachmentKey]) {
+    return oldKeys
+  }
+  const newKeys = row[attachmentKey].map((attachment: any) => attachment.key)
+  return oldKeys.filter((key: string) => newKeys.indexOf(key) === -1)
+}
+
+/**
+ * This will update any auto columns that are found on the row/table with the correct information based on
+ * time now and the current logged in user making the request.
+ * @param {Object} user The user to be used for an appId as well as the createdBy and createdAt fields.
+ * @param {Object} table The table which is to be used for the schema, as well as handling auto IDs incrementing.
+ * @param {Object} row The row which is to be updated with information for the auto columns.
+ * @param {Object} opts specific options for function to carry out optional features.
+ * @returns {{row: Object, table: Object}} The updated row and table, the table may need to be updated
+ * for automatic ID purposes.
+ */
+export function processAutoColumn(
+  user: ContextUser | null,
+  table: Table,
+  row: Row,
+  opts?: AutoColumnProcessingOpts
+) {
+  let noUser = !user || !user.userId
+  let isUserTable = table._id === InternalTables.USER_METADATA
+  let now = new Date().toISOString()
+  // if a row doesn't have a revision then it doesn't exist yet
+  const creating = !row._rev
+  // check its not user table, or whether any of the processing options have been disabled
+  const shouldUpdateUserFields =
+    !isUserTable && !opts?.reprocessing && !opts?.noAutoRelationships && !noUser
+  for (let [key, schema] of Object.entries(table.schema)) {
+    if (!schema.autocolumn) {
+      continue
+    }
+    if (!schema.subtype) {
+      schema = fixAutoColumnSubType(schema)
+    }
+    switch (schema.subtype) {
+      case AutoFieldSubTypes.CREATED_BY:
+        if (creating && shouldUpdateUserFields && user) {
+          row[key] = [user.userId]
+        }
+        break
+      case AutoFieldSubTypes.CREATED_AT:
+        if (creating) {
+          row[key] = now
+        }
+        break
+      case AutoFieldSubTypes.UPDATED_BY:
+        if (shouldUpdateUserFields && user) {
+          row[key] = [user.userId]
+        }
+        break
+      case AutoFieldSubTypes.UPDATED_AT:
+        row[key] = now
+        break
+      case AutoFieldSubTypes.AUTO_ID:
+        if (creating) {
+          schema.lastID = !schema.lastID ? BASE_AUTO_ID : schema.lastID + 1
+          row[key] = schema.lastID
+        }
+        break
+    }
+  }
+  return { table, row }
+}
+
+/**
+ * This will coerce a value to the correct types based on the type transform map
+ * @param {object} row The value to coerce
+ * @param {object} type The type fo coerce to
+ * @returns {object} The coerced value
+ */
+export function coerce(row: any, type: string) {
+  // no coercion specified for type, skip it
+  if (!TYPE_TRANSFORM_MAP[type]) {
+    return row
+  }
+  // eslint-disable-next-line no-prototype-builtins
+  if (TYPE_TRANSFORM_MAP[type].hasOwnProperty(row)) {
+    // @ts-ignore
+    return TYPE_TRANSFORM_MAP[type][row]
+  } else if (TYPE_TRANSFORM_MAP[type].parse) {
+    // @ts-ignore
+    return TYPE_TRANSFORM_MAP[type].parse(row)
+  }
+
+  return row
+}
+
+/**
+ * Given an input route this function will apply all the necessary pre-processing to it, such as coercion
+ * of column values or adding auto-column values.
+ * @param {object} user the user which is performing the input.
+ * @param {object} row the row which is being created/updated.
+ * @param {object} table the table which the row is being saved to.
+ * @param {object} opts some input processing options (like disabling auto-column relationships).
+ * @returns {object} the row which has been prepared to be written to the DB.
+ */
+export function inputProcessing(
+  user: ContextUser | null,
+  table: Table,
+  row: Row,
+  opts?: AutoColumnProcessingOpts
+) {
+  let clonedRow = cloneDeep(row)
+  // need to copy the table so it can be differenced on way out
+  const copiedTable = cloneDeep(table)
+  const dontCleanseKeys = ["type", "_id", "_rev", "tableId"]
+  for (let [key, value] of Object.entries(clonedRow)) {
+    const field = table.schema[key]
+    // cleanse fields that aren't in the schema
+    if (!field) {
+      if (dontCleanseKeys.indexOf(key) === -1) {
+        delete clonedRow[key]
+      }
+      continue
+    }
+    // remove any formula values, they are to be generated
+    if (field.type === FieldTypes.FORMULA) {
+      delete clonedRow[key]
+    }
+    // otherwise coerce what is there to correct types
+    else {
+      clonedRow[key] = coerce(value, field.type)
+    }
+
+    // remove any attachment urls, they are generated on read
+    if (field.type === FieldTypes.ATTACHMENT) {
+      const attachments = clonedRow[key]
+      if (attachments?.length) {
+        attachments.forEach((attachment: RowAttachment) => {
+          delete attachment.url
+        })
+      }
+    }
+  }
+
+  if (!clonedRow._id || !clonedRow._rev) {
+    clonedRow._id = row._id
+    clonedRow._rev = row._rev
+  }
+
+  // handle auto columns - this returns an object like {table, row}
+  return processAutoColumn(user, copiedTable, clonedRow, opts)
+}
+
+/**
+ * This function enriches the input rows with anything they are supposed to contain, for example
+ * link records or attachment links.
+ * @param {object} table the table from which these rows came from originally, this is used to determine
+ * the schema of the rows and then enrich.
+ * @param {object[]|object} rows the rows which are to be enriched.
+ * @param {object} opts used to set some options for the output, such as disabling relationship squashing.
+ * @returns {object[]|object} the enriched rows will be returned.
+ */
+export async function outputProcessing(
+  table: Table,
+  rows: Row[] | Row,
+  opts = { squash: true }
+) {
+  let wasArray = true
+  if (!(rows instanceof Array)) {
+    rows = [rows]
+    wasArray = false
+  }
+  // attach any linked row information
+  let enriched = await linkRows.attachFullLinkedDocs(table, rows as Row[])
+
+  // process formulas
+  enriched = processFormulas(table, enriched, { dynamic: true }) as Row[]
+
+  // set the attachments URLs
+  for (let [property, column] of Object.entries(table.schema)) {
+    if (column.type === FieldTypes.ATTACHMENT) {
+      for (let row of enriched) {
+        if (row[property] == null || !Array.isArray(row[property])) {
+          continue
+        }
+        row[property].forEach((attachment: RowAttachment) => {
+          attachment.url = objectStore.getAppFileUrl(attachment.key)
+        })
+      }
+    }
+  }
+  if (opts.squash) {
+    enriched = await linkRows.squashLinksToPrimaryDisplay(table, enriched)
+  }
+  return wasArray ? enriched : enriched[0]
+}
+
+/**
+ * Clean up any attachments that were attached to a row.
+ * @param {object} table The table from which a row is being removed.
+ * @param {any} row optional - the row being removed.
+ * @param {any} rows optional - if multiple rows being deleted can do this in bulk.
+ * @param {any} oldRow optional - if updating a row this will determine the difference.
+ * @param {any} oldTable optional - if updating a table, can supply the old table to look for
+ * deleted attachment columns.
+ * @return {Promise<void>} When all attachments have been removed this will return.
+ */
+export async function cleanupAttachments(
+  table: Table,
+  {
+    row,
+    rows,
+    oldRow,
+    oldTable,
+  }: { row?: Row; rows?: Row[]; oldRow?: Row; oldTable?: Table }
+): Promise<any> {
+  const appId = context.getAppId()
+  if (!dbCore.isProdAppID(appId)) {
+    const prodAppId = dbCore.getProdAppID(appId!)
+    // if prod exists, then don't allow deleting
+    const exists = await dbCore.dbExists(prodAppId)
+    if (exists) {
+      return
+    }
+  }
+  let files: string[] = []
+  function addFiles(row: Row, key: string) {
+    if (row[key]) {
+      files = files.concat(row[key].map((attachment: any) => attachment.key))
+    }
+  }
+  const schemaToUse = oldTable ? oldTable.schema : table.schema
+  for (let [key, schema] of Object.entries(schemaToUse)) {
+    if (schema.type !== FieldTypes.ATTACHMENT) {
+      continue
+    }
+    // old table had this column, new table doesn't - delete it
+    if (rows && oldTable && !table.schema[key]) {
+      rows.forEach(row => addFiles(row, key))
+    } else if (oldRow && row) {
+      // if updating, need to manage the differences
+      files = files.concat(getRemovedAttachmentKeys(oldRow, row, key))
+    } else if (row) {
+      addFiles(row, key)
+    } else if (rows) {
+      rows.forEach(row => addFiles(row, key))
+    }
+  }
+  if (files.length > 0) {
+    await objectStore.deleteFiles(ObjectStoreBuckets.APPS, files)
+  }
+}

package/src/utilities/rowProcessor/map.ts

@@ -0,0 +1,96 @@
+// @ts-nocheck
+import { FieldTypes } from "../../constants"
+
+/**
+ * A map of how we convert various properties in rows to each other based on the row type.
+ */
+export const TYPE_TRANSFORM_MAP: any = {
+  [FieldTypes.LINK]: {
+    "": [],
+    [null]: [],
+    [undefined]: undefined,
+    parse: link => {
+      if (Array.isArray(link) && typeof link[0] === "object") {
+        return link.map(el => (el && el._id ? el._id : el))
+      }
+      if (typeof link === "string") {
+        return [link]
+      }
+      return link
+    },
+  },
+  [FieldTypes.OPTIONS]: {
+    "": null,
+    [null]: null,
+    [undefined]: undefined,
+  },
+  [FieldTypes.ARRAY]: {
+    "": [],
+    [null]: [],
+    [undefined]: undefined,
+  },
+  [FieldTypes.STRING]: {
+    "": "",
+    [null]: "",
+    [undefined]: undefined,
+  },
+  [FieldTypes.BARCODEQR]: {
+    "": "",
+    [null]: "",
+    [undefined]: undefined,
+  },
+  [FieldTypes.FORMULA]: {
+    "": "",
+    [null]: "",
+    [undefined]: undefined,
+  },
+  [FieldTypes.LONGFORM]: {
+    "": "",
+    [null]: "",
+    [undefined]: undefined,
+  },
+  [FieldTypes.NUMBER]: {
+    "": null,
+    [null]: null,
+    [undefined]: undefined,
+    parse: n => parseFloat(n),
+  },
+  [FieldTypes.DATETIME]: {
+    "": null,
+    [undefined]: undefined,
+    [null]: null,
+    parse: date => {
+      if (date instanceof Date) {
+        return date.toISOString()
+      }
+      return date
+    },
+  },
+  [FieldTypes.ATTACHMENT]: {
+    "": [],
+    [null]: [],
+    [undefined]: undefined,
+  },
+  [FieldTypes.BOOLEAN]: {
+    "": null,
+    [null]: null,
+    [undefined]: undefined,
+    true: true,
+    false: false,
+  },
+  [FieldTypes.AUTO]: {
+    parse: () => undefined,
+  },
+  [FieldTypes.JSON]: {
+    parse: input => {
+      try {
+        if (input === "") {
+          return undefined
+        }
+        return JSON.parse(input)
+      } catch (err) {
+        return input
+      }
+    },
+  },
+}

package/src/utilities/rowProcessor/tests/utils.spec.ts

@@ -0,0 +1,57 @@
+import { fixAutoColumnSubType } from "../utils"
+import { AutoFieldDefaultNames, AutoFieldSubTypes } from "../../../constants"
+
+describe("rowProcessor utility", () => {
+  describe("fixAutoColumnSubType", () => {
+    let schema = {
+      name: "",
+      type: "link",
+      subtype: "", // missing subtype
+      icon: "ri-magic-line",
+      autocolumn: true,
+      constraints: { type: "array", presence: false },
+      tableId: "ta_users",
+      fieldName: "test-Updated By",
+      relationshipType: "many-to-many",
+      sortable: false,
+    }
+
+    it("updates the schema with the correct subtype", async () => {
+      schema.name = AutoFieldDefaultNames.CREATED_BY
+      expect(fixAutoColumnSubType(schema).subtype).toEqual(
+        AutoFieldSubTypes.CREATED_BY
+      )
+      schema.subtype = ""
+
+      schema.name = AutoFieldDefaultNames.UPDATED_BY
+      expect(fixAutoColumnSubType(schema).subtype).toEqual(
+        AutoFieldSubTypes.UPDATED_BY
+      )
+      schema.subtype = ""
+
+      schema.name = AutoFieldDefaultNames.CREATED_AT
+      expect(fixAutoColumnSubType(schema).subtype).toEqual(
+        AutoFieldSubTypes.CREATED_AT
+      )
+      schema.subtype = ""
+
+      schema.name = AutoFieldDefaultNames.UPDATED_AT
+      expect(fixAutoColumnSubType(schema).subtype).toEqual(
+        AutoFieldSubTypes.UPDATED_AT
+      )
+      schema.subtype = ""
+
+      schema.name = AutoFieldDefaultNames.AUTO_ID
+      expect(fixAutoColumnSubType(schema).subtype).toEqual(
+        AutoFieldSubTypes.AUTO_ID
+      )
+      schema.subtype = ""
+    })
+
+    it("returns the column if subtype exists", async () => {
+      schema.subtype = AutoFieldSubTypes.CREATED_BY
+      schema.name = AutoFieldDefaultNames.CREATED_AT
+      expect(fixAutoColumnSubType(schema)).toEqual(schema)
+    })
+  })
+})

package/src/utilities/rowProcessor/utils.ts

@@ -0,0 +1,96 @@
+import {
+  FieldTypes,
+  FormulaTypes,
+  AutoFieldDefaultNames,
+  AutoFieldSubTypes,
+} from "../../constants"
+import { processStringSync } from "@budibase/string-templates"
+import { FieldSchema, Table, Row } from "@budibase/types"
+
+/**
+ * If the subtype has been lost for any reason this works out what
+ * subtype the auto column should be.
+ */
+export function fixAutoColumnSubType(column: FieldSchema) {
+  if (!column.autocolumn || !column.name || column.subtype) {
+    return column
+  }
+  // the columns which get auto generated
+  if (column.name.endsWith(AutoFieldDefaultNames.CREATED_BY)) {
+    column.subtype = AutoFieldSubTypes.CREATED_BY
+  } else if (column.name.endsWith(AutoFieldDefaultNames.UPDATED_BY)) {
+    column.subtype = AutoFieldSubTypes.UPDATED_BY
+  } else if (column.name.endsWith(AutoFieldDefaultNames.CREATED_AT)) {
+    column.subtype = AutoFieldSubTypes.CREATED_AT
+  } else if (column.name.endsWith(AutoFieldDefaultNames.UPDATED_AT)) {
+    column.subtype = AutoFieldSubTypes.UPDATED_AT
+  } else if (column.name.endsWith(AutoFieldDefaultNames.AUTO_ID)) {
+    column.subtype = AutoFieldSubTypes.AUTO_ID
+  }
+  return column
+}
+
+/**
+ * Looks through the rows provided and finds formulas - which it then processes.
+ */
+export function processFormulas(
+  table: Table,
+  rows: Row[] | Row,
+  { dynamic, contextRows }: any = { dynamic: true }
+) {
+  const single = !Array.isArray(rows)
+  let rowArray: Row[]
+  if (single) {
+    rowArray = [rows]
+    contextRows = contextRows ? [contextRows] : contextRows
+  } else {
+    rowArray = rows
+  }
+  for (let [column, schema] of Object.entries(table.schema)) {
+    const isStatic = schema.formulaType === FormulaTypes.STATIC
+    if (
+      schema.type !== FieldTypes.FORMULA ||
+      (dynamic && isStatic) ||
+      (!dynamic && !isStatic)
+    ) {
+      continue
+    }
+    // iterate through rows and process formula
+    for (let i = 0; i < rowArray.length; i++) {
+      if (schema.formula) {
+        let row = rowArray[i]
+        let context = contextRows ? contextRows[i] : row
+        rowArray[i] = {
+          ...row,
+          [column]: processStringSync(schema.formula, context),
+        }
+      }
+    }
+  }
+  return single ? rowArray[0] : rowArray
+}
+
+/**
+ * Processes any date columns and ensures that those without the ignoreTimezones
+ * flag set are parsed as UTC rather than local time.
+ */
+export function processDates(table: Table, rows: Row[]) {
+  let datesWithTZ = []
+  for (let [column, schema] of Object.entries(table.schema)) {
+    if (schema.type !== FieldTypes.DATETIME) {
+      continue
+    }
+    if (!schema.timeOnly && !schema.ignoreTimezones) {
+      datesWithTZ.push(column)
+    }
+  }
+
+  for (let row of rows) {
+    for (let col of datesWithTZ) {
+      if (row[col] && typeof row[col] === "string" && !row[col].endsWith("Z")) {
+        row[col] = new Date(row[col]).toISOString()
+      }
+    }
+  }
+  return rows
+}

package/src/utilities/schema.ts

@@ -0,0 +1,141 @@
+import { FieldTypes } from "../constants"
+
+interface SchemaColumn {
+  readonly name: string
+  readonly type: FieldTypes
+  readonly autocolumn?: boolean
+}
+
+interface Schema {
+  readonly [index: string]: SchemaColumn
+}
+
+interface Row {
+  [index: string]: any
+}
+
+type Rows = Array<Row>
+
+interface SchemaValidation {
+  [index: string]: boolean
+}
+
+interface ValidationResults {
+  schemaValidation: SchemaValidation
+  allValid: boolean
+  invalidColumns: Array<string>
+}
+
+const PARSERS: any = {
+  [FieldTypes.NUMBER]: (attribute?: string) => {
+    if (!attribute) {
+      return attribute
+    }
+    return Number(attribute)
+  },
+  [FieldTypes.DATETIME]: (attribute?: string) => {
+    if (!attribute) {
+      return attribute
+    }
+    return new Date(attribute).toISOString()
+  },
+}
+
+export function isSchema(schema: any): schema is Schema {
+  return (
+    typeof schema === "object" &&
+    Object.values(schema).every(rawColumn => {
+      const column = rawColumn as SchemaColumn
+
+      return (
+        column !== null &&
+        typeof column === "object" &&
+        typeof column.type === "string" &&
+        Object.values(FieldTypes).includes(column.type as FieldTypes)
+      )
+    })
+  )
+}
+
+export function isRows(rows: any): rows is Rows {
+  return Array.isArray(rows) && rows.every(row => typeof row === "object")
+}
+
+export function validate(rows: Rows, schema: Schema): ValidationResults {
+  const results: ValidationResults = {
+    schemaValidation: {},
+    allValid: false,
+    invalidColumns: [],
+  }
+
+  rows.forEach(row => {
+    Object.entries(row).forEach(([columnName, columnData]) => {
+      const columnType = schema[columnName]?.type
+      const isAutoColumn = schema[columnName]?.autocolumn
+
+      // If the columnType is not a string, then it's not present in the schema, and should be added to the invalid columns array
+      if (typeof columnType !== "string") {
+        results.invalidColumns.push(columnName)
+      } else if (
+        // If there's no data for this field don't bother with further checks
+        // If the field is already marked as invalid there's no need for further checks
+        results.schemaValidation[columnName] === false ||
+        columnData == null ||
+        isAutoColumn
+      ) {
+        return
+      } else if (
+        columnType === FieldTypes.NUMBER &&
+        isNaN(Number(columnData))
+      ) {
+        // If provided must be a valid number
+        results.schemaValidation[columnName] = false
+      } else if (
+        // If provided must be a valid date
+        columnType === FieldTypes.DATETIME &&
+        isNaN(new Date(columnData).getTime())
+      ) {
+        results.schemaValidation[columnName] = false
+      } else {
+        results.schemaValidation[columnName] = true
+      }
+    })
+  })
+
+  results.allValid =
+    Object.values(results.schemaValidation).length > 0 &&
+    Object.values(results.schemaValidation).every(column => column)
+
+  // Select unique values
+  results.invalidColumns = [...new Set(results.invalidColumns)]
+  return results
+}
+
+export function parse(rows: Rows, schema: Schema): Rows {
+  return rows.map(row => {
+    const parsedRow: Row = {}
+
+    Object.entries(row).forEach(([columnName, columnData]) => {
+      if (!(columnName in schema) || schema[columnName]?.autocolumn) {
+        // Objects can be present in the row data but not in the schema, so make sure we don't proceed in such a case
+        return
+      }
+
+      const columnType = schema[columnName].type
+
+      if (columnType === FieldTypes.NUMBER) {
+        // If provided must be a valid number
+        parsedRow[columnName] = columnData ? Number(columnData) : columnData
+      } else if (columnType === FieldTypes.DATETIME) {
+        // If provided must be a valid date
+        parsedRow[columnName] = columnData
+          ? new Date(columnData).toISOString()
+          : columnData
+      } else {
+        parsedRow[columnName] = columnData
+      }
+    })
+
+    return parsedRow
+  })
+}